set cookie for room and peerUrl

.github/workflows/docker-bake.hcl

@@ -3,14 +3,14 @@ variable "BASE_IMAGE" {
 }
 
 group "default" {
-  targets = ["runner-base", "runner-builder"]
+  targets = ["runner"]
 }
 
 target "runner-base" {
   dockerfile = "containerfiles/runner-base.Containerfile"
   context    = "."
   args = {
-    BASE_IMAGE = BASE_IMAGE
+    BASE_IMAGE = "${BASE_IMAGE}"
   }
   cache-from = ["type=gha,scope=runner-base-pr"]
   cache-to = ["type=gha,scope=runner-base-pr,mode=max"]
@@ -30,3 +30,19 @@ target "runner-builder" {
     runner-base = "target:runner-base"
   }
 }
+
+target "runner" {
+  dockerfile = "containerfiles/runner.Containerfile"
+  context    = "."
+  args = {
+    RUNNER_BASE_IMAGE    = "runner-base:latest"
+    RUNNER_BUILDER_IMAGE = "runner-builder:latest"
+  }
+  cache-from = ["type=gha,scope=runner-pr"]
+  cache-to = ["type=gha,scope=runner-pr,mode=max"]
+  tags = ["nestri-runner"]
+  contexts = {
+    runner-base    = "target:runner-base"
+    runner-builder = "target:runner-builder"
+  }
+}

.github/workflows/play-standalone.yml

@@ -1,83 +0,0 @@
-name: Build Nestri standalone playsite
-
-on:
-  pull_request:
-    paths:
-      - "containerfiles/playsite.Containerfile"
-      - ".github/workflows/play-standalone.yml"
-      - "packages/play-standalone/**"
-      - "packages/input/**"
-  push:
-    branches: [ dev, production ]
-    paths:
-      - "containerfiles/playsite.Containerfile"
-      - ".github/workflows/play-standalone.yml"
-      - "packages/play-standalone/**"
-      - "packages/input/**"
-    tags:
-      - v*.*.*
-  release:
-    types: [ created ]
-
-env:
-  REGISTRY: ghcr.io
-  IMAGE_NAME: nestrilabs/nestri
-  BASE_TAG_PREFIX: playsite
-
-jobs:
-  build-docker-pr:
-    name: Build image on PR
-    runs-on: ubuntu-latest
-    if: ${{ github.event_name == 'pull_request' }}
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v4
-      - name: Setup Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      - name: Build Docker image
-        uses: docker/build-push-action@v5
-        with:
-          file: containerfiles/playsite.Containerfile
-          context: ./
-          push: false
-          load: true
-          tags: nestri:playsite
-
-  build-and-push-docker:
-    name: Build and push image
-    if: ${{ github.ref == 'refs/heads/dev' || github.ref == 'refs/heads/production' }}
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v4
-      - name: Log into registry ${{ env.REGISTRY }}
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ github.token }}
-      - name: Extract Container metadata
-        id: meta
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/${{ env.BASE_TAG_PREFIX }}
-          #
-          #tag on release, and a nightly build for 'dev'
-          tags: |
-            type=raw,value=nightly,enable={{is_default_branch}}
-            type=raw,value={{branch}}
-            type=raw,value=latest,enable=${{ github.ref == format('refs/heads/{0}', 'production') }}
-            type=semver,pattern={{version}}
-            type=semver,pattern={{major}}.{{minor}}
-            type=semver,pattern={{major}}
-      - name: Build Docker image
-        uses: docker/build-push-action@v5
-        with:
-          file: containerfiles/playsite.Containerfile
-          context: ./
-          push: true
-          tags: ${{ steps.meta.outputs.tags }}
-          labels: ${{ steps.meta.outputs.labels }}

.github/workflows/relay.yml

@@ -1,5 +1,6 @@
-name: Build Nestri relay
+#Tabs not spaces, you moron :)
 
+name: Build nestri:relay
 on:
     pull_request:
         paths:

.github/workflows/runner-bases.yml

@@ -1,73 +0,0 @@
-name: Build Nestri runner base images
-
-on: [ workflow_call ]
-
-env:
-  REGISTRY: ghcr.io
-  IMAGE_NAME: nestrilabs/nestri
-  BASE_IMAGE: docker.io/cachyos/cachyos:latest
-
-jobs:
-  build-and-push-bases:
-    name: Build and push images
-    if: ${{ github.ref == 'refs/heads/production' || github.ref == 'refs/heads/dev' }}
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-    strategy:
-      matrix:
-        variant:
-          - { suffix: "v2", base: "docker.io/cachyos/cachyos:latest" }
-          - { suffix: "v3", base: "docker.io/cachyos/cachyos-v3:latest" }
-          #- { suffix: "v4", base: "docker.io/cachyos/cachyos-v4:latest" } # Disabled until GHA has this
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v4
-      - name: Log into registry ${{ env.REGISTRY }}
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ github.token }}
-      - name: Setup Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      - name: Set Swap Space
-        uses: pierotofy/set-swap-space@master
-        with:
-          swap-size-gb: 20
-      - name: Build and push runner-base image
-        uses: docker/build-push-action@v6
-        with:
-          file: containerfiles/runner-base.Containerfile
-          context: ./
-          push: true
-          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/runner-base:latest-${{ matrix.variant.suffix }}
-          build-args: |
-            BASE_IMAGE=${{ matrix.variant.base }}
-          cache-from: type=gha,scope=runner-base-${{ matrix.variant.suffix }},mode=max
-          cache-to: type=gha,scope=runner-base-${{ matrix.variant.suffix }},mode=max
-          pull: true
-      - name: Build and push runner-builder image
-        uses: docker/build-push-action@v6
-        with:
-          file: containerfiles/runner-builder.Containerfile
-          context: ./
-          push: true
-          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/runner-builder:latest-${{ matrix.variant.suffix }}
-          build-args: |
-            RUNNER_BASE_IMAGE=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/runner-base:latest-${{ matrix.variant.suffix }}
-          cache-from: type=gha,scope=runner-builder-${{ matrix.variant.suffix }},mode=max
-          cache-to: type=gha,scope=runner-builder-${{ matrix.variant.suffix }},mode=max
-      - name: Build and push runner-common image
-        uses: docker/build-push-action@v6
-        with:
-          file: containerfiles/runner-common.Containerfile
-          context: ./
-          push: true
-          tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/runner-common:latest-${{ matrix.variant.suffix }}
-          build-args: |
-            RUNNER_BASE_IMAGE=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/runner-base:latest-${{ matrix.variant.suffix }}
-            RUNNER_BUILDER_IMAGE=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/runner-builder:latest-${{ matrix.variant.suffix }}
-          cache-from: type=gha,scope=runner-common-${{ matrix.variant.suffix }},mode=max
-          cache-to: type=gha,scope=runner-common-${{ matrix.variant.suffix }},mode=max

.github/workflows/runner-variants.yml

@@ -1,86 +0,0 @@
-name: Build Nestri runner image variants
-
-on:
-  workflow_dispatch:
-  schedule:
-    - cron: 7 0 * * 1,3,6  # Nightlies
-  push:
-    branches: [ dev, production ]
-    paths:
-      - "containerfiles/*runner.Containerfile"
-      - ".github/workflows/runner-variants.yml"
-      - "packages/scripts/**"
-      - "packages/configs/**"
-    tags:
-      - v*.*.*
-  release:
-    types: [ created ]
-
-env:
-  REGISTRY: ghcr.io
-  IMAGE_NAME: nestrilabs/nestri
-
-jobs:
-  bases:
-    uses: ./.github/workflows/runner-bases.yml
-    permissions:
-      contents: read
-      packages: write
-  build-and-push-variants:
-    needs: [ bases ]
-    name: Build and push images
-    if: ${{ github.ref == 'refs/heads/production' || github.ref == 'refs/heads/dev' }}
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-    strategy:
-      matrix:
-        variant:
-          - { suffix: "v2", base: "docker.io/cachyos/cachyos:latest" }
-          - { suffix: "v3", base: "docker.io/cachyos/cachyos-v3:latest" }
-          #- { suffix: "v4", base: "docker.io/cachyos/cachyos-v4:latest" } # Disabled until GHA has this
-        runner:
-          - steam
-          - heroic
-          - minecraft
-          # ADD MORE HERE AS NEEDED #
-    steps:
-      - name: Checkout repo
-        uses: actions/checkout@v4
-      - name: Log into registry ${{ env.REGISTRY }}
-        uses: docker/login-action@v3
-        with:
-          registry: ${{ env.REGISTRY }}
-          username: ${{ github.actor }}
-          password: ${{ github.token }}
-      - name: Extract runner metadata
-        id: meta-runner
-        uses: docker/metadata-action@v5
-        with:
-          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/runner
-          tags: |
-            type=raw,value=nightly-${{ matrix.runner }}-${{ matrix.variant.suffix }},enable={{is_default_branch}}
-            type=raw,value={{branch}}-${{ matrix.runner }}-${{ matrix.variant.suffix }}
-            type=raw,value=latest-${{ matrix.runner }}-${{ matrix.variant.suffix }},enable=${{ github.ref == format('refs/heads/{0}', 'production') }}
-            type=semver,pattern={{version}}-${{ matrix.runner }}-${{ matrix.variant.suffix }}
-            type=semver,pattern={{major}}.{{minor}}-${{ matrix.runner }}-${{ matrix.variant.suffix }}
-            type=semver,pattern={{major}}-${{ matrix.runner }}-${{ matrix.variant.suffix }}
-      - name: Setup Docker Buildx
-        uses: docker/setup-buildx-action@v3
-      - name: Set Swap Space
-        uses: pierotofy/set-swap-space@master
-        with:
-          swap-size-gb: 20
-      - name: Build and push runner image
-        uses: docker/build-push-action@v6
-        with:
-          file: containerfiles/${{ matrix.runner }}-runner.Containerfile
-          context: ./
-          push: true
-          tags: ${{ steps.meta-runner.outputs.tags }}
-          labels: ${{ steps.meta-runner.outputs.labels }}
-          build-args: |
-            RUNNER_COMMON_IMAGE=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/runner-common:latest-${{ matrix.variant.suffix }}
-          cache-from: type=gha,scope=runner-${{ matrix.runner }}-${{ matrix.variant.suffix }},mode=max
-          cache-to: type=gha,scope=runner-${{ matrix.runner }}-${{ matrix.variant.suffix }},mode=max

.github/workflows/runner.yml

@@ -0,0 +1,148 @@
+#Tabs not spaces, you moron :)
+
+name: Build nestri-runner
+
+on:
+    pull_request:
+        paths:
+            - "containerfiles/runner*.Containerfile"
+            - "packages/scripts/**"
+            - "packages/server/**"
+            - ".github/workflows/runner.yml"
+    schedule:
+        - cron: 7 0 * * 1,3,6 # Regularly to keep that build cache warm
+    push:
+        branches: [dev, production]
+        paths:
+            - "containerfiles/runner*.Containerfile"
+            - ".github/workflows/runner.yml"
+            - "packages/scripts/**"
+            - "packages/server/**"
+        tags:
+            - v*.*.*
+    release:
+        types: [created]
+
+env:
+    REGISTRY: ghcr.io
+    IMAGE_NAME: nestrilabs/nestri
+    BASE_IMAGE: docker.io/cachyos/cachyos:latest
+
+# This makes our release ci quit prematurely
+# concurrency:
+#     group: ci-${{ github.ref }}
+#     cancel-in-progress: true
+
+jobs:
+    build-docker-pr:
+        name: Build images on PR
+        runs-on: ubuntu-latest
+        permissions:
+            contents: read
+            packages: write
+        if: ${{ github.event_name == 'pull_request' }}
+        steps:
+            - 
+              name: Setup Docker Buildx
+              uses: docker/setup-buildx-action@v3
+            -
+              name: Set Swap Space
+              uses: pierotofy/set-swap-space@master
+              with:
+                swap-size-gb: 20
+            -
+              name: Build images using bake
+              uses: docker/bake-action@v6
+              env:
+                BASE_IMAGE: ${{ env.BASE_IMAGE }}
+              with:
+                files: |
+                  ./.github/workflows/docker-bake.hcl
+                targets: runner
+                push: false
+                load: true
+
+    build-and-push-docker:
+        name: Build and push images
+        if: ${{ github.ref == 'refs/heads/production' || github.ref == 'refs/heads/dev' }}
+        runs-on: ubuntu-latest
+        permissions:
+            contents: read
+            packages: write
+        strategy:
+            matrix:
+                variant:
+                    - { suffix: "", base: "docker.io/cachyos/cachyos:latest" }
+                    - { suffix: "-v3", base: "docker.io/cachyos/cachyos-v3:latest" }
+                    - { suffix: "-v4", base: "docker.io/cachyos/cachyos-v4:latest" }
+        steps:
+            -
+              name: Checkout repo
+              uses: actions/checkout@v4
+            - 
+              name: Log into registry ${{ env.REGISTRY }}
+              uses: docker/login-action@v3
+              with:
+                registry: ${{ env.REGISTRY }}
+                username: ${{ github.actor }}
+                password: ${{ github.token }}
+            - 
+              name: Extract runner metadata
+              id: meta-runner
+              uses: docker/metadata-action@v5
+              with:
+                images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/runner
+                tags: |
+                  type=raw,value=nightly${{ matrix.variant.suffix }},enable={{is_default_branch}}
+                  type=raw,value={{branch}}${{ matrix.variant.suffix }}
+                  type=raw,value=latest${{ matrix.variant.suffix }},enable=${{ github.ref == format('refs/heads/{0}', 'production') }}
+                  type=semver,pattern={{version}}${{ matrix.variant.suffix }}
+                  type=semver,pattern={{major}}.{{minor}}${{ matrix.variant.suffix }}
+                  type=semver,pattern={{major}}${{ matrix.variant.suffix }}
+            -
+              name: Setup Docker Buildx
+              uses: docker/setup-buildx-action@v3
+            - 
+              name: Set Swap Space
+              uses: pierotofy/set-swap-space@master
+              with:
+                swap-size-gb: 20
+            - 
+              name: Build and push runner-base image
+              uses: docker/build-push-action@v6
+              with:
+                file: containerfiles/runner-base.Containerfile
+                context: ./
+                push: true
+                tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/runner-base:latest${{ matrix.variant.suffix }}
+                build-args: |
+                  BASE_IMAGE=${{ matrix.variant.base }}
+                cache-from: type=gha,scope=runner-base${{ matrix.variant.suffix }},mode=max
+                cache-to: type=gha,scope=runner-base${{ matrix.variant.suffix }},mode=max
+                pull: ${{ github.event_name == 'schedule' }}
+            -
+              name: Build and push runner-builder image
+              uses: docker/build-push-action@v6
+              with:
+                file: containerfiles/runner-builder.Containerfile
+                context: ./
+                push: true
+                tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/runner-builder:latest${{ matrix.variant.suffix }}
+                build-args: |
+                  RUNNER_BASE_IMAGE=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/runner-base:latest${{ matrix.variant.suffix }}
+                cache-from: type=gha,scope=runner-builder${{ matrix.variant.suffix }},mode=max
+                cache-to: type=gha,scope=runner-builder${{ matrix.variant.suffix }},mode=max
+            -
+              name: Build and push runner image
+              uses: docker/build-push-action@v6
+              with:
+                file: containerfiles/runner.Containerfile
+                context: ./
+                push: true
+                tags: ${{ steps.meta-runner.outputs.tags }}
+                labels: ${{ steps.meta-runner.outputs.labels }}
+                build-args: |
+                  RUNNER_BASE_IMAGE=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/runner-base:latest${{ matrix.variant.suffix }}
+                  RUNNER_BUILDER_IMAGE=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}/runner-builder:latest${{ matrix.variant.suffix }}
+                cache-from: type=gha,scope=runner${{ matrix.variant.suffix }},mode=max
+                cache-to: type=gha,scope=runner${{ matrix.variant.suffix }},mode=max

buf.gen.yaml

@@ -15,4 +15,3 @@ plugins:
   # Rust (nestri-server)
   - remote: buf.build/community/neoeinstein-prost
     out: packages/server/src/proto
-    opt: flat_output_dir=true

containerfiles/heroic-runner.Containerfile

@@ -1,23 +0,0 @@
-# Container build arguments #
-ARG RUNNER_COMMON_IMAGE=runner-common:latest
-
-#*********************#
-# Final Runtime Stage #
-#*********************#
-FROM ${RUNNER_COMMON_IMAGE}
-
-### FLAVOR/VARIANT CONFIGURATION ###
-## HEROIC LAUNCHER ##
-RUN --mount=type=cache,target=/var/cache/pacman/pkg \
-    pacman -S --noconfirm heroic-games-launcher-bin && \
-    # Cleanup
-    paccache -rk1 && \
-    rm -rf /usr/share/{info,man,doc}/*
-
-## FLAVOR/VARIANT LAUNCH COMMAND ##
-ENV NESTRI_LAUNCH_CMD="heroic"
-### END OF FLAVOR/VARIANT CONFIGURATION ###
-
-### REQUIRED DEFAULT ENTRYPOINT FOR FLAVOR/VARIANT ###
-USER root
-ENTRYPOINT ["supervisord", "-c", "/etc/nestri/supervisord.conf"]

containerfiles/minecraft-runner.Containerfile

@@ -1,24 +0,0 @@
-# Container build arguments #
-ARG RUNNER_COMMON_IMAGE=runner-common:latest
-
-#*********************#
-# Final Runtime Stage #
-#*********************#
-FROM ${RUNNER_COMMON_IMAGE}
-
-### FLAVOR/VARIANT CONFIGURATION ###
-## MINECRAFT ##
-RUN --mount=type=cache,target=/var/cache/pacman/pkg \
-    pacman -S --noconfirm paru && \
-    sudo -H -u ${NESTRI_USER} paru -S --noconfirm aur/minecraft-launcher && \
-    # Cleanup
-    paccache -rk1 && \
-    rm -rf /usr/share/{info,man,doc}/*
-
-## FLAVOR/VARIANT LAUNCH COMMAND ##
-ENV NESTRI_LAUNCH_CMD="minecraft-launcher"
-### END OF FLAVOR/VARIANT CONFIGURATION ###
-
-### REQUIRED DEFAULT ENTRYPOINT FOR FLAVOR/VARIANT ###
-USER root
-ENTRYPOINT ["supervisord", "-c", "/etc/nestri/supervisord.conf"]

containerfiles/runner-builder.Containerfile

@@ -15,7 +15,7 @@ ENV CARGO_HOME=/usr/local/cargo \
 
 # Install build essentials and caching tools
 RUN --mount=type=cache,target=/var/cache/pacman/pkg \
-    pacman -S --noconfirm rustup git base-devel mold \
+    pacman -Sy --noconfirm rustup git base-devel mold \
     meson pkgconf cmake git gcc make
 
 # Override various linker with symlink so mold is forcefully used (ld, ld.lld, lld)
@@ -28,7 +28,7 @@ RUN rustup default stable
 
 # Install cargo-chef with proper caching
 RUN --mount=type=cache,target=${CARGO_HOME}/registry \
-    cargo install cargo-chef --locked
+    cargo install -j $(nproc) cargo-chef --locked
 
 #*******************************#
 # vimputti manager build stages #
@@ -38,10 +38,10 @@ WORKDIR /builder
 
 # Install build dependencies
 RUN --mount=type=cache,target=/var/cache/pacman/pkg \
-    pacman -S --noconfirm lib32-gcc-libs
+    pacman -Sy --noconfirm lib32-gcc-libs
 
 # Clone repository
-RUN git clone --depth 1 --rev "2fde5376b6b9a38cdbd94ccc6a80c9d29a81a417" https://github.com/DatCaptainHorse/vimputti.git
+RUN git clone --depth 1 --rev "9e8bfd0217eeab011c5afc368d3ea67a4c239e81" https://github.com/DatCaptainHorse/vimputti.git
 
 #--------------------------------------------------------------------
 FROM vimputti-manager-deps AS vimputti-manager-planner
@@ -83,7 +83,7 @@ WORKDIR /builder
 
 # Install build dependencies
 RUN --mount=type=cache,target=/var/cache/pacman/pkg \
-    pacman -S --noconfirm gst-plugins-good gst-plugin-rswebrtc
+    pacman -Sy --noconfirm gst-plugins-good gst-plugin-rswebrtc
 
 #--------------------------------------------------------------------
 FROM nestri-server-deps AS nestri-server-planner
@@ -123,14 +123,29 @@ WORKDIR /builder
 
 # Install build dependencies
 RUN --mount=type=cache,target=/var/cache/pacman/pkg \
-    pacman -S --noconfirm libxkbcommon wayland \
+    pacman -Sy --noconfirm libxkbcommon wayland \
     gst-plugins-good gst-plugins-bad libinput
 
 RUN --mount=type=cache,target=${CARGO_HOME}/registry \
     cargo install cargo-c
 
+# Grab cudart from NVIDIA..
+RUN wget https://developer.download.nvidia.com/compute/cuda/redist/cuda_cudart/linux-x86_64/cuda_cudart-linux-x86_64-13.0.96-archive.tar.xz -O cuda_cudart.tar.xz && \
+    mkdir cuda_cudart && tar -xf cuda_cudart.tar.xz -C cuda_cudart --strip-components=1 && \
+    cp cuda_cudart/lib/libcudart.so cuda_cudart/lib/libcudart.so.* /usr/lib/ && \
+    rm -r cuda_cudart && \
+    rm cuda_cudart.tar.xz
+
+# Grab cuda lib from NVIDIA (it's in driver package of all things..)
+RUN wget https://developer.download.nvidia.com/compute/cuda/redist/nvidia_driver/linux-x86_64/nvidia_driver-linux-x86_64-580.95.05-archive.tar.xz -O nvidia_driver.tar.xz && \
+    mkdir nvidia_driver && tar -xf nvidia_driver.tar.xz -C nvidia_driver --strip-components=1 && \
+    cp nvidia_driver/lib/libcuda.so.* /usr/lib/libcuda.so && \
+    ln -s /usr/lib/libcuda.so /usr/lib/libcuda.so.1 && \
+    rm -r nvidia_driver && \
+    rm nvidia_driver.tar.xz
+
 # Clone repository
-RUN git clone --depth 1 --rev "67b1183997fd7aaf57398e4b01bd64c4d2433c45" https://github.com/games-on-whales/gst-wayland-display.git
+RUN git clone --depth 1 --rev "afa853fa03e8403c83bbb3bc0cf39147ad46c266" https://github.com/games-on-whales/gst-wayland-display.git
 
 #--------------------------------------------------------------------
 FROM gst-wayland-deps AS gst-wayland-planner
@@ -148,7 +163,7 @@ COPY --from=gst-wayland-planner /builder/gst-wayland-display/recipe.json .
 
 # Cache dependencies using cargo-chef
 RUN --mount=type=cache,target=${CARGO_HOME}/registry \
-    cargo chef cook --release --recipe-path recipe.json --features cuda
+    cargo chef cook --release --recipe-path recipe.json
 
 
 ENV CARGO_TARGET_DIR=/builder/target
@@ -158,7 +173,7 @@ COPY --from=gst-wayland-planner /builder/gst-wayland-display/ .
 # Build and install directly to artifacts
 RUN --mount=type=cache,target=${CARGO_HOME}/registry \
     --mount=type=cache,target=/builder/target \
-    cargo cinstall --prefix=${ARTIFACTS} --release --features cuda
+    cargo cinstall --prefix=${ARTIFACTS} --release
 
 #*********************************#
 # Patched bubblewrap build stages #
@@ -168,7 +183,7 @@ WORKDIR /builder
 
 # Install build dependencies
 RUN --mount=type=cache,target=/var/cache/pacman/pkg \
-    pacman -S --noconfirm libtool libcap libselinux
+    pacman -Sy --noconfirm libtool libcap libselinux
 
 # Copy patch file from host
 COPY packages/patches/bubblewrap/ /builder/patches/
@@ -199,4 +214,5 @@ COPY --from=gst-wayland-cached-builder /artifacts/include/ /artifacts/include/
 COPY --from=vimputti-manager-cached-builder /artifacts/vimputti-manager /artifacts/bin/
 COPY --from=vimputti-manager-cached-builder /artifacts/libvimputti_shim_64.so /artifacts/lib64/libvimputti_shim.so
 COPY --from=vimputti-manager-cached-builder /artifacts/libvimputti_shim_32.so /artifacts/lib32/libvimputti_shim.so
+COPY --from=gst-wayland-deps /usr/lib/libcuda.so /usr/lib/libcuda.so.* /artifacts/lib/
 COPY --from=bubblewrap-builder /artifacts/bin/bwrap /artifacts/bin/

containerfiles/runner.Containerfile

@@ -2,9 +2,9 @@
 ARG RUNNER_BASE_IMAGE=runner-base:latest
 ARG RUNNER_BUILDER_IMAGE=runner-builder:latest
 
-#**********************#
-# Runtime Common Stage #
-#**********************#
+#*********************#
+# Final Runtime Stage #
+#*********************#
 FROM ${RUNNER_BASE_IMAGE} AS runtime
 FROM ${RUNNER_BUILDER_IMAGE} AS builder
 FROM runtime
@@ -12,11 +12,11 @@ FROM runtime
 ### Package Installation ###
 # Core system components
 RUN --mount=type=cache,target=/var/cache/pacman/pkg \
-    pacman -S --needed --noconfirm \
+    pacman -Sy --needed --noconfirm \
         vulkan-intel lib32-vulkan-intel vpl-gpu-rt \
         vulkan-radeon lib32-vulkan-radeon \
-        mesa lib32-mesa vulkan-mesa-layers lib32-vulkan-mesa-layers \
-        gtk3 lib32-gtk3 \
+        mesa lib32-mesa \
+        steam gtk3 lib32-gtk3 \
         sudo xorg-xwayland seatd libinput gamescope mangohud wlr-randr \
         pipewire pipewire-pulse pipewire-alsa wireplumber \
         noto-fonts-cjk supervisor jq pacman-contrib \
@@ -67,8 +67,10 @@ RUN mkdir -p /etc/pipewire/pipewire.conf.d && \
 COPY packages/configs/wireplumber.conf.d/* /etc/wireplumber/wireplumber.conf.d/
 COPY packages/configs/pipewire.conf.d/* /etc/pipewire/pipewire.conf.d/
 
-## MangoHud Config ##
-COPY packages/configs/MangoHud/MangoHud.conf /etc/nestri/configs/MangoHud/
+## Steam Configs - Proton (Experimental flavor) ##
+RUN mkdir -p "${NESTRI_HOME}/.local/share/Steam/config"
+
+COPY packages/configs/steam/config.vdf "${NESTRI_HOME}/.local/share/Steam/config/"
 
 ### Artifacts from Builder ###
 COPY --from=builder /artifacts/bin/nestri-server /usr/bin/
@@ -86,3 +88,7 @@ RUN chmod +x /etc/nestri/{envs.sh,entrypoint*.sh} && \
     setcap cap_net_admin+ep /usr/bin/vimputti-manager && \
     dbus-uuidgen > /etc/machine-id && \
     LANG=en_US.UTF-8 locale-gen
+
+# Root for most container engines, nestri-user compatible for apptainer without fakeroot
+USER root
+ENTRYPOINT ["supervisord", "-c", "/etc/nestri/supervisord.conf"]

containerfiles/steam-runner.Containerfile

@@ -1,27 +0,0 @@
-# Container build arguments #
-ARG RUNNER_COMMON_IMAGE=runner-common:latest
-
-#*********************#
-# Final Runtime Stage #
-#*********************#
-FROM ${RUNNER_COMMON_IMAGE}
-
-### FLAVOR/VARIANT CONFIGURATION ###
-## STEAM ##
-RUN --mount=type=cache,target=/var/cache/pacman/pkg \
-    pacman -S --noconfirm steam && \
-    # Cleanup
-    paccache -rk1 && \
-    rm -rf /usr/share/{info,man,doc}/*
-
-## Steam Configs - Proton (Experimental flavor) ##
-RUN mkdir -p "${NESTRI_HOME}/.local/share/Steam/config"
-COPY packages/configs/steam/config.vdf "${NESTRI_HOME}/.local/share/Steam/config/"
-
-## FLAVOR/VARIANT LAUNCH COMMAND ##
-ENV NESTRI_LAUNCH_CMD="steam -tenfoot -cef-force-gpu"
-### END OF FLAVOR/VARIANT CONFIGURATION ###
-
-### REQUIRED DEFAULT ENTRYPOINT FOR FLAVOR/VARIANT ###
-USER root
-ENTRYPOINT ["supervisord", "-c", "/etc/nestri/supervisord.conf"]

packages/configs/MangoHud/MangoHud.conf

@@ -1,48 +0,0 @@
-legacy_layout=false
-
-# common
-horizontal
-horizontal_stretch
-hud_no_margin
-no_small_font
-background_alpha=0.66
-round_corners=0
-background_color=000000
-font_size=24
-position=top-left
-engine_short_names
-
-# colors
-text_color=DFDFDF
-gpu_color=FF4E00
-cpu_color=00AA00
-engine_color=00AA00
-vram_color=00AA00
-ram_color=00AA00
-frametime_color=FF4E00
-
-# load colors
-cpu_load_color=DFDFDF,DF964D,DF3D3D
-gpu_load_color=DFDFDF,DF964D,DF3D3D
-
-# GPU and VRAM
-gpu_text=NESTRI
-gpu_stats
-gpu_load_change
-gpu_load_value=70,90
-
-vram
-
-# CPU and RAM
-cpu_text=CPU
-cpu_stats
-cpu_load_change
-cpu_load_value=70,90
-
-ram
-
-# FPS and timing
-fps
-fps_metrics=0.01
-
-frame_timing

packages/configs/pipewire.conf.d/nestri-low-latency.conf

@@ -2,6 +2,6 @@ context.properties = {
     default.clock.rate = 48000
     default.clock.allowed-rates = [48000]
     default.clock.min-quantum = 128
-    default.clock.max-quantum = 1024
-    default.clock.quantum = 512
+    default.clock.max-quantum = 256
+    default.clock.quantum = 128
 }

packages/configs/wireplumber.conf.d/nestri-low-latency.conf

@@ -22,7 +22,7 @@
       ],
       "apply_properties": {
         "pulse.min.req": 128,
-        "pulse.max.req": 1024,
+        "pulse.max.req": 256,
         "pulse.idle.timeout": 0
       }
     }

packages/input/.containerignore

@@ -1,3 +0,0 @@
-.idea/
-dist/
-node_modules/

packages/input/package.json

@@ -7,22 +7,24 @@
         ".": "./src/index.ts"
     },
     "devDependencies": {
-        "@bufbuild/buf": "^1.59.0",
-        "@bufbuild/protoc-gen-es": "^2.10.0"
+        "@bufbuild/buf": "^1.57.2",
+        "@bufbuild/protoc-gen-es": "^2.9.0"
     },
     "dependencies": {
-        "@bufbuild/protobuf": "^2.10.0",
-        "@chainsafe/libp2p-noise": "^17.0.0",
+        "@bufbuild/protobuf": "^2.9.0",
+        "@chainsafe/libp2p-noise": "^16.1.4",
         "@chainsafe/libp2p-quic": "^1.1.3",
-        "@chainsafe/libp2p-yamux": "^8.0.1",
-        "@libp2p/identify": "^4.0.5",
-        "@libp2p/interface": "^3.0.2",
-        "@libp2p/ping": "^3.0.5",
-        "@libp2p/websockets": "^10.0.6",
-        "@libp2p/webtransport": "^6.0.7",
-        "@libp2p/utils": "^7.0.5",
-        "@multiformats/multiaddr": "^13.0.1",
-        "libp2p": "^3.0.6",
-        "uint8arraylist": "^2.4.8"
+        "@chainsafe/libp2p-yamux": "^7.0.4",
+        "@libp2p/identify": "^3.0.39",
+        "@libp2p/interface": "^2.11.0",
+        "@libp2p/ping": "^2.0.37",
+        "@libp2p/websockets": "^9.2.19",
+        "@libp2p/webtransport": "^5.0.51",
+        "@multiformats/multiaddr": "^12.5.1",
+        "it-length-prefixed": "^10.0.1",
+        "it-pipe": "^3.0.1",
+        "libp2p": "^2.10.0",
+        "uint8arraylist": "^2.4.8",
+        "uint8arrays": "^5.1.0"
     }
 }

packages/input/src/controller.ts

@@ -1,15 +1,21 @@
 import { controllerButtonToLinuxEventCode } from "./codes";
 import { WebRTCStream } from "./webrtc-stream";
 import {
+  ProtoMessageBase,
+  ProtoMessageInput,
+  ProtoMessageInputSchema,
+} from "./proto/messages_pb";
+import {
+  ProtoInputSchema,
   ProtoControllerAttachSchema,
   ProtoControllerDetachSchema,
-  ProtoControllerStateBatchSchema,
-  ProtoControllerStateBatch,
+  ProtoControllerButtonSchema,
+  ProtoControllerTriggerSchema,
+  ProtoControllerAxisSchema,
+  ProtoControllerStickSchema,
   ProtoControllerRumble,
 } from "./proto/types_pb";
 import { create, toBinary, fromBinary } from "@bufbuild/protobuf";
-import { createMessage } from "./utils";
-import { ProtoMessageSchema } from "./proto/messages_pb";
 
 interface Props {
   webrtc: WebRTCStream;
@@ -17,7 +23,6 @@ interface Props {
 }
 
 interface GamepadState {
-  previousButtonState: Map<number, boolean>;
   buttonState: Map<number, boolean>;
   leftTrigger: number;
   rightTrigger: number;
@@ -29,17 +34,12 @@ interface GamepadState {
   dpadY: number;
 }
 
-enum PollState {
-  IDLE,
-  RUNNING,
-}
-
 export class Controller {
   protected wrtc: WebRTCStream;
+  protected slot: number;
   protected connected: boolean = false;
   protected gamepad: Gamepad | null = null;
-  protected state: GamepadState = {
-    previousButtonState: new Map<number, boolean>(),
+  protected lastState: GamepadState = {
     buttonState: new Map<number, boolean>(),
     leftTrigger: 0,
     rightTrigger: 0,
@@ -53,33 +53,17 @@ export class Controller {
   // TODO: As user configurable, set quite low now for decent controllers (not Nintendo ones :P)
   protected stickDeadzone: number = 2048; // 2048 / 32768 = ~0.06 (6% of stick range)
 
-  // Polling configuration
-  private readonly FULL_RATE_MS = 10; // 100 UPS
-  private readonly IDLE_THRESHOLD = 100; // ms before considering idle/hands off controller
-  private readonly FULL_INTERVAL = 250; // ms before sending full state occassionally, to verify inputs are synced
-
-  // Polling state
-  private pollingState: PollState = PollState.IDLE;
-  private lastInputTime: number = Date.now();
-  private lastFullTime: number = Date.now();
-  private pollInterval: any = null;
-
-  // Controller batch vars
-  private sequence: number = 0;
-  private readonly CHANGED_BUTTONS_STATE = 1 << 0;
-  private readonly CHANGED_LEFT_STICK_X = 1 << 1;
-  private readonly CHANGED_LEFT_STICK_Y = 1 << 2;
-  private readonly CHANGED_RIGHT_STICK_X = 1 << 3;
-  private readonly CHANGED_RIGHT_STICK_Y = 1 << 4;
-  private readonly CHANGED_LEFT_TRIGGER = 1 << 5;
-  private readonly CHANGED_RIGHT_TRIGGER = 1 << 6;
-  private readonly CHANGED_DPAD_X = 1 << 7;
-  private readonly CHANGED_DPAD_Y = 1 << 8;
-
-  private _dcHandler: ((data: ArrayBuffer) => void) | null = null;
+  private updateInterval = 10.0; // 100 updates per second
+  private _dcRumbleHandler: ((data: ArrayBuffer) => void) | null = null;
 
   constructor({ webrtc, e }: Props) {
     this.wrtc = webrtc;
+    this.slot = e.gamepad.index;
+
+    this.updateInterval = 1000 / webrtc.currentFrameRate;
+
+    // Gamepad connected
+    this.gamepad = e.gamepad;
 
     // Get vendor of gamepad from id string (i.e. "... Vendor: 054c Product: 09cc")
     const vendorMatch = e.gamepad.id.match(/Vendor:\s?([0-9a-fA-F]{4})/);
@@ -88,49 +72,34 @@ export class Controller {
     const productMatch = e.gamepad.id.match(/Product:\s?([0-9a-fA-F]{4})/);
     const productId = productMatch ? productMatch[1].toLowerCase() : "unknown";
 
-    // Listen to datachannel events from server
-    this._dcHandler = (data: ArrayBuffer) => {
-      if (!this.connected) return;
-      try {
-        // First decode the wrapper message
-        const uint8Data = new Uint8Array(data);
-        const messageWrapper = fromBinary(ProtoMessageSchema, uint8Data);
-
-        if (messageWrapper.payload.case === "controllerRumble") {
-          this.rumbleCallback(messageWrapper.payload.value);
-        } else if (messageWrapper.payload.case === "controllerAttach") {
-          if (this.gamepad) return; // already attached
-          const attachMsg = messageWrapper.payload.value;
-          // Gamepad connected succesfully
-          this.gamepad = e.gamepad;
-          console.log(
-            `Gamepad connected: ${e.gamepad.id}, local slot ${e.gamepad.index}, msg: ${attachMsg.sessionSlot}`,
-          );
-          this.run();
-        }
-      } catch (err) {
-        console.error("Error decoding datachannel message:", err);
-      }
-    };
-    this.wrtc.addDataChannelCallback(this._dcHandler);
-
-    const attachMsg = createMessage(
-      create(ProtoControllerAttachSchema, {
+    const attachMsg = create(ProtoInputSchema, {
+      $typeName: "proto.ProtoInput",
+      inputType: {
+        case: "controllerAttach",
+        value: create(ProtoControllerAttachSchema, {
+          type: "ControllerAttach",
           id: this.vendor_id_to_controller(vendorId, productId),
-        sessionSlot: e.gamepad.index,
-        sessionId: this.wrtc.getSessionID(),
+          slot: this.slot,
         }),
-      "controllerInput",
-    );
-    this.wrtc.sendBinary(toBinary(ProtoMessageSchema, attachMsg));
+      },
+    });
+    const message: ProtoMessageInput = {
+      $typeName: "proto.ProtoMessageInput",
+      messageBase: {
+        $typeName: "proto.ProtoMessageBase",
+        payloadType: "controllerInput",
+      } as ProtoMessageBase,
+      data: attachMsg,
+    };
+    this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message));
+
+    // Listen to feedback rumble events from server
+    this._dcRumbleHandler = (data: any) => this.rumbleCallback(data as ArrayBuffer);
+    this.wrtc.addDataChannelCallback(this._dcRumbleHandler);
 
     this.run();
   }
 
-  public getSlot(): number {
-    return this.gamepad.index;
-  }
-
   // Maps vendor id and product id to supported controller type
   // Currently supported: Sony (ps4, ps5), Microsoft (xbox360, xboxone), Nintendo (switchpro)
   // Default fallback to xbox360
@@ -180,334 +149,341 @@ export class Controller {
     return ((value - fromMin) * (toMax - toMin)) / (fromMax - fromMin) + toMin;
   }
 
-  private restartPolling() {
-    // Clear existing interval
-    if (this.pollInterval) {
-      clearInterval(this.pollInterval);
-      this.pollInterval = null;
-    }
-
-    // Restart with active polling
-    this.pollingState = PollState.RUNNING;
-    this.lastInputTime = Date.now();
-
-    // Start interval
-    this.pollInterval = setInterval(
-      () => this.pollGamepad(),
-      this.FULL_RATE_MS,
-    );
-  }
-
   private pollGamepad() {
-    if (!this.connected || !this.gamepad) return;
-
     const gamepads = navigator.getGamepads();
-    if (!gamepads[this.gamepad.index]) return;
-
-    this.gamepad = gamepads[this.gamepad.index];
-
-    // Collect state changes
-    const changedFields = this.collectStateChanges();
-
-    // Send batched changes update if there's changes
-    if (changedFields > 0) {
-      let send_type = 1;
-      const timeSinceFull = Date.now() - this.lastFullTime;
-      if (timeSinceFull > this.FULL_INTERVAL) {
-        send_type = 0;
-        this.lastFullTime = Date.now();
+    if (this.slot < gamepads.length) {
+      const gamepad = gamepads[this.slot];
+      if (gamepad) {
+        /* Button handling */
+        gamepad.buttons.forEach((button, index) => {
+          // Ignore d-pad buttons (12-15) as we handle those as axis
+          if (index >= 12 && index <= 15) return;
+          // ignore trigger buttons (6-7) as we handle those as axis
+          if (index === 6 || index === 7) return;
+          // If state differs, send
+          if (button.pressed !== this.lastState.buttonState.get(index)) {
+            const linuxCode = this.controllerButtonToVirtualKeyCode(index);
+            if (linuxCode === undefined) {
+              // Skip unmapped button index
+              this.lastState.buttonState.set(index, button.pressed);
+              return;
             }
 
-      this.sendBatchedState(changedFields, send_type);
-      this.lastInputTime = Date.now();
-      if (this.pollingState !== PollState.RUNNING) {
-        this.pollingState = PollState.RUNNING;
-      }
-    }
-
-    const timeSinceInput = Date.now() - this.lastInputTime;
-    if (timeSinceInput > this.IDLE_THRESHOLD) {
-      // Changing from running to idle..
-      if (this.pollingState === PollState.RUNNING) {
-        // Send full state on idle assumption
-        this.sendBatchedState(0xff, 0);
-        this.pollingState = PollState.IDLE;
-      }
-    }
-
-    this.state.buttonState.forEach((b, i) =>
-      this.state.previousButtonState.set(i, b),
+            const buttonProto = create(ProtoInputSchema, {
+              $typeName: "proto.ProtoInput",
+              inputType: {
+                case: "controllerButton",
+                value: create(ProtoControllerButtonSchema, {
+                  type: "ControllerButton",
+                  slot: this.slot,
+                  button: linuxCode,
+                  pressed: button.pressed,
+                }),
+              },
+            });
+            const buttonMessage: ProtoMessageInput = {
+              $typeName: "proto.ProtoMessageInput",
+              messageBase: {
+                $typeName: "proto.ProtoMessageBase",
+                payloadType: "controllerInput",
+              } as ProtoMessageBase,
+              data: buttonProto,
+            };
+            this.wrtc.sendBinary(
+              toBinary(ProtoMessageInputSchema, buttonMessage),
             );
+            // Store button state
+            this.lastState.buttonState.set(index, button.pressed);
           }
+        });
 
-  private collectStateChanges(): number {
-    let changedFields = 0;
-
-    // Collect analog values
+        /* Trigger handling */
+        // map trigger value from 0.0 to 1.0 to -32768 to 32767
         const leftTrigger = Math.round(
-      this.remapFromTo(
-        this.gamepad.buttons[6]?.value ?? 0,
-        0,
-        1,
-        -32768,
-        32767,
-      ),
+          this.remapFromTo(gamepad.buttons[6]?.value ?? 0, 0, 1, -32768, 32767),
         );
+        // If state differs, send
+        if (leftTrigger !== this.lastState.leftTrigger) {
+          const triggerProto = create(ProtoInputSchema, {
+            $typeName: "proto.ProtoInput",
+            inputType: {
+              case: "controllerTrigger",
+              value: create(ProtoControllerTriggerSchema, {
+                type: "ControllerTrigger",
+                slot: this.slot,
+                trigger: 0, // 0 = left, 1 = right
+                value: leftTrigger,
+              }),
+            },
+          });
+          const triggerMessage: ProtoMessageInput = {
+            $typeName: "proto.ProtoMessageInput",
+            messageBase: {
+              $typeName: "proto.ProtoMessageBase",
+              payloadType: "controllerInput",
+            } as ProtoMessageBase,
+            data: triggerProto,
+          };
+          this.lastState.leftTrigger = leftTrigger;
+          this.wrtc.sendBinary(
+            toBinary(ProtoMessageInputSchema, triggerMessage),
+          );
+        }
         const rightTrigger = Math.round(
-      this.remapFromTo(
-        this.gamepad.buttons[7]?.value ?? 0,
-        0,
-        1,
-        -32768,
-        32767,
-      ),
+          this.remapFromTo(gamepad.buttons[7]?.value ?? 0, 0, 1, -32768, 32767),
         );
+        // If state differs, send
+        if (rightTrigger !== this.lastState.rightTrigger) {
+          const triggerProto = create(ProtoInputSchema, {
+            $typeName: "proto.ProtoInput",
+            inputType: {
+              case: "controllerTrigger",
+              value: create(ProtoControllerTriggerSchema, {
+                type: "ControllerTrigger",
+                slot: this.slot,
+                trigger: 1, // 0 = left, 1 = right
+                value: rightTrigger,
+              }),
+            },
+          });
+          const triggerMessage: ProtoMessageInput = {
+            $typeName: "proto.ProtoMessageInput",
+            messageBase: {
+              $typeName: "proto.ProtoMessageBase",
+              payloadType: "controllerInput",
+            } as ProtoMessageBase,
+            data: triggerProto,
+          };
+          this.lastState.rightTrigger = rightTrigger;
+          this.wrtc.sendBinary(
+            toBinary(ProtoMessageInputSchema, triggerMessage),
+          );
+        }
 
-    const leftX = this.remapFromTo(
-      this.gamepad.axes[0] ?? 0,
-      -1,
-      1,
-      -32768,
-      32767,
-    );
-    const leftY = this.remapFromTo(
-      this.gamepad.axes[1] ?? 0,
-      -1,
-      1,
-      -32768,
-      32767,
-    );
+        /* DPad handling */
+        // We send dpad buttons as axis values -1 to 1 for left/up, right/down
+        const dpadLeft = gamepad.buttons[14]?.pressed ? 1 : 0;
+        const dpadRight = gamepad.buttons[15]?.pressed ? 1 : 0;
+        const dpadX = dpadLeft ? -1 : dpadRight ? 1 : 0;
+        if (dpadX !== this.lastState.dpadX) {
+          const dpadProto = create(ProtoInputSchema, {
+            $typeName: "proto.ProtoInput",
+            inputType: {
+              case: "controllerAxis",
+              value: create(ProtoControllerAxisSchema, {
+                type: "ControllerAxis",
+                slot: this.slot,
+                axis: 0, // 0 = dpadX, 1 = dpadY
+                value: dpadX,
+              }),
+            },
+          });
+          const dpadMessage: ProtoMessageInput = {
+            $typeName: "proto.ProtoMessageInput",
+            messageBase: {
+              $typeName: "proto.ProtoMessageBase",
+              payloadType: "controllerInput",
+            } as ProtoMessageBase,
+            data: dpadProto,
+          };
+          this.lastState.dpadX = dpadX;
+          this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, dpadMessage));
+        }
+
+        const dpadUp = gamepad.buttons[12]?.pressed ? 1 : 0;
+        const dpadDown = gamepad.buttons[13]?.pressed ? 1 : 0;
+        const dpadY = dpadUp ? -1 : dpadDown ? 1 : 0;
+        if (dpadY !== this.lastState.dpadY) {
+          const dpadProto = create(ProtoInputSchema, {
+            $typeName: "proto.ProtoInput",
+            inputType: {
+              case: "controllerAxis",
+              value: create(ProtoControllerAxisSchema, {
+                type: "ControllerAxis",
+                slot: this.slot,
+                axis: 1, // 0 = dpadX, 1 = dpadY
+                value: dpadY,
+              }),
+            },
+          });
+          const dpadMessage: ProtoMessageInput = {
+            $typeName: "proto.ProtoMessageInput",
+            messageBase: {
+              $typeName: "proto.ProtoMessageBase",
+              payloadType: "controllerInput",
+            } as ProtoMessageBase,
+            data: dpadProto,
+          };
+          this.lastState.dpadY = dpadY;
+          this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, dpadMessage));
+        }
+
+        /* Stick handling */
+        // stick values need to be mapped from -1.0 to 1.0 to -32768 to 32767
+        const leftX = this.remapFromTo(gamepad.axes[0] ?? 0, -1, 1, -32768, 32767);
+        const leftY = this.remapFromTo(gamepad.axes[1] ?? 0, -1, 1, -32768, 32767);
+        // Apply deadzone
         const sendLeftX =
           Math.abs(leftX) > this.stickDeadzone ? Math.round(leftX) : 0;
         const sendLeftY =
           Math.abs(leftY) > this.stickDeadzone ? Math.round(leftY) : 0;
+        // if outside deadzone, send normally if changed
+        // if moves inside deadzone, zero it if not inside deadzone last time
+        if (
+          sendLeftX !== this.lastState.leftX ||
+          sendLeftY !== this.lastState.leftY
+        ) {
+          // console.log("Sticks: ", sendLeftX, sendLeftY, sendRightX, sendRightY);
+          const stickProto = create(ProtoInputSchema, {
+            $typeName: "proto.ProtoInput",
+            inputType: {
+              case: "controllerStick",
+              value: create(ProtoControllerStickSchema, {
+                type: "ControllerStick",
+                slot: this.slot,
+                stick: 0, // 0 = left, 1 = right
+                x: sendLeftX,
+                y: sendLeftY,
+              }),
+            },
+          });
+          const stickMessage: ProtoMessageInput = {
+            $typeName: "proto.ProtoMessageInput",
+            messageBase: {
+              $typeName: "proto.ProtoMessageBase",
+              payloadType: "controllerInput",
+            } as ProtoMessageBase,
+            data: stickProto,
+          };
+          this.lastState.leftX = sendLeftX;
+          this.lastState.leftY = sendLeftY;
+          this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, stickMessage));
+        }
 
-    const rightX = this.remapFromTo(
-      this.gamepad.axes[2] ?? 0,
-      -1,
-      1,
-      -32768,
-      32767,
-    );
-    const rightY = this.remapFromTo(
-      this.gamepad.axes[3] ?? 0,
-      -1,
-      1,
-      -32768,
-      32767,
-    );
+        const rightX = this.remapFromTo(gamepad.axes[2] ?? 0, -1, 1, -32768, 32767);
+        const rightY = this.remapFromTo(gamepad.axes[3] ?? 0, -1, 1, -32768, 32767);
+        // Apply deadzone
         const sendRightX =
           Math.abs(rightX) > this.stickDeadzone ? Math.round(rightX) : 0;
         const sendRightY =
           Math.abs(rightY) > this.stickDeadzone ? Math.round(rightY) : 0;
-
-    const dpadX =
-      (this.gamepad.buttons[14]?.pressed ? -1 : 0) +
-      (this.gamepad.buttons[15]?.pressed ? 1 : 0);
-    const dpadY =
-      (this.gamepad.buttons[12]?.pressed ? -1 : 0) +
-      (this.gamepad.buttons[13]?.pressed ? 1 : 0);
-
-    // Check what changed
-    for (let i = 0; i < this.gamepad.buttons.length; i++) {
-      if (i >= 6 && i <= 7) continue; // Skip triggers
-      if (i >= 12 && i <= 15) continue; // Skip d-pad
-      if (this.state.buttonState.get(i) !== this.gamepad.buttons[i].pressed) {
-        changedFields |= this.CHANGED_BUTTONS_STATE;
-      }
-      this.state.buttonState.set(i, this.gamepad.buttons[i].pressed);
-    }
-    if (leftTrigger !== this.state.leftTrigger) {
-      changedFields |= this.CHANGED_LEFT_TRIGGER;
-    }
-    this.state.leftTrigger = leftTrigger;
-    if (rightTrigger !== this.state.rightTrigger) {
-      changedFields |= this.CHANGED_RIGHT_TRIGGER;
-    }
-    this.state.rightTrigger = rightTrigger;
-    if (sendLeftX !== this.state.leftX) {
-      changedFields |= this.CHANGED_LEFT_STICK_X;
-    }
-    this.state.leftX = sendLeftX;
-    if (sendLeftY !== this.state.leftY) {
-      changedFields |= this.CHANGED_LEFT_STICK_Y;
-    }
-    this.state.leftY = sendLeftY;
-    if (sendRightX !== this.state.rightX) {
-      changedFields |= this.CHANGED_RIGHT_STICK_X;
-    }
-    this.state.rightX = sendRightX;
-    if (sendRightY !== this.state.rightY) {
-      changedFields |= this.CHANGED_RIGHT_STICK_Y;
-    }
-    this.state.rightY = sendRightY;
-    if (dpadX !== this.state.dpadX) {
-      changedFields |= this.CHANGED_DPAD_X;
-    }
-    this.state.dpadX = dpadX;
-    if (dpadY !== this.state.dpadY) {
-      changedFields |= this.CHANGED_DPAD_Y;
-    }
-    this.state.dpadY = dpadY;
-
-    return changedFields;
-  }
-
-  private sendBatchedState(changedFields: number, updateType: number) {
-    // @ts-ignore
-    let message: ProtoControllerStateBatch = {
-      sessionSlot: this.gamepad.index,
-      sessionId: this.wrtc.getSessionID(),
-      updateType: updateType,
-      sequence: this.sequence++,
+        if (
+          sendRightX !== this.lastState.rightX ||
+          sendRightY !== this.lastState.rightY
+        ) {
+          const stickProto = create(ProtoInputSchema, {
+            $typeName: "proto.ProtoInput",
+            inputType: {
+              case: "controllerStick",
+              value: create(ProtoControllerStickSchema, {
+                type: "ControllerStick",
+                slot: this.slot,
+                stick: 1, // 0 = left, 1 = right
+                x: sendRightX,
+                y: sendRightY,
+              }),
+            },
+          });
+          const stickMessage: ProtoMessageInput = {
+            $typeName: "proto.ProtoMessageInput",
+            messageBase: {
+              $typeName: "proto.ProtoMessageBase",
+              payloadType: "controllerInput",
+            } as ProtoMessageBase,
+            data: stickProto,
           };
-
-    // For FULL_STATE, include everything
-    if (updateType === 0) {
-      message.changedFields = 0xff;
-
-      message.buttonChangedMask = Object.fromEntries(
-        Array.from(this.state.buttonState)
-          .map(
-            ([key, value]) =>
-              [this.controllerButtonToVirtualKeyCode(key), value] as const,
-          )
-          .filter(([code]) => code !== undefined),
-      );
-      message.leftStickX = this.state.leftX;
-      message.leftStickY = this.state.leftY;
-      message.rightStickX = this.state.rightX;
-      message.rightStickY = this.state.rightY;
-      message.leftTrigger = this.state.leftTrigger;
-      message.rightTrigger = this.state.rightTrigger;
-      message.dpadX = this.state.dpadX;
-      message.dpadY = this.state.dpadY;
+          this.lastState.rightX = sendRightX;
+          this.lastState.rightY = sendRightY;
+          this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, stickMessage));
         }
-    // For DELTA, only include changed fields
-    else {
-      message.changedFields = changedFields;
-
-      if (changedFields & this.CHANGED_BUTTONS_STATE) {
-        const currentStateMap = this.state.buttonState;
-        const previousStateMap = this.state.previousButtonState;
-        const allKeys = new Set([
-          // @ts-ignore
-          ...currentStateMap.keys(),
-          // @ts-ignore
-          ...previousStateMap.keys(),
-        ]);
-        message.buttonChangedMask = Object.fromEntries(
-          Array.from(allKeys)
-            .filter((key) => {
-              const newState = currentStateMap.get(key);
-              const oldState = previousStateMap.get(key);
-              return newState !== oldState;
-            })
-            .map((key) => {
-              const newValue = currentStateMap.get(key) ?? false;
-              return [
-                this.controllerButtonToVirtualKeyCode(key),
-                newValue,
-              ] as const;
-            })
-            .filter(([code]) => code !== undefined),
-        );
       }
-      if (changedFields & this.CHANGED_LEFT_STICK_X) {
-        message.leftStickX = this.state.leftX;
-      }
-      if (changedFields & this.CHANGED_LEFT_STICK_Y) {
-        message.leftStickY = this.state.leftY;
-      }
-      if (changedFields & this.CHANGED_RIGHT_STICK_X) {
-        message.rightStickX = this.state.rightX;
-      }
-      if (changedFields & this.CHANGED_RIGHT_STICK_Y) {
-        message.rightStickY = this.state.rightY;
-      }
-      if (changedFields & this.CHANGED_LEFT_TRIGGER) {
-        message.leftTrigger = this.state.leftTrigger;
-      }
-      if (changedFields & this.CHANGED_RIGHT_TRIGGER) {
-        message.rightTrigger = this.state.rightTrigger;
-      }
-      if (changedFields & this.CHANGED_DPAD_X) {
-        message.dpadX = this.state.dpadX;
-      }
-      if (changedFields & this.CHANGED_DPAD_Y) {
-        message.dpadY = this.state.dpadY;
     }
   }
 
-    // Send message
-    const batchMessage = createMessage(
-      create(
-        ProtoControllerStateBatchSchema,
-        message as ProtoControllerStateBatch,
-      ),
-      "controllerInput",
-    );
-    this.wrtc.sendBinary(toBinary(ProtoMessageSchema, batchMessage));
-  }
+  private loopInterval: any = null;
 
   public run() {
-    if (this.connected) this.stop();
+    if (this.connected)
+      this.stop();
 
     this.connected = true;
-
-    // Start with active polling
-    this.restartPolling();
+    // Poll gamepads in setInterval loop
+    this.loopInterval = setInterval(() => {
+      if (this.connected) this.pollGamepad();
+    }, this.updateInterval);
   }
 
   public stop() {
-    if (this.pollInterval) {
-      clearInterval(this.pollInterval);
-      this.pollInterval = null;
+    if (this.loopInterval) {
+      clearInterval(this.loopInterval);
+      this.loopInterval = null;
     }
     this.connected = false;
   }
 
+  public getSlot() {
+    return this.slot;
+  }
+
   public dispose() {
     this.stop();
     // Remove callback
-    if (this._dcHandler !== null) {
-      this.wrtc.removeDataChannelCallback(this._dcHandler);
-      this._dcHandler = null;
+    if (this._dcRumbleHandler !== null) {
+      this.wrtc.removeDataChannelCallback(this._dcRumbleHandler);
+      this._dcRumbleHandler = null;
     }
-    if (this.gamepad) {
     // Gamepad disconnected
-      const detachMsg = createMessage(
-        create(ProtoControllerDetachSchema, {
-          sessionSlot: this.gamepad.index,
+    const detachMsg = create(ProtoInputSchema, {
+      $typeName: "proto.ProtoInput",
+      inputType: {
+        case: "controllerDetach",
+        value: create(ProtoControllerDetachSchema, {
+          type: "ControllerDetach",
+          slot: this.slot,
         }),
-        "controllerInput",
-      );
-      this.wrtc.sendBinary(toBinary(ProtoMessageSchema, detachMsg));
-    }
+      },
+    });
+    const message: ProtoMessageInput = {
+      $typeName: "proto.ProtoMessageInput",
+      messageBase: {
+        $typeName: "proto.ProtoMessageBase",
+        payloadType: "controllerInput",
+      } as ProtoMessageBase,
+      data: detachMsg,
+    };
+    this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message));
   }
 
-  private controllerButtonToVirtualKeyCode(code: number): number | undefined {
+  private controllerButtonToVirtualKeyCode(code: number) {
     return controllerButtonToLinuxEventCode[code] || undefined;
   }
 
-  private rumbleCallback(rumbleMsg: ProtoControllerRumble) {
-    if (!this.connected || !this.gamepad) return;
+  private rumbleCallback(data: ArrayBuffer) {
+    // If not connected, ignore
+    if (!this.connected) return;
+    try {
+      // First decode the wrapper message
+      const uint8Data = new Uint8Array(data);
+      const messageWrapper = fromBinary(ProtoMessageInputSchema, uint8Data);
 
-    // Check if this rumble is for us
-    if (
-      rumbleMsg.sessionId !== this.wrtc.getSessionID() ||
-      rumbleMsg.sessionSlot !== this.gamepad.index
-    )
-      return;
+      // Check if it contains controller rumble data
+      if (messageWrapper.data?.inputType?.case === "controllerRumble") {
+        const rumbleMsg = messageWrapper.data.inputType.value as ProtoControllerRumble;
+
+        // Check if aimed at this controller slot
+        if (rumbleMsg.slot !== this.slot) return;
 
         // Trigger actual rumble
         // Need to remap from 0-65535 to 0.0-1.0 ranges
         const clampedLowFreq = Math.max(0, Math.min(65535, rumbleMsg.lowFrequency));
-    const rumbleLowFreq = this.remapFromTo(clampedLowFreq, 0, 65535, 0.0, 1.0);
-    const clampedHighFreq = Math.max(
+        const rumbleLowFreq = this.remapFromTo(
+          clampedLowFreq,
           0,
-      Math.min(65535, rumbleMsg.highFrequency),
+          65535,
+          0.0,
+          1.0,
         );
+        const clampedHighFreq = Math.max(0, Math.min(65535, rumbleMsg.highFrequency));
         const rumbleHighFreq = this.remapFromTo(
           clampedHighFreq,
           0,
@@ -518,14 +494,16 @@ export class Controller {
         // Cap to valid range (max 5000)
         const rumbleDuration = Math.max(0, Math.min(5000, rumbleMsg.duration));
         if (this.gamepad.vibrationActuator) {
-      this.gamepad.vibrationActuator
-        .playEffect("dual-rumble", {
+          this.gamepad.vibrationActuator.playEffect("dual-rumble", {
             startDelay: 0,
             duration: rumbleDuration,
             weakMagnitude: rumbleLowFreq,
             strongMagnitude: rumbleHighFreq,
-        })
-        .catch(console.error);
+          }).catch(console.error);
+        }
+      }
+    } catch (error) {
+      console.error("Failed to decode rumble message:", error);
     }
   }
 }

packages/input/src/keyboard.ts

@@ -1,9 +1,16 @@
-import { keyCodeToLinuxEventCode } from "./codes";
+import {keyCodeToLinuxEventCode} from "./codes"
 import {WebRTCStream} from "./webrtc-stream";
-import { ProtoKeyDownSchema, ProtoKeyUpSchema } from "./proto/types_pb";
+import {LatencyTracker} from "./latency";
+import {ProtoLatencyTracker, ProtoTimestampEntry} from "./proto/latency_tracker_pb";
+import {timestampFromDate} from "@bufbuild/protobuf/wkt";
+import {ProtoMessageBase, ProtoMessageInput, ProtoMessageInputSchema} from "./proto/messages_pb";
+import {
+  ProtoInput,
+  ProtoInputSchema,
+  ProtoKeyDownSchema,
+  ProtoKeyUpSchema,
+} from "./proto/types_pb";
 import {create, toBinary} from "@bufbuild/protobuf";
-import { createMessage } from "./utils";
-import { ProtoMessageSchema } from "./proto/messages_pb";
 
 interface Props {
   webrtc: WebRTCStream;
@@ -19,25 +26,36 @@ export class Keyboard {
 
   constructor({webrtc}: Props) {
     this.wrtc = webrtc;
-    this.keydownListener = this.createKeyboardListener((e: any) =>
-      create(ProtoKeyDownSchema, {
-        key: this.keyToVirtualKeyCode(e.code),
+    this.keydownListener = this.createKeyboardListener((e: any) => create(ProtoInputSchema, {
+      $typeName: "proto.ProtoInput",
+      inputType: {
+        case: "keyDown",
+        value: create(ProtoKeyDownSchema, {
+          type: "KeyDown",
+          key: this.keyToVirtualKeyCode(e.code)
         }),
-    );
-    this.keyupListener = this.createKeyboardListener((e: any) =>
-      create(ProtoKeyUpSchema, {
-        key: this.keyToVirtualKeyCode(e.code),
+      }
+    }));
+    this.keyupListener = this.createKeyboardListener((e: any) => create(ProtoInputSchema, {
+      $typeName: "proto.ProtoInput",
+      inputType: {
+        case: "keyUp",
+        value: create(ProtoKeyUpSchema, {
+          type: "KeyUp",
+          key: this.keyToVirtualKeyCode(e.code)
         }),
-    );
-    this.run();
+      }
+    }));
+    this.run()
   }
 
   private run() {
-    if (this.connected) this.stop();
+    if (this.connected)
+      this.stop()
 
-    this.connected = true;
-    document.addEventListener("keydown", this.keydownListener);
-    document.addEventListener("keyup", this.keyupListener);
+    this.connected = true
+    document.addEventListener("keydown", this.keydownListener, {passive: false});
+    document.addEventListener("keyup", this.keyupListener, {passive: false});
   }
 
   private stop() {
@@ -47,19 +65,42 @@ export class Keyboard {
   }
 
   // Helper function to create and return mouse listeners
-  private createKeyboardListener(
-    dataCreator: (e: Event) => any,
-  ): (e: Event) => void {
+  private createKeyboardListener(dataCreator: (e: Event) => ProtoInput): (e: Event) => void {
     return (e: Event) => {
       e.preventDefault();
       e.stopPropagation();
       // Prevent repeated key events from being sent (important for games)
-      if ((e as any).repeat) return;
+      if ((e as any).repeat)
+        return;
 
       const data = dataCreator(e as any);
 
-      const message = createMessage(data, "input");
-      this.wrtc.sendBinary(toBinary(ProtoMessageSchema, message));
+      // Latency tracking
+      const tracker = new LatencyTracker("input-keyboard");
+      tracker.addTimestamp("client_send");
+      const protoTracker: ProtoLatencyTracker = {
+        $typeName: "proto.ProtoLatencyTracker",
+        sequenceId: tracker.sequence_id,
+        timestamps: [],
+      };
+      for (const t of tracker.timestamps) {
+        protoTracker.timestamps.push({
+          $typeName: "proto.ProtoTimestampEntry",
+          stage: t.stage,
+          time: timestampFromDate(t.time),
+        } as ProtoTimestampEntry);
+      }
+
+      const message: ProtoMessageInput = {
+        $typeName: "proto.ProtoMessageInput",
+        messageBase: {
+          $typeName: "proto.ProtoMessageBase",
+          payloadType: "input",
+          latency: protoTracker,
+        } as ProtoMessageBase,
+        data: data,
+      };
+      this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message));
     };
   }
 
@@ -71,6 +112,6 @@ export class Keyboard {
   private keyToVirtualKeyCode(code: string) {
     // Treat Home key as Escape - TODO: Make user-configurable
     if (code === "Home") return 1;
-    return keyCodeToLinuxEventCode[code] || 0;
+    return keyCodeToLinuxEventCode[code] || undefined;
   }
 }

packages/input/src/messages.ts

@@ -0,0 +1,305 @@
+import { LatencyTracker } from "./latency";
+import { Uint8ArrayList } from "uint8arraylist";
+import { allocUnsafe } from "uint8arrays/alloc";
+import { pipe } from "it-pipe";
+import { decode, encode } from "it-length-prefixed";
+import { Stream } from "@libp2p/interface";
+
+export interface MessageBase {
+  payload_type: string;
+  latency?: LatencyTracker;
+}
+
+export interface MessageRaw extends MessageBase {
+  data: any;
+}
+
+export function NewMessageRaw(type: string, data: any): Uint8Array {
+  const msg = {
+    payload_type: type,
+    data: data,
+  };
+  return new TextEncoder().encode(JSON.stringify(msg));
+}
+
+export interface MessageICE extends MessageBase {
+  candidate: RTCIceCandidateInit;
+}
+
+export function NewMessageICE(
+  type: string,
+  candidate: RTCIceCandidateInit,
+): Uint8Array {
+  const msg = {
+    payload_type: type,
+    candidate: candidate,
+  };
+  return new TextEncoder().encode(JSON.stringify(msg));
+}
+
+export interface MessageSDP extends MessageBase {
+  sdp: RTCSessionDescriptionInit;
+}
+
+export function NewMessageSDP(
+  type: string,
+  sdp: RTCSessionDescriptionInit,
+): Uint8Array {
+  const msg = {
+    payload_type: type,
+    sdp: sdp,
+  };
+  return new TextEncoder().encode(JSON.stringify(msg));
+}
+
+const MAX_SIZE = 1024 * 1024; // 1MB
+const MAX_QUEUE_SIZE = 1000; // Maximum number of messages in the queue
+
+// Custom 4-byte length encoder
+export const length4ByteEncoder = (length: number) => {
+  const buf = allocUnsafe(4);
+
+  // Write the length as a 32-bit unsigned integer (4 bytes)
+  buf[0] = length >>> 24;
+  buf[1] = (length >>> 16) & 0xff;
+  buf[2] = (length >>> 8) & 0xff;
+  buf[3] = length & 0xff;
+
+  // Set the bytes property to 4
+  length4ByteEncoder.bytes = 4;
+
+  return buf;
+};
+length4ByteEncoder.bytes = 4;
+
+// Custom 4-byte length decoder
+export const length4ByteDecoder = (data: Uint8ArrayList) => {
+  if (data.byteLength < 4) {
+    // Not enough bytes to read the length
+    return -1;
+  }
+
+  // Read the length from the first 4 bytes
+  let length = 0;
+  length =
+    (data.subarray(0, 1)[0] >>> 0) * 0x1000000 +
+    (data.subarray(1, 2)[0] >>> 0) * 0x10000 +
+    (data.subarray(2, 3)[0] >>> 0) * 0x100 +
+    (data.subarray(3, 4)[0] >>> 0);
+
+  // Set bytes read to 4
+  length4ByteDecoder.bytes = 4;
+
+  return length;
+};
+length4ByteDecoder.bytes = 4;
+
+interface PromiseMessage {
+  data: Uint8Array;
+  resolve: () => void;
+  reject: (error: Error) => void;
+}
+
+export class SafeStream {
+  private stream: Stream;
+  private callbacks: Map<string, ((data: any) => void)[]> = new Map();
+  private isReading: boolean = false;
+  private isWriting: boolean = false;
+  private closed: boolean = false;
+  private messageQueue: PromiseMessage[] = [];
+  private writeLock = false;
+  private readRetries = 0;
+  private writeRetries = 0;
+  private readonly MAX_RETRIES = 5;
+
+  constructor(stream: Stream) {
+    this.stream = stream;
+    this.startReading();
+    this.startWriting();
+  }
+
+  private async startReading(): Promise<void> {
+    if (this.isReading || this.closed) return;
+
+    this.isReading = true;
+
+    try {
+      const source = this.stream.source;
+      const decodedSource = decode(source, {
+        maxDataLength: MAX_SIZE,
+        lengthDecoder: length4ByteDecoder,
+      });
+
+      for await (const chunk of decodedSource) {
+        if (this.closed) break;
+
+        this.readRetries = 0;
+
+        try {
+          const data = chunk.slice();
+          const message = JSON.parse(
+            new TextDecoder().decode(data),
+          ) as MessageBase;
+          const msgType = message.payload_type;
+
+          if (this.callbacks.has(msgType)) {
+            const handlers = this.callbacks.get(msgType)!;
+            for (const handler of handlers) {
+              try {
+                handler(message);
+              } catch (err) {
+                console.error(`Error in message handler for ${msgType}:`, err);
+              }
+            }
+          }
+        } catch (err) {
+          console.error("Error processing message:", err);
+        }
+      }
+    } catch (err) {
+      console.error("Stream reading error:", err);
+    } finally {
+      this.isReading = false;
+      this.readRetries++;
+
+      // If not closed, try to restart reading
+      if (!this.closed && this.readRetries < this.MAX_RETRIES)
+        setTimeout(() => this.startReading(), 100);
+      else if (this.readRetries >= this.MAX_RETRIES)
+        console.error(
+          "Max retries reached for reading stream, stopping attempts",
+        );
+    }
+  }
+
+  public registerCallback(
+    msgType: string,
+    callback: (data: any) => void,
+  ): void {
+    if (!this.callbacks.has(msgType)) {
+      this.callbacks.set(msgType, []);
+    }
+
+    this.callbacks.get(msgType)!.push(callback);
+  }
+
+  public removeCallback(msgType: string, callback: (data: any) => void): void {
+    if (this.callbacks.has(msgType)) {
+      const callbacks = this.callbacks.get(msgType)!;
+      const index = callbacks.indexOf(callback);
+
+      if (index !== -1) {
+        callbacks.splice(index, 1);
+      }
+
+      if (callbacks.length === 0) {
+        this.callbacks.delete(msgType);
+      }
+    }
+  }
+
+  private async startWriting(): Promise<void> {
+    if (this.isWriting || this.closed) return;
+
+    this.isWriting = true;
+
+    try {
+      // Create an async generator for real-time message processing
+      const messageSource = async function* (this: SafeStream) {
+        while (!this.closed) {
+          // Check if we have messages to send
+          if (this.messageQueue.length > 0) {
+            this.writeLock = true;
+
+            try {
+              const message = this.messageQueue[0];
+
+              // Encode the message
+              const encoded = encode([message.data], {
+                maxDataLength: MAX_SIZE,
+                lengthEncoder: length4ByteEncoder,
+              });
+
+              for await (const chunk of encoded) {
+                yield chunk;
+              }
+
+              // Remove message after successful sending
+              this.writeRetries = 0;
+              const sentMessage = this.messageQueue.shift();
+              if (sentMessage)
+                sentMessage.resolve();
+            } catch (err) {
+              console.error("Error encoding or sending message:", err);
+              const failedMessage = this.messageQueue.shift();
+              if (failedMessage)
+                failedMessage.reject(new Error(`Failed to send message: ${err}`));
+            } finally {
+              this.writeLock = false;
+            }
+          } else {
+            // No messages to send, wait for a short period
+            await new Promise((resolve) => setTimeout(resolve, 100));
+          }
+        }
+      }.bind(this);
+
+      await pipe(messageSource(), this.stream.sink).catch((err) => {
+        console.error("Sink error:", err);
+        this.isWriting = false;
+        this.writeRetries++;
+
+        // Try to restart if not closed
+        if (!this.closed && this.writeRetries < this.MAX_RETRIES) {
+          setTimeout(() => this.startWriting(), 1000);
+        } else if (this.writeRetries >= this.MAX_RETRIES) {
+          console.error("Max retries reached for writing to stream sink, stopping attempts");
+        }
+      });
+    } catch (err) {
+      console.error("Stream writing error:", err);
+      this.isWriting = false;
+      this.writeRetries++;
+
+      // Try to restart if not closed
+      if (!this.closed && this.writeRetries < this.MAX_RETRIES) {
+        setTimeout(() => this.startWriting(), 1000);
+      } else if (this.writeRetries >= this.MAX_RETRIES) {
+        console.error("Max retries reached for writing stream, stopping attempts");
+      }
+    }
+  }
+
+  public async writeMessage(message: Uint8Array): Promise<void> {
+    if (this.closed) {
+      throw new Error("Cannot write to closed stream");
+    }
+
+    // Validate message size before queuing
+    if (message.length > MAX_SIZE) {
+      throw new Error("Message size exceeds maximum size limit");
+    }
+
+    // Check if the message queue is too large
+    if (this.messageQueue.length >= MAX_QUEUE_SIZE) {
+      throw new Error("Message queue is full, cannot write message");
+    }
+
+    // Create a promise to resolve when the message is sent
+    return new Promise((resolve, reject) => {
+      this.messageQueue.push({ data: message, resolve, reject } as PromiseMessage);
+    });
+  }
+
+  public close(): void {
+    this.closed = true;
+    this.callbacks.clear();
+    // Reject pending messages
+    for (const msg of this.messageQueue)
+      msg.reject(new Error("Stream closed"));
+
+    this.messageQueue = [];
+    this.readRetries = 0;
+    this.writeRetries = 0;
+  }
+}

packages/input/src/mouse.ts

@@ -1,14 +1,18 @@
 import {WebRTCStream} from "./webrtc-stream";
+import {LatencyTracker} from "./latency";
+import {ProtoMessageInput, ProtoMessageBase, ProtoMessageInputSchema} from "./proto/messages_pb";
 import {
-  ProtoMouseKeyDownSchema,
-  ProtoMouseKeyUpSchema,
+  ProtoInput, ProtoInputSchema,
+  ProtoMouseKeyDown, ProtoMouseKeyDownSchema,
+  ProtoMouseKeyUp, ProtoMouseKeyUpSchema,
+  ProtoMouseMove,
   ProtoMouseMoveSchema,
-  ProtoMouseWheelSchema,
+  ProtoMouseWheel, ProtoMouseWheelSchema
 } from "./proto/types_pb";
 import {mouseButtonToLinuxEventCode} from "./codes";
+import {ProtoLatencyTracker, ProtoTimestampEntry} from "./proto/latency_tracker_pb";
 import {create, toBinary} from "@bufbuild/protobuf";
-import { createMessage } from "./utils";
-import { ProtoMessageSchema } from "./proto/messages_pb";
+import {timestampFromDate} from "@bufbuild/protobuf/wkt";
 
 interface Props {
   webrtc: WebRTCStream;
@@ -20,7 +24,7 @@ export class Mouse {
   protected canvas: HTMLCanvasElement;
   protected connected!: boolean;
 
-  private sendInterval = 10; // 100 updates per second
+  private sendInterval = 10 // 100 updates per second
 
   // Store references to event listeners
   private readonly mousemoveListener: (e: MouseEvent) => void;
@@ -35,6 +39,8 @@ export class Mouse {
     this.wrtc = webrtc;
     this.canvas = canvas;
 
+    this.sendInterval = 1000 / webrtc.currentFrameRate;
+
     this.mousemoveListener = (e: MouseEvent) => {
       e.preventDefault();
       e.stopPropagation();
@@ -42,48 +48,65 @@ export class Mouse {
       this.movementY += e.movementY;
     };
 
-    this.mousedownListener = this.createMouseListener((e: any) =>
-      create(ProtoMouseKeyDownSchema, {
-        key: this.keyToVirtualKeyCode(e.button),
+    this.mousedownListener = this.createMouseListener((e: any) => create(ProtoInputSchema, {
+      $typeName: "proto.ProtoInput",
+      inputType: {
+        case: "mouseKeyDown",
+        value: create(ProtoMouseKeyDownSchema, {
+          type: "MouseKeyDown",
+          key: this.keyToVirtualKeyCode(e.button)
         }),
-    );
-    this.mouseupListener = this.createMouseListener((e: any) =>
-      create(ProtoMouseKeyUpSchema, {
-        key: this.keyToVirtualKeyCode(e.button),
+      }
+    }));
+    this.mouseupListener = this.createMouseListener((e: any) => create(ProtoInputSchema, {
+      $typeName: "proto.ProtoInput",
+      inputType: {
+        case: "mouseKeyUp",
+        value: create(ProtoMouseKeyUpSchema, {
+          type: "MouseKeyUp",
+          key: this.keyToVirtualKeyCode(e.button)
         }),
-    );
-    this.mousewheelListener = this.createMouseListener((e: any) =>
-      create(ProtoMouseWheelSchema, {
+      }
+    }));
+    this.mousewheelListener = this.createMouseListener((e: any) => create(ProtoInputSchema, {
+      $typeName: "proto.ProtoInput",
+      inputType: {
+        case: "mouseWheel",
+        value: create(ProtoMouseWheelSchema, {
+          type: "MouseWheel",
           x: Math.round(e.deltaX),
           y: Math.round(e.deltaY),
         }),
-    );
+      }
+    }));
 
-    this.run();
+    this.run()
     this.startProcessing();
   }
 
   private run() {
     //calls all the other functions
     if (!document.pointerLockElement) {
-      console.log("no pointerlock");
+      console.log("no pointerlock")
       if (this.connected) {
-        this.stop();
+        this.stop()
       }
       return;
     }
 
     if (document.pointerLockElement == this.canvas) {
-      this.connected = true;
-      this.canvas.addEventListener("mousemove", this.mousemoveListener);
-      this.canvas.addEventListener("mousedown", this.mousedownListener);
-      this.canvas.addEventListener("mouseup", this.mouseupListener);
-      this.canvas.addEventListener("wheel", this.mousewheelListener);
+      this.connected = true
+      this.canvas.addEventListener("mousemove", this.mousemoveListener, {passive: false});
+      this.canvas.addEventListener("mousedown", this.mousedownListener, {passive: false});
+      this.canvas.addEventListener("mouseup", this.mouseupListener, {passive: false});
+      this.canvas.addEventListener("wheel", this.mousewheelListener, {passive: false});
+
     } else {
       if (this.connected) {
-        this.stop();
+        this.stop()
       }
     }
+
   }
 
   private stop() {
@@ -96,7 +119,7 @@ export class Mouse {
 
   private startProcessing() {
     setInterval(() => {
-      if (this.connected) {
+      if (this.connected && (this.movementX !== 0 || this.movementY !== 0)) {
         this.sendAggregatedMouseMove();
         this.movementX = 0;
         this.movementY = 0;
@@ -105,26 +128,79 @@ export class Mouse {
   }
 
   private sendAggregatedMouseMove() {
-    const data = create(ProtoMouseMoveSchema, {
+    const data = create(ProtoInputSchema, {
+      $typeName: "proto.ProtoInput",
+      inputType: {
+        case: "mouseMove",
+        value: create(ProtoMouseMoveSchema, {
+          type: "MouseMove",
           x: Math.round(this.movementX),
           y: Math.round(this.movementY),
+        }),
+      },
     });
 
-    const message = createMessage(data, "input");
-    this.wrtc.sendBinary(toBinary(ProtoMessageSchema, message));
+    // Latency tracking
+    const tracker = new LatencyTracker("input-mouse");
+    tracker.addTimestamp("client_send");
+    const protoTracker: ProtoLatencyTracker = {
+      $typeName: "proto.ProtoLatencyTracker",
+      sequenceId: tracker.sequence_id,
+      timestamps: [],
+    };
+    for (const t of tracker.timestamps) {
+      protoTracker.timestamps.push({
+        $typeName: "proto.ProtoTimestampEntry",
+        stage: t.stage,
+        time: timestampFromDate(t.time),
+      } as ProtoTimestampEntry);
+    }
+
+    const message: ProtoMessageInput = {
+      $typeName: "proto.ProtoMessageInput",
+      messageBase: {
+        $typeName: "proto.ProtoMessageBase",
+        payloadType: "input",
+        latency: protoTracker,
+      } as ProtoMessageBase,
+      data: data,
+    };
+    this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message));
   }
 
   // Helper function to create and return mouse listeners
-  private createMouseListener(
-    dataCreator: (e: Event) => any,
-  ): (e: Event) => void {
+  private createMouseListener(dataCreator: (e: Event) => ProtoInput): (e: Event) => void {
     return (e: Event) => {
       e.preventDefault();
       e.stopPropagation();
       const data = dataCreator(e as any);
 
-      const message = createMessage(data, "input");
-      this.wrtc.sendBinary(toBinary(ProtoMessageSchema, message));
+      // Latency tracking
+      const tracker = new LatencyTracker("input-mouse");
+      tracker.addTimestamp("client_send");
+      const protoTracker: ProtoLatencyTracker = {
+        $typeName: "proto.ProtoLatencyTracker",
+        sequenceId: tracker.sequence_id,
+        timestamps: [],
+      };
+      for (const t of tracker.timestamps) {
+        protoTracker.timestamps.push({
+          $typeName: "proto.ProtoTimestampEntry",
+          stage: t.stage,
+          time: timestampFromDate(t.time),
+        } as ProtoTimestampEntry);
+      }
+
+      const message: ProtoMessageInput = {
+        $typeName: "proto.ProtoMessageInput",
+        messageBase: {
+          $typeName: "proto.ProtoMessageBase",
+          payloadType: "input",
+          latency: protoTracker,
+        } as ProtoMessageBase,
+        data: data,
+      };
+      this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message));
     };
   }
 

packages/input/src/proto/latency_tracker_pb.ts

@@ -1,4 +1,4 @@
-// @generated by protoc-gen-es v2.10.1 with parameter "target=ts"
+// @generated by protoc-gen-es v2.9.0 with parameter "target=ts"
 // @generated from file latency_tracker.proto (package proto, syntax proto3)
 /* eslint-disable */
 

packages/input/src/proto/messages_pb.ts

@@ -1,10 +1,10 @@
-// @generated by protoc-gen-es v2.10.1 with parameter "target=ts"
+// @generated by protoc-gen-es v2.9.0 with parameter "target=ts"
 // @generated from file messages.proto (package proto, syntax proto3)
 /* eslint-disable */
 
 import type { GenFile, GenMessage } from "@bufbuild/protobuf/codegenv2";
 import { fileDesc, messageDesc } from "@bufbuild/protobuf/codegenv2";
-import type { ProtoClientDisconnected, ProtoClientRequestRoomStream, ProtoControllerAttach, ProtoControllerDetach, ProtoControllerRumble, ProtoControllerStateBatch, ProtoICE, ProtoKeyDown, ProtoKeyUp, ProtoMouseKeyDown, ProtoMouseKeyUp, ProtoMouseMove, ProtoMouseMoveAbs, ProtoMouseWheel, ProtoRaw, ProtoSDP, ProtoServerPushStream } from "./types_pb";
+import type { ProtoInput } from "./types_pb";
 import { file_types } from "./types_pb";
 import type { ProtoLatencyTracker } from "./latency_tracker_pb";
 import { file_latency_tracker } from "./latency_tracker_pb";
@@ -14,7 +14,7 @@ import type { Message } from "@bufbuild/protobuf";
  * Describes the file messages.proto.
  */
 export const file_messages: GenFile = /*@__PURE__*/
-  fileDesc("Cg5tZXNzYWdlcy5wcm90bxIFcHJvdG8iVQoQUHJvdG9NZXNzYWdlQmFzZRIUCgxwYXlsb2FkX3R5cGUYASABKAkSKwoHbGF0ZW5jeRgCIAEoCzIaLnByb3RvLlByb3RvTGF0ZW5jeVRyYWNrZXIipQcKDFByb3RvTWVzc2FnZRItCgxtZXNzYWdlX2Jhc2UYASABKAsyFy5wcm90by5Qcm90b01lc3NhZ2VCYXNlEisKCm1vdXNlX21vdmUYAiABKAsyFS5wcm90by5Qcm90b01vdXNlTW92ZUgAEjIKDm1vdXNlX21vdmVfYWJzGAMgASgLMhgucHJvdG8uUHJvdG9Nb3VzZU1vdmVBYnNIABItCgttb3VzZV93aGVlbBgEIAEoCzIWLnByb3RvLlByb3RvTW91c2VXaGVlbEgAEjIKDm1vdXNlX2tleV9kb3duGAUgASgLMhgucHJvdG8uUHJvdG9Nb3VzZUtleURvd25IABIuCgxtb3VzZV9rZXlfdXAYBiABKAsyFi5wcm90by5Qcm90b01vdXNlS2V5VXBIABInCghrZXlfZG93bhgHIAEoCzITLnByb3RvLlByb3RvS2V5RG93bkgAEiMKBmtleV91cBgIIAEoCzIRLnByb3RvLlByb3RvS2V5VXBIABI5ChFjb250cm9sbGVyX2F0dGFjaBgJIAEoCzIcLnByb3RvLlByb3RvQ29udHJvbGxlckF0dGFjaEgAEjkKEWNvbnRyb2xsZXJfZGV0YWNoGAogASgLMhwucHJvdG8uUHJvdG9Db250cm9sbGVyRGV0YWNoSAASOQoRY29udHJvbGxlcl9ydW1ibGUYCyABKAsyHC5wcm90by5Qcm90b0NvbnRyb2xsZXJSdW1ibGVIABJCChZjb250cm9sbGVyX3N0YXRlX2JhdGNoGAwgASgLMiAucHJvdG8uUHJvdG9Db250cm9sbGVyU3RhdGVCYXRjaEgAEh4KA2ljZRgUIAEoCzIPLnByb3RvLlByb3RvSUNFSAASHgoDc2RwGBUgASgLMg8ucHJvdG8uUHJvdG9TRFBIABIeCgNyYXcYFiABKAsyDy5wcm90by5Qcm90b1Jhd0gAEkkKGmNsaWVudF9yZXF1ZXN0X3Jvb21fc3RyZWFtGBcgASgLMiMucHJvdG8uUHJvdG9DbGllbnRSZXF1ZXN0Um9vbVN0cmVhbUgAEj0KE2NsaWVudF9kaXNjb25uZWN0ZWQYGCABKAsyHi5wcm90by5Qcm90b0NsaWVudERpc2Nvbm5lY3RlZEgAEjoKEnNlcnZlcl9wdXNoX3N0cmVhbRgZIAEoCzIcLnByb3RvLlByb3RvU2VydmVyUHVzaFN0cmVhbUgAQgkKB3BheWxvYWRCFloUcmVsYXkvaW50ZXJuYWwvcHJvdG9iBnByb3RvMw", [file_types, file_latency_tracker]);
+  fileDesc("Cg5tZXNzYWdlcy5wcm90bxIFcHJvdG8iVQoQUHJvdG9NZXNzYWdlQmFzZRIUCgxwYXlsb2FkX3R5cGUYASABKAkSKwoHbGF0ZW5jeRgCIAEoCzIaLnByb3RvLlByb3RvTGF0ZW5jeVRyYWNrZXIiYwoRUHJvdG9NZXNzYWdlSW5wdXQSLQoMbWVzc2FnZV9iYXNlGAEgASgLMhcucHJvdG8uUHJvdG9NZXNzYWdlQmFzZRIfCgRkYXRhGAIgASgLMhEucHJvdG8uUHJvdG9JbnB1dEIWWhRyZWxheS9pbnRlcm5hbC9wcm90b2IGcHJvdG8z", [file_types, file_latency_tracker]);
 
 /**
  * @generated from message proto.ProtoMessageBase
@@ -39,134 +39,24 @@ export const ProtoMessageBaseSchema: GenMessage<ProtoMessageBase> = /*@__PURE__*
   messageDesc(file_messages, 0);
 
 /**
- * @generated from message proto.ProtoMessage
+ * @generated from message proto.ProtoMessageInput
  */
-export type ProtoMessage = Message<"proto.ProtoMessage"> & {
+export type ProtoMessageInput = Message<"proto.ProtoMessageInput"> & {
   /**
    * @generated from field: proto.ProtoMessageBase message_base = 1;
    */
   messageBase?: ProtoMessageBase;
 
   /**
-   * @generated from oneof proto.ProtoMessage.payload
+   * @generated from field: proto.ProtoInput data = 2;
    */
-  payload: {
-    /**
-     * Input types
-     *
-     * @generated from field: proto.ProtoMouseMove mouse_move = 2;
-     */
-    value: ProtoMouseMove;
-    case: "mouseMove";
-  } | {
-    /**
-     * @generated from field: proto.ProtoMouseMoveAbs mouse_move_abs = 3;
-     */
-    value: ProtoMouseMoveAbs;
-    case: "mouseMoveAbs";
-  } | {
-    /**
-     * @generated from field: proto.ProtoMouseWheel mouse_wheel = 4;
-     */
-    value: ProtoMouseWheel;
-    case: "mouseWheel";
-  } | {
-    /**
-     * @generated from field: proto.ProtoMouseKeyDown mouse_key_down = 5;
-     */
-    value: ProtoMouseKeyDown;
-    case: "mouseKeyDown";
-  } | {
-    /**
-     * @generated from field: proto.ProtoMouseKeyUp mouse_key_up = 6;
-     */
-    value: ProtoMouseKeyUp;
-    case: "mouseKeyUp";
-  } | {
-    /**
-     * @generated from field: proto.ProtoKeyDown key_down = 7;
-     */
-    value: ProtoKeyDown;
-    case: "keyDown";
-  } | {
-    /**
-     * ProtoClipboard clipboard = 9;
-     *
-     * @generated from field: proto.ProtoKeyUp key_up = 8;
-     */
-    value: ProtoKeyUp;
-    case: "keyUp";
-  } | {
-    /**
-     * Controller input types
-     *
-     * @generated from field: proto.ProtoControllerAttach controller_attach = 9;
-     */
-    value: ProtoControllerAttach;
-    case: "controllerAttach";
-  } | {
-    /**
-     * @generated from field: proto.ProtoControllerDetach controller_detach = 10;
-     */
-    value: ProtoControllerDetach;
-    case: "controllerDetach";
-  } | {
-    /**
-     * @generated from field: proto.ProtoControllerRumble controller_rumble = 11;
-     */
-    value: ProtoControllerRumble;
-    case: "controllerRumble";
-  } | {
-    /**
-     * @generated from field: proto.ProtoControllerStateBatch controller_state_batch = 12;
-     */
-    value: ProtoControllerStateBatch;
-    case: "controllerStateBatch";
-  } | {
-    /**
-     * Signaling types
-     *
-     * @generated from field: proto.ProtoICE ice = 20;
-     */
-    value: ProtoICE;
-    case: "ice";
-  } | {
-    /**
-     * @generated from field: proto.ProtoSDP sdp = 21;
-     */
-    value: ProtoSDP;
-    case: "sdp";
-  } | {
-    /**
-     * @generated from field: proto.ProtoRaw raw = 22;
-     */
-    value: ProtoRaw;
-    case: "raw";
-  } | {
-    /**
-     * @generated from field: proto.ProtoClientRequestRoomStream client_request_room_stream = 23;
-     */
-    value: ProtoClientRequestRoomStream;
-    case: "clientRequestRoomStream";
-  } | {
-    /**
-     * @generated from field: proto.ProtoClientDisconnected client_disconnected = 24;
-     */
-    value: ProtoClientDisconnected;
-    case: "clientDisconnected";
-  } | {
-    /**
-     * @generated from field: proto.ProtoServerPushStream server_push_stream = 25;
-     */
-    value: ProtoServerPushStream;
-    case: "serverPushStream";
-  } | { case: undefined; value?: undefined };
+  data?: ProtoInput;
 };
 
 /**
- * Describes the message proto.ProtoMessage.
- * Use `create(ProtoMessageSchema)` to create a new message.
+ * Describes the message proto.ProtoMessageInput.
+ * Use `create(ProtoMessageInputSchema)` to create a new message.
  */
-export const ProtoMessageSchema: GenMessage<ProtoMessage> = /*@__PURE__*/
+export const ProtoMessageInputSchema: GenMessage<ProtoMessageInput> = /*@__PURE__*/
   messageDesc(file_messages, 1);
 

packages/input/src/proto/types_pb.ts

@@ -1,16 +1,16 @@
-// @generated by protoc-gen-es v2.10.1 with parameter "target=ts"
+// @generated by protoc-gen-es v2.9.0 with parameter "target=ts"
 // @generated from file types.proto (package proto, syntax proto3)
 /* eslint-disable */
 
-import type { GenEnum, GenFile, GenMessage } from "@bufbuild/protobuf/codegenv2";
-import { enumDesc, fileDesc, messageDesc } from "@bufbuild/protobuf/codegenv2";
+import type { GenFile, GenMessage } from "@bufbuild/protobuf/codegenv2";
+import { fileDesc, messageDesc } from "@bufbuild/protobuf/codegenv2";
 import type { Message } from "@bufbuild/protobuf";
 
 /**
  * Describes the file types.proto.
  */
 export const file_types: GenFile = /*@__PURE__*/
-  fileDesc("Cgt0eXBlcy5wcm90bxIFcHJvdG8iJgoOUHJvdG9Nb3VzZU1vdmUSCQoBeBgBIAEoBRIJCgF5GAIgASgFIikKEVByb3RvTW91c2VNb3ZlQWJzEgkKAXgYASABKAUSCQoBeRgCIAEoBSInCg9Qcm90b01vdXNlV2hlZWwSCQoBeBgBIAEoBRIJCgF5GAIgASgFIiAKEVByb3RvTW91c2VLZXlEb3duEgsKA2tleRgBIAEoBSIeCg9Qcm90b01vdXNlS2V5VXASCwoDa2V5GAEgASgFIhsKDFByb3RvS2V5RG93bhILCgNrZXkYASABKAUiGQoKUHJvdG9LZXlVcBILCgNrZXkYASABKAUiTQoVUHJvdG9Db250cm9sbGVyQXR0YWNoEgoKAmlkGAEgASgJEhQKDHNlc3Npb25fc2xvdBgCIAEoBRISCgpzZXNzaW9uX2lkGAMgASgJIkEKFVByb3RvQ29udHJvbGxlckRldGFjaBIUCgxzZXNzaW9uX3Nsb3QYASABKAUSEgoKc2Vzc2lvbl9pZBgCIAEoCSKCAQoVUHJvdG9Db250cm9sbGVyUnVtYmxlEhQKDHNlc3Npb25fc2xvdBgBIAEoBRISCgpzZXNzaW9uX2lkGAIgASgJEhUKDWxvd19mcmVxdWVuY3kYAyABKAUSFgoOaGlnaF9mcmVxdWVuY3kYBCABKAUSEAoIZHVyYXRpb24YBSABKAUi0AUKGVByb3RvQ29udHJvbGxlclN0YXRlQmF0Y2gSFAoMc2Vzc2lvbl9zbG90GAEgASgFEhIKCnNlc3Npb25faWQYAiABKAkSQAoLdXBkYXRlX3R5cGUYAyABKA4yKy5wcm90by5Qcm90b0NvbnRyb2xsZXJTdGF0ZUJhdGNoLlVwZGF0ZVR5cGUSEAoIc2VxdWVuY2UYBCABKA0SVAoTYnV0dG9uX2NoYW5nZWRfbWFzaxgFIAMoCzI3LnByb3RvLlByb3RvQ29udHJvbGxlclN0YXRlQmF0Y2guQnV0dG9uQ2hhbmdlZE1hc2tFbnRyeRIZCgxsZWZ0X3N0aWNrX3gYBiABKAVIAIgBARIZCgxsZWZ0X3N0aWNrX3kYByABKAVIAYgBARIaCg1yaWdodF9zdGlja194GAggASgFSAKIAQESGgoNcmlnaHRfc3RpY2tfeRgJIAEoBUgDiAEBEhkKDGxlZnRfdHJpZ2dlchgKIAEoBUgEiAEBEhoKDXJpZ2h0X3RyaWdnZXIYCyABKAVIBYgBARITCgZkcGFkX3gYDCABKAVIBogBARITCgZkcGFkX3kYDSABKAVIB4gBARIbCg5jaGFuZ2VkX2ZpZWxkcxgOIAEoDUgIiAEBGjgKFkJ1dHRvbkNoYW5nZWRNYXNrRW50cnkSCwoDa2V5GAEgASgFEg0KBXZhbHVlGAIgASgIOgI4ASInCgpVcGRhdGVUeXBlEg4KCkZVTExfU1RBVEUQABIJCgVERUxUQRABQg8KDV9sZWZ0X3N0aWNrX3hCDwoNX2xlZnRfc3RpY2tfeUIQCg5fcmlnaHRfc3RpY2tfeEIQCg5fcmlnaHRfc3RpY2tfeUIPCg1fbGVmdF90cmlnZ2VyQhAKDl9yaWdodF90cmlnZ2VyQgkKB19kcGFkX3hCCQoHX2RwYWRfeUIRCg9fY2hhbmdlZF9maWVsZHMiqgEKE1JUQ0ljZUNhbmRpZGF0ZUluaXQSEQoJY2FuZGlkYXRlGAEgASgJEhoKDXNkcE1MaW5lSW5kZXgYAiABKA1IAIgBARITCgZzZHBNaWQYAyABKAlIAYgBARIdChB1c2VybmFtZUZyYWdtZW50GAQgASgJSAKIAQFCEAoOX3NkcE1MaW5lSW5kZXhCCQoHX3NkcE1pZEITChFfdXNlcm5hbWVGcmFnbWVudCI2ChlSVENTZXNzaW9uRGVzY3JpcHRpb25Jbml0EgsKA3NkcBgBIAEoCRIMCgR0eXBlGAIgASgJIjkKCFByb3RvSUNFEi0KCWNhbmRpZGF0ZRgBIAEoCzIaLnByb3RvLlJUQ0ljZUNhbmRpZGF0ZUluaXQiOQoIUHJvdG9TRFASLQoDc2RwGAEgASgLMiAucHJvdG8uUlRDU2Vzc2lvbkRlc2NyaXB0aW9uSW5pdCIYCghQcm90b1JhdxIMCgRkYXRhGAEgASgJIkUKHFByb3RvQ2xpZW50UmVxdWVzdFJvb21TdHJlYW0SEQoJcm9vbV9uYW1lGAEgASgJEhIKCnNlc3Npb25faWQYAiABKAkiRwoXUHJvdG9DbGllbnREaXNjb25uZWN0ZWQSEgoKc2Vzc2lvbl9pZBgBIAEoCRIYChBjb250cm9sbGVyX3Nsb3RzGAIgAygFIioKFVByb3RvU2VydmVyUHVzaFN0cmVhbRIRCglyb29tX25hbWUYASABKAlCFloUcmVsYXkvaW50ZXJuYWwvcHJvdG9iBnByb3RvMw");
+  fileDesc("Cgt0eXBlcy5wcm90bxIFcHJvdG8iNAoOUHJvdG9Nb3VzZU1vdmUSDAoEdHlwZRgBIAEoCRIJCgF4GAIgASgFEgkKAXkYAyABKAUiNwoRUHJvdG9Nb3VzZU1vdmVBYnMSDAoEdHlwZRgBIAEoCRIJCgF4GAIgASgFEgkKAXkYAyABKAUiNQoPUHJvdG9Nb3VzZVdoZWVsEgwKBHR5cGUYASABKAkSCQoBeBgCIAEoBRIJCgF5GAMgASgFIi4KEVByb3RvTW91c2VLZXlEb3duEgwKBHR5cGUYASABKAkSCwoDa2V5GAIgASgFIiwKD1Byb3RvTW91c2VLZXlVcBIMCgR0eXBlGAEgASgJEgsKA2tleRgCIAEoBSIpCgxQcm90b0tleURvd24SDAoEdHlwZRgBIAEoCRILCgNrZXkYAiABKAUiJwoKUHJvdG9LZXlVcBIMCgR0eXBlGAEgASgJEgsKA2tleRgCIAEoBSI/ChVQcm90b0NvbnRyb2xsZXJBdHRhY2gSDAoEdHlwZRgBIAEoCRIKCgJpZBgCIAEoCRIMCgRzbG90GAMgASgFIjMKFVByb3RvQ29udHJvbGxlckRldGFjaBIMCgR0eXBlGAEgASgJEgwKBHNsb3QYAiABKAUiVAoVUHJvdG9Db250cm9sbGVyQnV0dG9uEgwKBHR5cGUYASABKAkSDAoEc2xvdBgCIAEoBRIOCgZidXR0b24YAyABKAUSDwoHcHJlc3NlZBgEIAEoCCJUChZQcm90b0NvbnRyb2xsZXJUcmlnZ2VyEgwKBHR5cGUYASABKAkSDAoEc2xvdBgCIAEoBRIPCgd0cmlnZ2VyGAMgASgFEg0KBXZhbHVlGAQgASgFIlcKFFByb3RvQ29udHJvbGxlclN0aWNrEgwKBHR5cGUYASABKAkSDAoEc2xvdBgCIAEoBRINCgVzdGljaxgDIAEoBRIJCgF4GAQgASgFEgkKAXkYBSABKAUiTgoTUHJvdG9Db250cm9sbGVyQXhpcxIMCgR0eXBlGAEgASgJEgwKBHNsb3QYAiABKAUSDAoEYXhpcxgDIAEoBRINCgV2YWx1ZRgEIAEoBSJ0ChVQcm90b0NvbnRyb2xsZXJSdW1ibGUSDAoEdHlwZRgBIAEoCRIMCgRzbG90GAIgASgFEhUKDWxvd19mcmVxdWVuY3kYAyABKAUSFgoOaGlnaF9mcmVxdWVuY3kYBCABKAUSEAoIZHVyYXRpb24YBSABKAUi9QUKClByb3RvSW5wdXQSKwoKbW91c2VfbW92ZRgBIAEoCzIVLnByb3RvLlByb3RvTW91c2VNb3ZlSAASMgoObW91c2VfbW92ZV9hYnMYAiABKAsyGC5wcm90by5Qcm90b01vdXNlTW92ZUFic0gAEi0KC21vdXNlX3doZWVsGAMgASgLMhYucHJvdG8uUHJvdG9Nb3VzZVdoZWVsSAASMgoObW91c2Vfa2V5X2Rvd24YBCABKAsyGC5wcm90by5Qcm90b01vdXNlS2V5RG93bkgAEi4KDG1vdXNlX2tleV91cBgFIAEoCzIWLnByb3RvLlByb3RvTW91c2VLZXlVcEgAEicKCGtleV9kb3duGAYgASgLMhMucHJvdG8uUHJvdG9LZXlEb3duSAASIwoGa2V5X3VwGAcgASgLMhEucHJvdG8uUHJvdG9LZXlVcEgAEjkKEWNvbnRyb2xsZXJfYXR0YWNoGAggASgLMhwucHJvdG8uUHJvdG9Db250cm9sbGVyQXR0YWNoSAASOQoRY29udHJvbGxlcl9kZXRhY2gYCSABKAsyHC5wcm90by5Qcm90b0NvbnRyb2xsZXJEZXRhY2hIABI5ChFjb250cm9sbGVyX2J1dHRvbhgKIAEoCzIcLnByb3RvLlByb3RvQ29udHJvbGxlckJ1dHRvbkgAEjsKEmNvbnRyb2xsZXJfdHJpZ2dlchgLIAEoCzIdLnByb3RvLlByb3RvQ29udHJvbGxlclRyaWdnZXJIABI3ChBjb250cm9sbGVyX3N0aWNrGAwgASgLMhsucHJvdG8uUHJvdG9Db250cm9sbGVyU3RpY2tIABI1Cg9jb250cm9sbGVyX2F4aXMYDSABKAsyGi5wcm90by5Qcm90b0NvbnRyb2xsZXJBeGlzSAASOQoRY29udHJvbGxlcl9ydW1ibGUYDiABKAsyHC5wcm90by5Qcm90b0NvbnRyb2xsZXJSdW1ibGVIAEIMCgppbnB1dF90eXBlQhZaFHJlbGF5L2ludGVybmFsL3Byb3RvYgZwcm90bzM");
 
 /**
  * MouseMove message
@@ -19,12 +19,19 @@ export const file_types: GenFile = /*@__PURE__*/
  */
 export type ProtoMouseMove = Message<"proto.ProtoMouseMove"> & {
   /**
-   * @generated from field: int32 x = 1;
+   * Fixed value "MouseMove"
+   *
+   * @generated from field: string type = 1;
+   */
+  type: string;
+
+  /**
+   * @generated from field: int32 x = 2;
    */
   x: number;
 
   /**
-   * @generated from field: int32 y = 2;
+   * @generated from field: int32 y = 3;
    */
   y: number;
 };
@@ -43,12 +50,19 @@ export const ProtoMouseMoveSchema: GenMessage<ProtoMouseMove> = /*@__PURE__*/
  */
 export type ProtoMouseMoveAbs = Message<"proto.ProtoMouseMoveAbs"> & {
   /**
-   * @generated from field: int32 x = 1;
+   * Fixed value "MouseMoveAbs"
+   *
+   * @generated from field: string type = 1;
+   */
+  type: string;
+
+  /**
+   * @generated from field: int32 x = 2;
    */
   x: number;
 
   /**
-   * @generated from field: int32 y = 2;
+   * @generated from field: int32 y = 3;
    */
   y: number;
 };
@@ -67,12 +81,19 @@ export const ProtoMouseMoveAbsSchema: GenMessage<ProtoMouseMoveAbs> = /*@__PURE_
  */
 export type ProtoMouseWheel = Message<"proto.ProtoMouseWheel"> & {
   /**
-   * @generated from field: int32 x = 1;
+   * Fixed value "MouseWheel"
+   *
+   * @generated from field: string type = 1;
+   */
+  type: string;
+
+  /**
+   * @generated from field: int32 x = 2;
    */
   x: number;
 
   /**
-   * @generated from field: int32 y = 2;
+   * @generated from field: int32 y = 3;
    */
   y: number;
 };
@@ -91,7 +112,14 @@ export const ProtoMouseWheelSchema: GenMessage<ProtoMouseWheel> = /*@__PURE__*/
  */
 export type ProtoMouseKeyDown = Message<"proto.ProtoMouseKeyDown"> & {
   /**
-   * @generated from field: int32 key = 1;
+   * Fixed value "MouseKeyDown"
+   *
+   * @generated from field: string type = 1;
+   */
+  type: string;
+
+  /**
+   * @generated from field: int32 key = 2;
    */
   key: number;
 };
@@ -110,7 +138,14 @@ export const ProtoMouseKeyDownSchema: GenMessage<ProtoMouseKeyDown> = /*@__PURE_
  */
 export type ProtoMouseKeyUp = Message<"proto.ProtoMouseKeyUp"> & {
   /**
-   * @generated from field: int32 key = 1;
+   * Fixed value "MouseKeyUp"
+   *
+   * @generated from field: string type = 1;
+   */
+  type: string;
+
+  /**
+   * @generated from field: int32 key = 2;
    */
   key: number;
 };
@@ -129,7 +164,14 @@ export const ProtoMouseKeyUpSchema: GenMessage<ProtoMouseKeyUp> = /*@__PURE__*/
  */
 export type ProtoKeyDown = Message<"proto.ProtoKeyDown"> & {
   /**
-   * @generated from field: int32 key = 1;
+   * Fixed value "KeyDown"
+   *
+   * @generated from field: string type = 1;
+   */
+  type: string;
+
+  /**
+   * @generated from field: int32 key = 2;
    */
   key: number;
 };
@@ -148,7 +190,14 @@ export const ProtoKeyDownSchema: GenMessage<ProtoKeyDown> = /*@__PURE__*/
  */
 export type ProtoKeyUp = Message<"proto.ProtoKeyUp"> & {
   /**
-   * @generated from field: int32 key = 1;
+   * Fixed value "KeyUp"
+   *
+   * @generated from field: string type = 1;
+   */
+  type: string;
+
+  /**
+   * @generated from field: int32 key = 2;
    */
   key: number;
 };
@@ -166,26 +215,26 @@ export const ProtoKeyUpSchema: GenMessage<ProtoKeyUp> = /*@__PURE__*/
  * @generated from message proto.ProtoControllerAttach
  */
 export type ProtoControllerAttach = Message<"proto.ProtoControllerAttach"> & {
+  /**
+   * Fixed value "ControllerAttach"
+   *
+   * @generated from field: string type = 1;
+   */
+  type: string;
+
   /**
    * One of the following enums: "ps", "xbox" or "switch"
    *
-   * @generated from field: string id = 1;
+   * @generated from field: string id = 2;
    */
   id: string;
 
   /**
-   * Session specific slot number (0-3)
+   * Slot number (0-3)
    *
-   * @generated from field: int32 session_slot = 2;
+   * @generated from field: int32 slot = 3;
    */
-  sessionSlot: number;
-
-  /**
-   * Session ID of the client
-   *
-   * @generated from field: string session_id = 3;
-   */
-  sessionId: string;
+  slot: number;
 };
 
 /**
@@ -202,18 +251,18 @@ export const ProtoControllerAttachSchema: GenMessage<ProtoControllerAttach> = /*
  */
 export type ProtoControllerDetach = Message<"proto.ProtoControllerDetach"> & {
   /**
-   * Session specific slot number (0-3)
+   * Fixed value "ControllerDetach"
    *
-   * @generated from field: int32 session_slot = 1;
+   * @generated from field: string type = 1;
    */
-  sessionSlot: number;
+  type: string;
 
   /**
-   * Session ID of the client
+   * Slot number (0-3)
    *
-   * @generated from field: string session_id = 2;
+   * @generated from field: int32 slot = 2;
    */
-  sessionId: string;
+  slot: number;
 };
 
 /**
@@ -223,6 +272,181 @@ export type ProtoControllerDetach = Message<"proto.ProtoControllerDetach"> & {
 export const ProtoControllerDetachSchema: GenMessage<ProtoControllerDetach> = /*@__PURE__*/
   messageDesc(file_types, 8);
 
+/**
+ * ControllerButton message
+ *
+ * @generated from message proto.ProtoControllerButton
+ */
+export type ProtoControllerButton = Message<"proto.ProtoControllerButton"> & {
+  /**
+   * Fixed value "ControllerButtons"
+   *
+   * @generated from field: string type = 1;
+   */
+  type: string;
+
+  /**
+   * Slot number (0-3)
+   *
+   * @generated from field: int32 slot = 2;
+   */
+  slot: number;
+
+  /**
+   * Button code (linux input event code)
+   *
+   * @generated from field: int32 button = 3;
+   */
+  button: number;
+
+  /**
+   * true if pressed, false if released
+   *
+   * @generated from field: bool pressed = 4;
+   */
+  pressed: boolean;
+};
+
+/**
+ * Describes the message proto.ProtoControllerButton.
+ * Use `create(ProtoControllerButtonSchema)` to create a new message.
+ */
+export const ProtoControllerButtonSchema: GenMessage<ProtoControllerButton> = /*@__PURE__*/
+  messageDesc(file_types, 9);
+
+/**
+ * ControllerTriggers message
+ *
+ * @generated from message proto.ProtoControllerTrigger
+ */
+export type ProtoControllerTrigger = Message<"proto.ProtoControllerTrigger"> & {
+  /**
+   * Fixed value "ControllerTriggers"
+   *
+   * @generated from field: string type = 1;
+   */
+  type: string;
+
+  /**
+   * Slot number (0-3)
+   *
+   * @generated from field: int32 slot = 2;
+   */
+  slot: number;
+
+  /**
+   * Trigger number (0 for left, 1 for right)
+   *
+   * @generated from field: int32 trigger = 3;
+   */
+  trigger: number;
+
+  /**
+   * trigger value (-32768 to 32767)
+   *
+   * @generated from field: int32 value = 4;
+   */
+  value: number;
+};
+
+/**
+ * Describes the message proto.ProtoControllerTrigger.
+ * Use `create(ProtoControllerTriggerSchema)` to create a new message.
+ */
+export const ProtoControllerTriggerSchema: GenMessage<ProtoControllerTrigger> = /*@__PURE__*/
+  messageDesc(file_types, 10);
+
+/**
+ * ControllerSticks message
+ *
+ * @generated from message proto.ProtoControllerStick
+ */
+export type ProtoControllerStick = Message<"proto.ProtoControllerStick"> & {
+  /**
+   * Fixed value "ControllerStick"
+   *
+   * @generated from field: string type = 1;
+   */
+  type: string;
+
+  /**
+   * Slot number (0-3)
+   *
+   * @generated from field: int32 slot = 2;
+   */
+  slot: number;
+
+  /**
+   * Stick number (0 for left, 1 for right)
+   *
+   * @generated from field: int32 stick = 3;
+   */
+  stick: number;
+
+  /**
+   * X axis value (-32768 to 32767)
+   *
+   * @generated from field: int32 x = 4;
+   */
+  x: number;
+
+  /**
+   * Y axis value (-32768 to 32767)
+   *
+   * @generated from field: int32 y = 5;
+   */
+  y: number;
+};
+
+/**
+ * Describes the message proto.ProtoControllerStick.
+ * Use `create(ProtoControllerStickSchema)` to create a new message.
+ */
+export const ProtoControllerStickSchema: GenMessage<ProtoControllerStick> = /*@__PURE__*/
+  messageDesc(file_types, 11);
+
+/**
+ * ControllerAxis message
+ *
+ * @generated from message proto.ProtoControllerAxis
+ */
+export type ProtoControllerAxis = Message<"proto.ProtoControllerAxis"> & {
+  /**
+   * Fixed value "ControllerAxis"
+   *
+   * @generated from field: string type = 1;
+   */
+  type: string;
+
+  /**
+   * Slot number (0-3)
+   *
+   * @generated from field: int32 slot = 2;
+   */
+  slot: number;
+
+  /**
+   * Axis number (0 for d-pad horizontal, 1 for d-pad vertical)
+   *
+   * @generated from field: int32 axis = 3;
+   */
+  axis: number;
+
+  /**
+   * axis value (-1 to 1)
+   *
+   * @generated from field: int32 value = 4;
+   */
+  value: number;
+};
+
+/**
+ * Describes the message proto.ProtoControllerAxis.
+ * Use `create(ProtoControllerAxisSchema)` to create a new message.
+ */
+export const ProtoControllerAxisSchema: GenMessage<ProtoControllerAxis> = /*@__PURE__*/
+  messageDesc(file_types, 12);
+
 /**
  * ControllerRumble message
  *
@@ -230,18 +454,18 @@ export const ProtoControllerDetachSchema: GenMessage<ProtoControllerDetach> = /*
  */
 export type ProtoControllerRumble = Message<"proto.ProtoControllerRumble"> & {
   /**
-   * Session specific slot number (0-3)
+   * Fixed value "ControllerRumble"
    *
-   * @generated from field: int32 session_slot = 1;
+   * @generated from field: string type = 1;
    */
-  sessionSlot: number;
+  type: string;
 
   /**
-   * Session ID of the client
+   * Slot number (0-3)
    *
-   * @generated from field: string session_id = 2;
+   * @generated from field: int32 slot = 2;
    */
-  sessionId: string;
+  slot: number;
 
   /**
    * Low frequency rumble (0-65535)
@@ -270,321 +494,108 @@ export type ProtoControllerRumble = Message<"proto.ProtoControllerRumble"> & {
  * Use `create(ProtoControllerRumbleSchema)` to create a new message.
  */
 export const ProtoControllerRumbleSchema: GenMessage<ProtoControllerRumble> = /*@__PURE__*/
-  messageDesc(file_types, 9);
-
-/**
- * ControllerStateBatch - single message containing full or partial controller state
- *
- * @generated from message proto.ProtoControllerStateBatch
- */
-export type ProtoControllerStateBatch = Message<"proto.ProtoControllerStateBatch"> & {
-  /**
-   * Session specific slot number (0-3)
-   *
-   * @generated from field: int32 session_slot = 1;
-   */
-  sessionSlot: number;
-
-  /**
-   * Session ID of the client
-   *
-   * @generated from field: string session_id = 2;
-   */
-  sessionId: string;
-
-  /**
-   * @generated from field: proto.ProtoControllerStateBatch.UpdateType update_type = 3;
-   */
-  updateType: ProtoControllerStateBatch_UpdateType;
-
-  /**
-   * Sequence number for packet loss detection
-   *
-   * @generated from field: uint32 sequence = 4;
-   */
-  sequence: number;
-
-  /**
-   * Button state map (Linux event codes)
-   *
-   * @generated from field: map<int32, bool> button_changed_mask = 5;
-   */
-  buttonChangedMask: { [key: number]: boolean };
-
-  /**
-   * Analog inputs
-   *
-   * -32768 to 32767
-   *
-   * @generated from field: optional int32 left_stick_x = 6;
-   */
-  leftStickX?: number;
-
-  /**
-   * -32768 to 32767
-   *
-   * @generated from field: optional int32 left_stick_y = 7;
-   */
-  leftStickY?: number;
-
-  /**
-   * -32768 to 32767
-   *
-   * @generated from field: optional int32 right_stick_x = 8;
-   */
-  rightStickX?: number;
-
-  /**
-   * -32768 to 32767
-   *
-   * @generated from field: optional int32 right_stick_y = 9;
-   */
-  rightStickY?: number;
-
-  /**
-   * -32768 to 32767
-   *
-   * @generated from field: optional int32 left_trigger = 10;
-   */
-  leftTrigger?: number;
-
-  /**
-   * -32768 to 32767
-   *
-   * @generated from field: optional int32 right_trigger = 11;
-   */
-  rightTrigger?: number;
-
-  /**
-   * -1, 0, or 1
-   *
-   * @generated from field: optional int32 dpad_x = 12;
-   */
-  dpadX?: number;
-
-  /**
-   * -1, 0, or 1
-   *
-   * @generated from field: optional int32 dpad_y = 13;
-   */
-  dpadY?: number;
-
-  /**
-   * Bitmask indicating which fields have changed
-   * Bit 0: button_changed_mask, Bit 1: left_stick_x, Bit 2: left_stick_y, etc.
-   *
-   * @generated from field: optional uint32 changed_fields = 14;
-   */
-  changedFields?: number;
-};
-
-/**
- * Describes the message proto.ProtoControllerStateBatch.
- * Use `create(ProtoControllerStateBatchSchema)` to create a new message.
- */
-export const ProtoControllerStateBatchSchema: GenMessage<ProtoControllerStateBatch> = /*@__PURE__*/
-  messageDesc(file_types, 10);
-
-/**
- * @generated from enum proto.ProtoControllerStateBatch.UpdateType
- */
-export enum ProtoControllerStateBatch_UpdateType {
-  /**
-   * Complete controller state
-   *
-   * @generated from enum value: FULL_STATE = 0;
-   */
-  FULL_STATE = 0,
-
-  /**
-   * Only changed fields
-   *
-   * @generated from enum value: DELTA = 1;
-   */
-  DELTA = 1,
-}
-
-/**
- * Describes the enum proto.ProtoControllerStateBatch.UpdateType.
- */
-export const ProtoControllerStateBatch_UpdateTypeSchema: GenEnum<ProtoControllerStateBatch_UpdateType> = /*@__PURE__*/
-  enumDesc(file_types, 10, 0);
-
-/**
- * @generated from message proto.RTCIceCandidateInit
- */
-export type RTCIceCandidateInit = Message<"proto.RTCIceCandidateInit"> & {
-  /**
-   * @generated from field: string candidate = 1;
-   */
-  candidate: string;
-
-  /**
-   * @generated from field: optional uint32 sdpMLineIndex = 2;
-   */
-  sdpMLineIndex?: number;
-
-  /**
-   * @generated from field: optional string sdpMid = 3;
-   */
-  sdpMid?: string;
-
-  /**
-   * @generated from field: optional string usernameFragment = 4;
-   */
-  usernameFragment?: string;
-};
-
-/**
- * Describes the message proto.RTCIceCandidateInit.
- * Use `create(RTCIceCandidateInitSchema)` to create a new message.
- */
-export const RTCIceCandidateInitSchema: GenMessage<RTCIceCandidateInit> = /*@__PURE__*/
-  messageDesc(file_types, 11);
-
-/**
- * @generated from message proto.RTCSessionDescriptionInit
- */
-export type RTCSessionDescriptionInit = Message<"proto.RTCSessionDescriptionInit"> & {
-  /**
-   * @generated from field: string sdp = 1;
-   */
-  sdp: string;
-
-  /**
-   * @generated from field: string type = 2;
-   */
-  type: string;
-};
-
-/**
- * Describes the message proto.RTCSessionDescriptionInit.
- * Use `create(RTCSessionDescriptionInitSchema)` to create a new message.
- */
-export const RTCSessionDescriptionInitSchema: GenMessage<RTCSessionDescriptionInit> = /*@__PURE__*/
-  messageDesc(file_types, 12);
-
-/**
- * ProtoICE message
- *
- * @generated from message proto.ProtoICE
- */
-export type ProtoICE = Message<"proto.ProtoICE"> & {
-  /**
-   * @generated from field: proto.RTCIceCandidateInit candidate = 1;
-   */
-  candidate?: RTCIceCandidateInit;
-};
-
-/**
- * Describes the message proto.ProtoICE.
- * Use `create(ProtoICESchema)` to create a new message.
- */
-export const ProtoICESchema: GenMessage<ProtoICE> = /*@__PURE__*/
   messageDesc(file_types, 13);
 
 /**
- * ProtoSDP message
+ * Union of all Input types
  *
- * @generated from message proto.ProtoSDP
+ * @generated from message proto.ProtoInput
  */
-export type ProtoSDP = Message<"proto.ProtoSDP"> & {
+export type ProtoInput = Message<"proto.ProtoInput"> & {
   /**
-   * @generated from field: proto.RTCSessionDescriptionInit sdp = 1;
+   * @generated from oneof proto.ProtoInput.input_type
    */
-  sdp?: RTCSessionDescriptionInit;
+  inputType: {
+    /**
+     * @generated from field: proto.ProtoMouseMove mouse_move = 1;
+     */
+    value: ProtoMouseMove;
+    case: "mouseMove";
+  } | {
+    /**
+     * @generated from field: proto.ProtoMouseMoveAbs mouse_move_abs = 2;
+     */
+    value: ProtoMouseMoveAbs;
+    case: "mouseMoveAbs";
+  } | {
+    /**
+     * @generated from field: proto.ProtoMouseWheel mouse_wheel = 3;
+     */
+    value: ProtoMouseWheel;
+    case: "mouseWheel";
+  } | {
+    /**
+     * @generated from field: proto.ProtoMouseKeyDown mouse_key_down = 4;
+     */
+    value: ProtoMouseKeyDown;
+    case: "mouseKeyDown";
+  } | {
+    /**
+     * @generated from field: proto.ProtoMouseKeyUp mouse_key_up = 5;
+     */
+    value: ProtoMouseKeyUp;
+    case: "mouseKeyUp";
+  } | {
+    /**
+     * @generated from field: proto.ProtoKeyDown key_down = 6;
+     */
+    value: ProtoKeyDown;
+    case: "keyDown";
+  } | {
+    /**
+     * @generated from field: proto.ProtoKeyUp key_up = 7;
+     */
+    value: ProtoKeyUp;
+    case: "keyUp";
+  } | {
+    /**
+     * @generated from field: proto.ProtoControllerAttach controller_attach = 8;
+     */
+    value: ProtoControllerAttach;
+    case: "controllerAttach";
+  } | {
+    /**
+     * @generated from field: proto.ProtoControllerDetach controller_detach = 9;
+     */
+    value: ProtoControllerDetach;
+    case: "controllerDetach";
+  } | {
+    /**
+     * @generated from field: proto.ProtoControllerButton controller_button = 10;
+     */
+    value: ProtoControllerButton;
+    case: "controllerButton";
+  } | {
+    /**
+     * @generated from field: proto.ProtoControllerTrigger controller_trigger = 11;
+     */
+    value: ProtoControllerTrigger;
+    case: "controllerTrigger";
+  } | {
+    /**
+     * @generated from field: proto.ProtoControllerStick controller_stick = 12;
+     */
+    value: ProtoControllerStick;
+    case: "controllerStick";
+  } | {
+    /**
+     * @generated from field: proto.ProtoControllerAxis controller_axis = 13;
+     */
+    value: ProtoControllerAxis;
+    case: "controllerAxis";
+  } | {
+    /**
+     * @generated from field: proto.ProtoControllerRumble controller_rumble = 14;
+     */
+    value: ProtoControllerRumble;
+    case: "controllerRumble";
+  } | { case: undefined; value?: undefined };
 };
 
 /**
- * Describes the message proto.ProtoSDP.
- * Use `create(ProtoSDPSchema)` to create a new message.
+ * Describes the message proto.ProtoInput.
+ * Use `create(ProtoInputSchema)` to create a new message.
  */
-export const ProtoSDPSchema: GenMessage<ProtoSDP> = /*@__PURE__*/
+export const ProtoInputSchema: GenMessage<ProtoInput> = /*@__PURE__*/
   messageDesc(file_types, 14);
 
-/**
- * ProtoRaw message
- *
- * @generated from message proto.ProtoRaw
- */
-export type ProtoRaw = Message<"proto.ProtoRaw"> & {
-  /**
-   * @generated from field: string data = 1;
-   */
-  data: string;
-};
-
-/**
- * Describes the message proto.ProtoRaw.
- * Use `create(ProtoRawSchema)` to create a new message.
- */
-export const ProtoRawSchema: GenMessage<ProtoRaw> = /*@__PURE__*/
-  messageDesc(file_types, 15);
-
-/**
- * ProtoClientRequestRoomStream message
- *
- * @generated from message proto.ProtoClientRequestRoomStream
- */
-export type ProtoClientRequestRoomStream = Message<"proto.ProtoClientRequestRoomStream"> & {
-  /**
-   * @generated from field: string room_name = 1;
-   */
-  roomName: string;
-
-  /**
-   * @generated from field: string session_id = 2;
-   */
-  sessionId: string;
-};
-
-/**
- * Describes the message proto.ProtoClientRequestRoomStream.
- * Use `create(ProtoClientRequestRoomStreamSchema)` to create a new message.
- */
-export const ProtoClientRequestRoomStreamSchema: GenMessage<ProtoClientRequestRoomStream> = /*@__PURE__*/
-  messageDesc(file_types, 16);
-
-/**
- * ProtoClientDisconnected message
- *
- * @generated from message proto.ProtoClientDisconnected
- */
-export type ProtoClientDisconnected = Message<"proto.ProtoClientDisconnected"> & {
-  /**
-   * @generated from field: string session_id = 1;
-   */
-  sessionId: string;
-
-  /**
-   * @generated from field: repeated int32 controller_slots = 2;
-   */
-  controllerSlots: number[];
-};
-
-/**
- * Describes the message proto.ProtoClientDisconnected.
- * Use `create(ProtoClientDisconnectedSchema)` to create a new message.
- */
-export const ProtoClientDisconnectedSchema: GenMessage<ProtoClientDisconnected> = /*@__PURE__*/
-  messageDesc(file_types, 17);
-
-/**
- * ProtoServerPushStream message
- *
- * @generated from message proto.ProtoServerPushStream
- */
-export type ProtoServerPushStream = Message<"proto.ProtoServerPushStream"> & {
-  /**
-   * @generated from field: string room_name = 1;
-   */
-  roomName: string;
-};
-
-/**
- * Describes the message proto.ProtoServerPushStream.
- * Use `create(ProtoServerPushStreamSchema)` to create a new message.
- */
-export const ProtoServerPushStreamSchema: GenMessage<ProtoServerPushStream> = /*@__PURE__*/
-  messageDesc(file_types, 18);
-

packages/input/src/streamwrapper.ts

@@ -1,81 +0,0 @@
-import { pbStream, type ProtobufStream } from "@libp2p/utils";
-import type { Stream } from "@libp2p/interface";
-import { bufbuildAdapter } from "./utils";
-import {
-  ProtoMessage,
-  ProtoMessageSchema,
-  ProtoMessageBase,
-} from "./proto/messages_pb";
-
-type MessageHandler = (
-  data: any,
-  base: ProtoMessageBase,
-) => void | Promise<void>;
-
-export class P2PMessageStream {
-  private pb: ProtobufStream;
-  private handlers = new Map<string, MessageHandler[]>();
-  private closed = false;
-  private readLoopRunning = false;
-
-  constructor(stream: Stream) {
-    this.pb = pbStream(stream);
-  }
-
-  public on(payloadType: string, handler: MessageHandler): void {
-    if (!this.handlers.has(payloadType)) {
-      this.handlers.set(payloadType, []);
-    }
-    this.handlers.get(payloadType)!.push(handler);
-
-    if (!this.readLoopRunning) this.startReading().catch(console.error);
-  }
-
-  private async startReading(): Promise<void> {
-    if (this.readLoopRunning || this.closed) return;
-    this.readLoopRunning = true;
-
-    while (!this.closed) {
-      try {
-        const msg: ProtoMessage = await this.pb.read(
-          bufbuildAdapter(ProtoMessageSchema),
-        );
-
-        const payloadType = msg.messageBase?.payloadType;
-        if (payloadType && this.handlers.has(payloadType)) {
-          const handlers = this.handlers.get(payloadType)!;
-          if (msg.payload.value) {
-            for (const handler of handlers) {
-              try {
-                await handler(msg.payload.value, msg.messageBase);
-              } catch (err) {
-                console.error(`Error in handler for ${payloadType}:`, err);
-              }
-            }
-          }
-        }
-      } catch (err) {
-        if (this.closed) break;
-        console.error("Stream read error:", err);
-        this.close();
-      }
-    }
-
-    this.readLoopRunning = false;
-  }
-
-  public async write(
-    message: ProtoMessage,
-    options?: { signal?: AbortSignal },
-  ): Promise<void> {
-    if (this.closed)
-      throw new Error("Cannot write to closed stream");
-
-    await this.pb.write(message, bufbuildAdapter(ProtoMessageSchema), options);
-  }
-
-  public close(): void {
-    this.closed = true;
-    this.handlers.clear();
-  }
-}

packages/input/src/utils.ts

@@ -1,95 +0,0 @@
-import { create, toBinary, fromBinary } from "@bufbuild/protobuf";
-import type { Message } from "@bufbuild/protobuf";
-import { Uint8ArrayList } from "uint8arraylist";
-import type { GenMessage } from "@bufbuild/protobuf/codegenv2";
-import { timestampFromDate } from "@bufbuild/protobuf/wkt";
-import {
-  ProtoLatencyTracker,
-  ProtoLatencyTrackerSchema,
-  ProtoTimestampEntrySchema,
-} from "./proto/latency_tracker_pb";
-import {
-  ProtoMessage,
-  ProtoMessageSchema,
-  ProtoMessageBaseSchema,
-} from "./proto/messages_pb";
-
-export function bufbuildAdapter<T extends Message>(schema: GenMessage<T>) {
-  return {
-    encode: (data: T): Uint8Array => {
-      return toBinary(schema, data);
-    },
-    decode: (data: Uint8Array | Uint8ArrayList): T => {
-      // Convert Uint8ArrayList to Uint8Array if needed
-      const bytes = data instanceof Uint8ArrayList ? data.subarray() : data;
-      return fromBinary(schema, bytes);
-    },
-  };
-}
-
-// Latency tracker helpers
-export function createLatencyTracker(sequenceId?: string): ProtoLatencyTracker {
-  return create(ProtoLatencyTrackerSchema, {
-    sequenceId: sequenceId || crypto.randomUUID(),
-    timestamps: [],
-  });
-}
-
-export function addLatencyTimestamp(
-  tracker: ProtoLatencyTracker,
-  stage: string,
-): ProtoLatencyTracker {
-  const entry = create(ProtoTimestampEntrySchema, {
-    stage,
-    time: timestampFromDate(new Date()),
-  });
-
-  return {
-    ...tracker,
-    timestamps: [...tracker.timestamps, entry],
-  };
-}
-
-interface CreateMessageOptions {
-  sequenceId?: string;
-}
-
-function derivePayloadCase(data: Message): string {
-  // Extract case from $typeName: "proto.ProtoICE" -> "ice"
-  // "proto.ProtoControllerAttach" -> "controllerAttach"
-  const typeName = data.$typeName;
-  if (!typeName)
-    throw new Error("Message has no $typeName");
-
-  // Remove "proto.Proto" prefix and convert first char to lowercase
-  const caseName = typeName.replace(/^proto\.Proto/, "");
-
-  // Convert PascalCase to camelCase
-  // If it's all caps (like SDP, ICE), lowercase everything
-  // Otherwise, just lowercase the first character
-  if (caseName === caseName.toUpperCase()) {
-    return caseName.toLowerCase();
-  }
-  return caseName.charAt(0).toLowerCase() + caseName.slice(1);
-}
-
-export function createMessage(
-  data: Message,
-  payloadType: string,
-  options?: CreateMessageOptions,
-): ProtoMessage {
-  const payloadCase = derivePayloadCase(data);
-
-  return create(ProtoMessageSchema, {
-    messageBase: create(ProtoMessageBaseSchema, {
-      payloadType,
-      latency: options?.sequenceId
-        ? createLatencyTracker(options.sequenceId)
-        : undefined,
-    }),
-    payload: {
-      case: payloadCase,
-      value: data,
-    } as any, // Type assertion needed for dynamic case
-  });
-}

packages/input/src/webrtc-stream.ts

@@ -1,3 +1,9 @@
+import {
+  NewMessageRaw,
+  NewMessageSDP,
+  NewMessageICE,
+  SafeStream,
+} from "./messages";
 import { webSockets } from "@libp2p/websockets";
 import { webTransport } from "@libp2p/webtransport";
 import { createLibp2p, Libp2p } from "libp2p";
@@ -7,38 +13,24 @@ import { identify } from "@libp2p/identify";
 import { multiaddr } from "@multiformats/multiaddr";
 import { Connection } from "@libp2p/interface";
 import { ping } from "@libp2p/ping";
-import { createMessage } from "./utils";
-import { create } from "@bufbuild/protobuf";
-import {
-  ProtoClientRequestRoomStream,
-  ProtoClientRequestRoomStreamSchema,
-  ProtoICE,
-  ProtoICESchema,
-  ProtoRaw,
-  ProtoSDP,
-  ProtoSDPSchema,
-} from "./proto/types_pb";
-import { P2PMessageStream } from "./streamwrapper";
 
 const NESTRI_PROTOCOL_STREAM_REQUEST = "/nestri-relay/stream-request/1.0.0";
 
 export class WebRTCStream {
-  private _sessionId: string | null = null;
   private _p2p: Libp2p | undefined = undefined;
   private _p2pConn: Connection | undefined = undefined;
-  private _msgStream: P2PMessageStream | undefined = undefined;
+  private _p2pSafeStream: SafeStream | undefined = undefined;
   private _pc: RTCPeerConnection | undefined = undefined;
   private _audioTrack: MediaStreamTrack | undefined = undefined;
   private _videoTrack: MediaStreamTrack | undefined = undefined;
   private _dataChannel: RTCDataChannel | undefined = undefined;
-  private _onConnected: ((stream: MediaStream | null) => void) | undefined =
-    undefined;
-  private _connectionTimer: NodeJS.Timeout | NodeJS.Timer | undefined =
-    undefined;
+  private _onConnected: ((stream: MediaStream | null) => void) | undefined = undefined;
+  private _connectionTimer: NodeJS.Timeout | NodeJS.Timer | undefined = undefined;
   private _serverURL: string | undefined = undefined;
   private _roomName: string | undefined = undefined;
   private _isConnected: boolean = false;
   private _dataChannelCallbacks: Array<(data: any) => void> = [];
+  currentFrameRate: number = 100;
 
   constructor(
     serverURL: string,
@@ -97,20 +89,14 @@ export class WebRTCStream {
         .newStream(NESTRI_PROTOCOL_STREAM_REQUEST)
         .catch(console.error);
       if (stream) {
-        this._msgStream = new P2PMessageStream(stream);
+        this._p2pSafeStream = new SafeStream(stream);
         console.log("Stream opened with peer");
 
         let iceHolder: RTCIceCandidateInit[] = [];
-        this._msgStream.on("ice-candidate", (data: ProtoICE) => {
-          const cand: RTCIceCandidateInit = {
-            candidate: data.candidate.candidate,
-            sdpMLineIndex: data.candidate.sdpMLineIndex,
-            sdpMid: data.candidate.sdpMid,
-            usernameFragment: data.candidate.usernameFragment,
-          };
+        this._p2pSafeStream.registerCallback("ice-candidate", (data) => {
           if (this._pc) {
             if (this._pc.remoteDescription) {
-              this._pc.addIceCandidate(cand).catch((err) => {
+              this._pc.addIceCandidate(data.candidate).catch((err) => {
                 console.error("Error adding ICE candidate:", err);
               });
               // Add held candidates
@@ -121,86 +107,45 @@ export class WebRTCStream {
               });
               iceHolder = [];
             } else {
-              iceHolder.push(cand);
+              iceHolder.push(data.candidate);
             }
+          } else {
+            iceHolder.push(data.candidate);
           }
         });
 
-        this._msgStream.on(
-          "session-assigned",
-          (data: ProtoClientRequestRoomStream) => {
-            this._sessionId = data.sessionId;
-            localStorage.setItem("nestri-session-id", this._sessionId);
-            console.log(
-              "Session ID assigned:",
-              this._sessionId,
-              "for room:",
-              data.roomName,
-            );
-          },
-        );
-
-        this._msgStream.on("offer", async (data: ProtoSDP) => {
+        this._p2pSafeStream.registerCallback("offer", async (data) => {
           if (!this._pc) {
             // Setup peer connection now
             this._setupPeerConnection();
           }
-          await this._pc!.setRemoteDescription({
-            sdp: data.sdp.sdp,
-            type: data.sdp.type as RTCSdpType,
-          });
-          // Add held candidates
-          iceHolder.forEach((candidate) => {
-            this._pc!.addIceCandidate(candidate).catch((err) => {
-              console.error("Error adding held ICE candidate:", err);
-            });
-          });
-          iceHolder = [];
-
+          await this._pc!.setRemoteDescription(data.sdp);
           // Create our answer
           const answer = await this._pc!.createAnswer();
           // Force stereo in Chromium browsers
           answer.sdp = this.forceOpusStereo(answer.sdp!);
           await this._pc!.setLocalDescription(answer);
           // Send answer back
-          const answerMsg = createMessage(
-            create(ProtoSDPSchema, {
-              sdp: answer,
-            }),
-            "answer",
-          );
-          await this._msgStream?.write(answerMsg);
+          const answerMsg = NewMessageSDP("answer", answer);
+          await this._p2pSafeStream?.writeMessage(answerMsg);
         });
 
-        this._msgStream.on("request-stream-offline", (msg: ProtoRaw) => {
-          console.warn("Stream is offline for room:", msg.data);
+        this._p2pSafeStream.registerCallback("request-stream-offline", (data) => {
+          console.warn("Stream is offline for room:", data.roomName);
           this._onConnected?.(null);
         });
 
-        const clientId = this.getSessionID();
-        if (clientId) {
-          console.debug("Using existing session ID:", clientId);
-        }
-
         // Send stream request
-        const requestMsg = createMessage(
-          create(ProtoClientRequestRoomStreamSchema, {
-            roomName: roomName,
-            sessionId: clientId ?? "",
-          }),
+        // marshal room name into json
+        const request = NewMessageRaw(
           "request-stream-room",
+          roomName,
         );
-        await this._msgStream.write(requestMsg);
+        await this._p2pSafeStream.writeMessage(request);
       }
     }
   }
 
-  public getSessionID(): string | null {
-    if (this._sessionId === null)
-      this._sessionId = localStorage.getItem("nestri-session-id");
-    return this._sessionId;
-  }
-
   // Forces opus to stereo in Chromium browsers, because of course
   private forceOpusStereo(SDP: string): string {
     // Look for "minptime=10;useinbandfec=1" and replace with "minptime=10;useinbandfec=1;stereo=1;sprop-stereo=1;"
@@ -255,16 +200,11 @@ export class WebRTCStream {
 
     this._pc.onicecandidate = (e) => {
       if (e.candidate) {
-        const iceMsg = createMessage(
-          create(ProtoICESchema, {
-            candidate: e.candidate,
-          }),
-          "ice-candidate",
+        const iceMsg = NewMessageICE("ice-candidate", e.candidate);
+        if (this._p2pSafeStream) {
+          this._p2pSafeStream.writeMessage(iceMsg).catch((err) =>
+            console.error("Error sending ICE candidate:", err),
           );
-        if (this._msgStream) {
-          this._msgStream
-            .write(iceMsg)
-            .catch((err) => console.error("Error sending ICE candidate:", err));
         } else {
           console.warn("P2P stream not established, cannot send ICE candidate");
         }
@@ -278,7 +218,8 @@ export class WebRTCStream {
   }
 
   private _checkConnectionState() {
-    if (!this._pc || !this._p2p || !this._p2pConn) return;
+    if (!this._pc || !this._p2p || !this._p2pConn)
+      return;
 
     console.debug("Checking connection state:", {
       connectionState: this._pc.connectionState,
@@ -301,8 +242,26 @@ export class WebRTCStream {
           this._onConnected(
             new MediaStream([this._audioTrack, this._videoTrack]),
           );
+
+          // Continuously set low-latency target
+          this._pc.getReceivers().forEach((receiver: RTCRtpReceiver) => {
+            let intervalLoop = setInterval(async () => {
+              if (
+                receiver.track.readyState !== "live" ||
+                (receiver.transport && receiver.transport.state !== "connected")
+              ) {
+                clearInterval(intervalLoop);
+                return;
+              } else {
+                // @ts-ignore
+                receiver.jitterBufferTarget = receiver.jitterBufferDelayHint = receiver.playoutDelayHint = 0;
+              }
+            }, 15);
+          });
         }
       }
+
+      this._gatherFrameRate();
     } else if (
       this._pc.connectionState === "failed" ||
       this._pc.connectionState === "closed" ||
@@ -327,9 +286,7 @@ export class WebRTCStream {
 
     // Attempt to reconnect only if not already connected
     if (!this._isConnected && this._serverURL && this._roomName) {
-      this._setup(this._serverURL, this._roomName).catch((err) =>
-        console.error("Reconnection failed:", err),
-      );
+      this._setup(this._serverURL, this._roomName).catch((err) => console.error("Reconnection failed:", err));
     }
   }
 
@@ -378,9 +335,7 @@ export class WebRTCStream {
   }
 
   public removeDataChannelCallback(callback: (data: any) => void) {
-    this._dataChannelCallbacks = this._dataChannelCallbacks.filter(
-      (cb) => cb !== callback,
-    );
+    this._dataChannelCallbacks = this._dataChannelCallbacks.filter(cb => cb !== callback);
   }
 
   private _setupDataChannelEvents() {
@@ -388,7 +343,7 @@ export class WebRTCStream {
 
     this._dataChannel.onclose = () => console.log("sendChannel has closed");
     this._dataChannel.onopen = () => console.log("sendChannel has opened");
-    this._dataChannel.onmessage = (event) => {
+    this._dataChannel.onmessage = (event => {
       // Parse as ProtoBuf message
       const data = event.data;
       // Call registered callback if exists
@@ -399,19 +354,14 @@ export class WebRTCStream {
           console.error("Error in data channel callback:", err);
         }
       });
-    };
+    });
   }
 
-  private async _gatherStats(): Promise<any> {
-    if (
-      this._pc === undefined ||
-      this._videoTrack === undefined ||
-      !this._isConnected
-    )
-      return null;
+  private _gatherFrameRate() {
+    if (this._pc === undefined || this._videoTrack === undefined) return;
 
-    return new Promise<any>((resolve) => {
-      // Keep trying to get stats until gotten
+    const videoInfoPromise = new Promise<{ fps: number }>((resolve) => {
+      // Keep trying to get fps until it's found
       const interval = setInterval(async () => {
         if (this._pc === undefined) {
           clearInterval(interval);
@@ -423,11 +373,15 @@ export class WebRTCStream {
           if (report.type === "inbound-rtp") {
             clearInterval(interval);
 
-            resolve({ pli: report.pliCount, nack: report.nackCount });
+            resolve({ fps: report.framesPerSecond });
           }
         });
       }, 250);
     });
+
+    videoInfoPromise.then((value) => {
+      this.currentFrameRate = value.fps;
+    });
   }
 
   // Send binary message through the data channel

packages/play-standalone/android/.gitignore

@@ -0,0 +1,101 @@
+# Using Android gitignore template: https://github.com/github/gitignore/blob/HEAD/Android.gitignore
+
+# Built application files
+*.apk
+*.aar
+*.ap_
+*.aab
+
+# Files for the ART/Dalvik VM
+*.dex
+
+# Java class files
+*.class
+
+# Generated files
+bin/
+gen/
+out/
+#  Uncomment the following line in case you need and you don't have the release build type files in your app
+# release/
+
+# Gradle files
+.gradle/
+build/
+
+# Local configuration file (sdk path, etc)
+local.properties
+
+# Proguard folder generated by Eclipse
+proguard/
+
+# Log Files
+*.log
+
+# Android Studio Navigation editor temp files
+.navigation/
+
+# Android Studio captures folder
+captures/
+
+# IntelliJ
+*.iml
+.idea/workspace.xml
+.idea/tasks.xml
+.idea/gradle.xml
+.idea/assetWizardSettings.xml
+.idea/dictionaries
+.idea/libraries
+# Android Studio 3 in .gitignore file.
+.idea/caches
+.idea/modules.xml
+# Comment next line if keeping position of elements in Navigation Editor is relevant for you
+.idea/navEditor.xml
+
+# Keystore files
+# Uncomment the following lines if you do not want to check your keystore files in.
+#*.jks
+#*.keystore
+
+# External native build folder generated in Android Studio 2.2 and later
+.externalNativeBuild
+.cxx/
+
+# Google Services (e.g. APIs or Firebase)
+# google-services.json
+
+# Freeline
+freeline.py
+freeline/
+freeline_project_description.json
+
+# fastlane
+fastlane/report.xml
+fastlane/Preview.html
+fastlane/screenshots
+fastlane/test_output
+fastlane/readme.md
+
+# Version control
+vcs.xml
+
+# lint
+lint/intermediates/
+lint/generated/
+lint/outputs/
+lint/tmp/
+# lint/reports/
+
+# Android Profiling
+*.hprof
+
+# Cordova plugins for Capacitor
+capacitor-cordova-android-plugins
+
+# Copied web assets
+app/src/main/assets/public
+
+# Generated Config files
+app/src/main/assets/capacitor.config.json
+app/src/main/assets/capacitor.plugins.json
+app/src/main/res/xml/config.xml

packages/play-standalone/android/app/.gitignore

@@ -0,0 +1,2 @@
+/build/*
+!/build/.npmkeep

packages/play-standalone/android/app/build.gradle

@@ -0,0 +1,54 @@
+apply plugin: 'com.android.application'
+
+android {
+    namespace "com.nestri.play"
+    compileSdk rootProject.ext.compileSdkVersion
+    defaultConfig {
+        applicationId "com.nestri.play"
+        minSdkVersion rootProject.ext.minSdkVersion
+        targetSdkVersion rootProject.ext.targetSdkVersion
+        versionCode 1
+        versionName "1.0"
+        testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
+        aaptOptions {
+             // Files and dirs to omit from the packaged assets dir, modified to accommodate modern web apps.
+             // Default: https://android.googlesource.com/platform/frameworks/base/+/282e181b58cf72b6ca770dc7ca5f91f135444502/tools/aapt/AaptAssets.cpp#61
+            ignoreAssetsPattern '!.svn:!.git:!.ds_store:!*.scc:.*:!CVS:!thumbs.db:!picasa.ini:!*~'
+        }
+    }
+    buildTypes {
+        release {
+            minifyEnabled false
+            proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro'
+        }
+    }
+}
+
+repositories {
+    flatDir{
+        dirs '../capacitor-cordova-android-plugins/src/main/libs', 'libs'
+    }
+}
+
+dependencies {
+    implementation fileTree(include: ['*.jar'], dir: 'libs')
+    implementation "androidx.appcompat:appcompat:$androidxAppCompatVersion"
+    implementation "androidx.coordinatorlayout:coordinatorlayout:$androidxCoordinatorLayoutVersion"
+    implementation "androidx.core:core-splashscreen:$coreSplashScreenVersion"
+    implementation project(':capacitor-android')
+    testImplementation "junit:junit:$junitVersion"
+    androidTestImplementation "androidx.test.ext:junit:$androidxJunitVersion"
+    androidTestImplementation "androidx.test.espresso:espresso-core:$androidxEspressoCoreVersion"
+    implementation project(':capacitor-cordova-android-plugins')
+}
+
+apply from: 'capacitor.build.gradle'
+
+try {
+    def servicesJSON = file('google-services.json')
+    if (servicesJSON.text) {
+        apply plugin: 'com.google.gms.google-services'
+    }
+} catch(Exception e) {
+    logger.info("google-services.json not found, google-services plugin not applied. Push Notifications won't work")
+}

packages/play-standalone/android/app/capacitor.build.gradle

@@ -0,0 +1,19 @@
+// DO NOT EDIT THIS FILE! IT IS GENERATED EACH TIME "capacitor update" IS RUN
+
+android {
+  compileOptions {
+      sourceCompatibility JavaVersion.VERSION_21
+      targetCompatibility JavaVersion.VERSION_21
+  }
+}
+
+apply from: "../capacitor-cordova-android-plugins/cordova.variables.gradle"
+dependencies {
+
+
+}
+
+
+if (hasProperty('postBuildExtras')) {
+  postBuildExtras()
+}

packages/play-standalone/android/app/proguard-rules.pro

@@ -0,0 +1,21 @@
+# Add project specific ProGuard rules here.
+# You can control the set of applied configuration files using the
+# proguardFiles setting in build.gradle.
+#
+# For more details, see
+#   http://developer.android.com/guide/developing/tools/proguard.html
+
+# If your project uses WebView with JS, uncomment the following
+# and specify the fully qualified class name to the JavaScript interface
+# class:
+#-keepclassmembers class fqcn.of.javascript.interface.for.webview {
+#   public *;
+#}
+
+# Uncomment this to preserve the line number information for
+# debugging stack traces.
+#-keepattributes SourceFile,LineNumberTable
+
+# If you keep the line number information, uncomment this to
+# hide the original source file name.
+#-renamesourcefileattribute SourceFile

packages/play-standalone/android/app/src/androidTest/java/com/getcapacitor/myapp/ExampleInstrumentedTest.java

@@ -0,0 +1,26 @@
+package com.getcapacitor.myapp;
+
+import static org.junit.Assert.*;
+
+import android.content.Context;
+import androidx.test.ext.junit.runners.AndroidJUnit4;
+import androidx.test.platform.app.InstrumentationRegistry;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+/**
+ * Instrumented test, which will execute on an Android device.
+ *
+ * @see <a href="http://d.android.com/tools/testing">Testing documentation</a>
+ */
+@RunWith(AndroidJUnit4.class)
+public class ExampleInstrumentedTest {
+
+    @Test
+    public void useAppContext() throws Exception {
+        // Context of the app under test.
+        Context appContext = InstrumentationRegistry.getInstrumentation().getTargetContext();
+
+        assertEquals("com.getcapacitor.app", appContext.getPackageName());
+    }
+}

packages/play-standalone/android/app/src/main/AndroidManifest.xml

@@ -0,0 +1,41 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android">
+
+    <application
+        android:allowBackup="true"
+        android:icon="@mipmap/ic_launcher"
+        android:label="@string/app_name"
+        android:roundIcon="@mipmap/ic_launcher_round"
+        android:supportsRtl="true"
+        android:theme="@style/AppTheme">
+
+        <activity
+            android:configChanges="orientation|keyboardHidden|keyboard|screenSize|locale|smallestScreenSize|screenLayout|uiMode|navigation"
+            android:name=".MainActivity"
+            android:label="@string/title_activity_main"
+            android:theme="@style/AppTheme.NoActionBarLaunch"
+            android:launchMode="singleTask"
+            android:exported="true">
+
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN" />
+                <category android:name="android.intent.category.LAUNCHER" />
+            </intent-filter>
+
+        </activity>
+
+        <provider
+            android:name="androidx.core.content.FileProvider"
+            android:authorities="${applicationId}.fileprovider"
+            android:exported="false"
+            android:grantUriPermissions="true">
+            <meta-data
+                android:name="android.support.FILE_PROVIDER_PATHS"
+                android:resource="@xml/file_paths"></meta-data>
+        </provider>
+    </application>
+
+    <!-- Permissions -->
+
+    <uses-permission android:name="android.permission.INTERNET" />
+</manifest>

packages/play-standalone/android/app/src/main/java/com/nestri/play/MainActivity.java

@@ -0,0 +1,5 @@
+package com.nestri.play;
+
+import com.getcapacitor.BridgeActivity;
+
+public class MainActivity extends BridgeActivity {}

packages/play-standalone/android/app/src/main/res/drawable-v24/ic_launcher_foreground.xml

@@ -0,0 +1,34 @@
+<vector xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:aapt="http://schemas.android.com/aapt"
+    android:width="108dp"
+    android:height="108dp"
+    android:viewportHeight="108"
+    android:viewportWidth="108">
+    <path
+        android:fillType="evenOdd"
+        android:pathData="M32,64C32,64 38.39,52.99 44.13,50.95C51.37,48.37 70.14,49.57 70.14,49.57L108.26,87.69L108,109.01L75.97,107.97L32,64Z"
+        android:strokeColor="#00000000"
+        android:strokeWidth="1">
+        <aapt:attr name="android:fillColor">
+            <gradient
+                android:endX="78.5885"
+                android:endY="90.9159"
+                android:startX="48.7653"
+                android:startY="61.0927"
+                android:type="linear">
+                <item
+                    android:color="#44000000"
+                    android:offset="0.0" />
+                <item
+                    android:color="#00000000"
+                    android:offset="1.0" />
+            </gradient>
+        </aapt:attr>
+    </path>
+    <path
+        android:fillColor="#FFFFFF"
+        android:fillType="nonZero"
+        android:pathData="M66.94,46.02L66.94,46.02C72.44,50.07 76,56.61 76,64L32,64C32,56.61 35.56,50.11 40.98,46.06L36.18,41.19C35.45,40.45 35.45,39.3 36.18,38.56C36.91,37.81 38.05,37.81 38.78,38.56L44.25,44.05C47.18,42.57 50.48,41.71 54,41.71C57.48,41.71 60.78,42.57 63.68,44.05L69.11,38.56C69.84,37.81 70.98,37.81 71.71,38.56C72.44,39.3 72.44,40.45 71.71,41.19L66.94,46.02ZM62.94,56.92C64.08,56.92 65,56.01 65,54.88C65,53.76 64.08,52.85 62.94,52.85C61.8,52.85 60.88,53.76 60.88,54.88C60.88,56.01 61.8,56.92 62.94,56.92ZM45.06,56.92C46.2,56.92 47.13,56.01 47.13,54.88C47.13,53.76 46.2,52.85 45.06,52.85C43.92,52.85 43,53.76 43,54.88C43,56.01 43.92,56.92 45.06,56.92Z"
+        android:strokeColor="#00000000"
+        android:strokeWidth="1" />
+</vector>

packages/play-standalone/android/app/src/main/res/drawable/ic_launcher_background.xml

@@ -0,0 +1,170 @@
+<?xml version="1.0" encoding="utf-8"?>
+<vector xmlns:android="http://schemas.android.com/apk/res/android"
+    android:width="108dp"
+    android:height="108dp"
+    android:viewportHeight="108"
+    android:viewportWidth="108">
+    <path
+        android:fillColor="#26A69A"
+        android:pathData="M0,0h108v108h-108z" />
+    <path
+        android:fillColor="#00000000"
+        android:pathData="M9,0L9,108"
+        android:strokeColor="#33FFFFFF"
+        android:strokeWidth="0.8" />
+    <path
+        android:fillColor="#00000000"
+        android:pathData="M19,0L19,108"
+        android:strokeColor="#33FFFFFF"
+        android:strokeWidth="0.8" />
+    <path
+        android:fillColor="#00000000"
+        android:pathData="M29,0L29,108"
+        android:strokeColor="#33FFFFFF"
+        android:strokeWidth="0.8" />
+    <path
+        android:fillColor="#00000000"
+        android:pathData="M39,0L39,108"
+        android:strokeColor="#33FFFFFF"
+        android:strokeWidth="0.8" />
+    <path
+        android:fillColor="#00000000"
+        android:pathData="M49,0L49,108"
+        android:strokeColor="#33FFFFFF"
+        android:strokeWidth="0.8" />
+    <path
+        android:fillColor="#00000000"
+        android:pathData="M59,0L59,108"
+        android:strokeColor="#33FFFFFF"
+        android:strokeWidth="0.8" />
+    <path
+        android:fillColor="#00000000"
+        android:pathData="M69,0L69,108"
+        android:strokeColor="#33FFFFFF"
+        android:strokeWidth="0.8" />
+    <path
+        android:fillColor="#00000000"
+        android:pathData="M79,0L79,108"
+        android:strokeColor="#33FFFFFF"
+        android:strokeWidth="0.8" />
+    <path
+        android:fillColor="#00000000"
+        android:pathData="M89,0L89,108"
+        android:strokeColor="#33FFFFFF"
+        android:strokeWidth="0.8" />
+    <path
+        android:fillColor="#00000000"
+        android:pathData="M99,0L99,108"
+        android:strokeColor="#33FFFFFF"
+        android:strokeWidth="0.8" />
+    <path
+        android:fillColor="#00000000"
+        android:pathData="M0,9L108,9"
+        android:strokeColor="#33FFFFFF"
+        android:strokeWidth="0.8" />
+    <path
+        android:fillColor="#00000000"
+        android:pathData="M0,19L108,19"
+        android:strokeColor="#33FFFFFF"
+        android:strokeWidth="0.8" />
+    <path
+        android:fillColor="#00000000"
+        android:pathData="M0,29L108,29"
+        android:strokeColor="#33FFFFFF"
+        android:strokeWidth="0.8" />
+    <path
+        android:fillColor="#00000000"
+        android:pathData="M0,39L108,39"
+        android:strokeColor="#33FFFFFF"
+        android:strokeWidth="0.8" />
+    <path
+        android:fillColor="#00000000"
+        android:pathData="M0,49L108,49"
+        android:strokeColor="#33FFFFFF"
+        android:strokeWidth="0.8" />
+    <path
+        android:fillColor="#00000000"
+        android:pathData="M0,59L108,59"
+        android:strokeColor="#33FFFFFF"
+        android:strokeWidth="0.8" />
+    <path
+        android:fillColor="#00000000"
+        android:pathData="M0,69L108,69"
+        android:strokeColor="#33FFFFFF"
+        android:strokeWidth="0.8" />
+    <path
+        android:fillColor="#00000000"
+        android:pathData="M0,79L108,79"
+        android:strokeColor="#33FFFFFF"
+        android:strokeWidth="0.8" />
+    <path
+        android:fillColor="#00000000"
+        android:pathData="M0,89L108,89"
+        android:strokeColor="#33FFFFFF"
+        android:strokeWidth="0.8" />
+    <path
+        android:fillColor="#00000000"
+        android:pathData="M0,99L108,99"
+        android:strokeColor="#33FFFFFF"
+        android:strokeWidth="0.8" />
+    <path
+        android:fillColor="#00000000"
+        android:pathData="M19,29L89,29"
+        android:strokeColor="#33FFFFFF"
+        android:strokeWidth="0.8" />
+    <path
+        android:fillColor="#00000000"
+        android:pathData="M19,39L89,39"
+        android:strokeColor="#33FFFFFF"
+        android:strokeWidth="0.8" />
+    <path
+        android:fillColor="#00000000"
+        android:pathData="M19,49L89,49"
+        android:strokeColor="#33FFFFFF"
+        android:strokeWidth="0.8" />
+    <path
+        android:fillColor="#00000000"
+        android:pathData="M19,59L89,59"
+        android:strokeColor="#33FFFFFF"
+        android:strokeWidth="0.8" />
+    <path
+        android:fillColor="#00000000"
+        android:pathData="M19,69L89,69"
+        android:strokeColor="#33FFFFFF"
+        android:strokeWidth="0.8" />
+    <path
+        android:fillColor="#00000000"
+        android:pathData="M19,79L89,79"
+        android:strokeColor="#33FFFFFF"
+        android:strokeWidth="0.8" />
+    <path
+        android:fillColor="#00000000"
+        android:pathData="M29,19L29,89"
+        android:strokeColor="#33FFFFFF"
+        android:strokeWidth="0.8" />
+    <path
+        android:fillColor="#00000000"
+        android:pathData="M39,19L39,89"
+        android:strokeColor="#33FFFFFF"
+        android:strokeWidth="0.8" />
+    <path
+        android:fillColor="#00000000"
+        android:pathData="M49,19L49,89"
+        android:strokeColor="#33FFFFFF"
+        android:strokeWidth="0.8" />
+    <path
+        android:fillColor="#00000000"
+        android:pathData="M59,19L59,89"
+        android:strokeColor="#33FFFFFF"
+        android:strokeWidth="0.8" />
+    <path
+        android:fillColor="#00000000"
+        android:pathData="M69,19L69,89"
+        android:strokeColor="#33FFFFFF"
+        android:strokeWidth="0.8" />
+    <path
+        android:fillColor="#00000000"
+        android:pathData="M79,19L79,89"
+        android:strokeColor="#33FFFFFF"
+        android:strokeWidth="0.8" />
+</vector>

packages/play-standalone/android/app/src/main/res/layout/activity_main.xml

@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="utf-8"?>
+<androidx.coordinatorlayout.widget.CoordinatorLayout xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:app="http://schemas.android.com/apk/res-auto"
+    xmlns:tools="http://schemas.android.com/tools"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent"
+    tools:context=".MainActivity">
+
+    <WebView
+        android:layout_width="match_parent"
+        android:layout_height="match_parent" />
+</androidx.coordinatorlayout.widget.CoordinatorLayout>

packages/play-standalone/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="utf-8"?>
+<adaptive-icon xmlns:android="http://schemas.android.com/apk/res/android">
+    <background android:drawable="@color/ic_launcher_background"/>
+    <foreground android:drawable="@mipmap/ic_launcher_foreground"/>
+</adaptive-icon>

packages/play-standalone/android/app/src/main/res/mipmap-anydpi-v26/ic_launcher_round.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="utf-8"?>
+<adaptive-icon xmlns:android="http://schemas.android.com/apk/res/android">
+    <background android:drawable="@color/ic_launcher_background"/>
+    <foreground android:drawable="@mipmap/ic_launcher_foreground"/>
+</adaptive-icon>

packages/play-standalone/android/app/src/main/res/values/ic_launcher_background.xml

@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="utf-8"?>
+<resources>
+    <color name="ic_launcher_background">#FFFFFF</color>
+</resources>

packages/play-standalone/android/app/src/main/res/values/strings.xml

@@ -0,0 +1,7 @@
+<?xml version='1.0' encoding='utf-8'?>
+<resources>
+    <string name="app_name">Nestri Play</string>
+    <string name="title_activity_main">Nestri Play</string>
+    <string name="package_name">com.nestri.play</string>
+    <string name="custom_url_scheme">com.nestri.play</string>
+</resources>

packages/play-standalone/android/app/src/main/res/values/styles.xml

@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="utf-8"?>
+<resources>
+
+    <!-- Base application theme. -->
+    <style name="AppTheme" parent="Theme.AppCompat.Light.DarkActionBar">
+        <!-- Customize your theme here. -->
+        <item name="colorPrimary">@color/colorPrimary</item>
+        <item name="colorPrimaryDark">@color/colorPrimaryDark</item>
+        <item name="colorAccent">@color/colorAccent</item>
+    </style>
+
+    <style name="AppTheme.NoActionBar" parent="Theme.AppCompat.DayNight.NoActionBar">
+        <item name="windowActionBar">false</item>
+        <item name="windowNoTitle">true</item>
+        <item name="android:background">@null</item>
+    </style>
+
+
+    <style name="AppTheme.NoActionBarLaunch" parent="Theme.SplashScreen">
+        <item name="android:background">@drawable/splash</item>
+    </style>
+</resources>

packages/play-standalone/android/app/src/main/res/xml/file_paths.xml

@@ -0,0 +1,5 @@
+<?xml version="1.0" encoding="utf-8"?>
+<paths xmlns:android="http://schemas.android.com/apk/res/android">
+    <external-path name="my_images" path="." />
+    <cache-path name="my_cache_images" path="." />
+</paths>

packages/play-standalone/android/app/src/test/java/com/getcapacitor/myapp/ExampleUnitTest.java

@@ -0,0 +1,18 @@
+package com.getcapacitor.myapp;
+
+import static org.junit.Assert.*;
+
+import org.junit.Test;
+
+/**
+ * Example local unit test, which will execute on the development machine (host).
+ *
+ * @see <a href="http://d.android.com/tools/testing">Testing documentation</a>
+ */
+public class ExampleUnitTest {
+
+    @Test
+    public void addition_isCorrect() throws Exception {
+        assertEquals(4, 2 + 2);
+    }
+}

packages/play-standalone/android/build.gradle

@@ -0,0 +1,29 @@
+// Top-level build file where you can add configuration options common to all sub-projects/modules.
+
+buildscript {
+    
+    repositories {
+        google()
+        mavenCentral()
+    }
+    dependencies {
+        classpath 'com.android.tools.build:gradle:8.7.2'
+        classpath 'com.google.gms:google-services:4.4.2'
+
+        // NOTE: Do not place your application dependencies here; they belong
+        // in the individual module build.gradle files
+    }
+}
+
+apply from: "variables.gradle"
+
+allprojects {
+    repositories {
+        google()
+        mavenCentral()
+    }
+}
+
+task clean(type: Delete) {
+    delete rootProject.buildDir
+}

packages/play-standalone/android/capacitor.settings.gradle

@@ -0,0 +1,3 @@
+// DO NOT EDIT THIS FILE! IT IS GENERATED EACH TIME "capacitor update" IS RUN
+include ':capacitor-android'
+project(':capacitor-android').projectDir = new File('../../../node_modules/@capacitor/android/capacitor')

packages/play-standalone/android/gradle.properties

@@ -0,0 +1,22 @@
+# Project-wide Gradle settings.
+
+# IDE (e.g. Android Studio) users:
+# Gradle settings configured through the IDE *will override*
+# any settings specified in this file.
+
+# For more details on how to configure your build environment visit
+# http://www.gradle.org/docs/current/userguide/build_environment.html
+
+# Specifies the JVM arguments used for the daemon process.
+# The setting is particularly useful for tweaking memory settings.
+org.gradle.jvmargs=-Xmx1536m
+
+# When configured, Gradle will run in incubating parallel mode.
+# This option should only be used with decoupled projects. More details, visit
+# http://www.gradle.org/docs/current/userguide/multi_project_builds.html#sec:decoupled_projects
+# org.gradle.parallel=true
+
+# AndroidX package structure to make it clearer which packages are bundled with the
+# Android operating system, and which are packaged with your app's APK
+# https://developer.android.com/topic/libraries/support-library/androidx-rn
+android.useAndroidX=true

packages/play-standalone/android/gradle/wrapper/gradle-wrapper.properties

@@ -0,0 +1,7 @@
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.11.1-all.zip
+networkTimeout=10000
+validateDistributionUrl=true
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists

packages/play-standalone/android/gradlew

@@ -0,0 +1,252 @@
+#!/bin/sh
+
+#
+# Copyright © 2015-2021 the original authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+
+##############################################################################
+#
+#   Gradle start up script for POSIX generated by Gradle.
+#
+#   Important for running:
+#
+#   (1) You need a POSIX-compliant shell to run this script. If your /bin/sh is
+#       noncompliant, but you have some other compliant shell such as ksh or
+#       bash, then to run this script, type that shell name before the whole
+#       command line, like:
+#
+#           ksh Gradle
+#
+#       Busybox and similar reduced shells will NOT work, because this script
+#       requires all of these POSIX shell features:
+#         * functions;
+#         * expansions «$var», «${var}», «${var:-default}», «${var+SET}»,
+#           «${var#prefix}», «${var%suffix}», and «$( cmd )»;
+#         * compound commands having a testable exit status, especially «case»;
+#         * various built-in commands including «command», «set», and «ulimit».
+#
+#   Important for patching:
+#
+#   (2) This script targets any POSIX shell, so it avoids extensions provided
+#       by Bash, Ksh, etc; in particular arrays are avoided.
+#
+#       The "traditional" practice of packing multiple parameters into a
+#       space-separated string is a well documented source of bugs and security
+#       problems, so this is (mostly) avoided, by progressively accumulating
+#       options in "$@", and eventually passing that to Java.
+#
+#       Where the inherited environment variables (DEFAULT_JVM_OPTS, JAVA_OPTS,
+#       and GRADLE_OPTS) rely on word-splitting, this is performed explicitly;
+#       see the in-line comments for details.
+#
+#       There are tweaks for specific operating systems such as AIX, CygWin,
+#       Darwin, MinGW, and NonStop.
+#
+#   (3) This script is generated from the Groovy template
+#       https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt
+#       within the Gradle project.
+#
+#       You can find Gradle at https://github.com/gradle/gradle/.
+#
+##############################################################################
+
+# Attempt to set APP_HOME
+
+# Resolve links: $0 may be a link
+app_path=$0
+
+# Need this for daisy-chained symlinks.
+while
+    APP_HOME=${app_path%"${app_path##*/}"}  # leaves a trailing /; empty if no leading path
+    [ -h "$app_path" ]
+do
+    ls=$( ls -ld "$app_path" )
+    link=${ls#*' -> '}
+    case $link in             #(
+      /*)   app_path=$link ;; #(
+      *)    app_path=$APP_HOME$link ;;
+    esac
+done
+
+# This is normally unused
+# shellcheck disable=SC2034
+APP_BASE_NAME=${0##*/}
+# Discard cd standard output in case $CDPATH is set (https://github.com/gradle/gradle/issues/25036)
+APP_HOME=$( cd -P "${APP_HOME:-./}" > /dev/null && printf '%s
+' "$PWD" ) || exit
+
+# Use the maximum available, or set MAX_FD != -1 to use that value.
+MAX_FD=maximum
+
+warn () {
+    echo "$*"
+} >&2
+
+die () {
+    echo
+    echo "$*"
+    echo
+    exit 1
+} >&2
+
+# OS specific support (must be 'true' or 'false').
+cygwin=false
+msys=false
+darwin=false
+nonstop=false
+case "$( uname )" in                #(
+  CYGWIN* )         cygwin=true  ;; #(
+  Darwin* )         darwin=true  ;; #(
+  MSYS* | MINGW* )  msys=true    ;; #(
+  NONSTOP* )        nonstop=true ;;
+esac
+
+CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
+
+
+# Determine the Java command to use to start the JVM.
+if [ -n "$JAVA_HOME" ] ; then
+    if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
+        # IBM's JDK on AIX uses strange locations for the executables
+        JAVACMD=$JAVA_HOME/jre/sh/java
+    else
+        JAVACMD=$JAVA_HOME/bin/java
+    fi
+    if [ ! -x "$JAVACMD" ] ; then
+        die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+    fi
+else
+    JAVACMD=java
+    if ! command -v java >/dev/null 2>&1
+    then
+        die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
+
+Please set the JAVA_HOME variable in your environment to match the
+location of your Java installation."
+    fi
+fi
+
+# Increase the maximum file descriptors if we can.
+if ! "$cygwin" && ! "$darwin" && ! "$nonstop" ; then
+    case $MAX_FD in #(
+      max*)
+        # In POSIX sh, ulimit -H is undefined. That's why the result is checked to see if it worked.
+        # shellcheck disable=SC2039,SC3045
+        MAX_FD=$( ulimit -H -n ) ||
+            warn "Could not query maximum file descriptor limit"
+    esac
+    case $MAX_FD in  #(
+      '' | soft) :;; #(
+      *)
+        # In POSIX sh, ulimit -n is undefined. That's why the result is checked to see if it worked.
+        # shellcheck disable=SC2039,SC3045
+        ulimit -n "$MAX_FD" ||
+            warn "Could not set maximum file descriptor limit to $MAX_FD"
+    esac
+fi
+
+# Collect all arguments for the java command, stacking in reverse order:
+#   * args from the command line
+#   * the main class name
+#   * -classpath
+#   * -D...appname settings
+#   * --module-path (only if needed)
+#   * DEFAULT_JVM_OPTS, JAVA_OPTS, and GRADLE_OPTS environment variables.
+
+# For Cygwin or MSYS, switch paths to Windows format before running java
+if "$cygwin" || "$msys" ; then
+    APP_HOME=$( cygpath --path --mixed "$APP_HOME" )
+    CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" )
+
+    JAVACMD=$( cygpath --unix "$JAVACMD" )
+
+    # Now convert the arguments - kludge to limit ourselves to /bin/sh
+    for arg do
+        if
+            case $arg in                                #(
+              -*)   false ;;                            # don't mess with options #(
+              /?*)  t=${arg#/} t=/${t%%/*}              # looks like a POSIX filepath
+                    [ -e "$t" ] ;;                      #(
+              *)    false ;;
+            esac
+        then
+            arg=$( cygpath --path --ignore --mixed "$arg" )
+        fi
+        # Roll the args list around exactly as many times as the number of
+        # args, so each arg winds up back in the position where it started, but
+        # possibly modified.
+        #
+        # NB: a `for` loop captures its iteration list before it begins, so
+        # changing the positional parameters here affects neither the number of
+        # iterations, nor the values presented in `arg`.
+        shift                   # remove old arg
+        set -- "$@" "$arg"      # push replacement arg
+    done
+fi
+
+
+# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"'
+
+# Collect all arguments for the java command:
+#   * DEFAULT_JVM_OPTS, JAVA_OPTS, JAVA_OPTS, and optsEnvironmentVar are not allowed to contain shell fragments,
+#     and any embedded shellness will be escaped.
+#   * For example: A user cannot expect ${Hostname} to be expanded, as it is an environment variable and will be
+#     treated as '${Hostname}' itself on the command line.
+
+set -- \
+        "-Dorg.gradle.appname=$APP_BASE_NAME" \
+        -classpath "$CLASSPATH" \
+        org.gradle.wrapper.GradleWrapperMain \
+        "$@"
+
+# Stop when "xargs" is not available.
+if ! command -v xargs >/dev/null 2>&1
+then
+    die "xargs is not available"
+fi
+
+# Use "xargs" to parse quoted args.
+#
+# With -n1 it outputs one arg per line, with the quotes and backslashes removed.
+#
+# In Bash we could simply go:
+#
+#   readarray ARGS < <( xargs -n1 <<<"$var" ) &&
+#   set -- "${ARGS[@]}" "$@"
+#
+# but POSIX shell has neither arrays nor command substitution, so instead we
+# post-process each arg (as a line of input to sed) to backslash-escape any
+# character that might be a shell metacharacter, then use eval to reverse
+# that process (while maintaining the separation between arguments), and wrap
+# the whole thing up as a single "set" statement.
+#
+# This will of course break if any of these variables contains a newline or
+# an unmatched quote.
+#
+
+eval "set -- $(
+        printf '%s\n' "$DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS" |
+        xargs -n1 |
+        sed ' s~[^-[:alnum:]+,./:=@_]~\\&~g; ' |
+        tr '\n' ' '
+    )" '"$@"'
+
+exec "$JAVACMD" "$@"

packages/play-standalone/android/gradlew.bat

@@ -0,0 +1,94 @@
+@rem
+@rem Copyright 2015 the original author or authors.
+@rem
+@rem Licensed under the Apache License, Version 2.0 (the "License");
+@rem you may not use this file except in compliance with the License.
+@rem You may obtain a copy of the License at
+@rem
+@rem      https://www.apache.org/licenses/LICENSE-2.0
+@rem
+@rem Unless required by applicable law or agreed to in writing, software
+@rem distributed under the License is distributed on an "AS IS" BASIS,
+@rem WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+@rem See the License for the specific language governing permissions and
+@rem limitations under the License.
+@rem
+@rem SPDX-License-Identifier: Apache-2.0
+@rem
+
+@if "%DEBUG%"=="" @echo off
+@rem ##########################################################################
+@rem
+@rem  Gradle startup script for Windows
+@rem
+@rem ##########################################################################
+
+@rem Set local scope for the variables with windows NT shell
+if "%OS%"=="Windows_NT" setlocal
+
+set DIRNAME=%~dp0
+if "%DIRNAME%"=="" set DIRNAME=.
+@rem This is normally unused
+set APP_BASE_NAME=%~n0
+set APP_HOME=%DIRNAME%
+
+@rem Resolve any "." and ".." in APP_HOME to make it shorter.
+for %%i in ("%APP_HOME%") do set APP_HOME=%%~fi
+
+@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
+set DEFAULT_JVM_OPTS="-Xmx64m" "-Xms64m"
+
+@rem Find java.exe
+if defined JAVA_HOME goto findJavaFromJavaHome
+
+set JAVA_EXE=java.exe
+%JAVA_EXE% -version >NUL 2>&1
+if %ERRORLEVEL% equ 0 goto execute
+
+echo. 1>&2
+echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 1>&2
+echo. 1>&2
+echo Please set the JAVA_HOME variable in your environment to match the 1>&2
+echo location of your Java installation. 1>&2
+
+goto fail
+
+:findJavaFromJavaHome
+set JAVA_HOME=%JAVA_HOME:"=%
+set JAVA_EXE=%JAVA_HOME%/bin/java.exe
+
+if exist "%JAVA_EXE%" goto execute
+
+echo. 1>&2
+echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 1>&2
+echo. 1>&2
+echo Please set the JAVA_HOME variable in your environment to match the 1>&2
+echo location of your Java installation. 1>&2
+
+goto fail
+
+:execute
+@rem Setup the command line
+
+set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
+
+
+@rem Execute Gradle
+"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %*
+
+:end
+@rem End local scope for the variables with windows NT shell
+if %ERRORLEVEL% equ 0 goto mainEnd
+
+:fail
+rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
+rem the _cmd.exe /c_ return code!
+set EXIT_CODE=%ERRORLEVEL%
+if %EXIT_CODE% equ 0 set EXIT_CODE=1
+if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE%
+exit /b %EXIT_CODE%
+
+:mainEnd
+if "%OS%"=="Windows_NT" endlocal
+
+:omega

packages/play-standalone/android/settings.gradle

@@ -0,0 +1,5 @@
+include ':app'
+include ':capacitor-cordova-android-plugins'
+project(':capacitor-cordova-android-plugins').projectDir = new File('./capacitor-cordova-android-plugins/')
+
+apply from: 'capacitor.settings.gradle'

packages/play-standalone/android/variables.gradle

@@ -0,0 +1,16 @@
+ext {
+    minSdkVersion = 23
+    compileSdkVersion = 35
+    targetSdkVersion = 35
+    androidxActivityVersion = '1.9.2'
+    androidxAppCompatVersion = '1.7.0'
+    androidxCoordinatorLayoutVersion = '1.2.0'
+    androidxCoreVersion = '1.15.0'
+    androidxFragmentVersion = '1.8.4'
+    coreSplashScreenVersion = '1.0.1'
+    androidxWebkitVersion = '1.12.1'
+    junitVersion = '4.13.2'
+    androidxJunitVersion = '1.2.1'
+    androidxEspressoCoreVersion = '3.6.1'
+    cordovaAndroidVersion = '10.1.1'
+}

packages/play-standalone/astro.config.mjs

@@ -2,12 +2,10 @@
 import { defineConfig, envField } from "astro/config";
 import node from "@astrojs/node";
 
+
 // https://astro.build/config
 export default defineConfig({
-  adapter: node({
-    mode: 'standalone',
-  }),
-  output: "server",
+  output: "static",
   server: {
     "host": "0.0.0.0",
     "port": 3000,

packages/play-standalone/capacitor.config.ts

@@ -0,0 +1,9 @@
+import type { CapacitorConfig } from '@capacitor/cli';
+
+const config: CapacitorConfig = {
+  appId: 'com.nestri.play',
+  appName: 'Nestri Play',
+  webDir: 'dist'
+};
+
+export default config;

packages/play-standalone/package.json

@@ -5,12 +5,18 @@
   "scripts": {
     "dev": "astro dev",
     "build": "astro build",
+    "build:cap": "astro build && npx cap copy",
     "preview": "astro preview",
-    "astro": "astro"
+    "astro": "astro",
+    "sync:android": "npm run build && npx cap sync android"
   },
   "dependencies": {
-    "@astrojs/node": "9.5.0",
+    "@astrojs/node": "^9.4.2",
+    "@capacitor/android": "^7.4.3",
+    "@capacitor/cli": "^7.4.3",
+    "@capacitor/core": "^7.4.3",
+    "@capacitor/ios": "^7.4.3",
     "@nestri/input": "*",
-    "astro": "5.15.1"
+    "astro": "5.14.5"
   }
 }

packages/play-standalone/src/pages/index.astro

@@ -0,0 +1,138 @@
+---
+import { ClientRouter } from "astro:transitions";
+import { navigate } from "astro:transitions/client";
+---
+<html lang="en">
+<head>
+  <meta charset="UTF-8">
+  <meta name="viewport" content="width=device-width">
+  <title>Nestri Play</title>
+  <ClientRouter />
+  <style>
+    body {
+      font-family: sans-serif;
+      display: flex;
+      justify-content: center;
+      align-items: center;
+      height: 100vh;
+      margin: 0;
+    }
+    form {
+      display: flex;
+      flex-direction: column;
+      gap: 1rem;
+      padding: 2rem;
+      border: 1px solid #ccc;
+      border-radius: 8px;
+    }
+    div {
+      display: flex;
+      flex-direction: column;
+    }
+    label {
+      margin-bottom: 0.5rem;
+    }
+    input {
+      padding: 0.5rem;
+      border: 1px solid #ccc;
+      border-radius: 4px;
+    }
+    button {
+      padding: 0.75rem;
+      border: none;
+      border-radius: 4px;
+      background-color: #007bff;
+      color: white;
+      cursor: pointer;
+    }
+    button:hover {
+      background-color: #0056b3;
+    }
+  </style>
+</head>
+<body>
+  <form id="join-form">
+    <h1>Nestri Play</h1>
+    <div>
+      <label for="room">Room</label>
+      <input type="text" id="room" name="room" required list="room-list">
+      <datalist id="room-list"></datalist>
+    </div>
+    <div>
+      <label for="peerURL">peerURL</label>
+      <input type="text" id="peerURL" name="peerURL" required list="peerURL-list">
+      <datalist id="peerURL-list"></datalist>
+    </div>
+    <button type="submit">Join</button>
+  </form>
+
+  <script>
+    import { navigate } from "astro:transitions/client";
+
+    const roomInput = document.getElementById('room') as HTMLInputElement;
+    const peerURLInput = document.getElementById('peerURL') as HTMLInputElement;
+    const roomList = document.getElementById('room-list');
+    const peerURLList = document.getElementById('peerURL-list');
+
+    // Load values from cookies
+    function getCookie(name) {
+      const value = `; ${document.cookie}`;
+      const parts = value.split(`; ${name}=`);
+      if (parts.length === 2) {
+        const cookieValue = parts.pop()?.split(';').shift();
+        return cookieValue ? decodeURIComponent(cookieValue) : undefined;
+      }
+    }
+
+    function setCookie(name, value, days) {
+      const d = new Date();
+      d.setTime(d.getTime() + (days*24*60*60*1000));
+      const expires = "expires="+ d.toUTCString();
+      document.cookie = name + "=" + encodeURIComponent(value) + ";" + expires + ";path=/";
+    }
+
+    const storedRooms = JSON.parse(getCookie('nestri-rooms') || '[]');
+    const storedPeerURLs = JSON.parse(getCookie('nestri-peerURLs') || '[]');
+
+    if (roomList) {
+      storedRooms.forEach(room => {
+        const option = document.createElement('option');
+        option.value = room;
+        roomList.appendChild(option);
+      });
+    }
+
+    if (peerURLList) {
+      storedPeerURLs.forEach(peerURL => {
+        const option = document.createElement('option');
+        option.value = peerURL;
+        peerURLList.appendChild(option);
+      });
+    }
+
+    if (storedRooms.length > 0 && roomInput) {
+      roomInput.value = storedRooms[0];
+    }
+    if (storedPeerURLs.length > 0 && peerURLInput) {
+      peerURLInput.value = storedPeerURLs[0];
+    }
+
+    document.getElementById('join-form')?.addEventListener('submit', function(event) {
+      event.preventDefault();
+      const room = roomInput.value;
+      const peerURL = peerURLInput.value;
+
+      // Save values to cookies
+      const newRooms = [room, ...storedRooms.filter(r => r !== room)].slice(0, 10);
+      const newPeerURLs = [peerURL, ...storedPeerURLs.filter(p => p !== peerURL)].slice(0, 10);
+
+      setCookie('nestri-rooms', JSON.stringify(newRooms), 365);
+      setCookie('nestri-peerURLs', JSON.stringify(newPeerURLs), 365);
+
+      if (room && peerURL) {
+        navigate(`/play/index.html?peerURL=${encodeURIComponent(peerURL)}#${room}`);
+      }
+    });
+  </script>
+</body>
+</html>

packages/play-standalone/src/pages/play.astro

@@ -1,6 +1,5 @@
 ---
 import DefaultLayout from "../layouts/DefaultLayout.astro";
-const { room } = Astro.params;
 
 // Passing of environment variables to the client side
 // gotta love node and it's ecosystem..
@@ -19,7 +18,7 @@ if (envs_map.size > 0) {
 <DefaultLayout>
 	<h1 id="offlineText" class="offline">Offline</h1>
 	<h1 id="loadingText" class="loading">Warming up the GPU...</h1>
-	<canvas id="playCanvas" class="playCanvas" data-room={room}></canvas>
+	<canvas id="playCanvas" class="playCanvas"></canvas>
 	<div id="ENVS" data-envs={envs}></div>
 </DefaultLayout>
 
@@ -42,9 +41,9 @@ if (envs_map.size > 0) {
 	const offlineText = document.getElementById("offlineText")! as HTMLHeadingElement;
 	const loadingText = document.getElementById("loadingText")! as HTMLHeadingElement;
 
-	const room = canvas.dataset.room;
+	const room = window.location.hash.substring(1);
 	if (!room || room.length <= 0) {
-		throw new Error("Room parameter is required");
+		throw new Error("Room parameter is required in URL hash");
 	}
 
 	offlineText.style.display = "flex";
@@ -90,7 +89,11 @@ if (envs_map.size > 0) {
   let nestriControllers: Controller[] = [];
 
   window.addEventListener("gamepadconnected", (e) => {
+    // Ignore gamepads with id including "nestri"
     console.log("Gamepad connected:", e.gamepad);
+    if (e.gamepad.id.toLowerCase().includes("nestri"))
+      return;
+
     const controller = new Controller({
       webrtc: stream,
       e: e,
@@ -99,6 +102,9 @@ if (envs_map.size > 0) {
   });
   window.addEventListener("gamepaddisconnected", (e) => {
     console.log("Gamepad disconnected:", e.gamepad);
+    if (e.gamepad.id.toLowerCase().includes("nestri"))
+      return;
+
     let disconnected = nestriControllers.find((c) => c.getSlot() === e.gamepad.index);
     if (disconnected) {
       disconnected.dispose();

packages/relay/go.mod

@@ -10,7 +10,7 @@ require (
 	github.com/oklog/ulid/v2 v2.1.1
 	github.com/pion/ice/v4 v4.0.10
 	github.com/pion/interceptor v0.1.41
-	github.com/pion/rtp v1.8.25
+	github.com/pion/rtp v1.8.24
 	github.com/pion/webrtc/v4 v4.1.6
 	github.com/prometheus/client_golang v1.23.2
 	google.golang.org/protobuf v1.36.10
@@ -30,17 +30,17 @@ require (
 	github.com/gorilla/websocket v1.5.3 // indirect
 	github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
 	github.com/huin/goupnp v1.3.0 // indirect
-	github.com/ipfs/go-cid v0.6.0 // indirect
+	github.com/ipfs/go-cid v0.5.0 // indirect
 	github.com/jackpal/go-nat-pmp v1.0.2 // indirect
 	github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
-	github.com/klauspost/compress v1.18.1 // indirect
+	github.com/klauspost/compress v1.18.0 // indirect
 	github.com/klauspost/cpuid/v2 v2.3.0 // indirect
 	github.com/koron/go-ssdp v0.1.0 // indirect
 	github.com/libp2p/go-buffer-pool v0.1.0 // indirect
 	github.com/libp2p/go-flow-metrics v0.3.0 // indirect
 	github.com/libp2p/go-libp2p-asn-util v0.4.1 // indirect
 	github.com/libp2p/go-msgio v0.3.0 // indirect
-	github.com/libp2p/go-netroute v0.4.0 // indirect
+	github.com/libp2p/go-netroute v0.3.0 // indirect
 	github.com/libp2p/go-yamux/v5 v5.1.0 // indirect
 	github.com/libp2p/zeroconf/v2 v2.2.0 // indirect
 	github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
@@ -71,13 +71,13 @@ require (
 	github.com/pion/sdp/v3 v3.0.16 // indirect
 	github.com/pion/srtp/v3 v3.0.8 // indirect
 	github.com/pion/stun v0.6.1 // indirect
-	github.com/pion/stun/v3 v3.0.1 // indirect
+	github.com/pion/stun/v3 v3.0.0 // indirect
 	github.com/pion/transport/v2 v2.2.10 // indirect
 	github.com/pion/transport/v3 v3.0.8 // indirect
-	github.com/pion/turn/v4 v4.1.2 // indirect
+	github.com/pion/turn/v4 v4.1.1 // indirect
 	github.com/prometheus/client_model v0.6.2 // indirect
-	github.com/prometheus/common v0.67.2 // indirect
-	github.com/prometheus/procfs v0.19.2 // indirect
+	github.com/prometheus/common v0.67.1 // indirect
+	github.com/prometheus/procfs v0.17.0 // indirect
 	github.com/quic-go/qpack v0.5.1 // indirect
 	github.com/quic-go/quic-go v0.55.0 // indirect
 	github.com/quic-go/webtransport-go v0.9.0 // indirect
@@ -91,12 +91,12 @@ require (
 	go.uber.org/zap v1.27.0 // indirect
 	go.yaml.in/yaml/v2 v2.4.3 // indirect
 	golang.org/x/crypto v0.43.0 // indirect
-	golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 // indirect
+	golang.org/x/exp v0.0.0-20251017212417-90e834f514db // indirect
 	golang.org/x/mod v0.29.0 // indirect
 	golang.org/x/net v0.46.0 // indirect
 	golang.org/x/sync v0.17.0 // indirect
 	golang.org/x/sys v0.37.0 // indirect
-	golang.org/x/telemetry v0.0.0-20251028164327-d7a2859f34e8 // indirect
+	golang.org/x/telemetry v0.0.0-20251014153721-24f779f6aaef // indirect
 	golang.org/x/text v0.30.0 // indirect
 	golang.org/x/time v0.14.0 // indirect
 	golang.org/x/tools v0.38.0 // indirect

packages/relay/go.sum

@@ -71,8 +71,8 @@ github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs
 github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc=
 github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8=
-github.com/ipfs/go-cid v0.6.0 h1:DlOReBV1xhHBhhfy/gBNNTSyfOM6rLiIx9J7A4DGf30=
-github.com/ipfs/go-cid v0.6.0/go.mod h1:NC4kS1LZjzfhK40UGmpXv5/qD2kcMzACYJNntCUiDhQ=
+github.com/ipfs/go-cid v0.5.0 h1:goEKKhaGm0ul11IHA7I6p1GmKz8kEYniqFopaB5Otwg=
+github.com/ipfs/go-cid v0.5.0/go.mod h1:0L7vmeNXpQpUS9vt+yEARkJ8rOg43DF3iPgn4GIN0mk=
 github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus=
 github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
 github.com/jbenet/go-temp-err-catcher v0.1.0 h1:zpb3ZH6wIE8Shj2sKS+khgRvf7T7RABoLk/+KKHggpk=
@@ -82,8 +82,8 @@ github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCV
 github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
-github.com/klauspost/compress v1.18.1 h1:bcSGx7UbpBqMChDtsF28Lw6v/G94LPrrbMbdC3JH2co=
-github.com/klauspost/compress v1.18.1/go.mod h1:ZQFFVG+MdnR0P+l6wpXgIL4NTtwiKIdBnrBd8Nrxr+0=
+github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
+github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
 github.com/klauspost/cpuid/v2 v2.3.0 h1:S4CRMLnYUhGeDFDqkGriYKdfoFlDnMtqTiI/sFzhA9Y=
 github.com/klauspost/cpuid/v2 v2.3.0/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0=
 github.com/koron/go-ssdp v0.1.0 h1:ckl5x5H6qSNFmi+wCuROvvGUu2FQnMbQrU95IHCcv3Y=
@@ -113,8 +113,8 @@ github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUI
 github.com/libp2p/go-libp2p-testing v0.12.0/go.mod h1:KcGDRXyN7sQCllucn1cOOS+Dmm7ujhfEyXQL5lvkcPg=
 github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0=
 github.com/libp2p/go-msgio v0.3.0/go.mod h1:nyRM819GmVaF9LX3l03RMh10QdOroF++NBbxAb0mmDM=
-github.com/libp2p/go-netroute v0.4.0 h1:sZZx9hyANYUx9PZyqcgE/E1GUG3iEtTZHUEvdtXT7/Q=
-github.com/libp2p/go-netroute v0.4.0/go.mod h1:Nkd5ShYgSMS5MUKy/MU2T57xFoOKvvLR92Lic48LEyA=
+github.com/libp2p/go-netroute v0.3.0 h1:nqPCXHmeNmgTJnktosJ/sIef9hvwYCrsLxXmfNks/oc=
+github.com/libp2p/go-netroute v0.3.0/go.mod h1:Nkd5ShYgSMS5MUKy/MU2T57xFoOKvvLR92Lic48LEyA=
 github.com/libp2p/go-reuseport v0.4.0 h1:nR5KU7hD0WxXCJbmw7r2rhRYruNRl2koHw8fQscQm2s=
 github.com/libp2p/go-reuseport v0.4.0/go.mod h1:ZtI03j/wO5hZVDFo2jKywN6bYKWLOy8Se6DrI2E1cLU=
 github.com/libp2p/go-yamux/v5 v5.1.0 h1:8Qlxj4E9JGJAQVW6+uj2o7mqkqsIVlSUGmTWhlXzoHE=
@@ -199,8 +199,8 @@ github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA=
 github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8=
 github.com/pion/rtcp v1.2.16 h1:fk1B1dNW4hsI78XUCljZJlC4kZOPk67mNRuQ0fcEkSo=
 github.com/pion/rtcp v1.2.16/go.mod h1:/as7VKfYbs5NIb4h6muQ35kQF/J0ZVNz2Z3xKoCBYOo=
-github.com/pion/rtp v1.8.25 h1:b8+y44GNbwOJTYWuVan7SglX/hMlicVCAtL50ztyZHw=
-github.com/pion/rtp v1.8.25/go.mod h1:rF5nS1GqbR7H/TCpKwylzeq6yDM+MM6k+On5EgeThEM=
+github.com/pion/rtp v1.8.24 h1:+ICyZXUQDv95EsHN70RrA4XKJf5MGWyC6QQc1u6/ynI=
+github.com/pion/rtp v1.8.24/go.mod h1:rF5nS1GqbR7H/TCpKwylzeq6yDM+MM6k+On5EgeThEM=
 github.com/pion/sctp v1.8.40 h1:bqbgWYOrUhsYItEnRObUYZuzvOMsVplS3oNgzedBlG8=
 github.com/pion/sctp v1.8.40/go.mod h1:SPBBUENXE6ThkEksN5ZavfAhFYll+h+66ZiG6IZQuzo=
 github.com/pion/sdp/v3 v3.0.16 h1:0dKzYO6gTAvuLaAKQkC02eCPjMIi4NuAr/ibAwrGDCo=
@@ -209,16 +209,16 @@ github.com/pion/srtp/v3 v3.0.8 h1:RjRrjcIeQsilPzxvdaElN0CpuQZdMvcl9VZ5UY9suUM=
 github.com/pion/srtp/v3 v3.0.8/go.mod h1:2Sq6YnDH7/UDCvkSoHSDNDeyBcFgWL0sAVycVbAsXFg=
 github.com/pion/stun v0.6.1 h1:8lp6YejULeHBF8NmV8e2787BogQhduZugh5PdhDyyN4=
 github.com/pion/stun v0.6.1/go.mod h1:/hO7APkX4hZKu/D0f2lHzNyvdkTGtIy3NDmLR7kSz/8=
-github.com/pion/stun/v3 v3.0.1 h1:jx1uUq6BdPihF0yF33Jj2mh+C9p0atY94IkdnW174kA=
-github.com/pion/stun/v3 v3.0.1/go.mod h1:RHnvlKFg+qHgoKIqtQWMOJF52wsImCAf/Jh5GjX+4Tw=
+github.com/pion/stun/v3 v3.0.0 h1:4h1gwhWLWuZWOJIJR9s2ferRO+W3zA/b6ijOI6mKzUw=
+github.com/pion/stun/v3 v3.0.0/go.mod h1:HvCN8txt8mwi4FBvS3EmDghW6aQJ24T+y+1TKjB5jyU=
 github.com/pion/transport/v2 v2.2.1/go.mod h1:cXXWavvCnFF6McHTft3DWS9iic2Mftcz1Aq29pGcU5g=
 github.com/pion/transport/v2 v2.2.4/go.mod h1:q2U/tf9FEfnSBGSW6w5Qp5PFWRLRj3NjLhCCgpRK4p0=
 github.com/pion/transport/v2 v2.2.10 h1:ucLBLE8nuxiHfvkFKnkDQRYWYfp8ejf4YBOPfaQpw6Q=
 github.com/pion/transport/v2 v2.2.10/go.mod h1:sq1kSLWs+cHW9E+2fJP95QudkzbK7wscs8yYgQToO5E=
 github.com/pion/transport/v3 v3.0.8 h1:oI3myyYnTKUSTthu/NZZ8eu2I5sHbxbUNNFW62olaYc=
 github.com/pion/transport/v3 v3.0.8/go.mod h1:+c2eewC5WJQHiAA46fkMMzoYZSuGzA/7E2FPrOYHctQ=
-github.com/pion/turn/v4 v4.1.2 h1:Em2svpl6aBFa88dLhxypMUzaLjC79kWZWx8FIov01cc=
-github.com/pion/turn/v4 v4.1.2/go.mod h1:ISYWfZYy0Z3tXzRpyYZHTL+U23yFQIspfxogdQ8pn9Y=
+github.com/pion/turn/v4 v4.1.1 h1:9UnY2HB99tpDyz3cVVZguSxcqkJ1DsTSZ+8TGruh4fc=
+github.com/pion/turn/v4 v4.1.1/go.mod h1:2123tHk1O++vmjI5VSD0awT50NywDAq5A2NNNU4Jjs8=
 github.com/pion/webrtc/v4 v4.1.6 h1:srHH2HwvCGwPba25EYJgUzgLqCQoXl1VCUnrGQMSzUw=
 github.com/pion/webrtc/v4 v4.1.6/go.mod h1:wKecGRlkl3ox/As/MYghJL+b/cVXMEhoPMJWPuGQFhU=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -231,11 +231,11 @@ github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:
 github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk=
 github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE=
 github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
-github.com/prometheus/common v0.67.2 h1:PcBAckGFTIHt2+L3I33uNRTlKTplNzFctXcWhPyAEN8=
-github.com/prometheus/common v0.67.2/go.mod h1:63W3KZb1JOKgcjlIr64WW/LvFGAqKPj0atm+knVGEko=
+github.com/prometheus/common v0.67.1 h1:OTSON1P4DNxzTg4hmKCc37o4ZAZDv0cfXLkOt0oEowI=
+github.com/prometheus/common v0.67.1/go.mod h1:RpmT9v35q2Y+lsieQsdOh5sXZ6ajUGC8NjZAmr8vb0Q=
 github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
-github.com/prometheus/procfs v0.19.2 h1:zUMhqEW66Ex7OXIiDkll3tl9a1ZdilUOd/F6ZXw4Vws=
-github.com/prometheus/procfs v0.19.2/go.mod h1:M0aotyiemPhBCM0z5w87kL22CxfcH05ZpYlu+b4J7mw=
+github.com/prometheus/procfs v0.17.0 h1:FuLQ+05u4ZI+SS/w9+BWEM2TXiHKsUQ9TADiRH7DuK0=
+github.com/prometheus/procfs v0.17.0/go.mod h1:oPQLaDAMRbA+u8H5Pbfq+dl3VDAvHxMUOVhe0wYB2zw=
 github.com/quic-go/qpack v0.5.1 h1:giqksBPnT/HDtZ6VhtFKgoLOWmlyo9Ei6u9PqzIMbhI=
 github.com/quic-go/qpack v0.5.1/go.mod h1:+PC4XFrEskIVkcLzpEkbLqq1uCoxPhQuvK5rH1ZgaEg=
 github.com/quic-go/quic-go v0.55.0 h1:zccPQIqYCXDt5NmcEabyYvOnomjs8Tlwl7tISjJh9Mk=
@@ -323,8 +323,8 @@ golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1m
 golang.org/x/crypto v0.43.0 h1:dduJYIi3A3KOfdGOHX8AVZ/jGiyPa3IbBozJ5kNuE04=
 golang.org/x/crypto v0.43.0/go.mod h1:BFbav4mRNlXJL4wNeejLpWxB7wMbc79PdRGhWKncxR0=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
-golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 h1:mgKeJMpvi0yx/sU5GsxQ7p6s2wtOnGAHZWCHUM4KGzY=
-golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70=
+golang.org/x/exp v0.0.0-20251017212417-90e834f514db h1:by6IehL4BH5k3e3SJmcoNbOobMey2SLpAF79iPOEBvw=
+golang.org/x/exp v0.0.0-20251017212417-90e834f514db/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70=
 golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
@@ -396,8 +396,8 @@ golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ=
 golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
-golang.org/x/telemetry v0.0.0-20251028164327-d7a2859f34e8 h1:DwMAzqwLj2rVin75cRFh1kfhwQY3hyHrU1oCEDZXPmQ=
-golang.org/x/telemetry v0.0.0-20251028164327-d7a2859f34e8/go.mod h1:Pi4ztBfryZoJEkyFTI5/Ocsu2jXyDr6iSdgJiYE/uwE=
+golang.org/x/telemetry v0.0.0-20251014153721-24f779f6aaef h1:5xFtU4tmJMJSxSeDlr1dgBff2tDXrq0laLdS1EA3LYw=
+golang.org/x/telemetry v0.0.0-20251014153721-24f779f6aaef/go.mod h1:Pi4ztBfryZoJEkyFTI5/Ocsu2jXyDr6iSdgJiYE/uwE=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=

packages/relay/internal/common/common.go

@@ -2,7 +2,6 @@ package common
 
 import (
 	"fmt"
-	"github.com/pion/interceptor/pkg/nack"
 	"log/slog"
 	"strconv"
 
@@ -27,125 +26,24 @@ func InitWebRTCAPI() error {
 	mediaEngine := &webrtc.MediaEngine{}
 
 	// Register our extensions
-	if err = RegisterExtensions(mediaEngine); err != nil {
+	if err := RegisterExtensions(mediaEngine); err != nil {
 		return fmt.Errorf("failed to register extensions: %w", err)
 	}
 
-	// Register codecs
-	for _, codec := range []webrtc.RTPCodecParameters{
-		{
-			RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus, ClockRate: 48000, Channels: 2, SDPFmtpLine: "minptime=10;useinbandfec=1"},
-			PayloadType:        111,
-		},
-	} {
-		if err = mediaEngine.RegisterCodec(codec, webrtc.RTPCodecTypeAudio); err != nil {
+	// Default codecs cover our needs
+	err = mediaEngine.RegisterDefaultCodecs()
+	if err != nil {
 		return err
 	}
-	}
-
-	videoRTCPFeedback := []webrtc.RTCPFeedback{{"nack", ""}, {"nack", "pli"}}
-	for _, codec := range []webrtc.RTPCodecParameters{
-		{
-			RTPCodecCapability: webrtc.RTPCodecCapability{
-				MimeType: webrtc.MimeTypeH264, ClockRate: 90000,
-				SDPFmtpLine:  "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42001f",
-				RTCPFeedback: videoRTCPFeedback,
-			},
-			PayloadType: 102,
-		},
-		{
-			RTPCodecCapability: webrtc.RTPCodecCapability{
-				MimeType: webrtc.MimeTypeH264, ClockRate: 90000,
-				SDPFmtpLine:  "level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42001f",
-				RTCPFeedback: videoRTCPFeedback,
-			},
-			PayloadType: 104,
-		},
-		{
-			RTPCodecCapability: webrtc.RTPCodecCapability{
-				MimeType: webrtc.MimeTypeH264, ClockRate: 90000,
-				SDPFmtpLine:  "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f",
-				RTCPFeedback: videoRTCPFeedback,
-			},
-			PayloadType: 106,
-		},
-		{
-			RTPCodecCapability: webrtc.RTPCodecCapability{
-				MimeType: webrtc.MimeTypeH264, ClockRate: 90000,
-				SDPFmtpLine:  "level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=42e01f",
-				RTCPFeedback: videoRTCPFeedback,
-			},
-			PayloadType: 108,
-		},
-		{
-			RTPCodecCapability: webrtc.RTPCodecCapability{
-				MimeType: webrtc.MimeTypeH264, ClockRate: 90000,
-				SDPFmtpLine:  "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=4d001f",
-				RTCPFeedback: videoRTCPFeedback,
-			},
-			PayloadType: 127,
-		},
-		{
-			RTPCodecCapability: webrtc.RTPCodecCapability{
-				MimeType:     webrtc.MimeTypeH264,
-				ClockRate:    90000,
-				SDPFmtpLine:  "level-asymmetry-allowed=1;packetization-mode=0;profile-level-id=4d001f",
-				RTCPFeedback: videoRTCPFeedback,
-			},
-			PayloadType: 39,
-		},
-		{
-			RTPCodecCapability: webrtc.RTPCodecCapability{
-				MimeType:     webrtc.MimeTypeH265,
-				ClockRate:    90000,
-				RTCPFeedback: videoRTCPFeedback,
-			},
-			PayloadType: 116,
-		},
-		{
-			RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeAV1, ClockRate: 90000, RTCPFeedback: videoRTCPFeedback},
-			PayloadType:        45,
-		},
-		{
-			RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP9, ClockRate: 90000, SDPFmtpLine: "profile-id=0", RTCPFeedback: videoRTCPFeedback},
-			PayloadType:        98,
-		},
-		{
-			RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP9, ClockRate: 90000, SDPFmtpLine: "profile-id=2", RTCPFeedback: videoRTCPFeedback},
-			PayloadType:        100,
-		},
-		{
-			RTPCodecCapability: webrtc.RTPCodecCapability{
-				MimeType: webrtc.MimeTypeH264, ClockRate: 90000,
-				SDPFmtpLine:  "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=64001f",
-				RTCPFeedback: videoRTCPFeedback,
-			},
-			PayloadType: 112,
-		},
-	} {
-		if err = mediaEngine.RegisterCodec(codec, webrtc.RTPCodecTypeVideo); err != nil {
-			return err
-		}
-	}
 
 	// Interceptor registry
 	interceptorRegistry := &interceptor.Registry{}
 
-	// Register our interceptors..
-	nackGenFactory, err := nack.NewGeneratorInterceptor()
+	// Use default set
+	err = webrtc.RegisterDefaultInterceptors(mediaEngine, interceptorRegistry)
 	if err != nil {
 		return err
 	}
-	interceptorRegistry.Add(nackGenFactory)
-	nackRespFactory, err := nack.NewResponderInterceptor()
-	if err != nil {
-		return err
-	}
-	interceptorRegistry.Add(nackRespFactory)
-
-	if err = webrtc.ConfigureRTCPReports(interceptorRegistry); err != nil {
-		return err
-	}
 
 	// Setting engine
 	settingEngine := webrtc.SettingEngine{}
@@ -155,7 +53,7 @@ func InitWebRTCAPI() error {
 
 	nat11IP := GetFlags().NAT11IP
 	if len(nat11IP) > 0 {
-		settingEngine.SetNAT1To1IPs([]string{nat11IP}, webrtc.ICECandidateTypeHost)
+		settingEngine.SetNAT1To1IPs([]string{nat11IP}, webrtc.ICECandidateTypeSrflx)
 		slog.Info("Using NAT 1:1 IP for WebRTC", "nat11_ip", nat11IP)
 	}
 

packages/relay/internal/common/flags.go

@@ -125,7 +125,7 @@ func getLocalIP() string {
 		return ""
 	}
 	for _, address := range addrs {
-		if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() && !ipnet.IP.IsPrivate() && !ipnet.IP.IsUnspecified() {
+		if ipnet, ok := address.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
 			if ipnet.IP.To4() != nil || ipnet.IP != nil {
 				return ipnet.IP.String()
 			}

packages/relay/internal/common/ice_helper.go

@@ -1,53 +0,0 @@
-package common
-
-import (
-	"log/slog"
-
-	"github.com/pion/webrtc/v4"
-)
-
-// ICEHelper holds webrtc.ICECandidateInit(s) until remote candidate is set for given webrtc.PeerConnection
-// Held candidates should be flushed at the end of negotiation to ensure all are available for connection
-type ICEHelper struct {
-	candidates []webrtc.ICECandidateInit
-	pc         *webrtc.PeerConnection
-}
-
-func NewICEHelper(pc *webrtc.PeerConnection) *ICEHelper {
-	return &ICEHelper{
-		pc:         pc,
-		candidates: make([]webrtc.ICECandidateInit, 0),
-	}
-}
-
-func (ice *ICEHelper) SetPeerConnection(pc *webrtc.PeerConnection) {
-	ice.pc = pc
-}
-
-func (ice *ICEHelper) AddCandidate(c webrtc.ICECandidateInit) {
-	if ice.pc != nil {
-		if ice.pc.RemoteDescription() != nil {
-			// Add immediately if remote is set
-			if err := ice.pc.AddICECandidate(c); err != nil {
-				slog.Error("Failed to add ICE candidate", "err", err)
-			}
-			// Also flush held candidates automatically
-			ice.FlushHeldCandidates()
-		} else {
-			// Hold in slice until remote is set
-			ice.candidates = append(ice.candidates, c)
-		}
-	}
-}
-
-func (ice *ICEHelper) FlushHeldCandidates() {
-	if ice.pc != nil && len(ice.candidates) > 0 {
-		for _, heldCandidate := range ice.candidates {
-			if err := ice.pc.AddICECandidate(heldCandidate); err != nil {
-				slog.Error("Failed to add held ICE candidate", "err", err)
-			}
-		}
-		// Clear the held candidates
-		ice.candidates = make([]webrtc.ICECandidateInit, 0)
-	}
-}

packages/relay/internal/common/safebufio.go

@@ -3,28 +3,16 @@ package common
 import (
 	"bufio"
 	"encoding/binary"
+	"encoding/json"
 	"errors"
 	"io"
-	gen "relay/internal/proto"
 	"sync"
 
 	"google.golang.org/protobuf/proto"
-	"google.golang.org/protobuf/reflect/protoreflect"
-	"google.golang.org/protobuf/types/known/timestamppb"
 )
 
-// readUvarint reads an unsigned varint from the reader
-func readUvarint(r io.ByteReader) (uint64, error) {
-	return binary.ReadUvarint(r)
-}
-
-// writeUvarint writes an unsigned varint to the writer
-func writeUvarint(w io.Writer, x uint64) error {
-	buf := make([]byte, binary.MaxVarintLen64)
-	n := binary.PutUvarint(buf, x)
-	_, err := w.Write(buf[:n])
-	return err
-}
+// MaxSize is the maximum allowed data size (1MB)
+const MaxSize = 1024 * 1024
 
 // SafeBufioRW wraps a bufio.ReadWriter for sending and receiving JSON and protobufs safely
 type SafeBufioRW struct {
@@ -36,6 +24,83 @@ func NewSafeBufioRW(brw *bufio.ReadWriter) *SafeBufioRW {
 	return &SafeBufioRW{brw: brw}
 }
 
+// SendJSON serializes the given data as JSON and sends it with a 4-byte length prefix
+func (bu *SafeBufioRW) SendJSON(data interface{}) error {
+	bu.mutex.Lock()
+	defer bu.mutex.Unlock()
+
+	jsonData, err := json.Marshal(data)
+	if err != nil {
+		return err
+	}
+
+	if len(jsonData) > MaxSize {
+		return errors.New("JSON data exceeds maximum size")
+	}
+
+	// Write the 4-byte length prefix
+	if err = binary.Write(bu.brw, binary.BigEndian, uint32(len(jsonData))); err != nil {
+		return err
+	}
+
+	// Write the JSON data
+	if _, err = bu.brw.Write(jsonData); err != nil {
+		return err
+	}
+
+	// Flush the writer to ensure data is sent
+	return bu.brw.Flush()
+}
+
+// ReceiveJSON reads a 4-byte length prefix, then reads and unmarshals the JSON
+func (bu *SafeBufioRW) ReceiveJSON(dest interface{}) error {
+	bu.mutex.RLock()
+	defer bu.mutex.RUnlock()
+
+	// Read the 4-byte length prefix
+	var length uint32
+	if err := binary.Read(bu.brw, binary.BigEndian, &length); err != nil {
+		return err
+	}
+
+	if length > MaxSize {
+		return errors.New("received JSON data exceeds maximum size")
+	}
+
+	// Read the JSON data
+	data := make([]byte, length)
+	if _, err := io.ReadFull(bu.brw, data); err != nil {
+		return err
+	}
+
+	return json.Unmarshal(data, dest)
+}
+
+// Receive reads a 4-byte length prefix, then reads the raw data
+func (bu *SafeBufioRW) Receive() ([]byte, error) {
+	bu.mutex.RLock()
+	defer bu.mutex.RUnlock()
+
+	// Read the 4-byte length prefix
+	var length uint32
+	if err := binary.Read(bu.brw, binary.BigEndian, &length); err != nil {
+		return nil, err
+	}
+
+	if length > MaxSize {
+		return nil, errors.New("received data exceeds maximum size")
+	}
+
+	// Read the raw data
+	data := make([]byte, length)
+	if _, err := io.ReadFull(bu.brw, data); err != nil {
+		return nil, err
+	}
+
+	return data, nil
+}
+
+// SendProto serializes the given protobuf message and sends it with a 4-byte length prefix
 func (bu *SafeBufioRW) SendProto(msg proto.Message) error {
 	bu.mutex.Lock()
 	defer bu.mutex.Unlock()
@@ -45,8 +110,12 @@ func (bu *SafeBufioRW) SendProto(msg proto.Message) error {
 		return err
 	}
 
-	// Write varint length prefix
-	if err := writeUvarint(bu.brw, uint64(len(protoData))); err != nil {
+	if len(protoData) > MaxSize {
+		return errors.New("protobuf data exceeds maximum size")
+	}
+
+	// Write the 4-byte length prefix
+	if err = binary.Write(bu.brw, binary.BigEndian, uint32(len(protoData))); err != nil {
 		return err
 	}
 
@@ -55,19 +124,25 @@ func (bu *SafeBufioRW) SendProto(msg proto.Message) error {
 		return err
 	}
 
+	// Flush the writer to ensure data is sent
 	return bu.brw.Flush()
 }
 
+// ReceiveProto reads a 4-byte length prefix, then reads and unmarshals the protobuf
 func (bu *SafeBufioRW) ReceiveProto(msg proto.Message) error {
 	bu.mutex.RLock()
 	defer bu.mutex.RUnlock()
 
-	// Read varint length prefix
-	length, err := readUvarint(bu.brw)
-	if err != nil {
+	// Read the 4-byte length prefix
+	var length uint32
+	if err := binary.Read(bu.brw, binary.BigEndian, &length); err != nil {
 		return err
 	}
 
+	if length > MaxSize {
+		return errors.New("received Protobuf data exceeds maximum size")
+	}
+
 	// Read the Protobuf data
 	data := make([]byte, length)
 	if _, err := io.ReadFull(bu.brw, data); err != nil {
@@ -77,51 +152,24 @@ func (bu *SafeBufioRW) ReceiveProto(msg proto.Message) error {
 	return proto.Unmarshal(data, msg)
 }
 
-type CreateMessageOptions struct {
-	SequenceID string
-	Latency    *gen.ProtoLatencyTracker
+// Write writes raw data to the underlying buffer
+func (bu *SafeBufioRW) Write(data []byte) (int, error) {
+	bu.mutex.Lock()
+	defer bu.mutex.Unlock()
+
+	if len(data) > MaxSize {
+		return 0, errors.New("data exceeds maximum size")
 	}
 
-func CreateMessage(payload proto.Message, payloadType string, opts *CreateMessageOptions) (*gen.ProtoMessage, error) {
-	msg := &gen.ProtoMessage{
-		MessageBase: &gen.ProtoMessageBase{
-			PayloadType: payloadType,
-		},
+	n, err := bu.brw.Write(data)
+	if err != nil {
+		return n, err
 	}
 
-	if opts != nil {
-		if opts.Latency != nil {
-			msg.MessageBase.Latency = opts.Latency
-		} else if opts.SequenceID != "" {
-			msg.MessageBase.Latency = &gen.ProtoLatencyTracker{
-				SequenceId: opts.SequenceID,
-				Timestamps: []*gen.ProtoTimestampEntry{
-					{
-						Stage: "created",
-						Time:  timestamppb.Now(),
-					},
-				},
-			}
-		}
+	// Flush the writer to ensure data is sent
+	if err = bu.brw.Flush(); err != nil {
+		return n, err
 	}
 
-	// Use reflection to set the oneof field automatically
-	msgReflect := msg.ProtoReflect()
-	payloadReflect := payload.ProtoReflect()
-
-	oneofDesc := msgReflect.Descriptor().Oneofs().ByName("payload")
-	if oneofDesc == nil {
-		return nil, errors.New("payload oneof not found")
-	}
-
-	fields := oneofDesc.Fields()
-	for i := 0; i < fields.Len(); i++ {
-		field := fields.Get(i)
-		if field.Message() != nil && field.Message().FullName() == payloadReflect.Descriptor().FullName() {
-			msgReflect.Set(field, protoreflect.ValueOfMessage(payloadReflect))
-			return msg, nil
-		}
-	}
-
-	return nil, errors.New("payload type not found in oneof")
+	return n, nil
 }

packages/relay/internal/connections/datachannel.go

@@ -31,18 +31,16 @@ func NewNestriDataChannel(dc *webrtc.DataChannel) *NestriDataChannel {
 		}
 
 		// Decode message
-		var base gen.ProtoMessage
+		var base gen.ProtoMessageInput
 		if err := proto.Unmarshal(msg.Data, &base); err != nil {
 			slog.Error("failed to decode binary DataChannel message", "err", err)
 			return
 		}
 
-		// Route based on PayloadType
-		if base.MessageBase != nil && len(base.MessageBase.PayloadType) > 0 {
-			if callback, ok := ndc.callbacks[base.MessageBase.PayloadType]; ok {
+		// Handle message type callback
+		if callback, ok := ndc.callbacks["input"]; ok {
 			go callback(msg.Data)
-			}
-		}
+		} // We don't care about unhandled messages
 	})
 
 	return ndc

packages/relay/internal/connections/messages.go

@@ -0,0 +1,94 @@
+package connections
+
+import (
+	"encoding/json"
+	"relay/internal/common"
+
+	"github.com/pion/webrtc/v4"
+)
+
+// MessageBase is the base type for any JSON message
+type MessageBase struct {
+	Type    string                 `json:"payload_type"`
+	Latency *common.LatencyTracker `json:"latency,omitempty"`
+}
+
+type MessageRaw struct {
+	MessageBase
+	Data json.RawMessage `json:"data"`
+}
+
+func NewMessageRaw(t string, data json.RawMessage) *MessageRaw {
+	return &MessageRaw{
+		MessageBase: MessageBase{
+			Type: t,
+		},
+		Data: data,
+	}
+}
+
+type MessageLog struct {
+	MessageBase
+	Level   string `json:"level"`
+	Message string `json:"message"`
+	Time    string `json:"time"`
+}
+
+func NewMessageLog(t string, level, message, time string) *MessageLog {
+	return &MessageLog{
+		MessageBase: MessageBase{
+			Type: t,
+		},
+		Level:   level,
+		Message: message,
+		Time:    time,
+	}
+}
+
+type MessageMetrics struct {
+	MessageBase
+	UsageCPU        float64 `json:"usage_cpu"`
+	UsageMemory     float64 `json:"usage_memory"`
+	Uptime          uint64  `json:"uptime"`
+	PipelineLatency float64 `json:"pipeline_latency"`
+}
+
+func NewMessageMetrics(t string, usageCPU, usageMemory float64, uptime uint64, pipelineLatency float64) *MessageMetrics {
+	return &MessageMetrics{
+		MessageBase: MessageBase{
+			Type: t,
+		},
+		UsageCPU:        usageCPU,
+		UsageMemory:     usageMemory,
+		Uptime:          uptime,
+		PipelineLatency: pipelineLatency,
+	}
+}
+
+type MessageICE struct {
+	MessageBase
+	Candidate webrtc.ICECandidateInit `json:"candidate"`
+}
+
+func NewMessageICE(t string, candidate webrtc.ICECandidateInit) *MessageICE {
+	return &MessageICE{
+		MessageBase: MessageBase{
+			Type: t,
+		},
+		Candidate: candidate,
+	}
+}
+
+type MessageSDP struct {
+	MessageBase
+	SDP webrtc.SessionDescription `json:"sdp"`
+}
+
+func NewMessageSDP(t string, sdp webrtc.SessionDescription) *MessageSDP {
+	return &MessageSDP{
+		MessageBase: MessageBase{
+			Type: t,
+		},
+		SDP: sdp,
+	}
+}

packages/relay/internal/core/core.go

@@ -20,9 +20,9 @@ import (
 	rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager"
 	"github.com/libp2p/go-libp2p/p2p/protocol/ping"
 	"github.com/libp2p/go-libp2p/p2p/security/noise"
-	p2pquic "github.com/libp2p/go-libp2p/p2p/transport/quic"
 	"github.com/libp2p/go-libp2p/p2p/transport/quicreuse"
 	"github.com/libp2p/go-libp2p/p2p/transport/tcp"
+	ws "github.com/libp2p/go-libp2p/p2p/transport/websocket"
 	webtransport "github.com/libp2p/go-libp2p/p2p/transport/webtransport"
 	"github.com/multiformats/go-multiaddr"
 	"github.com/oklog/ulid/v2"
@@ -91,10 +91,10 @@ func NewRelay(ctx context.Context, port int, identityKey crypto.PrivKey) (*Relay
 	listenAddrs := []string{
 		fmt.Sprintf("/ip4/0.0.0.0/tcp/%d", port),                      // IPv4 - Raw TCP
 		fmt.Sprintf("/ip6/::/tcp/%d", port),                           // IPv6 - Raw TCP
+		fmt.Sprintf("/ip4/0.0.0.0/tcp/%d/ws", port),                   // IPv4 - TCP WebSocket
+		fmt.Sprintf("/ip6/::/tcp/%d/ws", port),                        // IPv6 - TCP WebSocket
 		fmt.Sprintf("/ip4/0.0.0.0/udp/%d/quic-v1/webtransport", port), // IPv4 - UDP QUIC WebTransport
 		fmt.Sprintf("/ip6/::/udp/%d/quic-v1/webtransport", port),      // IPv6 - UDP QUIC WebTransport
-		fmt.Sprintf("/ip4/0.0.0.0/udp/%d/quic-v1", port),              // IPv4 - UDP Raw QUIC
-		fmt.Sprintf("/ip6/::/udp/%d/quic-v1", port),                   // IPv6 - UDP Raw QUIC
 	}
 
 	var muAddrs []multiaddr.Multiaddr
@@ -112,8 +112,8 @@ func NewRelay(ctx context.Context, port int, identityKey crypto.PrivKey) (*Relay
 		libp2p.Identity(identityKey),
 		// Enable required transports
 		libp2p.Transport(tcp.NewTCPTransport),
+		libp2p.Transport(ws.New),
 		libp2p.Transport(webtransport.New),
-		libp2p.Transport(p2pquic.NewTransport),
 		// Other options
 		libp2p.ListenAddrs(muAddrs...),
 		libp2p.Security(noise.ID, noise.New),

packages/relay/internal/core/protocol_stream.go

@@ -3,6 +3,7 @@ package core
 import (
 	"bufio"
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -11,10 +12,6 @@ import (
 	"relay/internal/connections"
 	"relay/internal/shared"
 
-	gen "relay/internal/proto"
-
-	"google.golang.org/protobuf/proto"
-
 	"github.com/libp2p/go-libp2p/core/network"
 	"github.com/libp2p/go-libp2p/core/peer"
 	"github.com/pion/rtp"
@@ -70,10 +67,9 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) {
 	safeBRW := common.NewSafeBufioRW(brw)
 
 	var currentRoomName string // Track the current room for this stream
-	iceHelper := common.NewICEHelper(nil)
+	iceHolder := make([]webrtc.ICECandidateInit, 0)
 	for {
-		var msgWrapper gen.ProtoMessage
-		err := safeBRW.ReceiveProto(&msgWrapper)
+		data, err := safeBRW.Receive()
 		if err != nil {
 			if errors.Is(err, io.EOF) || errors.Is(err, network.ErrReset) {
 				slog.Debug("Stream request connection closed by peer", "peer", stream.Conn().RemotePeer())
@@ -86,140 +82,79 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) {
 			return
 		}
 
-		if msgWrapper.MessageBase == nil {
-			slog.Error("No MessageBase in stream request")
-			_ = stream.Reset()
-			return
+		var baseMsg connections.MessageBase
+		if err = json.Unmarshal(data, &baseMsg); err != nil {
+			slog.Error("Failed to unmarshal base message", "err", err)
+			continue
 		}
 
-		switch msgWrapper.MessageBase.PayloadType {
+		switch baseMsg.Type {
 		case "request-stream-room":
-			reqMsg := msgWrapper.GetClientRequestRoomStream()
-			if reqMsg != nil {
-				currentRoomName = reqMsg.RoomName
-
-				// Generate session ID if not provided (first connection)
-				sessionID := reqMsg.SessionId
-				if sessionID == "" {
-					ulid, err := common.NewULID()
-					if err != nil {
-						slog.Error("Failed to generate session ID", "err", err)
+			var rawMsg connections.MessageRaw
+			if err = json.Unmarshal(data, &rawMsg); err != nil {
+				slog.Error("Failed to unmarshal raw message for room stream request", "err", err)
 				continue
 			}
-					sessionID = ulid.String()
-				}
 
-				slog.Info("Client session requested room stream", "session", sessionID, "room", reqMsg.RoomName)
-
-				// Send session ID back to client
-				sesMsg, err := common.CreateMessage(
-					&gen.ProtoClientRequestRoomStream{SessionId: sessionID, RoomName: reqMsg.RoomName},
-					"session-assigned", nil,
-				)
-				if err != nil {
-					slog.Error("Failed to create proto message", "err", err)
+			var roomName string
+			if err = json.Unmarshal(rawMsg.Data, &roomName); err != nil {
+				slog.Error("Failed to unmarshal room name from raw message", "err", err)
 				continue
 			}
-				if err = safeBRW.SendProto(sesMsg); err != nil {
-					slog.Error("Failed to send session assignment", "err", err)
-				}
 
-				slog.Info("Received stream request for room", "room", reqMsg.RoomName)
+			currentRoomName = roomName // Store the room name
+			slog.Info("Received stream request for room", "room", roomName)
 
-				room := sp.relay.GetRoomByName(reqMsg.RoomName)
+			room := sp.relay.GetRoomByName(roomName)
 			if room == nil || !room.IsOnline() || room.OwnerID != sp.relay.ID {
 				// TODO: Allow forward requests to other relays from here?
-					slog.Debug("Cannot provide stream for nil, offline or non-owned room", "room", reqMsg.RoomName, "is_online", room != nil && room.IsOnline(), "is_owner", room != nil && room.OwnerID == sp.relay.ID)
+				slog.Debug("Cannot provide stream for nil, offline or non-owned room", "room", roomName, "is_online", room != nil && room.IsOnline(), "is_owner", room != nil && room.OwnerID == sp.relay.ID)
 				// Respond with "request-stream-offline" message with room name
 				// TODO: Store the peer and send "online" message when the room comes online
-					rawMsg, err := common.CreateMessage(
-						&gen.ProtoRaw{
-							Data: reqMsg.RoomName,
-						},
-						"request-stream-offline", nil,
-					)
+				roomNameData, err := json.Marshal(roomName)
 				if err != nil {
-						slog.Error("Failed to create proto message", "err", err)
+					slog.Error("Failed to marshal room name for request stream offline", "room", roomName, "err", err)
 					continue
+				} else {
+					if err = safeBRW.SendJSON(connections.NewMessageRaw(
+						"request-stream-offline",
+						roomNameData,
+					)); err != nil {
+						slog.Error("Failed to send request stream offline message", "room", roomName, "err", err)
 					}
-					if err = safeBRW.SendProto(rawMsg); err != nil {
-						slog.Error("Failed to send request stream offline message", "room", reqMsg.RoomName, "err", err)
 				}
 				continue
 			}
 
 			pc, err := common.CreatePeerConnection(func() {
-					slog.Info("PeerConnection closed for requested stream", "room", reqMsg.RoomName)
+				slog.Info("PeerConnection closed for requested stream", "room", roomName)
 				// Cleanup the stream connection
-					if roomMap, ok := sp.servedConns.Get(reqMsg.RoomName); ok {
+				if roomMap, ok := sp.servedConns.Get(roomName); ok {
 					roomMap.Delete(stream.Conn().RemotePeer())
 					// If the room map is empty, delete it
 					if roomMap.Len() == 0 {
-							sp.servedConns.Delete(reqMsg.RoomName)
+						sp.servedConns.Delete(roomName)
 					}
 				}
 			})
 			if err != nil {
-					slog.Error("Failed to create PeerConnection for requested stream", "room", reqMsg.RoomName, "err", err)
+				slog.Error("Failed to create PeerConnection for requested stream", "room", roomName, "err", err)
 				continue
 			}
 
-				// Create participant for this viewer
-				participant, err := shared.NewParticipant(
-					sessionID,
-					stream.Conn().RemotePeer(),
-				)
-				if err != nil {
-					slog.Error("Failed to create participant", "room", reqMsg.RoomName, "err", err)
+			// Add tracks
+			if room.AudioTrack != nil {
+				if _, err = pc.AddTrack(room.AudioTrack); err != nil {
+					slog.Error("Failed to add audio track for requested stream", "room", roomName, "err", err)
 					continue
 				}
-
-				// Assign peer connection
-				participant.PeerConnection = pc
-				iceHelper.SetPeerConnection(pc)
-
-				// Add audio/video tracks
-				{
-					localTrack, err := webrtc.NewTrackLocalStaticRTP(
-						room.AudioCodec,
-						"participant-"+participant.ID.String(),
-						"participant-"+participant.ID.String()+"-audio",
-					)
-					if err != nil {
-						slog.Error("Failed to create track for stream request", "err", err)
-						return
 			}
-					participant.SetTrack(webrtc.RTPCodecTypeAudio, localTrack)
-					slog.Debug("Set audio track for requested stream", "room", room.Name)
+			if room.VideoTrack != nil {
+				if _, err = pc.AddTrack(room.VideoTrack); err != nil {
+					slog.Error("Failed to add video track for requested stream", "room", roomName, "err", err)
+					continue
 				}
-				{
-					localTrack, err := webrtc.NewTrackLocalStaticRTP(
-						room.VideoCodec,
-						"participant-"+participant.ID.String(),
-						"participant-"+participant.ID.String()+"-video",
-					)
-					if err != nil {
-						slog.Error("Failed to create track for stream request", "err", err)
-						return
 			}
-					participant.SetTrack(webrtc.RTPCodecTypeVideo, localTrack)
-					slog.Debug("Set video track for requested stream", "room", room.Name)
-				}
-
-				// Cleanup on disconnect
-				cleanupParticipantID := participant.ID
-				pc.OnConnectionStateChange(func(state webrtc.PeerConnectionState) {
-					if state == webrtc.PeerConnectionStateClosed ||
-						state == webrtc.PeerConnectionStateFailed ||
-						state == webrtc.PeerConnectionStateDisconnected {
-						slog.Info("Participant disconnected from room", "room", reqMsg.RoomName, "participant", cleanupParticipantID)
-						room.RemoveParticipantByID(cleanupParticipantID)
-						participant.Close()
-					} else if state == webrtc.PeerConnectionStateConnected {
-						// Add participant to room when connection is established
-						room.AddParticipant(participant)
-					}
-				})
 
 			// DataChannel setup
 			settingOrdered := true
@@ -229,36 +164,21 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) {
 				MaxRetransmits: &settingMaxRetransmits,
 			})
 			if err != nil {
-					slog.Error("Failed to create DataChannel for requested stream", "room", reqMsg.RoomName, "err", err)
+				slog.Error("Failed to create DataChannel for requested stream", "room", roomName, "err", err)
 				continue
 			}
 			ndc := connections.NewNestriDataChannel(dc)
 
 			ndc.RegisterOnOpen(func() {
-					slog.Debug("Relay DataChannel opened for requested stream", "room", reqMsg.RoomName)
+				slog.Debug("Relay DataChannel opened for requested stream", "room", roomName)
 			})
 			ndc.RegisterOnClose(func() {
-					slog.Debug("Relay DataChannel closed for requested stream", "room", reqMsg.RoomName)
+				slog.Debug("Relay DataChannel closed for requested stream", "room", roomName)
 			})
 			ndc.RegisterMessageCallback("input", func(data []byte) {
 				if room.DataChannel != nil {
 					if err = room.DataChannel.SendBinary(data); err != nil {
-							slog.Error("Failed to forward input message from mesh to upstream room", "room", reqMsg.RoomName, "err", err)
-						}
-					}
-				})
-				// Track controller input separately
-				ndc.RegisterMessageCallback("controllerInput", func(data []byte) {
-					// Parse the message to track controller slots for client sessions
-					var controllerMsgWrapper gen.ProtoMessage
-					if err = proto.Unmarshal(data, &controllerMsgWrapper); err != nil {
-						slog.Error("Failed to unmarshal controller input", "err", err)
-					}
-
-					// Forward to upstream room
-					if room.DataChannel != nil {
-						if err = room.DataChannel.SendBinary(data); err != nil {
-							slog.Error("Failed to forward controller input from mesh to upstream room", "room", reqMsg.RoomName, "err", err)
+						slog.Error("Failed to forward input message from mesh to upstream room", "room", roomName, "err", err)
 					}
 				}
 			})
@@ -269,28 +189,8 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) {
 					return
 				}
 
-					candInit := candidate.ToJSON()
-					var sdpMLineIndex *uint32
-					if candInit.SDPMLineIndex != nil {
-						idx := uint32(*candInit.SDPMLineIndex)
-						sdpMLineIndex = &idx
-					}
-					iceMsg, err := common.CreateMessage(
-						&gen.ProtoICE{
-							Candidate: &gen.RTCIceCandidateInit{
-								Candidate:     candInit.Candidate,
-								SdpMLineIndex: sdpMLineIndex,
-								SdpMid:        candInit.SDPMid,
-							},
-						},
-						"ice-candidate", nil,
-					)
-					if err != nil {
-						slog.Error("Failed to create proto message", "err", err)
-						return
-					}
-					if err = safeBRW.SendProto(iceMsg); err != nil {
-						slog.Error("Failed to send ICE candidate message for requested stream", "room", reqMsg.RoomName, "err", err)
+				if err = safeBRW.SendJSON(connections.NewMessageICE("ice-candidate", candidate.ToJSON())); err != nil {
+					slog.Error("Failed to send ICE candidate message for requested stream", "room", roomName, "err", err)
 					return
 				}
 			})
@@ -298,36 +198,23 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) {
 			// Create offer
 			offer, err := pc.CreateOffer(nil)
 			if err != nil {
-					slog.Error("Failed to create offer for requested stream", "room", reqMsg.RoomName, "err", err)
+				slog.Error("Failed to create offer for requested stream", "room", roomName, "err", err)
 				continue
 			}
 			if err = pc.SetLocalDescription(offer); err != nil {
-					slog.Error("Failed to set local description for requested stream", "room", reqMsg.RoomName, "err", err)
+				slog.Error("Failed to set local description for requested stream", "room", roomName, "err", err)
 				continue
 			}
-				offerMsg, err := common.CreateMessage(
-					&gen.ProtoSDP{
-						Sdp: &gen.RTCSessionDescriptionInit{
-							Sdp:  offer.SDP,
-							Type: offer.Type.String(),
-						},
-					},
-					"offer", nil,
-				)
-				if err != nil {
-					slog.Error("Failed to create proto message", "err", err)
-					continue
-				}
-				if err = safeBRW.SendProto(offerMsg); err != nil {
-					slog.Error("Failed to send offer for requested stream", "room", reqMsg.RoomName, "err", err)
+			if err = safeBRW.SendJSON(connections.NewMessageSDP("offer", offer)); err != nil {
+				slog.Error("Failed to send offer for requested stream", "room", roomName, "err", err)
 				continue
 			}
 
 			// Store the connection
-				roomMap, ok := sp.servedConns.Get(reqMsg.RoomName)
+			roomMap, ok := sp.servedConns.Get(roomName)
 			if !ok {
 				roomMap = common.NewSafeMap[peer.ID, *StreamConnection]()
-					sp.servedConns.Set(reqMsg.RoomName, roomMap)
+				sp.servedConns.Set(roomName, roomMap)
 			}
 			roomMap.Set(stream.Conn().RemotePeer(), &StreamConnection{
 				pc:  pc,
@@ -335,43 +222,50 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) {
 			})
 
 			slog.Debug("Sent offer for requested stream")
-			} else {
-				slog.Error("Could not get ClientRequestRoomStream for stream request")
-			}
 		case "ice-candidate":
-			iceMsg := msgWrapper.GetIce()
-			if iceMsg != nil {
-				cand := webrtc.ICECandidateInit{
-					Candidate:        iceMsg.Candidate.Candidate,
-					SDPMid:           iceMsg.Candidate.SdpMid,
-					UsernameFragment: iceMsg.Candidate.UsernameFragment,
+			var iceMsg connections.MessageICE
+			if err := json.Unmarshal(data, &iceMsg); err != nil {
+				slog.Error("Failed to unmarshal ICE message", "err", err)
+				continue
 			}
-				if iceMsg.Candidate.SdpMLineIndex != nil {
-					smollified := uint16(*iceMsg.Candidate.SdpMLineIndex)
-					cand.SDPMLineIndex = &smollified
+			// Use currentRoomName to get the connection from nested map
+			if len(currentRoomName) > 0 {
+				if roomMap, ok := sp.servedConns.Get(currentRoomName); ok {
+					if conn, ok := roomMap.Get(stream.Conn().RemotePeer()); ok && conn.pc.RemoteDescription() != nil {
+						if err := conn.pc.AddICECandidate(iceMsg.Candidate); err != nil {
+							slog.Error("Failed to add ICE candidate", "err", err)
 						}
-				iceHelper.AddCandidate(cand)
+						for _, heldIce := range iceHolder {
+							if err := conn.pc.AddICECandidate(heldIce); err != nil {
+								slog.Error("Failed to add held ICE candidate", "err", err)
+							}
+						}
+						// Clear the held candidates
+						iceHolder = make([]webrtc.ICECandidateInit, 0)
 					} else {
-				slog.Error("Could not GetIce from ice-candidate")
+						// Hold the candidate until remote description is set
+						iceHolder = append(iceHolder, iceMsg.Candidate)
+					}
+				}
+			} else {
+				// Hold the candidate until remote description is set
+				iceHolder = append(iceHolder, iceMsg.Candidate)
 			}
 		case "answer":
-			answerMsg := msgWrapper.GetSdp()
-			if answerMsg != nil {
-				ansSdp := webrtc.SessionDescription{
-					SDP:  answerMsg.Sdp.Sdp,
-					Type: webrtc.NewSDPType(answerMsg.Sdp.Type),
+			var answerMsg connections.MessageSDP
+			if err := json.Unmarshal(data, &answerMsg); err != nil {
+				slog.Error("Failed to unmarshal answer from signaling message", "err", err)
+				continue
 			}
 			// Use currentRoomName to get the connection from nested map
 			if len(currentRoomName) > 0 {
 				if roomMap, ok := sp.servedConns.Get(currentRoomName); ok {
 					if conn, ok := roomMap.Get(stream.Conn().RemotePeer()); ok {
-							if err = conn.pc.SetRemoteDescription(ansSdp); err != nil {
+						if err := conn.pc.SetRemoteDescription(answerMsg.SDP); err != nil {
 							slog.Error("Failed to set remote description for answer", "err", err)
 							continue
 						}
 						slog.Debug("Set remote description for answer")
-							// Flush held candidates now if missed before (race-condition)
-							iceHelper.FlushHeldCandidates()
 					} else {
 						slog.Warn("Received answer without active PeerConnection")
 					}
@@ -379,11 +273,199 @@ func (sp *StreamProtocol) handleStreamRequest(stream network.Stream) {
 			} else {
 				slog.Warn("Received answer without active PeerConnection")
 			}
+		}
+	}
+}
+
+// requestStream manages the internals of the stream request
+func (sp *StreamProtocol) requestStream(stream network.Stream, room *shared.Room) error {
+	brw := bufio.NewReadWriter(bufio.NewReader(stream), bufio.NewWriter(stream))
+	safeBRW := common.NewSafeBufioRW(brw)
+
+	slog.Debug("Requesting room stream from peer", "room", room.Name, "peer", stream.Conn().RemotePeer())
+
+	// Send room name to the remote peer
+	roomData, err := json.Marshal(room.Name)
+	if err != nil {
+		_ = stream.Close()
+		return fmt.Errorf("failed to marshal room name: %w", err)
+	}
+	if err = safeBRW.SendJSON(connections.NewMessageRaw(
+		"request-stream-room",
+		roomData,
+	)); err != nil {
+		_ = stream.Close()
+		return fmt.Errorf("failed to send room request: %w", err)
+	}
+
+	pc, err := common.CreatePeerConnection(func() {
+		slog.Info("Relay PeerConnection closed for requested stream", "room", room.Name)
+		_ = stream.Close() // ignore error as may be closed already
+		// Cleanup the stream connection
+		if ok := sp.requestedConns.Has(room.Name); ok {
+			sp.requestedConns.Delete(room.Name)
+		}
+	})
+	if err != nil {
+		_ = stream.Close()
+		return fmt.Errorf("failed to create PeerConnection: %w", err)
+	}
+
+	pc.OnTrack(func(track *webrtc.TrackRemote, receiver *webrtc.RTPReceiver) {
+		localTrack, _ := webrtc.NewTrackLocalStaticRTP(track.Codec().RTPCodecCapability, track.ID(), "relay-"+room.Name+"-"+track.Kind().String())
+		slog.Debug("Received track for requested stream", "room", room.Name, "track_kind", track.Kind().String())
+
+		room.SetTrack(track.Kind(), localTrack)
+
+		go func() {
+			for {
+				rtpPacket, _, err := track.ReadRTP()
+				if err != nil {
+					if !errors.Is(err, io.EOF) {
+						slog.Error("Failed to read RTP packet for requested stream room", "room", room.Name, "err", err)
+					}
+					break
+				}
+
+				err = localTrack.WriteRTP(rtpPacket)
+				if err != nil && !errors.Is(err, io.ErrClosedPipe) {
+					slog.Error("Failed to write RTP to local track for requested stream room", "room", room.Name, "err", err)
+					break
+				}
+			}
+		}()
+	})
+
+	pc.OnDataChannel(func(dc *webrtc.DataChannel) {
+		ndc := connections.NewNestriDataChannel(dc)
+		ndc.RegisterOnOpen(func() {
+			slog.Debug("Relay DataChannel opened for requested stream", "room", room.Name)
+		})
+		ndc.RegisterOnClose(func() {
+			slog.Debug("Relay DataChannel closed for requested stream", "room", room.Name)
+		})
+
+		// Set the DataChannel in the requestedConns map
+		if conn, ok := sp.requestedConns.Get(room.Name); ok {
+			conn.ndc = ndc
 		} else {
-				slog.Warn("Could not GetSdp from answer")
+			sp.requestedConns.Set(room.Name, &StreamConnection{
+				pc:  pc,
+				ndc: ndc,
+			})
+		}
+
+		// We do not handle any messages from upstream here
+	})
+
+	pc.OnICECandidate(func(candidate *webrtc.ICECandidate) {
+		if candidate == nil {
+			return
+		}
+
+		if err = safeBRW.SendJSON(connections.NewMessageICE(
+			"ice-candidate",
+			candidate.ToJSON(),
+		)); err != nil {
+			slog.Error("Failed to send ICE candidate message for requested stream", "room", room.Name, "err", err)
+			return
+		}
+	})
+
+	// Handle incoming messages (offer and candidates)
+	go func() {
+		iceHolder := make([]webrtc.ICECandidateInit, 0)
+
+		for {
+			data, err := safeBRW.Receive()
+			if err != nil {
+				if errors.Is(err, io.EOF) || errors.Is(err, network.ErrReset) {
+					slog.Debug("Connection for requested stream closed by peer", "room", room.Name)
+					return
+				}
+
+				slog.Error("Failed to receive data for requested stream", "room", room.Name, "err", err)
+				_ = stream.Reset()
+
+				return
+			}
+
+			var baseMsg connections.MessageBase
+			if err = json.Unmarshal(data, &baseMsg); err != nil {
+				slog.Error("Failed to unmarshal base message for requested stream", "room", room.Name, "err", err)
+				return
+			}
+
+			switch baseMsg.Type {
+			case "ice-candidate":
+				var iceMsg connections.MessageICE
+				if err = json.Unmarshal(data, &iceMsg); err != nil {
+					slog.Error("Failed to unmarshal ICE candidate for requested stream", "room", room.Name, "err", err)
+					continue
+				}
+				if conn, ok := sp.requestedConns.Get(room.Name); ok && conn.pc.RemoteDescription() != nil {
+					if err = conn.pc.AddICECandidate(iceMsg.Candidate); err != nil {
+						slog.Error("Failed to add ICE candidate for requested stream", "room", room.Name, "err", err)
+					}
+					// Add held candidates
+					for _, heldCandidate := range iceHolder {
+						if err = conn.pc.AddICECandidate(heldCandidate); err != nil {
+							slog.Error("Failed to add held ICE candidate for requested stream", "room", room.Name, "err", err)
 						}
 					}
+					// Clear the held candidates
+					iceHolder = make([]webrtc.ICECandidateInit, 0)
+				} else {
+					// Hold the candidate until remote description is set
+					iceHolder = append(iceHolder, iceMsg.Candidate)
 				}
+			case "offer":
+				var offerMsg connections.MessageSDP
+				if err = json.Unmarshal(data, &offerMsg); err != nil {
+					slog.Error("Failed to unmarshal offer for requested stream", "room", room.Name, "err", err)
+					continue
+				}
+				if err = pc.SetRemoteDescription(offerMsg.SDP); err != nil {
+					slog.Error("Failed to set remote description for requested stream", "room", room.Name, "err", err)
+					continue
+				}
+				answer, err := pc.CreateAnswer(nil)
+				if err != nil {
+					slog.Error("Failed to create answer for requested stream", "room", room.Name, "err", err)
+					if err = stream.Reset(); err != nil {
+						slog.Error("Failed to reset stream for requested stream", "err", err)
+					}
+					return
+				}
+				if err = pc.SetLocalDescription(answer); err != nil {
+					slog.Error("Failed to set local description for requested stream", "room", room.Name, "err", err)
+					if err = stream.Reset(); err != nil {
+						slog.Error("Failed to reset stream for requested stream", "err", err)
+					}
+					return
+				}
+				if err = safeBRW.SendJSON(connections.NewMessageSDP(
+					"answer",
+					answer,
+				)); err != nil {
+					slog.Error("Failed to send answer for requested stream", "room", room.Name, "err", err)
+					continue
+				}
+
+				// Store the connection
+				sp.requestedConns.Set(room.Name, &StreamConnection{
+					pc:  pc,
+					ndc: nil,
+				})
+
+				slog.Debug("Sent answer for requested stream", "room", room.Name)
+			default:
+				slog.Warn("Unknown signaling message type", "room", room.Name, "type", baseMsg.Type)
+			}
+		}
+	}()
+
+	return nil
 }
 
 // handleStreamPush manages a stream push from a node (nestri-server)
@@ -392,41 +474,44 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
 	safeBRW := common.NewSafeBufioRW(brw)
 
 	var room *shared.Room
-	iceHelper := common.NewICEHelper(nil)
+	iceHolder := make([]webrtc.ICECandidateInit, 0)
 	for {
-		var msgWrapper gen.ProtoMessage
-		err := safeBRW.ReceiveProto(&msgWrapper)
+		data, err := safeBRW.Receive()
 		if err != nil {
 			if errors.Is(err, io.EOF) || errors.Is(err, network.ErrReset) {
 				slog.Debug("Stream push connection closed by peer", "peer", stream.Conn().RemotePeer(), "error", err)
-				if room != nil {
-					room.Close()
-					sp.incomingConns.Delete(room.Name)
-				}
 				return
 			}
 
 			slog.Error("Failed to receive data for stream push", "err", err)
 			_ = stream.Reset()
-			if room != nil {
-				room.Close()
-				sp.incomingConns.Delete(room.Name)
-			}
+
 			return
 		}
 
-		if msgWrapper.MessageBase == nil {
-			slog.Error("No MessageBase in stream push")
+		var baseMsg connections.MessageBase
+		if err = json.Unmarshal(data, &baseMsg); err != nil {
+			slog.Error("Failed to unmarshal base message from base message", "err", err)
 			continue
 		}
 
-		switch msgWrapper.MessageBase.PayloadType {
+		switch baseMsg.Type {
 		case "push-stream-room":
-			pushMsg := msgWrapper.GetServerPushStream()
-			if pushMsg != nil {
-				slog.Info("Received stream push request for room", "room", pushMsg.RoomName)
+			var rawMsg connections.MessageRaw
+			if err = json.Unmarshal(data, &rawMsg); err != nil {
+				slog.Error("Failed to unmarshal room name from data", "err", err)
+				continue
+			}
 
-				room = sp.relay.GetRoomByName(pushMsg.RoomName)
+			var roomName string
+			if err = json.Unmarshal(rawMsg.Data, &roomName); err != nil {
+				slog.Error("Failed to unmarshal room name from raw message", "err", err)
+				continue
+			}
+
+			slog.Info("Received stream push request for room", "room", roomName)
+
+			room = sp.relay.GetRoomByName(roomName)
 			if room != nil {
 				if room.OwnerID != sp.relay.ID {
 					slog.Error("Cannot push a stream to non-owned room", "room", room.Name, "owner_id", room.OwnerID)
@@ -438,40 +523,42 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
 				}
 			} else {
 				// Create a new room if it doesn't exist
-					room = sp.relay.CreateRoom(pushMsg.RoomName)
+				room = sp.relay.CreateRoom(roomName)
 			}
 
 			// Respond with an OK with the room name
-				resMsg, err := common.CreateMessage(
-					&gen.ProtoServerPushStream{
-						RoomName: pushMsg.RoomName,
-					},
-					"push-stream-ok", nil,
-				)
+			roomData, err := json.Marshal(room.Name)
 			if err != nil {
-					slog.Error("Failed to create proto message", "err", err)
+				slog.Error("Failed to marshal room name for push stream response", "err", err)
 				continue
 			}
-				if err = safeBRW.SendProto(resMsg); err != nil {
+			if err = safeBRW.SendJSON(connections.NewMessageRaw(
+				"push-stream-ok",
+				roomData,
+			)); err != nil {
 				slog.Error("Failed to send push stream OK response", "room", room.Name, "err", err)
 				continue
 			}
-			} else {
-				slog.Error("Failed to GetServerPushStream in push-stream-room")
-			}
 		case "ice-candidate":
-			iceMsg := msgWrapper.GetIce()
-			if iceMsg != nil {
-				smollified := uint16(*iceMsg.Candidate.SdpMLineIndex)
-				cand := webrtc.ICECandidateInit{
-					Candidate:        iceMsg.Candidate.Candidate,
-					SDPMid:           iceMsg.Candidate.SdpMid,
-					SDPMLineIndex:    &smollified,
-					UsernameFragment: iceMsg.Candidate.UsernameFragment,
+			var iceMsg connections.MessageICE
+			if err = json.Unmarshal(data, &iceMsg); err != nil {
+				slog.Error("Failed to unmarshal ICE candidate from data", "err", err)
+				continue
 			}
-				iceHelper.AddCandidate(cand)
+			if conn, ok := sp.incomingConns.Get(room.Name); ok && conn.pc.RemoteDescription() != nil {
+				if err = conn.pc.AddICECandidate(iceMsg.Candidate); err != nil {
+					slog.Error("Failed to add ICE candidate for pushed stream", "err", err)
+				}
+				for _, heldIce := range iceHolder {
+					if err := conn.pc.AddICECandidate(heldIce); err != nil {
+						slog.Error("Failed to add held ICE candidate for pushed stream", "err", err)
+					}
+				}
+				// Clear the held candidates
+				iceHolder = make([]webrtc.ICECandidateInit, 0)
 			} else {
-				slog.Error("Failed to GetIce in pushed stream ice-candidate")
+				// Hold the candidate until remote description is set
+				iceHolder = append(iceHolder, iceMsg.Candidate)
 			}
 		case "offer":
 			// Make sure we have room set to push to (set by "push-stream-room")
@@ -480,12 +567,12 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
 				continue
 			}
 
-			offerMsg := msgWrapper.GetSdp()
-			if offerMsg != nil {
-				offSdp := webrtc.SessionDescription{
-					SDP:  offerMsg.Sdp.Sdp,
-					Type: webrtc.NewSDPType(offerMsg.Sdp.Type),
+			var offerMsg connections.MessageSDP
+			if err = json.Unmarshal(data, &offerMsg); err != nil {
+				slog.Error("Failed to unmarshal offer from data", "err", err)
+				continue
 			}
+
 			// Create PeerConnection for the incoming stream
 			pc, err := common.CreatePeerConnection(func() {
 				slog.Info("PeerConnection closed for pushed stream", "room", room.Name)
@@ -499,10 +586,6 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
 				continue
 			}
 
-				// Assign room peer connection
-				room.PeerConnection = pc
-				iceHelper.SetPeerConnection(pc)
-
 			pc.OnDataChannel(func(dc *webrtc.DataChannel) {
 				// TODO: Is this the best way to handle DataChannel? Should we just use the map directly?
 				room.DataChannel = connections.NewNestriDataChannel(dc)
@@ -512,24 +595,20 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
 				room.DataChannel.RegisterOnClose(func() {
 					slog.Debug("DataChannel closed for pushed stream", "room", room.Name)
 				})
-					// Handle controller feedback reverse-flow (like rumble events coming from game to client)
-					room.DataChannel.RegisterMessageCallback("controllerInput", func(data []byte) {
-						// Forward controller input to all viewers
+				room.DataChannel.RegisterMessageCallback("input", func(data []byte) {
+					if room.DataChannel != nil {
+						// Pass to servedConns DataChannels for this specific room
 						if roomMap, ok := sp.servedConns.Get(room.Name); ok {
 							roomMap.Range(func(peerID peer.ID, conn *StreamConnection) bool {
 								if conn.ndc != nil {
 									if err = conn.ndc.SendBinary(data); err != nil {
-										if errors.Is(err, io.ErrClosedPipe) {
-											slog.Warn("Failed to forward controller input to viewer, treating as disconnected", "err", err)
-											sp.relay.onPeerDisconnected(peerID)
-										} else {
-											slog.Error("Failed to forward controller input from pushed stream to viewer", "room", room.Name, "peer", peerID, "err", err)
+										slog.Error("Failed to forward input message from pushed stream to viewer", "room", room.Name, "peer", peerID, "err", err)
 									}
 								}
-								}
-								return true
+								return true // Continue iteration
 							})
 						}
+					}
 				})
 
 				// Set the DataChannel in the incomingConns map
@@ -548,29 +627,27 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
 					return
 				}
 
-					candInit := candidate.ToJSON()
-					biggified := uint32(*candInit.SDPMLineIndex)
-					iceMsg, err := common.CreateMessage(
-						&gen.ProtoICE{
-							Candidate: &gen.RTCIceCandidateInit{
-								Candidate:     candInit.Candidate,
-								SdpMLineIndex: &biggified,
-								SdpMid:        candInit.SDPMid,
-							},
-						},
-						"ice-candidate", nil,
-					)
-					if err != nil {
-						slog.Error("Failed to create proto message", "err", err)
-						return
-					}
-					if err = safeBRW.SendProto(iceMsg); err != nil {
+				if err = safeBRW.SendJSON(connections.NewMessageICE(
+					"ice-candidate",
+					candidate.ToJSON(),
+				)); err != nil {
 					slog.Error("Failed to send ICE candidate message for pushed stream", "room", room.Name, "err", err)
 					return
 				}
 			})
 
 			pc.OnTrack(func(remoteTrack *webrtc.TrackRemote, receiver *webrtc.RTPReceiver) {
+				localTrack, err := webrtc.NewTrackLocalStaticRTP(remoteTrack.Codec().RTPCodecCapability, remoteTrack.Kind().String(), fmt.Sprintf("nestri-%s-%s", room.Name, remoteTrack.Kind().String()))
+				if err != nil {
+					slog.Error("Failed to create local track for pushed stream", "room", room.Name, "track_kind", remoteTrack.Kind().String(), "err", err)
+					return
+				}
+
+				slog.Debug("Received track for pushed stream", "room", room.Name, "track_kind", remoteTrack.Kind().String())
+
+				// Set track for Room
+				room.SetTrack(remoteTrack.Kind(), localTrack)
+
 				// Prepare PlayoutDelayExtension so we don't need to recreate it for each packet
 				playoutExt := &rtp.PlayoutDelayExtension{
 					MinDelay: 0,
@@ -582,12 +659,6 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
 					return
 				}
 
-					if remoteTrack.Kind() == webrtc.RTPCodecTypeAudio {
-						room.AudioCodec = remoteTrack.Codec().RTPCodecCapability
-					} else if remoteTrack.Kind() == webrtc.RTPCodecTypeVideo {
-						room.VideoCodec = remoteTrack.Codec().RTPCodecCapability
-					}
-
 				for {
 					rtpPacket, _, err := remoteTrack.ReadRTP()
 					if err != nil {
@@ -599,27 +670,31 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
 
 					// Use PlayoutDelayExtension for low latency, if set for this track kind
 					if extID, ok := common.GetExtension(remoteTrack.Kind(), common.ExtensionPlayoutDelay); ok {
-							if err = rtpPacket.SetExtension(extID, playoutPayload); err != nil {
+						if err := rtpPacket.SetExtension(extID, playoutPayload); err != nil {
 							slog.Error("Failed to set PlayoutDelayExtension for room", "room", room.Name, "err", err)
 							continue
 						}
 					}
 
-						// Broadcast
-						room.BroadcastPacket(remoteTrack.Kind(), rtpPacket)
+					err = localTrack.WriteRTP(rtpPacket)
+					if err != nil && !errors.Is(err, io.ErrClosedPipe) {
+						slog.Error("Failed to write RTP to local track for room", "room", room.Name, "err", err)
+						break
+					}
 				}
 
 				slog.Debug("Track closed for room", "room", room.Name, "track_kind", remoteTrack.Kind().String())
+
+				// Cleanup the track from the room
+				room.SetTrack(remoteTrack.Kind(), nil)
 			})
 
 			// Set the remote description
-				if err = pc.SetRemoteDescription(offSdp); err != nil {
+			if err = pc.SetRemoteDescription(offerMsg.SDP); err != nil {
 				slog.Error("Failed to set remote description for pushed stream", "room", room.Name, "err", err)
 				continue
 			}
 			slog.Debug("Set remote description for pushed stream", "room", room.Name)
-				// Flush candidates now if they weren't before (race-condition)
-				iceHelper.FlushHeldCandidates()
 
 			// Create an answer
 			answer, err := pc.CreateAnswer(nil)
@@ -631,20 +706,10 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
 				slog.Error("Failed to set local description for pushed stream", "room", room.Name, "err", err)
 				continue
 			}
-				answerMsg, err := common.CreateMessage(
-					&gen.ProtoSDP{
-						Sdp: &gen.RTCSessionDescriptionInit{
-							Sdp:  answer.SDP,
-							Type: answer.Type.String(),
-						},
-					},
-					"answer", nil,
-				)
-				if err != nil {
-					slog.Error("Failed to create proto message", "err", err)
-					continue
-				}
-				if err = safeBRW.SendProto(answerMsg); err != nil {
+			if err = safeBRW.SendJSON(connections.NewMessageSDP(
+				"answer",
+				answer,
+			)); err != nil {
 				slog.Error("Failed to send answer for pushed stream", "room", room.Name, "err", err)
 			}
 
@@ -657,16 +722,15 @@ func (sp *StreamProtocol) handleStreamPush(stream network.Stream) {
 		}
 	}
 }
-}
 
 // --- Public Usable Methods ---
 
 // RequestStream sends a request to get room stream from another relay
 func (sp *StreamProtocol) RequestStream(ctx context.Context, room *shared.Room, peerID peer.ID) error {
-	_, err := sp.relay.Host.NewStream(ctx, peerID, protocolStreamRequest)
+	stream, err := sp.relay.Host.NewStream(ctx, peerID, protocolStreamRequest)
 	if err != nil {
 		return fmt.Errorf("failed to create stream: %w", err)
 	}
 
-	return nil /* TODO: This? */
+	return sp.requestStream(stream, room)
 }

packages/relay/internal/core/room.go

@@ -45,7 +45,7 @@ func (r *Relay) DeleteRoomIfEmpty(room *shared.Room) {
 	if room == nil {
 		return
 	}
-	if len(room.Participants) <= 0 && r.LocalRooms.Has(room.ID) {
+	if room.Participants.Len() == 0 && r.LocalRooms.Has(room.ID) {
 		slog.Debug("Deleting empty room without participants", "room", room.Name)
 		r.LocalRooms.Delete(room.ID)
 		err := room.PeerConnection.Close()

packages/relay/internal/core/state.go

@@ -129,11 +129,12 @@ func (r *Relay) onPeerConnected(peerID peer.ID) {
 
 // onPeerDisconnected marks a peer as disconnected in our status view and removes latency info
 func (r *Relay) onPeerDisconnected(peerID peer.ID) {
-	// Relay peer disconnect handling
 	slog.Info("Mesh peer disconnected, deleting from local peer map", "peer", peerID)
+	// Remove peer from local mesh peers
 	if r.Peers.Has(peerID) {
 		r.Peers.Delete(peerID)
 	}
+	// Remove any rooms associated with this peer
 	if r.Rooms.Has(peerID.String()) {
 		r.Rooms.Delete(peerID.String())
 	}
@@ -150,18 +151,18 @@ func (r *Relay) updateMeshRoomStates(peerID peer.ID, states []shared.RoomInfo) {
 		}
 
 		// If previously did not exist, but does now, request a connection if participants exist for our room
-		/*existed := r.Rooms.Has(state.ID.String())
+		existed := r.Rooms.Has(state.ID.String())
 		if !existed {
 			// Request connection to this peer if we have participants in our local room
 			if room, ok := r.LocalRooms.Get(state.ID); ok {
-				if len(room.Participants) > 0 {
+				if room.Participants.Len() > 0 {
 					slog.Debug("Got new remote room state, we locally have participants for, requesting stream", "room_name", room.Name, "peer", peerID)
 					if err := r.StreamProtocol.RequestStream(context.Background(), room, peerID); err != nil {
 						slog.Error("Failed to request stream for new remote room state", "room_name", room.Name, "peer", peerID, "err", err)
 					}
 				}
 			}
-		}*/
+		}
 
 		r.Rooms.Set(state.ID.String(), state)
 	}

packages/relay/internal/proto/messages.pb.go

@@ -73,47 +73,28 @@ func (x *ProtoMessageBase) GetLatency() *ProtoLatencyTracker {
 	return nil
 }
 
-type ProtoMessage struct {
+type ProtoMessageInput struct {
 	state         protoimpl.MessageState `protogen:"open.v1"`
 	MessageBase   *ProtoMessageBase      `protobuf:"bytes,1,opt,name=message_base,json=messageBase,proto3" json:"message_base,omitempty"`
-	// Types that are valid to be assigned to Payload:
-	//
-	//	*ProtoMessage_MouseMove
-	//	*ProtoMessage_MouseMoveAbs
-	//	*ProtoMessage_MouseWheel
-	//	*ProtoMessage_MouseKeyDown
-	//	*ProtoMessage_MouseKeyUp
-	//	*ProtoMessage_KeyDown
-	//	*ProtoMessage_KeyUp
-	//	*ProtoMessage_ControllerAttach
-	//	*ProtoMessage_ControllerDetach
-	//	*ProtoMessage_ControllerRumble
-	//	*ProtoMessage_ControllerStateBatch
-	//	*ProtoMessage_Ice
-	//	*ProtoMessage_Sdp
-	//	*ProtoMessage_Raw
-	//	*ProtoMessage_ClientRequestRoomStream
-	//	*ProtoMessage_ClientDisconnected
-	//	*ProtoMessage_ServerPushStream
-	Payload       isProtoMessage_Payload `protobuf_oneof:"payload"`
+	Data          *ProtoInput            `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"`
 	unknownFields protoimpl.UnknownFields
 	sizeCache     protoimpl.SizeCache
 }
 
-func (x *ProtoMessage) Reset() {
-	*x = ProtoMessage{}
+func (x *ProtoMessageInput) Reset() {
+	*x = ProtoMessageInput{}
 	mi := &file_messages_proto_msgTypes[1]
 	ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
 	ms.StoreMessageInfo(mi)
 }
 
-func (x *ProtoMessage) String() string {
+func (x *ProtoMessageInput) String() string {
 	return protoimpl.X.MessageStringOf(x)
 }
 
-func (*ProtoMessage) ProtoMessage() {}
+func (*ProtoMessageInput) ProtoMessage() {}
 
-func (x *ProtoMessage) ProtoReflect() protoreflect.Message {
+func (x *ProtoMessageInput) ProtoReflect() protoreflect.Message {
 	mi := &file_messages_proto_msgTypes[1]
 	if x != nil {
 		ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x))
@@ -125,287 +106,25 @@ func (x *ProtoMessage) ProtoReflect() protoreflect.Message {
 	return mi.MessageOf(x)
 }
 
-// Deprecated: Use ProtoMessage.ProtoReflect.Descriptor instead.
-func (*ProtoMessage) Descriptor() ([]byte, []int) {
+// Deprecated: Use ProtoMessageInput.ProtoReflect.Descriptor instead.
+func (*ProtoMessageInput) Descriptor() ([]byte, []int) {
 	return file_messages_proto_rawDescGZIP(), []int{1}
 }
 
-func (x *ProtoMessage) GetMessageBase() *ProtoMessageBase {
+func (x *ProtoMessageInput) GetMessageBase() *ProtoMessageBase {
 	if x != nil {
 		return x.MessageBase
 	}
 	return nil
 }
 
-func (x *ProtoMessage) GetPayload() isProtoMessage_Payload {
+func (x *ProtoMessageInput) GetData() *ProtoInput {
 	if x != nil {
-		return x.Payload
+		return x.Data
 	}
 	return nil
 }
 
-func (x *ProtoMessage) GetMouseMove() *ProtoMouseMove {
-	if x != nil {
-		if x, ok := x.Payload.(*ProtoMessage_MouseMove); ok {
-			return x.MouseMove
-		}
-	}
-	return nil
-}
-
-func (x *ProtoMessage) GetMouseMoveAbs() *ProtoMouseMoveAbs {
-	if x != nil {
-		if x, ok := x.Payload.(*ProtoMessage_MouseMoveAbs); ok {
-			return x.MouseMoveAbs
-		}
-	}
-	return nil
-}
-
-func (x *ProtoMessage) GetMouseWheel() *ProtoMouseWheel {
-	if x != nil {
-		if x, ok := x.Payload.(*ProtoMessage_MouseWheel); ok {
-			return x.MouseWheel
-		}
-	}
-	return nil
-}
-
-func (x *ProtoMessage) GetMouseKeyDown() *ProtoMouseKeyDown {
-	if x != nil {
-		if x, ok := x.Payload.(*ProtoMessage_MouseKeyDown); ok {
-			return x.MouseKeyDown
-		}
-	}
-	return nil
-}
-
-func (x *ProtoMessage) GetMouseKeyUp() *ProtoMouseKeyUp {
-	if x != nil {
-		if x, ok := x.Payload.(*ProtoMessage_MouseKeyUp); ok {
-			return x.MouseKeyUp
-		}
-	}
-	return nil
-}
-
-func (x *ProtoMessage) GetKeyDown() *ProtoKeyDown {
-	if x != nil {
-		if x, ok := x.Payload.(*ProtoMessage_KeyDown); ok {
-			return x.KeyDown
-		}
-	}
-	return nil
-}
-
-func (x *ProtoMessage) GetKeyUp() *ProtoKeyUp {
-	if x != nil {
-		if x, ok := x.Payload.(*ProtoMessage_KeyUp); ok {
-			return x.KeyUp
-		}
-	}
-	return nil
-}
-
-func (x *ProtoMessage) GetControllerAttach() *ProtoControllerAttach {
-	if x != nil {
-		if x, ok := x.Payload.(*ProtoMessage_ControllerAttach); ok {
-			return x.ControllerAttach
-		}
-	}
-	return nil
-}
-
-func (x *ProtoMessage) GetControllerDetach() *ProtoControllerDetach {
-	if x != nil {
-		if x, ok := x.Payload.(*ProtoMessage_ControllerDetach); ok {
-			return x.ControllerDetach
-		}
-	}
-	return nil
-}
-
-func (x *ProtoMessage) GetControllerRumble() *ProtoControllerRumble {
-	if x != nil {
-		if x, ok := x.Payload.(*ProtoMessage_ControllerRumble); ok {
-			return x.ControllerRumble
-		}
-	}
-	return nil
-}
-
-func (x *ProtoMessage) GetControllerStateBatch() *ProtoControllerStateBatch {
-	if x != nil {
-		if x, ok := x.Payload.(*ProtoMessage_ControllerStateBatch); ok {
-			return x.ControllerStateBatch
-		}
-	}
-	return nil
-}
-
-func (x *ProtoMessage) GetIce() *ProtoICE {
-	if x != nil {
-		if x, ok := x.Payload.(*ProtoMessage_Ice); ok {
-			return x.Ice
-		}
-	}
-	return nil
-}
-
-func (x *ProtoMessage) GetSdp() *ProtoSDP {
-	if x != nil {
-		if x, ok := x.Payload.(*ProtoMessage_Sdp); ok {
-			return x.Sdp
-		}
-	}
-	return nil
-}
-
-func (x *ProtoMessage) GetRaw() *ProtoRaw {
-	if x != nil {
-		if x, ok := x.Payload.(*ProtoMessage_Raw); ok {
-			return x.Raw
-		}
-	}
-	return nil
-}
-
-func (x *ProtoMessage) GetClientRequestRoomStream() *ProtoClientRequestRoomStream {
-	if x != nil {
-		if x, ok := x.Payload.(*ProtoMessage_ClientRequestRoomStream); ok {
-			return x.ClientRequestRoomStream
-		}
-	}
-	return nil
-}
-
-func (x *ProtoMessage) GetClientDisconnected() *ProtoClientDisconnected {
-	if x != nil {
-		if x, ok := x.Payload.(*ProtoMessage_ClientDisconnected); ok {
-			return x.ClientDisconnected
-		}
-	}
-	return nil
-}
-
-func (x *ProtoMessage) GetServerPushStream() *ProtoServerPushStream {
-	if x != nil {
-		if x, ok := x.Payload.(*ProtoMessage_ServerPushStream); ok {
-			return x.ServerPushStream
-		}
-	}
-	return nil
-}
-
-type isProtoMessage_Payload interface {
-	isProtoMessage_Payload()
-}
-
-type ProtoMessage_MouseMove struct {
-	// Input types
-	MouseMove *ProtoMouseMove `protobuf:"bytes,2,opt,name=mouse_move,json=mouseMove,proto3,oneof"`
-}
-
-type ProtoMessage_MouseMoveAbs struct {
-	MouseMoveAbs *ProtoMouseMoveAbs `protobuf:"bytes,3,opt,name=mouse_move_abs,json=mouseMoveAbs,proto3,oneof"`
-}
-
-type ProtoMessage_MouseWheel struct {
-	MouseWheel *ProtoMouseWheel `protobuf:"bytes,4,opt,name=mouse_wheel,json=mouseWheel,proto3,oneof"`
-}
-
-type ProtoMessage_MouseKeyDown struct {
-	MouseKeyDown *ProtoMouseKeyDown `protobuf:"bytes,5,opt,name=mouse_key_down,json=mouseKeyDown,proto3,oneof"`
-}
-
-type ProtoMessage_MouseKeyUp struct {
-	MouseKeyUp *ProtoMouseKeyUp `protobuf:"bytes,6,opt,name=mouse_key_up,json=mouseKeyUp,proto3,oneof"`
-}
-
-type ProtoMessage_KeyDown struct {
-	KeyDown *ProtoKeyDown `protobuf:"bytes,7,opt,name=key_down,json=keyDown,proto3,oneof"`
-}
-
-type ProtoMessage_KeyUp struct {
-	KeyUp *ProtoKeyUp `protobuf:"bytes,8,opt,name=key_up,json=keyUp,proto3,oneof"`
-}
-
-type ProtoMessage_ControllerAttach struct {
-	// Controller input types
-	ControllerAttach *ProtoControllerAttach `protobuf:"bytes,9,opt,name=controller_attach,json=controllerAttach,proto3,oneof"`
-}
-
-type ProtoMessage_ControllerDetach struct {
-	ControllerDetach *ProtoControllerDetach `protobuf:"bytes,10,opt,name=controller_detach,json=controllerDetach,proto3,oneof"`
-}
-
-type ProtoMessage_ControllerRumble struct {
-	ControllerRumble *ProtoControllerRumble `protobuf:"bytes,11,opt,name=controller_rumble,json=controllerRumble,proto3,oneof"`
-}
-
-type ProtoMessage_ControllerStateBatch struct {
-	ControllerStateBatch *ProtoControllerStateBatch `protobuf:"bytes,12,opt,name=controller_state_batch,json=controllerStateBatch,proto3,oneof"`
-}
-
-type ProtoMessage_Ice struct {
-	// Signaling types
-	Ice *ProtoICE `protobuf:"bytes,20,opt,name=ice,proto3,oneof"`
-}
-
-type ProtoMessage_Sdp struct {
-	Sdp *ProtoSDP `protobuf:"bytes,21,opt,name=sdp,proto3,oneof"`
-}
-
-type ProtoMessage_Raw struct {
-	Raw *ProtoRaw `protobuf:"bytes,22,opt,name=raw,proto3,oneof"`
-}
-
-type ProtoMessage_ClientRequestRoomStream struct {
-	ClientRequestRoomStream *ProtoClientRequestRoomStream `protobuf:"bytes,23,opt,name=client_request_room_stream,json=clientRequestRoomStream,proto3,oneof"`
-}
-
-type ProtoMessage_ClientDisconnected struct {
-	ClientDisconnected *ProtoClientDisconnected `protobuf:"bytes,24,opt,name=client_disconnected,json=clientDisconnected,proto3,oneof"`
-}
-
-type ProtoMessage_ServerPushStream struct {
-	ServerPushStream *ProtoServerPushStream `protobuf:"bytes,25,opt,name=server_push_stream,json=serverPushStream,proto3,oneof"`
-}
-
-func (*ProtoMessage_MouseMove) isProtoMessage_Payload() {}
-
-func (*ProtoMessage_MouseMoveAbs) isProtoMessage_Payload() {}
-
-func (*ProtoMessage_MouseWheel) isProtoMessage_Payload() {}
-
-func (*ProtoMessage_MouseKeyDown) isProtoMessage_Payload() {}
-
-func (*ProtoMessage_MouseKeyUp) isProtoMessage_Payload() {}
-
-func (*ProtoMessage_KeyDown) isProtoMessage_Payload() {}
-
-func (*ProtoMessage_KeyUp) isProtoMessage_Payload() {}
-
-func (*ProtoMessage_ControllerAttach) isProtoMessage_Payload() {}
-
-func (*ProtoMessage_ControllerDetach) isProtoMessage_Payload() {}
-
-func (*ProtoMessage_ControllerRumble) isProtoMessage_Payload() {}
-
-func (*ProtoMessage_ControllerStateBatch) isProtoMessage_Payload() {}
-
-func (*ProtoMessage_Ice) isProtoMessage_Payload() {}
-
-func (*ProtoMessage_Sdp) isProtoMessage_Payload() {}
-
-func (*ProtoMessage_Raw) isProtoMessage_Payload() {}
-
-func (*ProtoMessage_ClientRequestRoomStream) isProtoMessage_Payload() {}
-
-func (*ProtoMessage_ClientDisconnected) isProtoMessage_Payload() {}
-
-func (*ProtoMessage_ServerPushStream) isProtoMessage_Payload() {}
-
 var File_messages_proto protoreflect.FileDescriptor
 
 const file_messages_proto_rawDesc = "" +
@@ -413,31 +132,10 @@ const file_messages_proto_rawDesc = "" +
 	"\x0emessages.proto\x12\x05proto\x1a\vtypes.proto\x1a\x15latency_tracker.proto\"k\n" +
 	"\x10ProtoMessageBase\x12!\n" +
 	"\fpayload_type\x18\x01 \x01(\tR\vpayloadType\x124\n" +
-	"\alatency\x18\x02 \x01(\v2\x1a.proto.ProtoLatencyTrackerR\alatency\"\x9b\t\n" +
-	"\fProtoMessage\x12:\n" +
-	"\fmessage_base\x18\x01 \x01(\v2\x17.proto.ProtoMessageBaseR\vmessageBase\x126\n" +
-	"\n" +
-	"mouse_move\x18\x02 \x01(\v2\x15.proto.ProtoMouseMoveH\x00R\tmouseMove\x12@\n" +
-	"\x0emouse_move_abs\x18\x03 \x01(\v2\x18.proto.ProtoMouseMoveAbsH\x00R\fmouseMoveAbs\x129\n" +
-	"\vmouse_wheel\x18\x04 \x01(\v2\x16.proto.ProtoMouseWheelH\x00R\n" +
-	"mouseWheel\x12@\n" +
-	"\x0emouse_key_down\x18\x05 \x01(\v2\x18.proto.ProtoMouseKeyDownH\x00R\fmouseKeyDown\x12:\n" +
-	"\fmouse_key_up\x18\x06 \x01(\v2\x16.proto.ProtoMouseKeyUpH\x00R\n" +
-	"mouseKeyUp\x120\n" +
-	"\bkey_down\x18\a \x01(\v2\x13.proto.ProtoKeyDownH\x00R\akeyDown\x12*\n" +
-	"\x06key_up\x18\b \x01(\v2\x11.proto.ProtoKeyUpH\x00R\x05keyUp\x12K\n" +
-	"\x11controller_attach\x18\t \x01(\v2\x1c.proto.ProtoControllerAttachH\x00R\x10controllerAttach\x12K\n" +
-	"\x11controller_detach\x18\n" +
-	" \x01(\v2\x1c.proto.ProtoControllerDetachH\x00R\x10controllerDetach\x12K\n" +
-	"\x11controller_rumble\x18\v \x01(\v2\x1c.proto.ProtoControllerRumbleH\x00R\x10controllerRumble\x12X\n" +
-	"\x16controller_state_batch\x18\f \x01(\v2 .proto.ProtoControllerStateBatchH\x00R\x14controllerStateBatch\x12#\n" +
-	"\x03ice\x18\x14 \x01(\v2\x0f.proto.ProtoICEH\x00R\x03ice\x12#\n" +
-	"\x03sdp\x18\x15 \x01(\v2\x0f.proto.ProtoSDPH\x00R\x03sdp\x12#\n" +
-	"\x03raw\x18\x16 \x01(\v2\x0f.proto.ProtoRawH\x00R\x03raw\x12b\n" +
-	"\x1aclient_request_room_stream\x18\x17 \x01(\v2#.proto.ProtoClientRequestRoomStreamH\x00R\x17clientRequestRoomStream\x12Q\n" +
-	"\x13client_disconnected\x18\x18 \x01(\v2\x1e.proto.ProtoClientDisconnectedH\x00R\x12clientDisconnected\x12L\n" +
-	"\x12server_push_stream\x18\x19 \x01(\v2\x1c.proto.ProtoServerPushStreamH\x00R\x10serverPushStreamB\t\n" +
-	"\apayloadB\x16Z\x14relay/internal/protob\x06proto3"
+	"\alatency\x18\x02 \x01(\v2\x1a.proto.ProtoLatencyTrackerR\alatency\"v\n" +
+	"\x11ProtoMessageInput\x12:\n" +
+	"\fmessage_base\x18\x01 \x01(\v2\x17.proto.ProtoMessageBaseR\vmessageBase\x12%\n" +
+	"\x04data\x18\x02 \x01(\v2\x11.proto.ProtoInputR\x04dataB\x16Z\x14relay/internal/protob\x06proto3"
 
 var (
 	file_messages_proto_rawDescOnce sync.Once
@@ -454,51 +152,19 @@ func file_messages_proto_rawDescGZIP() []byte {
 var file_messages_proto_msgTypes = make([]protoimpl.MessageInfo, 2)
 var file_messages_proto_goTypes = []any{
 	(*ProtoMessageBase)(nil),    // 0: proto.ProtoMessageBase
-	(*ProtoMessage)(nil),                 // 1: proto.ProtoMessage
+	(*ProtoMessageInput)(nil),   // 1: proto.ProtoMessageInput
 	(*ProtoLatencyTracker)(nil), // 2: proto.ProtoLatencyTracker
-	(*ProtoMouseMove)(nil),               // 3: proto.ProtoMouseMove
-	(*ProtoMouseMoveAbs)(nil),            // 4: proto.ProtoMouseMoveAbs
-	(*ProtoMouseWheel)(nil),              // 5: proto.ProtoMouseWheel
-	(*ProtoMouseKeyDown)(nil),            // 6: proto.ProtoMouseKeyDown
-	(*ProtoMouseKeyUp)(nil),              // 7: proto.ProtoMouseKeyUp
-	(*ProtoKeyDown)(nil),                 // 8: proto.ProtoKeyDown
-	(*ProtoKeyUp)(nil),                   // 9: proto.ProtoKeyUp
-	(*ProtoControllerAttach)(nil),        // 10: proto.ProtoControllerAttach
-	(*ProtoControllerDetach)(nil),        // 11: proto.ProtoControllerDetach
-	(*ProtoControllerRumble)(nil),        // 12: proto.ProtoControllerRumble
-	(*ProtoControllerStateBatch)(nil),    // 13: proto.ProtoControllerStateBatch
-	(*ProtoICE)(nil),                     // 14: proto.ProtoICE
-	(*ProtoSDP)(nil),                     // 15: proto.ProtoSDP
-	(*ProtoRaw)(nil),                     // 16: proto.ProtoRaw
-	(*ProtoClientRequestRoomStream)(nil), // 17: proto.ProtoClientRequestRoomStream
-	(*ProtoClientDisconnected)(nil),      // 18: proto.ProtoClientDisconnected
-	(*ProtoServerPushStream)(nil),        // 19: proto.ProtoServerPushStream
+	(*ProtoInput)(nil),          // 3: proto.ProtoInput
 }
 var file_messages_proto_depIdxs = []int32{
 	2, // 0: proto.ProtoMessageBase.latency:type_name -> proto.ProtoLatencyTracker
-	0,  // 1: proto.ProtoMessage.message_base:type_name -> proto.ProtoMessageBase
-	3,  // 2: proto.ProtoMessage.mouse_move:type_name -> proto.ProtoMouseMove
-	4,  // 3: proto.ProtoMessage.mouse_move_abs:type_name -> proto.ProtoMouseMoveAbs
-	5,  // 4: proto.ProtoMessage.mouse_wheel:type_name -> proto.ProtoMouseWheel
-	6,  // 5: proto.ProtoMessage.mouse_key_down:type_name -> proto.ProtoMouseKeyDown
-	7,  // 6: proto.ProtoMessage.mouse_key_up:type_name -> proto.ProtoMouseKeyUp
-	8,  // 7: proto.ProtoMessage.key_down:type_name -> proto.ProtoKeyDown
-	9,  // 8: proto.ProtoMessage.key_up:type_name -> proto.ProtoKeyUp
-	10, // 9: proto.ProtoMessage.controller_attach:type_name -> proto.ProtoControllerAttach
-	11, // 10: proto.ProtoMessage.controller_detach:type_name -> proto.ProtoControllerDetach
-	12, // 11: proto.ProtoMessage.controller_rumble:type_name -> proto.ProtoControllerRumble
-	13, // 12: proto.ProtoMessage.controller_state_batch:type_name -> proto.ProtoControllerStateBatch
-	14, // 13: proto.ProtoMessage.ice:type_name -> proto.ProtoICE
-	15, // 14: proto.ProtoMessage.sdp:type_name -> proto.ProtoSDP
-	16, // 15: proto.ProtoMessage.raw:type_name -> proto.ProtoRaw
-	17, // 16: proto.ProtoMessage.client_request_room_stream:type_name -> proto.ProtoClientRequestRoomStream
-	18, // 17: proto.ProtoMessage.client_disconnected:type_name -> proto.ProtoClientDisconnected
-	19, // 18: proto.ProtoMessage.server_push_stream:type_name -> proto.ProtoServerPushStream
-	19, // [19:19] is the sub-list for method output_type
-	19, // [19:19] is the sub-list for method input_type
-	19, // [19:19] is the sub-list for extension type_name
-	19, // [19:19] is the sub-list for extension extendee
-	0,  // [0:19] is the sub-list for field type_name
+	0, // 1: proto.ProtoMessageInput.message_base:type_name -> proto.ProtoMessageBase
+	3, // 2: proto.ProtoMessageInput.data:type_name -> proto.ProtoInput
+	3, // [3:3] is the sub-list for method output_type
+	3, // [3:3] is the sub-list for method input_type
+	3, // [3:3] is the sub-list for extension type_name
+	3, // [3:3] is the sub-list for extension extendee
+	0, // [0:3] is the sub-list for field type_name
 }
 
 func init() { file_messages_proto_init() }
@@ -508,25 +174,6 @@ func file_messages_proto_init() {
 	}
 	file_types_proto_init()
 	file_latency_tracker_proto_init()
-	file_messages_proto_msgTypes[1].OneofWrappers = []any{
-		(*ProtoMessage_MouseMove)(nil),
-		(*ProtoMessage_MouseMoveAbs)(nil),
-		(*ProtoMessage_MouseWheel)(nil),
-		(*ProtoMessage_MouseKeyDown)(nil),
-		(*ProtoMessage_MouseKeyUp)(nil),
-		(*ProtoMessage_KeyDown)(nil),
-		(*ProtoMessage_KeyUp)(nil),
-		(*ProtoMessage_ControllerAttach)(nil),
-		(*ProtoMessage_ControllerDetach)(nil),
-		(*ProtoMessage_ControllerRumble)(nil),
-		(*ProtoMessage_ControllerStateBatch)(nil),
-		(*ProtoMessage_Ice)(nil),
-		(*ProtoMessage_Sdp)(nil),
-		(*ProtoMessage_Raw)(nil),
-		(*ProtoMessage_ClientRequestRoomStream)(nil),
-		(*ProtoMessage_ClientDisconnected)(nil),
-		(*ProtoMessage_ServerPushStream)(nil),
-	}
 	type x struct{}
 	out := protoimpl.TypeBuilder{
 		File: protoimpl.DescBuilder{