✨ feat: Add streaming support (#125)

This adds: - [x] Keyboard and mouse handling on the frontend - [x] Video and audio streaming from the backend to the frontend - [x] Input server that works with Websockets Update - 17/11 - [ ] Master docker container to run this - [ ] Steam runtime - [ ] Entrypoint.sh --------- Co-authored-by: Kristian Ollikainen <14197772+DatCaptainHorse@users.noreply.github.com> Co-authored-by: Kristian Ollikainen <DatCaptainHorse@users.noreply.github.com>
2025-12-12 16:55:37 +02:00 · 2024-12-08 14:54:56 +03:00
parent 5eb21eeadb
commit 379db1c87b
137 changed files with 12737 additions and 5234 deletions
--- a/packages/input/src/codes.ts
+++ b/packages/input/src/codes.ts
@@ -0,0 +1,113 @@
+export const keyCodeToLinuxEventCode: { [key: string]: number } = {
+    'KeyA': 30,
+    'KeyB': 48,
+    'KeyC': 46,
+    'KeyD': 32,
+    'KeyE': 18,
+    'KeyF': 33,
+    'KeyG': 34,
+    'KeyH': 35,
+    'KeyI': 23,
+    'KeyJ': 36,
+    'KeyK': 37,
+    'KeyL': 38,
+    'KeyM': 50,
+    'KeyN': 49,
+    'KeyO': 24,
+    'KeyP': 25,
+    'KeyQ': 16,
+    'KeyR': 19,
+    'KeyS': 31,
+    'KeyT': 20,
+    'KeyU': 22,
+    'KeyV': 47,
+    'KeyW': 17,
+    'KeyX': 45,
+    'KeyY': 21,
+    'KeyZ': 44,
+    'Digit1': 2,
+    'Digit2': 3,
+    'Digit3': 4,
+    'Digit4': 5,
+    'Digit5': 6,
+    'Digit6': 7,
+    'Digit7': 8,
+    'Digit8': 9,
+    'Digit9': 10,
+    'Digit0': 11,
+    'Enter': 28,
+    'Escape': 1,
+    'Backspace': 14,
+    'Tab': 15,
+    'Space': 57,
+    'Minus': 12,
+    'Equal': 13,
+    'BracketLeft': 26,
+    'BracketRight': 27,
+    'Backslash': 43,
+    'Semicolon': 39,
+    'Quote': 40,
+    'Backquote': 41,
+    'Comma': 51,
+    'Period': 52,
+    'Slash': 53,
+    'CapsLock': 58,
+    'F1': 59,
+    'F2': 60,
+    'F3': 61,
+    'F4': 62,
+    'F5': 63,
+    'F6': 64,
+    'F7': 65,
+    'F8': 66,
+    'F9': 67,
+    'F10': 68,
+    'F11': 87,
+    'F12': 88,
+    'Insert': 110,
+    'Delete': 111,
+    'ArrowUp': 103,
+    'ArrowDown': 108,
+    'ArrowLeft': 105,
+    'ArrowRight': 106,
+    'Home': 102,
+    'End': 107,
+    'PageUp': 104,
+    'PageDown': 109,
+    'NumLock': 69,
+    'ScrollLock': 70,
+    'Pause': 119,
+    'Numpad0': 82,
+    'Numpad1': 79,
+    'Numpad2': 80,
+    'Numpad3': 81,
+    'Numpad4': 75,
+    'Numpad5': 76,
+    'Numpad6': 77,
+    'Numpad7': 71,
+    'Numpad8': 72,
+    'Numpad9': 73,
+    'NumpadDivide': 98,
+    'NumpadMultiply': 55,
+    'NumpadSubtract': 74,
+    'NumpadAdd': 78,
+    'NumpadEnter': 96,
+    'NumpadDecimal': 83,
+    'ControlLeft': 29,
+    'ControlRight': 97,
+    'ShiftLeft': 42,
+    'ShiftRight': 54,
+    'AltLeft': 56,
+    'AltRight': 100,
+    //'MetaLeft': 125, // Disabled as will break input
+    //'MetaRight': 126, // Disabled as will break input
+    'ContextMenu': 127,
+};
+
+export const mouseButtonToLinuxEventCode: { [button: number]: number } = {
+    0: 272,
+    2: 273,
+    1: 274,
+    3: 275,
+    4: 276
+};
--- a/packages/input/src/index.ts
+++ b/packages/input/src/index.ts
@@ -0,0 +1,3 @@
+export * from "./keyboard"
+export * from "./mouse"
+export * from "./webrtc-stream"
--- a/packages/input/src/keyboard.ts
+++ b/packages/input/src/keyboard.ts
@@ -0,0 +1,96 @@
+import {type Input} from "./types"
+import {keyCodeToLinuxEventCode} from "./codes"
+import {MessageInput, encodeMessage} from "./messages";
+import {WebRTCStream} from "./webrtc-stream";
+import {LatencyTracker} from "./latency";
+
+interface Props {
+  webrtc: WebRTCStream;
+  canvas: HTMLCanvasElement;
+}
+
+export class Keyboard {
+  protected wrtc: WebRTCStream;
+  protected canvas: HTMLCanvasElement;
+  protected connected!: boolean;
+
+  // Store references to event listeners
+  private keydownListener: (e: KeyboardEvent) => void;
+  private keyupListener: (e: KeyboardEvent) => void;
+
+  constructor({webrtc, canvas}: Props) {
+    this.wrtc = webrtc;
+    this.canvas = canvas;
+    this.keydownListener = this.createKeyboardListener("keydown", (e: any) => ({
+      type: "KeyDown",
+      key: this.keyToVirtualKeyCode(e.code)
+    }));
+    this.keyupListener = this.createKeyboardListener("keyup", (e: any) => ({
+      type: "KeyUp",
+      key: this.keyToVirtualKeyCode(e.code)
+    }));
+    this.run()
+  }
+
+  private run() {
+    //calls all the other functions
+    if (!document.pointerLockElement) {
+      if (this.connected) {
+        this.stop()
+      }
+      return;
+    }
+
+    if (document.pointerLockElement == this.canvas) {
+      this.connected = true
+      document.addEventListener("keydown", this.keydownListener, {passive: false});
+      document.addEventListener("keyup", this.keyupListener, {passive: false});
+    } else {
+      if (this.connected) {
+        this.stop()
+      }
+    }
+  }
+
+  private stop() {
+    document.removeEventListener("keydown", this.keydownListener);
+    document.removeEventListener("keyup", this.keyupListener);
+    this.connected = false;
+  }
+
+  // Helper function to create and return mouse listeners
+  private createKeyboardListener(type: string, dataCreator: (e: Event) => Partial<Input>): (e: Event) => void {
+    return (e: Event) => {
+      e.preventDefault();
+      e.stopPropagation();
+      // Prevent repeated key events from being sent (important for games)
+      if ((e as any).repeat)
+        return;
+
+      const data = dataCreator(e as any); // type assertion because of the way dataCreator is used
+      const dataString = JSON.stringify({...data, type} as Input);
+
+      // Latency tracking
+      const tracker = new LatencyTracker("input-keyboard");
+      tracker.addTimestamp("client_send");
+      const message: MessageInput = {
+        payload_type: "input",
+        data: dataString,
+        latency: tracker,
+      };
+      this.wrtc.sendBinary(encodeMessage(message));
+    };
+  }
+
+  public dispose() {
+    document.exitPointerLock();
+    this.stop();
+    this.connected = false;
+  }
+
+  private keyToVirtualKeyCode(code: string) {
+    // Treat Home key as Escape - TODO: Make user-configurable
+    if (code === "Home") return 1;
+    return keyCodeToLinuxEventCode[code] || undefined;
+  }
+}
--- a/packages/input/src/latency.ts
+++ b/packages/input/src/latency.ts
@@ -0,0 +1,54 @@
+type TimestampEntry = {
+  stage: string;
+  time: Date;
+};
+
+export class LatencyTracker {
+  sequence_id: string;
+  timestamps: TimestampEntry[];
+  metadata?: Record<string, any>;
+
+  constructor(sequence_id: string, timestamps: TimestampEntry[] = [], metadata: Record<string, any> = {}) {
+    this.sequence_id = sequence_id;
+    this.timestamps = timestamps;
+    this.metadata = metadata;
+  }
+
+  addTimestamp(stage: string): void {
+    const timestamp: TimestampEntry = {
+      stage,
+      time: new Date(),
+    };
+    this.timestamps.push(timestamp);
+  }
+
+  // Calculates the total time between the first and last recorded timestamps.
+  getTotalLatency(): number {
+    if (this.timestamps.length < 2) return 0;
+
+    const times = this.timestamps.map((entry) => entry.time.getTime());
+    const minTime = Math.min(...times);
+    const maxTime = Math.max(...times);
+    return maxTime - minTime;
+  }
+
+  toJSON(): Record<string, any> {
+    return {
+      sequence_id: this.sequence_id,
+      timestamps: this.timestamps.map((entry) => ({
+        stage: entry.stage,
+        // Fill nanoseconds with zeros to match the expected format
+        time: entry.time.toISOString().replace(/\.(\d+)Z$/, ".$1000000Z"),
+      })),
+      metadata: this.metadata,
+    };
+  }
+
+  static fromJSON(json: any): LatencyTracker {
+    const timestamps: TimestampEntry[] = json.timestamps.map((ts: any) => ({
+      stage: ts.stage,
+      time: new Date(ts.time),
+    }));
+    return new LatencyTracker(json.sequence_id, timestamps, json.metadata);
+  }
+}
--- a/packages/input/src/messages.ts
+++ b/packages/input/src/messages.ts
@@ -0,0 +1,73 @@
+import {gzip, ungzip} from "pako";
+import {LatencyTracker} from "./latency";
+
+export interface MessageBase {
+  payload_type: string;
+}
+
+export interface MessageInput extends MessageBase {
+  payload_type: "input";
+  data: string;
+  latency?: LatencyTracker;
+}
+
+export interface MessageICE extends MessageBase {
+  payload_type: "ice";
+  candidate: RTCIceCandidateInit;
+}
+
+export interface MessageSDP extends MessageBase {
+  payload_type: "sdp";
+  sdp: RTCSessionDescriptionInit;
+}
+
+export enum JoinerType {
+  JoinerNode = 0,
+  JoinerClient = 1,
+}
+
+export interface MessageJoin extends MessageBase {
+  payload_type: "join";
+  joiner_type: JoinerType;
+}
+
+export enum AnswerType {
+  AnswerOffline = 0,
+  AnswerInUse,
+  AnswerOK
+}
+
+export interface MessageAnswer extends MessageBase {
+  payload_type: "answer";
+  answer_type: AnswerType;
+}
+
+function blobToUint8Array(blob: Blob): Promise<Uint8Array> {
+  return new Promise((resolve, reject) => {
+    const reader = new FileReader();
+    reader.onloadend = () => {
+      const arrayBuffer = reader.result as ArrayBuffer;
+      resolve(new Uint8Array(arrayBuffer));
+    };
+    reader.onerror = reject;
+    reader.readAsArrayBuffer(blob);
+  });
+}
+
+export function encodeMessage<T>(message: T): Uint8Array {
+  // Convert the message to JSON string
+  const json = JSON.stringify(message);
+  // Compress the JSON string using gzip
+  return gzip(json);
+}
+
+export async function decodeMessage<T>(data: Blob): Promise<T> {
+  // Convert the Blob to Uint8Array
+  const array = await blobToUint8Array(data);
+  // Decompress the gzip data
+  const decompressed = ungzip(array);
+  // Convert the Uint8Array to JSON string
+  const json = new TextDecoder().decode(decompressed);
+  // Parse the JSON string
+  return JSON.parse(json);
+}
--- a/packages/input/src/mouse.ts
+++ b/packages/input/src/mouse.ts
@@ -0,0 +1,112 @@
+import {type Input} from "./types"
+import {mouseButtonToLinuxEventCode} from "./codes"
+import {MessageInput, encodeMessage} from "./messages";
+import {WebRTCStream} from "./webrtc-stream";
+import {LatencyTracker} from "./latency";
+
+interface Props {
+  webrtc: WebRTCStream;
+  canvas: HTMLCanvasElement;
+}
+
+export class Mouse {
+  protected wrtc: WebRTCStream;
+  protected canvas: HTMLCanvasElement;
+  protected connected!: boolean;
+
+  // Store references to event listeners
+  private mousemoveListener: (e: MouseEvent) => void;
+  private mousedownListener: (e: MouseEvent) => void;
+  private mouseupListener: (e: MouseEvent) => void;
+  private mousewheelListener: (e: WheelEvent) => void;
+
+  constructor({webrtc, canvas}: Props) {
+    this.wrtc = webrtc;
+    this.canvas = canvas;
+
+    this.mousemoveListener = this.createMouseListener("mousemove", (e: any) => ({
+      type: "MouseMove",
+      x: e.movementX,
+      y: e.movementY
+    }));
+    this.mousedownListener = this.createMouseListener("mousedown", (e: any) => ({
+      type: "MouseKeyDown",
+      key: this.keyToVirtualKeyCode(e.button)
+    }));
+
+    this.mouseupListener = this.createMouseListener("mouseup", (e: any) => ({
+      type: "MouseKeyUp",
+      key: this.keyToVirtualKeyCode(e.button)
+    }));
+    this.mousewheelListener = this.createMouseListener("wheel", (e: any) => ({
+      type: "MouseWheel",
+      x: e.deltaX,
+      y: e.deltaY
+    }));
+
+    this.run()
+  }
+
+  private run() {
+    //calls all the other functions
+    if (!document.pointerLockElement) {
+      console.log("no pointerlock")
+      if (this.connected) {
+        this.stop()
+      }
+      return;
+    }
+
+    if (document.pointerLockElement == this.canvas) {
+      this.connected = true
+      this.canvas.addEventListener("mousemove", this.mousemoveListener, { passive: false });
+      this.canvas.addEventListener("mousedown", this.mousedownListener, { passive: false });
+      this.canvas.addEventListener("mouseup", this.mouseupListener, { passive: false });
+      this.canvas.addEventListener("wheel", this.mousewheelListener, { passive: false });
+
+    } else {
+      if (this.connected) {
+        this.stop()
+      }
+    }
+
+  }
+
+  private stop() {
+    this.canvas.removeEventListener("mousemove", this.mousemoveListener);
+    this.canvas.removeEventListener("mousedown", this.mousedownListener);
+    this.canvas.removeEventListener("mouseup", this.mouseupListener);
+    this.canvas.removeEventListener("wheel", this.mousewheelListener);
+    this.connected = false;
+  }
+
+  // Helper function to create and return mouse listeners
+  private createMouseListener(type: string, dataCreator: (e: Event) => Partial<Input>): (e: Event) => void {
+    return (e: Event) => {
+      e.preventDefault();
+      e.stopPropagation();
+      const data = dataCreator(e as any); // type assertion because of the way dataCreator is used
+      const dataString = JSON.stringify({...data, type} as Input);
+
+      // Latency tracking
+      const tracker = new LatencyTracker("input-mouse");
+      tracker.addTimestamp("client_send");
+      const message: MessageInput = {
+        payload_type: "input",
+        data: dataString,
+        latency: tracker,
+      };
+      this.wrtc.sendBinary(encodeMessage(message));
+    };
+  }
+
+  public dispose() {
+    document.exitPointerLock();
+    this.stop();
+    this.connected = false;
+  }
+
+  private keyToVirtualKeyCode(code: number) {
+    return mouseButtonToLinuxEventCode[code] || undefined;
+  }
+}
--- a/packages/input/src/types.ts
+++ b/packages/input/src/types.ts
@@ -0,0 +1,52 @@
+interface BaseInput {
+    timestamp?: number; // Add a timestamp for better context (optional)
+}
+
+interface MouseMove extends BaseInput {
+    type: "MouseMove";
+    x: number;
+    y: number;
+}
+
+interface MouseMoveAbs extends BaseInput {
+    type: "MouseMoveAbs";
+    x: number;
+    y: number;
+}
+
+interface MouseWheel extends BaseInput {
+    type: "MouseWheel";
+    x: number;
+    y: number;
+}
+
+interface MouseKeyDown extends BaseInput {
+    type: "MouseKeyDown";
+    key: number;
+}
+
+interface MouseKeyUp extends BaseInput {
+    type: "MouseKeyUp";
+    key: number;
+}
+
+interface KeyDown extends BaseInput {
+    type: "KeyDown";
+    key: number;
+}
+
+interface KeyUp extends BaseInput {
+    type: "KeyUp";
+    key: number;
+}
+
+
+export type Input =
+    | MouseMove
+    | MouseMoveAbs
+    | MouseWheel
+    | MouseKeyDown
+    | MouseKeyUp
+    | KeyDown
+    | KeyUp;
+
--- a/packages/input/src/webrtc-stream.ts
+++ b/packages/input/src/webrtc-stream.ts
@@ -0,0 +1,166 @@
+import {
+  MessageBase,
+  MessageICE,
+  MessageJoin,
+  MessageSDP,
+  MessageAnswer,
+  JoinerType,
+  AnswerType,
+  decodeMessage,
+  encodeMessage
+} from "./messages";
+
+export class WebRTCStream {
+  private _ws: WebSocket | undefined = undefined;
+  private _pc: RTCPeerConnection | undefined = undefined;
+  private _mediaStream: MediaStream | undefined = undefined;
+  private _dataChannel: RTCDataChannel | undefined = undefined;
+  private _onConnected: ((stream: MediaStream | null) => void) | undefined = undefined;
+
+  constructor(serverURL: string, roomName: string, connectedCallback: (stream: MediaStream | null) => void) {
+    // If roomName is not provided, return
+    if (roomName.length <= 0) {
+      console.error("Room name not provided");
+      return;
+    }
+
+    this._onConnected = connectedCallback;
+
+    console.log("Setting up WebSocket");
+    // Replace http/https with ws/wss
+    const wsURL = serverURL.replace(/^http/, "ws");
+    this._ws = new WebSocket(`${wsURL}/api/ws/${roomName}`);
+    this._ws.onopen = async () => {
+      console.log("WebSocket opened");
+
+      console.log("Setting up PeerConnection");
+      this._pc = new RTCPeerConnection({
+        iceServers: [
+          {
+            urls: "stun:stun.l.google.com:19302"
+          }
+        ],
+      });
+
+      this._pc.ontrack = (e) => {
+        console.log("Track received: ", e.track);
+        this._mediaStream = e.streams[e.streams.length - 1];
+      };
+
+      this._pc.onconnectionstatechange = () => {
+        console.log("Connection state: ", this._pc!.connectionState);
+        if (this._pc!.connectionState === "connected") {
+          if (this._onConnected && this._mediaStream)
+            this._onConnected(this._mediaStream);
+        }
+      };
+
+      this._pc.onicecandidate = (e) => {
+        if (e.candidate) {
+          const message: MessageICE = {
+            payload_type: "ice",
+            candidate: e.candidate
+          };
+          this._ws!.send(encodeMessage(message));
+        }
+      }
+
+      this._pc.ondatachannel = (e) => {
+        this._dataChannel = e.channel;
+        this._setupDataChannelEvents();
+      }
+
+      // Send join message
+      const joinMessage: MessageJoin = {
+        payload_type: "join",
+        joiner_type: JoinerType.JoinerClient
+      };
+      this._ws!.send(encodeMessage(joinMessage));
+    }
+
+    let iceHolder: RTCIceCandidateInit[] = [];
+
+    this._ws.onmessage = async (e) => {
+      // allow only binary
+      if (typeof e.data !== "object") return;
+      if (!e.data) return;
+      const message = await decodeMessage<MessageBase>(e.data);
+      switch (message.payload_type) {
+        case "sdp":
+          await this._pc!.setRemoteDescription((message as MessageSDP).sdp);
+          // Create our answer
+          const answer = await this._pc!.createAnswer();
+          // Force stereo in Chromium browsers
+          answer.sdp = this.forceOpusStereo(answer.sdp!);
+          await this._pc!.setLocalDescription(answer);
+          this._ws!.send(encodeMessage({
+            payload_type: "sdp",
+            sdp: answer
+          }));
+          break;
+        case "ice":
+          // If remote description is not set yet, hold the ICE candidates
+          if (this._pc!.remoteDescription) {
+            await this._pc!.addIceCandidate((message as MessageICE).candidate);
+            // Add held ICE candidates
+            for (const ice of iceHolder) {
+              await this._pc!.addIceCandidate(ice);
+            }
+            iceHolder = [];
+          } else {
+            iceHolder.push((message as MessageICE).candidate);
+          }
+          break;
+        case "answer":
+          switch ((message as MessageAnswer).answer_type) {
+            case AnswerType.AnswerOffline:
+              console.log("Room is offline");
+              // Call callback with null stream
+              if (this._onConnected)
+                this._onConnected(null);
+
+              break;
+            case AnswerType.AnswerInUse:
+              console.warn("Room is in use, we shouldn't even be getting this message");
+              break;
+            case AnswerType.AnswerOK:
+              console.log("Joining Room was successful");
+              break;
+          }
+          break;
+        default:
+          console.error("Unknown message type: ", message);
+      }
+    }
+
+    this._ws.onclose = () => {
+      console.log("WebSocket closed");
+    }
+
+    this._ws.onerror = (e) => {
+      console.error("WebSocket error: ", e);
+    }
+  }
+
+  // Forces opus to stereo in Chromium browsers, because of course
+  private forceOpusStereo(SDP: string): string {
+    // Look for "minptime=10;useinbandfec=1" and replace with "minptime=10;useinbandfec=1;stereo=1;sprop-stereo=1;"
+    return SDP.replace(/(minptime=10;useinbandfec=1)/, "$1;stereo=1;sprop-stereo=1;");
+  }
+
+  private _setupDataChannelEvents() {
+    if (!this._dataChannel) return;
+
+    this._dataChannel.onclose = () => console.log('sendChannel has closed')
+    this._dataChannel.onopen = () => console.log('sendChannel has opened')
+    this._dataChannel.onmessage = e => console.log(`Message from DataChannel '${this._dataChannel?.label}' payload '${e.data}'`)
+  }
+
+  // Send binary message through the data channel
+  public sendBinary(data: Uint8Array) {
+    if (this._dataChannel && this._dataChannel.readyState === "open")
+      this._dataChannel.send(data);
+    else
+      console.log("Data channel not open or not established.");
+  }
+}