github-mirrors/netris-nestri
1
0
Fork 0
mirror of https://github.com/nestriness/nestri.git synced 2025-12-12 08:45:38 +02:00
Code Issues Packages Projects Releases Wiki Activity

✨ feat: Add streaming support (#125)

Browse Source 
This adds:
- [x] Keyboard and mouse handling on the frontend
- [x] Video and audio streaming from the backend to the frontend
- [x] Input server that works with Websockets

Update - 17/11
- [ ] Master docker container to run this
- [ ] Steam runtime
- [ ] Entrypoint.sh

---------

Co-authored-by: Kristian Ollikainen <14197772+DatCaptainHorse@users.noreply.github.com>
Co-authored-by: Kristian Ollikainen <DatCaptainHorse@users.noreply.github.com>
This commit is contained in:
Wanjohi
2024-12-08 14:54:56 +03:00
committed by GitHub
parent 5eb21eeadb
commit 379db1c87b
137 changed files with 12737 additions and 5234 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
packages/certs/.gitignore vendored Normal file
View File

@@ -0,0 +1,3 @@
.terraform
relay_*
terraform.tfstate

61
packages/certs/.terraform.lock.hcl generated Normal file
View File

@@ -0,0 +1,61 @@
# This file is maintained automatically by "terraform init".
# Manual edits may be lost in future updates.
provider "registry.terraform.io/hashicorp/local" {
version = "2.5.2"
hashes = [
"h1:JlMZD6nYqJ8sSrFfEAH0Vk/SL8WLZRmFaMUF9PJK5wM=",
"zh:136299545178ce281c56f36965bf91c35407c11897f7082b3b983d86cb79b511",
"zh:3b4486858aa9cb8163378722b642c57c529b6c64bfbfc9461d940a84cd66ebea",
"zh:4855ee628ead847741aa4f4fc9bed50cfdbf197f2912775dd9fe7bc43fa077c0",
"zh:4b8cd2583d1edcac4011caafe8afb7a95e8110a607a1d5fb87d921178074a69b",
"zh:52084ddaff8c8cd3f9e7bcb7ce4dc1eab00602912c96da43c29b4762dc376038",
"zh:71562d330d3f92d79b2952ffdda0dad167e952e46200c767dd30c6af8d7c0ed3",
"zh:78d5eefdd9e494defcb3c68d282b8f96630502cac21d1ea161f53cfe9bb483b3",
"zh:805f81ade06ff68fa8b908d31892eaed5c180ae031c77ad35f82cb7a74b97cf4",
"zh:8b6b3ebeaaa8e38dd04e56996abe80db9be6f4c1df75ac3cccc77642899bd464",
"zh:ad07750576b99248037b897de71113cc19b1a8d0bc235eb99173cc83d0de3b1b",
"zh:b9f1c3bfadb74068f5c205292badb0661e17ac05eb23bfe8bd809691e4583d0e",
"zh:cc4cbcd67414fefb111c1bf7ab0bc4beb8c0b553d01719ad17de9a047adff4d1",
]
}
provider "registry.terraform.io/hashicorp/tls" {
version = "4.0.6"
hashes = [
"h1:dYSb3V94K5dDMtrBRLPzBpkMTPn+3cXZ/kIJdtFL+2M=",
"zh:10de0d8af02f2e578101688fd334da3849f56ea91b0d9bd5b1f7a243417fdda8",
"zh:37fc01f8b2bc9d5b055dc3e78bfd1beb7c42cfb776a4c81106e19c8911366297",
"zh:4578ca03d1dd0b7f572d96bd03f744be24c726bfd282173d54b100fd221608bb",
"zh:6c475491d1250050765a91a493ef330adc24689e8837a0f07da5a0e1269e11c1",
"zh:81bde94d53cdababa5b376bbc6947668be4c45ab655de7aa2e8e4736dfd52509",
"zh:abdce260840b7b050c4e401d4f75c7a199fafe58a8b213947a258f75ac18b3e8",
"zh:b754cebfc5184873840f16a642a7c9ef78c34dc246a8ae29e056c79939963c7a",
"zh:c928b66086078f9917aef0eec15982f2e337914c5c4dbc31dd4741403db7eb18",
"zh:cded27bee5f24de6f2ee0cfd1df46a7f88e84aaffc2ecbf3ff7094160f193d50",
"zh:d65eb3867e8f69aaf1b8bb53bd637c99c6b649ba3db16ded50fa9a01076d1a27",
"zh:ecb0c8b528c7a619fa71852bb3fb5c151d47576c5aab2bf3af4db52588722eeb",
"zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c",
]
}
provider "registry.terraform.io/vancluever/acme" {
version = "2.26.0"
constraints = "~> 2.0"
hashes = [
"h1:4Lk5cb2Fg1q1JEQf1jkrShjPC3ayukp4eFcdL4e+y0w=",
"zh:11f554916ee99d8930de6d7bb5a014ec636b53ef9ba35eea84b0d2522c78230f",
"zh:231c31271c25477c95e0a4972857b6d5e9d7c3a300cbc4b0948566d87bc46e04",
"zh:2ae165ca7a994a4c77801a82ebd9f2f6de33a4c8882381bea575b3385cc251d8",
"zh:2cf01e4694d81b24972f5dab8e5f374aa59100082ff6e2435615d9c0f24cc00e",
"zh:3de6f6f9d052dfaa5d5f366d7ca26bdebb42fc74b6e19325e67420c37ff630d3",
"zh:3fd2b4b680b970394e4d0d49c2a8e5365297e79cea418ce87197cc8bb456d8c7",
"zh:46ea249cc01dce23ff6c8f02106e693be3b046059834b60b670c45a8f4093980",
"zh:57cb181c73b6e7397744d885c788d8815ad6a43f07769e98c6327bbc37272896",
"zh:761f2adf3e63559bd279763eb91247cdebf31401d79853755453274f143cbb36",
"zh:c4a9905bf81d38201c080cb91ea85002194c47ca26619644628184a56c394b7d",
"zh:d6e3a757c357239edefb640807778fb69805b9ae5df84a811a2d505c51089367",
"zh:d713856e4a459e1091cbb19ffb830d25cd88953d3e54acd46db0729c77a531d8",
"zh:f7cb8dec263d0ee223737dad3b6fa8071258f41cfa9e0b8cf7f337f9f501fc3b",
]
}

24
packages/certs/README.md Normal file
View File

@@ -0,0 +1,24 @@
## Usage
1. Update the terraform.tfvars file with your domain and email.
2. Run `terraform init` to initialize the Terraform working directory.
3. Run `terraform plan` to see the planned changes.
4. Run `terraform apply` to create the resources and obtain the certificate.
Outputs
The configuration provides two sensitive outputs:
```bash
certificate_pem: The full certificate chain
private_key_pem: The private key for the certificate
```
These can be then be used in your `moq-relay` as it requires SSL/TLS certificates.
## Note
The generated certificate and key files are saved locally and ignored by git:
```git
.terraform
relay_*
```

7
packages/certs/input.tf Normal file
View File

@@ -0,0 +1,7 @@
variable "email" {
description = "Your email address, used for LetsEncrypt"
}
variable "domain" {
description = "domain name"
}

65
packages/certs/main.tf Normal file
View File

@@ -0,0 +1,65 @@
terraform {
required_providers {
acme = {
source = "vancluever/acme"
version = "~> 2.0"
}
}
}
provider "acme" {
server_url = "https://acme-v02.api.letsencrypt.org/directory"
}
resource "acme_registration" "reg" {
email_address = "wanjohiryan33@gmail.com"
}
resource "tls_private_key" "relay" {
algorithm = "ECDSA"
ecdsa_curve = "P256"
}
resource "acme_registration" "relay" {
account_key_pem = tls_private_key.relay.private_key_pem
email_address = var.email
}
resource "acme_certificate" "relay" {
account_key_pem = acme_registration.relay.account_key_pem
common_name = "relay.${var.domain}"
subject_alternative_names = ["*.relay.${var.domain}"]
key_type = tls_private_key.relay.ecdsa_curve
recursive_nameservers = ["8.8.8.8:53"]
dns_challenge {
provider = "route53"
}
}
# New resources to save certificate and private key
resource "local_file" "cert_file" {
content = "${acme_certificate.relay.certificate_pem}${acme_certificate.relay.issuer_pem}"
filename = "${path.module}/relay_cert.crt"
file_permission = "0644"
directory_permission = "0755"
}
resource "local_file" "key_file" {
content = acme_certificate.relay.private_key_pem
filename = "${path.module}/relay_key.key"
file_permission = "0600"
directory_permission = "0755"
}
# Outputs for certificate and private key
output "certificate_pem" {
value = "${acme_certificate.relay.certificate_pem}${acme_certificate.relay.issuer_pem}"
sensitive = true
}
output "private_key_pem" {
value = acme_certificate.relay.private_key_pem
sensitive = true
}

2
packages/certs/terraform.tfvars Normal file
View File

@@ -0,0 +1,2 @@
domain = "fst.so"
email = "wanjohiryan33@gmail.com"

1
packages/eslint-config/qwik.js
View File

@@ -43,6 +43,7 @@ module.exports = {
"prefer-spread": "off",
"no-case-declarations": "off",
"no-console": "off",
"qwik/no-use-visible-task": "off",
"@typescript-eslint/consistent-type-imports": "warn",
"@typescript-eslint/no-unnecessary-condition": "warn",
},

9
packages/input/package.json Normal file
View File

@@ -0,0 +1,9 @@
{
"name": "@nestri/input",
"version": "0.0.0",
"private": true,
"sideEffects": false,
"exports": {
".": "./src/index.ts"
}
}

113
packages/input/src/codes.ts Normal file
View File

@@ -0,0 +1,113 @@
export const keyCodeToLinuxEventCode: { [key: string]: number } = {
'KeyA': 30,
'KeyB': 48,
'KeyC': 46,
'KeyD': 32,
'KeyE': 18,
'KeyF': 33,
'KeyG': 34,
'KeyH': 35,
'KeyI': 23,
'KeyJ': 36,
'KeyK': 37,
'KeyL': 38,
'KeyM': 50,
'KeyN': 49,
'KeyO': 24,
'KeyP': 25,
'KeyQ': 16,
'KeyR': 19,
'KeyS': 31,
'KeyT': 20,
'KeyU': 22,
'KeyV': 47,
'KeyW': 17,
'KeyX': 45,
'KeyY': 21,
'KeyZ': 44,
'Digit1': 2,
'Digit2': 3,
'Digit3': 4,
'Digit4': 5,
'Digit5': 6,
'Digit6': 7,
'Digit7': 8,
'Digit8': 9,
'Digit9': 10,
'Digit0': 11,
'Enter': 28,
'Escape': 1,
'Backspace': 14,
'Tab': 15,
'Space': 57,
'Minus': 12,
'Equal': 13,
'BracketLeft': 26,
'BracketRight': 27,
'Backslash': 43,
'Semicolon': 39,
'Quote': 40,
'Backquote': 41,
'Comma': 51,
'Period': 52,
'Slash': 53,
'CapsLock': 58,
'F1': 59,
'F2': 60,
'F3': 61,
'F4': 62,
'F5': 63,
'F6': 64,
'F7': 65,
'F8': 66,
'F9': 67,
'F10': 68,
'F11': 87,
'F12': 88,
'Insert': 110,
'Delete': 111,
'ArrowUp': 103,
'ArrowDown': 108,
'ArrowLeft': 105,
'ArrowRight': 106,
'Home': 102,
'End': 107,
'PageUp': 104,
'PageDown': 109,
'NumLock': 69,
'ScrollLock': 70,
'Pause': 119,
'Numpad0': 82,
'Numpad1': 79,
'Numpad2': 80,
'Numpad3': 81,
'Numpad4': 75,
'Numpad5': 76,
'Numpad6': 77,
'Numpad7': 71,
'Numpad8': 72,
'Numpad9': 73,
'NumpadDivide': 98,
'NumpadMultiply': 55,
'NumpadSubtract': 74,
'NumpadAdd': 78,
'NumpadEnter': 96,
'NumpadDecimal': 83,
'ControlLeft': 29,
'ControlRight': 97,
'ShiftLeft': 42,
'ShiftRight': 54,
'AltLeft': 56,
'AltRight': 100,
//'MetaLeft': 125, // Disabled as will break input
//'MetaRight': 126, // Disabled as will break input
'ContextMenu': 127,
};
export const mouseButtonToLinuxEventCode: { [button: number]: number } = {
0: 272,
2: 273,
1: 274,
3: 275,
4: 276
};

3
packages/input/src/index.ts Normal file
View File

@@ -0,0 +1,3 @@
export * from "./keyboard"
export * from "./mouse"
export * from "./webrtc-stream"

96
packages/input/src/keyboard.ts Normal file
View File

@@ -0,0 +1,96 @@
import {type Input} from "./types"
import {keyCodeToLinuxEventCode} from "./codes"
import {MessageInput, encodeMessage} from "./messages";
import {WebRTCStream} from "./webrtc-stream";
import {LatencyTracker} from "./latency";
interface Props {
webrtc: WebRTCStream;
canvas: HTMLCanvasElement;
}
export class Keyboard {
protected wrtc: WebRTCStream;
protected canvas: HTMLCanvasElement;
protected connected!: boolean;
// Store references to event listeners
private keydownListener: (e: KeyboardEvent) => void;
private keyupListener: (e: KeyboardEvent) => void;
constructor({webrtc, canvas}: Props) {
this.wrtc = webrtc;
this.canvas = canvas;
this.keydownListener = this.createKeyboardListener("keydown", (e: any) => ({
type: "KeyDown",
key: this.keyToVirtualKeyCode(e.code)
}));
this.keyupListener = this.createKeyboardListener("keyup", (e: any) => ({
type: "KeyUp",
key: this.keyToVirtualKeyCode(e.code)
}));
this.run()
}
private run() {
//calls all the other functions
if (!document.pointerLockElement) {
if (this.connected) {
this.stop()
}
return;
}
if (document.pointerLockElement == this.canvas) {
this.connected = true
document.addEventListener("keydown", this.keydownListener, {passive: false});
document.addEventListener("keyup", this.keyupListener, {passive: false});
} else {
if (this.connected) {
this.stop()
}
}
}
private stop() {
document.removeEventListener("keydown", this.keydownListener);
document.removeEventListener("keyup", this.keyupListener);
this.connected = false;
}
// Helper function to create and return mouse listeners
private createKeyboardListener(type: string, dataCreator: (e: Event) => Partial<Input>): (e: Event) => void {
return (e: Event) => {
e.preventDefault();
e.stopPropagation();
// Prevent repeated key events from being sent (important for games)
if ((e as any).repeat)
return;
const data = dataCreator(e as any); // type assertion because of the way dataCreator is used
const dataString = JSON.stringify({...data, type} as Input);
// Latency tracking
const tracker = new LatencyTracker("input-keyboard");
tracker.addTimestamp("client_send");
const message: MessageInput = {
payload_type: "input",
data: dataString,
latency: tracker,
};
this.wrtc.sendBinary(encodeMessage(message));
};
}
public dispose() {
document.exitPointerLock();
this.stop();
this.connected = false;
}
private keyToVirtualKeyCode(code: string) {
// Treat Home key as Escape - TODO: Make user-configurable
if (code === "Home") return 1;
return keyCodeToLinuxEventCode[code] || undefined;
}
}

54
packages/input/src/latency.ts Normal file
View File

@@ -0,0 +1,54 @@
type TimestampEntry = {
stage: string;
time: Date;
};
export class LatencyTracker {
sequence_id: string;
timestamps: TimestampEntry[];
metadata?: Record<string, any>;
constructor(sequence_id: string, timestamps: TimestampEntry[] = [], metadata: Record<string, any> = {}) {
this.sequence_id = sequence_id;
this.timestamps = timestamps;
this.metadata = metadata;
}
addTimestamp(stage: string): void {
const timestamp: TimestampEntry = {
stage,
time: new Date(),
};
this.timestamps.push(timestamp);
}
// Calculates the total time between the first and last recorded timestamps.
getTotalLatency(): number {
if (this.timestamps.length < 2) return 0;
const times = this.timestamps.map((entry) => entry.time.getTime());
const minTime = Math.min(...times);
const maxTime = Math.max(...times);
return maxTime - minTime;
}
toJSON(): Record<string, any> {
return {
sequence_id: this.sequence_id,
timestamps: this.timestamps.map((entry) => ({
stage: entry.stage,
// Fill nanoseconds with zeros to match the expected format
time: entry.time.toISOString().replace(/\.(\d+)Z$/, ".$1000000Z"),
})),
metadata: this.metadata,
};
}
static fromJSON(json: any): LatencyTracker {
const timestamps: TimestampEntry[] = json.timestamps.map((ts: any) => ({
stage: ts.stage,
time: new Date(ts.time),
}));
return new LatencyTracker(json.sequence_id, timestamps, json.metadata);
}
}

73
packages/input/src/messages.ts Normal file
View File

@@ -0,0 +1,73 @@
import {gzip, ungzip} from "pako";
import {LatencyTracker} from "./latency";
export interface MessageBase {
payload_type: string;
}
export interface MessageInput extends MessageBase {
payload_type: "input";
data: string;
latency?: LatencyTracker;
}
export interface MessageICE extends MessageBase {
payload_type: "ice";
candidate: RTCIceCandidateInit;
}
export interface MessageSDP extends MessageBase {
payload_type: "sdp";
sdp: RTCSessionDescriptionInit;
}
export enum JoinerType {
JoinerNode = 0,
JoinerClient = 1,
}
export interface MessageJoin extends MessageBase {
payload_type: "join";
joiner_type: JoinerType;
}
export enum AnswerType {
AnswerOffline = 0,
AnswerInUse,
AnswerOK
}
export interface MessageAnswer extends MessageBase {
payload_type: "answer";
answer_type: AnswerType;
}
function blobToUint8Array(blob: Blob): Promise<Uint8Array> {
return new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onloadend = () => {
const arrayBuffer = reader.result as ArrayBuffer;
resolve(new Uint8Array(arrayBuffer));
};
reader.onerror = reject;
reader.readAsArrayBuffer(blob);
});
}
export function encodeMessage<T>(message: T): Uint8Array {
// Convert the message to JSON string
const json = JSON.stringify(message);
// Compress the JSON string using gzip
return gzip(json);
}
export async function decodeMessage<T>(data: Blob): Promise<T> {
// Convert the Blob to Uint8Array
const array = await blobToUint8Array(data);
// Decompress the gzip data
const decompressed = ungzip(array);
// Convert the Uint8Array to JSON string
const json = new TextDecoder().decode(decompressed);
// Parse the JSON string
return JSON.parse(json);
}

112
packages/input/src/mouse.ts Normal file
View File

@@ -0,0 +1,112 @@
import {type Input} from "./types"
import {mouseButtonToLinuxEventCode} from "./codes"
import {MessageInput, encodeMessage} from "./messages";
import {WebRTCStream} from "./webrtc-stream";
import {LatencyTracker} from "./latency";
interface Props {
webrtc: WebRTCStream;
canvas: HTMLCanvasElement;
}
export class Mouse {
protected wrtc: WebRTCStream;
protected canvas: HTMLCanvasElement;
protected connected!: boolean;
// Store references to event listeners
private mousemoveListener: (e: MouseEvent) => void;
private mousedownListener: (e: MouseEvent) => void;
private mouseupListener: (e: MouseEvent) => void;
private mousewheelListener: (e: WheelEvent) => void;
constructor({webrtc, canvas}: Props) {
this.wrtc = webrtc;
this.canvas = canvas;
this.mousemoveListener = this.createMouseListener("mousemove", (e: any) => ({
type: "MouseMove",
x: e.movementX,
y: e.movementY
}));
this.mousedownListener = this.createMouseListener("mousedown", (e: any) => ({
type: "MouseKeyDown",
key: this.keyToVirtualKeyCode(e.button)
}));
this.mouseupListener = this.createMouseListener("mouseup", (e: any) => ({
type: "MouseKeyUp",
key: this.keyToVirtualKeyCode(e.button)
}));
this.mousewheelListener = this.createMouseListener("wheel", (e: any) => ({
type: "MouseWheel",
x: e.deltaX,
y: e.deltaY
}));
this.run()
}
private run() {
//calls all the other functions
if (!document.pointerLockElement) {
console.log("no pointerlock")
if (this.connected) {
this.stop()
}
return;
}
if (document.pointerLockElement == this.canvas) {
this.connected = true
this.canvas.addEventListener("mousemove", this.mousemoveListener, { passive: false });
this.canvas.addEventListener("mousedown", this.mousedownListener, { passive: false });
this.canvas.addEventListener("mouseup", this.mouseupListener, { passive: false });
this.canvas.addEventListener("wheel", this.mousewheelListener, { passive: false });
} else {
if (this.connected) {
this.stop()
}
}
}
private stop() {
this.canvas.removeEventListener("mousemove", this.mousemoveListener);
this.canvas.removeEventListener("mousedown", this.mousedownListener);
this.canvas.removeEventListener("mouseup", this.mouseupListener);
this.canvas.removeEventListener("wheel", this.mousewheelListener);
this.connected = false;
}
// Helper function to create and return mouse listeners
private createMouseListener(type: string, dataCreator: (e: Event) => Partial<Input>): (e: Event) => void {
return (e: Event) => {
e.preventDefault();
e.stopPropagation();
const data = dataCreator(e as any); // type assertion because of the way dataCreator is used
const dataString = JSON.stringify({...data, type} as Input);
// Latency tracking
const tracker = new LatencyTracker("input-mouse");
tracker.addTimestamp("client_send");
const message: MessageInput = {
payload_type: "input",
data: dataString,
latency: tracker,
};
this.wrtc.sendBinary(encodeMessage(message));
};
}
public dispose() {
document.exitPointerLock();
this.stop();
this.connected = false;
}
private keyToVirtualKeyCode(code: number) {
return mouseButtonToLinuxEventCode[code] || undefined;
}
}

52
packages/input/src/types.ts Normal file
View File

@@ -0,0 +1,52 @@
interface BaseInput {
timestamp?: number; // Add a timestamp for better context (optional)
}
interface MouseMove extends BaseInput {
type: "MouseMove";
x: number;
y: number;
}
interface MouseMoveAbs extends BaseInput {
type: "MouseMoveAbs";
x: number;
y: number;
}
interface MouseWheel extends BaseInput {
type: "MouseWheel";
x: number;
y: number;
}
interface MouseKeyDown extends BaseInput {
type: "MouseKeyDown";
key: number;
}
interface MouseKeyUp extends BaseInput {
type: "MouseKeyUp";
key: number;
}
interface KeyDown extends BaseInput {
type: "KeyDown";
key: number;
}
interface KeyUp extends BaseInput {
type: "KeyUp";
key: number;
}
export type Input =
| MouseMove
| MouseMoveAbs
| MouseWheel
| MouseKeyDown
| MouseKeyUp
| KeyDown
| KeyUp;

166
packages/input/src/webrtc-stream.ts Normal file
View File

@@ -0,0 +1,166 @@
import {
MessageBase,
MessageICE,
MessageJoin,
MessageSDP,
MessageAnswer,
JoinerType,
AnswerType,
decodeMessage,
encodeMessage
} from "./messages";
export class WebRTCStream {
private _ws: WebSocket | undefined = undefined;
private _pc: RTCPeerConnection | undefined = undefined;
private _mediaStream: MediaStream | undefined = undefined;
private _dataChannel: RTCDataChannel | undefined = undefined;
private _onConnected: ((stream: MediaStream | null) => void) | undefined = undefined;
constructor(serverURL: string, roomName: string, connectedCallback: (stream: MediaStream | null) => void) {
// If roomName is not provided, return
if (roomName.length <= 0) {
console.error("Room name not provided");
return;
}
this._onConnected = connectedCallback;
console.log("Setting up WebSocket");
// Replace http/https with ws/wss
const wsURL = serverURL.replace(/^http/, "ws");
this._ws = new WebSocket(`${wsURL}/api/ws/${roomName}`);
this._ws.onopen = async () => {
console.log("WebSocket opened");
console.log("Setting up PeerConnection");
this._pc = new RTCPeerConnection({
iceServers: [
{
urls: "stun:stun.l.google.com:19302"
}
],
});
this._pc.ontrack = (e) => {
console.log("Track received: ", e.track);
this._mediaStream = e.streams[e.streams.length - 1];
};
this._pc.onconnectionstatechange = () => {
console.log("Connection state: ", this._pc!.connectionState);
if (this._pc!.connectionState === "connected") {
if (this._onConnected && this._mediaStream)
this._onConnected(this._mediaStream);
}
};
this._pc.onicecandidate = (e) => {
if (e.candidate) {
const message: MessageICE = {
payload_type: "ice",
candidate: e.candidate
};
this._ws!.send(encodeMessage(message));
}
}
this._pc.ondatachannel = (e) => {
this._dataChannel = e.channel;
this._setupDataChannelEvents();
}
// Send join message
const joinMessage: MessageJoin = {
payload_type: "join",
joiner_type: JoinerType.JoinerClient
};
this._ws!.send(encodeMessage(joinMessage));
}
let iceHolder: RTCIceCandidateInit[] = [];
this._ws.onmessage = async (e) => {
// allow only binary
if (typeof e.data !== "object") return;
if (!e.data) return;
const message = await decodeMessage<MessageBase>(e.data);
switch (message.payload_type) {
case "sdp":
await this._pc!.setRemoteDescription((message as MessageSDP).sdp);
// Create our answer
const answer = await this._pc!.createAnswer();
// Force stereo in Chromium browsers
answer.sdp = this.forceOpusStereo(answer.sdp!);
await this._pc!.setLocalDescription(answer);
this._ws!.send(encodeMessage({
payload_type: "sdp",
sdp: answer
}));
break;
case "ice":
// If remote description is not set yet, hold the ICE candidates
if (this._pc!.remoteDescription) {
await this._pc!.addIceCandidate((message as MessageICE).candidate);
// Add held ICE candidates
for (const ice of iceHolder) {
await this._pc!.addIceCandidate(ice);
}
iceHolder = [];
} else {
iceHolder.push((message as MessageICE).candidate);
}
break;
case "answer":
switch ((message as MessageAnswer).answer_type) {
case AnswerType.AnswerOffline:
console.log("Room is offline");
// Call callback with null stream
if (this._onConnected)
this._onConnected(null);
break;
case AnswerType.AnswerInUse:
console.warn("Room is in use, we shouldn't even be getting this message");
break;
case AnswerType.AnswerOK:
console.log("Joining Room was successful");
break;
}
break;
default:
console.error("Unknown message type: ", message);
}
}
this._ws.onclose = () => {
console.log("WebSocket closed");
}
this._ws.onerror = (e) => {
console.error("WebSocket error: ", e);
}
}
// Forces opus to stereo in Chromium browsers, because of course
private forceOpusStereo(SDP: string): string {
// Look for "minptime=10;useinbandfec=1" and replace with "minptime=10;useinbandfec=1;stereo=1;sprop-stereo=1;"
return SDP.replace(/(minptime=10;useinbandfec=1)/, "$1;stereo=1;sprop-stereo=1;");
}
private _setupDataChannelEvents() {
if (!this._dataChannel) return;
this._dataChannel.onclose = () => console.log('sendChannel has closed')
this._dataChannel.onopen = () => console.log('sendChannel has opened')
this._dataChannel.onmessage = e => console.log(`Message from DataChannel '${this._dataChannel?.label}' payload '${e.data}'`)
}
// Send binary message through the data channel
public sendBinary(data: Uint8Array) {
if (this._dataChannel && this._dataChannel.readyState === "open")
this._dataChannel.send(data);
else
console.log("Data channel not open or not established.");
}
}

32
packages/master/go.mod Normal file
View File

@@ -0,0 +1,32 @@
module master
go 1.23.3
require github.com/docker/docker v27.3.1+incompatible
require (
github.com/Microsoft/go-winio v0.4.14 // indirect
github.com/containerd/log v0.1.0 // indirect
github.com/distribution/reference v0.6.0 // indirect
github.com/docker/go-connections v0.5.0 // indirect
github.com/docker/go-units v0.5.0 // indirect
github.com/felixge/httpsnoop v1.0.4 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/moby/docker-image-spec v1.3.1 // indirect
github.com/moby/term v0.5.0 // indirect
github.com/morikuni/aec v1.0.0 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.0 // indirect
github.com/pkg/errors v0.9.1 // indirect
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0 // indirect
go.opentelemetry.io/otel v1.32.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.32.0 // indirect
go.opentelemetry.io/otel/metric v1.32.0 // indirect
go.opentelemetry.io/otel/sdk v1.32.0 // indirect
go.opentelemetry.io/otel/trace v1.32.0 // indirect
golang.org/x/sys v0.27.0 // indirect
golang.org/x/time v0.8.0 // indirect
gotest.tools/v3 v3.5.1 // indirect
)

123
packages/master/go.sum Normal file
View File

@@ -0,0 +1,123 @@
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
github.com/Microsoft/go-winio v0.4.14 h1:+hMXMk01us9KgxGb7ftKQt2Xpf5hH/yky+TDA+qxleU=
github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA=
github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8=
github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
github.com/docker/docker v27.3.1+incompatible h1:KttF0XoteNTicmUtBO0L2tP+J7FGRFTjaEF4k6WdhfI=
github.com/docker/docker v27.3.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.23.0 h1:ad0vkEBuk23VJzZR9nkLVG0YAoN9coASF1GusYX6AlU=
github.com/grpc-ecosystem/grpc-gateway/v2 v2.23.0/go.mod h1:igFoXX2ELCW06bol23DWPB5BEWfZISOzSP5K2sbLea0=
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0 h1:DheMAlT6POBP+gh8RUH19EOTnQIor5QE0uSRPtzCpSw=
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.57.0/go.mod h1:wZcGmeVO9nzP67aYSLDqXNWK87EZWhi7JWj1v7ZXf94=
go.opentelemetry.io/otel v1.32.0 h1:WnBN+Xjcteh0zdk01SVqV55d/m62NJLJdIyb4y/WO5U=
go.opentelemetry.io/otel v1.32.0/go.mod h1:00DCVSB0RQcnzlwyTfqtxSm+DRr9hpYrHjNGiBHVQIg=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.32.0 h1:IJFEoHiytixx8cMiVAO+GmHR6Frwu+u5Ur8njpFO6Ac=
go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.32.0/go.mod h1:3rHrKNtLIoS0oZwkY2vxi+oJcwFRWdtUyRII+so45p8=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.32.0 h1:cMyu9O88joYEaI47CnQkxO1XZdpoTF9fEnW2duIddhw=
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.32.0/go.mod h1:6Am3rn7P9TVVeXYG+wtcGE7IE1tsQ+bP3AuWcKt/gOI=
go.opentelemetry.io/otel/metric v1.32.0 h1:xV2umtmNcThh2/a/aCP+h64Xx5wsj8qqnkYZktzNa0M=
go.opentelemetry.io/otel/metric v1.32.0/go.mod h1:jH7CIbbK6SH2V2wE16W05BHCtIDzauciCRLoc/SyMv8=
go.opentelemetry.io/otel/sdk v1.32.0 h1:RNxepc9vK59A8XsgZQouW8ue8Gkb4jpWtJm9ge5lEG4=
go.opentelemetry.io/otel/sdk v1.32.0/go.mod h1:LqgegDBjKMmb2GC6/PrTnteJG39I8/vJCAP9LlJXEjU=
go.opentelemetry.io/otel/trace v1.32.0 h1:WIC9mYrXf8TmY/EXuULKc8hR17vE+Hjv2cssQDe03fM=
go.opentelemetry.io/otel/trace v1.32.0/go.mod h1:+i4rkvCraA+tG6AzwloGaCtkx53Fa+L+V8e9a7YvhT8=
go.opentelemetry.io/proto/otlp v1.3.1 h1:TrMUixzpM0yuc/znrFTP9MMRh8trP93mkCiDVeXrui0=
go.opentelemetry.io/proto/otlp v1.3.1/go.mod h1:0X1WI4de4ZsLrrJNLAQbFeLCm3T7yBkR0XqQ7niQU+8=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s=
golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.20.0 h1:gK/Kv2otX8gz+wn7Rmb3vT96ZwuoxnQlY+HlJVj7Qug=
golang.org/x/text v0.20.0/go.mod h1:D4IsuqiFMhST5bX19pQ9ikHC2GsaKyk/oF+pn3ducp4=
golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg=
golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28 h1:M0KvPgPmDZHPlbRbaNU1APr28TvwvvdUPlSv7PUvy8g=
google.golang.org/genproto/googleapis/api v0.0.0-20241104194629-dd2ea8efbc28/go.mod h1:dguCy7UOdZhTvLzDyt15+rOrawrpM4q7DD9dQ1P11P4=
google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28 h1:XVhgTWWV3kGQlwJHR3upFWZeTsei6Oks1apkZSeonIE=
google.golang.org/genproto/googleapis/rpc v0.0.0-20241104194629-dd2ea8efbc28/go.mod h1:GX3210XPVPUjJbTUbvwI8f2IpZDMZuPJWDzDuebbviI=
google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E=
google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA=
google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA=
google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU=
gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU=

80
packages/master/main.go Normal file
View File

@@ -0,0 +1,80 @@
package main
import (
"context"
"io"
"os"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/image"
"github.com/docker/docker/client"
"github.com/docker/docker/pkg/stdcopy"
)
func main() {
ctx := context.Background()
cli, err := client.NewClientWithOpts(client.FromEnv, client.WithAPIVersionNegotiation())
if err != nil {
panic(err)
}
defer cli.Close()
// Try to get the Docker version
_, err = cli.ServerVersion(ctx)
if err != nil {
// If an error occurs (e.g., Docker is not running), return false
panic(err)
}
// Download the image
containerName := "hello-world"
reader, err := cli.ImagePull(ctx, containerName, image.PullOptions{})
if err != nil {
panic(err)
}
defer reader.Close()
// cli.ImagePull is asynchronous.
// The reader needs to be read completely for the pull operation to complete.
// If stdout is not required, consider using io.Discard instead of os.Stdout.
io.Copy(os.Stdout, reader)
resp, err := cli.ContainerCreate(ctx, &container.Config{
Image: "hello-world",
},
nil, nil, nil, containerName)
if err != nil {
panic(err)
}
// Start the container
if err := cli.ContainerStart(ctx, resp.ID, container.StartOptions{}); err != nil {
panic(err)
}
// Wait for the container to finish and get its logs
statusCh, errCh := cli.ContainerWait(ctx, resp.ID, container.WaitConditionNotRunning)
select {
case err := <-errCh:
if err != nil {
panic(err)
}
case <-statusCh:
}
out, err := cli.ContainerLogs(ctx, resp.ID, container.LogsOptions{ShowStdout: true})
if err != nil {
panic(err)
}
stdcopy.StdCopy(os.Stdout, os.Stderr, out)
// Remove the container
if err := cli.ContainerRemove(ctx, resp.ID, container.RemoveOptions{}); err != nil {
panic(err)
}
}

2
packages/moq/.eslintrc.cjs
View File

@@ -8,7 +8,7 @@ module.exports = {
"prettier",
],
parser: "@typescript-eslint/parser",
plugins: ["@typescript-eslint", "prettier"],
plugins: ["@typescript-eslint", "prettier", "solid"],
root: true,
env: {
browser: true,

16
packages/moq/common/async.ts
View File

@@ -1,7 +1,7 @@
export class Deferred<T> {
promise: Promise<T>
resolve!: (value: T | PromiseLike<T>) => void
reject!: (reason: any) => void
reject!: (reason: unknown) => void
pending = true
constructor() {
@@ -35,16 +35,19 @@ export class Watch<T> {
update(v: T | ((v: T) => T)) {
if (!this.#next.pending) {
throw new Error("already closed")
throw new Error("closed")
}
// If we're given a function, call it with the current value
let value: T
if (v instanceof Function) {
v = v(this.#current[0])
value = v(this.#current[0])
} else {
value = v
}
const next = new Deferred<WatchNext<T>>()
this.#current = [v, next.promise]
this.#current = [value, next.promise]
this.#next.resolve(this.#current)
this.#next = next
}
@@ -53,6 +56,10 @@ export class Watch<T> {
this.#current[1] = undefined
this.#next.resolve(this.#current)
}
closed() {
return !this.#next.pending
}
}
// Wakes up a multiple consumers.
@@ -88,6 +95,7 @@ export class Queue<T> {
}
async push(v: T) {
if (this.#closed) throw new Error("closed")
const w = this.#stream.writable.getWriter()
await w.write(v)
w.releaseLock()

12
packages/moq/common/error.ts
View File

@@ -1,14 +1,14 @@
// I hate javascript
export function asError(e: any): Error {
export function asError(e: unknown): Error {
if (e instanceof Error) {
return e
} else if (typeof e === "string") {
return new Error(e)
} else {
return new Error(String(e))
}
if (typeof e === "string") {
return new Error(e)
}
return new Error(String(e))
}
export function isError(e: any): e is Error {
export function isError(e: unknown): e is Error {
return e instanceof Error
}

11
packages/moq/common/hex.ts Normal file
View File

@@ -0,0 +1,11 @@
export function decode(str: string): Uint8Array {
const bytes = new Uint8Array(str.length / 2)
for (let i = 0; i < bytes.length; i += 1) {
bytes[i] = Number.parseInt(str.slice(2 * i, 2 * i + 2), 16)
}
return bytes
}
export function encode(_bytes: Uint8Array): string {
throw "todo"
}

20
packages/moq/common/ring.ts
View File

@@ -2,8 +2,8 @@
enum STATE {
READ_POS = 0, // The current read position
WRITE_POS, // The current write position
LENGTH, // Clever way of saving the total number of enums values.
WRITE_POS = 1, // The current write position
LENGTH = 2, // Clever way of saving the total number of enums values.
}
interface FrameCopyToOptions {
@@ -62,16 +62,12 @@ export class Ring {
const readPos = Atomics.load(this.state, STATE.READ_POS)
const writePos = Atomics.load(this.state, STATE.WRITE_POS)
const startPos = writePos
let endPos = writePos + frame.numberOfFrames
const available = this.capacity - (writePos - readPos)
if (available <= 0) return 0
if (endPos > readPos + this.capacity) {
endPos = readPos + this.capacity
if (endPos <= startPos) {
// No space to write
return 0
}
}
const toWrite = Math.min(frame.numberOfFrames, available)
const startPos = writePos
const endPos = writePos + toWrite
const startIndex = startPos % this.capacity
const endIndex = endPos % this.capacity
@@ -114,7 +110,7 @@ export class Ring {
Atomics.store(this.state, STATE.WRITE_POS, endPos)
return endPos - startPos
return toWrite
}
read(dst: Float32Array[]): number {

69
packages/moq/contribute/audio.ts
View File

@@ -1,15 +1,67 @@
import { Deferred } from "../common/async"
import type { Frame } from "../karp/frame"
import type { Group, Track } from "../transfork"
import { Closed } from "../transfork/error"
const SUPPORTED = [
// TODO support AAC
// "mp4a"
"Opus",
]
export class Packer {
#source: MediaStreamTrackProcessor<AudioData>
#encoder: Encoder
#data: Track
#current?: Group
constructor(track: MediaStreamAudioTrack, encoder: Encoder, data: Track) {
this.#source = new MediaStreamTrackProcessor({ track })
this.#encoder = encoder
this.#data = data
}
async run() {
const output = new WritableStream({
write: (chunk) => this.#write(chunk),
close: () => this.#close(),
abort: (e) => this.#close(e),
})
return this.#source.readable.pipeThrough(this.#encoder.frames).pipeTo(output)
}
#write(frame: Frame) {
// TODO use a fixed interval instead of keyframes (audio)
// TODO actually just align with video
if (!this.#current || frame.type === "key") {
if (this.#current) {
this.#current.close()
}
this.#current = this.#data.appendGroup()
}
this.#current.writeFrame(frame.data)
}
#close(err?: unknown) {
const closed = Closed.from(err)
if (this.#current) {
this.#current.close(closed)
}
this.#data.close(closed)
}
}
export class Encoder {
#encoder!: AudioEncoder
#encoderConfig: AudioEncoderConfig
#decoderConfig?: AudioDecoderConfig
#decoderConfig = new Deferred<AudioDecoderConfig>()
frames: TransformStream<AudioData, AudioDecoderConfig | EncodedAudioChunk>
frames: TransformStream<AudioData, EncodedAudioChunk>
constructor(config: AudioEncoderConfig) {
this.#encoderConfig = config
@@ -21,7 +73,7 @@ export class Encoder {
})
}
#start(controller: TransformStreamDefaultController<AudioDecoderConfig | EncodedAudioChunk>) {
#start(controller: TransformStreamDefaultController<EncodedAudioChunk>) {
this.#encoder = new AudioEncoder({
output: (frame, metadata) => {
this.#enqueue(controller, frame, metadata)
@@ -40,17 +92,16 @@ export class Encoder {
}
#enqueue(
controller: TransformStreamDefaultController<AudioDecoderConfig | EncodedAudioChunk>,
controller: TransformStreamDefaultController<EncodedAudioChunk>,
frame: EncodedAudioChunk,
metadata?: EncodedAudioChunkMetadata,
) {
const config = metadata?.decoderConfig
if (config && !this.#decoderConfig) {
if (config && !this.#decoderConfig.pending) {
const config = metadata.decoderConfig
if (!config) throw new Error("missing decoder config")
controller.enqueue(config)
this.#decoderConfig = config
this.#decoderConfig.resolve(config)
}
controller.enqueue(frame)
@@ -72,4 +123,8 @@ export class Encoder {
get config() {
return this.#encoderConfig
}
async decoderConfig(): Promise<AudioDecoderConfig> {
return await this.#decoderConfig.promise
}
}

261
packages/moq/contribute/broadcast.ts
View File

@@ -1,15 +1,14 @@
import { Connection, SubscribeRecv } from "../transport"
import { asError } from "../common/error"
import { Segment } from "./segment"
import { Track } from "./track"
import * as Catalog from "../media/catalog"
import * as Catalog from "../karp/catalog"
import * as Transfork from "../transfork"
import * as Audio from "./audio"
import * as Video from "./video"
import { isAudioTrackSettings, isVideoTrackSettings } from "../common/settings"
export interface BroadcastConfig {
namespace: string
connection: Connection
path: string[]
media: MediaStream
id?: number
audio?: AudioEncoderConfig
video?: VideoEncoderConfig
@@ -21,221 +20,89 @@ export interface BroadcastConfigTrack {
}
export class Broadcast {
#tracks = new Map<string, Track>()
readonly config: BroadcastConfig
readonly catalog: Catalog.Root
readonly connection: Connection
readonly namespace: string
#running: Promise<void>
#config: BroadcastConfig
#path: string[]
constructor(config: BroadcastConfig) {
this.connection = config.connection
this.config = config
this.namespace = config.namespace
const id = config.id || new Date().getTime() / 1000
const tracks: Catalog.Track[] = []
this.#config = config
this.#path = config.path.concat(id.toString())
}
for (const media of this.config.media.getTracks()) {
const track = new Track(media, config)
this.#tracks.set(track.name, track)
async publish(connection: Transfork.Connection) {
const broadcast: Catalog.Broadcast = {
path: this.#config.path,
audio: [],
video: [],
}
for (const media of this.#config.media.getTracks()) {
const settings = media.getSettings()
const info = {
name: media.id, // TODO way too verbose
priority: media.kind === "video" ? 1 : 2,
}
const track = new Transfork.Track(this.#config.path.concat(info.name), info.priority)
if (isVideoTrackSettings(settings)) {
if (!config.video) {
if (!this.#config.video) {
throw new Error("no video configuration provided")
}
const video: Catalog.VideoTrack = {
namespace: this.namespace,
name: `${track.name}.m4s`,
initTrack: `${track.name}.mp4`,
selectionParams: {
mimeType: "video/mp4",
codec: config.video.codec,
width: settings.width,
height: settings.height,
framerate: settings.frameRate,
bitrate: config.video.bitrate,
},
const encoder = new Video.Encoder(this.#config.video)
const packer = new Video.Packer(media as MediaStreamVideoTrack, encoder, track)
// TODO handle error
packer.run().catch((err) => console.error("failed to run video packer: ", err))
const decoder = await encoder.decoderConfig()
const description = decoder.description ? new Uint8Array(decoder.description as ArrayBuffer) : undefined
const video: Catalog.Video = {
track: info,
codec: decoder.codec,
description: description,
resolution: { width: settings.width, height: settings.height },
frame_rate: settings.frameRate,
bitrate: this.#config.video.bitrate,
}
tracks.push(video)
broadcast.video.push(video)
} else if (isAudioTrackSettings(settings)) {
if (!config.audio) {
if (!this.#config.audio) {
throw new Error("no audio configuration provided")
}
const audio: Catalog.AudioTrack = {
namespace: this.namespace,
name: `${track.name}.m4s`,
initTrack: `${track.name}.mp4`,
selectionParams: {
mimeType: "audio/ogg",
codec: config.audio.codec,
samplerate: settings.sampleRate,
//sampleSize: settings.sampleSize,
channelConfig: `${settings.channelCount}`,
bitrate: config.audio.bitrate,
},
const encoder = new Audio.Encoder(this.#config.audio)
const packer = new Audio.Packer(media as MediaStreamAudioTrack, encoder, track)
packer.run().catch((err) => console.error("failed to run audio packer: ", err)) // TODO handle error
const decoder = await encoder.decoderConfig()
const audio: Catalog.Audio = {
track: info,
codec: decoder.codec,
sample_rate: settings.sampleRate,
channel_count: settings.channelCount,
bitrate: this.#config.audio.bitrate,
}
tracks.push(audio)
broadcast.audio.push(audio)
} else {
throw new Error(`unknown track type: ${media.kind}`)
}
connection.publish(track.reader())
}
this.catalog = {
version: 1,
streamingFormat: 1,
streamingFormatVersion: "0.2",
supportsDeltaUpdates: false,
commonTrackFields: {
packaging: "cmaf",
renderGroup: 1,
},
tracks,
}
const track = new Transfork.Track(this.#config.path.concat("catalog.json"), 0)
track.appendGroup().writeFrames(Catalog.encode(broadcast))
this.#running = this.#run()
connection.publish(track.reader())
}
async #run() {
await this.connection.announce(this.namespace)
for (;;) {
const subscriber = await this.connection.subscribed()
if (!subscriber) break
// Run an async task to serve each subscription.
this.#serveSubscribe(subscriber).catch((e) => {
const err = asError(e)
console.warn("failed to serve subscribe", err)
})
}
}
async #serveSubscribe(subscriber: SubscribeRecv) {
try {
const [base, ext] = splitExt(subscriber.track)
if (ext === "catalog") {
await this.#serveCatalog(subscriber, base)
} else if (ext === "mp4") {
await this.#serveInit(subscriber, base)
} else if (ext === "m4s") {
await this.#serveTrack(subscriber, base)
} else {
throw new Error(`unknown subscription: ${subscriber.track}`)
}
} catch (e) {
const err = asError(e)
await subscriber.close(1n, `failed to process subscribe: ${err.message}`)
} finally {
// TODO we can't close subscribers because there's no support for clean termination
// await subscriber.close()
}
}
async #serveCatalog(subscriber: SubscribeRecv, name: string) {
// We only support ".catalog"
if (name !== "") throw new Error(`unknown catalog: ${name}`)
const bytes = Catalog.encode(this.catalog)
// Send a SUBSCRIBE_OK
await subscriber.ack()
const stream = await subscriber.group({ group: 0 })
await stream.write({ object: 0, payload: bytes })
await stream.close()
}
async #serveInit(subscriber: SubscribeRecv, name: string) {
const track = this.#tracks.get(name)
if (!track) throw new Error(`no track with name ${subscriber.track}`)
// Send a SUBSCRIBE_OK
await subscriber.ack()
const init = await track.init()
const stream = await subscriber.group({ group: 0 })
await stream.write({ object: 0, payload: init })
await stream.close()
}
async #serveTrack(subscriber: SubscribeRecv, name: string) {
const track = this.#tracks.get(name)
if (!track) throw new Error(`no track with name ${subscriber.track}`)
// Send a SUBSCRIBE_OK
await subscriber.ack()
const segments = track.segments().getReader()
for (;;) {
const { value: segment, done } = await segments.read()
if (done) break
// Serve the segment and log any errors that occur.
this.#serveSegment(subscriber, segment).catch((e) => {
const err = asError(e)
console.warn("failed to serve segment", err)
})
}
}
async #serveSegment(subscriber: SubscribeRecv, segment: Segment) {
// Create a new stream for each segment.
const stream = await subscriber.group({
group: segment.id,
priority: 0, // TODO
})
let object = 0
// Pipe the segment to the stream.
const chunks = segment.chunks().getReader()
for (;;) {
const { value, done } = await chunks.read()
if (done) break
await stream.write({
object,
payload: value,
})
object += 1
}
await stream.close()
}
// Attach the captured video stream to the given video element.
attach(video: HTMLVideoElement) {
video.srcObject = this.config.media
}
close() {
// TODO implement publish close
}
// Returns the error message when the connection is closed
async closed(): Promise<Error> {
try {
await this.#running
return new Error("closed") // clean termination
} catch (e) {
return asError(e)
}
}
}
function splitExt(s: string): [string, string] {
const i = s.lastIndexOf(".")
if (i < 0) throw new Error(`no extension found`)
return [s.substring(0, i), s.substring(i + 1)]
close() {}
}

7
packages/moq/contribute/chunk.ts
View File

@@ -1,7 +0,0 @@
// Extends EncodedVideoChunk, allowing a new "init" type
export interface Chunk {
type: "init" | "key" | "delta"
timestamp: number // microseconds
duration: number // microseconds
data: Uint8Array
}

165
packages/moq/contribute/container.ts
View File

@@ -1,165 +0,0 @@
import * as MP4 from "../media/mp4"
import { Chunk } from "./chunk"
type DecoderConfig = AudioDecoderConfig | VideoDecoderConfig
type EncodedChunk = EncodedAudioChunk | EncodedVideoChunk
export class Container {
#mp4: MP4.ISOFile
#frame?: EncodedAudioChunk | EncodedVideoChunk // 1 frame buffer
#track?: number
#segment = 0
encode: TransformStream<DecoderConfig | EncodedChunk, Chunk>
constructor() {
this.#mp4 = new MP4.ISOFile()
this.#mp4.init()
this.encode = new TransformStream({
transform: (frame, controller) => {
if (isDecoderConfig(frame)) {
return this.#init(frame, controller)
} else {
return this.#enqueue(frame, controller)
}
},
})
}
#init(frame: DecoderConfig, controller: TransformStreamDefaultController<Chunk>) {
if (this.#track) throw new Error("duplicate decoder config")
let codec = frame.codec.substring(0, 4)
if (codec == "opus") {
codec = "Opus"
}
const options: MP4.TrackOptions = {
type: codec,
timescale: 1_000_000,
}
if (isVideoConfig(frame)) {
options.width = frame.codedWidth
options.height = frame.codedHeight
} else {
options.channel_count = frame.numberOfChannels
options.samplerate = frame.sampleRate
}
if (!frame.description) throw new Error("missing frame description")
const desc = frame.description as ArrayBufferLike
if (codec === "avc1") {
options.avcDecoderConfigRecord = desc
} else if (codec === "hev1") {
options.hevcDecoderConfigRecord = desc
} else if (codec === "Opus") {
// description is an identification header: https://datatracker.ietf.org/doc/html/rfc7845#section-5.1
// The first 8 bytes are the magic string "OpusHead", followed by what we actually want.
const dops = new MP4.BoxParser.dOpsBox(undefined)
// Annoyingly, the header is little endian while MP4 is big endian, so we have to parse.
const data = new MP4.Stream(desc, 8, MP4.Stream.LITTLE_ENDIAN)
dops.parse(data)
dops.Version = 0
options.description = dops
options.hdlr = "soun"
} else {
throw new Error(`unsupported codec: ${codec}`)
}
this.#track = this.#mp4.addTrack(options)
if (!this.#track) throw new Error("failed to initialize MP4 track")
const buffer = MP4.ISOFile.writeInitializationSegment(this.#mp4.ftyp!, this.#mp4.moov!, 0, 0)
const data = new Uint8Array(buffer)
controller.enqueue({
type: "init",
timestamp: 0,
duration: 0,
data,
})
}
#enqueue(frame: EncodedChunk, controller: TransformStreamDefaultController<Chunk>) {
// Check if we should create a new segment
if (frame.type == "key") {
this.#segment += 1
} else if (this.#segment == 0) {
throw new Error("must start with keyframe")
}
// We need a one frame buffer to compute the duration
if (!this.#frame) {
this.#frame = frame
return
}
const duration = frame.timestamp - this.#frame.timestamp
// TODO avoid this extra copy by writing to the mdat directly
// ...which means changing mp4box.js to take an offset instead of ArrayBuffer
const buffer = new Uint8Array(this.#frame.byteLength)
this.#frame.copyTo(buffer)
if (!this.#track) throw new Error("missing decoder config")
// Add the sample to the container
this.#mp4.addSample(this.#track, buffer, {
duration,
dts: this.#frame.timestamp,
cts: this.#frame.timestamp,
is_sync: this.#frame.type == "key",
})
const stream = new MP4.Stream(undefined, 0, MP4.Stream.BIG_ENDIAN)
// Moof and mdat atoms are written in pairs.
// TODO remove the moof/mdat from the Box to reclaim memory once everything works
for (;;) {
const moof = this.#mp4.moofs.shift()
const mdat = this.#mp4.mdats.shift()
if (!moof && !mdat) break
if (!moof) throw new Error("moof missing")
if (!mdat) throw new Error("mdat missing")
moof.write(stream)
mdat.write(stream)
}
// TODO avoid this extra copy by writing to the buffer provided in copyTo
const data = new Uint8Array(stream.buffer)
controller.enqueue({
type: this.#frame.type,
timestamp: this.#frame.timestamp,
duration: this.#frame.duration ?? 0,
data,
})
this.#frame = frame
}
/* TODO flush the last frame
#flush(controller: TransformStreamDefaultController<Chunk>) {
if (this.#frame) {
// TODO guess the duration
this.#enqueue(this.#frame, 0, controller)
}
}
*/
}
function isDecoderConfig(frame: DecoderConfig | EncodedChunk): frame is DecoderConfig {
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
return (frame as DecoderConfig).codec !== undefined
}
function isVideoConfig(frame: DecoderConfig): frame is VideoDecoderConfig {
return (frame as VideoDecoderConfig).codedWidth !== undefined
}

18
packages/moq/contribute/segment.ts
View File

@@ -1,10 +1,10 @@
import { Chunk } from "./chunk"
import type { Frame } from "../karp/frame"
export class Segment {
id: number
// Take in a stream of chunks
input: WritableStream<Chunk>
// Take in a stream of frames
input: WritableStream<Frame>
// Output a stream of bytes, which we fork for each new subscriber.
#cache: ReadableStream<Uint8Array>
@@ -16,16 +16,18 @@ export class Segment {
// Set a max size for each segment, dropping the tail if it gets too long.
// We tee the reader, so this limit applies to the FASTEST reader.
const backpressure = new ByteLengthQueuingStrategy({ highWaterMark: 8_000_000 })
const backpressure = new ByteLengthQueuingStrategy({
highWaterMark: 8_000_000,
})
const transport = new TransformStream<Chunk, Uint8Array>(
const transport = new TransformStream<Frame, Uint8Array>(
{
transform: (chunk: Chunk, controller) => {
transform: (frame: Frame, controller) => {
// Compute the max timestamp of the segment
this.timestamp = Math.max(chunk.timestamp + chunk.duration)
this.timestamp = Math.max(this.timestamp, frame.timestamp)
// Push the chunk to any listeners.
controller.enqueue(chunk.data)
controller.enqueue(frame.data)
},
},
undefined,

30
packages/moq/contribute/track.ts
View File

@@ -1,9 +1,8 @@
import { Segment } from "./segment"
import { Notify } from "../common/async"
import { Chunk } from "./chunk"
import { Container } from "./container"
import { BroadcastConfig } from "./broadcast"
import type { BroadcastConfig } from "./broadcast"
import { Segment } from "./segment"
import type { Frame } from "../karp/frame"
import * as Audio from "./audio"
import * as Video from "./video"
@@ -36,7 +35,6 @@ export class Track {
async #runAudio(track: MediaStreamAudioTrack, config: AudioEncoderConfig) {
const source = new MediaStreamTrackProcessor({ track })
const encoder = new Audio.Encoder(config)
const container = new Container()
// Split the container at keyframe boundaries
const segments = new WritableStream({
@@ -45,13 +43,12 @@ export class Track {
abort: (e) => this.#close(e),
})
return source.readable.pipeThrough(encoder.frames).pipeThrough(container.encode).pipeTo(segments)
return source.readable.pipeThrough(encoder.frames).pipeTo(segments)
}
async #runVideo(track: MediaStreamVideoTrack, config: VideoEncoderConfig) {
const source = new MediaStreamTrackProcessor({ track })
const encoder = new Video.Encoder(config)
const container = new Container()
// Split the container at keyframe boundaries
const segments = new WritableStream({
@@ -60,18 +57,12 @@ export class Track {
abort: (e) => this.#close(e),
})
return source.readable.pipeThrough(encoder.frames).pipeThrough(container.encode).pipeTo(segments)
return source.readable.pipeThrough(encoder.frames).pipeTo(segments)
}
async #write(chunk: Chunk) {
if (chunk.type === "init") {
this.#init = chunk.data
this.#notify.wake()
return
}
async #write(frame: Frame) {
let current = this.#segments.at(-1)
if (!current || chunk.type === "key") {
if (!current || frame.type === "key") {
if (current) {
await current.input.close()
}
@@ -88,7 +79,7 @@ export class Track {
const first = this.#segments[0]
// Expire after 10s
if (chunk.timestamp - first.timestamp < 10_000_000) break
if (frame.timestamp - first.timestamp < 10_000_000) break
this.#segments.shift()
this.#offset += 1
@@ -99,7 +90,7 @@ export class Track {
const writer = current.input.getWriter()
if ((writer.desiredSize || 0) > 0) {
await writer.write(chunk)
await writer.write(frame)
} else {
console.warn("dropping chunk", writer.desiredSize)
}
@@ -147,7 +138,8 @@ export class Track {
if (this.#error) {
controller.error(this.#error)
return
} else if (this.#closed) {
}
if (this.#closed) {
controller.close()
return
}

4
packages/moq/contribute/tsconfig.json
View File

@@ -9,10 +9,10 @@
"path": "../common"
},
{
"path": "../transport"
"path": "../transfork"
},
{
"path": "../media"
"path": "../karp"
}
]
}

70
packages/moq/contribute/video.ts
View File

@@ -1,3 +1,8 @@
import { Deferred } from "../common/async"
import type { Frame } from "../karp/frame"
import type { Group, Track } from "../transfork"
import { Closed } from "../transfork/error"
const SUPPORTED = [
"avc1", // H.264
"hev1", // HEVC (aka h.265)
@@ -8,10 +13,55 @@ export interface EncoderSupported {
codecs: string[]
}
export class Packer {
#source: MediaStreamTrackProcessor<VideoFrame>
#encoder: Encoder
#data: Track
#current?: Group
constructor(track: MediaStreamVideoTrack, encoder: Encoder, data: Track) {
this.#source = new MediaStreamTrackProcessor({ track })
this.#encoder = encoder
this.#data = data
}
async run() {
const output = new WritableStream({
write: (chunk) => this.#write(chunk),
close: () => this.#close(),
abort: (e) => this.#close(e),
})
return this.#source.readable.pipeThrough(this.#encoder.frames).pipeTo(output)
}
#write(frame: Frame) {
if (!this.#current || frame.type === "key") {
if (this.#current) {
this.#current.close()
}
this.#current = this.#data.appendGroup()
}
frame.encode(this.#current)
}
#close(err?: unknown) {
const closed = Closed.from(err)
if (this.#current) {
this.#current.close(closed)
}
this.#data.close(closed)
}
}
export class Encoder {
#encoder!: VideoEncoder
#encoderConfig: VideoEncoderConfig
#decoderConfig?: VideoDecoderConfig
#decoderConfig = new Deferred<VideoDecoderConfig>()
// true if we should insert a keyframe, undefined when the encoder should decide
#keyframeNext: true | undefined = true
@@ -20,7 +70,7 @@ export class Encoder {
#keyframeCounter = 0
// Converts raw rames to encoded frames.
frames: TransformStream<VideoFrame, VideoDecoderConfig | EncodedVideoChunk>
frames: TransformStream<VideoFrame, EncodedVideoChunk>
constructor(config: VideoEncoderConfig) {
config.bitrateMode ??= "constant"
@@ -53,12 +103,17 @@ export class Encoder {
return !!res.supported
}
async decoderConfig(): Promise<VideoDecoderConfig> {
return await this.#decoderConfig.promise
}
#start(controller: TransformStreamDefaultController<EncodedVideoChunk>) {
this.#encoder = new VideoEncoder({
output: (frame, metadata) => {
this.#enqueue(controller, frame, metadata)
},
error: (err) => {
this.#decoderConfig.reject(err)
throw err
},
})
@@ -77,23 +132,22 @@ export class Encoder {
}
#enqueue(
controller: TransformStreamDefaultController<VideoDecoderConfig | EncodedVideoChunk>,
controller: TransformStreamDefaultController<EncodedVideoChunk>,
frame: EncodedVideoChunk,
metadata?: EncodedVideoChunkMetadata,
) {
if (!this.#decoderConfig) {
if (this.#decoderConfig.pending) {
const config = metadata?.decoderConfig
if (!config) throw new Error("missing decoder config")
controller.enqueue(config)
this.#decoderConfig = config
this.#decoderConfig.resolve(config)
}
if (frame.type === "key") {
this.#keyframeCounter = 0
} else {
this.#keyframeCounter += 1
if (this.#keyframeCounter + this.#encoder.encodeQueueSize >= 2 * this.#encoderConfig.framerate!) {
const framesPerGop = this.#encoderConfig.framerate ? 2 * this.#encoderConfig.framerate : 60
if (this.#keyframeCounter + this.#encoder.encodeQueueSize >= framesPerGop) {
this.#keyframeNext = true
}
}

20
packages/moq/karp/catalog/audio.ts Normal file
View File

@@ -0,0 +1,20 @@
import { type Track, decodeTrack } from "./track"
export interface Audio {
track: Track
codec: string
sample_rate: number
channel_count: number
bitrate?: number
}
export function decodeAudio(o: unknown): o is Audio {
if (typeof o !== "object" || o === null) return false
const obj = o as Partial<Audio>
if (!decodeTrack(obj.track)) return false
if (typeof obj.codec !== "string") return false
if (typeof obj.sample_rate !== "number") return false
if (typeof obj.channel_count !== "number") return false
return true
}

62
packages/moq/karp/catalog/broadcast.ts Normal file
View File

@@ -0,0 +1,62 @@
import * as Transfork from "../../transfork"
import { type Audio, decodeAudio } from "./audio"
import { type Video, decodeVideo } from "./video"
export interface Broadcast {
path: string[]
video: Video[]
audio: Audio[]
}
export function encode(catalog: Broadcast): Uint8Array {
const encoder = new TextEncoder()
console.debug("encoding catalog", catalog)
const str = JSON.stringify(catalog)
return encoder.encode(str)
}
export function decode(path: string[], raw: Uint8Array): Broadcast {
const decoder = new TextDecoder()
const str = decoder.decode(raw)
const catalog = JSON.parse(str)
if (!decodeBroadcast(catalog)) {
console.error("invalid catalog", catalog)
throw new Error("invalid catalog")
}
catalog.path = path
return catalog
}
export async function fetch(connection: Transfork.Connection, path: string[]): Promise<Broadcast> {
const track = new Transfork.Track(path.concat("catalog.json"), 0)
const sub = await connection.subscribe(track)
try {
const segment = await sub.nextGroup()
if (!segment) throw new Error("no catalog data")
const frame = await segment.readFrame()
if (!frame) throw new Error("no catalog frame")
segment.close()
return decode(path, frame)
} finally {
sub.close()
}
}
export function decodeBroadcast(o: unknown): o is Broadcast {
if (typeof o !== "object" || o === null) return false
const catalog = o as Partial<Broadcast>
if (catalog.audio === undefined) catalog.audio = []
if (!Array.isArray(catalog.audio)) return false
if (!catalog.audio.every((track: unknown) => decodeAudio(track))) return false
if (catalog.video === undefined) catalog.video = []
if (!Array.isArray(catalog.video)) return false
if (!catalog.video.every((track: unknown) => decodeVideo(track))) return false
return true
}

7
packages/moq/karp/catalog/index.ts Normal file
View File

@@ -0,0 +1,7 @@
import type { Audio } from "./audio"
import { type Broadcast, decode, encode, fetch } from "./broadcast"
import type { Track } from "./track"
import type { Video } from "./video"
export type { Audio, Video, Track, Broadcast }
export { encode, decode, fetch }

15
packages/moq/karp/catalog/track.ts Normal file
View File

@@ -0,0 +1,15 @@
export type GroupOrder = "desc" | "asc"
export interface Track {
name: string
priority: number
}
export function decodeTrack(o: unknown): o is Track {
if (typeof o !== "object" || o === null) return false
const obj = o as Partial<Track>
if (typeof obj.name !== "string") return false
if (typeof obj.priority !== "number") return false
return true
}

29
packages/moq/karp/catalog/video.ts Normal file
View File

@@ -0,0 +1,29 @@
import * as Hex from "../../common/hex"
import { type Track, decodeTrack } from "./track"
export interface Video {
track: Track
codec: string
description?: Uint8Array
bitrate?: number
frame_rate?: number
resolution: Dimensions
}
export interface Dimensions {
width: number
height: number
}
export function decodeVideo(o: unknown): o is Video {
if (typeof o !== "object" || o === null) return false
const obj = o as Partial<Video>
if (!decodeTrack(obj.track)) return false
if (typeof obj.codec !== "string") return false
if (typeof obj.description !== "string") return false
obj.description = Hex.decode(obj.description)
return true
}

64
packages/moq/karp/frame.ts Normal file
View File

@@ -0,0 +1,64 @@
import type { Group, GroupReader } from "../transfork/model"
import { setVint62 } from "../transfork/stream"
export type FrameType = "key" | "delta"
export class Frame {
type: FrameType
timestamp: number
data: Uint8Array
constructor(type: FrameType, timestamp: number, data: Uint8Array) {
this.type = type
this.timestamp = timestamp
this.data = data
}
static async decode(group: GroupReader): Promise<Frame | undefined> {
const kind = group.index === 0 ? "key" : "delta"
const payload = await group.readFrame()
if (!payload) {
return undefined
}
const [timestamp, data] = decode_timestamp(payload)
return new Frame(kind, timestamp, data)
}
encode(group: Group) {
if ((group.length === 0) !== (this.type === "key")) {
throw new Error(`invalid ${this.type} position`)
}
let frame = new Uint8Array(8 + this.data.byteLength)
const size = setVint62(frame, BigInt(this.timestamp)).byteLength
frame.set(this.data, size)
frame = new Uint8Array(frame.buffer, 0, this.data.byteLength + size)
group.writeFrame(frame)
}
}
// QUIC VarInt
function decode_timestamp(buf: Uint8Array): [number, Uint8Array] {
const size = 1 << ((buf[0] & 0xc0) >> 6)
const view = new DataView(buf.buffer, buf.byteOffset, size)
const remain = new Uint8Array(buf.buffer, buf.byteOffset + size, buf.byteLength - size)
let v: number
if (size === 1) {
v = buf[0] & 0x3f
} else if (size === 2) {
v = view.getInt16(0) & 0x3fff
} else if (size === 4) {
v = view.getUint32(0) & 0x3fffffff
} else if (size === 8) {
// NOTE: Precision loss above 2^52
v = Number(view.getBigUint64(0) & 0x3fffffffffffffffn)
} else {
throw new Error("impossible")
}
return [v, remain]
}

6
packages/moq/media/tsconfig.json → packages/moq/karp/tsconfig.json
View File

@@ -1,12 +1,10 @@
{
"extends": "../tsconfig.json",
"include": ["."],
"compilerOptions": {
"types": ["mp4box"]
},
"compilerOptions": {},
"references": [
{
"path": "../transport"
"path": "../transfork"
},
{
"path": "../common"

218
packages/moq/media/catalog/index.ts
View File

@@ -1,218 +0,0 @@
import { Connection } from "../../transport"
import { asError } from "../../common/error"
export interface CommonTrackFields {
namespace?: string
packaging?: string
renderGroup?: number
altGroup?: number
}
export interface Root {
version: number
streamingFormat: number
streamingFormatVersion: string
supportsDeltaUpdates: boolean
commonTrackFields: CommonTrackFields
tracks: Track[]
}
export function encode(catalog: Root): Uint8Array {
const encoder = new TextEncoder()
const str = JSON.stringify(catalog)
return encoder.encode(str)
}
export function decode(raw: Uint8Array): Root {
const decoder = new TextDecoder()
const str = decoder.decode(raw)
const catalog = JSON.parse(str)
if (!isRoot(catalog)) {
throw new Error("invalid catalog")
}
// Merge common track fields into each track.
for (const track of catalog.tracks) {
track.altGroup ??= catalog.commonTrackFields.altGroup
track.namespace ??= catalog.commonTrackFields.namespace
track.packaging ??= catalog.commonTrackFields.packaging
track.renderGroup ??= catalog.commonTrackFields.renderGroup
}
return catalog
}
export async function fetch(connection: Connection, namespace: string): Promise<Root> {
const subscribe = await connection.subscribe(namespace, ".catalog")
try {
const segment = await subscribe.data()
if (!segment) throw new Error("no catalog data")
const chunk = await segment.read()
if (!chunk) throw new Error("no catalog chunk")
await segment.close()
await subscribe.close() // we done
if (chunk.payload instanceof Uint8Array) {
return decode(chunk.payload)
} else {
throw new Error("invalid catalog chunk")
}
} catch (e) {
const err = asError(e)
// Close the subscription after we're done.
await subscribe.close(1n, err.message)
throw err
}
}
export function isRoot(catalog: any): catalog is Root {
if (!isCatalogFieldValid(catalog, "packaging")) return false
if (!isCatalogFieldValid(catalog, "namespace")) return false
if (!Array.isArray(catalog.tracks)) return false
return catalog.tracks.every((track: any) => isTrack(track))
}
export interface Track {
namespace?: string
name: string
depends?: any[]
packaging?: string
renderGroup?: number
selectionParams: SelectionParams // technically optional but not really
altGroup?: number
initTrack?: string
initData?: string
}
export interface Mp4Track extends Track {
initTrack?: string
initData?: string
selectionParams: Mp4SelectionParams
}
export interface SelectionParams {
codec?: string
mimeType?: string
bitrate?: number
lang?: string
}
export interface Mp4SelectionParams extends SelectionParams {
mimeType: "video/mp4"
}
export interface AudioTrack extends Track {
name: string
selectionParams: AudioSelectionParams
}
export interface AudioSelectionParams extends SelectionParams {
samplerate: number
channelConfig: string
}
export interface VideoTrack extends Track {
name: string
selectionParams: VideoSelectionParams
temporalId?: number
spatialId?: number
}
export interface VideoSelectionParams extends SelectionParams {
width: number
height: number
displayWidth?: number
displayHeight?: number
framerate?: number
}
export function isTrack(track: any): track is Track {
if (typeof track.name !== "string") return false
return true
}
export function isMp4Track(track: any): track is Mp4Track {
if (!isTrack(track)) return false
if (typeof track.initTrack !== "string" && typeof track.initData !== "string") return false
if (typeof track.selectionParams.mimeType !== "string") return false
return true
}
export function isVideoTrack(track: any): track is VideoTrack {
if (!isTrack(track)) return false
return isVideoSelectionParams(track.selectionParams)
}
export function isVideoSelectionParams(params: any): params is VideoSelectionParams {
if (typeof params.width !== "number") return false
if (typeof params.height !== "number") return false
return true
}
export function isAudioTrack(track: any): track is AudioTrack {
if (!isTrack(track)) return false
return isAudioSelectionParams(track.selectionParams)
}
export function isAudioSelectionParams(params: any): params is AudioSelectionParams {
if (typeof params.channelConfig !== "string") return false
if (typeof params.samplerate !== "number") return false
return true
}
function isCatalogFieldValid(catalog: any, field: string): boolean {
//packaging,namespace if common would be listed in commonTrackFields but if fields
//in commonTrackFields are mentiond in Tracks , the fields in Tracks precedes
function isValidPackaging(packaging: any): boolean {
return packaging === "cmaf" || packaging === "loc"
}
function isValidNamespace(namespace: any): boolean {
return typeof namespace === "string"
}
let isValidField: (value: any) => boolean
if (field === "packaging") {
isValidField = isValidPackaging
} else if (field === "namespace") {
isValidField = isValidNamespace
} else {
throw new Error(`Invalid field: ${field}`)
}
if (catalog.commonTrackFields[field] !== undefined && !isValidField(catalog.commonTrackFields[field])) {
return false
}
for (const track of catalog.tracks) {
if (track[field] !== undefined && !isValidField(track[field])) {
return false
}
}
return true
}
export function isMediaTrack(track: any): track is Track {
if (track.name.toLowerCase().includes("audio") || track.name.toLowerCase().includes("video")) {
return true
}
if (track.selectionParams && track.selectionParams.codec) {
const codec = track.selectionParams.codec.toLowerCase()
const acceptedCodecs = ["mp4a", "avc1"]
for (const acceptedCodec of acceptedCodecs) {
if (codec.includes(acceptedCodec)) {
return true
}
}
}
return false
}

37
packages/moq/media/mp4/index.ts
View File

@@ -1,37 +0,0 @@
// Rename some stuff so it's on brand.
// We need a separate file so this file can use the rename too.
import * as MP4 from "./rename"
export * from "./rename"
export * from "./parser"
export function isAudioTrack(track: MP4.Track): track is MP4.AudioTrack {
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
return (track as MP4.AudioTrack).audio !== undefined
}
export function isVideoTrack(track: MP4.Track): track is MP4.VideoTrack {
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
return (track as MP4.VideoTrack).video !== undefined
}
// TODO contribute to mp4box
MP4.BoxParser.dOpsBox.prototype.write = function (stream: MP4.Stream) {
this.size = this.ChannelMappingFamily === 0 ? 11 : 13 + this.ChannelMapping!.length
this.writeHeader(stream)
stream.writeUint8(this.Version)
stream.writeUint8(this.OutputChannelCount)
stream.writeUint16(this.PreSkip)
stream.writeUint32(this.InputSampleRate)
stream.writeInt16(this.OutputGain)
stream.writeUint8(this.ChannelMappingFamily)
if (this.ChannelMappingFamily !== 0) {
stream.writeUint8(this.StreamCount!)
stream.writeUint8(this.CoupledCount!)
for (const mapping of this.ChannelMapping!) {
stream.writeUint8(mapping)
}
}
}

71
packages/moq/media/mp4/parser.ts
View File

@@ -1,71 +0,0 @@
import * as MP4 from "./index"
export interface Frame {
track: MP4.Track // The track this frame belongs to
sample: MP4.Sample // The actual sample contain the frame data
}
// Decode a MP4 container into individual samples.
export class Parser {
info!: MP4.Info
#mp4 = MP4.New()
#offset = 0
#samples: Array<Frame> = []
constructor(init: Uint8Array) {
this.#mp4.onError = (err) => {
console.error("MP4 error", err)
}
this.#mp4.onReady = (info: MP4.Info) => {
this.info = info
// Extract all of the tracks, because we don't know if it's audio or video.
for (const track of info.tracks) {
this.#mp4.setExtractionOptions(track.id, track, { nbSamples: 1 })
}
}
this.#mp4.onSamples = (_track_id: number, track: MP4.Track, samples: MP4.Sample[]) => {
for (const sample of samples) {
this.#samples.push({ track, sample })
}
}
this.#mp4.start()
// For some reason we need to modify the underlying ArrayBuffer with offset
const copy = new Uint8Array(init)
const buffer = copy.buffer as MP4.ArrayBuffer
buffer.fileStart = this.#offset
this.#mp4.appendBuffer(buffer)
this.#offset += buffer.byteLength
this.#mp4.flush()
if (!this.info) {
throw new Error("could not parse MP4 info")
}
}
decode(chunk: Uint8Array): Array<Frame> {
const copy = new Uint8Array(chunk)
// For some reason we need to modify the underlying ArrayBuffer with offset
const buffer = copy.buffer as MP4.ArrayBuffer
buffer.fileStart = this.#offset
// Parse the data
this.#mp4.appendBuffer(buffer)
this.#mp4.flush()
this.#offset += buffer.byteLength
const samples = [...this.#samples]
this.#samples.length = 0
return samples
}
}

13
packages/moq/media/mp4/rename.ts
View File

@@ -1,13 +0,0 @@
// Rename some stuff so it's on brand.
export { createFile as New, DataStream as Stream, ISOFile, BoxParser, Log } from "mp4box"
export type {
MP4ArrayBuffer as ArrayBuffer,
MP4Info as Info,
MP4Track as Track,
MP4AudioTrack as AudioTrack,
MP4VideoTrack as VideoTrack,
Sample,
TrackOptions,
SampleOptions,
} from "mp4box"

15
packages/moq/package.json
View File

@@ -1,5 +1,5 @@
{
"name": "@nestri/moq",
"name": "@nestri/libmoq",
"type": "module",
"version": "0.1.4",
"description": "Media over QUIC library",
@@ -7,23 +7,14 @@
"repository": "github:kixelated/moq-js",
"scripts": {
"build": "tsc -b && cp ../LICENSE* ./dist && cp ./README.md ./dist && cp ./package.json ./dist",
"lint": "eslint .",
"fmt": "prettier --write ."
"check": "biome check",
"fix": "biome check --write"
},
"devDependencies": {
"@types/audioworklet": "^0.0.50",
"@types/dom-mediacapture-transform": "^0.1.6",
"@types/dom-webcodecs": "^0.1.8",
"@typescript/lib-dom": "npm:@types/web@^0.0.115",
"@typescript-eslint/eslint-plugin": "^6.4.0",
"@typescript-eslint/parser": "^6.4.0",
"eslint": "^8.47.0",
"eslint-config-prettier": "^9.0.0",
"eslint-plugin-prettier": "^5.0.0",
"prettier": "^3.0.1",
"typescript": "^5.1.6"
},
"dependencies": {
"mp4box": "^0.5.2"
}
}

111
packages/moq/playback/audio.ts
View File

@@ -1,34 +1,57 @@
/// <reference types="vite/client" />
import * as Message from "./worker/message"
import { Ring, RingShared } from "../common/ring"
import type * as Catalog from "../karp/catalog"
import type { Frame } from "../karp/frame"
import type { Component } from "./timeline"
// This is a non-standard way of importing worklet/workers.
// Unfortunately, it's the only option because of a Vite bug: https://github.com/vitejs/vite/issues/11823
import workletURL from "./worklet/index.ts?worker&url"
// NOTE: This must be on the main thread
export class Audio {
context: AudioContext
worklet: Promise<AudioWorkletNode>
export class Renderer {
#context: AudioContext
#worklet: Promise<AudioWorkletNode>
constructor(config: Message.ConfigAudio) {
this.context = new AudioContext({
#ring: Ring
#ringShared: RingShared
#timeline: Component
#track: Catalog.Audio
#decoder!: AudioDecoder
#stream: TransformStream<Frame, AudioData>
constructor(track: Catalog.Audio, timeline: Component) {
this.#track = track
this.#context = new AudioContext({
latencyHint: "interactive",
sampleRate: config.sampleRate,
sampleRate: track.sample_rate,
})
this.worklet = this.load(config)
this.#worklet = this.load(track)
this.#timeline = timeline
this.#ringShared = new RingShared(2, track.sample_rate / 10) // 100ms
this.#ring = new Ring(this.#ringShared)
this.#stream = new TransformStream({
start: this.#start.bind(this),
transform: this.#transform.bind(this),
})
this.#run().catch((err) => console.error("failed to run audio renderer: ", err))
}
private async load(config: Message.ConfigAudio): Promise<AudioWorkletNode> {
private async load(catalog: Catalog.Audio): Promise<AudioWorkletNode> {
// Load the worklet source code.
await this.context.audioWorklet.addModule(workletURL)
await this.#context.audioWorklet.addModule(workletURL)
const volume = this.context.createGain()
const volume = this.#context.createGain()
volume.gain.value = 2.0
// Create the worklet
const worklet = new AudioWorkletNode(this.context, "renderer")
const worklet = new AudioWorkletNode(this.#context, "renderer")
worklet.port.addEventListener("message", this.on.bind(this))
worklet.onprocessorerror = (e: Event) => {
@@ -37,7 +60,13 @@ export class Audio {
// Connect the worklet to the volume node and then to the speakers
worklet.connect(volume)
volume.connect(this.context.destination)
volume.connect(this.#context.destination)
const config = {
sampleRate: catalog.sample_rate,
channelCount: catalog.channel_count,
ring: this.#ringShared,
}
worklet.port.postMessage({ config })
@@ -47,4 +76,58 @@ export class Audio {
private on(_event: MessageEvent) {
// TODO
}
play() {
this.#context.resume().catch((err) => console.warn("failed to resume audio context: ", err))
}
close() {
this.#context.close().catch((err) => console.warn("failed to close audio context: ", err))
}
#start(controller: TransformStreamDefaultController) {
this.#decoder = new AudioDecoder({
output: (frame: AudioData) => {
controller.enqueue(frame)
},
error: console.warn,
})
// We only support OPUS right now which doesn't need a description.
this.#decoder.configure({
codec: this.#track.codec,
sampleRate: this.#track.sample_rate,
numberOfChannels: this.#track.channel_count,
})
}
#transform(frame: Frame) {
const chunk = new EncodedAudioChunk({
type: frame.type,
timestamp: frame.timestamp,
data: frame.data,
})
this.#decoder.decode(chunk)
}
async #run() {
const reader = this.#timeline.frames.pipeThrough(this.#stream).getReader()
for (;;) {
const { value: frame, done } = await reader.read()
if (done) break
// Write audio samples to the ring buffer, dropping when there's no space.
const written = this.#ring.write(frame)
if (written < frame.numberOfFrames) {
/*
console.warn(
`droppped ${frame.numberOfFrames - written} audio samples`,
);
*/
}
}
}
}

114
packages/moq/playback/backend.ts
View File

@@ -1,114 +0,0 @@
/// <reference types="vite/client" />
import * as Message from "./worker/message"
import { Audio } from "./audio"
import MediaWorker from "./worker?worker"
import { RingShared } from "../common/ring"
import { Root, isAudioTrack } from "../media/catalog"
import { GroupHeader } from "../transport/objects"
export interface PlayerConfig {
canvas: OffscreenCanvas
catalog: Root
}
// This is a non-standard way of importing worklet/workers.
// Unfortunately, it's the only option because of a Vite bug: https://github.com/vitejs/vite/issues/11823
// Responsible for sending messages to the worker and worklet.
export default class Backend {
// General worker
#worker: Worker
// The audio context, which must be created on the main thread.
#audio?: Audio
constructor(config: PlayerConfig) {
// TODO does this block the main thread? If so, make this async
// @ts-expect-error: The Vite typing is wrong https://github.com/vitejs/vite/blob/22bd67d70a1390daae19ca33d7de162140d533d6/packages/vite/client.d.ts#L182
this.#worker = new MediaWorker({ format: "es" })
this.#worker.addEventListener("message", this.on.bind(this))
let sampleRate: number | undefined
let channels: number | undefined
for (const track of config.catalog.tracks) {
if (isAudioTrack(track)) {
if (sampleRate && track.selectionParams.samplerate !== sampleRate) {
throw new Error(`TODO multiple audio tracks with different sample rates`)
}
sampleRate = track.selectionParams.samplerate
// TODO properly handle weird channel configs
channels = Math.max(+track.selectionParams.channelConfig, channels ?? 0)
}
}
const msg: Message.Config = {}
// Only configure audio is we have an audio track
if (sampleRate && channels) {
msg.audio = {
channels: channels,
sampleRate: sampleRate,
ring: new RingShared(2, sampleRate / 10), // 100ms
}
this.#audio = new Audio(msg.audio)
}
// TODO only send the canvas if we have a video track
msg.video = {
canvas: config.canvas,
}
this.send({ config: msg }, msg.video.canvas)
}
async play() {
await this.#audio?.context.resume()
}
init(init: Init) {
this.send({ init })
}
segment(segment: Segment) {
this.send({ segment }, segment.stream)
}
async close() {
this.#worker.terminate()
await this.#audio?.context.close()
}
// Enforce we're sending valid types to the worker
private send(msg: Message.ToWorker, ...transfer: Transferable[]) {
//console.log("sent message from main to worker", msg)
this.#worker.postMessage(msg, transfer)
}
private on(e: MessageEvent) {
const msg = e.data as Message.FromWorker
// Don't print the verbose timeline message.
if (!msg.timeline) {
//console.log("received message from worker to main", msg)
}
}
}
export interface Init {
name: string // name of the init track
data: Uint8Array
}
export interface Segment {
init: string // name of the init track
kind: "audio" | "video"
header: GroupHeader
buffer: Uint8Array
stream: ReadableStream<Uint8Array>
}

148
packages/moq/playback/broadcast.ts Normal file
View File

@@ -0,0 +1,148 @@
import type * as Catalog from "../karp/catalog"
import type { Connection } from "../transfork/connection"
import { Track } from "../transfork"
import { Frame } from "../karp/frame"
import type { GroupReader } from "../transfork/model"
import * as Audio from "./audio"
import { Timeline } from "./timeline"
import * as Video from "./video"
// This class must be created on the main thread due to AudioContext.
export class Broadcast {
#connection: Connection
#catalog: Catalog.Broadcast
// Running is a promise that resolves when the player is closed.
// #close is called with no error, while #abort is called with an error.
#running: Promise<void>
// Timeline receives samples, buffering them and choosing the timestamp to render.
#timeline = new Timeline()
#audio?: Audio.Renderer
#video?: Video.Renderer
constructor(connection: Connection, catalog: Catalog.Broadcast, canvas: HTMLCanvasElement) {
this.#connection = connection
this.#catalog = catalog
const running = []
// Only configure audio is we have an audio track
const audio = (catalog.audio || []).at(0)
if (audio) {
this.#audio = new Audio.Renderer(audio, this.#timeline.audio)
running.push(this.#runAudio(audio))
}
const video = (catalog.video || []).at(0)
if (video) {
this.#video = new Video.Renderer(video, canvas, this.#timeline.video)
running.push(this.#runVideo(video))
}
// Async work
this.#running = Promise.race([...running])
}
async #runAudio(audio: Catalog.Audio) {
const track = new Track(this.#catalog.path.concat(audio.track.name), audio.track.priority)
const sub = await this.#connection.subscribe(track)
try {
for (;;) {
const group = await Promise.race([sub.nextGroup(), this.#running])
if (!group) break
this.#runAudioGroup(audio, group)
.catch(() => {})
.finally(() => group.close())
}
} finally {
sub.close()
}
}
async #runVideo(video: Catalog.Video) {
const track = new Track(this.#catalog.path.concat(video.track.name), video.track.priority)
const sub = await this.#connection.subscribe(track)
try {
for (;;) {
const group = await Promise.race([sub.nextGroup(), this.#running])
if (!group) break
this.#runVideoGroup(video, group)
.catch(() => {})
.finally(() => group.close())
}
} finally {
sub.close()
}
}
async #runAudioGroup(audio: Catalog.Audio, group: GroupReader) {
const timeline = this.#timeline.audio
// Create a queue that will contain each frame
const queue = new TransformStream<Frame>({})
const segment = queue.writable.getWriter()
// Add the segment to the timeline
const segments = timeline.segments.getWriter()
await segments.write({
sequence: group.id,
frames: queue.readable,
})
segments.releaseLock()
// Read each chunk, decoding the MP4 frames and adding them to the queue.
for (;;) {
const frame = await Frame.decode(group)
if (!frame) break
await segment.write(frame)
}
// We done.
await segment.close()
}
async #runVideoGroup(video: Catalog.Video, group: GroupReader) {
const timeline = this.#timeline.video
// Create a queue that will contain each MP4 frame.
const queue = new TransformStream<Frame>({})
const segment = queue.writable.getWriter()
// Add the segment to the timeline
const segments = timeline.segments.getWriter()
await segments.write({
sequence: group.id,
frames: queue.readable,
})
segments.releaseLock()
for (;;) {
const frame = await Frame.decode(group)
if (!frame) break
await segment.write(frame)
}
// We done.
await segment.close()
}
unmute() {
console.debug("unmuting audio")
this.#audio?.play()
}
close() {
this.#audio?.close()
this.#video?.close()
}
}

192
packages/moq/playback/index.ts
View File

@@ -1,190 +1,2 @@
import * as Message from "./worker/message"
import { Connection } from "../transport/connection"
import * as Catalog from "../media/catalog"
import { asError } from "../common/error"
import Backend from "./backend"
import { Client } from "../transport/client"
import { GroupReader } from "../transport/objects"
export type Range = Message.Range
export type Timeline = Message.Timeline
export interface PlayerConfig {
url: string
namespace: string
fingerprint?: string // URL to fetch TLS certificate fingerprint
canvas: HTMLCanvasElement
}
// This class must be created on the main thread due to AudioContext.
export class Player {
#backend: Backend
// A periodically updated timeline
//#timeline = new Watch<Timeline | undefined>(undefined)
#connection: Connection
#catalog: Catalog.Root
// Running is a promise that resolves when the player is closed.
// #close is called with no error, while #abort is called with an error.
#running: Promise<void>
#close!: () => void
#abort!: (err: Error) => void
private constructor(connection: Connection, catalog: Catalog.Root, backend: Backend) {
this.#connection = connection
this.#catalog = catalog
this.#backend = backend
const abort = new Promise<void>((resolve, reject) => {
this.#close = resolve
this.#abort = reject
})
// Async work
this.#running = Promise.race([this.#run(), abort]).catch(this.#close)
}
static async create(config: PlayerConfig): Promise<Player> {
const client = new Client({ url: config.url, fingerprint: config.fingerprint, role: "subscriber" })
const connection = await client.connect()
const catalog = await Catalog.fetch(connection, config.namespace)
console.log("catalog", catalog)
const canvas = config.canvas.transferControlToOffscreen()
const backend = new Backend({ canvas, catalog })
return new Player(connection, catalog, backend)
}
async #run() {
const inits = new Set<[string, string]>()
const tracks = new Array<Catalog.Track>()
for (const track of this.#catalog.tracks) {
if (!track.namespace) throw new Error("track has no namespace")
if (track.initTrack) inits.add([track.namespace, track.initTrack])
tracks.push(track)
}
// Call #runInit on each unique init track
// TODO do this in parallel with #runTrack to remove a round trip
await Promise.all(Array.from(inits).map((init) => this.#runInit(...init)))
// Call #runTrack on each track
await Promise.all(tracks.map((track) => this.#runTrack(track)))
}
async #runInit(namespace: string, name: string) {
const sub = await this.#connection.subscribe(namespace, name)
try {
const init = await Promise.race([sub.data(), this.#running])
if (!init) throw new Error("no init data")
// We don't care what type of reader we get, we just want the payload.
const chunk = await init.read()
if (!chunk) throw new Error("no init chunk")
if (!(chunk.payload instanceof Uint8Array)) throw new Error("invalid init chunk")
this.#backend.init({ data: chunk.payload, name })
} finally {
await sub.close()
}
}
async #runTrack(track: Catalog.Track) {
if (!track.namespace) throw new Error("track has no namespace")
const sub = await this.#connection.subscribe(track.namespace, track.name)
try {
for (;;) {
const segment = await Promise.race([sub.data(), this.#running])
if (!segment) break
if (!(segment instanceof GroupReader)) {
throw new Error(`expected group reader for segment: ${track.name}`)
}
const kind = Catalog.isVideoTrack(track) ? "video" : Catalog.isAudioTrack(track) ? "audio" : "unknown"
if (kind == "unknown") {
throw new Error(`unknown track kind: ${track.name}`)
}
if (!track.initTrack) {
throw new Error(`no init track for segment: ${track.name}`)
}
const [buffer, stream] = segment.stream.release()
this.#backend.segment({
init: track.initTrack,
kind,
header: segment.header,
buffer,
stream,
})
}
} catch (error) {
console.error("Error in #runTrack:", error)
} finally {
await sub.close()
}
}
getCatalog() {
return this.#catalog
}
#onMessage(msg: Message.FromWorker) {
if (msg.timeline) {
//this.#timeline.update(msg.timeline)
}
}
async close(err?: Error) {
if (err) this.#abort(err)
else this.#close()
if (this.#connection) this.#connection.close()
if (this.#backend) await this.#backend.close()
}
async closed(): Promise<Error | undefined> {
try {
await this.#running
} catch (e) {
return asError(e)
}
}
/*
play() {
this.#backend.play({ minBuffer: 0.5 }) // TODO configurable
}
seek(timestamp: number) {
this.#backend.seek({ timestamp })
}
*/
async play() {
await this.#backend.play()
}
/*
async *timeline() {
for (;;) {
const [timeline, next] = this.#timeline.value()
if (timeline) yield timeline
if (!next) break
await next
}
}
*/
}
export { Player } from "./player"
export type { PlayerConfig } from "./player"

63
packages/moq/playback/player.ts Normal file
View File

@@ -0,0 +1,63 @@
import * as Catalog from "../karp/catalog"
import type { Connection } from "../transfork/connection"
import { Broadcast } from "./broadcast"
export interface PlayerConfig {
connection: Connection
path: string[]
canvas: HTMLCanvasElement
}
// This class must be created on the main thread due to AudioContext.
export class Player {
#config: PlayerConfig
#running: Promise<void>
#active?: Broadcast
constructor(config: PlayerConfig) {
this.#config = config
this.#running = this.#run()
}
async #run() {
const announced = await this.#config.connection.announced(this.#config.path)
let activeId = -1
for (;;) {
const announce = await announced.next()
if (!announce) break
if (announce.path.length === this.#config.path.length) {
throw new Error("expected resumable broadcast")
}
const path = announce.path.slice(0, this.#config.path.length + 1)
const id = Number.parseInt(path[path.length - 1])
if (id <= activeId) continue
const catalog = await Catalog.fetch(this.#config.connection, path)
this.#active?.close()
this.#active = new Broadcast(this.#config.connection, catalog, this.#config.canvas)
activeId = id
}
this.#active?.close()
}
close() {
this.#config.connection.close()
this.#active?.close()
this.#active = undefined
}
async closed() {
await Promise.any([this.#running, this.#config.connection.closed()])
}
unmute() {
this.#active?.unmute()
}
}

15
packages/moq/playback/worker/timeline.ts → packages/moq/playback/timeline.ts
View File

@@ -1,5 +1,4 @@
import type { Frame } from "../../media/mp4"
export type { Frame }
import type { Frame } from "../karp/frame"
export interface Range {
start: number
@@ -48,7 +47,7 @@ export class Component {
// Get the next segment to render.
const segments = this.#segments.readable.getReader()
let res
let res: ReadableStreamReadResult<Segment> | ReadableStreamReadResult<Frame>
if (this.#current) {
// Get the next frame to render.
const frames = this.#current.frames.getReader()
@@ -85,17 +84,17 @@ export class Component {
// Our segment is older than the current, abandon it.
await value.frames.cancel("skipping segment; too old")
continue
} else {
// Our segment is newer than the current, cancel the old one.
await this.#current.frames.cancel("skipping segment; too slow")
}
// Our segment is newer than the current, cancel the old one.
await this.#current.frames.cancel("skipping segment; too slow")
}
this.#current = value
}
}
async #cancel(reason: any) {
async #cancel(reason: Error) {
if (this.#current) {
await this.#current.frames.cancel(reason)
}
@@ -111,8 +110,6 @@ export class Component {
}
// Return if a type is a segment or frame
// eslint-disable-next-line @typescript-eslint/no-redundant-type-constituents
function isSegment(value: Segment | Frame): value is Segment {
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
return (value as Segment).frames !== undefined
}

4
packages/moq/playback/tsconfig.json
View File

@@ -10,10 +10,10 @@
"path": "../common"
},
{
"path": "../transport"
"path": "../transfork"
},
{
"path": "../media"
"path": "../karp"
}
],
"paths": {

57
packages/moq/playback/worker/video.ts → packages/moq/playback/video.ts
View File

@@ -1,16 +1,18 @@
import { Frame, Component } from "./timeline"
import * as MP4 from "../../media/mp4"
import * as Message from "./message"
import type * as Catalog from "../karp/catalog"
import type { Frame } from "../karp/frame"
import type { Component } from "./timeline"
export class Renderer {
#canvas: OffscreenCanvas
#track: Catalog.Video
#canvas: HTMLCanvasElement
#timeline: Component
#decoder!: VideoDecoder
#queue: TransformStream<Frame, VideoFrame>
constructor(config: Message.ConfigVideo, timeline: Component) {
this.#canvas = config.canvas
constructor(track: Catalog.Video, canvas: HTMLCanvasElement, timeline: Component) {
this.#track = track
this.#canvas = canvas
this.#timeline = timeline
this.#queue = new TransformStream({
@@ -18,7 +20,11 @@ export class Renderer {
transform: this.#transform.bind(this),
})
this.#run().catch(console.error)
this.#run().catch((err) => console.error("failed to run video renderer: ", err))
}
close() {
// TODO
}
async #run() {
@@ -47,36 +53,21 @@ export class Renderer {
},
error: console.error,
})
this.#decoder.configure({
codec: this.#track.codec,
codedHeight: this.#track.resolution.height,
codedWidth: this.#track.resolution.width,
description: this.#track.description,
optimizeForLatency: true,
})
}
#transform(frame: Frame) {
// Configure the decoder with the first frame
if (this.#decoder.state !== "configured") {
const { sample, track } = frame
const desc = sample.description
const box = desc.avcC ?? desc.hvcC ?? desc.vpcC ?? desc.av1C
if (!box) throw new Error(`unsupported codec: ${track.codec}`)
const buffer = new MP4.Stream(undefined, 0, MP4.Stream.BIG_ENDIAN)
box.write(buffer)
const description = new Uint8Array(buffer.buffer, 8) // Remove the box header.
if (!MP4.isVideoTrack(track)) throw new Error("expected video track")
this.#decoder.configure({
codec: track.codec,
codedHeight: track.video.height,
codedWidth: track.video.width,
description,
// optimizeForLatency: true
})
}
const chunk = new EncodedVideoChunk({
type: frame.sample.is_sync ? "key" : "delta",
data: frame.sample.data,
timestamp: frame.sample.dts / frame.track.timescale,
type: frame.type,
data: frame.data,
timestamp: frame.timestamp,
})
this.#decoder.decode(chunk)

73
packages/moq/playback/worker/audio.ts
View File

@@ -1,73 +0,0 @@
import * as Message from "./message"
import { Ring } from "../../common/ring"
import { Component, Frame } from "./timeline"
import * as MP4 from "../../media/mp4"
// This is run in a worker.
export class Renderer {
#ring: Ring
#timeline: Component
#decoder!: AudioDecoder
#stream: TransformStream<Frame, AudioData>
constructor(config: Message.ConfigAudio, timeline: Component) {
this.#timeline = timeline
this.#ring = new Ring(config.ring)
this.#stream = new TransformStream({
start: this.#start.bind(this),
transform: this.#transform.bind(this),
})
this.#run().catch(console.error)
}
#start(controller: TransformStreamDefaultController) {
this.#decoder = new AudioDecoder({
output: (frame: AudioData) => {
controller.enqueue(frame)
},
error: console.warn,
})
}
#transform(frame: Frame) {
if (this.#decoder.state !== "configured") {
const track = frame.track
if (!MP4.isAudioTrack(track)) throw new Error("expected audio track")
// We only support OPUS right now which doesn't need a description.
this.#decoder.configure({
codec: track.codec,
sampleRate: track.audio.sample_rate,
numberOfChannels: track.audio.channel_count,
})
}
const chunk = new EncodedAudioChunk({
type: frame.sample.is_sync ? "key" : "delta",
timestamp: frame.sample.dts / frame.track.timescale,
duration: frame.sample.duration,
data: frame.sample.data,
})
this.#decoder.decode(chunk)
}
async #run() {
const reader = this.#timeline.frames.pipeThrough(this.#stream).getReader()
for (;;) {
const { value: frame, done } = await reader.read()
if (done) break
// Write audio samples to the ring buffer, dropping when there's no space.
const written = this.#ring.write(frame)
if (written < frame.numberOfFrames) {
console.warn(`droppped ${frame.numberOfFrames - written} audio samples`)
}
}
}
}

119
packages/moq/playback/worker/index.ts
View File

@@ -1,119 +0,0 @@
import { Timeline } from "./timeline"
import * as Audio from "./audio"
import * as Video from "./video"
import * as MP4 from "../../media/mp4"
import * as Message from "./message"
import { asError } from "../../common/error"
import { Deferred } from "../../common/async"
import { GroupReader, Reader } from "../../transport/objects"
class Worker {
// Timeline receives samples, buffering them and choosing the timestamp to render.
#timeline = new Timeline()
// A map of init tracks.
#inits = new Map<string, Deferred<Uint8Array>>()
// Renderer requests samples, rendering video frames and emitting audio frames.
#audio?: Audio.Renderer
#video?: Video.Renderer
on(e: MessageEvent) {
const msg = e.data as Message.ToWorker
if (msg.config) {
this.#onConfig(msg.config)
} else if (msg.init) {
// TODO buffer the init segmnet so we don't hold the stream open.
this.#onInit(msg.init)
} else if (msg.segment) {
this.#onSegment(msg.segment).catch(console.warn)
} else {
throw new Error(`unknown message: + ${JSON.stringify(msg)}`)
}
}
#onConfig(msg: Message.Config) {
if (msg.audio) {
this.#audio = new Audio.Renderer(msg.audio, this.#timeline.audio)
}
if (msg.video) {
this.#video = new Video.Renderer(msg.video, this.#timeline.video)
}
}
#onInit(msg: Message.Init) {
let init = this.#inits.get(msg.name)
if (!init) {
init = new Deferred()
this.#inits.set(msg.name, init)
}
init.resolve(msg.data)
}
async #onSegment(msg: Message.Segment) {
let init = this.#inits.get(msg.init)
if (!init) {
init = new Deferred()
this.#inits.set(msg.init, init)
}
// Create a new stream that we will use to decode.
const container = new MP4.Parser(await init.promise)
const timeline = msg.kind === "audio" ? this.#timeline.audio : this.#timeline.video
const reader = new GroupReader(msg.header, new Reader(msg.buffer, msg.stream))
// Create a queue that will contain each MP4 frame.
const queue = new TransformStream<MP4.Frame>({})
const segment = queue.writable.getWriter()
// Add the segment to the timeline
const segments = timeline.segments.getWriter()
await segments.write({
sequence: msg.header.group,
frames: queue.readable,
})
segments.releaseLock()
// Read each chunk, decoding the MP4 frames and adding them to the queue.
for (;;) {
const chunk = await reader.read()
if (!chunk) {
break
}
if (!(chunk.payload instanceof Uint8Array)) {
throw new Error(`invalid payload: ${chunk.payload}`)
}
const frames = container.decode(chunk.payload)
for (const frame of frames) {
await segment.write(frame)
}
}
// We done.
await segment.close()
}
}
// Pass all events to the worker
const worker = new Worker()
self.addEventListener("message", (msg) => {
try {
worker.on(msg)
} catch (e) {
const err = asError(e)
console.warn("worker error:", err)
}
})
// Validates this is an expected message
function _send(msg: Message.FromWorker) {
postMessage(msg)
}

98
packages/moq/playback/worker/message.ts
View File

@@ -1,98 +0,0 @@
import { GroupHeader } from "../../transport/objects"
import { RingShared } from "../../common/ring"
export interface Config {
audio?: ConfigAudio
video?: ConfigVideo
}
export interface ConfigAudio {
channels: number
sampleRate: number
ring: RingShared
}
export interface ConfigVideo {
canvas: OffscreenCanvas
}
export interface Init {
name: string // name of the init object
data: Uint8Array
}
export interface Segment {
init: string // name of the init object
kind: "audio" | "video"
header: GroupHeader
buffer: Uint8Array
stream: ReadableStream<Uint8Array>
}
/*
export interface Play {
// Start playback once the minimum buffer size has been reached.
minBuffer: number
}
export interface Seek {
timestamp: number
}
*/
// Sent periodically with the current timeline info.
export interface Timeline {
// The current playback position
timestamp?: number
// Audio specific information
audio: TimelineAudio
// Video specific information
video: TimelineVideo
}
export interface TimelineAudio {
buffer: Range[]
}
export interface TimelineVideo {
buffer: Range[]
}
export interface Range {
start: number
end: number
}
// Used to validate that only the correct messages can be sent.
// Any top level messages that can be sent to the worker.
export interface ToWorker {
// Sent to configure on startup.
config?: Config
// Sent on each init/data stream
init?: Init
segment?: Segment
/*
// Sent to control playback
play?: Play
seek?: Seek
*/
}
// Any top-level messages that can be sent from the worker.
export interface FromWorker {
// Sent back to the main thread regularly to update the UI
timeline?: Timeline
}
/*
interface ToWorklet {
config?: Audio.Config
}
*/

8
packages/moq/playback/worklet/index.ts
View File

@@ -1,6 +1,6 @@
// TODO add support for @/ to avoid relative imports
import { Ring } from "../../common/ring"
import * as Message from "./message"
import type * as Message from "./message"
class Renderer extends AudioWorkletProcessor {
ring?: Ring
@@ -26,17 +26,17 @@ class Renderer extends AudioWorkletProcessor {
}
// Inputs and outputs in groups of 128 samples.
process(inputs: Float32Array[][], outputs: Float32Array[][], _parameters: Record<string, Float32Array>): boolean {
process(_inputs: Float32Array[][], outputs: Float32Array[][], _parameters: Record<string, Float32Array>): boolean {
if (!this.ring) {
// Paused
return true
}
if (inputs.length != 1 && outputs.length != 1) {
if (outputs.length !== 1) {
throw new Error("only a single track is supported")
}
if (this.ring.size() == this.ring.capacity) {
if (this.ring.size() === this.ring.capacity) {
// This is a hack to clear any latency in the ring buffer.
// The proper solution is to play back slightly faster?
console.warn("resyncing ring buffer")

3
packages/moq/playback/worklet/message.ts
View File

@@ -1,4 +1,4 @@
import { RingShared } from "../../common/ring"
import type { RingShared } from "../../common/ring"
export interface From {
config?: Config
@@ -7,6 +7,5 @@ export interface From {
export interface Config {
channels: number
sampleRate: number
ring: RingShared
}

42
packages/moq/transport/client.ts → packages/moq/transfork/client.ts
View File

@@ -1,15 +1,11 @@
import * as Stream from "./stream"
import * as Setup from "./setup"
import * as Control from "./control"
import { Objects } from "./objects"
import * as Hex from "../common/hex"
import { Connection } from "./connection"
import * as Message from "./message"
import { Stream } from "./stream"
export interface ClientConfig {
url: string
// Parameters used to create the MoQ session
role: Setup.Role
// If set, the server fingerprint will be fetched from this URL.
// This is required to use self-signed certificates with Chrome (May 2023)
fingerprint?: string
@@ -39,28 +35,17 @@ export class Client {
const quic = new WebTransport(this.config.url, options)
await quic.ready
const stream = await quic.createBidirectionalStream()
const client = new Message.SessionClient([Message.Version.FORK_02])
const stream = await Stream.open(quic, client)
const writer = new Stream.Writer(stream.writable)
const reader = new Stream.Reader(new Uint8Array(), stream.readable)
const setup = new Setup.Stream(reader, writer)
// Send the setup message.
await setup.send.client({ versions: [Setup.Version.DRAFT_04], role: this.config.role })
// Receive the setup message.
// TODO verify the SETUP response.
const server = await setup.recv.server()
if (server.version != Setup.Version.DRAFT_04) {
const server = await Message.SessionServer.decode(stream.reader)
if (server.version !== Message.Version.FORK_02) {
throw new Error(`unsupported server version: ${server.version}`)
}
const control = new Control.Stream(reader, writer)
const objects = new Objects(quic)
console.log(`established connection: version=${server.version}`)
return new Connection(quic, control, objects)
return new Connection(quic, stream)
}
async #fetchFingerprint(url?: string): Promise<WebTransportHash | undefined> {
@@ -68,16 +53,11 @@ export class Client {
// TODO remove this fingerprint when Chrome WebTransport accepts the system CA
const response = await fetch(url)
const hexString = await response.text()
const hexBytes = new Uint8Array(hexString.length / 2)
for (let i = 0; i < hexBytes.length; i += 1) {
hexBytes[i] = parseInt(hexString.slice(2 * i, 2 * i + 2), 16)
}
const bytes = Hex.decode(await response.text())
return {
algorithm: "sha-256",
value: hexBytes,
value: bytes,
}
}
}

158
packages/moq/transfork/connection.ts Normal file
View File

@@ -0,0 +1,158 @@
import { asError } from "../common/error"
import * as Message from "./message"
import { Reader, Stream } from "./stream"
import type { Queue } from "../common/async"
import { Closed } from "./error"
import type { Track, TrackReader } from "./model"
import { Publisher } from "./publisher"
import { type Announced, Subscriber } from "./subscriber"
export class Connection {
// The established WebTransport session.
#quic: WebTransport
// Use to receive/send session messages.
#session: Stream
// Module for contributing tracks.
#publisher: Publisher
// Module for distributing tracks.
#subscriber: Subscriber
// Async work running in the background
#running: Promise<void>
constructor(quic: WebTransport, session: Stream) {
this.#quic = quic
this.#session = session
this.#publisher = new Publisher(this.#quic)
this.#subscriber = new Subscriber(this.#quic)
this.#running = this.#run()
}
close(code = 0, reason = "") {
this.#quic.close({ closeCode: code, reason })
}
async #run(): Promise<void> {
const session = this.#runSession().catch((err) => new Error("failed to run session: ", err))
const bidis = this.#runBidis().catch((err) => new Error("failed to run bidis: ", err))
const unis = this.#runUnis().catch((err) => new Error("failed to run unis: ", err))
await Promise.all([session, bidis, unis])
}
publish(track: TrackReader) {
this.#publisher.publish(track)
}
async announced(prefix: string[] = []): Promise<Queue<Announced>> {
return this.#subscriber.announced(prefix)
}
async subscribe(track: Track): Promise<TrackReader> {
return await this.#subscriber.subscribe(track)
}
async #runSession() {
// Receive messages until the connection is closed.
for (;;) {
const msg = await Message.SessionInfo.decode_maybe(this.#session.reader)
if (!msg) break
// TODO use the session info
}
}
async #runBidis() {
for (;;) {
const next = await Stream.accept(this.#quic)
if (!next) {
break
}
const [msg, stream] = next
this.#runBidi(msg, stream).catch((err) => stream.writer.reset(Closed.extract(err)))
}
}
async #runBidi(msg: Message.Bi, stream: Stream) {
console.debug("received bi stream: ", msg)
if (msg instanceof Message.SessionClient) {
throw new Error("duplicate session stream")
}
if (msg instanceof Message.AnnounceInterest) {
if (!this.#subscriber) {
throw new Error("not a subscriber")
}
return await this.#publisher.runAnnounce(msg, stream)
}
if (msg instanceof Message.Subscribe) {
if (!this.#publisher) {
throw new Error("not a publisher")
}
return await this.#publisher.runSubscribe(msg, stream)
}
if (msg instanceof Message.Datagrams) {
if (!this.#publisher) {
throw new Error("not a publisher")
}
return await this.#publisher.runDatagrams(msg, stream)
}
if (msg instanceof Message.Fetch) {
if (!this.#publisher) {
throw new Error("not a publisher")
}
return await this.#publisher.runFetch(msg, stream)
}
if (msg instanceof Message.InfoRequest) {
if (!this.#publisher) {
throw new Error("not a publisher")
}
return await this.#publisher.runInfo(msg, stream)
}
}
async #runUnis() {
for (;;) {
const next = await Reader.accept(this.#quic)
if (!next) {
break
}
const [msg, stream] = next
this.#runUni(msg, stream).catch((err) => stream.stop(Closed.extract(err)))
}
}
async #runUni(msg: Message.Uni, stream: Reader) {
console.debug("received uni stream: ", msg)
if (msg instanceof Message.Group) {
if (!this.#subscriber) {
throw new Error("not a subscriber")
}
return this.#subscriber.runGroup(msg, stream)
}
}
async closed(): Promise<Error> {
try {
await this.#running
return new Error("closed")
} catch (e) {
return asError(e)
}
}
}

20
packages/moq/transfork/error.ts Normal file
View File

@@ -0,0 +1,20 @@
export class Closed extends Error {
readonly code?: number
constructor(code?: number) {
super(`closed code=${code}`)
this.code = code
}
static from(err: unknown): Closed {
return new Closed(Closed.extract(err))
}
static extract(err: unknown): number {
if (err instanceof WebTransportError && err.streamErrorCode !== null) {
return err.streamErrorCode
}
return 0
}
}

45
packages/moq/transfork/frame.ts Normal file
View File

@@ -0,0 +1,45 @@
import type { Reader, Writer } from "./stream"
export class FrameReader {
#stream: Reader
constructor(stream: Reader) {
this.#stream = stream
}
// Returns the next frame
async read(): Promise<Uint8Array | undefined> {
if (await this.#stream.done()) return
const size = await this.#stream.u53()
const payload = await this.#stream.read(size)
return payload
}
async stop(code: number) {
await this.#stream.stop(code)
}
}
export class FrameWriter {
#stream: Writer
constructor(stream: Writer) {
this.#stream = stream
}
// Writes the next frame
async write(payload: Uint8Array) {
await this.#stream.u53(payload.byteLength)
await this.#stream.write(payload)
}
async close() {
await this.#stream.close()
}
async reset(code: number) {
await this.#stream.reset(code)
}
}

4
packages/moq/transport/index.ts → packages/moq/transfork/index.ts
View File

@@ -3,5 +3,5 @@ export type { ClientConfig } from "./client"
export { Connection } from "./connection"
export { SubscribeRecv, AnnounceSend } from "./publisher"
export { AnnounceRecv, SubscribeSend } from "./subscriber"
export { Track, Group } from "./model"
export { Announced } from "./subscriber"

428
packages/moq/transfork/message.ts Normal file
View File

@@ -0,0 +1,428 @@
import type { Reader, Writer } from "./stream"
export enum Version {
DRAFT_00 = 0xff000000,
DRAFT_01 = 0xff000001,
DRAFT_02 = 0xff000002,
DRAFT_03 = 0xff000003,
FORK_00 = 0xff0bad00,
FORK_01 = 0xff0bad01,
FORK_02 = 0xff0bad02,
}
export class Extensions {
entries: Map<bigint, Uint8Array>
constructor() {
this.entries = new Map()
}
set(id: bigint, value: Uint8Array) {
this.entries.set(id, value)
}
get(id: bigint): Uint8Array | undefined {
return this.entries.get(id)
}
remove(id: bigint): Uint8Array | undefined {
const value = this.entries.get(id)
this.entries.delete(id)
return value
}
async encode(w: Writer) {
await w.u53(this.entries.size)
for (const [id, value] of this.entries) {
await w.u62(id)
await w.u53(value.length)
await w.write(value)
}
}
static async decode(r: Reader): Promise<Extensions> {
const count = await r.u53()
const params = new Extensions()
for (let i = 0; i < count; i++) {
const id = await r.u62()
const size = await r.u53()
const value = await r.read(size)
if (params.entries.has(id)) {
throw new Error(`duplicate parameter id: ${id}`)
}
params.entries.set(id, value)
}
return params
}
}
export enum Order {
Any = 0,
Ascending = 1,
Descending = 2,
}
export class SessionClient {
versions: Version[]
extensions: Extensions
static StreamID = 0x0
constructor(versions: Version[], extensions = new Extensions()) {
this.versions = versions
this.extensions = extensions
}
async encode(w: Writer) {
await w.u53(this.versions.length)
for (const v of this.versions) {
await w.u53(v)
}
await this.extensions.encode(w)
}
static async decode(r: Reader): Promise<SessionClient> {
const versions = []
const count = await r.u53()
for (let i = 0; i < count; i++) {
versions.push(await r.u53())
}
const extensions = await Extensions.decode(r)
return new SessionClient(versions, extensions)
}
}
export class SessionServer {
version: Version
extensions: Extensions
constructor(version: Version, extensions = new Extensions()) {
this.version = version
this.extensions = extensions
}
async encode(w: Writer) {
await w.u53(this.version)
await this.extensions.encode(w)
}
static async decode(r: Reader): Promise<SessionServer> {
const version = await r.u53()
const extensions = await Extensions.decode(r)
return new SessionServer(version, extensions)
}
}
export class SessionInfo {
bitrate: number
constructor(bitrate: number) {
this.bitrate = bitrate
}
async encode(w: Writer) {
await w.u53(this.bitrate)
}
static async decode(r: Reader): Promise<SessionInfo> {
const bitrate = await r.u53()
return new SessionInfo(bitrate)
}
static async decode_maybe(r: Reader): Promise<SessionInfo | undefined> {
if (await r.done()) return
return await SessionInfo.decode(r)
}
}
export type AnnounceStatus = "active" | "closed"
export class Announce {
suffix: string[]
status: AnnounceStatus
constructor(suffix: string[], status: AnnounceStatus) {
this.suffix = suffix
this.status = status
}
async encode(w: Writer) {
await w.u53(this.status === "active" ? 1 : 0)
await w.path(this.suffix)
}
static async decode(r: Reader): Promise<Announce> {
const status = (await r.u53()) === 1 ? "active" : "closed"
const suffix = await r.path()
return new Announce(suffix, status)
}
static async decode_maybe(r: Reader): Promise<Announce | undefined> {
if (await r.done()) return
return await Announce.decode(r)
}
}
export class AnnounceInterest {
static StreamID = 0x1
constructor(public prefix: string[]) {}
async encode(w: Writer) {
await w.path(this.prefix)
}
static async decode(r: Reader): Promise<AnnounceInterest> {
const prefix = await r.path()
return new AnnounceInterest(prefix)
}
}
export class SubscribeUpdate {
priority: number
order = Order.Any
expires = 0 // ms
start?: bigint
end?: bigint
constructor(priority: number) {
this.priority = priority
}
async encode(w: Writer) {
await w.u53(this.priority)
await w.u53(this.order)
await w.u53(this.expires)
await w.u62(this.start ? this.start + 1n : 0n)
await w.u62(this.end ? this.end + 1n : 0n)
}
static async decode(r: Reader): Promise<SubscribeUpdate> {
const priority = await r.u53()
const order = await r.u53()
if (order > 2) {
throw new Error(`invalid order: ${order}`)
}
const expires = await r.u53()
const start = await r.u62()
const end = await r.u62()
const update = new SubscribeUpdate(priority)
update.order = order
update.expires = expires
update.start = start === 0n ? undefined : start - 1n
update.end = end === 0n ? undefined : end - 1n
return update
}
static async decode_maybe(r: Reader): Promise<SubscribeUpdate | undefined> {
if (await r.done()) return
return await SubscribeUpdate.decode(r)
}
}
export class Subscribe extends SubscribeUpdate {
id: bigint
path: string[]
static StreamID = 0x2
constructor(id: bigint, path: string[], priority: number) {
super(priority)
this.id = id
this.path = path
}
async encode(w: Writer) {
await w.u62(this.id)
await w.path(this.path)
await super.encode(w)
}
static async decode(r: Reader): Promise<Subscribe> {
const id = await r.u62()
const path = await r.path()
const update = await SubscribeUpdate.decode(r)
const subscribe = new Subscribe(id, path, update.priority)
subscribe.order = update.order
subscribe.expires = update.expires
subscribe.start = update.start
subscribe.end = update.end
return subscribe
}
}
export class Datagrams extends Subscribe {
static StreamID = 0x3
}
export class Info {
priority: number
order = Order.Descending
expires = 0
latest?: number
constructor(priority: number) {
this.priority = priority
}
async encode(w: Writer) {
await w.u53(this.priority)
await w.u53(this.order)
await w.u53(this.expires)
await w.u53(this.latest ? this.latest + 1 : 0)
}
static async decode(r: Reader): Promise<Info> {
const priority = await r.u53()
const order = await r.u53()
const latest = await r.u53()
const info = new Info(priority)
info.latest = latest === 0 ? undefined : latest - 1
info.order = order
return info
}
}
export class InfoRequest {
path: string[]
static StreamID = 0x5
constructor(path: string[]) {
this.path = path
}
async encode(w: Writer) {
await w.path(this.path)
}
static async decode(r: Reader): Promise<InfoRequest> {
const path = await r.path()
return new InfoRequest(path)
}
}
export class FetchUpdate {
priority: number
constructor(priority: number) {
this.priority = priority
}
async encode(w: Writer) {
await w.u53(this.priority)
}
static async decode(r: Reader): Promise<FetchUpdate> {
return new FetchUpdate(await r.u53())
}
static async decode_maybe(r: Reader): Promise<FetchUpdate | undefined> {
if (await r.done()) return
return await FetchUpdate.decode(r)
}
}
export class Fetch extends FetchUpdate {
path: string[]
static StreamID = 0x4
constructor(path: string[], priority: number) {
super(priority)
this.path = path
}
async encode(w: Writer) {
await w.path(this.path)
await super.encode(w)
}
static async decode(r: Reader): Promise<Fetch> {
const path = await r.path()
const update = await FetchUpdate.decode(r)
const fetch = new Fetch(path, update.priority)
return fetch
}
}
export class Group {
subscribe: bigint
sequence: number
static StreamID = 0x0
constructor(subscribe: bigint, sequence: number) {
this.subscribe = subscribe
this.sequence = sequence
}
async encode(w: Writer) {
await w.u62(this.subscribe)
await w.u53(this.sequence)
}
static async decode(r: Reader): Promise<Group> {
return new Group(await r.u62(), await r.u53())
}
}
export class GroupDrop {
sequence: number
count: number
error: number
constructor(sequence: number, count: number, error: number) {
this.sequence = sequence
this.count = count
this.error = error
}
async encode(w: Writer) {
await w.u53(this.sequence)
await w.u53(this.count)
await w.u53(this.error)
}
static async decode(r: Reader): Promise<GroupDrop> {
return new GroupDrop(await r.u53(), await r.u53(), await r.u53())
}
}
export class Frame {
payload: Uint8Array
constructor(payload: Uint8Array) {
this.payload = payload
}
async encode(w: Writer) {
await w.u53(this.payload.byteLength)
await w.write(this.payload)
}
static async decode(r: Reader): Promise<Frame> {
const size = await r.u53()
const payload = await r.read(size)
return new Frame(payload)
}
}
export type Bi = SessionClient | AnnounceInterest | Subscribe | Datagrams | Fetch | InfoRequest
export type Uni = Group

170
packages/moq/transfork/model.ts Normal file
View File

@@ -0,0 +1,170 @@
import { Watch } from "../common/async"
import { Closed } from "./error"
import { Order } from "./message"
export class Track {
readonly path: string[]
readonly priority: number
order = Order.Any
// TODO use an array
latest = new Watch<GroupReader | undefined>(undefined)
readers = 0
closed?: Closed
constructor(path: string[], priority: number) {
this.path = path
this.priority = priority
}
appendGroup(): Group {
const next = this.latest.value()[0]?.id ?? 0
return this.createGroup(next)
}
createGroup(sequence: number): Group {
if (this.closed) throw this.closed
const group = new Group(sequence)
const [current, _] = this.latest.value()
// TODO use an array
if (!current || current.id < sequence) {
const reader = new GroupReader(group)
this.latest.update(reader)
}
return group
}
close(closed = new Closed()) {
if (this.closed) return
this.closed = closed
this.latest.close()
}
reader(): TrackReader {
// VERY important that readers are closed to decrement the count
this.readers += 1
return new TrackReader(this)
}
}
export class TrackReader {
latest?: number
#track: Track
constructor(track: Track) {
this.#track = track
}
async nextGroup(): Promise<GroupReader | undefined> {
let [current, next] = this.#track.latest.value()
for (;;) {
if (current && this.latest !== current.id) {
this.latest = current.id
return current
}
if (this.#track.closed) throw this.#track.closed
if (!next) return
;[current, next] = await next
}
}
get path() {
return this.#track.path
}
get order() {
return this.#track.order
}
get priority() {
return this.#track.priority
}
close() {
this.#track.readers -= 1
if (this.#track.readers <= 0) this.#track.close()
}
}
export class Group {
readonly id: number
chunks = new Watch<Uint8Array[]>([])
readers = 0
closed?: Closed
constructor(id: number) {
this.id = id
}
writeFrame(frame: Uint8Array) {
if (this.closed) throw this.closed
this.chunks.update((chunks) => [...chunks, frame])
}
writeFrames(...frames: Uint8Array[]) {
if (this.closed) throw this.closed
this.chunks.update((chunks) => [...chunks, ...frames])
this.close()
}
reader(): GroupReader {
this.readers += 1
return new GroupReader(this)
}
get length(): number {
return this.chunks.value()[0].length
}
close(closed = new Closed()) {
if (this.closed) return
this.closed = closed
this.chunks.close()
}
}
export class GroupReader {
#group: Group
#index = 0
constructor(group: Group) {
this.#group = group
}
async readFrame(): Promise<Uint8Array | undefined> {
let [chunks, next] = this.#group.chunks.value()
for (;;) {
if (this.#index < chunks.length) {
this.#index += 1
return chunks[this.#index - 1]
}
if (this.#group.closed) throw this.#group.closed
if (!next) return
;[chunks, next] = await next
}
}
get index(): number {
return this.#index
}
get id(): number {
return this.#group.id
}
close() {
this.#group.readers -= 1
if (this.#group.readers <= 0) this.#group.close()
}
}

173
packages/moq/transfork/publisher.ts Normal file
View File

@@ -0,0 +1,173 @@
import { Watch } from "../common/async"
import { Closed } from "./error"
import * as Message from "./message"
import type { GroupReader, TrackReader } from "./model"
import { type Stream, Writer } from "./stream"
export class Publisher {
#quic: WebTransport
// Our announced broadcasts.
#announce = new Map<string[], TrackReader>()
// Their subscribed tracks.
#subscribe = new Map<bigint, Subscribed>()
constructor(quic: WebTransport) {
this.#quic = quic
}
// Publish a track
publish(track: TrackReader) {
if (this.#announce.has(track.path)) {
throw new Error(`already announced: ${track.path.toString()}`)
}
this.#announce.set(track.path, track)
// TODO: clean up announcements
// track.closed().then(() => this.#announce.delete(track.path))
}
#get(path: string[]): TrackReader | undefined {
return this.#announce.get(path)
}
async runAnnounce(msg: Message.AnnounceInterest, stream: Stream) {
for (const announce of this.#announce.values()) {
if (announce.path.length < msg.prefix.length) continue
const prefix = announce.path.slice(0, msg.prefix.length)
if (prefix !== msg.prefix) continue
const suffix = announce.path.slice(msg.prefix.length)
const active = new Message.Announce(suffix, "active")
await active.encode(stream.writer)
}
// TODO support updates.
// Until then, just keep the stream open.
await stream.reader.closed()
}
async runSubscribe(msg: Message.Subscribe, stream: Stream) {
if (this.#subscribe.has(msg.id)) {
throw new Error(`duplicate subscribe for id: ${msg.id}`)
}
const track = this.#get(msg.path)
if (!track) {
await stream.writer.reset(404)
return
}
const subscribe = new Subscribed(msg, track, this.#quic)
// TODO close the stream when done
subscribe.run().catch((err) => console.warn("failed to run subscribe: ", err))
try {
const info = new Message.Info(track.priority)
info.order = track.order
info.latest = track.latest
await info.encode(stream.writer)
for (;;) {
// TODO try_decode
const update = await Message.SubscribeUpdate.decode_maybe(stream.reader)
if (!update) {
subscribe.close()
break
}
// TODO use the update
}
} catch (err) {
subscribe.close(Closed.from(err))
}
}
async runDatagrams(msg: Message.Datagrams, stream: Stream) {
await stream.writer.reset(501)
throw new Error("datagrams not implemented")
}
async runFetch(msg: Message.Fetch, stream: Stream) {
await stream.writer.reset(501)
throw new Error("fetch not implemented")
}
async runInfo(msg: Message.InfoRequest, stream: Stream) {
const track = this.#get(msg.path)
if (!track) {
await stream.writer.reset(404)
return
}
const info = new Message.Info(track.priority)
info.order = track.order
info.latest = track.latest
await info.encode(stream.writer)
throw new Error("info not implemented")
}
}
class Subscribed {
#id: bigint
#track: TrackReader
#quic: WebTransport
#closed = new Watch<Closed | undefined>(undefined)
constructor(msg: Message.Subscribe, track: TrackReader, quic: WebTransport) {
this.#id = msg.id
this.#track = track
this.#quic = quic
}
async run() {
const closed = this.closed()
for (;;) {
const [group, done] = await Promise.all([this.#track.nextGroup(), closed])
if (done) return
if (!group) break
this.#runGroup(group).catch((err) => console.warn("failed to run group: ", err))
}
// TODO wait until all groups are done
this.close()
}
async #runGroup(group: GroupReader) {
const msg = new Message.Group(this.#id, group.id)
const stream = await Writer.open(this.#quic, msg)
for (;;) {
const frame = await group.readFrame()
if (!frame) break
await stream.u53(frame.byteLength)
await stream.write(frame)
}
}
close(err = new Closed()) {
this.#closed.update(err)
this.#track.close()
}
async closed(): Promise<Closed> {
let [closed, next] = this.#closed.value()
for (;;) {
if (closed !== undefined) return closed
if (!next) return new Closed()
;[closed, next] = await next
}
}
}

239
packages/moq/transport/stream.ts → packages/moq/transfork/stream.ts
View File

@@ -1,10 +1,93 @@
const MAX_U6 = Math.pow(2, 6) - 1
const MAX_U14 = Math.pow(2, 14) - 1
const MAX_U30 = Math.pow(2, 30) - 1
const MAX_U31 = Math.pow(2, 31) - 1
import * as Message from "./message"
const MAX_U6 = 2 ** 6 - 1
const MAX_U14 = 2 ** 14 - 1
const MAX_U30 = 2 ** 30 - 1
const MAX_U31 = 2 ** 31 - 1
const MAX_U53 = Number.MAX_SAFE_INTEGER
const MAX_U62: bigint = 2n ** 62n - 1n
export class Stream {
reader: Reader
writer: Writer
constructor(props: {
writable: WritableStream<Uint8Array>
readable: ReadableStream<Uint8Array>
}) {
this.writer = new Writer(props.writable)
this.reader = new Reader(props.readable)
}
static async accept(quic: WebTransport): Promise<[Message.Bi, Stream] | undefined> {
const reader = quic.incomingBidirectionalStreams.getReader()
const next = await reader.read()
reader.releaseLock()
if (next.done) return
const stream = new Stream(next.value)
let msg: Message.Bi
const typ = await stream.reader.u8()
if (typ === Message.SessionClient.StreamID) {
msg = await Message.SessionClient.decode(stream.reader)
} else if (typ === Message.AnnounceInterest.StreamID) {
msg = await Message.AnnounceInterest.decode(stream.reader)
} else if (typ === Message.Subscribe.StreamID) {
msg = await Message.Subscribe.decode(stream.reader)
} else if (typ === Message.Datagrams.StreamID) {
msg = await Message.Datagrams.decode(stream.reader)
} else if (typ === Message.Fetch.StreamID) {
msg = await Message.Fetch.decode(stream.reader)
} else if (typ === Message.InfoRequest.StreamID) {
msg = await Message.InfoRequest.decode(stream.reader)
} else {
throw new Error(`unknown stream type: ${typ}`)
}
console.debug("accepted stream", msg)
return [msg, stream]
}
static async open(quic: WebTransport, msg: Message.Bi): Promise<Stream> {
const stream = new Stream(await quic.createBidirectionalStream())
if (msg instanceof Message.SessionClient) {
await stream.writer.u8(Message.SessionClient.StreamID)
} else if (msg instanceof Message.AnnounceInterest) {
await stream.writer.u8(Message.AnnounceInterest.StreamID)
} else if (msg instanceof Message.Subscribe) {
await stream.writer.u8(Message.Subscribe.StreamID)
} else if (msg instanceof Message.Datagrams) {
await stream.writer.u8(Message.Datagrams.StreamID)
} else if (msg instanceof Message.Fetch) {
await stream.writer.u8(Message.Fetch.StreamID)
} else if (msg instanceof Message.InfoRequest) {
await stream.writer.u8(Message.InfoRequest.StreamID)
} else {
// Make sure we're not missing any types.
const _: never = msg
throw new Error("invalid message type")
}
await msg.encode(stream.writer)
console.debug("opened stream", msg)
return stream
}
async close(code?: number) {
if (code === undefined) {
await this.writer.close()
} else {
await this.writer.reset(code)
await this.reader.stop(code)
}
}
}
// Reader wraps a stream and provides convience methods for reading pieces from a stream
// Unfortunately we can't use a BYOB reader because it's not supported with WebTransport+WebWorkers yet.
export class Reader {
@@ -12,7 +95,7 @@ export class Reader {
#stream: ReadableStream<Uint8Array>
#reader: ReadableStreamDefaultReader<Uint8Array>
constructor(buffer: Uint8Array, stream: ReadableStream<Uint8Array>) {
constructor(stream: ReadableStream<Uint8Array>, buffer = new Uint8Array()) {
this.#buffer = buffer
this.#stream = stream
this.#reader = this.#stream.getReader()
@@ -27,7 +110,7 @@ export class Reader {
const buffer = new Uint8Array(result.value)
if (this.#buffer.byteLength == 0) {
if (this.#buffer.byteLength === 0) {
this.#buffer = buffer
} else {
const temp = new Uint8Array(this.#buffer.byteLength + buffer.byteLength)
@@ -57,14 +140,13 @@ export class Reader {
}
async read(size: number): Promise<Uint8Array> {
if (size == 0) return new Uint8Array()
if (size === 0) return new Uint8Array()
await this.#fillTo(size)
return this.#slice(size)
}
async readAll(): Promise<Uint8Array> {
// eslint-disable-next-line no-empty
while (await this.#fill()) {}
return this.#slice(this.#buffer.byteLength)
}
@@ -79,6 +161,17 @@ export class Reader {
return new TextDecoder().decode(buffer)
}
async path(): Promise<string[]> {
const parts = await this.u53()
const path = []
for (let i = 0; i < parts; i++) {
path.push(await this.string())
}
return path
}
async u8(): Promise<number> {
await this.#fillTo(1)
return this.#slice(1)[0]
@@ -99,30 +192,29 @@ export class Reader {
await this.#fillTo(1)
const size = (this.#buffer[0] & 0xc0) >> 6
if (size == 0) {
if (size === 0) {
const first = this.#slice(1)[0]
return BigInt(first) & 0x3fn
} else if (size == 1) {
}
if (size === 1) {
await this.#fillTo(2)
const slice = this.#slice(2)
const view = new DataView(slice.buffer, slice.byteOffset, slice.byteLength)
return BigInt(view.getInt16(0)) & 0x3fffn
} else if (size == 2) {
}
if (size === 2) {
await this.#fillTo(4)
const slice = this.#slice(4)
const view = new DataView(slice.buffer, slice.byteOffset, slice.byteLength)
return BigInt(view.getUint32(0)) & 0x3fffffffn
} else if (size == 3) {
await this.#fillTo(8)
const slice = this.#slice(8)
const view = new DataView(slice.buffer, slice.byteOffset, slice.byteLength)
return view.getBigUint64(0) & 0x3fffffffffffffffn
} else {
throw new Error("impossible")
}
await this.#fillTo(8)
const slice = this.#slice(8)
const view = new DataView(slice.buffer, slice.byteOffset, slice.byteLength)
return view.getBigUint64(0) & 0x3fffffffffffffffn
}
async done(): Promise<boolean> {
@@ -130,15 +222,38 @@ export class Reader {
return !(await this.#fill())
}
async close() {
async stop(code: number) {
this.#reader.releaseLock()
await this.#stream.cancel()
await this.#stream.cancel(code)
}
async closed() {
return this.#reader.closed
}
release(): [Uint8Array, ReadableStream<Uint8Array>] {
this.#reader.releaseLock()
return [this.#buffer, this.#stream]
}
static async accept(quic: WebTransport): Promise<[Message.Group, Reader] | undefined> {
const reader = quic.incomingUnidirectionalStreams.getReader()
const next = await reader.read()
reader.releaseLock()
if (next.done) return
const stream = new Reader(next.value)
let msg: Message.Uni
const typ = await stream.u8()
if (typ === Message.Group.StreamID) {
msg = await Message.Group.decode(stream)
} else {
throw new Error(`unknown stream type: ${typ}`)
}
return [msg, stream]
}
}
// Writer wraps a stream and writes chunks of data
@@ -170,7 +285,8 @@ export class Writer {
async u53(v: number) {
if (v < 0) {
throw new Error(`underflow, value is negative: ${v}`)
} else if (v > MAX_U53) {
}
if (v > MAX_U53) {
throw new Error(`overflow, value larger than 53-bits: ${v}`)
}
@@ -180,7 +296,8 @@ export class Writer {
async u62(v: bigint) {
if (v < 0) {
throw new Error(`underflow, value is negative: ${v}`)
} else if (v >= MAX_U62) {
}
if (v >= MAX_U62) {
throw new Error(`overflow, value larger than 62-bits: ${v}`)
}
@@ -197,72 +314,102 @@ export class Writer {
await this.write(data)
}
async path(path: string[]) {
await this.u53(path.length)
for (const part of path) {
await this.string(part)
}
}
async close() {
this.#writer.releaseLock()
await this.#stream.close()
}
async reset(code: number) {
this.#writer.releaseLock()
await this.#stream.abort(code)
}
release(): WritableStream<Uint8Array> {
this.#writer.releaseLock()
return this.#stream
}
static async open(quic: WebTransport, msg: Message.Uni): Promise<Writer> {
const stream = new Writer(await quic.createUnidirectionalStream())
if (msg instanceof Message.Group) {
await stream.u8(Message.Group.StreamID)
} else {
// Make sure we're not missing any types.
const _: never = msg
throw new Error("invalid message type")
}
return stream
}
}
function setUint8(dst: Uint8Array, v: number): Uint8Array {
export function setUint8(dst: Uint8Array, v: number): Uint8Array {
dst[0] = v
return dst.slice(0, 1)
}
function setUint16(dst: Uint8Array, v: number): Uint8Array {
export function setUint16(dst: Uint8Array, v: number): Uint8Array {
const view = new DataView(dst.buffer, dst.byteOffset, 2)
view.setUint16(0, v)
return new Uint8Array(view.buffer, view.byteOffset, view.byteLength)
}
function setInt32(dst: Uint8Array, v: number): Uint8Array {
export function setInt32(dst: Uint8Array, v: number): Uint8Array {
const view = new DataView(dst.buffer, dst.byteOffset, 4)
view.setInt32(0, v)
return new Uint8Array(view.buffer, view.byteOffset, view.byteLength)
}
function setUint32(dst: Uint8Array, v: number): Uint8Array {
export function setUint32(dst: Uint8Array, v: number): Uint8Array {
const view = new DataView(dst.buffer, dst.byteOffset, 4)
view.setUint32(0, v)
return new Uint8Array(view.buffer, view.byteOffset, view.byteLength)
}
function setVint53(dst: Uint8Array, v: number): Uint8Array {
export function setVint53(dst: Uint8Array, v: number): Uint8Array {
if (v <= MAX_U6) {
return setUint8(dst, v)
} else if (v <= MAX_U14) {
return setUint16(dst, v | 0x4000)
} else if (v <= MAX_U30) {
return setUint32(dst, v | 0x80000000)
} else if (v <= MAX_U53) {
return setUint64(dst, BigInt(v) | 0xc000000000000000n)
} else {
throw new Error(`overflow, value larger than 53-bits: ${v}`)
}
if (v <= MAX_U14) {
return setUint16(dst, v | 0x4000)
}
if (v <= MAX_U30) {
return setUint32(dst, v | 0x80000000)
}
if (v <= MAX_U53) {
return setUint64(dst, BigInt(v) | 0xc000000000000000n)
}
throw new Error(`overflow, value larger than 53-bits: ${v}`)
}
function setVint62(dst: Uint8Array, v: bigint): Uint8Array {
export function setVint62(dst: Uint8Array, v: bigint): Uint8Array {
if (v < MAX_U6) {
return setUint8(dst, Number(v))
} else if (v < MAX_U14) {
return setUint16(dst, Number(v) | 0x4000)
} else if (v <= MAX_U30) {
return setUint32(dst, Number(v) | 0x80000000)
} else if (v <= MAX_U62) {
return setUint64(dst, BigInt(v) | 0xc000000000000000n)
} else {
throw new Error(`overflow, value larger than 62-bits: ${v}`)
}
if (v < MAX_U14) {
return setUint16(dst, Number(v) | 0x4000)
}
if (v <= MAX_U30) {
return setUint32(dst, Number(v) | 0x80000000)
}
if (v <= MAX_U62) {
return setUint64(dst, BigInt(v) | 0xc000000000000000n)
}
throw new Error(`overflow, value larger than 62-bits: ${v}`)
}
function setUint64(dst: Uint8Array, v: bigint): Uint8Array {
export function setUint64(dst: Uint8Array, v: bigint): Uint8Array {
const view = new DataView(dst.buffer, dst.byteOffset, 8)
view.setBigUint64(0, v)

169
packages/moq/transfork/subscriber.ts Normal file
View File

@@ -0,0 +1,169 @@
import { Queue, Watch } from "../common/async"
import { Closed } from "./error"
import { FrameReader } from "./frame"
import * as Message from "./message"
import type { Track, TrackReader } from "./model"
import { type Reader, Stream } from "./stream"
export class Subscriber {
#quic: WebTransport
// Our subscribed tracks.
#subscribe = new Map<bigint, Subscribe>()
#subscribeNext = 0n
constructor(quic: WebTransport) {
this.#quic = quic
}
async announced(prefix: string[]): Promise<Queue<Announced>> {
const announced = new Queue<Announced>()
const msg = new Message.AnnounceInterest(prefix)
const stream = await Stream.open(this.#quic, msg)
this.runAnnounced(stream, announced, prefix)
.then(() => announced.close())
.catch((err) => announced.abort(err))
return announced
}
async runAnnounced(stream: Stream, announced: Queue<Announced>, prefix: string[]) {
const toggle: Map<string[], Announced> = new Map()
try {
for (;;) {
const announce = await Message.Announce.decode_maybe(stream.reader)
if (!announce) {
break
}
const existing = toggle.get(announce.suffix)
if (existing) {
if (announce.status === "active") {
throw new Error("duplicate announce")
}
existing.close()
toggle.delete(announce.suffix)
} else {
if (announce.status === "closed") {
throw new Error("unknown announce")
}
const path = prefix.concat(announce.suffix)
const item = new Announced(path)
await announced.push(item)
toggle.set(announce.suffix, item)
}
}
} finally {
for (const item of toggle.values()) {
item.close()
}
}
}
// TODO: Deduplicate identical subscribes
async subscribe(track: Track): Promise<TrackReader> {
const id = this.#subscribeNext++
const msg = new Message.Subscribe(id, track.path, track.priority)
const stream = await Stream.open(this.#quic, msg)
const subscribe = new Subscribe(id, stream, track)
this.#subscribe.set(subscribe.id, subscribe)
try {
const ok = await Message.Info.decode(stream.reader)
/*
for (;;) {
const dropped = await Message.GroupDrop.decode(stream.reader)
console.debug("dropped", dropped)
}
*/
return subscribe.track.reader()
} catch (err) {
console.error(err)
this.#subscribe.delete(subscribe.id)
await subscribe.close(Closed.from(err))
throw err
}
}
async runGroup(group: Message.Group, stream: Reader) {
const subscribe = this.#subscribe.get(group.subscribe)
if (!subscribe) return
const writer = subscribe.track.createGroup(group.sequence)
const reader = new FrameReader(stream)
for (;;) {
const frame = await reader.read()
if (!frame) break
writer.writeFrame(frame)
}
writer.close()
}
}
export class Announced {
readonly path: string[]
#closed = new Watch<Closed | undefined>(undefined)
constructor(path: string[]) {
this.path = path
}
close(err = new Closed()) {
this.#closed.update(err)
}
async closed(): Promise<Closed> {
let [closed, next] = this.#closed.value()
for (;;) {
if (closed !== undefined) return closed
if (!next) return new Closed()
;[closed, next] = await next
}
}
}
export class Subscribe {
readonly id: bigint
readonly track: Track
readonly stream: Stream
// A queue of received streams for this subscription.
#closed = new Watch<Closed | undefined>(undefined)
constructor(id: bigint, stream: Stream, track: Track) {
this.id = id
this.track = track
this.stream = stream
}
async run() {
try {
await this.closed()
await this.close()
} catch (err) {
await this.close(Closed.from(err))
}
}
async close(closed?: Closed) {
this.track.close(closed)
await this.stream.close(closed?.code)
}
async closed() {
await this.stream.reader.closed()
}
}

0
packages/moq/transport/tsconfig.json → packages/moq/transfork/tsconfig.json
View File

95
packages/moq/transport/connection.ts
View File

@@ -1,95 +0,0 @@
import * as Control from "./control"
import { Objects } from "./objects"
import { asError } from "../common/error"
import { Publisher } from "./publisher"
import { Subscriber } from "./subscriber"
export class Connection {
// The established WebTransport session.
#quic: WebTransport
// Use to receive/send control messages.
#control: Control.Stream
// Use to receive/send objects.
#objects: Objects
// Module for contributing tracks.
#publisher: Publisher
// Module for distributing tracks.
#subscriber: Subscriber
// Async work running in the background
#running: Promise<void>
constructor(quic: WebTransport, control: Control.Stream, objects: Objects) {
this.#quic = quic
this.#control = control
this.#objects = objects
this.#publisher = new Publisher(this.#control, this.#objects)
this.#subscriber = new Subscriber(this.#control, this.#objects)
this.#running = this.#run()
}
close(code = 0, reason = "") {
this.#quic.close({ closeCode: code, reason })
}
async #run(): Promise<void> {
await Promise.all([this.#runControl(), this.#runObjects()])
}
announce(namespace: string) {
return this.#publisher.announce(namespace)
}
announced() {
return this.#subscriber.announced()
}
subscribe(namespace: string, track: string) {
return this.#subscriber.subscribe(namespace, track)
}
subscribed() {
return this.#publisher.subscribed()
}
async #runControl() {
// Receive messages until the connection is closed.
for (;;) {
const msg = await this.#control.recv()
await this.#recv(msg)
}
}
async #runObjects() {
for (;;) {
const obj = await this.#objects.recv()
if (!obj) break
await this.#subscriber.recvObject(obj)
}
}
async #recv(msg: Control.Message) {
if (Control.isPublisher(msg)) {
await this.#subscriber.recv(msg)
} else {
await this.#publisher.recv(msg)
}
}
async closed(): Promise<Error> {
try {
await this.#running
return new Error("closed")
} catch (e) {
return asError(e)
}
}
}

550
packages/moq/transport/control.ts
View File

@@ -1,550 +0,0 @@
import { Reader, Writer } from "./stream"
export type Message = Subscriber | Publisher
// Sent by subscriber
export type Subscriber = Subscribe | Unsubscribe | AnnounceOk | AnnounceError
export function isSubscriber(m: Message): m is Subscriber {
return (
m.kind == Msg.Subscribe || m.kind == Msg.Unsubscribe || m.kind == Msg.AnnounceOk || m.kind == Msg.AnnounceError
)
}
// Sent by publisher
export type Publisher = SubscribeOk | SubscribeError | SubscribeDone | Announce | Unannounce
export function isPublisher(m: Message): m is Publisher {
return (
m.kind == Msg.SubscribeOk ||
m.kind == Msg.SubscribeError ||
m.kind == Msg.SubscribeDone ||
m.kind == Msg.Announce ||
m.kind == Msg.Unannounce
)
}
// I wish we didn't have to split Msg and Id into separate enums.
// However using the string in the message makes it easier to debug.
// We'll take the tiny performance hit until I'm better at Typescript.
export enum Msg {
// NOTE: object and setup are in other modules
Subscribe = "subscribe",
SubscribeOk = "subscribe_ok",
SubscribeError = "subscribe_error",
SubscribeDone = "subscribe_done",
Unsubscribe = "unsubscribe",
Announce = "announce",
AnnounceOk = "announce_ok",
AnnounceError = "announce_error",
Unannounce = "unannounce",
GoAway = "go_away",
}
enum Id {
// NOTE: object and setup are in other modules
// Object = 0,
// Setup = 1,
Subscribe = 0x3,
SubscribeOk = 0x4,
SubscribeError = 0x5,
SubscribeDone = 0xb,
Unsubscribe = 0xa,
Announce = 0x6,
AnnounceOk = 0x7,
AnnounceError = 0x8,
Unannounce = 0x9,
GoAway = 0x10,
}
export interface Subscribe {
kind: Msg.Subscribe
id: bigint
trackId: bigint
namespace: string
name: string
location: Location
params?: Parameters
}
export type Location = LatestGroup | LatestObject | AbsoluteStart | AbsoluteRange
export interface LatestGroup {
mode: "latest_group"
}
export interface LatestObject {
mode: "latest_object"
}
export interface AbsoluteStart {
mode: "absolute_start"
start_group: number
start_object: number
}
export interface AbsoluteRange {
mode: "absolute_range"
start_group: number
start_object: number
end_group: number
end_object: number
}
export type Parameters = Map<bigint, Uint8Array>
export interface SubscribeOk {
kind: Msg.SubscribeOk
id: bigint
expires: bigint
latest?: [number, number]
}
export interface SubscribeDone {
kind: Msg.SubscribeDone
id: bigint
code: bigint
reason: string
final?: [number, number]
}
export interface SubscribeError {
kind: Msg.SubscribeError
id: bigint
code: bigint
reason: string
}
export interface Unsubscribe {
kind: Msg.Unsubscribe
id: bigint
}
export interface Announce {
kind: Msg.Announce
namespace: string
params?: Parameters
}
export interface AnnounceOk {
kind: Msg.AnnounceOk
namespace: string
}
export interface AnnounceError {
kind: Msg.AnnounceError
namespace: string
code: bigint
reason: string
}
export interface Unannounce {
kind: Msg.Unannounce
namespace: string
}
export class Stream {
private decoder: Decoder
private encoder: Encoder
#mutex = Promise.resolve()
constructor(r: Reader, w: Writer) {
this.decoder = new Decoder(r)
this.encoder = new Encoder(w)
}
// Will error if two messages are read at once.
async recv(): Promise<Message> {
const msg = await this.decoder.message()
console.log("received message", msg)
return msg
}
async send(msg: Message) {
const unlock = await this.#lock()
try {
console.log("sending message", msg)
await this.encoder.message(msg)
} finally {
unlock()
}
}
async #lock() {
// Make a new promise that we can resolve later.
let done: () => void
const p = new Promise<void>((resolve) => {
done = () => resolve()
})
// Wait until the previous lock is done, then resolve our our lock.
const lock = this.#mutex.then(() => done)
// Save our lock as the next lock.
this.#mutex = p
// Return the lock.
return lock
}
}
export class Decoder {
r: Reader
constructor(r: Reader) {
this.r = r
}
private async msg(): Promise<Msg> {
const t = await this.r.u53()
switch (t) {
case Id.Subscribe:
return Msg.Subscribe
case Id.SubscribeOk:
return Msg.SubscribeOk
case Id.SubscribeDone:
return Msg.SubscribeDone
case Id.SubscribeError:
return Msg.SubscribeError
case Id.Unsubscribe:
return Msg.Unsubscribe
case Id.Announce:
return Msg.Announce
case Id.AnnounceOk:
return Msg.AnnounceOk
case Id.AnnounceError:
return Msg.AnnounceError
case Id.Unannounce:
return Msg.Unannounce
case Id.GoAway:
return Msg.GoAway
}
throw new Error(`unknown control message type: ${t}`)
}
async message(): Promise<Message> {
const t = await this.msg()
switch (t) {
case Msg.Subscribe:
return this.subscribe()
case Msg.SubscribeOk:
return this.subscribe_ok()
case Msg.SubscribeError:
return this.subscribe_error()
case Msg.SubscribeDone:
return this.subscribe_done()
case Msg.Unsubscribe:
return this.unsubscribe()
case Msg.Announce:
return this.announce()
case Msg.AnnounceOk:
return this.announce_ok()
case Msg.Unannounce:
return this.unannounce()
case Msg.AnnounceError:
return this.announce_error()
case Msg.GoAway:
throw new Error("TODO: implement go away")
}
}
private async subscribe(): Promise<Subscribe> {
return {
kind: Msg.Subscribe,
id: await this.r.u62(),
trackId: await this.r.u62(),
namespace: await this.r.string(),
name: await this.r.string(),
location: await this.location(),
params: await this.parameters(),
}
}
private async location(): Promise<Location> {
const mode = await this.r.u62()
if (mode == 1n) {
return {
mode: "latest_group",
}
} else if (mode == 2n) {
return {
mode: "latest_object",
}
} else if (mode == 3n) {
return {
mode: "absolute_start",
start_group: await this.r.u53(),
start_object: await this.r.u53(),
}
} else if (mode == 4n) {
return {
mode: "absolute_range",
start_group: await this.r.u53(),
start_object: await this.r.u53(),
end_group: await this.r.u53(),
end_object: await this.r.u53(),
}
} else {
throw new Error(`invalid filter type: ${mode}`)
}
}
private async parameters(): Promise<Parameters | undefined> {
const count = await this.r.u53()
if (count == 0) return undefined
const params = new Map<bigint, Uint8Array>()
for (let i = 0; i < count; i++) {
const id = await this.r.u62()
const size = await this.r.u53()
const value = await this.r.read(size)
if (params.has(id)) {
throw new Error(`duplicate parameter id: ${id}`)
}
params.set(id, value)
}
return params
}
private async subscribe_ok(): Promise<SubscribeOk> {
const id = await this.r.u62()
const expires = await this.r.u62()
let latest: [number, number] | undefined
const flag = await this.r.u8()
if (flag === 1) {
latest = [await this.r.u53(), await this.r.u53()]
} else if (flag !== 0) {
throw new Error(`invalid final flag: ${flag}`)
}
return {
kind: Msg.SubscribeOk,
id,
expires,
latest,
}
}
private async subscribe_done(): Promise<SubscribeDone> {
const id = await this.r.u62()
const code = await this.r.u62()
const reason = await this.r.string()
let final: [number, number] | undefined
const flag = await this.r.u8()
if (flag === 1) {
final = [await this.r.u53(), await this.r.u53()]
} else if (flag !== 0) {
throw new Error(`invalid final flag: ${flag}`)
}
return {
kind: Msg.SubscribeDone,
id,
code,
reason,
final,
}
}
private async subscribe_error(): Promise<SubscribeError> {
return {
kind: Msg.SubscribeError,
id: await this.r.u62(),
code: await this.r.u62(),
reason: await this.r.string(),
}
}
private async unsubscribe(): Promise<Unsubscribe> {
return {
kind: Msg.Unsubscribe,
id: await this.r.u62(),
}
}
private async announce(): Promise<Announce> {
const namespace = await this.r.string()
return {
kind: Msg.Announce,
namespace,
params: await this.parameters(),
}
}
private async announce_ok(): Promise<AnnounceOk> {
return {
kind: Msg.AnnounceOk,
namespace: await this.r.string(),
}
}
private async announce_error(): Promise<AnnounceError> {
return {
kind: Msg.AnnounceError,
namespace: await this.r.string(),
code: await this.r.u62(),
reason: await this.r.string(),
}
}
private async unannounce(): Promise<Unannounce> {
return {
kind: Msg.Unannounce,
namespace: await this.r.string(),
}
}
}
export class Encoder {
w: Writer
constructor(w: Writer) {
this.w = w
}
async message(m: Message) {
switch (m.kind) {
case Msg.Subscribe:
return this.subscribe(m)
case Msg.SubscribeOk:
return this.subscribe_ok(m)
case Msg.SubscribeError:
return this.subscribe_error(m)
case Msg.SubscribeDone:
return this.subscribe_done(m)
case Msg.Unsubscribe:
return this.unsubscribe(m)
case Msg.Announce:
return this.announce(m)
case Msg.AnnounceOk:
return this.announce_ok(m)
case Msg.AnnounceError:
return this.announce_error(m)
case Msg.Unannounce:
return this.unannounce(m)
}
}
async subscribe(s: Subscribe) {
await this.w.u53(Id.Subscribe)
await this.w.u62(s.id)
await this.w.u62(s.trackId)
await this.w.string(s.namespace)
await this.w.string(s.name)
await this.location(s.location)
await this.parameters(s.params)
}
private async location(l: Location) {
switch (l.mode) {
case "latest_group":
await this.w.u62(1n)
break
case "latest_object":
await this.w.u62(2n)
break
case "absolute_start":
await this.w.u62(3n)
await this.w.u53(l.start_group)
await this.w.u53(l.start_object)
break
case "absolute_range":
await this.w.u62(3n)
await this.w.u53(l.start_group)
await this.w.u53(l.start_object)
await this.w.u53(l.end_group)
await this.w.u53(l.end_object)
}
}
private async parameters(p: Parameters | undefined) {
if (!p) {
await this.w.u8(0)
return
}
await this.w.u53(p.size)
for (const [id, value] of p) {
await this.w.u62(id)
await this.w.u53(value.length)
await this.w.write(value)
}
}
async subscribe_ok(s: SubscribeOk) {
await this.w.u53(Id.SubscribeOk)
await this.w.u62(s.id)
await this.w.u62(s.expires)
if (s.latest !== undefined) {
await this.w.u8(1)
await this.w.u53(s.latest[0])
await this.w.u53(s.latest[1])
} else {
await this.w.u8(0)
}
}
async subscribe_done(s: SubscribeDone) {
await this.w.u53(Id.SubscribeDone)
await this.w.u62(s.id)
await this.w.u62(s.code)
await this.w.string(s.reason)
if (s.final !== undefined) {
await this.w.u8(1)
await this.w.u53(s.final[0])
await this.w.u53(s.final[1])
} else {
await this.w.u8(0)
}
}
async subscribe_error(s: SubscribeError) {
await this.w.u53(Id.SubscribeError)
await this.w.u62(s.id)
}
async unsubscribe(s: Unsubscribe) {
await this.w.u53(Id.Unsubscribe)
await this.w.u62(s.id)
}
async announce(a: Announce) {
await this.w.u53(Id.Announce)
await this.w.string(a.namespace)
await this.w.u53(0) // parameters
}
async announce_ok(a: AnnounceOk) {
await this.w.u53(Id.AnnounceOk)
await this.w.string(a.namespace)
}
async announce_error(a: AnnounceError) {
await this.w.u53(Id.AnnounceError)
await this.w.string(a.namespace)
await this.w.u62(a.code)
await this.w.string(a.reason)
}
async unannounce(a: Unannounce) {
await this.w.u53(Id.Unannounce)
await this.w.string(a.namespace)
}
}

307
packages/moq/transport/objects.ts
View File

@@ -1,307 +0,0 @@
import { Reader, Writer } from "./stream"
export { Reader, Writer }
export enum StreamType {
Object = 0x0,
Track = 0x50,
Group = 0x51,
}
export enum Status {
OBJECT_NULL = 1,
GROUP_NULL = 2,
GROUP_END = 3,
TRACK_END = 4,
}
export interface TrackHeader {
type: StreamType.Track
sub: bigint
track: bigint
priority: number // VarInt with a u32 maximum value
}
export interface TrackChunk {
group: number // The group sequence, as a number because 2^53 is enough.
object: number
payload: Uint8Array | Status
}
export interface GroupHeader {
type: StreamType.Group
sub: bigint
track: bigint
group: number // The group sequence, as a number because 2^53 is enough.
priority: number // VarInt with a u32 maximum value
}
export interface GroupChunk {
object: number
payload: Uint8Array | Status
}
export interface ObjectHeader {
type: StreamType.Object
sub: bigint
track: bigint
group: number
object: number
priority: number
status: number
}
export interface ObjectChunk {
payload: Uint8Array
}
type WriterType<T> = T extends TrackHeader
? TrackWriter
: T extends GroupHeader
? GroupWriter
: T extends ObjectHeader
? ObjectWriter
: never
export class Objects {
private quic: WebTransport
constructor(quic: WebTransport) {
this.quic = quic
}
async send<T extends TrackHeader | GroupHeader | ObjectHeader>(h: T): Promise<WriterType<T>> {
const stream = await this.quic.createUnidirectionalStream()
const w = new Writer(stream)
await w.u53(h.type)
await w.u62(h.sub)
await w.u62(h.track)
let res: WriterType<T>
if (h.type == StreamType.Object) {
await w.u53(h.group)
await w.u53(h.object)
await w.u53(h.priority)
await w.u53(h.status)
res = new ObjectWriter(h, w) as WriterType<T>
} else if (h.type === StreamType.Group) {
await w.u53(h.group)
await w.u53(h.priority)
res = new GroupWriter(h, w) as WriterType<T>
} else if (h.type === StreamType.Track) {
await w.u53(h.priority)
res = new TrackWriter(h, w) as WriterType<T>
} else {
throw new Error("unknown header type")
}
// console.trace("send object", res.header)
return res
}
async recv(): Promise<TrackReader | GroupReader | ObjectReader | undefined> {
const streams = this.quic.incomingUnidirectionalStreams.getReader()
const { value, done } = await streams.read()
streams.releaseLock()
if (done) return
const r = new Reader(new Uint8Array(), value)
const type = (await r.u53()) as StreamType
let res: TrackReader | GroupReader | ObjectReader
if (type == StreamType.Track) {
const h: TrackHeader = {
type,
sub: await r.u62(),
track: await r.u62(),
priority: await r.u53(),
}
res = new TrackReader(h, r)
} else if (type == StreamType.Group) {
const h: GroupHeader = {
type,
sub: await r.u62(),
track: await r.u62(),
group: await r.u53(),
priority: await r.u53(),
}
res = new GroupReader(h, r)
} else if (type == StreamType.Object) {
const h = {
type,
sub: await r.u62(),
track: await r.u62(),
group: await r.u53(),
object: await r.u53(),
status: await r.u53(),
priority: await r.u53(),
}
res = new ObjectReader(h, r)
} else {
throw new Error("unknown stream type")
}
// console.trace("receive object", res.header)
return res
}
}
export class TrackWriter {
constructor(
public header: TrackHeader,
public stream: Writer,
) {}
async write(c: TrackChunk) {
await this.stream.u53(c.group)
await this.stream.u53(c.object)
if (c.payload instanceof Uint8Array) {
await this.stream.u53(c.payload.byteLength)
await this.stream.write(c.payload)
} else {
// empty payload with status
await this.stream.u53(0)
await this.stream.u53(c.payload as number)
}
}
async close() {
await this.stream.close()
}
}
export class GroupWriter {
constructor(
public header: GroupHeader,
public stream: Writer,
) {}
async write(c: GroupChunk) {
await this.stream.u53(c.object)
if (c.payload instanceof Uint8Array) {
await this.stream.u53(c.payload.byteLength)
await this.stream.write(c.payload)
} else {
await this.stream.u53(0)
await this.stream.u53(c.payload as number)
}
}
async close() {
await this.stream.close()
}
}
export class ObjectWriter {
constructor(
public header: ObjectHeader,
public stream: Writer,
) {}
async write(c: ObjectChunk) {
await this.stream.write(c.payload)
}
async close() {
await this.stream.close()
}
}
export class TrackReader {
constructor(
public header: TrackHeader,
public stream: Reader,
) {}
async read(): Promise<TrackChunk | undefined> {
if (await this.stream.done()) {
return
}
const group = await this.stream.u53()
const object = await this.stream.u53()
const size = await this.stream.u53()
let payload
if (size == 0) {
payload = (await this.stream.u53()) as Status
} else {
payload = await this.stream.read(size)
}
return {
group,
object,
payload,
}
}
async close() {
await this.stream.close()
}
}
export class GroupReader {
constructor(
public header: GroupHeader,
public stream: Reader,
) {}
async read(): Promise<GroupChunk | undefined> {
if (await this.stream.done()) {
return
}
const object = await this.stream.u53()
const size = await this.stream.u53()
let payload
if (size == 0) {
payload = (await this.stream.u53()) as Status
} else {
payload = await this.stream.read(size)
}
return {
object,
payload,
}
}
async close() {
await this.stream.close()
}
}
export class ObjectReader {
constructor(
public header: ObjectHeader,
public stream: Reader,
) {}
// NOTE: Can only be called once.
async read(): Promise<ObjectChunk | undefined> {
if (await this.stream.done()) {
return
}
return {
payload: await this.stream.readAll(),
}
}
async close() {
await this.stream.close()
}
}

230
packages/moq/transport/publisher.ts
View File

@@ -1,230 +0,0 @@
import * as Control from "./control"
import { Queue, Watch } from "../common/async"
import { Objects, GroupWriter, ObjectWriter, StreamType, TrackWriter } from "./objects"
export class Publisher {
// Used to send control messages
#control: Control.Stream
// Use to send objects.
#objects: Objects
// Our announced tracks.
#announce = new Map<string, AnnounceSend>()
// Their subscribed tracks.
#subscribe = new Map<bigint, SubscribeRecv>()
#subscribeQueue = new Queue<SubscribeRecv>(Number.MAX_SAFE_INTEGER) // Unbounded queue in case there's no receiver
constructor(control: Control.Stream, objects: Objects) {
this.#control = control
this.#objects = objects
}
// Announce a track namespace.
async announce(namespace: string): Promise<AnnounceSend> {
if (this.#announce.has(namespace)) {
throw new Error(`already announce: ${namespace}`)
}
const announce = new AnnounceSend(this.#control, namespace)
this.#announce.set(namespace, announce)
await this.#control.send({
kind: Control.Msg.Announce,
namespace,
})
return announce
}
// Receive the next new subscription
async subscribed() {
return await this.#subscribeQueue.next()
}
async recv(msg: Control.Subscriber) {
if (msg.kind == Control.Msg.Subscribe) {
await this.recvSubscribe(msg)
} else if (msg.kind == Control.Msg.Unsubscribe) {
this.recvUnsubscribe(msg)
} else if (msg.kind == Control.Msg.AnnounceOk) {
this.recvAnnounceOk(msg)
} else if (msg.kind == Control.Msg.AnnounceError) {
this.recvAnnounceError(msg)
} else {
throw new Error(`unknown control message`) // impossible
}
}
recvAnnounceOk(msg: Control.AnnounceOk) {
const announce = this.#announce.get(msg.namespace)
if (!announce) {
throw new Error(`announce OK for unknown announce: ${msg.namespace}`)
}
announce.onOk()
}
recvAnnounceError(msg: Control.AnnounceError) {
const announce = this.#announce.get(msg.namespace)
if (!announce) {
// TODO debug this
console.warn(`announce error for unknown announce: ${msg.namespace}`)
return
}
announce.onError(msg.code, msg.reason)
}
async recvSubscribe(msg: Control.Subscribe) {
if (this.#subscribe.has(msg.id)) {
throw new Error(`duplicate subscribe for id: ${msg.id}`)
}
const subscribe = new SubscribeRecv(this.#control, this.#objects, msg)
this.#subscribe.set(msg.id, subscribe)
await this.#subscribeQueue.push(subscribe)
await this.#control.send({ kind: Control.Msg.SubscribeOk, id: msg.id, expires: 0n })
}
recvUnsubscribe(_msg: Control.Unsubscribe) {
throw new Error("TODO unsubscribe")
}
}
export class AnnounceSend {
#control: Control.Stream
readonly namespace: string
// The current state, updated by control messages.
#state = new Watch<"init" | "ack" | Error>("init")
constructor(control: Control.Stream, namespace: string) {
this.#control = control
this.namespace = namespace
}
async ok() {
for (;;) {
const [state, next] = this.#state.value()
if (state === "ack") return
if (state instanceof Error) throw state
if (!next) throw new Error("closed")
await next
}
}
async active() {
for (;;) {
const [state, next] = this.#state.value()
if (state instanceof Error) throw state
if (!next) return
await next
}
}
async close() {
// TODO implement unsubscribe
// await this.#inner.sendUnsubscribe()
}
closed() {
const [state, next] = this.#state.value()
return state instanceof Error || next == undefined
}
onOk() {
if (this.closed()) return
this.#state.update("ack")
}
onError(code: bigint, reason: string) {
if (this.closed()) return
const err = new Error(`ANNOUNCE_ERROR (${code})` + reason ? `: ${reason}` : "")
this.#state.update(err)
}
}
export class SubscribeRecv {
#control: Control.Stream
#objects: Objects
#id: bigint
#trackId: bigint
readonly namespace: string
readonly track: string
// The current state of the subscription.
#state: "init" | "ack" | "closed" = "init"
constructor(control: Control.Stream, objects: Objects, msg: Control.Subscribe) {
this.#control = control // so we can send messages
this.#objects = objects // so we can send objects
this.#id = msg.id
this.#trackId = msg.trackId
this.namespace = msg.namespace
this.track = msg.name
}
// Acknowledge the subscription as valid.
async ack() {
if (this.#state !== "init") return
this.#state = "ack"
// Send the control message.
return this.#control.send({ kind: Control.Msg.SubscribeOk, id: this.#id, expires: 0n })
}
// Close the subscription with an error.
async close(code = 0n, reason = "") {
if (this.#state === "closed") return
this.#state = "closed"
return this.#control.send({
kind: Control.Msg.SubscribeDone,
id: this.#id,
code,
reason,
})
}
// Create a writable data stream for the entire track
async serve(props?: { priority: number }): Promise<TrackWriter> {
return this.#objects.send({
type: StreamType.Track,
sub: this.#id,
track: this.#trackId,
priority: props?.priority ?? 0,
})
}
// Create a writable data stream for a group within the track
async group(props: { group: number; priority?: number }): Promise<GroupWriter> {
return this.#objects.send({
type: StreamType.Group,
sub: this.#id,
track: this.#trackId,
group: props.group,
priority: props.priority ?? 0,
})
}
// Create a writable data stream for a single object within the track
async object(props: { group: number; object: number; priority?: number }): Promise<ObjectWriter> {
return this.#objects.send({
type: StreamType.Object,
sub: this.#id,
track: this.#trackId,
group: props.group,
object: props.object,
priority: props.priority ?? 0,
status: 0,
})
}
}

163
packages/moq/transport/setup.ts
View File

@@ -1,163 +0,0 @@
import { Reader, Writer } from "./stream"
export type Message = Client | Server
export type Role = "publisher" | "subscriber" | "both"
export enum Version {
DRAFT_00 = 0xff000000,
DRAFT_01 = 0xff000001,
DRAFT_02 = 0xff000002,
DRAFT_03 = 0xff000003,
DRAFT_04 = 0xff000004,
KIXEL_00 = 0xbad00,
KIXEL_01 = 0xbad01,
}
// NOTE: These are forked from moq-transport-00.
// 1. messages lack a sized length
// 2. parameters are not optional and written in order (role + path)
// 3. role indicates local support only, not remote support
export interface Client {
versions: Version[]
role: Role
params?: Parameters
}
export interface Server {
version: Version
params?: Parameters
}
export class Stream {
recv: Decoder
send: Encoder
constructor(r: Reader, w: Writer) {
this.recv = new Decoder(r)
this.send = new Encoder(w)
}
}
export type Parameters = Map<bigint, Uint8Array>
export class Decoder {
r: Reader
constructor(r: Reader) {
this.r = r
}
async client(): Promise<Client> {
const type = await this.r.u53()
if (type !== 0x40) throw new Error(`client SETUP type must be 0x40, got ${type}`)
const count = await this.r.u53()
const versions = []
for (let i = 0; i < count; i++) {
const version = await this.r.u53()
versions.push(version)
}
const params = await this.parameters()
const role = this.role(params?.get(0n))
return {
versions,
role,
params,
}
}
async server(): Promise<Server> {
const type = await this.r.u53()
if (type !== 0x41) throw new Error(`server SETUP type must be 0x41, got ${type}`)
const version = await this.r.u53()
const params = await this.parameters()
return {
version,
params,
}
}
private async parameters(): Promise<Parameters | undefined> {
const count = await this.r.u53()
if (count == 0) return undefined
const params = new Map<bigint, Uint8Array>()
for (let i = 0; i < count; i++) {
const id = await this.r.u62()
const size = await this.r.u53()
const value = await this.r.read(size)
if (params.has(id)) {
throw new Error(`duplicate parameter id: ${id}`)
}
params.set(id, value)
}
return params
}
role(raw: Uint8Array | undefined): Role {
if (!raw) throw new Error("missing role parameter")
if (raw.length != 1) throw new Error("multi-byte varint not supported")
switch (raw[0]) {
case 1:
return "publisher"
case 2:
return "subscriber"
case 3:
return "both"
default:
throw new Error(`invalid role: ${raw[0]}`)
}
}
}
export class Encoder {
w: Writer
constructor(w: Writer) {
this.w = w
}
async client(c: Client) {
await this.w.u53(0x40)
await this.w.u53(c.versions.length)
for (const v of c.versions) {
await this.w.u53(v)
}
// I hate it
const params = c.params ?? new Map()
params.set(0n, new Uint8Array([c.role == "publisher" ? 1 : c.role == "subscriber" ? 2 : 3]))
await this.parameters(params)
}
async server(s: Server) {
await this.w.u53(0x41)
await this.w.u53(s.version)
await this.parameters(s.params)
}
private async parameters(p: Parameters | undefined) {
if (!p) {
await this.w.u8(0)
return
}
await this.w.u53(p.size)
for (const [id, value] of p) {
await this.w.u62(id)
await this.w.u53(value.length)
await this.w.write(value)
}
}
}

197
packages/moq/transport/subscriber.ts
View File

@@ -1,197 +0,0 @@
import * as Control from "./control"
import { Queue, Watch } from "../common/async"
import { Objects } from "./objects"
import type { TrackReader, GroupReader, ObjectReader } from "./objects"
export class Subscriber {
// Use to send control messages.
#control: Control.Stream
// Use to send objects.
#objects: Objects
// Announced broadcasts.
#announce = new Map<string, AnnounceRecv>()
#announceQueue = new Watch<AnnounceRecv[]>([])
// Our subscribed tracks.
#subscribe = new Map<bigint, SubscribeSend>()
#subscribeNext = 0n
constructor(control: Control.Stream, objects: Objects) {
this.#control = control
this.#objects = objects
}
announced(): Watch<AnnounceRecv[]> {
return this.#announceQueue
}
async recv(msg: Control.Publisher) {
if (msg.kind == Control.Msg.Announce) {
await this.recvAnnounce(msg)
} else if (msg.kind == Control.Msg.Unannounce) {
this.recvUnannounce(msg)
} else if (msg.kind == Control.Msg.SubscribeOk) {
this.recvSubscribeOk(msg)
} else if (msg.kind == Control.Msg.SubscribeError) {
await this.recvSubscribeError(msg)
} else if (msg.kind == Control.Msg.SubscribeDone) {
await this.recvSubscribeDone(msg)
} else {
throw new Error(`unknown control message`) // impossible
}
}
async recvAnnounce(msg: Control.Announce) {
if (this.#announce.has(msg.namespace)) {
throw new Error(`duplicate announce for namespace: ${msg.namespace}`)
}
await this.#control.send({ kind: Control.Msg.AnnounceOk, namespace: msg.namespace })
const announce = new AnnounceRecv(this.#control, msg.namespace)
this.#announce.set(msg.namespace, announce)
this.#announceQueue.update((queue) => [...queue, announce])
}
recvUnannounce(_msg: Control.Unannounce) {
throw new Error(`TODO Unannounce`)
}
async subscribe(namespace: string, track: string) {
const id = this.#subscribeNext++
const subscribe = new SubscribeSend(this.#control, id, namespace, track)
this.#subscribe.set(id, subscribe)
await this.#control.send({
kind: Control.Msg.Subscribe,
id,
trackId: id,
namespace,
name: track,
location: {
mode: "latest_group",
},
})
return subscribe
}
recvSubscribeOk(msg: Control.SubscribeOk) {
const subscribe = this.#subscribe.get(msg.id)
if (!subscribe) {
throw new Error(`subscribe ok for unknown id: ${msg.id}`)
}
subscribe.onOk()
}
async recvSubscribeError(msg: Control.SubscribeError) {
const subscribe = this.#subscribe.get(msg.id)
if (!subscribe) {
throw new Error(`subscribe error for unknown id: ${msg.id}`)
}
await subscribe.onError(msg.code, msg.reason)
}
async recvSubscribeDone(msg: Control.SubscribeDone) {
const subscribe = this.#subscribe.get(msg.id)
if (!subscribe) {
throw new Error(`subscribe error for unknown id: ${msg.id}`)
}
await subscribe.onError(msg.code, msg.reason)
}
async recvObject(reader: TrackReader | GroupReader | ObjectReader) {
const subscribe = this.#subscribe.get(reader.header.track)
if (!subscribe) {
throw new Error(`data for for unknown track: ${reader.header.track}`)
}
await subscribe.onData(reader)
}
}
export class AnnounceRecv {
#control: Control.Stream
readonly namespace: string
// The current state of the announce
#state: "init" | "ack" | "closed" = "init"
constructor(control: Control.Stream, namespace: string) {
this.#control = control // so we can send messages
this.namespace = namespace
}
// Acknowledge the subscription as valid.
async ok() {
if (this.#state !== "init") return
this.#state = "ack"
// Send the control message.
return this.#control.send({ kind: Control.Msg.AnnounceOk, namespace: this.namespace })
}
async close(code = 0n, reason = "") {
if (this.#state === "closed") return
this.#state = "closed"
return this.#control.send({ kind: Control.Msg.AnnounceError, namespace: this.namespace, code, reason })
}
}
export class SubscribeSend {
#control: Control.Stream
#id: bigint
readonly namespace: string
readonly track: string
// A queue of received streams for this subscription.
#data = new Queue<TrackReader | GroupReader | ObjectReader>()
constructor(control: Control.Stream, id: bigint, namespace: string, track: string) {
this.#control = control // so we can send messages
this.#id = id
this.namespace = namespace
this.track = track
}
async close(_code = 0n, _reason = "") {
// TODO implement unsubscribe
// await this.#inner.sendReset(code, reason)
}
onOk() {
// noop
}
async onError(code: bigint, reason: string) {
if (code == 0n) {
return await this.#data.close()
}
if (reason !== "") {
reason = `: ${reason}`
}
const err = new Error(`SUBSCRIBE_ERROR (${code})${reason}`)
return await this.#data.abort(err)
}
async onData(reader: TrackReader | GroupReader | ObjectReader) {
if (!this.#data.closed()) await this.#data.push(reader)
}
// Receive the next a readable data stream
async data() {
return await this.#data.next()
}
}

6
packages/moq/tsconfig.json
View File

@@ -14,7 +14,7 @@
"isolatedModules": true,
"types": [], // Don't automatically import any @types modules.
"lib": ["es2022", "dom"],
"typeRoots": ["./types", "../node_modules/@types"]
"typeRoots": ["../node_modules/@types"]
},
"references": [
{
@@ -30,10 +30,10 @@
"path": "./contribute"
},
{
"path": "./transport"
"path": "./transfork"
},
{
"path": "./media"
"path": "./karp"
}
],
"paths": {

1848
packages/moq/types/mp4box.d.ts vendored
View File

File diff suppressed because it is too large Load Diff

4
packages/moq/types/tsconfig.json
View File

@@ -1,4 +0,0 @@
{
"extends": "../tsconfig.json",
"include": ["."]
}

16
packages/relay/dev/Cargo.toml Normal file
View File

@@ -0,0 +1,16 @@
[package]
name = "dev"
version = "0.1.0"
edition = "2021"
[[bin]]
name = "nestri-test-server"
path = "src/main.rs"
[dependencies]
webrtc = "0.11.0"
tokio = { version = "1.41.1", features = ["full"] }
reqwest = { version = "0.12", features = ["json"] }
serde = { version = "1.0.215", features = ["derive"]}
serde_json = "1.0.133"

11
packages/relay/dev/Containerfile Normal file
View File

@@ -0,0 +1,11 @@
FROM archlinux:latest
RUN pacman -Syu --noconfirm
RUN pacman -Su --noconfirm \
gstreamer gst-plugins-base gst-plugins-good gst-plugin-rswebrtc
RUN pacman -Syu --noconfirm \
mesa mesa-utils xorg-xwayland vulkan-intel vpl-gpu-rt intel-media-driver gst-plugin-va gst-plugins-bad gst-plugin-fmp4 gst-plugin-qsv gst-plugin-pipewire
CMD [ "bash","-c", "gst-launch-1.0 videotestsrc ! openh264enc ! whip0. audiotestsrc ! opusenc ! whip0. whipclientsink name=whip0 signaller::whip-endpoint=http://localhost:8088/api/whip/test" ]

17
packages/relay/dev/server.sh Executable file
View File

@@ -0,0 +1,17 @@
#! /bin/bash -e
# sudo apt install build-essential -y
# To run tests, run the relay first - go run main.go.
# Run the docker container next - docker run --rm --init -d --device /dev/dri --network=host test-server
# Then run the nestri-test-server - cd packages/relay/dev cargo run
# Then run the frontend site, and navigate to http://localhost:5713/play/test
# Expected behavior, see some random messages on the browser's console tab
# And if you input works correctly, it should be logged to the console on the server-side of things
# docker build -t test-server -f Containerfile .
# docker run --rm --init -d --device /dev/dri --network=host test-server
# echo -e "Navigate to http://localhost:5713/play/test"

11
packages/relay/dev/src/main.rs Normal file
View File

@@ -0,0 +1,11 @@
mod room;
#[tokio::main]
async fn main() -> std::io::Result<()> {
let room = "test";
let base_url = "http://localhost:8088";
let mut room_handler = room::Room::new(room, base_url).await?;
room_handler.run().await?;
Ok(())
}

292
packages/relay/dev/src/room.rs Normal file
View File

@@ -0,0 +1,292 @@
use reqwest;
use std::collections::HashSet;
use std::io;
use std::sync::Arc;
use tokio::sync::mpsc;
use tokio::sync::Mutex;
use tokio::time::Duration;
use webrtc::api::interceptor_registry::register_default_interceptors;
// use std::collections::HashSet;
use serde::{Deserialize, Serialize};
use serde_json::from_str;
use webrtc::api::media_engine::MediaEngine;
use webrtc::api::APIBuilder;
use webrtc::data_channel::data_channel_message::DataChannelMessage;
use webrtc::ice_transport::ice_server::RTCIceServer;
use webrtc::interceptor::registry::Registry;
use webrtc::peer_connection::configuration::RTCConfiguration;
use webrtc::peer_connection::math_rand_alpha;
use webrtc::peer_connection::peer_connection_state::RTCPeerConnectionState;
use webrtc::peer_connection::sdp::session_description::RTCSessionDescription;
#[derive(Serialize, Deserialize, Debug)]
#[serde(tag = "type")]
enum InputMessage {
#[serde(rename = "mousemove")]
MouseMove { x: i32, y: i32 },
#[serde(rename = "mousemoveabs")]
MouseMoveAbs { x: i32, y: i32 },
#[serde(rename = "wheel")]
Wheel { x: f64, y: f64 },
#[serde(rename = "mousedown")]
MouseDown { key: i32 },
// Add other variants as needed
#[serde(rename = "mouseup")]
MouseUp { key: i32 },
#[serde(rename = "keydown")]
KeyDown { key: i32 },
#[serde(rename = "keyup")]
KeyUp { key: i32 },
}
pub struct Room {
peer_connection: Arc<webrtc::peer_connection::RTCPeerConnection>,
data_channel: Arc<webrtc::data_channel::RTCDataChannel>,
done_tx: mpsc::Sender<()>,
done_rx: mpsc::Receiver<()>,
base_url: String,
stream_name: String,
// pipeline: Arc<Mutex<gst::Pipeline>>,
}
impl Room {
pub async fn new(
stream_name: &str,
base_url: &str,
// pipeline: Arc<Mutex<gst::Pipeline>>,
) -> io::Result<Self> {
// Create a MediaEngine object to configure the supported codec
let mut m = MediaEngine::default();
// Register default codecs
let _ = m.register_default_codecs().map_err(map_to_io_error)?;
let mut registry = Registry::new();
// Use the default set of Interceptors
registry = register_default_interceptors(registry, &mut m).map_err(map_to_io_error)?;
// Create the API object with the MediaEngine
let api = APIBuilder::new()
.with_media_engine(m)
.with_interceptor_registry(registry)
.build();
// Prepare the configuration
let config = RTCConfiguration {
ice_servers: vec![RTCIceServer {
urls: vec!["stun:stun.l.google.com:19302".to_owned()],
..Default::default()
}],
..Default::default()
};
// Create a new RTCPeerConnection
let peer_connection = Arc::new(
api.new_peer_connection(config)
.await
.map_err(map_to_io_error)?,
);
// Create a datachannel with label 'data'
let data_channel = peer_connection
.create_data_channel("input", None)
.await
.map_err(map_to_io_error)?;
let (done_tx, done_rx) = mpsc::channel::<()>(1);
let done_tx_clone = done_tx.clone();
// Peer connection state change handler
peer_connection.on_peer_connection_state_change(Box::new(
move |s: RTCPeerConnectionState| {
println!("Peer Connection State has changed: {s}");
if s == RTCPeerConnectionState::Failed {
println!("Peer Connection has gone to failed exiting");
let _ = done_tx_clone.try_send(());
}
Box::pin(async {})
},
));
Ok(Self {
peer_connection,
// pipeline,
data_channel,
done_tx,
done_rx,
base_url: base_url.to_string(),
stream_name: stream_name.to_string(),
})
}
pub async fn run(&mut self) -> io::Result<()> {
// Create an async channel for sending events to the pipeline
let (event_tx, mut event_rx) = mpsc::channel(10);
// A shared state to track currently pressed keys
let pressed_keys = Arc::new(tokio::sync::Mutex::new(HashSet::new()));
// Spawn a task to process events for the pipeline
let pipeline_task = {
// let pipeline = Arc::clone(self.pipeline);
// let pipeline_clone = self.pipeline.clone();
tokio::spawn(async move {
while let Some(event) = event_rx.recv().await {
// let pipeline = pipeline_clone.lock().await;
// pipeline.send_event(event);
println!("Invoked an event: {}", event)
}
})
};
let data_channel = self.data_channel.clone();
//TODO: Handle heartbeats here
let d1 = Arc::clone(&self.data_channel);
data_channel.on_open(Box::new(move || {
println!("Data channel '{}'-'{}' open. Random messages will now be sent to any connected DataChannels every 5 seconds", d1.label(), d1.id());
let d2 = Arc::clone(&d1);
Box::pin(async move {
let mut result = std::io::Result::<usize>::Ok(0);
while result.is_ok() {
let timeout = tokio::time::sleep(Duration::from_secs(5));
tokio::pin!(timeout);
tokio::select! {
_ = timeout.as_mut() =>{
let message = math_rand_alpha(15);
println!("Sending '{message}'");
result = d2.send_text(message).await.map_err(map_to_io_error);
}
};
}
})
}));
// Data channel message handler
let d_label = data_channel.label().to_owned();
data_channel.on_message(Box::new(move |msg: DataChannelMessage| {
let msg_str = String::from_utf8(msg.data.to_vec()).unwrap();
println!("Message from DataChannel '{d_label}': '{msg_str}'");
let event_tx = event_tx.clone();
let pressed_keys = Arc::clone(&pressed_keys);
tokio::spawn(async move {
if let Ok(input_msg) = from_str::<InputMessage>(&msg_str) {
if let Some(event) = handle_input_message(input_msg, &pressed_keys).await {
event_tx.send(event).await.unwrap();
}
}
});
Box::pin(async {})
}));
// Create an offer to send to the browser
let offer = self
.peer_connection
.create_offer(None)
.await
.map_err(map_to_io_error)?;
// Create channel that is blocked until ICE Gathering is complete
let mut gather_complete = self.peer_connection.gathering_complete_promise().await;
// Sets the LocalDescription, and starts our UDP listeners
self.peer_connection
.set_local_description(offer)
.await
.map_err(map_to_io_error)?;
// Block until ICE Gathering is complete, disabling trickle ICE
// we do this because we only can exchange one signaling message
// in a production application you should exchange ICE Candidates via OnICECandidate
let _ = gather_complete.recv().await;
if let Some(local_description) = self.peer_connection.local_description().await {
let url = format!("{}/api/whep/{}", self.base_url, self.stream_name);
let response = reqwest::Client::new()
.post(&url)
.header("Content-Type", "application/sdp")
.body(local_description.sdp.clone()) // clone if you don't want to move offer.sdp
.send()
.await
.map_err(map_to_io_error)?;
let answer = response
.json::<RTCSessionDescription>()
.await
.map_err(map_to_io_error)?;
self.peer_connection
.set_remote_description(answer)
.await
.map_err(map_to_io_error)?;
} else {
println!("generate local_description failed!");
};
println!("Press ctrl-c to stop");
tokio::select! {
_ = self.done_rx.recv() => {
println!("received done signal!");
}
_ = tokio::signal::ctrl_c() => {
println!();
}
};
self.peer_connection
.close()
.await
.map_err(map_to_io_error)?;
//FIXME: Ctr + C is not working... i suspect it has something to do with this guy -- Do not forget to fix packages/server/room.rs as well
pipeline_task.await?;
Ok(())
}
}
fn map_to_io_error<E: std::fmt::Display>(e: E) -> io::Error {
io::Error::new(io::ErrorKind::Other, format!("{}", e))
}
async fn handle_input_message(
input_msg: InputMessage,
pressed_keys: &Arc<tokio::sync::Mutex<HashSet<i32>>>,
) -> Option<String> {
match input_msg {
InputMessage::MouseMove { x, y } => Some("MouseMoved".to_string()),
InputMessage::MouseMoveAbs { x, y } => Some("MouseMoveAbsolute".to_string()),
InputMessage::KeyDown { key } => {
let mut keys = pressed_keys.lock().await;
// If the key is already pressed, return to prevent key lockup
if keys.contains(&key) {
return None;
}
keys.insert(key);
Some("KeyDown".to_string())
}
InputMessage::KeyUp { key } => {
let mut keys = pressed_keys.lock().await;
// Remove the key from the pressed state when released
keys.remove(&key);
Some("KeyUp".to_string())
}
InputMessage::Wheel { x, y } => Some("Wheel".to_string()),
InputMessage::MouseDown { key } => Some("MouseDown".to_string()),
InputMessage::MouseUp { key } => Some("MouseUp".to_string()),
}
}

31
packages/relay/go.mod Normal file
View File

@@ -0,0 +1,31 @@
module relay
go 1.23
require (
github.com/google/uuid v1.6.0
github.com/gorilla/websocket v1.5.3
github.com/pion/interceptor v0.1.37
github.com/pion/webrtc/v4 v4.0.2
)
require (
github.com/pion/datachannel v1.5.9 // indirect
github.com/pion/dtls/v3 v3.0.4 // indirect
github.com/pion/ice/v4 v4.0.2 // indirect
github.com/pion/logging v0.2.2 // indirect
github.com/pion/mdns/v2 v2.0.7 // indirect
github.com/pion/randutil v0.1.0 // indirect
github.com/pion/rtcp v1.2.14 // indirect
github.com/pion/rtp v1.8.9 // indirect
github.com/pion/sctp v1.8.34 // indirect
github.com/pion/sdp/v3 v3.0.9 // indirect
github.com/pion/srtp/v3 v3.0.4 // indirect
github.com/pion/stun/v3 v3.0.0 // indirect
github.com/pion/transport/v3 v3.0.7 // indirect
github.com/pion/turn/v4 v4.0.0 // indirect
github.com/wlynxg/anet v0.0.5 // indirect
golang.org/x/crypto v0.29.0 // indirect
golang.org/x/net v0.31.0 // indirect
golang.org/x/sys v0.27.0 // indirect
)

62
packages/relay/go.sum Normal file
View File

@@ -0,0 +1,62 @@
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/pion/datachannel v1.5.9 h1:LpIWAOYPyDrXtU+BW7X0Yt/vGtYxtXQ8ql7dFfYUVZA=
github.com/pion/datachannel v1.5.9/go.mod h1:kDUuk4CU4Uxp82NH4LQZbISULkX/HtzKa4P7ldf9izE=
github.com/pion/dtls/v3 v3.0.4 h1:44CZekewMzfrn9pmGrj5BNnTMDCFwr+6sLH+cCuLM7U=
github.com/pion/dtls/v3 v3.0.4/go.mod h1:R373CsjxWqNPf6MEkfdy3aSe9niZvL/JaKlGeFphtMg=
github.com/pion/ice/v4 v4.0.2 h1:1JhBRX8iQLi0+TfcavTjPjI6GO41MFn4CeTBX+Y9h5s=
github.com/pion/ice/v4 v4.0.2/go.mod h1:DCdqyzgtsDNYN6/3U8044j3U7qsJ9KFJC92VnOWHvXg=
github.com/pion/interceptor v0.1.37 h1:aRA8Zpab/wE7/c0O3fh1PqY0AJI3fCSEM5lRWJVorwI=
github.com/pion/interceptor v0.1.37/go.mod h1:JzxbJ4umVTlZAf+/utHzNesY8tmRkM2lVmkS82TTj8Y=
github.com/pion/logging v0.2.2 h1:M9+AIj/+pxNsDfAT64+MAVgJO0rsyLnoJKCqf//DoeY=
github.com/pion/logging v0.2.2/go.mod h1:k0/tDVsRCX2Mb2ZEmTqNa7CWsQPc+YYCB7Q+5pahoms=
github.com/pion/mdns/v2 v2.0.7 h1:c9kM8ewCgjslaAmicYMFQIde2H9/lrZpjBkN8VwoVtM=
github.com/pion/mdns/v2 v2.0.7/go.mod h1:vAdSYNAT0Jy3Ru0zl2YiW3Rm/fJCwIeM0nToenfOJKA=
github.com/pion/randutil v0.1.0 h1:CFG1UdESneORglEsnimhUjf33Rwjubwj6xfiOXBa3mA=
github.com/pion/randutil v0.1.0/go.mod h1:XcJrSMMbbMRhASFVOlj/5hQial/Y8oH/HVo7TBZq+j8=
github.com/pion/rtcp v1.2.14 h1:KCkGV3vJ+4DAJmvP0vaQShsb0xkRfWkO540Gy102KyE=
github.com/pion/rtcp v1.2.14/go.mod h1:sn6qjxvnwyAkkPzPULIbVqSKI5Dv54Rv7VG0kNxh9L4=
github.com/pion/rtp v1.8.9 h1:E2HX740TZKaqdcPmf4pw6ZZuG8u5RlMMt+l3dxeu6Wk=
github.com/pion/rtp v1.8.9/go.mod h1:pBGHaFt/yW7bf1jjWAoUjpSNoDnw98KTMg+jWWvziqU=
github.com/pion/sctp v1.8.34 h1:rCuD3m53i0oGxCSp7FLQKvqVx0Nf5AUAHhMRXTTQjBc=
github.com/pion/sctp v1.8.34/go.mod h1:yWkCClkXlzVW7BXfI2PjrUGBwUI0CjXJBkhLt+sdo4U=
github.com/pion/sdp/v3 v3.0.9 h1:pX++dCHoHUwq43kuwf3PyJfHlwIj4hXA7Vrifiq0IJY=
github.com/pion/sdp/v3 v3.0.9/go.mod h1:B5xmvENq5IXJimIO4zfp6LAe1fD9N+kFv+V/1lOdz8M=
github.com/pion/srtp/v3 v3.0.4 h1:2Z6vDVxzrX3UHEgrUyIGM4rRouoC7v+NiF1IHtp9B5M=
github.com/pion/srtp/v3 v3.0.4/go.mod h1:1Jx3FwDoxpRaTh1oRV8A/6G1BnFL+QI82eK4ms8EEJQ=
github.com/pion/stun/v3 v3.0.0 h1:4h1gwhWLWuZWOJIJR9s2ferRO+W3zA/b6ijOI6mKzUw=
github.com/pion/stun/v3 v3.0.0/go.mod h1:HvCN8txt8mwi4FBvS3EmDghW6aQJ24T+y+1TKjB5jyU=
github.com/pion/transport/v3 v3.0.7 h1:iRbMH05BzSNwhILHoBoAPxoB9xQgOaJk+591KC9P1o0=
github.com/pion/transport/v3 v3.0.7/go.mod h1:YleKiTZ4vqNxVwh77Z0zytYi7rXHl7j6uPLGhhz9rwo=
github.com/pion/turn/v4 v4.0.0 h1:qxplo3Rxa9Yg1xXDxxH8xaqcyGUtbHYw4QSCvmFWvhM=
github.com/pion/turn/v4 v4.0.0/go.mod h1:MuPDkm15nYSklKpN8vWJ9W2M0PlyQZqYt1McGuxG7mA=
github.com/pion/webrtc/v4 v4.0.2 h1:fBwm5/hqSUybrCWl0DDBSTDrpbkcgkqpeLmXw9CsBQA=
github.com/pion/webrtc/v4 v4.0.2/go.mod h1:moylBT2A4dNoEaYBCdV1nThM3TLwRHzWszIG+eSPaqQ=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/wlynxg/anet v0.0.5 h1:J3VJGi1gvo0JwZ/P1/Yc/8p63SoW98B5dHkYDmpgvvU=
github.com/wlynxg/anet v0.0.5/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA=
golang.org/x/crypto v0.29.0 h1:L5SG1JTTXupVV3n6sUqMTeWbjAyfPwoda2DLX8J8FrQ=
golang.org/x/crypto v0.29.0/go.mod h1:+F4F4N5hv6v38hfeYwTdx20oUvLLc+QfrE9Ax9HtgRg=
golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo=
golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM=
golang.org/x/sys v0.27.0 h1:wBqf8DvsY9Y/2P8gAfPDEYNuS30J4lPHJxXSb/nJZ+s=
golang.org/x/sys v0.27.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

100
packages/relay/internal/common.go Normal file
View File

@@ -0,0 +1,100 @@
package relay
import (
"github.com/pion/interceptor"
"github.com/pion/webrtc/v4"
"log"
)
var globalWebRTCAPI *webrtc.API
var globalWebRTCConfig = webrtc.Configuration{
ICETransportPolicy: webrtc.ICETransportPolicyAll,
BundlePolicy: webrtc.BundlePolicyBalanced,
SDPSemantics: webrtc.SDPSemanticsUnifiedPlan,
}
func InitWebRTCAPI() error {
var err error
flags := GetFlags()
// Media engine
mediaEngine := &webrtc.MediaEngine{}
// Default codecs cover most of our needs
err = mediaEngine.RegisterDefaultCodecs()
if err != nil {
return err
}
// Add H.265 for special cases
videoRTCPFeedback := []webrtc.RTCPFeedback{{"goog-remb", ""}, {"ccm", "fir"}, {"nack", ""}, {"nack", "pli"}}
for _, codec := range []webrtc.RTPCodecParameters{
{
RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeH265, ClockRate: 90000, RTCPFeedback: videoRTCPFeedback},
PayloadType: 48,
},
{
RTPCodecCapability: webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeRTX, ClockRate: 90000, SDPFmtpLine: "apt=48"},
PayloadType: 49,
},
} {
if err := mediaEngine.RegisterCodec(codec, webrtc.RTPCodecTypeVideo); err != nil {
return err
}
}
// Interceptor registry
interceptorRegistry := &interceptor.Registry{}
// Use default set
err = webrtc.RegisterDefaultInterceptors(mediaEngine, interceptorRegistry)
if err != nil {
return err
}
// Setting engine
settingEngine := webrtc.SettingEngine{}
// New in v4, reduces CPU usage and latency when enabled
settingEngine.EnableSCTPZeroChecksum(true)
// Set the UDP port range used by WebRTC
err = settingEngine.SetEphemeralUDPPortRange(uint16(flags.WebRTCUDPStart), uint16(flags.WebRTCUDPEnd))
if err != nil {
return err
}
// Create a new API object with our customized settings
globalWebRTCAPI = webrtc.NewAPI(webrtc.WithMediaEngine(mediaEngine), webrtc.WithSettingEngine(settingEngine), webrtc.WithInterceptorRegistry(interceptorRegistry))
return nil
}
// GetWebRTCAPI returns the global WebRTC API
func GetWebRTCAPI() *webrtc.API {
return globalWebRTCAPI
}
// CreatePeerConnection sets up a new peer connection
func CreatePeerConnection(onClose func()) (*webrtc.PeerConnection, error) {
pc, err := globalWebRTCAPI.NewPeerConnection(globalWebRTCConfig)
if err != nil {
return nil, err
}
// Log connection state changes and handle failed/disconnected connections
pc.OnConnectionStateChange(func(connectionState webrtc.PeerConnectionState) {
// Close PeerConnection in cases
if connectionState == webrtc.PeerConnectionStateFailed ||
connectionState == webrtc.PeerConnectionStateDisconnected ||
connectionState == webrtc.PeerConnectionStateClosed {
err := pc.Close()
if err != nil {
log.Printf("Error closing PeerConnection: %s\n", err.Error())
}
onClose()
}
})
return pc, nil
}

72
packages/relay/internal/datachannel.go Normal file
View File

@@ -0,0 +1,72 @@
package relay
import (
"github.com/pion/webrtc/v4"
"log"
)
// NestriDataChannel is a custom data channel with callbacks
type NestriDataChannel struct {
*webrtc.DataChannel
binaryCallbacks map[string]OnMessageCallback // MessageBase type -> callback
}
// NewNestriDataChannel creates a new NestriDataChannel from *webrtc.DataChannel
func NewNestriDataChannel(dc *webrtc.DataChannel) *NestriDataChannel {
ndc := &NestriDataChannel{
DataChannel: dc,
binaryCallbacks: make(map[string]OnMessageCallback),
}
// Handler for incoming messages
ndc.OnMessage(func(msg webrtc.DataChannelMessage) {
// If string type message, ignore
if msg.IsString {
return
}
// Decode message
var base MessageBase
if err := DecodeMessage(msg.Data, &base); err != nil {
log.Printf("Failed to decode binary DataChannel message, reason: %s\n", err)
return
}
// Handle message type callback
if callback, ok := ndc.binaryCallbacks[base.PayloadType]; ok {
go callback(msg.Data)
} // TODO: Log unknown message type?
})
return ndc
}
// SendBinary sends a binary message to the data channel
func (ndc *NestriDataChannel) SendBinary(data []byte) error {
return ndc.Send(data)
}
// RegisterMessageCallback registers a callback for a given binary message type
func (ndc *NestriDataChannel) RegisterMessageCallback(msgType string, callback OnMessageCallback) {
if ndc.binaryCallbacks == nil {
ndc.binaryCallbacks = make(map[string]OnMessageCallback)
}
ndc.binaryCallbacks[msgType] = callback
}
// UnregisterMessageCallback removes the callback for a given binary message type
func (ndc *NestriDataChannel) UnregisterMessageCallback(msgType string) {
if ndc.binaryCallbacks != nil {
delete(ndc.binaryCallbacks, msgType)
}
}
// RegisterOnOpen registers a callback for the data channel opening
func (ndc *NestriDataChannel) RegisterOnOpen(callback func()) {
ndc.OnOpen(callback)
}
// RegisterOnClose registers a callback for the data channel closing
func (ndc *NestriDataChannel) RegisterOnClose(callback func()) {
ndc.OnClose(callback)
}

189
packages/relay/internal/egress.go Normal file
View File

@@ -0,0 +1,189 @@
package relay
import (
"github.com/pion/webrtc/v4"
"log"
)
func participantHandler(participant *Participant, room *Room) {
// Callback for closing PeerConnection
onPCClose := func() {
if GetFlags().Verbose {
log.Printf("Closed PeerConnection for participant: '%s'\n", participant.ID)
}
room.removeParticipantByID(participant.ID)
}
var err error
participant.PeerConnection, err = CreatePeerConnection(onPCClose)
if err != nil {
log.Printf("Failed to create PeerConnection for participant: '%s' - reason: %s\n", participant.ID, err)
return
}
// Data channel settings
settingOrdered := false
settingMaxRetransmits := uint16(0)
dc, err := participant.PeerConnection.CreateDataChannel("data", &webrtc.DataChannelInit{
Ordered: &settingOrdered,
MaxRetransmits: &settingMaxRetransmits,
})
if err != nil {
log.Printf("Failed to create data channel for participant: '%s' in room: '%s' - reason: %s\n", participant.ID, room.Name, err)
return
}
participant.DataChannel = NewNestriDataChannel(dc)
// Register channel opening handling
participant.DataChannel.RegisterOnOpen(func() {
if GetFlags().Verbose {
log.Printf("DataChannel open for participant: %s\n", participant.ID)
}
})
// Register channel closing handling
participant.DataChannel.RegisterOnClose(func() {
if GetFlags().Verbose {
log.Printf("DataChannel closed for participant: %s\n", participant.ID)
}
})
// Register text message handling
participant.DataChannel.RegisterMessageCallback("input", func(data []byte) {
// Send to room if it has a DataChannel
if room.DataChannel != nil {
// If debug mode, decode and add our timestamp, otherwise just send to room
if GetFlags().Debug {
var inputMsg MessageInput
if err = DecodeMessage(data, &inputMsg); err != nil {
log.Printf("Failed to decode input message from participant: '%s' in room: '%s' - reason: %s\n", participant.ID, room.Name, err)
return
}
inputMsg.LatencyTracker.AddTimestamp("relay_to_node")
// Encode and send
if data, err = EncodeMessage(inputMsg); err != nil {
log.Printf("Failed to encode input message for participant: '%s' in room: '%s' - reason: %s\n", participant.ID, room.Name, err)
return
}
if err = room.DataChannel.SendBinary(data); err != nil {
log.Printf("Failed to send input message to room: '%s' - reason: %s\n", room.Name, err)
}
} else {
if err = room.DataChannel.SendBinary(data); err != nil {
log.Printf("Failed to send input message to room: '%s' - reason: %s\n", room.Name, err)
}
}
}
})
participant.PeerConnection.OnICECandidate(func(candidate *webrtc.ICECandidate) {
if candidate == nil {
return
}
if GetFlags().Verbose {
log.Printf("ICE candidate for participant: '%s' in room: '%s'\n", participant.ID, room.Name)
}
err = participant.WebSocket.SendICECandidateMessageWS(candidate.ToJSON())
if err != nil {
log.Printf("Failed to send ICE candidate for participant: '%s' in room: '%s' - reason: %s\n", participant.ID, room.Name, err)
}
})
iceHolder := make([]webrtc.ICECandidateInit, 0)
// ICE callback
participant.WebSocket.RegisterMessageCallback("ice", func(data []byte) {
var iceMsg MessageICECandidate
if err = DecodeMessage(data, &iceMsg); err != nil {
log.Printf("Failed to decode ICE message from participant: '%s' in room: '%s' - reason: %s\n", participant.ID, room.Name, err)
return
}
candidate := webrtc.ICECandidateInit{
Candidate: iceMsg.Candidate.Candidate,
}
if participant.PeerConnection.RemoteDescription() != nil {
if err = participant.PeerConnection.AddICECandidate(candidate); err != nil {
log.Printf("Failed to add ICE candidate from participant: '%s' in room: '%s' - reason: %s\n", participant.ID, room.Name, err)
}
// Add held ICE candidates
for _, heldCandidate := range iceHolder {
if err = participant.PeerConnection.AddICECandidate(heldCandidate); err != nil {
log.Printf("Failed to add held ICE candidate from participant: '%s' in room: '%s' - reason: %s\n", participant.ID, room.Name, err)
}
}
iceHolder = nil
} else {
iceHolder = append(iceHolder, candidate)
}
})
// SDP answer callback
participant.WebSocket.RegisterMessageCallback("sdp", func(data []byte) {
var sdpMsg MessageSDP
if err = DecodeMessage(data, &sdpMsg); err != nil {
log.Printf("Failed to decode SDP message from participant: '%s' in room: '%s' - reason: %s\n", participant.ID, room.Name, err)
return
}
handleParticipantSDP(participant, sdpMsg)
})
// Log callback
participant.WebSocket.RegisterMessageCallback("log", func(data []byte) {
var logMsg MessageLog
if err = DecodeMessage(data, &logMsg); err != nil {
log.Printf("Failed to decode log message from participant: '%s' in room: '%s' - reason: %s\n", participant.ID, room.Name, err)
return
}
// TODO: Handle log message sending to metrics server
})
// Metrics callback
participant.WebSocket.RegisterMessageCallback("metrics", func(data []byte) {
// Ignore for now
})
participant.WebSocket.RegisterOnClose(func() {
if GetFlags().Verbose {
log.Printf("WebSocket closed for participant: '%s' in room: '%s'\n", participant.ID, room.Name)
}
// Remove from Room
room.removeParticipantByID(participant.ID)
})
log.Printf("Participant: '%s' in room: '%s' is now ready, sending an OK\n", participant.ID, room.Name)
if err = participant.WebSocket.SendAnswerMessageWS(AnswerOK); err != nil {
log.Printf("Failed to send OK answer for participant: '%s' in room: '%s' - reason: %s\n", participant.ID, room.Name, err)
}
// If room is already online, send also offer
if room.Online {
if room.AudioTrack != nil {
if err = participant.addTrack(&room.AudioTrack); err != nil {
log.Printf("Failed to add audio track for participant: '%s' in room: '%s' - reason: %s\n", participant.ID, room.Name, err)
}
}
if room.VideoTrack != nil {
if err = participant.addTrack(&room.VideoTrack); err != nil {
log.Printf("Failed to add video track for participant: '%s' in room: '%s' - reason: %s\n", participant.ID, room.Name, err)
}
}
if err = participant.signalOffer(); err != nil {
log.Printf("Failed to signal offer for participant: '%s' in room: '%s' - reason: %s\n", participant.ID, room.Name, err)
}
}
}
// SDP answer handler for participants
func handleParticipantSDP(participant *Participant, answerMsg MessageSDP) {
// Get SDP offer
sdpAnswer := answerMsg.SDP.SDP
// Set remote description
err := participant.PeerConnection.SetRemoteDescription(webrtc.SessionDescription{
Type: webrtc.SDPTypeAnswer,
SDP: sdpAnswer,
})
if err != nil {
log.Printf("Failed to set remote description for participant: '%s' - reason: %s\n", participant.ID, err)
}
}

82
packages/relay/internal/flags.go Normal file
View File

@@ -0,0 +1,82 @@
package relay
import (
"flag"
"log"
"os"
"strconv"
"github.com/pion/webrtc/v4"
)
var globalFlags *Flags
type Flags struct {
Verbose bool
Debug bool
EndpointPort int
WebRTCUDPStart int
WebRTCUDPEnd int
STUNServer string
}
func (flags *Flags) DebugLog() {
log.Println("Relay Flags:")
log.Println("> Verbose: ", flags.Verbose)
log.Println("> Debug: ", flags.Debug)
log.Println("> Endpoint Port: ", flags.EndpointPort)
log.Println("> WebRTC UDP Range Start: ", flags.WebRTCUDPStart)
log.Println("> WebRTC UDP Range End: ", flags.WebRTCUDPEnd)
log.Println("> WebRTC STUN Server: ", flags.STUNServer)
}
func getEnvAsInt(name string, defaultVal int) int {
valueStr := os.Getenv(name)
if value, err := strconv.Atoi(valueStr); err != nil {
return defaultVal
} else {
return value
}
}
func getEnvAsBool(name string, defaultVal bool) bool {
valueStr := os.Getenv(name)
val, err := strconv.ParseBool(valueStr)
if err != nil {
return defaultVal
}
return val
}
func getEnvAsString(name string, defaultVal string) string {
valueStr := os.Getenv(name)
if len(valueStr) == 0 {
return defaultVal
}
return valueStr
}
func InitFlags() {
// Create Flags struct
globalFlags = &Flags{}
// Get flags
flag.BoolVar(&globalFlags.Verbose, "verbose", getEnvAsBool("VERBOSE", false), "Verbose mode")
flag.BoolVar(&globalFlags.Debug, "debug", getEnvAsBool("DEBUG", false), "Debug mode")
flag.IntVar(&globalFlags.EndpointPort, "endpointPort", getEnvAsInt("ENDPOINT_PORT", 8088), "HTTP endpoint port")
flag.IntVar(&globalFlags.WebRTCUDPStart, "webrtcUDPStart", getEnvAsInt("WEBRTC_UDP_START", 10000), "WebRTC UDP port range start")
flag.IntVar(&globalFlags.WebRTCUDPEnd, "webrtcUDPEnd", getEnvAsInt("WEBRTC_UDP_END", 20000), "WebRTC UDP port range end")
flag.StringVar(&globalFlags.STUNServer, "stunServer", getEnvAsString("STUN_SERVER", "stun.l.google.com:19302"), "WebRTC STUN server")
// Parse flags
flag.Parse()
// ICE STUN servers
globalWebRTCConfig.ICEServers = []webrtc.ICEServer{
{
URLs: []string{"stun:" + globalFlags.STUNServer},
},
}
}
func GetFlags() *Flags {
return globalFlags
}

123
packages/relay/internal/http.go Normal file
View File

@@ -0,0 +1,123 @@
package relay
import (
"github.com/gorilla/websocket"
"log"
"net/http"
"strconv"
)
var httpMux *http.ServeMux
func InitHTTPEndpoint() {
// Create HTTP mux which serves our WS endpoint
httpMux = http.NewServeMux()
// Endpoints themselves
httpMux.Handle("/", http.NotFoundHandler())
httpMux.HandleFunc("/api/ws/{roomName}", corsAnyHandler(wsHandler))
// Get our serving port
port := GetFlags().EndpointPort
// Log and start the endpoint server
log.Println("Starting HTTP endpoint server on :", strconv.Itoa(port))
go func() {
log.Fatal((&http.Server{
Handler: httpMux,
Addr: ":" + strconv.Itoa(port),
}).ListenAndServe())
}()
}
// logHTTPError logs (if verbose) and sends an error code to requester
func logHTTPError(w http.ResponseWriter, err string, code int) {
if GetFlags().Verbose {
log.Println(err)
}
http.Error(w, err, code)
}
// corsAnyHandler allows any origin to access the endpoint
func corsAnyHandler(next func(w http.ResponseWriter, r *http.Request)) http.HandlerFunc {
return func(res http.ResponseWriter, req *http.Request) {
// Allow all origins
res.Header().Set("Access-Control-Allow-Origin", "*")
res.Header().Set("Access-Control-Allow-Methods", "*")
res.Header().Set("Access-Control-Allow-Headers", "*")
if req.Method != http.MethodOptions {
next(res, req)
}
}
}
// wsHandler is the handler for the /api/ws/{roomName} endpoint
func wsHandler(w http.ResponseWriter, r *http.Request) {
// Get given room name now
roomName := r.PathValue("roomName")
if len(roomName) <= 0 {
logHTTPError(w, "no room name given", http.StatusBadRequest)
return
}
// Get or create room in any case
room := GetOrCreateRoom(roomName)
// Upgrade to WebSocket
upgrader := websocket.Upgrader{
CheckOrigin: func(r *http.Request) bool {
return true
},
}
wsConn, err := upgrader.Upgrade(w, r, nil)
if err != nil {
logHTTPError(w, err.Error(), http.StatusInternalServerError)
return
}
// Create SafeWebSocket
ws := NewSafeWebSocket(wsConn)
// Assign message handler for join request
ws.RegisterMessageCallback("join", func(data []byte) {
var joinMsg MessageJoin
if err = DecodeMessage(data, &joinMsg); err != nil {
log.Printf("Failed to decode join message: %s\n", err)
return
}
if GetFlags().Verbose {
log.Printf("Join request for room: '%s' from: '%s'\n", room.Name, joinMsg.JoinerType.String())
}
// Handle join request, depending if it's from ingest/node or participant/client
switch joinMsg.JoinerType {
case JoinerNode:
// If room already online, send InUse answer
if room.Online {
if err = ws.SendAnswerMessageWS(AnswerInUse); err != nil {
log.Printf("Failed to send InUse answer for Room: '%s' - reason: %s\n", room.Name, err)
}
return
}
room.assignWebSocket(ws)
go ingestHandler(room)
case JoinerClient:
// Create participant and add to room regardless of online status
participant := NewParticipant(ws)
room.addParticipant(participant)
// If room not online, send Offline answer
if !room.Online {
if err = ws.SendAnswerMessageWS(AnswerOffline); err != nil {
log.Printf("Failed to send Offline answer for Room: '%s' - reason: %s\n", room.Name, err)
}
}
go participantHandler(participant, room)
default:
log.Printf("Unknown joiner type: %d\n", joinMsg.JoinerType)
}
// Unregister ourselves, if something happens on the other side they should just reconnect?
ws.UnregisterMessageCallback("join")
})
}

251
packages/relay/internal/ingest.go Normal file
View File

@@ -0,0 +1,251 @@
package relay
import (
"errors"
"fmt"
"github.com/pion/webrtc/v4"
"io"
"log"
"strings"
)
func ingestHandler(room *Room) {
// Callback for closing PeerConnection
onPCClose := func() {
if GetFlags().Verbose {
log.Printf("Closed PeerConnection for room: '%s'\n", room.Name)
}
room.Online = false
DeleteRoomIfEmpty(room)
}
var err error
room.PeerConnection, err = CreatePeerConnection(onPCClose)
if err != nil {
log.Printf("Failed to create PeerConnection for room: '%s' - reason: %s\n", room.Name, err)
return
}
room.PeerConnection.OnTrack(func(remoteTrack *webrtc.TrackRemote, receiver *webrtc.RTPReceiver) {
var localTrack *webrtc.TrackLocalStaticRTP
if remoteTrack.Kind() == webrtc.RTPCodecTypeVideo {
if GetFlags().Verbose {
log.Printf("Received video track for room: '%s'\n", room.Name)
}
localTrack, err = webrtc.NewTrackLocalStaticRTP(remoteTrack.Codec().RTPCodecCapability, "video", fmt.Sprint("nestri-", room.Name))
if err != nil {
log.Printf("Failed to create local video track for room: '%s' - reason: %s\n", room.Name, err)
return
}
room.VideoTrack = localTrack
} else if remoteTrack.Kind() == webrtc.RTPCodecTypeAudio {
if GetFlags().Verbose {
log.Printf("Received audio track for room: '%s'\n", room.Name)
}
localTrack, err = webrtc.NewTrackLocalStaticRTP(remoteTrack.Codec().RTPCodecCapability, "audio", fmt.Sprint("nestri-", room.Name))
if err != nil {
log.Printf("Failed to create local audio track for room: '%s' - reason: %s\n", room.Name, err)
return
}
room.AudioTrack = localTrack
}
// If both audio and video tracks are set, set online state
if room.AudioTrack != nil && room.VideoTrack != nil {
room.Online = true
if GetFlags().Verbose {
log.Printf("Room online and receiving: '%s' - signaling participants\n", room.Name)
}
room.signalParticipantsWithTracks()
}
rtpBuffer := make([]byte, 1400)
for {
read, _, err := remoteTrack.Read(rtpBuffer)
if err != nil {
// EOF is expected when stopping room
if !errors.Is(err, io.EOF) {
log.Printf("RTP read error from room: '%s' - reason: %s\n", room.Name, err)
}
break
}
_, err = localTrack.Write(rtpBuffer[:read])
if err != nil && !errors.Is(err, io.ErrClosedPipe) {
log.Printf("Failed to write RTP to local track for room: '%s' - reason: %s\n", room.Name, err)
break
}
}
if remoteTrack.Kind() == webrtc.RTPCodecTypeVideo {
room.VideoTrack = nil
} else if remoteTrack.Kind() == webrtc.RTPCodecTypeAudio {
room.AudioTrack = nil
}
if room.VideoTrack == nil && room.AudioTrack == nil {
room.Online = false
if GetFlags().Verbose {
log.Printf("Room offline and not receiving: '%s'\n", room.Name)
}
// Signal participants of room offline
room.signalParticipantsOffline()
DeleteRoomIfEmpty(room)
}
})
room.PeerConnection.OnDataChannel(func(dc *webrtc.DataChannel) {
room.DataChannel = NewNestriDataChannel(dc)
if GetFlags().Verbose {
log.Printf("New DataChannel for room: '%s' - '%s'\n", room.Name, room.DataChannel.Label())
}
// Register channel opening handling
room.DataChannel.RegisterOnOpen(func() {
if GetFlags().Verbose {
log.Printf("DataChannel for room: '%s' - '%s' open\n", room.Name, room.DataChannel.Label())
}
})
room.DataChannel.OnClose(func() {
if GetFlags().Verbose {
log.Printf("DataChannel for room: '%s' - '%s' closed\n", room.Name, room.DataChannel.Label())
}
})
// We do not handle any messages from ingest via DataChannel yet
})
room.PeerConnection.OnICECandidate(func(candidate *webrtc.ICECandidate) {
if candidate == nil {
return
}
if GetFlags().Verbose {
log.Printf("ICE candidate for room: '%s'\n", room.Name)
}
err = room.WebSocket.SendICECandidateMessageWS(candidate.ToJSON())
if err != nil {
log.Printf("Failed to send ICE candidate for room: '%s' - reason: %s\n", room.Name, err)
}
})
iceHolder := make([]webrtc.ICECandidateInit, 0)
// ICE callback
room.WebSocket.RegisterMessageCallback("ice", func(data []byte) {
var iceMsg MessageICECandidate
if err = DecodeMessage(data, &iceMsg); err != nil {
log.Printf("Failed to decode ICE candidate message from ingest for room: '%s' - reason: %s\n", room.Name, err)
return
}
candidate := webrtc.ICECandidateInit{
Candidate: iceMsg.Candidate.Candidate,
}
if room.PeerConnection != nil {
// If remote isn't set yet, store ICE candidates
if room.PeerConnection.RemoteDescription() != nil {
if err = room.PeerConnection.AddICECandidate(candidate); err != nil {
log.Printf("Failed to add ICE candidate for room: '%s' - reason: %s\n", room.Name, err)
}
// Add any held ICE candidates
for _, heldCandidate := range iceHolder {
if err = room.PeerConnection.AddICECandidate(heldCandidate); err != nil {
log.Printf("Failed to add held ICE candidate for room: '%s' - reason: %s\n", room.Name, err)
}
}
iceHolder = nil
} else {
iceHolder = append(iceHolder, candidate)
}
} else {
log.Printf("ICE candidate received before PeerConnection for room: '%s'\n", room.Name)
}
})
// SDP offer callback
room.WebSocket.RegisterMessageCallback("sdp", func(data []byte) {
var sdpMsg MessageSDP
if err = DecodeMessage(data, &sdpMsg); err != nil {
log.Printf("Failed to decode SDP message from ingest for room: '%s' - reason: %s\n", room.Name, err)
return
}
answer := handleIngestSDP(room, sdpMsg)
if answer != nil {
if err = room.WebSocket.SendSDPMessageWS(*answer); err != nil {
log.Printf("Failed to send SDP answer to ingest for room: '%s' - reason: %s\n", room.Name, err)
}
} else {
log.Printf("Failed to handle SDP message from ingest for room: '%s'\n", room.Name)
}
})
// Log callback
room.WebSocket.RegisterMessageCallback("log", func(data []byte) {
var logMsg MessageLog
if err = DecodeMessage(data, &logMsg); err != nil {
log.Printf("Failed to decode log message from ingest for room: '%s' - reason: %s\n", room.Name, err)
return
}
// TODO: Handle log message sending to metrics server
})
// Metrics callback
room.WebSocket.RegisterMessageCallback("metrics", func(data []byte) {
var metricsMsg MessageMetrics
if err = DecodeMessage(data, &metricsMsg); err != nil {
log.Printf("Failed to decode metrics message from ingest for room: '%s' - reason: %s\n", room.Name, err)
return
}
// TODO: Handle metrics message sending to metrics server
})
room.WebSocket.RegisterOnClose(func() {
// If PeerConnection is not open or does not exist, delete room
if (room.PeerConnection != nil && room.PeerConnection.ConnectionState() != webrtc.PeerConnectionStateConnected) ||
room.PeerConnection == nil {
DeleteRoomIfEmpty(room)
}
})
log.Printf("Room: '%s' is ready, sending an OK\n", room.Name)
if err = room.WebSocket.SendAnswerMessageWS(AnswerOK); err != nil {
log.Printf("Failed to send OK answer for room: '%s' - reason: %s\n", room.Name, err)
}
}
// SDP offer handler, returns SDP answer
func handleIngestSDP(room *Room, offerMsg MessageSDP) *webrtc.SessionDescription {
var err error
// Get SDP offer
sdpOffer := offerMsg.SDP.SDP
// Modify SDP offer to remove opus "sprop-maxcapturerate=24000" (fixes opus bad quality issue, present in GStreamer)
sdpOffer = strings.Replace(sdpOffer, ";sprop-maxcapturerate=24000", "", -1)
// Set new remote description
err = room.PeerConnection.SetRemoteDescription(webrtc.SessionDescription{
Type: webrtc.SDPTypeOffer,
SDP: sdpOffer,
})
if err != nil {
log.Printf("Failed to set remote description for room: '%s' - reason: %s\n", room.Name, err)
return nil
}
// Create SDP answer
answer, err := room.PeerConnection.CreateAnswer(nil)
if err != nil {
log.Printf("Failed to create SDP answer for room: '%s' - reason: %s\n", room.Name, err)
return nil
}
// Set local description
err = room.PeerConnection.SetLocalDescription(answer)
if err != nil {
log.Printf("Failed to set local description for room: '%s' - reason: %s\n", room.Name, err)
return nil
}
return &answer
}

114
packages/relay/internal/latency.go Normal file
View File

@@ -0,0 +1,114 @@
package relay
import (
"fmt"
"time"
)
type TimestampEntry struct {
Stage string `json:"stage"`
Time string `json:"time"` // ISO 8601 string
}
// LatencyTracker provides a generic structure for measuring time taken at various stages in message processing.
// It can be embedded in message structs for tracking the flow of data and calculating round-trip latency.
type LatencyTracker struct {
SequenceID string `json:"sequence_id"`
Timestamps []TimestampEntry `json:"timestamps"`
Metadata map[string]string `json:"metadata,omitempty"`
}
// NewLatencyTracker initializes a new LatencyTracker with the given sequence ID
func NewLatencyTracker(sequenceID string) *LatencyTracker {
return &LatencyTracker{
SequenceID: sequenceID,
Timestamps: make([]TimestampEntry, 0),
Metadata: make(map[string]string),
}
}
// AddTimestamp adds a new timestamp for a specific stage
func (lt *LatencyTracker) AddTimestamp(stage string) {
lt.Timestamps = append(lt.Timestamps, TimestampEntry{
Stage: stage,
// Ensure extremely precise UTC RFC3339 timestamps (down to nanoseconds)
Time: time.Now().UTC().Format(time.RFC3339Nano),
})
}
// TotalLatency calculates the total latency from the earliest to the latest timestamp
func (lt *LatencyTracker) TotalLatency() (int64, error) {
if len(lt.Timestamps) < 2 {
return 0, nil // Not enough timestamps to calculate latency
}
var earliest, latest time.Time
for _, ts := range lt.Timestamps {
t, err := time.Parse(time.RFC3339, ts.Time)
if err != nil {
return 0, err
}
if earliest.IsZero() || t.Before(earliest) {
earliest = t
}
if latest.IsZero() || t.After(latest) {
latest = t
}
}
return latest.Sub(earliest).Milliseconds(), nil
}
// PainPoints returns a list of stages where the duration exceeds the given threshold.
func (lt *LatencyTracker) PainPoints(threshold time.Duration) []string {
var painPoints []string
var lastStage string
var lastTime time.Time
for _, ts := range lt.Timestamps {
stage := ts.Stage
t := ts.Time
if lastStage == "" {
lastStage = stage
lastTime, _ = time.Parse(time.RFC3339, t)
continue
}
currentTime, _ := time.Parse(time.RFC3339, t)
if currentTime.Sub(lastTime) > threshold {
painPoints = append(painPoints, fmt.Sprintf("%s -> %s", lastStage, stage))
}
lastStage = stage
lastTime = currentTime
}
return painPoints
}
// StageLatency calculates the time taken between two specific stages.
func (lt *LatencyTracker) StageLatency(startStage, endStage string) (time.Duration, error) {
startTime, endTime := "", ""
for _, ts := range lt.Timestamps {
if ts.Stage == startStage {
startTime = ts.Time
}
if ts.Stage == endStage {
endTime = ts.Time
}
}
if startTime == "" || endTime == "" {
return 0, fmt.Errorf("missing timestamps for stages: %s -> %s", startStage, endStage)
}
start, err := time.Parse(time.RFC3339, startTime)
if err != nil {
return 0, err
}
end, err := time.Parse(time.RFC3339, endTime)
if err != nil {
return 0, err
}
return end.Sub(start), nil
}

227
packages/relay/internal/messages.go Normal file
View File

@@ -0,0 +1,227 @@
package relay
import (
"bytes"
"compress/gzip"
"encoding/json"
"fmt"
"github.com/pion/webrtc/v4"
"time"
)
// OnMessageCallback is a callback for binary messages of given type
type OnMessageCallback func(data []byte)
// MessageBase is the base type for WS/DC messages.
type MessageBase struct {
PayloadType string `json:"payload_type"`
LatencyTracker LatencyTracker `json:"latency_tracker,omitempty"`
}
// MessageInput represents an input message.
type MessageInput struct {
MessageBase
Data string `json:"data"`
}
// MessageLog represents a log message.
type MessageLog struct {
MessageBase
Level string `json:"level"`
Message string `json:"message"`
Time string `json:"time"`
}
// MessageMetrics represents a metrics/heartbeat message.
type MessageMetrics struct {
MessageBase
UsageCPU float64 `json:"usage_cpu"`
UsageMemory float64 `json:"usage_memory"`
Uptime uint64 `json:"uptime"`
PipelineLatency float64 `json:"pipeline_latency"`
}
// MessageICECandidate represents an ICE candidate message.
type MessageICECandidate struct {
MessageBase
Candidate webrtc.ICECandidateInit `json:"candidate"`
}
// MessageSDP represents an SDP message.
type MessageSDP struct {
MessageBase
SDP webrtc.SessionDescription `json:"sdp"`
}
// JoinerType is an enum for the type of incoming room joiner
type JoinerType int
const (
JoinerNode JoinerType = iota
JoinerClient
)
func (jt *JoinerType) String() string {
switch *jt {
case JoinerNode:
return "node"
case JoinerClient:
return "client"
default:
return "unknown"
}
}
// MessageJoin is used to tell us that either participant or ingest wants to join the room
type MessageJoin struct {
MessageBase
JoinerType JoinerType `json:"joiner_type"`
}
// AnswerType is an enum for the type of answer, signaling Room state for a joiner
type AnswerType int
const (
AnswerOffline AnswerType = iota // For participant/client, when the room is offline without stream
AnswerInUse // For ingest/node joiner, when the room is already in use by another ingest/node
AnswerOK // For both, when the join request is handled successfully
)
// MessageAnswer is used to send the answer to a join request
type MessageAnswer struct {
MessageBase
AnswerType AnswerType `json:"answer_type"`
}
// EncodeMessage encodes a message to be sent with gzip compression
func EncodeMessage(msg interface{}) ([]byte, error) {
// Marshal the message to JSON
data, err := json.Marshal(msg)
if err != nil {
return nil, fmt.Errorf("failed to encode message: %w", err)
}
// Gzip compress the JSON
var compressedData bytes.Buffer
writer := gzip.NewWriter(&compressedData)
_, err = writer.Write(data)
if err != nil {
return nil, fmt.Errorf("failed to compress message: %w", err)
}
if err := writer.Close(); err != nil {
return nil, fmt.Errorf("failed to finalize compression: %w", err)
}
return compressedData.Bytes(), nil
}
// DecodeMessage decodes a message received with gzip decompression
func DecodeMessage(data []byte, target interface{}) error {
// Gzip decompress the data
reader, err := gzip.NewReader(bytes.NewReader(data))
if err != nil {
return fmt.Errorf("failed to initialize decompression: %w", err)
}
defer func(reader *gzip.Reader) {
if err = reader.Close(); err != nil {
fmt.Printf("failed to close reader: %v\n", err)
}
}(reader)
// Decode the JSON
err = json.NewDecoder(reader).Decode(target)
if err != nil {
return fmt.Errorf("failed to decode message: %w", err)
}
return nil
}
// SendLogMessageWS sends a log message to the given WebSocket connection.
func (ws *SafeWebSocket) SendLogMessageWS(level, message string) error {
msg := MessageLog{
MessageBase: MessageBase{PayloadType: "log"},
Level: level,
Message: message,
Time: time.Now().Format(time.RFC3339),
}
encoded, err := EncodeMessage(msg)
if err != nil {
return fmt.Errorf("failed to encode log message: %w", err)
}
return ws.SendBinary(encoded)
}
// SendMetricsMessageWS sends a metrics message to the given WebSocket connection.
func (ws *SafeWebSocket) SendMetricsMessageWS(usageCPU, usageMemory float64, uptime uint64, pipelineLatency float64) error {
msg := MessageMetrics{
MessageBase: MessageBase{PayloadType: "metrics"},
UsageCPU: usageCPU,
UsageMemory: usageMemory,
Uptime: uptime,
PipelineLatency: pipelineLatency,
}
encoded, err := EncodeMessage(msg)
if err != nil {
return fmt.Errorf("failed to encode metrics message: %w", err)
}
return ws.SendBinary(encoded)
}
// SendICECandidateMessageWS sends an ICE candidate message to the given WebSocket connection.
func (ws *SafeWebSocket) SendICECandidateMessageWS(candidate webrtc.ICECandidateInit) error {
msg := MessageICECandidate{
MessageBase: MessageBase{PayloadType: "ice"},
Candidate: candidate,
}
encoded, err := EncodeMessage(msg)
if err != nil {
return fmt.Errorf("failed to encode ICE candidate message: %w", err)
}
return ws.SendBinary(encoded)
}
// SendSDPMessageWS sends an SDP message to the given WebSocket connection.
func (ws *SafeWebSocket) SendSDPMessageWS(sdp webrtc.SessionDescription) error {
msg := MessageSDP{
MessageBase: MessageBase{PayloadType: "sdp"},
SDP: sdp,
}
encoded, err := EncodeMessage(msg)
if err != nil {
return fmt.Errorf("failed to encode SDP message: %w", err)
}
return ws.SendBinary(encoded)
}
// SendAnswerMessageWS sends an answer message to the given WebSocket connection.
func (ws *SafeWebSocket) SendAnswerMessageWS(answer AnswerType) error {
msg := MessageAnswer{
MessageBase: MessageBase{PayloadType: "answer"},
AnswerType: answer,
}
encoded, err := EncodeMessage(msg)
if err != nil {
return fmt.Errorf("failed to encode answer message: %w", err)
}
return ws.SendBinary(encoded)
}
// SendInputMessageDC sends an input message to the given DataChannel connection.
func (ndc *NestriDataChannel) SendInputMessageDC(data string) error {
msg := MessageInput{
MessageBase: MessageBase{PayloadType: "input"},
Data: data,
}
encoded, err := EncodeMessage(msg)
if err != nil {
return fmt.Errorf("failed to encode input message: %w", err)
}
return ndc.SendBinary(encoded)
}

69
packages/relay/internal/participant.go Normal file
View File

@@ -0,0 +1,69 @@
package relay
import (
"fmt"
"github.com/google/uuid"
"github.com/pion/webrtc/v4"
"math/rand"
)
type Participant struct {
ID uuid.UUID //< Internal IDs are useful to keeping unique internal track and not have conflicts later
Name string
WebSocket *SafeWebSocket
PeerConnection *webrtc.PeerConnection
DataChannel *NestriDataChannel
}
func NewParticipant(ws *SafeWebSocket) *Participant {
return &Participant{
ID: uuid.New(),
Name: createRandomName(),
WebSocket: ws,
}
}
func (vw *Participant) addTrack(trackLocal *webrtc.TrackLocal) error {
rtpSender, err := vw.PeerConnection.AddTrack(*trackLocal)
if err != nil {
return err
}
go func() {
rtcpBuffer := make([]byte, 1400)
for {
if _, _, rtcpErr := rtpSender.Read(rtcpBuffer); rtcpErr != nil {
return
}
}
}()
return nil
}
func (vw *Participant) signalOffer() error {
if vw.PeerConnection == nil {
return fmt.Errorf("peer connection is nil for participant: '%s' - cannot signal offer", vw.ID)
}
offer, err := vw.PeerConnection.CreateOffer(nil)
if err != nil {
return err
}
err = vw.PeerConnection.SetLocalDescription(offer)
if err != nil {
return err
}
return vw.WebSocket.SendSDPMessageWS(offer)
}
var namesFirst = []string{"Happy", "Sad", "Angry", "Calm", "Excited", "Bored", "Confused", "Confident", "Curious", "Depressed", "Disappointed", "Embarrassed", "Energetic", "Fearful", "Frustrated", "Glad", "Guilty", "Hopeful", "Impatient", "Jealous", "Lonely", "Motivated", "Nervous", "Optimistic", "Pessimistic", "Proud", "Relaxed", "Shy", "Stressed", "Surprised", "Tired", "Worried"}
var namesSecond = []string{"Dragon", "Unicorn", "Troll", "Goblin", "Elf", "Dwarf", "Ogre", "Gnome", "Mermaid", "Siren", "Vampire", "Ghoul", "Werewolf", "Minotaur", "Centaur", "Griffin", "Phoenix", "Wyvern", "Hydra", "Kraken"}
func createRandomName() string {
randomFirst := namesFirst[rand.Intn(len(namesFirst))]
randomSecond := namesSecond[rand.Intn(len(namesSecond))]
return randomFirst + " " + randomSecond
}

179
packages/relay/internal/room.go Normal file
View File

@@ -0,0 +1,179 @@
package relay
import (
"github.com/google/uuid"
"github.com/pion/webrtc/v4"
"log"
"sync"
)
var Rooms = make(map[uuid.UUID]*Room) //< Room ID -> Room
var RoomsMutex = sync.RWMutex{}
func GetRoomByID(id uuid.UUID) *Room {
RoomsMutex.RLock()
defer RoomsMutex.RUnlock()
if room, ok := Rooms[id]; ok {
return room
}
return nil
}
func GetRoomByName(name string) *Room {
RoomsMutex.RLock()
defer RoomsMutex.RUnlock()
for _, room := range Rooms {
if room.Name == name {
return room
}
}
return nil
}
func GetOrCreateRoom(name string) *Room {
if room := GetRoomByName(name); room != nil {
return room
}
RoomsMutex.Lock()
room := NewRoom(name)
Rooms[room.ID] = room
if GetFlags().Verbose {
log.Printf("New room: '%s'\n", room.Name)
}
RoomsMutex.Unlock()
return room
}
func DeleteRoomIfEmpty(room *Room) {
room.ParticipantsMutex.RLock()
defer room.ParticipantsMutex.RUnlock()
if !room.Online && len(room.Participants) <= 0 {
RoomsMutex.Lock()
delete(Rooms, room.ID)
RoomsMutex.Unlock()
}
}
type Room struct {
ID uuid.UUID //< Internal IDs are useful to keeping unique internal track
Name string
Online bool //< Whether the room is currently online, i.e. receiving data from a nestri-server
WebSocket *SafeWebSocket
PeerConnection *webrtc.PeerConnection
AudioTrack webrtc.TrackLocal
VideoTrack webrtc.TrackLocal
DataChannel *NestriDataChannel
Participants map[uuid.UUID]*Participant
ParticipantsMutex sync.RWMutex
}
func NewRoom(name string) *Room {
return &Room{
ID: uuid.New(),
Name: name,
Online: false,
Participants: make(map[uuid.UUID]*Participant),
}
}
// Assigns a WebSocket connection to a Room
func (r *Room) assignWebSocket(ws *SafeWebSocket) {
// If WS already assigned, warn
if r.WebSocket != nil {
log.Printf("Warning: Room '%s' already has a WebSocket assigned\n", r.Name)
}
r.WebSocket = ws
}
// Adds a Participant to a Room
func (r *Room) addParticipant(participant *Participant) {
r.ParticipantsMutex.Lock()
r.Participants[participant.ID] = participant
r.ParticipantsMutex.Unlock()
}
// Removes a Participant from a Room by participant's ID.
// If Room is offline and this is the last participant, the room is deleted
func (r *Room) removeParticipantByID(pID uuid.UUID) {
r.ParticipantsMutex.Lock()
delete(r.Participants, pID)
r.ParticipantsMutex.Unlock()
DeleteRoomIfEmpty(r)
}
// Removes a Participant from a Room by participant's name.
// If Room is offline and this is the last participant, the room is deleted
func (r *Room) removeParticipantByName(pName string) {
r.ParticipantsMutex.Lock()
for id, p := range r.Participants {
if p.Name == pName {
delete(r.Participants, id)
break
}
}
r.ParticipantsMutex.Unlock()
DeleteRoomIfEmpty(r)
}
// Signals all participants with offer and add tracks to their PeerConnections
func (r *Room) signalParticipantsWithTracks() {
r.ParticipantsMutex.RLock()
for _, participant := range r.Participants {
// Add tracks to participant's PeerConnection
if r.AudioTrack != nil {
if err := participant.addTrack(&r.AudioTrack); err != nil {
log.Printf("Failed to add audio track to participant: '%s' - reason: %s\n", participant.ID, err)
}
}
if r.VideoTrack != nil {
if err := participant.addTrack(&r.VideoTrack); err != nil {
log.Printf("Failed to add video track to participant: '%s' - reason: %s\n", participant.ID, err)
}
}
// Signal participant with offer
if err := participant.signalOffer(); err != nil {
log.Printf("Error signaling participant: %v\n", err)
}
}
r.ParticipantsMutex.RUnlock()
}
// Signals all participants that the Room is offline
func (r *Room) signalParticipantsOffline() {
r.ParticipantsMutex.RLock()
for _, participant := range r.Participants {
if err := participant.WebSocket.SendAnswerMessageWS(AnswerOffline); err != nil {
log.Printf("Failed to send Offline answer for participant: '%s' - reason: %s\n", participant.ID, err)
}
}
r.ParticipantsMutex.RUnlock()
}
// Broadcasts a message to Room's Participant's - excluding one given ID of
func (r *Room) broadcastMessage(msg webrtc.DataChannelMessage, excludeID uuid.UUID) {
r.ParticipantsMutex.RLock()
for d, participant := range r.Participants {
if participant.DataChannel != nil {
if d != excludeID { // Don't send back to the sender
if err := participant.DataChannel.SendText(string(msg.Data)); err != nil {
log.Printf("Error broadcasting to %s: %v\n", participant.Name, err)
}
}
}
}
if r.DataChannel != nil {
if err := r.DataChannel.SendText(string(msg.Data)); err != nil {
log.Printf("Error broadcasting to Room: %v\n", err)
}
}
r.ParticipantsMutex.RUnlock()
}
// Sends message to Room (nestri-server)
func (r *Room) sendToRoom(msg webrtc.DataChannelMessage) {
if r.DataChannel != nil {
if err := r.DataChannel.SendText(string(msg.Data)); err != nil {
log.Printf("Error broadcasting to Room: %v\n", err)
}
}
}

114
packages/relay/internal/websocket.go Normal file
View File

@@ -0,0 +1,114 @@
package relay
import (
"github.com/gorilla/websocket"
"log"
"sync"
)
// SafeWebSocket is a websocket with a mutex
type SafeWebSocket struct {
*websocket.Conn
sync.Mutex
binaryCallbacks map[string]OnMessageCallback // MessageBase type -> callback
}
// NewSafeWebSocket creates a new SafeWebSocket from *websocket.Conn
func NewSafeWebSocket(conn *websocket.Conn) *SafeWebSocket {
ws := &SafeWebSocket{
Conn: conn,
binaryCallbacks: make(map[string]OnMessageCallback),
}
// Launch a goroutine to handle binary messages
go func() {
for {
// Read binary message
kind, data, err := ws.Conn.ReadMessage()
if websocket.IsUnexpectedCloseError(err, websocket.CloseGoingAway, websocket.CloseAbnormalClosure, websocket.CloseNoStatusReceived) {
// If unexpected close error, break
if GetFlags().Verbose {
log.Printf("Unexpected WebSocket close error, reason: %s\n", err)
}
break
} else if websocket.IsCloseError(err, websocket.CloseNormalClosure, websocket.CloseGoingAway, websocket.CloseAbnormalClosure, websocket.CloseNoStatusReceived) {
// If closing, just break
if GetFlags().Verbose {
log.Printf("WebSocket closing\n")
}
break
} else if err != nil {
log.Printf("Failed to read WebSocket message, reason: %s\n", err)
break
}
switch kind {
case websocket.TextMessage:
// Ignore, we use binary messages
continue
case websocket.BinaryMessage:
// Decode message
var msg MessageBase
if err = DecodeMessage(data, &msg); err != nil {
log.Printf("Failed to decode binary WebSocket message, reason: %s\n", err)
continue
}
// Handle message type callback
if callback, ok := ws.binaryCallbacks[msg.PayloadType]; ok {
callback(data)
} // TODO: Log unknown message type?
default:
log.Printf("Unknown WebSocket message type: %d\n", kind)
}
}
}()
return ws
}
// SendJSON writes JSON to a websocket with a mutex
func (ws *SafeWebSocket) SendJSON(v interface{}) error {
ws.Lock()
defer ws.Unlock()
return ws.Conn.WriteJSON(v)
}
// SendBinary writes binary to a websocket with a mutex
func (ws *SafeWebSocket) SendBinary(data []byte) error {
ws.Lock()
defer ws.Unlock()
return ws.Conn.WriteMessage(websocket.BinaryMessage, data)
}
// RegisterMessageCallback sets the callback for binary message of given type
func (ws *SafeWebSocket) RegisterMessageCallback(msgType string, callback OnMessageCallback) {
ws.Lock()
defer ws.Unlock()
if ws.binaryCallbacks == nil {
ws.binaryCallbacks = make(map[string]OnMessageCallback)
}
ws.binaryCallbacks[msgType] = callback
}
// UnregisterMessageCallback removes the callback for binary message of given type
func (ws *SafeWebSocket) UnregisterMessageCallback(msgType string) {
ws.Lock()
defer ws.Unlock()
if ws.binaryCallbacks != nil {
delete(ws.binaryCallbacks, msgType)
}
}
// RegisterOnClose sets the callback for websocket closing
func (ws *SafeWebSocket) RegisterOnClose(callback func()) {
ws.SetCloseHandler(func(code int, text string) error {
// Clear our callbacks
ws.Lock()
ws.binaryCallbacks = nil
ws.Unlock()
// Call the callback
callback()
return nil
})
}

32
packages/relay/main.go Normal file
View File

@@ -0,0 +1,32 @@
package main
import (
"log"
"os"
"os/signal"
relay "relay/internal"
"syscall"
)
func main() {
var err error
stopCh := make(chan os.Signal, 1)
signal.Notify(stopCh, os.Interrupt, syscall.SIGTERM)
// Get flags and log them
relay.InitFlags()
relay.GetFlags().DebugLog()
// Init WebRTC API
err = relay.InitWebRTCAPI()
if err != nil {
log.Fatal("Failed to initialize WebRTC API: ", err)
}
// Start our HTTP endpoints
relay.InitHTTPEndpoint()
// Wait for exit signal
<-stopCh
log.Println("Shutting down gracefully by signal...")
}

12
packages/scripts/envs.sh Normal file
View File

@@ -0,0 +1,12 @@
#!/bin/bash -e
export XDG_RUNTIME_DIR=/run/user/${UID}/
export WAYLAND_DISPLAY=wayland-0
export DISPLAY=:0
export $(dbus-launch)
# Fixes freezing issue
export PROTON_NO_FSYNC=1
# Our preferred prefix
export WINEPREFIX=${USER_HOME}/.nestripfx/

277
packages/scripts/gpu_helpers.sh Normal file
View File

@@ -0,0 +1,277 @@
#!/bin/bash -e
# Various helper functions for handling available GPUs
declare -ga gpu_map
declare -gA gpu_bus_map
declare -gA gpu_card_map
declare -gA gpu_product_map
declare -gA vendor_index_map
declare -gA vendor_full_map
# Map to help get shorter vendor identifiers
declare -A vendor_keywords=(
["advanced micro devices"]='amd'
["ati"]='amd'
["amd"]='amd'
["radeon"]='amd'
["nvidia"]='nvidia'
["intel"]='intel'
)
get_gpu_info() {
# Clear out previous data
gpu_map=()
gpu_bus_map=()
gpu_card_map=()
gpu_product_map=()
vendor_index_map=()
vendor_full_map=()
local vendor=""
local product=""
local bus_info=""
local vendor_full=""
while read -r line; do
line="${line##*( )}"
if [[ "${line,,}" =~ "vendor:" ]]; then
vendor=""
vendor_full=$(echo "$line" | awk '{$1=""; print $0}' | xargs)
# Look for short vendor keyword in line
for keyword in "${!vendor_keywords[@]}"; do
if [[ "${line,,}" == *"$keyword"* ]]; then
vendor="${vendor_keywords[$keyword]}"
break
fi
done
# If no vendor keywords match, use first word
if [[ -z "$vendor" ]]; then
vendor=$(echo "$vendor_full" | awk '{print tolower($1)}')
fi
elif [[ "${line,,}" =~ "product:" ]]; then
product=$(echo "$line" | awk '{$1=""; print $0}' | xargs)
elif [[ "${line,,}" =~ "bus info:" ]]; then
bus_info=$(echo "$line" | awk '{print $3}')
fi
if [[ -n "$vendor" && -n "$product" && -n "$bus_info" && ! "${line,,}" =~ \*"-display" ]]; then
# We have gathered all GPU info necessary, store it
# Check if vendor index is being tracked
if [[ -z "${vendor_index_map[$vendor]}" ]]; then
# Start new vendor index tracking
vendor_index_map[$vendor]=0
else
# Another GPU of same vendor, increment index
vendor_index_map[$vendor]="$((vendor_index_map[$vendor] + 1))"
fi
# Resolved GPU index
local gpu_index="${vendor_index_map[$vendor]}"
local gpu_key="$vendor:$gpu_index"
# Get /dev/dri/cardN of GPU
local gpu_card=$({ ls -1d /sys/bus/pci/devices/*${bus_info#pci@}/drm/*; } 2>&1 | grep card* | grep -oP '(?<=card)\d+')
# Store info in maps
gpu_map+=("$gpu_key")
gpu_bus_map["$gpu_key"]="$bus_info"
gpu_product_map["$gpu_key"]="$product"
vendor_full_map["$gpu_key"]="$vendor_full"
if [[ -n "$gpu_card" ]]; then
gpu_card_map["$gpu_key"]="$gpu_card"
fi
# Clear values for additional GPUs
vendor=""
product=""
bus_info=""
vendor_full=""
fi
if [[ "${line,,}" =~ \*"-display" ]]; then
# New GPU found before storing, clear incomplete values to prevent mixing
vendor=""
product=""
bus_info=""
vendor_full=""
fi
done < <(sudo lshw -c video)
}
check_and_populate_gpus() {
if [[ "${#gpu_map[@]}" -eq 0 ]]; then
get_gpu_info # Gather info incase info not gathered yet
if [[ "${#gpu_map[@]}" -eq 0 ]]; then
echo "No GPUs found on this system" >&2
return 1
fi
fi
}
check_selected_gpu() {
local selected_gpu="${1,,}"
if [[ ! " ${gpu_map[*]} " =~ " $selected_gpu " ]]; then
echo "No such GPU: '$selected_gpu'" >&2
return 1
fi
echo "$selected_gpu"
}
list_available_gpus() {
if ! check_and_populate_gpus; then
return 1
fi
echo "Available GPUs:" >&2
for gpu in "${gpu_map[@]}"; do
echo " [$gpu] \"${gpu_product_map[$gpu]}\" @[${gpu_bus_map[$gpu]}]"
done
}
convert_bus_id_to_xorg() {
local bus_info="$1"
IFS=":." read -ra bus_parts <<< "${bus_info#pci@????:}" # Remove "pci@" and the following 4 characters (domain)
# Check if bus_info has the correct format (at least 3 parts after removing domain)
if [[ "${#bus_parts[@]}" -lt 3 ]]; then
echo "Invalid bus info format: $bus_info" >&2
return 1
fi
# Convert each part from hexadecimal to decimal
bus_info_xorg="PCI:"
for part in "${bus_parts[@]}"; do
bus_info_xorg+="$((16#$part)):"
done
bus_info_xorg="${bus_info_xorg%:}" # Remove the trailing colon
echo "$bus_info_xorg"
}
print_gpu_info() {
if ! check_and_populate_gpus; then
return 1
fi
local selected_gpu
if ! selected_gpu=$(check_selected_gpu "$1"); then
return 1
fi
echo "[$selected_gpu]"
echo " Vendor: ${vendor_full_map[$selected_gpu]}"
echo " Product: ${gpu_product_map[$selected_gpu]}"
echo " Bus: ${gpu_bus_map[$selected_gpu]}"
# Check if card path was found
if [[ "${gpu_card_map[$selected_gpu]}" ]]; then
echo " Card: /dev/dri/card${gpu_card_map[$selected_gpu]}"
fi
echo
}
get_gpu_vendor() {
if ! check_and_populate_gpus; then
return 1
fi
local selected_gpu
if ! selected_gpu=$(check_selected_gpu "$1"); then
return 1
fi
echo "${selected_gpu%%:*}"
}
get_gpu_vendor_full() {
if ! check_and_populate_gpus; then
return 1
fi
local selected_gpu
if ! selected_gpu=$(check_selected_gpu "$1"); then
return 1
fi
echo "${vendor_full_map[$selected_gpu]}"
}
get_gpu_index() {
if ! check_and_populate_gpus; then
return 1
fi
local selected_gpu
if ! selected_gpu=$(check_selected_gpu "$1"); then
return 1
fi
echo "${selected_gpu#*:}"
}
get_gpu_product() {
if ! check_and_populate_gpus; then
return 1
fi
local selected_gpu
if ! selected_gpu=$(check_selected_gpu "$1"); then
return 1
fi
echo "${gpu_product_map[$selected_gpu]}"
}
get_gpu_bus() {
if ! check_and_populate_gpus; then
return 1
fi
local selected_gpu
if ! selected_gpu=$(check_selected_gpu "$1"); then
return 1
fi
echo "${gpu_bus_map[$selected_gpu]}"
}
get_gpu_bus_xorg() {
if ! check_and_populate_gpus; then
return 1
fi
local selected_gpu
if ! selected_gpu=$(check_selected_gpu "$1"); then
return 1
fi
echo $(convert_bus_id_to_xorg "${gpu_bus_map[$selected_gpu]}")
}
get_gpu_card() {
if ! check_and_populate_gpus; then
return 1
fi
local selected_gpu
if ! selected_gpu=$(check_selected_gpu "$1"); then
return 1
fi
# Check if card path was found
if [[ -z "${gpu_card_map[$selected_gpu]}" ]]; then
echo "No card device found for GPU: $selected_gpu" >&2
return 1
fi
echo "/dev/dri/card${gpu_card_map[$selected_gpu]}"
}

1
packages/server/.gitignore vendored Normal file
View File

@@ -0,0 +1 @@
*.mp4

28
packages/server/Cargo.toml Normal file
View File

@@ -0,0 +1,28 @@
[package]
name = "nestri-server"
version = "0.1.0"
edition = "2021"
[[bin]]
name = "nestri-server"
path = "src/main.rs"
[dependencies]
gst.workspace = true
gst-app.workspace = true
serde = {version = "1.0.214", features = ["derive"] }
tokio = { version = "1.41.0", features = ["full"] }
clap = { version = "4.5.20", features = ["env"] }
serde_json = "1.0.132"
webrtc = "0.11.0"
regex = "1.11.1"
rand = "0.8.5"
rustls = { version = "0.23.17", features = ["ring"] }
tokio-util = "0.7.12"
flate2 = "1.0.35"
tokio-tungstenite = { version = "0.24.0", features = ["native-tls"] }
log = { version = "0.4.22", features = ["std"] }
chrono = "0.4.38"
futures-util = "0.3.31"
num-derive = "0.4.2"
num-traits = "0.2.19"

Some files were not shown because too many files have changed in this diff Show More