✨ feat: Add streaming support (#125)

This adds:
- [x] Keyboard and mouse handling on the frontend
- [x] Video and audio streaming from the backend to the frontend
- [x] Input server that works with Websockets

Update - 17/11
- [ ] Master docker container to run this
- [ ] Steam runtime
- [ ] Entrypoint.sh

---------

Co-authored-by: Kristian Ollikainen <14197772+DatCaptainHorse@users.noreply.github.com>
Co-authored-by: Kristian Ollikainen <DatCaptainHorse@users.noreply.github.com>

packages/moq/karp/catalog/audio.ts

@@ -0,0 +1,20 @@
+import { type Track, decodeTrack } from "./track"
+
+export interface Audio {
+	track: Track
+	codec: string
+	sample_rate: number
+	channel_count: number
+	bitrate?: number
+}
+
+export function decodeAudio(o: unknown): o is Audio {
+	if (typeof o !== "object" || o === null) return false
+
+	const obj = o as Partial<Audio>
+	if (!decodeTrack(obj.track)) return false
+	if (typeof obj.codec !== "string") return false
+	if (typeof obj.sample_rate !== "number") return false
+	if (typeof obj.channel_count !== "number") return false
+	return true
+}

packages/moq/karp/catalog/broadcast.ts

@@ -0,0 +1,62 @@
+import * as Transfork from "../../transfork"
+import { type Audio, decodeAudio } from "./audio"
+import { type Video, decodeVideo } from "./video"
+
+export interface Broadcast {
+	path: string[]
+	video: Video[]
+	audio: Audio[]
+}
+
+export function encode(catalog: Broadcast): Uint8Array {
+	const encoder = new TextEncoder()
+	console.debug("encoding catalog", catalog)
+	const str = JSON.stringify(catalog)
+	return encoder.encode(str)
+}
+
+export function decode(path: string[], raw: Uint8Array): Broadcast {
+	const decoder = new TextDecoder()
+	const str = decoder.decode(raw)
+
+	const catalog = JSON.parse(str)
+	if (!decodeBroadcast(catalog)) {
+		console.error("invalid catalog", catalog)
+		throw new Error("invalid catalog")
+	}
+
+	catalog.path = path
+	return catalog
+}
+
+export async function fetch(connection: Transfork.Connection, path: string[]): Promise<Broadcast> {
+	const track = new Transfork.Track(path.concat("catalog.json"), 0)
+	const sub = await connection.subscribe(track)
+	try {
+		const segment = await sub.nextGroup()
+		if (!segment) throw new Error("no catalog data")
+
+		const frame = await segment.readFrame()
+		if (!frame) throw new Error("no catalog frame")
+
+		segment.close()
+		return decode(path, frame)
+	} finally {
+		sub.close()
+	}
+}
+
+export function decodeBroadcast(o: unknown): o is Broadcast {
+	if (typeof o !== "object" || o === null) return false
+
+	const catalog = o as Partial<Broadcast>
+	if (catalog.audio === undefined) catalog.audio = []
+	if (!Array.isArray(catalog.audio)) return false
+	if (!catalog.audio.every((track: unknown) => decodeAudio(track))) return false
+
+	if (catalog.video === undefined) catalog.video = []
+	if (!Array.isArray(catalog.video)) return false
+	if (!catalog.video.every((track: unknown) => decodeVideo(track))) return false
+
+	return true
+}

packages/moq/karp/catalog/index.ts

@@ -0,0 +1,7 @@
+import type { Audio } from "./audio"
+import { type Broadcast, decode, encode, fetch } from "./broadcast"
+import type { Track } from "./track"
+import type { Video } from "./video"
+
+export type { Audio, Video, Track, Broadcast }
+export { encode, decode, fetch }

packages/moq/karp/catalog/track.ts

@@ -0,0 +1,15 @@
+export type GroupOrder = "desc" | "asc"
+
+export interface Track {
+	name: string
+	priority: number
+}
+
+export function decodeTrack(o: unknown): o is Track {
+	if (typeof o !== "object" || o === null) return false
+
+	const obj = o as Partial<Track>
+	if (typeof obj.name !== "string") return false
+	if (typeof obj.priority !== "number") return false
+	return true
+}

packages/moq/karp/catalog/video.ts

@@ -0,0 +1,29 @@
+import * as Hex from "../../common/hex"
+import { type Track, decodeTrack } from "./track"
+
+export interface Video {
+	track: Track
+	codec: string
+	description?: Uint8Array
+	bitrate?: number
+	frame_rate?: number
+	resolution: Dimensions
+}
+
+export interface Dimensions {
+	width: number
+	height: number
+}
+
+export function decodeVideo(o: unknown): o is Video {
+	if (typeof o !== "object" || o === null) return false
+
+	const obj = o as Partial<Video>
+	if (!decodeTrack(obj.track)) return false
+	if (typeof obj.codec !== "string") return false
+	if (typeof obj.description !== "string") return false
+
+	obj.description = Hex.decode(obj.description)
+
+	return true
+}

packages/moq/karp/frame.ts

@@ -0,0 +1,64 @@
+import type { Group, GroupReader } from "../transfork/model"
+import { setVint62 } from "../transfork/stream"
+
+export type FrameType = "key" | "delta"
+
+export class Frame {
+	type: FrameType
+	timestamp: number
+	data: Uint8Array
+
+	constructor(type: FrameType, timestamp: number, data: Uint8Array) {
+		this.type = type
+		this.timestamp = timestamp
+		this.data = data
+	}
+
+	static async decode(group: GroupReader): Promise<Frame | undefined> {
+		const kind = group.index === 0 ? "key" : "delta"
+		const payload = await group.readFrame()
+		if (!payload) {
+			return undefined
+		}
+
+		const [timestamp, data] = decode_timestamp(payload)
+		return new Frame(kind, timestamp, data)
+	}
+
+	encode(group: Group) {
+		if ((group.length === 0) !== (this.type === "key")) {
+			throw new Error(`invalid ${this.type} position`)
+		}
+
+		let frame = new Uint8Array(8 + this.data.byteLength)
+		const size = setVint62(frame, BigInt(this.timestamp)).byteLength
+		frame.set(this.data, size)
+		frame = new Uint8Array(frame.buffer, 0, this.data.byteLength + size)
+
+		group.writeFrame(frame)
+	}
+}
+
+// QUIC VarInt
+function decode_timestamp(buf: Uint8Array): [number, Uint8Array] {
+	const size = 1 << ((buf[0] & 0xc0) >> 6)
+
+	const view = new DataView(buf.buffer, buf.byteOffset, size)
+	const remain = new Uint8Array(buf.buffer, buf.byteOffset + size, buf.byteLength - size)
+	let v: number
+
+	if (size === 1) {
+		v = buf[0] & 0x3f
+	} else if (size === 2) {
+		v = view.getInt16(0) & 0x3fff
+	} else if (size === 4) {
+		v = view.getUint32(0) & 0x3fffffff
+	} else if (size === 8) {
+		// NOTE: Precision loss above 2^52
+		v = Number(view.getBigUint64(0) & 0x3fffffffffffffffn)
+	} else {
+		throw new Error("impossible")
+	}
+
+	return [v, remain]
+}

packages/moq/karp/tsconfig.json

@@ -0,0 +1,13 @@
+{
+	"extends": "../tsconfig.json",
+	"include": ["."],
+	"compilerOptions": {},
+	"references": [
+		{
+			"path": "../transfork"
+		},
+		{
+			"path": "../common"
+		}
+	]
+}