✨ feat: Add streaming support (#125)

This adds:
- [x] Keyboard and mouse handling on the frontend
- [x] Video and audio streaming from the backend to the frontend
- [x] Input server that works with Websockets

Update - 17/11
- [ ] Master docker container to run this
- [ ] Steam runtime
- [ ] Entrypoint.sh

---------

Co-authored-by: Kristian Ollikainen <14197772+DatCaptainHorse@users.noreply.github.com>
Co-authored-by: Kristian Ollikainen <DatCaptainHorse@users.noreply.github.com>

packages/moq/contribute/video.ts

@@ -1,3 +1,8 @@
+import { Deferred } from "../common/async"
+import type { Frame } from "../karp/frame"
+import type { Group, Track } from "../transfork"
+import { Closed } from "../transfork/error"
+
 const SUPPORTED = [
 	"avc1", // H.264
 	"hev1", // HEVC (aka h.265)
@@ -8,10 +13,55 @@ export interface EncoderSupported {
 	codecs: string[]
 }
 
+export class Packer {
+	#source: MediaStreamTrackProcessor<VideoFrame>
+	#encoder: Encoder
+
+	#data: Track
+	#current?: Group
+
+	constructor(track: MediaStreamVideoTrack, encoder: Encoder, data: Track) {
+		this.#source = new MediaStreamTrackProcessor({ track })
+		this.#encoder = encoder
+		this.#data = data
+	}
+
+	async run() {
+		const output = new WritableStream({
+			write: (chunk) => this.#write(chunk),
+			close: () => this.#close(),
+			abort: (e) => this.#close(e),
+		})
+
+		return this.#source.readable.pipeThrough(this.#encoder.frames).pipeTo(output)
+	}
+
+	#write(frame: Frame) {
+		if (!this.#current || frame.type === "key") {
+			if (this.#current) {
+				this.#current.close()
+			}
+
+			this.#current = this.#data.appendGroup()
+		}
+
+		frame.encode(this.#current)
+	}
+
+	#close(err?: unknown) {
+		const closed = Closed.from(err)
+		if (this.#current) {
+			this.#current.close(closed)
+		}
+
+		this.#data.close(closed)
+	}
+}
+
 export class Encoder {
 	#encoder!: VideoEncoder
 	#encoderConfig: VideoEncoderConfig
-	#decoderConfig?: VideoDecoderConfig
+	#decoderConfig = new Deferred<VideoDecoderConfig>()
 
 	// true if we should insert a keyframe, undefined when the encoder should decide
 	#keyframeNext: true | undefined = true
@@ -20,7 +70,7 @@ export class Encoder {
 	#keyframeCounter = 0
 
 	// Converts raw rames to encoded frames.
-	frames: TransformStream<VideoFrame, VideoDecoderConfig | EncodedVideoChunk>
+	frames: TransformStream<VideoFrame, EncodedVideoChunk>
 
 	constructor(config: VideoEncoderConfig) {
 		config.bitrateMode ??= "constant"
@@ -53,12 +103,17 @@ export class Encoder {
 		return !!res.supported
 	}
 
+	async decoderConfig(): Promise<VideoDecoderConfig> {
+		return await this.#decoderConfig.promise
+	}
+
 	#start(controller: TransformStreamDefaultController<EncodedVideoChunk>) {
 		this.#encoder = new VideoEncoder({
 			output: (frame, metadata) => {
 				this.#enqueue(controller, frame, metadata)
 			},
 			error: (err) => {
+				this.#decoderConfig.reject(err)
 				throw err
 			},
 		})
@@ -77,23 +132,22 @@ export class Encoder {
 	}
 
 	#enqueue(
-		controller: TransformStreamDefaultController<VideoDecoderConfig | EncodedVideoChunk>,
+		controller: TransformStreamDefaultController<EncodedVideoChunk>,
 		frame: EncodedVideoChunk,
 		metadata?: EncodedVideoChunkMetadata,
 	) {
-		if (!this.#decoderConfig) {
+		if (this.#decoderConfig.pending) {
 			const config = metadata?.decoderConfig
 			if (!config) throw new Error("missing decoder config")
-
-			controller.enqueue(config)
-			this.#decoderConfig = config
+			this.#decoderConfig.resolve(config)
 		}
 
 		if (frame.type === "key") {
 			this.#keyframeCounter = 0
 		} else {
 			this.#keyframeCounter += 1
-			if (this.#keyframeCounter + this.#encoder.encodeQueueSize >= 2 * this.#encoderConfig.framerate!) {
+			const framesPerGop = this.#encoderConfig.framerate ? 2 * this.#encoderConfig.framerate : 60
+			if (this.#keyframeCounter + this.#encoder.encodeQueueSize >= framesPerGop) {
 				this.#keyframeNext = true
 			}
 		}