feat: Fully use protobuf, fix controller issues and cleanup (#305)

## Description
### First commit
Restructured protobuf schemas to make them easier to use across
languages, switched to using them in-place of JSON for signaling as
well, so there's no 2 different message formats flying about. Few new
message types to deal with clients and nestri-servers better (not final
format, may see changes still).

General cleanup of dead/unused code along some bug squashing and package
updates.

TODO for future commits:
- [x] Fix additional controllers not doing inputs (possibly needs
vimputti changes)
- [x] ~~Restructure relay protocols code a bit, to reduce bloatiness of
the currently single file for them, more code re-use.~~
- Gonna keep this PR somewhat manageable without poking more at relay..
- [x] ~~Try to fix issue where with multiple clients, static stream
content causes video to freeze until there's some movement.~~
- Was caused by server tuned profile being `throughput-performance`,
causing CPU latency to be too high.
- [x] Ponder the orb


### Second + third commit
Redid the controller polling handling and fixed multi-controller
handling in vimputti and nestri code sides. Remove some dead relay code
as well to clean up the protocol source file, we'll revisit the meshing
functionality later.

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
## Summary by CodeRabbit

* **New Features**
* Added software rendering option and MangoHud runtime config;
controller sessions now support reconnection and batched state updates
with persistent session IDs.

* **Bug Fixes**
* Restored previously-filtered NES-like gamepads so they connect
correctly.

* **Chores**
* Modernized dependencies and protobuf tooling, migrated to
protobuf-based messaging and streaming, and removed obsolete CUDA build
steps.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

---------

Co-authored-by: DatCaptainHorse <DatCaptainHorse@users.noreply.github.com>
This commit is contained in:
Kristian Ollikainen
2025-11-08 13:10:28 +02:00
committed by GitHub
parent 32341574dc
commit d87a0b35dd
50 changed files with 4413 additions and 3883 deletions

View File

@@ -1,16 +1,9 @@
import {keyCodeToLinuxEventCode} from "./codes"
import {WebRTCStream} from "./webrtc-stream";
import {LatencyTracker} from "./latency";
import {ProtoLatencyTracker, ProtoTimestampEntry} from "./proto/latency_tracker_pb";
import {timestampFromDate} from "@bufbuild/protobuf/wkt";
import {ProtoMessageBase, ProtoMessageInput, ProtoMessageInputSchema} from "./proto/messages_pb";
import {
ProtoInput,
ProtoInputSchema,
ProtoKeyDownSchema,
ProtoKeyUpSchema,
} from "./proto/types_pb";
import {create, toBinary} from "@bufbuild/protobuf";
import { keyCodeToLinuxEventCode } from "./codes";
import { WebRTCStream } from "./webrtc-stream";
import { ProtoKeyDownSchema, ProtoKeyUpSchema } from "./proto/types_pb";
import { create, toBinary } from "@bufbuild/protobuf";
import { createMessage } from "./utils";
import { ProtoMessageSchema } from "./proto/messages_pb";
interface Props {
webrtc: WebRTCStream;
@@ -24,38 +17,29 @@ export class Keyboard {
private readonly keydownListener: (e: KeyboardEvent) => void;
private readonly keyupListener: (e: KeyboardEvent) => void;
constructor({webrtc}: Props) {
constructor({ webrtc }: Props) {
this.wrtc = webrtc;
this.keydownListener = this.createKeyboardListener((e: any) => create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "keyDown",
value: create(ProtoKeyDownSchema, {
type: "KeyDown",
key: this.keyToVirtualKeyCode(e.code)
}),
}
}));
this.keyupListener = this.createKeyboardListener((e: any) => create(ProtoInputSchema, {
$typeName: "proto.ProtoInput",
inputType: {
case: "keyUp",
value: create(ProtoKeyUpSchema, {
type: "KeyUp",
key: this.keyToVirtualKeyCode(e.code)
}),
}
}));
this.run()
this.keydownListener = this.createKeyboardListener((e: any) =>
create(ProtoKeyDownSchema, {
key: this.keyToVirtualKeyCode(e.code),
}),
);
this.keyupListener = this.createKeyboardListener((e: any) =>
create(ProtoKeyUpSchema, {
key: this.keyToVirtualKeyCode(e.code),
}),
);
this.run();
}
private run() {
if (this.connected)
this.stop()
if (this.connected) this.stop();
this.connected = true
document.addEventListener("keydown", this.keydownListener, {passive: false});
document.addEventListener("keyup", this.keyupListener, {passive: false});
this.connected = true;
document.addEventListener("keydown", this.keydownListener, {
passive: false,
});
document.addEventListener("keyup", this.keyupListener, { passive: false });
}
private stop() {
@@ -65,42 +49,19 @@ export class Keyboard {
}
// Helper function to create and return mouse listeners
private createKeyboardListener(dataCreator: (e: Event) => ProtoInput): (e: Event) => void {
private createKeyboardListener(
dataCreator: (e: Event) => any,
): (e: Event) => void {
return (e: Event) => {
e.preventDefault();
e.stopPropagation();
// Prevent repeated key events from being sent (important for games)
if ((e as any).repeat)
return;
if ((e as any).repeat) return;
const data = dataCreator(e as any);
// Latency tracking
const tracker = new LatencyTracker("input-keyboard");
tracker.addTimestamp("client_send");
const protoTracker: ProtoLatencyTracker = {
$typeName: "proto.ProtoLatencyTracker",
sequenceId: tracker.sequence_id,
timestamps: [],
};
for (const t of tracker.timestamps) {
protoTracker.timestamps.push({
$typeName: "proto.ProtoTimestampEntry",
stage: t.stage,
time: timestampFromDate(t.time),
} as ProtoTimestampEntry);
}
const message: ProtoMessageInput = {
$typeName: "proto.ProtoMessageInput",
messageBase: {
$typeName: "proto.ProtoMessageBase",
payloadType: "input",
latency: protoTracker,
} as ProtoMessageBase,
data: data,
};
this.wrtc.sendBinary(toBinary(ProtoMessageInputSchema, message));
const message = createMessage(data, "input");
this.wrtc.sendBinary(toBinary(ProtoMessageSchema, message));
};
}
@@ -114,4 +75,4 @@ export class Keyboard {
if (code === "Home") return 1;
return keyCodeToLinuxEventCode[code] || undefined;
}
}
}