feat: Tidy things and improve nestri-server code behaviour

apps/www/package.json

@@ -26,6 +26,7 @@
     "lint": "eslint \"src/**/*.ts*\"",
     "preview": "qwik build preview && vite preview --open",
     "serve": "wrangler pages dev ./dist --compatibility-flags=nodejs_als",
+    "deno:serve": "deno run --allow-net --allow-read --allow-env server/entry.deno.js",
     "start": "vite --open --mode ssr",
     "qwik": "qwik"
   },

apps/www/src/routes/play/[id]/index.tsx

@@ -17,7 +17,7 @@ export default component$(() => {
       video = document.createElement("video");
       video.id = "stream-video-player";
       video.style.visibility = "hidden";
-      const webrtc = new WebRTCStream("http://localhost:8088", id, (mediaStream) => {
+      const webrtc = new WebRTCStream("https://relay.dathorse.com", id, (mediaStream) => {
         if (video && mediaStream && (video as HTMLVideoElement).srcObject === null) {
           console.log("Setting mediastream");
           (video as HTMLVideoElement).srcObject = mediaStream;
@@ -59,7 +59,7 @@ export default component$(() => {
               }
             });
 
-            document.addEventListener("pointerlockchange", (e) => {
+            document.addEventListener("pointerlockchange", () => {
               if (!canvas.value) return; // Ensure canvas is available
               // @ts-ignore
               if (document.pointerLockElement && !window.nestrimouse && !window.nestrikeyboard) {
@@ -115,9 +115,36 @@ export default component$(() => {
       onClick$={async () => {
         // @ts-ignore
         if (canvas.value && window.hasstream) {
-          // await element.value.requestFullscreen()
           // Do not use - unadjustedMovement: true - breaks input on linux
-          canvas.value.requestPointerLock();
+          await canvas.value.requestPointerLock();
+          await canvas.value.requestFullscreen()
+          if (document.fullscreenElement !== null) {
+            // @ts-ignore
+            if ('keyboard' in window.navigator && 'lock' in window.navigator.keyboard) {
+              const keys = [
+                "AltLeft",
+                "AltRight",
+                "Tab",
+                "Escape",
+                "ContextMenu",
+                "MetaLeft",
+                "MetaRight"
+              ];
+              console.log("requesting keyboard lock");
+              // @ts-ignore
+              window.navigator.keyboard.lock(keys).then(
+                () => {
+                  console.log("keyboard lock success");
+                }
+              ).catch(
+                (e: any) => {
+                  console.log("keyboard lock failed: ", e);
+                }
+              )
+            } else {
+              console.log("keyboard lock not supported, navigator is: ", window.navigator, navigator);
+            }
+          }
         }
       }}
       //TODO: go full screen, then lock on "landscape" screen-orientation on mobile

packages/input/src/keyboard.ts

@@ -1,90 +1,96 @@
 import {type Input} from "./types"
 import {keyCodeToLinuxEventCode} from "./codes"
-import {WebRTCStream, MessageInput, encodeMessage} from "./webrtc-stream";
+import {MessageInput, encodeMessage} from "./messages";
+import {WebRTCStream} from "./webrtc-stream";
+import {LatencyTracker} from "./latency";
 
 interface Props {
-    webrtc: WebRTCStream;
-    canvas: HTMLCanvasElement;
+  webrtc: WebRTCStream;
+  canvas: HTMLCanvasElement;
 }
 
 export class Keyboard {
-    protected wrtc: WebRTCStream;
-    protected canvas: HTMLCanvasElement;
-    protected connected!: boolean;
+  protected wrtc: WebRTCStream;
+  protected canvas: HTMLCanvasElement;
+  protected connected!: boolean;
 
-    // Store references to event listeners
-    private keydownListener: (e: KeyboardEvent) => void;
-    private keyupListener: (e: KeyboardEvent) => void;
+  // Store references to event listeners
+  private keydownListener: (e: KeyboardEvent) => void;
+  private keyupListener: (e: KeyboardEvent) => void;
 
-    constructor({webrtc, canvas}: Props) {
-        this.wrtc = webrtc;
-        this.canvas = canvas;
-        this.keydownListener = this.createKeyboardListener("keydown", (e: any) => ({
-            type: "KeyDown",
-            key: this.keyToVirtualKeyCode(e.code)
-        }));
-        this.keyupListener = this.createKeyboardListener("keyup", (e: any) => ({
-            type: "KeyUp",
-            key: this.keyToVirtualKeyCode(e.code)
-        }));
-        this.run()
-    }
-
-    private run() {
-        //calls all the other functions
-        if (!document.pointerLockElement) {
-            if (this.connected) {
-                this.stop()
-            }
-            return;
-        }
-
-        if (document.pointerLockElement == this.canvas) {
-            this.connected = true
-            document.addEventListener("keydown", this.keydownListener);
-            document.addEventListener("keyup", this.keyupListener);
-        } else {
-            if (this.connected) {
-                this.stop()
-            }
-        }
+  constructor({webrtc, canvas}: Props) {
+    this.wrtc = webrtc;
+    this.canvas = canvas;
+    this.keydownListener = this.createKeyboardListener("keydown", (e: any) => ({
+      type: "KeyDown",
+      key: this.keyToVirtualKeyCode(e.code)
+    }));
+    this.keyupListener = this.createKeyboardListener("keyup", (e: any) => ({
+      type: "KeyUp",
+      key: this.keyToVirtualKeyCode(e.code)
+    }));
+    this.run()
+  }
 
+  private run() {
+    //calls all the other functions
+    if (!document.pointerLockElement) {
+      if (this.connected) {
+        this.stop()
+      }
+      return;
     }
 
-    private stop() {
-        document.removeEventListener("keydown", this.keydownListener);
-        document.removeEventListener("keyup", this.keyupListener);
-        this.connected = false;
+    if (document.pointerLockElement == this.canvas) {
+      this.connected = true
+      document.addEventListener("keydown", this.keydownListener, {passive: false});
+      document.addEventListener("keyup", this.keyupListener, {passive: false});
+    } else {
+      if (this.connected) {
+        this.stop()
+      }
     }
+  }
 
-    // Helper function to create and return mouse listeners
-    private createKeyboardListener(type: string, dataCreator: (e: Event) => Partial<Input>): (e: Event) => void {
-        return (e: Event) => {
-            e.preventDefault();
-            e.stopPropagation();
-            // Prevent repeated key events from being sent (important for games)
-            if ((e as any).repeat)
-                return;
+  private stop() {
+    document.removeEventListener("keydown", this.keydownListener);
+    document.removeEventListener("keyup", this.keyupListener);
+    this.connected = false;
+  }
 
-            const data = dataCreator(e as any); // type assertion because of the way dataCreator is used
-            const dataString = JSON.stringify({...data, type} as Input);
-            const message: MessageInput = {
-                payload_type: "input",
-                data: dataString,
-            };
-            this.wrtc.sendBinary(encodeMessage(message));
-        };
-    }
+  // Helper function to create and return mouse listeners
+  private createKeyboardListener(type: string, dataCreator: (e: Event) => Partial<Input>): (e: Event) => void {
+    return (e: Event) => {
+      e.preventDefault();
+      e.stopPropagation();
+      // Prevent repeated key events from being sent (important for games)
+      if ((e as any).repeat)
+        return;
 
-    public dispose() {
-        document.exitPointerLock();
-        this.stop();
-        this.connected = false;
-    }
+      const data = dataCreator(e as any); // type assertion because of the way dataCreator is used
+      const dataString = JSON.stringify({...data, type} as Input);
 
-    private keyToVirtualKeyCode(code: string) {
-        // Treat Home key as Escape - TODO: Make user-configurable
-        if (code === "Home") return 1;
-        return keyCodeToLinuxEventCode[code] || undefined;
-    }
+      // Latency tracking
+      const tracker = new LatencyTracker("input-keyboard");
+      tracker.addTimestamp("client_send");
+      const message: MessageInput = {
+        payload_type: "input",
+        data: dataString,
+        latency: tracker,
+      };
+      this.wrtc.sendBinary(encodeMessage(message));
+    };
+  }
+
+  public dispose() {
+    document.exitPointerLock();
+    this.stop();
+    this.connected = false;
+  }
+
+  private keyToVirtualKeyCode(code: string) {
+    // Treat Home key as Escape - TODO: Make user-configurable
+    if (code === "Home") return 1;
+    return keyCodeToLinuxEventCode[code] || undefined;
+  }
 }

packages/input/src/latency.ts

@@ -0,0 +1,54 @@
+type TimestampEntry = {
+  stage: string;
+  time: Date;
+};
+
+export class LatencyTracker {
+  sequence_id: string;
+  timestamps: TimestampEntry[];
+  metadata?: Record<string, any>;
+
+  constructor(sequence_id: string, timestamps: TimestampEntry[] = [], metadata: Record<string, any> = {}) {
+    this.sequence_id = sequence_id;
+    this.timestamps = timestamps;
+    this.metadata = metadata;
+  }
+
+  addTimestamp(stage: string): void {
+    const timestamp: TimestampEntry = {
+      stage,
+      time: new Date(),
+    };
+    this.timestamps.push(timestamp);
+  }
+
+  // Calculates the total time between the first and last recorded timestamps.
+  getTotalLatency(): number {
+    if (this.timestamps.length < 2) return 0;
+
+    const times = this.timestamps.map((entry) => entry.time.getTime());
+    const minTime = Math.min(...times);
+    const maxTime = Math.max(...times);
+    return maxTime - minTime;
+  }
+
+  toJSON(): Record<string, any> {
+    return {
+      sequence_id: this.sequence_id,
+      timestamps: this.timestamps.map((entry) => ({
+        stage: entry.stage,
+        // Fill nanoseconds with zeros to match the expected format
+        time: entry.time.toISOString().replace(/\.(\d+)Z$/, ".$1000000Z"),
+      })),
+      metadata: this.metadata,
+    };
+  }
+
+  static fromJSON(json: any): LatencyTracker {
+    const timestamps: TimestampEntry[] = json.timestamps.map((ts: any) => ({
+      stage: ts.stage,
+      time: new Date(ts.time),
+    }));
+    return new LatencyTracker(json.sequence_id, timestamps, json.metadata);
+  }
+}

packages/input/src/messages.ts

@@ -0,0 +1,73 @@
+import {gzip, ungzip} from "pako";
+import {LatencyTracker} from "./latency";
+
+export interface MessageBase {
+  payload_type: string;
+}
+
+export interface MessageInput extends MessageBase {
+  payload_type: "input";
+  data: string;
+  latency?: LatencyTracker;
+}
+
+export interface MessageICE extends MessageBase {
+  payload_type: "ice";
+  candidate: RTCIceCandidateInit;
+}
+
+export interface MessageSDP extends MessageBase {
+  payload_type: "sdp";
+  sdp: RTCSessionDescriptionInit;
+}
+
+export enum JoinerType {
+  JoinerNode = 0,
+  JoinerClient = 1,
+}
+
+export interface MessageJoin extends MessageBase {
+  payload_type: "join";
+  joiner_type: JoinerType;
+}
+
+export enum AnswerType {
+  AnswerOffline = 0,
+  AnswerInUse,
+  AnswerOK
+}
+
+export interface MessageAnswer extends MessageBase {
+  payload_type: "answer";
+  answer_type: AnswerType;
+}
+
+function blobToUint8Array(blob: Blob): Promise<Uint8Array> {
+  return new Promise((resolve, reject) => {
+    const reader = new FileReader();
+    reader.onloadend = () => {
+      const arrayBuffer = reader.result as ArrayBuffer;
+      resolve(new Uint8Array(arrayBuffer));
+    };
+    reader.onerror = reject;
+    reader.readAsArrayBuffer(blob);
+  });
+}
+
+export function encodeMessage<T>(message: T): Uint8Array {
+  // Convert the message to JSON string
+  const json = JSON.stringify(message);
+  // Compress the JSON string using gzip
+  return gzip(json);
+}
+
+export async function decodeMessage<T>(data: Blob): Promise<T> {
+  // Convert the Blob to Uint8Array
+  const array = await blobToUint8Array(data);
+  // Decompress the gzip data
+  const decompressed = ungzip(array);
+  // Convert the Uint8Array to JSON string
+  const json = new TextDecoder().decode(decompressed);
+  // Parse the JSON string
+  return JSON.parse(json);
+}