perf(subtitle): remote caption caching

apps/app/src/components/use-tracks.tsx

@@ -8,17 +8,23 @@ import type {
   TextTrackListModeChangeEvent,
   VTTContent,
 } from "@vidstack/react";
-import { Notice } from "obsidian";
 import { useEffect, useMemo, useState } from "react";
 import { setDefaultLang } from "@/lib/lang/default-lang";
 import { WebiviewMediaProvider } from "@/lib/remote-player/provider";
-import { useMediaViewStore, useSettings } from "./context";
+import { toInfoKey } from "@/media-note/note-index/def";
+import { useMediaViewStore, usePlugin, useSettings } from "./context";
+
+function useSid() {
+  return useMediaViewStore((s) => (s.source ? toInfoKey(s.source.url) : null));
+}
 
 export function useRemoteTextTracks() {
   // const externalTextTracks = useMediaViewStore(({ textTracks }) => textTracks);
   const player = useMediaPlayer();
   const loaded = useMediaState("canPlay");
-  // const textTrackCache = useCaptionCache();
+  const sid = useSid();
+  const plugin = usePlugin();
+
   useEffect(() => {
     if (!player) return;
     const updateCaption = async (evt: TextTrackListModeChangeEvent) => {
@@ -30,16 +36,31 @@ export function useRemoteTextTracks() {
           track.mode === "showing" &&
           track.content === dummyVTTContent &&
           track.id &&
-          provider instanceof WebiviewMediaProvider
+          provider instanceof WebiviewMediaProvider &&
+          sid
         )
       )
         return;
       const id = track.id as string;
-      if (!loaded) {
-        new Notice("Cannot load remote captions before media is loaded");
-        return;
+      const cached = await plugin.cacheStore.getCaption(sid, id);
+      let vtt;
+      if (cached) {
+        vtt = cached.content;
+        console.debug("Caption data loaded from cache", sid, id);
+      } else {
+        if (!loaded) {
+          console.warn("Cannot load remote captions before media is loaded");
+          return;
+        }
+        vtt = await provider.media.methods.getTrack(id);
+        plugin.cacheStore.updateCaption(sid, id, vtt).then((v) => {
+          if (v) {
+            console.debug("Caption cached", sid, id);
+          } else {
+            console.error("Cannot cache caption", sid, id);
+          }
+        });
       }
-      const vtt = await provider.media.methods.getTrack(id);
       if (!vtt) return;
       player.textTracks.remove(track);
       player.textTracks.add({
@@ -58,7 +79,7 @@ export function useRemoteTextTracks() {
     return () => {
       player.textTracks.removeEventListener("mode-change", updateCaption);
     };
-  }, [player, loaded]);
+  }, [player, loaded, sid, plugin.cacheStore]);
 }
 
 export function useTextTracks() {
@@ -70,8 +91,23 @@ export function useTextTracks() {
   >([]);
   const defaultLang = useSettings((s) => s.defaultLanguage);
   const getDefaultLang = useSettings((s) => s.getDefaultLang);
+  const sid = useSid();
+  const plugin = usePlugin();
 
   const provider = useMediaProvider();
+  useEffect(() => {
+    if (!sid) return;
+    plugin.cacheStore.getCaptions(sid).then((data) => {
+      if (data.length === 0) return;
+      setRemoteTracks(
+        data.map(({ sid, data, ...info }) => ({
+          ...info,
+          content: dummyVTTContent,
+        })),
+      );
+      console.debug("Remote tracks loaded from cache", data.length);
+    });
+  }, [plugin.cacheStore, sid]);
   useEffect(() => {
     if (!(provider instanceof WebiviewMediaProvider)) return;
     return provider.media.on("mx-text-tracks", ({ payload: { tracks } }) => {
@@ -81,6 +117,17 @@ export function useTextTracks() {
       if (tracks.length !== 0) console.debug("Remote tracks loaded", tracks);
     });
   }, [provider]);
+  useEffect(() => {
+    if (!(provider instanceof WebiviewMediaProvider) || !sid) return;
+    return provider.media.on(
+      "mx-text-tracks",
+      async ({ payload: { tracks } }) => {
+        await plugin.cacheStore.saveCaptionList(sid, tracks);
+        if (tracks.length !== 0)
+          console.debug("Remote tracks cached", tracks.length);
+      },
+    );
+  }, [plugin.cacheStore, provider, sid]);
   return useMemo(
     () =>
       setDefaultLang(

apps/app/src/lib/json.ts

@@ -0,0 +1,7 @@
+export function json(strings: TemplateStringsArray, ...values: any[]) {
+  return strings.reduce((result, string, i) => {
+    const value = values[i];
+    const jsonValue = value !== undefined ? JSON.stringify(value) : "";
+    return result + string + jsonValue;
+  }, "");
+}

apps/app/src/lib/store.ts

@@ -0,0 +1,20 @@
+export async function jsonToGzipBlob(data: any) {
+  return await new Response(
+    new ReadableStream({
+      start(controller) {
+        controller.enqueue(JSON.stringify(data));
+        controller.close();
+      },
+    })
+      .pipeThrough(new TextEncoderStream())
+      .pipeThrough(new CompressionStream("gzip")),
+    // eslint-disable-next-line @typescript-eslint/naming-convention
+    { headers: { "Content-Type": "application/gzip" } },
+  ).blob();
+}
+export async function gzipBlobToJson<T = unknown>(blob: Blob) {
+  if (blob.type !== "application/gzip") throw new Error("Invalid blob type");
+  return (await new Response(
+    blob.stream().pipeThrough(new DecompressionStream("gzip")),
+  ).json()) as T;
+}

apps/app/src/mx-main.ts

@@ -43,6 +43,7 @@ import { createSettingsStore } from "./settings/def";
 import { MxSettingTabs } from "./settings/tab";
 import { initSwitcher } from "./switcher";
 import { registerTranscriptView } from "./transcript";
+import { CacheStore } from "./transcript/store";
 import { BilibiliRequestHacker } from "./web/bili-req";
 import { modifySession } from "./web/session";
 import { resolveMxProtocol } from "./web/url-match";
@@ -102,6 +103,7 @@ export default class MxPlugin extends Plugin {
   mediaNote = this.addChild(new MediaNoteIndex(this));
   playlist = this.addChild(new PlaylistIndex(this));
   biliReq = this.addChild(new BilibiliRequestHacker(this));
+  cacheStore = this.addChild(new CacheStore(this));
   leafOpener = this.addChild(new LeafOpener(this));
   recorderNote = this.addChild(new RecorderNote(this));
   handleMediaNote = handleMediaNote;

apps/app/src/transcript/store.ts

@@ -0,0 +1,129 @@
+/* eslint-disable @typescript-eslint/naming-convention */
+import type { TextTrackInit, VTTContent } from "@vidstack/react";
+import type { DBSchema, IDBPDatabase } from "idb";
+import { openDB } from "idb";
+import { Component } from "obsidian";
+import { createEventEmitter } from "@/lib/emitter";
+import { gzipBlobToJson, jsonToGzipBlob } from "@/lib/store";
+import type MxPlugin from "@/mx-main";
+
+interface MxCache extends DBSchema {
+  "caption-data": {
+    key: [string, string];
+    value: TextTrackInit & {
+      /** caption internal id */
+      id: string;
+      /** media source id */
+      sid: string;
+      data: {
+        blob: Blob;
+        cueCount: number;
+      } | null;
+    };
+    indexes: { "idx-sid": "sid" };
+  };
+}
+
+export class CacheStore extends Component {
+  constructor(public plugin: MxPlugin) {
+    super();
+  }
+
+  #db: IDBPDatabase<MxCache> | null = null;
+
+  get db(): Promise<IDBPDatabase<MxCache>> {
+    if (this.#db) return Promise.resolve(this.#db);
+    return new Promise((resolve, reject) => {
+      const unload = this.event.once("db-ready", (db) => {
+        resolve(db);
+        window.clearTimeout(timeoutId);
+      });
+      const timeoutId = window.setTimeout(() => {
+        reject(new Error("Timeout"));
+        unload();
+      }, 5e3);
+    });
+  }
+  async withDb<T>(
+    callback: (db: IDBPDatabase<MxCache>) => T,
+  ): Promise<Awaited<T>> {
+    const db = await this.db;
+    return await callback(db);
+  }
+
+  event = createEventEmitter<{
+    "db-ready": (db: IDBPDatabase<MxCache>) => void;
+  }>();
+
+  onload(): void {
+    openDB<MxCache>("mx-cache", 1, {
+      upgrade(db) {
+        const store = db.createObjectStore("caption-data", {
+          keyPath: ["sid", "id"],
+        });
+        store.createIndex("idx-sid", "sid", { unique: false });
+      },
+    }).then((db) => {
+      this.#db = db;
+      this.event.emit("db-ready", db);
+    });
+  }
+
+  async saveCaptionList(
+    sid: string,
+    data: (TextTrackInit & { id: string })[],
+  ): Promise<void> {
+    const tx = (await this.db).transaction("caption-data", "readwrite");
+    const store = tx.store;
+    await Promise.all(
+      data.map(async (item) => {
+        const prev = await store.get([sid, item.id]);
+        await store.put({ sid, ...item, data: prev?.data ?? null });
+      }),
+    );
+    await tx.done;
+  }
+  updateCaption = (sid: string, id: string, data: VTTContent | null) =>
+    this.withDb(async (db): Promise<boolean> => {
+      const blob = await jsonToGzipBlob(data);
+      const tx = db.transaction("caption-data", "readwrite");
+      const store = tx.store;
+      const item = await store.get([sid, id]);
+      if (!item) return false;
+      if (data) {
+        item.data = {
+          blob,
+          cueCount: data.cues?.length ?? -1,
+        };
+      } else {
+        item.data = null;
+      }
+      await store.put(item);
+      await tx.done;
+      return true;
+    });
+
+  async getCaptions(sid: string) {
+    const tx = (await this.db).transaction("caption-data", "readonly");
+    const store = tx.store;
+    const index = store.index("idx-sid");
+    const result = await index.getAllKeys(IDBKeyRange.only(sid));
+    if (result.length === 0) return [];
+    return Promise.all(result.map(async (key) => (await store.get(key))!));
+  }
+  getCaption = (sid: string, id: string) =>
+    this.withDb(async (db) => {
+      const data = await db.get("caption-data", [sid, id]);
+      if (!data) return null;
+      const { data: vttData, ...info } = data;
+      if (!vttData) return null;
+      return {
+        ...info,
+        content: await CacheStore.decompress(vttData.blob),
+      };
+    });
+
+  static decompress(blob: Blob) {
+    return gzipBlobToJson<VTTContent>(blob);
+  }
+}

apps/app/src/web/bili-req/index.ts

@@ -115,7 +115,6 @@ export class BilibiliRequestHacker extends Component {
           );
       }
       const { ipcRenderer } = require("electron");
-      console.log(channel.enable);
       await ipcRenderer.invoke(channel.enable, preloadScriptPath);
       this.register(() => {
         ipcRenderer.invoke(channel.disable);

apps/app/src/web/bili-req/scripts/preload.ts

@@ -1,4 +1,4 @@
-import { json } from "../base";
+import { json } from "../../../lib/json";
 
 // eslint-disable-next-line @typescript-eslint/naming-convention
 declare const __USERSCRIPT__: string;