From 48b31ce57f38479913332dad81fac83b1add4b3f Mon Sep 17 00:00:00 2001 From: aidenlx <31102694+aidenlx@users.noreply.github.com> Date: Fri, 5 Apr 2024 21:42:07 +0800 Subject: [PATCH] perf(subtitle): remote caption caching --- apps/app/src/components/use-tracks.tsx | 65 ++++++++-- apps/app/src/lib/json.ts | 7 + apps/app/src/lib/store.ts | 20 +++ apps/app/src/mx-main.ts | 2 + apps/app/src/transcript/store.ts | 129 +++++++++++++++++++ apps/app/src/web/bili-req/base.ts | 81 ------------ apps/app/src/web/bili-req/index.ts | 1 - apps/app/src/web/bili-req/scripts/preload.ts | 2 +- 8 files changed, 215 insertions(+), 92 deletions(-) create mode 100644 apps/app/src/lib/json.ts create mode 100644 apps/app/src/lib/store.ts create mode 100644 apps/app/src/transcript/store.ts delete mode 100644 apps/app/src/web/bili-req/base.ts diff --git a/apps/app/src/components/use-tracks.tsx b/apps/app/src/components/use-tracks.tsx index c9b5f069..cd68b8bb 100644 --- a/apps/app/src/components/use-tracks.tsx +++ b/apps/app/src/components/use-tracks.tsx @@ -8,17 +8,23 @@ import type { TextTrackListModeChangeEvent, VTTContent, } from "@vidstack/react"; -import { Notice } from "obsidian"; import { useEffect, useMemo, useState } from "react"; import { setDefaultLang } from "@/lib/lang/default-lang"; import { WebiviewMediaProvider } from "@/lib/remote-player/provider"; -import { useMediaViewStore, useSettings } from "./context"; +import { toInfoKey } from "@/media-note/note-index/def"; +import { useMediaViewStore, usePlugin, useSettings } from "./context"; + +function useSid() { + return useMediaViewStore((s) => (s.source ? toInfoKey(s.source.url) : null)); +} export function useRemoteTextTracks() { // const externalTextTracks = useMediaViewStore(({ textTracks }) => textTracks); const player = useMediaPlayer(); const loaded = useMediaState("canPlay"); - // const textTrackCache = useCaptionCache(); + const sid = useSid(); + const plugin = usePlugin(); + useEffect(() => { if (!player) return; const updateCaption = async (evt: TextTrackListModeChangeEvent) => { @@ -30,16 +36,31 @@ export function useRemoteTextTracks() { track.mode === "showing" && track.content === dummyVTTContent && track.id && - provider instanceof WebiviewMediaProvider + provider instanceof WebiviewMediaProvider && + sid ) ) return; const id = track.id as string; - if (!loaded) { - new Notice("Cannot load remote captions before media is loaded"); - return; + const cached = await plugin.cacheStore.getCaption(sid, id); + let vtt; + if (cached) { + vtt = cached.content; + console.debug("Caption data loaded from cache", sid, id); + } else { + if (!loaded) { + console.warn("Cannot load remote captions before media is loaded"); + return; + } + vtt = await provider.media.methods.getTrack(id); + plugin.cacheStore.updateCaption(sid, id, vtt).then((v) => { + if (v) { + console.debug("Caption cached", sid, id); + } else { + console.error("Cannot cache caption", sid, id); + } + }); } - const vtt = await provider.media.methods.getTrack(id); if (!vtt) return; player.textTracks.remove(track); player.textTracks.add({ @@ -58,7 +79,7 @@ export function useRemoteTextTracks() { return () => { player.textTracks.removeEventListener("mode-change", updateCaption); }; - }, [player, loaded]); + }, [player, loaded, sid, plugin.cacheStore]); } export function useTextTracks() { @@ -70,8 +91,23 @@ export function useTextTracks() { >([]); const defaultLang = useSettings((s) => s.defaultLanguage); const getDefaultLang = useSettings((s) => s.getDefaultLang); + const sid = useSid(); + const plugin = usePlugin(); const provider = useMediaProvider(); + useEffect(() => { + if (!sid) return; + plugin.cacheStore.getCaptions(sid).then((data) => { + if (data.length === 0) return; + setRemoteTracks( + data.map(({ sid, data, ...info }) => ({ + ...info, + content: dummyVTTContent, + })), + ); + console.debug("Remote tracks loaded from cache", data.length); + }); + }, [plugin.cacheStore, sid]); useEffect(() => { if (!(provider instanceof WebiviewMediaProvider)) return; return provider.media.on("mx-text-tracks", ({ payload: { tracks } }) => { @@ -81,6 +117,17 @@ export function useTextTracks() { if (tracks.length !== 0) console.debug("Remote tracks loaded", tracks); }); }, [provider]); + useEffect(() => { + if (!(provider instanceof WebiviewMediaProvider) || !sid) return; + return provider.media.on( + "mx-text-tracks", + async ({ payload: { tracks } }) => { + await plugin.cacheStore.saveCaptionList(sid, tracks); + if (tracks.length !== 0) + console.debug("Remote tracks cached", tracks.length); + }, + ); + }, [plugin.cacheStore, provider, sid]); return useMemo( () => setDefaultLang( diff --git a/apps/app/src/lib/json.ts b/apps/app/src/lib/json.ts new file mode 100644 index 00000000..925e6726 --- /dev/null +++ b/apps/app/src/lib/json.ts @@ -0,0 +1,7 @@ +export function json(strings: TemplateStringsArray, ...values: any[]) { + return strings.reduce((result, string, i) => { + const value = values[i]; + const jsonValue = value !== undefined ? JSON.stringify(value) : ""; + return result + string + jsonValue; + }, ""); +} diff --git a/apps/app/src/lib/store.ts b/apps/app/src/lib/store.ts new file mode 100644 index 00000000..877023da --- /dev/null +++ b/apps/app/src/lib/store.ts @@ -0,0 +1,20 @@ +export async function jsonToGzipBlob(data: any) { + return await new Response( + new ReadableStream({ + start(controller) { + controller.enqueue(JSON.stringify(data)); + controller.close(); + }, + }) + .pipeThrough(new TextEncoderStream()) + .pipeThrough(new CompressionStream("gzip")), + // eslint-disable-next-line @typescript-eslint/naming-convention + { headers: { "Content-Type": "application/gzip" } }, + ).blob(); +} +export async function gzipBlobToJson(blob: Blob) { + if (blob.type !== "application/gzip") throw new Error("Invalid blob type"); + return (await new Response( + blob.stream().pipeThrough(new DecompressionStream("gzip")), + ).json()) as T; +} diff --git a/apps/app/src/mx-main.ts b/apps/app/src/mx-main.ts index 3fd96027..f8285cc9 100644 --- a/apps/app/src/mx-main.ts +++ b/apps/app/src/mx-main.ts @@ -43,6 +43,7 @@ import { createSettingsStore } from "./settings/def"; import { MxSettingTabs } from "./settings/tab"; import { initSwitcher } from "./switcher"; import { registerTranscriptView } from "./transcript"; +import { CacheStore } from "./transcript/store"; import { BilibiliRequestHacker } from "./web/bili-req"; import { modifySession } from "./web/session"; import { resolveMxProtocol } from "./web/url-match"; @@ -102,6 +103,7 @@ export default class MxPlugin extends Plugin { mediaNote = this.addChild(new MediaNoteIndex(this)); playlist = this.addChild(new PlaylistIndex(this)); biliReq = this.addChild(new BilibiliRequestHacker(this)); + cacheStore = this.addChild(new CacheStore(this)); leafOpener = this.addChild(new LeafOpener(this)); recorderNote = this.addChild(new RecorderNote(this)); handleMediaNote = handleMediaNote; diff --git a/apps/app/src/transcript/store.ts b/apps/app/src/transcript/store.ts new file mode 100644 index 00000000..31ee8e1e --- /dev/null +++ b/apps/app/src/transcript/store.ts @@ -0,0 +1,129 @@ +/* eslint-disable @typescript-eslint/naming-convention */ +import type { TextTrackInit, VTTContent } from "@vidstack/react"; +import type { DBSchema, IDBPDatabase } from "idb"; +import { openDB } from "idb"; +import { Component } from "obsidian"; +import { createEventEmitter } from "@/lib/emitter"; +import { gzipBlobToJson, jsonToGzipBlob } from "@/lib/store"; +import type MxPlugin from "@/mx-main"; + +interface MxCache extends DBSchema { + "caption-data": { + key: [string, string]; + value: TextTrackInit & { + /** caption internal id */ + id: string; + /** media source id */ + sid: string; + data: { + blob: Blob; + cueCount: number; + } | null; + }; + indexes: { "idx-sid": "sid" }; + }; +} + +export class CacheStore extends Component { + constructor(public plugin: MxPlugin) { + super(); + } + + #db: IDBPDatabase | null = null; + + get db(): Promise> { + if (this.#db) return Promise.resolve(this.#db); + return new Promise((resolve, reject) => { + const unload = this.event.once("db-ready", (db) => { + resolve(db); + window.clearTimeout(timeoutId); + }); + const timeoutId = window.setTimeout(() => { + reject(new Error("Timeout")); + unload(); + }, 5e3); + }); + } + async withDb( + callback: (db: IDBPDatabase) => T, + ): Promise> { + const db = await this.db; + return await callback(db); + } + + event = createEventEmitter<{ + "db-ready": (db: IDBPDatabase) => void; + }>(); + + onload(): void { + openDB("mx-cache", 1, { + upgrade(db) { + const store = db.createObjectStore("caption-data", { + keyPath: ["sid", "id"], + }); + store.createIndex("idx-sid", "sid", { unique: false }); + }, + }).then((db) => { + this.#db = db; + this.event.emit("db-ready", db); + }); + } + + async saveCaptionList( + sid: string, + data: (TextTrackInit & { id: string })[], + ): Promise { + const tx = (await this.db).transaction("caption-data", "readwrite"); + const store = tx.store; + await Promise.all( + data.map(async (item) => { + const prev = await store.get([sid, item.id]); + await store.put({ sid, ...item, data: prev?.data ?? null }); + }), + ); + await tx.done; + } + updateCaption = (sid: string, id: string, data: VTTContent | null) => + this.withDb(async (db): Promise => { + const blob = await jsonToGzipBlob(data); + const tx = db.transaction("caption-data", "readwrite"); + const store = tx.store; + const item = await store.get([sid, id]); + if (!item) return false; + if (data) { + item.data = { + blob, + cueCount: data.cues?.length ?? -1, + }; + } else { + item.data = null; + } + await store.put(item); + await tx.done; + return true; + }); + + async getCaptions(sid: string) { + const tx = (await this.db).transaction("caption-data", "readonly"); + const store = tx.store; + const index = store.index("idx-sid"); + const result = await index.getAllKeys(IDBKeyRange.only(sid)); + if (result.length === 0) return []; + return Promise.all(result.map(async (key) => (await store.get(key))!)); + } + getCaption = (sid: string, id: string) => + this.withDb(async (db) => { + const data = await db.get("caption-data", [sid, id]); + if (!data) return null; + const { data: vttData, ...info } = data; + if (!vttData) return null; + return { + ...info, + content: await CacheStore.decompress(vttData.blob), + }; + }); + + static decompress(blob: Blob) { + return gzipBlobToJson(blob); + } +} diff --git a/apps/app/src/web/bili-req/base.ts b/apps/app/src/web/bili-req/base.ts deleted file mode 100644 index c00ab4be..00000000 --- a/apps/app/src/web/bili-req/base.ts +++ /dev/null @@ -1,81 +0,0 @@ -import type { App } from "obsidian"; -import { getPartition } from "@/lib/remote-player/const"; - -export function json(strings: TemplateStringsArray, ...values: any[]) { - return strings.reduce((result, string, i) => { - const value = values[i]; - const jsonValue = value !== undefined ? JSON.stringify(value) : ""; - return result + string + jsonValue; - }, ""); -} - -export const channelId = "mx:http_proxy"; -const playerV2 = { - type: "player_v2", - host: "api.bilibili.com", - pathnames: ["/x/player/v2", "/x/player/wbi/v2"], - filter: [ - "https://api.bilibili.com/x/player/v2*", - "https://api.bilibili.com/x/player/wbi/v2*", - ], - header: "player_v2", - types: ["xhr"], -} as const; -export function buildMainProcessScript(webContentId: number, app: App) { - const partition = getPartition(app.appId); - if (!partition) { - console.log("partition disabled, cannot watch requests"); - return ""; - } - return json` -const { session, webContents, net } = require("electron"); -const webviewSession = session.fromPartition(${partition}); -const webContent = webContents.fromId(${webContentId}); -webviewSession.webRequest.onSendHeaders( - { - urls: ${playerV2.filter}, type: ${playerV2.types} - }, ({url, method, requestHeaders, webContentsId}) => { - if (method !== "GET" || webContentsId===undefined) return; - webContent.send(${channelId}, {type:${playerV2.type} ,url, method, requestHeaders, webContentsId}); - }) -`.trim(); -} - -export type RequestInfo = Pick< - Electron.OnSendHeadersListenerDetails, - "url" | "method" | "requestHeaders" -> & { webContentsId: number; type: typeof playerV2.type }; - -export async function gzippedStreamToBlob( - readableStream: ReadableStream, - type: string, -): Promise { - return streamToBlob( - readableStream.pipeThrough(new DecompressionStream("gzip")), - type, - ); -} - -export function abToStream(ab: ArrayBuffer): ReadableStream { - return new ReadableStream({ - start(controller) { - controller.enqueue(new Uint8Array(ab)); - controller.close(); - }, - }); -} - -export async function streamToBlob( - readableStream: ReadableStream, - type: string, -): Promise { - const reader = readableStream.getReader(); - const chunks: Uint8Array[] = []; - // eslint-disable-next-line no-constant-condition - while (true) { - const { done, value } = await reader.read(); - if (done) break; - chunks.push(value); - } - return new Blob(chunks, { type }); -} diff --git a/apps/app/src/web/bili-req/index.ts b/apps/app/src/web/bili-req/index.ts index e2f7bd18..104d5812 100644 --- a/apps/app/src/web/bili-req/index.ts +++ b/apps/app/src/web/bili-req/index.ts @@ -115,7 +115,6 @@ export class BilibiliRequestHacker extends Component { ); } const { ipcRenderer } = require("electron"); - console.log(channel.enable); await ipcRenderer.invoke(channel.enable, preloadScriptPath); this.register(() => { ipcRenderer.invoke(channel.disable); diff --git a/apps/app/src/web/bili-req/scripts/preload.ts b/apps/app/src/web/bili-req/scripts/preload.ts index bff8640e..24e2b03a 100644 --- a/apps/app/src/web/bili-req/scripts/preload.ts +++ b/apps/app/src/web/bili-req/scripts/preload.ts @@ -1,4 +1,4 @@ -import { json } from "../base"; +import { json } from "../../../lib/json"; // eslint-disable-next-line @typescript-eslint/naming-convention declare const __USERSCRIPT__: string;