From 7c530c65f89a1698be6a142b73b31f0aadb5cb03 Mon Sep 17 00:00:00 2001 From: Jonny Burger Date: Wed, 2 Oct 2024 11:49:28 +0200 Subject: [PATCH] `@remotion/install-whisper-cpp`: Add support for `large-v3-turbo` --- .../download-whisper-model.mdx | 16 ++++++++-------- .../docs/install-whisper-cpp/transcribe.mdx | 18 +++++++++--------- .../src/download-whisper-model.ts | 2 ++ packages/install-whisper-cpp/src/transcribe.ts | 2 +- 4 files changed, 20 insertions(+), 18 deletions(-) diff --git a/packages/docs/docs/install-whisper-cpp/download-whisper-model.mdx b/packages/docs/docs/install-whisper-cpp/download-whisper-model.mdx index 8dd149703a9..09119d25f19 100644 --- a/packages/docs/docs/install-whisper-cpp/download-whisper-model.mdx +++ b/packages/docs/docs/install-whisper-cpp/download-whisper-model.mdx @@ -1,7 +1,7 @@ --- image: /generated/articles-docs-install-whisper-cpp-download-whisper-model.png title: downloadWhisperModel() -crumb: "@remotion/install-whisper-cpp" +crumb: '@remotion/install-whisper-cpp' --- # downloadWhisperModel() @@ -12,12 +12,12 @@ You should first install Whisper.cpp, for example through [`installWhisperCpp()` ```tsx twoslash title="install-whisper.mjs" // @module: esnext // @target: es2022 -import path from "path"; -import { downloadWhisperModel } from "@remotion/install-whisper-cpp"; +import path from 'path'; +import {downloadWhisperModel} from '@remotion/install-whisper-cpp'; -const { alreadyExisted } = await downloadWhisperModel({ - model: "medium.en", - folder: path.join(process.cwd(), "whisper.cpp"), +const {alreadyExisted} = await downloadWhisperModel({ + model: 'medium.en', + folder: path.join(process.cwd(), 'whisper.cpp'), }); ``` @@ -29,14 +29,14 @@ The folder to download the model to. The model will be downloaded into this fold ### `model` -The model to download. Possible values: `tiny`, `tiny.en`, `base`, `base.en`, `small`, `small.en`, `medium`, `medium.en`, `large-v1`, `large-v2`, `large-v3`. +The model to download. Possible values: `tiny`, `tiny.en`, `base`, `base.en`, `small`, `small.en`, `medium`, `medium.en`, `large-v1`, `large-v2`, `large-v3`, `large-v3-turbo`. ### `onProgress?` Act upon download progress. This is the function signature: ```tsx twoslash -import type { OnProgress } from "@remotion/install-whisper-cpp"; +import type {OnProgress} from '@remotion/install-whisper-cpp'; const onProgress: OnProgress = ( downloadedBytes: number, diff --git a/packages/docs/docs/install-whisper-cpp/transcribe.mdx b/packages/docs/docs/install-whisper-cpp/transcribe.mdx index 964e45290f2..0f34027c0cc 100644 --- a/packages/docs/docs/install-whisper-cpp/transcribe.mdx +++ b/packages/docs/docs/install-whisper-cpp/transcribe.mdx @@ -1,7 +1,7 @@ --- image: /generated/articles-docs-install-whisper-cpp-transcribe.png title: transcribe() -crumb: "@remotion/install-whisper-cpp" +crumb: '@remotion/install-whisper-cpp' --- # transcribe() @@ -16,13 +16,13 @@ This function only works with Whisper.cpp 1.5.5 or later, unless `tokenLevelTime ```tsx twoslash title="transcribe.mjs" // @module: esnext // @target: es2022 -import path from "path"; -import { transcribe } from "@remotion/install-whisper-cpp"; +import path from 'path'; +import {transcribe} from '@remotion/install-whisper-cpp'; -const { transcription } = await transcribe({ - inputPath: "/path/to/audio.wav", - whisperPath: path.join(process.cwd(), "whisper.cpp"), - model: "medium.en", +const {transcription} = await transcribe({ + inputPath: '/path/to/audio.wav', + whisperPath: path.join(process.cwd(), 'whisper.cpp'), + model: 'medium.en', tokenLevelTimestamps: true, }); @@ -67,7 +67,7 @@ _default: `base.en`_ Specify a specific Whisper model for the transcription. -Possible values: `tiny`, `tiny.en`, `base`, `base.en`, `small`, `small.en`, `medium`, `medium.en`, `large-v1`, `large-v2`, `large-v3`. +Possible values: `tiny`, `tiny.en`, `base`, `base.en`, `small`, `small.en`, `medium`, `medium.en`, `large-v1`, `large-v2`, `large-v3`, `large-v3-turbo`. Make sure the model you want to use exists in your `whisper.cpp/models` folder. You can ensure a specific model is available locally by utilizing the [downloadWhisperModel()](/docs/install-whisper-cpp/download-whisper-model) API. @@ -129,7 +129,7 @@ Listen for progress updates from the transcription process. The progress is a number between `0` and `1`. ```tsx twoslash -import type { TranscribeOnProgress } from "@remotion/install-whisper-cpp"; +import type {TranscribeOnProgress} from '@remotion/install-whisper-cpp'; const onProgress: TranscribeOnProgress = (progress) => { console.log(`Transcription progress: ${progress * 100}%`); diff --git a/packages/install-whisper-cpp/src/download-whisper-model.ts b/packages/install-whisper-cpp/src/download-whisper-model.ts index 9568e06cf18..c1752b1f5f0 100644 --- a/packages/install-whisper-cpp/src/download-whisper-model.ts +++ b/packages/install-whisper-cpp/src/download-whisper-model.ts @@ -14,6 +14,7 @@ const models = [ 'large-v1', 'large-v2', 'large-v3', + 'large-v3-turbo', ] as const; const modelSizes: {[key in WhisperModel]: number} = { @@ -22,6 +23,7 @@ const modelSizes: {[key in WhisperModel]: number} = { 'large-v1': 3094623691, 'large-v2': 3094623691, 'large-v3': 3095033483, + 'large-v3-turbo': 1624555275, small: 487601967, tiny: 77691713, 'small.en': 487614201, diff --git a/packages/install-whisper-cpp/src/transcribe.ts b/packages/install-whisper-cpp/src/transcribe.ts index 4e6b88888aa..cbe9d5b5a73 100644 --- a/packages/install-whisper-cpp/src/transcribe.ts +++ b/packages/install-whisper-cpp/src/transcribe.ts @@ -94,7 +94,7 @@ export type TranscribeOnProgress = (progress: number) => void; // https://github.com/ggerganov/whisper.cpp/blob/fe36c909715e6751277ddb020e7892c7670b61d4/examples/main/main.cpp#L989-L999 // https://github.com/remotion-dev/remotion/issues/4168 export const modelToDtw = (model: WhisperModel): string => { - if (model === 'large-v3') { + if (model === 'large-v3' || model === 'large-v3-turbo') { return 'large.v3'; }