From 0f803f9fe50e22c5160e8d394cff7871b3d7f3fe Mon Sep 17 00:00:00 2001 From: Albert Ho Date: Thu, 9 Nov 2023 12:02:37 -0800 Subject: [PATCH] update --- .github/workflows/react.yml | 8 ++-- binding/react/.gitignore | 1 - binding/react/README.md | 26 ++++------ binding/react/src/use_leopard.ts | 82 +++++++++++++++++--------------- demo/react/src/App.css | 2 +- demo/react/src/VoiceWidget.tsx | 24 ++++++---- 6 files changed, 71 insertions(+), 72 deletions(-) diff --git a/.github/workflows/react.yml b/.github/workflows/react.yml index 0ad31c3e..1eed6171 100644 --- a/.github/workflows/react.yml +++ b/.github/workflows/react.yml @@ -9,8 +9,8 @@ on: - '!binding/react/README.md' - 'lib/common/**' - 'lib/wasm/**' - - 'resources/**' - - '!resources/.lint/**' + - 'resources/.test/**' + - 'resources/audio_samples/**' - '.github/workflows/react.yml' pull_request: branches: [ master, 'v[0-9]+.[0-9]+' ] @@ -19,8 +19,8 @@ on: - '!binding/react/README.md' - 'lib/common/**' - 'lib/wasm/**' - - 'resources/**' - - '!resources/.lint/**' + - 'resources/.test/**' + - 'resources/audio_samples/**' - '.github/workflows/react.yml' defaults: diff --git a/binding/react/.gitignore b/binding/react/.gitignore index f477abdc..79a606db 100644 --- a/binding/react/.gitignore +++ b/binding/react/.gitignore @@ -8,4 +8,3 @@ cypress/**/*.wav test/*.pv test/*.js test/**/*.json -test/**/*.ppn diff --git a/binding/react/README.md b/binding/react/README.md index 63b17a0e..1a9296d5 100644 --- a/binding/react/README.md +++ b/binding/react/README.md @@ -106,31 +106,22 @@ const options = { } ``` -For processing, you may also consider transferring the buffer for performance: - -```typescript -const processOptions = { - transfer: true, - transferCallback: (data) => { pcm = data } -} -``` - ### Initialize Leopard Use `useLeopard` and `init` to initialize `Leopard`: ```typescript const { - init, + result, isLoaded, + error, + init, processFile, startRecording, stopRecording, isRecording, recordingElapsedSec, - result, release, - error, } = useLeopard(); const initLeopard = async () => { @@ -138,16 +129,15 @@ const initLeopard = async () => { "${ACCESS_KEY}", leopardModel, options, - processOptions - ) + ); } ``` In case of any errors, use the `error` state variable to check the error message. Use the `isLoaded` state variable to check if `Leopard` has loaded. -### Process Audio Frames +### Transcribe Audio -The audio that you want to transcribe can either be uploaded as a File object or recorded. +The audio that you want to transcribe can either be uploaded as a `File` object or recorded with a microphone. #### File Object @@ -177,7 +167,7 @@ await startRecording(); If `WebVoiceProcessor` has started correctly, `isRecording` will be set to true. -**Note**: By default, Leopard will only record for 2 minutes before stopping and processing the recording. This is to prevent buffer overflow. To increase this limit, call `startRecording` with the optional `maxRecordingSec` parameter: +**Note**: By default, Leopard will only record for 2 minutes before stopping and processing the buffered audio. This is to prevent unbounded memory usage. To increase this limit, call `startRecording` with the optional `maxRecordingSec` parameter: ```typescript const maxRecordingSec = 60 * 10 @@ -216,7 +206,7 @@ While running in a component, you can call `release` to clean up all resources u await release(); ``` -This will set `isLoaded` and `isRecording` to false and `error` to null. +This will set `isLoaded` and `isRecording` to false, `recordingElapsedSec` to 0, and `error` to null. If any arguments require changes, call `release`, then `init` again to initialize Leopard with the new settings. diff --git a/binding/react/src/use_leopard.ts b/binding/react/src/use_leopard.ts index 47655aff..ee245a5d 100644 --- a/binding/react/src/use_leopard.ts +++ b/binding/react/src/use_leopard.ts @@ -1,14 +1,14 @@ -// /* -// Copyright 2023 Picovoice Inc. -// -// You may not use this file except in compliance with the license. A copy of the license is located in the "LICENSE" -// file accompanying this source. -// -// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on -// an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the -// specific language governing permissions and limitations under the License. -// */ -// +/* + Copyright 2023 Picovoice Inc. + + You may not use this file except in compliance with the license. A copy of the license is located in the "LICENSE" + file accompanying this source. + + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + specific language governing permissions and limitations under the License. +*/ + import { useCallback, useEffect, useRef, useState } from 'react'; import { WebVoiceProcessor } from '@picovoice/web-voice-processor'; import { @@ -27,11 +27,7 @@ export const useLeopard = (): { init: ( accessKey: string, model: LeopardModel, - options?: LeopardOptions, - processOptions?: { - transfer?: boolean; - transferCallback?: (data: Int16Array) => void; - } + options?: LeopardOptions ) => Promise; processFile: (file: File) => Promise; startRecording: (maxRecordingSec?: number) => Promise; @@ -48,8 +44,7 @@ export const useLeopard = (): { const leopardRef = useRef(null); const audioDataRef = useRef([]); - const processOptionsRef = useRef({}); - const timerRef = useRef>(null); + const timerRef = useRef>(null); const recorderEngineRef = useRef({ onmessage: (event: any) => { switch (event.data.command) { @@ -66,11 +61,7 @@ export const useLeopard = (): { async ( accessKey: string, model: LeopardModel, - options: LeopardOptions = {}, - processOptions: { - transfer?: boolean; - transferCallback?: (data: Int16Array) => void; - } = {} + options: LeopardOptions = {} ): Promise => { try { if (!leopardRef.current) { @@ -80,7 +71,6 @@ export const useLeopard = (): { options ); - processOptionsRef.current = processOptions; setIsLoaded(true); setError(null); } @@ -100,10 +90,9 @@ export const useLeopard = (): { return; } - const processResult = await leopardRef.current.process( - pcm, - processOptionsRef.current - ); + const processResult = await leopardRef.current.process(pcm, { + transfer: true, + }); setResult(processResult); } catch (e: any) { setError(e); @@ -160,24 +149,29 @@ export const useLeopard = (): { const stopRecording = useCallback(async (): Promise => { try { + if (!isRecording) { + return; + } + if (timerRef.current) { clearInterval(timerRef.current); + timerRef.current = null; + } - await WebVoiceProcessor.unsubscribe(recorderEngineRef.current); - setIsRecording(false); - - const frames = new Int16Array(audioDataRef.current.length * 512); - for (let i = 0; i < audioDataRef.current.length; i++) { - frames.set(audioDataRef.current[i], i * 512); - } - await process(frames); + await WebVoiceProcessor.unsubscribe(recorderEngineRef.current); + setIsRecording(false); - audioDataRef.current = []; + const frames = new Int16Array(audioDataRef.current.length * 512); + for (let i = 0; i < audioDataRef.current.length; i++) { + frames.set(audioDataRef.current[i], i * 512); } + await process(frames); + + audioDataRef.current = []; } catch (e: any) { setError(e); } - }, []); + }, [isRecording]); const startRecording = useCallback( async (maxRecordingSec = DEFAULT_MAX_RECORDING_SEC): Promise => { @@ -188,8 +182,13 @@ export const useLeopard = (): { return; } + if (isRecording) { + return; + } + setError(null); audioDataRef.current = []; + setRecordingElapsedSec(0); try { await WebVoiceProcessor.subscribe(recorderEngineRef.current); @@ -210,7 +209,7 @@ export const useLeopard = (): { setError(e); } }, - [] + [isRecording] ); const release = useCallback(async (): Promise => { @@ -219,9 +218,14 @@ export const useLeopard = (): { await WebVoiceProcessor.unsubscribe(recorderEngineRef.current); leopardRef.current?.terminate(); leopardRef.current = null; + if (timerRef.current) { + clearInterval(timerRef.current); + timerRef.current = null; + } setError(null); setIsLoaded(false); setIsRecording(false); + setRecordingElapsedSec(0); } } catch (e: any) { setError(e); diff --git a/demo/react/src/App.css b/demo/react/src/App.css index 54414ae0..568301d1 100644 --- a/demo/react/src/App.css +++ b/demo/react/src/App.css @@ -12,7 +12,7 @@ button { .release-button, input { padding: 0; - margin: 0; + margin: 0 0.25rem; font-size: medium; } diff --git a/demo/react/src/VoiceWidget.tsx b/demo/react/src/VoiceWidget.tsx index 298b0690..38993e68 100644 --- a/demo/react/src/VoiceWidget.tsx +++ b/demo/react/src/VoiceWidget.tsx @@ -1,22 +1,23 @@ -import React, { useCallback, useRef } from "react"; +import React, { useCallback, useState, useRef } from "react"; import { useLeopard } from "@picovoice/leopard-react"; import leopardModel from "./lib/leopardModel"; export default function VoiceWidget() { const accessKeyRef = useRef(""); + const [isBusy, setIsBusy] = useState(false); const { - init, result, isLoaded, + error, + init, processFile, startRecording, stopRecording, isRecording, recordingElapsedSec, release, - error, } = useLeopard(); const initEngine = useCallback(async () => { @@ -24,24 +25,25 @@ export default function VoiceWidget() { return; } + setIsBusy(true); await init( accessKeyRef.current, leopardModel, { enableAutomaticPunctuation: true, - }, - { - transfer: true, } ); + setIsBusy(false); }, [init]); const toggleRecord = async () => { + setIsBusy(true); if (isRecording) { await stopRecording(); } else { await startRecording(); } + setIsBusy(false); }; return ( @@ -54,6 +56,7 @@ export default function VoiceWidget() { { accessKeyRef.current = e.target.value; }} @@ -61,14 +64,14 @@ export default function VoiceWidget() { @@ -87,9 +90,12 @@ export default function VoiceWidget() { name="audio-file" onChange={async (e) => { if (!!e.target.files?.length) { + setIsBusy(true); await processFile(e.target.files[0]); + setIsBusy(false); } }} + disabled={!isLoaded || isBusy} />

OR @@ -100,7 +106,7 @@ export default function VoiceWidget() { {recordingElapsedSec}s

-

Transcript: