feat(speech): add text to speech

- update packages
- add speech button
- add filesystem utils
- add ssml as i18next context
- add settings for voice selection
- refactor chapter sections providing

.env.template

@@ -0,0 +1,3 @@
+# Azure Cognitive Service (Text-To-Speech)
+REACT_APP_AZURE_COGNITIVE_TTS_SUBSCRIPTION_KEY=""
+REACT_APP_AZURE_COGNITIVE_TTS_SERVICE_REGION=""

.gitignore

@@ -1,5 +1,10 @@
 # See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
 
+
+# cache
+.eslintcache
+.cache
+
 # dependencies
 /node_modules
 /.pnp

.prettierrc.json

@@ -4,5 +4,13 @@
   "useTabs": false,
   "semi": true,
   "singleQuote": true,
-  "trailingComma": "all"
+  "trailingComma": "all",
+  "overrides": [
+    {
+      "files": ["src/data/chapter/*.tsx"],
+      "options": {
+        "printWidth": 120
+      }
+    }
+  ]
 }

capacitor.config.json

@@ -19,5 +19,5 @@
       "splashFullScreen": true,
       "splashImmersive": true
     }
-  },
+  }
 }

package.json

@@ -27,10 +27,11 @@
   "dependencies": {
     "@capacitor/android": "^3.0.0-beta.1",
     "@capacitor/core": "^3.0.0-beta.1",
+    "@capacitor/filesystem": "^0.5.1",
     "@capacitor/ios": "^3.0.0-beta.1",
     "@capacitor/splash-screen": "^0.3.1",
     "@capacitor/status-bar": "^0.4.0",
-    "@capacitor/storage": "^0.3.0",
+    "@capacitor/storage": "^0.3.5",
     "@ionic/react": "^5.5.2",
     "@ionic/react-router": "^5.5.2",
     "@ionic/storage": "^2.3.1",
@@ -39,10 +40,12 @@
     "@testing-library/react": "^11.2.3",
     "@testing-library/user-event": "^12.6.0",
     "connected-react-router": "6.8.0",
-    "i18next": "19.8.4",
+    "i18next": "^19.9.1",
     "i18next-browser-languagedetector": "6.0.1",
     "ionicons": "5.3.0",
     "lunr": "^2.3.9",
+    "md5": "^2.3.0",
+    "microsoft-cognitiveservices-speech-sdk": "^1.15.1",
     "react": "17.0.1",
     "react-dom": "17.0.1",
     "react-i18next": "11.8.5",

src/App.tsx

@@ -77,9 +77,7 @@ const App: React.FC = () => {
               <Route path="/credits" render={() => <CreditsPage />} exact />
               <Route
                 path="/chapter/:chapterId/:sectionId"
-                render={(props) => {
-                  return <ChapterPage {...props} />;
-                }}
+                render={() => <ChapterPage />}
                 exact
               />
               <Route path="/" render={() => <Redirect to="/home" />} exact />

src/components/Chapters/ChapterViewPage.tsx

@@ -6,21 +6,9 @@ import {
   IonRow,
   IonText,
 } from '@ionic/react';
-import React, { ReactNode } from 'react';
-import { ChapterId } from '../../utils/chapters';
-
-interface ComponentProps {
-  chapterId: ChapterId;
-  content: ReactNode | string;
-  color?: string;
-  sectionId: string;
-  subTitle?: string;
-  title: string;
-}
-
-export const ChapterViewPage: React.FC<ComponentProps> = (props) => {
-  const { color, content, subTitle, title } = props;
+import React from 'react';
 
+export const ChapterViewPage = ({ color, content, subTitle, title }) => {
   return (
     <IonGrid class="ion-padding ion-text-justify">
       <IonRow class="ion-justify-content-center">
@@ -31,7 +19,7 @@ export const ChapterViewPage: React.FC<ComponentProps> = (props) => {
           <IonText>
             <h2 className="chapter-title">{title}</h2>
           </IonText>
-          {content}
+          {content()}
         </IonCol>
       </IonRow>
     </IonGrid>

src/components/Chapters/index.tsx

@@ -46,10 +46,8 @@ export const Chapter: React.FC<ContainerProps> = (props) => {
 
   return (
     <ChapterViewPage
-      chapterId={chapterId}
       color={color}
-      content={chapterView.page.content}
-      sectionId={sectionId}
+      content={chapterView.page.Content}
       subTitle={title.chapter}
       title={title.section}
     ></ChapterViewPage>

src/components/SpeechFabButton/SpeechFabButton.tsx

@@ -0,0 +1,33 @@
+import { IonFab, IonFabButton } from '@ionic/react';
+import React from 'react';
+import { useParams } from 'react-router';
+import { ChapterId } from '../../utils/chapters';
+import { formatSecondsToTimeMinutes } from '../../utils/format';
+import { useSpeechAudio } from './hooks/useSpeechAudio';
+import { SpeechFabButtonContent } from './SpeechFabButtonContent';
+
+interface RouteChapterProps {
+  chapterId: ChapterId;
+  sectionId: string;
+}
+
+export const SpeechFabButton = () => {
+  const { chapterId, sectionId } = useParams<RouteChapterProps>();
+  const { isLoading, isPlaying, pause, play, restTime } = useSpeechAudio(
+    chapterId,
+    sectionId,
+  );
+
+  return (
+    <IonFab vertical="top" horizontal="end" edge slot="fixed">
+      <IonFabButton
+        color="secondary"
+        onClick={() => (isPlaying ? pause() : play())}
+      >
+        <SpeechFabButtonContent isLoading={isLoading} isPlaying={isPlaying}>
+          {formatSecondsToTimeMinutes(restTime)}
+        </SpeechFabButtonContent>
+      </IonFabButton>
+    </IonFab>
+  );
+};

src/components/SpeechFabButton/SpeechFabButtonContent.tsx

@@ -0,0 +1,33 @@
+import { IonIcon, IonSpinner } from '@ionic/react';
+import { pause, play } from 'ionicons/icons';
+import React from 'react';
+import './styles.css';
+
+type Props = {
+  isLoading?: boolean;
+  isPlaying?: boolean;
+};
+
+export const SpeechFabButtonContent: React.FC<Props> = ({
+  isLoading = false,
+  isPlaying = false,
+  children,
+}) => {
+  return (
+    <>
+      {isLoading ? (
+        <IonSpinner />
+      ) : isPlaying ? (
+        <>
+          <IonIcon
+            className={children ? 'audio-fab-button-show-time' : ''}
+            icon={pause}
+          />
+          {children}
+        </>
+      ) : (
+        <IonIcon icon={play} />
+      )}
+    </>
+  );
+};

src/components/SpeechFabButton/hooks/useSpeechAudio.ts

@@ -0,0 +1,26 @@
+import { useEffect } from 'react';
+import { ChapterId } from '../../../utils/chapters';
+import { useChapterSectionSsml } from '../../../utils/hooks/useChapterSectionSsml';
+import { useAudio } from '../../../utils/hooks/useAudio';
+import { useSpeechSource } from './useSpeechSource';
+
+export const useSpeechAudio = (chapterId: ChapterId, sectionId: string) => {
+  const { filesystem, speak } = useChapterSectionSsml(chapterId, sectionId);
+
+  const { isLoading, load, source } = useSpeechSource(speak, filesystem.path);
+  const { isPlaying, pause, play, restTime } = useAudio(source);
+
+  useEffect(() => {
+    pause();
+    // eslint-disable-next-line react-hooks/exhaustive-deps
+  }, [speak]);
+
+  const playAudio = async () => {
+    if (!source) {
+      await load();
+    }
+    play();
+  };
+
+  return { isLoading, isPlaying, play: playAudio, pause, restTime };
+};

src/components/SpeechFabButton/hooks/useSpeechSource.ts

@@ -0,0 +1,32 @@
+import { useEffect, useState } from 'react';
+import {
+  existsFileInCache,
+  readFileFromCache,
+} from '../../../utils/filesystem';
+import { synthesizeSpeechToFile } from '../utils';
+
+export const useSpeechSource = (ssml: string, path: string) => {
+  const [source, setSource] = useState('');
+  const [isLoading, setIsLoading] = useState(false);
+
+  useEffect(() => {
+    setSource('');
+  }, [path]);
+
+  const load = async () => {
+    setIsLoading(true);
+    try {
+      const existFile = await existsFileInCache(path);
+      if (!existFile) {
+        await synthesizeSpeechToFile(ssml, path);
+      }
+      const file = await readFileFromCache(path);
+      setSource(file);
+    } catch (error) {
+      console.error(error);
+    }
+    setIsLoading(false);
+  };
+
+  return { isLoading, load, source };
+};

src/components/SpeechFabButton/index.tsx

@@ -0,0 +1 @@
+export { SpeechFabButton } from './SpeechFabButton';

src/components/SpeechFabButton/styles.css

@@ -0,0 +1,4 @@
+.audio-fab-button-show-time {
+  opacity: 0.25;
+  position: absolute;
+}

src/components/SpeechFabButton/utils.ts

@@ -0,0 +1,46 @@
+import {
+  AudioConfig,
+  AudioOutputStream,
+  SpeechConfig,
+  SpeechSynthesizer,
+} from 'microsoft-cognitiveservices-speech-sdk';
+import { writeAudioBufferToCache } from '../../utils/filesystem';
+
+const getAudioConfig = () => {
+  const stream = AudioOutputStream.createPullStream();
+  const audioConfig = AudioConfig.fromStreamOutput(stream);
+  return audioConfig;
+};
+
+const getSpeechConfig = () => {
+  const speechConfig = SpeechConfig.fromSubscription(
+    process.env.REACT_APP_AZURE_COGNITIVE_TTS_SUBSCRIPTION_KEY || '',
+    process.env.REACT_APP_AZURE_COGNITIVE_TTS_SERVICE_REGION || '',
+  );
+  return speechConfig;
+};
+
+export const synthesizeSpeechToFile = async (ssml: string, path: string) => {
+  const audioConfig = getAudioConfig();
+  const speechConfig = getSpeechConfig();
+  const synthesizer = new SpeechSynthesizer(speechConfig, audioConfig);
+
+  return new Promise<ArrayBuffer>((resolve, reject) =>
+    synthesizer.speakSsmlAsync(
+      ssml,
+      async (result) => {
+        synthesizer.close();
+        const { audioData, errorDetails } = result;
+        if (!audioData) {
+          reject(errorDetails);
+        }
+        await writeAudioBufferToCache(path, audioData, 'audio/wav');
+        resolve(audioData);
+      },
+      (error) => {
+        synthesizer.close();
+        reject(error);
+      },
+    ),
+  );
+};

src/data/chapter/chapter.tsx

@@ -1,3 +1,5 @@
+export type ChapterContext = '' | 'ssml';
+
 export const CHAPTER_COLOR = new Map([
   ['01', 'accent-step-0100'],
   ['02', 'accent-step-0200'],