Add caption renderer

src/components/CaptionRenderer/CaptionRenderer.tsx

@@ -0,0 +1,113 @@
+import React, { useCallback, useEffect, useState } from 'react';
+import { makeStyles } from '@material-ui/core/styles';
+import { TwilioCaptionResult } from './CaptionTypes';
+import { Typography } from '@material-ui/core';
+import useParticipants from '../../hooks/useParticipants/useParticipants';
+import useTracks from '../../hooks/useTracks/useTracks';
+import { useAppState } from '../../state';
+
+interface Caption {
+  identity: string;
+  id: string;
+  timestamp: number;
+  transcript: string;
+}
+
+const useStyles = makeStyles({
+  captionContainer: {
+    position: 'fixed',
+    left: '15%',
+    right: '15%',
+    top: 'calc(100% - 300px)',
+    zIndex: 100,
+  },
+  caption: {
+    color: 'white',
+    background: 'rgba(0, 0, 0, 0.8)',
+    padding: '0.2em',
+    display: 'inline-block',
+  },
+});
+
+export function CaptionRenderer() {
+  const classes = useStyles();
+  const [captions, setCaptions] = useState<Caption[]>([]);
+  const participants = useParticipants();
+  const transcriberParticipant = participants.find(p => p.identity === 'media-transcriber');
+  const transcriberTracks = useTracks(transcriberParticipant);
+  const transcriberDataTrack = transcriberTracks.find(track => track.kind === 'data');
+  const { displayCaptions } = useAppState();
+
+  const registerResult = useCallback((result: TwilioCaptionResult) => {
+    if (result.transcriptionResponse.TranscriptEvent.Transcript.Results.length) {
+      const transcript = result.transcriptionResponse.TranscriptEvent.Transcript.Results[0].Alternatives[0].Transcript;
+      const id = result.transcriptionResponse.TranscriptEvent.Transcript.Results[0].ResultId;
+      const timestamp = Date.now();
+      const identity = result.participantIdentity;
+
+      setCaptions(prevCaptions => {
+        // Make a copy of the caption array, keeping only the 4 most recent captions
+        const arrayCopy = prevCaptions.slice(-4);
+
+        const existingID = arrayCopy.find(item => item.id === id);
+        if (existingID) {
+          const existingIdIndex = arrayCopy.indexOf(existingID);
+          arrayCopy[existingIdIndex] = { transcript, id, timestamp, identity };
+        } else {
+          arrayCopy.push({ transcript, id, timestamp, identity });
+        }
+
+        return arrayCopy;
+      });
+    }
+  }, []);
+
+  useEffect(() => {
+    if (transcriberDataTrack) {
+      const handleMessage = (message: string) => {
+        try {
+          registerResult(JSON.parse(message));
+        } catch (e) {
+          console.log('received unexpected dataTrack message: ', message);
+        }
+      };
+      transcriberDataTrack.on('message', handleMessage);
+
+      return () => {
+        transcriberDataTrack.on('message', handleMessage);
+      };
+    }
+  }, [transcriberDataTrack, registerResult]);
+
+  // Every second, we go through the captions, and remove any that are older than ten seconds
+  useEffect(() => {
+    const intervalId = setInterval(() => {
+      setCaptions(prevCaptions => {
+        const now = Date.now();
+        const filteredCaptions = prevCaptions.filter(caption => caption.timestamp > now - 10000);
+        if (filteredCaptions.length !== prevCaptions.length) {
+          return filteredCaptions;
+        } else {
+          return prevCaptions;
+        }
+      });
+    }, 1000);
+    return () => {
+      clearInterval(intervalId);
+    };
+  }, []);
+
+  if (!displayCaptions) return null;
+
+  return (
+    <div className={classes.captionContainer}>
+      {captions.map(caption => (
+        <div>
+          <Typography variant="h6" key={caption.id} className={classes.caption}>
+            {caption.identity}: {caption.transcript}
+          </Typography>
+        </div>
+      ))}
+    </div>
+  );
+}

src/components/CaptionRenderer/CaptionTypes.ts

@@ -0,0 +1,37 @@
+export interface TwilioCaptionResult {
+  transcriptionResponse: TranscriptionResponse;
+  participantIdentity: string;
+}
+
+export interface TranscriptionResponse {
+  TranscriptEvent: TranscriptEvent;
+}
+
+export interface TranscriptEvent {
+  Transcript: Transcript;
+}
+
+export interface Transcript {
+  Results: Result[];
+}
+
+export interface Result {
+  Alternatives: Alternative[];
+  EndTime: number;
+  IsPartial: boolean;
+  ResultId: string;
+  StartTime: number;
+}
+
+export interface Alternative {
+  Items: Item[];
+  Transcript: string;
+}
+
+export interface Item {
+  Content: string;
+  EndTime: number;
+  StartTime: number;
+  Type: string;
+  VocabularyFilterMatch: boolean;
+}

src/components/MenuBar/Menu/Menu.tsx

@@ -1,6 +1,7 @@
 import React, { useState, useRef } from 'react';
 import AboutDialog from '../../AboutDialog/AboutDialog';
 import BackgroundIcon from '../../../icons/BackgroundIcon';
+import ClosedCaptionsIcon from '@material-ui/icons/ClosedCaption';
 import DeviceSelectionDialog from '../../DeviceSelectionDialog/DeviceSelectionDialog';
 import ExpandMoreIcon from '@material-ui/icons/ExpandMore';
 import InfoIconOutlined from '../../../icons/InfoIconOutlined';
@@ -34,7 +35,7 @@ export default function Menu(props: { buttonClassName?: string }) {
   const [menuOpen, setMenuOpen] = useState(false);
   const [settingsOpen, setSettingsOpen] = useState(false);
 
-  const { isFetching, updateRecordingRules, roomType } = useAppState();
+  const { isFetching, updateRecordingRules, roomType, displayCaptions, setDisplayCaptions } = useAppState();
   const { setIsChatWindowOpen } = useChatContext();
   const isRecording = useIsRecording();
   const { room, setIsBackgroundSelectionOpen } = useVideoContext();
@@ -133,6 +134,13 @@ export default function Menu(props: { buttonClassName?: string }) {
           <Typography variant="body1">Room Monitor</Typography>
         </MenuItem>
 
+        <MenuItem onClick={() => setDisplayCaptions(prevDisplayCaptions => !prevDisplayCaptions)}>
+          <IconContainer>
+            <ClosedCaptionsIcon />
+          </IconContainer>
+          <Typography variant="body1">{displayCaptions ? 'Hide Captions' : 'Show Captions'}</Typography>
+        </MenuItem>
+
         <MenuItem onClick={() => setAboutOpen(true)}>
           <IconContainer>
             <InfoIconOutlined />

src/components/ParticipantList/ParticipantList.tsx

@@ -45,7 +45,7 @@ export default function ParticipantList() {
   const classes = useStyles();
   const { room } = useVideoContext();
   const localParticipant = room!.localParticipant;
-  const participants = useParticipants();
+  const participants = useParticipants().filter(p => p.identity !== 'media-transcriber');
   const [selectedParticipant, setSelectedParticipant] = useSelectedParticipant();
   const screenShareParticipant = useScreenShareParticipant();
   const mainParticipant = useMainParticipant();

src/components/Room/Room.tsx

@@ -1,6 +1,7 @@
 import React from 'react';
 import clsx from 'clsx';
 import { makeStyles, Theme } from '@material-ui/core';
+import { CaptionRenderer } from '../CaptionRenderer/CaptionRenderer';
 import ChatWindow from '../ChatWindow/ChatWindow';
 import ParticipantList from '../ParticipantList/ParticipantList';
 import MainParticipant from '../MainParticipant/MainParticipant';
@@ -42,6 +43,7 @@ export default function Room() {
       <ParticipantList />
       <ChatWindow />
       <BackgroundSelectionDialog />
+      <CaptionRenderer />
     </div>
   );
 }

src/hooks/useMainParticipant/useMainParticipant.tsx

@@ -8,7 +8,7 @@ export default function useMainParticipant() {
   const [selectedParticipant] = useSelectedParticipant();
   const screenShareParticipant = useScreenShareParticipant();
   const dominantSpeaker = useDominantSpeaker();
-  const participants = useParticipants();
+  const participants = useParticipants().filter(p => p.identity !== 'media-transcriber');
   const { room } = useVideoContext();
   const localParticipant = room?.localParticipant;
   const remoteScreenShareParticipant = screenShareParticipant !== localParticipant ? screenShareParticipant : null;

src/hooks/useTracks/useTracks.test.ts

@@ -0,0 +1,58 @@
+import { renderHook, act } from '@testing-library/react-hooks';
+import EventEmitter from 'events';
+import useTracks from './useTracks';
+
+describe('the useTracks hook', () => {
+  let mockParticipant: any;
+
+  beforeEach(() => {
+    mockParticipant = new EventEmitter();
+    mockParticipant.tracks = new Map([
+      [0, { track: 'track1' }],
+      [1, { track: null }],
+      [2, { track: 'track2' }],
+    ]);
+  });
+
+  it('should return an array of mockParticipant.tracks by default, filtering out null tracks', () => {
+    const { result } = renderHook(() => useTracks(mockParticipant));
+    expect(result.current).toEqual(['track1', 'track2']);
+  });
+
+  it('should respond to "trackSubscribed" events', async () => {
+    const { result } = renderHook(() => useTracks(mockParticipant));
+    act(() => {
+      mockParticipant.emit('trackSubscribed', 'newMockTrack');
+    });
+    expect(result.current).toEqual(['track1', 'track2', 'newMockTrack']);
+  });
+
+  it('should respond to "trackUnsubscribed" events', async () => {
+    const { result } = renderHook(() => useTracks(mockParticipant));
+    act(() => {
+      mockParticipant.emit('trackUnsubscribed', 'track1');
+    });
+    expect(result.current).toEqual(['track2']);
+  });
+
+  it('should return a new set of tracks if the participant changes', () => {
+    const { result, rerender } = renderHook(({ participant }) => useTracks(participant), {
+      initialProps: { participant: mockParticipant },
+    });
+    expect(result.current).toEqual(['track1', 'track2']);
+    mockParticipant = new EventEmitter();
+    mockParticipant.tracks = new Map([
+      [0, { track: 'track3' }],
+      [1, { track: 'track4' }],
+    ]);
+    rerender({ participant: mockParticipant });
+    expect(result.current).toEqual(['track3', 'track4']);
+  });
+
+  it('should clean up listeners on unmount', () => {
+    const { unmount } = renderHook(() => useTracks(mockParticipant));
+    unmount();
+    expect(mockParticipant.listenerCount('trackSubscribed')).toBe(0);
+    expect(mockParticipant.listenerCount('trackUnsubscribed')).toBe(0);
+  });
+});

src/hooks/useTracks/useTracks.ts

@@ -0,0 +1,29 @@
+import { RemoteParticipant, RemoteTrack } from 'twilio-video';
+import { useEffect, useState } from 'react';
+
+export default function useTracks(participant: RemoteParticipant | undefined) {
+  const [tracks, setTracks] = useState<RemoteTrack[]>([]);
+
+  useEffect(() => {
+    if (participant) {
+      const subscribedTracks = Array.from(participant.tracks.values())
+        .filter(trackPublication => trackPublication.track !== null)
+        .map(trackPublication => trackPublication.track!);
+
+      setTracks(subscribedTracks);
+
+      const handleTrackSubscribed = (track: RemoteTrack) => setTracks(prevTracks => [...prevTracks, track]);
+      const handleTrackUnsubscribed = (track: RemoteTrack) =>
+        setTracks(prevTracks => prevTracks.filter(t => t !== track));
+
+      participant.on('trackSubscribed', handleTrackSubscribed);
+      participant.on('trackUnsubscribed', handleTrackUnsubscribed);
+      return () => {
+        participant.off('trackSubscribed', handleTrackSubscribed);
+        participant.off('trackUnsubscribed', handleTrackUnsubscribed);
+      };
+    }
+  }, [participant]);
+
+  return tracks;
+}

src/state/index.tsx

@@ -22,6 +22,8 @@ export interface StateContextType {
   dispatchSetting: React.Dispatch<SettingsAction>;
   roomType?: RoomType;
   updateRecordingRules(room_sid: string, rules: RecordingRules): Promise<object>;
+  displayCaptions: boolean;
+  setDisplayCaptions: React.Dispatch<React.SetStateAction<boolean>>;
 }
 
 export const StateContext = createContext<StateContextType>(null!);
@@ -41,6 +43,7 @@ export default function AppStateProvider(props: React.PropsWithChildren<{}>) {
   const [activeSinkId, setActiveSinkId] = useActiveSinkId();
   const [settings, dispatchSetting] = useReducer(settingsReducer, initialSettings);
   const [roomType, setRoomType] = useState<RoomType>();
+  const [displayCaptions, setDisplayCaptions] = useState(false);
 
   let contextValue = {
     error,
@@ -51,6 +54,8 @@ export default function AppStateProvider(props: React.PropsWithChildren<{}>) {
     settings,
     dispatchSetting,
     roomType,
+    displayCaptions,
+    setDisplayCaptions,
   } as StateContextType;
 
   if (process.env.REACT_APP_SET_AUTH === 'firebase') {