feat(audio) [closes #230] Screenshare audio control (#248)

* feat(ui): present mic volume icon * feat(ui): improve mic volume display * refactor(ui): nest mic audio as a channel * fix(ui): prevent volume control from reappearing for returning peers * refactor(audio): update specific audio channel states * refactor(audio): use enum for audio channel name * refactor(types): improve audio type names * feat(audio): wire up screen share audio * refactor(networking): always provide stream metadata * fix(audio): remove screen audio when stream ends * fix(audio): stop audio when removing it * feat(audio): show appropriate icon for channel * fix(audio): clean up audio for leaving peers consistently * fix(audio): use up-to-date peerAudios reference * refactor(audio): simplify audio state updating * refactor(audio): use functional setState to update peer list * refactor(variables): rename peerAudios to peerAudioChannels * refactor(types): consolidate stream types * refactor(types): require stream type metadata
jeremyckahn · Apr 2, 2024 · 05b4615 · 05b4615
1 parent 89abe71
commit 05b4615
Show file tree

Hide file tree

Showing 13 changed files with 345 additions and 122 deletions.
diff --git a/src/components/AudioVolume/AudioVolume.tsx b/src/components/AudioVolume/AudioVolume.tsx
@@ -1,16 +1,25 @@
 import { useState, useEffect } from 'react'
 import Slider from '@mui/material/Slider'
 import Box from '@mui/material/Box'
+import Paper from '@mui/material/Paper'
 import ListItemIcon from '@mui/material/ListItemIcon'
-import VolumeUp from '@mui/icons-material/VolumeUp'
-import VolumeDown from '@mui/icons-material/VolumeDown'
-import VolumeMute from '@mui/icons-material/VolumeMute'
+import VolumeUpIcon from '@mui/icons-material/VolumeUp'
+import VolumeDownIcon from '@mui/icons-material/VolumeDown'
+import VolumeMuteIcon from '@mui/icons-material/VolumeMute'
+import MicIcon from '@mui/icons-material/Mic'
+import LaptopWindowsIcon from '@mui/icons-material/LaptopWindows'
+import Tooltip from '@mui/material/Tooltip'
+import { AudioChannelName } from 'models/chat'
 
 interface AudioVolumeProps {
   audioEl: HTMLAudioElement
+  audioChannelName: AudioChannelName
 }
 
-export const AudioVolume = ({ audioEl }: AudioVolumeProps) => {
+export const AudioVolume = ({
+  audioEl,
+  audioChannelName,
+}: AudioVolumeProps) => {
   const [audioVolume, setAudioVolume] = useState(audioEl.volume)
 
   useEffect(() => {
@@ -32,27 +41,48 @@ export const AudioVolume = ({ audioEl }: AudioVolumeProps) => {
 
   const formatLabelValue = () => `${Math.round(audioVolume * 100)}%`
 
-  let VolumeIcon = VolumeUp
+  let VolumeIcon = VolumeUpIcon
 
   if (audioVolume === 0) {
-    VolumeIcon = VolumeMute
+    VolumeIcon = VolumeMuteIcon
   } else if (audioVolume < 0.5) {
-    VolumeIcon = VolumeDown
+    VolumeIcon = VolumeDownIcon
   }
 
   return (
-    <Box sx={{ display: 'flex', pt: 1, pr: 3, alignItems: 'center' }}>
-      <ListItemIcon>
-        <VolumeIcon sx={{ cursor: 'pointer' }} onClick={handleIconClick} />
+    <Paper
+      sx={{
+        alignItems: 'center',
+        display: 'flex',
+        mt: 1.5,
+        pl: 2,
+        pr: 3,
+        py: 1,
+      }}
+    >
+      <ListItemIcon sx={{ cursor: 'pointer' }} onClick={handleIconClick}>
+        <VolumeIcon fontSize="small" />
+        {audioChannelName === AudioChannelName.MICROPHONE && (
+          <Tooltip title="Their microphone volume">
+            <MicIcon fontSize="small" sx={{ ml: 1, mr: 2 }} />
+          </Tooltip>
+        )}
+        {audioChannelName === AudioChannelName.SCREEN_SHARE && (
+          <Tooltip title="Their screen's volume">
+            <LaptopWindowsIcon fontSize="small" sx={{ ml: 1, mr: 2 }} />
+          </Tooltip>
+        )}
       </ListItemIcon>
-      <Slider
-        aria-label="Volume"
-        getAriaValueText={formatLabelValue}
-        valueLabelFormat={formatLabelValue}
-        valueLabelDisplay="auto"
-        onChange={handleSliderChange}
-        value={audioVolume * 100}
-      ></Slider>
-    </Box>
+      <Box display="flex" width={1}>
+        <Slider
+          aria-label="Volume"
+          getAriaValueText={formatLabelValue}
+          valueLabelFormat={formatLabelValue}
+          valueLabelDisplay="auto"
+          onChange={handleSliderChange}
+          value={audioVolume * 100}
+        ></Slider>
+      </Box>
+    </Paper>
   )
 }
diff --git a/src/components/Room/PeerVideo.tsx b/src/components/Room/PeerVideo.tsx
@@ -3,7 +3,7 @@ import Paper from '@mui/material/Paper'
 import Tooltip from '@mui/material/Tooltip'
 
 import { PeerNameDisplay } from 'components/PeerNameDisplay'
-import { VideoStreamType } from 'models/chat'
+import { StreamType } from 'models/chat'
 
 import { SelectedPeerStream } from './RoomVideoDisplay'
 
@@ -13,13 +13,13 @@ interface PeerVideoProps {
   numberOfVideos: number
   onVideoClick?: (
     userId: string,
-    videoStreamType: VideoStreamType,
+    streamType: StreamType,
     videoStream: MediaStream
   ) => void
   selectedPeerStream: SelectedPeerStream | null
   userId: string
   videoStream: MediaStream
-  videoStreamType: VideoStreamType
+  streamType: StreamType
 }
 
 // Adapted from https://www.geeksforgeeks.org/find-the-next-perfect-square-greater-than-a-given-number/
@@ -37,7 +37,7 @@ export const PeerVideo = ({
   userId,
   selectedPeerStream,
   videoStream,
-  videoStreamType,
+  streamType,
 }: PeerVideoProps) => {
   const videoRef = useRef<HTMLVideoElement>(null)
 
@@ -47,13 +47,14 @@ export const PeerVideo = ({
 
     video.autoplay = true
     video.srcObject = videoStream
+    video.muted = true
   }, [videoRef, videoStream])
 
   const cols = Math.sqrt(nextPerfectSquare(numberOfVideos - 1))
   const rows = Math.ceil(numberOfVideos / cols)
 
   const handleVideoClick = () => {
-    onVideoClick?.(userId, videoStreamType, videoStream)
+    onVideoClick?.(userId, streamType, videoStream)
   }
 
   return (

diff --git a/src/components/Room/RoomVideoDisplay.tsx b/src/components/Room/RoomVideoDisplay.tsx
@@ -4,7 +4,7 @@ import Paper from '@mui/material/Paper'
 
 import { RoomContext } from 'contexts/RoomContext'
 import { ShellContext } from 'contexts/ShellContext'
-import { Peer, VideoStreamType } from 'models/chat'
+import { Peer, StreamType } from 'models/chat'
 
 import { PeerVideo } from './PeerVideo'
 
@@ -16,7 +16,7 @@ interface PeerWithVideo {
 
 export interface SelectedPeerStream {
   peerId: string
-  videoStreamType: VideoStreamType
+  streamType: StreamType
   videoStream: MediaStream
 }
 
@@ -105,13 +105,13 @@ export const RoomVideoDisplay = ({
 
   const handleVideoClick = (
     peerId: string,
-    videoStreamType: VideoStreamType,
+    streamType: StreamType,
     videoStream: MediaStream
   ) => {
     if (selectedPeerStream?.videoStream === videoStream) {
       setSelectedPeerStream(null)
     } else if (numberOfVideos > 1) {
-      setSelectedPeerStream({ peerId, videoStreamType, videoStream })
+      setSelectedPeerStream({ peerId, streamType, videoStream })
     }
   }
 
@@ -139,7 +139,7 @@ export const RoomVideoDisplay = ({
             userId={selectedPeerStream.peerId}
             selectedPeerStream={selectedPeerStream}
             videoStream={selectedPeerStream.videoStream}
-            videoStreamType={selectedPeerStream.videoStreamType}
+            streamType={selectedPeerStream.streamType}
           />
         </Box>
       )}
@@ -168,7 +168,7 @@ export const RoomVideoDisplay = ({
             userId={userId}
             selectedPeerStream={selectedPeerStream}
             videoStream={selfVideoStream}
-            videoStreamType={VideoStreamType.WEBCAM}
+            streamType={StreamType.WEBCAM}
           />
         )}
         {selfScreenStream && (
@@ -179,7 +179,7 @@ export const RoomVideoDisplay = ({
             userId={userId}
             selectedPeerStream={selectedPeerStream}
             videoStream={selfScreenStream}
-            videoStreamType={VideoStreamType.SCREEN_SHARE}
+            streamType={StreamType.SCREEN_SHARE}
           />
         )}
         {peersWithVideo.map(peerWithVideo => (
@@ -191,7 +191,7 @@ export const RoomVideoDisplay = ({
                 userId={peerWithVideo.peer.userId}
                 selectedPeerStream={selectedPeerStream}
                 videoStream={peerWithVideo.videoStream}
-                videoStreamType={VideoStreamType.WEBCAM}
+                streamType={StreamType.WEBCAM}
               />
             )}
             {peerWithVideo.screenStream && (
@@ -201,7 +201,7 @@ export const RoomVideoDisplay = ({
                 userId={peerWithVideo.peer.userId}
                 selectedPeerStream={selectedPeerStream}
                 videoStream={peerWithVideo.screenStream}
-                videoStreamType={VideoStreamType.SCREEN_SHARE}
+                streamType={StreamType.SCREEN_SHARE}
               />
             )}
           </Fragment>

diff --git a/src/components/Room/useRoom.ts b/src/components/Room/useRoom.ts
@@ -23,6 +23,7 @@ import {
   TypingStatus,
   Peer,
   PeerVerificationState,
+  AudioChannelName,
 } from 'models/chat'
 import { getPeerName, usePeerNameDisplay } from 'components/PeerNameDisplay'
 import { Audio } from 'lib/Audio'
@@ -269,7 +270,10 @@ export function useRoom(
           userId,
           publicKey,
           customUsername,
-          audioState: AudioState.STOPPED,
+          audioChannelState: {
+            [AudioChannelName.MICROPHONE]: AudioState.STOPPED,
+            [AudioChannelName.SCREEN_SHARE]: AudioState.STOPPED,
+          },
           videoState: VideoState.STOPPED,
           screenShareState: ScreenShareState.NOT_SHARING,
           offeredFileId: null,