diff --git a/packages/teleport/src/DesktopSession/DesktopSession.story.tsx b/packages/teleport/src/DesktopSession/DesktopSession.story.tsx index 8374a89f3..2f65a91ab 100644 --- a/packages/teleport/src/DesktopSession/DesktopSession.story.tsx +++ b/packages/teleport/src/DesktopSession/DesktopSession.story.tsx @@ -65,6 +65,7 @@ const props: State = { onContextMenu: () => false, onMouseEnter: () => {}, onClipboardData: () => {}, + windowOnFocus: () => {}, webauthn: { errorText: '', requested: false, diff --git a/packages/teleport/src/DesktopSession/DesktopSession.tsx b/packages/teleport/src/DesktopSession/DesktopSession.tsx index 5c5ae16bd..189ab7346 100644 --- a/packages/teleport/src/DesktopSession/DesktopSession.tsx +++ b/packages/teleport/src/DesktopSession/DesktopSession.tsx @@ -123,6 +123,7 @@ function Session(props: PropsWithChildren) { onMouseWheelScroll, onContextMenu, onMouseEnter, + windowOnFocus, } = props; const clipboardSharingActive = @@ -193,6 +194,7 @@ function Session(props: PropsWithChildren) { onMouseWheelScroll={onMouseWheelScroll} onContextMenu={onContextMenu} onMouseEnter={onMouseEnter} + windowOnFocus={windowOnFocus} /> ); diff --git a/packages/teleport/src/DesktopSession/useTdpClientCanvas.tsx b/packages/teleport/src/DesktopSession/useTdpClientCanvas.tsx index dd29bf2dc..f9b143347 100644 --- a/packages/teleport/src/DesktopSession/useTdpClientCanvas.tsx +++ b/packages/teleport/src/DesktopSession/useTdpClientCanvas.tsx @@ -33,7 +33,6 @@ export default function useTdpClientCanvas(props: Props) { } = props; const [tdpClient, setTdpClient] = useState(null); const initialTdpConnectionSucceeded = useRef(false); - const latestClipboardData = useRef(null); useEffect(() => { const { width, height } = getDisplaySize(); @@ -70,17 +69,8 @@ export default function useTdpClientCanvas(props: Props) { // Default TdpClientEvent.TDP_CLIPBOARD_DATA handler. const onClipboardData = (clipboardData: ClipboardData) => { - if ( - enableClipboardSharing && - document.hasFocus() && - clipboardData.data !== latestClipboardData.current - ) { - navigator.clipboard.writeText(clipboardData.data).then(() => { - // Set latestClipboardText.current to whatever we got, so that - // next time onMouseEnter fires we don't try to send it back to - // the remote machine. - latestClipboardData.current = clipboardData.data; - }); + if (enableClipboardSharing && document.hasFocus() && clipboardData.data) { + navigator.clipboard.writeText(clipboardData.data); } }; @@ -150,26 +140,12 @@ export default function useTdpClientCanvas(props: Props) { const sendLocalClipboardToRemote = (cli: TdpClient) => { // We must check that the DOM is focused or navigator.clipboard.readText throws an error. - // We check that initialTdpConnectionSucceeded so that we don't mistakenly send clipboard data - // to a backend that isn't ready for it yet, which would fail silently. - if ( - enableClipboardSharing && - document.hasFocus() && - initialTdpConnectionSucceeded.current - ) { + if (enableClipboardSharing && document.hasFocus()) { navigator.clipboard.readText().then(text => { - if (text != latestClipboardData.current) { - // Wrap in try catch so that lastCopiedClipboardText is only - // updated if sendClipboardData succeeds. - // eslint-disable-next-line no-useless-catch - try { - cli.sendClipboardData({ - data: text, - }); - latestClipboardData.current = text; - } catch (e) { - throw e; - } + if (text) { + cli.sendClipboardData({ + data: text, + }); } }); } @@ -182,6 +158,13 @@ export default function useTdpClientCanvas(props: Props) { sendLocalClipboardToRemote(cli); }; + // onMouseEnter does not fire in certain situations, so ensure we cover all of our bases by adding a window level + // onfocus handler. See https://github.com/gravitational/webapps/issues/626 for further details. + const windowOnFocus = (cli: TdpClient, e: FocusEvent) => { + e.preventDefault(); + sendLocalClipboardToRemote(cli); + }; + return { tdpClient, onPngFrame, @@ -197,6 +180,7 @@ export default function useTdpClientCanvas(props: Props) { onMouseWheelScroll, onContextMenu, onMouseEnter, + windowOnFocus, }; } diff --git a/packages/teleport/src/Player/DesktopPlayer.tsx b/packages/teleport/src/Player/DesktopPlayer.tsx index 17946ae0f..5f55aae31 100644 --- a/packages/teleport/src/Player/DesktopPlayer.tsx +++ b/packages/teleport/src/Player/DesktopPlayer.tsx @@ -1,9 +1,10 @@ -import React, { useState, useEffect } from 'react'; +import React, { useEffect, useState } from 'react'; import styled from 'styled-components'; -import { throttle } from 'lodash'; -import { dateToUtc } from 'shared/services/loc'; -import { format } from 'date-fns'; + +import { Indicator, Box, Alert } from 'design'; + +import useAttempt from 'shared/hooks/useAttemptNext'; import cfg from 'teleport/config'; import { PlayerClient, PlayerClientEvent } from 'teleport/lib/tdp'; @@ -11,7 +12,7 @@ import { PngFrame, ClientScreenSpec } from 'teleport/lib/tdp/codec'; import { getAccessToken, getHostName } from 'teleport/services/api'; import TdpClientCanvas from 'teleport/components/TdpClientCanvas'; -import ProgressBar from './ProgressBar'; +import { ProgressBarDesktop } from './ProgressBar'; export const DesktopPlayer = ({ sid, @@ -22,99 +23,60 @@ export const DesktopPlayer = ({ clusterId: string; durationMs: number; }) => { - const { playerClient, tdpCliOnPngFrame, tdpCliOnClientScreenSpec } = - useDesktopPlayer({ - sid, - clusterId, - }); + const { + playerClient, + tdpCliOnPngFrame, + tdpCliOnClientScreenSpec, + tdpCliOnWsClose, + tdpCliOnTdpError, + attempt, + } = useDesktopPlayer({ + sid, + clusterId, + }); + + const displayCanvas = attempt.status === 'success' || attempt.status === ''; + const displayProgressBar = attempt.status !== 'processing'; return ( + {attempt.status === 'processing' && ( + + + + )} + + {attempt.status === 'failed' && ( + + )} + true} // overflow: 'hidden' is needed to prevent the canvas from outgrowing the container due to some weird css flex idiosyncracy. // See https://gaurav5430.medium.com/css-flex-positioning-gotchas-child-expands-to-more-than-the-width-allowed-by-the-parent-799c37428dd6. - style={{ display: 'flex', flexGrow: 1, overflow: 'hidden' }} + style={{ + alignSelf: 'center', + overflow: 'hidden', + display: displayCanvas ? 'flex' : 'none', + }} + /> + - ); }; -export const ProgressBarDesktop = (props: { - playerClient: PlayerClient; - durationMs: number; -}) => { - const { playerClient, durationMs } = props; - - const toHuman = (currentMs: number) => { - return format(dateToUtc(new Date(currentMs)), 'mm:ss'); - }; - - const [state, setState] = useState({ - max: durationMs, - min: 0, - current: 0, // the recording always starts at 0 ms - time: toHuman(0), - isPlaying: true, // determines whether play or pause symbol is shown - }); - - useEffect(() => { - playerClient.addListener(PlayerClientEvent.TOGGLE_PLAY_PAUSE, () => { - // setState({...state, isPlaying: !state.isPlaying}) doesn't work because - // the listener is added when state == initialState, and that initialState - // value is effectively hardcoded into its logic. - setState(prevState => { - return { ...prevState, isPlaying: !prevState.isPlaying }; - }); - }); - - const throttledUpdateCurrentTime = throttle( - currentTimeMs => { - setState(prevState => { - return { - ...prevState, - current: currentTimeMs, - time: toHuman(currentTimeMs), - }; - }); - }, - // Magic number to throttle progress bar updates so that the playback is smoother. - 50 - ); - - playerClient.addListener( - PlayerClientEvent.UPDATE_CURRENT_TIME, - currentTimeMs => throttledUpdateCurrentTime(currentTimeMs) - ); - - playerClient.addListener(PlayerClientEvent.SESSION_END, () => { - throttledUpdateCurrentTime.cancel(); - // TODO(isaiah): Make this smoother - // https://github.com/gravitational/webapps/issues/579 - setState(prevState => { - return { ...prevState, current: durationMs }; - }); - }); - - return () => { - throttledUpdateCurrentTime.cancel(); - playerClient.nuke(); - }; - }, [playerClient]); - - return ( - playerClient.togglePlayPause()} - move={() => {}} - /> - ); -}; - const useDesktopPlayer = ({ sid, clusterId, @@ -122,13 +84,21 @@ const useDesktopPlayer = ({ sid: string; clusterId: string; }) => { - const playerClient = new PlayerClient( - cfg.api.desktopPlaybackWsAddr - .replace(':fqdn', getHostName()) - .replace(':clusterId', clusterId) - .replace(':sid', sid) - .replace(':token', getAccessToken()) - ); + const [playerClient, setPlayerClient] = useState(null); + // attempt.status === '' means the playback ended gracefully + const { attempt, setAttempt } = useAttempt('processing'); + + useEffect(() => { + setPlayerClient( + new PlayerClient( + cfg.api.desktopPlaybackWsAddr + .replace(':fqdn', getHostName()) + .replace(':clusterId', clusterId) + .replace(':sid', sid) + .replace(':token', getAccessToken()) + ) + ); + }, [clusterId, sid]); const tdpCliOnPngFrame = ( ctx: CanvasRenderingContext2D, @@ -141,20 +111,96 @@ const useDesktopPlayer = ({ canvas: HTMLCanvasElement, spec: ClientScreenSpec ) => { + const styledPlayer = canvas.parentElement; + const progressBar = styledPlayer.children.namedItem('progressBarDesktop'); + + const fullWidth = styledPlayer.clientWidth; + const fullHeight = styledPlayer.clientHeight - progressBar.clientHeight; + const originalAspectRatio = spec.width / spec.height; + const currentAspectRatio = fullWidth / fullHeight; + + if (originalAspectRatio > currentAspectRatio) { + // Use the full width of the screen and scale the height. + canvas.style.height = `${(fullWidth * spec.height) / spec.width}px`; + } else if (originalAspectRatio < currentAspectRatio) { + // Use the full height of the screen and scale the width. + canvas.style.width = `${(fullHeight * spec.width) / spec.height}px`; + } + canvas.width = spec.width; canvas.height = spec.height; + + setAttempt({ status: 'success' }); + }; + + useEffect(() => { + if (playerClient) { + playerClient.addListener(PlayerClientEvent.SESSION_END, () => { + setAttempt({ status: '' }); + }); + + playerClient.addListener( + PlayerClientEvent.PLAYBACK_ERROR, + (err: Error) => { + setAttempt({ + status: 'failed', + statusText: `There was an error while playing this session: ${err.message}`, + }); + } + ); + + return () => { + playerClient.nuke(); + }; + } + }, [playerClient]); + + // If the websocket closed for some reason other than the session playback ending, + // as signaled by the server (which sets prevAttempt.status = '' in + // the PlayerClientEvent.SESSION_END event handler), or a TDP message from the server + // signalling an error, assume some sort of network or playback error and alert the user. + const tdpCliOnWsClose = () => { + setAttempt(prevAttempt => { + if (prevAttempt.status !== '' && prevAttempt.status !== 'failed') { + return { + status: 'failed', + statusText: 'connection to the server failed for an unknown reason', + }; + } + return prevAttempt; + }); + }; + + const tdpCliOnTdpError = (err: Error) => { + setAttempt({ + status: 'failed', + statusText: err.message, + }); }; return { playerClient, tdpCliOnPngFrame, tdpCliOnClientScreenSpec, + tdpCliOnWsClose, + tdpCliOnTdpError, + attempt, }; }; const StyledPlayer = styled.div` display: flex; flex-direction: column; + justify-content: center; width: 100%; height: 100%; `; + +const DesktopPlayerAlert = styled(Alert)` + align-self: center; + min-width: 450px; + + // Overrides StyledPlayer container's justify-content + // https://stackoverflow.com/a/34063808/6277051 + margin-bottom: auto; +`; diff --git a/packages/teleport/src/Player/ProgressBar/ProgressBar.tsx b/packages/teleport/src/Player/ProgressBar/ProgressBar.tsx index cbe9844ab..a24759074 100644 --- a/packages/teleport/src/Player/ProgressBar/ProgressBar.tsx +++ b/packages/teleport/src/Player/ProgressBar/ProgressBar.tsx @@ -23,7 +23,7 @@ import Slider from './Slider'; export default function ProgressBar(props: ProgressBarProps) { const Icon = props.isPlaying ? Icons.CirclePause : Icons.CirclePlay; return ( - + @@ -51,6 +51,8 @@ export type ProgressBarProps = { current: number; move: (value: any) => void; toggle: () => void; + style?: React.CSSProperties; + id?: string; }; const SliderContainer = styled.div` diff --git a/packages/teleport/src/Player/ProgressBar/ProgressBarDesktop.tsx b/packages/teleport/src/Player/ProgressBar/ProgressBarDesktop.tsx new file mode 100644 index 000000000..376da8df2 --- /dev/null +++ b/packages/teleport/src/Player/ProgressBar/ProgressBarDesktop.tsx @@ -0,0 +1,134 @@ +import React, { useState, useEffect, useRef } from 'react'; + +import { throttle } from 'lodash'; +import { dateToUtc } from 'shared/services/loc'; +import { format } from 'date-fns'; + +import { + PlayerClient, + PlayerClientEvent, + TdpClientEvent, +} from 'teleport/lib/tdp'; + +import ProgressBar from './ProgressBar'; + +export const ProgressBarDesktop = (props: { + playerClient: PlayerClient; + durationMs: number; + style?: React.CSSProperties; + id?: string; +}) => { + const { playerClient, durationMs } = props; + const intervalRef = useRef(); + + const toHuman = (currentMs: number) => { + return format(dateToUtc(new Date(currentMs)), 'mm:ss'); + }; + + const [state, setState] = useState({ + max: durationMs, + min: 0, + current: 0, // the recording always starts at 0 ms + time: toHuman(0), + isPlaying: true, // determines whether play or pause symbol is shown + }); + + // updateCurrentTime is a helper function to update the state variable. + // It should be used within a setState, like + // setState(prevState => { + // return updateCurrentTime(prevState, newTime) + // }) + const updateCurrentTime = ( + prevState: typeof state, + currentTimeMs: number + ) => { + return { + ...prevState, + current: currentTimeMs, + time: toHuman(currentTimeMs), + }; + }; + + useEffect(() => { + if (playerClient) { + // Starts the smoothing interval, which smooths out the progress of the progress bar. + // This ensures the bar continues to progress even during playbacks where there are long + // intervals between TDP events sent to us by the server. The interval should be active + // whenever the playback is in "play" mode. + const smoothOutProgress = () => { + const smoothingInterval = 25; + + intervalRef.current = setInterval(() => { + setState(prevState => { + const nextTimeMs = prevState.current + smoothingInterval; + if (nextTimeMs <= durationMs) { + return updateCurrentTime(prevState, nextTimeMs); + } else { + stopProgress(); + return updateCurrentTime(prevState, durationMs); + } + }); + }, smoothingInterval); + }; + + // The player always starts in play mode, so call this initially. + smoothOutProgress(); + + // Clears the smoothing interval and cancels any throttled updates, + // should be called when the playback is paused or ended. + const stopProgress = () => { + throttledUpdateCurrentTime.cancel(); + clearInterval(intervalRef.current); + }; + + const throttledUpdateCurrentTime = throttle( + currentTimeMs => { + setState(prevState => { + return updateCurrentTime(prevState, currentTimeMs); + }); + }, + // Magic number to throttle progress bar updates caused by TDP events + // so that the playback is smoother. + 50 + ); + + // Listens for UPDATE_CURRENT_TIME events which coincide with + // TDP events sent to the playerClient by the server. + playerClient.addListener( + PlayerClientEvent.UPDATE_CURRENT_TIME, + currentTimeMs => throttledUpdateCurrentTime(currentTimeMs) + ); + + playerClient.addListener(PlayerClientEvent.TOGGLE_PLAY_PAUSE, () => { + // setState({...state, isPlaying: !state.isPlaying}) doesn't work because + // the listener is added when state == initialState, and that initialState + // value is effectively hardcoded into its logic. + setState(prevState => { + if (prevState.isPlaying) { + // pause + stopProgress(); + } else { + // play + smoothOutProgress(); + } + return { ...prevState, isPlaying: !prevState.isPlaying }; + }); + }); + + return () => { + playerClient.nuke(); + stopProgress(); + }; + } + }, [playerClient]); + + return ( + playerClient.togglePlayPause()} + move={() => {}} + style={props.style} + id={props.id} + /> + ); +}; diff --git a/packages/teleport/src/Player/ProgressBar/index.tsx b/packages/teleport/src/Player/ProgressBar/index.tsx index ef4fc98ac..480df13f9 100644 --- a/packages/teleport/src/Player/ProgressBar/index.tsx +++ b/packages/teleport/src/Player/ProgressBar/index.tsx @@ -16,6 +16,7 @@ limitations under the License. import ProgressBar from './ProgressBar'; import ProgressBarTty from './ProgressBarTty'; +import { ProgressBarDesktop } from './ProgressBarDesktop'; export default ProgressBar; -export { ProgressBarTty }; +export { ProgressBarTty, ProgressBarDesktop }; diff --git a/packages/teleport/src/components/TdpClientCanvas/TdpClientCanvas.tsx b/packages/teleport/src/components/TdpClientCanvas/TdpClientCanvas.tsx index 3af3d63d0..1dd579649 100644 --- a/packages/teleport/src/components/TdpClientCanvas/TdpClientCanvas.tsx +++ b/packages/teleport/src/components/TdpClientCanvas/TdpClientCanvas.tsx @@ -38,6 +38,7 @@ export default function TdpClientCanvas(props: Props) { onMouseWheelScroll, onContextMenu, onMouseEnter, + windowOnFocus, style, } = props; @@ -261,11 +262,25 @@ export default function TdpClientCanvas(props: Props) { } return () => { - if (onMouseEnter) - canvas.removeEventListener('onmouseenter', _onmouseenter); + if (onMouseEnter) canvas.removeEventListener('mouseenter', _onmouseenter); }; }, [onMouseEnter]); + useEffect(() => { + const _windowonfocus = (e: FocusEvent) => { + // Checking for (canvasRef.current.style.display !== 'none') ensures windowOnFocus behaves + // like the other passed event listeners, namely it isn't called if the TdpClientCanvas isn't displayed. + if (canvasRef.current.style.display !== 'none') windowOnFocus(tdpCli, e); + }; + if (windowOnFocus) { + window.onfocus = _windowonfocus; + } + + return () => { + if (windowOnFocus) window.removeEventListener('focus', _windowonfocus); + }; + }, [windowOnFocus]); + return ; } @@ -295,5 +310,6 @@ export type Props = { onMouseWheelScroll?: (cli: TdpClient, e: WheelEvent) => void; onContextMenu?: () => boolean; onMouseEnter?: (cli: TdpClient, e: MouseEvent) => void; + windowOnFocus?: (cli: TdpClient, e: FocusEvent) => void; style?: CSSProperties; }; diff --git a/packages/teleport/src/lib/tdp/client.ts b/packages/teleport/src/lib/tdp/client.ts index 484889f59..0f211e4fb 100644 --- a/packages/teleport/src/lib/tdp/client.ts +++ b/packages/teleport/src/lib/tdp/client.ts @@ -228,6 +228,8 @@ export default class Client extends EventEmitterWebAuthnSender { // Ensures full cleanup of this object. // Note that it removes all listeners first and then cleans up the socket, // so don't call this if your calling object is relying on listeners. + // It's safe to call this multiple times, calls subsequent to the first call + // will simply do nothing. nuke() { this.removeAllListeners(); this.socket?.close(); diff --git a/packages/teleport/src/lib/tdp/playerClient.ts b/packages/teleport/src/lib/tdp/playerClient.ts index e0c190dc5..2cd1c344d 100644 --- a/packages/teleport/src/lib/tdp/playerClient.ts +++ b/packages/teleport/src/lib/tdp/playerClient.ts @@ -24,6 +24,7 @@ export enum PlayerClientEvent { TOGGLE_PLAY_PAUSE = 'play/pause', UPDATE_CURRENT_TIME = 'time', SESSION_END = 'end', + PLAYBACK_ERROR = 'playback error', } export class PlayerClient extends Client { @@ -42,13 +43,16 @@ export class PlayerClient extends Client { // Overrides Client implementation. processMessage(buffer: ArrayBuffer) { const json = JSON.parse(this.textDecoder.decode(buffer)); + if (json.message === 'end') { this.emit(PlayerClientEvent.SESSION_END); - return; + } else if (json.message === 'error') { + this.emit(PlayerClientEvent.PLAYBACK_ERROR, new Error(json.errorText)); + } else { + const ms = json.ms; + this.emit(PlayerClientEvent.UPDATE_CURRENT_TIME, ms); + super.processMessage(base64ToArrayBuffer(json.message)); } - const ms = json.ms; - super.processMessage(base64ToArrayBuffer(json.message)); - this.emit(PlayerClientEvent.UPDATE_CURRENT_TIME, ms); } // Overrides Client implementation.