diff --git a/packages/core/src/buffer-until-first-frame.ts b/packages/core/src/buffer-until-first-frame.ts index 31bd50a4818..3153d0d80cd 100644 --- a/packages/core/src/buffer-until-first-frame.ts +++ b/packages/core/src/buffer-until-first-frame.ts @@ -4,60 +4,72 @@ import {useBufferState} from './use-buffer-state'; export const useBufferUntilFirstFrame = ({ mediaRef, mediaType, + onVariableFpsVideoDetected, }: { mediaRef: React.RefObject; mediaType: 'video' | 'audio'; + onVariableFpsVideoDetected: () => void; }) => { const bufferingRef = useRef(false); const {delayPlayback} = useBufferState(); - const bufferUntilFirstFrame = useCallback(() => { - if (mediaType !== 'video') { - return; - } + const bufferUntilFirstFrame = useCallback( + (requestedTime: number) => { + if (mediaType !== 'video') { + return; + } - const current = mediaRef.current as HTMLVideoElement | null; + const current = mediaRef.current as HTMLVideoElement | null; - if (!current) { - return; - } + if (!current) { + return; + } - if (!current.requestVideoFrameCallback) { - return; - } + if (!current.requestVideoFrameCallback) { + return; + } - bufferingRef.current = true; + bufferingRef.current = true; - const playback = delayPlayback(); + const playback = delayPlayback(); - const unblock = () => { - playback.unblock(); - current.removeEventListener('ended', unblock, { - // @ts-expect-error - once: true, - }); - current.removeEventListener('pause', unblock, { - // @ts-expect-error - once: true, - }); - bufferingRef.current = false; - }; + const unblock = () => { + playback.unblock(); + current.removeEventListener('ended', unblock, { + // @ts-expect-error + once: true, + }); + current.removeEventListener('pause', unblock, { + // @ts-expect-error + once: true, + }); + bufferingRef.current = false; + }; - const onEndedOrPause = () => { - unblock(); - }; - - current.requestVideoFrameCallback(() => { - // Safari often seeks and then stalls. - // This makes sure that the video actually starts playing. - current.requestVideoFrameCallback(() => { + const onEndedOrPause = () => { unblock(); + }; + + current.requestVideoFrameCallback((_, info) => { + const differenceFromRequested = Math.abs( + info.mediaTime - requestedTime, + ); + if (differenceFromRequested > 0.5) { + onVariableFpsVideoDetected(); + } + + // Safari often seeks and then stalls. + // This makes sure that the video actually starts playing. + current.requestVideoFrameCallback(() => { + unblock(); + }); }); - }); - current.addEventListener('ended', onEndedOrPause, {once: true}); - current.addEventListener('pause', onEndedOrPause, {once: true}); - }, [delayPlayback, mediaRef, mediaType]); + current.addEventListener('ended', onEndedOrPause, {once: true}); + current.addEventListener('pause', onEndedOrPause, {once: true}); + }, + [delayPlayback, mediaRef, mediaType, onVariableFpsVideoDetected], + ); return useMemo(() => { return { diff --git a/packages/core/src/use-media-playback.ts b/packages/core/src/use-media-playback.ts index 00c450135ce..d773d94a43f 100644 --- a/packages/core/src/use-media-playback.ts +++ b/packages/core/src/use-media-playback.ts @@ -1,5 +1,5 @@ import type {RefObject} from 'react'; -import {useContext, useEffect} from 'react'; +import {useCallback, useContext, useEffect, useRef} from 'react'; import {useMediaStartsAt} from './audio/use-audio-frame.js'; import {useBufferUntilFirstFrame} from './buffer-until-first-frame.js'; import {BufferingContextReact} from './buffering.js'; @@ -60,6 +60,7 @@ export const useMediaPlayback = ({ const buffering = useContext(BufferingContextReact); const {fps} = useVideoConfig(); const mediaStartsAt = useMediaStartsAt(); + const lastSeekDueToShift = useRef(null); if (!buffering) { throw new Error( @@ -82,9 +83,27 @@ export const useMediaPlayback = ({ isPremounting, }); + const isVariableFpsVideoMap = useRef>({}); + + const onVariableFpsVideoDetected = useCallback(() => { + if (!src) { + return; + } + + if (debugSeeking) { + // eslint-disable-next-line no-console + console.log( + `Detected ${src} as a variable FPS video. Disabling buffering while seeking.`, + ); + } + + isVariableFpsVideoMap.current[src] = true; + }, [debugSeeking, src]); + const {bufferUntilFirstFrame, isBuffering} = useBufferUntilFirstFrame({ mediaRef, mediaType, + onVariableFpsVideoDetected, }); const playbackRate = localPlaybackRate * globalPlaybackRate; @@ -143,36 +162,52 @@ export const useMediaPlayback = ({ !Number.isNaN(duration) && Number.isFinite(duration) ? Math.min(duration, desiredUnclampedTime) : desiredUnclampedTime; - const isTime = mediaRef.current.currentTime; + + const mediaTagTime = mediaRef.current.currentTime; const rvcTime = currentTime.current ?? null; - const timeShiftMediaTag = Math.abs(shouldBeTime - isTime); + const isVariableFpsVideo = isVariableFpsVideoMap.current[src]; + + const timeShiftMediaTag = Math.abs(shouldBeTime - mediaTagTime); const timeShiftRvcTag = rvcTime ? Math.abs(shouldBeTime - rvcTime) : null; - const timeShift = timeShiftRvcTag ? timeShiftRvcTag : timeShiftMediaTag; + const timeShift = + timeShiftRvcTag && !isVariableFpsVideo + ? timeShiftRvcTag + : timeShiftMediaTag; if (debugSeeking) { // eslint-disable-next-line no-console console.log({ - isTime, + mediaTagTime, rvcTime, shouldBeTime, state: mediaRef.current.readyState, playing: !mediaRef.current.paused, + isVariableFpsVideo, }); } - if (timeShift > acceptableTimeShiftButLessThanDuration) { + if ( + timeShift > acceptableTimeShiftButLessThanDuration && + lastSeekDueToShift.current !== shouldBeTime + ) { // If scrubbing around, adjust timing // or if time shift is bigger than 0.45sec if (debugSeeking) { // eslint-disable-next-line no-console - console.log('Seeking', {shouldBeTime, isTime, rvcTime, timeShift}); + console.log('Seeking', { + shouldBeTime, + isTime: mediaTagTime, + rvcTime, + timeShift, + }); } seek(mediaRef, shouldBeTime); - if (playing) { - bufferUntilFirstFrame(); + lastSeekDueToShift.current = shouldBeTime; + if (playing && !isVariableFpsVideo) { + bufferUntilFirstFrame(shouldBeTime); if (mediaRef.current.paused) { playAndHandleNotAllowedError(mediaRef, mediaType); } @@ -220,7 +255,9 @@ export const useMediaPlayback = ({ } playAndHandleNotAllowedError(mediaRef, mediaType); - bufferUntilFirstFrame(); + if (!isVariableFpsVideo) { + bufferUntilFirstFrame(shouldBeTime); + } } }, [ absoluteFrame, diff --git a/packages/example/public/variablefps-no-duration.webm b/packages/example/public/variablefps-no-duration.webm new file mode 100644 index 00000000000..017d0d9a8d4 Binary files /dev/null and b/packages/example/public/variablefps-no-duration.webm differ diff --git a/packages/it-tests/src/rendering/get-video-metadata.test.ts b/packages/it-tests/src/rendering/get-video-metadata.test.ts index 30327926d39..a3559743a85 100644 --- a/packages/it-tests/src/rendering/get-video-metadata.test.ts +++ b/packages/it-tests/src/rendering/get-video-metadata.test.ts @@ -100,3 +100,19 @@ test("Should return an error due to using a audio file", async () => { ); } }); + +test("Should not return duration in variable fps video", async () => { + const video = path.join( + __dirname, + "..", + "..", + "..", + "example", + "public", + "variablefps-no-duration.webm" + ); + expect(existsSync(video)).toEqual(true); + + const hi = await getVideoMetadata(video, { logLevel: "verbose" }); + console.log(hi); +}); diff --git a/packages/player-example/public/video.webm b/packages/player-example/public/video.webm new file mode 100644 index 00000000000..017d0d9a8d4 Binary files /dev/null and b/packages/player-example/public/video.webm differ diff --git a/packages/renderer/rust/ffmpeg.rs b/packages/renderer/rust/ffmpeg.rs index e484683177b..afff26c5575 100644 --- a/packages/renderer/rust/ffmpeg.rs +++ b/packages/renderer/rust/ffmpeg.rs @@ -7,6 +7,7 @@ use crate::payloads::payloads::{ KnownAudioCodecs, KnownCodecs, KnownColorSpaces, OpenVideoStats, VideoMetadata, }; use std::fs::File; +use std::i64; use std::io::{BufReader, ErrorKind}; extern crate ffmpeg_next as remotionffmpeg; use remotionffmpeg::{codec, encoder, format, media, Rational}; @@ -301,13 +302,17 @@ pub fn get_video_metadata(file_path: &str) -> Result None, + _ => Some(duration as f64 / remotionffmpeg::ffi::AV_TIME_BASE as f64), + }; #[allow(non_snake_case)] let supportsSeeking = match video_codec_name { KnownCodecs::H264 => { - if durationInSeconds < 5.0 { + if durationInSeconds.is_some() && durationInSeconds.unwrap() < 5.0 { true } else { let f = File::open(file_path).unwrap(); diff --git a/packages/renderer/rust/payloads.rs b/packages/renderer/rust/payloads.rs index 48ca15a1e1e..c68bf4379d6 100644 --- a/packages/renderer/rust/payloads.rs +++ b/packages/renderer/rust/payloads.rs @@ -221,7 +221,7 @@ pub mod payloads { pub fps: f32, pub width: u32, pub height: u32, - pub durationInSeconds: f64, + pub durationInSeconds: Option, pub codec: KnownCodecs, pub canPlayInVideoTag: bool, pub supportsSeeking: bool,