@remotion + @remotion/renderer: Detect variable fps video and handle it better

packages/core/src/buffer-until-first-frame.ts

@@ -4,60 +4,72 @@ import {useBufferState} from './use-buffer-state';
 export const useBufferUntilFirstFrame = ({
 	mediaRef,
 	mediaType,
+	onVariableFpsVideoDetected,
 }: {
 	mediaRef: React.RefObject<HTMLVideoElement | HTMLAudioElement>;
 	mediaType: 'video' | 'audio';
+	onVariableFpsVideoDetected: () => void;
 }) => {
 	const bufferingRef = useRef<boolean>(false);
 	const {delayPlayback} = useBufferState();
 
-	const bufferUntilFirstFrame = useCallback(() => {
-		if (mediaType !== 'video') {
-			return;
-		}
+	const bufferUntilFirstFrame = useCallback(
+		(requestedTime: number) => {
+			if (mediaType !== 'video') {
+				return;
+			}
 
-		const current = mediaRef.current as HTMLVideoElement | null;
+			const current = mediaRef.current as HTMLVideoElement | null;
 
-		if (!current) {
-			return;
-		}
+			if (!current) {
+				return;
+			}
 
-		if (!current.requestVideoFrameCallback) {
-			return;
-		}
+			if (!current.requestVideoFrameCallback) {
+				return;
+			}
 
-		bufferingRef.current = true;
+			bufferingRef.current = true;
 
-		const playback = delayPlayback();
+			const playback = delayPlayback();
 
-		const unblock = () => {
-			playback.unblock();
-			current.removeEventListener('ended', unblock, {
-				// @ts-expect-error
-				once: true,
-			});
-			current.removeEventListener('pause', unblock, {
-				// @ts-expect-error
-				once: true,
-			});
-			bufferingRef.current = false;
-		};
+			const unblock = () => {
+				playback.unblock();
+				current.removeEventListener('ended', unblock, {
+					// @ts-expect-error
+					once: true,
+				});
+				current.removeEventListener('pause', unblock, {
+					// @ts-expect-error
+					once: true,
+				});
+				bufferingRef.current = false;
+			};
 
-		const onEndedOrPause = () => {
-			unblock();
-		};
-
-		current.requestVideoFrameCallback(() => {
-			// Safari often seeks and then stalls.
-			// This makes sure that the video actually starts playing.
-			current.requestVideoFrameCallback(() => {
+			const onEndedOrPause = () => {
 				unblock();
+			};
+
+			current.requestVideoFrameCallback((_, info) => {
+				const differenceFromRequested = Math.abs(
+					info.mediaTime - requestedTime,
+				);
+				if (differenceFromRequested > 0.5) {
+					onVariableFpsVideoDetected();
+				}
+
+				// Safari often seeks and then stalls.
+				// This makes sure that the video actually starts playing.
+				current.requestVideoFrameCallback(() => {
+					unblock();
+				});
 			});
-		});
 
-		current.addEventListener('ended', onEndedOrPause, {once: true});
-		current.addEventListener('pause', onEndedOrPause, {once: true});
-	}, [delayPlayback, mediaRef, mediaType]);
+			current.addEventListener('ended', onEndedOrPause, {once: true});
+			current.addEventListener('pause', onEndedOrPause, {once: true});
+		},
+		[delayPlayback, mediaRef, mediaType, onVariableFpsVideoDetected],
+	);
 
 	return useMemo(() => {
 		return {

packages/core/src/use-media-playback.ts

@@ -1,5 +1,5 @@
 import type {RefObject} from 'react';
-import {useContext, useEffect} from 'react';
+import {useCallback, useContext, useEffect, useRef} from 'react';
 import {useMediaStartsAt} from './audio/use-audio-frame.js';
 import {useBufferUntilFirstFrame} from './buffer-until-first-frame.js';
 import {BufferingContextReact} from './buffering.js';
@@ -60,6 +60,7 @@ export const useMediaPlayback = ({
 	const buffering = useContext(BufferingContextReact);
 	const {fps} = useVideoConfig();
 	const mediaStartsAt = useMediaStartsAt();
+	const lastSeekDueToShift = useRef<number | null>(null);
 
 	if (!buffering) {
 		throw new Error(
@@ -82,9 +83,27 @@ export const useMediaPlayback = ({
 		isPremounting,
 	});
 
+	const isVariableFpsVideoMap = useRef<Record<string, boolean>>({});
+
+	const onVariableFpsVideoDetected = useCallback(() => {
+		if (!src) {
+			return;
+		}
+
+		if (debugSeeking) {
+			// eslint-disable-next-line no-console
+			console.log(
+				`Detected ${src} as a variable FPS video. Disabling buffering while seeking.`,
+			);
+		}
+
+		isVariableFpsVideoMap.current[src] = true;
+	}, [debugSeeking, src]);
+
 	const {bufferUntilFirstFrame, isBuffering} = useBufferUntilFirstFrame({
 		mediaRef,
 		mediaType,
+		onVariableFpsVideoDetected,
 	});
 
 	const playbackRate = localPlaybackRate * globalPlaybackRate;
@@ -143,36 +162,52 @@ export const useMediaPlayback = ({
 			!Number.isNaN(duration) && Number.isFinite(duration)
 				? Math.min(duration, desiredUnclampedTime)
 				: desiredUnclampedTime;
-		const isTime = mediaRef.current.currentTime;
+
+		const mediaTagTime = mediaRef.current.currentTime;
 		const rvcTime = currentTime.current ?? null;
 
-		const timeShiftMediaTag = Math.abs(shouldBeTime - isTime);
+		const isVariableFpsVideo = isVariableFpsVideoMap.current[src];
+
+		const timeShiftMediaTag = Math.abs(shouldBeTime - mediaTagTime);
 		const timeShiftRvcTag = rvcTime ? Math.abs(shouldBeTime - rvcTime) : null;
-		const timeShift = timeShiftRvcTag ? timeShiftRvcTag : timeShiftMediaTag;
+		const timeShift =
+			timeShiftRvcTag && !isVariableFpsVideo
+				? timeShiftRvcTag
+				: timeShiftMediaTag;
 
 		if (debugSeeking) {
 			// eslint-disable-next-line no-console
 			console.log({
-				isTime,
+				mediaTagTime,
 				rvcTime,
 				shouldBeTime,
 				state: mediaRef.current.readyState,
 				playing: !mediaRef.current.paused,
+				isVariableFpsVideo,
 			});
 		}
 
-		if (timeShift > acceptableTimeShiftButLessThanDuration) {
+		if (
+			timeShift > acceptableTimeShiftButLessThanDuration &&
+			lastSeekDueToShift.current !== shouldBeTime
+		) {
 			// If scrubbing around, adjust timing
 			// or if time shift is bigger than 0.45sec
 
 			if (debugSeeking) {
 				// eslint-disable-next-line no-console
-				console.log('Seeking', {shouldBeTime, isTime, rvcTime, timeShift});
+				console.log('Seeking', {
+					shouldBeTime,
+					isTime: mediaTagTime,
+					rvcTime,
+					timeShift,
+				});
 			}
 
 			seek(mediaRef, shouldBeTime);
-			if (playing) {
-				bufferUntilFirstFrame();
+			lastSeekDueToShift.current = shouldBeTime;
+			if (playing && !isVariableFpsVideo) {
+				bufferUntilFirstFrame(shouldBeTime);
 				if (mediaRef.current.paused) {
 					playAndHandleNotAllowedError(mediaRef, mediaType);
 				}
@@ -220,7 +255,9 @@ export const useMediaPlayback = ({
 			}
 
 			playAndHandleNotAllowedError(mediaRef, mediaType);
-			bufferUntilFirstFrame();
+			if (!isVariableFpsVideo) {
+				bufferUntilFirstFrame(shouldBeTime);
+			}
 		}
 	}, [
 		absoluteFrame,

packages/it-tests/src/rendering/get-video-metadata.test.ts

@@ -100,3 +100,19 @@ test("Should return an error due to using a audio file", async () => {
     );
   }
 });
+
+test("Should not return duration in variable fps video", async () => {
+  const video = path.join(
+    __dirname,
+    "..",
+    "..",
+    "..",
+    "example",
+    "public",
+    "variablefps-no-duration.webm"
+  );
+  expect(existsSync(video)).toEqual(true);
+
+  const hi = await getVideoMetadata(video, { logLevel: "verbose" });
+  console.log(hi);
+});

packages/renderer/rust/ffmpeg.rs

@@ -7,6 +7,7 @@ use crate::payloads::payloads::{
     KnownAudioCodecs, KnownCodecs, KnownColorSpaces, OpenVideoStats, VideoMetadata,
 };
 use std::fs::File;
+use std::i64;
 use std::io::{BufReader, ErrorKind};
 extern crate ffmpeg_next as remotionffmpeg;
 use remotionffmpeg::{codec, encoder, format, media, Rational};
@@ -301,13 +302,17 @@ pub fn get_video_metadata(file_path: &str) -> Result<VideoMetadata, ErrorWithBac
         .map_err(|e| e.to_string())?;
 
     // Get the duration
+    let duration = input.duration();
     #[allow(non_snake_case)]
-    let durationInSeconds = input.duration() as f64 / remotionffmpeg::ffi::AV_TIME_BASE as f64;
+    let durationInSeconds = match duration {
+        i64::MIN => None,
+        _ => Some(duration as f64 / remotionffmpeg::ffi::AV_TIME_BASE as f64),
+    };
 
     #[allow(non_snake_case)]
     let supportsSeeking = match video_codec_name {
         KnownCodecs::H264 => {
-            if durationInSeconds < 5.0 {
+            if durationInSeconds.is_some() && durationInSeconds.unwrap() < 5.0 {
                 true
             } else {
                 let f = File::open(file_path).unwrap();

packages/renderer/rust/payloads.rs

@@ -221,7 +221,7 @@ pub mod payloads {
         pub fps: f32,
         pub width: u32,
         pub height: u32,
-        pub durationInSeconds: f64,
+        pub durationInSeconds: Option<f64>,
         pub codec: KnownCodecs,
         pub canPlayInVideoTag: bool,
         pub supportsSeeking: bool,