Merge pull request #4279 from remotion-dev/create-cues

packages/example/src/Encoder/SrcEncoder.tsx

@@ -16,7 +16,6 @@ import {
 import React, {useCallback, useRef, useState} from 'react';
 import {flushSync} from 'react-dom';
 import {AbsoluteFill} from 'remotion';
-import {getMicroSecondsAheadOfTrack} from './ahead-of-track';
 import {fitElementSizeInContainer} from './fit-element-size-in-container';
 
 const CANVAS_WIDTH = 1024 / 4;
@@ -101,8 +100,6 @@ export const SrcEncoder: React.FC<{
 
 	const i = useRef(0);
 
-	const trackProgresses = useRef<Record<number, number>>({});
-
 	const onVideoFrame = useCallback(
 		async (inputFrame: VideoFrame, track: VideoTrack) => {
 			i.current++;
@@ -167,14 +164,6 @@ export const SrcEncoder: React.FC<{
 		[setState],
 	);
 
-	const getFramesInEncodingQueue = useCallback(() => {
-		return stateRef.current.videoFrames - stateRef.current.encodedVideoFrames;
-	}, []);
-
-	const getFramesInAudioQueue = useCallback(() => {
-		return stateRef.current.audioFrames - stateRef.current.encodedAudioFrames;
-	}, []);
-
 	const onVideoTrack = useCallback(
 		(mediaState: MediaFn): OnVideoTrack =>
 			async (track) => {
@@ -192,8 +181,6 @@ export const SrcEncoder: React.FC<{
 					},
 					width: track.codedWidth,
 					height: track.codedHeight,
-					// TODO: Unhardcode
-					defaultDuration: 2658,
 					codecId: 'V_VP8',
 				});
 
@@ -213,6 +200,10 @@ export const SrcEncoder: React.FC<{
 							}));
 						});
 					},
+					onError: (err) => {
+						// TODO: Do error handling
+						console.log(err);
+					},
 				});
 				if (videoEncoder === null) {
 					setState((s) => ({
@@ -229,6 +220,10 @@ export const SrcEncoder: React.FC<{
 						await videoEncoder.encodeFrame(frame);
 						frame.close();
 					},
+					onError: (err) => {
+						// TODO: Do error handling
+						console.log(err);
+					},
 				});
 				if (videoDecoder === null) {
 					setState((s) => ({
@@ -246,27 +241,10 @@ export const SrcEncoder: React.FC<{
 				});
 
 				return async (chunk) => {
-					while (
-						getMicroSecondsAheadOfTrack(trackProgresses.current, trackNumber) >
-						// 2 seconds
-						1_000_000
-					) {
-						await new Promise<void>((r) => {
-							setTimeout(r, 100);
-						});
-					}
-					trackProgresses.current[trackNumber] = chunk.timestamp;
-
-					while (getFramesInEncodingQueue() > 15) {
-						await new Promise<void>((r) => {
-							setTimeout(r, 100);
-						});
-					}
-
 					await videoDecoder.processSample(chunk);
 				};
 			},
-		[getFramesInEncodingQueue, onVideoFrame, setState, trackProgresses],
+		[onVideoFrame, setState],
 	);
 
 	const onAudioTrack = useCallback(
@@ -292,6 +270,10 @@ export const SrcEncoder: React.FC<{
 					},
 					sampleRate: track.sampleRate,
 					numberOfChannels: track.numberOfChannels,
+					onError: (err) => {
+						// TODO: Do error handling
+						console.log(err);
+					},
 				});
 
 				if (!audioEncoder) {
@@ -313,6 +295,7 @@ export const SrcEncoder: React.FC<{
 						frame.close();
 					},
 					onError(error) {
+						// TODO: Do better error handling
 						setState((s) => ({...s, audioError: error}));
 					},
 				});
@@ -333,26 +316,10 @@ export const SrcEncoder: React.FC<{
 				});
 
 				return async (audioSample) => {
-					while (
-						getMicroSecondsAheadOfTrack(trackProgresses.current, trackNumber) >
-						// 2 seconds
-						1_000_000
-					) {
-						await new Promise<void>((r) => {
-							setTimeout(r, 100);
-						});
-					}
-					trackProgresses.current[trackNumber] = audioSample.timestamp;
-
-					while (getFramesInAudioQueue() > 15) {
-						await new Promise<void>((r) => {
-							setTimeout(r, 100);
-						});
-					}
 					await audioDecoder.processSample(audioSample);
 				};
 			},
-		[getFramesInAudioQueue, setState, trackProgresses],
+		[setState],
 	);
 
 	const onClick = useCallback(() => {

packages/media-parser/src/create/create-media.ts

@@ -109,6 +109,7 @@ export const createMedia = async (
 
 	const clusterOffset = w.getWrittenByteCount();
 	let currentCluster = await makeCluster(w, 0);
+	// TODO: Also create a `Cues` seek element
 	seeks.push({
 		hexString: matroskaElements.Cluster,
 		byte: clusterOffset - seekHeadOffset,

packages/media-parser/src/create/matroska-trackentry.ts

@@ -152,7 +152,6 @@ export type MakeTrackVideo = {
 	color: MatroskaColorParams;
 	width: number;
 	height: number;
-	defaultDuration: number;
 	trackNumber: number;
 	codecId: string;
 	type: 'video';
@@ -240,7 +239,6 @@ export const makeMatroskaVideoTrackEntryBytes = ({
 	color,
 	width,
 	height,
-	defaultDuration,
 	trackNumber,
 	codecId,
 }: MakeTrackVideo) => {
@@ -295,14 +293,6 @@ export const makeMatroskaVideoTrackEntryBytes = ({
 				},
 				minVintWidth: null,
 			},
-			{
-				type: 'DefaultDuration',
-				value: {
-					value: defaultDuration,
-					byteLength: null,
-				},
-				minVintWidth: null,
-			},
 			makeMatroskaVideoBytes({
 				color,
 				width,

packages/webcodecs/src/audio-decoder.ts

@@ -1,5 +1,12 @@
 import type {AudioSample, AudioTrack} from '@remotion/media-parser';
-import {decoderWaitForDequeue, decoderWaitForFinish} from './wait-for-dequeue';
+
+export type WebCodecsAudioDecoder = {
+	processSample: (audioSample: AudioSample) => Promise<void>;
+	waitForFinish: () => Promise<void>;
+	close: () => void;
+	getQueueSize: () => number;
+	flush: () => Promise<void>;
+};
 
 export const createAudioDecoder = async ({
 	track,
@@ -9,7 +16,7 @@ export const createAudioDecoder = async ({
 	track: AudioTrack;
 	onFrame: (frame: AudioData) => Promise<void>;
 	onError: (error: DOMException) => void;
-}) => {
+}): Promise<WebCodecsAudioDecoder | null> => {
 	if (typeof AudioDecoder === 'undefined') {
 		return null;
 	}
@@ -20,26 +27,56 @@ export const createAudioDecoder = async ({
 		return null;
 	}
 
-	let prom = Promise.resolve();
+	let outputQueue = Promise.resolve();
+	let outputQueueSize = 0;
+	let dequeueResolver = () => {};
 
 	const audioDecoder = new AudioDecoder({
 		output(inputFrame) {
-			// TODO: Should make this a "decoder queue size as well"
-			prom = prom.then(() => onFrame(inputFrame));
+			outputQueueSize++;
+			outputQueue = outputQueue
+				.then(() => onFrame(inputFrame))
+				.then(() => {
+					dequeueResolver();
+					outputQueueSize--;
+					return Promise.resolve();
+				});
 		},
 		error(error) {
 			onError(error);
 		},
 	});
 
+	const getQueueSize = () => {
+		return audioDecoder.decodeQueueSize + outputQueueSize;
+	};
+
 	audioDecoder.configure(config);
 
+	const waitForDequeue = async () => {
+		await new Promise<void>((r) => {
+			dequeueResolver = r;
+			// @ts-expect-error exists
+			audioDecoder.addEventListener('dequeue', () => r(), {
+				once: true,
+			});
+		});
+	};
+
+	const waitForFinish = async () => {
+		while (getQueueSize() > 0) {
+			await waitForDequeue();
+		}
+	};
+
 	const processSample = async (audioSample: AudioSample) => {
 		if (audioDecoder.state === 'closed') {
 			return;
 		}
 
-		await decoderWaitForDequeue(audioDecoder);
+		while (getQueueSize() > 10) {
+			await waitForDequeue();
+		}
 
 		// Don't flush, it messes up the audio
 
@@ -55,11 +92,16 @@ export const createAudioDecoder = async ({
 			return queue;
 		},
 		waitForFinish: async () => {
-			await decoderWaitForFinish(audioDecoder);
-			await prom;
+			await audioDecoder.flush();
+			await waitForFinish();
+			await outputQueue;
 		},
 		close: () => {
 			audioDecoder.close();
 		},
+		getQueueSize,
+		flush: async () => {
+			await audioDecoder.flush();
+		},
 	};
 };

packages/webcodecs/src/audio-encoder.ts

@@ -1,26 +1,43 @@
-import {encoderWaitForDequeue, encoderWaitForFinish} from './wait-for-dequeue';
+export type WebCodecsAudioEncoder = {
+	encodeFrame: (audioData: AudioData) => Promise<void>;
+	waitForFinish: () => Promise<void>;
+	close: () => void;
+	getQueueSize: () => number;
+	flush: () => Promise<void>;
+};
 
 export const createAudioEncoder = async ({
 	onChunk,
 	sampleRate,
 	numberOfChannels,
+	onError,
 }: {
 	onChunk: (chunk: EncodedAudioChunk) => Promise<void>;
 	sampleRate: number;
 	numberOfChannels: number;
-}) => {
+	onError: (error: DOMException) => void;
+}): Promise<WebCodecsAudioEncoder | null> => {
 	if (typeof AudioEncoder === 'undefined') {
 		return null;
 	}
 
 	let prom = Promise.resolve();
+	let outputQueue = 0;
+	let dequeueResolver = () => {};
 
 	const encoder = new AudioEncoder({
 		output: (chunk) => {
-			prom = prom.then(() => onChunk(chunk));
+			outputQueue++;
+			prom = prom
+				.then(() => onChunk(chunk))
+				.then(() => {
+					outputQueue--;
+					dequeueResolver();
+					return Promise.resolve();
+				});
 		},
 		error(error) {
-			console.error(error);
+			onError(error);
 		},
 	});
 
@@ -37,14 +54,38 @@ export const createAudioEncoder = async ({
 		return null;
 	}
 
+	const getQueueSize = () => {
+		return encoder.encodeQueueSize + outputQueue;
+	};
+
 	encoder.configure(audioEncoderConfig);
 
+	const waitForDequeue = async () => {
+		await new Promise<void>((r) => {
+			dequeueResolver = r;
+
+			// @ts-expect-error exists
+			encoder.addEventListener('dequeue', () => r(), {
+				once: true,
+			});
+		});
+	};
+
+	const waitForFinish = async () => {
+		while (getQueueSize() > 0) {
+			await waitForDequeue();
+		}
+	};
+
 	const encodeFrame = async (audioData: AudioData) => {
-		await encoderWaitForDequeue(encoder);
 		if (encoder.state === 'closed') {
 			return;
 		}
 
+		while (getQueueSize() > 10) {
+			await waitForDequeue();
+		}
+
 		encoder.encode(audioData);
 	};
 
@@ -56,11 +97,16 @@ export const createAudioEncoder = async ({
 			return queue;
 		},
 		waitForFinish: async () => {
-			await encoderWaitForFinish(encoder);
+			await encoder.flush();
+			await waitForFinish();
 			await prom;
 		},
 		close: () => {
 			encoder.close();
 		},
+		getQueueSize,
+		flush: async () => {
+			await encoder.flush();
+		},
 	};
 };