Merge pull request #4271 from remotion-dev/media-parser-multiple-clus…

…ters

packages/media-parser/src/create/cluster-segment.ts

@@ -8,15 +8,15 @@ import {
 
 export const CLUSTER_MIN_VINT_WIDTH = 8;
 
-export const createClusterSegment = () => {
+export const createClusterSegment = (timestamp: number) => {
 	return makeMatroskaBytes({
 		type: 'Cluster',
 		value: [
 			{
 				type: 'Timestamp',
-				minVintWidth: 4,
+				minVintWidth: null,
 				value: {
-					value: 0,
+					value: timestamp,
 					byteLength: null,
 				},
 			},

packages/media-parser/src/create/cluster.ts

@@ -0,0 +1,64 @@
+import {getVariableInt} from '../boxes/webm/ebml';
+import {matroskaToHex} from '../boxes/webm/make-header';
+import {matroskaElements} from '../boxes/webm/segments/all-segments';
+import type {Writer} from '../writers/writer';
+import {
+	CLUSTER_MIN_VINT_WIDTH,
+	createClusterSegment,
+	makeSimpleBlock,
+} from './cluster-segment';
+import {CREATE_TIME_SCALE} from './timescale';
+
+const maxClusterTimestamp = 2 ** 15;
+
+const timestampToClusterTimestamp = (timestamp: number) => {
+	return Math.round((timestamp / CREATE_TIME_SCALE) * 1000);
+};
+
+export const makeCluster = async (w: Writer, timestamp: number) => {
+	const cluster = createClusterSegment(timestampToClusterTimestamp(timestamp));
+	const clusterVIntPosition =
+		w.getWrittenByteCount() +
+		cluster.offsets.offset +
+		matroskaToHex(matroskaElements.Cluster).byteLength;
+
+	let clusterSize = cluster.bytes.byteLength;
+	await w.write(cluster.bytes);
+
+	const addSample = async (chunk: EncodedVideoChunk, trackNumber: number) => {
+		const arr = new Uint8Array(chunk.byteLength);
+		chunk.copyTo(arr);
+		const timecodeRelativeToCluster =
+			timestampToClusterTimestamp(chunk.timestamp) -
+			timestampToClusterTimestamp(timestamp);
+		if (timecodeRelativeToCluster > maxClusterTimestamp) {
+			throw new Error('timecodeRelativeToCluster is too big');
+		}
+
+		const keyframe = chunk.type === 'key';
+		const simpleBlock = makeSimpleBlock({
+			bytes: arr,
+			invisible: false,
+			keyframe,
+			lacing: 0,
+			trackNumber,
+			timecodeRelativeToCluster,
+		});
+
+		clusterSize += simpleBlock.byteLength;
+		await w.updateDataAt(
+			clusterVIntPosition,
+			getVariableInt(clusterSize, CLUSTER_MIN_VINT_WIDTH),
+		);
+		await w.write(simpleBlock);
+	};
+
+	const shouldMakeNewCluster = (chunk: EncodedVideoChunk) => {
+		const newTimestamp = timestampToClusterTimestamp(chunk.timestamp);
+		const oldTimestamp = timestampToClusterTimestamp(timestamp);
+
+		return newTimestamp - oldTimestamp >= 2000 && chunk.type === 'key';
+	};
+
+	return {addSample, shouldMakeNewCluster};
+};

packages/media-parser/src/create/create-media.ts

@@ -1,17 +1,8 @@
-import {getVariableInt} from '../boxes/webm/ebml';
-import {
-	combineUint8Arrays,
-	matroskaToHex,
-	padMatroskaBytes,
-} from '../boxes/webm/make-header';
+import {combineUint8Arrays} from '../boxes/webm/make-header';
 import type {BytesAndOffset} from '../boxes/webm/segments/all-segments';
-import {matroskaElements} from '../boxes/webm/segments/all-segments';
 import type {WriterInterface} from '../writers/writer';
-import {
-	CLUSTER_MIN_VINT_WIDTH,
-	createClusterSegment,
-	makeSimpleBlock,
-} from './cluster-segment';
+import {makeCluster} from './cluster';
+import {makeDurationWithPadding} from './make-duration-with-padding';
 import {makeMatroskaHeader} from './matroska-header';
 import {makeMatroskaInfo} from './matroska-info';
 import {createMatroskaSegment} from './matroska-segment';
@@ -21,6 +12,7 @@ import {
 	makeMatroskaTracks,
 	makeMatroskaVideoTrackEntryBytes,
 } from './matroska-trackentry';
+import {CREATE_TIME_SCALE} from './timescale';
 
 export type MediaFn = {
 	save: () => Promise<void>;
@@ -43,9 +35,7 @@ export const createMedia = async (
 	const w = await writer.createContent();
 	await w.write(header.bytes);
 	const matroskaInfo = makeMatroskaInfo({
-		timescale: 1_000_000,
-		// TODO: Hardcoded
-		duration: 2658,
+		timescale: CREATE_TIME_SCALE,
 	});
 
 	const currentTracks: BytesAndOffset[] = [];
@@ -74,50 +64,24 @@ export const createMedia = async (
 
 	await w.write(matroskaSegment.bytes);
 
-	const cluster = createClusterSegment();
-	const clusterVIntPosition =
-		w.getWrittenByteCount() +
-		cluster.offsets.offset +
-		matroskaToHex(matroskaElements.Cluster).byteLength;
+	let currentCluster = await makeCluster(w, 0);
 
-	let clusterSize = cluster.bytes.byteLength;
-	await w.write(cluster.bytes);
+	const getClusterOrMakeNew = async (chunk: EncodedVideoChunk) => {
+		if (!currentCluster.shouldMakeNewCluster(chunk)) {
+			return currentCluster;
+		}
+
+		currentCluster = await makeCluster(w, chunk.timestamp);
+		return currentCluster;
+	};
 
 	const addSample = async (chunk: EncodedVideoChunk, trackNumber: number) => {
-		const arr = new Uint8Array(chunk.byteLength);
-		chunk.copyTo(arr);
-		const simpleBlock = makeSimpleBlock({
-			bytes: arr,
-			invisible: false,
-			keyframe: chunk.type === 'key',
-			lacing: 0,
-			trackNumber,
-			// TODO: Maybe this is bad, because it's in microseconds, but should be in timescale
-			// Maybe it only works by coincidence
-			timecodeRelativeToCluster: Math.round(chunk.timestamp / 1000),
-		});
-
-		clusterSize += simpleBlock.byteLength;
-		await w.updateDataAt(
-			clusterVIntPosition,
-			getVariableInt(clusterSize, CLUSTER_MIN_VINT_WIDTH),
-		);
-		await w.write(simpleBlock);
+		const cluster = await getClusterOrMakeNew(chunk);
+		return cluster.addSample(chunk, trackNumber);
 	};
 
 	const updateDuration = async (newDuration: number) => {
-		const blocks = padMatroskaBytes(
-			{
-				type: 'Duration',
-				value: {
-					value: newDuration,
-					size: '64',
-				},
-				minVintWidth: null,
-			},
-			// TODO: That's too much padding
-			1000,
-		);
+		const blocks = makeDurationWithPadding(newDuration);
 		await w.updateDataAt(
 			durationOffset,
 			combineUint8Arrays(blocks.map((b) => b.bytes)),

packages/media-parser/src/create/make-duration-with-padding.ts

@@ -0,0 +1,15 @@
+import {padMatroskaBytes} from '../boxes/webm/make-header';
+
+export const makeDurationWithPadding = (newDuration: number) => {
+	return padMatroskaBytes(
+		{
+			type: 'Duration',
+			value: {
+				value: newDuration,
+				size: '64',
+			},
+			minVintWidth: null,
+		},
+		100,
+	);
+};

packages/media-parser/src/create/matroska-info.ts

@@ -1,12 +1,7 @@
-import {makeMatroskaBytes, padMatroskaBytes} from '../boxes/webm/make-header';
+import {makeMatroskaBytes} from '../boxes/webm/make-header';
+import {makeDurationWithPadding} from './make-duration-with-padding';
 
-export const makeMatroskaInfo = ({
-	timescale,
-	duration,
-}: {
-	timescale: number;
-	duration: number;
-}) => {
+export const makeMatroskaInfo = ({timescale}: {timescale: number}) => {
 	return makeMatroskaBytes({
 		type: 'Info',
 		value: [
@@ -28,18 +23,7 @@ export const makeMatroskaInfo = ({
 				value: '@remotion/media-parser',
 				minVintWidth: null,
 			},
-			...padMatroskaBytes(
-				{
-					type: 'Duration',
-					value: {
-						value: duration,
-						size: '64',
-					},
-					minVintWidth: null,
-				},
-				// TODO: That's too much padding
-				1000,
-			),
+			...makeDurationWithPadding(0),
 		],
 		minVintWidth: null,
 	});

packages/media-parser/src/create/timescale.ts

@@ -0,0 +1 @@
+export const CREATE_TIME_SCALE = 1_000_000;