@remotion/video-parser: Web + Node API, Parse more Matroska fields

.github/workflows/push.yml

@@ -19,7 +19,7 @@ jobs:
           node-version: 16
       - uses: oven-sh/setup-bun@v1
         with:
-          bun-version: 1.1.16
+          bun-version: 1.1.20
       - run: |
           corepack enable
       - name: Get pnpm store directory
@@ -71,7 +71,7 @@ jobs:
           node-version: 16
       - uses: oven-sh/setup-bun@v1
         with:
-          bun-version: 1.1.16
+          bun-version: 1.1.20
       - run: |
           corepack enable
       - name: Get pnpm store directory
@@ -138,7 +138,7 @@ jobs:
           node-version: 16
       - uses: oven-sh/setup-bun@v1
         with:
-          bun-version: 1.1.16
+          bun-version: 1.1.20
       - run: |
           corepack enable
       - name: Get pnpm store directory
@@ -187,7 +187,7 @@ jobs:
           node-version: ${{ matrix.node_version }}
       - uses: oven-sh/setup-bun@v1
         with:
-          bun-version: 1.1.16
+          bun-version: 1.1.20
       - run: |
           corepack enable
       - name: Get pnpm store directory

packages/renderer/src/index.ts

@@ -146,6 +146,7 @@ import {
 	supportedAudioCodecs,
 } from './options/audio-codec';
 import {getShouldRenderAudio} from './render-has-audio';
+import {exampleVideos} from './test/example-videos';
 import {toMegabytes} from './to-megabytes';
 import {validatePuppeteerTimeout} from './validate-puppeteer-timeout';
 import {validateBitrate} from './validate-videobitrate';
@@ -240,6 +241,7 @@ export const RenderInternals = {
 	codecSupportsMedia,
 	toMegabytes,
 	internalEnsureBrowser,
+	exampleVideos,
 };
 
 // Warn of potential performance issues with Apple Silicon (M1 chip under Rosetta)

packages/video-parser/.npmignore

@@ -0,0 +1 @@
+dist/test

packages/video-parser/package.json

@@ -9,9 +9,12 @@
 	"scripts": {
 		"formatting": "prettier src --check",
 		"lint": "eslint src --ext ts,tsx",
-		"test": "bun test",
+		"test": "bun test src/test",
 		"watch": "tsc -w"
 	},
+	"devDependencies": {
+		"@remotion/renderer": "workspace:*"
+	},
 	"bugs": {
 		"url": "https://github.com/remotion-dev/remotion/issues"
 	},

packages/video-parser/src/boxes/iso-base-media/stsd/samples.ts

@@ -17,10 +17,10 @@ type AudioSample = SampleBase & {
 	compressionId: number;
 	packetSize: number;
 	sampleRate: number;
-	samplesPerPacket: number;
-	bytesPerPacket: number;
-	bytesPerFrame: number;
-	bitsPerSample: number;
+	samplesPerPacket: number | null;
+	bytesPerPacket: number | null;
+	bytesPerFrame: number | null;
+	bitsPerSample: number | null;
 };
 
 type VideoSample = SampleBase & {
@@ -154,45 +154,79 @@ export const processSample = ({
 	}
 
 	if (isAudio) {
-		if (version !== 1) {
-			throw new Error(`Unsupported version ${version}`);
+		if (version === 0) {
+			const numberOfChannels = iterator.getUint16();
+			const sampleSize = iterator.getUint16();
+			const compressionId = iterator.getUint16();
+			const packetSize = iterator.getUint16();
+			const sampleRate = iterator.getFixedPoint1616Number();
+
+			const bytesRemainingInBox =
+				boxSize - (iterator.counter.getOffset() - fileOffset);
+			iterator.discard(bytesRemainingInBox);
+
+			return {
+				sample: {
+					format: boxFormat,
+					offset: fileOffset,
+					dataReferenceIndex,
+					version,
+					revisionLevel,
+					vendor: [...Array.from(new Uint8Array(vendor))],
+					size: boxSize,
+					type: 'audio',
+					numberOfChannels,
+					sampleSize,
+					compressionId,
+					packetSize,
+					sampleRate,
+					samplesPerPacket: null,
+					bytesPerPacket: null,
+					bytesPerFrame: null,
+					bitsPerSample: null,
+				},
+			};
 		}
 
-		const numberOfChannels = iterator.getUint16();
-		const sampleSize = iterator.getUint16();
-		const compressionId = iterator.getUint16();
-		const packetSize = iterator.getUint16();
-		const sampleRate = iterator.getFixedPoint1616Number();
-		const samplesPerPacket = iterator.getUint16();
-		const bytesPerPacket = iterator.getUint16();
-		const bytesPerFrame = iterator.getUint16();
-		const bitsPerSample = iterator.getUint16();
+		if (version === 1) {
+			const numberOfChannels = iterator.getUint16();
+			const sampleSize = iterator.getUint16();
+			const compressionId = iterator.getUint16();
+			const packetSize = iterator.getUint16();
+			const sampleRate = iterator.getFixedPoint1616Number();
+			const samplesPerPacket = iterator.getUint16();
+			const bytesPerPacket = iterator.getUint16();
+			const bytesPerFrame = iterator.getUint16();
+			const bitsPerSample = iterator.getUint16();
 
-		const bytesRemainingInBox =
-			boxSize - (iterator.counter.getOffset() - fileOffset);
-		iterator.discard(bytesRemainingInBox);
+			const bytesRemainingInBox =
+				boxSize - (iterator.counter.getOffset() - fileOffset);
+			iterator.discard(bytesRemainingInBox);
 
-		return {
-			sample: {
-				format: boxFormat,
-				offset: fileOffset,
-				dataReferenceIndex,
-				version,
-				revisionLevel,
-				vendor: [...Array.from(new Uint8Array(vendor))],
-				size: boxSize,
-				type: 'audio',
-				numberOfChannels,
-				sampleSize,
-				compressionId,
-				packetSize,
-				sampleRate,
-				samplesPerPacket,
-				bytesPerPacket,
-				bytesPerFrame,
-				bitsPerSample,
-			},
-		};
+			return {
+				sample: {
+					format: boxFormat,
+					offset: fileOffset,
+					dataReferenceIndex,
+					version,
+					revisionLevel,
+					vendor: [...Array.from(new Uint8Array(vendor))],
+					size: boxSize,
+					type: 'audio',
+					numberOfChannels,
+					sampleSize,
+					compressionId,
+					packetSize,
+					sampleRate,
+					samplesPerPacket,
+					bytesPerPacket,
+					bytesPerFrame,
+					bitsPerSample,
+				},
+			};
+		}
+
+		throw new Error(`Unsupported version ${version}`);
 	}
 
 	if (isVideo) {

packages/video-parser/src/boxes/webm/segments.ts

@@ -15,6 +15,34 @@ import {
 } from './segments/seek-position';
 import type {TimestampScaleSegment} from './segments/timestamp-scale';
 import {parseTimestampScaleSegment} from './segments/timestamp-scale';
+import type {
+	AlphaModeSegment,
+	CodecSegment,
+	DefaultDurationSegment,
+	FlagLacingSegment,
+	HeightSegment,
+	LanguageSegment,
+	TrackEntrySegment,
+	TrackNumberSegment,
+	TrackTypeSegment,
+	TrackUIDSegment,
+	VideoSegment,
+	WidthSegment,
+} from './segments/track-entry';
+import {
+	parseAlphaModeSegment,
+	parseCodecSegment,
+	parseDefaultDurationSegment,
+	parseFlagLacing,
+	parseHeightSegment,
+	parseLanguageSegment,
+	parseTrackEntry,
+	parseTrackNumber,
+	parseTrackTypeSegment,
+	parseTrackUID,
+	parseVideoSegment,
+	parseWidthSegment,
+} from './segments/track-entry';
 import type {TracksSegment} from './segments/tracks';
 import {parseTracksSegment} from './segments/tracks';
 import type {UnknownSegment} from './segments/unknown';
@@ -36,11 +64,30 @@ export type MatroskaSegment =
 	| MuxingAppSegment
 	| WritingAppSegment
 	| DurationSegment
-	| TracksSegment;
+	| TracksSegment
+	| TrackEntrySegment
+	| TrackNumberSegment
+	| TrackUIDSegment
+	| FlagLacingSegment
+	| LanguageSegment
+	| CodecSegment
+	| TrackTypeSegment
+	| DefaultDurationSegment
+	| VideoSegment
+	| WidthSegment
+	| HeightSegment
+	| AlphaModeSegment;
 
 export const expectSegment = (iterator: BufferIterator): MatroskaSegment => {
 	const segmentId = iterator.getMatroskaSegmentId();
 
+	if (segmentId === '0x') {
+		return {
+			type: 'unknown-segment',
+			id: segmentId,
+		};
+	}
+
 	if (segmentId === '0x18538067') {
 		return parseMainSegment(iterator);
 	}
@@ -69,22 +116,70 @@ export const expectSegment = (iterator: BufferIterator): MatroskaSegment => {
 		return parseTimestampScaleSegment(iterator);
 	}
 
-	if (segmentId === '0x4d808c') {
+	if (segmentId.startsWith('0x4d80')) {
 		return parseMuxingSegment(iterator);
 	}
 
-	if (segmentId === '0x57418c') {
+	if (segmentId.startsWith('0x5741')) {
 		return parseWritingSegment(iterator);
 	}
 
-	if (segmentId === '0x448988') {
+	if (segmentId.startsWith('0x4489')) {
 		return parseDurationSegment(iterator);
 	}
 
 	if (segmentId === '0x1654ae6b') {
 		return parseTracksSegment(iterator);
 	}
 
+	if (segmentId === '0xae') {
+		return parseTrackEntry(iterator);
+	}
+
+	if (segmentId === '0xd7') {
+		return parseTrackNumber(iterator);
+	}
+
+	if (segmentId === '0x73c5') {
+		return parseTrackUID(iterator);
+	}
+
+	if (segmentId === '0x9c') {
+		return parseFlagLacing(iterator);
+	}
+
+	if (segmentId === '0x22b59c') {
+		return parseLanguageSegment(iterator);
+	}
+
+	if (segmentId === '0x86') {
+		return parseCodecSegment(iterator);
+	}
+
+	if (segmentId === '0x83') {
+		return parseTrackTypeSegment(iterator);
+	}
+
+	if (segmentId === '0x23e383') {
+		return parseDefaultDurationSegment(iterator);
+	}
+
+	if (segmentId === '0xe0') {
+		return parseVideoSegment(iterator);
+	}
+
+	if (segmentId === '0xb0') {
+		return parseWidthSegment(iterator);
+	}
+
+	if (segmentId === '0xba') {
+		return parseHeightSegment(iterator);
+	}
+
+	if (segmentId === '0x53c0') {
+		return parseAlphaModeSegment(iterator);
+	}
+
 	const length = iterator.getVint(8);
 
 	const bytesRemaining = iterator.byteLength() - iterator.counter.getOffset();