@remotion/media-parser: Get tracks and packets, apply rotation, new API signature

packages/browser-renderer/.eslintrc

@@ -0,0 +1,3 @@
+{
+  "extends": "@jonny"
+}

packages/browser-renderer/README.md

@@ -0,0 +1,18 @@
+# @remotion/browser-renderer
+
+Private experimental package
+
+[![NPM Downloads](https://img.shields.io/npm/dm/@remotion/browser-renderer.svg?style=flat&color=black&label=Downloads)](https://npmcharts.com/compare/@remotion/browser-renderer?minimal=true)
+
+## Installation
+
+```bash
+npm install @remotion/browser-renderer --save-exact
+```
+
+When installing a Remotion package, make sure to align the version of all `remotion` and `@remotion/*` packages to the same version.
+Remove the `^` character from the version number to use the exact version.
+
+## Usage
+
+This is an internal package and has no documentation.

packages/browser-renderer/package.json

@@ -0,0 +1,34 @@
+{
+	"name": "@remotion/browser-renderer",
+	"version": "4.0.194",
+	"main": "dist/index.js",
+	"sideEffects": false,
+	"repository": {
+		"url": "https://github.com/remotion-dev/remotion/tree/main/packages/browser-renderer"
+	},
+	"bugs": {
+		"url": "https://github.com/remotion-dev/remotion/issues"
+	},
+	"scripts": {
+		"formatting": "prettier src --check",
+		"lint": "eslint src --ext ts,tsx",
+		"build": "tsc -d",
+		"watch": "tsc -w"
+	},
+	"files": [
+		"dist"
+	],
+	"author": "Jonny Burger <[email protected]>",
+	"license": "SEE LICENSE IN LICENSE.md",
+	"dependencies": {
+		"mp4box": "0.5.2"
+	},
+	"peerDependencies": {},
+	"devDependencies": {
+		"@remotion/media-parser": "workspace:*",
+		"@types/dom-webcodecs": "0.1.11"
+	},
+	"keywords": [],
+	"private": true,
+	"description": "Private experimental package"
+}

packages/browser-renderer/src/create-decoder.ts

@@ -0,0 +1,41 @@
+export const createDecoder = ({
+	onProgress,
+	onFrame,
+}: {
+	onProgress: (decoded: number) => void;
+	onFrame: (frame: VideoFrame, keyframe: boolean) => void;
+}) => {
+	let decodedFrames = 0;
+	let nextKeyFrameTimestamp = 0;
+
+	const decoder = new VideoDecoder({
+		async output(inputFrame) {
+			const bitmap = await createImageBitmap(inputFrame);
+
+			const keyFrameEveryHowManySeconds = 2;
+			let keyFrame = false;
+			if (inputFrame.timestamp >= nextKeyFrameTimestamp) {
+				keyFrame = true;
+				nextKeyFrameTimestamp =
+					inputFrame.timestamp + keyFrameEveryHowManySeconds * 1e6;
+			}
+
+			const outputFrame = new VideoFrame(bitmap, {
+				timestamp: inputFrame.timestamp,
+				duration: inputFrame.duration as number,
+			});
+
+			onFrame(outputFrame, keyFrame);
+
+			inputFrame.close();
+
+			decodedFrames++;
+			onProgress(decodedFrames);
+		},
+		error(error) {
+			console.error(error);
+		},
+	});
+
+	return {decoder};
+};

packages/browser-renderer/src/create-encoder.ts

@@ -0,0 +1,64 @@
+import {createFile} from 'mp4box';
+
+const timescale = 90000;
+
+export const createEncoder = ({
+	width,
+	height,
+	onProgress,
+}: {
+	width: number;
+	height: number;
+	onProgress: (encoded: number) => void;
+}) => {
+	let trackID: number | null = null;
+
+	let encodedFrames = 0;
+
+	const outputMp4 = createFile();
+
+	const encoder = new VideoEncoder({
+		output(chunk, metadata) {
+			const uint8 = new Uint8Array(chunk.byteLength);
+			chunk.copyTo(uint8);
+
+			if (trackID === null) {
+				trackID = outputMp4.addTrack({
+					width,
+					height,
+					timescale,
+					avcDecoderConfigRecord: (
+						metadata!.decoderConfig as VideoDecoderConfig
+					).description,
+				});
+			}
+
+			if (chunk.duration === null) {
+				throw new Error('No duration in the chunk');
+			}
+
+			const sampleDuration = chunk.duration / (1_000_000 / timescale);
+
+			outputMp4.addSample(trackID, uint8, {
+				duration: sampleDuration,
+				is_sync: chunk.type === 'key',
+			});
+
+			encodedFrames++;
+			onProgress(encodedFrames);
+		},
+		error(error) {
+			console.error(error);
+		},
+	});
+
+	encoder.configure({
+		codec: 'avc1.4d0034',
+		width,
+		height,
+		hardwareAcceleration: 'prefer-hardware',
+		bitrate: 20_000_000,
+	});
+
+	return {encoder, outputMp4};
+};

packages/browser-renderer/src/get-description.ts

@@ -0,0 +1,24 @@
+import type {MP4File, MP4MediaTrack} from 'mp4box';
+import {DataStream} from 'mp4box';
+
+export const getDescription = ({
+	mp4File,
+	track,
+}: {
+	mp4File: MP4File;
+	track: MP4MediaTrack;
+}) => {
+	const trak = mp4File.getTrackById(track.id);
+	for (const entry of trak.mdia.minf.stbl.stsd.entries) {
+		if (entry.avcC || entry.hvcC) {
+			const stream = new DataStream(undefined, 0, DataStream.BIG_ENDIAN);
+			if (entry.avcC) {
+				entry.avcC.write(stream);
+			} else if (entry.hvcC) {
+				entry.hvcC.write(stream);
+			}
+
+			return new Uint8Array(stream.buffer, 8); // Remove the box header.
+		}
+	}
+};

packages/browser-renderer/src/get-samples.ts

@@ -0,0 +1,31 @@
+import type {MP4File, MP4MediaTrack, Sample} from 'mp4box';
+
+export const getSamples = ({
+	mp4File,
+	track,
+}: {
+	mp4File: MP4File;
+	track: MP4MediaTrack;
+}) => {
+	mp4File.setExtractionOptions(track.id, null, {
+		nbSamples: Infinity,
+	});
+
+	return new Promise<Sample[]>((resolve, reject) => {
+		mp4File.onSamples = (track_id, _ref, samples) => {
+			if (track_id === track.id) {
+				resolve(samples);
+				mp4File.onSamples = undefined;
+				mp4File.onError = undefined;
+			}
+		};
+
+		mp4File.onError = (e) => {
+			reject(e);
+			mp4File.onSamples = undefined;
+			mp4File.onError = undefined;
+		};
+
+		mp4File.start();
+	});
+};

packages/browser-renderer/src/index.ts

@@ -0,0 +1,2 @@
+export {reencodeVideo} from './reencode-video';
+export {parseVideo} from './video-parser';

packages/browser-renderer/src/load-mp4-file.ts

@@ -0,0 +1,48 @@
+import type {MP4ArrayBuffer, MP4File, MP4Info} from 'mp4box';
+import {createFile} from 'mp4box';
+
+type ReturnType = {
+	mp4File: MP4File;
+	info: MP4Info;
+};
+
+const readFile = (file: File): Promise<MP4ArrayBuffer> => {
+	return new Promise<MP4ArrayBuffer>((resolve, reject) => {
+		const reader = new FileReader();
+		reader.onload = () => {
+			(reader.result as MP4ArrayBuffer).fileStart = 0;
+			resolve(reader.result as MP4ArrayBuffer);
+			reader.onerror = null;
+			reader.onload = null;
+		};
+
+		reader.onerror = (error) => {
+			reject(error);
+			reader.onerror = null;
+			reader.onload = null;
+		};
+
+		reader.readAsArrayBuffer(file);
+	});
+};
+
+export const loadMp4File = async (file: File) => {
+	const mp4File = createFile();
+
+	const arrayBuffer = await readFile(file);
+
+	const prom = new Promise<ReturnType>((resolve, reject) => {
+		mp4File.onError = (error) => {
+			reject(error);
+		};
+
+		mp4File.onReady = (info) => {
+			resolve({mp4File, info});
+		};
+	});
+
+	mp4File.appendBuffer(arrayBuffer);
+	mp4File.flush();
+
+	return prom;
+};

packages/browser-renderer/src/reencode-video.ts

@@ -0,0 +1,66 @@
+import {createDecoder} from './create-decoder';
+import {createEncoder} from './create-encoder';
+import {getDescription} from './get-description';
+import {getSamples} from './get-samples';
+import {loadMp4File} from './load-mp4-file';
+
+export const reencodeVideo = async (file: File) => {
+	const {info, mp4File} = await loadMp4File(file);
+	const track = info.videoTracks[0];
+
+	if (!track) {
+		throw new Error('No video track found in the file');
+	}
+
+	const {encoder, outputMp4} = createEncoder({
+		width: track.track_width,
+		height: track.track_height,
+		onProgress: (encoded) => {
+			const encodingProgress = Math.round((100 * encoded) / track.nb_samples);
+			console.log(`Encoding frame ${encoded} (${encodingProgress}%)`);
+		},
+	});
+
+	const {decoder} = createDecoder({
+		onFrame: (frame, keyframe) => {
+			encoder.encode(frame, {keyFrame: keyframe});
+			frame.close();
+		},
+		onProgress: (decoded) => {
+			const decodingProgress = Math.round((100 * decoded) / track.nb_samples);
+			console.log(`Decoding frame ${decoded} (${decodingProgress}%)`);
+		},
+	});
+
+	decoder.configure({
+		codec: track.codec,
+		codedWidth: track.track_width,
+		codedHeight: track.track_height,
+		hardwareAcceleration: 'prefer-hardware',
+		description: getDescription({mp4File, track}),
+	});
+
+	const samples = await getSamples({mp4File, track});
+	console.log(samples);
+
+	for (const sample of samples) {
+		const duration = (sample.duration * 1_000_000) / sample.timescale;
+		const timestamp = (sample.cts * 1_000_000) / sample.timescale;
+
+		const chunk = new EncodedVideoChunk({
+			type: sample.is_sync ? 'key' : 'delta',
+			timestamp,
+			duration,
+			data: sample.data,
+		});
+
+		decoder.decode(chunk);
+	}
+
+	await decoder.flush();
+	await encoder.flush();
+	encoder.close();
+	decoder.close();
+
+	outputMp4.save('mp4box.mp4');
+};