alright

packages/example/src/Encoder/SrcEncoder.tsx

@@ -179,11 +179,27 @@ export const SrcEncoder: React.FC<{
 			if (!mediaState) {
 				throw new Error('mediaState is null');
 			}
+
+			await mediaState.addTrack({
+				type: 'video',
+				color: {
+					transferChracteristics: 'bt709',
+					matrixCoefficients: 'bt709',
+					primaries: 'bt709',
+					fullRange: true,
+				},
+				width: 1920,
+				height: 1080,
+				defaultDuration: 2658,
+				trackNumber: 1,
+				codecId: 'V_VP8',
+			});
+
 			const videoEncoder = await createVideoEncoder({
 				width: track.displayAspectWidth,
 				height: track.displayAspectHeight,
 				onChunk: async (chunk) => {
-					await mediaState.addSample(chunk, 1);
+					// await mediaState.addSample(chunk, 1);
 					const newDuration = Math.round(
 						(chunk.timestamp + (chunk.duration ?? 0)) / 1000,
 					);
@@ -238,6 +254,13 @@ export const SrcEncoder: React.FC<{
 			if (!mediaState) {
 				throw new Error('mediaState is null');
 			}
+
+			await mediaState.addTrack({
+				type: 'audio',
+				trackNumber: 2,
+				codecId: 'A_OPUS',
+			});
+
 			const audioEncoder = await createAudioEncoder({
 				onChunk: async (chunk) => {
 					await mediaState.addSample(chunk, 2);
@@ -249,6 +272,8 @@ export const SrcEncoder: React.FC<{
 						}));
 					});
 				},
+				sampleRate: track.sampleRate,
+				numberOfChannels: track.numberOfChannels,
 			});
 
 			if (!audioEncoder) {
@@ -283,10 +308,8 @@ export const SrcEncoder: React.FC<{
 			}
 
 			return async (audioSample) => {
-				audioDecoder.processSample(audioSample);
-				flushSync(() => {
-					setState((s) => ({...s, audioFrames: s.audioFrames + 1}));
-				});
+				// await mediaState.addSample(new EncodedAudioChunk(audioSample), 2);
+				await audioDecoder.processSample(audioSample);
 			};
 		},
 		[mediaState, setState],

packages/media-parser/src/create/create-media.ts

@@ -4,6 +4,7 @@ import {
 	matroskaToHex,
 	padMatroskaBytes,
 } from '../boxes/webm/make-header';
+import type {BytesAndOffset} from '../boxes/webm/segments/all-segments';
 import {matroskaElements} from '../boxes/webm/segments/all-segments';
 import type {WriterInterface} from '../writers/writer';
 import {
@@ -14,6 +15,7 @@ import {
 import {makeMatroskaHeader} from './matroska-header';
 import {makeMatroskaInfo} from './matroska-info';
 import {createMatroskaSegment} from './matroska-segment';
+import type {MakeTrackAudio, MakeTrackVideo} from './matroska-trackentry';
 import {
 	makeMatroskaAudioTrackEntryBytes,
 	makeMatroskaTracks,
@@ -24,6 +26,7 @@ export type MediaFn = {
 	save: () => Promise<void>;
 	addSample: (chunk: EncodedVideoChunk, trackNumber: number) => Promise<void>;
 	updateDuration: (duration: number) => Promise<void>;
+	addTrack: (track: MakeTrackAudio | MakeTrackVideo) => Promise<void>;
 };
 
 export const createMedia = async (
@@ -37,37 +40,31 @@ export const createMedia = async (
 		timescale: 1_000_000,
 		duration: 2658,
 	});
-	const matroskaVideoTrackEntry = makeMatroskaVideoTrackEntryBytes({
-		color: {
-			transferChracteristics: 'bt709',
-			matrixCoefficients: 'bt709',
-			primaries: 'bt709',
-			fullRange: true,
-		},
-		width: 1920,
-		height: 1080,
-		defaultDuration: 2658,
-		trackNumber: 1,
-		codecId: 'V_VP8',
-	});
-	const matroskaAudioTrackEntry = makeMatroskaAudioTrackEntryBytes({
-		trackNumber: 2,
-		codecId: 'A_OPUS',
-	});
-	const matroskaTracks = makeMatroskaTracks([
-		matroskaVideoTrackEntry,
-		matroskaAudioTrackEntry,
+
+	const currentTracks: BytesAndOffset[] = [];
+
+	const matroskaTracks = makeMatroskaTracks(currentTracks);
+	const matroskaSegment = createMatroskaSegment([
+		matroskaInfo,
+		...matroskaTracks,
 	]);
-	const matroskaSegment = createMatroskaSegment([matroskaInfo, matroskaTracks]);
 
 	const durationOffset =
 		(matroskaSegment.offsets.children[0].children.find(
 			(c) => c.field === 'Duration',
 		)?.offset ?? 0) + w.getWrittenByteCount();
+	const tracksOffset =
+		(matroskaSegment.offsets.children.find((o) => o.field === 'Tracks')
+			?.offset ?? 0) + w.getWrittenByteCount();
+
 	if (!durationOffset) {
 		throw new Error('could not get duration offset');
 	}
 
+	if (!tracksOffset) {
+		throw new Error('could not get tracks offset');
+	}
+
 	await w.write(matroskaSegment.bytes);
 
 	const cluster = createClusterSegment();
@@ -92,6 +89,7 @@ export const createMedia = async (
 			// Maybe it only works by coincidence
 			timecodeRelativeToCluster: Math.round(chunk.timestamp / 1000),
 		});
+
 		clusterSize += simpleBlock.byteLength;
 		await w.updateDataAt(
 			clusterVIntPosition,
@@ -118,6 +116,16 @@ export const createMedia = async (
 		);
 	};
 
+	const addTrack = async (track: BytesAndOffset) => {
+		currentTracks.push(track);
+		const newTracks = makeMatroskaTracks(currentTracks);
+
+		await w.updateDataAt(
+			tracksOffset,
+			combineUint8Arrays(newTracks.map((b) => b.bytes)),
+		);
+	};
+
 	let operationProm = Promise.resolve();
 
 	return {
@@ -132,5 +140,14 @@ export const createMedia = async (
 			operationProm = operationProm.then(() => updateDuration(duration));
 			return operationProm;
 		},
+		addTrack: (track) => {
+			const bytes =
+				track.type === 'video'
+					? makeMatroskaVideoTrackEntryBytes(track)
+					: makeMatroskaAudioTrackEntryBytes(track);
+
+			operationProm = operationProm.then(() => addTrack(bytes));
+			return operationProm;
+		},
 	};
 };

packages/media-parser/src/create/matroska-trackentry.ts

@@ -1,4 +1,4 @@
-import {makeMatroskaBytes} from '../boxes/webm/make-header';
+import {makeMatroskaBytes, padMatroskaBytes} from '../boxes/webm/make-header';
 import type {BytesAndOffset} from '../boxes/webm/segments/all-segments';
 
 export type MatroskaColorParams = {
@@ -140,13 +140,26 @@ export const makeMatroskaVideoBytes = ({
 	});
 };
 
+export type MakeTrackAudio = {
+	trackNumber: number;
+	codecId: string;
+	type: 'audio';
+};
+
+export type MakeTrackVideo = {
+	color: MatroskaColorParams;
+	width: number;
+	height: number;
+	defaultDuration: number;
+	trackNumber: number;
+	codecId: string;
+	type: 'video';
+};
+
 export const makeMatroskaAudioTrackEntryBytes = ({
 	trackNumber,
 	codecId,
-}: {
-	trackNumber: number;
-	codecId: string;
-}) => {
+}: MakeTrackAudio) => {
 	return makeMatroskaBytes({
 		type: 'TrackEntry',
 		minVintWidth: null,
@@ -200,7 +213,7 @@ export const makeMatroskaAudioTrackEntryBytes = ({
 						type: 'SamplingFrequency',
 						minVintWidth: null,
 						value: {
-							value: 44100,
+							value: 48000,
 							size: '64',
 						},
 					},
@@ -226,14 +239,7 @@ export const makeMatroskaVideoTrackEntryBytes = ({
 	defaultDuration,
 	trackNumber,
 	codecId,
-}: {
-	color: MatroskaColorParams;
-	width: number;
-	height: number;
-	defaultDuration: number;
-	trackNumber: number;
-	codecId: string;
-}) => {
+}: MakeTrackVideo) => {
 	return makeMatroskaBytes({
 		type: 'TrackEntry',
 		minVintWidth: null,
@@ -303,9 +309,12 @@ export const makeMatroskaVideoTrackEntryBytes = ({
 };
 
 export const makeMatroskaTracks = (tracks: BytesAndOffset[]) => {
-	return makeMatroskaBytes({
-		type: 'Tracks',
-		value: tracks,
-		minVintWidth: null,
-	});
+	return padMatroskaBytes(
+		makeMatroskaBytes({
+			type: 'Tracks',
+			value: tracks,
+			minVintWidth: null,
+		}),
+		1000,
+	);
 };

packages/webcodecs/src/audio-decoder.ts

@@ -42,7 +42,8 @@ export const createAudioDecoder = async ({
 			await audioDecoder.flush();
 		}
 
-		audioDecoder.decode(new EncodedAudioChunk(audioSample));
+		const chunk = new EncodedAudioChunk(audioSample);
+		audioDecoder.decode(chunk);
 	};
 
 	let queue = Promise.resolve();

packages/webcodecs/src/audio-encoder.ts

@@ -2,8 +2,12 @@ import {encoderWaitForDequeue} from './wait-for-dequeue';
 
 export const createAudioEncoder = async ({
 	onChunk,
+	sampleRate,
+	numberOfChannels,
 }: {
 	onChunk: (chunk: EncodedAudioChunk) => void;
+	sampleRate: number;
+	numberOfChannels: number;
 }) => {
 	if (typeof AudioEncoder === 'undefined') {
 		return null;
@@ -20,10 +24,8 @@ export const createAudioEncoder = async ({
 
 	const audioEncoderConfig: AudioEncoderConfig = {
 		codec: 'opus',
-		// TODO: Hardcoded
-		numberOfChannels: 2,
-		// TODO: Hardcoded and fails if wrong
-		sampleRate: 48000,
+		numberOfChannels,
+		sampleRate,
 		bitrate: 128000,
 	};
 
@@ -36,8 +38,20 @@ export const createAudioEncoder = async ({
 	encoder.configure(audioEncoderConfig);
 
 	const encodeFrame = async (audioData: AudioData) => {
+		console.log({
+			audioData: audioData.timestamp,
+			t: audioData.timestamp,
+			d: audioData.numberOfFrames,
+			s: audioData.numberOfChannels,
+			a: audioData.sampleRate,
+			f: audioData.format,
+		});
 		await encoderWaitForDequeue(encoder);
+		if (encoder.state === 'closed') {
+			return;
+		}
 
+		console.log(audioData.duration, audioData.timestamp);
 		encoder.encode(audioData);
 	};
 

packages/webcodecs/src/video-encoder.ts

@@ -39,6 +39,10 @@ export const createVideoEncoder = async ({
 
 	const encodeFrame = async (frame: VideoFrame) => {
 		await encoderWaitForDequeue(encoder);
+		if (encoder.state === 'closed') {
+			return;
+		}
+
 		encoder.encode(frame, {
 			keyFrame: framesProcessed % 40 === 0,
 		});