From 8c892de7be4b969c31bd6efcf72d94e1cfc47837 Mon Sep 17 00:00:00 2001 From: Jonny Burger Date: Sun, 1 Sep 2024 15:29:30 +0200 Subject: [PATCH] alright --- packages/example/src/Encoder/SrcEncoder.tsx | 33 +++++++++-- .../media-parser/src/create/create-media.ts | 59 ++++++++++++------- .../src/create/matroska-trackentry.ts | 47 +++++++++------ packages/webcodecs/src/audio-decoder.ts | 3 +- packages/webcodecs/src/audio-encoder.ts | 22 +++++-- packages/webcodecs/src/video-encoder.ts | 4 ++ 6 files changed, 118 insertions(+), 50 deletions(-) diff --git a/packages/example/src/Encoder/SrcEncoder.tsx b/packages/example/src/Encoder/SrcEncoder.tsx index 9249945e630..c68c04a7813 100644 --- a/packages/example/src/Encoder/SrcEncoder.tsx +++ b/packages/example/src/Encoder/SrcEncoder.tsx @@ -179,11 +179,27 @@ export const SrcEncoder: React.FC<{ if (!mediaState) { throw new Error('mediaState is null'); } + + await mediaState.addTrack({ + type: 'video', + color: { + transferChracteristics: 'bt709', + matrixCoefficients: 'bt709', + primaries: 'bt709', + fullRange: true, + }, + width: 1920, + height: 1080, + defaultDuration: 2658, + trackNumber: 1, + codecId: 'V_VP8', + }); + const videoEncoder = await createVideoEncoder({ width: track.displayAspectWidth, height: track.displayAspectHeight, onChunk: async (chunk) => { - await mediaState.addSample(chunk, 1); + // await mediaState.addSample(chunk, 1); const newDuration = Math.round( (chunk.timestamp + (chunk.duration ?? 0)) / 1000, ); @@ -238,6 +254,13 @@ export const SrcEncoder: React.FC<{ if (!mediaState) { throw new Error('mediaState is null'); } + + await mediaState.addTrack({ + type: 'audio', + trackNumber: 2, + codecId: 'A_OPUS', + }); + const audioEncoder = await createAudioEncoder({ onChunk: async (chunk) => { await mediaState.addSample(chunk, 2); @@ -249,6 +272,8 @@ export const SrcEncoder: React.FC<{ })); }); }, + sampleRate: track.sampleRate, + numberOfChannels: track.numberOfChannels, }); if (!audioEncoder) { @@ -283,10 +308,8 @@ export const SrcEncoder: React.FC<{ } return async (audioSample) => { - audioDecoder.processSample(audioSample); - flushSync(() => { - setState((s) => ({...s, audioFrames: s.audioFrames + 1})); - }); + // await mediaState.addSample(new EncodedAudioChunk(audioSample), 2); + await audioDecoder.processSample(audioSample); }; }, [mediaState, setState], diff --git a/packages/media-parser/src/create/create-media.ts b/packages/media-parser/src/create/create-media.ts index d1bedc96a0e..756890ac888 100644 --- a/packages/media-parser/src/create/create-media.ts +++ b/packages/media-parser/src/create/create-media.ts @@ -4,6 +4,7 @@ import { matroskaToHex, padMatroskaBytes, } from '../boxes/webm/make-header'; +import type {BytesAndOffset} from '../boxes/webm/segments/all-segments'; import {matroskaElements} from '../boxes/webm/segments/all-segments'; import type {WriterInterface} from '../writers/writer'; import { @@ -14,6 +15,7 @@ import { import {makeMatroskaHeader} from './matroska-header'; import {makeMatroskaInfo} from './matroska-info'; import {createMatroskaSegment} from './matroska-segment'; +import type {MakeTrackAudio, MakeTrackVideo} from './matroska-trackentry'; import { makeMatroskaAudioTrackEntryBytes, makeMatroskaTracks, @@ -24,6 +26,7 @@ export type MediaFn = { save: () => Promise; addSample: (chunk: EncodedVideoChunk, trackNumber: number) => Promise; updateDuration: (duration: number) => Promise; + addTrack: (track: MakeTrackAudio | MakeTrackVideo) => Promise; }; export const createMedia = async ( @@ -37,37 +40,31 @@ export const createMedia = async ( timescale: 1_000_000, duration: 2658, }); - const matroskaVideoTrackEntry = makeMatroskaVideoTrackEntryBytes({ - color: { - transferChracteristics: 'bt709', - matrixCoefficients: 'bt709', - primaries: 'bt709', - fullRange: true, - }, - width: 1920, - height: 1080, - defaultDuration: 2658, - trackNumber: 1, - codecId: 'V_VP8', - }); - const matroskaAudioTrackEntry = makeMatroskaAudioTrackEntryBytes({ - trackNumber: 2, - codecId: 'A_OPUS', - }); - const matroskaTracks = makeMatroskaTracks([ - matroskaVideoTrackEntry, - matroskaAudioTrackEntry, + + const currentTracks: BytesAndOffset[] = []; + + const matroskaTracks = makeMatroskaTracks(currentTracks); + const matroskaSegment = createMatroskaSegment([ + matroskaInfo, + ...matroskaTracks, ]); - const matroskaSegment = createMatroskaSegment([matroskaInfo, matroskaTracks]); const durationOffset = (matroskaSegment.offsets.children[0].children.find( (c) => c.field === 'Duration', )?.offset ?? 0) + w.getWrittenByteCount(); + const tracksOffset = + (matroskaSegment.offsets.children.find((o) => o.field === 'Tracks') + ?.offset ?? 0) + w.getWrittenByteCount(); + if (!durationOffset) { throw new Error('could not get duration offset'); } + if (!tracksOffset) { + throw new Error('could not get tracks offset'); + } + await w.write(matroskaSegment.bytes); const cluster = createClusterSegment(); @@ -92,6 +89,7 @@ export const createMedia = async ( // Maybe it only works by coincidence timecodeRelativeToCluster: Math.round(chunk.timestamp / 1000), }); + clusterSize += simpleBlock.byteLength; await w.updateDataAt( clusterVIntPosition, @@ -118,6 +116,16 @@ export const createMedia = async ( ); }; + const addTrack = async (track: BytesAndOffset) => { + currentTracks.push(track); + const newTracks = makeMatroskaTracks(currentTracks); + + await w.updateDataAt( + tracksOffset, + combineUint8Arrays(newTracks.map((b) => b.bytes)), + ); + }; + let operationProm = Promise.resolve(); return { @@ -132,5 +140,14 @@ export const createMedia = async ( operationProm = operationProm.then(() => updateDuration(duration)); return operationProm; }, + addTrack: (track) => { + const bytes = + track.type === 'video' + ? makeMatroskaVideoTrackEntryBytes(track) + : makeMatroskaAudioTrackEntryBytes(track); + + operationProm = operationProm.then(() => addTrack(bytes)); + return operationProm; + }, }; }; diff --git a/packages/media-parser/src/create/matroska-trackentry.ts b/packages/media-parser/src/create/matroska-trackentry.ts index 41df4d88868..5eb7c99c242 100644 --- a/packages/media-parser/src/create/matroska-trackentry.ts +++ b/packages/media-parser/src/create/matroska-trackentry.ts @@ -1,4 +1,4 @@ -import {makeMatroskaBytes} from '../boxes/webm/make-header'; +import {makeMatroskaBytes, padMatroskaBytes} from '../boxes/webm/make-header'; import type {BytesAndOffset} from '../boxes/webm/segments/all-segments'; export type MatroskaColorParams = { @@ -140,13 +140,26 @@ export const makeMatroskaVideoBytes = ({ }); }; +export type MakeTrackAudio = { + trackNumber: number; + codecId: string; + type: 'audio'; +}; + +export type MakeTrackVideo = { + color: MatroskaColorParams; + width: number; + height: number; + defaultDuration: number; + trackNumber: number; + codecId: string; + type: 'video'; +}; + export const makeMatroskaAudioTrackEntryBytes = ({ trackNumber, codecId, -}: { - trackNumber: number; - codecId: string; -}) => { +}: MakeTrackAudio) => { return makeMatroskaBytes({ type: 'TrackEntry', minVintWidth: null, @@ -200,7 +213,7 @@ export const makeMatroskaAudioTrackEntryBytes = ({ type: 'SamplingFrequency', minVintWidth: null, value: { - value: 44100, + value: 48000, size: '64', }, }, @@ -226,14 +239,7 @@ export const makeMatroskaVideoTrackEntryBytes = ({ defaultDuration, trackNumber, codecId, -}: { - color: MatroskaColorParams; - width: number; - height: number; - defaultDuration: number; - trackNumber: number; - codecId: string; -}) => { +}: MakeTrackVideo) => { return makeMatroskaBytes({ type: 'TrackEntry', minVintWidth: null, @@ -303,9 +309,12 @@ export const makeMatroskaVideoTrackEntryBytes = ({ }; export const makeMatroskaTracks = (tracks: BytesAndOffset[]) => { - return makeMatroskaBytes({ - type: 'Tracks', - value: tracks, - minVintWidth: null, - }); + return padMatroskaBytes( + makeMatroskaBytes({ + type: 'Tracks', + value: tracks, + minVintWidth: null, + }), + 1000, + ); }; diff --git a/packages/webcodecs/src/audio-decoder.ts b/packages/webcodecs/src/audio-decoder.ts index b81cef1594b..a0a724fa210 100644 --- a/packages/webcodecs/src/audio-decoder.ts +++ b/packages/webcodecs/src/audio-decoder.ts @@ -42,7 +42,8 @@ export const createAudioDecoder = async ({ await audioDecoder.flush(); } - audioDecoder.decode(new EncodedAudioChunk(audioSample)); + const chunk = new EncodedAudioChunk(audioSample); + audioDecoder.decode(chunk); }; let queue = Promise.resolve(); diff --git a/packages/webcodecs/src/audio-encoder.ts b/packages/webcodecs/src/audio-encoder.ts index c8f2f71a7f3..5891c2628d4 100644 --- a/packages/webcodecs/src/audio-encoder.ts +++ b/packages/webcodecs/src/audio-encoder.ts @@ -2,8 +2,12 @@ import {encoderWaitForDequeue} from './wait-for-dequeue'; export const createAudioEncoder = async ({ onChunk, + sampleRate, + numberOfChannels, }: { onChunk: (chunk: EncodedAudioChunk) => void; + sampleRate: number; + numberOfChannels: number; }) => { if (typeof AudioEncoder === 'undefined') { return null; @@ -20,10 +24,8 @@ export const createAudioEncoder = async ({ const audioEncoderConfig: AudioEncoderConfig = { codec: 'opus', - // TODO: Hardcoded - numberOfChannels: 2, - // TODO: Hardcoded and fails if wrong - sampleRate: 48000, + numberOfChannels, + sampleRate, bitrate: 128000, }; @@ -36,8 +38,20 @@ export const createAudioEncoder = async ({ encoder.configure(audioEncoderConfig); const encodeFrame = async (audioData: AudioData) => { + console.log({ + audioData: audioData.timestamp, + t: audioData.timestamp, + d: audioData.numberOfFrames, + s: audioData.numberOfChannels, + a: audioData.sampleRate, + f: audioData.format, + }); await encoderWaitForDequeue(encoder); + if (encoder.state === 'closed') { + return; + } + console.log(audioData.duration, audioData.timestamp); encoder.encode(audioData); }; diff --git a/packages/webcodecs/src/video-encoder.ts b/packages/webcodecs/src/video-encoder.ts index 1d7ff4dc9bf..c54e40078ab 100644 --- a/packages/webcodecs/src/video-encoder.ts +++ b/packages/webcodecs/src/video-encoder.ts @@ -39,6 +39,10 @@ export const createVideoEncoder = async ({ const encodeFrame = async (frame: VideoFrame) => { await encoderWaitForDequeue(encoder); + if (encoder.state === 'closed') { + return; + } + encoder.encode(frame, { keyFrame: framesProcessed % 40 === 0, });