From 5a35f2e2e49843125af472424a283f10439d5bc3 Mon Sep 17 00:00:00 2001 From: Jonny Burger Date: Tue, 3 Sep 2024 15:26:11 +0200 Subject: [PATCH 1/2] unhardcode stuff --- .../media-parser/src/create/create-media.ts | 31 +++++-------------- .../src/create/make-duration-with-padding.ts | 15 +++++++++ .../media-parser/src/create/matroska-info.ts | 24 +++----------- packages/media-parser/src/create/timescale.ts | 1 + 4 files changed, 28 insertions(+), 43 deletions(-) create mode 100644 packages/media-parser/src/create/make-duration-with-padding.ts create mode 100644 packages/media-parser/src/create/timescale.ts diff --git a/packages/media-parser/src/create/create-media.ts b/packages/media-parser/src/create/create-media.ts index 63d05c2cc0f..9ffc7fed5bb 100644 --- a/packages/media-parser/src/create/create-media.ts +++ b/packages/media-parser/src/create/create-media.ts @@ -1,9 +1,5 @@ import {getVariableInt} from '../boxes/webm/ebml'; -import { - combineUint8Arrays, - matroskaToHex, - padMatroskaBytes, -} from '../boxes/webm/make-header'; +import {combineUint8Arrays, matroskaToHex} from '../boxes/webm/make-header'; import type {BytesAndOffset} from '../boxes/webm/segments/all-segments'; import {matroskaElements} from '../boxes/webm/segments/all-segments'; import type {WriterInterface} from '../writers/writer'; @@ -12,6 +8,7 @@ import { createClusterSegment, makeSimpleBlock, } from './cluster-segment'; +import {makeDurationWithPadding} from './make-duration-with-padding'; import {makeMatroskaHeader} from './matroska-header'; import {makeMatroskaInfo} from './matroska-info'; import {createMatroskaSegment} from './matroska-segment'; @@ -21,6 +18,7 @@ import { makeMatroskaTracks, makeMatroskaVideoTrackEntryBytes, } from './matroska-trackentry'; +import {CREATE_TIME_SCALE} from './timescale'; export type MediaFn = { save: () => Promise; @@ -43,9 +41,7 @@ export const createMedia = async ( const w = await writer.createContent(); await w.write(header.bytes); const matroskaInfo = makeMatroskaInfo({ - timescale: 1_000_000, - // TODO: Hardcoded - duration: 2658, + timescale: CREATE_TIME_SCALE, }); const currentTracks: BytesAndOffset[] = []; @@ -92,9 +88,9 @@ export const createMedia = async ( keyframe: chunk.type === 'key', lacing: 0, trackNumber, - // TODO: Maybe this is bad, because it's in microseconds, but should be in timescale - // Maybe it only works by coincidence - timecodeRelativeToCluster: Math.round(chunk.timestamp / 1000), + timecodeRelativeToCluster: Math.round( + (chunk.timestamp / CREATE_TIME_SCALE) * 1000, + ), }); clusterSize += simpleBlock.byteLength; @@ -106,18 +102,7 @@ export const createMedia = async ( }; const updateDuration = async (newDuration: number) => { - const blocks = padMatroskaBytes( - { - type: 'Duration', - value: { - value: newDuration, - size: '64', - }, - minVintWidth: null, - }, - // TODO: That's too much padding - 1000, - ); + const blocks = makeDurationWithPadding(newDuration); await w.updateDataAt( durationOffset, combineUint8Arrays(blocks.map((b) => b.bytes)), diff --git a/packages/media-parser/src/create/make-duration-with-padding.ts b/packages/media-parser/src/create/make-duration-with-padding.ts new file mode 100644 index 00000000000..3f6f41b7b4d --- /dev/null +++ b/packages/media-parser/src/create/make-duration-with-padding.ts @@ -0,0 +1,15 @@ +import {padMatroskaBytes} from '../boxes/webm/make-header'; + +export const makeDurationWithPadding = (newDuration: number) => { + return padMatroskaBytes( + { + type: 'Duration', + value: { + value: newDuration, + size: '64', + }, + minVintWidth: null, + }, + 100, + ); +}; diff --git a/packages/media-parser/src/create/matroska-info.ts b/packages/media-parser/src/create/matroska-info.ts index 77b172b1f73..cd0884b83c0 100644 --- a/packages/media-parser/src/create/matroska-info.ts +++ b/packages/media-parser/src/create/matroska-info.ts @@ -1,12 +1,7 @@ -import {makeMatroskaBytes, padMatroskaBytes} from '../boxes/webm/make-header'; +import {makeMatroskaBytes} from '../boxes/webm/make-header'; +import {makeDurationWithPadding} from './make-duration-with-padding'; -export const makeMatroskaInfo = ({ - timescale, - duration, -}: { - timescale: number; - duration: number; -}) => { +export const makeMatroskaInfo = ({timescale}: {timescale: number}) => { return makeMatroskaBytes({ type: 'Info', value: [ @@ -28,18 +23,7 @@ export const makeMatroskaInfo = ({ value: '@remotion/media-parser', minVintWidth: null, }, - ...padMatroskaBytes( - { - type: 'Duration', - value: { - value: duration, - size: '64', - }, - minVintWidth: null, - }, - // TODO: That's too much padding - 1000, - ), + ...makeDurationWithPadding(0), ], minVintWidth: null, }); diff --git a/packages/media-parser/src/create/timescale.ts b/packages/media-parser/src/create/timescale.ts new file mode 100644 index 00000000000..1f50dc02d2c --- /dev/null +++ b/packages/media-parser/src/create/timescale.ts @@ -0,0 +1 @@ +export const CREATE_TIME_SCALE = 1_000_000; From 603eebed4e4bc5950cd4fa520440df06a86239b7 Mon Sep 17 00:00:00 2001 From: Jonny Burger Date: Tue, 3 Sep 2024 16:11:24 +0200 Subject: [PATCH 2/2] create multiple clusters --- .../src/create/cluster-segment.ts | 6 +- packages/media-parser/src/create/cluster.ts | 64 +++++++++++++++++++ .../media-parser/src/create/create-media.ts | 47 ++++---------- 3 files changed, 80 insertions(+), 37 deletions(-) create mode 100644 packages/media-parser/src/create/cluster.ts diff --git a/packages/media-parser/src/create/cluster-segment.ts b/packages/media-parser/src/create/cluster-segment.ts index 8d733fca2bd..69f20270c89 100644 --- a/packages/media-parser/src/create/cluster-segment.ts +++ b/packages/media-parser/src/create/cluster-segment.ts @@ -8,15 +8,15 @@ import { export const CLUSTER_MIN_VINT_WIDTH = 8; -export const createClusterSegment = () => { +export const createClusterSegment = (timestamp: number) => { return makeMatroskaBytes({ type: 'Cluster', value: [ { type: 'Timestamp', - minVintWidth: 4, + minVintWidth: null, value: { - value: 0, + value: timestamp, byteLength: null, }, }, diff --git a/packages/media-parser/src/create/cluster.ts b/packages/media-parser/src/create/cluster.ts new file mode 100644 index 00000000000..71f1eef7118 --- /dev/null +++ b/packages/media-parser/src/create/cluster.ts @@ -0,0 +1,64 @@ +import {getVariableInt} from '../boxes/webm/ebml'; +import {matroskaToHex} from '../boxes/webm/make-header'; +import {matroskaElements} from '../boxes/webm/segments/all-segments'; +import type {Writer} from '../writers/writer'; +import { + CLUSTER_MIN_VINT_WIDTH, + createClusterSegment, + makeSimpleBlock, +} from './cluster-segment'; +import {CREATE_TIME_SCALE} from './timescale'; + +const maxClusterTimestamp = 2 ** 15; + +const timestampToClusterTimestamp = (timestamp: number) => { + return Math.round((timestamp / CREATE_TIME_SCALE) * 1000); +}; + +export const makeCluster = async (w: Writer, timestamp: number) => { + const cluster = createClusterSegment(timestampToClusterTimestamp(timestamp)); + const clusterVIntPosition = + w.getWrittenByteCount() + + cluster.offsets.offset + + matroskaToHex(matroskaElements.Cluster).byteLength; + + let clusterSize = cluster.bytes.byteLength; + await w.write(cluster.bytes); + + const addSample = async (chunk: EncodedVideoChunk, trackNumber: number) => { + const arr = new Uint8Array(chunk.byteLength); + chunk.copyTo(arr); + const timecodeRelativeToCluster = + timestampToClusterTimestamp(chunk.timestamp) - + timestampToClusterTimestamp(timestamp); + if (timecodeRelativeToCluster > maxClusterTimestamp) { + throw new Error('timecodeRelativeToCluster is too big'); + } + + const keyframe = chunk.type === 'key'; + const simpleBlock = makeSimpleBlock({ + bytes: arr, + invisible: false, + keyframe, + lacing: 0, + trackNumber, + timecodeRelativeToCluster, + }); + + clusterSize += simpleBlock.byteLength; + await w.updateDataAt( + clusterVIntPosition, + getVariableInt(clusterSize, CLUSTER_MIN_VINT_WIDTH), + ); + await w.write(simpleBlock); + }; + + const shouldMakeNewCluster = (chunk: EncodedVideoChunk) => { + const newTimestamp = timestampToClusterTimestamp(chunk.timestamp); + const oldTimestamp = timestampToClusterTimestamp(timestamp); + + return newTimestamp - oldTimestamp >= 2000 && chunk.type === 'key'; + }; + + return {addSample, shouldMakeNewCluster}; +}; diff --git a/packages/media-parser/src/create/create-media.ts b/packages/media-parser/src/create/create-media.ts index 9ffc7fed5bb..583e59b4e59 100644 --- a/packages/media-parser/src/create/create-media.ts +++ b/packages/media-parser/src/create/create-media.ts @@ -1,13 +1,7 @@ -import {getVariableInt} from '../boxes/webm/ebml'; -import {combineUint8Arrays, matroskaToHex} from '../boxes/webm/make-header'; +import {combineUint8Arrays} from '../boxes/webm/make-header'; import type {BytesAndOffset} from '../boxes/webm/segments/all-segments'; -import {matroskaElements} from '../boxes/webm/segments/all-segments'; import type {WriterInterface} from '../writers/writer'; -import { - CLUSTER_MIN_VINT_WIDTH, - createClusterSegment, - makeSimpleBlock, -} from './cluster-segment'; +import {makeCluster} from './cluster'; import {makeDurationWithPadding} from './make-duration-with-padding'; import {makeMatroskaHeader} from './matroska-header'; import {makeMatroskaInfo} from './matroska-info'; @@ -70,35 +64,20 @@ export const createMedia = async ( await w.write(matroskaSegment.bytes); - const cluster = createClusterSegment(); - const clusterVIntPosition = - w.getWrittenByteCount() + - cluster.offsets.offset + - matroskaToHex(matroskaElements.Cluster).byteLength; + let currentCluster = await makeCluster(w, 0); + + const getClusterOrMakeNew = async (chunk: EncodedVideoChunk) => { + if (!currentCluster.shouldMakeNewCluster(chunk)) { + return currentCluster; + } - let clusterSize = cluster.bytes.byteLength; - await w.write(cluster.bytes); + currentCluster = await makeCluster(w, chunk.timestamp); + return currentCluster; + }; const addSample = async (chunk: EncodedVideoChunk, trackNumber: number) => { - const arr = new Uint8Array(chunk.byteLength); - chunk.copyTo(arr); - const simpleBlock = makeSimpleBlock({ - bytes: arr, - invisible: false, - keyframe: chunk.type === 'key', - lacing: 0, - trackNumber, - timecodeRelativeToCluster: Math.round( - (chunk.timestamp / CREATE_TIME_SCALE) * 1000, - ), - }); - - clusterSize += simpleBlock.byteLength; - await w.updateDataAt( - clusterVIntPosition, - getVariableInt(clusterSize, CLUSTER_MIN_VINT_WIDTH), - ); - await w.write(simpleBlock); + const cluster = await getClusterOrMakeNew(chunk); + return cluster.addSample(chunk, trackNumber); }; const updateDuration = async (newDuration: number) => {