From 79e6186b238e494218522fcebc8769f3d61f0911 Mon Sep 17 00:00:00 2001 From: Jonny Burger Date: Wed, 12 Jun 2024 16:33:48 +0200 Subject: [PATCH 01/36] Start working on captions --- packages/core/src/CompositionManager.tsx | 9 ++++- packages/core/src/index.ts | 1 + packages/core/src/no-react.ts | 6 +++- .../src/assets/calculate-asset-positions.ts | 17 +++++++--- .../src/assets/convert-assets-to-file-urls.ts | 6 ++-- .../assets/download-and-map-assets-to-file.ts | 6 ++-- packages/renderer/src/assets/types.ts | 10 +++--- packages/renderer/src/compress-assets.ts | 8 ++--- packages/renderer/src/filter-asset-types.ts | 17 ++++++++++ packages/renderer/src/render-frames.ts | 33 +++++++++++-------- .../src/test/asset-calculation.test.tsx | 7 +++- .../src/test/asset-compression.test.ts | 7 ++-- .../src/test/dont-skip-assets.test.ts | 7 +++- 13 files changed, 94 insertions(+), 40 deletions(-) create mode 100644 packages/renderer/src/filter-asset-types.ts diff --git a/packages/core/src/CompositionManager.tsx b/packages/core/src/CompositionManager.tsx index bc83c93337a..1381d103b34 100644 --- a/packages/core/src/CompositionManager.tsx +++ b/packages/core/src/CompositionManager.tsx @@ -126,7 +126,7 @@ export type TSequence = { premountDisplay: number | null; } & EnhancedTSequenceData; -export type TRenderAsset = { +export type AudioOrVideoAsset = { type: 'audio' | 'video'; src: string; id: string; @@ -139,6 +139,13 @@ export type TRenderAsset = { audioStartFrame: number; }; +export type ArtifactAsset = { + type: 'artifact'; + id: string; +}; + +export type TRenderAsset = AudioOrVideoAsset | ArtifactAsset; + export const compositionsRef = React.createRef<{ getCompositions: () => AnyComposition[]; }>(); diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index a083bc3af17..461e2711673 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -103,6 +103,7 @@ export { export { AnyCompMetadata, AnyComposition, + AudioOrVideoAsset, SmallTCompMetadata, TCompMetadata, TRenderAsset, diff --git a/packages/core/src/no-react.ts b/packages/core/src/no-react.ts index 6a81eed547d..4559b5f4f72 100644 --- a/packages/core/src/no-react.ts +++ b/packages/core/src/no-react.ts @@ -1,4 +1,8 @@ -export type {TRenderAsset} from './CompositionManager'; +export type { + ArtifactAsset, + AudioOrVideoAsset, + TRenderAsset, +} from './CompositionManager'; export type {ClipRegion} from './NativeLayers'; export { EasingFunction, diff --git a/packages/renderer/src/assets/calculate-asset-positions.ts b/packages/renderer/src/assets/calculate-asset-positions.ts index b8f7fa1d3d4..ecadc15a266 100644 --- a/packages/renderer/src/assets/calculate-asset-positions.ts +++ b/packages/renderer/src/assets/calculate-asset-positions.ts @@ -1,4 +1,5 @@ -import type {TRenderAsset} from 'remotion/no-react'; +import type {AudioOrVideoAsset, TRenderAsset} from 'remotion/no-react'; +import {onlyAudioAndVideoAssets} from '../filter-asset-types'; import {resolveAssetSrc} from '../resolve-asset-src'; import {convertAssetToFlattenedVolume} from './flatten-volume-array'; import type {Assets, MediaAsset, UnsafeAsset} from './types'; @@ -18,9 +19,13 @@ const findFrom = (target: TRenderAsset[], renderAsset: TRenderAsset) => { return true; }; -const copyAndDeduplicateAssets = (renderAssets: TRenderAsset[]) => { - const deduplicated: TRenderAsset[] = []; - for (const renderAsset of renderAssets) { +const copyAndDeduplicateAssets = ( + renderAssets: TRenderAsset[], +): AudioOrVideoAsset[] => { + const onlyAudioAndVideo = onlyAudioAndVideoAssets(renderAssets); + const deduplicated: AudioOrVideoAsset[] = []; + + for (const renderAsset of onlyAudioAndVideo) { if (!deduplicated.find((d) => d.id === renderAsset.id)) { deduplicated.push(renderAsset); } @@ -29,7 +34,9 @@ const copyAndDeduplicateAssets = (renderAssets: TRenderAsset[]) => { return deduplicated; }; -export const calculateAssetPositions = (frames: TRenderAsset[][]): Assets => { +export const calculateAssetPositions = ( + frames: AudioOrVideoAsset[][], +): Assets => { const assets: UnsafeAsset[] = []; for (let frame = 0; frame < frames.length; frame++) { diff --git a/packages/renderer/src/assets/convert-assets-to-file-urls.ts b/packages/renderer/src/assets/convert-assets-to-file-urls.ts index 1d75a95bec1..3f90b22ac82 100644 --- a/packages/renderer/src/assets/convert-assets-to-file-urls.ts +++ b/packages/renderer/src/assets/convert-assets-to-file-urls.ts @@ -1,4 +1,4 @@ -import type {TRenderAsset} from 'remotion/no-react'; +import type {AudioOrVideoAsset} from 'remotion/no-react'; import type {LogLevel} from '../log-level'; import type {FrameAndAssets} from '../render-frames'; import type {RenderMediaOnDownload} from './download-and-map-assets-to-file'; @@ -25,9 +25,9 @@ export const convertAssetsToFileUrls = async ({ downloadMap: DownloadMap; indent: boolean; logLevel: LogLevel; -}): Promise => { +}): Promise => { const chunks = chunk(assets, 1000); - const results: TRenderAsset[][][] = []; + const results: AudioOrVideoAsset[][][] = []; for (const ch of chunks) { const assetPromises = ch.map((frame) => { diff --git a/packages/renderer/src/assets/download-and-map-assets-to-file.ts b/packages/renderer/src/assets/download-and-map-assets-to-file.ts index a34fa32217a..db8c3ebaa11 100644 --- a/packages/renderer/src/assets/download-and-map-assets-to-file.ts +++ b/packages/renderer/src/assets/download-and-map-assets-to-file.ts @@ -1,6 +1,6 @@ import fs from 'node:fs'; import path, {extname} from 'node:path'; -import type {TRenderAsset} from 'remotion/no-react'; +import type {AudioOrVideoAsset} from 'remotion/no-react'; import {random} from 'remotion/no-react'; import {isAssetCompressed} from '../compress-assets'; import {ensureOutputDirectory} from '../ensure-output-directory'; @@ -350,12 +350,12 @@ export const downloadAndMapAssetsToFileUrl = async ({ logLevel, indent, }: { - renderAsset: TRenderAsset; + renderAsset: AudioOrVideoAsset; onDownload: RenderMediaOnDownload | null; downloadMap: DownloadMap; logLevel: LogLevel; indent: boolean; -}): Promise => { +}): Promise => { const cleanup = attachDownloadListenerToEmitter(downloadMap, onDownload); const newSrc = await downloadAsset({ src: renderAsset.src, diff --git a/packages/renderer/src/assets/types.ts b/packages/renderer/src/assets/types.ts index 22624f43918..f22289f6aa6 100644 --- a/packages/renderer/src/assets/types.ts +++ b/packages/renderer/src/assets/types.ts @@ -1,10 +1,10 @@ -import type {TRenderAsset} from 'remotion/no-react'; +import type {AudioOrVideoAsset} from 'remotion/no-react'; // An unsafe asset is an asset with looser types, which occurs // during construction of the asset list. Prefer the MediaAsset // type instead. export type UnsafeAsset = Omit< - TRenderAsset, + AudioOrVideoAsset, 'frame' | 'id' | 'volume' | 'mediaFrame' | 'audioStartFrom' > & { startInVideo: number; @@ -27,9 +27,9 @@ export type MediaAsset = Omit & { }; export const uncompressMediaAsset = ( - allRenderAssets: TRenderAsset[], - assetToUncompress: TRenderAsset, -): TRenderAsset => { + allRenderAssets: AudioOrVideoAsset[], + assetToUncompress: AudioOrVideoAsset, +): AudioOrVideoAsset => { const isCompressed = assetToUncompress.src.match(/same-as-(.*)-([0-9]+)$/); if (!isCompressed) { return assetToUncompress; diff --git a/packages/renderer/src/compress-assets.ts b/packages/renderer/src/compress-assets.ts index b0d6c28aaf0..d2ce0fe50d3 100644 --- a/packages/renderer/src/compress-assets.ts +++ b/packages/renderer/src/compress-assets.ts @@ -3,12 +3,12 @@ * Since we track the `src` property for every frame, Node.JS can run out of memory easily. Instead of duplicating the src for every frame, we save memory by replacing the full base 64 encoded data with a string `same-as-[asset-id]-[frame]` referencing a previous asset with the same src. */ -import type {TRenderAsset} from 'remotion/no-react'; +import type {AudioOrVideoAsset} from 'remotion/no-react'; export const compressAsset = ( - previousRenderAssets: TRenderAsset[], - newRenderAsset: TRenderAsset, -): TRenderAsset => { + previousRenderAssets: AudioOrVideoAsset[], + newRenderAsset: AudioOrVideoAsset, +): AudioOrVideoAsset => { if (newRenderAsset.src.length < 400) { return newRenderAsset; } diff --git a/packages/renderer/src/filter-asset-types.ts b/packages/renderer/src/filter-asset-types.ts new file mode 100644 index 00000000000..b7fe68b32e8 --- /dev/null +++ b/packages/renderer/src/filter-asset-types.ts @@ -0,0 +1,17 @@ +import type { + ArtifactAsset, + AudioOrVideoAsset, + TRenderAsset, +} from 'remotion/no-react'; + +export const onlyAudioAndVideoAssets = ( + assets: TRenderAsset[], +): AudioOrVideoAsset[] => { + return assets.filter( + (asset) => asset.type === 'audio' || asset.type === 'video', + ) as AudioOrVideoAsset[]; +}; + +export const onlyArtifact = (assets: TRenderAsset[]): ArtifactAsset[] => { + return assets.filter((asset) => asset.type === 'artifact') as ArtifactAsset[]; +}; diff --git a/packages/renderer/src/render-frames.ts b/packages/renderer/src/render-frames.ts index b1885eee45b..d29f1f06664 100644 --- a/packages/renderer/src/render-frames.ts +++ b/packages/renderer/src/render-frames.ts @@ -2,8 +2,7 @@ import fs from 'node:fs'; import path from 'node:path'; import {performance} from 'perf_hooks'; // eslint-disable-next-line no-restricted-imports -import type {TAsset} from 'remotion'; -import type {VideoConfig} from 'remotion/no-react'; +import type {AudioOrVideoAsset, VideoConfig} from 'remotion/no-react'; import {NoReactInternals} from 'remotion/no-react'; import type {RenderMediaOnDownload} from './assets/download-and-map-assets-to-file'; import {downloadAndMapAssetsToFileUrl} from './assets/download-and-map-assets-to-file'; @@ -23,6 +22,7 @@ import type {Compositor} from './compositor/compositor'; import {compressAsset} from './compress-assets'; import {cycleBrowserTabs} from './cycle-browser-tabs'; import {handleJavascriptException} from './error-handling/handle-javascript-exception'; +import {onlyArtifact, onlyAudioAndVideoAssets} from './filter-asset-types'; import {findRemotionRoot} from './find-closest-package-json'; import type {FrameRange} from './frame-range'; import {getActualConcurrency} from './get-concurrency'; @@ -144,7 +144,7 @@ type InnerRenderFramesOptions = { export type FrameAndAssets = { frame: number; - assets: TAsset[]; + assets: AudioOrVideoAsset[]; }; export type RenderFramesOptions = { @@ -488,20 +488,24 @@ const innerRenderFrames = async ({ stopPerfMeasure(id); - const compressedAssets = collectedAssets.map((asset) => - compressAsset( - assets - .filter(truthy) - .map((a) => a.assets) - .flat(2), - asset, - ), - ); + const previousRenderAssets = assets + .filter(truthy) + .map((a) => a.assets) + .flat(2); + + const audioAndVideoAssets = onlyAudioAndVideoAssets(collectedAssets); + const artifactAssets = onlyArtifact(collectedAssets); + console.log(artifactAssets); + + const compressedAssets = audioAndVideoAssets.map((asset) => { + return compressAsset(previousRenderAssets, asset); + }); + assets.push({ assets: compressedAssets, frame, }); - compressedAssets.forEach((renderAsset) => { + for (const renderAsset of compressedAssets) { downloadAndMapAssetsToFileUrl({ renderAsset, onDownload, @@ -518,7 +522,8 @@ const innerRenderFrames = async ({ ), ); }); - }); + } + if (!assetsOnly) { framesRendered++; onFrameUpdate?.(framesRendered, frame, performance.now() - startTime); diff --git a/packages/renderer/src/test/asset-calculation.test.tsx b/packages/renderer/src/test/asset-calculation.test.tsx index d1bfc76a42d..1f40f9f33cb 100644 --- a/packages/renderer/src/test/asset-calculation.test.tsx +++ b/packages/renderer/src/test/asset-calculation.test.tsx @@ -7,6 +7,7 @@ import {Audio, interpolate, Sequence, useCurrentFrame, Video} from 'remotion'; import {expect, test} from 'vitest'; import {calculateAssetPositions} from '../assets/calculate-asset-positions'; import type {MediaAsset} from '../assets/types'; +import {onlyAudioAndVideoAssets} from '../filter-asset-types'; import {getAssetsForMarkup} from './get-assets-for-markup'; const basicConfig = { @@ -19,7 +20,11 @@ const basicConfig = { const getPositions = async (Markup: React.FC) => { const assets = await getAssetsForMarkup(Markup, basicConfig); - return calculateAssetPositions(assets); + const onlyAudioAndVideo = assets.map((ass) => { + return onlyAudioAndVideoAssets(ass); + }); + + return calculateAssetPositions(onlyAudioAndVideo); }; const withoutId = (asset: MediaAsset) => { diff --git a/packages/renderer/src/test/asset-compression.test.ts b/packages/renderer/src/test/asset-compression.test.ts index 36ee4d08fc6..0e22d40d577 100644 --- a/packages/renderer/src/test/asset-compression.test.ts +++ b/packages/renderer/src/test/asset-compression.test.ts @@ -2,6 +2,7 @@ import type {TRenderAsset} from 'remotion'; import {expect, test} from 'vitest'; import {calculateAssetPositions} from '../assets/calculate-asset-positions'; import {compressAsset} from '../compress-assets'; +import {onlyAudioAndVideoAssets} from '../filter-asset-types'; test('Should compress and uncompress assets', () => { const uncompressed: TRenderAsset[] = [ @@ -35,8 +36,10 @@ test('Should compress and uncompress assets', () => { ], ].flat(1); - const compressedAssets = uncompressed.map((asset, i) => { - return compressAsset(uncompressed.slice(0, i), asset); + const onlyAudioAndVideo = onlyAudioAndVideoAssets(uncompressed); + + const compressedAssets = onlyAudioAndVideo.map((asset, i) => { + return compressAsset(onlyAudioAndVideo.slice(0, i), asset); }); expect(compressedAssets[0].src).toBe(String('x').repeat(1000)); diff --git a/packages/renderer/src/test/dont-skip-assets.test.ts b/packages/renderer/src/test/dont-skip-assets.test.ts index 84ac0290bab..3f12d1cbae6 100644 --- a/packages/renderer/src/test/dont-skip-assets.test.ts +++ b/packages/renderer/src/test/dont-skip-assets.test.ts @@ -1,6 +1,7 @@ import type {TRenderAsset} from 'remotion'; import {expect, test} from 'vitest'; import {calculateAssetPositions} from '../assets/calculate-asset-positions'; +import {onlyAudioAndVideoAssets} from '../filter-asset-types'; type Truthy = T extends false | '' | 0 | null | undefined ? never : T; // from lodash @@ -9,7 +10,11 @@ function truthy(value: T): value is Truthy { } test('Dont skip assets', () => { - const assetPositions = calculateAssetPositions(mock); + const onlyAudioAndVideo = mock.map((m) => { + return onlyAudioAndVideoAssets(m); + }); + + const assetPositions = calculateAssetPositions(onlyAudioAndVideo); expect(assetPositions).toEqual([ { src: 'http://localhost:3000/4793bac32f610ffba8197b8a3422456f.mp3', From e769c8204fb4a3798e167891fd368ce0957ac179 Mon Sep 17 00:00:00 2001 From: Jonny Burger Date: Wed, 12 Jun 2024 16:38:37 +0200 Subject: [PATCH 02/36] hmm --- .../src/assets/convert-assets-to-file-urls.ts | 2 +- packages/renderer/src/render-frames.ts | 19 ++++++++++++------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/packages/renderer/src/assets/convert-assets-to-file-urls.ts b/packages/renderer/src/assets/convert-assets-to-file-urls.ts index 3f90b22ac82..ef3b6ad1b4a 100644 --- a/packages/renderer/src/assets/convert-assets-to-file-urls.ts +++ b/packages/renderer/src/assets/convert-assets-to-file-urls.ts @@ -31,7 +31,7 @@ export const convertAssetsToFileUrls = async ({ for (const ch of chunks) { const assetPromises = ch.map((frame) => { - const frameAssetPromises = frame.assets.map((a) => { + const frameAssetPromises = frame.audioAndVideoAssets.map((a) => { return downloadAndMapAssetsToFileUrl({ renderAsset: a, onDownload, diff --git a/packages/renderer/src/render-frames.ts b/packages/renderer/src/render-frames.ts index d29f1f06664..e587592ecb9 100644 --- a/packages/renderer/src/render-frames.ts +++ b/packages/renderer/src/render-frames.ts @@ -2,7 +2,11 @@ import fs from 'node:fs'; import path from 'node:path'; import {performance} from 'perf_hooks'; // eslint-disable-next-line no-restricted-imports -import type {AudioOrVideoAsset, VideoConfig} from 'remotion/no-react'; +import type { + ArtifactAsset, + AudioOrVideoAsset, + VideoConfig, +} from 'remotion/no-react'; import {NoReactInternals} from 'remotion/no-react'; import type {RenderMediaOnDownload} from './assets/download-and-map-assets-to-file'; import {downloadAndMapAssetsToFileUrl} from './assets/download-and-map-assets-to-file'; @@ -144,7 +148,8 @@ type InnerRenderFramesOptions = { export type FrameAndAssets = { frame: number; - assets: AudioOrVideoAsset[]; + audioAndVideoAssets: AudioOrVideoAsset[]; + artifactAssets: ArtifactAsset[]; }; export type RenderFramesOptions = { @@ -488,22 +493,22 @@ const innerRenderFrames = async ({ stopPerfMeasure(id); - const previousRenderAssets = assets + const previousAudioRenderAssets = assets .filter(truthy) - .map((a) => a.assets) + .map((a) => a.audioAndVideoAssets) .flat(2); const audioAndVideoAssets = onlyAudioAndVideoAssets(collectedAssets); const artifactAssets = onlyArtifact(collectedAssets); - console.log(artifactAssets); const compressedAssets = audioAndVideoAssets.map((asset) => { - return compressAsset(previousRenderAssets, asset); + return compressAsset(previousAudioRenderAssets, asset); }); assets.push({ - assets: compressedAssets, + audioAndVideoAssets: compressedAssets, frame, + artifactAssets, }); for (const renderAsset of compressedAssets) { downloadAndMapAssetsToFileUrl({ From ef5552d490e65916035e73727d95329139da0855 Mon Sep 17 00:00:00 2001 From: Jonny Burger Date: Wed, 12 Jun 2024 16:47:22 +0200 Subject: [PATCH 03/36] Artifact API --- packages/core/src/Artifact.tsx | 32 ++++++++++++++++++++++++ packages/core/src/CompositionManager.tsx | 2 ++ packages/core/src/index.ts | 2 ++ 3 files changed, 36 insertions(+) create mode 100644 packages/core/src/Artifact.tsx diff --git a/packages/core/src/Artifact.tsx b/packages/core/src/Artifact.tsx new file mode 100644 index 00000000000..9c38bf60d7f --- /dev/null +++ b/packages/core/src/Artifact.tsx @@ -0,0 +1,32 @@ +import type React from 'react'; +import {useContext, useEffect, useState} from 'react'; +import {RenderAssetManager} from './RenderAssetManager'; + +// TODO: Not register in development? +export const Artifact: React.FC<{ + filename: string; + content: string; +}> = ({filename, content}) => { + // TODO: Validate filename and content + const {registerRenderAsset, unregisterRenderAsset} = + useContext(RenderAssetManager); + + const [id] = useState(() => { + return String(Math.random()); + }); + + useEffect(() => { + registerRenderAsset({ + type: 'artifact', + id, + content, + filename, + }); + + return () => { + return unregisterRenderAsset(id); + }; + }, [content, filename, id, registerRenderAsset, unregisterRenderAsset]); + + return null; +}; diff --git a/packages/core/src/CompositionManager.tsx b/packages/core/src/CompositionManager.tsx index 1381d103b34..b21e617883a 100644 --- a/packages/core/src/CompositionManager.tsx +++ b/packages/core/src/CompositionManager.tsx @@ -142,6 +142,8 @@ export type AudioOrVideoAsset = { export type ArtifactAsset = { type: 'artifact'; id: string; + filename: string; + content: string; }; export type TRenderAsset = AudioOrVideoAsset | ArtifactAsset; diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index 461e2711673..f52ac894315 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1,3 +1,4 @@ +import {Artifact} from './Artifact.js'; import './asset-types.js'; import {Clipper} from './Clipper.js'; import type {Codec} from './codec.js'; @@ -170,6 +171,7 @@ export const Experimental = { * @see [Documentation](https://www.remotion.dev/docs/null) */ Null, + Artifact, useIsPlayer, }; From b95253e2a5f1e1f7091301a3fa3adeeb8989a62c Mon Sep 17 00:00:00 2001 From: Jonny Burger Date: Wed, 12 Jun 2024 17:07:45 +0200 Subject: [PATCH 04/36] artifact --- packages/core/src/Artifact.tsx | 16 +++++++++++++--- packages/core/src/CompositionManager.tsx | 1 + packages/example/src/Root.tsx | 19 +++++++++++++++---- .../src/SubtitleArtifact/SubtitleArtifact.tsx | 6 ++++++ packages/renderer/src/render-frames.ts | 18 ++++++++++++++++++ 5 files changed, 53 insertions(+), 7 deletions(-) create mode 100644 packages/example/src/SubtitleArtifact/SubtitleArtifact.tsx diff --git a/packages/core/src/Artifact.tsx b/packages/core/src/Artifact.tsx index 9c38bf60d7f..b2fd295a435 100644 --- a/packages/core/src/Artifact.tsx +++ b/packages/core/src/Artifact.tsx @@ -1,15 +1,17 @@ import type React from 'react'; import {useContext, useEffect, useState} from 'react'; import {RenderAssetManager} from './RenderAssetManager'; +import {useCurrentFrame} from './use-current-frame'; // TODO: Not register in development? export const Artifact: React.FC<{ - filename: string; - content: string; + readonly filename: string; + readonly content: string; }> = ({filename, content}) => { // TODO: Validate filename and content const {registerRenderAsset, unregisterRenderAsset} = useContext(RenderAssetManager); + const frame = useCurrentFrame(); const [id] = useState(() => { return String(Math.random()); @@ -21,12 +23,20 @@ export const Artifact: React.FC<{ id, content, filename, + frame, }); return () => { return unregisterRenderAsset(id); }; - }, [content, filename, id, registerRenderAsset, unregisterRenderAsset]); + }, [ + content, + filename, + frame, + id, + registerRenderAsset, + unregisterRenderAsset, + ]); return null; }; diff --git a/packages/core/src/CompositionManager.tsx b/packages/core/src/CompositionManager.tsx index b21e617883a..7bed4510d9d 100644 --- a/packages/core/src/CompositionManager.tsx +++ b/packages/core/src/CompositionManager.tsx @@ -144,6 +144,7 @@ export type ArtifactAsset = { id: string; filename: string; content: string; + frame: number; }; export type TRenderAsset = AudioOrVideoAsset | ArtifactAsset; diff --git a/packages/example/src/Root.tsx b/packages/example/src/Root.tsx index 20e943ef7f0..ee5ed32611f 100644 --- a/packages/example/src/Root.tsx +++ b/packages/example/src/Root.tsx @@ -4,9 +4,9 @@ import { CalculateMetadataFunction, Composition, Folder, - Still, getInputProps, staticFile, + Still, } from 'remotion'; import {z} from 'zod'; import {TwentyTwoKHzAudio} from './22KhzAudio'; @@ -32,8 +32,8 @@ import {Layers} from './Layers'; import {ManyAudio} from './ManyAudio'; import {MissingImg} from './MissingImg'; import { - OffthreadRemoteVideo, calculateMetadataFn, + OffthreadRemoteVideo, } from './OffthreadRemoteVideo/OffthreadRemoteVideo'; import {OrbScene} from './Orb'; import {ShapesMorph} from './Paths/ShapesMorph'; @@ -47,8 +47,8 @@ import RiveVehicle from './Rive/RiveExample'; import {ScalePath} from './ScalePath'; import { ArrayTest, - SchemaTest, schemaArrayTestSchema, + SchemaTest, schemaTestSchema, } from './SchemaTest'; import {Scripts} from './Scripts'; @@ -74,6 +74,7 @@ import { } from './StudioApis/SaveDefaultProps'; import {TriggerCalculateMetadata} from './StudioApis/TriggerCalculateMetadata'; import {WriteStaticFile} from './StudioApis/WriteStaticFile'; +import './style.css'; import {Tailwind} from './Tailwind'; import {TenFrameTester} from './TenFrameTester'; import {TextStroke} from './TextStroke'; @@ -90,7 +91,6 @@ import {VideoSpeed} from './VideoSpeed'; import {VideoTesting} from './VideoTesting'; import {WarpDemoOuter} from './WarpText'; import {WarpDemo2} from './WarpText/demo2'; -import './style.css'; import {WatchStaticDemo} from './watch-static'; if (alias !== 'alias') { throw new Error('should support TS aliases'); @@ -98,6 +98,7 @@ if (alias !== 'alias') { // @ts-expect-error no types import styles from './styles.module.scss'; +import {SubtitleArtifact} from './SubtitleArtifact/SubtitleArtifact'; if (!styles.hithere) { throw new Error('should support SCSS modules'); @@ -1309,6 +1310,16 @@ export const Index: React.FC = () => { defaultProps={{color: 'green'}} /> + + + ); }; diff --git a/packages/example/src/SubtitleArtifact/SubtitleArtifact.tsx b/packages/example/src/SubtitleArtifact/SubtitleArtifact.tsx new file mode 100644 index 00000000000..56e892fdcfb --- /dev/null +++ b/packages/example/src/SubtitleArtifact/SubtitleArtifact.tsx @@ -0,0 +1,6 @@ +import React from 'react'; +import {Experimental} from 'remotion'; + +export const SubtitleArtifact: React.FC = () => { + return ; +}; diff --git a/packages/renderer/src/render-frames.ts b/packages/renderer/src/render-frames.ts index e587592ecb9..34c5746f487 100644 --- a/packages/renderer/src/render-frames.ts +++ b/packages/renderer/src/render-frames.ts @@ -498,9 +498,27 @@ const innerRenderFrames = async ({ .map((a) => a.audioAndVideoAssets) .flat(2); + const previousArtifactAssets = assets + .filter(truthy) + .map((a) => a.artifactAssets) + .flat(2); + const audioAndVideoAssets = onlyAudioAndVideoAssets(collectedAssets); const artifactAssets = onlyArtifact(collectedAssets); + for (const artifact of artifactAssets) { + for (const previousArtifact of previousArtifactAssets) { + if (artifact.filename === previousArtifact.filename) { + reject( + new Error( + `An artifact with output "${artifact.filename}" was already registered at frame ${previousArtifact.frame}, but now registered again at frame ${artifact.frame}.`, + ), + ); + return; + } + } + } + const compressedAssets = audioAndVideoAssets.map((asset) => { return compressAsset(previousAudioRenderAssets, asset); }); From 8b0550b1d3d82158f3a9ee9a3bbecc3b3d8e5ca8 Mon Sep 17 00:00:00 2001 From: Jonny Burger Date: Wed, 12 Jun 2024 17:10:13 +0200 Subject: [PATCH 05/36] Update SubtitleArtifact.tsx --- .../example/src/SubtitleArtifact/SubtitleArtifact.tsx | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/example/src/SubtitleArtifact/SubtitleArtifact.tsx b/packages/example/src/SubtitleArtifact/SubtitleArtifact.tsx index 56e892fdcfb..fa36b1dc18a 100644 --- a/packages/example/src/SubtitleArtifact/SubtitleArtifact.tsx +++ b/packages/example/src/SubtitleArtifact/SubtitleArtifact.tsx @@ -1,6 +1,10 @@ import React from 'react'; -import {Experimental} from 'remotion'; +import {Experimental, useCurrentFrame} from 'remotion'; export const SubtitleArtifact: React.FC = () => { - return ; + const frame = useCurrentFrame(); + + return frame === 0 ? ( + + ) : null; }; From 367224479d83a5031173bbf51e3da3bbe48df5ab Mon Sep 17 00:00:00 2001 From: Jonny Burger Date: Wed, 12 Jun 2024 17:55:44 +0200 Subject: [PATCH 06/36] scaffold our artifact --- packages/cli/src/benchmark.ts | 1 + packages/cli/src/render-flows/render.ts | 7 ++++ .../functions/render-media-single-thread.ts | 12 +++++- packages/lambda/src/functions/renderer.ts | 12 +++++- packages/renderer/src/index.ts | 2 +- packages/renderer/src/render-frames.ts | 39 ++++++++++++++----- packages/renderer/src/render-media.ts | 7 ++++ 7 files changed, 68 insertions(+), 12 deletions(-) diff --git a/packages/cli/src/benchmark.ts b/packages/cli/src/benchmark.ts index 83967681f41..2db1ea5cca4 100644 --- a/packages/cli/src/benchmark.ts +++ b/packages/cli/src/benchmark.ts @@ -489,6 +489,7 @@ export const benchmarkCommand = async ( }).value, compositionStart: 0, onBrowserDownload, + onArtifact: () => undefined, }, (run, progress) => { benchmarkProgress.update( diff --git a/packages/cli/src/render-flows/render.ts b/packages/cli/src/render-flows/render.ts index f3493511577..2e61f875f1d 100644 --- a/packages/cli/src/render-flows/render.ts +++ b/packages/cli/src/render-flows/render.ts @@ -11,6 +11,7 @@ import type { FrameRange, LogLevel, NumberOfGifLoops, + OnArtifact, PixelFormat, ProResProfile, RenderMediaOnDownload, @@ -402,6 +403,10 @@ export const renderVideoFlow = async ({ logLevel, ); + const onArtifact: OnArtifact = (artifact) => { + console.log('artifact', artifact); + }; + const absoluteSeparateAudioTo = separateAudioTo === null ? null : path.resolve(separateAudioTo); const exists = existsSync(absoluteOutputFile); @@ -491,6 +496,7 @@ export const renderVideoFlow = async ({ compositionStart: 0, forSeamlessAacConcatenation, onBrowserDownload, + onArtifact, }); Log.info({indent, logLevel}, chalk.blue(`▶ ${absoluteOutputFile}`)); @@ -580,6 +586,7 @@ export const renderVideoFlow = async ({ forSeamlessAacConcatenation, compositionStart: 0, onBrowserDownload, + onArtifact, }); if (!updatesDontOverwrite) { updateRenderProgress({newline: true, printToConsole: true}); diff --git a/packages/cloudrun/src/functions/render-media-single-thread.ts b/packages/cloudrun/src/functions/render-media-single-thread.ts index cd6b677f066..fcdc80d1531 100644 --- a/packages/cloudrun/src/functions/render-media-single-thread.ts +++ b/packages/cloudrun/src/functions/render-media-single-thread.ts @@ -1,6 +1,10 @@ import type * as ff from '@google-cloud/functions-framework'; import {Storage} from '@google-cloud/storage'; -import type {ChromiumOptions, RenderMediaOnProgress} from '@remotion/renderer'; +import type { + ChromiumOptions, + OnArtifact, + RenderMediaOnProgress, +} from '@remotion/renderer'; import {RenderInternals} from '@remotion/renderer'; import {NoReactInternals} from 'remotion/no-react'; import {VERSION} from 'remotion/version'; @@ -59,6 +63,11 @@ export const renderMediaSingleThread = async ( enableMultiProcessOnLinux: true, }; + const onArtifact: OnArtifact = (artifact) => { + // TODO: Do something with the artifact + console.log(artifact); + }; + await RenderInternals.internalRenderMedia({ composition: { ...composition, @@ -126,6 +135,7 @@ export const renderMediaSingleThread = async ( onBrowserDownload: () => { throw new Error('Should not download a browser in Cloud Run'); }, + onArtifact, }); const storage = new Storage(); diff --git a/packages/lambda/src/functions/renderer.ts b/packages/lambda/src/functions/renderer.ts index 8e15291dfa3..ac45bec1aa2 100644 --- a/packages/lambda/src/functions/renderer.ts +++ b/packages/lambda/src/functions/renderer.ts @@ -1,4 +1,9 @@ -import type {AudioCodec, BrowserLog, Codec} from '@remotion/renderer'; +import type { + AudioCodec, + BrowserLog, + Codec, + OnArtifact, +} from '@remotion/renderer'; import {RenderInternals} from '@remotion/renderer'; import fs from 'node:fs'; import path from 'node:path'; @@ -167,6 +172,10 @@ const renderHandler = async ({ params.everyNthFrame, ); + const onArtifact: OnArtifact = (artifact) => { + console.log(artifact); + }; + await new Promise((resolve, reject) => { RenderInternals.internalRenderMedia({ repro: false, @@ -271,6 +280,7 @@ const renderHandler = async ({ onBrowserDownload: () => { throw new Error('Should not download a browser in Lambda'); }, + onArtifact, }) .then(({slowestFrames}) => { RenderInternals.Log.verbose( diff --git a/packages/renderer/src/index.ts b/packages/renderer/src/index.ts index d30ac4b6e7b..ac072fa0339 100644 --- a/packages/renderer/src/index.ts +++ b/packages/renderer/src/index.ts @@ -108,7 +108,7 @@ export {X264Preset} from './options/x264-preset'; export {PixelFormat} from './pixel-format'; export {RemotionServer} from './prepare-server'; export {ProResProfile} from './prores-profile'; -export {RenderFramesOptions, renderFrames} from './render-frames'; +export {OnArtifact, RenderFramesOptions, renderFrames} from './render-frames'; export { InternalRenderMediaOptions, RenderMediaOnProgress, diff --git a/packages/renderer/src/render-frames.ts b/packages/renderer/src/render-frames.ts index 34c5746f487..8ac752ef349 100644 --- a/packages/renderer/src/render-frames.ts +++ b/packages/renderer/src/render-frames.ts @@ -2,11 +2,7 @@ import fs from 'node:fs'; import path from 'node:path'; import {performance} from 'perf_hooks'; // eslint-disable-next-line no-restricted-imports -import type { - ArtifactAsset, - AudioOrVideoAsset, - VideoConfig, -} from 'remotion/no-react'; +import type {AudioOrVideoAsset, VideoConfig} from 'remotion/no-react'; import {NoReactInternals} from 'remotion/no-react'; import type {RenderMediaOnDownload} from './assets/download-and-map-assets-to-file'; import {downloadAndMapAssetsToFileUrl} from './assets/download-and-map-assets-to-file'; @@ -71,7 +67,9 @@ import {wrapWithErrorHandling} from './wrap-with-error-handling'; const MAX_RETRIES_PER_FRAME = 1; -export type InternalRenderFramesOptions = { +export type OnArtifact = (asset: EmittedAsset) => void; + +type InternalRenderFramesOptions = { onStart: null | ((data: OnStartData) => void); onFrameUpdate: | null @@ -105,8 +103,15 @@ export type InternalRenderFramesOptions = { serializedResolvedPropsWithCustomSchema: string; parallelEncodingEnabled: boolean; compositionStart: number; + onArtifact: OnArtifact | null; } & ToOptions; +type EmittedAsset = { + filename: string; + content: string | Buffer; + frame: number; +}; + type InnerRenderFramesOptions = { onStart: null | ((data: OnStartData) => void); onFrameUpdate: @@ -123,6 +128,7 @@ type InnerRenderFramesOptions = { everyNthFrame: number; onBrowserLog: null | ((log: BrowserLog) => void); onFrameBuffer: null | ((buffer: Buffer, frame: number) => void); + onArtifact: OnArtifact | null; onDownload: RenderMediaOnDownload | null; timeoutInMilliseconds: number; scale: number; @@ -146,10 +152,15 @@ type InnerRenderFramesOptions = { compositionStart: number; } & ToOptions; +type ArtifactWithoutContent = { + frame: number; + filename: string; +}; + export type FrameAndAssets = { frame: number; audioAndVideoAssets: AudioOrVideoAsset[]; - artifactAssets: ArtifactAsset[]; + artifactAssets: ArtifactWithoutContent[]; }; export type RenderFramesOptions = { @@ -190,6 +201,7 @@ export type RenderFramesOptions = { composition: VideoConfig; muted?: boolean; concurrency?: number | string | null; + onArtifact?: OnArtifact | null; serveUrl: string; } & Partial>; @@ -511,7 +523,7 @@ const innerRenderFrames = async ({ if (artifact.filename === previousArtifact.filename) { reject( new Error( - `An artifact with output "${artifact.filename}" was already registered at frame ${previousArtifact.frame}, but now registered again at frame ${artifact.frame}.`, + `An artifact with output "${artifact.filename}" was already registered at frame ${previousArtifact.frame}, but now registered again at frame ${artifact.frame}. An artifact`, ), ); return; @@ -526,7 +538,12 @@ const innerRenderFrames = async ({ assets.push({ audioAndVideoAssets: compressedAssets, frame, - artifactAssets, + artifactAssets: artifactAssets.map((a) => { + return { + frame: a.frame, + filename: a.filename, + }; + }), }); for (const renderAsset of compressedAssets) { downloadAndMapAssetsToFileUrl({ @@ -784,6 +801,7 @@ const internalRenderFramesRaw = ({ forSeamlessAacConcatenation, compositionStart, onBrowserDownload, + onArtifact, }: InternalRenderFramesOptions): Promise => { validateDimension( composition.height, @@ -913,6 +931,7 @@ const internalRenderFramesRaw = ({ forSeamlessAacConcatenation, compositionStart, onBrowserDownload, + onArtifact, }); }), ]) @@ -1007,6 +1026,7 @@ export const renderFrames = ( offthreadVideoCacheSizeInBytes, binariesDirectory, onBrowserDownload, + onArtifact, } = options; if (!composition) { @@ -1078,5 +1098,6 @@ export const renderFrames = ( onBrowserDownload: onBrowserDownload ?? defaultBrowserDownloadProgress({indent, logLevel, api: 'renderFrames()'}), + onArtifact: onArtifact ?? null, }); }; diff --git a/packages/renderer/src/render-media.ts b/packages/renderer/src/render-media.ts index fce7039b5a7..191035cdc7f 100644 --- a/packages/renderer/src/render-media.ts +++ b/packages/renderer/src/render-media.ts @@ -53,6 +53,7 @@ import {prespawnFfmpeg} from './prespawn-ffmpeg'; import {shouldUseParallelEncoding} from './prestitcher-memory-usage'; import type {ProResProfile} from './prores-profile'; import {validateSelectedCodecAndProResCombination} from './prores-profile'; +import type {OnArtifact} from './render-frames'; import {internalRenderFrames} from './render-frames'; import { disableRepro, @@ -125,6 +126,7 @@ export type InternalRenderMediaOptions = { concurrency: number | string | null; binariesDirectory: string | null; compositionStart: number; + onArtifact: OnArtifact | null; } & MoreRenderMediaOptions; type Prettify = { @@ -179,6 +181,7 @@ export type RenderMediaOptions = Prettify<{ colorSpace?: ColorSpace; repro?: boolean; binariesDirectory?: string | null; + onArtifact?: OnArtifact; }> & Partial; @@ -241,6 +244,7 @@ const internalRenderMediaRaw = ({ forSeamlessAacConcatenation, compositionStart, onBrowserDownload, + onArtifact, }: InternalRenderMediaOptions): Promise => { if (repro) { enableRepro({ @@ -649,6 +653,7 @@ const internalRenderMediaRaw = ({ compositionStart, forSeamlessAacConcatenation, onBrowserDownload, + onArtifact, }); return renderFramesProc; @@ -863,6 +868,7 @@ export const renderMedia = ({ separateAudioTo, forSeamlessAacConcatenation, onBrowserDownload, + onArtifact, }: RenderMediaOptions): Promise => { if (quality !== undefined) { console.warn( @@ -937,6 +943,7 @@ export const renderMedia = ({ onBrowserDownload: onBrowserDownload ?? defaultBrowserDownloadProgress({indent, logLevel, api: 'renderMedia()'}), + onArtifact: onArtifact ?? null, // TODO: In the future, introduce this as a public API when launching the distributed rendering API compositionStart: 0, }); From d654062a8d6d3b1184b7b9a6b6d5e79c8c9c6ac2 Mon Sep 17 00:00:00 2001 From: Jonny Burger Date: Wed, 12 Jun 2024 17:58:45 +0200 Subject: [PATCH 07/36] do something with the artifact --- packages/cli/src/render-flows/render.ts | 4 ++-- packages/renderer/src/render-frames.ts | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/packages/cli/src/render-flows/render.ts b/packages/cli/src/render-flows/render.ts index 2e61f875f1d..6fbb8faa795 100644 --- a/packages/cli/src/render-flows/render.ts +++ b/packages/cli/src/render-flows/render.ts @@ -29,7 +29,7 @@ import type { RenderingProgressInput, StitchingProgressInput, } from '@remotion/studio-server'; -import fs, {existsSync} from 'node:fs'; +import fs, {existsSync, writeFileSync} from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import {NoReactInternals} from 'remotion/no-react'; @@ -404,7 +404,7 @@ export const renderVideoFlow = async ({ ); const onArtifact: OnArtifact = (artifact) => { - console.log('artifact', artifact); + writeFileSync(artifact.filename, artifact.content); }; const absoluteSeparateAudioTo = diff --git a/packages/renderer/src/render-frames.ts b/packages/renderer/src/render-frames.ts index 8ac752ef349..15ef6b4ea3a 100644 --- a/packages/renderer/src/render-frames.ts +++ b/packages/renderer/src/render-frames.ts @@ -239,6 +239,7 @@ const innerRenderFrames = async ({ parallelEncodingEnabled, compositionStart, forSeamlessAacConcatenation, + onArtifact, }: Omit< InnerRenderFramesOptions, 'offthreadVideoCacheSizeInBytes' @@ -529,6 +530,8 @@ const innerRenderFrames = async ({ return; } } + + onArtifact?.(artifact); } const compressedAssets = audioAndVideoAssets.map((asset) => { From c2bb5e99535a2f999e397601aaf7ec7b0d4b8346 Mon Sep 17 00:00:00 2001 From: Jonny Burger Date: Thu, 13 Jun 2024 10:50:04 +0200 Subject: [PATCH 08/36] general CLI handler --- packages/cli/src/on-artifact.ts | 6 ++++++ packages/cli/src/render-flows/render.ts | 12 ++++-------- packages/core/src/index.ts | 3 +-- .../src/SubtitleArtifact/SubtitleArtifact.tsx | 4 ++-- packages/renderer/src/render-frames.ts | 2 +- 5 files changed, 14 insertions(+), 13 deletions(-) create mode 100644 packages/cli/src/on-artifact.ts diff --git a/packages/cli/src/on-artifact.ts b/packages/cli/src/on-artifact.ts new file mode 100644 index 00000000000..869d7d40f35 --- /dev/null +++ b/packages/cli/src/on-artifact.ts @@ -0,0 +1,6 @@ +import type {OnArtifact} from '@remotion/renderer'; +import {writeFileSync} from 'fs'; + +export const onArtifactOnCli: OnArtifact = (artifact) => { + writeFileSync(artifact.filename, artifact.content); +}; diff --git a/packages/cli/src/render-flows/render.ts b/packages/cli/src/render-flows/render.ts index 6fbb8faa795..bbb22fbe550 100644 --- a/packages/cli/src/render-flows/render.ts +++ b/packages/cli/src/render-flows/render.ts @@ -11,7 +11,6 @@ import type { FrameRange, LogLevel, NumberOfGifLoops, - OnArtifact, PixelFormat, ProResProfile, RenderMediaOnDownload, @@ -29,7 +28,7 @@ import type { RenderingProgressInput, StitchingProgressInput, } from '@remotion/studio-server'; -import fs, {existsSync, writeFileSync} from 'node:fs'; +import fs, {existsSync} from 'node:fs'; import os from 'node:os'; import path from 'node:path'; import {NoReactInternals} from 'remotion/no-react'; @@ -43,6 +42,7 @@ import {makeHyperlink} from '../hyperlinks/make-link'; import {getVideoImageFormat} from '../image-formats'; import {Log} from '../log'; import {makeOnDownload} from '../make-on-download'; +import {onArtifactOnCli} from '../on-artifact'; import {parsedCli, quietFlagProvided} from '../parsed-cli'; import { LABEL_WIDTH, @@ -403,10 +403,6 @@ export const renderVideoFlow = async ({ logLevel, ); - const onArtifact: OnArtifact = (artifact) => { - writeFileSync(artifact.filename, artifact.content); - }; - const absoluteSeparateAudioTo = separateAudioTo === null ? null : path.resolve(separateAudioTo); const exists = existsSync(absoluteOutputFile); @@ -496,7 +492,7 @@ export const renderVideoFlow = async ({ compositionStart: 0, forSeamlessAacConcatenation, onBrowserDownload, - onArtifact, + onArtifact: onArtifactOnCli, }); Log.info({indent, logLevel}, chalk.blue(`▶ ${absoluteOutputFile}`)); @@ -586,7 +582,7 @@ export const renderVideoFlow = async ({ forSeamlessAacConcatenation, compositionStart: 0, onBrowserDownload, - onArtifact, + onArtifact: onArtifactOnCli, }); if (!updatesDontOverwrite) { updateRenderProgress({newline: true, printToConsole: true}); diff --git a/packages/core/src/index.ts b/packages/core/src/index.ts index f52ac894315..88ed8ff801b 100644 --- a/packages/core/src/index.ts +++ b/packages/core/src/index.ts @@ -1,4 +1,3 @@ -import {Artifact} from './Artifact.js'; import './asset-types.js'; import {Clipper} from './Clipper.js'; import type {Codec} from './codec.js'; @@ -92,6 +91,7 @@ export type BundleState = checkMultipleRemotionVersions(); export * from './AbsoluteFill.js'; +export {Artifact} from './Artifact.js'; export * from './audio/index.js'; export {cancelRender} from './cancel-render.js'; export { @@ -171,7 +171,6 @@ export const Experimental = { * @see [Documentation](https://www.remotion.dev/docs/null) */ Null, - Artifact, useIsPlayer, }; diff --git a/packages/example/src/SubtitleArtifact/SubtitleArtifact.tsx b/packages/example/src/SubtitleArtifact/SubtitleArtifact.tsx index fa36b1dc18a..14ca160c160 100644 --- a/packages/example/src/SubtitleArtifact/SubtitleArtifact.tsx +++ b/packages/example/src/SubtitleArtifact/SubtitleArtifact.tsx @@ -1,10 +1,10 @@ import React from 'react'; -import {Experimental, useCurrentFrame} from 'remotion'; +import {Artifact, useCurrentFrame} from 'remotion'; export const SubtitleArtifact: React.FC = () => { const frame = useCurrentFrame(); return frame === 0 ? ( - + ) : null; }; diff --git a/packages/renderer/src/render-frames.ts b/packages/renderer/src/render-frames.ts index 15ef6b4ea3a..6e753eddffc 100644 --- a/packages/renderer/src/render-frames.ts +++ b/packages/renderer/src/render-frames.ts @@ -108,7 +108,7 @@ type InternalRenderFramesOptions = { type EmittedAsset = { filename: string; - content: string | Buffer; + content: string | Uint8Array; frame: number; }; From fcbcac71bd289c77474aceacfa011d1d4c4b56b9 Mon Sep 17 00:00:00 2001 From: Jonny Burger Date: Thu, 13 Jun 2024 12:59:47 +0200 Subject: [PATCH 09/36] artifact state --- packages/cli/src/on-artifact.ts | 30 ++++++++++++++++++++++-- packages/cli/src/progress-types.ts | 3 +++ packages/cli/src/render-flows/render.ts | 15 +++++++++--- packages/studio-shared/src/index.ts | 2 ++ packages/studio-shared/src/render-job.ts | 11 +++++++++ 5 files changed, 56 insertions(+), 5 deletions(-) diff --git a/packages/cli/src/on-artifact.ts b/packages/cli/src/on-artifact.ts index 869d7d40f35..715d025cd55 100644 --- a/packages/cli/src/on-artifact.ts +++ b/packages/cli/src/on-artifact.ts @@ -1,6 +1,32 @@ import type {OnArtifact} from '@remotion/renderer'; +import type {ArtifactProgress} from '@remotion/studio-shared'; import {writeFileSync} from 'fs'; +import path from 'path'; -export const onArtifactOnCli: OnArtifact = (artifact) => { - writeFileSync(artifact.filename, artifact.content); +export const handleOnArtifact = ( + onProgress: (artifact: ArtifactProgress) => void, +) => { + const progress: ArtifactProgress = {received: []}; + + const onArtifact: OnArtifact = (artifact) => { + // TODO: Normalize backslashes on Windows + const absoluteOutputDestination = path.join( + process.cwd(), + artifact.filename, + ); + writeFileSync(artifact.filename, artifact.content); + + progress.received.push({ + absoluteOutputDestination, + filename: artifact.filename, + sizeInBytes: artifact.content.length, + }); + }; + + onProgress(progress); + + return { + onArtifact, + initialProgress: progress, + }; }; diff --git a/packages/cli/src/progress-types.ts b/packages/cli/src/progress-types.ts index 58fe3bb4a5d..497f4d07662 100644 --- a/packages/cli/src/progress-types.ts +++ b/packages/cli/src/progress-types.ts @@ -12,4 +12,7 @@ export const initialAggregateRenderProgress = (): AggregateRenderProgress => ({ bytes: 0, doneIn: null, }, + artifactState: { + received: [], + }, }); diff --git a/packages/cli/src/render-flows/render.ts b/packages/cli/src/render-flows/render.ts index bbb22fbe550..82c82600850 100644 --- a/packages/cli/src/render-flows/render.ts +++ b/packages/cli/src/render-flows/render.ts @@ -42,7 +42,7 @@ import {makeHyperlink} from '../hyperlinks/make-link'; import {getVideoImageFormat} from '../image-formats'; import {Log} from '../log'; import {makeOnDownload} from '../make-on-download'; -import {onArtifactOnCli} from '../on-artifact'; +import {handleOnArtifact} from '../on-artifact'; import {parsedCli, quietFlagProvided} from '../parsed-cli'; import { LABEL_WIDTH, @@ -220,6 +220,13 @@ export const renderVideoFlow = async ({ doneIn: null, }; + let {onArtifact, initialProgress: artifactState} = handleOnArtifact( + (progress) => { + artifactState = progress; + updateRenderProgress({newline: false, printToConsole: true}); + }, + ); + const updateRenderProgress = ({ newline, printToConsole, @@ -233,6 +240,7 @@ export const renderVideoFlow = async ({ downloads, bundling: bundlingProgress, copyingState, + artifactState, }; const {output, message, progress} = makeRenderingAndStitchingProgress({ @@ -430,6 +438,7 @@ export const renderVideoFlow = async ({ codec: shouldOutputImageSequence ? undefined : codec, uiImageFormat, }); + if (shouldOutputImageSequence) { fs.mkdirSync(absoluteOutputFile, { recursive: true, @@ -492,7 +501,7 @@ export const renderVideoFlow = async ({ compositionStart: 0, forSeamlessAacConcatenation, onBrowserDownload, - onArtifact: onArtifactOnCli, + onArtifact, }); Log.info({indent, logLevel}, chalk.blue(`▶ ${absoluteOutputFile}`)); @@ -582,7 +591,7 @@ export const renderVideoFlow = async ({ forSeamlessAacConcatenation, compositionStart: 0, onBrowserDownload, - onArtifact: onArtifactOnCli, + onArtifact, }); if (!updatesDontOverwrite) { updateRenderProgress({newline: true, printToConsole: true}); diff --git a/packages/studio-shared/src/index.ts b/packages/studio-shared/src/index.ts index 11bf724e0d7..7e66eac1070 100644 --- a/packages/studio-shared/src/index.ts +++ b/packages/studio-shared/src/index.ts @@ -49,10 +49,12 @@ export {ProjectInfo} from './project-info'; export type {RenderDefaults} from './render-defaults'; export { AggregateRenderProgress, + ArtifactProgress, BundlingState, CopyingState, DownloadProgress, JobProgressCallback, + ReceivedAsset, RenderJob, RenderJobWithCleanup, RenderingProgressInput, diff --git a/packages/studio-shared/src/render-job.ts b/packages/studio-shared/src/render-job.ts index 6ff3b0c1f8b..86eefd9e7e2 100644 --- a/packages/studio-shared/src/render-job.ts +++ b/packages/studio-shared/src/render-job.ts @@ -56,6 +56,17 @@ export type AggregateRenderProgress = { downloads: DownloadProgress[]; bundling: BundlingState; copyingState: CopyingState; + artifactState: ArtifactProgress; +}; + +export type ReceivedAsset = { + filename: string; + absoluteOutputDestination: string; + sizeInBytes: number; +}; + +export type ArtifactProgress = { + received: ReceivedAsset[]; }; export type JobProgressCallback = ( From dd1710fede5b099d0442d4cb6a6223f04e79bf4f Mon Sep 17 00:00:00 2001 From: Jonny Burger Date: Fri, 14 Jun 2024 17:25:46 +0200 Subject: [PATCH 10/36] add some tests to it --- packages/core/src/Artifact.tsx | 1 + packages/core/src/RenderAssetManager.tsx | 2 + packages/core/src/internals.ts | 2 + .../core/src/validation/validate-artifact.ts | 41 +++++++++ packages/renderer/src/test/artifacts.test.tsx | 88 +++++++++++++++++++ .../src/test/get-assets-for-markup.tsx | 1 + 6 files changed, 135 insertions(+) create mode 100644 packages/core/src/validation/validate-artifact.ts create mode 100644 packages/renderer/src/test/artifacts.test.tsx diff --git a/packages/core/src/Artifact.tsx b/packages/core/src/Artifact.tsx index b2fd295a435..489e47685d4 100644 --- a/packages/core/src/Artifact.tsx +++ b/packages/core/src/Artifact.tsx @@ -11,6 +11,7 @@ export const Artifact: React.FC<{ // TODO: Validate filename and content const {registerRenderAsset, unregisterRenderAsset} = useContext(RenderAssetManager); + const frame = useCurrentFrame(); const [id] = useState(() => { diff --git a/packages/core/src/RenderAssetManager.tsx b/packages/core/src/RenderAssetManager.tsx index 43588b15e6a..87b41d67dad 100644 --- a/packages/core/src/RenderAssetManager.tsx +++ b/packages/core/src/RenderAssetManager.tsx @@ -6,6 +6,7 @@ import { useState, } from 'react'; import type {TRenderAsset} from './CompositionManager.js'; +import {validateRenderAsset} from './validation/validate-artifact.js'; export type RenderAssetManagerContext = { registerRenderAsset: (renderAsset: TRenderAsset) => void; @@ -26,6 +27,7 @@ export const RenderAssetManagerProvider: React.FC<{ const [renderAssets, setRenderAssets] = useState([]); const registerRenderAsset = useCallback((renderAsset: TRenderAsset) => { + validateRenderAsset(renderAsset); setRenderAssets((assets) => { return [...assets, renderAsset]; }); diff --git a/packages/core/src/internals.ts b/packages/core/src/internals.ts index 656c475d3f8..3a9342b779d 100644 --- a/packages/core/src/internals.ts +++ b/packages/core/src/internals.ts @@ -75,6 +75,7 @@ import { import {useLazyComponent} from './use-lazy-component.js'; import {useUnsafeVideoConfig} from './use-unsafe-video-config.js'; import {useVideo} from './use-video.js'; +import {validateRenderAsset} from './validation/validate-artifact.js'; import { invalidCompositionErrorMessage, isCompositionIdValid, @@ -161,6 +162,7 @@ export const Internals = { calculateScale, editorPropsProviderRef, PROPS_UPDATED_EXTERNALLY, + validateRenderAsset, } as const; export type { diff --git a/packages/core/src/validation/validate-artifact.ts b/packages/core/src/validation/validate-artifact.ts new file mode 100644 index 00000000000..162e177b093 --- /dev/null +++ b/packages/core/src/validation/validate-artifact.ts @@ -0,0 +1,41 @@ +import type {TRenderAsset} from '../CompositionManager'; + +export const validateArtifactFilename = (filename: unknown) => { + if (typeof filename !== 'string') { + throw new TypeError( + `The "filename" must be a string, but you passed a value of type ${typeof filename}`, + ); + } + + if (filename.trim() === '') { + throw new Error('The `filename` must not be empty'); + } + + if (!filename.match(/^([0-9a-zA-Z-!_.*'()/:&$@=;+,?]+)/g)) { + throw new Error( + 'The `filename` must match "/^([0-9a-zA-Z-!_.*\'()/:&$@=;+,?]+)/g". Use forward slashes only, even on Windows.', + ); + } +}; + +const validateContent = (content: unknown) => { + if (typeof content !== 'string') { + throw new TypeError( + `The "content" must be a string, but you passed a value of type ${typeof content}`, + ); + } + + if (content.trim() === '') { + throw new Error('The `content` must not be empty'); + } +}; + +export const validateRenderAsset = (artifact: TRenderAsset) => { + // We don't have validation for it yet + if (artifact.type !== 'artifact') { + return; + } + + validateArtifactFilename(artifact.filename); + validateContent(artifact.content); +}; diff --git a/packages/renderer/src/test/artifacts.test.tsx b/packages/renderer/src/test/artifacts.test.tsx new file mode 100644 index 00000000000..879787c735c --- /dev/null +++ b/packages/renderer/src/test/artifacts.test.tsx @@ -0,0 +1,88 @@ +/* eslint-disable no-restricted-imports */ +/** + * @vitest-environment jsdom + */ +import React from 'react'; +import {Artifact, useCurrentFrame} from 'remotion'; +import {expect, test} from 'vitest'; +import {getAssetsForMarkup} from './get-assets-for-markup'; + +const basicConfig = { + width: 1080, + height: 1080, + fps: 30, + durationInFrames: 60, + id: 'hithere', +}; + +test('Should be able to collect artifacts', async () => { + const Markup: React.FC = () => { + const frame = useCurrentFrame(); + + return ( +
+ {frame === 1 ? ( + + ) : null} +
+ ); + }; + + const [first, second] = await getAssetsForMarkup(Markup, basicConfig); + expect(first).toEqual([]); + expect(second.length).toEqual(1); + const sec = second[0]; + if (sec.type !== 'artifact') { + throw new Error('Expected artifact'); + } + + expect(sec.filename).toEqual('hi.txt'); + expect(sec.content).toEqual('hi there'); +}); + +test('Should throw on invalid artifact', async () => { + const Markup: React.FC = () => { + const frame = useCurrentFrame(); + + return ( +
+ {frame === 1 ? ( + + ) : null} +
+ ); + }; + + try { + await getAssetsForMarkup(Markup, basicConfig); + } catch (err) { + expect(err).toMatch(/The "filename" must match/); + } +}); + +test('Should throw on when missing content', async () => { + const Markup: React.FC = () => { + const frame = useCurrentFrame(); + + return ( +
+ {frame === 1 ? ( + + ) : null} +
+ ); + }; + + try { + await getAssetsForMarkup(Markup, basicConfig); + } catch (err) { + expect(err).toMatch(/The "content" must be a string/); + } +}); diff --git a/packages/renderer/src/test/get-assets-for-markup.tsx b/packages/renderer/src/test/get-assets-for-markup.tsx index 93000ae7195..6b935fbecd5 100644 --- a/packages/renderer/src/test/get-assets-for-markup.tsx +++ b/packages/renderer/src/test/get-assets-for-markup.tsx @@ -52,6 +52,7 @@ export const getAssetsForMarkup = async ( const [renderAssets, setAssets] = useState([]); const registerRenderAsset = useCallback((renderAsset: TRenderAsset) => { + Internals.validateRenderAsset(renderAsset); setAssets((assts) => { return [...assts, renderAsset]; }); From 2fc2eb9ab431e6d1905e3000ef19133088faa161 Mon Sep 17 00:00:00 2001 From: Jonny Burger Date: Fri, 14 Jun 2024 17:42:41 +0200 Subject: [PATCH 11/36] cool artifacts --- packages/cli/src/on-artifact.ts | 8 +++---- packages/cli/src/progress-bar.ts | 28 ++++++++++++++++++++++++- packages/cli/src/render-flows/render.ts | 13 ++++++------ 3 files changed, 38 insertions(+), 11 deletions(-) diff --git a/packages/cli/src/on-artifact.ts b/packages/cli/src/on-artifact.ts index 715d025cd55..c93f287faa2 100644 --- a/packages/cli/src/on-artifact.ts +++ b/packages/cli/src/on-artifact.ts @@ -4,9 +4,10 @@ import {writeFileSync} from 'fs'; import path from 'path'; export const handleOnArtifact = ( + artifactState: ArtifactProgress, onProgress: (artifact: ArtifactProgress) => void, ) => { - const progress: ArtifactProgress = {received: []}; + const initialProgress = {...artifactState}; const onArtifact: OnArtifact = (artifact) => { // TODO: Normalize backslashes on Windows @@ -16,17 +17,16 @@ export const handleOnArtifact = ( ); writeFileSync(artifact.filename, artifact.content); - progress.received.push({ + initialProgress.received.push({ absoluteOutputDestination, filename: artifact.filename, sizeInBytes: artifact.content.length, }); }; - onProgress(progress); + onProgress(initialProgress); return { onArtifact, - initialProgress: progress, }; }; diff --git a/packages/cli/src/progress-bar.ts b/packages/cli/src/progress-bar.ts index cc57ecd88ea..20cc2ca1b18 100644 --- a/packages/cli/src/progress-bar.ts +++ b/packages/cli/src/progress-bar.ts @@ -8,6 +8,7 @@ import type { StitchingProgressInput, } from '@remotion/studio-server'; import {StudioServerInternals} from '@remotion/studio-server'; +import {formatBytes, type ArtifactProgress} from '@remotion/studio-shared'; import {chalk} from './chalk'; import { getFileSizeDownloadBar, @@ -192,6 +193,30 @@ const makeRenderingProgress = ({ .join(' '); }; +const makeArtifactProgress = (artifactState: ArtifactProgress) => { + const {received} = artifactState; + if (received.length === 0) { + return null; + } + + return received + .map((artifact) => { + return [ + chalk.blue('+'.padEnd(LABEL_WIDTH)), + chalk.blue( + makeHyperlink({ + url: 'file://' + artifact.absoluteOutputDestination, + fallback: artifact.filename, + text: artifact.filename, + }), + ), + chalk.gray(`${formatBytes(artifact.sizeInBytes)}`), + ].join(' '); + }) + .filter(truthy) + .join('\n'); +}; + export const getRightLabelWidth = (totalFrames: number) => { return `${totalFrames}/${totalFrames}`.length; }; @@ -237,7 +262,7 @@ export const makeRenderingAndStitchingProgress = ({ progress: number; message: string; } => { - const {rendering, stitching, downloads, bundling} = prog; + const {rendering, stitching, downloads, bundling, artifactState} = prog; const output = [ rendering ? makeRenderingProgress(rendering) : null, makeMultiDownloadProgress(downloads, rendering?.totalFrames ?? 0), @@ -247,6 +272,7 @@ export const makeRenderingAndStitchingProgress = ({ stitchingProgress: stitching, isUsingParallelEncoding, }), + makeArtifactProgress(artifactState), ] .filter(truthy) .join('\n'); diff --git a/packages/cli/src/render-flows/render.ts b/packages/cli/src/render-flows/render.ts index 82c82600850..89983aeb578 100644 --- a/packages/cli/src/render-flows/render.ts +++ b/packages/cli/src/render-flows/render.ts @@ -28,6 +28,7 @@ import type { RenderingProgressInput, StitchingProgressInput, } from '@remotion/studio-server'; +import type {ArtifactProgress} from '@remotion/studio-shared'; import fs, {existsSync} from 'node:fs'; import os from 'node:os'; import path from 'node:path'; @@ -220,12 +221,7 @@ export const renderVideoFlow = async ({ doneIn: null, }; - let {onArtifact, initialProgress: artifactState} = handleOnArtifact( - (progress) => { - artifactState = progress; - updateRenderProgress({newline: false, printToConsole: true}); - }, - ); + let artifactState: ArtifactProgress = {received: []}; const updateRenderProgress = ({ newline, @@ -254,6 +250,11 @@ export const renderVideoFlow = async ({ } }; + const {onArtifact} = handleOnArtifact(artifactState, (progress) => { + artifactState = progress; + updateRenderProgress({newline: false, printToConsole: true}); + }); + const {urlOrBundle, cleanup: cleanupBundle} = await bundleOnCliOrTakeServeUrl( { fullPath: fullEntryPoint, From d7a75ec463dd3dfb17fe7e5f5e071e5c736b0949 Mon Sep 17 00:00:00 2001 From: Jonny Burger Date: Mon, 17 Jun 2024 17:08:11 +0200 Subject: [PATCH 12/36] artifact name --- packages/cli/src/on-artifact.ts | 3 +- packages/lambda/src/functions/helpers/io.ts | 2 +- .../src/functions/helpers/stream-renderer.ts | 27 ++++++++- packages/lambda/src/functions/launch.ts | 56 ++++++++++++++++++- packages/lambda/src/functions/renderer.ts | 27 ++++++++- .../src/functions/streaming/streaming.ts | 10 ++++ packages/lambda/src/shared/constants.ts | 2 + packages/renderer/src/index.ts | 7 ++- packages/renderer/src/render-frames.ts | 4 +- 9 files changed, 128 insertions(+), 10 deletions(-) diff --git a/packages/cli/src/on-artifact.ts b/packages/cli/src/on-artifact.ts index c93f287faa2..7fa19e4c811 100644 --- a/packages/cli/src/on-artifact.ts +++ b/packages/cli/src/on-artifact.ts @@ -10,10 +10,9 @@ export const handleOnArtifact = ( const initialProgress = {...artifactState}; const onArtifact: OnArtifact = (artifact) => { - // TODO: Normalize backslashes on Windows const absoluteOutputDestination = path.join( process.cwd(), - artifact.filename, + artifact.filename.replace('/', path.sep), ); writeFileSync(artifact.filename, artifact.content); diff --git a/packages/lambda/src/functions/helpers/io.ts b/packages/lambda/src/functions/helpers/io.ts index 09efc03e7a5..adc8d2f1909 100644 --- a/packages/lambda/src/functions/helpers/io.ts +++ b/packages/lambda/src/functions/helpers/io.ts @@ -99,7 +99,7 @@ export const lambdaDeleteFile = async ({ type LambdaWriteFileInput = { bucketName: string; key: string; - body: ReadStream | string; + body: ReadStream | string | Uint8Array; region: AwsRegion; privacy: Privacy; expectedBucketOwner: string | null; diff --git a/packages/lambda/src/functions/helpers/stream-renderer.ts b/packages/lambda/src/functions/helpers/stream-renderer.ts index 94fe98cada5..e7a041aaaa3 100644 --- a/packages/lambda/src/functions/helpers/stream-renderer.ts +++ b/packages/lambda/src/functions/helpers/stream-renderer.ts @@ -1,4 +1,4 @@ -import type {LogLevel} from '@remotion/renderer'; +import type {EmittedAsset, LogLevel} from '@remotion/renderer'; import {RenderInternals} from '@remotion/renderer'; import {writeFileSync} from 'fs'; import {join} from 'path'; @@ -26,6 +26,7 @@ const streamRenderer = ({ overallProgress, files, logLevel, + onArtifact, }: { payload: LambdaPayload; functionName: string; @@ -33,6 +34,7 @@ const streamRenderer = ({ overallProgress: OverallProgressHelper; files: string[]; logLevel: LogLevel; + onArtifact: (asset: EmittedAsset) => {alreadyExisted: boolean}; }) => { if (payload.type !== LambdaRoutines.renderer) { throw new Error('Expected renderer type'); @@ -97,6 +99,25 @@ const streamRenderer = ({ return; } + if (message.type === 'artifact-emitted') { + RenderInternals.Log.info( + {indent: false, logLevel}, + `Received artifact on frame ${message.payload.artifact.frame}:`, + message.payload.artifact.filename, + message.payload.artifact.content.length + 'bytes.', + ); + const {alreadyExisted} = onArtifact(message.payload.artifact); + if (alreadyExisted) { + return resolve({ + type: 'error', + error: `Chunk ${payload.chunk} emitted an asset filename ${message.payload.artifact.filename} at frame ${message.payload.artifact.frame} but there is already another artifact with the same name.`, + shouldRetry: false, + }); + } + + return; + } + if (message.type === 'error-occurred') { overallProgress.addErrorWithoutUpload(message.payload.errorInfo); overallProgress.setFrames({ @@ -162,6 +183,7 @@ export const streamRendererFunctionWithRetry = async ({ outdir, overallProgress, logLevel, + onArtifact, }: { payload: LambdaPayload; functionName: string; @@ -169,6 +191,7 @@ export const streamRendererFunctionWithRetry = async ({ overallProgress: OverallProgressHelper; files: string[]; logLevel: LogLevel; + onArtifact: (asset: EmittedAsset) => {alreadyExisted: boolean}; }): Promise => { if (payload.type !== LambdaRoutines.renderer) { throw new Error('Expected renderer type'); @@ -181,6 +204,7 @@ export const streamRendererFunctionWithRetry = async ({ overallProgress, payload, logLevel, + onArtifact, }); if (result.type === 'error') { @@ -205,6 +229,7 @@ export const streamRendererFunctionWithRetry = async ({ retriesLeft: payload.retriesLeft - 1, }, logLevel, + onArtifact, }); } }; diff --git a/packages/lambda/src/functions/launch.ts b/packages/lambda/src/functions/launch.ts index 78c8d2e58ae..078d0def3db 100644 --- a/packages/lambda/src/functions/launch.ts +++ b/packages/lambda/src/functions/launch.ts @@ -1,10 +1,10 @@ /* eslint-disable @typescript-eslint/no-use-before-define */ -import type {LogOptions} from '@remotion/renderer'; +import type {EmittedAsset, LogOptions} from '@remotion/renderer'; import {RenderInternals} from '@remotion/renderer'; import {existsSync, mkdirSync, rmSync} from 'fs'; import {join} from 'path'; import {VERSION} from 'remotion/version'; -import type {EventEmitter} from 'stream'; +import {type EventEmitter} from 'stream'; import { compressInputProps, decompressInputProps, @@ -20,6 +20,7 @@ import { CONCAT_FOLDER_TOKEN, LambdaRoutines, MAX_FUNCTIONS_PER_RENDER, + artifactName, } from '../shared/constants'; import {DOCS_URL} from '../shared/docs-url'; import {invokeWebhook} from '../shared/invoke-webhook'; @@ -41,6 +42,7 @@ import { getBrowserInstance, } from './helpers/get-browser-instance'; import {getCurrentRegionInFunction} from './helpers/get-current-region'; +import {lambdaWriteFile} from './helpers/io'; import {mergeChunksAndFinishRender} from './helpers/merge-chunks'; import type {OverallProgressHelper} from './helpers/overall-render-progress'; import {makeOverallRenderProgress} from './helpers/overall-render-progress'; @@ -54,6 +56,11 @@ type Options = { getRemainingTimeInMillis: () => number; }; +type ReceivedAsset = { + filename: string; + sizeInBytes: number; +}; + const innerLaunchHandler = async ({ functionName, params, @@ -354,6 +361,50 @@ const innerLaunchHandler = async ({ const files: string[] = []; + const artifacts: ReceivedAsset[] = []; + + const onArtifact = (artifact: EmittedAsset): {alreadyExisted: boolean} => { + if (artifacts.find((a) => a.filename === artifact.filename)) { + return {alreadyExisted: true}; + } + + artifacts.push({ + filename: artifact.filename, + sizeInBytes: artifact.content.length, + }); + + const start = Date.now(); + RenderInternals.Log.info( + {indent: false, logLevel: params.logLevel}, + 'Writing artifact ' + artifact.filename + ' to S3', + ); + lambdaWriteFile({ + bucketName: renderBucketName, + key: artifactName(renderMetadata.renderId, artifact.filename), + body: artifact.content, + region: getCurrentRegionInFunction(), + privacy: params.privacy, + expectedBucketOwner: options.expectedBucketOwner, + downloadBehavior: params.downloadBehavior, + customCredentials, + }) + .then(() => { + RenderInternals.Log.info( + {indent: false, logLevel: params.logLevel}, + `Wrote artifact to S3 in ${Date.now() - start}ms`, + ); + }) + .catch((err) => { + // TODO: Handle error + RenderInternals.Log.error( + {indent: false, logLevel: params.logLevel}, + 'Failed to write artifact to S3', + err, + ); + }); + return {alreadyExisted: false}; + }; + await Promise.all( lambdaPayloads.map(async (payload) => { await streamRendererFunctionWithRetry({ @@ -363,6 +414,7 @@ const innerLaunchHandler = async ({ overallProgress, payload, logLevel: params.logLevel, + onArtifact, }); }), ); diff --git a/packages/lambda/src/functions/renderer.ts b/packages/lambda/src/functions/renderer.ts index 8bae9be4b9b..439c63136d5 100644 --- a/packages/lambda/src/functions/renderer.ts +++ b/packages/lambda/src/functions/renderer.ts @@ -173,7 +173,32 @@ const renderHandler = async ({ ); const onArtifact: OnArtifact = (artifact) => { - console.log(artifact); + RenderInternals.Log.info( + {indent: false, logLevel: params.logLevel}, + `Received artifact on frame ${artifact.frame}:`, + artifact.filename, + artifact.content.length + 'bytes. Streaming to main function', + ); + const startTimestamp = Date.now(); + onStream({ + type: 'artifact-emitted', + payload: { + artifact, + }, + }) + .then(() => { + RenderInternals.Log.info( + {indent: false, logLevel: params.logLevel}, + `Streaming artifact ${artifact.filename} to main function took ${Date.now() - startTimestamp}ms`, + ); + }) + .catch((e) => { + RenderInternals.Log.error( + {indent: false, logLevel: params.logLevel}, + `Error streaming artifact ${artifact.filename} to main function`, + e, + ); + }); }; await new Promise((resolve, reject) => { diff --git a/packages/lambda/src/functions/streaming/streaming.ts b/packages/lambda/src/functions/streaming/streaming.ts index f7d8eea4e13..71966376ef3 100644 --- a/packages/lambda/src/functions/streaming/streaming.ts +++ b/packages/lambda/src/functions/streaming/streaming.ts @@ -1,3 +1,4 @@ +import type {EmittedAsset} from '@remotion/renderer'; import {makeStreamPayloadMessage} from '@remotion/streaming'; import type {LambdaErrorInfo} from '../helpers/write-lambda-error'; import type {RenderStillLambdaResponsePayload} from '../still'; @@ -10,6 +11,7 @@ const audioChunkRendered = 'audio-chunk-rendered' as const; const chunkComplete = 'chunk-complete' as const; const stillRendered = 'still-rendered' as const; const lambdaInvoked = 'lambda-invoked' as const; +const artifactEmitted = 'artifact-emitted' as const; const messageTypes = { '1': {type: framesRendered}, @@ -20,6 +22,7 @@ const messageTypes = { '6': {type: stillRendered}, '7': {type: chunkComplete}, '8': {type: lambdaInvoked}, + '9': {type: artifactEmitted}, } as const; export type MessageTypeId = keyof typeof messageTypes; @@ -34,6 +37,7 @@ export const formatMap: {[key in MessageType]: 'json' | 'binary'} = { [stillRendered]: 'json', [chunkComplete]: 'json', [lambdaInvoked]: 'json', + [artifactEmitted]: 'json', }; export type StreamingPayload = @@ -82,6 +86,12 @@ export type StreamingPayload = payload: { attempt: number; }; + } + | { + type: typeof artifactEmitted; + payload: { + artifact: EmittedAsset; + }; }; export const messageTypeIdToMessageType = ( diff --git a/packages/lambda/src/shared/constants.ts b/packages/lambda/src/shared/constants.ts index 8abd7bd221d..3591297884c 100644 --- a/packages/lambda/src/shared/constants.ts +++ b/packages/lambda/src/shared/constants.ts @@ -85,6 +85,8 @@ export const outName = (renderId: string, extension: string) => `${rendersPrefix(renderId)}/out.${extension}`; export const outStillName = (renderId: string, imageFormat: StillImageFormat) => `${rendersPrefix(renderId)}/out.${imageFormat}`; +export const artifactName = (renderId: string, name: string) => + `${rendersPrefix(renderId)}/artifacts/${name}`; export const customOutName = ( renderId: string, bucketName: string, diff --git a/packages/renderer/src/index.ts b/packages/renderer/src/index.ts index ac072fa0339..9b3973414ee 100644 --- a/packages/renderer/src/index.ts +++ b/packages/renderer/src/index.ts @@ -108,7 +108,12 @@ export {X264Preset} from './options/x264-preset'; export {PixelFormat} from './pixel-format'; export {RemotionServer} from './prepare-server'; export {ProResProfile} from './prores-profile'; -export {OnArtifact, RenderFramesOptions, renderFrames} from './render-frames'; +export { + EmittedArtifact as EmittedAsset, + OnArtifact, + RenderFramesOptions, + renderFrames, +} from './render-frames'; export { InternalRenderMediaOptions, RenderMediaOnProgress, diff --git a/packages/renderer/src/render-frames.ts b/packages/renderer/src/render-frames.ts index 6e753eddffc..5b8dc820233 100644 --- a/packages/renderer/src/render-frames.ts +++ b/packages/renderer/src/render-frames.ts @@ -67,7 +67,7 @@ import {wrapWithErrorHandling} from './wrap-with-error-handling'; const MAX_RETRIES_PER_FRAME = 1; -export type OnArtifact = (asset: EmittedAsset) => void; +export type OnArtifact = (asset: EmittedArtifact) => void; type InternalRenderFramesOptions = { onStart: null | ((data: OnStartData) => void); @@ -106,7 +106,7 @@ type InternalRenderFramesOptions = { onArtifact: OnArtifact | null; } & ToOptions; -type EmittedAsset = { +export type EmittedArtifact = { filename: string; content: string | Uint8Array; frame: number; From e6f27c263d35b7a161e4ed75cd3903e7a2df3c12 Mon Sep 17 00:00:00 2001 From: Jonny Burger Date: Mon, 17 Jun 2024 17:29:20 +0200 Subject: [PATCH 13/36] let's go --- packages/example/runlambda.sh | 2 +- .../src/cli/commands/render/progress.ts | 27 +++++++++++++++++++ .../helpers/create-post-render-data.ts | 1 + .../src/functions/helpers/get-progress.ts | 2 ++ .../helpers/overall-render-progress.ts | 17 ++++++++++++ packages/lambda/src/functions/launch.ts | 22 +++++++-------- packages/lambda/src/shared/constants.ts | 3 +++ 7 files changed, 62 insertions(+), 12 deletions(-) diff --git a/packages/example/runlambda.sh b/packages/example/runlambda.sh index 43c2eb94300..935c37f18bd 100644 --- a/packages/example/runlambda.sh +++ b/packages/example/runlambda.sh @@ -7,4 +7,4 @@ cd example bunx remotion lambda functions rmall -f bunx remotion lambda functions deploy --memory=3000 --disk=10000 bunx remotion lambda sites create --site-name=testbed-v6 --log=verbose --enable-folder-expiry -bunx remotion lambda render testbed-v6 OffthreadRemoteVideo --log=verbose --delete-after="1-day" +bunx remotion lambda render testbed-v6 subtitle --log=verbose --delete-after="1-day" diff --git a/packages/lambda/src/cli/commands/render/progress.ts b/packages/lambda/src/cli/commands/render/progress.ts index 2ee2a50c77f..316a6a14fe4 100644 --- a/packages/lambda/src/cli/commands/render/progress.ts +++ b/packages/lambda/src/cli/commands/render/progress.ts @@ -200,6 +200,32 @@ const makeTopRow = (overall: RenderProgress) => { return CliInternals.chalk.gray(str); }; +const makeArtifactProgress = (progress: RenderProgress) => { + const {artifactProgress} = progress; + if (artifactProgress.length === 0) { + return null; + } + + return artifactProgress + .map((artifact) => { + return [ + CliInternals.chalk.blue('+'.padEnd(CliInternals.LABEL_WIDTH)), + CliInternals.chalk.blue( + CliInternals.makeHyperlink({ + url: artifact.s3Url, + fallback: artifact.filename, + text: artifact.filename, + }), + ), + CliInternals.chalk.gray( + `${CliInternals.formatBytes(artifact.sizeInBytes)}`, + ), + ].join(' '); + }) + .filter(truthy) + .join('\n'); +}; + export const makeProgressString = ({ downloadInfo, overall, @@ -213,6 +239,7 @@ export const makeProgressString = ({ ...makeInvokeProgress(overall), ...makeRenderProgress(overall), makeCombinationProgress(overall), + makeArtifactProgress(overall), downloadInfo ? makeDownloadProgress(downloadInfo) : null, ] .filter(NoReactInternals.truthy) diff --git a/packages/lambda/src/functions/helpers/create-post-render-data.ts b/packages/lambda/src/functions/helpers/create-post-render-data.ts index ba9ffd508d0..f5a550063ed 100644 --- a/packages/lambda/src/functions/helpers/create-post-render-data.ts +++ b/packages/lambda/src/functions/helpers/create-post-render-data.ts @@ -103,5 +103,6 @@ export const createPostRenderData = ({ deleteAfter: renderMetadata.deleteAfter, estimatedBillingDurationInMilliseconds, timeToCombine: timeToCombine ?? null, + artifactProgress: overallProgress.receivedAssets, }; }; diff --git a/packages/lambda/src/functions/helpers/get-progress.ts b/packages/lambda/src/functions/helpers/get-progress.ts index 0f3928027ce..0a77693adc6 100644 --- a/packages/lambda/src/functions/helpers/get-progress.ts +++ b/packages/lambda/src/functions/helpers/get-progress.ts @@ -110,6 +110,7 @@ export const getProgress = async ({ compositionValidated: overallProgress.compositionValidated, functionLaunched: overallProgress.functionLaunched, serveUrlOpened: overallProgress.serveUrlOpened, + artifactProgress: overallProgress.receivedAssets, }; } @@ -257,5 +258,6 @@ export const getProgress = async ({ compositionValidated: overallProgress.compositionValidated, functionLaunched: overallProgress.functionLaunched, serveUrlOpened: overallProgress.serveUrlOpened, + artifactProgress: overallProgress.receivedAssets, }; }; diff --git a/packages/lambda/src/functions/helpers/overall-render-progress.ts b/packages/lambda/src/functions/helpers/overall-render-progress.ts index ecc6557e89c..68a793d98c2 100644 --- a/packages/lambda/src/functions/helpers/overall-render-progress.ts +++ b/packages/lambda/src/functions/helpers/overall-render-progress.ts @@ -26,6 +26,13 @@ export type OverallRenderProgress = { functionLaunched: number; serveUrlOpened: number | null; compositionValidated: number | null; + receivedAssets: ReceivedAsset[]; +}; + +export type ReceivedAsset = { + filename: string; + sizeInBytes: number; + s3Url: string; }; export type OverallProgressHelper = { @@ -55,6 +62,8 @@ export type OverallProgressHelper = { get: () => OverallRenderProgress; setServeUrlOpened: (timestamp: number) => void; setCompositionValidated: (timestamp: number) => void; + addReceivedAsset: (asset: ReceivedAsset) => void; + getReceivedAssets: () => ReceivedAsset[]; }; export const makeInitialOverallRenderProgress = ( @@ -78,6 +87,7 @@ export const makeInitialOverallRenderProgress = ( functionLaunched: Date.now(), serveUrlOpened: null, compositionValidated: null, + receivedAssets: [], }; }; @@ -274,6 +284,13 @@ export const makeOverallRenderProgress = ({ renderProgress.retries.push(retry); upload(); }, + addReceivedAsset(asset) { + renderProgress.receivedAssets.push(asset); + upload(); + }, + getReceivedAssets() { + return renderProgress.receivedAssets; + }, get: () => renderProgress, }; }; diff --git a/packages/lambda/src/functions/launch.ts b/packages/lambda/src/functions/launch.ts index 078d0def3db..f42a3873c29 100644 --- a/packages/lambda/src/functions/launch.ts +++ b/packages/lambda/src/functions/launch.ts @@ -56,11 +56,6 @@ type Options = { getRemainingTimeInMillis: () => number; }; -type ReceivedAsset = { - filename: string; - sizeInBytes: number; -}; - const innerLaunchHandler = async ({ functionName, params, @@ -361,16 +356,21 @@ const innerLaunchHandler = async ({ const files: string[] = []; - const artifacts: ReceivedAsset[] = []; - const onArtifact = (artifact: EmittedAsset): {alreadyExisted: boolean} => { - if (artifacts.find((a) => a.filename === artifact.filename)) { + if ( + overallProgress + .getReceivedAssets() + .find((a) => a.filename === artifact.filename) + ) { return {alreadyExisted: true}; } - artifacts.push({ + const region = getCurrentRegionInFunction(); + const s3Key = artifactName(renderMetadata.renderId, artifact.filename); + overallProgress.addReceivedAsset({ filename: artifact.filename, sizeInBytes: artifact.content.length, + s3Url: `https://s3.${region}.amazonaws.com/${renderBucketName}/${s3Key}`, }); const start = Date.now(); @@ -380,9 +380,9 @@ const innerLaunchHandler = async ({ ); lambdaWriteFile({ bucketName: renderBucketName, - key: artifactName(renderMetadata.renderId, artifact.filename), + key: s3Key, body: artifact.content, - region: getCurrentRegionInFunction(), + region, privacy: params.privacy, expectedBucketOwner: options.expectedBucketOwner, downloadBehavior: params.downloadBehavior, diff --git a/packages/lambda/src/shared/constants.ts b/packages/lambda/src/shared/constants.ts index 3591297884c..6418b7b2979 100644 --- a/packages/lambda/src/shared/constants.ts +++ b/packages/lambda/src/shared/constants.ts @@ -14,6 +14,7 @@ import type { import type {BrowserSafeApis} from '@remotion/renderer/client'; import type {ChunkRetry} from '../functions/helpers/get-retry-stats'; import type {DeleteAfter} from '../functions/helpers/lifecycle'; +import type {ReceivedAsset} from '../functions/helpers/overall-render-progress'; import type {EnhancedErrorInfo} from '../functions/helpers/write-lambda-error'; import type {AwsRegion} from '../pricing/aws-regions'; import type { @@ -414,6 +415,7 @@ export type PostRenderData = { estimatedBillingDurationInMilliseconds: number; deleteAfter: DeleteAfter | null; timeToCombine: number | null; + artifactProgress: ReceivedAsset[]; }; export type CostsInfo = { @@ -469,6 +471,7 @@ export type RenderProgress = { functionLaunched: number; serveUrlOpened: number | null; compositionValidated: number | null; + artifactProgress: ReceivedAsset[]; }; export type Privacy = 'public' | 'private' | 'no-acl'; From fb2e525d44a0d42183c0f2727483f1aa7f6f87e2 Mon Sep 17 00:00:00 2001 From: Jonny Burger Date: Tue, 18 Jun 2024 17:41:52 +0200 Subject: [PATCH 14/36] artifact --- packages/core/src/Artifact.tsx | 10 +- packages/docs/docs/artifact.mdx | 62 + .../docs/miscellaneous/linux-dependencies.mdx | 1 + packages/docs/sidebars.js | 1 + packages/docs/src/data/articles.ts | 5046 +++++++++-------- .../generated/articles-docs-artifact.png | Bin 0 -> 32218 bytes ...-docs-miscellaneous-linux-dependencies.png | Bin 0 -> 37868 bytes 7 files changed, 2602 insertions(+), 2518 deletions(-) create mode 100644 packages/docs/docs/artifact.mdx create mode 100644 packages/docs/static/generated/articles-docs-artifact.png create mode 100644 packages/docs/static/generated/articles-docs-miscellaneous-linux-dependencies.png diff --git a/packages/core/src/Artifact.tsx b/packages/core/src/Artifact.tsx index 489e47685d4..cca65033b33 100644 --- a/packages/core/src/Artifact.tsx +++ b/packages/core/src/Artifact.tsx @@ -1,17 +1,18 @@ import type React from 'react'; import {useContext, useEffect, useState} from 'react'; import {RenderAssetManager} from './RenderAssetManager'; +import {getRemotionEnvironment} from './get-remotion-environment'; import {useCurrentFrame} from './use-current-frame'; -// TODO: Not register in development? export const Artifact: React.FC<{ readonly filename: string; readonly content: string; }> = ({filename, content}) => { - // TODO: Validate filename and content const {registerRenderAsset, unregisterRenderAsset} = useContext(RenderAssetManager); + const [env] = useState(() => getRemotionEnvironment()); + const frame = useCurrentFrame(); const [id] = useState(() => { @@ -19,6 +20,10 @@ export const Artifact: React.FC<{ }); useEffect(() => { + if (!env.isRendering) { + return; + } + registerRenderAsset({ type: 'artifact', id, @@ -32,6 +37,7 @@ export const Artifact: React.FC<{ }; }, [ content, + env.isRendering, filename, frame, id, diff --git a/packages/docs/docs/artifact.mdx b/packages/docs/docs/artifact.mdx new file mode 100644 index 00000000000..146c33e5068 --- /dev/null +++ b/packages/docs/docs/artifact.mdx @@ -0,0 +1,62 @@ +--- +image: /generated/articles-docs-artifact.png +title: "" +crumb: API +--- + +# `` + +By rendering an `` tag in your Remotion markup, an extra file will get emitted during rendering. + +```tsx twoslash title="MyComp.tsx" +import { Artifact, useCurrentFrame } from "remotion"; + +export const MyComp: React.FC = () => { + const frame = useCurrentFrame(); + + return frame === 0 ? ( + + ) : null; +}; +``` + +If rendered on the CLI or via the Studio, this will emit an additional file: + +``` +$ npx remotion render MyComp ++ out/MyComp.mp4 ++ my-file.txt (12B) +``` + +It is allowed for a composition to emit multiple files. +However, the file names must be unique. + +The component will get evaluated on every frame, which means if you want to emit just one file, only render it on one frame. + +```tsx twoslash title="❌ Will generate an asset on every frame and throw an error because the file name is not unique" +import { Artifact, useCurrentFrame } from "remotion"; + +export const MyComp: React.FC = () => { + const frame = useCurrentFrame(); + + return frame === 0 ? ( + + ) : null; +}; +``` + +## API + +### `filename` + +A string that is the name of the file that will be emitted. +Use forward slashes only, even on Windows. +Must match the regex `/^([0-9a-zA-Z-!_.*'()/:&$@=;+,?]+)/g`. + +### `content` + +A string or `UInt8Array` that is the content of the file that will be emitted. + +## See also + +- [Source code for this component](https://github.com/remotion-dev/remotion/blob/main/packages/core/src/components/Artifact.tsx) diff --git a/packages/docs/docs/miscellaneous/linux-dependencies.mdx b/packages/docs/docs/miscellaneous/linux-dependencies.mdx index 16721224e49..91dd7421e7e 100644 --- a/packages/docs/docs/miscellaneous/linux-dependencies.mdx +++ b/packages/docs/docs/miscellaneous/linux-dependencies.mdx @@ -1,4 +1,5 @@ --- +image: /generated/articles-docs-miscellaneous-linux-dependencies.png sidebar_label: Linux Dependencies title: Linux Dependencies crumb: "FAQ" diff --git a/packages/docs/sidebars.js b/packages/docs/sidebars.js index bd1c4e1aca2..3f56d1f2350 100644 --- a/packages/docs/sidebars.js +++ b/packages/docs/sidebars.js @@ -84,6 +84,7 @@ module.exports = { items: [ "absolute-fill", "audio", + "artifact", "calculate-metadata", "cancel-render", "composition", diff --git a/packages/docs/src/data/articles.ts b/packages/docs/src/data/articles.ts index ceb3701af32..42e185e56a3 100644 --- a/packages/docs/src/data/articles.ts +++ b/packages/docs/src/data/articles.ts @@ -1,73 +1,80 @@ export const articles = [ { - id: "2-0-migration", - title: "v2.0 Migration", - relativePath: "docs/2-0-migration.mdx", - compId: "articles-docs-2-0-migration", - crumb: "Version Upgrade", + id: "figma", + title: "Import from Figma", + relativePath: "docs/figma.mdx", + compId: "articles-docs-figma", + crumb: "The best of both", }, { - id: "3-0-migration", - title: "v3.0 Migration", - relativePath: "docs/3-0-migration.mdx", - compId: "articles-docs-3-0-migration", - crumb: "Version Upgrade", + id: "env-variables", + title: "Environment variables", + relativePath: "docs/env-variables.mdx", + compId: "articles-docs-env-variables", + crumb: "How To", }, { - id: "4-0-alpha", - title: "v4.0 Alpha", - relativePath: "docs/4-0-alpha.mdx", - compId: "articles-docs-4-0-alpha", - crumb: "Version Upgrade", + id: "skia", + title: "@remotion/skia", + relativePath: "docs/skia/skia.mdx", + compId: "articles-docs-skia-skia", + crumb: null, }, { - id: "4-0-migration", - title: "v4.0 Migration", - relativePath: "docs/4-0-migration.mdx", - compId: "articles-docs-4-0-migration", - crumb: "Version Upgrade", + id: "skia/enable-skia", + title: "enableSkia()", + relativePath: "docs/skia/enable-skia.mdx", + compId: "articles-docs-skia-enable-skia", + crumb: "@remotion/skia", }, { - id: "5-0-migration", - title: "v5.0 Migration", - relativePath: "docs/5-0-migration.mdx", - compId: "articles-docs-5-0-migration", - crumb: "Version Upgrade", + id: "skia/skia-canvas", + title: "", + relativePath: "docs/skia/skia-canvas.mdx", + compId: "articles-docs-skia-skia-canvas", + crumb: "@remotion/skia", }, { - id: "absolute-fill", - title: "", - relativePath: "docs/absolute-fill.mdx", - compId: "articles-docs-absolute-fill", + id: "spring", + title: "spring()", + relativePath: "docs/spring.mdx", + compId: "articles-docs-spring", crumb: "API", }, { - id: "acknowledgements", - title: "Acknowledgements", - relativePath: "docs/acknowledgements.mdx", - compId: "articles-docs-acknowledgements", - crumb: "Credits", + id: "audio-visualization", + title: "Audio visualization", + relativePath: "docs/audio-visualization.mdx", + compId: "articles-docs-audio-visualization", + crumb: "Techniques", }, { - id: "after-effects", - title: "Import from After Effects", - relativePath: "docs/after-effects.mdx", - compId: "articles-docs-after-effects", - crumb: "Integrations", + id: "react-native", + title: "React Native", + relativePath: "docs/react-native.mdx", + compId: "articles-docs-react-native", + crumb: null, }, { - id: "animating-properties", - title: "Animating properties", - relativePath: "docs/animating-properties.mdx", - compId: "articles-docs-animating-properties", - crumb: "The basics", + id: "use-current-scale", + title: "useCurrentScale()", + relativePath: "docs/use-current-scale.mdx", + compId: "articles-docs-use-current-scale", + crumb: "API", }, { - id: "animation-math", - title: "Animation math", - relativePath: "docs/animation-math.mdx", - compId: "articles-docs-animation-math", - crumb: "Techniques", + id: "config", + title: "Configuration file", + relativePath: "docs/config.mdx", + compId: "articles-docs-config", + crumb: "remotion.config.ts", + }, + { + id: "make-transform", + title: "makeTransform()", + relativePath: "docs/animation-utils/make-transform.mdx", + compId: "articles-docs-animation-utils-make-transform", + crumb: "@remotion/animation-utils", }, { id: "animation-utils/index", @@ -84,46 +91,25 @@ export const articles = [ crumb: "@remotion/animation-utils", }, { - id: "make-transform", - title: "makeTransform()", - relativePath: "docs/animation-utils/make-transform.mdx", - compId: "articles-docs-animation-utils-make-transform", - crumb: "@remotion/animation-utils", - }, - { - id: "api", - title: "API overview", - relativePath: "docs/api.mdx", - compId: "articles-docs-api", - crumb: null, - }, - { - id: "get-help", - title: "Get help", - relativePath: "docs/ask-for-help.mdx", - compId: "articles-docs-ask-for-help", - crumb: null, - }, - { - id: "audio-visualization", - title: "Audio visualization", - relativePath: "docs/audio-visualization.mdx", - compId: "articles-docs-audio-visualization", - crumb: "Techniques", + id: "4-0-alpha", + title: "v4.0 Alpha", + relativePath: "docs/4-0-alpha.mdx", + compId: "articles-docs-4-0-alpha", + crumb: "Version Upgrade", }, { - id: "audio", - title: "