diff --git a/packages/app/src/app/api/resolve/providers/comfyui/index.ts b/packages/app/src/app/api/resolve/providers/comfyui/index.ts index b3982eeb..a5a4f3fa 100644 --- a/packages/app/src/app/api/resolve/providers/comfyui/index.ts +++ b/packages/app/src/app/api/resolve/providers/comfyui/index.ts @@ -2,6 +2,7 @@ import { ResolveRequest } from '@aitube/clapper-services' import { ClapAssetSource, ClapSegmentCategory, + ClapWorkflowCategory, generateSeed, } from '@aitube/clap' import { TimelineSegment } from '@aitube/timeline' @@ -44,27 +45,37 @@ export async function resolveSegment( : undefined ).init() - if (request.segment.category === ClapSegmentCategory.STORYBOARD) { - const imageGenerationWorkflow = request.settings.imageGenerationWorkflow - - if (!imageGenerationWorkflow.inputValues[ClapperComfyUiInputIds.PROMPT]) { + if ( + [ClapSegmentCategory.STORYBOARD, ClapSegmentCategory.VIDEO].includes( + request.segment.category + ) + ) { + const clapWorkflow = { + [ClapSegmentCategory.STORYBOARD]: + request.settings.imageGenerationWorkflow, + [ClapSegmentCategory.VIDEO]: request.settings.videoGenerationWorkflow, + }[request.segment.category] + + if ( + clapWorkflow.category === ClapWorkflowCategory.IMAGE_GENERATION && + !clapWorkflow.inputValues[ClapperComfyUiInputIds.PROMPT] + ) { throw new Error( `This workflow doesn't seem to have an input required by Clapper (e.g. a node with an input called "prompt")` ) } - if (!imageGenerationWorkflow.inputValues[ClapperComfyUiInputIds.OUTPUT]) { + if (!clapWorkflow.inputValues[ClapperComfyUiInputIds.OUTPUT]) { throw new Error( `This workflow doesn't seem to have a node output required by Clapper (e.g. a 'Save Image' node)` ) } const comfyApiWorkflowPromptBuilder = createPromptBuilder( - ComfyUIWorkflowApiGraph.fromString(imageGenerationWorkflow.data) + ComfyUIWorkflowApiGraph.fromString(clapWorkflow.data) ) - const { inputFields, inputValues } = - request.settings.imageGenerationWorkflow + const { inputFields, inputValues } = clapWorkflow inputFields.forEach((inputField) => { comfyApiWorkflowPromptBuilder.input( @@ -79,13 +90,21 @@ export async function resolveSegment( [ClapperComfyUiInputIds.WIDTH, request.meta.width], [ClapperComfyUiInputIds.HEIGHT, request.meta.height], [ClapperComfyUiInputIds.SEED, generateSeed()], + [ + ClapperComfyUiInputIds.IMAGE, + request.prompts.video.image.split(';base64,')?.[1], + ], ] mainInputs.forEach((mainInput) => { - const inputId = (inputValues[mainInput[0]] as ClapInputValueObject).id - const inputValue = mainInput[1] - if (inputId) { - comfyApiWorkflowPromptBuilder.input(inputId, inputValue) + if ( + inputValues[mainInput[0]]?.id && + inputValues[mainInput[0]]?.id != ClapperComfyUiInputIds.NULL + ) { + comfyApiWorkflowPromptBuilder.input( + inputValues[mainInput[0]]?.id, + mainInput[1] + ) } }) @@ -114,21 +133,32 @@ export async function resolveSegment( throw new Error(`failed to run the pipeline (no output)`) } - const imagePaths = rawOutput[ClapperComfyUiInputIds.OUTPUT]?.images.map( - (img: any) => api.getPathImage(img) - ) + const getAssetPaths = (rawOutput) => { + if (clapWorkflow.category == ClapWorkflowCategory.VIDEO_GENERATION) { + return ( + rawOutput[ClapperComfyUiInputIds.OUTPUT]?.videos || + rawOutput[ClapperComfyUiInputIds.OUTPUT]?.gifs || + rawOutput[ClapperComfyUiInputIds.OUTPUT]?.images + ).map((asset: any) => api.getPathImage(asset)) + } else { + return rawOutput[ClapperComfyUiInputIds.OUTPUT]?.images.map( + (img: any) => api.getPathImage(img) + ) + } + } + const assetPaths = getAssetPaths(rawOutput) - console.log(`imagePaths:`, imagePaths) + console.log(`assetPaths:`, assetPaths) - const imagePath = imagePaths.at(0) - if (!imagePath) { + const assetPath = assetPaths.at(0) + if (!assetPath) { throw new Error(`failed to run the pipeline (no image)`) } // TODO: check what the imagePath looks like exactly - const assetUrl = await decodeOutput(imagePath) + const assetUrl = await decodeOutput(assetPath) - console.log(`assetUrl:`, imagePaths) + console.log(`assetUrl:`, assetPath) segment.assetUrl = assetUrl segment.assetSourceType = ClapAssetSource.DATA diff --git a/packages/app/src/app/api/resolve/providers/comfyui/utils.spec.ts b/packages/app/src/app/api/resolve/providers/comfyui/utils.spec.ts index c637db6f..489cfd0c 100644 --- a/packages/app/src/app/api/resolve/providers/comfyui/utils.spec.ts +++ b/packages/app/src/app/api/resolve/providers/comfyui/utils.spec.ts @@ -239,36 +239,66 @@ test('should return the correct inputs by node id', () => { name: 'seed', value: 156680208700286, id: '3.inputs.seed', - nodeId: '3', + node: { + id: '3', + name: 'KSampler', + type: 'KSampler', + }, }, { type: 'number', name: 'steps', value: 20, id: '3.inputs.steps', - nodeId: '3', + node: { + id: '3', + name: 'KSampler', + type: 'KSampler', + }, + }, + { + type: 'number', + name: 'cfg', + value: 8, + id: '3.inputs.cfg', + node: { + id: '3', + name: 'KSampler', + type: 'KSampler', + }, }, - { type: 'number', name: 'cfg', value: 8, id: '3.inputs.cfg', nodeId: '3' }, { type: 'string', name: 'sampler_name', value: 'euler', id: '3.inputs.sampler_name', - nodeId: '3', + node: { + id: '3', + name: 'KSampler', + type: 'KSampler', + }, }, { type: 'string', name: 'scheduler', value: 'normal', id: '3.inputs.scheduler', - nodeId: '3', + node: { + id: '3', + name: 'KSampler', + type: 'KSampler', + }, }, { type: 'number', name: 'denoise', value: 1, id: '3.inputs.denoise', - nodeId: '3', + node: { + id: '3', + name: 'KSampler', + type: 'KSampler', + }, }, ]) @@ -278,7 +308,11 @@ test('should return the correct inputs by node id', () => { name: 'ckpt_name', value: 'v1-5-pruned-emaonly.ckpt', id: '4.inputs.ckpt_name', - nodeId: '4', + node: { + id: '4', + name: 'Load Checkpoint', + type: 'CheckpointLoaderSimple', + }, }, ]) @@ -288,21 +322,33 @@ test('should return the correct inputs by node id', () => { name: 'width', value: 512, id: '5.inputs.width', - nodeId: '5', + node: { + id: '5', + name: 'Empty Latent Image', + type: 'EmptyLatentImage', + }, }, { type: 'number', name: 'height', value: 512, id: '5.inputs.height', - nodeId: '5', + node: { + id: '5', + name: 'Empty Latent Image', + type: 'EmptyLatentImage', + }, }, { type: 'number', name: 'batch_size', value: 1, id: '5.inputs.batch_size', - nodeId: '5', + node: { + id: '5', + name: 'Empty Latent Image', + type: 'EmptyLatentImage', + }, }, ]) @@ -313,7 +359,11 @@ test('should return the correct inputs by node id', () => { value: 'beautiful scenery nature glass bottle landscape, , purple galaxy bottle,', id: '6.inputs.text', - nodeId: '6', + node: { + id: '6', + name: 'CLIP Text Encode (Prompt)', + type: 'CLIPTextEncode', + }, }, ]) @@ -329,7 +379,11 @@ test('should detect the correct positive and negative prompt inputs', () => { expect(positivePromptInput).toEqual([ { id: '6.inputs.text', - nodeId: '6', + node: { + id: '6', + name: 'CLIP Text Encode (Prompt)', + type: 'CLIPTextEncode', + }, name: 'text', type: 'string', value: @@ -340,7 +394,11 @@ test('should detect the correct positive and negative prompt inputs', () => { expect(negativePromptInput).toEqual([ { id: '7.inputs.text', - nodeId: '7', + node: { + id: '7', + name: 'CLIP Text Encode (Prompt)', + type: 'CLIPTextEncode', + }, name: 'text', type: 'string', value: 'text, watermark', @@ -364,7 +422,11 @@ test('should detect the correct positive and negative prompt inputs using clappe expect(positivePromptInput).toEqual([ { id: '6.inputs.text', - nodeId: '6', + node: { + id: '6', + name: 'CLIP Text Encode (Prompt)', + type: 'CLIPTextEncode', + }, type: 'string', name: 'text', value: '@clapper/prompt', @@ -374,7 +436,11 @@ test('should detect the correct positive and negative prompt inputs using clappe expect(negativePromptInput).toEqual([ { id: '7.inputs.text', - nodeId: '7', + node: { + id: '7', + name: 'CLIP Text Encode (Prompt)', + type: 'CLIPTextEncode', + }, type: 'string', name: 'text', value: '@clapper/negative', @@ -398,7 +464,11 @@ test('should correctly search workflow inputs', () => { ).toEqual([ { id: '6.inputs.text', - nodeId: '6', + node: { + id: '6', + name: 'CLIP Text Encode (Prompt)', + type: 'CLIPTextEncode', + }, name: 'text', type: 'string', value: @@ -406,7 +476,11 @@ test('should correctly search workflow inputs', () => { }, { id: '7.inputs.text', - nodeId: '7', + node: { + id: '7', + name: 'CLIP Text Encode (Prompt)', + type: 'CLIPTextEncode', + }, name: 'text', type: 'string', value: 'text, watermark', @@ -421,7 +495,11 @@ test('should correctly search workflow inputs', () => { ).toEqual([ { id: '6.inputs.text', - nodeId: '6', + node: { + id: '6', + name: 'CLIP Text Encode (Prompt)', + type: 'CLIPTextEncode', + }, name: 'text', type: 'string', value: @@ -437,7 +515,11 @@ test('should correctly search workflow inputs', () => { ).toEqual([ { id: '6.inputs.text', - nodeId: '6', + node: { + id: '6', + name: 'CLIP Text Encode (Prompt)', + type: 'CLIPTextEncode', + }, name: 'text', type: 'string', value: @@ -472,6 +554,7 @@ test('should create the PromptBuilder', () => { ClapperComfyUiInputIds.NEGATIVE_PROMPT, [ClapperComfyUiInputIds.WIDTH]: ClapperComfyUiInputIds.WIDTH, [ClapperComfyUiInputIds.HEIGHT]: ClapperComfyUiInputIds.HEIGHT, + [ClapperComfyUiInputIds.SEED]: ClapperComfyUiInputIds.SEED, }) expect(promptBuilder.prompt).toEqual(workflowRaw) }) diff --git a/packages/app/src/app/api/resolve/providers/comfyui/utils.ts b/packages/app/src/app/api/resolve/providers/comfyui/utils.ts index 56a85101..d5e68b18 100644 --- a/packages/app/src/app/api/resolve/providers/comfyui/utils.ts +++ b/packages/app/src/app/api/resolve/providers/comfyui/utils.ts @@ -17,7 +17,9 @@ export enum ClapperComfyUiInputIds { WIDTH = '@clapper/width', HEIGHT = '@clapper/height', SEED = '@clapper/seed', + IMAGE = '@clapper/image', OUTPUT = '@clapper/output', + NULL = '@clapper/null', } type NodeRawData = { @@ -38,11 +40,15 @@ type INPUT_TYPES = 'string' | 'number' export type ComfyUiWorkflowApiNodeInput = { id: string - nodeId: string // Infered primitive type of the input based on its value type: INPUT_TYPES name: string value: any + node: { + id: string + name?: string + type?: string + } } export class ComfyUIWorkflowApiGraphEdge { @@ -270,7 +276,8 @@ export class ComfyUIWorkflowApiGraph { for (const node of nodesWithInputs) { const inputSchemas = this.getInputsByNodeId(node.id) inputSchemas?.forEach((inputSchema) => { - inputs[`${inputSchema.nodeId}.inputs.${inputSchema.name}`] = inputSchema + inputs[`${inputSchema.node.id}.inputs.${inputSchema.name}`] = + inputSchema }) } @@ -341,7 +348,11 @@ export class ComfyUIWorkflowApiGraph { name: name, value: value, id: `${nodeId}.inputs.${name}`, - nodeId, + node: { + id: nodeId, + type: nodeData.class_type, + name: nodeData?._meta?.title, + }, }) } @@ -357,6 +368,9 @@ export class ComfyUIWorkflowApiGraph { type?: string | RegExp // By name of node input name?: string | RegExp + // Based on the node + nodeType?: string | RegExp + nodeName?: string | RegExp // If any output of the node containing the input // is targeting another node's input with the given // name @@ -368,14 +382,14 @@ export class ComfyUIWorkflowApiGraph { // Helper function to match string or RegExp const matches = ( - value: string, + value: string | undefined, query: string | RegExp | undefined ): boolean => { if (!query) return true if (typeof query === 'string') { return value === query } else if (query instanceof RegExp) { - return query.test(value) + return query.test(value || '') } return false } @@ -384,8 +398,10 @@ export class ComfyUIWorkflowApiGraph { (input) => (matches(input.type, query.type) && matches(input.name, query.name) && + matches(input.node.name, query.nodeName) && + matches(input.node.type, query.nodeType) && (!query.nodeOutputToNodeInput || - this.nodes[input.nodeId].outboundEdges.some((edge) => + this.nodes[input.node.id].outboundEdges.some((edge) => matches(edge.relationship, query.nodeOutputToNodeInput) )) && !query.value) || @@ -406,8 +422,8 @@ export class ComfyUIWorkflowApiGraph { } const input = inputs[inputKey] if (!input) return - if (!this.nodes[input.nodeId].inputs) this.nodes[input.nodeId].inputs = {} - this.nodes[input.nodeId].inputs![input.name] = value + if (!this.nodes[input.node.id].inputs) this.nodes[input.node.id].inputs = {} + this.nodes[input.node.id].inputs![input.name] = value } toJson(): Record { @@ -467,7 +483,7 @@ export function findPromptInputsFromWorkflow( export function findNegativePromptInputsFromWorkflow( workflow: ComfyUIWorkflowApiGraph -) { +): ComfyUiWorkflowApiNodeInput[] { return unionBy( workflow.findInput({ name: /.*(text|prompt|negative).*/i, @@ -481,7 +497,9 @@ export function findNegativePromptInputsFromWorkflow( ) } -export function findWidthInputsFromWorkflow(workflow: ComfyUIWorkflowApiGraph) { +export function findWidthInputsFromWorkflow( + workflow: ComfyUIWorkflowApiGraph +): ComfyUiWorkflowApiNodeInput[] { return unionBy( workflow.findInput({ name: /.*(width).*/i, @@ -500,7 +518,7 @@ export function findWidthInputsFromWorkflow(workflow: ComfyUIWorkflowApiGraph) { export function findHeightInputsFromWorkflow( workflow: ComfyUIWorkflowApiGraph -) { +): ComfyUiWorkflowApiNodeInput[] { return unionBy( workflow.findInput({ name: /.*(height).*/i, @@ -517,7 +535,9 @@ export function findHeightInputsFromWorkflow( ) } -export function findSeedInputsFromWorkflow(workflow: ComfyUIWorkflowApiGraph) { +export function findSeedInputsFromWorkflow( + workflow: ComfyUIWorkflowApiGraph +): ComfyUiWorkflowApiNodeInput[] { return unionBy( workflow.findInput({ name: /.*(seed).*/i, @@ -534,17 +554,62 @@ export function findSeedInputsFromWorkflow(workflow: ComfyUIWorkflowApiGraph) { ) } -/** - * Returns input fields / input values required by ComfyUi - * @param workflow - */ +export function findImageInputsFromWorkflow( + workflow: ComfyUIWorkflowApiGraph +): ComfyUiWorkflowApiNodeInput[] { + return unionBy( + workflow.findInput({ + value: (value) => /.*\@clapper\/image.*/i.test(value), + }), + 'id' + ) +} -export function getInputsFromComfyUiWorkflow(workflowString: string): { +export const getMainInputIdsByClapWorkflowCategory = ( + category: ClapWorkflowCategory +) => { + switch (category) { + case ClapWorkflowCategory.VIDEO_GENERATION: { + return [ + ClapperComfyUiInputIds.IMAGE, + ClapperComfyUiInputIds.WIDTH, + ClapperComfyUiInputIds.HEIGHT, + ClapperComfyUiInputIds.SEED, + ClapperComfyUiInputIds.OUTPUT, + ] + } + default: { + return [ + ClapperComfyUiInputIds.PROMPT, + ClapperComfyUiInputIds.NEGATIVE_PROMPT, + ClapperComfyUiInputIds.WIDTH, + ClapperComfyUiInputIds.HEIGHT, + ClapperComfyUiInputIds.SEED, + ClapperComfyUiInputIds.OUTPUT, + ] + } + } +} + +export const MainClapWorkflowInputsLabels = { + [ClapperComfyUiInputIds.PROMPT]: 'Prompt node input', + [ClapperComfyUiInputIds.NEGATIVE_PROMPT]: 'Negative prompt node input', + [ClapperComfyUiInputIds.WIDTH]: 'Width node input', + [ClapperComfyUiInputIds.HEIGHT]: 'Height node input', + [ClapperComfyUiInputIds.SEED]: 'Seed node input', + [ClapperComfyUiInputIds.IMAGE]: 'Image node input', + [ClapperComfyUiInputIds.OUTPUT]: 'Output node', +} + +export function getMainInputsFromComfyUiWorkflow( + workflowString: string, + category: ClapWorkflowCategory +): { inputFields: ClapInputFields inputValues: ClapInputValues } { const workflowGraph = ComfyUIWorkflowApiGraph.fromString(workflowString) - + const mainInputsIds = getMainInputIdsByClapWorkflowCategory(category) const nodes = workflowGraph.getNodes() const textInputs = workflowGraph.findInput({ type: 'string', @@ -560,37 +625,44 @@ export function getInputsFromComfyUiWorkflow(workflowString: string): { const widthNodeInputs = findWidthInputsFromWorkflow(workflowGraph) const heightNodeInputs = findHeightInputsFromWorkflow(workflowGraph) const seedNodeInputs = findSeedInputsFromWorkflow(workflowGraph) + const imageNodeInputs = findImageInputsFromWorkflow(workflowGraph) const outputNode = workflowGraph.getOutputNode() const inputValues = { [ClapperComfyUiInputIds.PROMPT]: promptNodeInputs?.[0] ? { id: promptNodeInputs?.[0].id, - label: `${promptNodeInputs?.[0].id} (from node ${promptNodeInputs?.[0].nodeId})`, + label: `${promptNodeInputs?.[0].id} (from node ${promptNodeInputs?.[0].node.id})`, } : undefined, [ClapperComfyUiInputIds.NEGATIVE_PROMPT]: negativePromptNodeInputs?.[0] ? { id: negativePromptNodeInputs?.[0].id, - label: `${negativePromptNodeInputs?.[0].id} (from node ${negativePromptNodeInputs?.[0].nodeId})`, + label: `${negativePromptNodeInputs?.[0].id} (from node ${negativePromptNodeInputs?.[0].node.id})`, } : undefined, [ClapperComfyUiInputIds.WIDTH]: widthNodeInputs?.[0] ? { id: widthNodeInputs?.[0].id, - label: `${widthNodeInputs?.[0].id} (from node ${widthNodeInputs?.[0].nodeId})`, + label: `${widthNodeInputs?.[0].id} (from node ${widthNodeInputs?.[0].node.id})`, } : undefined, [ClapperComfyUiInputIds.HEIGHT]: heightNodeInputs?.[0] ? { id: heightNodeInputs?.[0].id, - label: `${heightNodeInputs?.[0].id} (from node ${heightNodeInputs?.[0].nodeId})`, + label: `${heightNodeInputs?.[0].id} (from node ${heightNodeInputs?.[0].node.id})`, } : undefined, [ClapperComfyUiInputIds.SEED]: seedNodeInputs?.[0] ? { id: seedNodeInputs?.[0].id, - label: `${seedNodeInputs?.[0].id} (from node ${seedNodeInputs?.[0].nodeId})`, + label: `${seedNodeInputs?.[0].id} (from node ${seedNodeInputs?.[0].node.id})`, + } + : undefined, + [ClapperComfyUiInputIds.IMAGE]: imageNodeInputs?.[0] + ? { + id: imageNodeInputs?.[0].id, + label: `${imageNodeInputs?.[0].id} (from node ${imageNodeInputs?.[0].node.id})`, } : undefined, [ClapperComfyUiInputIds.OUTPUT]: outputNode @@ -601,133 +673,213 @@ export function getInputsFromComfyUiWorkflow(workflowString: string): { : undefined, } - const inputLabels = { - [ClapperComfyUiInputIds.PROMPT]: 'Prompt node input', - [ClapperComfyUiInputIds.NEGATIVE_PROMPT]: 'Negative prompt node input', - [ClapperComfyUiInputIds.WIDTH]: 'Width node input', - [ClapperComfyUiInputIds.HEIGHT]: 'Height node input', - [ClapperComfyUiInputIds.SEED]: 'Seed node input', - [ClapperComfyUiInputIds.OUTPUT]: 'Output node', + const getOptionsItems = (inputs: ComfyUiWorkflowApiNodeInput[]) => { + return [ + ...inputs, + { + id: ClapperComfyUiInputIds.NULL, + name: 'Unset', + node: { + id: null, + }, + }, + ].map((p) => { + const item = { + id: p.id, + label: + p.id === ClapperComfyUiInputIds.NULL + ? `Unset` + : `${p.name} (from node ${p.node.id})`, + } + return { + ...item, + value: item, + } + }) } - const inputFields: ClapInputField<{ - options?: { - id: string - label: string - value: any - }[] - mainInput?: any - }>[] = [ - // Required fields that should be available in the workflow, otherwise - // Clapper doesn't know how to input its settings (prompts, dimensions, etc) - { - id: '@clapper/mainInputs', - label: 'Main settings', - type: 'group' as any, - category: ClapInputCategory.UNKNOWN, - description: 'Main inputs', - defaultValue: '', - inputFields: [ - { + const inputFields: any = [] + mainInputsIds.forEach((mainInput) => { + switch (mainInput) { + case ClapperComfyUiInputIds.PROMPT: { + inputFields.push({ id: ClapperComfyUiInputIds.PROMPT, - label: inputLabels[ClapperComfyUiInputIds.PROMPT], + label: MainClapWorkflowInputsLabels[ClapperComfyUiInputIds.PROMPT], type: 'nodeInput' as any, category: ClapInputCategory.PROMPT, description: 'The input where Clapper will put the segment prompt', defaultValue: '', metadata: { - options: unionBy(promptNodeInputs, textInputs, 'id').map((p) => ({ - id: p.id, - label: `${p.name} (from node ${p.nodeId})`, - value: p.id, - })), + options: getOptionsItems( + unionBy(promptNodeInputs, textInputs, 'id') + ), }, - }, - { + }) + return + } + case ClapperComfyUiInputIds.NEGATIVE_PROMPT: { + inputFields.push({ id: ClapperComfyUiInputIds.NEGATIVE_PROMPT, - label: inputLabels[ClapperComfyUiInputIds.NEGATIVE_PROMPT], + label: + MainClapWorkflowInputsLabels[ + ClapperComfyUiInputIds.NEGATIVE_PROMPT + ], type: 'nodeInput' as any, category: ClapInputCategory.PROMPT, description: 'The node input where Clapper will put the segment negative prompt', defaultValue: '', metadata: { - options: unionBy(negativePromptNodeInputs, textInputs, 'id').map( - (p) => ({ - id: p.id, - label: `${p.name} (from node ${p.nodeId})`, - value: p.id, - }) + options: getOptionsItems( + unionBy(negativePromptNodeInputs, textInputs, 'id') ), }, - }, - { + }) + return + } + case ClapperComfyUiInputIds.WIDTH: { + inputFields.push({ id: ClapperComfyUiInputIds.WIDTH, - label: inputLabels[ClapperComfyUiInputIds.WIDTH], + label: MainClapWorkflowInputsLabels[ClapperComfyUiInputIds.WIDTH], type: 'nodeInput' as any, category: ClapInputCategory.WIDTH, description: 'The node input where Clapper will put the required image width', defaultValue: '', metadata: { - options: unionBy(widthNodeInputs, dimensionInputs, 'id').map( - (p) => ({ - id: p.id, - label: `${p.name} (from node ${p.nodeId})`, - value: p.id, - }) + options: getOptionsItems( + unionBy(widthNodeInputs, dimensionInputs, 'id') ), }, - }, - { + }) + return + } + case ClapperComfyUiInputIds.HEIGHT: { + inputFields.push({ id: ClapperComfyUiInputIds.HEIGHT, - label: inputLabels[ClapperComfyUiInputIds.HEIGHT], + label: MainClapWorkflowInputsLabels[ClapperComfyUiInputIds.HEIGHT], type: 'nodeInput' as any, description: 'The node input where Clapper will put the required image height', category: ClapInputCategory.HEIGHT, defaultValue: 1000, metadata: { - options: unionBy(heightNodeInputs, dimensionInputs, 'id').map( - (p) => ({ - id: p.id, - label: `${p.name} (from node ${p.nodeId})`, - value: p.id, - }) + options: getOptionsItems( + unionBy(heightNodeInputs, dimensionInputs, 'id') ), }, - }, - { + }) + return + } + case ClapperComfyUiInputIds.SEED: { + inputFields.push({ id: ClapperComfyUiInputIds.SEED, - label: inputLabels[ClapperComfyUiInputIds.SEED], + label: MainClapWorkflowInputsLabels[ClapperComfyUiInputIds.SEED], type: 'nodeInput' as any, category: ClapInputCategory.UNKNOWN, description: 'The node input where Clapper will set a seed', defaultValue: '', metadata: { - options: seedNodeInputs.map((p) => ({ - id: p.id, - label: `${p.name} (from node ${p.nodeId})`, - value: p.id, - })), + options: getOptionsItems(seedNodeInputs), }, - }, - { + }) + return + } + case ClapperComfyUiInputIds.IMAGE: { + inputFields.push({ + id: ClapperComfyUiInputIds.IMAGE, + label: MainClapWorkflowInputsLabels[ClapperComfyUiInputIds.IMAGE], + type: 'nodeInput' as any, + category: ClapInputCategory.UNKNOWN, + description: 'The node input where Clapper will set an image', + defaultValue: '', + metadata: { + options: getOptionsItems(imageNodeInputs), + tooltip: { + message: ` + Clapper doesn't support file/upload node inputs; + use a string input from where Clapper can load a base64 + data URI string (e.g. the "Load Image From Base64" node's + "data" input in the default video workflow). + `, + type: 'info', + }, + }, + }) + return + } + case ClapperComfyUiInputIds.OUTPUT: { + inputFields.push({ id: ClapperComfyUiInputIds.OUTPUT, - label: inputLabels[ClapperComfyUiInputIds.OUTPUT], + label: MainClapWorkflowInputsLabels[ClapperComfyUiInputIds.OUTPUT], type: 'node' as any, category: ClapInputCategory.UNKNOWN, description: 'The node from which Clapper will get the output image', defaultValue: '', metadata: { - options: nodes.map((p) => ({ - id: p.id, - label: `${p._meta?.title || 'Untitled node'} (id: ${p.id})`, - value: p.id, - })), + options: nodes.map((p) => { + const item = { + id: p.id, + label: `${p._meta?.title || 'Untitled node'} (id: ${p.id})`, + } + return { + ...item, + value: item, + } + }), }, - }, - ], + }) + return + } + } + }) + + return { + inputFields, + inputValues, + } +} + +/** + * Returns input fields / input values required by ComfyUi + * @param workflow + */ +export function getInputsFromComfyUiWorkflow( + workflowString: string, + category: ClapWorkflowCategory +): { + inputFields: ClapInputFields + inputValues: ClapInputValues +} { + const workflowGraph = ComfyUIWorkflowApiGraph.fromString(workflowString) + const { inputFields: mainInputFields, inputValues: mainInputValues } = + getMainInputsFromComfyUiWorkflow(workflowString, category) + + const inputValues = { + ...mainInputValues, + } as any + + const inputFields: ClapInputField<{ + options?: { + id: string + label: string + value: any + }[] + tooltip?: { + type: string + message: string + } + }>[] = [ + // Required fields that should be available in the workflow, otherwise + // Clapper doesn't know how to input its settings (prompts, dimensions, etc) + { + id: '@clapper/mainInputs', + label: 'Main settings', + type: 'group' as any, + category: ClapInputCategory.UNKNOWN, + description: 'Main inputs', + defaultValue: '', + inputFields: mainInputFields, }, // Other input fields based on the workflow nodes { @@ -762,7 +914,13 @@ export function getInputsFromComfyUiWorkflow(workflowString: string): { description: '', defaultValue: input.value, metadata: { - mainInput: inputLabels[mainInputKey as string], + tooltip: MainClapWorkflowInputsLabels[mainInputKey as string] + ? { + type: 'warning', + message: `This value will be overwritten by Clapper because it is + used as "${MainClapWorkflowInputsLabels[mainInputKey as string]}".`, + } + : undefined, }, } }), @@ -816,26 +974,51 @@ export function createPromptBuilder( } export function convertComfyUiWorkflowApiToClapWorkflow( - workflowString: string + workflowString: string, + category: ClapWorkflowCategory = ClapWorkflowCategory.IMAGE_GENERATION ): ClapWorkflow { try { - const { inputFields, inputValues } = - getInputsFromComfyUiWorkflow(workflowString) - return { - id: 'comfyui://settings.comfyWorkflowForImage', - label: 'Custom Image Workflow', - description: 'Custom ComfyUI workflow to generate images', - tags: ['custom', 'image generation'], - author: 'You', - thumbnailUrl: '', - nonCommercial: false, - engine: ClapWorkflowEngine.COMFYUI_WORKFLOW, - provider: ClapWorkflowProvider.COMFYUI, - category: ClapWorkflowCategory.IMAGE_GENERATION, - data: workflowString, - schema: '', - inputFields, - inputValues, + const { inputFields, inputValues } = getInputsFromComfyUiWorkflow( + workflowString, + category + ) + switch (category) { + case ClapWorkflowCategory.VIDEO_GENERATION: { + return { + id: 'comfyui://settings.comfyWorkflowForVideo', + label: 'Custom Video Workflow', + description: 'Custom ComfyUI workflow to generate videos', + tags: ['custom', 'video generation'], + author: 'You', + thumbnailUrl: '', + nonCommercial: false, + engine: ClapWorkflowEngine.COMFYUI_WORKFLOW, + provider: ClapWorkflowProvider.COMFYUI, + category, + data: workflowString, + schema: '', + inputFields, + inputValues, + } + } + default: { + return { + id: 'comfyui://settings.comfyWorkflowForImage', + label: 'Custom Image Workflow', + description: 'Custom ComfyUI workflow to generate images', + tags: ['custom', 'image generation'], + author: 'You', + thumbnailUrl: '', + nonCommercial: false, + engine: ClapWorkflowEngine.COMFYUI_WORKFLOW, + provider: ClapWorkflowProvider.COMFYUI, + category, + data: workflowString, + schema: '', + inputFields, + inputValues, + } + } } } catch (e) { throw e diff --git a/packages/app/src/components/forms/FormComfyUIWorkflowSettings.tsx b/packages/app/src/components/forms/FormComfyUIWorkflowSettings.tsx index 2c984ea1..c8f1fcb5 100644 --- a/packages/app/src/components/forms/FormComfyUIWorkflowSettings.tsx +++ b/packages/app/src/components/forms/FormComfyUIWorkflowSettings.tsx @@ -4,7 +4,7 @@ import { GrDocumentConfig } from 'react-icons/gr' import debounce from 'lodash/debounce' import { FormSelect } from './FormSelect' import { Tooltip, TooltipContent, TooltipTrigger } from '../ui/tooltip' -import { MdWarning } from 'react-icons/md' +import { MdInfo, MdWarning } from 'react-icons/md' import { ClapInputField, ClapWorkflow } from '@aitube/clap' import { FormArea } from './FormArea' import { useCallback, useState } from 'react' @@ -12,13 +12,16 @@ import { ComfyUIWorkflowApiGraph, convertComfyUiWorkflowApiToClapWorkflow, } from '@/app/api/resolve/providers/comfyui/utils' +import clsx from 'clsx' export function FormComfyUIWorkflowSettings({ + label, clapWorkflow, defaultClapWorkflow, onChange, className, }: { + label: string clapWorkflow: ClapWorkflow defaultClapWorkflow: ClapWorkflow onChange: (clapWorkflow: ClapWorkflow) => void @@ -44,7 +47,9 @@ export function FormComfyUIWorkflowSettings({ if (ComfyUIWorkflowApiGraph.isValidWorkflow(json)) { setErrors({ ...errors, workflow: null }) debouncedOnChangeClapWorkflow( - structuredClone(convertComfyUiWorkflowApiToClapWorkflow(json)) + structuredClone( + convertComfyUiWorkflowApiToClapWorkflow(json, clapWorkflow.category) + ) ) } else { setErrors({ ...errors, workflow: 'Please, provide a valid workflow.' }) @@ -69,6 +74,34 @@ export function FormComfyUIWorkflowSettings({ } } + const renderInputTooltip = (tooltip) => { + return ( + + +
+
+ {tooltip.type == 'info' ? ( + + ) : ( + + )} +
+
+
+ +

+ {tooltip.message} +

+
+
+ ) + } + const renderInputFields = ( inputFields: ClapInputField[], inputValues: Record @@ -94,20 +127,30 @@ export function FormComfyUIWorkflowSettings({ case 'nodeInput': case 'node': { return ( - - handleOnChangeInputValue(inputField.id, value) - } - /> + > + + handleOnChangeInputValue(inputField.id, value) + } + /> + {inputField.metadata?.tooltip && + renderInputTooltip(inputField.metadata.tooltip)} + ) } default: { @@ -126,21 +169,8 @@ export function FormComfyUIWorkflowSettings({ } className="pr-8" /> - {inputField.metadata?.mainInput && ( - - -
- -
-
- -

- This value will be overwritten by Clapper because it is - used as "{inputField.metadata?.mainInput} ". -

-
-
- )} + {inputField.metadata?.tooltip && + renderInputTooltip(inputField.metadata.tooltip)} ) } @@ -151,7 +181,7 @@ export function FormComfyUIWorkflowSettings({ return ( <> s.comfyWorkflowForVideo) + const comfyClapWorkflow = useSettings((s) => s.comfyClapWorkflowForVideo) const setComfyWorkflowForVideo = useSettings( - (s) => s.setComfyWorkflowForVideo + (s) => s.setComfyClapWorkflowForVideo ) const videoPromptPrefix = useSettings((s) => s.videoPromptPrefix) @@ -25,6 +26,10 @@ export function SettingsSectionVideo() { (s) => s.setMaxVideosToGenerateInParallel ) + const onChangeClapWorkflow = (clapWorkflowUpdated: ClapWorkflow) => { + setComfyWorkflowForVideo(clapWorkflowUpdated) + } + return (
@@ -56,12 +61,12 @@ export function SettingsSectionVideo() { onChange={setVideoNegativePrompt} /> -
diff --git a/packages/app/src/services/editors/workflow-editor/useDynamicWorkflows.ts b/packages/app/src/services/editors/workflow-editor/useDynamicWorkflows.ts index 591683d8..4c430e9d 100644 --- a/packages/app/src/services/editors/workflow-editor/useDynamicWorkflows.ts +++ b/packages/app/src/services/editors/workflow-editor/useDynamicWorkflows.ts @@ -14,7 +14,9 @@ export function useDynamicWorkflows() { const comfyClapWorkflowForImage = useSettings( (s) => s.comfyClapWorkflowForImage ) - const comfyWorkflowForVideo = useSettings((s) => s.comfyWorkflowForVideo) + const comfyClapWorkflowForVideo = useSettings( + (s) => s.comfyClapWorkflowForVideo + ) const comfyWorkflowForMusic = useSettings((s) => s.comfyWorkflowForMusic) const comfyWorkflowForSound = useSettings((s) => s.comfyWorkflowForSound) const comfyWorkflowForVoice = useSettings((s) => s.comfyWorkflowForVoice) @@ -27,7 +29,7 @@ export function useDynamicWorkflows() { updateAvailableWorkflows() }, [ comfyClapWorkflowForImage, - comfyWorkflowForVideo, + comfyClapWorkflowForVideo, comfyWorkflowForMusic, comfyWorkflowForSound, comfyWorkflowForVoice, diff --git a/packages/app/src/services/editors/workflow-editor/workflows/comfyui/getComfyWorkflow.ts b/packages/app/src/services/editors/workflow-editor/workflows/comfyui/getComfyWorkflow.ts index 4e2d144d..0935a867 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/comfyui/getComfyWorkflow.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/comfyui/getComfyWorkflow.ts @@ -1,19 +1,16 @@ import { ClapSegmentCategory } from '@aitube/clap' -import { getVideoPrompt } from '@aitube/engine' - -import { ComfyNode, ResolveRequest } from '@aitube/clapper-services' import { useSettings } from '@/services' export function getComfyWorkflow(category: ClapSegmentCategory) { const settings = useSettings.getState() - let comfyWorkflow = '{}' + let comfyWorkflow if (category === ClapSegmentCategory.STORYBOARD) { - comfyWorkflow = settings.comfyClapWorkflowForImage?.data + comfyWorkflow = settings.comfyClapWorkflowForImage } else if (category === ClapSegmentCategory.VIDEO) { - comfyWorkflow = settings.comfyWorkflowForVideo + comfyWorkflow = settings.comfyClapWorkflowForVideo } - return JSON.stringify(comfyWorkflow) + return JSON.stringify(comfyWorkflow || {}) } diff --git a/packages/app/src/services/editors/workflow-editor/workflows/comfyui/index.ts b/packages/app/src/services/editors/workflow-editor/workflows/comfyui/index.ts index d3a68be8..15f5b492 100644 --- a/packages/app/src/services/editors/workflow-editor/workflows/comfyui/index.ts +++ b/packages/app/src/services/editors/workflow-editor/workflows/comfyui/index.ts @@ -7,7 +7,6 @@ import { import { genericImage, genericPrompt } from '../common/defaultValues' import { text_to_image_demo_workflow } from '../common/comfyui/text_to_image_demo_workflow' -import { getComfyWorkflow } from './getComfyWorkflow' import { useSettings } from '@/services' // ------------------------------------------------------------------------------ @@ -80,10 +79,12 @@ export async function getDynamicComfyuiWorkflows(): Promise { engine: ClapWorkflowEngine.COMFYUI_WORKFLOW, provider: ClapWorkflowProvider.COMFYUI, category: ClapWorkflowCategory.VIDEO_GENERATION, - data: settings.comfyWorkflowForVideo, + data: settings.comfyClapWorkflowForVideo.data, schema: '', - inputFields: [genericImage], - inputValues: { + inputFields: settings.comfyClapWorkflowForVideo.inputFields || [ + genericImage, + ], + inputValues: settings.comfyClapWorkflowForVideo.inputValues || { [genericImage.id]: genericImage.defaultValue, }, }, diff --git a/packages/app/src/services/io/useIO.ts b/packages/app/src/services/io/useIO.ts index 6c951bb1..4f34e189 100644 --- a/packages/app/src/services/io/useIO.ts +++ b/packages/app/src/services/io/useIO.ts @@ -549,8 +549,6 @@ export const useIO = create((set, get) => ({ isExportableToFile && id !== ignoreThisVideoSegmentId ) - console.log('segments:', segments) - const videos: FFMPegVideoInput[] = [] const audios: FFMPegAudioInput[] = [] @@ -569,7 +567,6 @@ export const useIO = create((set, get) => ({ assetSourceType = ClapAssetSource.PATH if (filePath.startsWith('video/')) { - console.log('adding video') videos.push({ data: base64DataUriToUint8Array(segment.assetUrl), startTimeInMs: segment.startTimeInMs, @@ -610,10 +607,8 @@ export const useIO = create((set, get) => ({ ) const videoBlob = new Blob([fullVideo], { type: 'video/mp4' }) - - task.success() - saveAnyFile(videoBlob, 'my_project.mp4') + task.success() }, saveZipFile: async () => { diff --git a/packages/app/src/services/settings/getDefaultSettingsState.ts b/packages/app/src/services/settings/getDefaultSettingsState.ts index 9ce176d6..52639274 100644 --- a/packages/app/src/services/settings/getDefaultSettingsState.ts +++ b/packages/app/src/services/settings/getDefaultSettingsState.ts @@ -2,6 +2,7 @@ import { RenderingStrategy } from '@aitube/timeline' import { ComfyIcuAccelerator, SettingsState } from '@aitube/clapper-services' import { defaultWorkflowForImages } from './workflows/image' +import { defaultWorkflowForVideos } from './workflows/video' export function getDefaultSettingsState(): SettingsState { const state: SettingsState = { @@ -81,7 +82,7 @@ export function getDefaultSettingsState(): SettingsState { maxVideosToGenerateInParallel: 1, comfyClapWorkflowForImage: defaultWorkflowForImages, - comfyWorkflowForVideo: '{}', + comfyClapWorkflowForVideo: defaultWorkflowForVideos, comfyWorkflowForVoice: '{}', comfyWorkflowForSound: '{}', comfyWorkflowForMusic: '{}', diff --git a/packages/app/src/services/settings/useSettings.ts b/packages/app/src/services/settings/useSettings.ts index 9f6a324c..83d4522f 100644 --- a/packages/app/src/services/settings/useSettings.ts +++ b/packages/app/src/services/settings/useSettings.ts @@ -675,13 +675,27 @@ export const useSettings = create()( console.error(e) } }, - setComfyWorkflowForVideo: (comfyWorkflowForVideo?: string) => { - set({ - comfyWorkflowForVideo: getValidComfyWorkflowTemplate( - comfyWorkflowForVideo, - getDefaultSettingsState().comfyWorkflowForVideo - ), - }) + setComfyClapWorkflowForVideo: ( + comfyClapWorkflowForVideo?: ClapWorkflow + ) => { + try { + if (!comfyClapWorkflowForVideo) throw new Error('Invalid workflow') + const currentWorkflowString = get().videoGenerationWorkflow + const currentWorkflow = parseWorkflow( + currentWorkflowString, + ClapWorkflowCategory.VIDEO_GENERATION + ) + const newWorkflow = comfyClapWorkflowForVideo + set({ + comfyClapWorkflowForVideo: newWorkflow, + videoGenerationWorkflow: + currentWorkflow.engine === ClapWorkflowEngine.COMFYUI_WORKFLOW + ? JSON.stringify(newWorkflow) + : currentWorkflowString, + }) + } catch (e) { + console.error(e) + } }, setComfyWorkflowForVoice: (comfyWorkflowForVoice?: string) => { set({ @@ -1017,9 +1031,9 @@ export const useSettings = create()( comfyClapWorkflowForImage: state.comfyClapWorkflowForImage || defaultSettings.comfyClapWorkflowForImage, - comfyWorkflowForVideo: - state.comfyWorkflowForVideo || - defaultSettings.comfyWorkflowForVideo, + comfyClapWorkflowForVideo: + state.comfyClapWorkflowForVideo || + defaultSettings.comfyClapWorkflowForVideo, comfyWorkflowForVoice: state.comfyWorkflowForVoice || defaultSettings.comfyWorkflowForVoice, diff --git a/packages/app/src/services/settings/workflows/image.ts b/packages/app/src/services/settings/workflows/image.ts index 29f97a6e..29d3feeb 100644 --- a/packages/app/src/services/settings/workflows/image.ts +++ b/packages/app/src/services/settings/workflows/image.ts @@ -1,4 +1,5 @@ import { convertComfyUiWorkflowApiToClapWorkflow } from '@/app/api/resolve/providers/comfyui/utils' +import { ClapWorkflowCategory } from '@aitube/clap' // https://replicate.com/fofr/any-comfyui-workflow/examples#ps4f5zcthdrgm0cfqh586zed7r export const defaultWorkflowForImages = convertComfyUiWorkflowApiToClapWorkflow( @@ -81,5 +82,6 @@ export const defaultWorkflowForImages = convertComfyUiWorkflowApiToClapWorkflow( title: 'Save Image', }, }, - }) + }), + ClapWorkflowCategory.IMAGE_GENERATION ) diff --git a/packages/app/src/services/settings/workflows/parseWorkflow.ts b/packages/app/src/services/settings/workflows/parseWorkflow.ts index c6eb1ffd..20373140 100644 --- a/packages/app/src/services/settings/workflows/parseWorkflow.ts +++ b/packages/app/src/services/settings/workflows/parseWorkflow.ts @@ -54,7 +54,10 @@ export function parseWorkflow( const { inputFields: defaultInputFields, inputValues: defaultInputValues, - } = convertComfyUiWorkflowApiToClapWorkflow(maybeWorkflow.data) + } = convertComfyUiWorkflowApiToClapWorkflow( + maybeWorkflow.data, + maybeWorkflow.category + ) // Use the already existing inputFields/inputValues, otherwise use the default // ones based on the raw comfyui workflow data maybeWorkflow.inputFields = diff --git a/packages/app/src/services/settings/workflows/video.ts b/packages/app/src/services/settings/workflows/video.ts new file mode 100644 index 00000000..88266bf0 --- /dev/null +++ b/packages/app/src/services/settings/workflows/video.ts @@ -0,0 +1,116 @@ +/** + * https://comfyanonymous.github.io/ComfyUI_examples/video/ + * + * Modifications: + * - Replaced 'Load Image' node by 'Load Image From Base64' + */ + +import { convertComfyUiWorkflowApiToClapWorkflow } from '@/app/api/resolve/providers/comfyui/utils' +import { ClapWorkflowCategory } from '@aitube/clap' + +export const defaultWorkflowForVideos = convertComfyUiWorkflowApiToClapWorkflow( + JSON.stringify({ + '3': { + inputs: { + seed: 524650296754258, + steps: 20, + cfg: 2.5, + sampler_name: 'euler', + scheduler: 'karras', + denoise: 1, + model: ['14', 0], + positive: ['12', 0], + negative: ['12', 1], + latent_image: ['12', 2], + }, + class_type: 'KSampler', + _meta: { + title: 'KSampler', + }, + }, + '8': { + inputs: { + samples: ['3', 0], + vae: ['15', 2], + }, + class_type: 'VAEDecode', + _meta: { + title: 'VAE Decode', + }, + }, + '12': { + inputs: { + width: 1024, + height: 576, + video_frames: 14, + motion_bucket_id: 127, + fps: 6, + augmentation_level: 0, + clip_vision: ['15', 1], + init_image: ['24', 0], + vae: ['15', 2], + }, + class_type: 'SVD_img2vid_Conditioning', + _meta: { + title: 'SVD_img2vid_Conditioning', + }, + }, + '14': { + inputs: { + min_cfg: 1, + model: ['15', 0], + }, + class_type: 'VideoLinearCFGGuidance', + _meta: { + title: 'VideoLinearCFGGuidance', + }, + }, + '15': { + inputs: { + ckpt_name: 'svd/svd.safetensors', + }, + class_type: 'ImageOnlyCheckpointLoader', + _meta: { + title: 'Image Only Checkpoint Loader (img2vid model)', + }, + }, + '23': { + inputs: { + image: 'download.jpeg', + upload: 'image', + }, + class_type: 'LoadImage', + _meta: { + title: 'Load Image', + }, + }, + '24': { + inputs: { + data: '@clapper/image', + }, + class_type: 'LoadImageFromBase64', + _meta: { + title: 'Load Image From Base64', + }, + }, + '25': { + inputs: { + frame_rate: 8, + loop_count: 0, + filename_prefix: 'AnimateDiff', + format: 'video/h264-mp4', + pix_fmt: 'yuv420p', + crf: 0, + save_metadata: false, + pingpong: false, + save_output: true, + images: ['8', 0], + }, + class_type: 'VHS_VideoCombine', + _meta: { + title: 'Video Combine 🎥🅥🅗🅢', + }, + }, + }), + ClapWorkflowCategory.VIDEO_GENERATION +) diff --git a/packages/clapper-services/src/settings.ts b/packages/clapper-services/src/settings.ts index d3f9443a..1c09b0a6 100644 --- a/packages/clapper-services/src/settings.ts +++ b/packages/clapper-services/src/settings.ts @@ -65,7 +65,7 @@ export type BaseSettings = { // ------------ COMFY UI CLAP WORKFLOWS ----------------- comfyClapWorkflowForImage: ClapWorkflow - comfyWorkflowForVideo: string + comfyClapWorkflowForVideo: ClapWorkflow comfyWorkflowForVoice: string comfyWorkflowForSound: string comfyWorkflowForMusic: string @@ -200,7 +200,7 @@ export type SettingsControls = { setMaxVideosToGenerateInParallel: (maxVideosToGenerateInParallel?: number) => void setComfyClapWorkflowForImage: (comfyClapWorkflowForImage?: ClapWorkflow) => void - setComfyWorkflowForVideo: (comfyWorkflowForVideo?: string) => void + setComfyClapWorkflowForVideo: (comfyClapWorkflowForVideo?: ClapWorkflow) => void setComfyWorkflowForVoice: (comfyWorkflowForVoice?: string) => void setComfyWorkflowForSound: (comfyWorkflowForSound?: string) => void setComfyWorkflowForMusic: (comfyWorkflowForMusic?: string) => void