Skip to content

Commit

Permalink
Merge branch 'main' of github.com:jbilcke-hf/clapper
Browse files Browse the repository at this point in the history
  • Loading branch information
@jbilcke-hf
jbilcke-hf committed Aug 28, 2024
2 parents 1b1405d + 56d8237 commit aa5b0b4
Show file tree
Hide file tree
Showing 16 changed files with 695 additions and 232 deletions.
70 changes: 50 additions & 20 deletions packages/app/src/app/api/resolve/providers/comfyui/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ import { ResolveRequest } from '@aitube/clapper-services'
import {
ClapAssetSource,
ClapSegmentCategory,
ClapWorkflowCategory,
generateSeed,
} from '@aitube/clap'
import { TimelineSegment } from '@aitube/timeline'
Expand Down Expand Up @@ -44,27 +45,37 @@ export async function resolveSegment(
: undefined
).init()

if (request.segment.category === ClapSegmentCategory.STORYBOARD) {
const imageGenerationWorkflow = request.settings.imageGenerationWorkflow

if (!imageGenerationWorkflow.inputValues[ClapperComfyUiInputIds.PROMPT]) {
if (
[ClapSegmentCategory.STORYBOARD, ClapSegmentCategory.VIDEO].includes(
request.segment.category
)
) {
const clapWorkflow = {
[ClapSegmentCategory.STORYBOARD]:
request.settings.imageGenerationWorkflow,
[ClapSegmentCategory.VIDEO]: request.settings.videoGenerationWorkflow,
}[request.segment.category]

if (
clapWorkflow.category === ClapWorkflowCategory.IMAGE_GENERATION &&
!clapWorkflow.inputValues[ClapperComfyUiInputIds.PROMPT]
) {
throw new Error(
`This workflow doesn't seem to have an input required by Clapper (e.g. a node with an input called "prompt")`
)
}

if (!imageGenerationWorkflow.inputValues[ClapperComfyUiInputIds.OUTPUT]) {
if (!clapWorkflow.inputValues[ClapperComfyUiInputIds.OUTPUT]) {
throw new Error(
`This workflow doesn't seem to have a node output required by Clapper (e.g. a 'Save Image' node)`
)
}

const comfyApiWorkflowPromptBuilder = createPromptBuilder(
ComfyUIWorkflowApiGraph.fromString(imageGenerationWorkflow.data)
ComfyUIWorkflowApiGraph.fromString(clapWorkflow.data)
)

const { inputFields, inputValues } =
request.settings.imageGenerationWorkflow
const { inputFields, inputValues } = clapWorkflow

inputFields.forEach((inputField) => {
comfyApiWorkflowPromptBuilder.input(
Expand All @@ -79,13 +90,21 @@ export async function resolveSegment(
[ClapperComfyUiInputIds.WIDTH, request.meta.width],
[ClapperComfyUiInputIds.HEIGHT, request.meta.height],
[ClapperComfyUiInputIds.SEED, generateSeed()],
[
ClapperComfyUiInputIds.IMAGE,
request.prompts.video.image.split(';base64,')?.[1],
],
]

mainInputs.forEach((mainInput) => {
const inputId = (inputValues[mainInput[0]] as ClapInputValueObject).id
const inputValue = mainInput[1]
if (inputId) {
comfyApiWorkflowPromptBuilder.input(inputId, inputValue)
if (
inputValues[mainInput[0]]?.id &&
inputValues[mainInput[0]]?.id != ClapperComfyUiInputIds.NULL
) {
comfyApiWorkflowPromptBuilder.input(
inputValues[mainInput[0]]?.id,
mainInput[1]
)
}
})

Expand Down Expand Up @@ -114,21 +133,32 @@ export async function resolveSegment(
throw new Error(`failed to run the pipeline (no output)`)
}

const imagePaths = rawOutput[ClapperComfyUiInputIds.OUTPUT]?.images.map(
(img: any) => api.getPathImage(img)
)
const getAssetPaths = (rawOutput) => {
if (clapWorkflow.category == ClapWorkflowCategory.VIDEO_GENERATION) {
return (
rawOutput[ClapperComfyUiInputIds.OUTPUT]?.videos ||
rawOutput[ClapperComfyUiInputIds.OUTPUT]?.gifs ||
rawOutput[ClapperComfyUiInputIds.OUTPUT]?.images
).map((asset: any) => api.getPathImage(asset))
} else {
return rawOutput[ClapperComfyUiInputIds.OUTPUT]?.images.map(
(img: any) => api.getPathImage(img)
)
}
}
const assetPaths = getAssetPaths(rawOutput)

console.log(`imagePaths:`, imagePaths)
console.log(`assetPaths:`, assetPaths)

const imagePath = imagePaths.at(0)
if (!imagePath) {
const assetPath = assetPaths.at(0)
if (!assetPath) {
throw new Error(`failed to run the pipeline (no image)`)
}

// TODO: check what the imagePath looks like exactly
const assetUrl = await decodeOutput(imagePath)
const assetUrl = await decodeOutput(assetPath)

console.log(`assetUrl:`, imagePaths)
console.log(`assetUrl:`, assetPath)
segment.assetUrl = assetUrl
segment.assetSourceType = ClapAssetSource.DATA

Expand Down
121 changes: 102 additions & 19 deletions packages/app/src/app/api/resolve/providers/comfyui/utils.spec.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -239,36 +239,66 @@ test('should return the correct inputs by node id', () => {
name: 'seed',
value: 156680208700286,
id: '3.inputs.seed',
nodeId: '3',
node: {
id: '3',
name: 'KSampler',
type: 'KSampler',
},
},
{
type: 'number',
name: 'steps',
value: 20,
id: '3.inputs.steps',
nodeId: '3',
node: {
id: '3',
name: 'KSampler',
type: 'KSampler',
},
},
{
type: 'number',
name: 'cfg',
value: 8,
id: '3.inputs.cfg',
node: {
id: '3',
name: 'KSampler',
type: 'KSampler',
},
},
{ type: 'number', name: 'cfg', value: 8, id: '3.inputs.cfg', nodeId: '3' },
{
type: 'string',
name: 'sampler_name',
value: 'euler',
id: '3.inputs.sampler_name',
nodeId: '3',
node: {
id: '3',
name: 'KSampler',
type: 'KSampler',
},
},
{
type: 'string',
name: 'scheduler',
value: 'normal',
id: '3.inputs.scheduler',
nodeId: '3',
node: {
id: '3',
name: 'KSampler',
type: 'KSampler',
},
},
{
type: 'number',
name: 'denoise',
value: 1,
id: '3.inputs.denoise',
nodeId: '3',
node: {
id: '3',
name: 'KSampler',
type: 'KSampler',
},
},
])

Expand All @@ -278,7 +308,11 @@ test('should return the correct inputs by node id', () => {
name: 'ckpt_name',
value: 'v1-5-pruned-emaonly.ckpt',
id: '4.inputs.ckpt_name',
nodeId: '4',
node: {
id: '4',
name: 'Load Checkpoint',
type: 'CheckpointLoaderSimple',
},
},
])

Expand All @@ -288,21 +322,33 @@ test('should return the correct inputs by node id', () => {
name: 'width',
value: 512,
id: '5.inputs.width',
nodeId: '5',
node: {
id: '5',
name: 'Empty Latent Image',
type: 'EmptyLatentImage',
},
},
{
type: 'number',
name: 'height',
value: 512,
id: '5.inputs.height',
nodeId: '5',
node: {
id: '5',
name: 'Empty Latent Image',
type: 'EmptyLatentImage',
},
},
{
type: 'number',
name: 'batch_size',
value: 1,
id: '5.inputs.batch_size',
nodeId: '5',
node: {
id: '5',
name: 'Empty Latent Image',
type: 'EmptyLatentImage',
},
},
])

Expand All @@ -313,7 +359,11 @@ test('should return the correct inputs by node id', () => {
value:
'beautiful scenery nature glass bottle landscape, , purple galaxy bottle,',
id: '6.inputs.text',
nodeId: '6',
node: {
id: '6',
name: 'CLIP Text Encode (Prompt)',
type: 'CLIPTextEncode',
},
},
])

Expand All @@ -329,7 +379,11 @@ test('should detect the correct positive and negative prompt inputs', () => {
expect(positivePromptInput).toEqual([
{
id: '6.inputs.text',
nodeId: '6',
node: {
id: '6',
name: 'CLIP Text Encode (Prompt)',
type: 'CLIPTextEncode',
},
name: 'text',
type: 'string',
value:
Expand All @@ -340,7 +394,11 @@ test('should detect the correct positive and negative prompt inputs', () => {
expect(negativePromptInput).toEqual([
{
id: '7.inputs.text',
nodeId: '7',
node: {
id: '7',
name: 'CLIP Text Encode (Prompt)',
type: 'CLIPTextEncode',
},
name: 'text',
type: 'string',
value: 'text, watermark',
Expand All @@ -364,7 +422,11 @@ test('should detect the correct positive and negative prompt inputs using clappe
expect(positivePromptInput).toEqual([
{
id: '6.inputs.text',
nodeId: '6',
node: {
id: '6',
name: 'CLIP Text Encode (Prompt)',
type: 'CLIPTextEncode',
},
type: 'string',
name: 'text',
value: '@clapper/prompt',
Expand All @@ -374,7 +436,11 @@ test('should detect the correct positive and negative prompt inputs using clappe
expect(negativePromptInput).toEqual([
{
id: '7.inputs.text',
nodeId: '7',
node: {
id: '7',
name: 'CLIP Text Encode (Prompt)',
type: 'CLIPTextEncode',
},
type: 'string',
name: 'text',
value: '@clapper/negative',
Expand All @@ -398,15 +464,23 @@ test('should correctly search workflow inputs', () => {
).toEqual([
{
id: '6.inputs.text',
nodeId: '6',
node: {
id: '6',
name: 'CLIP Text Encode (Prompt)',
type: 'CLIPTextEncode',
},
name: 'text',
type: 'string',
value:
'beautiful scenery nature glass bottle landscape, , purple galaxy bottle,',
},
{
id: '7.inputs.text',
nodeId: '7',
node: {
id: '7',
name: 'CLIP Text Encode (Prompt)',
type: 'CLIPTextEncode',
},
name: 'text',
type: 'string',
value: 'text, watermark',
Expand All @@ -421,7 +495,11 @@ test('should correctly search workflow inputs', () => {
).toEqual([
{
id: '6.inputs.text',
nodeId: '6',
node: {
id: '6',
name: 'CLIP Text Encode (Prompt)',
type: 'CLIPTextEncode',
},
name: 'text',
type: 'string',
value:
Expand All @@ -437,7 +515,11 @@ test('should correctly search workflow inputs', () => {
).toEqual([
{
id: '6.inputs.text',
nodeId: '6',
node: {
id: '6',
name: 'CLIP Text Encode (Prompt)',
type: 'CLIPTextEncode',
},
name: 'text',
type: 'string',
value:
Expand Down Expand Up @@ -472,6 +554,7 @@ test('should create the PromptBuilder', () => {
ClapperComfyUiInputIds.NEGATIVE_PROMPT,
[ClapperComfyUiInputIds.WIDTH]: ClapperComfyUiInputIds.WIDTH,
[ClapperComfyUiInputIds.HEIGHT]: ClapperComfyUiInputIds.HEIGHT,
[ClapperComfyUiInputIds.SEED]: ClapperComfyUiInputIds.SEED,
})
expect(promptBuilder.prompt).toEqual(workflowRaw)
})
Expand Down
Loading

0 comments on commit aa5b0b4

Please sign in to comment.