flux img2img controlnet channels error

[wen020]

Describe the bug

When I use flux's img2img controlnet for inference, a channel error occurs.

Reproduction

import numpy as np
import torch
import cv2
from PIL import Image
from diffusers.utils import load_image
from diffusers import FluxControlNetImg2ImgPipeline, FluxControlNetPipeline
from diffusers import FluxControlNetModel
from controlnet_aux import HEDdetector

base_model = "black-forest-labs/FLUX.1-dev"
controlnet_model = "Xlabs-AI/flux-controlnet-hed-diffusers"
controlnet = FluxControlNetModel.from_pretrained(
  controlnet_model,
  torch_dtype=torch.bfloat16,
  use_safetensors=True,
)
pipe = FluxControlNetImg2ImgPipeline.from_pretrained(
    base_model, controlnet=controlnet, torch_dtype=torch.bfloat16
)
pipe.load_lora_weights("./toonystarkKoreanWebtoonFlux_fluxLoraAlpha.safetensors")

pipe.enable_sequential_cpu_offload()

hed = HEDdetector.from_pretrained("lllyasviel/Annotators")

image_source = load_image("./03.jpeg")
control_image = hed(image_source)
control_image = control_image.resize(image_source.size)
if control_image.mode != 'RGB':
    control_image = control_image.convert('RGB')
control_image.save(f"./hed_03.png")

prompt = "bird, cool, futuristic"
image = pipe(
    prompt,
    image=image_source,
    control_image=control_image,
    control_guidance_start=0.2,
    control_guidance_end=0.8,
    controlnet_conditioning_scale=0.5,
    num_inference_steps=50,
    guidance_scale=6,
).images[0]
image.save("flux.png")

Logs

---------------------------------------------------------------------------
RuntimeError                              Traceback (most recent call last)
Cell In[13], line 2
      1 prompt = "bird, cool, futuristic"
----> 2 image = pipe(
      3     prompt,
      4     image=image_source,
      5     control_image=control_image,
      6     control_guidance_start=0.2,
      7     control_guidance_end=0.8,
      8     controlnet_conditioning_scale=0.5,
      9     num_inference_steps=50,
     10     guidance_scale=6,
     11 ).images[0]
     12 image.save("flux.png")

File /opt/conda/lib/python3.11/site-packages/torch/utils/_contextlib.py:115, in context_decorator.<locals>.decorate_context(*args, **kwargs)
    112 @functools.wraps(func)
    113 def decorate_context(*args, **kwargs):
    114     with ctx_factory():
--> 115         return func(*args, **kwargs)

File /opt/conda/lib/python3.11/site-packages/diffusers/pipelines/flux/pipeline_flux_controlnet_image_to_image.py:924, in FluxControlNetImg2ImgPipeline.__call__(self, prompt, prompt_2, image, control_image, height, width, strength, num_inference_steps, timesteps, guidance_scale, control_guidance_start, control_guidance_end, control_mode, controlnet_conditioning_scale, num_images_per_prompt, generator, latents, prompt_embeds, pooled_prompt_embeds, output_type, return_dict, joint_attention_kwargs, callback_on_step_end, callback_on_step_end_tensor_inputs, max_sequence_length)
    921         controlnet_cond_scale = controlnet_cond_scale[0]
    922     cond_scale = controlnet_cond_scale * controlnet_keep[i]
--> 924 controlnet_block_samples, controlnet_single_block_samples = self.controlnet(
    925     hidden_states=latents,
    926     controlnet_cond=control_image,
    927     controlnet_mode=control_mode,
    928     conditioning_scale=cond_scale,
    929     timestep=timestep / 1000,
    930     guidance=guidance,
    931     pooled_projections=pooled_prompt_embeds,
    932     encoder_hidden_states=prompt_embeds,
    933     txt_ids=text_ids,
    934     img_ids=latent_image_ids,
    935     joint_attention_kwargs=self.joint_attention_kwargs,
    936     return_dict=False,
    937 )
    939 guidance = (
    940     torch.tensor([guidance_scale], device=device) if self.transformer.config.guidance_embeds else None
    941 )
    942 guidance = guidance.expand(latents.shape[0]) if guidance is not None else None

File /opt/conda/lib/python3.11/site-packages/torch/nn/modules/module.py:1511, in Module._wrapped_call_impl(self, *args, **kwargs)
   1509     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1510 else:
-> 1511     return self._call_impl(*args, **kwargs)

File /opt/conda/lib/python3.11/site-packages/torch/nn/modules/module.py:1520, in Module._call_impl(self, *args, **kwargs)
   1515 # If we don't have any hooks, we want to skip the rest of the logic in
   1516 # this function, and just call forward.
   1517 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1518         or _global_backward_pre_hooks or _global_backward_hooks
   1519         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1520     return forward_call(*args, **kwargs)
   1522 try:
   1523     result = None

File /opt/conda/lib/python3.11/site-packages/accelerate/hooks.py:170, in add_hook_to_module.<locals>.new_forward(module, *args, **kwargs)
    168         output = module._old_forward(*args, **kwargs)
    169 else:
--> 170     output = module._old_forward(*args, **kwargs)
    171 return module._hf_hook.post_forward(module, output)

File /opt/conda/lib/python3.11/site-packages/diffusers/models/controlnets/controlnet_flux.py:281, in FluxControlNetModel.forward(self, hidden_states, controlnet_cond, controlnet_mode, conditioning_scale, encoder_hidden_states, pooled_projections, timestep, img_ids, txt_ids, guidance, joint_attention_kwargs, return_dict)
    278 hidden_states = self.x_embedder(hidden_states)
    280 if self.input_hint_block is not None:
--> 281     controlnet_cond = self.input_hint_block(controlnet_cond)
    282     batch_size, channels, height_pw, width_pw = controlnet_cond.shape
    283     height = height_pw // self.config.patch_size

File /opt/conda/lib/python3.11/site-packages/torch/nn/modules/module.py:1511, in Module._wrapped_call_impl(self, *args, **kwargs)
   1509     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1510 else:
-> 1511     return self._call_impl(*args, **kwargs)

File /opt/conda/lib/python3.11/site-packages/torch/nn/modules/module.py:1520, in Module._call_impl(self, *args, **kwargs)
   1515 # If we don't have any hooks, we want to skip the rest of the logic in
   1516 # this function, and just call forward.
   1517 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1518         or _global_backward_pre_hooks or _global_backward_hooks
   1519         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1520     return forward_call(*args, **kwargs)
   1522 try:
   1523     result = None

File /opt/conda/lib/python3.11/site-packages/diffusers/models/controlnets/controlnet.py:99, in ControlNetConditioningEmbedding.forward(self, conditioning)
     98 def forward(self, conditioning):
---> 99     embedding = self.conv_in(conditioning)
    100     embedding = F.silu(embedding)
    102     for block in self.blocks:

File /opt/conda/lib/python3.11/site-packages/torch/nn/modules/module.py:1511, in Module._wrapped_call_impl(self, *args, **kwargs)
   1509     return self._compiled_call_impl(*args, **kwargs)  # type: ignore[misc]
   1510 else:
-> 1511     return self._call_impl(*args, **kwargs)

File /opt/conda/lib/python3.11/site-packages/torch/nn/modules/module.py:1520, in Module._call_impl(self, *args, **kwargs)
   1515 # If we don't have any hooks, we want to skip the rest of the logic in
   1516 # this function, and just call forward.
   1517 if not (self._backward_hooks or self._backward_pre_hooks or self._forward_hooks or self._forward_pre_hooks
   1518         or _global_backward_pre_hooks or _global_backward_hooks
   1519         or _global_forward_hooks or _global_forward_pre_hooks):
-> 1520     return forward_call(*args, **kwargs)
   1522 try:
   1523     result = None

File /opt/conda/lib/python3.11/site-packages/accelerate/hooks.py:170, in add_hook_to_module.<locals>.new_forward(module, *args, **kwargs)
    168         output = module._old_forward(*args, **kwargs)
    169 else:
--> 170     output = module._old_forward(*args, **kwargs)
    171 return module._hf_hook.post_forward(module, output)

File /opt/conda/lib/python3.11/site-packages/torch/nn/modules/conv.py:460, in Conv2d.forward(self, input)
    459 def forward(self, input: Tensor) -> Tensor:
--> 460     return self._conv_forward(input, self.weight, self.bias)

File /opt/conda/lib/python3.11/site-packages/torch/nn/modules/conv.py:456, in Conv2d._conv_forward(self, input, weight, bias)
    452 if self.padding_mode != 'zeros':
    453     return F.conv2d(F.pad(input, self._reversed_padding_repeated_twice, mode=self.padding_mode),
    454                     weight, bias, self.stride,
    455                     _pair(0), self.dilation, self.groups)
--> 456 return F.conv2d(input, weight, bias, self.stride,
    457                 self.padding, self.dilation, self.groups)

RuntimeError: Given groups=1, weight of size [16, 3, 3, 3], expected input[1, 1, 4096, 64] to have 3 channels, but got 1 channels instead

System Info

latest diffusers

Who can help?

@yiyixuxu @sayakpaul

[deepak-lenka]

The error occurs because the control image has only 1 channel (grayscale) while the Flux ControlNet model expects a 3-channel (RGB) image, causing a channel mismatch during the image-to-image generation process.

[wen020]

I found that the image has 3 channels

[wen020]

I checked that the control image is 3-channel, but the problem still exists

[wen020]

@sayakpaul

[sayakpaul]

Please format the code snippet because it's incorrect. Once done, let us know

[wen020]

@sayakpaul

import numpy as np
import torch
import cv2
from PIL import Image
from diffusers.utils import load_image
from diffusers import FluxControlNetImg2ImgPipeline, FluxControlNetPipeline
from diffusers import FluxControlNetModel
from controlnet_aux import HEDdetector

base_model = "black-forest-labs/FLUX.1-dev"
controlnet_model = "Xlabs-AI/flux-controlnet-hed-diffusers"
controlnet = FluxControlNetModel.from_pretrained(
    controlnet_model,
    torch_dtype=torch.bfloat16,
    use_safetensors=True,
    )
pipe = FluxControlNetImg2ImgPipeline.from_pretrained(
    base_model, controlnet=controlnet, torch_dtype=torch.bfloat16
    )
pipe.load_lora_weights("./toonystarkKoreanWebtoonFlux_fluxLoraAlpha.safetensors")

pipe.enable_sequential_cpu_offload()

hed = HEDdetector.from_pretrained("lllyasviel/Annotators")

image_source = load_image("./bird.png")
control_image = hed(image_source)
control_image = control_image.resize(image_source.size)
if control_image.mode != 'RGB':
    control_image = control_image.convert('RGB')
    control_image.save(f"./hed.png")

prompt = "bird, cool, futuristic"
image = pipe(
    prompt,
    image=image_source,
    control_image=control_image,
    control_guidance_start=0.2,
    control_guidance_end=0.8,
    controlnet_conditioning_scale=0.5,
    num_inference_steps=50,
    guidance_scale=6,
    ).images[0]
image.save("flux.png")

bird.png

[github-actions]

This issue has been automatically marked as stale because it has not had recent activity. If you think this still needs to be addressed please comment on this thread.

Please note that issues that do not follow the contributing guidelines are likely to be ignored.

[GinoBelief]

@wen020 have you got a solution? I am having exactly the same issue.

[github-actions]

This issue has been automatically marked as stale because it has not had recent activity. If you think this still needs to be addressed please comment on this thread.

Please note that issues that do not follow the contributing guidelines are likely to be ignored.

[yiyixuxu]

the flux controlnet img2ing does not support x-lab controlnet yet, anyone interested in adding the support?
need to apply same change as in #9687

[BadrKebrit]

I will try to take care of this issue.