Using multiple ControlNets at the same time error
Hi, the documentation references that you can use multiple controlnets at the same time passing them as a list.
Is there any example about this? Thank you in advance, this controlnet model is amazing!
Example:
self.controlnet_canny = SD3ControlNetModel.from_pretrained("InstantX/SD3-Controlnet-Canny", torch_dtype=torch.float16)
self.controlnet_impainting = SD3ControlNetModel.from_pretrained("alimama-creative/SD3-Controlnet-Inpainting", use_safetensors=True, extra_conditioning_channels=1)
self.pipe = StableDiffusion3ControlNetInpaintingPipeline.from_pretrained(
"stabilityai/stable-diffusion-3-medium-diffusers",
controlnet=[self.controlnet_impainting,self.controlnet_canny],
torch_dtype=torch.float16,
)
Documentation reference in the source code:
class StableDiffusion3ControlNetInpaintingPipeline(DiffusionPipeline, SD3LoraLoaderMixin, FromSingleFileMixin):
r"""
Args:
(...)
controlnet ([`SD3ControlNetModel`] or `List[SD3ControlNetModel]` or [`SD3MultiControlNetModel`]):
Provides additional conditioning to the `unet` during the denoising process. If you set multiple
ControlNets as a list, the outputs from each ControlNet are added together to create one combined
additional conditioning.
"""
Errror I get given the example:
AttributeError Traceback (most recent call last)
Cell In[12], line 1
----> 1 stable_diffussion_model = SD_wrapper("mps")
Cell In[11], line 21
18 self.controlnet_canny = SD3ControlNetModel.from_pretrained("InstantX/SD3-Controlnet-Canny", torch_dtype=torch.float16)
19 self.controlnet_impainting = SD3ControlNetModel.from_pretrained("alimama-creative/SD3-Controlnet-Inpainting", use_safetensors=True, extra_conditioning_channels=1)
---> 21 self.pipe = StableDiffusion3ControlNetInpaintingPipeline.from_pretrained(
22 "stabilityai/stable-diffusion-3-medium-diffusers",
23 controlnet=[self.controlnet_impainting,self.controlnet_canny],
24 torch_dtype=torch.float16,
25 )
27 self.generator = torch.Generator(device=self.device).manual_seed(manual_seed)
29 self.pipe.text_encoder.to(torch.float16)
File ~/Documents/Projects/StableDiffControlnet_test/.venv/lib/python3.10/site-packages/huggingface_hub/utils/_validators.py:114, in validate_hf_hub_args.._inner_fn(*args, **kwargs)
111 if check_use_auth_token:
112 kwargs = smoothly_deprecate_use_auth_token(fn_name=fn.name, has_token=has_token, kwargs=kwargs)
--> 114 return fn(*args, **kwargs)
File ~/Documents/Projects/StableDiffControlnet_test/.venv/lib/python3.10/site-packages/diffusers/pipelines/pipeline_utils.py:976, in DiffusionPipeline.from_pretrained(cls, pretrained_model_name_or_path, **kwargs)
971 raise ValueError(
972 f"Pipeline {pipeline_class} expected {expected_modules}, but only {passed_modules} were passed."
973 )
975 # 10. Instantiate the pipeline
--> 976 model = pipeline_class(**init_kwargs)
978 # 11. Save where the model was instantiated from
979 model.register_to_config(_name_or_path=pretrained_model_name_or_path)
File ~/Documents/Projects/StableDiffControlnet_test/.venv/lib/python3.10/site-packages/diffusers/pipelines/controlnet_sd3/pipeline_stable_diffusion_3_controlnet_inpainting.py:221, in StableDiffusion3ControlNetInpaintingPipeline.init(self, transformer, scheduler, vae, text_encoder, tokenizer, text_encoder_2, tokenizer_2, text_encoder_3, tokenizer_3, controlnet)
204 def init(
205 self,
206 transformer: SD3Transformer2DModel,
(...)
217 ],
218 ):
219 super().init()
--> 221 self.register_modules(
222 vae=vae,
223 text_encoder=text_encoder,
224 text_encoder_2=text_encoder_2,
225 text_encoder_3=text_encoder_3,
226 tokenizer=tokenizer,
227 tokenizer_2=tokenizer_2,
228 tokenizer_3=tokenizer_3,
229 transformer=transformer,
230 scheduler=scheduler,
231 controlnet=controlnet,
232 )
233 self.vae_scale_factor = (
234 2 ** (len(self.vae.config.block_out_channels) - 1) if hasattr(self, "vae") and self.vae is not None else 8
235 )
236 self.image_processor = VaeImageProcessor(
237 vae_scale_factor=self.vae_scale_factor, do_resize=True, do_convert_rgb=True, do_normalize=True
238 )
File ~/Documents/Projects/StableDiffControlnet_test/.venv/lib/python3.10/site-packages/diffusers/pipelines/pipeline_utils.py:165, in DiffusionPipeline.register_modules(self, **kwargs)
163 register_dict = {name: (None, None)}
164 else:
--> 165 library, class_name = _fetch_class_library_tuple(module)
166 register_dict = {name: (library, class_name)}
168 # save model index config
File ~/Documents/Projects/StableDiffControlnet_test/.venv/lib/python3.10/site-packages/diffusers/pipelines/pipeline_loading_utils.py:756, in _fetch_class_library_tuple(module)
754 # register the config from the original module, not the dynamo compiled one
755 not_compiled_module = _unwrap_model(module)
--> 756 library = not_compiled_module.module.split(".")[0]
758 # check if the module is a pipeline module
759 module_path_items = not_compiled_module.module.split(".")
AttributeError: 'list' object has no attribute 'module'