[ctrlnet] fix gradio (#229)

* [ctrlnet] fix gradio

* fix controlnet bug;

Signed-off-by: lawrence-cj <[email protected]>

---------

Signed-off-by: lawrence-cj <[email protected]>
Co-authored-by: lawrence-cj <[email protected]>

Author: yujincheng08

app/sana_controlnet_pipeline.py

@@ -17,6 +17,7 @@
 from dataclasses import dataclass, field
 from typing import Optional, Tuple
 
+import cv2
 import numpy as np
 import pyrallis
 import torch
@@ -282,7 +283,6 @@ def forward(
                     )
                 else:
                     z = latents.to(self.device)
-                model_kwargs = dict(data_info={"img_hw": hw, "aspect_ratio": ar}, mask=emb_masks)
 
                 # control signal
                 if isinstance(ref_image, str):
@@ -302,7 +302,10 @@ def forward(
                     self.config.vae.vae_type, self.vae, control_signal, self.config.vae.sample_posterior, self.device
                 )
 
-                model_kwargs["control_signal"] = control_signal_latent
+                model_kwargs = dict(
+                    data_info={"img_hw": hw, "aspect_ratio": ar, "control_signal": control_signal_latent},
+                    mask=emb_masks,
+                )
 
                 if self.vis_sampler == "flow_euler":
                     flow_solver = FlowEuler(

configs/sana_controlnet_config/Sana_1600M_1024px_controlnet_bf16.yaml

@@ -40,6 +40,7 @@ vae:
   vae_latent_dim: 32
   vae_downsample_rate: 32
   sample_posterior: true
+  weight_dtype: bf16
 # text encoder
 text_encoder:
   text_encoder_name: gemma-2-2b-it