Update app.py

app.py

@@ -13,13 +13,13 @@ import numpy as np
 from PIL import Image
 import torch
 from diffusers import AutoencoderKL, StableDiffusionXLPipeline, EulerAncestralDiscreteScheduler
+from transformers import CLIPTextModelWithProjection, CLIPTextModel
 from typing import Tuple
 import paramiko
 import datetime
 from gradio import themes
 from image_gen_aux import UpscaleWithModel
 from ip_adapter import IPAdapterXL
-
 from huggingface_hub import snapshot_download
 
 torch.backends.cuda.matmul.allow_tf32 = False
@@ -28,7 +28,7 @@ torch.backends.cuda.matmul.allow_fp16_reduced_precision_reduction = False
 torch.backends.cudnn.allow_tf32 = False
 torch.backends.cudnn.deterministic = False
 torch.backends.cudnn.benchmark = False
-torch.backends.cuda.preferred_blas_library="cublas"
+#torch.backends.cuda.preferred_blas_library="cublas"
 # torch.backends.cuda.preferred_linalg_library="cusolver"
 
 torch.set_float32_matmul_precision("highest")
@@ -92,7 +92,6 @@ local_folder = os.path.join(local_repo_path, subfolder)
 local_folder2 = os.path.join(local_repo_path, subfolder2) # Path to the ip_adapter dir
 ip_ckpt = os.path.join(local_folder2, "ip-adapter_sdxl_vit-h.bin") # Correct path
 
-#sampling_schedule = AysSchedules["StableDiffusionXLTimesteps"]
 upscaler = UpscaleWithModel.from_pretrained("Kim2091/ClearRealityV1").to(torch.device("cuda:0"))
 
 def apply_style(style_name: str, positive: str, negative: str = "") -> Tuple[str, str]:
@@ -106,20 +105,7 @@ def apply_style(style_name: str, positive: str, negative: str = "") -> Tuple[str
 
 def load_and_prepare_model():
     #vae = AutoencoderKL.from_pretrained("ford442/sdxl-vae-bf16", safety_checker=None)
-    vaeX = AutoencoderKL.from_pretrained("stabilityai/sdxl-vae", safety_checker=None,use_safetensors=False, token=HF_TOKEN)
-    #vae = AutoencoderKL.from_pretrained('cross-attention/asymmetric-autoencoder-kl-x-2',use_safetensors=False)
-    #vae = AutoencoderKL.from_single_file('https://huggingface.co/ford442/sdxl-vae-bf16/mySLR/myslrVAE_v10.safetensors')
-    #vaeX = AutoencoderKL.from_pretrained("stabilityai/sd-vae-ft-mse",use_safetensors=True)
-    #vaeX = AutoencoderKL.from_pretrained('ford442/Juggernaut-XI-v11-fp32',subfolder='vae')   #  ,use_safetensors=True  FAILS
-    #vaeX = AutoencoderKL.from_pretrained('ford442/RealVisXL_V5.0_FP64',subfolder='vae').to(torch.bfloat16)   #  ,use_safetensors=True  FAILS
-    #unetX = UNet2DConditionModel.from_pretrained('ford442/RealVisXL_V5.0_BF16',subfolder='unet').to(torch.bfloat16)   #  ,use_safetensors=True  FAILS
-  #  vae = AutoencoderKL.from_pretrained("BeastHF/MyBack_SDXL_Juggernaut_XL_VAE/MyBack_SDXL_Juggernaut_XL_VAE_V10(version_X).safetensors",safety_checker=None).to(torch.bfloat16)
-    #sched = EulerAncestralDiscreteScheduler.from_pretrained("SG161222/RealVisXL_V5.0", subfolder='scheduler',beta_schedule="scaled_linear", steps_offset=1,timestep_spacing="trailing"))
-    #sched = EulerAncestralDiscreteScheduler.from_pretrained("SG161222/RealVisXL_V5.0", subfolder='scheduler', steps_offset=1,timestep_spacing="trailing")
-    sched = EulerAncestralDiscreteScheduler.from_pretrained('ford442/RealVisXL_V5.0_BF16', subfolder='scheduler',beta_schedule="scaled_linear", token=HF_TOKEN) #, beta_start=0.00085, beta_end=0.012, steps_offset=1,use_karras_sigmas=True, token=HF_TOKEN)
-    #sched = EulerAncestralDiscreteScheduler.from_pretrained('ford442/RealVisXL_V5.0_BF16', subfolder='scheduler',beta_schedule="scaled_linear")
-    #pipeX = StableDiffusionXLPipeline.from_pretrained("SG161222/RealVisXL_V5.0").to(torch.bfloat16)
-    #pipeX = StableDiffusionXLPipeline.from_pretrained("ford442/Juggernaut-XI-v11-fp32",use_safetensors=True)
+    vaeX = AutoencoderKL.from_pretrained("stabilityai/sdxl-vae", safety_checker=None, use_safetensors=False, low_cpu_mem_usage=False, torch_dtype=torch.float32, token=True) #.to(device).to(torch.bfloat16) #.to(device=device, dtype=torch.bfloat16)
     pipe = StableDiffusionXLPipeline.from_pretrained(
         'ford442/RealVisXL_V5.0_BF16',
        #'ford442/Juggernaut-XI-v11-fp32',
@@ -130,22 +116,10 @@ def load_and_prepare_model():
        # custom_pipeline="lpw_stable_diffusion_xl",
         #use_safetensors=True,
         token=HF_TOKEN,
-        #  vae=AutoencoderKL.from_pretrained("BeastHF/MyBack_SDXL_Juggernaut_XL_VAE/MyBack_SDXL_Juggernaut_XL_VAE_V10(version_X).safetensors",repo_type='model',safety_checker=None),
-     #   vae=AutoencoderKL.from_pretrained("stabilityai/sdxl-vae",repo_type='model',safety_checker=None, torch_dtype=torch.float32),
-       # vae=AutoencoderKL.from_pretrained("ford442/sdxl-vae-bf16",repo_type='model',safety_checker=None),
-        #vae=vae,
-        #unet=pipeX.unet,
-        #scheduler = sched,
-      #  scheduler = EulerAncestralDiscreteScheduler.from_config(pipeX.scheduler.config, beta_schedule="scaled_linear", beta_start=0.00085, beta_end=0.012, steps_offset=1)
-        #scheduler=EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config, beta_schedule="scaled_linear", beta_start=0.00085, beta_end=0.012, steps_offset =1)
+        text_encoder=None,
+        text_encoder_2=None,
+        vae=None,
     )
-    #pipe.vae = AsymmetricAutoencoderKL.from_pretrained('cross-attention/asymmetric-autoencoder-kl-x-2').to(torch.bfloat16)   #  ,use_safetensors=True  FAILS
-            
-    #pipe.vae = AutoencoderKL.from_pretrained('ford442/Juggernaut-XI-v11-fp32',subfolder='vae')   #  ,use_safetensors=True  FAILS
-    
-    #pipe.vae = AutoencoderKL.from_pretrained('stabilityai/sdxl-vae-bf16',subfolder='vae')  
-    #pipe.vae = AutoencoderKL.from_pretrained('stabilityai/sdxl-vae',subfolder='vae',force_upcast=False,scaling_factor= 0.182158767676)  
-    #pipe.vae.to(torch.bfloat16)
     
     '''
     scaling_factor (`float`, *optional*, defaults to 0.18215):
@@ -162,30 +136,13 @@ def load_and_prepare_model():
         
     '''
     
-    #sched = EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config, beta_schedule="scaled_linear",use_karras_sigmas=True, algorithm_type="dpmsolver++")
-    #pipe.scheduler = EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config, beta_schedule="scaled_linear", beta_start=0.00085, beta_end=0.012, steps_offset=1)
-    #pipe.scheduler = DPMSolverMultistepScheduler.from_pretrained('SG161222/RealVisXL_V5.0', subfolder='scheduler', algorithm_type='sde-dpmsolver++')
+    pipe.to(device=device, dtype=torch.bfloat16)
     pipe.vae = vaeX #.to(torch.bfloat16)
-    #pipe.unet = unetX
-    
     pipe.vae.do_resize=False
     #pipe.vae.do_rescale=False
     #pipe.vae.do_convert_rgb=True
-    pipe.vae.vae_scale_factor=8
-
-    pipe.scheduler = sched
-    #pipe.vae=vae.to(torch.bfloat16)
-    #pipe.unet=pipeX.unet
-    #pipe.scheduler=EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config, beta_schedule="scaled_linear", beta_start=0.00085, beta_end=0.012, steps_offset=1)
-    #pipe.scheduler=EulerAncestralDiscreteScheduler.from_pretrained('ford442/RealVisXL_V5.0_BF16', subfolder='scheduler',beta_schedule="scaled_linear")
-
-    pipe.to(device=device, dtype=torch.bfloat16)
-    #pipe.to(torch.bfloat16)
-    #apply_hidiffusion(pipe)
-
-    #pipe.unet.set_default_attn_processor()
+    pipe.vae.vae_scale_factor=8    #pipe.unet.set_default_attn_processor()
     pipe.vae.set_default_attn_processor()
-    
     print(f'Pipeline: ')
     #print(f'_optional_components: {pipe._optional_components}')
     #print(f'watermark: {pipe.watermark}')
@@ -195,19 +152,14 @@ def load_and_prepare_model():
     #print(f'UNET: {pipe.unet}')
     pipe.watermark=None
     pipe.safety_checker=None  
-    #pipe.to(torch.device("cuda:0"))
-    #pipe.vae.to(torch.bfloat16)
-    #pipe.to(device, torch.bfloat16)
-    #del pipeX
-    #sched = EulerAncestralDiscreteScheduler.from_config(pipe.scheduler.config, beta_schedule="scaled_linear", algorithm_type="dpmsolver++")
-    #sched = DPMSolverMultistepScheduler.from_config(pipe.scheduler.config, beta_schedule="linear", algorithm_type="dpmsolver++")
-    #sched = DDIMScheduler.from_config(pipe.scheduler.config)
     return pipe
     
 # Preload and compile both models
 pipe = load_and_prepare_model()
 ip_model = IPAdapterXL(pipe, local_folder, ip_ckpt, device)
-
+text_encoder=CLIPTextModel.from_pretrained('ford442/RealVisXL_V5.0_BF16', subfolder='text_encoder',token=True).to(device=device, dtype=torch.bfloat16)
+text_encoder_2=CLIPTextModelWithProjection.from_pretrained('ford442/RealVisXL_V5.0_BF16', subfolder='text_encoder_2',token=True).to(device=device, dtype=torch.bfloat16)
+   
 MAX_SEED = np.iinfo(np.int32).max
 
 neg_prompt_2 = " 'non-photorealistic':1.5, 'unrealistic skin','unattractive face':1.3, 'low quality':1.1, ('dull color scheme', 'dull colors', 'digital noise':1.2),'amateurish', 'poorly drawn face':1.3, 'poorly drawn', 'distorted face', 'low resolution', 'simplistic' "
@@ -270,6 +222,8 @@ def generate_30(
     samples=1,
     progress=gr.Progress(track_tqdm=True)  # Add progress as a keyword argument
 ):
+    pipe.text_encoder=text_encoder
+    pipe.text_encoder_2=text_encoder_2
     seed = random.randint(0, MAX_SEED)
     generator = torch.Generator(device='cuda').manual_seed(seed)
     if latent_file is not None:  # Check if a latent file is provided
@@ -298,7 +252,6 @@ def generate_30(
         filename= f'rv_IP_{timestamp}.png'
         print("-- using image file --")
         print('-- generating image --')
-        #with torch.no_grad():
         sd_image = ip_model.generate(
                 pil_image_1=sd_image_a,
                 pil_image_2=sd_image_b,
@@ -322,7 +275,7 @@ def generate_30(
         sd_image[0].save(filename,optimize=False,compress_level=0)
         upload_to_ftp(filename)
         uploadNote(prompt,num_inference_steps,guidance_scale,timestamp)
-        #torch.setfloat32_matmul_precision("medium")
+        torch.setfloat32_matmul_precision("medium")
         with torch.no_grad():
             upscale = upscaler(sd_image, tiling=True, tile_width=256, tile_height=256)
         downscale1 = upscale.resize((upscale.width // 4, upscale.height // 4), Image.LANCZOS)
@@ -359,6 +312,8 @@ def generate_60(
     samples=1,
     progress=gr.Progress(track_tqdm=True)  # Add progress as a keyword argument
 ):
+    pipe.text_encoder=text_encoder
+    pipe.text_encoder_2=text_encoder_2
     seed = random.randint(0, MAX_SEED)
     generator = torch.Generator(device='cuda').manual_seed(seed)
     if latent_file is not None:  # Check if a latent file is provided
@@ -387,7 +342,6 @@ def generate_60(
         filename= f'rv_IP_{timestamp}.png'
         print("-- using image file --")
         print('-- generating image --')
-        #with torch.no_grad():
         sd_image = ip_model.generate(
                 pil_image_1=sd_image_a,
                 pil_image_2=sd_image_b,
@@ -411,7 +365,7 @@ def generate_60(
         sd_image[0].save(filename,optimize=False,compress_level=0)
         upload_to_ftp(filename)
         uploadNote(prompt,num_inference_steps,guidance_scale,timestamp)
-        #torch.setfloat32_matmul_precision("medium")
+        torch.setfloat32_matmul_precision("medium")
         with torch.no_grad():
             upscale = upscaler(sd_image, tiling=True, tile_width=256, tile_height=256)
         downscale1 = upscale.resize((upscale.width // 4, upscale.height // 4), Image.LANCZOS)
@@ -448,6 +402,8 @@ def generate_90(
     samples=1,
     progress=gr.Progress(track_tqdm=True)  # Add progress as a keyword argument
 ):
+    pipe.text_encoder=text_encoder
+    pipe.text_encoder_2=text_encoder_2
     seed = random.randint(0, MAX_SEED)
     generator = torch.Generator(device='cuda').manual_seed(seed)
     if latent_file is not None:  # Check if a latent file is provided
@@ -500,7 +456,7 @@ def generate_90(
         sd_image[0].save(filename,optimize=False,compress_level=0)
         upload_to_ftp(filename)
         uploadNote(prompt,num_inference_steps,guidance_scale,timestamp)
-        #torch.setfloat32_matmul_precision("medium")
+        torch.setfloat32_matmul_precision("medium")
         with torch.no_grad():
             upscale = upscaler(sd_image, tiling=True, tile_width=256, tile_height=256)
         downscale1 = upscale.resize((upscale.width // 4, upscale.height // 4), Image.LANCZOS)