|
--- |
|
base_model: stabilityai/stable-diffusion-xl-base-1.0 |
|
library_name: diffusers |
|
license: openrail++ |
|
tags: |
|
- text-to-image |
|
- text-to-image |
|
- diffusers-training |
|
- diffusers |
|
- lora |
|
- template:sd-lora |
|
- stable-diffusion-xl |
|
- stable-diffusion-xl-diffusers |
|
instance_prompt: <leaf microstructure> |
|
widget: [] |
|
--- |
|
|
|
# SDXL Fine-tuned with Leaf Images |
|
|
|
## Model description |
|
|
|
These are LoRA adaption weights for the SDXL-base-1.0 model. |
|
|
|
## Trigger keywords |
|
|
|
The following image were used during fine-tuning using the keyword \<leaf microstructure\>: |
|
|
|
 |
|
|
|
You should use <leaf microstructure> to trigger the image generation. |
|
|
|
## How to use |
|
|
|
Defining some helper functions: |
|
|
|
```python |
|
from diffusers import DiffusionPipeline |
|
import torch |
|
import os |
|
from datetime import datetime |
|
from PIL import Image |
|
|
|
def generate_filename(base_name, extension=".png"): |
|
timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") |
|
return f"{base_name}_{timestamp}{extension}" |
|
|
|
def save_image(image, directory, base_name="image_grid"): |
|
|
|
filename = generate_filename(base_name) |
|
file_path = os.path.join(directory, filename) |
|
image.save(file_path) |
|
print(f"Image saved as {file_path}") |
|
|
|
def image_grid(imgs, rows, cols, save=True, save_dir='generated_images', base_name="image_grid", |
|
save_individual_files=False): |
|
|
|
if not os.path.exists(save_dir): |
|
os.makedirs(save_dir) |
|
|
|
assert len(imgs) == rows * cols |
|
|
|
w, h = imgs[0].size |
|
grid = Image.new('RGB', size=(cols * w, rows * h)) |
|
grid_w, grid_h = grid.size |
|
|
|
for i, img in enumerate(imgs): |
|
grid.paste(img, box=(i % cols * w, i // cols * h)) |
|
if save_individual_files: |
|
save_image(img, save_dir, base_name=base_name+f'_{i}-of-{len(imgs)}_') |
|
|
|
if save and save_dir: |
|
save_image(grid, save_dir, base_name) |
|
|
|
return grid |
|
``` |
|
|
|
### Text-to-image |
|
|
|
Model loading: |
|
|
|
```python |
|
|
|
import torch |
|
from diffusers import DiffusionPipeline, AutoencoderKL |
|
|
|
repo_id='lamm-mit/SDXL-leaf-inspired' |
|
|
|
vae = AutoencoderKL.from_pretrained("madebyollin/sdxl-vae-fp16-fix", torch_dtype=torch.float16) |
|
base = DiffusionPipeline.from_pretrained( |
|
"stabilityai/stable-diffusion-xl-base-1.0", |
|
vae=vae, |
|
torch_dtype=torch.float16, |
|
variant="fp16", |
|
use_safetensors=True |
|
) |
|
base.load_lora_weights(repo_id) |
|
_ = base.to("cuda") |
|
|
|
refiner = DiffusionPipeline.from_pretrained( |
|
"stabilityai/stable-diffusion-xl-refiner-1.0", |
|
text_encoder_2=base.text_encoder_2, |
|
vae=base.vae, |
|
torch_dtype=torch.float16, |
|
use_safetensors=True, |
|
variant="fp16", |
|
) |
|
refiner.to("cuda") |
|
``` |
|
|
|
Image generation: |
|
|
|
```python |
|
prompt = "a vase that resembles a <leaf microstructure>, high quality" |
|
|
|
num_samples = 4 |
|
num_rows = 4 |
|
guidance_scale = 15 |
|
|
|
all_images = [] |
|
|
|
for _ in range(num_rows): |
|
# Define how many steps and what % of steps to be run on each experts (80/20) |
|
n_steps = 25 |
|
high_noise_frac = 0.8 |
|
|
|
# run both experts |
|
image = base( |
|
prompt=prompt, |
|
num_inference_steps=n_steps, guidance_scale=guidance_scale, |
|
denoising_end=high_noise_frac,num_images_per_prompt=num_samples, |
|
output_type="latent", |
|
).images |
|
image = refiner( |
|
prompt=prompt, |
|
num_inference_steps=n_steps, guidance_scale=guidance_scale, |
|
denoising_start=high_noise_frac,num_images_per_prompt=num_samples, |
|
image=image, |
|
).images |
|
|
|
all_images.extend(image) |
|
|
|
grid = image_grid(all_images, num_rows, num_samples, |
|
save_individual_files=True, |
|
) |
|
grid |
|
``` |
|
|
|
|
|
 |
|
|
|
|
|
|
