# install import glob import gradio as gr import os import subprocess if os.getenv('SYSTEM') == 'spaces': # subprocess.run('pip install pyembree'.split()) subprocess.run( 'pip install --no-index --no-cache-dir pytorch3d -f https://dl.fbaipublicfiles.com/pytorch3d/packaging/wheels/py38_cu116_pyt1130/download.html' .split() ) subprocess.run("python setup.py build_ext --inplace".split(), cwd="./lib/common/libmesh/") subprocess.run("python setup.py build_ext --inplace".split(), cwd="./lib/common/libvoxelize/") from apps.infer import generate_model, generate_video # running description = ''' # Unconstrained & Detailed Clothed Human Digitization (ECON + ControlNet) ### ECON: Explicit Clothed humans Optimized via Normal integration (CVPR 2023, Highlight)

  • Colab Notebook Google Colab
  • Blender Plugin Blender
  • Docker Image Docker
  • Windows Setup

Twitter Follow
#### Citation ``` @inproceedings{xiu2023econ, title = {{ECON: Explicit Clothed humans Optimized via Normal integration}}, author = {Xiu, Yuliang and Yang, Jinlong and Cao, Xu and Tzionas, Dimitrios and Black, Michael J.}, booktitle = {Proceedings of the IEEE/CVF Conference on Computer Vision and Pattern Recognition (CVPR)}, month = {June}, year = {2023}, } ```
More #### Acknowledgments: - [controlnet-openpose](https://huggingface.co/spaces/diffusers/controlnet-openpose) - [TEXTure](https://huggingface.co/spaces/TEXTurePaper/TEXTure) #### Image Credits * [Pinterest](https://www.pinterest.com/search/pins/?q=parkour&rs=sitelinks_searchbox) #### Related works * [ICON @ MPI-IS](https://icon.is.tue.mpg.de/) * [MonoPort @ USC](https://xiuyuliang.cn/monoport) * [Phorhum @ Google](https://phorhum.github.io/) * [PIFuHD @ Meta](https://shunsukesaito.github.io/PIFuHD/) * [PaMIR @ Tsinghua](http://www.liuyebin.com/pamir/pamir.html)

Generate pose & prompt-guided images / Upload photos / Use examples → Submit Image (~3min) → Generate Video (~3min)

''' from controlnet_aux import OpenposeDetector from diffusers import StableDiffusionControlNetPipeline, ControlNetModel from diffusers import UniPCMultistepScheduler import gradio as gr import torch import base64 from io import BytesIO from PIL import Image # live conditioning canvas_html = "" load_js = """ async () => { const url = "https://huggingface.co/datasets/radames/gradio-components/raw/main/pose-gradio.js" fetch(url) .then(res => res.text()) .then(text => { const script = document.createElement('script'); script.type = "module" script.src = URL.createObjectURL(new Blob([text], { type: 'application/javascript' })); document.head.appendChild(script); }); } """ get_js_image = """ async (image_in_img, prompt, image_file_live_opt, live_conditioning) => { const canvasEl = document.getElementById("canvas-root"); const data = canvasEl? canvasEl._data : null; return [image_in_img, prompt, image_file_live_opt, data] } """ # Constants low_threshold = 100 high_threshold = 200 default_step = 50 cached = False # Models pose_model = OpenposeDetector.from_pretrained("lllyasviel/ControlNet") controlnet = ControlNetModel.from_pretrained( "lllyasviel/sd-controlnet-openpose", torch_dtype=torch.float16 ) pipe = StableDiffusionControlNetPipeline.from_pretrained( "runwayml/stable-diffusion-v1-5", controlnet=controlnet, safety_checker=None, torch_dtype=torch.float16 ) pipe.scheduler = UniPCMultistepScheduler.from_config(pipe.scheduler.config) # This command loads the individual model components on GPU on-demand. So, we don't # need to explicitly call pipe.to("cuda"). pipe.enable_model_cpu_offload() # xformers pipe.enable_xformers_memory_efficient_attention() # Generator seed, generator = torch.manual_seed(0) hint_prompts = ''' Hints:
best quality, extremely detailed, solid color background, super detail, high detail, edge lighting, soft focus, light and dark contrast, 8k, edge lighting, 3d, c4d, blender, oc renderer, ultra high definition, 3d rendering ''' def get_pose(image): return pose_model(image) # def generate_texture(input_shape, text, seed, guidance_scale): # iface = gr.Interface.load("spaces/TEXTurePaper/TEXTure") # output_shape = iface(input_shape, text, seed, guidance_scale) # return output_shape def generate_images(image, prompt, image_file_live_opt='file', live_conditioning=None): if image is None and 'image' not in live_conditioning: raise gr.Error("Please provide an image") try: if image_file_live_opt == 'file': pose = get_pose(image) elif image_file_live_opt == 'webcam': base64_img = live_conditioning['image'] image_data = base64.b64decode(base64_img.split(',')[1]) pose = Image.open(BytesIO(image_data)).convert('RGB').resize((512, 512)) output = pipe( prompt, pose, generator=generator, num_images_per_prompt=3, num_inference_steps=20, ) all_outputs = [] all_outputs.append(pose) for image in output.images: all_outputs.append(image) return all_outputs, all_outputs except Exception as e: raise gr.Error(str(e)) def toggle(choice): if choice == "file": return gr.update(visible=True, value=None), gr.update(visible=False, value=None) elif choice == "webcam": return gr.update(visible=False, value=None), gr.update(visible=True, value=canvas_html) examples_pose = glob.glob('examples/pose/*') examples_cloth = glob.glob('examples/cloth/*') with gr.Blocks() as demo: gr.Markdown(description) out_lst = [] with gr.Row(): with gr.Column(): with gr.Row(): live_conditioning = gr.JSON(value={}, visible=False) with gr.Column(): image_file_live_opt = gr.Radio(["file", "webcam"], value="file", label="How would you like to upload your image?") with gr.Row(): image_in_img = gr.Image( source="upload", visible=True, type="pil", label="Image for Pose" ) canvas = gr.HTML(None, elem_id="canvas_html", visible=False) image_file_live_opt.change( fn=toggle, inputs=[image_file_live_opt], outputs=[image_in_img, canvas], queue=False ) prompt = gr.Textbox( label="Enter your prompt to synthesise the image", max_lines=10, placeholder="best quality, extremely detailed", ) gr.Markdown(hint_prompts) with gr.Column(): gallery = gr.Gallery(label="Generated Images").style(grid=[2], height="auto") gallery_cache = gr.State() gr.Markdown( '''
Click the target generated image for Reconstruction.
↓
''' ) inp = gr.Image(type="filepath", label="Input Image for Reconstruction") fitting_step = gr.inputs.Slider( 10, 100, step=10, label='Fitting steps (Slower yet Better-aligned SMPL-X)', default=default_step ) with gr.Row(): btn_sample = gr.Button("Generate Image") btn_submit = gr.Button("Submit Image (~3min)") btn_sample.click( fn=generate_images, inputs=[image_in_img, prompt, image_file_live_opt, live_conditioning], outputs=[gallery, gallery_cache], _js=get_js_image ) def get_select_index(cache, evt: gr.SelectData): return cache[evt.index] gallery.select( fn=get_select_index, inputs=[gallery_cache], outputs=[inp], ) with gr.Row(): gr.Examples( examples=list(examples_pose), inputs=[inp], cache_examples=cached, fn=None, outputs=None, label="Hard Pose Examples" ) gr.Examples( examples=list(examples_cloth), inputs=[inp], cache_examples=cached, fn=None, outputs=None, label="Loose Cloth Examples" ) out_vid = gr.Video(label="Shared on Twitter with #ECON") with gr.Column(): overlap_inp = gr.Image(type="filepath", label="Image Normal Overlap").style(height=400) out_final = gr.Model3D( clear_color=[0.0, 0.0, 0.0, 0.0], label="Clothed human", elem_id="avatar" ) out_smpl = gr.Model3D( clear_color=[0.0, 0.0, 0.0, 0.0], label="SMPL-X body", elem_id="avatar" ) vis_tensor_path = gr.State() with gr.Row(): btn_video = gr.Button("Generate Video (~3min)") out_lst = [out_smpl, out_final, overlap_inp, vis_tensor_path] btn_video.click( fn=generate_video, inputs=[vis_tensor_path], outputs=[out_vid], ) btn_submit.click(fn=generate_model, inputs=[inp, fitting_step], outputs=out_lst) demo.load(None, None, None, _js=load_js) if __name__ == "__main__": demo.queue(concurrency_count=1) demo.launch(debug=True, enable_queue=True)