Spaces:
Sleeping
Sleeping
File size: 3,800 Bytes
3552447 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 |
import gradio as gr
import open_clip
import numpy as np
import torch
import pandas as pd
import os
open_clip_model, _, preprocess = open_clip.create_model_and_transforms(
'ViT-B-32',
pretrained='./open_clip_pytorch_model.bin')
debiased_model, _, _ = open_clip.create_model_and_transforms(
'ViT-B-32',
pretrained='./debiased_openclip.pt')
open_clip_model.eval()
debiased_model.eval()
tokenizer = open_clip.get_tokenizer('ViT-B-32')
def get_clip_scores(images, candidates, w=1):
images = images / np.sqrt(np.sum(images**2, axis=1, keepdims=True))
candidates = candidates / np.sqrt(np.sum(candidates**2, axis=1, keepdims=True))
per = w*np.clip(np.sum(images * candidates, axis=1), 0, None)
return per
def predict(text1, text2, input_img):
with torch.no_grad():
image = preprocess(input_img)
image= image.unsqueeze(0)
image_features = open_clip_model.encode_image(image)
debiased_image_features = debiased_model.encode_image(image)
texts = tokenizer([text1])
texts2 = tokenizer([text2])
text_features = open_clip_model.encode_text(texts)
debiased_text_features = debiased_model.encode_text(texts)
# print(image_features.size(), text_features.size())
# print(debiased_image_features.size(), debiased_text_features.size())
score = get_clip_scores(image_features.numpy(), text_features.numpy())
debiased_score = get_clip_scores(debiased_image_features.numpy(), debiased_text_features.numpy())
text_features2 = open_clip_model.encode_text(texts2)
debiased_text_features2 = debiased_model.encode_text(texts2)
score2 = get_clip_scores(image_features.numpy(), text_features2.numpy())
debiased_score2 = get_clip_scores(debiased_image_features.numpy(), debiased_text_features2.numpy())
print(score, score2)
data = {'label': ["OpenCLIP for text1", "Debiased CLIP for text1",
"OpenCLIP for text2", "Debiased CLIP for text2"
],
'score': [score[0], debiased_score[0], score2[0], debiased_score2[0]]
}
print(pd.DataFrame.from_dict(data))
return pd.DataFrame.from_dict(data)
# gradio_app = gr.Interface(
# predict,
# inputs=["text", "text",
# gr.Image(label="Select Image", sources=['upload', 'webcam'], type="pil"),
# ],
# outputs=gr.BarPlot(x="label",
# y="score",
# title="CLIP Score and Debiased Score",
# vertical=False,
# x_title=None
# ),
# title="Parrot Bias in CLIP!! (Both CLIP models are ViT-B-32)",
# )
with gr.Blocks() as demo:
gr.Markdown("# Parrot Bias in CLIP!! (Both CLIP models are ViT-B-32)")
with gr.Row():
im = gr.Image(label="Select Image",
sources=['upload', 'webcam'],
type="pil",
height=450)
with gr.Row():
txt_1 = gr.Textbox(label="Input Text")
txt_2 = gr.Textbox(label="Input Text 2")
bar = gr.BarPlot(x="label", y="score",
title="CLIP Score and Debiased Score",
vertical=False, x_title=None)
btn = gr.Button(value="Submit")
btn.click(predict, inputs=[txt_1, txt_2, im], outputs=[bar])
gr.Markdown("## Examples (from https://joaanna.github.io/disentangling_spelling_in_clip/)")
gr.Examples(
[["A mug cup", "An iPad",os.path.join(os.path.dirname(__file__), "examples/IMG_2938.jpg")],
["A hat", "bad",os.path.join(os.path.dirname(__file__), "examples/IMG_3066.jpg")]],
[txt_1, txt_2, im],
fn=predict,
outputs=bar,
cache_examples=True,
)
if __name__ == "__main__":
demo.launch(show_api=False,share=True) |