Update README.md
Browse files
README.md
CHANGED
@@ -39,7 +39,48 @@ This is the model card of a 🤗 transformers model that has been pushed on the
|
|
39 |
|
40 |
### Direct Use
|
41 |
|
42 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
43 |
|
44 |
[More Information Needed]
|
45 |
|
|
|
39 |
|
40 |
### Direct Use
|
41 |
|
42 |
+
```python
|
43 |
+
from transformers import Qwen2VLForConditionalGeneration, AutoTokenizer, AutoProcessor
|
44 |
+
from qwen_vl_utils import process_vision_info
|
45 |
+
processor = AutoProcessor.from_pretrained("davanstrien/qwen2-vl-finetune")
|
46 |
+
model = Qwen2VLForConditionalGeneration.from_pretrained("davanstrien/qwen2-vl-finetune", torch_dtype="auto", device_map="auto")
|
47 |
+
messages = [
|
48 |
+
{
|
49 |
+
"role": "user",
|
50 |
+
"content": [
|
51 |
+
{
|
52 |
+
"type": "image",
|
53 |
+
"image": "image.jpg",
|
54 |
+
},
|
55 |
+
{"type": "text", "text": "<GENERATE_QUERY>"},
|
56 |
+
],
|
57 |
+
}
|
58 |
+
]
|
59 |
+
|
60 |
+
# Preparation for inference
|
61 |
+
text = processor.apply_chat_template(
|
62 |
+
messages, tokenize=False, add_generation_prompt=True
|
63 |
+
)
|
64 |
+
image_inputs, video_inputs = process_vision_info(messages)
|
65 |
+
inputs = processor(
|
66 |
+
text=[text],
|
67 |
+
images=image_inputs,
|
68 |
+
videos=video_inputs,
|
69 |
+
padding=True,
|
70 |
+
return_tensors="pt",
|
71 |
+
)
|
72 |
+
inputs = inputs.to("cuda")
|
73 |
+
|
74 |
+
# Inference: Generation of the output
|
75 |
+
generated_ids = model.generate(**inputs, max_new_tokens=128)
|
76 |
+
generated_ids_trimmed = [
|
77 |
+
out_ids[len(in_ids) :] for in_ids, out_ids in zip(inputs.input_ids, generated_ids)
|
78 |
+
]
|
79 |
+
output_text = processor.batch_decode(
|
80 |
+
generated_ids_trimmed, skip_special_tokens=True, clean_up_tokenization_spaces=False
|
81 |
+
)
|
82 |
+
print(output_text)
|
83 |
+
```
|
84 |
|
85 |
[More Information Needed]
|
86 |
|