Update README.md

README.md

@@ -31,8 +31,11 @@ Built upon Ovis1.5, **Ovis1.6** further enhances high-resolution image processin
 | Ovis MLLMs        | ViT         | LLM                |                          Model Weights                          | Demo                                                             |
 |:------------------|:-----------:|:------------------:|:---------------------------------------------------------------:|:----------------------------------------------------------------:|
 | Ovis1.6-Gemma2-9B | Siglip-400M | Gemma2-9B-It       | [Huggingface](https://huggingface.co/AIDC-AI/Ovis1.6-Gemma2-9B) | [Space](https://huggingface.co/spaces/AIDC-AI/Ovis1.6-Gemma2-9B) |
+| Ovis1.6-Llama3.2-3B | Siglip-400M | Llama-3.2-3B-Instruct       | [Huggingface](https://huggingface.co/AIDC-AI/Ovis1.6-Llama3.2-3B) | [Space](https://huggingface.co/spaces/AIDC-AI/Ovis1.6-Llama3.2-3B) |
+| Ovis1.6-Gemma2-9B-GPTQ-Int4 | Siglip-400M | Gemma2-9B-It       | [Huggingface](https://huggingface.co/AIDC-AI/Ovis1.6-Gemma2-9B-GPTQ-Int4) | - |
+| Ovis1.6-Llama3.2-3B-GPTQ-Int4 | Siglip-400M | Llama-3.2-3B-Instruct       | [Huggingface](https://huggingface.co/AIDC-AI/Ovis1.6-Llama3.2-3B-GPTQ-Int4) | - |
 
-## Quantized Model: GPTQ-Int4
+## Quantized Model
 We quantized Ovis1.6 with AutoGPTQ. Follow these steps to run it.
 
 ### Installation
@@ -45,29 +48,28 @@ pip install numpy==1.24.3 transformers==4.44.2 pillow==10.3.0 gekko pandas
 ```
 2. Build AutoGPTQ: We customized AutoGPTQ to support Ovis model quantization. You need to build from source to install the customized version.
 ```bash
-git clone https://github.com/kq-chen/AutoGPTQ.git
+git clone https://github.com/AIDC-AI/AutoGPTQ.git
 cd AutoGPTQ
 pip install -vvv --no-build-isolation -e .
 ```
 Check [this](https://github.com/AutoGPTQ/AutoGPTQ/issues/194) first if you are building inside a Docker container.
 
 ### Usage
-Below is a code snippet to run Ovis1.6-Gemma2-9B-GPTQ-Int4 with multimodal inputs. For additional usage instructions, including inference wrapper and Gradio UI, please refer to [Ovis GitHub](https://github.com/AIDC-AI/Ovis?tab=readme-ov-file#inference).
+Below is a code snippet to run **Ovis1.6-Gemma2-9B-GPTQ-Int4** with multimodal inputs. For additional usage instructions, including inference wrapper and Gradio UI, please refer to [Ovis GitHub](https://github.com/AIDC-AI/Ovis?tab=readme-ov-file#inference).
 ```python
 import torch
 from PIL import Image
 from transformers import GenerationConfig
-from auto_gptq.modeling import OvisGPTQForCausalLM
+from auto_gptq.modeling import OvisGemma2GPTQForCausalLM
 
 # load model
 load_device = "cuda:0" # customize load device
-model = OvisGPTQForCausalLM.from_pretrained(
-    "TryingHard/Ovis1.6-Gemma2-9B-GPTQ-Int4",
+model = OvisGemma2GPTQForCausalLM.from_pretrained(
+    "AIDC-AI/Ovis1.6-Gemma2-9B-GPTQ-Int4",
     device=load_device,
-    multimodal_max_length=8192,
     trust_remote_code=True
 )
-model.model.generation_config = GenerationConfig.from_pretrained("TryingHard/Ovis1.6-Gemma2-9B-GPTQ-Int4")
+model.model.generation_config = GenerationConfig.from_pretrained("AIDC-AI/Ovis1.6-Gemma2-9B-GPTQ-Int4")
 text_tokenizer = model.get_text_tokenizer()
 visual_tokenizer = model.get_visual_tokenizer()
 
@@ -156,14 +158,14 @@ for i in range(len(batch_input_ids)):
 
 
 ## Quantize Your Own Ovis Model with AutoGPTQ
-We provide a demonstration code snippet for you to quantize your own fine-tuned Ovis model. Before running the code, you need to **follow the ABOVE installation steps** to obtain an environment for quantization.
+We provide a demonstration code snippet for you to quantize your own fine-tuned **Ovis1.6-Gemma2-9B** model. Before running the code, you need to **follow the ABOVE installation steps** to obtain an environment for quantization.
 ```python
 from typing import Dict, Sequence, Union, List
 import copy
 import logging
 
 from auto_gptq import BaseQuantizeConfig
-from auto_gptq.modeling import OvisGPTQForCausalLM
+from auto_gptq.modeling import OvisGemma2GPTQForCausalLM
 import torch
 from torch.utils.data import Dataset, DataLoader
 from PIL import Image
@@ -187,13 +189,13 @@ quantize_config = BaseQuantizeConfig(
 
 
 # Load model
-model = OvisGPTQForCausalLM.from_pretrained(
+model = OvisGemma2GPTQForCausalLM.from_pretrained(
     model_path,
     quantize_config,
     torch_dtype=torch.bfloat16,
     multimodal_max_length=8192,
     trust_remote_code=True
-)
+).cuda()
 print(f"Model Loaded!")
 
 
@@ -325,7 +327,7 @@ print(f"Dataloader Loaded!")
 
 
 # start quantizing
-model.quantize(train_loader, cache_examples_on_gpu=False, samples_dtype=torch.bfloat16) # do not change samples_dtype
+model.quantize(train_loader, cache_examples_on_gpu=False)
 print(f"Model Quantized! Now Saving...")
 
 model.save_quantized(quantize_save_path, use_safetensors=True)