Update app.py

app.py

@@ -45,17 +45,25 @@ def generate_response(instruction, input_text):
 
         # 在函数内部导入需要 GPU 的库
         import torch
-        from transformers import AutoTokenizer, AutoModelForCausalLM
+        from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
         from peft import PeftModel
 
+        # 创建量化配置
+        bnb_config = BitsAndBytesConfig(
+            load_in_4bit=True,
+            bnb_4bit_use_double_quant=True,
+            bnb_4bit_quant_type="nf4",
+            bnb_4bit_compute_dtype=torch.float16
+        )
+
         # 加载分词器
         tokenizer = AutoTokenizer.from_pretrained(base_model_name, use_auth_token=hf_token)
 
         # 加载基础模型
         base_model = AutoModelForCausalLM.from_pretrained(
             base_model_name,
+            quantization_config=bnb_config,
             device_map="auto",
-            torch_dtype=torch.float16,
             use_auth_token=hf_token,
             trust_remote_code=True
         )