Use quantization

app.py

@@ -15,8 +15,8 @@ quant_config = BitsAndBytesConfig(
     bnb_4bit_use_double_quant=False,
 )
 
-model = MistralForCausalLM.from_pretrained("mistralai/Mistral-7B-v0.1", torch_dtype=float16).to("cuda")
-    #quantization_config=quant_config,)#.to("cuda")
+model = MistralForCausalLM.from_pretrained("mistralai/Mistral-7B-v0.1",
+    quantization_config=quant_config)
     #device_map="auto",)
 model = PeftModel.from_pretrained(model, "lang-uk/dragoman").to("cuda")
 tokenizer = AutoTokenizer.from_pretrained("mistralai/Mistral-7B-v0.1", use_fast=False, add_bos_token=False)