Update app.py

app.py

@@ -17,17 +17,27 @@ model, tokenizer = FastLanguageModel.from_pretrained(
     trust_remote_code=True,
     # token = "hf_...", # use one if using gated models like meta-llama/Llama-2-7b-hf
 )
-
+FastLanguageModel.for_inference(model)
 print("model loaded")
 
 
 streamer = TextStreamer(tokenizer, timeout=10.0, skip_prompt=True, skip_special_tokens = True)
 
-def generate_text(prompt, max_length, top_p, top_k):
-    inputs = tokenizer([prompt], return_tensors="pt")
+stored_message = gr.State([])
+
+def generate_text(prompt, max_length, top_p, top_k, messages):
+
+    messages.append({"role": "user", "content": prompt})           
+        
+    input_ids = tokenizer.apply_chat_template(
+        messages,
+        add_generation_prompt = True,
+        return_tensors = "pt",
+    )
+
 
     generate_kwargs = dict(
-        inputs,
+        input_ids,
         max_length=int(max_length),top_p=float(top_p), do_sample=True, top_k=int(top_k), streamer=streamer
     )
 
@@ -39,7 +49,8 @@ def generate_text(prompt, max_length, top_p, top_k):
     for text in streamer:
         generated_text.append(text)
         yield "".join(generated_text)
-
+    
+    messages.append({"role": "assistant", "content": "".join(generated_text)})   
 
 description = """
 # Deploy our LLM
@@ -49,6 +60,7 @@ inputs = [
     gr.Textbox(label="max-lenth generation", value=100),
     gr.Slider(0.0, 1.0, label="top-p value", value=0.95),
     gr.Textbox(label="top-k", value=50,),
+    stored_message
 ]
 outputs = [gr.Textbox(label="Generated Text")]