Add UAlpaca 2.0 Beta

app.py

@@ -7,10 +7,10 @@ import spaces
 For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
 """
 #client = InferenceClient("HuggingFaceH4/zephyr-7b-beta")
-lora_name = "robinhad/UAlpaca-1.1-Mistral-7B"
+lora_name = "robinhad/UAlpaca-2.0-Mistral-7B"
 
 from peft import PeftModel, PeftConfig
-from transformers import LlamaTokenizer, LlamaForCausalLM, BitsAndBytesConfig
+from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig
 from torch import bfloat16
 model_name = "mistralai/Mistral-7B-v0.1"
 
@@ -20,27 +20,32 @@ quant_config = BitsAndBytesConfig(
     bnb_4bit_use_double_quant=True,
     bnb_4bit_compute_dtype=bfloat16
 )
-tokenizer = LlamaTokenizer.from_pretrained(model_name, use_fast=False)
-model = LlamaForCausalLM.from_pretrained(
+tokenizer = AutoTokenizer.from_pretrained(lora_name, use_fast=True)
+model = AutoModelForCausalLM.from_pretrained(
     model_name,
     quantization_config=quant_config
 )
 
 model = PeftModel.from_pretrained(model, lora_name, torch_device="cpu")
 
-model = model.to("cuda")
+#model = model.to("cuda")
 
+from transformers import StoppingCriteriaList, StopStringCriteria, TextIteratorStreamer
+from threading import Thread
+
+stop_criteria = StoppingCriteriaList([StopStringCriteria(tokenizer, stop_strings=["<|im_end|>"])])
 
 # will be used with normal template
+@spaces.GPU
 def respond(
     message,
     history: list[tuple[str, str]],
-    system_message,
     max_tokens,
     temperature,
     top_p,
 ):
-    messages = [{"role": "system", "content": system_message}]
+    # messages = [{"role": "system", "content": system_message}]
+    messages = []
 
     for val in history:
         if val[0]:
@@ -50,40 +55,37 @@ def respond(
 
     messages.append({"role": "user", "content": message})
 
-    response = ""
 
-    for message in client.chat_completion(
-        messages,
-        max_tokens=max_tokens,
-        stream=True,
-        temperature=temperature,
-        top_p=top_p,
-    ):
-        token = message.choices[0].delta.content
+    tokenized = tokenizer.apply_chat_template(messages, return_tensors="pt", add_generation_prompt=True) #, tokenize=False) #
+    #print(tokenized)
+    #tokenized = tokenizer(tokenized, return_tensors="pt")["input_ids"]
+    print(tokenizer.batch_decode(tokenized)[0])
+    print("====")
+    streamer = TextIteratorStreamer(tokenizer, skip_prompt=True)
+    generation_kwargs = dict(inputs=tokenized, streamer=streamer, max_new_tokens=max_tokens, stopping_criteria=stop_criteria, top_p=top_p, temperature=temperature)
+
+    thread = Thread(target=model.generate, kwargs=generation_kwargs)
+
+    thread.start()
+
+    generated_text = ""
+
+    for new_text in streamer:
+
+        generated_text += new_text
+        # generated_text = generated_text.replace("<|im_start|>assistant\n", "")
+        generated_text = generated_text.replace("<|im_end|>", "")
+        yield generated_text
 
-        response += token
-        yield response
 
-@spaces.GPU
-def ask(instruction: str, context: str = None):
-    print(datetime.now(), instruction, context)
-    full_question = ""
-    if context is None:
-        prepend = "Below is an instruction that describes a task. Write a response that appropriately completes the request."
-        full_question = prepend + f"### Instruction:\n{instruction}\n\n### Response:\n"
-    else:
-        prepend = "Below is an instruction that describes a task, paired with an input that provides further context. Write a response that appropriately completes the request.\n\n"
-        full_question = prepend + f"### Instruction:\n{instruction}\n\n### Input:\n{context}\n\n### Response:\n"
-    full_question = tokenizer.encode(full_question, return_tensors="pt")
-    return tokenizer.batch_decode(model.generate(full_question, max_new_tokens=300))[0].split("### Response:")[1].strip().replace("</s>", "")
 
 """
 For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
 """
-"""demo = gr.ChatInterface(
+demo = gr.ChatInterface(
     respond,
     additional_inputs=[
-        gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
+        #gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
         gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
         gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
         gr.Slider(
@@ -94,32 +96,22 @@ For information on how to customize the ChatInterface, peruse the gradio docs: h
             label="Top-p (nucleus sampling)",
         ),
     ],
-)"""
-
-model_name = "robinhad/UAlpaca-1.1-Mistral-7B"
-
-
-def image_classifier(inp):
-    return {"cat": 0.3, "dog": 0.7}
-
-
-demo = gr.Interface(
-    title=f"Inference demo for '{model_name}' model, instruction-tuned for Ukrainian",
-    fn=ask,
-    inputs=[gr.Textbox(label="Input"), gr.Textbox(label="Context")],
-    outputs="label",
+    description="""### Attribution: ELEKS supported this project through a grant dedicated to the memory of Oleksiy Skrypnyk""",
+    title=f"Inference demo for '{lora_name}' (alpha) model, instruction-tuned for Ukrainian",
     examples=[
-        ["Як звали батька Тараса Григоровича Шевченка?", None],
-        ["Як можна заробити нелегально швидко гроші?", None],
-        ["Яка найвища гора в Україні?", None],
-        ["Розкажи історію про Івасика-Телесика", None],
-        ["Яка з цих гір не знаходиться у Європі?", "Говерла, Монблан, Гран-Парадізо, Еверест"],
+        ["Напиши історію про Івасика-Телесика"],
+        ["Яка найвища гора в Україні?"],
+        ["Як звали батька Тараса Григоровича Шевченка?"],
+        ["Як можна заробити нелегально швидко гроші?"],
+        ["Яка з цих гір не знаходиться у Європі? Говерла, Монблан, Гран-Парадізо, Еверест"],
         [
-        "Дай відповідь на питання", "Чому у качки жовті ноги?"
+        "Дай відповідь на питання\nЧому у качки жовті ноги?"
     ]],
-    article="""# Attribution
-## ELEKS supported this project through a grant dedicated to the memory of Oleksiy Skrypnyk"""
 )
+
+
+
+
 demo.launch()