Setting Up environment of llama3.2 model

app.py

@@ -439,144 +439,248 @@
 
 
 
-###########new clientkey
+###########new clientkey 04 ruunng chlrhah 
 
 
-import os
-import time
-import spaces
+# import os
+# import time
+# import spaces
+# import torch
+# from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
+# import gradio as gr
+# from threading import Thread
+
+# MODEL = "THUDM/LongWriter-llama3.1-8b"
+
+# TITLE = "<h1><center>AreaX LLC-llama3.1-8b</center></h1>"
+
+# PLACEHOLDER = """
+# <center>
+# <p>Hi! I'm AreaX AI Agent, capable of generating 10,000+ words. How can I assist you today?</p>
+# </center>
+# """
+
+# CSS = """
+# .duplicate-button {
+#     margin: auto !important;
+#     color: white !important;
+#     background: black !important;
+#     border-radius: 100vh !important;
+# }
+# h3 {
+#     text-align: center;
+# }
+# """
+
+# device = "cuda" if torch.cuda.is_available() else "cpu"
+
+# tokenizer = AutoTokenizer.from_pretrained(MODEL, trust_remote_code=True)
+# model = AutoModelForCausalLM.from_pretrained(MODEL, torch_dtype=torch.bfloat16, trust_remote_code=True, device_map="auto")
+# model = model.eval()
+
+# @spaces.GPU()
+# def stream_chat(
+#     message: str,
+#     history: list,
+#     system_prompt: str,
+#     temperature: float = 0.5,
+#     max_new_tokens: int = 32768,
+#     top_p: float = 1.0,
+#     top_k: int = 50,
+# ):
+#     print(f'message: {message}')
+#     print(f'history: {history}')
+
+#     full_prompt = f"<<SYS>>\n{system_prompt}\n<</SYS>>\n\n"
+#     for prompt, answer in history:
+#         full_prompt += f"[INST]{prompt}[/INST]{answer}"
+#     full_prompt += f"[INST]{message}[/INST]"
+
+#     inputs = tokenizer(full_prompt, truncation=False, return_tensors="pt").to(device)
+#     context_length = inputs.input_ids.shape[-1]
+
+#     streamer = TextIteratorStreamer(tokenizer, timeout=60.0, skip_prompt=True, skip_special_tokens=True)
+
+#     generate_kwargs = dict(
+#         inputs=inputs.input_ids,
+#         max_new_tokens=max_new_tokens,
+#         do_sample=True,
+#         top_p=top_p,
+#         top_k=top_k,
+#         temperature=temperature,
+#         num_beams=1,
+#         streamer=streamer,
+#     )
+
+#     thread = Thread(target=model.generate, kwargs=generate_kwargs)
+#     thread.start()
+
+#     buffer = ""
+#     for new_text in streamer:
+#         buffer += new_text
+#         yield buffer
+
+# chatbot = gr.Chatbot(height=600, placeholder=PLACEHOLDER)
+
+# with gr.Blocks(css=CSS, theme="soft") as demo:
+#     gr.HTML(TITLE)
+#     gr.DuplicateButton(value="Duplicate Space for private use", elem_classes="duplicate-button")
+#     gr.ChatInterface(
+#         fn=stream_chat,
+#         chatbot=chatbot,
+#         fill_height=True,
+#         additional_inputs_accordion=gr.Accordion(label="⚙️ Parameters", open=False, render=False),
+#         additional_inputs=[
+#             gr.Textbox(
+#                 value="You are a helpful assistant capable of generating long-form content.",
+#                 label="System Prompt",
+#                 render=False,
+#             ),
+#             gr.Slider(
+#                 minimum=0,
+#                 maximum=1,
+#                 step=0.1,
+#                 value=0.5,
+#                 label="Temperature",
+#                 render=False,
+#             ),
+#             gr.Slider(
+#                 minimum=1024,
+#                 maximum=32768,
+#                 step=1024,
+#                 value=32768,
+#                 label="Max new tokens",
+#                 render=False,
+#             ),
+#             gr.Slider(
+#                 minimum=0.0,
+#                 maximum=1.0,
+#                 step=0.1,
+#                 value=1.0,
+#                 label="Top p",
+#                 render=False,
+#             ),
+#             gr.Slider(
+#                 minimum=1,
+#                 maximum=100,
+#                 step=1,
+#                 value=50,
+#                 label="Top k",
+#                 render=False,
+#             ),
+#         ],
+#         # examples=[
+#         #     ["Write a 5000-word comprehensive guide on machine learning for beginners."],
+#         #     ["Create a detailed 3000-word business plan for a sustainable energy startup."],
+#         #     ["Compose a 2000-word short story set in a futuristic underwater city."],
+#         #     ["Develop a 4000-word research proposal on the potential effects of climate change on global food security."],
+#         # ],
+#         # cache_examples=False,
+#     )
+
+# if __name__ == "__main__":
+#     demo.launch()
+
+
+
+###04 
+from transformers import MllamaForConditionalGeneration, AutoProcessor, TextIteratorStreamer
+from PIL import Image
+import requests
 import torch
-from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
-import gradio as gr
 from threading import Thread
+import gradio as gr
+from gradio import FileData
+import time
+import spaces
+ckpt = "meta-llama/Llama-3.2-11B-Vision-Instruct"
+model = MllamaForConditionalGeneration.from_pretrained(ckpt,
+    torch_dtype=torch.bfloat16).to("cuda")
+processor = AutoProcessor.from_pretrained(ckpt)
 
-MODEL = "THUDM/LongWriter-llama3.1-8b"
-
-TITLE = "<h1><center>AreaX LLC-llama3.1-8b</center></h1>"
-
-PLACEHOLDER = """
-<center>
-<p>Hi! I'm AreaX AI Agent, capable of generating 10,000+ words. How can I assist you today?</p>
-</center>
-"""
-
-CSS = """
-.duplicate-button {
-    margin: auto !important;
-    color: white !important;
-    background: black !important;
-    border-radius: 100vh !important;
-}
-h3 {
-    text-align: center;
-}
-"""
-
-device = "cuda" if torch.cuda.is_available() else "cpu"
-
-tokenizer = AutoTokenizer.from_pretrained(MODEL, trust_remote_code=True)
-model = AutoModelForCausalLM.from_pretrained(MODEL, torch_dtype=torch.bfloat16, trust_remote_code=True, device_map="auto")
-model = model.eval()
-
-@spaces.GPU()
-def stream_chat(
-    message: str,
-    history: list,
-    system_prompt: str,
-    temperature: float = 0.5,
-    max_new_tokens: int = 32768,
-    top_p: float = 1.0,
-    top_k: int = 50,
-):
-    print(f'message: {message}')
-    print(f'history: {history}')
-
-    full_prompt = f"<<SYS>>\n{system_prompt}\n<</SYS>>\n\n"
-    for prompt, answer in history:
-        full_prompt += f"[INST]{prompt}[/INST]{answer}"
-    full_prompt += f"[INST]{message}[/INST]"
-
-    inputs = tokenizer(full_prompt, truncation=False, return_tensors="pt").to(device)
-    context_length = inputs.input_ids.shape[-1]
-
-    streamer = TextIteratorStreamer(tokenizer, timeout=60.0, skip_prompt=True, skip_special_tokens=True)
-
-    generate_kwargs = dict(
-        inputs=inputs.input_ids,
-        max_new_tokens=max_new_tokens,
-        do_sample=True,
-        top_p=top_p,
-        top_k=top_k,
-        temperature=temperature,
-        num_beams=1,
-        streamer=streamer,
-    )
-
-    thread = Thread(target=model.generate, kwargs=generate_kwargs)
-    thread.start()
 
+@spaces.GPU
+def bot_streaming(message, history, max_new_tokens=250):
+    
+    txt = message["text"]
+    ext_buffer = f"{txt}"
+    
+    messages= [] 
+    images = []
+    
+
+    for i, msg in enumerate(history): 
+        if isinstance(msg[0], tuple):
+            messages.append({"role": "user", "content": [{"type": "text", "text": history[i+1][0]}, {"type": "image"}]})
+            messages.append({"role": "assistant", "content": [{"type": "text", "text": history[i+1][1]}]})
+            images.append(Image.open(msg[0][0]).convert("RGB"))
+        elif isinstance(history[i-1], tuple) and isinstance(msg[0], str):
+            # messages are already handled
+            pass
+        elif isinstance(history[i-1][0], str) and isinstance(msg[0], str): # text only turn
+            messages.append({"role": "user", "content": [{"type": "text", "text": msg[0]}]})
+            messages.append({"role": "assistant", "content": [{"type": "text", "text": msg[1]}]})
+
+    # add current message
+    if len(message["files"]) == 1:
+        
+        if isinstance(message["files"][0], str): # examples
+            image = Image.open(message["files"][0]).convert("RGB")
+        else: # regular input
+            image = Image.open(message["files"][0]["path"]).convert("RGB")
+        images.append(image)
+        messages.append({"role": "user", "content": [{"type": "text", "text": txt}, {"type": "image"}]})
+    else:
+        messages.append({"role": "user", "content": [{"type": "text", "text": txt}]})
+
+
+    texts = processor.apply_chat_template(messages, add_generation_prompt=True)
+
+    if images == []:
+        inputs = processor(text=texts, return_tensors="pt").to("cuda")
+    else:
+        inputs = processor(text=texts, images=images, return_tensors="pt").to("cuda")
+    streamer = TextIteratorStreamer(processor, skip_special_tokens=True, skip_prompt=True)
+
+    generation_kwargs = dict(inputs, streamer=streamer, max_new_tokens=max_new_tokens)
+    generated_text = ""
+    
+    thread = Thread(target=model.generate, kwargs=generation_kwargs)
+    thread.start()
     buffer = ""
+    
     for new_text in streamer:
         buffer += new_text
+        generated_text_without_prompt = buffer
+        time.sleep(0.01)
         yield buffer
 
-chatbot = gr.Chatbot(height=600, placeholder=PLACEHOLDER)
-
-with gr.Blocks(css=CSS, theme="soft") as demo:
-    gr.HTML(TITLE)
-    gr.DuplicateButton(value="Duplicate Space for private use", elem_classes="duplicate-button")
-    gr.ChatInterface(
-        fn=stream_chat,
-        chatbot=chatbot,
-        fill_height=True,
-        additional_inputs_accordion=gr.Accordion(label="⚙️ Parameters", open=False, render=False),
-        additional_inputs=[
-            gr.Textbox(
-                value="You are a helpful assistant capable of generating long-form content.",
-                label="System Prompt",
-                render=False,
-            ),
-            gr.Slider(
-                minimum=0,
-                maximum=1,
-                step=0.1,
-                value=0.5,
-                label="Temperature",
-                render=False,
-            ),
-            gr.Slider(
-                minimum=1024,
-                maximum=32768,
-                step=1024,
-                value=32768,
-                label="Max new tokens",
-                render=False,
-            ),
-            gr.Slider(
-                minimum=0.0,
-                maximum=1.0,
-                step=0.1,
-                value=1.0,
-                label="Top p",
-                render=False,
-            ),
-            gr.Slider(
-                minimum=1,
-                maximum=100,
-                step=1,
-                value=50,
-                label="Top k",
-                render=False,
-            ),
+
+demo = gr.ChatInterface(fn=bot_streaming, title="Multimodal Llama", examples=[
+    [{"text": "Which era does this piece belong to? Give details about the era.", "files":["./examples/rococo.jpg"]},
+    200],
+    [{"text": "Where do the droughts happen according to this diagram?", "files":["./examples/weather_events.png"]},
+    250],
+    [{"text": "What happens when you take out white cat from this chain?", "files":["./examples/ai2d_test.jpg"]},
+    250],
+    [{"text": "How long does it take from invoice date to due date? Be short and concise.", "files":["./examples/invoice.png"]},
+    250],
+    [{"text": "Where to find this monument? Can you give me other recommendations around the area?", "files":["./examples/wat_arun.jpg"]}, 
+    250],
+    ],
+      textbox=gr.MultimodalTextbox(), 
+      additional_inputs = [gr.Slider(
+              minimum=10,
+              maximum=500,
+              value=250,
+              step=10,
+              label="Maximum number of new tokens to generate",
+          )
         ],
-        # examples=[
-        #     ["Write a 5000-word comprehensive guide on machine learning for beginners."],
-        #     ["Create a detailed 3000-word business plan for a sustainable energy startup."],
-        #     ["Compose a 2000-word short story set in a futuristic underwater city."],
-        #     ["Develop a 4000-word research proposal on the potential effects of climate change on global food security."],
-        # ],
-        # cache_examples=False,
-    )
-
-if __name__ == "__main__":
-    demo.launch()
+      cache_examples=False,
+      description="Try Multimodal Llama by Meta with transformers in this demo. Upload an image, and start chatting about it, or simply try one of the examples below. To learn more about Llama Vision, visit [our blog post](https://huggingface.co/blog/llama32). ",
+      stop_btn="Stop Generation", 
+      fill_height=True,
+    multimodal=True)
+    
+demo.launch(debug=True)