Spaces:

MadsGalsgaard
/

Project-W

Sleeping

File size: 19,697 Bytes

# import gradio as gr
# from huggingface_hub import InferenceClient

# """
# For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
# """
# client = InferenceClient("meta-llama/Meta-Llama-3-8B-Instruct")

# ## None type 
# def respond(
#     message: str,
#     history: list[tuple[str, str]],  # This will not be used
#     system_message: str,
#     max_tokens: int,
#     temperature: float,
#     top_p: float,
# ):
#     messages = [{"role": "system", "content": system_message}]
    
#     # Append only the latest user message





#     messages.append({"role": "user", "content": message})

#     response = ""

#     try:
#         # Generate response from the model
#         for message in client.chat_completion(
#             messages,
#             max_tokens=max_tokens,
#             stream=True,
#             temperature=temperature,
#             top_p=top_p,
#         ):
#             if message.choices[0].delta.content is not None:
#                 token = message.choices[0].delta.content
#                 response += token
#             yield response
#     except Exception as e:
#         yield f"An error occurred: {e}"
#     ],
# )


# if __name__ == "__main__":
#     demo.launch()


##Running smothly CHATBOT

# import gradio as gr
# from huggingface_hub import InferenceClient

# """
# For more information on `huggingface_hub` Inference API support, please check the docs: https://huggingface.co/docs/huggingface_hub/v0.22.2/en/guides/inference
# """
# client = InferenceClient("meta-llama/Meta-Llama-3-8B-Instruct")

# def respond(
#     message: str,
#     history: list[tuple[str, str]],  # This will not be used
#     system_message: str,
#     max_tokens: int,
#     temperature: float,
#     top_p: float,
# ):
#     # Build the messages list
#     messages = [{"role": "system", "content": system_message}]
#     messages.append({"role": "user", "content": message})

#     response = ""

#     try:
#         # Generate response from the model
#         for msg in client.chat_completion(
#             messages=messages,
#             max_tokens=max_tokens,
#             stream=True,
#             temperature=temperature,
#             top_p=top_p,
#         ):
#             if msg.choices[0].delta.content is not None:
#                 token = msg.choices[0].delta.content
#                 response += token
#             yield response
#     except Exception as e:
#         yield f"An error occurred: {e}"

# """
# For information on how to customize the ChatInterface, peruse the gradio docs: https://www.gradio.app/docs/chatinterface
# """
# demo = gr.ChatInterface(
#     respond,
#     additional_inputs=[
#         gr.Textbox(value="You are a friendly Chatbot.", label="System message"),
#         gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
#         gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
#         gr.Slider(
#             minimum=0.1,
#             maximum=1.0,
#             value=0.95,
#             step=0.05,
#             label="Top-p (nucleus sampling)",
#         ),
#     ],
# )

# if __name__ == "__main__":
#     demo.launch()


####03 3.1 8b

# import os
# import time
# import spaces
# import torch
# from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer, BitsAndBytesConfig
# import gradio as gr
# from threading import Thread

# MODEL_LIST = ["meta-llama/Meta-Llama-3.1-8B-Instruct"]
# HF_TOKEN = os.environ.get("HF_API_TOKEN",None)
# print(HF_TOKEN,"######$$$$$$$$$$$$$$$")
# MODEL = os.environ.get("MODEL_ID","meta-llama/Meta-Llama-3.1-8B-Instruct")

# TITLE = "<h1><center>Meta-Llama3.1-8B</center></h1>"

# PLACEHOLDER = """
# <center>
# <p>Hi! How can I help you today?</p>
# </center>
# """


# CSS = """
# .duplicate-button {
#     margin: auto !important;
#     color: white !important;
#     background: black !important;
#     border-radius: 100vh !important;
# }
# h3 {
#     text-align: center;
# }
# """

# device = "cuda" # for GPU usage or "cpu" for CPU usage

# quantization_config = BitsAndBytesConfig(
#     load_in_4bit=True,
#     bnb_4bit_compute_dtype=torch.bfloat16,
#     bnb_4bit_use_double_quant=True,
#     bnb_4bit_quant_type= "nf4")

# tokenizer = AutoTokenizer.from_pretrained(MODEL)
# model = AutoModelForCausalLM.from_pretrained(
#     MODEL,
#     torch_dtype=torch.bfloat16,
#     device_map="auto",
#     quantization_config=quantization_config)

# @spaces.GPU()
# def stream_chat(
#     message: str, 
#     history: list,
#     system_prompt: str,
#     temperature: float = 0.8, 
#     max_new_tokens: int = 1024, 
#     top_p: float = 1.0, 
#     top_k: int = 20, 
#     penalty: float = 1.2,
# ):
#     print(f'message: {message}')
#     print(f'history: {history}')

#     conversation = [
#         {"role": "system", "content": system_prompt}
#     ]
#     for prompt, answer in history:
#         conversation.extend([
#             {"role": "user", "content": prompt}, 
#             {"role": "assistant", "content": answer},
#         ])

#     conversation.append({"role": "user", "content": message})

#     input_ids = tokenizer.apply_chat_template(conversation, add_generation_prompt=True, return_tensors="pt").to(model.device)
    
#     streamer = TextIteratorStreamer(tokenizer, timeout=60.0, skip_prompt=True, skip_special_tokens=True)
    
#     generate_kwargs = dict(
#         input_ids=input_ids, 
#         max_new_tokens = max_new_tokens,
#         do_sample = False if temperature == 0 else True,
#         top_p = top_p,
#         top_k = top_k,
#         temperature = temperature,
#         repetition_penalty=penalty,
#         eos_token_id=[128001,128008,128009],
#         streamer=streamer,
#     )

#     with torch.no_grad():
#         thread = Thread(target=model.generate, kwargs=generate_kwargs)
#         thread.start()
        
#     buffer = ""
#     for new_text in streamer:
#         buffer += new_text
#         yield buffer

            
# chatbot = gr.Chatbot(height=600, placeholder=PLACEHOLDER)

# with gr.Blocks(css=CSS, theme="soft") as demo:
#     gr.HTML(TITLE)
#     gr.DuplicateButton(value="Duplicate Space for private use", elem_classes="duplicate-button")
#     gr.ChatInterface(
#         fn=stream_chat,
#         chatbot=chatbot,
#         fill_height=True,
#         additional_inputs_accordion=gr.Accordion(label="⚙️ Parameters", open=False, render=False),
#         additional_inputs=[
#             gr.Textbox(
#                 value="You are a helpful assistant",
#                 label="System Prompt",
#                 render=False,
#             ),
#             gr.Slider(
#                 minimum=0,
#                 maximum=1,
#                 step=0.1,
#                 value=0.8,
#                 label="Temperature",
#                 render=False,
#             ),
#             gr.Slider(
#                 minimum=128,
#                 maximum=8192,
#                 step=1,
#                 value=1024,
#                 label="Max new tokens",
#                 render=False,
#             ),
#             gr.Slider(
#                 minimum=0.0,
#                 maximum=1.0,
#                 step=0.1,
#                 value=1.0,
#                 label="top_p",
#                 render=False,
#             ),
#             gr.Slider(
#                 minimum=1,
#                 maximum=20,
#                 step=1,
#                 value=20,
#                 label="top_k",
#                 render=False,
#             ),
#             gr.Slider(
#                 minimum=0.0,
#                 maximum=2.0,
#                 step=0.1,
#                 value=1.2,
#                 label="Repetition penalty",
#                 render=False,
#             ),
#         ],
#         examples=[
#             ["Help me study vocabulary: write a sentence for me to fill in the blank, and I'll try to pick the correct option."],
#             ["What are 5 creative things I could do with my kids' art? I don't want to throw them away, but it's also so much clutter."],
#             ["Tell me a random fun fact about the Roman Empire."],
#             ["Show me a code snippet of a website's sticky header in CSS and JavaScript."],
#         ],
#         cache_examples=False,
#     )


# if __name__ == "__main__":
#     demo.launch()



###########new clientkey 04 ruunng chlrhah 


# import os
# import time
# import spaces
# import torch
# from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer
# import gradio as gr
# from threading import Thread

# MODEL = "THUDM/LongWriter-llama3.1-8b"

# TITLE = "<h1><center>AreaX LLC-llama3.1-8b</center></h1>"

# PLACEHOLDER = """
# <center>
# <p>Hi! I'm AreaX AI Agent, capable of generating 10,000+ words. How can I assist you today?</p>
# </center>
# """

# CSS = """
# .duplicate-button {
#     margin: auto !important;
#     color: white !important;
#     background: black !important;
#     border-radius: 100vh !important;
# }
# h3 {
#     text-align: center;
# }
# """

# device = "cuda" if torch.cuda.is_available() else "cpu"

# tokenizer = AutoTokenizer.from_pretrained(MODEL, trust_remote_code=True)
# model = AutoModelForCausalLM.from_pretrained(MODEL, torch_dtype=torch.bfloat16, trust_remote_code=True, device_map="auto")
# model = model.eval()

# @spaces.GPU()
# def stream_chat(
#     message: str,
#     history: list,
#     system_prompt: str,
#     temperature: float = 0.5,
#     max_new_tokens: int = 32768,
#     top_p: float = 1.0,
#     top_k: int = 50,
# ):
#     print(f'message: {message}')
#     print(f'history: {history}')

#     full_prompt = f"<<SYS>>\n{system_prompt}\n<</SYS>>\n\n"
#     for prompt, answer in history:
#         full_prompt += f"[INST]{prompt}[/INST]{answer}"
#     full_prompt += f"[INST]{message}[/INST]"

#     inputs = tokenizer(full_prompt, truncation=False, return_tensors="pt").to(device)
#     context_length = inputs.input_ids.shape[-1]

#     streamer = TextIteratorStreamer(tokenizer, timeout=60.0, skip_prompt=True, skip_special_tokens=True)

#     generate_kwargs = dict(
#         inputs=inputs.input_ids,
#         max_new_tokens=max_new_tokens,
#         do_sample=True,
#         top_p=top_p,
#         top_k=top_k,
#         temperature=temperature,
#         num_beams=1,
#         streamer=streamer,
#     )

#     thread = Thread(target=model.generate, kwargs=generate_kwargs)
#     thread.start()

#     buffer = ""
#     for new_text in streamer:
#         buffer += new_text
#         yield buffer

# chatbot = gr.Chatbot(height=600, placeholder=PLACEHOLDER)

# with gr.Blocks(css=CSS, theme="soft") as demo:
#     gr.HTML(TITLE)
#     gr.DuplicateButton(value="Duplicate Space for private use", elem_classes="duplicate-button")
#     gr.ChatInterface(
#         fn=stream_chat,
#         chatbot=chatbot,
#         fill_height=True,
#         additional_inputs_accordion=gr.Accordion(label="⚙️ Parameters", open=False, render=False),
#         additional_inputs=[
#             gr.Textbox(
#                 value="You are a helpful assistant capable of generating long-form content.",
#                 label="System Prompt",
#                 render=False,
#             ),
#             gr.Slider(
#                 minimum=0,
#                 maximum=1,
#                 step=0.1,
#                 value=0.5,
#                 label="Temperature",
#                 render=False,
#             ),
#             gr.Slider(
#                 minimum=1024,
#                 maximum=32768,
#                 step=1024,
#                 value=32768,
#                 label="Max new tokens",
#                 render=False,
#             ),
#             gr.Slider(
#                 minimum=0.0,
#                 maximum=1.0,
#                 step=0.1,
#                 value=1.0,
#                 label="Top p",
#                 render=False,
#             ),
#             gr.Slider(
#                 minimum=1,
#                 maximum=100,
#                 step=1,
#                 value=50,
#                 label="Top k",
#                 render=False,
#             ),
#         ],
#         # examples=[
#         #     ["Write a 5000-word comprehensive guide on machine learning for beginners."],
#         #     ["Create a detailed 3000-word business plan for a sustainable energy startup."],
#         #     ["Compose a 2000-word short story set in a futuristic underwater city."],
#         #     ["Develop a 4000-word research proposal on the potential effects of climate change on global food security."],
#         # ],
#         # cache_examples=False,
#     )

# if __name__ == "__main__":
#     demo.launch()



# ###OCT04 LLAMA3.2 Vision Model
from transformers import MllamaForConditionalGeneration, AutoProcessor, TextIteratorStreamer
from PIL import Image
import requests
import torch
from threading import Thread
import gradio as gr
from gradio import FileData
import time
import os
import spaces
from huggingface_hub import login
login(token=os.getenv("HF_API_TOKEN"))
# ckpt = "meta-llama/Llama-3.2-11B-Vision-Instruct"
# model = MllamaForConditionalGeneration.from_pretrained(ckpt,
#     torch_dtype=torch.bfloat16).to("cuda")
# processor = AutoProcessor.from_pretrained(ckpt)


# @spaces.GPU
# def bot_streaming(message, history, max_new_tokens=250):
    
#     txt = message["text"]
#     ext_buffer = f"{txt}"
    
#     messages= [] 
#     images = []
    

#     for i, msg in enumerate(history): 
#         if isinstance(msg[0], tuple):
#             messages.append({"role": "user", "content": [{"type": "text", "text": history[i+1][0]}, {"type": "image"}]})
#             messages.append({"role": "assistant", "content": [{"type": "text", "text": history[i+1][1]}]})
#             images.append(Image.open(msg[0][0]).convert("RGB"))
#         elif isinstance(history[i-1], tuple) and isinstance(msg[0], str):
#             # messages are already handled
#             pass
#         elif isinstance(history[i-1][0], str) and isinstance(msg[0], str): # text only turn
#             messages.append({"role": "user", "content": [{"type": "text", "text": msg[0]}]})
#             messages.append({"role": "assistant", "content": [{"type": "text", "text": msg[1]}]})

#     # add current message
#     if len(message["files"]) == 1:
        
#         if isinstance(message["files"][0], str): # examples
#             image = Image.open(message["files"][0]).convert("RGB")
#         else: # regular input
#             image = Image.open(message["files"][0]["path"]).convert("RGB")
#         images.append(image)
#         messages.append({"role": "user", "content": [{"type": "text", "text": txt}, {"type": "image"}]})
#     else:
#         messages.append({"role": "user", "content": [{"type": "text", "text": txt}]})


#     texts = processor.apply_chat_template(messages, add_generation_prompt=True)

#     if images == []:
#         inputs = processor(text=texts, return_tensors="pt").to("cuda")
#     else:
#         inputs = processor(text=texts, images=images, return_tensors="pt").to("cuda")
#     streamer = TextIteratorStreamer(processor, skip_special_tokens=True, skip_prompt=True)

#     generation_kwargs = dict(inputs, streamer=streamer, max_new_tokens=max_new_tokens)
#     generated_text = ""
    
#     thread = Thread(target=model.generate, kwargs=generation_kwargs)
#     thread.start()
#     buffer = ""
    
#     for new_text in streamer:
#         buffer += new_text
#         generated_text_without_prompt = buffer
#         time.sleep(0.01)
#         yield buffer


# demo = gr.ChatInterface(fn=bot_streaming, title="Multimodal Llama",
#       textbox=gr.MultimodalTextbox(), 
#       additional_inputs = [gr.Slider(
#               minimum=10,
#               maximum=500,
#               value=250,
#               step=10,
#               label="Maximum number of new tokens to generate",
#           )
#         ],
#       cache_examples=False,
#       description="Try Multimodal Llama by Meta with transformers in this demo. Upload an image, and start chatting about it, or simply try one of the examples below. To learn more about Llama Vision, visit [our blog post](https://huggingface.co/blog/llama32). ",
#       stop_btn="Stop Generation", 
#       fill_height=True,
#     multimodal=True)
    
# demo.launch(debug=True,live=True)

ckpt = "meta-llama/Llama-3.2-11B-Vision-Instruct"
model = MllamaForConditionalGeneration.from_pretrained(ckpt, torch_dtype=torch.bfloat16).to("cuda")
processor = AutoProcessor.from_pretrained(ckpt)

@spaces.GPU
def bot_streaming(message, history, max_new_tokens=1000):
    txt = message["text"]
    ext_buffer = f"{txt}"
    
    messages = [] 
    images = []
    
    # Process history messages
    for i, msg in enumerate(history): 
        if isinstance(msg[0], tuple):
            messages.append({"role": "user", "content": [{"type": "text", "text": history[i+1][0]}, {"type": "image"}]})
            messages.append({"role": "assistant", "content": [{"type": "text", "text": history[i+1][1]}]})
            images.append(Image.open(msg[0][0]).convert("RGB"))
        elif isinstance(history[i-1], tuple) and isinstance(msg[0], str):
            pass  # Previous messages already handled
        elif isinstance(history[i-1][0], str) and isinstance(msg[0], str):  # Text-only turn
            messages.append({"role": "user", "content": [{"type": "text", "text": msg[0]}]})
            messages.append({"role": "assistant", "content": [{"type": "text", "text": msg[1]}]})

    # Add current message
    if len(message["files"]) == 1:
        if isinstance(message["files"][0], str):  # Example images
            image = Image.open(message["files"][0]).convert("RGB")
        else:  # Regular input
            image = Image.open(message["files"][0]["path"]).convert("RGB")
        images.append(image)
        messages.append({"role": "user", "content": [{"type": "text", "text": txt}, {"type": "image"}]})
    else:
        messages.append({"role": "user", "content": [{"type": "text", "text": txt}]})

    # Prepare input for the model
    texts = processor.apply_chat_template(messages, add_generation_prompt=True)

    if not images:
        inputs = processor(text=texts, return_tensors="pt").to("cuda")
    else:
        inputs = processor(text=texts, images=images, return_tensors="pt").to("cuda")
    
    streamer = TextIteratorStreamer(processor, skip_special_tokens=True, skip_prompt=True)

    generation_kwargs = dict(inputs, streamer=streamer, max_new_tokens=max_new_tokens)
    generated_text = ""
    
    # Start text generation in a separate thread
    thread = Thread(target=model.generate, kwargs=generation_kwargs)
    thread.start()
    buffer = ""
    
    for new_text in streamer:
        buffer += new_text
        time.sleep(0.01)  # Small delay to simulate streaming
        yield buffer

# Gradio interface setup
demo = gr.ChatInterface(
    fn=bot_streaming, 
    title="AreaX-Llama3.2-11B-Vision", 
    textbox=gr.MultimodalTextbox(), 
    additional_inputs=[
        gr.Slider(
            minimum=10,
            maximum=500,
            value=250,
            step=10,
            label="Maximum number of new tokens to generate",
        )
    ],
    cache_examples=False,
    description="Try AreaX Llama3.2-11B Vision Model by Meta with transformers in this demo. Upload an image, and start chatting about it, or simply type your question.",
    stop_btn="Stop Generation", 
    fill_height=True,
    multimodal=True
)

demo.launch(debug=True,share=True)