import os from threading import Thread from typing import Iterator import gradio as gr import spaces import torch from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer MAX_MAX_NEW_TOKENS = 2048 DEFAULT_MAX_NEW_TOKENS = 1024 MAX_INPUT_TOKEN_LENGTH = int(os.getenv("MAX_INPUT_TOKEN_LENGTH", "8192")) DESCRIPTION = """\ # Playground with Ghost 8B Beta (p) **Ghost 8B Beta** is a large language model developed with goals that include excellent multilingual support, superior knowledge capabilities, and cost-effectiveness. The model comes in two context length versions, 8k and 128k, along with multilingual function tools support by default. The languages supported are 🇺🇸 English, 🇫🇷 French, 🇮🇹 Italian, 🇪🇸 Spanish, 🇵🇹 Portuguese, 🇩🇪 German, 🇻🇳 Vietnamese, 🇰🇷 Korean and 🇨🇳 Chinese. 📋 Note: current model version is "disl-0x5-8k" (10 Jul 2024), context length 8k and current status is "moderating / previewing". For detailed information about the model, see [here](https://ghost-x.org/docs/models/ghost-8b-beta/). Try to experience it the way you want! """ PLACEHOLDER = """

👻 Ghost 8B Beta

Ask and share whatever you want ~

""" LICENSE = """

--- Ghost 8B Beta may give inaccurate information, including information about people, so please verify Ghost 8B Beta's answers. [Ghost 8B Beta](https://ghost-x.org/docs/models/ghost-8b-beta/) by [Ghost X](https://ghost-x.org). """ EXAMPLES = [ [ "Explain the concept of quantum entanglement and its implications for quantum computing." ], ["Comment le mouvement des Lumières a-t-il influencé la Révolution française ?"], ["Quale fu l'impatto del Rinascimento italiano sull'arte e la cultura europea?"], [ "Spiega il funzionamento e le applicazioni della spettroscopia Raman in chimica analitica." ], [ "Explique el teorema de incompletitud de Gödel y sus implicaciones en la lógica matemática." ], [ "Descreva o processo de meiose celular e sua importância na variabilidade genética." ], [ "Giải thích nguyên lý hoạt động của máy học sâu (deep learning) trong trí tuệ nhân tạo và ứng dụng của nó trong xử lý ngôn ngữ tự nhiên." ], ["조선 시대의 신분제도가 한국 사회에 미친 영향을 분석하시오."], ["分析丝绸之路对中国古代文化交流和经济发展的影响。"], ] if not torch.cuda.is_available(): DESCRIPTION += "\n

Running on CPU 🥶 This demo does not work on CPU.

" if torch.cuda.is_available(): model_id = "lamhieu/ghost-8b-beta-disl-0x5-8k" model_tk = os.getenv("HF_TOKEN", None) model = AutoModelForCausalLM.from_pretrained( model_id, device_map="auto", trust_remote_code=True, token=model_tk, ) tokenizer = AutoTokenizer.from_pretrained( model_id, trust_remote_code=True, token=model_tk, ) @spaces.GPU(duration=60) def generate( message: str, chat_history: list[tuple[str, str]], system_prompt: str, max_new_tokens: int = 1024, temperature: float = 0.4, top_p: float = 0.95, top_k: int = 50, repetition_penalty: float = 1.0, ) -> Iterator[str]: conversation = [] if system_prompt: conversation.append({"role": "system", "content": system_prompt}) for user, assistant in chat_history: conversation.extend( [ {"role": "user", "content": user}, {"role": "assistant", "content": assistant}, ] ) conversation.append({"role": "user", "content": message}) input_ids = tokenizer.apply_chat_template( conversation, add_generation_prompt=True, return_tensors="pt" ) input_ids = input_ids.to(model.device) if input_ids.shape[1] > MAX_INPUT_TOKEN_LENGTH: input_ids = input_ids[:, -MAX_INPUT_TOKEN_LENGTH:] gr.Warning( f"Trimmed input from conversation as it was longer than {MAX_INPUT_TOKEN_LENGTH} tokens." ) streamer = TextIteratorStreamer( tokenizer, timeout=10.0, skip_prompt=True, skip_special_tokens=True ) generate_kwargs = dict( input_ids=input_ids, streamer=streamer, max_new_tokens=max_new_tokens, do_sample=True, top_p=top_p, top_k=top_k, temperature=temperature, repetition_penalty=repetition_penalty, ) t = Thread(target=model.generate, kwargs=generate_kwargs) t.start() outputs = [] for text in streamer: outputs.append(text) yield "".join(outputs) chatbot = gr.Chatbot(height=400, placeholder=PLACEHOLDER, label="Ghost 8B Beta") chat_interface = gr.ChatInterface( fn=generate, chatbot=chatbot, fill_height=True, additional_inputs=[ gr.Textbox(label="System prompt", lines=6), gr.Slider( label="Max new tokens", minimum=1, maximum=MAX_MAX_NEW_TOKENS, step=1, value=DEFAULT_MAX_NEW_TOKENS, ), gr.Slider( label="Temperature", minimum=0.1, maximum=2.0, step=0.1, value=0.4, ), gr.Slider( label="Top-p (nucleus sampling)", minimum=0.05, maximum=1.0, step=0.05, value=0.95, ), gr.Slider( label="Top-k", minimum=1, maximum=100, step=1, value=50, ), gr.Slider( label="Repetition penalty", minimum=1.0, maximum=2.0, step=0.05, value=1.0, ), ], stop_btn=None, cache_examples=False, examples=EXAMPLES, ) with gr.Blocks(fill_height=True, css="style.css") as demo: gr.Markdown(DESCRIPTION) chat_interface.render() gr.Markdown(LICENSE) if __name__ == "__main__": # demo.queue(max_size=20).launch() demo.launch()