Spaces:
Sleeping
Sleeping
File size: 2,885 Bytes
04e5516 fd91f3c 04e5516 f4ffb0a 04e5516 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 |
import spaces
import os
import torch
from transformers import AutoTokenizer, AutoModelForCausalLM, pipeline
import gradio as gr
huggingface_token = os.getenv("HUGGINGFACE_TOKEN")
if not huggingface_token:
pass
print("no HUGGINGFACE_TOKEN if you need set secret ")
#raise ValueError("HUGGINGFACE_TOKEN environment variable is not set")
model_id = "openbmb/MiniCPM-2B-dpo-bf16"
device = "auto" # torch.device("cuda" if torch.cuda.is_available() else "cpu")
dtype = torch.bfloat16
tokenizer = AutoTokenizer.from_pretrained(model_id,trust_remote_code=True)
print(model_id,device,dtype)
histories = []
#model = None
def call_generate_text(prompt, system_message="You are a helpful assistant."):
if prompt =="":
print("empty prompt return")
return ""
global histories
#global model
#if model != None:# and model.is_cuda:
# print("Model is alive")
#else:
# model = AutoModelForCausalLM.from_pretrained(
# model_id, token=huggingface_token ,torch_dtype=dtype,device_map=device
#)
messages = [
{"role": "system", "content": system_message},
]
messages += histories
user_message = {"role": "user", "content": prompt}
messages += [user_message]
try:
text = generate_text(messages)
histories += [user_message,{"role": "assistant", "content": text}]
#model.to("cpu")
return text
except RuntimeError as e:
print(f"An unexpected error occurred: {e}")
#model = None
return ""
iface = gr.Interface(
fn=call_generate_text,
inputs=[
gr.Textbox(lines=3, label="Input Prompt"),
gr.Textbox(lines=2, label="System Message", value="あなたは親切なアシスタントで常に日本語で返答します。"),
],
outputs=gr.Textbox(label="Generated Text"),
title=f"{model_id}",
description=f"{model_id} CPU",
)
print("Initialized")
model = AutoModelForCausalLM.from_pretrained(
model_id, token=huggingface_token ,torch_dtype=dtype,device_map=device,trust_remote_code=True
)
def generate_text(messages):
#model.to("cuda")
text_generator = pipeline("text-generation", model=model, tokenizer=tokenizer,torch_dtype=dtype,device_map=device) #pipeline has not to(device)
result = text_generator(messages, max_new_tokens=256, do_sample=True, temperature=0.7)
generated_output = result[0]["generated_text"]
if isinstance(generated_output, list):
for message in reversed(generated_output):
if message.get("role") == "assistant":
content= message.get("content", "No content found.")
return content
return "No assistant response found."
else:
return "Unexpected output format."
if __name__ == "__main__":
print("Main")
iface.launch() |