Spaces:
Runtime error
Runtime error
| from ctransformers import AutoModelForCausalLM,AutoConfig,AutoTokenizer | |
| from transformers import TextIteratorStreamer | |
| import torch | |
| import gradio as gr | |
| from threading import Thread | |
| hub_name = "StarkWizard/Mistral-7b-instruct-cairo-instruct-GGUF" | |
| model_file = "Mistral-7b-instruct-cairo-instruct.Q4_k.gguf" | |
| DEVICE,hw,layers = ("cpu",True,0) if torch.cuda.is_available() else ("cpu",False,0) | |
| print("loading LLM") | |
| # Load model | |
| config = AutoConfig.from_pretrained("TheBloke/Mistral-7B-v0.1-GGUF") | |
| config.max_seq_len = 4096 | |
| config.max_answer_len= 1024 | |
| model = AutoModelForCausalLM.from_pretrained(hub_name, model_file=model_file, model_type="mistral", gpu_layers=layers, | |
| config=config, | |
| compress_pos_emb=2, | |
| top_k=4000, | |
| top_p=0.99, | |
| temperature=0.0001, | |
| do_sample=True, | |
| ) | |
| def fmt_history(history) -> str: | |
| return "\n".join(["User: \"{usr_query}\", Assistant: \"{your_resp}\"".format( | |
| usr_query=usr_query.replace("\n",""), your_resp=your_resp.format("\n","")) | |
| for usr_query, your_resp in history]) | |
| def run_generation(user_text, top_p, temperature, top_k, max_new_tokens): | |
| text =f""" | |
| [INST] | |
| <<SYS>> | |
| A student asks you a question about Cairo 1. Provide a concise answer to the student's questions,do not expand the subject of the question, do not introduce any new topics or new question not provided by the student. | |
| Make sure the explanations never be longer than 300 words.Don’t justify your answers. Don’t give information not mentioned in the CONTEXT INFORMATION.provide only one solution <SYS>> | |
| Question: I'm working in Cairo 1 :{user_text} | |
| [/INST] | |
| """ | |
| model_output = "" | |
| for text in model(text, stream=True,max_new_tokens=max_new_tokens,top_p=top_p,top_k=top_k,temperature=temperature): | |
| model_output += text | |
| yield model_output | |
| return model_output | |
| def reset_textbox(): | |
| return gr.update(value='') | |
| with gr.Blocks() as demo: | |
| duplicate_link = "https://huggingface.co/spaces/joaogante/transformers_streaming?duplicate=true" | |
| gr.Markdown( | |
| "# 🔥 Mistral Cairo 🔥\n" | |
| f"[{hub_name}](https://huggingface.co/{hub_name})\n\n" | |
| ) | |
| with gr.Row(): | |
| with gr.Column(scale=4): | |
| # user_text = gr.Textbox( | |
| # placeholder="Write an email about an alpaca that likes flan", | |
| # label="User input" | |
| # ) | |
| # model_output = gr.Markdown(label="Model output", lines=10, interactive=False) | |
| # button_submit = gr.Button(value="Submit") | |
| chatbot = gr.Chatbot() | |
| msg = gr.Textbox() | |
| clear = gr.Button("Clear") | |
| def user(user_message, history): | |
| return "", history + [[user_message, None]] | |
| def respond(history): | |
| message = history[-1][0] | |
| print(f"User: {message}") | |
| print(f"top_p {top_p.value}, temperature {temperature.value}, top_k {top_k.value}, max_new_tokens {max_new_tokens.value}") | |
| bot_message = run_generation(message,top_p.value, temperature.value, top_k.value, max_new_tokens.value) | |
| for character in bot_message: | |
| history[-1][1] = character | |
| yield history | |
| msg.submit(user, [msg, chatbot], [msg, chatbot], queue=False).then(respond, chatbot, chatbot) | |
| clear.click(lambda: None, None, chatbot, queue=False) | |
| with gr.Column(scale=1): | |
| max_new_tokens = gr.Slider( | |
| minimum=1, maximum=2000, value=2000, step=1, interactive=True, label="Max New Tokens", | |
| ) | |
| top_p = gr.Slider( | |
| minimum=0.05, maximum=1.0, value=0.99, step=0.05, interactive=True, label="Top-p (nucleus sampling)", | |
| ) | |
| top_k = gr.Slider( | |
| minimum=40, maximum=5000, value=4000, step=10, interactive=True, label="Top-k", | |
| ) | |
| temperature = gr.Slider( | |
| minimum=0.01, maximum=0.4, value=0.0001, step=0.1, interactive=True, label="Temperature", | |
| ) | |
| # user_text.submit(run_generation, [user_text, top_p, temperature, top_k, max_new_tokens], model_output) | |
| # button_submit.click(run_generation, [user_text, top_p, temperature, top_k, max_new_tokens], model_output) | |
| #demo.queue(max_size=32).launch(enable_queue=True) | |
| demo.queue() | |
| demo.launch() | |