Update run.py

run.py

@@ -2,6 +2,26 @@ import gradio as gr
 import random
 import time
 
+endpoint_url = "https://ko60a2m26ylqgri6.us-east-1.aws.endpoints.huggingface.cloud"
+
+
+# Streaming Client
+client = InferenceClient(endpoint_url, token=hf_token)
+
+# generation parameter
+gen_kwargs = dict(
+    max_new_tokens=1024,
+    top_k=30,
+    top_p=0.9,
+    temperature=0.2,
+    repetition_penalty=1.05, #1.02
+    stop=["\nUser:", "<|endoftext|>", "</s>"],
+)
+# prompt
+# prompt = "What can you do in Nuremberg, Germany? Give me 3 Tips"
+
+
+
 with gr.Blocks() as demo:
     chatbot = gr.Chatbot()
     msg = gr.Textbox()
@@ -11,11 +31,10 @@ with gr.Blocks() as demo:
         return "", history + [[user_message, None]]
 
     def bot(history):
-        bot_message = random.choice(["How are you?", "I love you", "I'm very hungry"])
+        bot_message = client.text_generation(history, stream=True, details=True, **gen_kwargs)
         history[-1][1] = ""
         for character in bot_message:
             history[-1][1] += character
-            time.sleep(0.05)
             yield history
 
     msg.submit(user, [msg, chatbot], [msg, chatbot], queue=False).then(