add stream

app.py

@@ -5,12 +5,12 @@ model = "Qwen/Qwen2-7B-Instruct-GGUF"
 llm = Llama.from_pretrained(
     repo_id=model,
     filename="qwen2-7b-instruct-q4_k_m.gguf",
-    verbose=False,
+    verbose=True,
     use_mmap=False,
     use_mlock=True,
     n_threads=2,
     n_threads_batch=2,
-    n_ctx=40000,
+    n_ctx=8000,
 )
 
 
@@ -32,13 +32,26 @@ def respond(
 
     messages.append({"role": "user", "content": message})
 
-    response = llm.create_chat_completion(
-        messages=messages,
+    # response = llm.create_chat_completion(
+    #     messages=messages,
+    #     max_tokens=max_tokens,
+    #     temperature=temperature,
+    #     top_p=top_p,
+    # )
+    # return response["choices"][0]["message"]["content"]
+    response = ""
+
+    for message in llm.create_chat_completion(
+        messages,
         max_tokens=max_tokens,
+        stream=True,
         temperature=temperature,
         top_p=top_p,
-    )
-    return response["choices"][0]["message"]["content"]
+    ):
+        token = message.choices[0].delta.content
+
+        response += token
+        yield response
 
 
 demo = gr.ChatInterface(
@@ -48,7 +61,7 @@ demo = gr.ChatInterface(
             value="You are a helpful assistant.",
             label="System message",
         ),
-        gr.Slider(minimum=1, maximum=2048, value=512, step=1, label="Max new tokens"),
+        gr.Slider(minimum=1, maximum=2048, value=1024, step=1, label="Max new tokens"),
         gr.Slider(minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"),
         gr.Slider(
             minimum=0.1,