fixed streaming issue

app_modules/init.py

@@ -23,7 +23,7 @@ load_dotenv(found_dotenv, override=False)
 init_settings()
 
 
-def app_init(lc_serve: bool = False):
+def app_init():
     # https://github.com/huggingface/transformers/issues/17611
     os.environ["CURL_CA_BUNDLE"] = ""
 
@@ -69,7 +69,7 @@ def app_init(lc_serve: bool = False):
     print(f"Completed in {end - start:.3f}s")
 
     start = timer()
-    llm_loader = LLMLoader(llm_model_type, lc_serve)
+    llm_loader = LLMLoader(llm_model_type)
     llm_loader.init(n_threds=n_threds, hf_pipeline_device_type=hf_pipeline_device_type)
     qa_chain = QAChain(vectorstore, llm_loader)
     end = timer()

app_modules/llm_loader.py

@@ -91,10 +91,13 @@ class LLMLoader:
     max_tokens_limit: int
     lock: any
 
-    def __init__(self, llm_model_type, lc_serve: bool = False):
+    def __init__(self, llm_model_type):
         self.llm_model_type = llm_model_type
         self.llm = None
-        self.streamer = TextIteratorStreamer("")
+        self.streamer = TextIteratorStreamer(
+            "",
+            for_huggingface=True,
+        )
         self.max_tokens_limit = 2048
         self.search_kwargs = {"k": 4}
         self.lock = threading.Lock()

server.py

@@ -11,7 +11,7 @@ from app_modules.init import app_init
 from app_modules.llm_chat_chain import ChatChain
 from app_modules.utils import print_llm_response
 
-llm_loader, qa_chain = app_init(__name__ != "__main__")
+llm_loader, qa_chain = app_init()
 
 chat_history_enabled = os.environ.get("CHAT_HISTORY_ENABLED") == "true"