fixed bug for lc_serve

app_modules/init.py

@@ -23,7 +23,7 @@ load_dotenv(found_dotenv, override=False)
 init_settings()
 
 
-def app_init():
+def app_init(lc_serve: bool = False):
     # https://github.com/huggingface/transformers/issues/17611
     os.environ["CURL_CA_BUNDLE"] = ""
 
@@ -69,7 +69,7 @@ def app_init():
     print(f"Completed in {end - start:.3f}s")
 
     start = timer()
-    llm_loader = LLMLoader(llm_model_type)
+    llm_loader = LLMLoader(llm_model_type, lc_serve)
     llm_loader.init(n_threds=n_threds, hf_pipeline_device_type=hf_pipeline_device_type)
     qa_chain = QAChain(vectorstore, llm_loader)
     end = timer()

app_modules/llm_loader.py

@@ -90,10 +90,12 @@ class LLMLoader:
     streamer: any
     max_tokens_limit: int
 
-    def __init__(self, llm_model_type, max_tokens_limit: int = 2048):
+    def __init__(
+        self, llm_model_type, max_tokens_limit: int = 2048, lc_serve: bool = False
+    ):
         self.llm_model_type = llm_model_type
         self.llm = None
-        self.streamer = None
+        self.streamer = None if lc_serve else TextIteratorStreamer("")
         self.max_tokens_limit = max_tokens_limit
         self.search_kwargs = {"k": 4}
 

server.py

@@ -11,7 +11,7 @@ from app_modules.init import app_init
 from app_modules.llm_chat_chain import ChatChain
 from app_modules.utils import print_llm_response
 
-llm_loader, qa_chain = app_init()
+llm_loader, qa_chain = app_init(True)
 
 chat_history_enabled = os.environ.get("CHAT_HISTORY_ENABLED") == "true"