get latest code from chat-with-pci-dss-v4

.env.example

@@ -8,6 +8,9 @@ LLM_MODEL_TYPE=huggingface
 
 OPENAI_API_KEY=
 
+# if unset, default to "gpt-4"
+OPENAI_MODEL_NAME=
+
 # cpu, mps or cuda:0 - if unset, use whatever detected
 HF_EMBEDDINGS_DEVICE_TYPE=
 HF_PIPELINE_DEVICE_TYPE=
@@ -16,9 +19,11 @@ HF_PIPELINE_DEVICE_TYPE=
 # LOAD_QUANTIZED_MODEL=4bit
 # LOAD_QUANTIZED_MODEL=8bit
 
+DISABLE_MODEL_PRELOADING=false
 CHAT_HISTORY_ENABLED=true
 SHOW_PARAM_SETTINGS=false
-PDF_FILE_BASE_URL=https://ai-engd.netlify.app/pdfs/books/
+SHARE_GRADIO_APP=false
+PDF_FILE_BASE_URL=https://ai-engd.netlify.app/pdfs/pci_dss_v4/
 
 # if unset, default to "hkunlp/instructor-xl"
 HF_EMBEDDINGS_MODEL_NAME="hkunlp/instructor-large"
@@ -26,6 +31,8 @@ HF_EMBEDDINGS_MODEL_NAME="hkunlp/instructor-large"
 # number of cpu cores - used to set n_threads for GPT4ALL & LlamaCpp models
 NUMBER_OF_CPU_CORES=
 
+HUGGINGFACE_AUTH_TOKEN=
+
 USING_TORCH_BFLOAT16=true
 # HUGGINGFACE_MODEL_NAME_OR_PATH="databricks/dolly-v2-3b"
 # HUGGINGFACE_MODEL_NAME_OR_PATH="databricks/dolly-v2-7b"
@@ -36,14 +43,14 @@ USING_TORCH_BFLOAT16=true
 # HUGGINGFACE_MODEL_NAME_OR_PATH="TheBloke/vicuna-7B-1.1-HF"
 # HUGGINGFACE_MODEL_NAME_OR_PATH="nomic-ai/gpt4all-j"
 # HUGGINGFACE_MODEL_NAME_OR_PATH="nomic-ai/gpt4all-falcon"
-HUGGINGFACE_MODEL_NAME_OR_PATH="lmsys/fastchat-t5-3b-v1.0"
+# HUGGINGFACE_MODEL_NAME_OR_PATH="lmsys/fastchat-t5-3b-v1.0"
+HUGGINGFACE_MODEL_NAME_OR_PATH="meta-llama/Llama-2-7b-chat-hf"
+# HUGGINGFACE_MODEL_NAME_OR_PATH="meta-llama/Llama-2-13b-chat-hf"
+# HUGGINGFACE_MODEL_NAME_OR_PATH="meta-llama/Llama-2-70b-chat-hf"
 
-# STABLELM_MODEL_NAME_OR_PATH="./models/stablelm-base-alpha-7b"
-# STABLELM_MODEL_NAME_OR_PATH="./models/stablelm-tuned-alpha-7b"
 STABLELM_MODEL_NAME_OR_PATH="OpenAssistant/stablelm-7b-sft-v7-epoch-3"
 
-# MOSAICML_MODEL_NAME_OR_PATH="mosaicml/mpt-7b-instruct"
-MOSAICML_MODEL_NAME_OR_PATH="mosaicml/mpt-1b-redpajama-200b-dolly"
+MOSAICML_MODEL_NAME_OR_PATH="mosaicml/mpt-7b-instruct"
 
 FALCON_MODEL_NAME_OR_PATH="tiiuae/falcon-7b-instruct"
 
@@ -66,6 +73,6 @@ TOKENIZERS_PARALLELISM=true
 
 # env variables for ingesting source PDF files
 SOURCE_PDFS_PATH="./data/pdfs/"
-SOURCE_URLS=
+SOURCE_URLS="./data/pci_dss_urls.txt"
 CHUNCK_SIZE=1024
 CHUNK_OVERLAP=512

.gitattributes

@@ -35,3 +35,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
 data/faiss_1024_512/index.faiss filter=lfs diff=lfs merge=lfs -text
 data/faiss_1024_512/index.pkl filter=lfs diff=lfs merge=lfs -text
+data/pci_dss_v4/index.faiss filter=lfs diff=lfs merge=lfs -text
+data/pci_dss_v4/index.pkl filter=lfs diff=lfs merge=lfs -text

Makefile

@@ -2,6 +2,13 @@
 start:
 	python app.py
 	
+serve:
+ifeq ("$(PORT)", "")
+	JINA_HIDE_SURVEY=1 TRANSFORMERS_OFFLINE=1 python -m lcserve deploy local server
+else
+	JINA_HIDE_SURVEY=1 TRANSFORMERS_OFFLINE=1 python -m lcserve deploy local server --port=${PORT}
+endif
+
 test:
 	PYTORCH_MPS_HIGH_WATERMARK_RATIO=0.0 TRANSFORMERS_OFFLINE=1 python test.py
 
@@ -21,9 +28,14 @@ format:
 	black .
 
 install:
-	CXX=g++-11  CC=gcc-11 pip install -U -r requirements.txt
-	pip show langchain llama-cpp-python transformers
-	
-mac-install:
 	pip install -U -r requirements.txt
 	pip show langchain transformers
+	
+install-extra:
+	CXX=g++-11  CC=gcc-11 pip install -U -r requirements_extra.txt
+	pip show langchain llama-cpp-python transformers
+	
+install-extra-mac:
+	# brew install llvm libomp
+	CXX=/usr/local/opt/llvm/bin/clang++ CC=/usr/local/opt/llvm/bin/clang pip install -U -r requirements_extra.txt
+	pip show langchain llama-cpp-python transformers

app.py

@@ -33,6 +33,7 @@ using_faiss = os.environ.get("FAISS_INDEX_PATH") is not None
 llm_model_type = os.environ.get("LLM_MODEL_TYPE")
 chat_history_enabled = os.environ.get("CHAT_HISTORY_ENABLED") == "true"
 show_param_settings = os.environ.get("SHOW_PARAM_SETTINGS") == "true"
+share_gradio_app = os.environ.get("SHARE_GRADIO_APP") == "true"
 
 
 streaming_enabled = True  # llm_model_type in ["openai", "llamacpp"]
@@ -76,7 +77,9 @@ def qa(chatbot):
 
     def task(question, chat_history):
         start = timer()
-        ret = qa_chain.call({"question": question, "chat_history": chat_history}, q)
+        ret = qa_chain.call(
+            {"question": question, "chat_history": chat_history}, None, q
+        )
         end = timer()
 
         print(f"Completed in {end - start:.3f}s")
@@ -203,7 +206,7 @@ with gr.Blocks(css=customCSS, theme=small_and_beautiful_theme) as demo:
     ).then(qa, chatbot, chatbot)
 
     submitBtn.click(
-        chat, [user_input, chatbot], [user_input, chatbot], queue=True
+        chat, [user_input, chatbot], [user_input, chatbot], queue=True, api_name="chat"
     ).then(qa, chatbot, chatbot)
 
     def reset():
@@ -213,7 +216,8 @@ with gr.Blocks(css=customCSS, theme=small_and_beautiful_theme) as demo:
         reset,
         outputs=[user_input, chatbot],
         show_progress=True,
+        api_name="reset",
     )
 
-demo.title = "Chat with AI Books"
-demo.queue(concurrency_count=1).launch()
+demo.title = "Chat with PCI DSS v4"
+demo.queue(concurrency_count=1).launch(share=share_gradio_app)

app_modules/presets.py

@@ -3,15 +3,19 @@ import os
 
 import gradio as gr
 
+from app_modules.utils import *
+
 using_openai = os.environ.get("LLM_MODEL_TYPE") == "openai"
 href = (
     "https://openai.com/gpt-4"
     if using_openai
     else "https://huggingface.co/lmsys/fastchat-t5-3b-v1.0"
 )
-model = "OpenAI GPT-4" if using_openai else "lmsys/fastchat-t5-3b-v1.0"
+model = (
+    "OpenAI GPT-4" if using_openai else os.environ.get("HUGGINGFACE_MODEL_NAME_OR_PATH")
+)
 
-title = """<h1 align="left" style="min-width:200px; margin-top:0;"> Chat with AI Books </h1>"""
+title = """<h1 align="left" style="min-width:200px; margin-top:0;"> Chat with PCI DSS v4 </h1>"""
 
 description_top = f"""\
 <div align="left">

app_modules/qa_chain.py

@@ -1,7 +1,9 @@
 import os
 import sys
+import time
 import urllib
 from queue import Queue
+from threading import Thread
 from typing import Any, Optional
 
 import torch
@@ -78,6 +80,9 @@ class TextIteratorStreamer(TextStreamer, StreamingStdOutCallbackHandler):
         # print("resetting TextIteratorStreamer")
         self.text_queue = q if q is not None else Queue()
 
+    def empty(self):
+        return self.text_queue.empty()
+
 
 class QAChain:
     llm_model_type: str
@@ -177,6 +182,17 @@ class QAChain:
                 MODEL_NAME_OR_PATH = os.environ.get("HUGGINGFACE_MODEL_NAME_OR_PATH")
                 print(f"            loading model: {MODEL_NAME_OR_PATH}")
 
+                hf_auth_token = os.environ.get("HUGGINGFACE_AUTH_TOKEN")
+                transformers_offline = os.environ.get("TRANSFORMERS_OFFLINE") == "1"
+                token = (
+                    hf_auth_token
+                    if hf_auth_token is not None
+                    and len(hf_auth_token) > 0
+                    and not transformers_offline
+                    else None
+                )
+                print(f"            HF auth token: {str(token)[-5:]}")
+
                 is_t5 = "t5" in MODEL_NAME_OR_PATH
                 temperature = (
                     0.01
@@ -192,20 +208,26 @@ class QAChain:
                 padding_side = "left"  # if "dolly" in MODEL_NAME_OR_PATH else None
 
                 config = AutoConfig.from_pretrained(
-                    MODEL_NAME_OR_PATH, trust_remote_code=True
+                    MODEL_NAME_OR_PATH,
+                    trust_remote_code=True,
+                    token=token,
                 )
                 # config.attn_config["attn_impl"] = "triton"
                 # config.max_seq_len = 4096
                 config.init_device = hf_pipeline_device_type
 
                 tokenizer = (
-                    T5Tokenizer.from_pretrained(MODEL_NAME_OR_PATH)
+                    T5Tokenizer.from_pretrained(
+                        MODEL_NAME_OR_PATH,
+                        token=token,
+                    )
                     if is_t5
                     else AutoTokenizer.from_pretrained(
                         MODEL_NAME_OR_PATH,
                         use_fast=use_fast,
                         trust_remote_code=True,
                         padding_side=padding_side,
+                        token=token,
                     )
                 )
 
@@ -228,6 +250,7 @@ class QAChain:
                             config=config,
                             quantization_config=double_quant_config,
                             trust_remote_code=True,
+                            token=token,
                         )
                         if is_t5
                         else AutoModelForCausalLM.from_pretrained(
@@ -235,6 +258,7 @@ class QAChain:
                             config=config,
                             quantization_config=double_quant_config,
                             trust_remote_code=True,
+                            token=token,
                         )
                     )
 
@@ -310,11 +334,34 @@ class QAChain:
                         temperature=temperature,
                         return_full_text=True,
                         repetition_penalty=repetition_penalty,
+                        token=token,
                     )
                 else:
+                    if os.environ.get("DISABLE_MODEL_PRELOADING") != "true":
+                        use_auth_token = None
+                        model = (
+                            AutoModelForSeq2SeqLM.from_pretrained(
+                                MODEL_NAME_OR_PATH,
+                                config=config,
+                                trust_remote_code=True,
+                                token=token,
+                            )
+                            if is_t5
+                            else AutoModelForCausalLM.from_pretrained(
+                                MODEL_NAME_OR_PATH,
+                                config=config,
+                                trust_remote_code=True,
+                                token=token,
+                            )
+                        )
+                        print(f"Model memory footprint: {model.get_memory_footprint()}")
+                    else:
+                        use_auth_token = token
+                        model = MODEL_NAME_OR_PATH
+
                     pipe = pipeline(
-                        task,  # model=model,
-                        model=MODEL_NAME_OR_PATH,
+                        task,
+                        model=model,
                         tokenizer=tokenizer,
                         streamer=self.streamer,
                         return_full_text=return_full_text,  # langchain expects the full text
@@ -322,11 +369,11 @@ class QAChain:
                         torch_dtype=torch_dtype,
                         max_new_tokens=2048,
                         trust_remote_code=True,
-                        # verbose=True,
                         temperature=temperature,
                         top_p=0.95,
                         top_k=0,  # select from top 0 tokens (because zero, relies on top_p)
                         repetition_penalty=1.115,
+                        token=use_auth_token,
                     )
 
                 self.llm = HuggingFacePipeline(pipeline=pipe, callbacks=callbacks)
@@ -386,7 +433,7 @@ class QAChain:
                 self.search_kwargs = (
                     {"k": 8} if "30b" in MODEL_NAME_OR_PATH else self.search_kwargs
                 )
-                repetition_penalty = 1.0005 if "30b" in MODEL_NAME_OR_PATH else 1.02
+                repetition_penalty = 1.05 if "30b" in MODEL_NAME_OR_PATH else 1.02
 
                 pipe = (
                     pipeline(
@@ -524,7 +571,7 @@ class QAChain:
 
         return qa
 
-    def call(self, inputs, q: Queue = None, tracing: bool = False):
+    def call(self, inputs, streaming_handler, q: Queue = None, tracing: bool = False):
         print(inputs)
 
         if self.streamer is not None and isinstance(
@@ -533,7 +580,15 @@ class QAChain:
             self.streamer.reset(q)
 
         qa = self.get_chain(tracing)
-        result = qa(inputs)
+        result = (
+            self._run_qa_chain(
+                qa,
+                inputs,
+                streaming_handler,
+            )
+            if streaming_handler is not None
+            else qa(inputs)
+        )
 
         result["answer"] = remove_extra_spaces(result["answer"])
 
@@ -546,3 +601,31 @@ class QAChain:
                 doc.metadata["url"] = f"{base_url}{urllib.parse.quote(title)}"
 
         return result
+
+    def _run_qa_chain(self, qa, inputs, streaming_handler):
+        que = Queue()
+
+        t = Thread(
+            target=lambda qa, inputs, q, sh: q.put(qa(inputs, callbacks=[sh])),
+            args=(qa, inputs, que, streaming_handler),
+        )
+        t.start()
+
+        if self.streamer is not None and isinstance(
+            self.streamer, TextIteratorStreamer
+        ):
+            count = 2 if len(inputs.get("chat_history")) > 0 else 1
+
+            while count > 0:
+                try:
+                    for token in self.streamer:
+                        streaming_handler.on_llm_new_token(token)
+
+                    self.streamer.reset()
+                    count -= 1
+                except Exception:
+                    print("nothing generated yet - retry in 0.5s")
+                    time.sleep(0.5)
+
+        t.join()
+        return que.get()

app_modules/utils.py

@@ -88,7 +88,9 @@ def print_llm_response(llm_response):
             + " Source: "
             + str(metadata["url"] if "url" in metadata else metadata["source"])
         )
-        print(source.page_content)
+        print(
+            source["page_content"] if "page_content" in source else source.page_content
+        )
 
 
 def get_device_types():

data/pci_dss_v4/index.faiss

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98e8c49e6c3ef2bcd0b258fb51ffe58fa92a63544b672f1c0c75857593afa2a8
+size 5987373

data/pci_dss_v4/index.pkl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8467d3647bf272f11151a512f54515ef6dd83f2081686156a437132380b28b4b
+size 2035755

data/questions.txt

@@ -1,4 +1,3 @@
-What's AI?
-life in AI era
-machine learning
-generative model
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f61185685e79b9b115f1b3d34c6bca2913174a18c014b210e749e419beb2211
+size 60

requirements.txt

@@ -6,14 +6,11 @@ socksio
 tqdm
 colorama
 accelerate
-Pygments
-llama_index
 langchain
 torch
-langchain
-protobuf==3.20.*
+langchain-serve
+protobuf
 faiss-cpu
-chromadb 
 sentence_transformers 
 InstructorEmbedding
 python-dotenv
@@ -25,8 +22,6 @@ git+https://github.com/huggingface/transformers.git
 SentencePiece
 isort
 black
-llama-cpp-python
-pyllamacpp
 pygpt4all
 tiktoken
 safetensors
@@ -34,4 +29,5 @@ xformers
 bitsandbytes
 einops
 gevent
-pydantic >= 1.10.11
+pydantic >= 1.10.11
+pypdf

requirements_extra.txt

@@ -0,0 +1,3 @@
+llama-cpp-python
+pyllamacpp
+chromadb

server.py

@@ -0,0 +1,109 @@
+"""Main entrypoint for the app."""
+import json
+import os
+import time
+from queue import Queue
+from timeit import default_timer as timer
+from typing import List, Optional
+
+from langchain.embeddings import HuggingFaceInstructEmbeddings
+from langchain.vectorstores.chroma import Chroma
+from langchain.vectorstores.faiss import FAISS
+from lcserve import serving
+from pydantic import BaseModel
+
+from app_modules.presets import *
+from app_modules.qa_chain import QAChain
+from app_modules.utils import *
+
+# Constants
+init_settings()
+
+# https://github.com/huggingface/transformers/issues/17611
+os.environ["CURL_CA_BUNDLE"] = ""
+
+hf_embeddings_device_type, hf_pipeline_device_type = get_device_types()
+print(f"hf_embeddings_device_type: {hf_embeddings_device_type}")
+print(f"hf_pipeline_device_type: {hf_pipeline_device_type}")
+
+hf_embeddings_model_name = (
+    os.environ.get("HF_EMBEDDINGS_MODEL_NAME") or "hkunlp/instructor-xl"
+)
+n_threds = int(os.environ.get("NUMBER_OF_CPU_CORES") or "4")
+index_path = os.environ.get("FAISS_INDEX_PATH") or os.environ.get("CHROMADB_INDEX_PATH")
+using_faiss = os.environ.get("FAISS_INDEX_PATH") is not None
+llm_model_type = os.environ.get("LLM_MODEL_TYPE")
+chat_history_enabled = os.environ.get("CHAT_HISTORY_ENABLED") == "true"
+show_param_settings = os.environ.get("SHOW_PARAM_SETTINGS") == "true"
+share_gradio_app = os.environ.get("SHARE_GRADIO_APP") == "true"
+
+
+streaming_enabled = True  # llm_model_type in ["openai", "llamacpp"]
+
+start = timer()
+embeddings = HuggingFaceInstructEmbeddings(
+    model_name=hf_embeddings_model_name,
+    model_kwargs={"device": hf_embeddings_device_type},
+)
+end = timer()
+
+print(f"Completed in {end - start:.3f}s")
+
+start = timer()
+
+print(f"Load index from {index_path} with {'FAISS' if using_faiss else 'Chroma'}")
+
+if not os.path.isdir(index_path):
+    raise ValueError(f"{index_path} does not exist!")
+elif using_faiss:
+    vectorstore = FAISS.load_local(index_path, embeddings)
+else:
+    vectorstore = Chroma(embedding_function=embeddings, persist_directory=index_path)
+
+end = timer()
+
+print(f"Completed in {end - start:.3f}s")
+
+start = timer()
+qa_chain = QAChain(vectorstore, llm_model_type)
+qa_chain.init(n_threds=n_threds, hf_pipeline_device_type=hf_pipeline_device_type)
+end = timer()
+print(f"Completed in {end - start:.3f}s")
+
+
+class ChatResponse(BaseModel):
+    """Chat response schema."""
+
+    token: Optional[str] = None
+    error: Optional[str] = None
+    sourceDocs: Optional[List] = None
+
+
+@serving(websocket=True)
+def chat(question: str, history: Optional[List], **kwargs) -> str:
+    # Get the `streaming_handler` from `kwargs`. This is used to stream data to the client.
+    streaming_handler = kwargs.get("streaming_handler") if streaming_enabled else None
+    chat_history = []
+    if chat_history_enabled:
+        for element in history:
+            item = (element[0] or "", element[1] or "")
+            chat_history.append(item)
+
+    start = timer()
+    result = qa_chain.call(
+        {"question": question, "chat_history": chat_history}, streaming_handler
+    )
+    end = timer()
+    print(f"Completed in {end - start:.3f}s")
+
+    resp = ChatResponse(sourceDocs=result["source_documents"])
+
+    if not streaming_enabled:
+        resp.token = remove_extra_spaces(result["answer"])
+        print(resp.token)
+
+    return json.dumps(resp.dict())
+
+
+if __name__ == "__main__":
+    print_llm_response(json.loads(chat("What is PCI DSS?", [])))

test.py

@@ -29,8 +29,9 @@ hf_embeddings_model_name = (
     os.environ.get("HF_EMBEDDINGS_MODEL_NAME") or "hkunlp/instructor-xl"
 )
 n_threds = int(os.environ.get("NUMBER_OF_CPU_CORES") or "4")
-index_path = os.environ.get("FAISS_INDEX_PATH") or os.environ.get("CHROMADB_INDEX_PATH")
-using_faiss = os.environ.get("FAISS_INDEX_PATH") is not None
+faiss_index_path = os.environ.get("FAISS_INDEX_PATH") or ""
+using_faiss = len(faiss_index_path) > 0
+index_path = faiss_index_path if using_faiss else os.environ.get("CHROMADB_INDEX_PATH")
 llm_model_type = os.environ.get("LLM_MODEL_TYPE")
 chatting = len(sys.argv) > 1 and sys.argv[1] == "chat"
 questions_file_path = os.environ.get("QUESTIONS_FILE_PATH")

test.sh

@@ -11,56 +11,69 @@ echo Using extension: $EXT
 
 [ ! -f .env ] || export $(grep -v '^#' .env | xargs)
 
-LLM_MODEL_TYPE=huggingface
+export LLM_MODEL_TYPE=openai
+export OPENAI_MODEL_NAME="gpt-3.5-turbo"
+echo Testing openai-${OPENAI_MODEL_NAME}
+python test.py 2>&1 | tee ./data/logs/openai-${OPENAI_MODEL_NAME}_${EXT}.log
 
-HUGGINGFACE_MODEL_NAME_OR_PATH="lmsys/fastchat-t5-3b-v1.0"
+export OPENAI_MODEL_NAME="gpt-4"
+echo Testing openai-${OPENAI_MODEL_NAME}
+python test.py 2>&1 | tee ./data/logs/openai-${OPENAI_MODEL_NAME}_${EXT}.log
+
+export LLM_MODEL_TYPE=huggingface
+
+export HUGGINGFACE_MODEL_NAME_OR_PATH="lmsys/fastchat-t5-3b-v1.0"
 echo Testing $HUGGINGFACE_MODEL_NAME_OR_PATH
 python test.py 2>&1 | tee ./data/logs/fastchat-t5-3b-v1.0_${EXT}.log
 
 
-HUGGINGFACE_MODEL_NAME_OR_PATH="TheBloke/wizardLM-7B-HF"
+export HUGGINGFACE_MODEL_NAME_OR_PATH="TheBloke/wizardLM-7B-HF"
 echo Testing $HUGGINGFACE_MODEL_NAME_OR_PATH
 python test.py 2>&1 | tee ./data/logs/wizardLM-7B-HF_${EXT}.log
 
 
-HUGGINGFACE_MODEL_NAME_OR_PATH="TheBloke/vicuna-7B-1.1-HF"
+export HUGGINGFACE_MODEL_NAME_OR_PATH="TheBloke/vicuna-7B-1.1-HF"
 echo Testing $HUGGINGFACE_MODEL_NAME_OR_PATH
 python test.py 2>&1 | tee ./data/logs/vicuna-7B-1.1-HF_${EXT}.log
 
 
-HUGGINGFACE_MODEL_NAME_OR_PATH="nomic-ai/gpt4all-j"
+export HUGGINGFACE_MODEL_NAME_OR_PATH="nomic-ai/gpt4all-j"
 echo Testing $HUGGINGFACE_MODEL_NAME_OR_PATH
 python test.py 2>&1 | tee ./data/logs/gpt4all-j_${EXT}.log
 
 
-# HUGGINGFACE_MODEL_NAME_OR_PATH="nomic-ai/gpt4all-falcon"
+# export HUGGINGFACE_MODEL_NAME_OR_PATH="nomic-ai/gpt4all-falcon"
 # echo Testing $HUGGINGFACE_MODEL_NAME_OR_PATH
 # python test.py 2>&1 | tee ./data/logs/gpt4all-falcon_${EXT}.log
 
-LLM_MODEL_TYPE=stablelm
+export LLM_MODEL_TYPE=stablelm
 
-STABLELM_MODEL_NAME_OR_PATH="stabilityai/stablelm-tuned-alpha-7b"
-echo Testing $STABLELM_MODEL_NAME_OR_PATH
-python test.py 2>&1 | tee ./data/logs/stablelm-tuned-alpha-7b_${EXT}.log
+# export STABLELM_MODEL_NAME_OR_PATH="stabilityai/stablelm-tuned-alpha-7b"
+# echo Testing $STABLELM_MODEL_NAME_OR_PATH
+# python test.py 2>&1 | tee ./data/logs/stablelm-tuned-alpha-7b_${EXT}.log
 
 
-STABLELM_MODEL_NAME_OR_PATH="OpenAssistant/stablelm-7b-sft-v7-epoch-3"
+export STABLELM_MODEL_NAME_OR_PATH="OpenAssistant/stablelm-7b-sft-v7-epoch-3"
 echo Testing $STABLELM_MODEL_NAME_OR_PATH
 python test.py 2>&1 | tee ./data/logs/stablelm-7b-sft-v7-epoch-3_${EXT}.log
 
 
-LLM_MODEL_TYPE=mosaicml
-MOSAICML_MODEL_NAME_OR_PATH="nomic-ai/gpt4all-mpt"
+export LLM_MODEL_TYPE=mosaicml
+export MOSAICML_MODEL_NAME_OR_PATH="mosaicml/mpt-7b-instruct"
 echo Testing $MOSAICML_MODEL_NAME_OR_PATH
-python test.py 2>&1 | tee ./data/logs/gpt4all-mpt_${EXT}.log
+python test.py 2>&1 | tee ./data/logs/mpt-7b-instruct_${EXT}.log
 
 
-LLM_MODEL_TYPE=huggingface
-HUGGINGFACE_MODEL_NAME_OR_PATH="HuggingFaceH4/starchat-beta"
+# export MOSAICML_MODEL_NAME_OR_PATH="mosaicml/mpt-30b-instruct"
+# echo Testing $MOSAICML_MODEL_NAME_OR_PATH
+# LOAD_QUANTIZED_MODEL=4bit python test.py 2>&1 | tee ./data/logs/mpt-30b-instruct_${EXT}.log
+
+export LLM_MODEL_TYPE=huggingface
+export HUGGINGFACE_MODEL_NAME_OR_PATH="HuggingFaceH4/starchat-beta"
 echo Testing $HUGGINGFACE_MODEL_NAME_OR_PATH
 LOAD_QUANTIZED_MODEL=8bit python test.py 2>&1 | tee ./data/logs/starchat-beta_${EXT}.log
 
 
-HUGGINGFACE_MODEL_NAME_OR_PATH="../../models/starcoder"
-echo Testing $HUGGINGFACE_MODEL_NAME_OR_PATH
-LOAD_QUANTIZED_MODEL=8bit python test.py 2>&1 | tee ./data/logs/starcoder_${EXT}.log
+# export HUGGINGFACE_MODEL_NAME_OR_PATH="../../models/starcoder"
+# echo Testing $HUGGINGFACE_MODEL_NAME_OR_PATH
+# LOAD_QUANTIZED_MODEL=8bit python test.py 2>&1 | tee ./data/logs/starcoder_${EXT}.log