update

.env

@@ -0,0 +1,6 @@
+PERSIST_DIRECTORY=db
+MODEL_TYPE=dolly-v2-3b
+MODEL_PATH=databricks/dolly-v2-3b
+EMBEDDINGS_MODEL_NAME=sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
+MODEL_N_CTX=1000
+TARGET_SOURCE_CHUNKS=4

app.py

@@ -1,7 +1,264 @@
+# Disclamer: This code is not written by me. Its taken from https://github.com/imartinez/privateGPT/pull/91. 
+# All credit goes to `vnk8071` as I mentioned in the video.
+# As this code was still in the pull request while I was creating the video, did some modifications so that it works for me locally.
+import os 
+
+os.system('pip install -e ./langchain')
+
 import gradio as gr
+from dotenv import load_dotenv
+from langchain.callbacks.streaming_stdout import StreamingStdOutCallbackHandler
+from langchain.chains import RetrievalQA
+from langchain.embeddings import LlamaCppEmbeddings
+# from langchain.llms import GPT4All, LlamaCpp
+from langchain.vectorstores import Chroma
+from langchain.embeddings.huggingface import HuggingFaceEmbeddings
+from langchain.embeddings import HuggingFaceEmbeddings, HuggingFaceInstructEmbeddings#, SentenceTransformerEmbeddings 
+from langchain.prompts.prompt import PromptTemplate
+from langchain import PromptTemplate, LLMChain
+from langchain.llms import HuggingFacePipeline
+
+from training.generate import InstructionTextGenerationPipeline, load_model_tokenizer_for_generate
+# from googletrans import Translator
+# translator = Translator()
+
+load_dotenv()
+
+embeddings_model_name = os.environ.get("EMBEDDINGS_MODEL_NAME")
+persist_directory = os.environ.get('PERSIST_DIRECTORY')
+
+model_type = os.environ.get('MODEL_TYPE')
+model_path = os.environ.get('MODEL_PATH')
+model_n_ctx = int(os.environ.get('MODEL_N_CTX'))
+target_source_chunks = int(os.environ.get('TARGET_SOURCE_CHUNKS',4))
+
+# PERSIST_DIRECTORY=db
+# MODEL_TYPE=dolly-v2-3b
+# MODEL_PATH=/media/siiva/DataStore/LLMs/cache/dolly-v2-3b
+# EMBEDDINGS_MODEL_NAME=sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2
+# MODEL_N_CTX=1000
+# TARGET_SOURCE_CHUNKS=4
+
+
+from constants import CHROMA_SETTINGS
+# embeddings_model_name = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"
+# persist_directory = "db"
+# model_type = "dolly-v2-3b"
+# model_path = "/media/siiva/DataStore/LLMs/cache/dolly-v2-3b"
+# target_source_chunks = 3 
+# model_n_ctx = 1000
+
+embeddings = HuggingFaceEmbeddings(model_name=embeddings_model_name)
+db = Chroma(persist_directory=persist_directory, embedding_function=embeddings, client_settings=CHROMA_SETTINGS)
+retriever = db.as_retriever(search_kwargs={"k": target_source_chunks})
+# Prepare the LLM
+callbacks = [StreamingStdOutCallbackHandler()]
+
+match model_type:
+    case "dolly-v2-3b":
+        model, tokenizer = load_model_tokenizer_for_generate(model_path)
+        llm = HuggingFacePipeline(
+            pipeline=InstructionTextGenerationPipeline(
+                # Return the full text, because this is what the HuggingFacePipeline expects.
+                model=model, tokenizer=tokenizer, return_full_text=True, task="text-generation", max_new_tokens=model_n_ctx))#, max_new_tokens=model_n_ctx
+                #))
+    case "GPT4All":
+        llm = GPT4All(model=model_path, n_ctx=model_n_ctx, backend='gptj', callbacks=callbacks, verbose=False)
+    case _default:
+        print(f"Model {model_type} not supported!")
+        exit;    
+qa = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=retriever, return_source_documents=True)
+
+server_error_msg = "**NETWORK ERROR DUE TO HIGH TRAFFIC. PLEASE REGENERATE OR REFRESH THIS PAGE.**"
+
+def clear_history(request: gr.Request):
+    state = None
+    return ([], state, "")
+
+def post_process_code(code):
+    sep = "\n```"
+    if sep in code:
+        blocks = code.split(sep)
+        if len(blocks) % 2 == 1:
+            for i in range(1, len(blocks), 2):
+                blocks[i] = blocks[i].replace("\\_", "_")
+        code = sep.join(blocks)
+    return code
+
+def post_process_answer(answer, source):
+    answer += f"<br><br>Source: {source}"
+    answer = answer.replace("\n", "<br>")
+    return answer
+
+def predict(
+    question: str,
+    # system_content: str,
+    # embeddings_model_name: str,
+    # persist_directory: str,
+    # model_type: str,
+    # model_path: str,
+    # model_n_ctx: int,
+    # target_source_chunks: int,
+    chatbot: list = [],
+    history: list = [],
+):
+    # try:
+    # embeddings_model_name = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"
+    # persist_directory = "db"
+    # model_type = "dolly-v2-3b"
+    # model_path = "/media/siiva/DataStore/LLMs/cache/dolly-v2-3b"
+    # target_source_chunks = 3 
+    # model_n_ctx = 1000
+
+    # embeddings = HuggingFaceEmbeddings(model_name=embeddings_model_name)
+    # db = Chroma(persist_directory=persist_directory, embedding_function=embeddings, client_settings=CHROMA_SETTINGS)
+    # retriever = db.as_retriever(search_kwargs={"k": target_source_chunks})
+    # # Prepare the LLM
+    # callbacks = [StreamingStdOutCallbackHandler()]
+    
+    # match model_type:
+    #     case "dolly-v2-3b":
+    #         model, tokenizer = load_model_tokenizer_for_generate(model_path)
+    #         llm = HuggingFacePipeline(
+    #             pipeline=InstructionTextGenerationPipeline(
+    #                 # Return the full text, because this is what the HuggingFacePipeline expects.
+    #                 model=model, tokenizer=tokenizer, return_full_text=True, task="text-generation", max_new_tokens=model_n_ctx
+    #                 ))
+    #     case "GPT4All":
+    #         llm = GPT4All(model=model_path, n_ctx=model_n_ctx, backend='gptj', callbacks=callbacks, verbose=False)
+    #     case _default:
+    #         print(f"Model {model_type} not supported!")
+    #         exit;    
+    # qa = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=retriever, return_source_documents=True)
+    
+    # Get the answer from the chain
+    # prompt = system_content + f"\n Question: {question}"
+    prompt = f"{question}"
+    # res = qa(prompt)    
+
+    no_input_prompt = PromptTemplate(input_variables=[], template=prompt, dest_language='en')#src_language='id', 
+    no_input_prompt.format()
+
+    query = no_input_prompt.translate()
+
+
+    # prompt_trans = translator.translate(prompt, src='en', dest='id')
+    # print(prompt_trans.text)
+    
+    # result = qa({"question": query, "chat_history": chat_history})
+    llm_response = qa(query)
+
+    answer, docs = llm_response['result'], llm_response['source_documents']
+    no_input_prompt = PromptTemplate(input_variables=[], template=answer, dest_language='id')
+    no_input_prompt.format()
+    answer = no_input_prompt.translate()
+    # answer = post_process_answer(answer, docs)
+    history.append(question)
+    history.append(answer)
+    chatbot = [(history[i], history[i + 1]) for i in range(0, len(history), 2)]
+    return chatbot, history
+    
+    # except Exception as e:
+    #     history.append("")
+    #     answer = server_error_msg + f" (error_code: 503)"
+    #     history.append(answer)
+    #     chatbot = [(history[i], history[i + 1]) for i in range(0, len(history), 2)]
+    #     return chatbot, history
+
+def reset_textbox():
+    return gr.update(value="")
+
+title = """<h1 align="center">Chat with QuGPT 🤖</h1>"""
+
+# def add_text(history, text):
+#     history = history + [(text, None)]
+#     return history, ""
+
+def bot(history):
+    response = "**That's cool!**"
+    history[-1][1] = response
+    return history
+
+with gr.Blocks(
+    css="""
+    footer .svelte-1lyswbr {display: none !important;}
+    #col_container {margin-left: auto; margin-right: auto;}
+    #chatbot .wrap.svelte-13f7djk {height: 70vh; max-height: 70vh}
+    #chatbot .message.user.svelte-13f7djk.svelte-13f7djk {width:fit-content; background:orange; border-bottom-right-radius:0}
+    #chatbot .message.bot.svelte-13f7djk.svelte-13f7djk {width:fit-content; padding-left: 16px; border-bottom-left-radius:0}
+    #chatbot .pre {border:2px solid white;}
+    pre {
+    white-space: pre-wrap;       /* Since CSS 2.1 */
+    white-space: -moz-pre-wrap;  /* Mozilla, since 1999 */
+    white-space: -pre-wrap;      /* Opera 4-6 */
+    white-space: -o-pre-wrap;    /* Opera 7 */
+    word-wrap: break-word;       /* Internet Explorer 5.5+ */
+    }
+    """
+) as demo:
+    gr.HTML(title)
+    with gr.Row():
+        # with gr.Column(elem_id="col_container", scale=0.3):
+            # with gr.Accordion("Prompt", open=True):
+            #     system_content = gr.Textbox(value="You are QuGPT which built with LangChain and dolly-v2 and sentence-transformer.", show_label=False)
+            # with gr.Accordion("Config", open=True):
+            # embeddings_model_name = "sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2"#gr.Textbox(value="sentence-transformers/paraphrase-multilingual-MiniLM-L12-v2", label="embeddings_model_name")
+            # persist_directory = "db" #gr.Textbox(value="db", label="persist_directory")
+            # model_type = "dolly-v2-3b" #gr.Textbox(value="dolly-v2-3b", label="model_type")
+            # model_path = "/media/siiva/DataStore/LLMs/cache/dolly-v2-3b" #gr.Textbox(value="/media/siiva/DataStore/LLMs/cache/dolly-v2-3b", label="model_path")
+            # target_source_chunks = 3 
+            #     # gr.Slider(
+            #     #     minimum=1,
+            #     #     maximum=5,
+            #     #     value=2,
+            #     #     step=1,
+            #     #     interactive=True,
+            #     #     label="target_source_chunks",
+            #     # )
+
+            # model_n_ctx = 1000
+                # gr.Slider(
+                #     minimum=32,
+                #     maximum=4096,
+                #     value=1000,
+                #     step=32,
+                #     interactive=True,
+                #     label="model_n_ctx",
+                # )
+                
+        with gr.Column(elem_id="col_container"):
+            chatbot = gr.Chatbot(elem_id="chatbot", label="QuGPT")
+            question = gr.Textbox(placeholder="Ask something", show_label=False, value="")
+            state = gr.State([])
+            with gr.Row():
+                with gr.Column():
+                    submit_btn = gr.Button(value="🚀 Send")
+                with gr.Column():
+                    clear_btn = gr.Button(value="🗑️ Clear history")
+                
+    question.submit(
+        predict,
+        # [question, system_content, embeddings_model_name, persist_directory, model_type, model_path, model_n_ctx, target_source_chunks, chatbot, state],
+        [question, chatbot, state],
+
+        [chatbot, state],
+    )
+    submit_btn.click(
+        predict,
+        # [question, system_content, embeddings_model_name, persist_directory, model_type, model_path, model_n_ctx, target_source_chunks, chatbot, state],
+        [question, chatbot, state],
+        [chatbot, state],
+    )
+    submit_btn.click(reset_textbox, [], [question])
+    clear_btn.click(clear_history, None, [chatbot, state, question])
+    question.submit(reset_textbox, [], [question])
+    # demo.queue(concurrency_count=10, status_update_rate="auto")
 
-def greet(name):
-    return "Hello " + name + "!!"
+    # question.submit(predict, [question, system_content, embeddings_model_name, persist_directory, model_type, model_path, model_n_ctx, target_source_chunks, chatbot, state], [chatbot, state]).then(
+    #     predict, chatbot
+    # )
+    
+    
+    #demo.launch(server_name=args.server_name, server_port=args.server_port, share=args.share, debug=args.debug)
+    demo.launch()
 
-iface = gr.Interface(fn=greet, inputs="text", outputs="text")
-iface.launch()

requirements.txt

@@ -0,0 +1,17 @@
+chromadb==0.3.21
+duckdb==0.7.1
+googletrans==3.1.0a0
+gradio==3.28.3
+gradio_client==0.2.0
+huggingface-hub==0.13.4
+pypdf==3.8.1
+python-dotenv==1.0.0
+sentence-transformers==2.2.2
+tiktoken==0.3.3
+tokenizers==0.13.3
+torch==2.0.0
+transformers @ git+https://github.com/huggingface/transformers@ef42c2c487260c2a0111fa9d17f2507d84ddedea
+unstructured==0.6.2
+xformers==0.0.19
+
+