Using csv agent, some code for custom vector store based agents too.

app.py

@@ -1,7 +1,9 @@
 import asyncio
 import glob
 import os
+import shutil
 import time
+import traceback
 
 import gradio as gr
 from dotenv import load_dotenv
@@ -40,13 +42,18 @@ def run_model(text):
     global grader, grader_qa
     start_time = time.time()
     print("start time:" + str(start_time))
-    response = grader_qa.chain(text)
+    try:
+        response = grader_qa.agent.run(text)
+    except Exception as e:
+        response = "I need a break. Please ask me again in a few minutes"
+        print(traceback.format_exc())
+
     sources = []
-    for document in response['source_documents']:
-        sources.append(str(document.metadata))
+    # for document in response['source_documents']:
+    #     sources.append(str(document.metadata))
 
     source = ','.join(set(sources))
-    response = response['answer'] + '\nSources: ' + str(len(sources))
+    # response = response['answer'] + '\nSources: ' + str(len(sources))
     end_time = time.time()
     # # If response contains string `SOURCES:`, then add a \n before `SOURCES`
     # if "SOURCES:" in response:
@@ -171,6 +178,31 @@ def reset_data(history):
     return history
 
 
+def get_output_dir(orig_name):
+    script_dir = os.path.dirname(os.path.abspath(__file__))
+    output_dir = os.path.join(script_dir, 'output', orig_name)
+    return output_dir
+
+
+def upload_grading_results(file, history):
+    global grader, grader_qa
+    # Delete output folder and save the file in output folder
+    if os.path.isdir('output'):
+        shutil.rmtree('output')
+    os.mkdir('output')
+    if os.path.isdir('vector_stores'):
+        shutil.rmtree('vector_stores')
+    os.mkdir('vector_stores')
+    # get current path
+    path = os.path.join("output", os.path.basename(file.name))
+    # Copy the uploaded file from its temporary location to the desired location
+    shutil.copyfile(file.name, path)
+    grader = Grader(qa_model)
+    grader_qa = GraderQA(grader, embeddings)
+    history = [(None, 'Grading results uploaded successfully')]
+    return history
+
+
 def bot(history):
     return get_grading_status(history)
 
@@ -203,6 +235,7 @@ with gr.Blocks() as demo:
                 label="Ask questions about how students did on the discussion",
                 placeholder="Enter text and press enter, or upload an image", lines=1
             )
+        upload = gr.UploadButton(label="Upload grading results", type="file", file_types=["csv"], scale=0.5)
         ask = gr.Button(value="Ask", variant="secondary", scale=1)
 
     chatbot.value = get_first_message([])
@@ -231,6 +264,9 @@ with gr.Blocks() as demo:
     reset.click(reset_data, inputs=[chatbot], outputs=[chatbot], postprocess=False, show_progress=True, ).success(
         bot, chatbot, chatbot)
 
+    upload.upload(upload_grading_results, inputs=[upload, chatbot], outputs=[chatbot], postprocess=False, ).then(
+        bot, chatbot, chatbot)
+
 if __name__ == "__main__":
     demo.queue()
     demo.queue(concurrency_count=5)

grader.py

@@ -25,7 +25,8 @@ class Grader:
         self.model = model
         self.rubric_file = 'docs/rubric_data.json'
         self.discussions_file_path = "docs/discussion_entries.json"
-        self.fieldnames = ['student_name', 'total_score', 'student_feedback', 'grader_comments', 'summary']
+        self.fieldnames = ['student_name', 'total_score', 'score_breakdown', 'grader_comments', 'student_feedback',
+                           'summary']
         self.docs = self.get_html_files()
         self.llm = ChatOpenAI(temperature=0, model_name=model)
         self.parser: PydanticOutputParser = self.create_parser()
@@ -42,16 +43,18 @@ class Grader:
     class ToolArgsSchema(BaseModel):
         student_name: Optional[str] = Field(description="The name of the student")
         total_score: int = Field(description="The grade of the student's answer")
-        student_feedback: Optional[str] = Field(
-            description="The developmental feedback from Grader's point of view to the student, some examples are: 'Great work, ...', 'Although, your submission is relevant to the question, it doesn't answer the question entirely...'. Give customized feedback based on student's answer")
+        score_breakdown: Optional[str] = Field(description="The grade split breakup based on rubric")
         grader_comments: Optional[str] = Field(
             description="The grade split breakup based on rubric added as grader's one liner customized comments to explain how the grade was calculated for that particular student's answer")
+        student_feedback: Optional[str] = Field(
+            description="The developmental feedback from Grader's point of view to the student, some examples are: 'Great work, ...', 'Although, your submission is relevant to the question, it doesn't answer the question entirely...'. Give customized feedback based on student's answer")
         summary: Optional[str] = Field(
             description="The overall summary of the student's answer outlining key points from the student's answer based on the rubric which can be used as a portion of a vectorstore, used to answer summary based questions about all the discussions")
 
         class Config:
             schema_extra = {
-                "required": ["student_name", "total_score", "student_feedback", "grader_comments", "summary"]
+                "required": ["student_name", "total_score", "score_breakdown", "grader_comments", "student_feedback",
+                             "summary"]
             }
 
     def create_parser(self):

grader_qa.py

@@ -1,19 +1,13 @@
-import os
-
 from langchain import FAISS
-from langchain.chains import ConversationalRetrievalChain
-from langchain.chat_models import ChatOpenAI
-from langchain.document_loaders import CSVLoader
+from langchain import LLMMathChain
+from langchain.agents import AgentType, create_csv_agent
+from langchain.chains import RetrievalQA
+from langchain.chains.question_answering import load_qa_chain
 from langchain.memory import ConversationBufferMemory
 from langchain.prompts import ChatPromptTemplate, HumanMessagePromptTemplate, SystemMessagePromptTemplate
-from langchain.text_splitter import RecursiveCharacterTextSplitter
-
+from langchain.tools import Tool
 
-def search_index_from_docs(source_chunks, embeddings):
-    # print("source chunks: " + str(len(source_chunks)))
-    # print("embeddings: " + str(embeddings))
-    search_index = FAISS.from_documents(source_chunks, embeddings)
-    return search_index
+import utils
 
 
 def get_chat_history(inputs) -> str:
@@ -27,108 +21,143 @@ class GraderQA:
     def __init__(self, grader, embeddings):
         self.grader = grader
         self.llm = self.grader.llm
-        self.index_file = "vector_stores/canvas-discussions.faiss"
-        self.pickle_file = "vector_stores/canvas-discussions.pkl"
+        self.folder_path = "vector_stores/"
+        self.summary_index_name = "canvas-discussions-summary"
+        self.summary_index_file = "vector_stores/canvas-discussions-summary.faiss"
+        self.summary_pickle_file = "vector_stores/canvas-discussions-summary.pkl"
+        self.qa_index_name = "canvas-discussions-qa"
+        self.qa_index_file = "vector_stores/canvas-discussions-qa.faiss"
+        self.qa_pickle_file = "vector_stores/canvas-discussions-qa.pkl"
+        self.summary_docs = utils.get_csv_files(self.grader.csv, source_column='student_name')
+        self.qa_docs = utils.get_csv_files(self.grader.csv, source_column='student_name',
+                                           field_names=['student_name', 'total_score', 'score_breakdown'])
         self.rubric_text = grader.rubric_text
-        self.search_index = self.get_search_index(embeddings)
+        self.summary_index = self.get_search_index(embeddings)
+        self.qa_index = self.get_qa_index(embeddings)
         self.chain = self.create_chain(embeddings)
+        self.qa_chain = self.create_qa_chain()
+        self.math_chain = self.create_math_chain()
+        self.tools = self.get_tools()
+        self.memory = ConversationBufferMemory(memory_key='chat_history',
+                                               return_messages=True,
+                                               output_key='answer')
+        self.agent = self.create_agent()
         self.tokens = None
         self.question = None
 
+    def load_all_indexes(self, embeddings):
+        return self.get_search_index(embeddings), self.get_qa_index(embeddings)
+
     def get_search_index(self, embeddings):
-        if os.path.isfile(self.pickle_file) and os.path.isfile(self.index_file) and os.path.getsize(
-                self.pickle_file) > 0:
+        if utils.index_exists(self.summary_pickle_file, self.summary_index_file):
             # Load index from pickle file
-            search_index = self.load_index(embeddings)
+            search_index = utils.load_index(self.folder_path, self.summary_index_name, embeddings)
         else:
-            search_index = self.create_index(embeddings)
+            search_index = utils.create_index(self.folder_path, self.summary_index_name, embeddings, self.summary_docs)
             print("Created index")
         return search_index
 
-    def load_index(self, embeddings):
-        # Load index
-        db = FAISS.load_local(
-            folder_path="vector_stores/",
-            index_name="canvas-discussions", embeddings=embeddings,
-        )
-        print("Loaded index")
-        return db
-
-    def create_index(self, embeddings):
-        source_chunks = self.create_chunk_documents()
-        search_index = search_index_from_docs(source_chunks, embeddings)
-        FAISS.save_local(search_index, folder_path="vector_stores/", index_name="canvas-discussions")
+    def get_qa_index(self, embeddings):
+        if utils.index_exists(self.qa_pickle_file, self.qa_index_file):
+            # Load index from pickle file
+            search_index = utils.load_index(self.folder_path, self.qa_index_name, embeddings)
+        else:
+            search_index = utils.create_index(self.folder_path, self.qa_index_name, embeddings, self.qa_docs)
+            print("Created index")
         return search_index
 
-    def create_chunk_documents(self):
-        sources = self.fetch_data_for_embeddings()
-
-        splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=0)
-
-        source_chunks = splitter.split_documents(sources)
+    def create_chain(self, embeddings):
+        if not self.summary_index:
+            self.summary_index = self.get_search_index(embeddings)
 
-        print("chunks: " + str(len(source_chunks)))
-        print("sources: " + str(len(sources)))
+        question_prompt, combine_prompt = self.create_map_reduce_prompt()
+        # create agent, 1 chain for summary based question, 2nd chain for semantic retrieval based question
+        qa_chain = load_qa_chain(self.llm, chain_type="map_reduce", question_prompt=question_prompt,
+                                 combine_prompt=combine_prompt, verbose=True)
 
-        return source_chunks
+        chain = RetrievalQA(combine_documents_chain=qa_chain,
+                            retriever=self.summary_index.as_retriever(search_type='mmr',
+                                                                      search_kwargs={'lambda_mult': 1, 'fetch_k': 50,
+                                                                                     'k': 30}),
+                            return_source_documents=True, verbose=True, )
+        return chain
 
-    def fetch_data_for_embeddings(self):
-        document_list = self.get_csv_files()
-        print("document list: " + str(len(document_list)))
-        return document_list
+    def create_qa_chain(self):
+        qa = RetrievalQA.from_chain_type(llm=self.llm, chain_type="stuff",
+                                         retriever=self.qa_index.as_retriever(search_type='mmr',
+                                                                              search_kwargs={'lambda_mult': 1,
+                                                                                             'fetch_k': 50,
+                                                                                             'k': 30}), verbose=True)
+        return qa
 
-    def get_csv_files(self):
-        loader = CSVLoader(file_path=self.grader.csv, source_column="student_name")
-        document_list = loader.load()
-        return document_list
+    def create_math_chain(self):
+        return LLMMathChain.from_llm(llm=self.llm, verbose=True)
 
-    def create_chain(self, embeddings):
-        if not self.search_index:
-            self.search_index = self.load_index(embeddings)
+    def get_tools(self):
+        tools = [
+            Tool(
+                name="Grading Score Results",
+                func=self.run_qa_chain,
+                description="useful when you need to answer questions related to GRADES, SCORING or SCORE BREAKDOWN(INDIVIDUAL OR OVERALL) based questions from the grading results of the canvas discussion. Use this more often because this has a higher accuracy about the SCORING and GRADES of the students."
+            ),
+            Tool(
+                name="Summary",
+                func=self.run_summary_chain,
+                description="useful when you need to answer summary based questions for all students' grading results for the canvas discussion where the question is complicated and ONLY WHEN the answer is not directly available in the grading score results"
+            ),
+            Tool(
+                name="Calculator",
+                func=self.run_math_chain,
+                description="Useful for when you need to compute mathematical expressions"
+            )
+        ]
+        return tools
 
-        question_prompt, combine_prompt = self.create_map_reduce_prompt()
-        # create agent, 1 chain for summary based question, 2nd chain for semantic retrieval based question
+    def create_agent(self):
+        # Initialize a Conversational Agent with the existing chain as a tool
+        # planner = load_chat_planner(self.llm)
+        #
+        # # agent = initialize_agent(self.tools, self.llm, agent=AgentType.CHAT_CONVERSATIONAL_REACT_DESCRIPTION, verbose=True, memory=self.memory)
+        # executor = load_agent_executor(self.llm,self.tools, verbose=True)
+        #
+        #
+        # agent = PlanAndExecute(planner=planner, executor=executor, verbose=True)
+        # agent = initialize_agent(
+        #     self.tools, self.llm, agent=AgentType.ZERO_SHOT_REACT_DESCRIPTION, verbose=True
+        # )
 
-        chain = ConversationalRetrievalChain.from_llm(llm=self.llm, chain_type='map_reduce',
-                                                      retriever=self.search_index.as_retriever(search_type='mmr',
-                                                                                               search_kwargs={
-                                                                                                   'lambda_mult': 1,
-                                                                                                   'fetch_k': 50,
-                                                                                                   'k': 30}),
-                                                      return_source_documents=True,
-                                                      verbose=True,
-                                                      memory=ConversationBufferMemory(memory_key='chat_history',
-                                                                                      return_messages=True,
-                                                                                      output_key='answer'),
-                                                      condense_question_llm=ChatOpenAI(temperature=0,
-                                                                                       model='gpt-3.5-turbo'),
-                                                      combine_docs_chain_kwargs={"question_prompt": question_prompt,
-                                                                                 "combine_prompt": combine_prompt})
-        return chain
+        agent = create_csv_agent(
+            self.llm,
+            self.grader.csv,
+            verbose=True,
+            agent_type=AgentType.ZERO_SHOT_REACT_DESCRIPTION,
+        )
+        return agent
 
     def create_map_reduce_prompt(self):
-        system_template = f"""Use the following portion of a long grading results document to answer the question BUT ONLY FOR THE STUDENT MENTIONED. Use the following examples to take guidance on how to answer the question.
+        system_template = f"""Use the following student's grading result document to answer a summary based question. The question will always be related to the overall grading results, feedback, score, summary of student responses for the discussion. But the answer will ALWAYS be specific to the student based on the question. There are examples to help you understand how to answer the question.
+        ______________________
+        Grading Result For:
+        {{context}}
+        ______________________
+        Use the following examples to take guidance on how to answer the question.
         Examples:
         Question: How many students participated in the discussion?
+        Rephrased question: Did this student participate in the discussion?
         Answer: This student participated in the discussion./This student did not participate in the discussion.
         Question: What was the average score for the discussion?
+        Rephrased question: What was the score for this student for the discussion?
         Answer: This student received a score of 10/10 for the discussion.
         Question: How many students received a full score?/How many students did not receive a full score?
+        Rephrased question: Did this student receive a full score?
         Answer: This student received a full score./This student did not receive a full score.
         Question: How many students lost marks in X category of the rubric?
+        Rephrased question: Did this student lose marks in X category of the rubric?
         Answer: This student lost marks in X category of the rubric./This student did not lose marks in X category of the rubric.
         Question: Give me 3 best responses received for the discussion.
+        Rephrased question: What were the 3 best responses received for the discussion?
         Answer: This student gave the following responses for the discussion and received a score of 10/10.
-        
-        
-        ______________________
-        Grading Result For:
-        {{context}}
         ______________________
-        Following are the instructions and rubric of the discussion post for reference, used to grade the discussion.
-        ----------------
-        Instructions and Rubric:
-        {self.rubric_text}
         """
         messages = [
             SystemMessagePromptTemplate.from_template(system_template),
@@ -136,7 +165,7 @@ class GraderQA:
         ]
         CHAT_QUESTION_PROMPT = ChatPromptTemplate.from_messages(messages)
         system_template = """You are Canvas Discussions Grading + Feedback QA Bot. Have a conversation with a human, answering the questions about the grading results, feedback, answers as accurately as possible.
-        Use the following answers for each student to answer the users question as accurately as possible. 
+        Use the following answers for each student to answer the users question as accurately as possible.
         You are an expert at basic calculations and answering questions on grading results and can answer the following questions with ease.
         If you don't know the answer, just say that you don't know. Don't try to make up an answer.
         ______________________
@@ -150,8 +179,8 @@ class GraderQA:
 
     def create_prompt(self):
         system_template = f"""You are Canvas Discussions Grading + Feedback QA Bot. Have a conversation with a human, answering the questions about the grading results, feedback, answers as accurately as possible.
-        You are a grading assistant who graded the canvas discussions to create the following grading results and feedback. 
-        Use the following instruction, rubric of the discussion which were used to grade the discussions and refine the answer if needed.  
+        You are a grading assistant who graded the canvas discussions to create the following grading results and feedback.
+        Use the following instruction, rubric of the discussion which were used to grade the discussions and refine the answer if needed.
         ----------------
         {self.rubric_text}
         ----------------
@@ -165,24 +194,43 @@ class GraderQA:
 
     def get_tokens(self):
         total_tokens = 0
-        for doc in self.docs:
-            chat_prompt = self.prompt.format(context=doc, question=self.question)
-
-            num_tokens = self.llm.get_num_tokens(chat_prompt)
-            total_tokens += num_tokens
+        # for doc in self.docs:
+        #     chat_prompt = self.prompt.format(context=doc, question=self.question)
+        #
+        #     num_tokens = self.llm.get_num_tokens(chat_prompt)
+        #     total_tokens += num_tokens
 
-            # summary = self.llm(summary_prompt)
+        # summary = self.llm(summary_prompt)
 
-            # print (f"Summary: {summary.strip()}")
-            # print ("\n")
+        # print (f"Summary: {summary.strip()}")
+        # print ("\n")
         return total_tokens
 
     def run_qa_chain(self, question):
+        self.question = question
+        self.get_tokens()
+        answer = self.qa_chain.run(question)
+        return answer
+
+    def run_summary_chain(self, question):
         self.question = question
         self.get_tokens()
         answer = self.chain(question)
         return answer
 
+    def run_math_chain(self, question):
+        self.question = question
+        self.get_tokens()
+        answer = self.math_chain.run(question)
+        return answer
+
+
+def search_index_from_docs(source_chunks, embeddings):
+    # print("source chunks: " + str(len(source_chunks)))
+    # print("embeddings: " + str(embeddings))
+    search_index = FAISS.from_documents(source_chunks, embeddings)
+    return search_index
+
 # system_template = """You are Canvas Discussions Grading + Feedback QA Bot. Have a conversation with a human, answering the following questions as best you can.
 # You are a grading assistant who graded the canvas discussions to create the following grading results and feedback. Use the following pieces of the grading results and feedback to answer the users question.
 # Use the following pieces of context to answer the users question.

utils.py

@@ -2,6 +2,11 @@ import os
 import shutil
 import time
 
+from langchain import FAISS
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+
+from custom_csv_loader import CSVLoader
+
 
 def reset_folder(destination):
     # synchrnously and recursively delete the destination folder and all its contents, donot return until done
@@ -12,3 +17,61 @@ def reset_folder(destination):
     os.mkdir(destination)
     while not os.path.isdir(destination):
         time.sleep(4)
+
+
+def search_index_from_docs(source_chunks, embeddings):
+    # print("source chunks: " + str(len(source_chunks)))
+    # print("embeddings: " + str(embeddings))
+    search_index = FAISS.from_documents(source_chunks, embeddings)
+    return search_index
+
+
+def load_index(folder_path, index_name, embeddings):
+    # Load index
+    db = FAISS.load_local(
+        folder_path=folder_path,
+        index_name=index_name, embeddings=embeddings,
+    )
+    print("Loaded index")
+    return db
+
+
+def fetch_data_for_embeddings(document_list):
+    print("document list: " + str(len(document_list)))
+    return document_list
+
+
+def create_chunk_documents(document_list):
+    sources = fetch_data_for_embeddings(document_list)
+
+    splitter = RecursiveCharacterTextSplitter(chunk_size=500, chunk_overlap=0)
+
+    source_chunks = splitter.split_documents(sources)
+
+    print("chunks: " + str(len(source_chunks)))
+    print("sources: " + str(len(sources)))
+
+    return source_chunks
+
+
+def create_index(folder_path, index_name, embeddings, document_list):
+    source_chunks = create_chunk_documents(document_list)
+    search_index = search_index_from_docs(source_chunks, embeddings)
+    FAISS.save_local(search_index, folder_path=folder_path, index_name=index_name)
+    return search_index
+
+
+def get_csv_files(csv_file, source_column, field_names=None):
+    loader = None
+    if field_names:
+        loader = CSVLoader(file_path=csv_file, source_column=source_column,
+                           csv_args={'fieldnames': field_names, 'restkey': 'restkey'})
+    else:
+        loader = CSVLoader(file_path=csv_file, source_column=source_column, )
+    document_list = loader.load()
+    return document_list
+
+
+def index_exists(pickle_file, index_file):
+    return os.path.isfile(pickle_file) and os.path.isfile(index_file) and os.path.getsize(
+        pickle_file) > 0