updated index

app.py

@@ -15,6 +15,8 @@ from pinecone_text.sparse import BM25Encoder
 from langchain_huggingface import HuggingFaceEmbeddings
 from langchain_community.retrievers import PineconeHybridSearchRetriever
 from langchain_groq import ChatGroq
+from langchain.retrievers import ContextualCompressionRetriever
+from langchain.retrievers.document_compressors import FlashrankRerank
 
 # Load environment variables
 load_dotenv(".env")
@@ -53,8 +55,8 @@ def initialize_pinecone(index_name: str):
 ##################################################
 
 # Initialize Pinecone index and BM25 encoder
-pinecone_index = initialize_pinecone("mbzuai-policies")
-bm25 = BM25Encoder().load("./mbzuai-policies.json")
+pinecone_index = initialize_pinecone("updated-mbzuai-policies")
+bm25 = BM25Encoder().load("./new_mbzuai-policies.json")
 
 ##################################################
 ##################################################
@@ -74,6 +76,13 @@ retriever = PineconeHybridSearchRetriever(
 # Initialize LLM
 llm = ChatGroq(model="llama-3.1-70b-versatile", temperature=0, max_tokens=1024, max_retries=2)
 
+
+# Initialize Reranker
+compressor = FlashrankRerank()
+compression_retriever = ContextualCompressionRetriever(
+    base_compressor=compressor, base_retriever=retriever
+)
+
 # Contextualization prompt and retriever
 contextualize_q_system_prompt = """Given a chat history and the latest user question \
 which might reference context in the chat history, formulate a standalone question \
@@ -87,7 +96,7 @@ contextualize_q_prompt = ChatPromptTemplate.from_messages(
         ("human", "{input}")
     ]
 )
-history_aware_retriever = create_history_aware_retriever(llm, retriever, contextualize_q_prompt)
+history_aware_retriever = create_history_aware_retriever(llm, compression_retriever, contextualize_q_prompt)
 
 # QA system prompt and chain
 qa_system_prompt = """You are a highly skilled information retrieval assistant. Use the following context to answer questions effectively. \

requirements.txt

@@ -99,3 +99,4 @@ wget==3.2
 wsproto==1.2.0
 yarl==1.9.4
 zipp==3.19.2
+flaskrank