App

README.md

@@ -1,13 +1 @@
----
-title: Youtube Retrieval Qa
-emoji: 📚
-colorFrom: indigo
-colorTo: red
-sdk: gradio
-sdk_version: 3.32.0
-app_file: app.py
-pinned: false
-license: mit
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+# youtube-retrieval-qa

app.py

@@ -0,0 +1,65 @@
+import os 
+from typing import List
+import gradio as gr
+from qa.manager import YoutubeQA 
+
+DESCRIPTION = """
+
+<h1> <center> 🤗 Hello. This App will help you do questions on youtube videos.</center> </h1>
+
+<h4>
+Follow this steps to use 😉:
+</h4>
+
+<ol>
+  <li>Set your OpenAI Key</li>
+  <li>Set your Youtube URL</li>
+  <li>Ask!</li>
+</ol> 
+"""
+
+qa = YoutubeQA()
+
+def set_openai_key(key: str):    
+    os.environ["OPENAI_API_KEY"] = key
+    # Set status field to Not Ready
+    return gr.update(lines=1, value="Not Ready 🥴")
+
+def instanciate_retriver(url: str):
+    qa.load_model()
+    qa.load_vector_store(url)    
+    qa.load_retriever()   
+    # Set status field to Ready 
+    return gr.update(lines=1, value="Ready 😎")
+
+def respond(message: str, chat_history: List[str]):
+    bot_message = qa.run(message)  
+    chat_history.append((message, bot_message))
+    return "", chat_history
+
+with gr.Blocks() as app:
+
+    gr.Markdown(DESCRIPTION)
+    with gr.Tab("QA"):
+        status = gr.Textbox(label="🤔 Vector DB Status:", interactive=False)            
+        chatbot = gr.Chatbot(label="🤖 Bot Answer:")
+        question = gr.Textbox(label="🕵️‍♀️ Question:", placeholder="Write your question here and press enter")    
+        clear = gr.Button("Clear")
+        question.submit(respond, [question, chatbot], [question, chatbot])
+        clear.click(lambda: None, None, chatbot, queue=False)
+
+    with gr.Tab("Youtube URL"):
+        url = gr.Textbox(label="🎞️ URL:", lines=1, placeholder="Set your Youtube URL here...")
+        url_button = gr.Button("Set URL")
+
+    with gr.Tab("OpenAI Key"):
+        key = gr.Textbox(label="🔑 Key:", type="password", placeholder="Set your OpenAI Key here...")
+        key_button = gr.Button("Set Key")
+
+    #with gr.Accordion("Click me. About this App"):
+    #    gr.Markdown("Look at me...")
+
+    url_button.click(instanciate_retriver, inputs=url, outputs=status)
+    key_button.click(set_openai_key, inputs=key, outputs=status)      
+
+app.launch()

qa/chains.py

@@ -0,0 +1,6 @@
+from typing import Callable
+
+def retrieval_qa(llm: Callable, retriever: Callable) -> Callable: 
+    from langchain.chains import RetrievalQA
+    qa = RetrievalQA.from_chain_type(llm=llm, chain_type="stuff", retriever=retriever)
+    return qa

qa/loader.py

@@ -0,0 +1,7 @@
+from typing import List
+
+def youtube_doc_loader(url: str) -> List:
+    from langchain.document_loaders import YoutubeLoader
+    loader = YoutubeLoader.from_youtube_url(url, add_video_info=False)
+    data = loader.load()
+    return data

qa/manager.py

@@ -0,0 +1,24 @@
+from qa.chains import retrieval_qa
+from qa.loader import youtube_doc_loader
+from qa.model import load_llm
+from qa.split import split_document
+from qa.vector_store import create_vector_store
+
+class YoutubeQA:
+
+    def __init__(self):
+        pass
+
+    def load_model(self) -> None:
+        self.llm = load_llm()
+
+    def load_vector_store(self, url: str) -> None:
+        data = youtube_doc_loader(url=url)
+        docs = split_document(data=data)
+        self.retriver = create_vector_store(docs=docs)
+     
+    def load_retriever(self) -> None:
+        self.retrieval_qa = retrieval_qa(llm=self.llm, retriever=self.retriver)
+
+    def run(self, question: str) -> str:
+        return self.retrieval_qa.run(question)

qa/model.py

@@ -0,0 +1,6 @@
+from typing import Callable
+
+def load_llm(temperature: float = 0.0, model: str = 'gpt-3.5-turbo') -> Callable:
+	from langchain.chat_models import ChatOpenAI
+	llm = ChatOpenAI(temperature=temperature, model=model)
+	return llm

qa/split.py

@@ -0,0 +1,7 @@
+from typing import List
+
+def split_document(data: List, chunk_size: int = 3000) -> List:
+	from langchain.text_splitter import RecursiveCharacterTextSplitter
+	text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=400)
+	docs = text_splitter.split_documents(data)
+	return docs

qa/vector_store.py

@@ -0,0 +1,25 @@
+from typing import Callable, List
+
+def create_vector_store(
+    docs: List,
+    metric: str = 'cos',
+    top_k: int = 4
+) -> Callable:
+    
+    from langchain.vectorstores import FAISS
+    from langchain.embeddings.openai import OpenAIEmbeddings
+    
+    embeddings = OpenAIEmbeddings()
+    
+    # Embed your documents and combine with the raw text in a pseudo db. 
+    # Note: This will make an API call to OpenAI
+    docsearch = FAISS.from_documents(docs, embeddings)
+    
+    # Retriver object
+    retriever = docsearch.as_retriever()
+    
+    # Retriver configs
+    retriever.search_kwargs['distance_metric'] = metric
+    retriever.search_kwargs['k'] = top_k
+
+    return retriever

requirements.txt

@@ -0,0 +1,5 @@
+langchain 
+openai 
+youtube-transcript-api 
+faiss-cpu 
+tiktoken