Spaces:
Sleeping
Sleeping
File size: 936 Bytes
346b8db |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 |
from langchain_community.document_loaders import PyPDFLoader,DirectoryLoader
from langchain.embeddings import HuggingFaceEmbeddings
from langchain.text_splitter import RecursiveCharacterTextSplitter
from langchain_community.vectorstores import FAISS
loader = DirectoryLoader('/content/data', glob="./*.pdf", loader_cls=PyPDFLoader)
documents = loader.load()
text_splitter = RecursiveCharacterTextSplitter(chunk_size=1024, chunk_overlap=200)
texts = text_splitter.split_documents(documents)
embedings = HuggingFaceEmbeddings(model_name="nomic-ai/nomic-embed-text-v1",model_kwargs={"trust_remote_code":True,"revision":"289f532e14dbbbd5a04753fa58739e9ba766f3c7"})
# Creates vector embeddings and saves it in the FAISS DB
faiss_db = FAISS.from_documents(texts, embedings)
#vectordb=Chroma.from_documents(document_chunks,embedding=embedings)
# Saves and export the vector embeddings databse
faiss_db.save_local("/content/ipc_vector_db")
|