code/modules/embedding_model_loader.py

from langchain_community.embeddings import OpenAIEmbeddings
from langchain_community.embeddings import HuggingFaceEmbeddings
from langchain_community.embeddings import LlamaCppEmbeddings

try:
    from modules.constants import *
except:
    from constants import *
import os


class EmbeddingModelLoader:
    def __init__(self, config):
        self.config = config

    def load_embedding_model(self):
        if self.config["embedding_options"]["model"] in ["text-embedding-ada-002"]:
            embedding_model = OpenAIEmbeddings(
                deployment="SL-document_embedder",
                model=self.config["embedding_options"]["model"],
                show_progress_bar=True,
                openai_api_key=OPENAI_API_KEY,
                disallowed_special=(),
            )
        else:
            embedding_model = HuggingFaceEmbeddings(
                model_name=self.config["embedding_options"]["model"],
                model_kwargs={
                    "device": "cpu",
                    "token": f"{HUGGINGFACE_TOKEN}",
                    "trust_remote_code": True,
                },
            )
            # embedding_model = LlamaCppEmbeddings(
            #     model_path=os.path.abspath("storage/llama-7b.ggmlv3.q4_0.bin")
            # )

        return embedding_model