remove hard coded values

code/modules/config/{user_config.yml → project_config.yml}

@@ -1,3 +1,7 @@
 retriever:
   retriever_hf_paths:
     RAGatouille: "XThomasBU/Colbert_Index"
+
+metadata:
+  metada_links: ["https://dl4ds.github.io/sp2024/lectures/", "https://dl4ds.github.io/sp2024/schedule/"]
+  slide_base_link: "https://dl4ds.github.io"

code/modules/dataloader/data_loader.py

@@ -222,8 +222,7 @@ class ChunkProcessor:
 
     def chunk_docs(self, file_reader, uploaded_files, weblinks):
         addl_metadata = get_metadata(
-            "https://dl4ds.github.io/sp2024/lectures/",
-            "https://dl4ds.github.io/sp2024/schedule/",
+            *self.config["metadata"]["metada_links"], self.config
         )  # For any additional metadata
 
         # remove already processed files if reparse_files is False
@@ -426,6 +425,12 @@ if __name__ == "__main__":
     with open("../code/modules/config/config.yml", "r") as f:
         config = yaml.safe_load(f)
 
+    with open("../code/modules/config/project_config.yml", "r") as f:
+        project_config = yaml.safe_load(f)
+
+    # Combine project config with the main config
+    config.update(project_config)
+
     STORAGE_DIR = os.path.join(BASE_DIR, config["vectorstore"]["data_path"])
     uploaded_files = [
         os.path.join(STORAGE_DIR, file)
@@ -434,6 +439,7 @@ if __name__ == "__main__":
     ]
 
     data_loader = DataLoader(config, logger=logger)
+    # Just for testing
     document_chunks, document_names, documents, document_metadata = (
         data_loader.get_chunks(
             [

code/modules/dataloader/helpers.py

@@ -21,7 +21,8 @@ def get_base_url(url):
     return base_url
 
 
-def get_metadata(lectures_url, schedule_url):
+### THIS FUNCTION IS NOT GENERALIZABLE.. IT IS SPECIFIC TO THE COURSE WEBSITE ###
+def get_metadata(lectures_url, schedule_url, config):
     """
     Function to get the lecture metadata from the lectures and schedule URLs.
     """
@@ -50,7 +51,9 @@ def get_metadata(lectures_url, schedule_url):
             slides_link_tag = description_div.find("a", title="Download slides")
             slides_link = slides_link_tag["href"].strip() if slides_link_tag else None
             slides_link = (
-                f"https://dl4ds.github.io{slides_link}" if slides_link else None
+                f"{config['metadata']['slide_base_link']}{slides_link}"
+                if slides_link
+                else None
             )
             if slides_link:
                 date_mapping[slides_link] = date
@@ -70,7 +73,9 @@ def get_metadata(lectures_url, schedule_url):
             slides_link_tag = block.find("a", title="Download slides")
             slides_link = slides_link_tag["href"].strip() if slides_link_tag else None
             slides_link = (
-                f"https://dl4ds.github.io{slides_link}" if slides_link else None
+                f"{config['metadata']['slide_base_link']}{slides_link}"
+                if slides_link
+                else None
             )
 
             # Extract the link to the lecture recording

code/modules/vectorstore/store_manager.py

@@ -168,19 +168,21 @@ if __name__ == "__main__":
 
     with open("modules/config/config.yml", "r") as f:
         config = yaml.safe_load(f)
-    with open("modules/config/user_config.yml", "r") as f:
-        user_config = yaml.safe_load(f)
+    with open("modules/config/project_config.yml", "r") as f:
+        project_config = yaml.safe_load(f)
+
+    # combine the two configs
+    config.update(project_config)
     print(config)
-    print(user_config)
     print(f"Trying to create database with config: {config}")
     vector_db = VectorStoreManager(config)
     if config["vectorstore"]["load_from_HF"]:
         if (
             config["vectorstore"]["db_option"]
-            in user_config["retriever"]["retriever_hf_paths"]
+            in config["retriever"]["retriever_hf_paths"]
         ):
             vector_db.load_from_HF(
-                HF_PATH=user_config["retriever"]["retriever_hf_paths"][
+                HF_PATH=config["retriever"]["retriever_hf_paths"][
                     config["vectorstore"]["db_option"]
                 ]
             )

docs/setup.md

@@ -124,4 +124,4 @@ CHAINLIT_URL=<your_chainlit_url>
 # Configuration
 
 The configuration file `code/modules/config.yaml` contains the parameters that control the behaviour of your app.
-The configuration file `code/modules/user_config.yaml` contains user-defined parameters.
+The configuration file `code/modules/project_config.yaml` contains project-specific parameters.