Refactor PDFProcessor to skip pages that appear to be part of the references section

utils/pdf_processor.py

@@ -12,6 +12,7 @@ import datetime
 from slugify import slugify
 import json
 from PIL import Image
+import re
 
 
 logger = logging.getLogger(__name__)
@@ -48,6 +49,58 @@ class PDFProcessor:
             logger.error(f"Error rendering page {page_num} from {file_path}: {str(e)}")
             return None
 
+    def is_references_page(self, text: str) -> bool:
+        """
+        Check if the page appears to be a references/bibliography page.
+        """
+        # Common section headers for references
+        ref_headers = [
+            r"^references\s*$",
+            r"^bibliography\s*$",
+            r"^works cited\s*$",
+            r"^citations\s*$",
+            r"^cited literature\s*$",
+        ]
+
+        # Check first few lines of the page
+        first_lines = text.lower().split("\n")[:3]
+        first_block = " ".join(first_lines)
+
+        # Check for reference headers
+        for header in ref_headers:
+            if re.search(header, first_block, re.IGNORECASE):
+                return True
+
+        # Check for reference-like patterns (e.g., [1] Author, et al.)
+        ref_patterns = [
+            r"^\[\d+\]",  # [1] style
+            r"^\d+\.",  # 1. style
+            r"^[A-Z][a-z]+,\s+[A-Z]\.",  # Author, I. style
+        ]
+
+        ref_pattern_count = 0
+        lines = text.split("\n")[:10]  # Check first 10 lines
+        for line in lines:
+            line = line.strip()
+            if any(re.match(pattern, line) for pattern in ref_patterns):
+                ref_pattern_count += 1
+
+        # If multiple reference-like patterns are found, likely a references page
+        return ref_pattern_count >= 3
+
+    def detect_references_start(self, doc: fitz.Document) -> Optional[int]:
+        """
+        Detect the page where references section starts.
+        Returns the page number or None if not found.
+        """
+        for page_num in range(len(doc)):
+            page = doc[page_num]
+            text = page.get_text()
+            if self.is_references_page(text):
+                logger.info(f"Detected references section starting at page {page_num}")
+                return page_num
+        return None
+
     def process_pdfs(self, file_paths: List[str], collection_name: str) -> str:
         """Process multiple PDF files and store their content."""
         processed_docs = []
@@ -56,7 +109,9 @@ class PDFProcessor:
             try:
                 doc_data = self.extract_text_from_pdf(file_path)
                 processed_docs.append(doc_data)
-                logger.info(f"Successfully processed {file_path}")
+                logger.info(
+                    f"Successfully processed {file_path} ({doc_data['content_pages']} content pages)"
+                )
             except Exception as e:
                 logger.error(f"Error processing {file_path}: {str(e)}")
                 continue
@@ -83,12 +138,30 @@ class PDFProcessor:
         try:
             doc = fitz.open(file_path)
 
+            # Find references section start
+            refs_start = self.detect_references_start(doc)
+
             # Extract text from all pages with page tracking
             text = ""
             pages = {}
             for page_num in range(len(doc)):
+                # Skip if this is after references section starts
+                if refs_start is not None and page_num >= refs_start:
+                    logger.info(
+                        f"Skipping page {page_num} as it appears to be part of references"
+                    )
+                    continue
+
                 page_text = doc[page_num].get_text()
-                pages[str(page_num)] = page_text  # Convert page_num to string for JSON
+
+                # Extra check to catch references if they weren't caught by the initial scan
+                if page_num > 0 and self.is_references_page(page_text):
+                    logger.info(
+                        f"Detected references content on page {page_num}, skipping"
+                    )
+                    continue
+
+                pages[str(page_num)] = page_text
                 text += page_text + "\n"
 
             # Extract metadata
@@ -110,6 +183,7 @@ class PDFProcessor:
                 "source_file": file_path,
                 "pages": pages,
                 "page_count": len(doc),
+                "content_pages": len(pages),  # Number of pages excluding references
             }
 
             doc.close()