Upload 169 files

App_Function_Libraries/Audio/Audio_Files.py

@@ -117,16 +117,15 @@ def process_audio_files(audio_urls, audio_file, whisper_model, api_name, api_key
     progress = []
     all_transcriptions = []
     all_summaries = []
-    #v2
+    temp_files = []  # Keep track of temporary files
+
     def format_transcription_with_timestamps(segments):
         if keep_timestamps:
             formatted_segments = []
             for segment in segments:
                 start = segment.get('Time_Start', 0)
                 end = segment.get('Time_End', 0)
-                text = segment.get('Text', '').strip()  # Ensure text is stripped of leading/trailing spaces
-
-                # Add the formatted timestamp and text to the list, followed by a newline
+                text = segment.get('Text', '').strip()
                 formatted_segments.append(f"[{start:.2f}-{end:.2f}] {text}")
 
             # Join the segments with a newline to ensure proper formatting
@@ -191,205 +190,64 @@ def process_audio_files(audio_urls, audio_file, whisper_model, api_name, api_key
             'language': chunk_language
         }
 
-        # Process multiple URLs
-        urls = [url.strip() for url in audio_urls.split('\n') if url.strip()]
-
-        for i, url in enumerate(urls):
-            update_progress(f"Processing URL {i + 1}/{len(urls)}: {url}")
-
-            # Download and process audio file
-            audio_file_path = download_audio_file(url, use_cookies, cookies)
-            if not os.path.exists(audio_file_path):
-                update_progress(f"Downloaded file not found: {audio_file_path}")
-                failed_count += 1
-                log_counter(
-                    metric_name="audio_files_failed_total",
-                    labels={"whisper_model": whisper_model, "api_name": api_name},
-                    value=1
-                )
-                continue
-
-            temp_files.append(audio_file_path)
-            update_progress("Audio file downloaded successfully.")
-
-            # Re-encode MP3 to fix potential issues
-            reencoded_mp3_path = reencode_mp3(audio_file_path)
-            if not os.path.exists(reencoded_mp3_path):
-                update_progress(f"Re-encoded file not found: {reencoded_mp3_path}")
-                failed_count += 1
-                log_counter(
-                    metric_name="audio_files_failed_total",
-                    labels={"whisper_model": whisper_model, "api_name": api_name},
-                    value=1
-                )
-                continue
-
-            temp_files.append(reencoded_mp3_path)
-
-            # Convert re-encoded MP3 to WAV
-            wav_file_path = convert_mp3_to_wav(reencoded_mp3_path)
-            if not os.path.exists(wav_file_path):
-                update_progress(f"Converted WAV file not found: {wav_file_path}")
-                failed_count += 1
-                log_counter(
-                    metric_name="audio_files_failed_total",
-                    labels={"whisper_model": whisper_model, "api_name": api_name},
-                    value=1
-                )
-                continue
-
-            temp_files.append(wav_file_path)
-
-            # Initialize transcription
-            transcription = ""
-
-            # Transcribe audio
-            if diarize:
-                segments = speech_to_text(wav_file_path, whisper_model=whisper_model, diarize=True)
-            else:
-                segments = speech_to_text(wav_file_path, whisper_model=whisper_model)
-
-            # Handle segments nested under 'segments' key
-            if isinstance(segments, dict) and 'segments' in segments:
-                segments = segments['segments']
-
-            if isinstance(segments, list):
-                # Log first 5 segments for debugging
-                logging.debug(f"Segments before formatting: {segments[:5]}")
-                transcription = format_transcription_with_timestamps(segments)
-                logging.debug(f"Formatted transcription (first 500 chars): {transcription[:500]}")
-                update_progress("Audio transcribed successfully.")
-            else:
-                update_progress("Unexpected segments format received from speech_to_text.")
-                logging.error(f"Unexpected segments format: {segments}")
-                failed_count += 1
-                log_counter(
-                    metric_name="audio_files_failed_total",
-                    labels={"whisper_model": whisper_model, "api_name": api_name},
-                    value=1
-                )
-                continue
-
-            if not transcription.strip():
-                update_progress("Transcription is empty.")
-                failed_count += 1
-                log_counter(
-                    metric_name="audio_files_failed_total",
-                    labels={"whisper_model": whisper_model, "api_name": api_name},
-                    value=1
-                )
-            else:
-                # Apply chunking
-                chunked_text = improved_chunking_process(transcription, chunk_options)
-
-                # Summarize
-                logging.debug(f"Audio Transcription API Name: {api_name}")
-                if api_name:
-                    try:
-                        summary = perform_summarization(api_name, chunked_text, custom_prompt_input, api_key)
-                        update_progress("Audio summarized successfully.")
-                    except Exception as e:
-                        logging.error(f"Error during summarization: {str(e)}")
-                        summary = "Summary generation failed"
-                        failed_count += 1
-                        log_counter(
-                            metric_name="audio_files_failed_total",
-                            labels={"whisper_model": whisper_model, "api_name": api_name},
-                            value=1
-                        )
-                else:
-                    summary = "No summary available (API not provided)"
+        # Process URLs if provided
+        if audio_urls:
+            urls = [url.strip() for url in audio_urls.split('\n') if url.strip()]
+            for i, url in enumerate(urls):
+                try:
+                    update_progress(f"Processing URL {i + 1}/{len(urls)}: {url}")
 
-                all_transcriptions.append(transcription)
-                all_summaries.append(summary)
+                    # Download and process audio file
+                    audio_file_path = download_audio_file(url, use_cookies, cookies)
+                    if not audio_file_path:
+                        raise FileNotFoundError(f"Failed to download audio from URL: {url}")
 
-                # Use custom_title if provided, otherwise use the original filename
-                title = custom_title if custom_title else os.path.basename(wav_file_path)
-
-                # Add to database
-                add_media_with_keywords(
-                    url=url,
-                    title=title,
-                    media_type='audio',
-                    content=transcription,
-                    keywords=custom_keywords,
-                    prompt=custom_prompt_input,
-                    summary=summary,
-                    transcription_model=whisper_model,
-                    author="Unknown",
-                    ingestion_date=datetime.now().strftime('%Y-%m-%d')
-                )
-                update_progress("Audio file processed and added to database.")
-                processed_count += 1
-                log_counter(
-                    metric_name="audio_files_processed_total",
-                    labels={"whisper_model": whisper_model, "api_name": api_name},
-                    value=1
-                )
-
-        # Process uploaded file if provided
-        if audio_file:
-            url = generate_unique_id()
-            if os.path.getsize(audio_file.name) > MAX_FILE_SIZE:
-                update_progress(
-                    f"Uploaded file size exceeds the maximum limit of {MAX_FILE_SIZE / (1024 * 1024):.2f}MB. Skipping this file.")
-            else:
-                try:
-                    # Re-encode MP3 to fix potential issues
-                    reencoded_mp3_path = reencode_mp3(audio_file.name)
-                    if not os.path.exists(reencoded_mp3_path):
-                        update_progress(f"Re-encoded file not found: {reencoded_mp3_path}")
-                        return update_progress("Processing failed: Re-encoded file not found"), "", ""
+                    temp_files.append(audio_file_path)
 
+                    # Process the audio file
+                    reencoded_mp3_path = reencode_mp3(audio_file_path)
                     temp_files.append(reencoded_mp3_path)
 
-                    # Convert re-encoded MP3 to WAV
                     wav_file_path = convert_mp3_to_wav(reencoded_mp3_path)
-                    if not os.path.exists(wav_file_path):
-                        update_progress(f"Converted WAV file not found: {wav_file_path}")
-                        return update_progress("Processing failed: Converted WAV file not found"), "", ""
-
                     temp_files.append(wav_file_path)
 
-                    # Initialize transcription
-                    transcription = ""
-
-                    if diarize:
-                        segments = speech_to_text(wav_file_path, whisper_model=whisper_model, diarize=True)
-                    else:
-                        segments = speech_to_text(wav_file_path, whisper_model=whisper_model)
+                    # Transcribe audio
+                    segments = speech_to_text(wav_file_path, whisper_model=whisper_model, diarize=diarize)
 
-                    # Handle segments nested under 'segments' key
+                    # Handle segments format
                     if isinstance(segments, dict) and 'segments' in segments:
                         segments = segments['segments']
 
-                    if isinstance(segments, list):
-                        transcription = format_transcription_with_timestamps(segments)
-                    else:
-                        update_progress("Unexpected segments format received from speech_to_text.")
-                        logging.error(f"Unexpected segments format: {segments}")
+                    if not isinstance(segments, list):
+                        raise ValueError("Unexpected segments format received from speech_to_text")
 
-                    chunked_text = improved_chunking_process(transcription, chunk_options)
+                    transcription = format_transcription_with_timestamps(segments)
+                    if not transcription.strip():
+                        raise ValueError("Empty transcription generated")
 
-                    logging.debug(f"Audio Transcription API Name: {api_name}")
-                    if api_name:
+                    # Initialize summary with default value
+                    summary = "No summary available"
+
+                    # Attempt summarization if API is provided
+                    if api_name and api_name.lower() != "none":
                         try:
-                            summary = perform_summarization(api_name, chunked_text, custom_prompt_input, api_key)
+                            chunked_text = improved_chunking_process(transcription, chunk_options)
+                            summary_result = perform_summarization(api_name, chunked_text, custom_prompt_input, api_key)
+                            if summary_result:
+                                summary = summary_result
                             update_progress("Audio summarized successfully.")
                         except Exception as e:
-                            logging.error(f"Error during summarization: {str(e)}")
+                            logging.error(f"Summarization failed: {str(e)}")
                             summary = "Summary generation failed"
-                    else:
-                        summary = "No summary available (API not provided)"
 
+                    # Add to results
                     all_transcriptions.append(transcription)
                     all_summaries.append(summary)
 
-                    # Use custom_title if provided, otherwise use the original filename
+                    # Add to database
                     title = custom_title if custom_title else os.path.basename(wav_file_path)
-
                     add_media_with_keywords(
-                        url="Uploaded File",
+                        url=url,
                         title=title,
                         media_type='audio',
                         content=transcription,
@@ -400,65 +258,112 @@ def process_audio_files(audio_urls, audio_file, whisper_model, api_name, api_key
                         author="Unknown",
                         ingestion_date=datetime.now().strftime('%Y-%m-%d')
                     )
-                    update_progress("Uploaded file processed and added to database.")
+
                     processed_count += 1
-                    log_counter(
-                        metric_name="audio_files_processed_total",
-                        labels={"whisper_model": whisper_model, "api_name": api_name},
-                        value=1
-                    )
+                    update_progress(f"Successfully processed URL {i + 1}")
+                    log_counter("audio_files_processed_total", 1, {"whisper_model": whisper_model, "api_name": api_name})
+
                 except Exception as e:
-                    update_progress(f"Error processing uploaded file: {str(e)}")
-                    logging.error(f"Error processing uploaded file: {str(e)}")
                     failed_count += 1
-                    log_counter(
-                        metric_name="audio_files_failed_total",
-                        labels={"whisper_model": whisper_model, "api_name": api_name},
-                        value=1
-                    )
-                    return update_progress("Processing failed: Error processing uploaded file"), "", ""
-        # Final cleanup
-        if not keep_original:
-            cleanup_files()
+                    update_progress(f"Failed to process URL {i + 1}: {str(e)}")
+                    log_counter("audio_files_failed_total", 1, {"whisper_model": whisper_model, "api_name": api_name})
+                    continue
 
-        end_time = time.time()
-        processing_time = end_time - start_time
-        # Log processing time
-        log_histogram(
-            metric_name="audio_processing_time_seconds",
-            value=processing_time,
-            labels={"whisper_model": whisper_model, "api_name": api_name}
-        )
+        # Process uploaded file if provided
+        if audio_file:
+            try:
+                update_progress("Processing uploaded file...")
+                if os.path.getsize(audio_file.name) > MAX_FILE_SIZE:
+                    raise ValueError(f"File size exceeds maximum limit of {MAX_FILE_SIZE / (1024 * 1024):.2f}MB")
 
-        # Optionally, log total counts
-        log_counter(
-            metric_name="total_audio_files_processed",
-            labels={"whisper_model": whisper_model, "api_name": api_name},
-            value=processed_count
-        )
+                reencoded_mp3_path = reencode_mp3(audio_file.name)
+                temp_files.append(reencoded_mp3_path)
 
-        log_counter(
-            metric_name="total_audio_files_failed",
-            labels={"whisper_model": whisper_model, "api_name": api_name},
-            value=failed_count
-        )
+                wav_file_path = convert_mp3_to_wav(reencoded_mp3_path)
+                temp_files.append(wav_file_path)
+
+                # Transcribe audio
+                segments = speech_to_text(wav_file_path, whisper_model=whisper_model, diarize=diarize)
+
+                if isinstance(segments, dict) and 'segments' in segments:
+                    segments = segments['segments']
+
+                if not isinstance(segments, list):
+                    raise ValueError("Unexpected segments format received from speech_to_text")
 
+                transcription = format_transcription_with_timestamps(segments)
+                if not transcription.strip():
+                    raise ValueError("Empty transcription generated")
+
+                # Initialize summary with default value
+                summary = "No summary available"
 
-        final_progress = update_progress("All processing complete.")
-        final_transcriptions = "\n\n".join(all_transcriptions)
-        final_summaries = "\n\n".join(all_summaries)
+                # Attempt summarization if API is provided
+                if api_name and api_name.lower() != "none":
+                    try:
+                        chunked_text = improved_chunking_process(transcription, chunk_options)
+                        summary_result = perform_summarization(api_name, chunked_text, custom_prompt_input, api_key)
+                        if summary_result:
+                            summary = summary_result
+                        update_progress("Audio summarized successfully.")
+                    except Exception as e:
+                        logging.error(f"Summarization failed: {str(e)}")
+                        summary = "Summary generation failed"
+
+                # Add to results
+                all_transcriptions.append(transcription)
+                all_summaries.append(summary)
+
+                # Add to database
+                title = custom_title if custom_title else os.path.basename(wav_file_path)
+                add_media_with_keywords(
+                    url="Uploaded File",
+                    title=title,
+                    media_type='audio',
+                    content=transcription,
+                    keywords=custom_keywords,
+                    prompt=custom_prompt_input,
+                    summary=summary,
+                    transcription_model=whisper_model,
+                    author="Unknown",
+                    ingestion_date=datetime.now().strftime('%Y-%m-%d')
+                )
+
+                processed_count += 1
+                update_progress("Successfully processed uploaded file")
+                log_counter("audio_files_processed_total", 1, {"whisper_model": whisper_model, "api_name": api_name})
+
+            except Exception as e:
+                failed_count += 1
+                update_progress(f"Failed to process uploaded file: {str(e)}")
+                log_counter("audio_files_failed_total", 1, {"whisper_model": whisper_model, "api_name": api_name})
+
+        # Cleanup temporary files
+        if not keep_original:
+            cleanup_files()
+
+        # Log processing metrics
+        processing_time = time.time() - start_time
+        log_histogram("audio_processing_time_seconds", processing_time,
+                     {"whisper_model": whisper_model, "api_name": api_name})
+        log_counter("total_audio_files_processed", processed_count,
+                   {"whisper_model": whisper_model, "api_name": api_name})
+        log_counter("total_audio_files_failed", failed_count,
+                   {"whisper_model": whisper_model, "api_name": api_name})
+
+        # Prepare final output
+        final_progress = update_progress(f"Processing complete. Processed: {processed_count}, Failed: {failed_count}")
+        final_transcriptions = "\n\n".join(all_transcriptions) if all_transcriptions else "No transcriptions available"
+        final_summaries = "\n\n".join(all_summaries) if all_summaries else "No summaries available"
 
         return final_progress, final_transcriptions, final_summaries
 
     except Exception as e:
-        logging.error(f"Error processing audio files: {str(e)}")
-        log_counter(
-            metric_name="audio_files_failed_total",
-            labels={"whisper_model": whisper_model, "api_name": api_name},
-            value=1
-        )
-        cleanup_files()
-        return update_progress(f"Processing failed: {str(e)}"), "", ""
+        logging.error(f"Error in process_audio_files: {str(e)}")
+        log_counter("audio_files_failed_total", 1, {"whisper_model": whisper_model, "api_name": api_name})
+        if not keep_original:
+            cleanup_files()
+        return update_progress(f"Processing failed: {str(e)}"), "No transcriptions available", "No summaries available"
 
 
 def format_transcription_with_timestamps(segments, keep_timestamps):

App_Function_Libraries/Audio/Audio_Transcription_Lib.py

@@ -332,4 +332,4 @@ def save_audio_temp(audio_data, sample_rate=16000):
 
 #
 #
-#######################################################################################################################
+#######################################################################################################################

App_Function_Libraries/Benchmarks_Evaluations/ms_g_eval.py

@@ -24,7 +24,7 @@ from tenacity import (
     wait_random_exponential,
 )
 
-from App_Function_Libraries.Chat import chat_api_call
+from App_Function_Libraries.Chat.Chat_Functions import chat_api_call
 
 #
 #######################################################################################################################

App_Function_Libraries/Books/Book_Ingestion_Lib.py

@@ -18,6 +18,9 @@ import tempfile
 import zipfile
 from datetime import datetime
 import logging
+import xml.etree.ElementTree as ET
+import html2text
+import csv
 #
 # External Imports
 import ebooklib
@@ -241,109 +244,244 @@ def process_zip_file(zip_file,
     return "\n".join(results)
 
 
-def import_file_handler(file,
-                        title,
-                        author,
-                        keywords,
-                        system_prompt,
-                        custom_prompt,
-                        auto_summarize,
-                        api_name,
-                        api_key,
-                        max_chunk_size,
-                        chunk_overlap,
-                        custom_chapter_pattern
-                        ):
+def import_html(file_path, title=None, author=None, keywords=None, **kwargs):
+    """
+    Imports an HTML file and converts it to markdown format.
+    """
     try:
-        log_counter("file_import_attempt", labels={"file_name": file.name})
-
-        # Handle max_chunk_size
-        if isinstance(max_chunk_size, str):
-            max_chunk_size = int(max_chunk_size) if max_chunk_size.strip() else 4000
-        elif not isinstance(max_chunk_size, int):
-            max_chunk_size = 4000  # Default value if not a string or int
-
-        # Handle chunk_overlap
-        if isinstance(chunk_overlap, str):
-            chunk_overlap = int(chunk_overlap) if chunk_overlap.strip() else 0
-        elif not isinstance(chunk_overlap, int):
-            chunk_overlap = 0  # Default value if not a string or int
-
-        chunk_options = {
-            'method': 'chapter',
-            'max_size': max_chunk_size,
-            'overlap': chunk_overlap,
-            'custom_chapter_pattern': custom_chapter_pattern if custom_chapter_pattern else None
-        }
+        logging.info(f"Importing HTML file from {file_path}")
+        h = html2text.HTML2Text()
+        h.ignore_links = False
 
-        if file is None:
-            log_counter("file_import_error", labels={"error": "No file uploaded"})
-            return "No file uploaded."
+        with open(file_path, 'r', encoding='utf-8') as file:
+            html_content = file.read()
 
-        file_path = file.name
-        if not os.path.exists(file_path):
-            log_counter("file_import_error", labels={"error": "File not found", "file_name": file.name})
-            return "Uploaded file not found."
+        markdown_content = h.handle(html_content)
 
-        start_time = datetime.now()
+        # Extract title from HTML if not provided
+        if not title:
+            soup = BeautifulSoup(html_content, 'html.parser')
+            title_tag = soup.find('title')
+            title = title_tag.string if title_tag else os.path.basename(file_path)
 
-        if file_path.lower().endswith('.epub'):
-            status = import_epub(
-                file_path,
-                title,
-                author,
-                keywords,
-                custom_prompt=custom_prompt,
-                system_prompt=system_prompt,
-                summary=None,
-                auto_summarize=auto_summarize,
-                api_name=api_name,
-                api_key=api_key,
-                chunk_options=chunk_options,
-                custom_chapter_pattern=custom_chapter_pattern
-            )
-            log_counter("epub_import_success", labels={"file_name": file.name})
-            result = f"📚 EPUB Imported Successfully:\n{status}"
-        elif file.name.lower().endswith('.zip'):
-            status = process_zip_file(
-                zip_file=file,
-                title=title,
-                author=author,
-                keywords=keywords,
-                custom_prompt=custom_prompt,
-                system_prompt=system_prompt,
-                summary=None,
-                auto_summarize=auto_summarize,
-                api_name=api_name,
-                api_key=api_key,
-                chunk_options=chunk_options
-            )
-            log_counter("zip_import_success", labels={"file_name": file.name})
-            result = f"📦 ZIP Processed Successfully:\n{status}"
-        elif file.name.lower().endswith(('.chm', '.html', '.pdf', '.xml', '.opml')):
-            file_type = file.name.split('.')[-1].upper()
-            log_counter("unsupported_file_type", labels={"file_type": file_type})
-            result = f"{file_type} file import is not yet supported."
-        else:
-            log_counter("unsupported_file_type", labels={"file_type": file.name.split('.')[-1]})
-            result = "❌ Unsupported file type. Please upload an `.epub` file or a `.zip` file containing `.epub` files."
+        return process_markdown_content(markdown_content, file_path, title, author, keywords, **kwargs)
 
-        end_time = datetime.now()
-        processing_time = (end_time - start_time).total_seconds()
-        log_histogram("file_import_duration", processing_time, labels={"file_name": file.name})
+    except Exception as e:
+        logging.exception(f"Error importing HTML file: {str(e)}")
+        raise
+
+
+def import_xml(file_path, title=None, author=None, keywords=None, **kwargs):
+    """
+    Imports an XML file and converts it to markdown format.
+    """
+    try:
+        logging.info(f"Importing XML file from {file_path}")
+        tree = ET.parse(file_path)
+        root = tree.getroot()
+
+        # Convert XML to markdown
+        markdown_content = xml_to_markdown(root)
+
+        return process_markdown_content(markdown_content, file_path, title, author, keywords, **kwargs)
+
+    except Exception as e:
+        logging.exception(f"Error importing XML file: {str(e)}")
+        raise
+
+
+def import_opml(file_path, title=None, author=None, keywords=None, **kwargs):
+    """
+    Imports an OPML file and converts it to markdown format.
+    """
+    try:
+        logging.info(f"Importing OPML file from {file_path}")
+        tree = ET.parse(file_path)
+        root = tree.getroot()
+
+        # Extract title from OPML if not provided
+        if not title:
+            title_elem = root.find(".//title")
+            title = title_elem.text if title_elem is not None else os.path.basename(file_path)
+
+        # Convert OPML to markdown
+        markdown_content = opml_to_markdown(root)
+
+        return process_markdown_content(markdown_content, file_path, title, author, keywords, **kwargs)
+
+    except Exception as e:
+        logging.exception(f"Error importing OPML file: {str(e)}")
+        raise
+
+
+def xml_to_markdown(element, level=0):
+    """
+    Recursively converts XML elements to markdown format.
+    """
+    markdown = ""
+
+    # Add element name as heading
+    if level > 0:
+        markdown += f"{'#' * min(level, 6)} {element.tag}\n\n"
+
+    # Add element text if it exists
+    if element.text and element.text.strip():
+        markdown += f"{element.text.strip()}\n\n"
+
+    # Process child elements
+    for child in element:
+        markdown += xml_to_markdown(child, level + 1)
+
+    return markdown
 
+
+def opml_to_markdown(root):
+    """
+    Converts OPML structure to markdown format.
+    """
+    markdown = "# Table of Contents\n\n"
+
+    def process_outline(outline, level=0):
+        result = ""
+        for item in outline.findall("outline"):
+            text = item.get("text", "")
+            result += f"{'  ' * level}- {text}\n"
+            result += process_outline(item, level + 1)
         return result
 
+    body = root.find(".//body")
+    if body is not None:
+        markdown += process_outline(body)
+
+    return markdown
+
+
+def process_markdown_content(markdown_content, file_path, title, author, keywords, **kwargs):
+    """
+    Processes markdown content and adds it to the database.
+    """
+    info_dict = {
+        'title': title or os.path.basename(file_path),
+        'uploader': author or "Unknown",
+        'ingestion_date': datetime.now().strftime('%Y-%m-%d')
+    }
+
+    # Create segments (you may want to adjust the chunking method)
+    segments = [{'Text': markdown_content}]
+
+    # Add to database
+    result = add_media_to_database(
+        url=file_path,
+        info_dict=info_dict,
+        segments=segments,
+        summary=kwargs.get('summary', "No summary provided"),
+        keywords=keywords.split(',') if keywords else [],
+        custom_prompt_input=kwargs.get('custom_prompt'),
+        whisper_model="Imported",
+        media_type="document",
+        overwrite=False
+    )
+
+    return f"Document '{title}' imported successfully. Database result: {result}"
+
+
+def import_file_handler(files,
+                       author,
+                       keywords,
+                       system_prompt,
+                       custom_prompt,
+                       auto_summarize,
+                       api_name,
+                       api_key,
+                       max_chunk_size,
+                       chunk_overlap,
+                       custom_chapter_pattern):
+    try:
+        if not files:
+            return "No files uploaded."
+
+        # Convert single file to list for consistent processing
+        if not isinstance(files, list):
+            files = [files]
+
+        results = []
+        for file in files:
+            log_counter("file_import_attempt", labels={"file_name": file.name})
+
+            # Handle max_chunk_size and chunk_overlap
+            chunk_size = int(max_chunk_size) if isinstance(max_chunk_size, (str, int)) else 4000
+            overlap = int(chunk_overlap) if isinstance(chunk_overlap, (str, int)) else 0
+
+            chunk_options = {
+                'method': 'chapter',
+                'max_size': chunk_size,
+                'overlap': overlap,
+                'custom_chapter_pattern': custom_chapter_pattern if custom_chapter_pattern else None
+            }
+
+            file_path = file.name
+            if not os.path.exists(file_path):
+                results.append(f"❌ File not found: {file.name}")
+                continue
+
+            start_time = datetime.now()
+
+            # Extract title from filename
+            title = os.path.splitext(os.path.basename(file_path))[0]
+
+            if file_path.lower().endswith('.epub'):
+                status = import_epub(
+                    file_path,
+                    title=title,  # Use filename as title
+                    author=author,
+                    keywords=keywords,
+                    custom_prompt=custom_prompt,
+                    system_prompt=system_prompt,
+                    summary=None,
+                    auto_summarize=auto_summarize,
+                    api_name=api_name,
+                    api_key=api_key,
+                    chunk_options=chunk_options,
+                    custom_chapter_pattern=custom_chapter_pattern
+                )
+                log_counter("epub_import_success", labels={"file_name": file.name})
+                results.append(f"📚 {file.name}: {status}")
+
+            elif file_path.lower().endswith('.zip'):
+                status = process_zip_file(
+                    zip_file=file,
+                    title=None,  # Let each file use its own name
+                    author=author,
+                    keywords=keywords,
+                    custom_prompt=custom_prompt,
+                    system_prompt=system_prompt,
+                    summary=None,
+                    auto_summarize=auto_summarize,
+                    api_name=api_name,
+                    api_key=api_key,
+                    chunk_options=chunk_options
+                )
+                log_counter("zip_import_success", labels={"file_name": file.name})
+                results.append(f"📦 {file.name}: {status}")
+            else:
+                results.append(f"❌ Unsupported file type: {file.name}")
+                continue
+
+            end_time = datetime.now()
+            processing_time = (end_time - start_time).total_seconds()
+            log_histogram("file_import_duration", processing_time, labels={"file_name": file.name})
+
+        return "\n\n".join(results)
+
     except ValueError as ve:
         logging.exception(f"Error parsing input values: {str(ve)}")
-        log_counter("file_import_error", labels={"error": "Invalid input", "file_name": file.name})
         return f"❌ Error: Invalid input for chunk size or overlap. Please enter valid numbers."
     except Exception as e:
         logging.exception(f"Error during file import: {str(e)}")
-        log_counter("file_import_error", labels={"error": str(e), "file_name": file.name})
         return f"❌ Error during import: {str(e)}"
 
 
+
 def read_epub(file_path):
     """
     Reads and extracts text from an EPUB file.
@@ -424,9 +562,9 @@ def ingest_text_file(file_path, title=None, author=None, keywords=None):
 
         # Add the text file to the database
         add_media_with_keywords(
-            url=file_path,
+            url="its_a_book",
             title=title,
-            media_type='document',
+            media_type='book',
             content=content,
             keywords=keywords,
             prompt='No prompt for text files',

App_Function_Libraries/Chat/Chat_Functions.py

@@ -0,0 +1,453 @@
+# Chat_Functions.py
+# Chat functions for interacting with the LLMs as chatbots
+import base64
+# Imports
+import json
+import logging
+import os
+import re
+import sqlite3
+import tempfile
+import time
+from datetime import datetime
+from pathlib import Path
+#
+# External Imports
+#
+# Local Imports
+from App_Function_Libraries.DB.DB_Manager import start_new_conversation, delete_messages_in_conversation, save_message
+from App_Function_Libraries.DB.RAG_QA_Chat_DB import get_db_connection, get_conversation_name
+from App_Function_Libraries.LLM_API_Calls import chat_with_openai, chat_with_anthropic, chat_with_cohere, \
+    chat_with_groq, chat_with_openrouter, chat_with_deepseek, chat_with_mistral, chat_with_huggingface
+from App_Function_Libraries.LLM_API_Calls_Local import chat_with_aphrodite, chat_with_local_llm, chat_with_ollama, \
+    chat_with_kobold, chat_with_llama, chat_with_oobabooga, chat_with_tabbyapi, chat_with_vllm, chat_with_custom_openai
+from App_Function_Libraries.DB.SQLite_DB import load_media_content
+from App_Function_Libraries.Utils.Utils import generate_unique_filename, load_and_log_configs
+from App_Function_Libraries.Metrics.metrics_logger import log_counter, log_histogram
+#
+####################################################################################################
+#
+# Functions:
+
+def approximate_token_count(history):
+    total_text = ''
+    for user_msg, bot_msg in history:
+        if user_msg:
+            total_text += user_msg + ' '
+        if bot_msg:
+            total_text += bot_msg + ' '
+    total_tokens = len(total_text.split())
+    return total_tokens
+
+def chat_api_call(api_endpoint, api_key, input_data, prompt, temp, system_message=None):
+    log_counter("chat_api_call_attempt", labels={"api_endpoint": api_endpoint})
+    start_time = time.time()
+    if not api_key:
+        api_key = None
+    model = None
+    try:
+        logging.info(f"Debug - Chat API Call - API Endpoint: {api_endpoint}")
+        logging.info(f"Debug - Chat API Call - API Key: {api_key}")
+        logging.info(f"Debug - Chat chat_api_call - API Endpoint: {api_endpoint}")
+        if api_endpoint.lower() == 'openai':
+            response = chat_with_openai(api_key, input_data, prompt, temp, system_message)
+
+        elif api_endpoint.lower() == 'anthropic':
+            # Retrieve the model from config
+            loaded_config_data = load_and_log_configs()
+            model = loaded_config_data['models']['anthropic'] if loaded_config_data else None
+            response = chat_with_anthropic(
+                api_key=api_key,
+                input_data=input_data,
+                model=model,
+                custom_prompt_arg=prompt,
+                system_prompt=system_message
+            )
+
+        elif api_endpoint.lower() == "cohere":
+            response = chat_with_cohere(
+                api_key,
+                input_data,
+                model=model,
+                custom_prompt_arg=prompt,
+                system_prompt=system_message,
+                temp=temp
+            )
+
+        elif api_endpoint.lower() == "groq":
+            response = chat_with_groq(api_key, input_data, prompt, temp, system_message)
+
+        elif api_endpoint.lower() == "openrouter":
+            response = chat_with_openrouter(api_key, input_data, prompt, temp, system_message)
+
+        elif api_endpoint.lower() == "deepseek":
+            response = chat_with_deepseek(api_key, input_data, prompt, temp, system_message)
+
+        elif api_endpoint.lower() == "mistral":
+            response = chat_with_mistral(api_key, input_data, prompt, temp, system_message)
+
+        elif api_endpoint.lower() == "llama.cpp":
+            response = chat_with_llama(input_data, prompt, temp, None, api_key, system_message)
+        elif api_endpoint.lower() == "kobold":
+            response = chat_with_kobold(input_data, api_key, prompt, temp, system_message)
+
+        elif api_endpoint.lower() == "ooba":
+            response = chat_with_oobabooga(input_data, api_key, prompt, temp, system_message)
+
+        elif api_endpoint.lower() == "tabbyapi":
+            response = chat_with_tabbyapi(input_data, prompt, temp, system_message)
+
+        elif api_endpoint.lower() == "vllm":
+            response = chat_with_vllm(input_data, prompt, system_message)
+
+        elif api_endpoint.lower() == "local-llm":
+            response = chat_with_local_llm(input_data, prompt, temp, system_message)
+
+        elif api_endpoint.lower() == "huggingface":
+            response = chat_with_huggingface(api_key, input_data, prompt, temp)  # , system_message)
+
+        elif api_endpoint.lower() == "ollama":
+            response = chat_with_ollama(input_data, prompt, None, api_key, temp, system_message)
+
+        elif api_endpoint.lower() == "aphrodite":
+            response = chat_with_aphrodite(input_data, prompt, temp, system_message)
+
+        elif api_endpoint.lower() == "custom-openai-api":
+            response = chat_with_custom_openai(api_key, input_data, prompt, temp, system_message)
+
+        else:
+            raise ValueError(f"Unsupported API endpoint: {api_endpoint}")
+
+        call_duration = time.time() - start_time
+        log_histogram("chat_api_call_duration", call_duration, labels={"api_endpoint": api_endpoint})
+        log_counter("chat_api_call_success", labels={"api_endpoint": api_endpoint})
+        return response
+
+    except Exception as e:
+        log_counter("chat_api_call_error", labels={"api_endpoint": api_endpoint, "error": str(e)})
+        logging.error(f"Error in chat function: {str(e)}")
+        return f"An error occurred: {str(e)}"
+
+
+def chat(message, history, media_content, selected_parts, api_endpoint, api_key, prompt, temperature,
+         system_message=None):
+    log_counter("chat_attempt", labels={"api_endpoint": api_endpoint})
+    start_time = time.time()
+    try:
+        logging.info(f"Debug - Chat Function - Message: {message}")
+        logging.info(f"Debug - Chat Function - Media Content: {media_content}")
+        logging.info(f"Debug - Chat Function - Selected Parts: {selected_parts}")
+        logging.info(f"Debug - Chat Function - API Endpoint: {api_endpoint}")
+        # logging.info(f"Debug - Chat Function - Prompt: {prompt}")
+
+        # Ensure selected_parts is a list
+        if not isinstance(selected_parts, (list, tuple)):
+            selected_parts = [selected_parts] if selected_parts else []
+
+        # logging.debug(f"Debug - Chat Function - Selected Parts (after check): {selected_parts}")
+
+        # Combine the selected parts of the media content
+        combined_content = "\n\n".join(
+            [f"{part.capitalize()}: {media_content.get(part, '')}" for part in selected_parts if part in media_content])
+        # Print first 500 chars
+        # logging.debug(f"Debug - Chat Function - Combined Content: {combined_content[:500]}...")
+
+        # Prepare the input for the API
+        input_data = f"{combined_content}\n\n" if combined_content else ""
+        for old_message, old_response in history:
+            input_data += f"{old_message}\nAssistant: {old_response}\n\n"
+        input_data += f"{message}\n"
+
+        if system_message:
+            print(f"System message: {system_message}")
+            logging.debug(f"Debug - Chat Function - System Message: {system_message}")
+        temperature = float(temperature) if temperature else 0.7
+        temp = temperature
+
+        logging.debug(f"Debug - Chat Function - Temperature: {temperature}")
+        logging.debug(f"Debug - Chat Function - API Key: {api_key[:10]}")
+        logging.debug(f"Debug - Chat Function - Prompt: {prompt}")
+
+        # Use the existing API request code based on the selected endpoint
+        response = chat_api_call(api_endpoint, api_key, input_data, prompt, temp, system_message)
+
+        chat_duration = time.time() - start_time
+        log_histogram("chat_duration", chat_duration, labels={"api_endpoint": api_endpoint})
+        log_counter("chat_success", labels={"api_endpoint": api_endpoint})
+        return response
+    except Exception as e:
+        log_counter("chat_error", labels={"api_endpoint": api_endpoint, "error": str(e)})
+        logging.error(f"Error in chat function: {str(e)}")
+        return f"An error occurred: {str(e)}"
+
+
+def save_chat_history_to_db_wrapper(chatbot, conversation_id, media_content, media_name=None):
+    log_counter("save_chat_history_to_db_attempt")
+    start_time = time.time()
+    logging.info(f"Attempting to save chat history. Media content type: {type(media_content)}")
+
+    try:
+        # First check if we can access the database
+        try:
+            with get_db_connection() as conn:
+                cursor = conn.cursor()
+                cursor.execute("SELECT 1")
+        except sqlite3.DatabaseError as db_error:
+            logging.error(f"Database is corrupted or inaccessible: {str(db_error)}")
+            return conversation_id, "Database error: The database file appears to be corrupted. Please contact support."
+
+        # Now attempt the save
+        if not conversation_id:
+            # Only for new conversations, not updates
+            media_id = None
+            if isinstance(media_content, dict) and 'content' in media_content:
+                try:
+                    content = media_content['content']
+                    content_json = content if isinstance(content, dict) else json.loads(content)
+                    media_id = content_json.get('webpage_url')
+                    media_name = media_name or content_json.get('title', 'Unnamed Media')
+                except (json.JSONDecodeError, AttributeError) as e:
+                    logging.error(f"Error processing media content: {str(e)}")
+                    media_id = "unknown_media"
+                    media_name = media_name or "Unnamed Media"
+            else:
+                media_id = "unknown_media"
+                media_name = media_name or "Unnamed Media"
+
+            timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+            conversation_title = f"{media_name}_{timestamp}"
+            conversation_id = start_new_conversation(title=conversation_title, media_id=media_id)
+            logging.info(f"Created new conversation with ID: {conversation_id}")
+
+        # For both new and existing conversations
+        try:
+            delete_messages_in_conversation(conversation_id)
+            for user_msg, assistant_msg in chatbot:
+                if user_msg:
+                    save_message(conversation_id, "user", user_msg)
+                if assistant_msg:
+                    save_message(conversation_id, "assistant", assistant_msg)
+        except sqlite3.DatabaseError as db_error:
+            logging.error(f"Database error during message save: {str(db_error)}")
+            return conversation_id, "Database error: Unable to save messages. Please try again or contact support."
+
+        save_duration = time.time() - start_time
+        log_histogram("save_chat_history_to_db_duration", save_duration)
+        log_counter("save_chat_history_to_db_success")
+
+        return conversation_id, "Chat history saved successfully!"
+
+    except Exception as e:
+        log_counter("save_chat_history_to_db_error", labels={"error": str(e)})
+        error_message = f"Failed to save chat history: {str(e)}"
+        logging.error(error_message, exc_info=True)
+        return conversation_id, error_message
+
+
+def save_chat_history(history, conversation_id, media_content):
+    log_counter("save_chat_history_attempt")
+    start_time = time.time()
+    try:
+        content, conversation_name = generate_chat_history_content(history, conversation_id, media_content)
+
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        safe_conversation_name = re.sub(r'[^a-zA-Z0-9_-]', '_', conversation_name)
+        base_filename = f"{safe_conversation_name}_{timestamp}.json"
+
+        # Create a temporary file
+        with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.json') as temp_file:
+            temp_file.write(content)
+            temp_file_path = temp_file.name
+
+        # Generate a unique filename
+        unique_filename = generate_unique_filename(os.path.dirname(temp_file_path), base_filename)
+        final_path = os.path.join(os.path.dirname(temp_file_path), unique_filename)
+
+        # Rename the temporary file to the unique filename
+        os.rename(temp_file_path, final_path)
+
+        save_duration = time.time() - start_time
+        log_histogram("save_chat_history_duration", save_duration)
+        log_counter("save_chat_history_success")
+        return final_path
+    except Exception as e:
+        log_counter("save_chat_history_error", labels={"error": str(e)})
+        logging.error(f"Error saving chat history: {str(e)}")
+        return None
+
+
+def generate_chat_history_content(history, conversation_id, media_content):
+    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+
+    conversation_name = get_conversation_name(conversation_id)
+
+    if not conversation_name:
+        media_name = extract_media_name(media_content)
+        if media_name:
+            conversation_name = f"{media_name}-chat"
+        else:
+            conversation_name = f"chat-{timestamp}"  # Fallback name
+
+    chat_data = {
+        "conversation_id": conversation_id,
+        "conversation_name": conversation_name,
+        "timestamp": timestamp,
+        "history": [
+            {
+                "role": "user" if i % 2 == 0 else "bot",
+                "content": msg[0] if isinstance(msg, tuple) else msg
+            }
+            for i, msg in enumerate(history)
+        ]
+    }
+
+    return json.dumps(chat_data, indent=2), conversation_name
+
+
+def extract_media_name(media_content):
+    if isinstance(media_content, dict):
+        content = media_content.get('content', {})
+        if isinstance(content, str):
+            try:
+                content = json.loads(content)
+            except json.JSONDecodeError:
+                logging.warning("Failed to parse media_content JSON string")
+                return None
+
+        # Try to extract title from the content
+        if isinstance(content, dict):
+            return content.get('title') or content.get('name')
+
+    logging.warning(f"Unexpected media_content format: {type(media_content)}")
+    return None
+
+
+def update_chat_content(selected_item, use_content, use_summary, use_prompt, item_mapping):
+    log_counter("update_chat_content_attempt")
+    start_time = time.time()
+    logging.debug(f"Debug - Update Chat Content - Selected Item: {selected_item}\n")
+    logging.debug(f"Debug - Update Chat Content - Use Content: {use_content}\n\n\n\n")
+    logging.debug(f"Debug - Update Chat Content - Use Summary: {use_summary}\n\n")
+    logging.debug(f"Debug - Update Chat Content - Use Prompt: {use_prompt}\n\n")
+    logging.debug(f"Debug - Update Chat Content - Item Mapping: {item_mapping}\n\n")
+
+    if selected_item and selected_item in item_mapping:
+        media_id = item_mapping[selected_item]
+        content = load_media_content(media_id)
+        selected_parts = []
+        if use_content and "content" in content:
+            selected_parts.append("content")
+        if use_summary and "summary" in content:
+            selected_parts.append("summary")
+        if use_prompt and "prompt" in content:
+            selected_parts.append("prompt")
+
+        # Modified debug print
+        if isinstance(content, dict):
+            print(f"Debug - Update Chat Content - Content keys: {list(content.keys())}")
+            for key, value in content.items():
+                print(f"Debug - Update Chat Content - {key} (first 500 char): {str(value)[:500]}\n\n\n\n")
+        else:
+            print(f"Debug - Update Chat Content - Content(first 500 char): {str(content)[:500]}\n\n\n\n")
+
+        print(f"Debug - Update Chat Content - Selected Parts: {selected_parts}")
+        update_duration = time.time() - start_time
+        log_histogram("update_chat_content_duration", update_duration)
+        log_counter("update_chat_content_success")
+        return content, selected_parts
+    else:
+        log_counter("update_chat_content_error", labels={"error": str("No item selected or item not in mapping")})
+        print(f"Debug - Update Chat Content - No item selected or item not in mapping")
+        return {}, []
+
+#
+# End of Chat functions
+#######################################################################################################################
+
+
+#######################################################################################################################
+#
+# Character Card Functions
+
+CHARACTERS_FILE = Path('.', 'Helper_Scripts', 'Character_Cards', 'Characters.json')
+
+
+def save_character(character_data):
+    log_counter("save_character_attempt")
+    start_time = time.time()
+    characters_file = os.path.join(os.path.dirname(__file__), '..', 'Helper_Scripts', 'Character_Cards', 'Characters.json')
+    characters_dir = os.path.dirname(characters_file)
+
+    try:
+        if os.path.exists(characters_file):
+            with open(characters_file, 'r') as f:
+                characters = json.load(f)
+        else:
+            characters = {}
+
+        char_name = character_data['name']
+
+        # Save the image separately if it exists
+        if 'image' in character_data:
+            img_data = base64.b64decode(character_data['image'])
+            img_filename = f"{char_name.replace(' ', '_')}.png"
+            img_path = os.path.join(characters_dir, img_filename)
+            with open(img_path, 'wb') as f:
+                f.write(img_data)
+            character_data['image_path'] = os.path.abspath(img_path)
+            del character_data['image']  # Remove the base64 image data from the JSON
+
+        characters[char_name] = character_data
+
+        with open(characters_file, 'w') as f:
+            json.dump(characters, f, indent=2)
+
+        save_duration = time.time() - start_time
+        log_histogram("save_character_duration", save_duration)
+        log_counter("save_character_success")
+        logging.info(f"Character '{char_name}' saved successfully.")
+    except Exception as e:
+        log_counter("save_character_error", labels={"error": str(e)})
+        logging.error(f"Error saving character: {str(e)}")
+
+
+def load_characters():
+    log_counter("load_characters_attempt")
+    start_time = time.time()
+    try:
+        characters_file = os.path.join(os.path.dirname(__file__), '..', 'Helper_Scripts', 'Character_Cards', 'Characters.json')
+        if os.path.exists(characters_file):
+            with open(characters_file, 'r') as f:
+                characters = json.load(f)
+            logging.debug(f"Loaded {len(characters)} characters from {characters_file}")
+            load_duration = time.time() - start_time
+            log_histogram("load_characters_duration", load_duration)
+            log_counter("load_characters_success", labels={"character_count": len(characters)})
+            return characters
+        else:
+            logging.warning(f"Characters file not found: {characters_file}")
+            return {}
+    except Exception as e:
+        log_counter("load_characters_error", labels={"error": str(e)})
+        return {}
+
+
+
+def get_character_names():
+    log_counter("get_character_names_attempt")
+    start_time = time.time()
+    try:
+        characters = load_characters()
+        names = list(characters.keys())
+        get_names_duration = time.time() - start_time
+        log_histogram("get_character_names_duration", get_names_duration)
+        log_counter("get_character_names_success", labels={"name_count": len(names)})
+        return names
+    except Exception as e:
+        log_counter("get_character_names_error", labels={"error": str(e)})
+        logging.error(f"Error getting character names: {str(e)}")
+        return []
+
+#
+# End of Chat.py
+##########################################################################################################################

App_Function_Libraries/Chunk_Lib.py

@@ -11,6 +11,7 @@ import json
 import logging
 import re
 from typing import Any, Dict, List, Optional, Tuple
+import xml.etree.ElementTree as ET
 #
 # Import 3rd party
 from openai import OpenAI
@@ -23,7 +24,6 @@ from sklearn.feature_extraction.text import TfidfVectorizer
 from sklearn.metrics.pairwise import cosine_similarity
 #
 # Import Local
-from App_Function_Libraries.Tokenization_Methods_Lib import openai_tokenize
 from App_Function_Libraries.Utils.Utils import load_comprehensive_config
 #
 #######################################################################################################################
@@ -106,6 +106,7 @@ def load_document(file_path: str) -> str:
 
 def improved_chunking_process(text: str, chunk_options: Dict[str, Any] = None) -> List[Dict[str, Any]]:
     logging.debug("Improved chunking process started...")
+    logging.debug(f"Received chunk_options: {chunk_options}")
 
     # Extract JSON metadata if present
     json_content = {}
@@ -125,49 +126,70 @@ def improved_chunking_process(text: str, chunk_options: Dict[str, Any] = None) -
         text = text[len(header_text):].strip()
         logging.debug(f"Extracted header text: {header_text}")
 
-    options = chunk_options.copy() if chunk_options else {}
+    # Make a copy of chunk_options and ensure values are correct types
+    options = {}
     if chunk_options:
-        options.update(chunk_options)
-
-    chunk_method = options.get('method', 'words')
-    max_size = options.get('max_size', 2000)
-    overlap = options.get('overlap', 0)
-    language = options.get('language', None)
+        try:
+            options['method'] = str(chunk_options.get('method', 'words'))
+            options['max_size'] = int(chunk_options.get('max_size', 2000))
+            options['overlap'] = int(chunk_options.get('overlap', 0))
+            # Handle language specially - it can be None
+            lang = chunk_options.get('language')
+            options['language'] = str(lang) if lang is not None else None
+            logging.debug(f"Processed options: {options}")
+        except Exception as e:
+            logging.error(f"Error processing chunk options: {e}")
+            raise
+    else:
+        options = {'method': 'words', 'max_size': 2000, 'overlap': 0, 'language': None}
+        logging.debug("Using default options")
 
-    if language is None:
-        language = detect_language(text)
+    if options.get('language') is None:
+        detected_lang = detect_language(text)
+        options['language'] = str(detected_lang)
+        logging.debug(f"Detected language: {options['language']}")
 
-    if chunk_method == 'json':
-        chunks = chunk_text_by_json(text, max_size=max_size, overlap=overlap)
-    else:
-        chunks = chunk_text(text, chunk_method, max_size, overlap, language)
+    try:
+        if options['method'] == 'json':
+            chunks = chunk_text_by_json(text, max_size=options['max_size'], overlap=options['overlap'])
+        else:
+            chunks = chunk_text(text, options['method'], options['max_size'], options['overlap'], options['language'])
+        logging.debug(f"Created {len(chunks)} chunks using method {options['method']}")
+    except Exception as e:
+        logging.error(f"Error in chunking process: {e}")
+        raise
 
     chunks_with_metadata = []
     total_chunks = len(chunks)
-    for i, chunk in enumerate(chunks):
-        metadata = {
-            'chunk_index': i + 1,
-            'total_chunks': total_chunks,
-            'chunk_method': chunk_method,
-            'max_size': max_size,
-            'overlap': overlap,
-            'language': language,
-            'relative_position': (i + 1) / total_chunks
-        }
-        metadata.update(json_content)  # Add the extracted JSON content to metadata
-        metadata['header_text'] = header_text  # Add the header text to metadata
-
-        if chunk_method == 'json':
-            chunk_text_content = json.dumps(chunk['json'], ensure_ascii=False)
-        else:
-            chunk_text_content = chunk
+    try:
+        for i, chunk in enumerate(chunks):
+            metadata = {
+                'chunk_index': i + 1,
+                'total_chunks': total_chunks,
+                'chunk_method': options['method'],
+                'max_size': options['max_size'],
+                'overlap': options['overlap'],
+                'language': options['language'],
+                'relative_position': float((i + 1) / total_chunks)
+            }
+            metadata.update(json_content)
+            metadata['header_text'] = header_text
+
+            if options['method'] == 'json':
+                chunk_text_content = json.dumps(chunk['json'], ensure_ascii=False)
+            else:
+                chunk_text_content = chunk
 
-        chunks_with_metadata.append({
-            'text': chunk_text_content,
-            'metadata': metadata
-        })
+            chunks_with_metadata.append({
+                'text': chunk_text_content,
+                'metadata': metadata
+            })
 
-    return chunks_with_metadata
+        logging.debug(f"Successfully created metadata for all chunks")
+        return chunks_with_metadata
+    except Exception as e:
+        logging.error(f"Error creating chunk metadata: {e}")
+        raise
 
 
 def multi_level_chunking(text: str, method: str, max_size: int, overlap: int, language: str) -> List[str]:
@@ -220,24 +242,35 @@ def determine_chunk_position(relative_position: float) -> str:
 
 def chunk_text_by_words(text: str, max_words: int = 300, overlap: int = 0, language: str = None) -> List[str]:
     logging.debug("chunk_text_by_words...")
-    if language is None:
-        language = detect_language(text)
-
-    if language.startswith('zh'):  # Chinese
-        import jieba
-        words = list(jieba.cut(text))
-    elif language == 'ja':  # Japanese
-        import fugashi
-        tagger = fugashi.Tagger()
-        words = [word.surface for word in tagger(text)]
-    else:  # Default to simple splitting for other languages
-        words = text.split()
+    logging.debug(f"Parameters: max_words={max_words}, overlap={overlap}, language={language}")
 
-    chunks = []
-    for i in range(0, len(words), max_words - overlap):
-        chunk = ' '.join(words[i:i + max_words])
-        chunks.append(chunk)
-    return post_process_chunks(chunks)
+    try:
+        if language is None:
+            language = detect_language(text)
+            logging.debug(f"Detected language: {language}")
+
+        if language.startswith('zh'):  # Chinese
+            import jieba
+            words = list(jieba.cut(text))
+        elif language == 'ja':  # Japanese
+            import fugashi
+            tagger = fugashi.Tagger()
+            words = [word.surface for word in tagger(text)]
+        else:  # Default to simple splitting for other languages
+            words = text.split()
+
+        logging.debug(f"Total words: {len(words)}")
+
+        chunks = []
+        for i in range(0, len(words), max_words - overlap):
+            chunk = ' '.join(words[i:i + max_words])
+            chunks.append(chunk)
+            logging.debug(f"Created chunk {len(chunks)} with {len(chunk.split())} words")
+
+        return post_process_chunks(chunks)
+    except Exception as e:
+        logging.error(f"Error in chunk_text_by_words: {e}")
+        raise
 
 
 def chunk_text_by_sentences(text: str, max_sentences: int = 10, overlap: int = 0, language: str = None) -> List[str]:
@@ -338,24 +371,24 @@ def get_chunk_metadata(chunk: str, full_text: str, chunk_type: str = "generic",
     """
     chunk_length = len(chunk)
     start_index = full_text.find(chunk)
-    end_index = start_index + chunk_length if start_index != -1 else None
+    end_index = start_index + chunk_length if start_index != -1 else -1
 
     # Calculate a hash for the chunk
     chunk_hash = hashlib.md5(chunk.encode()).hexdigest()
 
     metadata = {
-        'start_index': start_index,
-        'end_index': end_index,
-        'word_count': len(chunk.split()),
-        'char_count': chunk_length,
+        'start_index': int(start_index),
+        'end_index': int(end_index),
+        'word_count': int(len(chunk.split())),
+        'char_count': int(chunk_length),
         'chunk_type': chunk_type,
         'language': language,
         'chunk_hash': chunk_hash,
-        'relative_position': start_index / len(full_text) if len(full_text) > 0 and start_index != -1 else 0
+        'relative_position': float(start_index / len(full_text) if len(full_text) > 0 and start_index != -1 else 0)
     }
 
     if chunk_type == "chapter":
-        metadata['chapter_number'] = chapter_number
+        metadata['chapter_number'] = int(chapter_number) if chapter_number is not None else None
         metadata['chapter_pattern'] = chapter_pattern
 
     return metadata
@@ -943,6 +976,151 @@ def chunk_ebook_by_chapters(text: str, chunk_options: Dict[str, Any]) -> List[Di
 #
 # End of ebook chapter chunking
 #######################################################################################################################
+#
+# XML Chunking
+
+def extract_xml_structure(element, path=""):
+    """
+    Recursively extract XML structure and content.
+    Returns a list of (path, text) tuples.
+    """
+    results = []
+    current_path = f"{path}/{element.tag}" if path else element.tag
+
+    # Get direct text content
+    if element.text and element.text.strip():
+        results.append((current_path, element.text.strip()))
+
+    # Process attributes if any
+    if element.attrib:
+        for key, value in element.attrib.items():
+            results.append((f"{current_path}/@{key}", value))
+
+    # Process child elements
+    for child in element:
+        results.extend(extract_xml_structure(child, current_path))
+
+    return results
+
+
+def chunk_xml(xml_text: str, chunk_options: Dict[str, Any]) -> List[Dict[str, Any]]:
+    """
+    Enhanced XML chunking that preserves structure and hierarchy.
+    Processes XML content into chunks while maintaining structural context.
+
+    Args:
+        xml_text (str): The XML content as a string
+        chunk_options (Dict[str, Any]): Configuration options including:
+            - max_size (int): Maximum chunk size (default: 1000)
+            - overlap (int): Number of overlapping elements (default: 0)
+            - method (str): Chunking method (default: 'xml')
+            - language (str): Content language (default: 'english')
+
+    Returns:
+        List[Dict[str, Any]]: List of chunks, each containing:
+            - text: The chunk content
+            - metadata: Chunk metadata including XML paths and chunking info
+    """
+    logging.debug("Starting XML chunking process...")
+
+    try:
+        # Parse XML content
+        root = ET.fromstring(xml_text)
+        chunks = []
+
+        # Get chunking parameters with defaults
+        max_size = chunk_options.get('max_size', 1000)
+        overlap = chunk_options.get('overlap', 0)
+        language = chunk_options.get('language', 'english')
+
+        logging.debug(f"Chunking parameters - max_size: {max_size}, overlap: {overlap}, language: {language}")
+
+        # Extract full structure with hierarchy
+        xml_content = extract_xml_structure(root)
+        logging.debug(f"Extracted {len(xml_content)} XML elements")
+
+        # Initialize chunking variables
+        current_chunk = []
+        current_size = 0
+        chunk_count = 0
+
+        # Process XML content into chunks
+        for path, content in xml_content:
+            # Calculate content size (by words)
+            content_size = len(content.split())
+
+            # Check if adding this content would exceed max_size
+            if current_size + content_size > max_size and current_chunk:
+                # Create chunk from current content
+                chunk_text = '\n'.join(f"{p}: {c}" for p, c in current_chunk)
+                chunk_count += 1
+
+                # Create chunk with metadata
+                chunks.append({
+                    'text': chunk_text,
+                    'metadata': {
+                        'paths': [p for p, _ in current_chunk],
+                        'chunk_method': 'xml',
+                        'chunk_index': chunk_count,
+                        'max_size': max_size,
+                        'overlap': overlap,
+                        'language': language,
+                        'root_tag': root.tag,
+                        'xml_attributes': dict(root.attrib)
+                    }
+                })
+
+                # Handle overlap if specified
+                if overlap > 0:
+                    # Keep last few items for overlap
+                    overlap_items = current_chunk[-overlap:]
+                    current_chunk = overlap_items
+                    current_size = sum(len(c.split()) for _, c in overlap_items)
+                    logging.debug(f"Created overlap chunk with {len(overlap_items)} items")
+                else:
+                    current_chunk = []
+                    current_size = 0
+
+            # Add current content to chunk
+            current_chunk.append((path, content))
+            current_size += content_size
+
+        # Process final chunk if content remains
+        if current_chunk:
+            chunk_text = '\n'.join(f"{p}: {c}" for p, c in current_chunk)
+            chunk_count += 1
+
+            chunks.append({
+                'text': chunk_text,
+                'metadata': {
+                    'paths': [p for p, _ in current_chunk],
+                    'chunk_method': 'xml',
+                    'chunk_index': chunk_count,
+                    'max_size': max_size,
+                    'overlap': overlap,
+                    'language': language,
+                    'root_tag': root.tag,
+                    'xml_attributes': dict(root.attrib)
+                }
+            })
+
+        # Update total chunks count in metadata
+        for chunk in chunks:
+            chunk['metadata']['total_chunks'] = chunk_count
+
+        logging.debug(f"XML chunking complete. Created {len(chunks)} chunks")
+        return chunks
+
+    except ET.ParseError as e:
+        logging.error(f"XML parsing error: {str(e)}")
+        raise
+    except Exception as e:
+        logging.error(f"Unexpected error during XML chunking: {str(e)}")
+        raise
+
+#
+# End of XML Chunking
+#######################################################################################################################
 
 #######################################################################################################################
 #

App_Function_Libraries/DB/Character_Chat_DB.py

@@ -1,701 +1,1059 @@
-# character_chat_db.py
-# Database functions for managing character cards and chat histories.
-# #
-# Imports
-import configparser
-import sqlite3
-import json
-import os
-import sys
-from typing import List, Dict, Optional, Tuple, Any, Union
-
-from App_Function_Libraries.Utils.Utils import get_database_dir, get_project_relative_path, get_database_path
-import logging
-
-#
-#######################################################################################################################
-#
-#
-
-def ensure_database_directory():
-    os.makedirs(get_database_dir(), exist_ok=True)
-
-ensure_database_directory()
-
-
-# Construct the path to the config file
-config_path = get_project_relative_path('Config_Files/config.txt')
-
-# Read the config file
-config = configparser.ConfigParser()
-config.read(config_path)
-
-# Get the chat db path from the config, or use the default if not specified
-chat_DB_PATH = config.get('Database', 'chatDB_path', fallback=get_database_path('chatDB.db'))
-print(f"Chat Database path: {chat_DB_PATH}")
-
-########################################################################################################
-#
-# Functions
-
-# FIXME - Setup properly and test/add documentation for its existence...
-def initialize_database():
-    """Initialize the SQLite database with required tables and FTS5 virtual tables."""
-    conn = None
-    try:
-        conn = sqlite3.connect(chat_DB_PATH)
-        cursor = conn.cursor()
-
-        # Enable foreign key constraints
-        cursor.execute("PRAGMA foreign_keys = ON;")
-
-        # Create CharacterCards table with V2 fields
-        cursor.execute("""
-        CREATE TABLE IF NOT EXISTS CharacterCards (
-            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            name TEXT UNIQUE NOT NULL,
-            description TEXT,
-            personality TEXT,
-            scenario TEXT,
-            image BLOB,
-            post_history_instructions TEXT,
-            first_mes TEXT,
-            mes_example TEXT,
-            creator_notes TEXT,
-            system_prompt TEXT,
-            alternate_greetings TEXT,
-            tags TEXT,
-            creator TEXT,
-            character_version TEXT,
-            extensions TEXT,
-            created_at DATETIME DEFAULT CURRENT_TIMESTAMP
-        );
-        """)
-
-        # Create CharacterChats table
-        cursor.execute("""
-        CREATE TABLE IF NOT EXISTS CharacterChats (
-            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            character_id INTEGER NOT NULL,
-            conversation_name TEXT,
-            chat_history TEXT,
-            is_snapshot BOOLEAN DEFAULT FALSE,
-            created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
-            FOREIGN KEY (character_id) REFERENCES CharacterCards(id) ON DELETE CASCADE
-        );
-        """)
-
-        # Create FTS5 virtual table for CharacterChats
-        cursor.execute("""
-        CREATE VIRTUAL TABLE IF NOT EXISTS CharacterChats_fts USING fts5(
-            conversation_name,
-            chat_history,
-            content='CharacterChats',
-            content_rowid='id'
-        );
-        """)
-
-        # Create triggers to keep FTS5 table in sync with CharacterChats
-        cursor.executescript("""
-        CREATE TRIGGER IF NOT EXISTS CharacterChats_ai AFTER INSERT ON CharacterChats BEGIN
-            INSERT INTO CharacterChats_fts(rowid, conversation_name, chat_history)
-            VALUES (new.id, new.conversation_name, new.chat_history);
-        END;
-
-        CREATE TRIGGER IF NOT EXISTS CharacterChats_ad AFTER DELETE ON CharacterChats BEGIN
-            DELETE FROM CharacterChats_fts WHERE rowid = old.id;
-        END;
-
-        CREATE TRIGGER IF NOT EXISTS CharacterChats_au AFTER UPDATE ON CharacterChats BEGIN
-            UPDATE CharacterChats_fts SET conversation_name = new.conversation_name, chat_history = new.chat_history
-            WHERE rowid = new.id;
-        END;
-        """)
-
-        # Create ChatKeywords table
-        cursor.execute("""
-        CREATE TABLE IF NOT EXISTS ChatKeywords (
-            chat_id INTEGER NOT NULL,
-            keyword TEXT NOT NULL,
-            FOREIGN KEY (chat_id) REFERENCES CharacterChats(id) ON DELETE CASCADE
-        );
-        """)
-
-        # Create indexes for faster searches
-        cursor.execute("""
-        CREATE INDEX IF NOT EXISTS idx_chatkeywords_keyword ON ChatKeywords(keyword);
-        """)
-        cursor.execute("""
-        CREATE INDEX IF NOT EXISTS idx_chatkeywords_chat_id ON ChatKeywords(chat_id);
-        """)
-
-        conn.commit()
-        logging.info("Database initialized successfully.")
-    except sqlite3.Error as e:
-        logging.error(f"SQLite error occurred during database initialization: {e}")
-        if conn:
-            conn.rollback()
-        raise
-    except Exception as e:
-        logging.error(f"Unexpected error occurred during database initialization: {e}")
-        if conn:
-            conn.rollback()
-        raise
-    finally:
-        if conn:
-            conn.close()
-
-# Call initialize_database() at the start of your application
-def setup_chat_database():
-    try:
-        initialize_database()
-    except Exception as e:
-        logging.critical(f"Failed to initialize database: {e}")
-        sys.exit(1)
-
-setup_chat_database()
-
-########################################################################################################
-#
-# Character Card handling
-
-def parse_character_card(card_data: Dict[str, Any]) -> Dict[str, Any]:
-    """Parse and validate a character card according to V2 specification."""
-    v2_data = {
-        'name': card_data.get('name', ''),
-        'description': card_data.get('description', ''),
-        'personality': card_data.get('personality', ''),
-        'scenario': card_data.get('scenario', ''),
-        'first_mes': card_data.get('first_mes', ''),
-        'mes_example': card_data.get('mes_example', ''),
-        'creator_notes': card_data.get('creator_notes', ''),
-        'system_prompt': card_data.get('system_prompt', ''),
-        'post_history_instructions': card_data.get('post_history_instructions', ''),
-        'alternate_greetings': json.dumps(card_data.get('alternate_greetings', [])),
-        'tags': json.dumps(card_data.get('tags', [])),
-        'creator': card_data.get('creator', ''),
-        'character_version': card_data.get('character_version', ''),
-        'extensions': json.dumps(card_data.get('extensions', {}))
-    }
-
-    # Handle 'image' separately as it might be binary data
-    if 'image' in card_data:
-        v2_data['image'] = card_data['image']
-
-    return v2_data
-
-
-def add_character_card(card_data: Dict[str, Any]) -> Optional[int]:
-    """Add or update a character card in the database."""
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    try:
-        parsed_card = parse_character_card(card_data)
-
-        # Check if character already exists
-        cursor.execute("SELECT id FROM CharacterCards WHERE name = ?", (parsed_card['name'],))
-        row = cursor.fetchone()
-
-        if row:
-            # Update existing character
-            character_id = row[0]
-            update_query = """
-                UPDATE CharacterCards
-                SET description = ?, personality = ?, scenario = ?, image = ?, 
-                    post_history_instructions = ?, first_mes = ?, mes_example = ?,
-                    creator_notes = ?, system_prompt = ?, alternate_greetings = ?,
-                    tags = ?, creator = ?, character_version = ?, extensions = ?
-                WHERE id = ?
-            """
-            cursor.execute(update_query, (
-                parsed_card['description'], parsed_card['personality'], parsed_card['scenario'],
-                parsed_card['image'], parsed_card['post_history_instructions'], parsed_card['first_mes'],
-                parsed_card['mes_example'], parsed_card['creator_notes'], parsed_card['system_prompt'],
-                parsed_card['alternate_greetings'], parsed_card['tags'], parsed_card['creator'],
-                parsed_card['character_version'], parsed_card['extensions'], character_id
-            ))
-        else:
-            # Insert new character
-            insert_query = """
-                INSERT INTO CharacterCards (name, description, personality, scenario, image, 
-                post_history_instructions, first_mes, mes_example, creator_notes, system_prompt, 
-                alternate_greetings, tags, creator, character_version, extensions)
-                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-            """
-            cursor.execute(insert_query, (
-                parsed_card['name'], parsed_card['description'], parsed_card['personality'],
-                parsed_card['scenario'], parsed_card['image'], parsed_card['post_history_instructions'],
-                parsed_card['first_mes'], parsed_card['mes_example'], parsed_card['creator_notes'],
-                parsed_card['system_prompt'], parsed_card['alternate_greetings'], parsed_card['tags'],
-                parsed_card['creator'], parsed_card['character_version'], parsed_card['extensions']
-            ))
-            character_id = cursor.lastrowid
-
-        conn.commit()
-        return character_id
-    except sqlite3.IntegrityError as e:
-        logging.error(f"Error adding character card: {e}")
-        return None
-    except Exception as e:
-        logging.error(f"Unexpected error adding character card: {e}")
-        return None
-    finally:
-        conn.close()
-
-# def add_character_card(card_data: Dict) -> Optional[int]:
-#     """Add or update a character card in the database.
-#
-#     Returns the ID of the inserted character or None if failed.
-#     """
-#     conn = sqlite3.connect(chat_DB_PATH)
-#     cursor = conn.cursor()
-#     try:
-#         # Ensure all required fields are present
-#         required_fields = ['name', 'description', 'personality', 'scenario', 'image', 'post_history_instructions', 'first_message']
-#         for field in required_fields:
-#             if field not in card_data:
-#                 card_data[field] = ''  # Assign empty string if field is missing
-#
-#         # Check if character already exists
-#         cursor.execute("SELECT id FROM CharacterCards WHERE name = ?", (card_data['name'],))
-#         row = cursor.fetchone()
-#
-#         if row:
-#             # Update existing character
-#             character_id = row[0]
-#             cursor.execute("""
-#                 UPDATE CharacterCards
-#                 SET description = ?, personality = ?, scenario = ?, image = ?, post_history_instructions = ?, first_message = ?
-#                 WHERE id = ?
-#             """, (
-#                 card_data['description'],
-#                 card_data['personality'],
-#                 card_data['scenario'],
-#                 card_data['image'],
-#                 card_data['post_history_instructions'],
-#                 card_data['first_message'],
-#                 character_id
-#             ))
-#         else:
-#             # Insert new character
-#             cursor.execute("""
-#                 INSERT INTO CharacterCards (name, description, personality, scenario, image, post_history_instructions, first_message)
-#                 VALUES (?, ?, ?, ?, ?, ?, ?)
-#             """, (
-#                 card_data['name'],
-#                 card_data['description'],
-#                 card_data['personality'],
-#                 card_data['scenario'],
-#                 card_data['image'],
-#                 card_data['post_history_instructions'],
-#                 card_data['first_message']
-#             ))
-#             character_id = cursor.lastrowid
-#
-#         conn.commit()
-#         return cursor.lastrowid
-#     except sqlite3.IntegrityError as e:
-#         logging.error(f"Error adding character card: {e}")
-#         return None
-#     except Exception as e:
-#         logging.error(f"Unexpected error adding character card: {e}")
-#         return None
-#     finally:
-#         conn.close()
-
-
-def get_character_cards() -> List[Dict]:
-    """Retrieve all character cards from the database."""
-    logging.debug(f"Fetching characters from DB: {chat_DB_PATH}")
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    cursor.execute("SELECT * FROM CharacterCards")
-    rows = cursor.fetchall()
-    columns = [description[0] for description in cursor.description]
-    conn.close()
-    characters = [dict(zip(columns, row)) for row in rows]
-    #logging.debug(f"Characters fetched from DB: {characters}")
-    return characters
-
-
-def get_character_card_by_id(character_id: Union[int, Dict[str, Any]]) -> Optional[Dict[str, Any]]:
-    """
-    Retrieve a single character card by its ID.
-
-    Args:
-        character_id: Can be either an integer ID or a dictionary containing character data.
-
-    Returns:
-        A dictionary containing the character card data, or None if not found.
-    """
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    try:
-        if isinstance(character_id, dict):
-            # If a dictionary is passed, assume it's already a character card
-            return character_id
-        elif isinstance(character_id, int):
-            # If an integer is passed, fetch the character from the database
-            cursor.execute("SELECT * FROM CharacterCards WHERE id = ?", (character_id,))
-            row = cursor.fetchone()
-            if row:
-                columns = [description[0] for description in cursor.description]
-                return dict(zip(columns, row))
-        else:
-            logging.warning(f"Invalid type for character_id: {type(character_id)}")
-        return None
-    except Exception as e:
-        logging.error(f"Error in get_character_card_by_id: {e}")
-        return None
-    finally:
-        conn.close()
-
-
-def update_character_card(character_id: int, card_data: Dict) -> bool:
-    """Update an existing character card."""
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    try:
-        cursor.execute("""
-            UPDATE CharacterCards
-            SET name = ?, description = ?, personality = ?, scenario = ?, image = ?, post_history_instructions = ?, first_message = ?
-            WHERE id = ?
-        """, (
-            card_data.get('name'),
-            card_data.get('description'),
-            card_data.get('personality'),
-            card_data.get('scenario'),
-            card_data.get('image'),
-            card_data.get('post_history_instructions', ''),
-            card_data.get('first_message', "Hello! I'm ready to chat."),
-            character_id
-        ))
-        conn.commit()
-        return cursor.rowcount > 0
-    except sqlite3.IntegrityError as e:
-        logging.error(f"Error updating character card: {e}")
-        return False
-    finally:
-        conn.close()
-
-
-def delete_character_card(character_id: int) -> bool:
-    """Delete a character card and its associated chats."""
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    try:
-        # Delete associated chats first due to foreign key constraint
-        cursor.execute("DELETE FROM CharacterChats WHERE character_id = ?", (character_id,))
-        cursor.execute("DELETE FROM CharacterCards WHERE id = ?", (character_id,))
-        conn.commit()
-        return cursor.rowcount > 0
-    except sqlite3.Error as e:
-        logging.error(f"Error deleting character card: {e}")
-        return False
-    finally:
-        conn.close()
-
-
-def add_character_chat(character_id: int, conversation_name: str, chat_history: List[Tuple[str, str]], keywords: Optional[List[str]] = None, is_snapshot: bool = False) -> Optional[int]:
-    """
-    Add a new chat history for a character, optionally associating keywords.
-
-    Args:
-        character_id (int): The ID of the character.
-        conversation_name (str): Name of the conversation.
-        chat_history (List[Tuple[str, str]]): List of (user, bot) message tuples.
-        keywords (Optional[List[str]]): List of keywords to associate with this chat.
-        is_snapshot (bool, optional): Whether this chat is a snapshot.
-
-    Returns:
-        Optional[int]: The ID of the inserted chat or None if failed.
-    """
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    try:
-        chat_history_json = json.dumps(chat_history)
-        cursor.execute("""
-            INSERT INTO CharacterChats (character_id, conversation_name, chat_history, is_snapshot)
-            VALUES (?, ?, ?, ?)
-        """, (
-            character_id,
-            conversation_name,
-            chat_history_json,
-            is_snapshot
-        ))
-        chat_id = cursor.lastrowid
-
-        if keywords:
-            # Insert keywords into ChatKeywords table
-            keyword_records = [(chat_id, keyword.strip().lower()) for keyword in keywords]
-            cursor.executemany("""
-                INSERT INTO ChatKeywords (chat_id, keyword)
-                VALUES (?, ?)
-            """, keyword_records)
-
-        conn.commit()
-        return chat_id
-    except sqlite3.Error as e:
-        logging.error(f"Error adding character chat: {e}")
-        return None
-    finally:
-        conn.close()
-
-
-def get_character_chats(character_id: Optional[int] = None) -> List[Dict]:
-    """Retrieve all chats, or chats for a specific character if character_id is provided."""
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    if character_id is not None:
-        cursor.execute("SELECT * FROM CharacterChats WHERE character_id = ?", (character_id,))
-    else:
-        cursor.execute("SELECT * FROM CharacterChats")
-    rows = cursor.fetchall()
-    columns = [description[0] for description in cursor.description]
-    conn.close()
-    return [dict(zip(columns, row)) for row in rows]
-
-
-def get_character_chat_by_id(chat_id: int) -> Optional[Dict]:
-    """Retrieve a single chat by its ID."""
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    cursor.execute("SELECT * FROM CharacterChats WHERE id = ?", (chat_id,))
-    row = cursor.fetchone()
-    conn.close()
-    if row:
-        columns = [description[0] for description in cursor.description]
-        chat = dict(zip(columns, row))
-        chat['chat_history'] = json.loads(chat['chat_history'])
-        return chat
-    return None
-
-
-def search_character_chats(query: str, character_id: Optional[int] = None) -> Tuple[List[Dict], str]:
-    """
-    Search for character chats using FTS5, optionally filtered by character_id.
-
-    Args:
-        query (str): The search query.
-        character_id (Optional[int]): The ID of the character to filter chats by.
-
-    Returns:
-        Tuple[List[Dict], str]: A list of matching chats and a status message.
-    """
-    if not query.strip():
-        return [], "Please enter a search query."
-
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    try:
-        if character_id is not None:
-            # Search with character_id filter
-            cursor.execute("""
-                SELECT CharacterChats.id, CharacterChats.conversation_name, CharacterChats.chat_history
-                FROM CharacterChats_fts
-                JOIN CharacterChats ON CharacterChats_fts.rowid = CharacterChats.id
-                WHERE CharacterChats_fts MATCH ? AND CharacterChats.character_id = ?
-                ORDER BY rank
-            """, (query, character_id))
-        else:
-            # Search without character_id filter
-            cursor.execute("""
-                SELECT CharacterChats.id, CharacterChats.conversation_name, CharacterChats.chat_history
-                FROM CharacterChats_fts
-                JOIN CharacterChats ON CharacterChats_fts.rowid = CharacterChats.id
-                WHERE CharacterChats_fts MATCH ?
-                ORDER BY rank
-            """, (query,))
-
-        rows = cursor.fetchall()
-        columns = [description[0] for description in cursor.description]
-        results = [dict(zip(columns, row)) for row in rows]
-
-        if character_id is not None:
-            status_message = f"Found {len(results)} chat(s) matching '{query}' for the selected character."
-        else:
-            status_message = f"Found {len(results)} chat(s) matching '{query}' across all characters."
-
-        return results, status_message
-    except Exception as e:
-        logging.error(f"Error searching chats with FTS5: {e}")
-        return [], f"Error occurred during search: {e}"
-    finally:
-        conn.close()
-
-def update_character_chat(chat_id: int, chat_history: List[Tuple[str, str]]) -> bool:
-    """Update an existing chat history."""
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    try:
-        chat_history_json = json.dumps(chat_history)
-        cursor.execute("""
-            UPDATE CharacterChats
-            SET chat_history = ?
-            WHERE id = ?
-        """, (
-            chat_history_json,
-            chat_id
-        ))
-        conn.commit()
-        return cursor.rowcount > 0
-    except sqlite3.Error as e:
-        logging.error(f"Error updating character chat: {e}")
-        return False
-    finally:
-        conn.close()
-
-
-def delete_character_chat(chat_id: int) -> bool:
-    """Delete a specific chat."""
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    try:
-        cursor.execute("DELETE FROM CharacterChats WHERE id = ?", (chat_id,))
-        conn.commit()
-        return cursor.rowcount > 0
-    except sqlite3.Error as e:
-        logging.error(f"Error deleting character chat: {e}")
-        return False
-    finally:
-        conn.close()
-
-def fetch_keywords_for_chats(keywords: List[str]) -> List[int]:
-    """
-    Fetch chat IDs associated with any of the specified keywords.
-
-    Args:
-        keywords (List[str]): List of keywords to search for.
-
-    Returns:
-        List[int]: List of chat IDs associated with the keywords.
-    """
-    if not keywords:
-        return []
-
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    try:
-        # Construct the WHERE clause to search for each keyword
-        keyword_clauses = " OR ".join(["keyword = ?"] * len(keywords))
-        sql_query = f"SELECT DISTINCT chat_id FROM ChatKeywords WHERE {keyword_clauses}"
-        cursor.execute(sql_query, keywords)
-        rows = cursor.fetchall()
-        chat_ids = [row[0] for row in rows]
-        return chat_ids
-    except Exception as e:
-        logging.error(f"Error in fetch_keywords_for_chats: {e}")
-        return []
-    finally:
-        conn.close()
-
-def save_chat_history_to_character_db(character_id: int, conversation_name: str, chat_history: List[Tuple[str, str]]) -> Optional[int]:
-    """Save chat history to the CharacterChats table.
-
-    Returns the ID of the inserted chat or None if failed.
-    """
-    return add_character_chat(character_id, conversation_name, chat_history)
-
-def migrate_chat_to_media_db():
-    pass
-
-
-def search_db(query: str, fields: List[str], where_clause: str = "", page: int = 1, results_per_page: int = 5) -> List[Dict[str, Any]]:
-    """
-    Perform a full-text search on specified fields with optional filtering and pagination.
-
-    Args:
-        query (str): The search query.
-        fields (List[str]): List of fields to search in.
-        where_clause (str, optional): Additional SQL WHERE clause to filter results.
-        page (int, optional): Page number for pagination.
-        results_per_page (int, optional): Number of results per page.
-
-    Returns:
-        List[Dict[str, Any]]: List of matching chat records with content and metadata.
-    """
-    if not query.strip():
-        return []
-
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    try:
-        # Construct the MATCH query for FTS5
-        match_query = " AND ".join(fields) + f" MATCH ?"
-        # Adjust the query with the fields
-        fts_query = f"""
-            SELECT CharacterChats.id, CharacterChats.conversation_name, CharacterChats.chat_history
-            FROM CharacterChats_fts
-            JOIN CharacterChats ON CharacterChats_fts.rowid = CharacterChats.id
-            WHERE {match_query}
-        """
-        if where_clause:
-            fts_query += f" AND ({where_clause})"
-        fts_query += " ORDER BY rank LIMIT ? OFFSET ?"
-        offset = (page - 1) * results_per_page
-        cursor.execute(fts_query, (query, results_per_page, offset))
-        rows = cursor.fetchall()
-        columns = [description[0] for description in cursor.description]
-        results = [dict(zip(columns, row)) for row in rows]
-        return results
-    except Exception as e:
-        logging.error(f"Error in search_db: {e}")
-        return []
-    finally:
-        conn.close()
-
-
-def perform_full_text_search_chat(query: str, relevant_chat_ids: List[int], page: int = 1, results_per_page: int = 5) -> \
-List[Dict[str, Any]]:
-    """
-    Perform a full-text search within the specified chat IDs using FTS5.
-
-    Args:
-        query (str): The user's query.
-        relevant_chat_ids (List[int]): List of chat IDs to search within.
-        page (int): Pagination page number.
-        results_per_page (int): Number of results per page.
-
-    Returns:
-        List[Dict[str, Any]]: List of search results with content and metadata.
-    """
-    try:
-        # Construct a WHERE clause to limit the search to relevant chat IDs
-        where_clause = " OR ".join([f"media_id = {chat_id}" for chat_id in relevant_chat_ids])
-        if not where_clause:
-            where_clause = "1"  # No restriction if no chat IDs
-
-        # Perform full-text search using FTS5
-        fts_results = search_db(query, ["content"], where_clause, page=page, results_per_page=results_per_page)
-
-        filtered_fts_results = [
-            {
-                "content": result['content'],
-                "metadata": {"media_id": result['id']}
-            }
-            for result in fts_results
-            if result['id'] in relevant_chat_ids
-        ]
-        return filtered_fts_results
-    except Exception as e:
-        logging.error(f"Error in perform_full_text_search_chat: {str(e)}")
-        return []
-
-
-def fetch_all_chats() -> List[Dict[str, Any]]:
-    """
-    Fetch all chat messages from the database.
-
-    Returns:
-        List[Dict[str, Any]]: List of chat messages with relevant metadata.
-    """
-    try:
-        chats = get_character_chats()  # Modify this function to retrieve all chats
-        return chats
-    except Exception as e:
-        logging.error(f"Error fetching all chats: {str(e)}")
-        return []
-
-#
-# End of Character_Chat_DB.py
-#######################################################################################################################
+# character_chat_db.py
+# Database functions for managing character cards and chat histories.
+# #
+# Imports
+import configparser
+import sqlite3
+import json
+import os
+import sys
+from typing import List, Dict, Optional, Tuple, Any, Union
+
+from App_Function_Libraries.Utils.Utils import get_database_dir, get_project_relative_path, get_database_path
+from Tests.Chat_APIs.Chat_APIs_Integration_test import logging
+
+#
+#######################################################################################################################
+#
+#
+
+def ensure_database_directory():
+    os.makedirs(get_database_dir(), exist_ok=True)
+
+ensure_database_directory()
+
+
+# Construct the path to the config file
+config_path = get_project_relative_path('Config_Files/config.txt')
+
+# Read the config file
+config = configparser.ConfigParser()
+config.read(config_path)
+
+# Get the chat db path from the config, or use the default if not specified
+chat_DB_PATH = config.get('Database', 'chatDB_path', fallback=get_database_path('chatDB.db'))
+print(f"Chat Database path: {chat_DB_PATH}")
+
+########################################################################################################
+#
+# Functions
+
+# FIXME - Setup properly and test/add documentation for its existence...
+def initialize_database():
+    """Initialize the SQLite database with required tables and FTS5 virtual tables."""
+    conn = None
+    try:
+        conn = sqlite3.connect(chat_DB_PATH)
+        cursor = conn.cursor()
+
+        # Enable foreign key constraints
+        cursor.execute("PRAGMA foreign_keys = ON;")
+
+        # Create CharacterCards table with V2 fields
+        cursor.execute("""
+        CREATE TABLE IF NOT EXISTS CharacterCards (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            name TEXT UNIQUE NOT NULL,
+            description TEXT,
+            personality TEXT,
+            scenario TEXT,
+            image BLOB,
+            post_history_instructions TEXT,
+            first_mes TEXT,
+            mes_example TEXT,
+            creator_notes TEXT,
+            system_prompt TEXT,
+            alternate_greetings TEXT,
+            tags TEXT,
+            creator TEXT,
+            character_version TEXT,
+            extensions TEXT,
+            created_at DATETIME DEFAULT CURRENT_TIMESTAMP
+        );
+        """)
+
+        # Create FTS5 virtual table for CharacterCards
+        cursor.execute("""
+        CREATE VIRTUAL TABLE IF NOT EXISTS CharacterCards_fts USING fts5(
+            name,
+            description,
+            personality,
+            scenario,
+            system_prompt,
+            content='CharacterCards',
+            content_rowid='id'
+        );
+        """)
+
+        # Create triggers to keep FTS5 table in sync with CharacterCards
+        cursor.executescript("""
+        CREATE TRIGGER IF NOT EXISTS CharacterCards_ai AFTER INSERT ON CharacterCards BEGIN
+            INSERT INTO CharacterCards_fts(
+                rowid,
+                name,
+                description,
+                personality,
+                scenario,
+                system_prompt
+            ) VALUES (
+                new.id,
+                new.name,
+                new.description,
+                new.personality,
+                new.scenario,
+                new.system_prompt
+            );
+        END;
+
+        CREATE TRIGGER IF NOT EXISTS CharacterCards_ad AFTER DELETE ON CharacterCards BEGIN
+            DELETE FROM CharacterCards_fts WHERE rowid = old.id;
+        END;
+
+        CREATE TRIGGER IF NOT EXISTS CharacterCards_au AFTER UPDATE ON CharacterCards BEGIN
+            UPDATE CharacterCards_fts SET
+                name = new.name,
+                description = new.description,
+                personality = new.personality,
+                scenario = new.scenario,
+                system_prompt = new.system_prompt
+            WHERE rowid = new.id;
+        END;
+        """)
+
+        # Create CharacterChats table
+        cursor.execute("""
+        CREATE TABLE IF NOT EXISTS CharacterChats (
+            id INTEGER PRIMARY KEY AUTOINCREMENT,
+            character_id INTEGER NOT NULL,
+            conversation_name TEXT,
+            chat_history TEXT,
+            is_snapshot BOOLEAN DEFAULT FALSE,
+            created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
+            FOREIGN KEY (character_id) REFERENCES CharacterCards(id) ON DELETE CASCADE
+        );
+        """)
+
+        # Create FTS5 virtual table for CharacterChats
+        cursor.execute("""
+        CREATE VIRTUAL TABLE IF NOT EXISTS CharacterChats_fts USING fts5(
+            conversation_name,
+            chat_history,
+            content='CharacterChats',
+            content_rowid='id'
+        );
+        """)
+
+        # Create triggers to keep FTS5 table in sync with CharacterChats
+        cursor.executescript("""
+        CREATE TRIGGER IF NOT EXISTS CharacterChats_ai AFTER INSERT ON CharacterChats BEGIN
+            INSERT INTO CharacterChats_fts(rowid, conversation_name, chat_history)
+            VALUES (new.id, new.conversation_name, new.chat_history);
+        END;
+
+        CREATE TRIGGER IF NOT EXISTS CharacterChats_ad AFTER DELETE ON CharacterChats BEGIN
+            DELETE FROM CharacterChats_fts WHERE rowid = old.id;
+        END;
+
+        CREATE TRIGGER IF NOT EXISTS CharacterChats_au AFTER UPDATE ON CharacterChats BEGIN
+            UPDATE CharacterChats_fts SET conversation_name = new.conversation_name, chat_history = new.chat_history
+            WHERE rowid = new.id;
+        END;
+        """)
+
+        # Create ChatKeywords table
+        cursor.execute("""
+        CREATE TABLE IF NOT EXISTS ChatKeywords (
+            chat_id INTEGER NOT NULL,
+            keyword TEXT NOT NULL,
+            FOREIGN KEY (chat_id) REFERENCES CharacterChats(id) ON DELETE CASCADE
+        );
+        """)
+
+        # Create indexes for faster searches
+        cursor.execute("""
+        CREATE INDEX IF NOT EXISTS idx_chatkeywords_keyword ON ChatKeywords(keyword);
+        """)
+        cursor.execute("""
+        CREATE INDEX IF NOT EXISTS idx_chatkeywords_chat_id ON ChatKeywords(chat_id);
+        """)
+
+        conn.commit()
+        logging.info("Database initialized successfully.")
+    except sqlite3.Error as e:
+        logging.error(f"SQLite error occurred during database initialization: {e}")
+        if conn:
+            conn.rollback()
+        raise
+    except Exception as e:
+        logging.error(f"Unexpected error occurred during database initialization: {e}")
+        if conn:
+            conn.rollback()
+        raise
+    finally:
+        if conn:
+            conn.close()
+
+# Call initialize_database() at the start of your application
+def setup_chat_database():
+    try:
+        initialize_database()
+    except Exception as e:
+        logging.critical(f"Failed to initialize database: {e}")
+        sys.exit(1)
+
+setup_chat_database()
+
+
+########################################################################################################
+#
+# Character Card handling
+
+def parse_character_card(card_data: Dict[str, Any]) -> Dict[str, Any]:
+    """Parse and validate a character card according to V2 specification."""
+    v2_data = {
+        'name': card_data.get('name', ''),
+        'description': card_data.get('description', ''),
+        'personality': card_data.get('personality', ''),
+        'scenario': card_data.get('scenario', ''),
+        'first_mes': card_data.get('first_mes', ''),
+        'mes_example': card_data.get('mes_example', ''),
+        'creator_notes': card_data.get('creator_notes', ''),
+        'system_prompt': card_data.get('system_prompt', ''),
+        'post_history_instructions': card_data.get('post_history_instructions', ''),
+        'alternate_greetings': json.dumps(card_data.get('alternate_greetings', [])),
+        'tags': json.dumps(card_data.get('tags', [])),
+        'creator': card_data.get('creator', ''),
+        'character_version': card_data.get('character_version', ''),
+        'extensions': json.dumps(card_data.get('extensions', {}))
+    }
+
+    # Handle 'image' separately as it might be binary data
+    if 'image' in card_data:
+        v2_data['image'] = card_data['image']
+
+    return v2_data
+
+
+def add_character_card(card_data: Dict[str, Any]) -> Optional[int]:
+    """Add or update a character card in the database."""
+    conn = sqlite3.connect(chat_DB_PATH)
+    cursor = conn.cursor()
+    try:
+        parsed_card = parse_character_card(card_data)
+
+        # Check if character already exists
+        cursor.execute("SELECT id FROM CharacterCards WHERE name = ?", (parsed_card['name'],))
+        row = cursor.fetchone()
+
+        if row:
+            # Update existing character
+            character_id = row[0]
+            update_query = """
+                UPDATE CharacterCards
+                SET description = ?, personality = ?, scenario = ?, image = ?, 
+                    post_history_instructions = ?, first_mes = ?, mes_example = ?,
+                    creator_notes = ?, system_prompt = ?, alternate_greetings = ?,
+                    tags = ?, creator = ?, character_version = ?, extensions = ?
+                WHERE id = ?
+            """
+            cursor.execute(update_query, (
+                parsed_card['description'], parsed_card['personality'], parsed_card['scenario'],
+                parsed_card['image'], parsed_card['post_history_instructions'], parsed_card['first_mes'],
+                parsed_card['mes_example'], parsed_card['creator_notes'], parsed_card['system_prompt'],
+                parsed_card['alternate_greetings'], parsed_card['tags'], parsed_card['creator'],
+                parsed_card['character_version'], parsed_card['extensions'], character_id
+            ))
+        else:
+            # Insert new character
+            insert_query = """
+                INSERT INTO CharacterCards (name, description, personality, scenario, image, 
+                post_history_instructions, first_mes, mes_example, creator_notes, system_prompt, 
+                alternate_greetings, tags, creator, character_version, extensions)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+            """
+            cursor.execute(insert_query, (
+                parsed_card['name'], parsed_card['description'], parsed_card['personality'],
+                parsed_card['scenario'], parsed_card['image'], parsed_card['post_history_instructions'],
+                parsed_card['first_mes'], parsed_card['mes_example'], parsed_card['creator_notes'],
+                parsed_card['system_prompt'], parsed_card['alternate_greetings'], parsed_card['tags'],
+                parsed_card['creator'], parsed_card['character_version'], parsed_card['extensions']
+            ))
+            character_id = cursor.lastrowid
+
+        conn.commit()
+        return character_id
+    except sqlite3.IntegrityError as e:
+        logging.error(f"Error adding character card: {e}")
+        return None
+    except Exception as e:
+        logging.error(f"Unexpected error adding character card: {e}")
+        return None
+    finally:
+        conn.close()
+
+# def add_character_card(card_data: Dict) -> Optional[int]:
+#     """Add or update a character card in the database.
+#
+#     Returns the ID of the inserted character or None if failed.
+#     """
+#     conn = sqlite3.connect(chat_DB_PATH)
+#     cursor = conn.cursor()
+#     try:
+#         # Ensure all required fields are present
+#         required_fields = ['name', 'description', 'personality', 'scenario', 'image', 'post_history_instructions', 'first_message']
+#         for field in required_fields:
+#             if field not in card_data:
+#                 card_data[field] = ''  # Assign empty string if field is missing
+#
+#         # Check if character already exists
+#         cursor.execute("SELECT id FROM CharacterCards WHERE name = ?", (card_data['name'],))
+#         row = cursor.fetchone()
+#
+#         if row:
+#             # Update existing character
+#             character_id = row[0]
+#             cursor.execute("""
+#                 UPDATE CharacterCards
+#                 SET description = ?, personality = ?, scenario = ?, image = ?, post_history_instructions = ?, first_message = ?
+#                 WHERE id = ?
+#             """, (
+#                 card_data['description'],
+#                 card_data['personality'],
+#                 card_data['scenario'],
+#                 card_data['image'],
+#                 card_data['post_history_instructions'],
+#                 card_data['first_message'],
+#                 character_id
+#             ))
+#         else:
+#             # Insert new character
+#             cursor.execute("""
+#                 INSERT INTO CharacterCards (name, description, personality, scenario, image, post_history_instructions, first_message)
+#                 VALUES (?, ?, ?, ?, ?, ?, ?)
+#             """, (
+#                 card_data['name'],
+#                 card_data['description'],
+#                 card_data['personality'],
+#                 card_data['scenario'],
+#                 card_data['image'],
+#                 card_data['post_history_instructions'],
+#                 card_data['first_message']
+#             ))
+#             character_id = cursor.lastrowid
+#
+#         conn.commit()
+#         return cursor.lastrowid
+#     except sqlite3.IntegrityError as e:
+#         logging.error(f"Error adding character card: {e}")
+#         return None
+#     except Exception as e:
+#         logging.error(f"Unexpected error adding character card: {e}")
+#         return None
+#     finally:
+#         conn.close()
+
+
+def get_character_cards() -> List[Dict]:
+    """Retrieve all character cards from the database."""
+    logging.debug(f"Fetching characters from DB: {chat_DB_PATH}")
+    conn = sqlite3.connect(chat_DB_PATH)
+    cursor = conn.cursor()
+    cursor.execute("SELECT * FROM CharacterCards")
+    rows = cursor.fetchall()
+    columns = [description[0] for description in cursor.description]
+    conn.close()
+    characters = [dict(zip(columns, row)) for row in rows]
+    #logging.debug(f"Characters fetched from DB: {characters}")
+    return characters
+
+
+def get_character_card_by_id(character_id: Union[int, Dict[str, Any]]) -> Optional[Dict[str, Any]]:
+    """
+    Retrieve a single character card by its ID.
+
+    Args:
+        character_id: Can be either an integer ID or a dictionary containing character data.
+
+    Returns:
+        A dictionary containing the character card data, or None if not found.
+    """
+    conn = sqlite3.connect(chat_DB_PATH)
+    cursor = conn.cursor()
+    try:
+        if isinstance(character_id, dict):
+            # If a dictionary is passed, assume it's already a character card
+            return character_id
+        elif isinstance(character_id, int):
+            # If an integer is passed, fetch the character from the database
+            cursor.execute("SELECT * FROM CharacterCards WHERE id = ?", (character_id,))
+            row = cursor.fetchone()
+            if row:
+                columns = [description[0] for description in cursor.description]
+                return dict(zip(columns, row))
+        else:
+            logging.warning(f"Invalid type for character_id: {type(character_id)}")
+        return None
+    except Exception as e:
+        logging.error(f"Error in get_character_card_by_id: {e}")
+        return None
+    finally:
+        conn.close()
+
+
+def update_character_card(character_id: int, card_data: Dict) -> bool:
+    """Update an existing character card."""
+    conn = sqlite3.connect(chat_DB_PATH)
+    cursor = conn.cursor()
+    try:
+        cursor.execute("""
+            UPDATE CharacterCards
+            SET name = ?, description = ?, personality = ?, scenario = ?, image = ?, post_history_instructions = ?, first_message = ?
+            WHERE id = ?
+        """, (
+            card_data.get('name'),
+            card_data.get('description'),
+            card_data.get('personality'),
+            card_data.get('scenario'),
+            card_data.get('image'),
+            card_data.get('post_history_instructions', ''),
+            card_data.get('first_message', "Hello! I'm ready to chat."),
+            character_id
+        ))
+        conn.commit()
+        return cursor.rowcount > 0
+    except sqlite3.IntegrityError as e:
+        logging.error(f"Error updating character card: {e}")
+        return False
+    finally:
+        conn.close()
+
+
+def delete_character_card(character_id: int) -> bool:
+    """Delete a character card and its associated chats."""
+    conn = sqlite3.connect(chat_DB_PATH)
+    cursor = conn.cursor()
+    try:
+        # Delete associated chats first due to foreign key constraint
+        cursor.execute("DELETE FROM CharacterChats WHERE character_id = ?", (character_id,))
+        cursor.execute("DELETE FROM CharacterCards WHERE id = ?", (character_id,))
+        conn.commit()
+        return cursor.rowcount > 0
+    except sqlite3.Error as e:
+        logging.error(f"Error deleting character card: {e}")
+        return False
+    finally:
+        conn.close()
+
+
+def add_character_chat(character_id: int, conversation_name: str, chat_history: List[Tuple[str, str]], keywords: Optional[List[str]] = None, is_snapshot: bool = False) -> Optional[int]:
+    """
+    Add a new chat history for a character, optionally associating keywords.
+
+    Args:
+        character_id (int): The ID of the character.
+        conversation_name (str): Name of the conversation.
+        chat_history (List[Tuple[str, str]]): List of (user, bot) message tuples.
+        keywords (Optional[List[str]]): List of keywords to associate with this chat.
+        is_snapshot (bool, optional): Whether this chat is a snapshot.
+
+    Returns:
+        Optional[int]: The ID of the inserted chat or None if failed.
+    """
+    conn = sqlite3.connect(chat_DB_PATH)
+    cursor = conn.cursor()
+    try:
+        chat_history_json = json.dumps(chat_history)
+        cursor.execute("""
+            INSERT INTO CharacterChats (character_id, conversation_name, chat_history, is_snapshot)
+            VALUES (?, ?, ?, ?)
+        """, (
+            character_id,
+            conversation_name,
+            chat_history_json,
+            is_snapshot
+        ))
+        chat_id = cursor.lastrowid
+
+        if keywords:
+            # Insert keywords into ChatKeywords table
+            keyword_records = [(chat_id, keyword.strip().lower()) for keyword in keywords]
+            cursor.executemany("""
+                INSERT INTO ChatKeywords (chat_id, keyword)
+                VALUES (?, ?)
+            """, keyword_records)
+
+        conn.commit()
+        return chat_id
+    except sqlite3.Error as e:
+        logging.error(f"Error adding character chat: {e}")
+        return None
+    finally:
+        conn.close()
+
+
+def get_character_chats(character_id: Optional[int] = None) -> List[Dict]:
+    """Retrieve all chats, or chats for a specific character if character_id is provided."""
+    conn = sqlite3.connect(chat_DB_PATH)
+    cursor = conn.cursor()
+    if character_id is not None:
+        cursor.execute("SELECT * FROM CharacterChats WHERE character_id = ?", (character_id,))
+    else:
+        cursor.execute("SELECT * FROM CharacterChats")
+    rows = cursor.fetchall()
+    columns = [description[0] for description in cursor.description]
+    conn.close()
+    return [dict(zip(columns, row)) for row in rows]
+
+
+def get_character_chat_by_id(chat_id: int) -> Optional[Dict]:
+    """Retrieve a single chat by its ID."""
+    conn = sqlite3.connect(chat_DB_PATH)
+    cursor = conn.cursor()
+    cursor.execute("SELECT * FROM CharacterChats WHERE id = ?", (chat_id,))
+    row = cursor.fetchone()
+    conn.close()
+    if row:
+        columns = [description[0] for description in cursor.description]
+        chat = dict(zip(columns, row))
+        chat['chat_history'] = json.loads(chat['chat_history'])
+        return chat
+    return None
+
+
+def search_character_chats(query: str, character_id: Optional[int] = None) -> Tuple[List[Dict], str]:
+    """
+    Search for character chats using FTS5, optionally filtered by character_id.
+
+    Args:
+        query (str): The search query.
+        character_id (Optional[int]): The ID of the character to filter chats by.
+
+    Returns:
+        Tuple[List[Dict], str]: A list of matching chats and a status message.
+    """
+    if not query.strip():
+        return [], "Please enter a search query."
+
+    conn = sqlite3.connect(chat_DB_PATH)
+    cursor = conn.cursor()
+    try:
+        if character_id is not None:
+            # Search with character_id filter
+            cursor.execute("""
+                SELECT CharacterChats.id, CharacterChats.conversation_name, CharacterChats.chat_history
+                FROM CharacterChats_fts
+                JOIN CharacterChats ON CharacterChats_fts.rowid = CharacterChats.id
+                WHERE CharacterChats_fts MATCH ? AND CharacterChats.character_id = ?
+                ORDER BY rank
+            """, (query, character_id))
+        else:
+            # Search without character_id filter
+            cursor.execute("""
+                SELECT CharacterChats.id, CharacterChats.conversation_name, CharacterChats.chat_history
+                FROM CharacterChats_fts
+                JOIN CharacterChats ON CharacterChats_fts.rowid = CharacterChats.id
+                WHERE CharacterChats_fts MATCH ?
+                ORDER BY rank
+            """, (query,))
+
+        rows = cursor.fetchall()
+        columns = [description[0] for description in cursor.description]
+        results = [dict(zip(columns, row)) for row in rows]
+
+        if character_id is not None:
+            status_message = f"Found {len(results)} chat(s) matching '{query}' for the selected character."
+        else:
+            status_message = f"Found {len(results)} chat(s) matching '{query}' across all characters."
+
+        return results, status_message
+    except Exception as e:
+        logging.error(f"Error searching chats with FTS5: {e}")
+        return [], f"Error occurred during search: {e}"
+    finally:
+        conn.close()
+
+def update_character_chat(chat_id: int, chat_history: List[Tuple[str, str]]) -> bool:
+    """Update an existing chat history."""
+    conn = sqlite3.connect(chat_DB_PATH)
+    cursor = conn.cursor()
+    try:
+        chat_history_json = json.dumps(chat_history)
+        cursor.execute("""
+            UPDATE CharacterChats
+            SET chat_history = ?
+            WHERE id = ?
+        """, (
+            chat_history_json,
+            chat_id
+        ))
+        conn.commit()
+        return cursor.rowcount > 0
+    except sqlite3.Error as e:
+        logging.error(f"Error updating character chat: {e}")
+        return False
+    finally:
+        conn.close()
+
+
+def delete_character_chat(chat_id: int) -> bool:
+    """Delete a specific chat."""
+    conn = sqlite3.connect(chat_DB_PATH)
+    cursor = conn.cursor()
+    try:
+        cursor.execute("DELETE FROM CharacterChats WHERE id = ?", (chat_id,))
+        conn.commit()
+        return cursor.rowcount > 0
+    except sqlite3.Error as e:
+        logging.error(f"Error deleting character chat: {e}")
+        return False
+    finally:
+        conn.close()
+
+
+def fetch_keywords_for_chats(keywords: List[str]) -> List[int]:
+    """
+    Fetch chat IDs associated with any of the specified keywords.
+
+    Args:
+        keywords (List[str]): List of keywords to search for.
+
+    Returns:
+        List[int]: List of chat IDs associated with the keywords.
+    """
+    if not keywords:
+        return []
+
+    conn = sqlite3.connect(chat_DB_PATH)
+    cursor = conn.cursor()
+    try:
+        # Construct the WHERE clause to search for each keyword
+        keyword_clauses = " OR ".join(["keyword = ?"] * len(keywords))
+        sql_query = f"SELECT DISTINCT chat_id FROM ChatKeywords WHERE {keyword_clauses}"
+        cursor.execute(sql_query, keywords)
+        rows = cursor.fetchall()
+        chat_ids = [row[0] for row in rows]
+        return chat_ids
+    except Exception as e:
+        logging.error(f"Error in fetch_keywords_for_chats: {e}")
+        return []
+    finally:
+        conn.close()
+
+
+def save_chat_history_to_character_db(character_id: int, conversation_name: str, chat_history: List[Tuple[str, str]]) -> Optional[int]:
+    """Save chat history to the CharacterChats table.
+
+    Returns the ID of the inserted chat or None if failed.
+    """
+    return add_character_chat(character_id, conversation_name, chat_history)
+
+
+def search_db(query: str, fields: List[str], where_clause: str = "", page: int = 1, results_per_page: int = 5) -> List[Dict[str, Any]]:
+    """
+    Perform a full-text search on specified fields with optional filtering and pagination.
+
+    Args:
+        query (str): The search query.
+        fields (List[str]): List of fields to search in.
+        where_clause (str, optional): Additional SQL WHERE clause to filter results.
+        page (int, optional): Page number for pagination.
+        results_per_page (int, optional): Number of results per page.
+
+    Returns:
+        List[Dict[str, Any]]: List of matching chat records with content and metadata.
+    """
+    if not query.strip():
+        return []
+
+    conn = sqlite3.connect(chat_DB_PATH)
+    cursor = conn.cursor()
+    try:
+        # Construct the MATCH query for FTS5
+        match_query = " AND ".join(fields) + f" MATCH ?"
+        # Adjust the query with the fields
+        fts_query = f"""
+            SELECT CharacterChats.id, CharacterChats.conversation_name, CharacterChats.chat_history
+            FROM CharacterChats_fts
+            JOIN CharacterChats ON CharacterChats_fts.rowid = CharacterChats.id
+            WHERE {match_query}
+        """
+        if where_clause:
+            fts_query += f" AND ({where_clause})"
+        fts_query += " ORDER BY rank LIMIT ? OFFSET ?"
+        offset = (page - 1) * results_per_page
+        cursor.execute(fts_query, (query, results_per_page, offset))
+        rows = cursor.fetchall()
+        columns = [description[0] for description in cursor.description]
+        results = [dict(zip(columns, row)) for row in rows]
+        return results
+    except Exception as e:
+        logging.error(f"Error in search_db: {e}")
+        return []
+    finally:
+        conn.close()
+
+
+def perform_full_text_search_chat(query: str, relevant_chat_ids: List[int], page: int = 1, results_per_page: int = 5) -> \
+List[Dict[str, Any]]:
+    """
+    Perform a full-text search within the specified chat IDs using FTS5.
+
+    Args:
+        query (str): The user's query.
+        relevant_chat_ids (List[int]): List of chat IDs to search within.
+        page (int): Pagination page number.
+        results_per_page (int): Number of results per page.
+
+    Returns:
+        List[Dict[str, Any]]: List of search results with content and metadata.
+    """
+    try:
+        # Construct a WHERE clause to limit the search to relevant chat IDs
+        where_clause = " OR ".join([f"media_id = {chat_id}" for chat_id in relevant_chat_ids])
+        if not where_clause:
+            where_clause = "1"  # No restriction if no chat IDs
+
+        # Perform full-text search using FTS5
+        fts_results = search_db(query, ["content"], where_clause, page=page, results_per_page=results_per_page)
+
+        filtered_fts_results = [
+            {
+                "content": result['content'],
+                "metadata": {"media_id": result['id']}
+            }
+            for result in fts_results
+            if result['id'] in relevant_chat_ids
+        ]
+        return filtered_fts_results
+    except Exception as e:
+        logging.error(f"Error in perform_full_text_search_chat: {str(e)}")
+        return []
+
+
+def fetch_all_chats() -> List[Dict[str, Any]]:
+    """
+    Fetch all chat messages from the database.
+
+    Returns:
+        List[Dict[str, Any]]: List of chat messages with relevant metadata.
+    """
+    try:
+        chats = get_character_chats()  # Modify this function to retrieve all chats
+        return chats
+    except Exception as e:
+        logging.error(f"Error fetching all chats: {str(e)}")
+        return []
+
+
+def search_character_chat(query: str, fts_top_k: int = 10, relevant_media_ids: List[str] = None) -> List[Dict[str, Any]]:
+    """
+    Perform a full-text search on the Character Chat database.
+
+    Args:
+        query: Search query string.
+        fts_top_k: Maximum number of results to return.
+        relevant_media_ids: Optional list of character IDs to filter results.
+
+    Returns:
+        List of search results with content and metadata.
+    """
+    if not query.strip():
+        return []
+
+    try:
+        # Construct a WHERE clause to limit the search to relevant character IDs
+        where_clause = ""
+        if relevant_media_ids:
+            placeholders = ','.join(['?'] * len(relevant_media_ids))
+            where_clause = f"CharacterChats.character_id IN ({placeholders})"
+
+        # Perform full-text search using existing search_db function
+        results = search_db(query, ["conversation_name", "chat_history"], where_clause, results_per_page=fts_top_k)
+
+        # Format results
+        formatted_results = []
+        for r in results:
+            formatted_results.append({
+                "content": r['chat_history'],
+                "metadata": {
+                    "chat_id": r['id'],
+                    "conversation_name": r['conversation_name'],
+                    "character_id": r['character_id']
+                }
+            })
+
+        return formatted_results
+
+    except Exception as e:
+        logging.error(f"Error in search_character_chat: {e}")
+        return []
+
+
+def search_character_cards(query: str, fts_top_k: int = 10, relevant_media_ids: List[str] = None) -> List[Dict[str, Any]]:
+    """
+    Perform a full-text search on the Character Cards database.
+
+    Args:
+        query: Search query string.
+        fts_top_k: Maximum number of results to return.
+        relevant_media_ids: Optional list of character IDs to filter results.
+
+    Returns:
+        List of search results with content and metadata.
+    """
+    if not query.strip():
+        return []
+
+    try:
+        conn = sqlite3.connect(chat_DB_PATH)
+        cursor = conn.cursor()
+
+        # Construct the query
+        sql_query = """
+            SELECT CharacterCards.id, CharacterCards.name, CharacterCards.description, CharacterCards.personality, CharacterCards.scenario
+            FROM CharacterCards_fts
+            JOIN CharacterCards ON CharacterCards_fts.rowid = CharacterCards.id
+            WHERE CharacterCards_fts MATCH ?
+        """
+
+        params = [query]
+
+        # Add filtering by character IDs if provided
+        if relevant_media_ids:
+            placeholders = ','.join(['?'] * len(relevant_media_ids))
+            sql_query += f" AND CharacterCards.id IN ({placeholders})"
+            params.extend(relevant_media_ids)
+
+        sql_query += " LIMIT ?"
+        params.append(fts_top_k)
+
+        cursor.execute(sql_query, params)
+        rows = cursor.fetchall()
+        columns = [description[0] for description in cursor.description]
+
+        results = [dict(zip(columns, row)) for row in rows]
+
+        # Format results
+        formatted_results = []
+        for r in results:
+            content = f"Name: {r['name']}\nDescription: {r['description']}\nPersonality: {r['personality']}\nScenario: {r['scenario']}"
+            formatted_results.append({
+                "content": content,
+                "metadata": {
+                    "character_id": r['id'],
+                    "name": r['name']
+                }
+            })
+
+        return formatted_results
+
+    except Exception as e:
+        logging.error(f"Error in search_character_cards: {e}")
+        return []
+    finally:
+        conn.close()
+
+
+def fetch_character_ids_by_keywords(keywords: List[str]) -> List[int]:
+    """
+    Fetch character IDs associated with any of the specified keywords.
+
+    Args:
+        keywords (List[str]): List of keywords to search for.
+
+    Returns:
+        List[int]: List of character IDs associated with the keywords.
+    """
+    if not keywords:
+        return []
+
+    conn = sqlite3.connect(chat_DB_PATH)
+    cursor = conn.cursor()
+    try:
+        # Assuming 'tags' column in CharacterCards table stores tags as JSON array
+        placeholders = ','.join(['?'] * len(keywords))
+        sql_query = f"""
+            SELECT DISTINCT id FROM CharacterCards
+            WHERE EXISTS (
+                SELECT 1 FROM json_each(tags)
+                WHERE json_each.value IN ({placeholders})
+            )
+        """
+        cursor.execute(sql_query, keywords)
+        rows = cursor.fetchall()
+        character_ids = [row[0] for row in rows]
+        return character_ids
+    except Exception as e:
+        logging.error(f"Error in fetch_character_ids_by_keywords: {e}")
+        return []
+    finally:
+        conn.close()
+
+
+###################################################################
+#
+# Character Keywords
+
+def view_char_keywords():
+    try:
+        with sqlite3.connect(chat_DB_PATH) as conn:
+            cursor = conn.cursor()
+            cursor.execute("""
+                SELECT DISTINCT keyword 
+                FROM CharacterCards 
+                CROSS JOIN json_each(tags) 
+                WHERE json_valid(tags)
+                ORDER BY keyword
+            """)
+            keywords = cursor.fetchall()
+            if keywords:
+                keyword_list = [k[0] for k in keywords]
+                return "### Current Character Keywords:\n" + "\n".join(
+                    [f"- {k}" for k in keyword_list])
+            return "No keywords found."
+    except Exception as e:
+        return f"Error retrieving keywords: {str(e)}"
+
+
+def add_char_keywords(name: str, keywords: str):
+    try:
+        keywords_list = [k.strip() for k in keywords.split(",") if k.strip()]
+        with sqlite3.connect('character_chat.db') as conn:
+            cursor = conn.cursor()
+            cursor.execute(
+                "SELECT tags FROM CharacterCards WHERE name = ?",
+                (name,)
+            )
+            result = cursor.fetchone()
+            if not result:
+                return "Character not found."
+
+            current_tags = result[0] if result[0] else "[]"
+            current_keywords = set(current_tags[1:-1].split(',')) if current_tags != "[]" else set()
+            updated_keywords = current_keywords.union(set(keywords_list))
+
+            cursor.execute(
+                "UPDATE CharacterCards SET tags = ? WHERE name = ?",
+                (str(list(updated_keywords)), name)
+            )
+            conn.commit()
+            return f"Successfully added keywords to character {name}"
+    except Exception as e:
+        return f"Error adding keywords: {str(e)}"
+
+
+def delete_char_keyword(char_name: str, keyword: str) -> str:
+    """
+    Delete a keyword from a character's tags.
+
+    Args:
+        char_name (str): The name of the character
+        keyword (str): The keyword to delete
+
+    Returns:
+        str: Success/failure message
+    """
+    try:
+        with sqlite3.connect(chat_DB_PATH) as conn:
+            cursor = conn.cursor()
+
+            # First, check if the character exists
+            cursor.execute("SELECT tags FROM CharacterCards WHERE name = ?", (char_name,))
+            result = cursor.fetchone()
+
+            if not result:
+                return f"Character '{char_name}' not found."
+
+            # Parse existing tags
+            current_tags = json.loads(result[0]) if result[0] else []
+
+            if keyword not in current_tags:
+                return f"Keyword '{keyword}' not found in character '{char_name}' tags."
+
+            # Remove the keyword
+            updated_tags = [tag for tag in current_tags if tag != keyword]
+
+            # Update the character's tags
+            cursor.execute(
+                "UPDATE CharacterCards SET tags = ? WHERE name = ?",
+                (json.dumps(updated_tags), char_name)
+            )
+            conn.commit()
+
+            logging.info(f"Keyword '{keyword}' deleted from character '{char_name}'")
+            return f"Successfully deleted keyword '{keyword}' from character '{char_name}'."
+
+    except Exception as e:
+        error_msg = f"Error deleting keyword: {str(e)}"
+        logging.error(error_msg)
+        return error_msg
+
+
+def export_char_keywords_to_csv() -> Tuple[str, str]:
+    """
+    Export all character keywords to a CSV file with associated metadata.
+
+    Returns:
+        Tuple[str, str]: (status_message, file_path)
+    """
+    import csv
+    from tempfile import NamedTemporaryFile
+    from datetime import datetime
+
+    try:
+        # Create a temporary CSV file
+        temp_file = NamedTemporaryFile(mode='w+', delete=False, suffix='.csv', newline='')
+
+        with sqlite3.connect(chat_DB_PATH) as conn:
+            cursor = conn.cursor()
+
+            # Get all characters and their tags
+            cursor.execute("""
+                SELECT 
+                    name,
+                    tags,
+                    (SELECT COUNT(*) FROM CharacterChats WHERE CharacterChats.character_id = CharacterCards.id) as chat_count
+                FROM CharacterCards
+                WHERE json_valid(tags)
+                ORDER BY name
+            """)
+
+            results = cursor.fetchall()
+
+            # Process the results to create rows for the CSV
+            csv_rows = []
+            for name, tags_json, chat_count in results:
+                tags = json.loads(tags_json) if tags_json else []
+                for tag in tags:
+                    csv_rows.append([
+                        tag,  # keyword
+                        name,  # character name
+                        chat_count  # number of chats
+                    ])
+
+            # Write to CSV
+            writer = csv.writer(temp_file)
+            writer.writerow(['Keyword', 'Character Name', 'Number of Chats'])
+            writer.writerows(csv_rows)
+
+        temp_file.close()
+
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        status_msg = f"Successfully exported {len(csv_rows)} character keyword entries to CSV."
+        logging.info(status_msg)
+
+        return status_msg, temp_file.name
+
+    except Exception as e:
+        error_msg = f"Error exporting keywords: {str(e)}"
+        logging.error(error_msg)
+        return error_msg, ""
+
+#
+# End of Character chat keyword functions
+######################################################
+
+
+#
+# End of Character_Chat_DB.py
+#######################################################################################################################

App_Function_Libraries/DB/DB_Backups.py

@@ -0,0 +1,160 @@
+# Backup_Manager.py
+#
+# Imports:
+import os
+import shutil
+import sqlite3
+from datetime import datetime
+import logging
+#
+# Local Imports:
+from App_Function_Libraries.DB.Character_Chat_DB import chat_DB_PATH
+from App_Function_Libraries.DB.RAG_QA_Chat_DB import get_rag_qa_db_path
+from App_Function_Libraries.Utils.Utils import get_project_relative_path
+#
+# End of Imports
+#######################################################################################################################
+#
+# Functions:
+
+def init_backup_directory(backup_base_dir: str, db_name: str) -> str:
+    """Initialize backup directory for a specific database."""
+    backup_dir = os.path.join(backup_base_dir, db_name)
+    os.makedirs(backup_dir, exist_ok=True)
+    return backup_dir
+
+
+def create_backup(db_path: str, backup_dir: str, db_name: str) -> str:
+    """Create a full backup of the database."""
+    try:
+        db_path = os.path.abspath(db_path)
+        backup_dir = os.path.abspath(backup_dir)
+
+        logging.info(f"Creating backup:")
+        logging.info(f"  DB Path: {db_path}")
+        logging.info(f"  Backup Dir: {backup_dir}")
+        logging.info(f"  DB Name: {db_name}")
+
+        # Create subdirectory based on db_name
+        specific_backup_dir = os.path.join(backup_dir, db_name)
+        os.makedirs(specific_backup_dir, exist_ok=True)
+
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        backup_file = os.path.join(specific_backup_dir, f"{db_name}_backup_{timestamp}.db")
+        logging.info(f"  Full backup path: {backup_file}")
+
+        # Create a backup using SQLite's backup API
+        with sqlite3.connect(db_path) as source, \
+                sqlite3.connect(backup_file) as target:
+            source.backup(target)
+
+        logging.info(f"Backup created successfully: {backup_file}")
+        return f"Backup created: {backup_file}"
+    except Exception as e:
+        error_msg = f"Failed to create backup: {str(e)}"
+        logging.error(error_msg)
+        return error_msg
+
+
+def create_incremental_backup(db_path: str, backup_dir: str, db_name: str) -> str:
+    """Create an incremental backup using VACUUM INTO."""
+    try:
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        backup_file = os.path.join(backup_dir,
+                                   f"{db_name}_incremental_{timestamp}.sqlib")
+
+        with sqlite3.connect(db_path) as conn:
+            conn.execute(f"VACUUM INTO '{backup_file}'")
+
+        logging.info(f"Incremental backup created: {backup_file}")
+        return f"Incremental backup created: {backup_file}"
+    except Exception as e:
+        error_msg = f"Failed to create incremental backup: {str(e)}"
+        logging.error(error_msg)
+        return error_msg
+
+
+def list_backups(backup_dir: str) -> str:
+    """List all available backups."""
+    try:
+        backups = [f for f in os.listdir(backup_dir)
+                   if f.endswith(('.db', '.sqlib'))]
+        backups.sort(reverse=True)  # Most recent first
+        return "\n".join(backups) if backups else "No backups found"
+    except Exception as e:
+        error_msg = f"Failed to list backups: {str(e)}"
+        logging.error(error_msg)
+        return error_msg
+
+
+def restore_single_db_backup(db_path: str, backup_dir: str, db_name: str, backup_name: str) -> str:
+    """Restore database from a backup file."""
+    try:
+        logging.info(f"Restoring backup: {backup_name}")
+        backup_path = os.path.join(backup_dir, backup_name)
+        if not os.path.exists(backup_path):
+            logging.error(f"Backup file not found: {backup_name}")
+            return f"Backup file not found: {backup_name}"
+
+        # Create a timestamp for the current db
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        current_backup = os.path.join(backup_dir,
+                                      f"{db_name}_pre_restore_{timestamp}.db")
+
+        # Backup current database before restore
+        logging.info(f"Creating backup of current database: {current_backup}")
+        shutil.copy2(db_path, current_backup)
+
+        # Restore the backup
+        logging.info(f"Restoring database from {backup_name}")
+        shutil.copy2(backup_path, db_path)
+
+        logging.info(f"Database restored from {backup_name}")
+        return f"Database restored from {backup_name}"
+    except Exception as e:
+        error_msg = f"Failed to restore backup: {str(e)}"
+        logging.error(error_msg)
+        return error_msg
+
+
+def setup_backup_config():
+    """Setup configuration for database backups."""
+    backup_base_dir = get_project_relative_path('tldw_DB_Backups')
+    logging.info(f"Base backup directory: {os.path.abspath(backup_base_dir)}")
+
+    # RAG Chat DB configuration
+    rag_db_path = get_rag_qa_db_path()
+    rag_backup_dir = os.path.join(backup_base_dir, 'rag_chat')
+    os.makedirs(rag_backup_dir, exist_ok=True)
+    logging.info(f"RAG backup directory: {os.path.abspath(rag_backup_dir)}")
+
+    rag_db_config = {
+        'db_path': rag_db_path,
+        'backup_dir': rag_backup_dir,  # Make sure we use the full path
+        'db_name': 'rag_qa'
+    }
+
+    # Character Chat DB configuration
+    char_backup_dir = os.path.join(backup_base_dir, 'character_chat')
+    os.makedirs(char_backup_dir, exist_ok=True)
+    logging.info(f"Character backup directory: {os.path.abspath(char_backup_dir)}")
+
+    char_db_config = {
+        'db_path': chat_DB_PATH,
+        'backup_dir': char_backup_dir,  # Make sure we use the full path
+        'db_name': 'chatDB'
+    }
+
+    # Media DB configuration (based on your logs)
+    media_backup_dir = os.path.join(backup_base_dir, 'media')
+    os.makedirs(media_backup_dir, exist_ok=True)
+    logging.info(f"Media backup directory: {os.path.abspath(media_backup_dir)}")
+
+    media_db_config = {
+        'db_path': os.path.join(os.path.dirname(chat_DB_PATH), 'media_summary.db'),
+        'backup_dir': media_backup_dir,
+        'db_name': 'media'
+    }
+
+    return rag_db_config, char_db_config, media_db_config
+

App_Function_Libraries/DB/DB_Manager.py

@@ -13,11 +13,14 @@ from elasticsearch import Elasticsearch
 #
 # Import your existing SQLite functions
 from App_Function_Libraries.DB.SQLite_DB import DatabaseError
+from App_Function_Libraries.DB.Prompts_DB import list_prompts as sqlite_list_prompts, \
+    fetch_prompt_details as sqlite_fetch_prompt_details, add_prompt as sqlite_add_prompt, \
+    search_prompts as sqlite_search_prompts, add_or_update_prompt as sqlite_add_or_update_prompt, \
+    load_prompt_details as sqlite_load_prompt_details, insert_prompt_to_db as sqlite_insert_prompt_to_db, \
+    delete_prompt as sqlite_delete_prompt
 from App_Function_Libraries.DB.SQLite_DB import (
     update_media_content as sqlite_update_media_content,
-    list_prompts as sqlite_list_prompts,
     search_and_display as sqlite_search_and_display,
-    fetch_prompt_details as sqlite_fetch_prompt_details,
     keywords_browser_interface as sqlite_keywords_browser_interface,
     add_keyword as sqlite_add_keyword,
     delete_keyword as sqlite_delete_keyword,
@@ -25,31 +28,17 @@ from App_Function_Libraries.DB.SQLite_DB import (
     ingest_article_to_db as sqlite_ingest_article_to_db,
     add_media_to_database as sqlite_add_media_to_database,
     import_obsidian_note_to_db as sqlite_import_obsidian_note_to_db,
-    add_prompt as sqlite_add_prompt,
-    delete_chat_message as sqlite_delete_chat_message,
-    update_chat_message as sqlite_update_chat_message,
-    add_chat_message as sqlite_add_chat_message,
-    get_chat_messages as sqlite_get_chat_messages,
-    search_chat_conversations as sqlite_search_chat_conversations,
-    create_chat_conversation as sqlite_create_chat_conversation,
-    save_chat_history_to_database as sqlite_save_chat_history_to_database,
     view_database as sqlite_view_database,
     get_transcripts as sqlite_get_transcripts,
     get_trashed_items as sqlite_get_trashed_items,
     user_delete_item as sqlite_user_delete_item,
     empty_trash as sqlite_empty_trash,
     create_automated_backup as sqlite_create_automated_backup,
-    add_or_update_prompt as sqlite_add_or_update_prompt,
-    load_prompt_details as sqlite_load_prompt_details,
-    load_preset_prompts as sqlite_load_preset_prompts,
-    insert_prompt_to_db as sqlite_insert_prompt_to_db,
-    delete_prompt as sqlite_delete_prompt,
     search_and_display_items as sqlite_search_and_display_items,
-    get_conversation_name as sqlite_get_conversation_name,
     add_media_with_keywords as sqlite_add_media_with_keywords,
     check_media_and_whisper_model as sqlite_check_media_and_whisper_model, \
     create_document_version as sqlite_create_document_version,
-    get_document_version as sqlite_get_document_version, sqlite_search_db, add_media_chunk as sqlite_add_media_chunk,
+    get_document_version as sqlite_get_document_version, search_media_db as sqlite_search_media_db, add_media_chunk as sqlite_add_media_chunk,
     sqlite_update_fts_for_media, get_unprocessed_media as sqlite_get_unprocessed_media, fetch_item_details as sqlite_fetch_item_details, \
     search_media_database as sqlite_search_media_database, mark_as_trash as sqlite_mark_as_trash, \
     get_media_transcripts as sqlite_get_media_transcripts, get_specific_transcript as sqlite_get_specific_transcript, \
@@ -60,23 +49,35 @@ from App_Function_Libraries.DB.SQLite_DB import (
     delete_specific_prompt as sqlite_delete_specific_prompt,
     fetch_keywords_for_media as sqlite_fetch_keywords_for_media, \
     update_keywords_for_media as sqlite_update_keywords_for_media, check_media_exists as sqlite_check_media_exists, \
-    search_prompts as sqlite_search_prompts, get_media_content as sqlite_get_media_content, \
-    get_paginated_files as sqlite_get_paginated_files, get_media_title as sqlite_get_media_title, \
-    get_all_content_from_database as sqlite_get_all_content_from_database,
-    get_next_media_id as sqlite_get_next_media_id, \
-    batch_insert_chunks as sqlite_batch_insert_chunks, Database, save_workflow_chat_to_db as sqlite_save_workflow_chat_to_db, \
-    get_workflow_chat as sqlite_get_workflow_chat, update_media_content_with_version as sqlite_update_media_content_with_version, \
+    get_media_content as sqlite_get_media_content, get_paginated_files as sqlite_get_paginated_files, \
+    get_media_title as sqlite_get_media_title, get_all_content_from_database as sqlite_get_all_content_from_database, \
+    get_next_media_id as sqlite_get_next_media_id, batch_insert_chunks as sqlite_batch_insert_chunks, Database, \
+    save_workflow_chat_to_db as sqlite_save_workflow_chat_to_db, get_workflow_chat as sqlite_get_workflow_chat, \
+    update_media_content_with_version as sqlite_update_media_content_with_version, \
     check_existing_media as sqlite_check_existing_media, get_all_document_versions as sqlite_get_all_document_versions, \
     fetch_paginated_data as sqlite_fetch_paginated_data, get_latest_transcription as sqlite_get_latest_transcription, \
     mark_media_as_processed as sqlite_mark_media_as_processed,
 )
+from App_Function_Libraries.DB.RAG_QA_Chat_DB import start_new_conversation as sqlite_start_new_conversation, \
+    save_message as sqlite_save_message, load_chat_history as sqlite_load_chat_history, \
+    get_all_conversations as sqlite_get_all_conversations, get_notes_by_keywords as sqlite_get_notes_by_keywords, \
+    get_note_by_id as sqlite_get_note_by_id, update_note as sqlite_update_note, save_notes as sqlite_save_notes, \
+    clear_keywords_from_note as sqlite_clear_keywords_from_note, add_keywords_to_note as sqlite_add_keywords_to_note, \
+    add_keywords_to_conversation as sqlite_add_keywords_to_conversation, \
+    get_keywords_for_note as sqlite_get_keywords_for_note, delete_note as sqlite_delete_note, \
+    search_conversations_by_keywords as sqlite_search_conversations_by_keywords, \
+    delete_conversation as sqlite_delete_conversation, get_conversation_title as sqlite_get_conversation_title, \
+    update_conversation_title as sqlite_update_conversation_title, \
+    fetch_all_conversations as sqlite_fetch_all_conversations, fetch_all_notes as sqlite_fetch_all_notes, \
+    fetch_conversations_by_ids as sqlite_fetch_conversations_by_ids, fetch_notes_by_ids as sqlite_fetch_notes_by_ids, \
+    delete_messages_in_conversation as sqlite_delete_messages_in_conversation, \
+    get_conversation_text as sqlite_get_conversation_text, search_notes_titles as sqlite_search_notes_titles
 from App_Function_Libraries.DB.Character_Chat_DB import (
     add_character_card as sqlite_add_character_card, get_character_cards as sqlite_get_character_cards, \
     get_character_card_by_id as sqlite_get_character_card_by_id, update_character_card as sqlite_update_character_card, \
     delete_character_card as sqlite_delete_character_card, add_character_chat as sqlite_add_character_chat, \
     get_character_chats as sqlite_get_character_chats, get_character_chat_by_id as sqlite_get_character_chat_by_id, \
-    update_character_chat as sqlite_update_character_chat, delete_character_chat as sqlite_delete_character_chat, \
-    migrate_chat_to_media_db as sqlite_migrate_chat_to_media_db,
+    update_character_chat as sqlite_update_character_chat, delete_character_chat as sqlite_delete_character_chat
 )
 #
 # Local Imports
@@ -214,9 +215,9 @@ print(f"Database path: {db.db_path}")
 #
 # DB Search functions
 
-def search_db(search_query: str, search_fields: List[str], keywords: str, page: int = 1, results_per_page: int = 10):
+def search_media_db(search_query: str, search_fields: List[str], keywords: str, page: int = 1, results_per_page: int = 10):
     if db_type == 'sqlite':
-        return sqlite_search_db(search_query, search_fields, keywords, page, results_per_page)
+        return sqlite_search_media_db(search_query, search_fields, keywords, page, results_per_page)
     elif db_type == 'elasticsearch':
         # Implement Elasticsearch version when available
         raise NotImplementedError("Elasticsearch version of search_db not yet implemented")
@@ -500,13 +501,6 @@ def load_prompt_details(*args, **kwargs):
         # Implement Elasticsearch version
         raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
 
-def load_preset_prompts(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_load_preset_prompts()
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
 def insert_prompt_to_db(*args, **kwargs):
     if db_type == 'sqlite':
         return sqlite_insert_prompt_to_db(*args, **kwargs)
@@ -539,7 +533,6 @@ def mark_as_trash(media_id: int) -> None:
     else:
         raise ValueError(f"Unsupported database type: {db_type}")
 
-
 def get_latest_transcription(*args, **kwargs):
     if db_type == 'sqlite':
         return sqlite_get_latest_transcription(*args, **kwargs)
@@ -721,62 +714,132 @@ def fetch_keywords_for_media(*args, **kwargs):
 #
 # Chat-related Functions
 
-def delete_chat_message(*args, **kwargs):
+def search_notes_titles(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_search_notes_titles(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+def save_message(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_save_message(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+def load_chat_history(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_load_chat_history(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+def start_new_conversation(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_start_new_conversation(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+def get_all_conversations(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_get_all_conversations(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+def get_notes_by_keywords(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_get_notes_by_keywords(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+def get_note_by_id(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_get_note_by_id(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+def add_keywords_to_conversation(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_add_keywords_to_conversation(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
+
+def get_keywords_for_note(*args, **kwargs):
     if db_type == 'sqlite':
-        return sqlite_delete_chat_message(*args, **kwargs)
+        return sqlite_get_keywords_for_note(*args, **kwargs)
     elif db_type == 'elasticsearch':
         # Implement Elasticsearch version
         raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
 
-def update_chat_message(*args, **kwargs):
+def delete_note(*args, **kwargs):
     if db_type == 'sqlite':
-        return sqlite_update_chat_message(*args, **kwargs)
+        return sqlite_delete_note(*args, **kwargs)
     elif db_type == 'elasticsearch':
         # Implement Elasticsearch version
         raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
 
-def add_chat_message(*args, **kwargs):
+def search_conversations_by_keywords(*args, **kwargs):
     if db_type == 'sqlite':
-        return sqlite_add_chat_message(*args, **kwargs)
+        return sqlite_search_conversations_by_keywords(*args, **kwargs)
     elif db_type == 'elasticsearch':
         # Implement Elasticsearch version
         raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
 
-def get_chat_messages(*args, **kwargs):
+def delete_conversation(*args, **kwargs):
     if db_type == 'sqlite':
-        return sqlite_get_chat_messages(*args, **kwargs)
+        return sqlite_delete_conversation(*args, **kwargs)
     elif db_type == 'elasticsearch':
         # Implement Elasticsearch version
         raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
 
-def search_chat_conversations(*args, **kwargs):
+def get_conversation_title(*args, **kwargs):
     if db_type == 'sqlite':
-        return sqlite_search_chat_conversations(*args, **kwargs)
+        return sqlite_get_conversation_title(*args, **kwargs)
     elif db_type == 'elasticsearch':
         # Implement Elasticsearch version
         raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
 
-def create_chat_conversation(*args, **kwargs):
+def update_conversation_title(*args, **kwargs):
     if db_type == 'sqlite':
-        return sqlite_create_chat_conversation(*args, **kwargs)
+        return sqlite_update_conversation_title(*args, **kwargs)
     elif db_type == 'elasticsearch':
         # Implement Elasticsearch version
         raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
 
-def save_chat_history_to_database(*args, **kwargs):
+def fetch_all_conversations(*args, **kwargs):
     if db_type == 'sqlite':
-        return sqlite_save_chat_history_to_database(*args, **kwargs)
+        return sqlite_fetch_all_conversations()
     elif db_type == 'elasticsearch':
         # Implement Elasticsearch version
         raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
 
-def get_conversation_name(*args, **kwargs):
+def fetch_all_notes(*args, **kwargs):
     if db_type == 'sqlite':
-        return sqlite_get_conversation_name(*args, **kwargs)
+        return sqlite_fetch_all_notes()
     elif db_type == 'elasticsearch':
         # Implement Elasticsearch version
         raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
 
+def delete_messages_in_conversation(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_delete_messages_in_conversation(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of delete_messages_in_conversation not yet implemented")
+
+def get_conversation_text(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_get_conversation_text(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of get_conversation_text not yet implemented")
+
 #
 # End of Chat-related Functions
 ############################################################################################################
@@ -856,12 +919,54 @@ def delete_character_chat(*args, **kwargs):
         # Implement Elasticsearch version
         raise NotImplementedError("Elasticsearch version of delete_character_chat not yet implemented")
 
-def migrate_chat_to_media_db(*args, **kwargs):
+def update_note(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_update_note(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of update_note not yet implemented")
+
+def save_notes(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_save_notes(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of save_notes not yet implemented")
+
+def clear_keywords(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_clear_keywords_from_note(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of clear_keywords not yet implemented")
+
+def clear_keywords_from_note(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_clear_keywords_from_note(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of clear_keywords_from_note not yet implemented")
+
+def add_keywords_to_note(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_add_keywords_to_note(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of add_keywords_to_note not yet implemented")
+
+def fetch_conversations_by_ids(*args, **kwargs):
+    if db_type == 'sqlite':
+        return sqlite_fetch_conversations_by_ids(*args, **kwargs)
+    elif db_type == 'elasticsearch':
+        # Implement Elasticsearch version
+        raise NotImplementedError("Elasticsearch version of fetch_conversations_by_ids not yet implemented")
+
+def fetch_notes_by_ids(*args, **kwargs):
     if db_type == 'sqlite':
-        return sqlite_migrate_chat_to_media_db(*args, **kwargs)
+        return sqlite_fetch_notes_by_ids(*args, **kwargs)
     elif db_type == 'elasticsearch':
         # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of migrate_chat_to_media_db not yet implemented")
+        raise NotImplementedError("Elasticsearch version of fetch_notes_by_ids not yet implemented")
 
 #
 # End of Character Chat-related Functions

App_Function_Libraries/DB/Prompts_DB.py

@@ -0,0 +1,626 @@
+# Prompts_DB.py
+# Description: Functions to manage the prompts database.
+#
+# Imports
+import sqlite3
+import logging
+#
+# External Imports
+import re
+from typing import Tuple
+#
+# Local Imports
+from App_Function_Libraries.Utils.Utils import get_database_path
+#
+#######################################################################################################################
+#
+# Functions to manage prompts DB
+
+def create_prompts_db():
+    logging.debug("create_prompts_db: Creating prompts database.")
+    with sqlite3.connect(get_database_path('prompts.db')) as conn:
+        cursor = conn.cursor()
+        cursor.executescript('''
+            CREATE TABLE IF NOT EXISTS Prompts (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                name TEXT NOT NULL UNIQUE,
+                author TEXT,
+                details TEXT,
+                system TEXT,
+                user TEXT
+            );
+            CREATE TABLE IF NOT EXISTS Keywords (
+                id INTEGER PRIMARY KEY AUTOINCREMENT,
+                keyword TEXT NOT NULL UNIQUE COLLATE NOCASE
+            );
+            CREATE TABLE IF NOT EXISTS PromptKeywords (
+                prompt_id INTEGER,
+                keyword_id INTEGER,
+                FOREIGN KEY (prompt_id) REFERENCES Prompts (id),
+                FOREIGN KEY (keyword_id) REFERENCES Keywords (id),
+                PRIMARY KEY (prompt_id, keyword_id)
+            );
+            CREATE INDEX IF NOT EXISTS idx_keywords_keyword ON Keywords(keyword);
+            CREATE INDEX IF NOT EXISTS idx_promptkeywords_prompt_id ON PromptKeywords(prompt_id);
+            CREATE INDEX IF NOT EXISTS idx_promptkeywords_keyword_id ON PromptKeywords(keyword_id);
+        ''')
+
+# FIXME - dirty hack that should be removed later...
+# Migration function to add the 'author' column to the Prompts table
+def add_author_column_to_prompts():
+    with sqlite3.connect(get_database_path('prompts.db')) as conn:
+        cursor = conn.cursor()
+        # Check if 'author' column already exists
+        cursor.execute("PRAGMA table_info(Prompts)")
+        columns = [col[1] for col in cursor.fetchall()]
+
+        if 'author' not in columns:
+            # Add the 'author' column
+            cursor.execute('ALTER TABLE Prompts ADD COLUMN author TEXT')
+            print("Author column added to Prompts table.")
+        else:
+            print("Author column already exists in Prompts table.")
+
+add_author_column_to_prompts()
+
+def normalize_keyword(keyword):
+    return re.sub(r'\s+', ' ', keyword.strip().lower())
+
+
+# FIXME - update calls to this function to use the new args
+def add_prompt(name, author, details, system=None, user=None, keywords=None):
+    logging.debug(f"add_prompt: Adding prompt with name: {name}, author: {author}, system: {system}, user: {user}, keywords: {keywords}")
+    if not name:
+        logging.error("add_prompt: A name is required.")
+        return "A name is required."
+
+    try:
+        with sqlite3.connect(get_database_path('prompts.db')) as conn:
+            cursor = conn.cursor()
+            cursor.execute('''
+                INSERT INTO Prompts (name, author, details, system, user)
+                VALUES (?, ?, ?, ?, ?)
+            ''', (name, author, details, system, user))
+            prompt_id = cursor.lastrowid
+
+            if keywords:
+                normalized_keywords = [normalize_keyword(k) for k in keywords if k.strip()]
+                for keyword in set(normalized_keywords):  # Use set to remove duplicates
+                    cursor.execute('''
+                        INSERT OR IGNORE INTO Keywords (keyword) VALUES (?)
+                    ''', (keyword,))
+                    cursor.execute('SELECT id FROM Keywords WHERE keyword = ?', (keyword,))
+                    keyword_id = cursor.fetchone()[0]
+                    cursor.execute('''
+                        INSERT OR IGNORE INTO PromptKeywords (prompt_id, keyword_id) VALUES (?, ?)
+                    ''', (prompt_id, keyword_id))
+        return "Prompt added successfully."
+    except sqlite3.IntegrityError:
+        return "Prompt with this name already exists."
+    except sqlite3.Error as e:
+        return f"Database error: {e}"
+
+
+def fetch_prompt_details(name):
+    logging.debug(f"fetch_prompt_details: Fetching details for prompt: {name}")
+    with sqlite3.connect(get_database_path('prompts.db')) as conn:
+        cursor = conn.cursor()
+        cursor.execute('''
+            SELECT p.name, p.author, p.details, p.system, p.user, GROUP_CONCAT(k.keyword, ', ') as keywords
+            FROM Prompts p
+            LEFT JOIN PromptKeywords pk ON p.id = pk.prompt_id
+            LEFT JOIN Keywords k ON pk.keyword_id = k.id
+            WHERE p.name = ?
+            GROUP BY p.id
+        ''', (name,))
+        return cursor.fetchone()
+
+
+def list_prompts(page=1, per_page=10):
+    logging.debug(f"list_prompts: Listing prompts for page {page} with {per_page} prompts per page.")
+    offset = (page - 1) * per_page
+    with sqlite3.connect(get_database_path('prompts.db')) as conn:
+        cursor = conn.cursor()
+        cursor.execute('SELECT name FROM Prompts LIMIT ? OFFSET ?', (per_page, offset))
+        prompts = [row[0] for row in cursor.fetchall()]
+
+        # Get total count of prompts
+        cursor.execute('SELECT COUNT(*) FROM Prompts')
+        total_count = cursor.fetchone()[0]
+
+    total_pages = (total_count + per_page - 1) // per_page
+    return prompts, total_pages, page
+
+
+def insert_prompt_to_db(title, author, description, system_prompt, user_prompt, keywords=None):
+    return add_prompt(title, author, description, system_prompt, user_prompt, keywords)
+
+
+def get_prompt_db_connection():
+    prompt_db_path = get_database_path('prompts.db')
+    return sqlite3.connect(prompt_db_path)
+
+
+def search_prompts(query):
+    logging.debug(f"search_prompts: Searching prompts with query: {query}")
+    try:
+        with get_prompt_db_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute("""
+                SELECT p.name, p.details, p.system, p.user, GROUP_CONCAT(k.keyword, ', ') as keywords
+                FROM Prompts p
+                LEFT JOIN PromptKeywords pk ON p.id = pk.prompt_id
+                LEFT JOIN Keywords k ON pk.keyword_id = k.id
+                WHERE p.name LIKE ? OR p.details LIKE ? OR p.system LIKE ? OR p.user LIKE ? OR k.keyword LIKE ?
+                GROUP BY p.id
+                ORDER BY p.name
+            """, (f'%{query}%', f'%{query}%', f'%{query}%', f'%{query}%', f'%{query}%'))
+            return cursor.fetchall()
+    except sqlite3.Error as e:
+        logging.error(f"Error searching prompts: {e}")
+        return []
+
+
+def search_prompts_by_keyword(keyword, page=1, per_page=10):
+    logging.debug(f"search_prompts_by_keyword: Searching prompts by keyword: {keyword}")
+    normalized_keyword = normalize_keyword(keyword)
+    offset = (page - 1) * per_page
+    with sqlite3.connect(get_database_path('prompts.db')) as conn:
+        cursor = conn.cursor()
+        cursor.execute('''
+            SELECT DISTINCT p.name
+            FROM Prompts p
+            JOIN PromptKeywords pk ON p.id = pk.prompt_id
+            JOIN Keywords k ON pk.keyword_id = k.id
+            WHERE k.keyword LIKE ?
+            LIMIT ? OFFSET ?
+        ''', ('%' + normalized_keyword + '%', per_page, offset))
+        prompts = [row[0] for row in cursor.fetchall()]
+
+        # Get total count of matching prompts
+        cursor.execute('''
+            SELECT COUNT(DISTINCT p.id)
+            FROM Prompts p
+            JOIN PromptKeywords pk ON p.id = pk.prompt_id
+            JOIN Keywords k ON pk.keyword_id = k.id
+            WHERE k.keyword LIKE ?
+        ''', ('%' + normalized_keyword + '%',))
+        total_count = cursor.fetchone()[0]
+
+    total_pages = (total_count + per_page - 1) // per_page
+    return prompts, total_pages, page
+
+
+def update_prompt_keywords(prompt_name, new_keywords):
+    logging.debug(f"update_prompt_keywords: Updating keywords for prompt: {prompt_name}")
+    try:
+        with sqlite3.connect(get_database_path('prompts.db')) as conn:
+            cursor = conn.cursor()
+
+            cursor.execute('SELECT id FROM Prompts WHERE name = ?', (prompt_name,))
+            prompt_id = cursor.fetchone()
+            if not prompt_id:
+                return "Prompt not found."
+            prompt_id = prompt_id[0]
+
+            cursor.execute('DELETE FROM PromptKeywords WHERE prompt_id = ?', (prompt_id,))
+
+            normalized_keywords = [normalize_keyword(k) for k in new_keywords if k.strip()]
+            for keyword in set(normalized_keywords):  # Use set to remove duplicates
+                cursor.execute('INSERT OR IGNORE INTO Keywords (keyword) VALUES (?)', (keyword,))
+                cursor.execute('SELECT id FROM Keywords WHERE keyword = ?', (keyword,))
+                keyword_id = cursor.fetchone()[0]
+                cursor.execute('INSERT INTO PromptKeywords (prompt_id, keyword_id) VALUES (?, ?)',
+                               (prompt_id, keyword_id))
+
+            # Remove unused keywords
+            cursor.execute('''
+                DELETE FROM Keywords
+                WHERE id NOT IN (SELECT DISTINCT keyword_id FROM PromptKeywords)
+            ''')
+        return "Keywords updated successfully."
+    except sqlite3.Error as e:
+        return f"Database error: {e}"
+
+
+def add_or_update_prompt(title, author, description, system_prompt, user_prompt, keywords=None):
+    logging.debug(f"add_or_update_prompt: Adding or updating prompt: {title}")
+    if not title:
+        return "Error: Title is required."
+
+    existing_prompt = fetch_prompt_details(title)
+    if existing_prompt:
+        # Update existing prompt
+        result = update_prompt_in_db(title, author, description, system_prompt, user_prompt)
+        if "successfully" in result:
+            # Update keywords if the prompt update was successful
+            keyword_result = update_prompt_keywords(title, keywords or [])
+            result += f" {keyword_result}"
+    else:
+        # Insert new prompt
+        result = insert_prompt_to_db(title, author, description, system_prompt, user_prompt, keywords)
+
+    return result
+
+
+def load_prompt_details(selected_prompt):
+    logging.debug(f"load_prompt_details: Loading prompt details for {selected_prompt}")
+    if selected_prompt:
+        details = fetch_prompt_details(selected_prompt)
+        if details:
+            return details[0], details[1], details[2], details[3], details[4], details[5]
+    return "", "", "", "", "", ""
+
+
+def update_prompt_in_db(title, author, description, system_prompt, user_prompt):
+    logging.debug(f"update_prompt_in_db: Updating prompt: {title}")
+    try:
+        with sqlite3.connect(get_database_path('prompts.db')) as conn:
+            cursor = conn.cursor()
+            cursor.execute(
+                "UPDATE Prompts SET author = ?, details = ?, system = ?, user = ? WHERE name = ?",
+                (author, description, system_prompt, user_prompt, title)
+            )
+            if cursor.rowcount == 0:
+                return "No prompt found with the given title."
+        return "Prompt updated successfully!"
+    except sqlite3.Error as e:
+        return f"Error updating prompt: {e}"
+
+
+def delete_prompt(prompt_id):
+    logging.debug(f"delete_prompt: Deleting prompt with ID: {prompt_id}")
+    try:
+        with sqlite3.connect(get_database_path('prompts.db')) as conn:
+            cursor = conn.cursor()
+
+            # Delete associated keywords
+            cursor.execute("DELETE FROM PromptKeywords WHERE prompt_id = ?", (prompt_id,))
+
+            # Delete the prompt
+            cursor.execute("DELETE FROM Prompts WHERE id = ?", (prompt_id,))
+
+            if cursor.rowcount == 0:
+                return f"No prompt found with ID {prompt_id}"
+            else:
+                conn.commit()
+                return f"Prompt with ID {prompt_id} has been successfully deleted"
+    except sqlite3.Error as e:
+        return f"An error occurred: {e}"
+
+
+def delete_prompt_keyword(keyword: str) -> str:
+    """
+    Delete a keyword and its associations from the prompts database.
+
+    Args:
+        keyword (str): The keyword to delete
+
+    Returns:
+        str: Success/failure message
+    """
+    logging.debug(f"delete_prompt_keyword: Deleting keyword: {keyword}")
+    try:
+        with sqlite3.connect(get_database_path('prompts.db')) as conn:
+            cursor = conn.cursor()
+
+            # First normalize the keyword
+            normalized_keyword = normalize_keyword(keyword)
+
+            # Get the keyword ID
+            cursor.execute("SELECT id FROM Keywords WHERE keyword = ?", (normalized_keyword,))
+            result = cursor.fetchone()
+
+            if not result:
+                return f"Keyword '{keyword}' not found."
+
+            keyword_id = result[0]
+
+            # Delete keyword associations from PromptKeywords
+            cursor.execute("DELETE FROM PromptKeywords WHERE keyword_id = ?", (keyword_id,))
+
+            # Delete the keyword itself
+            cursor.execute("DELETE FROM Keywords WHERE id = ?", (keyword_id,))
+
+            # Get the number of affected prompts
+            affected_prompts = cursor.rowcount
+
+            conn.commit()
+
+            logging.info(f"Keyword '{keyword}' deleted successfully")
+            return f"Successfully deleted keyword '{keyword}' and removed it from {affected_prompts} prompts."
+
+    except sqlite3.Error as e:
+        error_msg = f"Database error deleting keyword: {str(e)}"
+        logging.error(error_msg)
+        return error_msg
+    except Exception as e:
+        error_msg = f"Error deleting keyword: {str(e)}"
+        logging.error(error_msg)
+        return error_msg
+
+
+def export_prompt_keywords_to_csv() -> Tuple[str, str]:
+    """
+    Export all prompt keywords to a CSV file with associated metadata.
+
+    Returns:
+        Tuple[str, str]: (status_message, file_path)
+    """
+    import csv
+    import tempfile
+    import os
+    from datetime import datetime
+
+    logging.debug("export_prompt_keywords_to_csv: Starting export")
+    try:
+        # Create a temporary file with a specific name in the system's temp directory
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        temp_dir = tempfile.gettempdir()
+        file_path = os.path.join(temp_dir, f'prompt_keywords_export_{timestamp}.csv')
+
+        with sqlite3.connect(get_database_path('prompts.db')) as conn:
+            cursor = conn.cursor()
+
+            # Get keywords with related prompt information
+            query = '''
+            SELECT 
+                k.keyword,
+                GROUP_CONCAT(p.name, ' | ') as prompt_names,
+                COUNT(DISTINCT p.id) as num_prompts,
+                GROUP_CONCAT(DISTINCT p.author, ' | ') as authors
+            FROM Keywords k
+            LEFT JOIN PromptKeywords pk ON k.id = pk.keyword_id
+            LEFT JOIN Prompts p ON pk.prompt_id = p.id
+            GROUP BY k.id, k.keyword
+            ORDER BY k.keyword
+            '''
+
+            cursor.execute(query)
+            results = cursor.fetchall()
+
+            # Write to CSV
+            with open(file_path, 'w', newline='', encoding='utf-8') as csvfile:
+                writer = csv.writer(csvfile)
+                writer.writerow([
+                    'Keyword',
+                    'Associated Prompts',
+                    'Number of Prompts',
+                    'Authors'
+                ])
+
+                for row in results:
+                    writer.writerow([
+                        row[0],  # keyword
+                        row[1] if row[1] else '',  # prompt_names (may be None)
+                        row[2],  # num_prompts
+                        row[3] if row[3] else ''  # authors (may be None)
+                    ])
+
+        status_msg = f"Successfully exported {len(results)} prompt keywords to CSV."
+        logging.info(status_msg)
+
+        return status_msg, file_path
+
+    except sqlite3.Error as e:
+        error_msg = f"Database error exporting keywords: {str(e)}"
+        logging.error(error_msg)
+        return error_msg, "None"
+    except Exception as e:
+        error_msg = f"Error exporting keywords: {str(e)}"
+        logging.error(error_msg)
+        return error_msg, "None"
+
+
+def view_prompt_keywords() -> str:
+    """
+    View all keywords currently in the prompts database.
+
+    Returns:
+        str: Markdown formatted string of all keywords
+    """
+    logging.debug("view_prompt_keywords: Retrieving all keywords")
+    try:
+        with sqlite3.connect(get_database_path('prompts.db')) as conn:
+            cursor = conn.cursor()
+            cursor.execute("""
+                SELECT k.keyword, COUNT(DISTINCT pk.prompt_id) as prompt_count 
+                FROM Keywords k
+                LEFT JOIN PromptKeywords pk ON k.id = pk.keyword_id
+                GROUP BY k.id, k.keyword
+                ORDER BY k.keyword
+            """)
+
+            keywords = cursor.fetchall()
+            if keywords:
+                keyword_list = [f"- {k[0]} ({k[1]} prompts)" for k in keywords]
+                return "### Current Prompt Keywords:\n" + "\n".join(keyword_list)
+            return "No keywords found."
+
+    except Exception as e:
+        error_msg = f"Error retrieving keywords: {str(e)}"
+        logging.error(error_msg)
+        return error_msg
+
+
+def export_prompts(
+        export_format='csv',
+        filter_keywords=None,
+        include_system=True,
+        include_user=True,
+        include_details=True,
+        include_author=True,
+        include_keywords=True,
+        markdown_template=None
+) -> Tuple[str, str]:
+    """
+    Export prompts to CSV or Markdown with configurable options.
+
+    Args:
+        export_format (str): 'csv' or 'markdown'
+        filter_keywords (List[str], optional): Keywords to filter prompts by
+        include_system (bool): Include system prompts in export
+        include_user (bool): Include user prompts in export
+        include_details (bool): Include prompt details/descriptions
+        include_author (bool): Include author information
+        include_keywords (bool): Include associated keywords
+        markdown_template (str, optional): Template for markdown export
+
+    Returns:
+        Tuple[str, str]: (status_message, file_path)
+    """
+    import csv
+    import tempfile
+    import os
+    import zipfile
+    from datetime import datetime
+
+    try:
+        # Get prompts data
+        with get_prompt_db_connection() as conn:
+            cursor = conn.cursor()
+
+            # Build query based on included fields
+            select_fields = ['p.name']
+            if include_author:
+                select_fields.append('p.author')
+            if include_details:
+                select_fields.append('p.details')
+            if include_system:
+                select_fields.append('p.system')
+            if include_user:
+                select_fields.append('p.user')
+
+            query = f"""
+                SELECT DISTINCT {', '.join(select_fields)}
+                FROM Prompts p
+            """
+
+            # Add keyword filtering if specified
+            if filter_keywords:
+                placeholders = ','.join(['?' for _ in filter_keywords])
+                query += f"""
+                    JOIN PromptKeywords pk ON p.id = pk.prompt_id
+                    JOIN Keywords k ON pk.keyword_id = k.id
+                    WHERE k.keyword IN ({placeholders})
+                """
+
+            cursor.execute(query, filter_keywords if filter_keywords else ())
+            prompts = cursor.fetchall()
+
+            # Get keywords for each prompt if needed
+            if include_keywords:
+                prompt_keywords = {}
+                for prompt in prompts:
+                    cursor.execute("""
+                        SELECT k.keyword
+                        FROM Keywords k
+                        JOIN PromptKeywords pk ON k.id = pk.keyword_id
+                        JOIN Prompts p ON pk.prompt_id = p.id
+                        WHERE p.name = ?
+                    """, (prompt[0],))
+                    prompt_keywords[prompt[0]] = [row[0] for row in cursor.fetchall()]
+
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+
+        if export_format == 'csv':
+            # Export as CSV
+            temp_file = os.path.join(tempfile.gettempdir(), f'prompts_export_{timestamp}.csv')
+            with open(temp_file, 'w', newline='', encoding='utf-8') as csvfile:
+                writer = csv.writer(csvfile)
+
+                # Write header
+                header = ['Name']
+                if include_author:
+                    header.append('Author')
+                if include_details:
+                    header.append('Details')
+                if include_system:
+                    header.append('System Prompt')
+                if include_user:
+                    header.append('User Prompt')
+                if include_keywords:
+                    header.append('Keywords')
+                writer.writerow(header)
+
+                # Write data
+                for prompt in prompts:
+                    row = list(prompt)
+                    if include_keywords:
+                        row.append(', '.join(prompt_keywords.get(prompt[0], [])))
+                    writer.writerow(row)
+
+            return f"Successfully exported {len(prompts)} prompts to CSV.", temp_file
+
+        else:
+            # Export as Markdown files in ZIP
+            temp_dir = tempfile.mkdtemp()
+            zip_path = os.path.join(tempfile.gettempdir(), f'prompts_export_{timestamp}.zip')
+
+            # Define markdown templates
+            templates = {
+                "Basic Template": """# {title}
+{author_section}
+{details_section}
+{system_section}
+{user_section}
+{keywords_section}
+""",
+                "Detailed Template": """# {title}
+
+## Author
+{author_section}
+
+## Description
+{details_section}
+
+## System Prompt
+{system_section}
+
+## User Prompt
+{user_section}
+
+## Keywords
+{keywords_section}
+"""
+            }
+
+            template = templates.get(markdown_template, markdown_template or templates["Basic Template"])
+
+            with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
+                for prompt in prompts:
+                    # Create markdown content
+                    md_content = template.format(
+                        title=prompt[0],
+                        author_section=f"Author: {prompt[1]}" if include_author else "",
+                        details_section=prompt[2] if include_details else "",
+                        system_section=prompt[3] if include_system else "",
+                        user_section=prompt[4] if include_user else "",
+                        keywords_section=', '.join(prompt_keywords.get(prompt[0], [])) if include_keywords else ""
+                    )
+
+                    # Create safe filename
+                    safe_filename = re.sub(r'[^\w\-_\. ]', '_', prompt[0])
+                    md_path = os.path.join(temp_dir, f"{safe_filename}.md")
+
+                    # Write markdown file
+                    with open(md_path, 'w', encoding='utf-8') as f:
+                        f.write(md_content)
+
+                    # Add to ZIP
+                    zipf.write(md_path, os.path.basename(md_path))
+
+            return f"Successfully exported {len(prompts)} prompts to Markdown files.", zip_path
+
+    except Exception as e:
+        error_msg = f"Error exporting prompts: {str(e)}"
+        logging.error(error_msg)
+        return error_msg, "None"
+
+
+create_prompts_db()
+
+#
+# End of Propmts_DB.py
+#######################################################################################################################
+

App_Function_Libraries/DB/RAG_QA_Chat_DB.py

@@ -4,39 +4,37 @@
 # Imports
 import configparser
 import logging
+import os
 import re
 import sqlite3
 import uuid
 from contextlib import contextmanager
 from datetime import datetime
-
-from App_Function_Libraries.Utils.Utils import get_project_relative_path, get_database_path
-
+from pathlib import Path
+from typing import List, Dict, Any, Tuple, Optional
 #
 # External Imports
 # (No external imports)
 #
 # Local Imports
-# (No additional local imports)
+from App_Function_Libraries.Utils.Utils import get_project_relative_path, get_project_root
+
 #
 ########################################################################################################################
 #
 # Functions:
 
-# Construct the path to the config file
-config_path = get_project_relative_path('Config_Files/config.txt')
-
-# Read the config file
-config = configparser.ConfigParser()
-config.read(config_path)
-
-# Get the SQLite path from the config, or use the default if not specified
-if config.has_section('Database') and config.has_option('Database', 'rag_qa_db_path'):
-    rag_qa_db_path = config.get('Database', 'rag_qa_db_path')
-else:
-    rag_qa_db_path = get_database_path('RAG_QA_Chat.db')
-
-print(f"RAG QA Chat Database path: {rag_qa_db_path}")
+def get_rag_qa_db_path():
+    config_path = os.path.join(get_project_root(), 'Config_Files', 'config.txt')
+    config = configparser.ConfigParser()
+    config.read(config_path)
+    if config.has_section('Database') and config.has_option('Database', 'rag_qa_db_path'):
+        rag_qa_db_path = config.get('Database', 'rag_qa_db_path')
+        if not os.path.isabs(rag_qa_db_path):
+            rag_qa_db_path = get_project_relative_path(rag_qa_db_path)
+        return rag_qa_db_path
+    else:
+        raise ValueError("Database path not found in config file")
 
 # Set up logging
 logging.basicConfig(level=logging.INFO)
@@ -58,7 +56,9 @@ CREATE TABLE IF NOT EXISTS conversation_metadata (
     conversation_id TEXT PRIMARY KEY,
     created_at DATETIME NOT NULL,
     last_updated DATETIME NOT NULL,
-    title TEXT NOT NULL
+    title TEXT NOT NULL,
+    media_id INTEGER,
+    rating INTEGER CHECK(rating BETWEEN 1 AND 3)
 );
 
 -- Table for storing keywords
@@ -122,19 +122,137 @@ CREATE INDEX IF NOT EXISTS idx_rag_qa_keyword_collections_parent_id ON rag_qa_ke
 CREATE INDEX IF NOT EXISTS idx_rag_qa_collection_keywords_collection_id ON rag_qa_collection_keywords(collection_id);
 CREATE INDEX IF NOT EXISTS idx_rag_qa_collection_keywords_keyword_id ON rag_qa_collection_keywords(keyword_id);
 
--- Full-text search virtual table for chat content
-CREATE VIRTUAL TABLE IF NOT EXISTS rag_qa_chats_fts USING fts5(conversation_id, timestamp, role, content);
+-- Full-text search virtual tables
+CREATE VIRTUAL TABLE IF NOT EXISTS rag_qa_chats_fts USING fts5(
+    content,
+    content='rag_qa_chats',
+    content_rowid='id'
+);
+
+-- FTS table for conversation metadata
+CREATE VIRTUAL TABLE IF NOT EXISTS conversation_metadata_fts USING fts5(
+    title,
+    content='conversation_metadata',
+    content_rowid='rowid'
+);
+
+-- FTS table for keywords
+CREATE VIRTUAL TABLE IF NOT EXISTS rag_qa_keywords_fts USING fts5(
+    keyword,
+    content='rag_qa_keywords',
+    content_rowid='id'
+);
+
+-- FTS table for keyword collections
+CREATE VIRTUAL TABLE IF NOT EXISTS rag_qa_keyword_collections_fts USING fts5(
+    name,
+    content='rag_qa_keyword_collections',
+    content_rowid='id'
+);
+
+-- FTS table for notes
+CREATE VIRTUAL TABLE IF NOT EXISTS rag_qa_notes_fts USING fts5(
+    title,
+    content,
+    content='rag_qa_notes',
+    content_rowid='id'
+);
+-- FTS table for notes (modified to include both title and content)
+CREATE VIRTUAL TABLE IF NOT EXISTS rag_qa_notes_fts USING fts5(
+    title,
+    content,
+    content='rag_qa_notes',
+    content_rowid='id'
+);
 
--- Trigger to keep the FTS table up to date
+-- Triggers for maintaining FTS indexes
+-- Triggers for rag_qa_chats
 CREATE TRIGGER IF NOT EXISTS rag_qa_chats_ai AFTER INSERT ON rag_qa_chats BEGIN
-  INSERT INTO rag_qa_chats_fts(conversation_id, timestamp, role, content) VALUES (new.conversation_id, new.timestamp, new.role, new.content);
+    INSERT INTO rag_qa_chats_fts(rowid, content) 
+    VALUES (new.id, new.content);
+END;
+
+CREATE TRIGGER IF NOT EXISTS rag_qa_chats_au AFTER UPDATE ON rag_qa_chats BEGIN
+    UPDATE rag_qa_chats_fts 
+    SET content = new.content
+    WHERE rowid = old.id;
+END;
+
+CREATE TRIGGER IF NOT EXISTS rag_qa_chats_ad AFTER DELETE ON rag_qa_chats BEGIN
+    DELETE FROM rag_qa_chats_fts WHERE rowid = old.id;
+END;
+
+-- Triggers for conversation_metadata
+CREATE TRIGGER IF NOT EXISTS conversation_metadata_ai AFTER INSERT ON conversation_metadata BEGIN
+    INSERT INTO conversation_metadata_fts(rowid, title) 
+    VALUES (new.rowid, new.title);
+END;
+
+CREATE TRIGGER IF NOT EXISTS conversation_metadata_au AFTER UPDATE ON conversation_metadata BEGIN
+    UPDATE conversation_metadata_fts 
+    SET title = new.title
+    WHERE rowid = old.rowid;
+END;
+
+CREATE TRIGGER IF NOT EXISTS conversation_metadata_ad AFTER DELETE ON conversation_metadata BEGIN
+    DELETE FROM conversation_metadata_fts WHERE rowid = old.rowid;
+END;
+
+-- Triggers for rag_qa_keywords
+CREATE TRIGGER IF NOT EXISTS rag_qa_keywords_ai AFTER INSERT ON rag_qa_keywords BEGIN
+    INSERT INTO rag_qa_keywords_fts(rowid, keyword) 
+    VALUES (new.id, new.keyword);
+END;
+
+CREATE TRIGGER IF NOT EXISTS rag_qa_keywords_au AFTER UPDATE ON rag_qa_keywords BEGIN
+    UPDATE rag_qa_keywords_fts 
+    SET keyword = new.keyword
+    WHERE rowid = old.id;
+END;
+
+CREATE TRIGGER IF NOT EXISTS rag_qa_keywords_ad AFTER DELETE ON rag_qa_keywords BEGIN
+    DELETE FROM rag_qa_keywords_fts WHERE rowid = old.id;
+END;
+
+-- Triggers for rag_qa_keyword_collections
+CREATE TRIGGER IF NOT EXISTS rag_qa_keyword_collections_ai AFTER INSERT ON rag_qa_keyword_collections BEGIN
+    INSERT INTO rag_qa_keyword_collections_fts(rowid, name) 
+    VALUES (new.id, new.name);
+END;
+
+CREATE TRIGGER IF NOT EXISTS rag_qa_keyword_collections_au AFTER UPDATE ON rag_qa_keyword_collections BEGIN
+    UPDATE rag_qa_keyword_collections_fts 
+    SET name = new.name
+    WHERE rowid = old.id;
+END;
+
+CREATE TRIGGER IF NOT EXISTS rag_qa_keyword_collections_ad AFTER DELETE ON rag_qa_keyword_collections BEGIN
+    DELETE FROM rag_qa_keyword_collections_fts WHERE rowid = old.id;
+END;
+
+-- Triggers for rag_qa_notes
+CREATE TRIGGER IF NOT EXISTS rag_qa_notes_ai AFTER INSERT ON rag_qa_notes BEGIN
+    INSERT INTO rag_qa_notes_fts(rowid, title, content) 
+    VALUES (new.id, new.title, new.content);
+END;
+
+CREATE TRIGGER IF NOT EXISTS rag_qa_notes_au AFTER UPDATE ON rag_qa_notes BEGIN
+    UPDATE rag_qa_notes_fts 
+    SET title = new.title,
+        content = new.content
+    WHERE rowid = old.id;
+END;
+
+CREATE TRIGGER IF NOT EXISTS rag_qa_notes_ad AFTER DELETE ON rag_qa_notes BEGIN
+    DELETE FROM rag_qa_notes_fts WHERE rowid = old.id;
 END;
 '''
 
 # Database connection management
 @contextmanager
 def get_db_connection():
-    conn = sqlite3.connect(rag_qa_db_path)
+    db_path = get_rag_qa_db_path()
+    conn = sqlite3.connect(db_path)
     try:
         yield conn
     finally:
@@ -168,10 +286,43 @@ def execute_query(query, params=None, conn=None):
             conn.commit()
             return cursor.fetchall()
 
+
 def create_tables():
+    """Create database tables and initialize FTS indexes."""
     with get_db_connection() as conn:
-        conn.executescript(SCHEMA_SQL)
-    logger.info("All RAG QA Chat tables created successfully")
+        cursor = conn.cursor()
+        # Execute the SCHEMA_SQL to create tables and triggers
+        cursor.executescript(SCHEMA_SQL)
+
+        # Check and populate all FTS tables
+        fts_tables = [
+            ('rag_qa_notes_fts', 'rag_qa_notes', ['title', 'content']),
+            ('rag_qa_chats_fts', 'rag_qa_chats', ['content']),
+            ('conversation_metadata_fts', 'conversation_metadata', ['title']),
+            ('rag_qa_keywords_fts', 'rag_qa_keywords', ['keyword']),
+            ('rag_qa_keyword_collections_fts', 'rag_qa_keyword_collections', ['name'])
+        ]
+
+        for fts_table, source_table, columns in fts_tables:
+            # Check if FTS table needs population
+            cursor.execute(f"SELECT COUNT(*) FROM {fts_table}")
+            fts_count = cursor.fetchone()[0]
+            cursor.execute(f"SELECT COUNT(*) FROM {source_table}")
+            source_count = cursor.fetchone()[0]
+
+            if fts_count != source_count:
+                # Repopulate FTS table
+                logger.info(f"Repopulating {fts_table}")
+                cursor.execute(f"DELETE FROM {fts_table}")
+                columns_str = ', '.join(columns)
+                source_columns = ', '.join([f"id" if source_table != 'conversation_metadata' else "rowid"] + columns)
+                cursor.execute(f"""
+                    INSERT INTO {fts_table}(rowid, {columns_str})
+                    SELECT {source_columns} FROM {source_table}
+                """)
+
+        logger.info("All RAG QA Chat tables and triggers created successfully")
+
 
 # Initialize the database
 create_tables()
@@ -197,6 +348,7 @@ def validate_keyword(keyword):
         raise ValueError("Keyword contains invalid characters")
     return keyword.strip()
 
+
 def validate_collection_name(name):
     if not isinstance(name, str):
         raise ValueError("Collection name must be a string")
@@ -208,6 +360,7 @@ def validate_collection_name(name):
         raise ValueError("Collection name contains invalid characters")
     return name.strip()
 
+
 # Core functions
 def add_keyword(keyword, conn=None):
     try:
@@ -222,6 +375,7 @@ def add_keyword(keyword, conn=None):
         logger.error(f"Error adding keyword '{keyword}': {e}")
         raise
 
+
 def create_keyword_collection(name, parent_id=None):
     try:
         validated_name = validate_collection_name(name)
@@ -235,6 +389,7 @@ def create_keyword_collection(name, parent_id=None):
         logger.error(f"Error creating keyword collection '{name}': {e}")
         raise
 
+
 def add_keyword_to_collection(collection_name, keyword):
     try:
         validated_collection_name = validate_collection_name(collection_name)
@@ -259,6 +414,7 @@ def add_keyword_to_collection(collection_name, keyword):
         logger.error(f"Error adding keyword '{keyword}' to collection '{collection_name}': {e}")
         raise
 
+
 def add_keywords_to_conversation(conversation_id, keywords):
     if not isinstance(keywords, (list, tuple)):
         raise ValueError("Keywords must be a list or tuple")
@@ -282,6 +438,23 @@ def add_keywords_to_conversation(conversation_id, keywords):
         logger.error(f"Error adding keywords to conversation '{conversation_id}': {e}")
         raise
 
+
+def view_rag_keywords():
+    try:
+        rag_db_path = get_rag_qa_db_path()
+        with sqlite3.connect(rag_db_path) as conn:
+            cursor = conn.cursor()
+            cursor.execute("SELECT keyword FROM rag_qa_keywords ORDER BY keyword")
+            keywords = cursor.fetchall()
+            if keywords:
+                keyword_list = [k[0] for k in keywords]
+                return "### Current RAG QA Keywords:\n" + "\n".join(
+                    [f"- {k}" for k in keyword_list])
+            return "No keywords found."
+    except Exception as e:
+        return f"Error retrieving keywords: {str(e)}"
+
+
 def get_keywords_for_conversation(conversation_id):
     try:
         query = '''
@@ -298,6 +471,7 @@ def get_keywords_for_conversation(conversation_id):
         logger.error(f"Error getting keywords for conversation '{conversation_id}': {e}")
         raise
 
+
 def get_keywords_for_collection(collection_name):
     try:
         query = '''
@@ -315,6 +489,116 @@ def get_keywords_for_collection(collection_name):
         logger.error(f"Error getting keywords for collection '{collection_name}': {e}")
         raise
 
+
+def delete_rag_keyword(keyword: str) -> str:
+    """
+    Delete a keyword from the RAG QA database and all its associations.
+
+    Args:
+        keyword (str): The keyword to delete
+
+    Returns:
+        str: Success/failure message
+    """
+    try:
+        # Validate the keyword
+        validated_keyword = validate_keyword(keyword)
+
+        with transaction() as conn:
+            # First, get the keyword ID
+            cursor = conn.cursor()
+            cursor.execute("SELECT id FROM rag_qa_keywords WHERE keyword = ?", (validated_keyword,))
+            result = cursor.fetchone()
+
+            if not result:
+                return f"Keyword '{validated_keyword}' not found."
+
+            keyword_id = result[0]
+
+            # Delete from all associated tables
+            cursor.execute("DELETE FROM rag_qa_conversation_keywords WHERE keyword_id = ?", (keyword_id,))
+            cursor.execute("DELETE FROM rag_qa_collection_keywords WHERE keyword_id = ?", (keyword_id,))
+            cursor.execute("DELETE FROM rag_qa_note_keywords WHERE keyword_id = ?", (keyword_id,))
+
+            # Finally, delete the keyword itself
+            cursor.execute("DELETE FROM rag_qa_keywords WHERE id = ?", (keyword_id,))
+
+            logger.info(f"Keyword '{validated_keyword}' deleted successfully")
+            return f"Successfully deleted keyword '{validated_keyword}' and all its associations."
+
+    except ValueError as e:
+        error_msg = f"Invalid keyword: {str(e)}"
+        logger.error(error_msg)
+        return error_msg
+    except Exception as e:
+        error_msg = f"Error deleting keyword: {str(e)}"
+        logger.error(error_msg)
+        return error_msg
+
+
+def export_rag_keywords_to_csv() -> Tuple[str, str]:
+    """
+    Export all RAG QA keywords to a CSV file.
+
+    Returns:
+        Tuple[str, str]: (status_message, file_path)
+    """
+    import csv
+    from tempfile import NamedTemporaryFile
+    from datetime import datetime
+
+    try:
+        # Create a temporary CSV file
+        temp_file = NamedTemporaryFile(mode='w+', delete=False, suffix='.csv', newline='')
+
+        with transaction() as conn:
+            cursor = conn.cursor()
+
+            # Get all keywords and their associations
+            query = """
+            SELECT 
+                k.keyword,
+                GROUP_CONCAT(DISTINCT c.name) as collections,
+                COUNT(DISTINCT ck.conversation_id) as num_conversations,
+                COUNT(DISTINCT nk.note_id) as num_notes
+            FROM rag_qa_keywords k
+            LEFT JOIN rag_qa_collection_keywords col_k ON k.id = col_k.keyword_id
+            LEFT JOIN rag_qa_keyword_collections c ON col_k.collection_id = c.id
+            LEFT JOIN rag_qa_conversation_keywords ck ON k.id = ck.keyword_id
+            LEFT JOIN rag_qa_note_keywords nk ON k.id = nk.keyword_id
+            GROUP BY k.id, k.keyword
+            ORDER BY k.keyword
+            """
+
+            cursor.execute(query)
+            results = cursor.fetchall()
+
+            # Write to CSV
+            writer = csv.writer(temp_file)
+            writer.writerow(['Keyword', 'Collections', 'Number of Conversations', 'Number of Notes'])
+
+            for row in results:
+                writer.writerow([
+                    row[0],  # keyword
+                    row[1] if row[1] else '',  # collections (may be None)
+                    row[2],  # num_conversations
+                    row[3]  # num_notes
+                ])
+
+        temp_file.close()
+
+        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
+        status_msg = f"Successfully exported {len(results)} keywords to CSV."
+        logger.info(status_msg)
+
+        return status_msg, temp_file.name
+
+    except Exception as e:
+        error_msg = f"Error exporting keywords: {str(e)}"
+        logger.error(error_msg)
+        return error_msg, ""
+
+
 #
 # End of Keyword-related functions
 ###################################################
@@ -339,6 +623,7 @@ def save_notes(conversation_id, title, content):
         logger.error(f"Error saving notes for conversation '{conversation_id}': {e}")
         raise
 
+
 def update_note(note_id, title, content):
     try:
         query = "UPDATE rag_qa_notes SET title = ?, content = ?, timestamp = ? WHERE id = ?"
@@ -349,6 +634,121 @@ def update_note(note_id, title, content):
         logger.error(f"Error updating note ID '{note_id}': {e}")
         raise
 
+
+def search_notes_titles(search_term: str, page: int = 1, results_per_page: int = 20, connection=None) -> Tuple[
+    List[Tuple], int, int]:
+    """
+    Search note titles using full-text search. Returns all notes if search_term is empty.
+
+    Args:
+        search_term (str): The search term for note titles. If empty, returns all notes.
+        page (int, optional): Page number for pagination. Defaults to 1.
+        results_per_page (int, optional): Number of results per page. Defaults to 20.
+        connection (sqlite3.Connection, optional): Database connection. Uses new connection if not provided.
+
+    Returns:
+        Tuple[List[Tuple], int, int]: Tuple containing:
+            - List of tuples: (note_id, title, content, timestamp, conversation_id)
+            - Total number of pages
+            - Total count of matching records
+
+    Raises:
+        ValueError: If page number is less than 1
+        sqlite3.Error: If there's a database error
+    """
+    if page < 1:
+        raise ValueError("Page number must be 1 or greater.")
+
+    offset = (page - 1) * results_per_page
+
+    def execute_search(conn):
+        cursor = conn.cursor()
+
+        # Debug: Show table contents
+        cursor.execute("SELECT title FROM rag_qa_notes")
+        main_titles = cursor.fetchall()
+        logger.debug(f"Main table titles: {main_titles}")
+
+        cursor.execute("SELECT title FROM rag_qa_notes_fts")
+        fts_titles = cursor.fetchall()
+        logger.debug(f"FTS table titles: {fts_titles}")
+
+        if not search_term.strip():
+            # Query for all notes
+            cursor.execute(
+                """
+                SELECT COUNT(*)
+                FROM rag_qa_notes
+                """
+            )
+            total_count = cursor.fetchone()[0]
+
+            cursor.execute(
+                """
+                SELECT id, title, content, timestamp, conversation_id
+                FROM rag_qa_notes
+                ORDER BY timestamp DESC
+                LIMIT ? OFFSET ?
+                """,
+                (results_per_page, offset)
+            )
+            results = cursor.fetchall()
+        else:
+            # Search query
+            search_term_clean = search_term.strip().lower()
+
+            # Test direct FTS search
+            cursor.execute(
+                """
+                SELECT COUNT(*)
+                FROM rag_qa_notes n
+                JOIN rag_qa_notes_fts fts ON n.id = fts.rowid
+                WHERE fts.title MATCH ?
+                """,
+                (search_term_clean,)
+            )
+            total_count = cursor.fetchone()[0]
+
+            cursor.execute(
+                """
+                SELECT 
+                    n.id,
+                    n.title,
+                    n.content,
+                    n.timestamp,
+                    n.conversation_id
+                FROM rag_qa_notes n
+                JOIN rag_qa_notes_fts fts ON n.id = fts.rowid
+                WHERE fts.title MATCH ?
+                ORDER BY rank
+                LIMIT ? OFFSET ?
+                """,
+                (search_term_clean, results_per_page, offset)
+            )
+            results = cursor.fetchall()
+
+            logger.debug(f"Search term: {search_term_clean}")
+            logger.debug(f"Results: {results}")
+
+        total_pages = max(1, (total_count + results_per_page - 1) // results_per_page)
+        logger.info(f"Found {total_count} matching notes for search term '{search_term}'")
+
+        return results, total_pages, total_count
+
+    try:
+        if connection:
+            return execute_search(connection)
+        else:
+            with get_db_connection() as conn:
+                return execute_search(conn)
+
+    except sqlite3.Error as e:
+        logger.error(f"Database error in search_notes_titles: {str(e)}")
+        logger.error(f"Search term: {search_term}")
+        raise sqlite3.Error(f"Error searching notes: {str(e)}")
+
+
+
 def get_notes(conversation_id):
     """Retrieve notes for a given conversation."""
     try:
@@ -361,6 +761,7 @@ def get_notes(conversation_id):
         logger.error(f"Error getting notes for conversation '{conversation_id}': {e}")
         raise
 
+
 def get_note_by_id(note_id):
     try:
         query = "SELECT id, title, content FROM rag_qa_notes WHERE id = ?"
@@ -370,9 +771,21 @@ def get_note_by_id(note_id):
         logger.error(f"Error getting note by ID '{note_id}': {e}")
         raise
 
+
 def get_notes_by_keywords(keywords, page=1, page_size=20):
     try:
-        placeholders = ','.join(['?'] * len(keywords))
+        # Handle empty or invalid keywords
+        if not keywords or not isinstance(keywords, (list, tuple)) or len(keywords) == 0:
+            return [], 0, 0
+
+        # Convert all keywords to strings and strip them
+        clean_keywords = [str(k).strip() for k in keywords if k is not None and str(k).strip()]
+
+        # If no valid keywords after cleaning, return empty result
+        if not clean_keywords:
+            return [], 0, 0
+
+        placeholders = ','.join(['?'] * len(clean_keywords))
         query = f'''
         SELECT n.id, n.title, n.content, n.timestamp
         FROM rag_qa_notes n
@@ -381,14 +794,15 @@ def get_notes_by_keywords(keywords, page=1, page_size=20):
         WHERE k.keyword IN ({placeholders})
         ORDER BY n.timestamp DESC
         '''
-        results, total_pages, total_count = get_paginated_results(query, tuple(keywords), page, page_size)
-        logger.info(f"Retrieved {len(results)} notes matching keywords: {', '.join(keywords)} (page {page} of {total_pages})")
+        results, total_pages, total_count = get_paginated_results(query, tuple(clean_keywords), page, page_size)
+        logger.info(f"Retrieved {len(results)} notes matching keywords: {', '.join(clean_keywords)} (page {page} of {total_pages})")
         notes = [(row[0], row[1], row[2], row[3]) for row in results]
         return notes, total_pages, total_count
     except Exception as e:
         logger.error(f"Error getting notes by keywords: {e}")
         raise
 
+
 def get_notes_by_keyword_collection(collection_name, page=1, page_size=20):
     try:
         query = '''
@@ -501,9 +915,10 @@ def delete_note(note_id):
 #
 # Chat-related functions
 
-def save_message(conversation_id, role, content):
+def save_message(conversation_id, role, content, timestamp=None):
     try:
-        timestamp = datetime.now().isoformat()
+        if timestamp is None:
+            timestamp = datetime.now().isoformat()
         query = "INSERT INTO rag_qa_chats (conversation_id, timestamp, role, content) VALUES (?, ?, ?, ?)"
         execute_query(query, (conversation_id, timestamp, role, content))
 
@@ -516,29 +931,103 @@ def save_message(conversation_id, role, content):
         logger.error(f"Error saving message for conversation '{conversation_id}': {e}")
         raise
 
-def start_new_conversation(title="Untitled Conversation"):
+
+def start_new_conversation(title="Untitled Conversation", media_id=None):
     try:
         conversation_id = str(uuid.uuid4())
-        query = "INSERT INTO conversation_metadata (conversation_id, created_at, last_updated, title) VALUES (?, ?, ?, ?)"
+        query = """
+            INSERT INTO conversation_metadata (
+                conversation_id, created_at, last_updated, title, media_id, rating
+            ) VALUES (?, ?, ?, ?, ?, ?)
+        """
         now = datetime.now().isoformat()
-        execute_query(query, (conversation_id, now, now, title))
-        logger.info(f"New conversation '{conversation_id}' started with title '{title}'")
+        # Set initial rating to NULL
+        execute_query(query, (conversation_id, now, now, title, media_id, None))
+        logger.info(f"New conversation '{conversation_id}' started with title '{title}' and media_id '{media_id}'")
         return conversation_id
     except Exception as e:
         logger.error(f"Error starting new conversation: {e}")
         raise
 
+
 def get_all_conversations(page=1, page_size=20):
     try:
-        query = "SELECT conversation_id, title FROM conversation_metadata ORDER BY last_updated DESC"
-        results, total_pages, total_count = get_paginated_results(query, page=page, page_size=page_size)
-        conversations = [(row[0], row[1]) for row in results]
-        logger.info(f"Retrieved {len(conversations)} conversations (page {page} of {total_pages})")
-        return conversations, total_pages, total_count
+        query = """
+            SELECT conversation_id, title, media_id, rating
+            FROM conversation_metadata
+            ORDER BY last_updated DESC
+            LIMIT ? OFFSET ?
+        """
+
+        count_query = "SELECT COUNT(*) FROM conversation_metadata"
+        db_path = get_rag_qa_db_path()
+        with sqlite3.connect(db_path) as conn:
+            cursor = conn.cursor()
+
+            # Get total count
+            cursor.execute(count_query)
+            total_count = cursor.fetchone()[0]
+            total_pages = (total_count + page_size - 1) // page_size
+
+            # Get page of results
+            offset = (page - 1) * page_size
+            cursor.execute(query, (page_size, offset))
+            results = cursor.fetchall()
+
+            conversations = [{
+                'conversation_id': row[0],
+                'title': row[1],
+                'media_id': row[2],
+                'rating': row[3]  # Include rating
+            } for row in results]
+            return conversations, total_pages, total_count
+    except Exception as e:
+        logging.error(f"Error getting conversations: {e}")
+        raise
+
+
+def get_all_notes(page=1, page_size=20):
+    try:
+        query = """
+        SELECT n.id, n.conversation_id, n.title, n.content, n.timestamp,
+               cm.title as conversation_title, cm.media_id
+        FROM rag_qa_notes n
+        LEFT JOIN conversation_metadata cm ON n.conversation_id = cm.conversation_id
+        ORDER BY n.timestamp DESC
+        LIMIT ? OFFSET ?
+        """
+
+        count_query = "SELECT COUNT(*) FROM rag_qa_notes"
+        db_path = get_rag_qa_db_path()
+        with sqlite3.connect(db_path) as conn:
+            cursor = conn.cursor()
+
+            # Get total count
+            cursor.execute(count_query)
+            total_count = cursor.fetchone()[0]
+            total_pages = (total_count + page_size - 1) // page_size
+
+            # Get page of results
+            offset = (page - 1) * page_size
+            cursor.execute(query, (page_size, offset))
+            results = cursor.fetchall()
+
+            notes = [{
+                'id': row[0],
+                'conversation_id': row[1],
+                'title': row[2],
+                'content': row[3],
+                'timestamp': row[4],
+                'conversation_title': row[5],
+                'media_id': row[6]
+            } for row in results]
+
+        return notes, total_pages, total_count
     except Exception as e:
-        logger.error(f"Error getting conversations: {e}")
+        logging.error(f"Error getting notes: {e}")
         raise
 
+
 # Pagination helper function
 def get_paginated_results(query, params=None, page=1, page_size=20):
     try:
@@ -564,6 +1053,7 @@ def get_paginated_results(query, params=None, page=1, page_size=20):
         logger.error(f"Error retrieving paginated results: {e}")
         raise
 
+
 def get_all_collections(page=1, page_size=20):
     try:
         query = "SELECT name FROM rag_qa_keyword_collections"
@@ -575,24 +1065,79 @@ def get_all_collections(page=1, page_size=20):
         logger.error(f"Error getting collections: {e}")
         raise
 
-def search_conversations_by_keywords(keywords, page=1, page_size=20):
+
+def search_conversations_by_keywords(keywords=None, title_query=None, content_query=None, page=1, page_size=20):
     try:
-        placeholders = ','.join(['?' for _ in keywords])
-        query = f'''
-        SELECT DISTINCT cm.conversation_id, cm.title
+        # Base query starts with conversation metadata
+        query = """
+        SELECT DISTINCT cm.conversation_id, cm.title, cm.last_updated
         FROM conversation_metadata cm
-        JOIN rag_qa_conversation_keywords ck ON cm.conversation_id = ck.conversation_id
-        JOIN rag_qa_keywords k ON ck.keyword_id = k.id
-        WHERE k.keyword IN ({placeholders})
-        '''
-        results, total_pages, total_count = get_paginated_results(query, tuple(keywords), page, page_size)
-        logger.info(
-            f"Found {total_count} conversations matching keywords: {', '.join(keywords)} (page {page} of {total_pages})")
-        return results, total_pages, total_count
+        WHERE 1=1
+        """
+        params = []
+
+        # Add content search if provided
+        if content_query and isinstance(content_query, str) and content_query.strip():
+            query += """
+            AND EXISTS (
+                SELECT 1 FROM rag_qa_chats_fts
+                WHERE rag_qa_chats_fts.content MATCH ?
+                AND rag_qa_chats_fts.rowid IN (
+                    SELECT id FROM rag_qa_chats 
+                    WHERE conversation_id = cm.conversation_id
+                )
+            )
+            """
+            params.append(content_query.strip())
+
+        # Add title search if provided
+        if title_query and isinstance(title_query, str) and title_query.strip():
+            query += """
+            AND EXISTS (
+                SELECT 1 FROM conversation_metadata_fts
+                WHERE conversation_metadata_fts.title MATCH ?
+                AND conversation_metadata_fts.rowid = cm.rowid
+            )
+            """
+            params.append(title_query.strip())
+
+        # Add keyword search if provided
+        if keywords and isinstance(keywords, (list, tuple)) and len(keywords) > 0:
+            # Convert all keywords to strings and strip them
+            clean_keywords = [str(k).strip() for k in keywords if k is not None and str(k).strip()]
+            if clean_keywords:  # Only add to query if we have valid keywords
+                placeholders = ','.join(['?' for _ in clean_keywords])
+                query += f"""
+                AND EXISTS (
+                    SELECT 1 FROM rag_qa_conversation_keywords ck
+                    JOIN rag_qa_keywords k ON ck.keyword_id = k.id
+                    WHERE ck.conversation_id = cm.conversation_id
+                    AND k.keyword IN ({placeholders})
+                )
+                """
+                params.extend(clean_keywords)
+
+        # Add ordering
+        query += " ORDER BY cm.last_updated DESC"
+
+        results, total_pages, total_count = get_paginated_results(query, tuple(params), page, page_size)
+
+        conversations = [
+            {
+                'conversation_id': row[0],
+                'title': row[1],
+                'last_updated': row[2]
+            }
+            for row in results
+        ]
+
+        return conversations, total_pages, total_count
+
     except Exception as e:
-        logger.error(f"Error searching conversations by keywords {keywords}: {e}")
+        logger.error(f"Error searching conversations: {e}")
         raise
 
+
 def load_chat_history(conversation_id, page=1, page_size=50):
     try:
         query = "SELECT role, content FROM rag_qa_chats WHERE conversation_id = ? ORDER BY timestamp"
@@ -604,6 +1149,7 @@ def load_chat_history(conversation_id, page=1, page_size=50):
         logger.error(f"Error loading chat history for conversation '{conversation_id}': {e}")
         raise
 
+
 def update_conversation_title(conversation_id, new_title):
     """Update the title of a conversation."""
     try:
@@ -614,6 +1160,59 @@ def update_conversation_title(conversation_id, new_title):
         logger.error(f"Error updating conversation title: {e}")
         raise
 
+
+def delete_messages_in_conversation(conversation_id):
+    """Helper function to delete all messages in a conversation."""
+    try:
+        execute_query("DELETE FROM rag_qa_chats WHERE conversation_id = ?", (conversation_id,))
+        logging.info(f"Messages in conversation '{conversation_id}' deleted successfully.")
+    except Exception as e:
+        logging.error(f"Error deleting messages in conversation '{conversation_id}': {e}")
+        raise
+
+
+def get_conversation_title(conversation_id):
+    """Helper function to get the conversation title."""
+    query = "SELECT title FROM conversation_metadata WHERE conversation_id = ?"
+    result = execute_query(query, (conversation_id,))
+    if result:
+        return result[0][0]
+    else:
+        return "Untitled Conversation"
+
+
+def get_conversation_text(conversation_id):
+    try:
+        query = """
+        SELECT role, content
+        FROM rag_qa_chats
+        WHERE conversation_id = ?
+        ORDER BY timestamp ASC
+        """
+
+        messages = []
+        # Use the connection as a context manager
+        db_path = get_rag_qa_db_path()
+        with sqlite3.connect(db_path) as conn:
+            cursor = conn.cursor()
+            cursor.execute(query, (conversation_id,))
+            messages = cursor.fetchall()
+
+        return "\n\n".join([f"{msg[0]}: {msg[1]}" for msg in messages])
+    except Exception as e:
+        logger.error(f"Error getting conversation text: {e}")
+        raise
+
+
+def get_conversation_details(conversation_id):
+    query = "SELECT title, media_id, rating FROM conversation_metadata WHERE conversation_id = ?"
+    result = execute_query(query, (conversation_id,))
+    if result:
+        return {'title': result[0][0], 'media_id': result[0][1], 'rating': result[0][2]}
+    else:
+        return {'title': "Untitled Conversation", 'media_id': None, 'rating': None}
+
+
 def delete_conversation(conversation_id):
     """Delete a conversation and its associated messages and notes."""
     try:
@@ -633,11 +1232,203 @@ def delete_conversation(conversation_id):
         logger.error(f"Error deleting conversation '{conversation_id}': {e}")
         raise
 
+def set_conversation_rating(conversation_id, rating):
+    """Set the rating for a conversation."""
+    # Validate rating
+    if rating not in [1, 2, 3]:
+        raise ValueError('Rating must be an integer between 1 and 3.')
+    try:
+        query = "UPDATE conversation_metadata SET rating = ? WHERE conversation_id = ?"
+        execute_query(query, (rating, conversation_id))
+        logger.info(f"Rating for conversation '{conversation_id}' set to {rating}")
+    except Exception as e:
+        logger.error(f"Error setting rating for conversation '{conversation_id}': {e}")
+        raise
+
+def get_conversation_rating(conversation_id):
+    """Get the rating of a conversation."""
+    try:
+        query = "SELECT rating FROM conversation_metadata WHERE conversation_id = ?"
+        result = execute_query(query, (conversation_id,))
+        if result:
+            rating = result[0][0]
+            logger.info(f"Rating for conversation '{conversation_id}' is {rating}")
+            return rating
+        else:
+            logger.warning(f"Conversation '{conversation_id}' not found.")
+            return None
+    except Exception as e:
+        logger.error(f"Error getting rating for conversation '{conversation_id}': {e}")
+        raise
+
+
+def get_conversation_name(conversation_id: str) -> str:
+    """
+    Retrieves the title/name of a conversation from the conversation_metadata table.
+
+    Args:
+        conversation_id (str): The unique identifier of the conversation
+
+    Returns:
+        str: The title of the conversation if found, "Untitled Conversation" if not found
+
+    Raises:
+        sqlite3.Error: If there's a database error
+    """
+    try:
+        with get_db_connection() as conn:
+            cursor = conn.cursor()
+            cursor.execute(
+                "SELECT title FROM conversation_metadata WHERE conversation_id = ?",
+                (conversation_id,)
+            )
+            result = cursor.fetchone()
+
+            if result:
+                return result[0]
+            else:
+                logging.warning(f"No conversation found with ID: {conversation_id}")
+                return "Untitled Conversation"
+
+    except sqlite3.Error as e:
+        logging.error(f"Database error retrieving conversation name for ID {conversation_id}: {e}")
+        raise
+    except Exception as e:
+        logging.error(f"Unexpected error retrieving conversation name for ID {conversation_id}: {e}")
+        raise
+
+
+def search_rag_chat(query: str, fts_top_k: int = 10, relevant_media_ids: List[str] = None) -> List[Dict[str, Any]]:
+    """
+    Perform a full-text search on the RAG Chat database.
+
+    Args:
+        query: Search query string.
+        fts_top_k: Maximum number of results to return.
+        relevant_media_ids: Optional list of media IDs to filter results.
+
+    Returns:
+        List of search results with content and metadata.
+    """
+    if not query.strip():
+        return []
+
+    try:
+        db_path = get_rag_qa_db_path()
+        with sqlite3.connect(db_path) as conn:
+            cursor = conn.cursor()
+            # Perform the full-text search using the FTS virtual table
+            cursor.execute("""
+                SELECT rag_qa_chats.id, rag_qa_chats.conversation_id, rag_qa_chats.role, rag_qa_chats.content
+                FROM rag_qa_chats_fts
+                JOIN rag_qa_chats ON rag_qa_chats_fts.rowid = rag_qa_chats.id
+                WHERE rag_qa_chats_fts MATCH ?
+                LIMIT ?
+            """, (query, fts_top_k))
+
+            rows = cursor.fetchall()
+            columns = [description[0] for description in cursor.description]
+            results = [dict(zip(columns, row)) for row in rows]
+
+            # Filter by relevant_media_ids if provided
+            if relevant_media_ids is not None:
+                results = [
+                    r for r in results
+                    if get_conversation_details(r['conversation_id']).get('media_id') in relevant_media_ids
+                ]
+
+            # Format results
+            formatted_results = [
+                {
+                    "content": r['content'],
+                    "metadata": {
+                        "conversation_id": r['conversation_id'],
+                        "role": r['role'],
+                        "media_id": get_conversation_details(r['conversation_id']).get('media_id')
+                    }
+                }
+                for r in results
+            ]
+            return formatted_results
+
+    except Exception as e:
+        logging.error(f"Error in search_rag_chat: {e}")
+        return []
+
+
+def search_rag_notes(query: str, fts_top_k: int = 10, relevant_media_ids: List[str] = None) -> List[Dict[str, Any]]:
+    """
+    Perform a full-text search on the RAG Notes database.
+
+    Args:
+        query: Search query string.
+        fts_top_k: Maximum number of results to return.
+        relevant_media_ids: Optional list of media IDs to filter results.
+
+    Returns:
+        List of search results with content and metadata.
+    """
+    if not query.strip():
+        return []
+
+    try:
+        db_path = get_rag_qa_db_path()
+        with sqlite3.connect(db_path) as conn:
+            cursor = conn.cursor()
+            # Perform the full-text search using the FTS virtual table
+            cursor.execute("""
+                SELECT rag_qa_notes.id, rag_qa_notes.title, rag_qa_notes.content, rag_qa_notes.conversation_id
+                FROM rag_qa_notes_fts
+                JOIN rag_qa_notes ON rag_qa_notes_fts.rowid = rag_qa_notes.id
+                WHERE rag_qa_notes_fts MATCH ?
+                LIMIT ?
+            """, (query, fts_top_k))
+
+            rows = cursor.fetchall()
+            columns = [description[0] for description in cursor.description]
+            results = [dict(zip(columns, row)) for row in rows]
+
+            # Filter by relevant_media_ids if provided
+            if relevant_media_ids is not None:
+                results = [
+                    r for r in results
+                    if get_conversation_details(r['conversation_id']).get('media_id') in relevant_media_ids
+                ]
+
+            # Format results
+            formatted_results = [
+                {
+                    "content": r['content'],
+                    "metadata": {
+                        "note_id": r['id'],
+                        "title": r['title'],
+                        "conversation_id": r['conversation_id'],
+                        "media_id": get_conversation_details(r['conversation_id']).get('media_id')
+                    }
+                }
+                for r in results
+            ]
+            return formatted_results
+
+    except Exception as e:
+        logging.error(f"Error in search_rag_notes: {e}")
+        return []
+
 #
 # End of Chat-related functions
 ###################################################
 
 
+###################################################
+#
+# Import functions
+
+
+#
+# End of Import functions
+###################################################
+
+
 ###################################################
 #
 # Functions to export DB data

App_Function_Libraries/DB/SQLite_DB.py

@@ -21,7 +21,7 @@ import configparser
 # 11. browse_items(search_query, search_type)
 # 12. fetch_item_details(media_id: int)
 # 13. add_media_version(media_id: int, prompt: str, summary: str)
-# 14. search_db(search_query: str, search_fields: List[str], keywords: str, page: int = 1, results_per_page: int = 10)
+# 14. search_media_db(search_query: str, search_fields: List[str], keywords: str, page: int = 1, results_per_page: int = 10)
 # 15. search_and_display(search_query, search_fields, keywords, page)
 # 16. display_details(index, results)
 # 17. get_details(index, dataframe)
@@ -55,12 +55,14 @@ import re
 import shutil
 import sqlite3
 import threading
+import time
 import traceback
 from contextlib import contextmanager
 from datetime import datetime, timedelta
 from typing import List, Tuple, Dict, Any, Optional
 from urllib.parse import quote
 
+from App_Function_Libraries.Metrics.metrics_logger import log_counter, log_histogram
 # Local Libraries
 from App_Function_Libraries.Utils.Utils import get_project_relative_path, get_database_path, \
     get_database_dir
@@ -342,27 +344,6 @@ def create_tables(db) -> None:
         )
         ''',
         '''
-        CREATE TABLE IF NOT EXISTS ChatConversations (
-            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            media_id INTEGER,
-            media_name TEXT,
-            conversation_name TEXT,
-            created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
-            updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
-            FOREIGN KEY (media_id) REFERENCES Media(id)
-        )
-        ''',
-        '''
-        CREATE TABLE IF NOT EXISTS ChatMessages (
-            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            conversation_id INTEGER,
-            sender TEXT,
-            message TEXT,
-            timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
-            FOREIGN KEY (conversation_id) REFERENCES ChatConversations(id)
-        )
-        ''',
-        '''
         CREATE TABLE IF NOT EXISTS Transcripts (
             id INTEGER PRIMARY KEY AUTOINCREMENT,
             media_id INTEGER,
@@ -421,8 +402,6 @@ def create_tables(db) -> None:
         'CREATE INDEX IF NOT EXISTS idx_mediakeywords_keyword_id ON MediaKeywords(keyword_id)',
         'CREATE INDEX IF NOT EXISTS idx_media_version_media_id ON MediaVersion(media_id)',
         'CREATE INDEX IF NOT EXISTS idx_mediamodifications_media_id ON MediaModifications(media_id)',
-        'CREATE INDEX IF NOT EXISTS idx_chatconversations_media_id ON ChatConversations(media_id)',
-        'CREATE INDEX IF NOT EXISTS idx_chatmessages_conversation_id ON ChatMessages(conversation_id)',
         'CREATE INDEX IF NOT EXISTS idx_media_is_trash ON Media(is_trash)',
         'CREATE INDEX IF NOT EXISTS idx_mediachunks_media_id ON MediaChunks(media_id)',
         'CREATE INDEX IF NOT EXISTS idx_unvectorized_media_chunks_media_id ON UnvectorizedMediaChunks(media_id)',
@@ -606,7 +585,10 @@ def mark_media_as_processed(database, media_id):
 # Function to add media with keywords
 def add_media_with_keywords(url, title, media_type, content, keywords, prompt, summary, transcription_model, author,
                             ingestion_date):
+    log_counter("add_media_with_keywords_attempt")
+    start_time = time.time()
     logging.debug(f"Entering add_media_with_keywords: URL={url}, Title={title}")
+
     # Set default values for missing fields
     if url is None:
         url = 'localhost'
@@ -622,10 +604,17 @@ def add_media_with_keywords(url, title, media_type, content, keywords, prompt, s
     author = author or 'Unknown'
     ingestion_date = ingestion_date or datetime.now().strftime('%Y-%m-%d')
 
-    if media_type not in ['article', 'audio', 'document', 'mediawiki_article', 'mediawiki_dump', 'obsidian_note', 'podcast', 'text', 'video', 'unknown']:
-        raise InputError("Invalid media type. Allowed types: article, audio file, document, obsidian_note podcast, text, video, unknown.")
+    if media_type not in ['article', 'audio', 'book', 'document', 'mediawiki_article', 'mediawiki_dump',
+                          'obsidian_note', 'podcast', 'text', 'video', 'unknown']:
+        log_counter("add_media_with_keywords_error", labels={"error_type": "InvalidMediaType"})
+        duration = time.time() - start_time
+        log_histogram("add_media_with_keywords_duration", duration)
+        raise InputError("Invalid media type. Allowed types: article, audio file, document, obsidian_note, podcast, text, video, unknown.")
 
     if ingestion_date and not is_valid_date(ingestion_date):
+        log_counter("add_media_with_keywords_error", labels={"error_type": "InvalidDateFormat"})
+        duration = time.time() - start_time
+        log_histogram("add_media_with_keywords_duration", duration)
         raise InputError("Invalid ingestion date format. Use YYYY-MM-DD.")
 
     # Handle keywords as either string or list
@@ -654,6 +643,7 @@ def add_media_with_keywords(url, title, media_type, content, keywords, prompt, s
             logging.debug(f"Existing media ID for {url}: {existing_media_id}")
 
             if existing_media_id:
+                # Update existing media
                 media_id = existing_media_id
                 logging.debug(f"Updating existing media with ID: {media_id}")
                 cursor.execute('''
@@ -661,7 +651,9 @@ def add_media_with_keywords(url, title, media_type, content, keywords, prompt, s
                 SET content = ?, transcription_model = ?, type = ?, author = ?, ingestion_date = ?
                 WHERE id = ?
                 ''', (content, transcription_model, media_type, author, ingestion_date, media_id))
+                log_counter("add_media_with_keywords_update")
             else:
+                # Insert new media
                 logging.debug("Inserting new media")
                 cursor.execute('''
                 INSERT INTO Media (url, title, type, content, author, ingestion_date, transcription_model)
@@ -669,6 +661,7 @@ def add_media_with_keywords(url, title, media_type, content, keywords, prompt, s
                 ''', (url, title, media_type, content, author, ingestion_date, transcription_model))
                 media_id = cursor.lastrowid
                 logging.debug(f"New media inserted with ID: {media_id}")
+                log_counter("add_media_with_keywords_insert")
 
             cursor.execute('''
             INSERT INTO MediaModifications (media_id, prompt, summary, modification_date)
@@ -698,13 +691,23 @@ def add_media_with_keywords(url, title, media_type, content, keywords, prompt, s
             conn.commit()
             logging.info(f"Media '{title}' successfully added/updated with ID: {media_id}")
 
-        return media_id, f"Media '{title}' added/updated successfully with keywords: {', '.join(keyword_list)}"
+            duration = time.time() - start_time
+            log_histogram("add_media_with_keywords_duration", duration)
+            log_counter("add_media_with_keywords_success")
+
+            return media_id, f"Media '{title}' added/updated successfully with keywords: {', '.join(keyword_list)}"
 
     except sqlite3.Error as e:
         logging.error(f"SQL Error in add_media_with_keywords: {e}")
+        duration = time.time() - start_time
+        log_histogram("add_media_with_keywords_duration", duration)
+        log_counter("add_media_with_keywords_error", labels={"error_type": "SQLiteError"})
         raise DatabaseError(f"Error adding media with keywords: {e}")
     except Exception as e:
         logging.error(f"Unexpected Error in add_media_with_keywords: {e}")
+        duration = time.time() - start_time
+        log_histogram("add_media_with_keywords_duration", duration)
+        log_counter("add_media_with_keywords_error", labels={"error_type": type(e).__name__})
         raise DatabaseError(f"Unexpected error: {e}")
 
 
@@ -779,7 +782,13 @@ def ingest_article_to_db(url, title, author, content, keywords, summary, ingesti
 
 # Function to add a keyword
 def add_keyword(keyword: str) -> int:
+    log_counter("add_keyword_attempt")
+    start_time = time.time()
+
     if not keyword.strip():
+        log_counter("add_keyword_error", labels={"error_type": "EmptyKeyword"})
+        duration = time.time() - start_time
+        log_histogram("add_keyword_duration", duration)
         raise DatabaseError("Keyword cannot be empty")
 
     keyword = keyword.strip().lower()
@@ -801,18 +810,32 @@ def add_keyword(keyword: str) -> int:
 
             logging.info(f"Keyword '{keyword}' added or updated with ID: {keyword_id}")
             conn.commit()
+
+            duration = time.time() - start_time
+            log_histogram("add_keyword_duration", duration)
+            log_counter("add_keyword_success")
+
             return keyword_id
         except sqlite3.IntegrityError as e:
             logging.error(f"Integrity error adding keyword: {e}")
+            duration = time.time() - start_time
+            log_histogram("add_keyword_duration", duration)
+            log_counter("add_keyword_error", labels={"error_type": "IntegrityError"})
             raise DatabaseError(f"Integrity error adding keyword: {e}")
         except sqlite3.Error as e:
             logging.error(f"Error adding keyword: {e}")
+            duration = time.time() - start_time
+            log_histogram("add_keyword_duration", duration)
+            log_counter("add_keyword_error", labels={"error_type": "SQLiteError"})
             raise DatabaseError(f"Error adding keyword: {e}")
 
 
 
 # Function to delete a keyword
 def delete_keyword(keyword: str) -> str:
+    log_counter("delete_keyword_attempt")
+    start_time = time.time()
+
     keyword = keyword.strip().lower()
     with db.get_connection() as conn:
         cursor = conn.cursor()
@@ -823,10 +846,23 @@ def delete_keyword(keyword: str) -> str:
                 cursor.execute('DELETE FROM Keywords WHERE keyword = ?', (keyword,))
                 cursor.execute('DELETE FROM keyword_fts WHERE rowid = ?', (keyword_id[0],))
                 conn.commit()
+
+                duration = time.time() - start_time
+                log_histogram("delete_keyword_duration", duration)
+                log_counter("delete_keyword_success")
+
                 return f"Keyword '{keyword}' deleted successfully."
             else:
+                duration = time.time() - start_time
+                log_histogram("delete_keyword_duration", duration)
+                log_counter("delete_keyword_not_found")
+
                 return f"Keyword '{keyword}' not found."
         except sqlite3.Error as e:
+            duration = time.time() - start_time
+            log_histogram("delete_keyword_duration", duration)
+            log_counter("delete_keyword_error", labels={"error_type": type(e).__name__})
+            logging.error(f"Error deleting keyword: {e}")
             raise DatabaseError(f"Error deleting keyword: {e}")
 
 
@@ -1000,7 +1036,7 @@ def add_media_version(conn, media_id: int, prompt: str, summary: str) -> None:
 
 
 # Function to search the database with advanced options, including keyword search and full-text search
-def sqlite_search_db(search_query: str, search_fields: List[str], keywords: str, page: int = 1, results_per_page: int = 10, connection=None):
+def search_media_db(search_query: str, search_fields: List[str], keywords: str, page: int = 1, results_per_page: int = 20, connection=None):
     if page < 1:
         raise ValueError("Page number must be 1 or greater.")
 
@@ -1055,7 +1091,7 @@ def sqlite_search_db(search_query: str, search_fields: List[str], keywords: str,
 
 # Gradio function to handle user input and display results with pagination, with better feedback
 def search_and_display(search_query, search_fields, keywords, page):
-    results = sqlite_search_db(search_query, search_fields, keywords, page)
+    results = search_media_db(search_query, search_fields, keywords, page)
 
     if isinstance(results, pd.DataFrame):
         # Convert DataFrame to a list of tuples or lists
@@ -1133,7 +1169,7 @@ def format_results(results):
 # Function to export search results to CSV or markdown with pagination
 def export_to_file(search_query: str, search_fields: List[str], keyword: str, page: int = 1, results_per_file: int = 1000, export_format: str = 'csv'):
     try:
-        results = sqlite_search_db(search_query, search_fields, keyword, page, results_per_file)
+        results = search_media_db(search_query, search_fields, keyword, page, results_per_file)
         if not results:
             return "No results found to export."
 
@@ -1381,303 +1417,6 @@ def schedule_chunking(media_id: int, content: str, media_name: str):
 #######################################################################################################################
 
 
-#######################################################################################################################
-#
-# Functions to manage prompts DB
-
-def create_prompts_db():
-    logging.debug("create_prompts_db: Creating prompts database.")
-    with sqlite3.connect(get_database_path('prompts.db')) as conn:
-        cursor = conn.cursor()
-        cursor.executescript('''
-            CREATE TABLE IF NOT EXISTS Prompts (
-                id INTEGER PRIMARY KEY AUTOINCREMENT,
-                name TEXT NOT NULL UNIQUE,
-                author TEXT,
-                details TEXT,
-                system TEXT,
-                user TEXT
-            );
-            CREATE TABLE IF NOT EXISTS Keywords (
-                id INTEGER PRIMARY KEY AUTOINCREMENT,
-                keyword TEXT NOT NULL UNIQUE COLLATE NOCASE
-            );
-            CREATE TABLE IF NOT EXISTS PromptKeywords (
-                prompt_id INTEGER,
-                keyword_id INTEGER,
-                FOREIGN KEY (prompt_id) REFERENCES Prompts (id),
-                FOREIGN KEY (keyword_id) REFERENCES Keywords (id),
-                PRIMARY KEY (prompt_id, keyword_id)
-            );
-            CREATE INDEX IF NOT EXISTS idx_keywords_keyword ON Keywords(keyword);
-            CREATE INDEX IF NOT EXISTS idx_promptkeywords_prompt_id ON PromptKeywords(prompt_id);
-            CREATE INDEX IF NOT EXISTS idx_promptkeywords_keyword_id ON PromptKeywords(keyword_id);
-        ''')
-
-# FIXME - dirty hack that should be removed later...
-# Migration function to add the 'author' column to the Prompts table
-def add_author_column_to_prompts():
-    with sqlite3.connect(get_database_path('prompts.db')) as conn:
-        cursor = conn.cursor()
-        # Check if 'author' column already exists
-        cursor.execute("PRAGMA table_info(Prompts)")
-        columns = [col[1] for col in cursor.fetchall()]
-
-        if 'author' not in columns:
-            # Add the 'author' column
-            cursor.execute('ALTER TABLE Prompts ADD COLUMN author TEXT')
-            print("Author column added to Prompts table.")
-        else:
-            print("Author column already exists in Prompts table.")
-
-add_author_column_to_prompts()
-
-def normalize_keyword(keyword):
-    return re.sub(r'\s+', ' ', keyword.strip().lower())
-
-
-# FIXME - update calls to this function to use the new args
-def add_prompt(name, author, details, system=None, user=None, keywords=None):
-    logging.debug(f"add_prompt: Adding prompt with name: {name}, author: {author}, system: {system}, user: {user}, keywords: {keywords}")
-    if not name:
-        logging.error("add_prompt: A name is required.")
-        return "A name is required."
-
-    try:
-        with sqlite3.connect(get_database_path('prompts.db')) as conn:
-            cursor = conn.cursor()
-            cursor.execute('''
-                INSERT INTO Prompts (name, author, details, system, user)
-                VALUES (?, ?, ?, ?, ?)
-            ''', (name, author, details, system, user))
-            prompt_id = cursor.lastrowid
-
-            if keywords:
-                normalized_keywords = [normalize_keyword(k) for k in keywords if k.strip()]
-                for keyword in set(normalized_keywords):  # Use set to remove duplicates
-                    cursor.execute('''
-                        INSERT OR IGNORE INTO Keywords (keyword) VALUES (?)
-                    ''', (keyword,))
-                    cursor.execute('SELECT id FROM Keywords WHERE keyword = ?', (keyword,))
-                    keyword_id = cursor.fetchone()[0]
-                    cursor.execute('''
-                        INSERT OR IGNORE INTO PromptKeywords (prompt_id, keyword_id) VALUES (?, ?)
-                    ''', (prompt_id, keyword_id))
-        return "Prompt added successfully."
-    except sqlite3.IntegrityError:
-        return "Prompt with this name already exists."
-    except sqlite3.Error as e:
-        return f"Database error: {e}"
-
-
-def fetch_prompt_details(name):
-    logging.debug(f"fetch_prompt_details: Fetching details for prompt: {name}")
-    with sqlite3.connect(get_database_path('prompts.db')) as conn:
-        cursor = conn.cursor()
-        cursor.execute('''
-            SELECT p.name, p.author, p.details, p.system, p.user, GROUP_CONCAT(k.keyword, ', ') as keywords
-            FROM Prompts p
-            LEFT JOIN PromptKeywords pk ON p.id = pk.prompt_id
-            LEFT JOIN Keywords k ON pk.keyword_id = k.id
-            WHERE p.name = ?
-            GROUP BY p.id
-        ''', (name,))
-        return cursor.fetchone()
-
-
-def list_prompts(page=1, per_page=10):
-    logging.debug(f"list_prompts: Listing prompts for page {page} with {per_page} prompts per page.")
-    offset = (page - 1) * per_page
-    with sqlite3.connect(get_database_path('prompts.db')) as conn:
-        cursor = conn.cursor()
-        cursor.execute('SELECT name FROM Prompts LIMIT ? OFFSET ?', (per_page, offset))
-        prompts = [row[0] for row in cursor.fetchall()]
-
-        # Get total count of prompts
-        cursor.execute('SELECT COUNT(*) FROM Prompts')
-        total_count = cursor.fetchone()[0]
-
-    total_pages = (total_count + per_page - 1) // per_page
-    return prompts, total_pages, page
-
-# This will not scale. For a large number of prompts, use a more efficient method.
-# FIXME - see above statement.
-def load_preset_prompts():
-    logging.debug("load_preset_prompts: Loading preset prompts.")
-    try:
-        with sqlite3.connect(get_database_path('prompts.db')) as conn:
-            cursor = conn.cursor()
-            cursor.execute('SELECT name FROM Prompts ORDER BY name ASC')
-            prompts = [row[0] for row in cursor.fetchall()]
-        return prompts
-    except sqlite3.Error as e:
-        print(f"Database error: {e}")
-        return []
-
-
-def insert_prompt_to_db(title, author, description, system_prompt, user_prompt, keywords=None):
-    return add_prompt(title, author, description, system_prompt, user_prompt, keywords)
-
-
-def get_prompt_db_connection():
-    prompt_db_path = get_database_path('prompts.db')
-    return sqlite3.connect(prompt_db_path)
-
-
-def search_prompts(query):
-    logging.debug(f"search_prompts: Searching prompts with query: {query}")
-    try:
-        with get_prompt_db_connection() as conn:
-            cursor = conn.cursor()
-            cursor.execute("""
-                SELECT p.name, p.details, p.system, p.user, GROUP_CONCAT(k.keyword, ', ') as keywords
-                FROM Prompts p
-                LEFT JOIN PromptKeywords pk ON p.id = pk.prompt_id
-                LEFT JOIN Keywords k ON pk.keyword_id = k.id
-                WHERE p.name LIKE ? OR p.details LIKE ? OR p.system LIKE ? OR p.user LIKE ? OR k.keyword LIKE ?
-                GROUP BY p.id
-                ORDER BY p.name
-            """, (f'%{query}%', f'%{query}%', f'%{query}%', f'%{query}%', f'%{query}%'))
-            return cursor.fetchall()
-    except sqlite3.Error as e:
-        logging.error(f"Error searching prompts: {e}")
-        return []
-
-
-def search_prompts_by_keyword(keyword, page=1, per_page=10):
-    logging.debug(f"search_prompts_by_keyword: Searching prompts by keyword: {keyword}")
-    normalized_keyword = normalize_keyword(keyword)
-    offset = (page - 1) * per_page
-    with sqlite3.connect(get_database_path('prompts.db')) as conn:
-        cursor = conn.cursor()
-        cursor.execute('''
-            SELECT DISTINCT p.name
-            FROM Prompts p
-            JOIN PromptKeywords pk ON p.id = pk.prompt_id
-            JOIN Keywords k ON pk.keyword_id = k.id
-            WHERE k.keyword LIKE ?
-            LIMIT ? OFFSET ?
-        ''', ('%' + normalized_keyword + '%', per_page, offset))
-        prompts = [row[0] for row in cursor.fetchall()]
-
-        # Get total count of matching prompts
-        cursor.execute('''
-            SELECT COUNT(DISTINCT p.id)
-            FROM Prompts p
-            JOIN PromptKeywords pk ON p.id = pk.prompt_id
-            JOIN Keywords k ON pk.keyword_id = k.id
-            WHERE k.keyword LIKE ?
-        ''', ('%' + normalized_keyword + '%',))
-        total_count = cursor.fetchone()[0]
-
-    total_pages = (total_count + per_page - 1) // per_page
-    return prompts, total_pages, page
-
-
-def update_prompt_keywords(prompt_name, new_keywords):
-    logging.debug(f"update_prompt_keywords: Updating keywords for prompt: {prompt_name}")
-    try:
-        with sqlite3.connect(get_database_path('prompts.db')) as conn:
-            cursor = conn.cursor()
-
-            cursor.execute('SELECT id FROM Prompts WHERE name = ?', (prompt_name,))
-            prompt_id = cursor.fetchone()
-            if not prompt_id:
-                return "Prompt not found."
-            prompt_id = prompt_id[0]
-
-            cursor.execute('DELETE FROM PromptKeywords WHERE prompt_id = ?', (prompt_id,))
-
-            normalized_keywords = [normalize_keyword(k) for k in new_keywords if k.strip()]
-            for keyword in set(normalized_keywords):  # Use set to remove duplicates
-                cursor.execute('INSERT OR IGNORE INTO Keywords (keyword) VALUES (?)', (keyword,))
-                cursor.execute('SELECT id FROM Keywords WHERE keyword = ?', (keyword,))
-                keyword_id = cursor.fetchone()[0]
-                cursor.execute('INSERT INTO PromptKeywords (prompt_id, keyword_id) VALUES (?, ?)',
-                               (prompt_id, keyword_id))
-
-            # Remove unused keywords
-            cursor.execute('''
-                DELETE FROM Keywords
-                WHERE id NOT IN (SELECT DISTINCT keyword_id FROM PromptKeywords)
-            ''')
-        return "Keywords updated successfully."
-    except sqlite3.Error as e:
-        return f"Database error: {e}"
-
-
-def add_or_update_prompt(title, author, description, system_prompt, user_prompt, keywords=None):
-    logging.debug(f"add_or_update_prompt: Adding or updating prompt: {title}")
-    if not title:
-        return "Error: Title is required."
-
-    existing_prompt = fetch_prompt_details(title)
-    if existing_prompt:
-        # Update existing prompt
-        result = update_prompt_in_db(title, author, description, system_prompt, user_prompt)
-        if "successfully" in result:
-            # Update keywords if the prompt update was successful
-            keyword_result = update_prompt_keywords(title, keywords or [])
-            result += f" {keyword_result}"
-    else:
-        # Insert new prompt
-        result = insert_prompt_to_db(title, author, description, system_prompt, user_prompt, keywords)
-
-    return result
-
-
-def load_prompt_details(selected_prompt):
-    logging.debug(f"load_prompt_details: Loading prompt details for {selected_prompt}")
-    if selected_prompt:
-        details = fetch_prompt_details(selected_prompt)
-        if details:
-            return details[0], details[1], details[2], details[3], details[4], details[5]
-    return "", "", "", "", "", ""
-
-
-def update_prompt_in_db(title, author, description, system_prompt, user_prompt):
-    logging.debug(f"update_prompt_in_db: Updating prompt: {title}")
-    try:
-        with sqlite3.connect(get_database_path('prompts.db')) as conn:
-            cursor = conn.cursor()
-            cursor.execute(
-                "UPDATE Prompts SET author = ?, details = ?, system = ?, user = ? WHERE name = ?",
-                (author, description, system_prompt, user_prompt, title)
-            )
-            if cursor.rowcount == 0:
-                return "No prompt found with the given title."
-        return "Prompt updated successfully!"
-    except sqlite3.Error as e:
-        return f"Error updating prompt: {e}"
-
-
-create_prompts_db()
-
-def delete_prompt(prompt_id):
-    logging.debug(f"delete_prompt: Deleting prompt with ID: {prompt_id}")
-    try:
-        with sqlite3.connect(get_database_path('prompts.db')) as conn:
-            cursor = conn.cursor()
-
-            # Delete associated keywords
-            cursor.execute("DELETE FROM PromptKeywords WHERE prompt_id = ?", (prompt_id,))
-
-            # Delete the prompt
-            cursor.execute("DELETE FROM Prompts WHERE id = ?", (prompt_id,))
-
-            if cursor.rowcount == 0:
-                return f"No prompt found with ID {prompt_id}"
-            else:
-                conn.commit()
-                return f"Prompt with ID {prompt_id} has been successfully deleted"
-    except sqlite3.Error as e:
-        return f"An error occurred: {e}"
-
-#
-#
-#######################################################################################################################
-
-
 #######################################################################################################################
 #
 # Function to fetch/update media content
@@ -2020,204 +1759,6 @@ def import_obsidian_note_to_db(note_data):
 #######################################################################################################################
 
 
-#######################################################################################################################
-#
-# Chat-related Functions
-
-
-
-def create_chat_conversation(media_id, conversation_name):
-    try:
-        with db.get_connection() as conn:
-            cursor = conn.cursor()
-            cursor.execute('''
-                INSERT INTO ChatConversations (media_id, conversation_name, created_at, updated_at)
-                VALUES (?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
-            ''', (media_id, conversation_name))
-            conn.commit()
-            return cursor.lastrowid
-    except sqlite3.Error as e:
-        logging.error(f"Error creating chat conversation: {e}")
-        raise DatabaseError(f"Error creating chat conversation: {e}")
-
-
-def add_chat_message(conversation_id: int, sender: str, message: str) -> int:
-    try:
-        with db.get_connection() as conn:
-            cursor = conn.cursor()
-            cursor.execute('''
-                INSERT INTO ChatMessages (conversation_id, sender, message)
-                VALUES (?, ?, ?)
-            ''', (conversation_id, sender, message))
-            conn.commit()
-            return cursor.lastrowid
-    except sqlite3.Error as e:
-        logging.error(f"Error adding chat message: {e}")
-        raise DatabaseError(f"Error adding chat message: {e}")
-
-
-def get_chat_messages(conversation_id: int) -> List[Dict[str, Any]]:
-    try:
-        with db.get_connection() as conn:
-            cursor = conn.cursor()
-            cursor.execute('''
-                SELECT id, sender, message, timestamp
-                FROM ChatMessages
-                WHERE conversation_id = ?
-                ORDER BY timestamp ASC
-            ''', (conversation_id,))
-            messages = cursor.fetchall()
-            return [
-                {
-                    'id': msg[0],
-                    'sender': msg[1],
-                    'message': msg[2],
-                    'timestamp': msg[3]
-                }
-                for msg in messages
-            ]
-    except sqlite3.Error as e:
-        logging.error(f"Error retrieving chat messages: {e}")
-        raise DatabaseError(f"Error retrieving chat messages: {e}")
-
-
-def search_chat_conversations(search_query: str) -> List[Dict[str, Any]]:
-    try:
-        with db.get_connection() as conn:
-            cursor = conn.cursor()
-            cursor.execute('''
-                SELECT cc.id, cc.media_id, cc.conversation_name, cc.created_at, m.title as media_title
-                FROM ChatConversations cc
-                LEFT JOIN Media m ON cc.media_id = m.id
-                WHERE cc.conversation_name LIKE ? OR m.title LIKE ?
-                ORDER BY cc.updated_at DESC
-            ''', (f'%{search_query}%', f'%{search_query}%'))
-            conversations = cursor.fetchall()
-            return [
-                {
-                    'id': conv[0],
-                    'media_id': conv[1],
-                    'conversation_name': conv[2],
-                    'created_at': conv[3],
-                    'media_title': conv[4] or "Unknown Media"
-                }
-                for conv in conversations
-            ]
-    except sqlite3.Error as e:
-        logging.error(f"Error searching chat conversations: {e}")
-        return []
-
-
-def update_chat_message(message_id: int, new_message: str) -> None:
-    try:
-        with db.get_connection() as conn:
-            cursor = conn.cursor()
-            cursor.execute('''
-                UPDATE ChatMessages
-                SET message = ?, timestamp = CURRENT_TIMESTAMP
-                WHERE id = ?
-            ''', (new_message, message_id))
-            conn.commit()
-    except sqlite3.Error as e:
-        logging.error(f"Error updating chat message: {e}")
-        raise DatabaseError(f"Error updating chat message: {e}")
-
-
-def delete_chat_message(message_id: int) -> None:
-    try:
-        with db.get_connection() as conn:
-            cursor = conn.cursor()
-            cursor.execute('DELETE FROM ChatMessages WHERE id = ?', (message_id,))
-            conn.commit()
-    except sqlite3.Error as e:
-        logging.error(f"Error deleting chat message: {e}")
-        raise DatabaseError(f"Error deleting chat message: {e}")
-
-
-def save_chat_history_to_database(chatbot, conversation_id, media_id, media_name, conversation_name):
-    try:
-        with db.get_connection() as conn:
-            cursor = conn.cursor()
-
-            # If conversation_id is None, create a new conversation
-            if conversation_id is None:
-                cursor.execute('''
-                    INSERT INTO ChatConversations (media_id, media_name, conversation_name, created_at, updated_at)
-                    VALUES (?, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
-                ''', (media_id, media_name, conversation_name))
-                conversation_id = cursor.lastrowid
-            else:
-                # If conversation exists, update the media_name
-                cursor.execute('''
-                    UPDATE ChatConversations
-                    SET media_name = ?, updated_at = CURRENT_TIMESTAMP
-                    WHERE id = ?
-                ''', (media_name, conversation_id))
-
-            # Save each message in the chatbot history
-            for i, (user_msg, ai_msg) in enumerate(chatbot):
-                cursor.execute('''
-                    INSERT INTO ChatMessages (conversation_id, sender, message, timestamp)
-                    VALUES (?, ?, ?, CURRENT_TIMESTAMP)
-                ''', (conversation_id, 'user', user_msg))
-
-                cursor.execute('''
-                    INSERT INTO ChatMessages (conversation_id, sender, message, timestamp)
-                    VALUES (?, ?, ?, CURRENT_TIMESTAMP)
-                ''', (conversation_id, 'ai', ai_msg))
-
-            # Update the conversation's updated_at timestamp
-            cursor.execute('''
-                UPDATE ChatConversations
-                SET updated_at = CURRENT_TIMESTAMP
-                WHERE id = ?
-            ''', (conversation_id,))
-
-            conn.commit()
-
-        return conversation_id
-    except Exception as e:
-        logging.error(f"Error saving chat history to database: {str(e)}")
-        raise
-
-
-def get_conversation_name(conversation_id):
-    if conversation_id is None:
-        return None
-
-    try:
-        with sqlite3.connect('media_summary.db') as conn:  # Replace with your actual database name
-            cursor = conn.cursor()
-
-            query = """
-            SELECT conversation_name, media_name
-            FROM ChatConversations
-            WHERE id = ?
-            """
-
-            cursor.execute(query, (conversation_id,))
-            result = cursor.fetchone()
-
-            if result:
-                conversation_name, media_name = result
-                if conversation_name:
-                    return conversation_name
-                elif media_name:
-                    return f"{media_name}-chat"
-
-            return None  # Return None if no result found
-    except sqlite3.Error as e:
-        logging.error(f"Database error in get_conversation_name: {e}")
-        return None
-    except Exception as e:
-        logging.error(f"Unexpected error in get_conversation_name: {e}")
-        return None
-
-#
-# End of Chat-related Functions
-#######################################################################################################################
-
-
 #######################################################################################################################
 #
 # Functions to Compare Transcripts
@@ -2837,29 +2378,42 @@ def process_chunks(database, chunks: List[Dict], media_id: int, batch_size: int
     :param media_id: ID of the media these chunks belong to
     :param batch_size: Number of chunks to process in each batch
     """
+    log_counter("process_chunks_attempt", labels={"media_id": media_id})
+    start_time = time.time()
     total_chunks = len(chunks)
     processed_chunks = 0
 
-    for i in range(0, total_chunks, batch_size):
-        batch = chunks[i:i + batch_size]
-        chunk_data = [
-            (media_id, chunk['text'], chunk['start_index'], chunk['end_index'])
-            for chunk in batch
-        ]
-
-        try:
-            database.execute_many(
-                "INSERT INTO MediaChunks (media_id, chunk_text, start_index, end_index) VALUES (?, ?, ?, ?)",
-                chunk_data
-            )
-            processed_chunks += len(batch)
-            logging.info(f"Processed {processed_chunks}/{total_chunks} chunks for media_id {media_id}")
-        except Exception as e:
-            logging.error(f"Error inserting chunk batch for media_id {media_id}: {e}")
-            # Optionally, you could raise an exception here to stop processing
-            # raise
+    try:
+        for i in range(0, total_chunks, batch_size):
+            batch = chunks[i:i + batch_size]
+            chunk_data = [
+                (media_id, chunk['text'], chunk['start_index'], chunk['end_index'])
+                for chunk in batch
+            ]
 
-    logging.info(f"Finished processing all {total_chunks} chunks for media_id {media_id}")
+            try:
+                database.execute_many(
+                    "INSERT INTO MediaChunks (media_id, chunk_text, start_index, end_index) VALUES (?, ?, ?, ?)",
+                    chunk_data
+                )
+                processed_chunks += len(batch)
+                logging.info(f"Processed {processed_chunks}/{total_chunks} chunks for media_id {media_id}")
+                log_counter("process_chunks_batch_success", labels={"media_id": media_id})
+            except Exception as e:
+                logging.error(f"Error inserting chunk batch for media_id {media_id}: {e}")
+                log_counter("process_chunks_batch_error", labels={"media_id": media_id, "error_type": type(e).__name__})
+                # Optionally, you could raise an exception here to stop processing
+                # raise
+
+            logging.info(f"Finished processing all {total_chunks} chunks for media_id {media_id}")
+            duration = time.time() - start_time
+            log_histogram("process_chunks_duration", duration, labels={"media_id": media_id})
+            log_counter("process_chunks_success", labels={"media_id": media_id})
+    except Exception as e:
+        duration = time.time() - start_time
+        log_histogram("process_chunks_duration", duration, labels={"media_id": media_id})
+        log_counter("process_chunks_error", labels={"media_id": media_id, "error_type": type(e).__name__})
+        logging.error(f"Error processing chunks for media_id {media_id}: {e}")
 
 
 # Usage example:
@@ -2995,46 +2549,48 @@ def update_media_table(db):
 #
 # Workflow Functions
 
+# Workflow Functions
 def save_workflow_chat_to_db(chat_history, workflow_name, conversation_id=None):
-    try:
-        with db.get_connection() as conn:
-            cursor = conn.cursor()
-
-            if conversation_id is None:
-                # Create a new conversation
-                conversation_name = f"{workflow_name}_Workflow_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
-                cursor.execute('''
-                    INSERT INTO ChatConversations (media_id, media_name, conversation_name, created_at, updated_at)
-                    VALUES (NULL, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
-                ''', (workflow_name, conversation_name))
-                conversation_id = cursor.lastrowid
-            else:
-                # Update existing conversation
-                cursor.execute('''
-                    UPDATE ChatConversations
-                    SET updated_at = CURRENT_TIMESTAMP
-                    WHERE id = ?
-                ''', (conversation_id,))
-
-            # Save messages
-            for user_msg, ai_msg in chat_history:
-                if user_msg:
-                    cursor.execute('''
-                        INSERT INTO ChatMessages (conversation_id, sender, message, timestamp)
-                        VALUES (?, 'user', ?, CURRENT_TIMESTAMP)
-                    ''', (conversation_id, user_msg))
-                if ai_msg:
-                    cursor.execute('''
-                        INSERT INTO ChatMessages (conversation_id, sender, message, timestamp)
-                        VALUES (?, 'ai', ?, CURRENT_TIMESTAMP)
-                    ''', (conversation_id, ai_msg))
-
-            conn.commit()
-
-        return conversation_id, f"Chat saved successfully! Conversation ID: {conversation_id}"
-    except Exception as e:
-        logging.error(f"Error saving workflow chat to database: {str(e)}")
-        return None, f"Error saving chat to database: {str(e)}"
+    pass
+#     try:
+#         with db.get_connection() as conn:
+#             cursor = conn.cursor()
+#
+#             if conversation_id is None:
+#                 # Create a new conversation
+#                 conversation_name = f"{workflow_name}_Workflow_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+#                 cursor.execute('''
+#                     INSERT INTO ChatConversations (media_id, media_name, conversation_name, created_at, updated_at)
+#                     VALUES (NULL, ?, ?, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
+#                 ''', (workflow_name, conversation_name))
+#                 conversation_id = cursor.lastrowid
+#             else:
+#                 # Update existing conversation
+#                 cursor.execute('''
+#                     UPDATE ChatConversations
+#                     SET updated_at = CURRENT_TIMESTAMP
+#                     WHERE id = ?
+#                 ''', (conversation_id,))
+#
+#             # Save messages
+#             for user_msg, ai_msg in chat_history:
+#                 if user_msg:
+#                     cursor.execute('''
+#                         INSERT INTO ChatMessages (conversation_id, sender, message, timestamp)
+#                         VALUES (?, 'user', ?, CURRENT_TIMESTAMP)
+#                     ''', (conversation_id, user_msg))
+#                 if ai_msg:
+#                     cursor.execute('''
+#                         INSERT INTO ChatMessages (conversation_id, sender, message, timestamp)
+#                         VALUES (?, 'ai', ?, CURRENT_TIMESTAMP)
+#                     ''', (conversation_id, ai_msg))
+#
+#             conn.commit()
+#
+#         return conversation_id, f"Chat saved successfully! Conversation ID: {conversation_id}"
+#     except Exception as e:
+#         logging.error(f"Error saving workflow chat to database: {str(e)}")
+#         return None, f"Error saving chat to database: {str(e)}"
 
 
 def get_workflow_chat(conversation_id):

App_Function_Libraries/Gradio_Related.py

@@ -1,420 +1,600 @@
-# Gradio_Related.py
-#########################################
-# Gradio UI Functions Library
-# I fucking hate Gradio.
-#
-#########################################
-#
-# Built-In Imports
-import logging
-import os
-import webbrowser
-
-#
-# Import 3rd-Party Libraries
-import gradio as gr
-#
-# Local Imports
-from App_Function_Libraries.DB.DB_Manager import get_db_config
-from App_Function_Libraries.Gradio_UI.Arxiv_tab import create_arxiv_tab
-from App_Function_Libraries.Gradio_UI.Audio_ingestion_tab import create_audio_processing_tab
-from App_Function_Libraries.Gradio_UI.Book_Ingestion_tab import create_import_book_tab
-from App_Function_Libraries.Gradio_UI.Character_Chat_tab import create_character_card_interaction_tab, create_character_chat_mgmt_tab, create_custom_character_card_tab, \
-    create_character_card_validation_tab, create_export_characters_tab
-from App_Function_Libraries.Gradio_UI.Character_interaction_tab import create_narrator_controlled_conversation_tab, \
-    create_multiple_character_chat_tab
-from App_Function_Libraries.Gradio_UI.Chat_ui import create_chat_management_tab, \
-    create_chat_interface_four, create_chat_interface_multi_api, create_chat_interface_stacked, create_chat_interface
-from App_Function_Libraries.Gradio_UI.Config_tab import create_config_editor_tab
-from App_Function_Libraries.Gradio_UI.Explain_summarize_tab import create_summarize_explain_tab
-from App_Function_Libraries.Gradio_UI.Export_Functionality import create_export_tab
-from App_Function_Libraries.Gradio_UI.Backup_Functionality import create_backup_tab, create_view_backups_tab, \
-    create_restore_backup_tab
-from App_Function_Libraries.Gradio_UI.Import_Functionality import create_import_single_prompt_tab, \
-    create_import_obsidian_vault_tab, create_import_item_tab, create_import_multiple_prompts_tab
-from App_Function_Libraries.Gradio_UI.Introduction_tab import create_introduction_tab
-from App_Function_Libraries.Gradio_UI.Keywords import create_view_keywords_tab, create_add_keyword_tab, \
-    create_delete_keyword_tab, create_export_keywords_tab
-from App_Function_Libraries.Gradio_UI.Live_Recording import create_live_recording_tab
-#from App_Function_Libraries.Gradio_UI.Llamafile_tab import create_chat_with_llamafile_tab
-#from App_Function_Libraries.Gradio_UI.MMLU_Pro_tab import create_mmlu_pro_tab
-from App_Function_Libraries.Gradio_UI.Media_edit import create_prompt_clone_tab, create_prompt_edit_tab, \
-    create_media_edit_and_clone_tab, create_media_edit_tab
-from App_Function_Libraries.Gradio_UI.Media_wiki_tab import create_mediawiki_import_tab, create_mediawiki_config_tab
-from App_Function_Libraries.Gradio_UI.PDF_ingestion_tab import create_pdf_ingestion_tab, create_pdf_ingestion_test_tab
-from App_Function_Libraries.Gradio_UI.Plaintext_tab_import import create_plain_text_import_tab
-from App_Function_Libraries.Gradio_UI.Podcast_tab import create_podcast_tab
-from App_Function_Libraries.Gradio_UI.Prompt_Suggestion_tab import create_prompt_suggestion_tab
-from App_Function_Libraries.Gradio_UI.RAG_QA_Chat_tab import create_rag_qa_chat_tab, create_rag_qa_notes_management_tab, \
-    create_rag_qa_chat_management_tab
-from App_Function_Libraries.Gradio_UI.Re_summarize_tab import create_resummary_tab
-from App_Function_Libraries.Gradio_UI.Search_Tab import create_prompt_search_tab, \
-    create_search_summaries_tab, create_search_tab
-from App_Function_Libraries.Gradio_UI.RAG_Chat_tab import create_rag_tab
-from App_Function_Libraries.Gradio_UI.Embeddings_tab import create_embeddings_tab, create_view_embeddings_tab, \
-    create_purge_embeddings_tab
-from App_Function_Libraries.Gradio_UI.Trash import create_view_trash_tab, create_empty_trash_tab, \
-    create_delete_trash_tab, create_search_and_mark_trash_tab
-from App_Function_Libraries.Gradio_UI.Utilities import create_utilities_yt_timestamp_tab, create_utilities_yt_audio_tab, \
-    create_utilities_yt_video_tab
-from App_Function_Libraries.Gradio_UI.Video_transcription_tab import create_video_transcription_tab
-from App_Function_Libraries.Gradio_UI.View_tab import create_manage_items_tab
-from App_Function_Libraries.Gradio_UI.Website_scraping_tab import create_website_scraping_tab
-from App_Function_Libraries.Gradio_UI.Chat_Workflows import chat_workflows_tab
-from App_Function_Libraries.Gradio_UI.View_DB_Items_tab import create_prompt_view_tab, \
-    create_view_all_with_versions_tab, create_viewing_tab
-#
-# Gradio UI Imports
-from App_Function_Libraries.Gradio_UI.Evaluations_Benchmarks_tab import create_geval_tab, create_infinite_bench_tab
-#from App_Function_Libraries.Local_LLM.Local_LLM_huggingface import create_huggingface_tab
-#from App_Function_Libraries.Local_LLM.Local_LLM_ollama import create_ollama_tab
-#
-#######################################################################################################################
-# Function Definitions
-#
-
-
-# Disable Gradio Analytics
-os.environ['GRADIO_ANALYTICS_ENABLED'] = 'False'
-
-
-custom_prompt_input = None
-server_mode = False
-share_public = False
-custom_prompt_summarize_bulleted_notes = ("""
-                    <s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
-                        **Bulleted Note Creation Guidelines**
-
-                        **Headings**:
-                        - Based on referenced topics, not categories like quotes or terms
-                        - Surrounded by **bold** formatting 
-                        - Not listed as bullet points
-                        - No space between headings and list items underneath
-
-                        **Emphasis**:
-                        - **Important terms** set in bold font
-                        - **Text ending in a colon**: also bolded
-
-                        **Review**:
-                        - Ensure adherence to specified format
-                        - Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]
-                    """)
-#
-# End of globals
-#######################################################################################################################
-#
-# Start of Video/Audio Transcription and Summarization Functions
-#
-# Functions:
-# FIXME
-#
-#
-################################################################################################################
-# Functions for Re-Summarization
-#
-# Functions:
-# FIXME
-# End of Re-Summarization Functions
-#
-############################################################################################################################################################################################################################
-#
-# Explain/Summarize This Tab
-#
-# Functions:
-# FIXME
-#
-#
-############################################################################################################################################################################################################################
-#
-# Transcript Comparison Tab
-#
-# Functions:
-# FIXME
-#
-#
-###########################################################################################################################################################################################################################
-#
-# Search Tab
-#
-# Functions:
-# FIXME
-#
-# End of Search Tab Functions
-#
-##############################################################################################################################################################################################################################
-#
-# Llamafile Tab
-#
-# Functions:
-# FIXME
-#
-# End of Llamafile Tab Functions
-##############################################################################################################################################################################################################################
-#
-# Chat Interface Tab Functions
-#
-# Functions:
-# FIXME
-#
-#
-# End of Chat Interface Tab Functions
-################################################################################################################################################################################################################################
-#
-# Media Edit Tab Functions
-# Functions:
-# Fixme
-# create_media_edit_tab():
-##### Trash Tab
-# FIXME
-# Functions:
-#
-# End of Media Edit Tab Functions
-################################################################################################################
-#
-# Import Items Tab Functions
-#
-# Functions:
-#FIXME
-# End of Import Items Tab Functions
-################################################################################################################
-#
-# Export Items Tab Functions
-#
-# Functions:
-# FIXME
-#
-#
-# End of Export Items Tab Functions
-################################################################################################################
-#
-# Keyword Management Tab Functions
-#
-# Functions:
-#  create_view_keywords_tab():
-# FIXME
-#
-# End of Keyword Management Tab Functions
-################################################################################################################
-#
-# Document Editing Tab Functions
-#
-# Functions:
-#   #FIXME
-#
-#
-################################################################################################################
-#
-# Utilities Tab Functions
-# Functions:
-#   create_utilities_yt_video_tab():
-# #FIXME
-
-#
-# End of Utilities Tab Functions
-################################################################################################################
-
-# FIXME - Prompt sample box
-#
-# # Sample data
-# prompts_category_1 = [
-#     "What are the key points discussed in the video?",
-#     "Summarize the main arguments made by the speaker.",
-#     "Describe the conclusions of the study presented."
-# ]
-#
-# prompts_category_2 = [
-#     "How does the proposed solution address the problem?",
-#     "What are the implications of the findings?",
-#     "Can you explain the theory behind the observed phenomenon?"
-# ]
-#
-# all_prompts2 = prompts_category_1 + prompts_category_2
-
-
-def launch_ui(share_public=None, server_mode=False):
-    webbrowser.open_new_tab('http://127.0.0.1:7860/?__theme=dark')
-    share=share_public
-    css = """
-    .result-box {
-        margin-bottom: 20px;
-        border: 1px solid #ddd;
-        padding: 10px;
-    }
-    .result-box.error {
-        border-color: #ff0000;
-        background-color: #ffeeee;
-    }
-    .transcription, .summary {
-        max-height: 800px;
-        overflow-y: auto;
-        border: 1px solid #eee;
-        padding: 10px;
-        margin-top: 10px;
-    }
-    """
-
-    with gr.Blocks(theme='bethecloud/storj_theme',css=css) as iface:
-        gr.HTML(
-            """
-            <script>
-            document.addEventListener('DOMContentLoaded', (event) => {
-                document.body.classList.add('dark');
-                document.querySelector('gradio-app').style.backgroundColor = 'var(--color-background-primary)';
-            });
-            </script>
-            """
-        )
-        db_config = get_db_config()
-        db_type = db_config['type']
-        gr.Markdown(f"# tl/dw: Your LLM-powered Research Multi-tool")
-        gr.Markdown(f"(Using {db_type.capitalize()} Database)")
-        with gr.Tabs():
-            with gr.TabItem("Transcription / Summarization / Ingestion", id="ingestion-grouping", visible=True):
-                with gr.Tabs():
-                    create_video_transcription_tab()
-                    #create_audio_processing_tab()
-                    #create_podcast_tab()
-                    #create_import_book_tab()
-                    ##create_plain_text_import_tab()
-                    #create_website_scraping_tab()
-                    #create_pdf_ingestion_tab()
-                    #create_pdf_ingestion_test_tab()
-                    #create_resummary_tab()
-                    #create_summarize_explain_tab()
-                    #create_live_recording_tab()
-                    #create_arxiv_tab()
-
-            #with gr.TabItem("Text Search", id="text search", visible=True):
-                #create_search_tab()
-                #create_search_summaries_tab()
-
-            #with gr.TabItem("RAG Chat/Search", id="RAG Chat Notes group", visible=True):
-                #create_rag_tab()
-                #create_rag_qa_chat_tab()
-                #create_rag_qa_notes_management_tab()
-                #create_rag_qa_chat_management_tab()
-
-            #with gr.TabItem("Chat with an LLM", id="LLM Chat group", visible=True):
-                #create_chat_interface()
-                #create_chat_interface_stacked()
-                #create_chat_interface_multi_api()
-                #create_chat_interface_four()
-                #create_chat_with_llamafile_tab()
-                #create_chat_management_tab()
-                #chat_workflows_tab()
-
-
-            #with gr.TabItem("Character Chat", id="character chat group", visible=True):
-                #create_character_card_interaction_tab()
-                #create_character_chat_mgmt_tab()
-                #create_custom_character_card_tab()
-                #create_character_card_validation_tab()
-                #create_multiple_character_chat_tab()
-                #create_narrator_controlled_conversation_tab()
-                #create_export_characters_tab()
-
-
-            #with gr.TabItem("View DB Items", id="view db items group", visible=True):
-                # This one works
-                #create_view_all_with_versions_tab()
-                # This one is WIP
-                #create_viewing_tab()
-                #create_prompt_view_tab()
-
-
-            #with gr.TabItem("Prompts", id='view prompts group', visible=True):
-                #create_prompt_view_tab()
-                #create_prompt_search_tab()
-                #create_prompt_edit_tab()
-                #create_prompt_clone_tab()
-                #create_prompt_suggestion_tab()
-
-
-            #with gr.TabItem("Manage / Edit Existing Items", id="manage group", visible=True):
-                #create_media_edit_tab()
-                #create_manage_items_tab()
-                #create_media_edit_and_clone_tab()
-                # FIXME
-                #create_compare_transcripts_tab()
-
-
-            #with gr.TabItem("Embeddings Management", id="embeddings group", visible=True):
-                #create_embeddings_tab()
-                #create_view_embeddings_tab()
-                #create_purge_embeddings_tab()
-
-            #with gr.TabItem("Writing Tools", id="writing_tools group", visible=True):
-                #from App_Function_Libraries.Gradio_UI.Writing_tab import create_document_feedback_tab
-                #create_document_feedback_tab()
-                #from App_Function_Libraries.Gradio_UI.Writing_tab import create_grammar_style_check_tab
-                #create_grammar_style_check_tab()
-                #from App_Function_Libraries.Gradio_UI.Writing_tab import create_tone_adjustment_tab
-                #create_tone_adjustment_tab()
-                #from App_Function_Libraries.Gradio_UI.Writing_tab import create_creative_writing_tab
-                #create_creative_writing_tab()
-                #from App_Function_Libraries.Gradio_UI.Writing_tab import create_mikupad_tab
-                #create_mikupad_tab()
-
-
-            #with gr.TabItem("Keywords", id="keywords group", visible=True):
-                #create_view_keywords_tab()
-                #create_add_keyword_tab()
-                #create_delete_keyword_tab()
-                #create_export_keywords_tab()
-
-            #with gr.TabItem("Import", id="import group", visible=True):
-                #create_import_item_tab()
-                #create_import_obsidian_vault_tab()
-                #create_import_single_prompt_tab()
-                #create_import_multiple_prompts_tab()
-                #create_mediawiki_import_tab()
-                #create_mediawiki_config_tab()
-
-            #with gr.TabItem("Export", id="export group", visible=True):
-                #create_export_tab()
-
-            #with gr.TabItem("Backup Management", id="backup group", visible=True):
-                #create_backup_tab()
-                #create_view_backups_tab()
-                #create_restore_backup_tab()
-
-            #with gr.TabItem("Utilities", id="util group", visible=True):
-                #create_utilities_yt_video_tab()
-                #create_utilities_yt_audio_tab()
-                #create_utilities_yt_timestamp_tab()
-
-            #with gr.TabItem("Local LLM", id="local llm group", visible=True):
-                #create_chat_with_llamafile_tab()
-                #create_ollama_tab()
-                #create_huggingface_tab()
-
-            #with gr.TabItem("Trashcan", id="trashcan group", visible=True):
-                #create_search_and_mark_trash_tab()
-                #create_view_trash_tab()
-                #create_delete_trash_tab()
-                #create_empty_trash_tab()
-
-            #with gr.TabItem("Evaluations", id="eval", visible=True):
-                #create_geval_tab()
-                #create_infinite_bench_tab()
-                # FIXME
-                #create_mmlu_pro_tab()
-
-            #with gr.TabItem("Introduction/Help", id="introduction group", visible=True):
-                #create_introduction_tab()
-
-            #with gr.TabItem("Config Editor", id="config group"):
-                #create_config_editor_tab()
-
-    # Launch the interface
-    server_port_variable = 7860
-    os.environ['GRADIO_ANALYTICS_ENABLED'] = 'False'
-    if share==True:
-        iface.launch(share=True)
-    elif server_mode and not share_public:
-        iface.launch(share=False, server_name="0.0.0.0", server_port=server_port_variable, )
-    else:
-        try:
-            iface.launch(share=False, server_name="0.0.0.0", server_port=server_port_variable, )
-        except Exception as e:
-            logging.error(f"Error launching interface: {str(e)}")
+# Gradio_Related.py
+#########################################
+# Gradio UI Functions Library
+# I fucking hate Gradio.
+#
+#########################################
+#
+# Built-In Imports
+import logging
+import os
+import webbrowser
+#
+# Import 3rd-Party Libraries
+import gradio as gr
+#
+# Local Imports
+from App_Function_Libraries.DB.DB_Manager import get_db_config, backup_dir
+from App_Function_Libraries.DB.RAG_QA_Chat_DB import create_tables
+from App_Function_Libraries.Gradio_UI.Anki_tab import create_anki_validation_tab, create_anki_generator_tab
+from App_Function_Libraries.Gradio_UI.Arxiv_tab import create_arxiv_tab
+from App_Function_Libraries.Gradio_UI.Audio_ingestion_tab import create_audio_processing_tab
+from App_Function_Libraries.Gradio_UI.Backup_RAG_Notes_Character_Chat_tab import create_database_management_interface
+from App_Function_Libraries.Gradio_UI.Book_Ingestion_tab import create_import_book_tab
+from App_Function_Libraries.Gradio_UI.Character_Chat_tab import create_character_card_interaction_tab, create_character_chat_mgmt_tab, create_custom_character_card_tab, \
+    create_character_card_validation_tab, create_export_characters_tab
+from App_Function_Libraries.Gradio_UI.Character_interaction_tab import create_narrator_controlled_conversation_tab, \
+    create_multiple_character_chat_tab
+from App_Function_Libraries.Gradio_UI.Chat_ui import create_chat_interface_four, create_chat_interface_multi_api, \
+    create_chat_interface_stacked, create_chat_interface
+from App_Function_Libraries.Gradio_UI.Config_tab import create_config_editor_tab
+from App_Function_Libraries.Gradio_UI.Explain_summarize_tab import create_summarize_explain_tab
+from App_Function_Libraries.Gradio_UI.Export_Functionality import create_rag_export_tab, create_export_tabs
+#from App_Function_Libraries.Gradio_UI.Backup_Functionality import create_backup_tab, create_view_backups_tab, \
+#    create_restore_backup_tab
+from App_Function_Libraries.Gradio_UI.Import_Functionality import create_import_single_prompt_tab, \
+    create_import_obsidian_vault_tab, create_import_item_tab, create_import_multiple_prompts_tab, \
+    create_conversation_import_tab
+from App_Function_Libraries.Gradio_UI.Introduction_tab import create_introduction_tab
+from App_Function_Libraries.Gradio_UI.Keywords import create_view_keywords_tab, create_add_keyword_tab, \
+    create_delete_keyword_tab, create_export_keywords_tab, create_rag_qa_keywords_tab, create_character_keywords_tab, \
+    create_meta_keywords_tab, create_prompt_keywords_tab
+from App_Function_Libraries.Gradio_UI.Live_Recording import create_live_recording_tab
+from App_Function_Libraries.Gradio_UI.Llamafile_tab import create_chat_with_llamafile_tab
+#from App_Function_Libraries.Gradio_UI.MMLU_Pro_tab import create_mmlu_pro_tab
+from App_Function_Libraries.Gradio_UI.Media_edit import create_prompt_clone_tab, create_prompt_edit_tab, \
+    create_media_edit_and_clone_tab, create_media_edit_tab
+from App_Function_Libraries.Gradio_UI.Media_wiki_tab import create_mediawiki_import_tab, create_mediawiki_config_tab
+from App_Function_Libraries.Gradio_UI.Mind_Map_tab import create_mindmap_tab
+from App_Function_Libraries.Gradio_UI.PDF_ingestion_tab import create_pdf_ingestion_tab, create_pdf_ingestion_test_tab
+from App_Function_Libraries.Gradio_UI.Plaintext_tab_import import create_plain_text_import_tab
+from App_Function_Libraries.Gradio_UI.Podcast_tab import create_podcast_tab
+from App_Function_Libraries.Gradio_UI.Prompt_Suggestion_tab import create_prompt_suggestion_tab
+from App_Function_Libraries.Gradio_UI.RAG_QA_Chat_tab import create_rag_qa_chat_tab, create_rag_qa_notes_management_tab, \
+    create_rag_qa_chat_management_tab
+from App_Function_Libraries.Gradio_UI.Re_summarize_tab import create_resummary_tab
+from App_Function_Libraries.Gradio_UI.Search_Tab import create_prompt_search_tab, \
+    create_search_summaries_tab, create_search_tab
+from App_Function_Libraries.Gradio_UI.RAG_Chat_tab import create_rag_tab
+from App_Function_Libraries.Gradio_UI.Embeddings_tab import create_embeddings_tab, create_view_embeddings_tab, \
+    create_purge_embeddings_tab
+from App_Function_Libraries.Gradio_UI.Semantic_Scholar_tab import create_semantic_scholar_tab
+from App_Function_Libraries.Gradio_UI.Trash import create_view_trash_tab, create_empty_trash_tab, \
+    create_delete_trash_tab, create_search_and_mark_trash_tab
+from App_Function_Libraries.Gradio_UI.Utilities import create_utilities_yt_timestamp_tab, create_utilities_yt_audio_tab, \
+    create_utilities_yt_video_tab
+from App_Function_Libraries.Gradio_UI.Video_transcription_tab import create_video_transcription_tab
+from App_Function_Libraries.Gradio_UI.View_tab import create_manage_items_tab
+from App_Function_Libraries.Gradio_UI.Website_scraping_tab import create_website_scraping_tab
+from App_Function_Libraries.Gradio_UI.Workflows_tab import chat_workflows_tab
+from App_Function_Libraries.Gradio_UI.View_DB_Items_tab import create_view_all_mediadb_with_versions_tab, \
+    create_viewing_mediadb_tab, create_view_all_rag_notes_tab, create_viewing_ragdb_tab, \
+    create_mediadb_keyword_search_tab, create_ragdb_keyword_items_tab
+from App_Function_Libraries.Gradio_UI.Prompts_tab import create_prompt_view_tab, create_prompts_export_tab
+#
+# Gradio UI Imports
+from App_Function_Libraries.Gradio_UI.Evaluations_Benchmarks_tab import create_geval_tab, create_infinite_bench_tab
+from App_Function_Libraries.Gradio_UI.XML_Ingestion_Tab import create_xml_import_tab
+#from App_Function_Libraries.Local_LLM.Local_LLM_huggingface import create_huggingface_tab
+from App_Function_Libraries.Local_LLM.Local_LLM_ollama import create_ollama_tab
+from App_Function_Libraries.Utils.Utils import load_and_log_configs
+
+#
+#######################################################################################################################
+# Function Definitions
+#
+
+
+# Disable Gradio Analytics
+os.environ['GRADIO_ANALYTICS_ENABLED'] = 'False'
+
+
+custom_prompt_input = None
+server_mode = False
+share_public = False
+custom_prompt_summarize_bulleted_notes = ("""
+                    <s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
+                        **Bulleted Note Creation Guidelines**
+
+                        **Headings**:
+                        - Based on referenced topics, not categories like quotes or terms
+                        - Surrounded by **bold** formatting 
+                        - Not listed as bullet points
+                        - No space between headings and list items underneath
+
+                        **Emphasis**:
+                        - **Important terms** set in bold font
+                        - **Text ending in a colon**: also bolded
+
+                        **Review**:
+                        - Ensure adherence to specified format
+                        - Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]
+                    """)
+#
+# End of globals
+#######################################################################################################################
+#
+# Start of Video/Audio Transcription and Summarization Functions
+#
+# Functions:
+# FIXME
+#
+#
+################################################################################################################
+# Functions for Re-Summarization
+#
+# Functions:
+# FIXME
+# End of Re-Summarization Functions
+#
+############################################################################################################################################################################################################################
+#
+# Explain/Summarize This Tab
+#
+# Functions:
+# FIXME
+#
+#
+############################################################################################################################################################################################################################
+#
+# Transcript Comparison Tab
+#
+# Functions:
+# FIXME
+#
+#
+###########################################################################################################################################################################################################################
+#
+# Search Tab
+#
+# Functions:
+# FIXME
+#
+# End of Search Tab Functions
+#
+##############################################################################################################################################################################################################################
+#
+# Llamafile Tab
+#
+# Functions:
+# FIXME
+#
+# End of Llamafile Tab Functions
+##############################################################################################################################################################################################################################
+#
+# Chat Interface Tab Functions
+#
+# Functions:
+# FIXME
+#
+#
+# End of Chat Interface Tab Functions
+################################################################################################################################################################################################################################
+#
+# Media Edit Tab Functions
+# Functions:
+# Fixme
+# create_media_edit_tab():
+##### Trash Tab
+# FIXME
+# Functions:
+#
+# End of Media Edit Tab Functions
+################################################################################################################
+#
+# Import Items Tab Functions
+#
+# Functions:
+#FIXME
+# End of Import Items Tab Functions
+################################################################################################################
+#
+# Export Items Tab Functions
+#
+# Functions:
+# FIXME
+#
+#
+# End of Export Items Tab Functions
+################################################################################################################
+#
+# Keyword Management Tab Functions
+#
+# Functions:
+#  create_view_keywords_tab():
+# FIXME
+#
+# End of Keyword Management Tab Functions
+################################################################################################################
+#
+# Document Editing Tab Functions
+#
+# Functions:
+#   #FIXME
+#
+#
+################################################################################################################
+#
+# Utilities Tab Functions
+# Functions:
+#   create_utilities_yt_video_tab():
+# #FIXME
+
+#
+# End of Utilities Tab Functions
+################################################################################################################
+
+# FIXME - Prompt sample box
+#
+# # Sample data
+# prompts_category_1 = [
+#     "What are the key points discussed in the video?",
+#     "Summarize the main arguments made by the speaker.",
+#     "Describe the conclusions of the study presented."
+# ]
+#
+# prompts_category_2 = [
+#     "How does the proposed solution address the problem?",
+#     "What are the implications of the findings?",
+#     "Can you explain the theory behind the observed phenomenon?"
+# ]
+#
+# all_prompts2 = prompts_category_1 + prompts_category_2
+
+
+
+#######################################################################################################################
+#
+# Migration Script
+import sqlite3
+import uuid
+import logging
+import os
+from datetime import datetime
+import shutil
+
+# def migrate_media_db_to_rag_chat_db(media_db_path, rag_chat_db_path):
+#     # Check if migration is needed
+#     if not os.path.exists(media_db_path):
+#         logging.info("Media DB does not exist. No migration needed.")
+#         return
+#
+#     # Optional: Check if migration has already been completed
+#     migration_flag = os.path.join(os.path.dirname(rag_chat_db_path), 'migration_completed.flag')
+#     if os.path.exists(migration_flag):
+#         logging.info("Migration already completed. Skipping migration.")
+#         return
+#
+#     # Backup databases
+#     backup_database(media_db_path)
+#     backup_database(rag_chat_db_path)
+#
+#     # Connect to both databases
+#     try:
+#         media_conn = sqlite3.connect(media_db_path)
+#         rag_conn = sqlite3.connect(rag_chat_db_path)
+#
+#         # Enable foreign key support
+#         media_conn.execute('PRAGMA foreign_keys = ON;')
+#         rag_conn.execute('PRAGMA foreign_keys = ON;')
+#
+#         media_cursor = media_conn.cursor()
+#         rag_cursor = rag_conn.cursor()
+#
+#         # Begin transaction
+#         rag_conn.execute('BEGIN TRANSACTION;')
+#
+#         # Extract conversations from media DB
+#         media_cursor.execute('''
+#             SELECT id, media_id, media_name, conversation_name, created_at, updated_at
+#             FROM ChatConversations
+#         ''')
+#         conversations = media_cursor.fetchall()
+#
+#         for conv in conversations:
+#             old_conv_id, media_id, media_name, conversation_name, created_at, updated_at = conv
+#
+#             # Convert timestamps if necessary
+#             created_at = parse_timestamp(created_at)
+#             updated_at = parse_timestamp(updated_at)
+#
+#             # Generate a new conversation_id
+#             conversation_id = str(uuid.uuid4())
+#             title = conversation_name or (f"{media_name}-chat" if media_name else "Untitled Conversation")
+#
+#             # Insert into conversation_metadata
+#             rag_cursor.execute('''
+#                 INSERT INTO conversation_metadata (conversation_id, created_at, last_updated, title, media_id)
+#                 VALUES (?, ?, ?, ?, ?)
+#             ''', (conversation_id, created_at, updated_at, title, media_id))
+#
+#             # Extract messages from media DB
+#             media_cursor.execute('''
+#                 SELECT sender, message, timestamp
+#                 FROM ChatMessages
+#                 WHERE conversation_id = ?
+#                 ORDER BY timestamp ASC
+#             ''', (old_conv_id,))
+#             messages = media_cursor.fetchall()
+#
+#             for msg in messages:
+#                 sender, content, timestamp = msg
+#
+#                 # Convert timestamp if necessary
+#                 timestamp = parse_timestamp(timestamp)
+#
+#                 role = sender  # Assuming 'sender' is 'user' or 'ai'
+#
+#                 # Insert message into rag_qa_chats
+#                 rag_cursor.execute('''
+#                     INSERT INTO rag_qa_chats (conversation_id, timestamp, role, content)
+#                     VALUES (?, ?, ?, ?)
+#                 ''', (conversation_id, timestamp, role, content))
+#
+#         # Commit transaction
+#         rag_conn.commit()
+#         logging.info("Migration completed successfully.")
+#
+#         # Mark migration as complete
+#         with open(migration_flag, 'w') as f:
+#             f.write('Migration completed on ' + datetime.now().isoformat())
+#
+#     except Exception as e:
+#         # Rollback transaction in case of error
+#         rag_conn.rollback()
+#         logging.error(f"Error during migration: {e}")
+#         raise
+#     finally:
+#         media_conn.close()
+#         rag_conn.close()
+
+def backup_database(db_path):
+    backup_path = db_path + '.backup'
+    if not os.path.exists(backup_path):
+        shutil.copyfile(db_path, backup_path)
+        logging.info(f"Database backed up to {backup_path}")
+    else:
+        logging.info(f"Backup already exists at {backup_path}")
+
+def parse_timestamp(timestamp_value):
+    """
+    Parses the timestamp from the old database and converts it to a standard format.
+    Adjust this function based on the actual format of your timestamps.
+    """
+    try:
+        # Attempt to parse ISO format
+        return datetime.fromisoformat(timestamp_value).isoformat()
+    except ValueError:
+        # Handle other timestamp formats if necessary
+        # For example, if timestamps are in Unix epoch format
+        try:
+            timestamp_float = float(timestamp_value)
+            return datetime.fromtimestamp(timestamp_float).isoformat()
+        except ValueError:
+            # Default to current time if parsing fails
+            logging.warning(f"Unable to parse timestamp '{timestamp_value}', using current time.")
+            return datetime.now().isoformat()
+
+#
+# End of Migration Script
+#######################################################################################################################
+
+
+#######################################################################################################################
+#
+# Launch UI Function
+def launch_ui(share_public=None, server_mode=False):
+    webbrowser.open_new_tab('http://127.0.0.1:7860/?__theme=dark')
+    share=share_public
+    css = """
+    .result-box {
+        margin-bottom: 20px;
+        border: 1px solid #ddd;
+        padding: 10px;
+    }
+    .result-box.error {
+        border-color: #ff0000;
+        background-color: #ffeeee;
+    }
+    .transcription, .summary {
+        max-height: 800px;
+        overflow-y: auto;
+        border: 1px solid #eee;
+        padding: 10px;
+        margin-top: 10px;
+    }
+    """
+
+    config = load_and_log_configs()
+    # Get database paths from config
+    db_config = config['db_config']
+    media_db_path = db_config['sqlite_path']
+    character_chat_db_path = os.path.join(os.path.dirname(media_db_path), "chatDB.db")
+    rag_chat_db_path = os.path.join(os.path.dirname(media_db_path), "rag_qa.db")
+    # Initialize the RAG Chat DB (create tables and update schema)
+    create_tables()
+
+    # Migrate data from the media DB to the RAG Chat DB
+    #migrate_media_db_to_rag_chat_db(media_db_path, rag_chat_db_path)
+
+
+    with gr.Blocks(theme='bethecloud/storj_theme',css=css) as iface:
+        gr.HTML(
+            """
+            <script>
+            document.addEventListener('DOMContentLoaded', (event) => {
+                document.body.classList.add('dark');
+                document.querySelector('gradio-app').style.backgroundColor = 'var(--color-background-primary)';
+            });
+            </script>
+            """
+        )
+        db_config = get_db_config()
+        db_type = db_config['type']
+        gr.Markdown(f"# tl/dw: Your LLM-powered Research Multi-tool")
+        gr.Markdown(f"(Using {db_type.capitalize()} Database)")
+        with gr.Tabs():
+            with gr.TabItem("Transcribe / Analyze / Ingestion", id="ingestion-grouping", visible=True):
+                with gr.Tabs():
+                    create_video_transcription_tab()
+                    create_audio_processing_tab()
+                    create_podcast_tab()
+                    create_import_book_tab()
+                    create_plain_text_import_tab()
+                    create_xml_import_tab()
+                    create_website_scraping_tab()
+                    create_pdf_ingestion_tab()
+                    create_pdf_ingestion_test_tab()
+                    create_resummary_tab()
+                    create_summarize_explain_tab()
+                    create_live_recording_tab()
+                    create_arxiv_tab()
+                    create_semantic_scholar_tab()
+
+            with gr.TabItem("RAG Chat/Search", id="RAG Chat Notes group", visible=True):
+                create_rag_tab()
+                create_rag_qa_chat_tab()
+                create_rag_qa_notes_management_tab()
+                create_rag_qa_chat_management_tab()
+
+            with gr.TabItem("Chat with an LLM", id="LLM Chat group", visible=True):
+                create_chat_interface()
+                create_chat_interface_stacked()
+                create_chat_interface_multi_api()
+                create_chat_interface_four()
+                chat_workflows_tab()
+
+            with gr.TabItem("Character Chat", id="character chat group", visible=True):
+                create_character_card_interaction_tab()
+                create_character_chat_mgmt_tab()
+                create_custom_character_card_tab()
+                create_character_card_validation_tab()
+                create_multiple_character_chat_tab()
+                create_narrator_controlled_conversation_tab()
+                create_export_characters_tab()
+
+            with gr.TabItem("Writing Tools", id="writing_tools group", visible=True):
+                from App_Function_Libraries.Gradio_UI.Writing_tab import create_document_feedback_tab
+                create_document_feedback_tab()
+                from App_Function_Libraries.Gradio_UI.Writing_tab import create_grammar_style_check_tab
+                create_grammar_style_check_tab()
+                from App_Function_Libraries.Gradio_UI.Writing_tab import create_tone_adjustment_tab
+                create_tone_adjustment_tab()
+                from App_Function_Libraries.Gradio_UI.Writing_tab import create_creative_writing_tab
+                create_creative_writing_tab()
+                from App_Function_Libraries.Gradio_UI.Writing_tab import create_mikupad_tab
+                create_mikupad_tab()
+
+            with gr.TabItem("Search/View DB Items", id="view db items group", visible=True):
+                create_search_tab()
+                create_search_summaries_tab()
+                create_view_all_mediadb_with_versions_tab()
+                create_viewing_mediadb_tab()
+                create_mediadb_keyword_search_tab()
+                create_view_all_rag_notes_tab()
+                create_viewing_ragdb_tab()
+                create_ragdb_keyword_items_tab()
+
+            with gr.TabItem("Prompts", id='view prompts group', visible=True):
+                with gr.Tabs():
+                    create_prompt_view_tab()
+                    create_prompt_search_tab()
+                    create_prompt_edit_tab()
+                    create_prompt_clone_tab()
+                    create_prompt_suggestion_tab()
+                    create_prompts_export_tab()
+
+            with gr.TabItem("Manage Media DB Items", id="manage group", visible=True):
+                create_media_edit_tab()
+                create_manage_items_tab()
+                create_media_edit_and_clone_tab()
+
+            with gr.TabItem("Embeddings Management", id="embeddings group", visible=True):
+                create_embeddings_tab()
+                create_view_embeddings_tab()
+                create_purge_embeddings_tab()
+
+            with gr.TabItem("Keywords", id="keywords group", visible=True):
+                create_view_keywords_tab()
+                create_add_keyword_tab()
+                create_delete_keyword_tab()
+                create_export_keywords_tab()
+                create_character_keywords_tab()
+                create_rag_qa_keywords_tab()
+                create_meta_keywords_tab()
+                create_prompt_keywords_tab()
+
+            with gr.TabItem("Import", id="import group", visible=True):
+                create_import_item_tab()
+                create_import_obsidian_vault_tab()
+                create_import_single_prompt_tab()
+                create_import_multiple_prompts_tab()
+                create_mediawiki_import_tab()
+                create_mediawiki_config_tab()
+                create_conversation_import_tab()
+
+            with gr.TabItem("Export", id="export group", visible=True):
+                create_export_tabs()
+
+
+            with gr.TabItem("Database Management", id="database_management_group", visible=True):
+                create_database_management_interface(
+                    media_db_config={
+                        'db_path': media_db_path,
+                        'backup_dir': backup_dir
+                    },
+                    rag_db_config={
+                        'db_path': rag_chat_db_path,
+                        'backup_dir': backup_dir
+                    },
+                    char_db_config={
+                        'db_path': character_chat_db_path,
+                        'backup_dir': backup_dir
+                    }
+                )
+
+            with gr.TabItem("Utilities", id="util group", visible=True):
+                create_mindmap_tab()
+                create_utilities_yt_video_tab()
+                create_utilities_yt_audio_tab()
+                create_utilities_yt_timestamp_tab()
+
+            with gr.TabItem("Anki Deck Creation/Validation", id="anki group", visible=True):
+                create_anki_generator_tab()
+                create_anki_validation_tab()
+
+            with gr.TabItem("Local LLM", id="local llm group", visible=True):
+                create_chat_with_llamafile_tab()
+                create_ollama_tab()
+                #create_huggingface_tab()
+
+            with gr.TabItem("Trashcan", id="trashcan group", visible=True):
+                create_search_and_mark_trash_tab()
+                create_view_trash_tab()
+                create_delete_trash_tab()
+                create_empty_trash_tab()
+
+            with gr.TabItem("Evaluations", id="eval", visible=True):
+                create_geval_tab()
+                create_infinite_bench_tab()
+                # FIXME
+                #create_mmlu_pro_tab()
+
+            with gr.TabItem("Introduction/Help", id="introduction group", visible=True):
+                create_introduction_tab()
+
+            with gr.TabItem("Config Editor", id="config group"):
+                create_config_editor_tab()
+
+    # Launch the interface
+    server_port_variable = 7860
+    os.environ['GRADIO_ANALYTICS_ENABLED'] = 'False'
+    if share==True:
+        iface.launch(share=True)
+    elif server_mode and not share_public:
+        iface.launch(share=False, server_name="0.0.0.0", server_port=server_port_variable, )
+    else:
+        try:
+            iface.launch(share=False, server_name="0.0.0.0", server_port=server_port_variable, )
+        except Exception as e:
+            logging.error(f"Error launching interface: {str(e)}")

App_Function_Libraries/Gradio_UI/Anki_tab.py

@@ -0,0 +1,921 @@
+# Anki_Validation_tab.py
+# Description: Gradio functions for the Anki Validation tab
+#
+# Imports
+import json
+import logging
+import os
+import tempfile
+from typing import Optional, Tuple, List, Dict
+#
+# External Imports
+import genanki
+import gradio as gr
+#
+# Local Imports
+from App_Function_Libraries.Chat.Chat_Functions import approximate_token_count, update_chat_content, save_chat_history, \
+    save_chat_history_to_db_wrapper
+from App_Function_Libraries.DB.DB_Manager import list_prompts
+from App_Function_Libraries.Gradio_UI.Chat_ui import update_dropdown_multiple, chat_wrapper, update_selected_parts, \
+    search_conversations, regenerate_last_message, load_conversation, debug_output
+from App_Function_Libraries.Third_Party.Anki import sanitize_html, generate_card_choices, \
+    export_cards, load_card_for_editing, handle_file_upload, \
+    validate_for_ui, update_card_with_validation, update_card_choices, enhanced_file_upload, \
+    handle_validation
+from App_Function_Libraries.Utils.Utils import default_api_endpoint, global_api_endpoints, format_api_name
+#
+############################################################################################################
+#
+# Functions:
+
+def create_anki_validation_tab():
+    with gr.TabItem("Anki Flashcard Validation", visible=True):
+        gr.Markdown("# Anki Flashcard Validation and Editor")
+
+        # State variables for internal tracking
+        current_card_data = gr.State({})
+        preview_update_flag = gr.State(False)
+
+        with gr.Row():
+            # Left Column: Input and Validation
+            with gr.Column(scale=1):
+                gr.Markdown("## Import or Create Flashcards")
+
+                input_type = gr.Radio(
+                    choices=["JSON", "APKG"],
+                    label="Input Type",
+                    value="JSON"
+                )
+
+                with gr.Group() as json_input_group:
+                    flashcard_input = gr.TextArea(
+                        label="Enter Flashcards (JSON format)",
+                        placeholder='''{
+    "cards": [
+        {
+            "id": "CARD_001",
+            "type": "basic",
+            "front": "What is the capital of France?",
+            "back": "Paris",
+            "tags": ["geography", "europe"],
+            "note": "Remember: City of Light"
+        }
+    ]
+}''',
+                        lines=10
+                    )
+
+                    import_json = gr.File(
+                        label="Or Import JSON File",
+                        file_types=[".json"]
+                    )
+
+                with gr.Group(visible=False) as apkg_input_group:
+                    import_apkg = gr.File(
+                        label="Import APKG File",
+                        file_types=[".apkg"]
+                    )
+                    deck_info = gr.JSON(
+                        label="Deck Information",
+                        visible=False
+                    )
+
+                validate_button = gr.Button("Validate Flashcards")
+
+            # Right Column: Validation Results and Editor
+            with gr.Column(scale=1):
+                gr.Markdown("## Validation Results")
+                validation_status = gr.Markdown("")
+
+                with gr.Accordion("Validation Rules", open=False):
+                    gr.Markdown("""
+                    ### Required Fields:
+                    - Unique ID
+                    - Card Type (basic, cloze, reverse)
+                    - Front content
+                    - Back content
+                    - At least one tag
+
+                    ### Content Rules:
+                    - No empty fields
+                    - Front side should be a clear question/prompt
+                    - Back side should contain complete answer
+                    - Cloze deletions must have valid syntax
+                    - No duplicate IDs
+
+                    ### Image Rules:
+                    - Valid image tags
+                    - Supported formats (JPG, PNG, GIF)
+                    - Base64 encoded or valid URL
+
+                    ### APKG-specific Rules:
+                    - Valid SQLite database structure
+                    - Media files properly referenced
+                    - Note types match Anki standards
+                    - Card templates are well-formed
+                    """)
+
+        with gr.Row():
+            # Card Editor
+            gr.Markdown("## Card Editor")
+        with gr.Row():
+            with gr.Column(scale=1):
+                with gr.Accordion("Edit Individual Cards", open=True):
+                    card_selector = gr.Dropdown(
+                        label="Select Card to Edit",
+                        choices=[],
+                        interactive=True
+                    )
+
+                    card_type = gr.Radio(
+                        choices=["basic", "cloze", "reverse"],
+                        label="Card Type",
+                        value="basic"
+                    )
+
+                    # Front content with preview
+                    with gr.Group():
+                        gr.Markdown("### Front Content")
+                        front_content = gr.TextArea(
+                            label="Content (HTML supported)",
+                            lines=3
+                        )
+                        front_preview = gr.HTML(
+                            label="Preview"
+                        )
+
+                    # Back content with preview
+                    with gr.Group():
+                        gr.Markdown("### Back Content")
+                        back_content = gr.TextArea(
+                            label="Content (HTML supported)",
+                            lines=3
+                        )
+                        back_preview = gr.HTML(
+                            label="Preview"
+                        )
+
+                    tags_input = gr.TextArea(
+                        label="Tags (comma-separated)",
+                        lines=1
+                    )
+
+                    notes_input = gr.TextArea(
+                        label="Additional Notes",
+                        lines=2
+                    )
+
+                    with gr.Row():
+                        update_card_button = gr.Button("Update Card")
+                        delete_card_button = gr.Button("Delete Card", variant="stop")
+
+        with gr.Row():
+            with gr.Column(scale=1):
+                # Export Options
+                gr.Markdown("## Export Options")
+                export_format = gr.Radio(
+                    choices=["Anki CSV", "JSON", "Plain Text"],
+                    label="Export Format",
+                    value="Anki CSV"
+                )
+                export_button = gr.Button("Export Valid Cards")
+                export_file = gr.File(label="Download Validated Cards")
+                export_status = gr.Markdown("")
+            with gr.Column(scale=1):
+                gr.Markdown("## Export Instructions")
+                gr.Markdown("""
+                ### Anki CSV Format:
+                - Front, Back, Tags, Type, Note
+                - Use for importing into Anki
+                - Images preserved as HTML
+
+                ### JSON Format:
+                - JSON array of cards
+                - Images as base64 or URLs
+                - Use for custom processing
+
+                ### Plain Text Format:
+                - Question and Answer pairs
+                - Images represented as [IMG] placeholder
+                - Use for manual review
+                """)
+
+        def update_preview(content):
+            """Update preview with sanitized content."""
+            if not content:
+                return ""
+            return sanitize_html(content)
+
+        # Event handlers
+        def validation_chain(content: str) -> Tuple[str, List[str]]:
+            """Combined validation and card choice update."""
+            validation_message = validate_for_ui(content)
+            card_choices = update_card_choices(content)
+            return validation_message, card_choices
+
+        def delete_card(card_selection, current_content):
+            """Delete selected card and return updated content."""
+            if not card_selection or not current_content:
+                return current_content, "No card selected", []
+
+            try:
+                data = json.loads(current_content)
+                selected_id = card_selection.split(" - ")[0]
+
+                data['cards'] = [card for card in data['cards'] if card['id'] != selected_id]
+                new_content = json.dumps(data, indent=2)
+
+                return (
+                    new_content,
+                    "Card deleted successfully!",
+                    generate_card_choices(new_content)
+                )
+
+            except Exception as e:
+                return current_content, f"Error deleting card: {str(e)}", []
+
+        def process_validation_result(is_valid, message):
+            """Process validation result into a formatted markdown string."""
+            if is_valid:
+                return f"✅ {message}"
+            else:
+                return f"❌ {message}"
+
+        # Register event handlers
+        input_type.change(
+            fn=lambda t: (
+                gr.update(visible=t == "JSON"),
+                gr.update(visible=t == "APKG"),
+                gr.update(visible=t == "APKG")
+            ),
+            inputs=[input_type],
+            outputs=[json_input_group, apkg_input_group, deck_info]
+        )
+
+        # File upload handlers
+        import_json.upload(
+            fn=handle_file_upload,
+            inputs=[import_json, input_type],
+            outputs=[
+                flashcard_input,
+                deck_info,
+                validation_status,
+                card_selector
+            ]
+        )
+
+        import_apkg.upload(
+            fn=enhanced_file_upload,
+            inputs=[import_apkg, input_type],
+            outputs=[
+                flashcard_input,
+                deck_info,
+                validation_status,
+                card_selector
+            ]
+        )
+
+        # Validation handler
+        validate_button.click(
+            fn=lambda content, input_format: (
+                handle_validation(content, input_format),
+                generate_card_choices(content) if content else []
+            ),
+            inputs=[flashcard_input, input_type],
+            outputs=[validation_status, card_selector]
+        )
+
+        # Card editing handlers
+        # Card selector change event
+        card_selector.change(
+            fn=load_card_for_editing,
+            inputs=[card_selector, flashcard_input],
+            outputs=[
+                card_type,
+                front_content,
+                back_content,
+                tags_input,
+                notes_input,
+                front_preview,
+                back_preview
+            ]
+        )
+
+        # Live preview updates
+        front_content.change(
+            fn=update_preview,
+            inputs=[front_content],
+            outputs=[front_preview]
+        )
+
+        back_content.change(
+            fn=update_preview,
+            inputs=[back_content],
+            outputs=[back_preview]
+        )
+
+        # Card update handler
+        update_card_button.click(
+            fn=update_card_with_validation,
+            inputs=[
+                flashcard_input,
+                card_selector,
+                card_type,
+                front_content,
+                back_content,
+                tags_input,
+                notes_input
+            ],
+            outputs=[
+                flashcard_input,
+                validation_status,
+                card_selector
+            ]
+        )
+
+        # Delete card handler
+        delete_card_button.click(
+            fn=delete_card,
+            inputs=[card_selector, flashcard_input],
+            outputs=[flashcard_input, validation_status, card_selector]
+        )
+
+        # Export handler
+        export_button.click(
+            fn=export_cards,
+            inputs=[flashcard_input, export_format],
+            outputs=[export_status, export_file]
+        )
+
+        return (
+            flashcard_input,
+            import_json,
+            import_apkg,
+            validate_button,
+            validation_status,
+            card_selector,
+            card_type,
+            front_content,
+            back_content,
+            front_preview,
+            back_preview,
+            tags_input,
+            notes_input,
+            update_card_button,
+            delete_card_button,
+            export_format,
+            export_button,
+            export_file,
+            export_status,
+            deck_info
+        )
+
+
+def create_anki_generator_tab():
+    with gr.TabItem("Anki Deck Generator", visible=True):
+        try:
+            default_value = None
+            if default_api_endpoint:
+                if default_api_endpoint in global_api_endpoints:
+                    default_value = format_api_name(default_api_endpoint)
+                else:
+                    logging.warning(f"Default API endpoint '{default_api_endpoint}' not found in global_api_endpoints")
+        except Exception as e:
+            logging.error(f"Error setting default API endpoint: {str(e)}")
+            default_value = None
+        custom_css = """
+        .chatbot-container .message-wrap .message {
+            font-size: 14px !important;
+        }
+        """
+        with gr.TabItem("LLM Chat & Anki Deck Creation", visible=True):
+            gr.Markdown("# Chat with an LLM to help you come up with Questions/Answers for an Anki Deck")
+            chat_history = gr.State([])
+            media_content = gr.State({})
+            selected_parts = gr.State([])
+            conversation_id = gr.State(None)
+            initial_prompts, total_pages, current_page = list_prompts(page=1, per_page=10)
+
+            with gr.Row():
+                with gr.Column(scale=1):
+                    search_query_input = gr.Textbox(
+                        label="Search Query",
+                        placeholder="Enter your search query here..."
+                    )
+                    search_type_input = gr.Radio(
+                        choices=["Title", "Content", "Author", "Keyword"],
+                        value="Keyword",
+                        label="Search By"
+                    )
+                    keyword_filter_input = gr.Textbox(
+                        label="Filter by Keywords (comma-separated)",
+                        placeholder="ml, ai, python, etc..."
+                    )
+                    search_button = gr.Button("Search")
+                    items_output = gr.Dropdown(label="Select Item", choices=[], interactive=True)
+                    item_mapping = gr.State({})
+                    with gr.Row():
+                        use_content = gr.Checkbox(label="Use Content")
+                        use_summary = gr.Checkbox(label="Use Summary")
+                        use_prompt = gr.Checkbox(label="Use Prompt")
+                        save_conversation = gr.Checkbox(label="Save Conversation", value=False, visible=True)
+                    with gr.Row():
+                        temperature = gr.Slider(label="Temperature", minimum=0.00, maximum=1.0, step=0.05, value=0.7)
+                    with gr.Row():
+                        conversation_search = gr.Textbox(label="Search Conversations")
+                    with gr.Row():
+                        search_conversations_btn = gr.Button("Search Conversations")
+                    with gr.Row():
+                        previous_conversations = gr.Dropdown(label="Select Conversation", choices=[], interactive=True)
+                    with gr.Row():
+                        load_conversations_btn = gr.Button("Load Selected Conversation")
+
+                    # Refactored API selection dropdown
+                    api_endpoint = gr.Dropdown(
+                        choices=["None"] + [format_api_name(api) for api in global_api_endpoints],
+                        value=default_value,
+                        label="API for Chat Interaction (Optional)"
+                    )
+                    api_key = gr.Textbox(label="API Key (if required)", type="password")
+                    custom_prompt_checkbox = gr.Checkbox(label="Use a Custom Prompt",
+                                                         value=False,
+                                                         visible=True)
+                    preset_prompt_checkbox = gr.Checkbox(label="Use a Pre-set Prompt",
+                                                         value=False,
+                                                         visible=True)
+                    with gr.Row(visible=False) as preset_prompt_controls:
+                        prev_prompt_page = gr.Button("Previous")
+                        next_prompt_page = gr.Button("Next")
+                        current_prompt_page_text = gr.Text(f"Page {current_page} of {total_pages}")
+                        current_prompt_page_state = gr.State(value=1)
+
+                    preset_prompt = gr.Dropdown(
+                        label="Select Preset Prompt",
+                        choices=initial_prompts
+                    )
+                    user_prompt = gr.Textbox(label="Custom Prompt",
+                                             placeholder="Enter custom prompt here",
+                                             lines=3,
+                                             visible=False)
+                    system_prompt_input = gr.Textbox(label="System Prompt",
+                                                     value="You are a helpful AI assitant",
+                                                     lines=3,
+                                                     visible=False)
+                with gr.Column(scale=2):
+                    chatbot = gr.Chatbot(height=800, elem_classes="chatbot-container")
+                    msg = gr.Textbox(label="Enter your message")
+                    submit = gr.Button("Submit")
+                    regenerate_button = gr.Button("Regenerate Last Message")
+                    token_count_display = gr.Number(label="Approximate Token Count", value=0, interactive=False)
+                    clear_chat_button = gr.Button("Clear Chat")
+
+                    chat_media_name = gr.Textbox(label="Custom Chat Name(optional)")
+                    save_chat_history_to_db = gr.Button("Save Chat History to DataBase")
+                    save_status = gr.Textbox(label="Save Status", interactive=False)
+                    save_chat_history_as_file = gr.Button("Save Chat History as File")
+                    download_file = gr.File(label="Download Chat History")
+
+            search_button.click(
+                fn=update_dropdown_multiple,
+                inputs=[search_query_input, search_type_input, keyword_filter_input],
+                outputs=[items_output, item_mapping]
+            )
+
+            def update_prompt_visibility(custom_prompt_checked, preset_prompt_checked):
+                user_prompt_visible = custom_prompt_checked
+                system_prompt_visible = custom_prompt_checked
+                preset_prompt_visible = preset_prompt_checked
+                preset_prompt_controls_visible = preset_prompt_checked
+                return (
+                    gr.update(visible=user_prompt_visible, interactive=user_prompt_visible),
+                    gr.update(visible=system_prompt_visible, interactive=system_prompt_visible),
+                    gr.update(visible=preset_prompt_visible, interactive=preset_prompt_visible),
+                    gr.update(visible=preset_prompt_controls_visible)
+                )
+
+            def update_prompt_page(direction, current_page_val):
+                new_page = current_page_val + direction
+                if new_page < 1:
+                    new_page = 1
+                prompts, total_pages, _ = list_prompts(page=new_page, per_page=20)
+                if new_page > total_pages:
+                    new_page = total_pages
+                    prompts, total_pages, _ = list_prompts(page=new_page, per_page=20)
+                return (
+                    gr.update(choices=prompts),
+                    gr.update(value=f"Page {new_page} of {total_pages}"),
+                    new_page
+                )
+
+            def clear_chat():
+                return [], None  # Return empty list for chatbot and None for conversation_id
+
+            custom_prompt_checkbox.change(
+                update_prompt_visibility,
+                inputs=[custom_prompt_checkbox, preset_prompt_checkbox],
+                outputs=[user_prompt, system_prompt_input, preset_prompt, preset_prompt_controls]
+            )
+
+            preset_prompt_checkbox.change(
+                update_prompt_visibility,
+                inputs=[custom_prompt_checkbox, preset_prompt_checkbox],
+                outputs=[user_prompt, system_prompt_input, preset_prompt, preset_prompt_controls]
+            )
+
+            prev_prompt_page.click(
+                lambda x: update_prompt_page(-1, x),
+                inputs=[current_prompt_page_state],
+                outputs=[preset_prompt, current_prompt_page_text, current_prompt_page_state]
+            )
+
+            next_prompt_page.click(
+                lambda x: update_prompt_page(1, x),
+                inputs=[current_prompt_page_state],
+                outputs=[preset_prompt, current_prompt_page_text, current_prompt_page_state]
+            )
+
+            submit.click(
+                chat_wrapper,
+                inputs=[msg, chatbot, media_content, selected_parts, api_endpoint, api_key, user_prompt,
+                        conversation_id,
+                        save_conversation, temperature, system_prompt_input],
+                outputs=[msg, chatbot, conversation_id]
+            ).then(  # Clear the message box after submission
+                lambda x: gr.update(value=""),
+                inputs=[chatbot],
+                outputs=[msg]
+            ).then(  # Clear the user prompt after the first message
+                lambda: (gr.update(value=""), gr.update(value="")),
+                outputs=[user_prompt, system_prompt_input]
+            ).then(
+                lambda history: approximate_token_count(history),
+                inputs=[chatbot],
+                outputs=[token_count_display]
+            )
+
+
+            clear_chat_button.click(
+                clear_chat,
+                outputs=[chatbot, conversation_id]
+            )
+
+            items_output.change(
+                update_chat_content,
+                inputs=[items_output, use_content, use_summary, use_prompt, item_mapping],
+                outputs=[media_content, selected_parts]
+            )
+
+            use_content.change(update_selected_parts, inputs=[use_content, use_summary, use_prompt],
+                               outputs=[selected_parts])
+            use_summary.change(update_selected_parts, inputs=[use_content, use_summary, use_prompt],
+                               outputs=[selected_parts])
+            use_prompt.change(update_selected_parts, inputs=[use_content, use_summary, use_prompt],
+                              outputs=[selected_parts])
+            items_output.change(debug_output, inputs=[media_content, selected_parts], outputs=[])
+
+            search_conversations_btn.click(
+                search_conversations,
+                inputs=[conversation_search],
+                outputs=[previous_conversations]
+            )
+
+            load_conversations_btn.click(
+                clear_chat,
+                outputs=[chatbot, chat_history]
+            ).then(
+                load_conversation,
+                inputs=[previous_conversations],
+                outputs=[chatbot, conversation_id]
+            )
+
+            previous_conversations.change(
+                load_conversation,
+                inputs=[previous_conversations],
+                outputs=[chat_history]
+            )
+
+            save_chat_history_as_file.click(
+                save_chat_history,
+                inputs=[chatbot, conversation_id],
+                outputs=[download_file]
+            )
+
+            save_chat_history_to_db.click(
+                save_chat_history_to_db_wrapper,
+                inputs=[chatbot, conversation_id, media_content, chat_media_name],
+                outputs=[conversation_id, gr.Textbox(label="Save Status")]
+            )
+
+            regenerate_button.click(
+                regenerate_last_message,
+                inputs=[chatbot, media_content, selected_parts, api_endpoint, api_key, user_prompt, temperature,
+                        system_prompt_input],
+                outputs=[chatbot, save_status]
+            ).then(
+                lambda history: approximate_token_count(history),
+                inputs=[chatbot],
+                outputs=[token_count_display]
+            )
+        gr.Markdown("# Create Anki Deck")
+
+        with gr.Row():
+            # Left Column: Deck Settings
+            with gr.Column(scale=1):
+                gr.Markdown("## Deck Settings")
+                deck_name = gr.Textbox(
+                    label="Deck Name",
+                    placeholder="My Study Deck",
+                    value="My Study Deck"
+                )
+
+                deck_description = gr.Textbox(
+                    label="Deck Description",
+                    placeholder="Description of your deck",
+                    lines=2
+                )
+
+                note_type = gr.Radio(
+                    choices=["Basic", "Basic (and reversed)", "Cloze"],
+                    label="Note Type",
+                    value="Basic"
+                )
+
+                # Card Fields based on note type
+                with gr.Group() as basic_fields:
+                    front_template = gr.Textbox(
+                        label="Front Template (HTML)",
+                        value="{{Front}}",
+                        lines=3
+                    )
+                    back_template = gr.Textbox(
+                        label="Back Template (HTML)",
+                        value="{{FrontSide}}<hr id='answer'>{{Back}}",
+                        lines=3
+                    )
+
+                with gr.Group() as cloze_fields:
+                    cloze_template = gr.Textbox(
+                        label="Cloze Template (HTML)",
+                        value="{{cloze:Text}}",
+                        lines=3,
+                        visible=False
+                    )
+
+                css_styling = gr.Textbox(
+                    label="Card Styling (CSS)",
+                    value=".card {\n font-family: arial;\n font-size: 20px;\n text-align: center;\n color: black;\n background-color: white;\n}\n\n.cloze {\n font-weight: bold;\n color: blue;\n}",
+                    lines=5
+                )
+
+            # Right Column: Card Creation
+            with gr.Column(scale=1):
+                gr.Markdown("## Add Cards")
+
+                with gr.Group() as basic_input:
+                    front_content = gr.TextArea(
+                        label="Front Content",
+                        placeholder="Question or prompt",
+                        lines=3
+                    )
+                    back_content = gr.TextArea(
+                        label="Back Content",
+                        placeholder="Answer",
+                        lines=3
+                    )
+
+                with gr.Group() as cloze_input:
+                    cloze_content = gr.TextArea(
+                        label="Cloze Content",
+                        placeholder="Text with {{c1::cloze}} deletions",
+                        lines=3,
+                        visible=False
+                    )
+
+                tags_input = gr.TextArea(
+                    label="Tags (comma-separated)",
+                    placeholder="tag1, tag2, tag3",
+                    lines=1
+                )
+
+                add_card_btn = gr.Button("Add Card")
+
+                cards_list = gr.JSON(
+                    label="Cards in Deck",
+                    value={"cards": []}
+                )
+
+                clear_cards_btn = gr.Button("Clear All Cards", variant="stop")
+
+        with gr.Row():
+            generate_deck_btn = gr.Button("Generate Deck", variant="primary")
+            download_deck = gr.File(label="Download Deck")
+            generation_status = gr.Markdown("")
+
+        def update_note_type_fields(note_type: str):
+            if note_type == "Cloze":
+                return {
+                    basic_input: gr.update(visible=False),
+                    cloze_input: gr.update(visible=True),
+                    basic_fields: gr.update(visible=False),
+                    cloze_fields: gr.update(visible=True)
+                }
+            else:
+                return {
+                    basic_input: gr.update(visible=True),
+                    cloze_input: gr.update(visible=False),
+                    basic_fields: gr.update(visible=True),
+                    cloze_fields: gr.update(visible=False)
+                }
+
+        def add_card(note_type: str, front: str, back: str, cloze: str, tags: str, current_cards: Dict[str, List]):
+            if not current_cards:
+                current_cards = {"cards": []}
+
+            cards_data = current_cards["cards"]
+
+            # Process tags
+            card_tags = [tag.strip() for tag in tags.split(',') if tag.strip()]
+
+            new_card = {
+                "id": f"CARD_{len(cards_data) + 1}",
+                "tags": card_tags
+            }
+
+            if note_type == "Cloze":
+                if not cloze or "{{c" not in cloze:
+                    return current_cards, "❌ Invalid cloze format. Use {{c1::text}} syntax."
+                new_card.update({
+                    "type": "cloze",
+                    "content": cloze
+                })
+            else:
+                if not front or not back:
+                    return current_cards, "❌ Both front and back content are required."
+                new_card.update({
+                    "type": "basic",
+                    "front": front,
+                    "back": back,
+                    "is_reverse": note_type == "Basic (and reversed)"
+                })
+
+            cards_data.append(new_card)
+            return {"cards": cards_data}, "✅ Card added successfully!"
+
+        def clear_cards() -> Tuple[Dict[str, List], str]:
+            return {"cards": []}, "✅ All cards cleared!"
+
+        def generate_anki_deck(
+            deck_name: str,
+            deck_description: str,
+            note_type: str,
+            front_template: str,
+            back_template: str,
+            cloze_template: str,
+            css: str,
+            cards_data: Dict[str, List]
+        ) -> Tuple[Optional[str], str]:
+            try:
+                if not cards_data or not cards_data.get("cards"):
+                    return None, "❌ No cards to generate deck from!"
+
+                # Create model based on note type
+                if note_type == "Cloze":
+                    model = genanki.Model(
+                        1483883320,  # Random model ID
+                        'Cloze Model',
+                        fields=[
+                            {'name': 'Text'},
+                            {'name': 'Back Extra'}
+                        ],
+                        templates=[{
+                            'name': 'Cloze Card',
+                            'qfmt': cloze_template,
+                            'afmt': cloze_template + '<br><hr id="extra">{{Back Extra}}'
+                        }],
+                        css=css,
+                        # FIXME CLOZE DOESNT EXIST
+                        model_type=1
+                    )
+                else:
+                    templates = [{
+                        'name': 'Card 1',
+                        'qfmt': front_template,
+                        'afmt': back_template
+                    }]
+
+                    if note_type == "Basic (and reversed)":
+                        templates.append({
+                            'name': 'Card 2',
+                            'qfmt': '{{Back}}',
+                            'afmt': '{{FrontSide}}<hr id="answer">{{Front}}'
+                        })
+
+                    model = genanki.Model(
+                        1607392319,  # Random model ID
+                        'Basic Model',
+                        fields=[
+                            {'name': 'Front'},
+                            {'name': 'Back'}
+                        ],
+                        templates=templates,
+                        css=css
+                    )
+
+                # Create deck
+                deck = genanki.Deck(
+                    2059400110,  # Random deck ID
+                    deck_name,
+                    description=deck_description
+                )
+
+                # Add cards to deck
+                for card in cards_data["cards"]:
+                    if card["type"] == "cloze":
+                        note = genanki.Note(
+                            model=model,
+                            fields=[card["content"], ""],
+                            tags=card["tags"]
+                        )
+                    else:
+                        note = genanki.Note(
+                            model=model,
+                            fields=[card["front"], card["back"]],
+                            tags=card["tags"]
+                        )
+                    deck.add_note(note)
+
+                # Save deck to temporary file
+                temp_dir = tempfile.mkdtemp()
+                deck_path = os.path.join(temp_dir, f"{deck_name}.apkg")
+                genanki.Package(deck).write_to_file(deck_path)
+
+                return deck_path, "✅ Deck generated successfully!"
+
+            except Exception as e:
+                return None, f"❌ Error generating deck: {str(e)}"
+
+        # Register event handlers
+        note_type.change(
+            fn=update_note_type_fields,
+            inputs=[note_type],
+            outputs=[basic_input, cloze_input, basic_fields, cloze_fields]
+        )
+
+        add_card_btn.click(
+            fn=add_card,
+            inputs=[
+                note_type,
+                front_content,
+                back_content,
+                cloze_content,
+                tags_input,
+                cards_list
+            ],
+            outputs=[cards_list, generation_status]
+        )
+
+        clear_cards_btn.click(
+            fn=clear_cards,
+            inputs=[],
+            outputs=[cards_list, generation_status]
+        )
+
+        generate_deck_btn.click(
+            fn=generate_anki_deck,
+            inputs=[
+                deck_name,
+                deck_description,
+                note_type,
+                front_template,
+                back_template,
+                cloze_template,
+                css_styling,
+                cards_list
+            ],
+            outputs=[download_deck, generation_status]
+        )
+
+
+        return (
+            deck_name,
+            deck_description,
+            note_type,
+            front_template,
+            back_template,
+            cloze_template,
+            css_styling,
+            front_content,
+            back_content,
+            cloze_content,
+            tags_input,
+            cards_list,
+            add_card_btn,
+            clear_cards_btn,
+            generate_deck_btn,
+            download_deck,
+            generation_status
+        )
+
+#
+# End of Anki_Validation_tab.py
+############################################################################################################

App_Function_Libraries/Gradio_UI/Audio_ingestion_tab.py

@@ -2,16 +2,18 @@
 # Description: Gradio UI for ingesting audio files into the database
 #
 # Imports
+import logging
 #
 # External Imports
 import gradio as gr
 #
 # Local Imports
 from App_Function_Libraries.Audio.Audio_Files import process_audio_files
-from App_Function_Libraries.DB.DB_Manager import load_preset_prompts
+from App_Function_Libraries.DB.DB_Manager import list_prompts
 from App_Function_Libraries.Gradio_UI.Chat_ui import update_user_prompt
 from App_Function_Libraries.Gradio_UI.Gradio_Shared import whisper_models
-from App_Function_Libraries.Utils.Utils import cleanup_temp_files
+from App_Function_Libraries.Utils.Utils import cleanup_temp_files, default_api_endpoint, global_api_endpoints, \
+    format_api_name
 # Import metrics logging
 from App_Function_Libraries.Metrics.metrics_logger import log_counter, log_histogram
 from App_Function_Libraries.Metrics.logger_config import logger
@@ -22,6 +24,18 @@ from App_Function_Libraries.Metrics.logger_config import logger
 def create_audio_processing_tab():
     with gr.TabItem("Audio File Transcription + Summarization", visible=True):
         gr.Markdown("# Transcribe & Summarize Audio Files from URLs or Local Files!")
+        # Get and validate default value
+        try:
+            default_value = None
+            if default_api_endpoint:
+                if default_api_endpoint in global_api_endpoints:
+                    default_value = format_api_name(default_api_endpoint)
+                else:
+                    logging.warning(f"Default API endpoint '{default_api_endpoint}' not found in global_api_endpoints")
+        except Exception as e:
+            logging.error(f"Error setting default API endpoint: {str(e)}")
+            default_value = None
+
         with gr.Row():
             with gr.Column():
                 audio_url_input = gr.Textbox(label="Audio File URL(s)", placeholder="Enter the URL(s) of the audio file(s), one per line")
@@ -46,54 +60,133 @@ def create_audio_processing_tab():
                 keep_timestamps_input = gr.Checkbox(label="Keep Timestamps", value=True)
 
                 with gr.Row():
-                    custom_prompt_checkbox = gr.Checkbox(label="Use a Custom Prompt",
-                                                     value=False,
-                                                     visible=True)
-                    preset_prompt_checkbox = gr.Checkbox(label="Use a pre-set Prompt",
-                                                     value=False,
-                                                     visible=True)
+                    custom_prompt_checkbox = gr.Checkbox(
+                        label="Use a Custom Prompt",
+                        value=False,
+                        visible=True
+                    )
+                    preset_prompt_checkbox = gr.Checkbox(
+                        label="Use a pre-set Prompt",
+                        value=False,
+                        visible=True
+                    )
+
+                # Initialize state variables for pagination
+                current_page_state = gr.State(value=1)
+                total_pages_state = gr.State(value=1)
+
+                with gr.Row():
+                    # Add pagination controls
+                    preset_prompt = gr.Dropdown(
+                        label="Select Preset Prompt",
+                        choices=[],
+                        visible=False
+                    )
                 with gr.Row():
-                    preset_prompt = gr.Dropdown(label="Select Preset Prompt",
-                                                choices=load_preset_prompts(),
-                                                visible=False)
+                    prev_page_button = gr.Button("Previous Page", visible=False)
+                    page_display = gr.Markdown("Page 1 of X", visible=False)
+                    next_page_button = gr.Button("Next Page", visible=False)
+
                 with gr.Row():
-                    custom_prompt_input = gr.Textbox(label="Custom Prompt",
-                                                     placeholder="Enter custom prompt here",
-                                                     lines=3,
-                                                     visible=False)
+                    custom_prompt_input = gr.Textbox(
+                        label="Custom Prompt",
+                        placeholder="Enter custom prompt here",
+                        lines=3,
+                        visible=False
+                    )
                 with gr.Row():
-                    system_prompt_input = gr.Textbox(label="System Prompt",
-                                                     value="""<s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
-**Bulleted Note Creation Guidelines**
-
-**Headings**:
-- Based on referenced topics, not categories like quotes or terms
-- Surrounded by **bold** formatting 
-- Not listed as bullet points
-- No space between headings and list items underneath
-
-**Emphasis**:
-- **Important terms** set in bold font
-- **Text ending in a colon**: also bolded
-
-**Review**:
-- Ensure adherence to specified format
-- Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]
-""",
-                                                     lines=3,
-                                                     visible=False)
+                    system_prompt_input = gr.Textbox(
+                        label="System Prompt",
+                        value="""<s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhere to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
+    **Bulleted Note Creation Guidelines**
+
+    **Headings**:
+    - Based on referenced topics, not categories like quotes or terms
+    - Surrounded by **bold** formatting 
+    - Not listed as bullet points
+    - No space between headings and list items underneath
+
+    **Emphasis**:
+    - **Important terms** set in bold font
+    - **Text ending in a colon**: also bolded
+
+    **Review**:
+    - Ensure adherence to specified format
+    - Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]
+    """,
+                        lines=3,
+                        visible=False
+                    )
 
                 custom_prompt_checkbox.change(
                     fn=lambda x: (gr.update(visible=x), gr.update(visible=x)),
                     inputs=[custom_prompt_checkbox],
                     outputs=[custom_prompt_input, system_prompt_input]
                 )
+
+                # Handle preset prompt checkbox change
+                def on_preset_prompt_checkbox_change(is_checked):
+                    if is_checked:
+                        prompts, total_pages, current_page = list_prompts(page=1, per_page=10)
+                        page_display_text = f"Page {current_page} of {total_pages}"
+                        return (
+                            gr.update(visible=True, interactive=True, choices=prompts),  # preset_prompt
+                            gr.update(visible=True),  # prev_page_button
+                            gr.update(visible=True),  # next_page_button
+                            gr.update(value=page_display_text, visible=True),  # page_display
+                            current_page,  # current_page_state
+                            total_pages   # total_pages_state
+                        )
+                    else:
+                        return (
+                            gr.update(visible=False, interactive=False),  # preset_prompt
+                            gr.update(visible=False),  # prev_page_button
+                            gr.update(visible=False),  # next_page_button
+                            gr.update(visible=False),  # page_display
+                            1,  # current_page_state
+                            1   # total_pages_state
+                        )
+
                 preset_prompt_checkbox.change(
-                    fn=lambda x: gr.update(visible=x),
+                    fn=on_preset_prompt_checkbox_change,
                     inputs=[preset_prompt_checkbox],
-                    outputs=[preset_prompt]
+                    outputs=[preset_prompt, prev_page_button, next_page_button, page_display, current_page_state, total_pages_state]
+                )
+
+                # Pagination button functions
+                def on_prev_page_click(current_page, total_pages):
+                    new_page = max(current_page - 1, 1)
+                    prompts, total_pages, current_page = list_prompts(page=new_page, per_page=10)
+                    page_display_text = f"Page {current_page} of {total_pages}"
+                    return (
+                        gr.update(choices=prompts),
+                        gr.update(value=page_display_text),
+                        current_page
+                    )
+
+                prev_page_button.click(
+                    fn=on_prev_page_click,
+                    inputs=[current_page_state, total_pages_state],
+                    outputs=[preset_prompt, page_display, current_page_state]
                 )
 
+                def on_next_page_click(current_page, total_pages):
+                    new_page = min(current_page + 1, total_pages)
+                    prompts, total_pages, current_page = list_prompts(page=new_page, per_page=10)
+                    page_display_text = f"Page {current_page} of {total_pages}"
+                    return (
+                        gr.update(choices=prompts),
+                        gr.update(value=page_display_text),
+                        current_page
+                    )
+
+                next_page_button.click(
+                    fn=on_next_page_click,
+                    inputs=[current_page_state, total_pages_state],
+                    outputs=[preset_prompt, page_display, current_page_state]
+                )
+
+                # Update prompts when a preset is selected
                 def update_prompts(preset_name):
                     prompts = update_user_prompt(preset_name)
                     return (
@@ -103,15 +196,14 @@ def create_audio_processing_tab():
 
                 preset_prompt.change(
                     update_prompts,
-                    inputs=preset_prompt,
+                    inputs=[preset_prompt],
                     outputs=[custom_prompt_input, system_prompt_input]
                 )
-
+                # Refactored API selection dropdown
                 api_name_input = gr.Dropdown(
-                    choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral", "OpenRouter",
-                             "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM","ollama", "HuggingFace", "Custom-OpenAI-API"],
-                    value=None,
-                    label="API for Summarization (Optional)"
+                    choices=["None"] + [format_api_name(api) for api in global_api_endpoints],
+                    value=default_value,
+                    label="API for Summarization/Analysis (Optional)"
                 )
                 api_key_input = gr.Textbox(label="API Key (if required)", placeholder="Enter your API key here", type="password")
                 custom_keywords_input = gr.Textbox(label="Custom Keywords", placeholder="Enter custom keywords, comma-separated")

App_Function_Libraries/Gradio_UI/Backup_Functionality.py

@@ -14,7 +14,7 @@ from App_Function_Libraries.DB.DB_Manager import create_automated_backup, db_pat
 #
 # Functions:
 
-def create_backup():
+def create_db_backup():
     backup_file = create_automated_backup(db_path, backup_dir)
     return f"Backup created: {backup_file}"
 
@@ -42,18 +42,7 @@ def create_backup_tab():
                 create_button = gr.Button("Create Backup")
                 create_output = gr.Textbox(label="Result")
             with gr.Column():
-                create_button.click(create_backup, inputs=[], outputs=create_output)
-
-
-def create_view_backups_tab():
-    with gr.TabItem("View Backups", visible=True):
-        gr.Markdown("# Browse available backups")
-        with gr.Row():
-            with gr.Column():
-                view_button = gr.Button("View Backups")
-            with gr.Column():
-                backup_list = gr.Textbox(label="Available Backups")
-                view_button.click(list_backups, inputs=[], outputs=backup_list)
+                create_button.click(create_db_backup, inputs=[], outputs=create_output)
 
 
 def create_restore_backup_tab():

App_Function_Libraries/Gradio_UI/Backup_RAG_Notes_Character_Chat_tab.py

@@ -0,0 +1,195 @@
+# Backup_Functionality.py
+# Functionality for managing database backups
+#
+# Imports:
+import os
+import shutil
+import gradio as gr
+from typing import Dict, List
+#
+# Local Imports:
+from App_Function_Libraries.DB.DB_Manager import create_automated_backup
+from App_Function_Libraries.DB.DB_Backups import create_backup, create_incremental_backup, restore_single_db_backup
+
+
+#
+# End of Imports
+#######################################################################################################################
+#
+# Functions:
+
+def get_db_specific_backups(backup_dir: str, db_name: str) -> List[str]:
+    """Get list of backups specific to a database."""
+    all_backups = [f for f in os.listdir(backup_dir) if f.endswith(('.db', '.sqlib'))]
+    db_specific_backups = [
+        backup for backup in all_backups
+        if backup.startswith(f"{db_name}_")
+    ]
+    return sorted(db_specific_backups, reverse=True)  # Most recent first
+
+def create_backup_tab(db_path: str, backup_dir: str, db_name: str):
+    """Create the backup creation tab for a database."""
+    gr.Markdown("## Create Database Backup")
+    gr.Markdown(f"This will create a backup in the directory: `{backup_dir}`")
+    with gr.Row():
+        with gr.Column():
+            #automated_backup_btn = gr.Button("Create Simple Backup")
+            full_backup_btn = gr.Button("Create Full Backup")
+            incr_backup_btn = gr.Button("Create Incremental Backup")
+        with gr.Column():
+            backup_output = gr.Textbox(label="Result")
+
+    def create_db_backup():
+        backup_file = create_automated_backup(db_path, backup_dir)
+        return f"Backup created: {backup_file}"
+
+    # automated_backup_btn.click(
+    #     fn=create_db_backup,
+    #     inputs=[],
+    #     outputs=[backup_output]
+    # )
+    full_backup_btn.click(
+        fn=lambda: create_backup(db_path, backup_dir, db_name),
+        inputs=[],
+        outputs=[backup_output]
+    )
+    incr_backup_btn.click(
+        fn=lambda: create_incremental_backup(db_path, backup_dir, db_name),
+        inputs=[],
+        outputs=[backup_output]
+    )
+
+def create_view_backups_tab(backup_dir: str, db_name: str):
+    """Create the backup viewing tab for a database."""
+    gr.Markdown("## Available Backups")
+    with gr.Row():
+        with gr.Column():
+            view_btn = gr.Button("Refresh Backup List")
+        with gr.Column():
+            backup_list = gr.Textbox(label="Available Backups")
+
+    def list_db_backups():
+        """List backups specific to this database."""
+        backups = get_db_specific_backups(backup_dir, db_name)
+        return "\n".join(backups) if backups else f"No backups found for {db_name} database"
+
+    view_btn.click(
+        fn=list_db_backups,
+        inputs=[],
+        outputs=[backup_list]
+    )
+
+def validate_backup_name(backup_name: str, db_name: str) -> bool:
+    """Validate that the backup name matches the database being restored."""
+    # Check if backup name starts with the database name prefix and has valid extension
+    valid_prefixes = [
+        f"{db_name}_backup_",    # Full backup prefix
+        f"{db_name}_incremental_"  # Incremental backup prefix
+    ]
+    has_valid_prefix = any(backup_name.startswith(prefix) for prefix in valid_prefixes)
+    has_valid_extension = backup_name.endswith(('.db', '.sqlib'))
+    return has_valid_prefix and has_valid_extension
+
+def create_restore_backup_tab(db_path: str, backup_dir: str, db_name: str):
+    """Create the backup restoration tab for a database."""
+    gr.Markdown("## Restore Database")
+    gr.Markdown("⚠️ **Warning**: Restoring a backup will overwrite the current database.")
+    with gr.Row():
+        with gr.Column():
+            backup_input = gr.Textbox(label="Backup Filename")
+            restore_btn = gr.Button("Restore", variant="primary")
+        with gr.Column():
+            restore_output = gr.Textbox(label="Result")
+
+    def secure_restore(backup_name: str) -> str:
+        """Restore backup with validation checks."""
+        if not backup_name:
+            return "Please enter a backup filename"
+
+        # Validate backup name format
+        if not validate_backup_name(backup_name, db_name):
+            return f"Invalid backup file. Please select a backup file that starts with '{db_name}_backup_' or '{db_name}_incremental_'"
+
+        # Check if backup exists
+        backup_path = os.path.join(backup_dir, backup_name)
+        if not os.path.exists(backup_path):
+            return f"Backup file not found: {backup_name}"
+
+        # Proceed with restore
+        return restore_single_db_backup(db_path, backup_dir, db_name, backup_name)
+
+    restore_btn.click(
+        fn=secure_restore,
+        inputs=[backup_input],
+        outputs=[restore_output]
+    )
+
+def create_media_db_tabs(db_config: Dict[str, str]):
+    """Create all tabs for the Media database."""
+    create_backup_tab(
+        db_path=db_config['db_path'],
+        backup_dir=db_config['backup_dir'],
+        db_name='media'
+    )
+    create_view_backups_tab(
+        backup_dir=db_config['backup_dir'],
+        db_name='media'
+    )
+    create_restore_backup_tab(
+        db_path=db_config['db_path'],
+        backup_dir=db_config['backup_dir'],
+        db_name='media'
+    )
+
+def create_rag_chat_tabs(db_config: Dict[str, str]):
+    """Create all tabs for the RAG Chat database."""
+    create_backup_tab(
+        db_path=db_config['db_path'],
+        backup_dir=db_config['backup_dir'],
+        db_name='rag_qa'  # Updated to match DB_Manager.py
+    )
+    create_view_backups_tab(
+        backup_dir=db_config['backup_dir'],
+        db_name='rag_qa'  # Updated to match DB_Manager.py
+    )
+    create_restore_backup_tab(
+        db_path=db_config['db_path'],
+        backup_dir=db_config['backup_dir'],
+        db_name='rag_qa'  # Updated to match DB_Manager.py
+    )
+
+def create_character_chat_tabs(db_config: Dict[str, str]):
+    """Create all tabs for the Character Chat database."""
+    create_backup_tab(
+        db_path=db_config['db_path'],
+        backup_dir=db_config['backup_dir'],
+        db_name='chatDB'  # Updated to match DB_Manager.py
+    )
+    create_view_backups_tab(
+        backup_dir=db_config['backup_dir'],
+        db_name='chatDB'  # Updated to match DB_Manager.py
+    )
+    create_restore_backup_tab(
+        db_path=db_config['db_path'],
+        backup_dir=db_config['backup_dir'],
+        db_name='chatDB'
+    )
+
+def create_database_management_interface(
+        media_db_config: Dict[str, str],
+        rag_db_config: Dict[str, str],
+        char_db_config: Dict[str, str]
+):
+    """Create the main database management interface with tabs for each database."""
+    with gr.TabItem("Media Database", id="media_db_group", visible=True):
+        create_media_db_tabs(media_db_config)
+
+    with gr.TabItem("RAG Chat Database", id="rag_chat_group", visible=True):
+        create_rag_chat_tabs(rag_db_config)
+
+    with gr.TabItem("Character Chat Database", id="character_chat_group", visible=True):
+        create_character_chat_tabs(char_db_config)
+
+#
+# End of Functions
+#######################################################################################################################

App_Function_Libraries/Gradio_UI/Book_Ingestion_tab.py

@@ -8,69 +8,113 @@
 #
 ####################
 # Imports
+import logging
 #
 # External Imports
 import gradio as gr
 #
 # Local Imports
-from App_Function_Libraries.Books.Book_Ingestion_Lib import process_zip_file, import_epub, import_file_handler
+from App_Function_Libraries.Books.Book_Ingestion_Lib import import_file_handler
+from App_Function_Libraries.Utils.Utils import default_api_endpoint, global_api_endpoints, format_api_name
 #
 ########################################################################################################################
 #
 # Functions:
 
-
-
 def create_import_book_tab():
+    try:
+        default_value = None
+        if default_api_endpoint:
+            if default_api_endpoint in global_api_endpoints:
+                default_value = format_api_name(default_api_endpoint)
+            else:
+                logging.warning(f"Default API endpoint '{default_api_endpoint}' not found in global_api_endpoints")
+    except Exception as e:
+        logging.error(f"Error setting default API endpoint: {str(e)}")
+        default_value = None
+
     with gr.TabItem("Ebook(epub) Files", visible=True):
         with gr.Row():
             with gr.Column():
                 gr.Markdown("# Import .epub files")
-                gr.Markdown("Upload a single .epub file or a .zip file containing multiple .epub files")
+                gr.Markdown("Upload multiple .epub files or a .zip file containing multiple .epub files")
                 gr.Markdown(
                     "🔗 **How to remove DRM from your ebooks:** [Reddit Guide](https://www.reddit.com/r/Calibre/comments/1ck4w8e/2024_guide_on_removing_drm_from_kobo_kindle_ebooks/)")
-                import_file = gr.File(label="Upload file for import", file_types=[".epub", ".zip"])
-                title_input = gr.Textbox(label="Title", placeholder="Enter the title of the content (for single files)")
-                author_input = gr.Textbox(label="Author", placeholder="Enter the author's name (for single files)")
-                keywords_input = gr.Textbox(label="Keywords (like genre or publish year)",
-                                            placeholder="Enter keywords, comma-separated")
-                system_prompt_input = gr.Textbox(label="System Prompt", lines=3,
-                                                 value=""""
-                                                    <s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
-                                                    **Bulleted Note Creation Guidelines**
-
-                                                    **Headings**:
-                                                    - Based on referenced topics, not categories like quotes or terms
-                                                    - Surrounded by **bold** formatting 
-                                                    - Not listed as bullet points
-                                                    - No space between headings and list items underneath
 
-                                                    **Emphasis**:
-                                                    - **Important terms** set in bold font
-                                                    - **Text ending in a colon**: also bolded
+                # Updated to support multiple files
+                import_files = gr.File(
+                    label="Upload files for import",
+                    file_count="multiple",
+                    file_types=[".epub", ".zip", ".html", ".htm", ".xml", ".opml"]
+                )
 
-                                                    **Review**:
-                                                    - Ensure adherence to specified format
-                                                    - Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]
-                                                """, )
-                custom_prompt_input = gr.Textbox(label="Custom User Prompt",
-                                                 placeholder="Enter a custom user prompt for summarization (optional)")
+                # Optional fields for overriding auto-extracted metadata
+                author_input = gr.Textbox(
+                    label="Author Override (optional)",
+                    placeholder="Enter author name to override auto-extracted metadata"
+                )
+                keywords_input = gr.Textbox(
+                    label="Keywords (like genre or publish year)",
+                    placeholder="Enter keywords, comma-separated - will be applied to all uploaded books"
+                )
+                system_prompt_input = gr.Textbox(
+                    label="System Prompt",
+                    lines=3,
+                    value=""""
+                        <s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
+                        **Bulleted Note Creation Guidelines**
+                        
+                        **Headings**:
+                        - Based on referenced topics, not categories like quotes or terms
+                        - Surrounded by **bold** formatting 
+                        - Not listed as bullet points
+                        - No space between headings and list items underneath
+                        
+                        **Emphasis**:
+                        - **Important terms** set in bold font
+                        - **Text ending in a colon**: also bolded
+                        
+                        **Review**:
+                        - Ensure adherence to specified format
+                        - Do not reference these instructions in your response.</s>[INST]
+                    """
+                )
+                custom_prompt_input = gr.Textbox(
+                    label="Custom User Prompt",
+                    placeholder="Enter a custom user prompt for summarization (optional)"
+                )
                 auto_summarize_checkbox = gr.Checkbox(label="Auto-summarize", value=False)
+
+                # API configuration
                 api_name_input = gr.Dropdown(
-                    choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral",
-                             "OpenRouter", "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace"],
-                    label="API for Auto-summarization"
+                    choices=["None"] + [format_api_name(api) for api in global_api_endpoints],
+                    value=default_value,
+                    label="API for Summarization/Analysis (Optional)"
                 )
                 api_key_input = gr.Textbox(label="API Key", type="password")
 
                 # Chunking options
-                max_chunk_size = gr.Slider(minimum=100, maximum=2000, value=500, step=50, label="Max Chunk Size")
-                chunk_overlap = gr.Slider(minimum=0, maximum=500, value=200, step=10, label="Chunk Overlap")
-                custom_chapter_pattern = gr.Textbox(label="Custom Chapter Pattern (optional)",
-                                                    placeholder="Enter a custom regex pattern for chapter detection")
+                max_chunk_size = gr.Slider(
+                    minimum=100,
+                    maximum=2000,
+                    value=500,
+                    step=50,
+                    label="Max Chunk Size"
+                )
+                chunk_overlap = gr.Slider(
+                    minimum=0,
+                    maximum=500,
+                    value=200,
+                    step=10,
+                    label="Chunk Overlap"
+                )
+                custom_chapter_pattern = gr.Textbox(
+                    label="Custom Chapter Pattern (optional)",
+                    placeholder="Enter a custom regex pattern for chapter detection"
+                )
 
+                import_button = gr.Button("Import eBooks")
 
-                import_button = gr.Button("Import eBook(s)")
             with gr.Column():
                 with gr.Row():
                     import_output = gr.Textbox(label="Import Status", lines=10, interactive=False)
@@ -78,10 +122,10 @@ def create_import_book_tab():
         import_button.click(
             fn=import_file_handler,
             inputs=[
-                import_file,
-                title_input,
+                import_files,  # Now handles multiple files
                 author_input,
                 keywords_input,
+                system_prompt_input,
                 custom_prompt_input,
                 auto_summarize_checkbox,
                 api_name_input,
@@ -93,8 +137,8 @@ def create_import_book_tab():
             outputs=import_output
         )
 
-    return import_file, title_input, author_input, keywords_input, system_prompt_input, custom_prompt_input, auto_summarize_checkbox, api_name_input, api_key_input, import_button, import_output
+    return import_files, author_input, keywords_input, system_prompt_input, custom_prompt_input, auto_summarize_checkbox, api_name_input, api_key_input, import_button, import_output
 
 #
 # End of File
-########################################################################################################################
+########################################################################################################################

App_Function_Libraries/Gradio_UI/Character_Chat_tab.py

@@ -2,10 +2,10 @@
 # Description: Library for character card import functions
 #
 # Imports
+from datetime import datetime
 import re
 import tempfile
 import uuid
-from datetime import datetime
 import json
 import logging
 import io
@@ -21,7 +21,7 @@ import gradio as gr
 from App_Function_Libraries.Character_Chat.Character_Chat_Lib import validate_character_book, validate_v2_card, \
     replace_placeholders, replace_user_placeholder, extract_json_from_image, parse_character_book, \
     load_chat_and_character, load_chat_history, load_character_and_image, extract_character_id, load_character_wrapper
-from App_Function_Libraries.Chat import chat
+from App_Function_Libraries.Chat.Chat_Functions import chat, approximate_token_count
 from App_Function_Libraries.DB.Character_Chat_DB import (
     add_character_card,
     get_character_cards,
@@ -32,9 +32,12 @@ from App_Function_Libraries.DB.Character_Chat_DB import (
     update_character_chat,
     delete_character_chat,
     delete_character_card,
-    update_character_card, search_character_chats,
+    update_character_card, search_character_chats, save_chat_history_to_character_db,
 )
-from App_Function_Libraries.Utils.Utils import sanitize_user_input
+from App_Function_Libraries.Utils.Utils import sanitize_user_input, format_api_name, global_api_endpoints, \
+    default_api_endpoint, load_comprehensive_config
+
+
 #
 ############################################################################################################
 #
@@ -252,8 +255,37 @@ def export_all_characters():
 # Gradio tabs
 
 def create_character_card_interaction_tab():
+    try:
+        default_value = None
+        if default_api_endpoint:
+            if default_api_endpoint in global_api_endpoints:
+                default_value = format_api_name(default_api_endpoint)
+            else:
+                logging.warning(f"Default API endpoint '{default_api_endpoint}' not found in global_api_endpoints")
+    except Exception as e:
+        logging.error(f"Error setting default API endpoint: {str(e)}")
+        default_value = None
     with gr.TabItem("Chat with a Character Card", visible=True):
         gr.Markdown("# Chat with a Character Card")
+        with gr.Row():
+            with gr.Column(scale=1):
+                # Checkbox to Decide Whether to Save Chats by Default
+                config = load_comprehensive_config()
+                auto_save_value = config.get('auto-save', 'save_character_chats', fallback='False')
+                auto_save_checkbox = gr.Checkbox(label="Save chats automatically", value=auto_save_value)
+                chat_media_name = gr.Textbox(label="Custom Chat Name (optional)", visible=True)
+                save_chat_history_to_db = gr.Button("Save Chat History to Database")
+                save_status = gr.Textbox(label="Status", interactive=False)
+            with gr.Column(scale=2):
+                gr.Markdown("## Search and Load Existing Chats")
+                chat_search_query = gr.Textbox(
+                    label="Search Chats",
+                    placeholder="Enter chat name or keywords to search"
+                )
+                chat_search_button = gr.Button("Search Chats")
+                chat_search_dropdown = gr.Dropdown(label="Search Results", choices=[], visible=False)
+                load_chat_button = gr.Button("Load Selected Chat", visible=False)
+
         with gr.Row():
             with gr.Column(scale=1):
                 character_image = gr.Image(label="Character Image", type="pil")
@@ -265,13 +297,10 @@ def create_character_card_interaction_tab():
                 load_characters_button = gr.Button("Load Existing Characters")
                 character_dropdown = gr.Dropdown(label="Select Character", choices=[])
                 user_name_input = gr.Textbox(label="Your Name", placeholder="Enter your name here")
+                # Refactored API selection dropdown
                 api_name_input = gr.Dropdown(
-                    choices=[
-                        "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral",
-                        "OpenRouter", "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace",
-                        "Custom-OpenAI-API"
-                    ],
-                    value="HuggingFace",
+                    choices=["None"] + [format_api_name(api) for api in global_api_endpoints],
+                    value=default_value,
                     label="API for Interaction (Mandatory)"
                 )
                 api_key_input = gr.Textbox(
@@ -281,24 +310,8 @@ def create_character_card_interaction_tab():
                 temperature_slider = gr.Slider(
                     minimum=0.0, maximum=2.0, value=0.7, step=0.05, label="Temperature"
                 )
-                import_chat_button = gr.Button("Import Chat History")
                 chat_file_upload = gr.File(label="Upload Chat History JSON", visible=True)
-
-                # Chat History Import and Search
-                gr.Markdown("## Search and Load Existing Chats")
-                chat_search_query = gr.Textbox(
-                    label="Search Chats",
-                    placeholder="Enter chat name or keywords to search"
-                )
-                chat_search_button = gr.Button("Search Chats")
-                chat_search_dropdown = gr.Dropdown(label="Search Results", choices=[], visible=False)
-                load_chat_button = gr.Button("Load Selected Chat", visible=False)
-
-                # Checkbox to Decide Whether to Save Chats by Default
-                auto_save_checkbox = gr.Checkbox(label="Save chats automatically", value=True)
-                chat_media_name = gr.Textbox(label="Custom Chat Name (optional)", visible=True)
-                save_chat_history_to_db = gr.Button("Save Chat History to Database")
-                save_status = gr.Textbox(label="Save Status", interactive=False)
+                import_chat_button = gr.Button("Import Chat History")
 
             with gr.Column(scale=2):
                 chat_history = gr.Chatbot(label="Conversation", height=800)
@@ -307,6 +320,7 @@ def create_character_card_interaction_tab():
                 answer_for_me_button = gr.Button("Answer for Me")
                 continue_talking_button = gr.Button("Continue Talking")
                 regenerate_button = gr.Button("Regenerate Last Message")
+                token_count_display = gr.Number(label="Approximate Token Count", value=0, interactive=False)
                 clear_chat_button = gr.Button("Clear Chat")
                 save_snapshot_button = gr.Button("Save Chat Snapshot")
                 update_chat_dropdown = gr.Dropdown(label="Select Chat to Update", choices=[], visible=False)
@@ -491,23 +505,114 @@ def create_character_card_interaction_tab():
 
             return history, save_status
 
+        def validate_chat_history(chat_history: List[Tuple[Optional[str], str]]) -> bool:
+            """
+            Validate the chat history format and content.
+
+            Args:
+                chat_history: List of message tuples (user_message, bot_message)
+
+            Returns:
+                bool: True if valid, False if invalid
+            """
+            if not isinstance(chat_history, list):
+                return False
+
+            for entry in chat_history:
+                if not isinstance(entry, tuple) or len(entry) != 2:
+                    return False
+                # First element can be None (for system messages) or str
+                if not (entry[0] is None or isinstance(entry[0], str)):
+                    return False
+                # Second element (bot response) must be str and not empty
+                if not isinstance(entry[1], str) or not entry[1].strip():
+                    return False
+
+            return True
+
+        def sanitize_conversation_name(name: str) -> str:
+            """
+            Sanitize the conversation name.
+
+            Args:
+                name: Raw conversation name
+
+            Returns:
+                str: Sanitized conversation name
+            """
+            # Remove any non-alphanumeric characters except spaces and basic punctuation
+            sanitized = re.sub(r'[^a-zA-Z0-9\s\-_.]', '', name)
+            # Limit length
+            sanitized = sanitized[:100]
+            # Ensure it's not empty
+            if not sanitized.strip():
+                sanitized = f"Chat_{datetime.now().strftime('%Y%m%d_%H%M%S')}"
+            return sanitized
+
         def save_chat_history_to_db_wrapper(
-            chat_history, conversation_id, media_content,
-            chat_media_name, char_data, auto_save
-        ):
-            if not char_data or not chat_history:
-                return "No character or chat history available.", ""
+                chat_history: List[Tuple[Optional[str], str]],
+                conversation_id: str,
+                media_content: Dict,
+                chat_media_name: str,
+                char_data: Dict,
+                auto_save: bool
+        ) -> Tuple[str, str]:
+            """
+            Save chat history to the database with validation.
 
-            character_id = char_data.get('id')
-            if not character_id:
-                return "Character ID not found.", ""
+            Args:
+                chat_history: List of message tuples
+                conversation_id: Current conversation ID
+                media_content: Media content metadata
+                chat_media_name: Custom name for the chat
+                char_data: Character data dictionary
+                auto_save: Auto-save flag
 
-            conversation_name = chat_media_name or f"Chat {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
-            chat_id = add_character_chat(character_id, conversation_name, chat_history)
-            if chat_id:
-                return f"Chat saved successfully with ID {chat_id}.", ""
-            else:
-                return "Failed to save chat.", ""
+            Returns:
+                Tuple[str, str]: (status message, detail message)
+            """
+            try:
+                # Basic input validation
+                if not chat_history:
+                    return "No chat history to save.", ""
+
+                if not validate_chat_history(chat_history):
+                    return "Invalid chat history format.", "Please ensure the chat history is valid."
+
+                if not char_data:
+                    return "No character selected.", "Please select a character first."
+
+                character_id = char_data.get('id')
+                if not character_id:
+                    return "Invalid character data: No character ID found.", ""
+
+                # Sanitize and prepare conversation name
+                conversation_name = sanitize_conversation_name(
+                    chat_media_name if chat_media_name.strip()
+                    else f"Chat with {char_data.get('name', 'Unknown')} - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
+                )
+
+                # Save to the database using your existing function
+                chat_id = save_chat_history_to_character_db(
+                    character_id=character_id,
+                    conversation_name=conversation_name,
+                    chat_history=chat_history
+                )
+
+                if chat_id:
+                    success_message = (
+                        f"Chat saved successfully!\n"
+                        f"ID: {chat_id}\n"
+                        f"Name: {conversation_name}\n"
+                        f"Messages: {len(chat_history)}"
+                    )
+                    return success_message, ""
+                else:
+                    return "Failed to save chat to database.", "Database operation failed."
+
+            except Exception as e:
+                logging.error(f"Error saving chat history: {str(e)}", exc_info=True)
+                return f"Error saving chat: {str(e)}", "Please check the logs for more details."
 
         def update_character_info(name):
             return load_character_and_image(name, user_name.value)
@@ -871,6 +976,10 @@ def create_character_card_interaction_tab():
                 auto_save_checkbox
             ],
             outputs=[chat_history, save_status]
+        ).then(
+            lambda history: approximate_token_count(history),
+            inputs=[chat_history],
+            outputs=[token_count_display]
         )
 
         continue_talking_button.click(
@@ -885,6 +994,10 @@ def create_character_card_interaction_tab():
                 auto_save_checkbox
             ],
             outputs=[chat_history, save_status]
+        ).then(
+            lambda history: approximate_token_count(history),
+            inputs=[chat_history],
+            outputs=[token_count_display]
         )
 
         import_card_button.click(
@@ -903,6 +1016,10 @@ def create_character_card_interaction_tab():
             fn=clear_chat_history,
             inputs=[character_data, user_name_input],
             outputs=[chat_history, character_data]
+        ).then(
+            lambda history: approximate_token_count(history),
+            inputs=[chat_history],
+            outputs=[token_count_display]
         )
 
         character_dropdown.change(
@@ -928,7 +1045,13 @@ def create_character_card_interaction_tab():
                 auto_save_checkbox
             ],
             outputs=[chat_history, save_status]
-        ).then(lambda: "", outputs=user_input)
+        ).then(
+            lambda: "", outputs=user_input
+        ).then(
+            lambda history: approximate_token_count(history),
+            inputs=[chat_history],
+            outputs=[token_count_display]
+        )
 
         regenerate_button.click(
             fn=regenerate_last_message,
@@ -942,6 +1065,10 @@ def create_character_card_interaction_tab():
                 auto_save_checkbox
             ],
             outputs=[chat_history, save_status]
+        ).then(
+            lambda history: approximate_token_count(history),
+            inputs=[chat_history],
+            outputs=[token_count_display]
         )
 
         import_chat_button.click(
@@ -951,8 +1078,12 @@ def create_character_card_interaction_tab():
 
         chat_file_upload.change(
             fn=import_chat_history,
-            inputs=[chat_file_upload, chat_history, character_data],
+            inputs=[chat_file_upload, chat_history, character_data, user_name_input],
             outputs=[chat_history, character_data, save_status]
+        ).then(
+            lambda history: approximate_token_count(history),
+            inputs=[chat_history],
+            outputs=[token_count_display]
         )
 
         save_chat_history_to_db.click(
@@ -1009,6 +1140,10 @@ def create_character_card_interaction_tab():
             fn=load_selected_chat_from_search,
             inputs=[chat_search_dropdown, user_name_input],
             outputs=[character_data, chat_history, character_image, save_status]
+        ).then(
+            lambda history: approximate_token_count(history),
+            inputs=[chat_history],
+            outputs=[token_count_display]
         )
 
         # Show Load Chat Button when a chat is selected
@@ -1023,8 +1158,8 @@ def create_character_card_interaction_tab():
 
 
 def create_character_chat_mgmt_tab():
-    with gr.TabItem("Character and Chat Management", visible=True):
-        gr.Markdown("# Character and Chat Management")
+    with gr.TabItem("Character Chat Management", visible=True):
+        gr.Markdown("# Character Chat Management")
 
         with gr.Row():
             # Left Column: Character Import and Chat Management
@@ -1057,13 +1192,17 @@ def create_character_chat_mgmt_tab():
             gr.Markdown("## Chat Management")
             select_chat = gr.Dropdown(label="Select Chat", choices=[], visible=False, interactive=True)
             load_chat_button = gr.Button("Load Selected Chat", visible=False)
-            conversation_list = gr.Dropdown(label="Select Conversation or Character", choices=[])
+            conversation_list = gr.Dropdown(label="Select Conversation", choices=[])
             conversation_mapping = gr.State({})
 
         with gr.Tabs():
             with gr.TabItem("Edit", visible=True):
                 chat_content = gr.TextArea(label="Chat/Character Content (JSON)", lines=20, max_lines=50)
                 save_button = gr.Button("Save Changes")
+                export_chat_button = gr.Button("Export Current Conversation", variant="secondary")
+                export_all_chats_button = gr.Button("Export All Character Conversations", variant="secondary")
+                export_file = gr.File(label="Downloaded File", visible=False)
+                export_status = gr.Markdown("")
                 delete_button = gr.Button("Delete Conversation/Character", variant="stop")
 
             with gr.TabItem("Preview", visible=True):
@@ -1306,6 +1445,90 @@ def create_character_chat_mgmt_tab():
 
             return "Import results:\n" + "\n".join(results)
 
+        def export_current_conversation(selected_chat):
+            if not selected_chat:
+                return "Please select a conversation to export.", None
+
+            try:
+                chat_id = int(selected_chat.split('(ID: ')[1].rstrip(')'))
+                chat = get_character_chat_by_id(chat_id)
+
+                if not chat:
+                    return "Selected chat not found.", None
+
+                # Ensure chat_history is properly parsed
+                chat_history = chat['chat_history']
+                if isinstance(chat_history, str):
+                    chat_history = json.loads(chat_history)
+
+                export_data = {
+                    "conversation_id": chat['id'],
+                    "conversation_name": chat['conversation_name'],
+                    "character_id": chat['character_id'],
+                    "chat_history": chat_history,
+                    "exported_at": datetime.now().isoformat()
+                }
+
+                # Convert to JSON string
+                json_str = json.dumps(export_data, indent=2, ensure_ascii=False)
+
+                # Create file name
+                file_name = f"conversation_{chat['id']}_{chat['conversation_name']}.json"
+
+                # Return file for download
+                return "Conversation exported successfully!", (file_name, json_str, "application/json")
+
+            except Exception as e:
+                logging.error(f"Error exporting conversation: {e}")
+                return f"Error exporting conversation: {str(e)}", None
+
+        def export_all_character_conversations(character_selection):
+            if not character_selection:
+                return "Please select a character first.", None
+
+            try:
+                character_id = int(character_selection.split('(ID: ')[1].rstrip(')'))
+                character = get_character_card_by_id(character_id)
+                chats = get_character_chats(character_id=character_id)
+
+                if not chats:
+                    return "No conversations found for this character.", None
+
+                # Process chat histories
+                conversations = []
+                for chat in chats:
+                    chat_history = chat['chat_history']
+                    if isinstance(chat_history, str):
+                        chat_history = json.loads(chat_history)
+
+                    conversations.append({
+                        "conversation_id": chat['id'],
+                        "conversation_name": chat['conversation_name'],
+                        "chat_history": chat_history
+                    })
+
+                export_data = {
+                    "character": {
+                        "id": character['id'],
+                        "name": character['name']
+                    },
+                    "conversations": conversations,
+                    "exported_at": datetime.now().isoformat()
+                }
+
+                # Convert to JSON string
+                json_str = json.dumps(export_data, indent=2, ensure_ascii=False)
+
+                # Create file name
+                file_name = f"all_conversations_{character['name']}_{character['id']}.json"
+
+                # Return file for download
+                return "All conversations exported successfully!", (file_name, json_str, "application/json")
+
+            except Exception as e:
+                logging.error(f"Error exporting all conversations: {e}")
+                return f"Error exporting conversations: {str(e)}", None
+
         # Register new callback for character import
         import_characters_button.click(
             fn=import_multiple_characters,
@@ -1368,6 +1591,18 @@ def create_character_chat_mgmt_tab():
             outputs=select_character
         )
 
+        export_chat_button.click(
+            fn=export_current_conversation,
+            inputs=[select_chat],
+            outputs=[export_status, export_file]
+        )
+
+        export_all_chats_button.click(
+            fn=export_all_character_conversations,
+            inputs=[select_character],
+            outputs=[export_status, export_file]
+        )
+
         return (
             character_files, import_characters_button, import_status,
             search_query, search_button, search_results, search_status,

App_Function_Libraries/Gradio_UI/Character_interaction_tab.py

@@ -17,9 +17,12 @@ import gradio as gr
 from PIL import Image
 #
 # Local Imports
-from App_Function_Libraries.Chat import chat, load_characters, save_chat_history_to_db_wrapper
+from App_Function_Libraries.Chat.Chat_Functions import chat, load_characters, save_chat_history_to_db_wrapper
 from App_Function_Libraries.Gradio_UI.Chat_ui import chat_wrapper
 from App_Function_Libraries.Gradio_UI.Writing_tab import generate_writing_feedback
+from App_Function_Libraries.Utils.Utils import default_api_endpoint, format_api_name, global_api_endpoints
+
+
 #
 ########################################################################################################################
 #
@@ -253,6 +256,16 @@ def character_interaction(character1: str, character2: str, api_endpoint: str, a
 
 
 def create_multiple_character_chat_tab():
+    try:
+        default_value = None
+        if default_api_endpoint:
+            if default_api_endpoint in global_api_endpoints:
+                default_value = format_api_name(default_api_endpoint)
+            else:
+                logging.warning(f"Default API endpoint '{default_api_endpoint}' not found in global_api_endpoints")
+    except Exception as e:
+        logging.error(f"Error setting default API endpoint: {str(e)}")
+        default_value = None
     with gr.TabItem("Multi-Character Chat", visible=True):
         characters, conversation, current_character, other_character = character_interaction_setup()
 
@@ -264,13 +277,12 @@ def create_multiple_character_chat_tab():
                 character_selectors = [gr.Dropdown(label=f"Character {i + 1}", choices=list(characters.keys())) for i in
                                        range(4)]
 
-            api_endpoint = gr.Dropdown(label="API Endpoint",
-                                       choices=["Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek",
-                                                "Mistral",
-                                                "OpenRouter", "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM",
-                                                "ollama", "HuggingFace",
-                                                "Custom-OpenAI-API"],
-                                        value="HuggingFace")
+            # Refactored API selection dropdown
+            api_endpoint = gr.Dropdown(
+                choices=["None"] + [format_api_name(api) for api in global_api_endpoints],
+                value=default_value,
+                label="API for Interaction (Optional)"
+            )
             api_key = gr.Textbox(label="API Key (if required)", type="password")
             temperature = gr.Slider(label="Temperature", minimum=0.1, maximum=1.0, step=0.1, value=0.7)
             scenario = gr.Textbox(label="Scenario (optional)", lines=3)
@@ -393,17 +405,26 @@ def create_multiple_character_chat_tab():
 
 # From `Fuzzlewumper` on Reddit.
 def create_narrator_controlled_conversation_tab():
+    try:
+        default_value = None
+        if default_api_endpoint:
+            if default_api_endpoint in global_api_endpoints:
+                default_value = format_api_name(default_api_endpoint)
+            else:
+                logging.warning(f"Default API endpoint '{default_api_endpoint}' not found in global_api_endpoints")
+    except Exception as e:
+        logging.error(f"Error setting default API endpoint: {str(e)}")
+        default_value = None
     with gr.TabItem("Narrator-Controlled Conversation", visible=True):
         gr.Markdown("# Narrator-Controlled Conversation")
 
         with gr.Row():
             with gr.Column(scale=1):
+                # Refactored API selection dropdown
                 api_endpoint = gr.Dropdown(
-                    label="API Endpoint",
-                    choices=["Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral",
-                             "OpenRouter", "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace",
-                             "Custom-OpenAI-API"],
-                    value="HuggingFace"
+                    choices=["None"] + [format_api_name(api) for api in global_api_endpoints],
+                    value=default_value,
+                    label="API for Chat Interaction (Optional)"
                 )
                 api_key = gr.Textbox(label="API Key (if required)", type="password")
                 temperature = gr.Slider(label="Temperature", minimum=0.1, maximum=1.0, step=0.1, value=0.7)

App_Function_Libraries/Gradio_UI/Chat_ui.py

@@ -2,23 +2,25 @@
 # Description: Chat interface functions for Gradio
 #
 # Imports
-import html
-import json
 import logging
 import os
 import sqlite3
+import time
 from datetime import datetime
 #
 # External Imports
 import gradio as gr
 #
 # Local Imports
-from App_Function_Libraries.Chat import chat, save_chat_history, update_chat_content, save_chat_history_to_db_wrapper
-from App_Function_Libraries.DB.DB_Manager import add_chat_message, search_chat_conversations, create_chat_conversation, \
-    get_chat_messages, update_chat_message, delete_chat_message, load_preset_prompts, db
+from App_Function_Libraries.Chat.Chat_Functions import approximate_token_count, chat, save_chat_history, \
+    update_chat_content, save_chat_history_to_db_wrapper
+from App_Function_Libraries.DB.DB_Manager import db, load_chat_history, start_new_conversation, \
+    save_message, search_conversations_by_keywords, \
+    get_all_conversations, delete_messages_in_conversation, search_media_db, list_prompts
+from App_Function_Libraries.DB.RAG_QA_Chat_DB import get_db_connection
 from App_Function_Libraries.Gradio_UI.Gradio_Shared import update_dropdown, update_user_prompt
-
-
+from App_Function_Libraries.Metrics.metrics_logger import log_counter, log_histogram
+from App_Function_Libraries.Utils.Utils import default_api_endpoint, format_api_name, global_api_endpoints
 #
 #
 ########################################################################################################################
@@ -91,10 +93,9 @@ def chat_wrapper(message, history, media_content, selected_parts, api_endpoint,
                 # Create a new conversation
                 media_id = media_content.get('id', None)
                 conversation_name = f"Chat about {media_content.get('title', 'Unknown Media')} - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
-                conversation_id = create_chat_conversation(media_id, conversation_name)
-
+                conversation_id = start_new_conversation(title=conversation_name, media_id=media_id)
             # Add user message to the database
-            user_message_id = add_chat_message(conversation_id, "user", message)
+            user_message_id = save_message(conversation_id, role="user", content=message)
 
         # Include the selected parts and custom_prompt only for the first message
         if not history and selected_parts:
@@ -113,7 +114,7 @@ def chat_wrapper(message, history, media_content, selected_parts, api_endpoint,
 
         if save_conversation:
             # Add assistant message to the database
-            add_chat_message(conversation_id, "assistant", bot_message)
+            save_message(conversation_id, role="assistant", content=bot_message)
 
         # Update history
         new_history = history + [(message, bot_message)]
@@ -123,51 +124,57 @@ def chat_wrapper(message, history, media_content, selected_parts, api_endpoint,
         logging.error(f"Error in chat wrapper: {str(e)}")
         return "An error occurred.", history, conversation_id
 
+
 def search_conversations(query):
+    """Convert existing chat search to use RAG chat functions"""
     try:
-        conversations = search_chat_conversations(query)
-        if not conversations:
-            print(f"Debug - Search Conversations - No results found for query: {query}")
+        # Use the RAG search function - search by title if given a query
+        if query and query.strip():
+            results, _, _ = search_conversations_by_keywords(
+                title_query=query.strip()
+            )
+        else:
+            # Get all conversations if no query
+            results, _, _ = get_all_conversations()
+
+        if not results:
             return gr.update(choices=[])
 
+        # Format choices to match existing UI format
         conversation_options = [
-            (f"{c['conversation_name']} (Media: {c['media_title']}, ID: {c['id']})", c['id'])
-            for c in conversations
+            (f"{conv['title']} (ID: {conv['conversation_id'][:8]})", conv['conversation_id'])
+            for conv in results
         ]
-        print(f"Debug - Search Conversations - Options: {conversation_options}")
+
         return gr.update(choices=conversation_options)
     except Exception as e:
-        print(f"Debug - Search Conversations - Error: {str(e)}")
+        logging.error(f"Error searching conversations: {str(e)}")
         return gr.update(choices=[])
 
 
 def load_conversation(conversation_id):
+    """Convert existing load to use RAG chat functions"""
     if not conversation_id:
         return [], None
 
-    messages = get_chat_messages(conversation_id)
-    history = [
-        (msg['message'], None) if msg['sender'] == 'user' else (None, msg['message'])
-        for msg in messages
-    ]
-    return history, conversation_id
-
-
-def update_message_in_chat(message_id, new_text, history):
-    update_chat_message(message_id, new_text)
-    updated_history = [(msg1, msg2) if msg1[1] != message_id and msg2[1] != message_id
-                       else ((new_text, msg1[1]) if msg1[1] == message_id else (new_text, msg2[1]))
-                       for msg1, msg2 in history]
-    return updated_history
+    try:
+        # Use RAG load function
+        messages, _, _ = load_chat_history(conversation_id)
 
+        # Convert to chatbot history format
+        history = [
+            (content, None) if role == 'user' else (None, content)
+            for role, content in messages
+        ]
 
-def delete_message_from_chat(message_id, history):
-    delete_chat_message(message_id)
-    updated_history = [(msg1, msg2) for msg1, msg2 in history if msg1[1] != message_id and msg2[1] != message_id]
-    return updated_history
+        return history, conversation_id
+    except Exception as e:
+        logging.error(f"Error loading conversation: {str(e)}")
+        return [], None
 
 
-def regenerate_last_message(history, media_content, selected_parts, api_endpoint, api_key, custom_prompt, temperature, system_prompt):
+def regenerate_last_message(history, media_content, selected_parts, api_endpoint, api_key, custom_prompt, temperature,
+                            system_prompt):
     if not history:
         return history, "No messages to regenerate."
 
@@ -200,7 +207,56 @@ def regenerate_last_message(history, media_content, selected_parts, api_endpoint
 
     return new_history, "Last message regenerated successfully."
 
+
+def update_dropdown_multiple(query, search_type, keywords=""):
+    """Updated function to handle multiple search results using search_media_db"""
+    try:
+        # Define search fields based on search type
+        search_fields = []
+        if search_type.lower() == "keyword":
+            # When searching by keyword, we'll search across multiple fields
+            search_fields = ["title", "content", "author"]
+        else:
+            # Otherwise use the specific field
+            search_fields = [search_type.lower()]
+
+        # Perform the search
+        results = search_media_db(
+            search_query=query,
+            search_fields=search_fields,
+            keywords=keywords,
+            page=1,
+            results_per_page=50  # Adjust as needed
+        )
+
+        # Process results
+        item_map = {}
+        formatted_results = []
+
+        for row in results:
+            id, url, title, type_, content, author, date, prompt, summary = row
+            # Create a display text that shows relevant info
+            display_text = f"{title} - {author or 'Unknown'} ({date})"
+            formatted_results.append(display_text)
+            item_map[display_text] = id
+
+        return gr.update(choices=formatted_results), item_map
+    except Exception as e:
+        logging.error(f"Error in update_dropdown_multiple: {str(e)}")
+        return gr.update(choices=[]), {}
+
+
 def create_chat_interface():
+    try:
+        default_value = None
+        if default_api_endpoint:
+            if default_api_endpoint in global_api_endpoints:
+                default_value = format_api_name(default_api_endpoint)
+            else:
+                logging.warning(f"Default API endpoint '{default_api_endpoint}' not found in global_api_endpoints")
+    except Exception as e:
+        logging.error(f"Error setting default API endpoint: {str(e)}")
+        default_value = None
     custom_css = """
     .chatbot-container .message-wrap .message {
         font-size: 14px !important;
@@ -215,9 +271,19 @@ def create_chat_interface():
 
         with gr.Row():
             with gr.Column(scale=1):
-                search_query_input = gr.Textbox(label="Search Query", placeholder="Enter your search query here...")
-                search_type_input = gr.Radio(choices=["Title", "URL", "Keyword", "Content"], value="Title",
-                                             label="Search By")
+                search_query_input = gr.Textbox(
+                    label="Search Query",
+                    placeholder="Enter your search query here..."
+                )
+                search_type_input = gr.Radio(
+                    choices=["Title", "Content", "Author", "Keyword"],
+                    value="Keyword",
+                    label="Search By"
+                )
+                keyword_filter_input = gr.Textbox(
+                    label="Filter by Keywords (comma-separated)",
+                    placeholder="ml, ai, python, etc..."
+                )
                 search_button = gr.Button("Search")
                 items_output = gr.Dropdown(label="Select Item", choices=[], interactive=True)
                 item_mapping = gr.State({})
@@ -237,53 +303,60 @@ def create_chat_interface():
                 with gr.Row():
                     load_conversations_btn = gr.Button("Load Selected Conversation")
 
-                api_endpoint = gr.Dropdown(label="Select API Endpoint",
-                                           choices=["Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek",
-                                                    "Mistral", "OpenRouter",
-                                                    "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama",
-                                                    "HuggingFace"])
+                # Refactored API selection dropdown
+                api_endpoint = gr.Dropdown(
+                    choices=["None"] + [format_api_name(api) for api in global_api_endpoints],
+                    value=default_value,
+                    label="API for Chat Interaction (Optional)"
+                )
                 api_key = gr.Textbox(label="API Key (if required)", type="password")
+
+                # Initialize state variables for pagination
+                current_page_state = gr.State(value=1)
+                total_pages_state = gr.State(value=1)
+
                 custom_prompt_checkbox = gr.Checkbox(label="Use a Custom Prompt",
                                                      value=False,
                                                      visible=True)
                 preset_prompt_checkbox = gr.Checkbox(label="Use a pre-set Prompt",
                                                      value=False,
                                                      visible=True)
-                preset_prompt = gr.Dropdown(label="Select Preset Prompt",
-                                            choices=load_preset_prompts(),
-                                            visible=False)
-                user_prompt = gr.Textbox(label="Custom Prompt",
-                                         placeholder="Enter custom prompt here",
-                                         lines=3,
-                                         visible=False)
-                system_prompt_input = gr.Textbox(label="System Prompt",
-                                                 value="You are a helpful AI assitant",
-                                                 lines=3,
-                                                 visible=False)
+                with gr.Row():
+                    # Add pagination controls
+                    preset_prompt = gr.Dropdown(label="Select Preset Prompt",
+                                                choices=[],
+                                                visible=False)
+                with gr.Row():
+                    prev_page_button = gr.Button("Previous Page", visible=False)
+                    page_display = gr.Markdown("Page 1 of X", visible=False)
+                    next_page_button = gr.Button("Next Page", visible=False)
+                    system_prompt_input = gr.Textbox(label="System Prompt",
+                                                     value="You are a helpful AI assistant",
+                                                     lines=3,
+                                                     visible=False)
+                with gr.Row():
+                    user_prompt = gr.Textbox(label="Custom Prompt",
+                                             placeholder="Enter custom prompt here",
+                                             lines=3,
+                                             visible=False)
             with gr.Column(scale=2):
-                chatbot = gr.Chatbot(height=600, elem_classes="chatbot-container")
+                chatbot = gr.Chatbot(height=800, elem_classes="chatbot-container")
                 msg = gr.Textbox(label="Enter your message")
                 submit = gr.Button("Submit")
                 regenerate_button = gr.Button("Regenerate Last Message")
+                token_count_display = gr.Number(label="Approximate Token Count", value=0, interactive=False)
                 clear_chat_button = gr.Button("Clear Chat")
 
-                edit_message_id = gr.Number(label="Message ID to Edit", visible=False)
-                edit_message_text = gr.Textbox(label="Edit Message", visible=False)
-                update_message_button = gr.Button("Update Message", visible=False)
-
-                delete_message_id = gr.Number(label="Message ID to Delete", visible=False)
-                delete_message_button = gr.Button("Delete Message", visible=False)
-
                 chat_media_name = gr.Textbox(label="Custom Chat Name(optional)")
                 save_chat_history_to_db = gr.Button("Save Chat History to DataBase")
+                save_status = gr.Textbox(label="Save Status", interactive=False)
                 save_chat_history_as_file = gr.Button("Save Chat History as File")
                 download_file = gr.File(label="Download Chat History")
-                save_status = gr.Textbox(label="Save Status", interactive=False)
 
         # Restore original functionality
         search_button.click(
-            fn=update_dropdown,
-            inputs=[search_query_input, search_type_input],
+            fn=update_dropdown_multiple,
+            inputs=[search_query_input, search_type_input, keyword_filter_input],
             outputs=[items_output, item_mapping]
         )
 
@@ -314,21 +387,72 @@ def create_chat_interface():
             clear_chat,
             outputs=[chatbot, conversation_id]
         )
+
+        # Function to handle preset prompt checkbox change
+        def on_preset_prompt_checkbox_change(is_checked):
+            if is_checked:
+                prompts, total_pages, current_page = list_prompts(page=1, per_page=20)
+                page_display_text = f"Page {current_page} of {total_pages}"
+                return (
+                    gr.update(visible=True, interactive=True, choices=prompts),  # preset_prompt
+                    gr.update(visible=True),  # prev_page_button
+                    gr.update(visible=True),  # next_page_button
+                    gr.update(value=page_display_text, visible=True),  # page_display
+                    current_page,  # current_page_state
+                    total_pages   # total_pages_state
+                )
+            else:
+                return (
+                    gr.update(visible=False, interactive=False),  # preset_prompt
+                    gr.update(visible=False),  # prev_page_button
+                    gr.update(visible=False),  # next_page_button
+                    gr.update(visible=False),  # page_display
+                    1,  # current_page_state
+                    1   # total_pages_state
+                )
+
+        preset_prompt_checkbox.change(
+            fn=on_preset_prompt_checkbox_change,
+            inputs=[preset_prompt_checkbox],
+            outputs=[preset_prompt, prev_page_button, next_page_button, page_display, current_page_state, total_pages_state]
+        )
+
+        def on_prev_page_click(current_page, total_pages):
+            new_page = max(current_page - 1, 1)
+            prompts, total_pages, current_page = list_prompts(page=new_page, per_page=20)
+            page_display_text = f"Page {current_page} of {total_pages}"
+            return gr.update(choices=prompts), gr.update(value=page_display_text), current_page
+
+        prev_page_button.click(
+            fn=on_prev_page_click,
+            inputs=[current_page_state, total_pages_state],
+            outputs=[preset_prompt, page_display, current_page_state]
+        )
+
+        def on_next_page_click(current_page, total_pages):
+            new_page = min(current_page + 1, total_pages)
+            prompts, total_pages, current_page = list_prompts(page=new_page, per_page=20)
+            page_display_text = f"Page {current_page} of {total_pages}"
+            return gr.update(choices=prompts), gr.update(value=page_display_text), current_page
+
+        next_page_button.click(
+            fn=on_next_page_click,
+            inputs=[current_page_state, total_pages_state],
+            outputs=[preset_prompt, page_display, current_page_state]
+        )
+
         preset_prompt.change(
             update_prompts,
-            inputs=preset_prompt,
+            inputs=[preset_prompt],
             outputs=[user_prompt, system_prompt_input]
         )
+
         custom_prompt_checkbox.change(
             fn=lambda x: (gr.update(visible=x), gr.update(visible=x)),
             inputs=[custom_prompt_checkbox],
             outputs=[user_prompt, system_prompt_input]
         )
-        preset_prompt_checkbox.change(
-            fn=lambda x: gr.update(visible=x),
-            inputs=[preset_prompt_checkbox],
-            outputs=[preset_prompt]
-        )
+
         submit.click(
             chat_wrapper,
             inputs=[msg, chatbot, media_content, selected_parts, api_endpoint, api_key, user_prompt, conversation_id,
@@ -341,6 +465,10 @@ def create_chat_interface():
         ).then(  # Clear the user prompt after the first message
             lambda: (gr.update(value=""), gr.update(value="")),
             outputs=[user_prompt, system_prompt_input]
+        ).then(
+            lambda history: approximate_token_count(history),
+            inputs=[chatbot],
+            outputs=[token_count_display]
         )
 
         items_output.change(
@@ -348,6 +476,7 @@ def create_chat_interface():
             inputs=[items_output, use_content, use_summary, use_prompt, item_mapping],
             outputs=[media_content, selected_parts]
         )
+
         use_content.change(update_selected_parts, inputs=[use_content, use_summary, use_prompt],
                            outputs=[selected_parts])
         use_summary.change(update_selected_parts, inputs=[use_content, use_summary, use_prompt],
@@ -377,18 +506,6 @@ def create_chat_interface():
             outputs=[chat_history]
         )
 
-        update_message_button.click(
-            update_message_in_chat,
-            inputs=[edit_message_id, edit_message_text, chat_history],
-            outputs=[chatbot]
-        )
-
-        delete_message_button.click(
-            delete_message_from_chat,
-            inputs=[delete_message_id, chat_history],
-            outputs=[chatbot]
-        )
-
         save_chat_history_as_file.click(
             save_chat_history,
             inputs=[chatbot, conversation_id],
@@ -403,15 +520,28 @@ def create_chat_interface():
 
         regenerate_button.click(
             regenerate_last_message,
-            inputs=[chatbot, media_content, selected_parts, api_endpoint, api_key, user_prompt, temperature, system_prompt_input],
+            inputs=[chatbot, media_content, selected_parts, api_endpoint, api_key, user_prompt, temperature,
+                    system_prompt_input],
             outputs=[chatbot, save_status]
+        ).then(
+            lambda history: approximate_token_count(history),
+            inputs=[chatbot],
+            outputs=[token_count_display]
         )
 
-        chatbot.select(show_edit_message, None, [edit_message_text, edit_message_id, update_message_button])
-        chatbot.select(show_delete_message, None, [delete_message_id, delete_message_button])
-
 
 def create_chat_interface_stacked():
+    try:
+        default_value = None
+        if default_api_endpoint:
+            if default_api_endpoint in global_api_endpoints:
+                default_value = format_api_name(default_api_endpoint)
+            else:
+                logging.warning(f"Default API endpoint '{default_api_endpoint}' not found in global_api_endpoints")
+    except Exception as e:
+        logging.error(f"Error setting default API endpoint: {str(e)}")
+        default_value = None
+
     custom_css = """
     .chatbot-container .message-wrap .message {
         font-size: 14px !important;
@@ -426,9 +556,19 @@ def create_chat_interface_stacked():
 
         with gr.Row():
             with gr.Column():
-                search_query_input = gr.Textbox(label="Search Query", placeholder="Enter your search query here...")
-                search_type_input = gr.Radio(choices=["Title", "URL", "Keyword", "Content"], value="Title",
-                                             label="Search By")
+                search_query_input = gr.Textbox(
+                    label="Search Query",
+                    placeholder="Enter your search query here..."
+                )
+                search_type_input = gr.Radio(
+                    choices=["Title", "Content", "Author", "Keyword"],
+                    value="Keyword",
+                    label="Search By"
+                )
+                keyword_filter_input = gr.Textbox(
+                    label="Filter by Keywords (comma-separated)",
+                    placeholder="ml, ai, python, etc..."
+                )
                 search_button = gr.Button("Search")
                 items_output = gr.Dropdown(label="Select Item", choices=[], interactive=True)
                 item_mapping = gr.State({})
@@ -446,45 +586,165 @@ def create_chat_interface_stacked():
                     search_conversations_btn = gr.Button("Search Conversations")
                     load_conversations_btn = gr.Button("Load Selected Conversation")
             with gr.Column():
-                api_endpoint = gr.Dropdown(label="Select API Endpoint",
-                                           choices=["Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek",
-                                                    "OpenRouter", "Mistral", "Llama.cpp", "Kobold", "Ooba", "Tabbyapi",
-                                                    "VLLM", "ollama", "HuggingFace"])
+                # Refactored API selection dropdown
+                api_endpoint = gr.Dropdown(
+                    choices=["None"] + [format_api_name(api) for api in global_api_endpoints],
+                    value=default_value,
+                    label="API for Chat Interaction (Optional)"
+                )
                 api_key = gr.Textbox(label="API Key (if required)", type="password")
-                preset_prompt = gr.Dropdown(label="Select Preset Prompt",
-                                            choices=load_preset_prompts(),
-                                            visible=True)
-                system_prompt = gr.Textbox(label="System Prompt",
-                                           value="You are a helpful AI assistant.",
-                                           lines=3,
-                                           visible=True)
-                user_prompt = gr.Textbox(label="Custom User Prompt",
-                                         placeholder="Enter custom prompt here",
-                                         lines=3,
-                                         visible=True)
+
+                # Initialize state variables for pagination
+                current_page_state = gr.State(value=1)
+                total_pages_state = gr.State(value=1)
+
+                custom_prompt_checkbox = gr.Checkbox(
+                    label="Use a Custom Prompt",
+                    value=False,
+                    visible=True
+                )
+                preset_prompt_checkbox = gr.Checkbox(
+                    label="Use a pre-set Prompt",
+                    value=False,
+                    visible=True
+                )
+
+                with gr.Row():
+                    preset_prompt = gr.Dropdown(
+                        label="Select Preset Prompt",
+                        choices=[],
+                        visible=False
+                    )
+                with gr.Row():
+                    prev_page_button = gr.Button("Previous Page", visible=False)
+                    page_display = gr.Markdown("Page 1 of X", visible=False)
+                    next_page_button = gr.Button("Next Page", visible=False)
+
+                system_prompt = gr.Textbox(
+                    label="System Prompt",
+                    value="You are a helpful AI assistant.",
+                    lines=4,
+                    visible=False
+                )
+                user_prompt = gr.Textbox(
+                    label="Custom User Prompt",
+                    placeholder="Enter custom prompt here",
+                    lines=4,
+                    visible=False
+                )
                 gr.Markdown("Scroll down for the chat window...")
         with gr.Row():
             with gr.Column(scale=1):
-                chatbot = gr.Chatbot(height=600, elem_classes="chatbot-container")
+                chatbot = gr.Chatbot(height=800, elem_classes="chatbot-container")
                 msg = gr.Textbox(label="Enter your message")
         with gr.Row():
             with gr.Column():
                 submit = gr.Button("Submit")
                 regenerate_button = gr.Button("Regenerate Last Message")
+                token_count_display = gr.Number(label="Approximate Token Count", value=0, interactive=False)
                 clear_chat_button = gr.Button("Clear Chat")
                 chat_media_name = gr.Textbox(label="Custom Chat Name(optional)", visible=True)
                 save_chat_history_to_db = gr.Button("Save Chat History to DataBase")
+                save_status = gr.Textbox(label="Save Status", interactive=False)
                 save_chat_history_as_file = gr.Button("Save Chat History as File")
             with gr.Column():
                 download_file = gr.File(label="Download Chat History")
 
         # Restore original functionality
         search_button.click(
-            fn=update_dropdown,
-            inputs=[search_query_input, search_type_input],
+            fn=update_dropdown_multiple,
+            inputs=[search_query_input, search_type_input, keyword_filter_input],
             outputs=[items_output, item_mapping]
         )
 
+        def search_conversations(query):
+            try:
+                # Use RAG search with title search
+                if query and query.strip():
+                    results, _, _ = search_conversations_by_keywords(title_query=query.strip())
+                else:
+                    results, _, _ = get_all_conversations()
+
+                if not results:
+                    return gr.update(choices=[])
+
+                # Format choices to match UI
+                conversation_options = [
+                    (f"{conv['title']} (ID: {conv['conversation_id'][:8]})", conv['conversation_id'])
+                    for conv in results
+                ]
+
+                return gr.update(choices=conversation_options)
+            except Exception as e:
+                logging.error(f"Error searching conversations: {str(e)}")
+                return gr.update(choices=[])
+
+        def load_conversation(conversation_id):
+            if not conversation_id:
+                return [], None
+
+            try:
+                # Use RAG load function
+                messages, _, _ = load_chat_history(conversation_id)
+
+                # Convert to chatbot history format
+                history = [
+                    (content, None) if role == 'user' else (None, content)
+                    for role, content in messages
+                ]
+
+                return history, conversation_id
+            except Exception as e:
+                logging.error(f"Error loading conversation: {str(e)}")
+                return [], None
+
+        def save_chat_history_to_db_wrapper(chatbot, conversation_id, media_content, chat_name=None):
+            log_counter("save_chat_history_to_db_attempt")
+            start_time = time.time()
+            logging.info(f"Attempting to save chat history. Media content type: {type(media_content)}")
+
+            try:
+                # First check if we can access the database
+                try:
+                    with get_db_connection() as conn:
+                        cursor = conn.cursor()
+                        cursor.execute("SELECT 1")
+                except sqlite3.DatabaseError as db_error:
+                    logging.error(f"Database is corrupted or inaccessible: {str(db_error)}")
+                    return conversation_id, gr.update(
+                        value="Database error: The database file appears to be corrupted. Please contact support.")
+
+                # For both new and existing conversations
+                try:
+                    if not conversation_id:
+                        title = chat_name if chat_name else "Untitled Conversation"
+                        conversation_id = start_new_conversation(title=title)
+                        logging.info(f"Created new conversation with ID: {conversation_id}")
+
+                    # Update existing messages
+                    delete_messages_in_conversation(conversation_id)
+                    for user_msg, assistant_msg in chatbot:
+                        if user_msg:
+                            save_message(conversation_id, "user", user_msg)
+                        if assistant_msg:
+                            save_message(conversation_id, "assistant", assistant_msg)
+                except sqlite3.DatabaseError as db_error:
+                    logging.error(f"Database error during message save: {str(db_error)}")
+                    return conversation_id, gr.update(
+                        value="Database error: Unable to save messages. Please try again or contact support.")
+
+                save_duration = time.time() - start_time
+                log_histogram("save_chat_history_to_db_duration", save_duration)
+                log_counter("save_chat_history_to_db_success")
+
+                return conversation_id, gr.update(value="Chat history saved successfully!")
+
+            except Exception as e:
+                log_counter("save_chat_history_to_db_error", labels={"error": str(e)})
+                error_message = f"Failed to save chat history: {str(e)}"
+                logging.error(error_message, exc_info=True)
+                return conversation_id, gr.update(value=error_message)
+
         def update_prompts(preset_name):
             prompts = update_user_prompt(preset_name)
             return (
@@ -492,13 +752,85 @@ def create_chat_interface_stacked():
                 gr.update(value=prompts["system_prompt"], visible=True)
             )
 
+        def clear_chat():
+            return [], None, 0  # Empty history, conversation_id, and token count
+
         clear_chat_button.click(
             clear_chat,
-            outputs=[chatbot, conversation_id]
+            outputs=[chatbot, conversation_id, token_count_display]
         )
+
+        # Handle custom prompt checkbox change
+        def on_custom_prompt_checkbox_change(is_checked):
+            return (
+                gr.update(visible=is_checked),
+                gr.update(visible=is_checked)
+            )
+
+        custom_prompt_checkbox.change(
+            fn=on_custom_prompt_checkbox_change,
+            inputs=[custom_prompt_checkbox],
+            outputs=[user_prompt, system_prompt]
+        )
+
+        # Handle preset prompt checkbox change
+        def on_preset_prompt_checkbox_change(is_checked):
+            if is_checked:
+                prompts, total_pages, current_page = list_prompts(page=1, per_page=20)
+                page_display_text = f"Page {current_page} of {total_pages}"
+                return (
+                    gr.update(visible=True, interactive=True, choices=prompts),  # preset_prompt
+                    gr.update(visible=True),  # prev_page_button
+                    gr.update(visible=True),  # next_page_button
+                    gr.update(value=page_display_text, visible=True),  # page_display
+                    current_page,  # current_page_state
+                    total_pages   # total_pages_state
+                )
+            else:
+                return (
+                    gr.update(visible=False, interactive=False),  # preset_prompt
+                    gr.update(visible=False),  # prev_page_button
+                    gr.update(visible=False),  # next_page_button
+                    gr.update(visible=False),  # page_display
+                    1,  # current_page_state
+                    1   # total_pages_state
+                )
+
+        preset_prompt_checkbox.change(
+            fn=on_preset_prompt_checkbox_change,
+            inputs=[preset_prompt_checkbox],
+            outputs=[preset_prompt, prev_page_button, next_page_button, page_display, current_page_state, total_pages_state]
+        )
+
+        # Pagination button functions
+        def on_prev_page_click(current_page, total_pages):
+            new_page = max(current_page - 1, 1)
+            prompts, total_pages, current_page = list_prompts(page=new_page, per_page=20)
+            page_display_text = f"Page {current_page} of {total_pages}"
+            return gr.update(choices=prompts), gr.update(value=page_display_text), current_page
+
+        prev_page_button.click(
+            fn=on_prev_page_click,
+            inputs=[current_page_state, total_pages_state],
+            outputs=[preset_prompt, page_display, current_page_state]
+        )
+
+        def on_next_page_click(current_page, total_pages):
+            new_page = min(current_page + 1, total_pages)
+            prompts, total_pages, current_page = list_prompts(page=new_page, per_page=20)
+            page_display_text = f"Page {current_page} of {total_pages}"
+            return gr.update(choices=prompts), gr.update(value=page_display_text), current_page
+
+        next_page_button.click(
+            fn=on_next_page_click,
+            inputs=[current_page_state, total_pages_state],
+            outputs=[preset_prompt, page_display, current_page_state]
+        )
+
+        # Update prompts when a preset is selected
         preset_prompt.change(
             update_prompts,
-            inputs=preset_prompt,
+            inputs=[preset_prompt],
             outputs=[user_prompt, system_prompt]
         )
 
@@ -507,13 +839,14 @@ def create_chat_interface_stacked():
             inputs=[msg, chatbot, media_content, selected_parts, api_endpoint, api_key, user_prompt,
                     conversation_id, save_conversation, temp, system_prompt],
             outputs=[msg, chatbot, conversation_id]
-        ).then(  # Clear the message box after submission
+        ).then(
             lambda x: gr.update(value=""),
             inputs=[chatbot],
             outputs=[msg]
-        ).then(  # Clear the user prompt after the first message
-            lambda: gr.update(value=""),
-            outputs=[user_prompt, system_prompt]
+        ).then(
+            lambda history: approximate_token_count(history),
+            inputs=[chatbot],
+            outputs=[token_count_display]
         )
 
         items_output.change(
@@ -559,18 +892,31 @@ def create_chat_interface_stacked():
         save_chat_history_to_db.click(
             save_chat_history_to_db_wrapper,
             inputs=[chatbot, conversation_id, media_content, chat_media_name],
-            outputs=[conversation_id, gr.Textbox(label="Save Status")]
+            outputs=[conversation_id, save_status]
         )
 
         regenerate_button.click(
             regenerate_last_message,
             inputs=[chatbot, media_content, selected_parts, api_endpoint, api_key, user_prompt, temp, system_prompt],
             outputs=[chatbot, gr.Textbox(label="Regenerate Status")]
+        ).then(
+            lambda history: approximate_token_count(history),
+            inputs=[chatbot],
+            outputs=[token_count_display]
         )
 
 
-# FIXME - System prompts
 def create_chat_interface_multi_api():
+    try:
+        default_value = None
+        if default_api_endpoint:
+            if default_api_endpoint in global_api_endpoints:
+                default_value = format_api_name(default_api_endpoint)
+            else:
+                logging.warning(f"Default API endpoint '{default_api_endpoint}' not found in global_api_endpoints")
+    except Exception as e:
+        logging.error(f"Error setting default API endpoint: {str(e)}")
+        default_value = None
     custom_css = """
     .chatbot-container .message-wrap .message {
         font-size: 14px !important;
@@ -596,9 +942,31 @@ def create_chat_interface_multi_api():
                     use_summary = gr.Checkbox(label="Use Summary")
                     use_prompt = gr.Checkbox(label="Use Prompt")
             with gr.Column():
-                preset_prompt = gr.Dropdown(label="Select Preset Prompt", choices=load_preset_prompts(), visible=True)
-                system_prompt = gr.Textbox(label="System Prompt", value="You are a helpful AI assistant.", lines=5)
-                user_prompt = gr.Textbox(label="Modify Prompt (Prefixed to your message every time)", lines=5, value="", visible=True)
+                # Initialize state variables for pagination
+                current_page_state = gr.State(value=1)
+                total_pages_state = gr.State(value=1)
+
+                custom_prompt_checkbox = gr.Checkbox(label="Use a Custom Prompt",
+                                                     value=False,
+                                                     visible=True)
+                preset_prompt_checkbox = gr.Checkbox(label="Use a pre-set Prompt",
+                                                     value=False,
+                                                     visible=True)
+                with gr.Row():
+                    # Add pagination controls
+                    preset_prompt = gr.Dropdown(label="Select Preset Prompt",
+                                                choices=[],
+                                                visible=False)
+                with gr.Row():
+                    prev_page_button = gr.Button("Previous Page", visible=False)
+                    page_display = gr.Markdown("Page 1 of X", visible=False)
+                    next_page_button = gr.Button("Next Page", visible=False)
+                system_prompt = gr.Textbox(label="System Prompt",
+                                           value="You are a helpful AI assistant.",
+                                           lines=5,
+                                           visible=True)
+                user_prompt = gr.Textbox(label="Modify Prompt (Prefixed to your message every time)", lines=5,
+                                         value="", visible=True)
 
         with gr.Row():
             chatbots = []
@@ -606,17 +974,23 @@ def create_chat_interface_multi_api():
             api_keys = []
             temperatures = []
             regenerate_buttons = []
+            token_count_displays = []
             for i in range(3):
                 with gr.Column():
                     gr.Markdown(f"### Chat Window {i + 1}")
-                    api_endpoint = gr.Dropdown(label=f"API Endpoint {i + 1}",
-                                               choices=["Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq",
-                                                        "DeepSeek", "Mistral", "OpenRouter", "Llama.cpp", "Kobold",
-                                                        "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace"])
+                    # Refactored API selection dropdown
+                    api_endpoint = gr.Dropdown(
+                        choices=["None"] + [format_api_name(api) for api in global_api_endpoints],
+                        value=default_value,
+                        label="API for Chat Interaction (Optional)"
+                    )
                     api_key = gr.Textbox(label=f"API Key {i + 1} (if required)", type="password")
                     temperature = gr.Slider(label=f"Temperature {i + 1}", minimum=0.0, maximum=1.0, step=0.05,
                                             value=0.7)
                     chatbot = gr.Chatbot(height=800, elem_classes="chat-window")
+                    token_count_display = gr.Number(label=f"Approximate Token Count {i + 1}", value=0,
+                                                    interactive=False)
+                    token_count_displays.append(token_count_display)
                     regenerate_button = gr.Button(f"Regenerate Last Message {i + 1}")
                     chatbots.append(chatbot)
                     api_endpoints.append(api_endpoint)
@@ -642,16 +1016,103 @@ def create_chat_interface_multi_api():
             outputs=[items_output, item_mapping]
         )
 
+        def update_prompts(preset_name):
+            prompts = update_user_prompt(preset_name)
+            return (
+                gr.update(value=prompts["user_prompt"], visible=True),
+                gr.update(value=prompts["system_prompt"], visible=True)
+            )
+
+        def on_custom_prompt_checkbox_change(is_checked):
+            return (
+                gr.update(visible=is_checked),
+                gr.update(visible=is_checked)
+            )
+
+        custom_prompt_checkbox.change(
+            fn=on_custom_prompt_checkbox_change,
+            inputs=[custom_prompt_checkbox],
+            outputs=[user_prompt, system_prompt]
+        )
+
+        def clear_all_chats():
+            return [[]] * 3 + [[]] * 3 + [0] * 3
+
+        clear_chat_button.click(
+            clear_all_chats,
+            outputs=chatbots + chat_history + token_count_displays
+        )
+
+        def on_preset_prompt_checkbox_change(is_checked):
+            if is_checked:
+                prompts, total_pages, current_page = list_prompts(page=1, per_page=10)
+                page_display_text = f"Page {current_page} of {total_pages}"
+                return (
+                    gr.update(visible=True, interactive=True, choices=prompts),  # preset_prompt
+                    gr.update(visible=True),  # prev_page_button
+                    gr.update(visible=True),  # next_page_button
+                    gr.update(value=page_display_text, visible=True),  # page_display
+                    current_page,  # current_page_state
+                    total_pages   # total_pages_state
+                )
+            else:
+                return (
+                    gr.update(visible=False, interactive=False),  # preset_prompt
+                    gr.update(visible=False),  # prev_page_button
+                    gr.update(visible=False),  # next_page_button
+                    gr.update(visible=False),  # page_display
+                    1,  # current_page_state
+                    1   # total_pages_state
+                )
+
         preset_prompt.change(update_user_prompt, inputs=preset_prompt, outputs=user_prompt)
 
+        preset_prompt_checkbox.change(
+            fn=on_preset_prompt_checkbox_change,
+            inputs=[preset_prompt_checkbox],
+            outputs=[preset_prompt, prev_page_button, next_page_button, page_display, current_page_state,
+                     total_pages_state]
+        )
+
+        def on_prev_page_click(current_page, total_pages):
+            new_page = max(current_page - 1, 1)
+            prompts, total_pages, current_page = list_prompts(page=new_page, per_page=10)
+            page_display_text = f"Page {current_page} of {total_pages}"
+            return gr.update(choices=prompts), gr.update(value=page_display_text), current_page
+
+        prev_page_button.click(
+            fn=on_prev_page_click,
+            inputs=[current_page_state, total_pages_state],
+            outputs=[preset_prompt, page_display, current_page_state]
+        )
+
+        def on_next_page_click(current_page, total_pages):
+            new_page = min(current_page + 1, total_pages)
+            prompts, total_pages, current_page = list_prompts(page=new_page, per_page=10)
+            page_display_text = f"Page {current_page} of {total_pages}"
+            return gr.update(choices=prompts), gr.update(value=page_display_text), current_page
+
+        next_page_button.click(
+            fn=on_next_page_click,
+            inputs=[current_page_state, total_pages_state],
+            outputs=[preset_prompt, page_display, current_page_state]
+        )
+
+        # Update prompts when a preset is selected
+        preset_prompt.change(
+            update_prompts,
+            inputs=[preset_prompt],
+            outputs=[user_prompt, system_prompt]
+        )
 
         def clear_all_chats():
-            return [[]] * 3 + [[]] * 3
+            return [[]] * 3 + [[]] * 3 + [0] * 3
 
         clear_chat_button.click(
             clear_all_chats,
-            outputs=chatbots + chat_history
+            outputs=chatbots + chat_history + token_count_displays
         )
+
         def chat_wrapper_multi(message, custom_prompt, system_prompt, *args):
             chat_histories = args[:3]
             chatbots = args[3:6]
@@ -681,6 +1142,11 @@ def create_chat_interface_multi_api():
 
             return [gr.update(value="")] + new_chatbots + new_chat_histories
 
+        def update_token_counts(*histories):
+            token_counts = []
+            for history in histories:
+                token_counts.append(approximate_token_count(history))
+            return token_counts
 
         def regenerate_last_message(chat_history, chatbot, media_content, selected_parts, api_endpoint, api_key, custom_prompt, temperature, system_prompt):
             if not chat_history:
@@ -717,8 +1183,13 @@ def create_chat_interface_multi_api():
         for i in range(3):
             regenerate_buttons[i].click(
                 regenerate_last_message,
-                inputs=[chat_history[i], chatbots[i], media_content, selected_parts, api_endpoints[i], api_keys[i], user_prompt, temperatures[i], system_prompt],
+                inputs=[chat_history[i], chatbots[i], media_content, selected_parts, api_endpoints[i], api_keys[i],
+                        user_prompt, temperatures[i], system_prompt],
                 outputs=[chatbots[i], chat_history[i], gr.Textbox(label=f"Regenerate Status {i + 1}")]
+            ).then(
+                lambda history: approximate_token_count(history),
+                inputs=[chat_history[i]],
+                outputs=[token_count_displays[i]]
             )
 
         # In the create_chat_interface_multi_api function:
@@ -731,6 +1202,10 @@ def create_chat_interface_multi_api():
         ).then(
             lambda: (gr.update(value=""), gr.update(value="")),
             outputs=[msg, user_prompt]
+        ).then(
+            update_token_counts,
+            inputs=chat_history,
+            outputs=token_count_displays
         )
 
         items_output.change(
@@ -747,8 +1222,17 @@ def create_chat_interface_multi_api():
             )
 
 
-
 def create_chat_interface_four():
+    try:
+        default_value = None
+        if default_api_endpoint:
+            if default_api_endpoint in global_api_endpoints:
+                default_value = format_api_name(default_api_endpoint)
+            else:
+                logging.warning(f"Default API endpoint '{default_api_endpoint}' not found in global_api_endpoints")
+    except Exception as e:
+        logging.error(f"Error setting default API endpoint: {str(e)}")
+        default_value = None
     custom_css = """
     .chatbot-container .message-wrap .message {
         font-size: 14px !important;
@@ -762,17 +1246,32 @@ def create_chat_interface_four():
     with gr.TabItem("Four Independent API Chats", visible=True):
         gr.Markdown("# Four Independent API Chat Interfaces")
 
+        # Initialize prompts during component creation
+        prompts, total_pages, current_page = list_prompts(page=1, per_page=10)
+        current_page_state = gr.State(value=current_page)
+        total_pages_state = gr.State(value=total_pages)
+        page_display_text = f"Page {current_page} of {total_pages}"
+
         with gr.Row():
             with gr.Column():
                 preset_prompt = gr.Dropdown(
-                    label="Select Preset Prompt",
-                    choices=load_preset_prompts(),
+                    label="Select Preset Prompt (This will be prefixed to your messages, recommend copy/pasting and then clearing the User Prompt box)",
+                    choices=prompts,
                     visible=True
                 )
+                prev_page_button = gr.Button("Previous Page", visible=True)
+                page_display = gr.Markdown(page_display_text, visible=True)
+                next_page_button = gr.Button("Next Page", visible=True)
                 user_prompt = gr.Textbox(
-                    label="Modify Prompt",
+                    label="Modify User Prompt",
                     lines=3
                 )
+                system_prompt = gr.Textbox(
+                    label="System Prompt",
+                    value="You are a helpful AI assistant.",
+                    lines=3
+                )
+
             with gr.Column():
                 gr.Markdown("Scroll down for the chat windows...")
 
@@ -781,13 +1280,11 @@ def create_chat_interface_four():
         def create_single_chat_interface(index, user_prompt_component):
             with gr.Column():
                 gr.Markdown(f"### Chat Window {index + 1}")
+                # Refactored API selection dropdown
                 api_endpoint = gr.Dropdown(
-                    label=f"API Endpoint {index + 1}",
-                    choices=[
-                        "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq",
-                        "DeepSeek", "Mistral", "OpenRouter", "Llama.cpp", "Kobold",
-                        "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace"
-                    ]
+                    choices=["None"] + [format_api_name(api) for api in global_api_endpoints],
+                    value=default_value,
+                    label="API for Chat Interaction (Optional)"
                 )
                 api_key = gr.Textbox(
                     label=f"API Key {index + 1} (if required)",
@@ -804,6 +1301,8 @@ def create_chat_interface_four():
                 msg = gr.Textbox(label=f"Enter your message for Chat {index + 1}")
                 submit = gr.Button(f"Submit to Chat {index + 1}")
                 regenerate_button = gr.Button(f"Regenerate Last Message {index + 1}")
+                token_count_display = gr.Number(label=f"Approximate Token Count {index + 1}", value=0,
+                                                interactive=False)
                 clear_chat_button = gr.Button(f"Clear Chat {index + 1}")
 
                 # State to maintain chat history
@@ -819,7 +1318,8 @@ def create_chat_interface_four():
                     'submit': submit,
                     'regenerate_button': regenerate_button,
                     'clear_chat_button': clear_chat_button,
-                    'chat_history': chat_history
+                    'chat_history': chat_history,
+                    'token_count_display': token_count_display
                 })
 
         # Create four chat interfaces arranged in a 2x2 grid
@@ -830,10 +1330,47 @@ def create_chat_interface_four():
                         create_single_chat_interface(i * 2 + j, user_prompt)
 
         # Update user_prompt based on preset_prompt selection
+        def update_prompts(preset_name):
+            prompts = update_user_prompt(preset_name)
+            return gr.update(value=prompts["user_prompt"]), gr.update(value=prompts["system_prompt"])
+
         preset_prompt.change(
-            fn=update_user_prompt,
-            inputs=preset_prompt,
-            outputs=user_prompt
+            fn=update_prompts,
+            inputs=[preset_prompt],
+            outputs=[user_prompt, system_prompt]
+        )
+
+        # Pagination button functions
+        def on_prev_page_click(current_page, total_pages):
+            new_page = max(current_page - 1, 1)
+            prompts, total_pages, current_page = list_prompts(page=new_page, per_page=10)
+            page_display_text = f"Page {current_page} of {total_pages}"
+            return (
+                gr.update(choices=prompts),
+                gr.update(value=page_display_text),
+                current_page
+            )
+
+        prev_page_button.click(
+            fn=on_prev_page_click,
+            inputs=[current_page_state, total_pages_state],
+            outputs=[preset_prompt, page_display, current_page_state]
+        )
+
+        def on_next_page_click(current_page, total_pages):
+            new_page = min(current_page + 1, total_pages)
+            prompts, total_pages, current_page = list_prompts(page=new_page, per_page=10)
+            page_display_text = f"Page {current_page} of {total_pages}"
+            return (
+                gr.update(choices=prompts),
+                gr.update(value=page_display_text),
+                current_page
+            )
+
+        next_page_button.click(
+            fn=on_next_page_click,
+            inputs=[current_page_state, total_pages_state],
+            outputs=[preset_prompt, page_display, current_page_state]
         )
 
         def chat_wrapper_single(message, chat_history, api_endpoint, api_key, temperature, user_prompt):
@@ -913,6 +1450,10 @@ def create_chat_interface_four():
                     interface['chatbot'],
                     interface['chat_history']
                 ]
+            ).then(
+                lambda history: approximate_token_count(history),
+                inputs=[interface['chat_history']],
+                outputs=[interface['token_count_display']]
             )
 
             interface['regenerate_button'].click(
@@ -929,12 +1470,18 @@ def create_chat_interface_four():
                     interface['chat_history'],
                     gr.Textbox(label="Regenerate Status")
                 ]
+            ).then(
+                lambda history: approximate_token_count(history),
+                inputs=[interface['chat_history']],
+                outputs=[interface['token_count_display']]
             )
 
+            def clear_chat_single():
+                return [], [], 0
+
             interface['clear_chat_button'].click(
                 clear_chat_single,
-                inputs=[],
-                outputs=[interface['chatbot'], interface['chat_history']]
+                outputs=[interface['chatbot'], interface['chat_history'], interface['token_count_display']]
             )
 
 
@@ -953,233 +1500,11 @@ def chat_wrapper_single(message, chat_history, chatbot, api_endpoint, api_key, t
 
     return new_msg, updated_chatbot, new_history, new_conv_id
 
-
-# FIXME - Finish implementing functions + testing/valdidation
-def create_chat_management_tab():
-    with gr.TabItem("Chat Management", visible=True):
-        gr.Markdown("# Chat Management")
-
-        with gr.Row():
-            search_query = gr.Textbox(label="Search Conversations")
-            search_button = gr.Button("Search")
-
-        conversation_list = gr.Dropdown(label="Select Conversation", choices=[])
-        conversation_mapping = gr.State({})
-
-        with gr.Tabs():
-            with gr.TabItem("Edit", visible=True):
-                chat_content = gr.TextArea(label="Chat Content (JSON)", lines=20, max_lines=50)
-                save_button = gr.Button("Save Changes")
-                delete_button = gr.Button("Delete Conversation", variant="stop")
-
-            with gr.TabItem("Preview", visible=True):
-                chat_preview = gr.HTML(label="Chat Preview")
-        result_message = gr.Markdown("")
-
-        def search_conversations(query):
-            conversations = search_chat_conversations(query)
-            choices = [f"{conv['conversation_name']} (Media: {conv['media_title']}, ID: {conv['id']})" for conv in
-                       conversations]
-            mapping = {choice: conv['id'] for choice, conv in zip(choices, conversations)}
-            return gr.update(choices=choices), mapping
-
-        def load_conversations(selected, conversation_mapping):
-            logging.info(f"Selected: {selected}")
-            logging.info(f"Conversation mapping: {conversation_mapping}")
-
-            try:
-                if selected and selected in conversation_mapping:
-                    conversation_id = conversation_mapping[selected]
-                    messages = get_chat_messages(conversation_id)
-                    conversation_data = {
-                        "conversation_id": conversation_id,
-                        "messages": messages
-                    }
-                    json_content = json.dumps(conversation_data, indent=2)
-
-                    # Create HTML preview
-                    html_preview = "<div style='max-height: 500px; overflow-y: auto;'>"
-                    for msg in messages:
-                        sender_style = "background-color: #e6f3ff;" if msg[
-                                                                           'sender'] == 'user' else "background-color: #f0f0f0;"
-                        html_preview += f"<div style='margin-bottom: 10px; padding: 10px; border-radius: 5px; {sender_style}'>"
-                        html_preview += f"<strong>{msg['sender']}:</strong> {html.escape(msg['message'])}<br>"
-                        html_preview += f"<small>Timestamp: {msg['timestamp']}</small>"
-                        html_preview += "</div>"
-                    html_preview += "</div>"
-
-                    logging.info("Returning json_content and html_preview")
-                    return json_content, html_preview
-                else:
-                    logging.warning("No conversation selected or not in mapping")
-                    return "", "<p>No conversation selected</p>"
-            except Exception as e:
-                logging.error(f"Error in load_conversations: {str(e)}")
-                return f"Error: {str(e)}", "<p>Error loading conversation</p>"
-
-        def validate_conversation_json(content):
-            try:
-                data = json.loads(content)
-                if not isinstance(data, dict):
-                    return False, "Invalid JSON structure: root should be an object"
-                if "conversation_id" not in data or not isinstance(data["conversation_id"], int):
-                    return False, "Missing or invalid conversation_id"
-                if "messages" not in data or not isinstance(data["messages"], list):
-                    return False, "Missing or invalid messages array"
-                for msg in data["messages"]:
-                    if not all(key in msg for key in ["sender", "message"]):
-                        return False, "Invalid message structure: missing required fields"
-                return True, data
-            except json.JSONDecodeError as e:
-                return False, f"Invalid JSON: {str(e)}"
-
-        def save_conversation(selected, conversation_mapping, content):
-            if not selected or selected not in conversation_mapping:
-                return "Please select a conversation before saving.", "<p>No changes made</p>"
-
-            conversation_id = conversation_mapping[selected]
-            is_valid, result = validate_conversation_json(content)
-
-            if not is_valid:
-                return f"Error: {result}", "<p>No changes made due to error</p>"
-
-            conversation_data = result
-            if conversation_data["conversation_id"] != conversation_id:
-                return "Error: Conversation ID mismatch.", "<p>No changes made due to ID mismatch</p>"
-
-            try:
-                with db.get_connection() as conn:
-                    conn.execute("BEGIN TRANSACTION")
-                    cursor = conn.cursor()
-
-                    # Backup original conversation
-                    cursor.execute("SELECT * FROM ChatMessages WHERE conversation_id = ?", (conversation_id,))
-                    original_messages = cursor.fetchall()
-                    backup_data = json.dumps({"conversation_id": conversation_id, "messages": original_messages})
-
-                    # You might want to save this backup_data somewhere
-
-                    # Delete existing messages
-                    cursor.execute("DELETE FROM ChatMessages WHERE conversation_id = ?", (conversation_id,))
-
-                    # Insert updated messages
-                    for message in conversation_data["messages"]:
-                        cursor.execute('''
-                            INSERT INTO ChatMessages (conversation_id, sender, message, timestamp)
-                            VALUES (?, ?, ?, COALESCE(?, CURRENT_TIMESTAMP))
-                        ''', (conversation_id, message["sender"], message["message"], message.get("timestamp")))
-
-                    conn.commit()
-
-                    # Create updated HTML preview
-                    html_preview = "<div style='max-height: 500px; overflow-y: auto;'>"
-                    for msg in conversation_data["messages"]:
-                        sender_style = "background-color: #e6f3ff;" if msg[
-                                                                           'sender'] == 'user' else "background-color: #f0f0f0;"
-                        html_preview += f"<div style='margin-bottom: 10px; padding: 10px; border-radius: 5px; {sender_style}'>"
-                        html_preview += f"<strong>{msg['sender']}:</strong> {html.escape(msg['message'])}<br>"
-                        html_preview += f"<small>Timestamp: {msg.get('timestamp', 'N/A')}</small>"
-                        html_preview += "</div>"
-                    html_preview += "</div>"
-
-                    return "Conversation updated successfully.", html_preview
-            except sqlite3.Error as e:
-                conn.rollback()
-                logging.error(f"Database error in save_conversation: {e}")
-                return f"Error updating conversation: {str(e)}", "<p>Error occurred while saving</p>"
-            except Exception as e:
-                conn.rollback()
-                logging.error(f"Unexpected error in save_conversation: {e}")
-                return f"Unexpected error: {str(e)}", "<p>Unexpected error occurred</p>"
-
-        def delete_conversation(selected, conversation_mapping):
-            if not selected or selected not in conversation_mapping:
-                return "Please select a conversation before deleting.", "<p>No changes made</p>", gr.update(choices=[])
-
-            conversation_id = conversation_mapping[selected]
-
-            try:
-                with db.get_connection() as conn:
-                    cursor = conn.cursor()
-
-                    # Delete messages associated with the conversation
-                    cursor.execute("DELETE FROM ChatMessages WHERE conversation_id = ?", (conversation_id,))
-
-                    # Delete the conversation itself
-                    cursor.execute("DELETE FROM ChatConversations WHERE id = ?", (conversation_id,))
-
-                    conn.commit()
-
-                # Update the conversation list
-                remaining_conversations = [choice for choice in conversation_mapping.keys() if choice != selected]
-                updated_mapping = {choice: conversation_mapping[choice] for choice in remaining_conversations}
-
-                return "Conversation deleted successfully.", "<p>Conversation deleted</p>", gr.update(choices=remaining_conversations)
-            except sqlite3.Error as e:
-                conn.rollback()
-                logging.error(f"Database error in delete_conversation: {e}")
-                return f"Error deleting conversation: {str(e)}", "<p>Error occurred while deleting</p>", gr.update()
-            except Exception as e:
-                conn.rollback()
-                logging.error(f"Unexpected error in delete_conversation: {e}")
-                return f"Unexpected error: {str(e)}", "<p>Unexpected error occurred</p>", gr.update()
-
-        def parse_formatted_content(formatted_content):
-            lines = formatted_content.split('\n')
-            conversation_id = int(lines[0].split(': ')[1])
-            timestamp = lines[1].split(': ')[1]
-            history = []
-            current_role = None
-            current_content = None
-            for line in lines[3:]:
-                if line.startswith("Role: "):
-                    if current_role is not None:
-                        history.append({"role": current_role, "content": ["", current_content]})
-                    current_role = line.split(': ')[1]
-                elif line.startswith("Content: "):
-                    current_content = line.split(': ', 1)[1]
-            if current_role is not None:
-                history.append({"role": current_role, "content": ["", current_content]})
-            return json.dumps({
-                "conversation_id": conversation_id,
-                "timestamp": timestamp,
-                "history": history
-            }, indent=2)
-
-        search_button.click(
-            search_conversations,
-            inputs=[search_query],
-            outputs=[conversation_list, conversation_mapping]
-        )
-
-        conversation_list.change(
-            load_conversations,
-            inputs=[conversation_list, conversation_mapping],
-            outputs=[chat_content, chat_preview]
-        )
-
-        save_button.click(
-            save_conversation,
-            inputs=[conversation_list, conversation_mapping, chat_content],
-            outputs=[result_message, chat_preview]
-        )
-
-        delete_button.click(
-            delete_conversation,
-            inputs=[conversation_list, conversation_mapping],
-            outputs=[result_message, chat_preview, conversation_list]
-        )
-
-    return search_query, search_button, conversation_list, conversation_mapping, chat_content, save_button, delete_button, result_message, chat_preview
-
-
-
 # Mock function to simulate LLM processing
 def process_with_llm(workflow, context, prompt, api_endpoint, api_key):
     api_key_snippet = api_key[:5] + "..." if api_key else "Not provided"
     return f"LLM output using {api_endpoint} (API Key: {api_key_snippet}) for {workflow} with context: {context[:30]}... and prompt: {prompt[:30]}..."
 
-
 #
 # End of Chat_ui.py
 #######################################################################################################################

App_Function_Libraries/Gradio_UI/Embeddings_tab.py

@@ -4,6 +4,7 @@
 # Imports
 import json
 import logging
+import os
 #
 # External Imports
 import gradio as gr
@@ -11,26 +12,58 @@ import numpy as np
 from tqdm import tqdm
 #
 # Local Imports
-from App_Function_Libraries.DB.DB_Manager import get_all_content_from_database
+from App_Function_Libraries.DB.DB_Manager import get_all_content_from_database, get_all_conversations, \
+    get_conversation_text, get_note_by_id
+from App_Function_Libraries.DB.RAG_QA_Chat_DB import get_all_notes
 from App_Function_Libraries.RAG.ChromaDB_Library import chroma_client, \
     store_in_chroma, situate_context
 from App_Function_Libraries.RAG.Embeddings_Create import create_embedding, create_embeddings_batch
 from App_Function_Libraries.Chunk_Lib import improved_chunking_process, chunk_for_embedding
+from App_Function_Libraries.Utils.Utils import load_and_log_configs
+
+
 #
 ########################################################################################################################
 #
 # Functions:
 
 def create_embeddings_tab():
+    # Load configuration first
+    config = load_and_log_configs()
+    if not config:
+        raise ValueError("Could not load configuration")
+
+    # Get database paths from config
+    db_config = config['db_config']
+    media_db_path = db_config['sqlite_path']
+    rag_qa_db_path = os.path.join(os.path.dirname(media_db_path), "rag_qa.db")
+    character_chat_db_path = os.path.join(os.path.dirname(media_db_path), "chatDB.db")
+    chroma_db_path = db_config['chroma_db_path']
+
     with gr.TabItem("Create Embeddings", visible=True):
         gr.Markdown("# Create Embeddings for All Content")
 
         with gr.Row():
             with gr.Column():
+                # Database selection at the top
+                database_selection = gr.Radio(
+                    choices=["Media DB", "RAG Chat", "Character Chat"],
+                    label="Select Content Source",
+                    value="Media DB",
+                    info="Choose which database to create embeddings from"
+                )
+
+                # Add database path display
+                current_db_path = gr.Textbox(
+                    label="Current Database Path",
+                    value=media_db_path,
+                    interactive=False
+                )
+
                 embedding_provider = gr.Radio(
                     choices=["huggingface", "local", "openai"],
                     label="Select Embedding Provider",
-                    value="huggingface"
+                    value=config['embedding_config']['embedding_provider'] or "huggingface"
                 )
                 gr.Markdown("Note: Local provider requires a running Llama.cpp/llamafile server.")
                 gr.Markdown("OpenAI provider requires a valid API key.")
@@ -65,22 +98,24 @@ def create_embeddings_tab():
 
                 embedding_api_url = gr.Textbox(
                     label="API URL (for local provider)",
-                    value="http://localhost:8080/embedding",
+                    value=config['embedding_config']['embedding_api_url'],
                     visible=False
                 )
 
-                # Add chunking options
+                # Add chunking options with config defaults
                 chunking_method = gr.Dropdown(
                     choices=["words", "sentences", "paragraphs", "tokens", "semantic"],
                     label="Chunking Method",
                     value="words"
                 )
                 max_chunk_size = gr.Slider(
-                    minimum=1, maximum=8000, step=1, value=500,
+                    minimum=1, maximum=8000, step=1,
+                    value=config['embedding_config']['chunk_size'],
                     label="Max Chunk Size"
                 )
                 chunk_overlap = gr.Slider(
-                    minimum=0, maximum=4000, step=1, value=200,
+                    minimum=0, maximum=4000, step=1,
+                    value=config['embedding_config']['overlap'],
                     label="Chunk Overlap"
                 )
                 adaptive_chunking = gr.Checkbox(
@@ -92,6 +127,7 @@ def create_embeddings_tab():
 
             with gr.Column():
                 status_output = gr.Textbox(label="Status", lines=10)
+                progress = gr.Progress()
 
         def update_provider_options(provider):
             if provider == "huggingface":
@@ -107,23 +143,54 @@ def create_embeddings_tab():
             else:
                 return gr.update(visible=False)
 
-        embedding_provider.change(
-            fn=update_provider_options,
-            inputs=[embedding_provider],
-            outputs=[huggingface_model, openai_model, custom_embedding_model, embedding_api_url]
-        )
-
-        huggingface_model.change(
-            fn=update_huggingface_options,
-            inputs=[huggingface_model],
-            outputs=[custom_embedding_model]
-        )
+        def update_database_path(database_type):
+            if database_type == "Media DB":
+                return media_db_path
+            elif database_type == "RAG Chat":
+                return rag_qa_db_path
+            else:  # Character Chat
+                return character_chat_db_path
 
-        def create_all_embeddings(provider, hf_model, openai_model, custom_model, api_url, method, max_size, overlap, adaptive):
+        def create_all_embeddings(provider, hf_model, openai_model, custom_model, api_url, method,
+                                max_size, overlap, adaptive, database_type, progress=gr.Progress()):
             try:
-                all_content = get_all_content_from_database()
+                # Initialize content based on database selection
+                if database_type == "Media DB":
+                    all_content = get_all_content_from_database()
+                    content_type = "media"
+                elif database_type == "RAG Chat":
+                    all_content = []
+                    page = 1
+                    while True:
+                        conversations, total_pages, _ = get_all_conversations(page=page)
+                        if not conversations:
+                            break
+                        all_content.extend([{
+                            'id': conv['conversation_id'],
+                            'content': get_conversation_text(conv['conversation_id']),
+                            'title': conv['title'],
+                            'type': 'conversation'
+                        } for conv in conversations])
+                        progress(page / total_pages, desc=f"Loading conversations... Page {page}/{total_pages}")
+                        page += 1
+                else:  # Character Chat
+                    all_content = []
+                    page = 1
+                    while True:
+                        notes, total_pages, _ = get_all_notes(page=page)
+                        if not notes:
+                            break
+                        all_content.extend([{
+                            'id': note['id'],
+                            'content': f"{note['title']}\n\n{note['content']}",
+                            'conversation_id': note['conversation_id'],
+                            'type': 'note'
+                        } for note in notes])
+                        progress(page / total_pages, desc=f"Loading notes... Page {page}/{total_pages}")
+                        page += 1
+
                 if not all_content:
-                    return "No content found in the database."
+                    return "No content found in the selected database."
 
                 chunk_options = {
                     'method': method,
@@ -132,7 +199,7 @@ def create_embeddings_tab():
                     'adaptive': adaptive
                 }
 
-                collection_name = "all_content_embeddings"
+                collection_name = f"{database_type.lower().replace(' ', '_')}_embeddings"
                 collection = chroma_client.get_or_create_collection(name=collection_name)
 
                 # Determine the model to use
@@ -141,55 +208,113 @@ def create_embeddings_tab():
                 elif provider == "openai":
                     model = openai_model
                 else:
-                    model = custom_model
+                    model = api_url
+
+                total_items = len(all_content)
+                for idx, item in enumerate(all_content):
+                    progress((idx + 1) / total_items, desc=f"Processing item {idx + 1} of {total_items}")
 
-                for item in all_content:
-                    media_id = item['id']
+                    content_id = item['id']
                     text = item['content']
 
                     chunks = improved_chunking_process(text, chunk_options)
-                    for i, chunk in enumerate(chunks):
+                    for chunk_idx, chunk in enumerate(chunks):
                         chunk_text = chunk['text']
-                        chunk_id = f"doc_{media_id}_chunk_{i}"
-
-                        existing = collection.get(ids=[chunk_id])
-                        if existing['ids']:
+                        chunk_id = f"{database_type.lower()}_{content_id}_chunk_{chunk_idx}"
+
+                        try:
+                            embedding = create_embedding(chunk_text, provider, model, api_url)
+                            metadata = {
+                                'content_id': str(content_id),
+                                'chunk_index': int(chunk_idx),
+                                'total_chunks': int(len(chunks)),
+                                'chunking_method': method,
+                                'max_chunk_size': int(max_size),
+                                'chunk_overlap': int(overlap),
+                                'adaptive_chunking': bool(adaptive),
+                                'embedding_model': model,
+                                'embedding_provider': provider,
+                                'content_type': item.get('type', 'media'),
+                                'conversation_id': item.get('conversation_id'),
+                                **{k: (int(v) if isinstance(v, str) and v.isdigit() else v)
+                                   for k, v in chunk['metadata'].items()}
+                            }
+                            store_in_chroma(collection_name, [chunk_text], [embedding], [chunk_id], [metadata])
+
+                        except Exception as e:
+                            logging.error(f"Error processing chunk {chunk_id}: {str(e)}")
                             continue
 
-                        embedding = create_embedding(chunk_text, provider, model, api_url)
-                        metadata = {
-                            "media_id": str(media_id),
-                            "chunk_index": i,
-                            "total_chunks": len(chunks),
-                            "chunking_method": method,
-                            "max_chunk_size": max_size,
-                            "chunk_overlap": overlap,
-                            "adaptive_chunking": adaptive,
-                            "embedding_model": model,
-                            "embedding_provider": provider,
-                            **chunk['metadata']
-                        }
-                        store_in_chroma(collection_name, [chunk_text], [embedding], [chunk_id], [metadata])
-
-                return "Embeddings created and stored successfully for all content."
+                return f"Embeddings created and stored successfully for all {database_type} content."
             except Exception as e:
                 logging.error(f"Error during embedding creation: {str(e)}")
                 return f"Error: {str(e)}"
 
+        # Event handlers
+        embedding_provider.change(
+            fn=update_provider_options,
+            inputs=[embedding_provider],
+            outputs=[huggingface_model, openai_model, custom_embedding_model, embedding_api_url]
+        )
+
+        huggingface_model.change(
+            fn=update_huggingface_options,
+            inputs=[huggingface_model],
+            outputs=[custom_embedding_model]
+        )
+
+        database_selection.change(
+            fn=update_database_path,
+            inputs=[database_selection],
+            outputs=[current_db_path]
+        )
+
         create_button.click(
             fn=create_all_embeddings,
-            inputs=[embedding_provider, huggingface_model, openai_model, custom_embedding_model, embedding_api_url,
-                    chunking_method, max_chunk_size, chunk_overlap, adaptive_chunking],
+            inputs=[
+                embedding_provider, huggingface_model, openai_model, custom_embedding_model,
+                embedding_api_url, chunking_method, max_chunk_size, chunk_overlap,
+                adaptive_chunking, database_selection
+            ],
             outputs=status_output
         )
 
 
 def create_view_embeddings_tab():
+    # Load configuration first
+    config = load_and_log_configs()
+    if not config:
+        raise ValueError("Could not load configuration")
+
+    # Get database paths from config
+    db_config = config['db_config']
+    media_db_path = db_config['sqlite_path']
+    rag_qa_db_path = os.path.join(os.path.dirname(media_db_path), "rag_chat.db")
+    character_chat_db_path = os.path.join(os.path.dirname(media_db_path), "character_chat.db")
+    chroma_db_path = db_config['chroma_db_path']
+
     with gr.TabItem("View/Update Embeddings", visible=True):
         gr.Markdown("# View and Update Embeddings")
-        item_mapping = gr.State({})
+        # Initialize item_mapping as a Gradio State
+
+
         with gr.Row():
             with gr.Column():
+                # Add database selection
+                database_selection = gr.Radio(
+                    choices=["Media DB", "RAG Chat", "Character Chat"],
+                    label="Select Content Source",
+                    value="Media DB",
+                    info="Choose which database to view embeddings from"
+                )
+
+                # Add database path display
+                current_db_path = gr.Textbox(
+                    label="Current Database Path",
+                    value=media_db_path,
+                    interactive=False
+                )
+
                 item_dropdown = gr.Dropdown(label="Select Item", choices=[], interactive=True)
                 refresh_button = gr.Button("Refresh Item List")
                 embedding_status = gr.Textbox(label="Embedding Status", interactive=False)
@@ -236,9 +361,10 @@ def create_view_embeddings_tab():
 
                 embedding_api_url = gr.Textbox(
                     label="API URL (for local provider)",
-                    value="http://localhost:8080/embedding",
+                    value=config['embedding_config']['embedding_api_url'],
                     visible=False
                 )
+
                 chunking_method = gr.Dropdown(
                     choices=["words", "sentences", "paragraphs", "tokens", "semantic"],
                     label="Chunking Method",
@@ -267,15 +393,45 @@ def create_view_embeddings_tab():
                 )
                 contextual_api_key = gr.Textbox(label="API Key", lines=1)
 
-        def get_items_with_embedding_status():
+        item_mapping = gr.State(value={})
+
+        def update_database_path(database_type):
+            if database_type == "Media DB":
+                return media_db_path
+            elif database_type == "RAG Chat":
+                return rag_qa_db_path
+            else:  # Character Chat
+                return character_chat_db_path
+
+        def get_items_with_embedding_status(database_type):
             try:
-                items = get_all_content_from_database()
-                collection = chroma_client.get_or_create_collection(name="all_content_embeddings")
+                # Get items based on database selection
+                if database_type == "Media DB":
+                    items = get_all_content_from_database()
+                elif database_type == "RAG Chat":
+                    conversations, _, _ = get_all_conversations(page=1)
+                    items = [{
+                        'id': conv['conversation_id'],
+                        'title': conv['title'],
+                        'type': 'conversation'
+                    } for conv in conversations]
+                else:  # Character Chat
+                    notes, _, _ = get_all_notes(page=1)
+                    items = [{
+                        'id': note['id'],
+                        'title': note['title'],
+                        'type': 'note'
+                    } for note in notes]
+
+                collection_name = f"{database_type.lower().replace(' ', '_')}_embeddings"
+                collection = chroma_client.get_or_create_collection(name=collection_name)
+
                 choices = []
                 new_item_mapping = {}
                 for item in items:
                     try:
-                        result = collection.get(ids=[f"doc_{item['id']}_chunk_0"])
+                        chunk_id = f"{database_type.lower()}_{item['id']}_chunk_0"
+                        result = collection.get(ids=[chunk_id])
                         embedding_exists = result is not None and result.get('ids') and len(result['ids']) > 0
                         status = "Embedding exists" if embedding_exists else "No embedding"
                     except Exception as e:
@@ -303,40 +459,62 @@ def create_view_embeddings_tab():
             else:
                 return gr.update(visible=False)
 
-        def check_embedding_status(selected_item, item_mapping):
+        def check_embedding_status(selected_item, database_type, item_mapping):
             if not selected_item:
                 return "Please select an item", "", ""
 
+            if item_mapping is None:
+                # If mapping is None, try to refresh it
+                try:
+                    _, item_mapping = get_items_with_embedding_status(database_type)
+                except Exception as e:
+                    return f"Error initializing item mapping: {str(e)}", "", ""
+
             try:
                 item_id = item_mapping.get(selected_item)
                 if item_id is None:
                     return f"Invalid item selected: {selected_item}", "", ""
 
                 item_title = selected_item.rsplit(' (', 1)[0]
-                collection = chroma_client.get_or_create_collection(name="all_content_embeddings")
+                collection_name = f"{database_type.lower().replace(' ', '_')}_embeddings"
+                collection = chroma_client.get_or_create_collection(name=collection_name)
+                chunk_id = f"{database_type.lower()}_{item_id}_chunk_0"
+
+                try:
+                    result = collection.get(ids=[chunk_id], include=["embeddings", "metadatas"])
+                except Exception as e:
+                    logging.error(f"ChromaDB get error: {str(e)}")
+                    return f"Error retrieving embedding for '{item_title}': {str(e)}", "", ""
 
-                result = collection.get(ids=[f"doc_{item_id}_chunk_0"], include=["embeddings", "metadatas"])
-                logging.info(f"ChromaDB result for item '{item_title}' (ID: {item_id}): {result}")
+                # Check if result exists and has the expected structure
+                if not result or not isinstance(result, dict):
+                    return f"No embedding found for item '{item_title}' (ID: {item_id})", "", ""
 
-                if not result['ids']:
+                # Check if we have any results
+                if not result.get('ids') or len(result['ids']) == 0:
                     return f"No embedding found for item '{item_title}' (ID: {item_id})", "", ""
 
-                if not result['embeddings'] or not result['embeddings'][0]:
+                # Check if embeddings exist
+                if not result.get('embeddings') or not result['embeddings'][0]:
                     return f"Embedding data missing for item '{item_title}' (ID: {item_id})", "", ""
 
                 embedding = result['embeddings'][0]
-                metadata = result['metadatas'][0] if result['metadatas'] else {}
+                metadata = result.get('metadatas', [{}])[0] if result.get('metadatas') else {}
                 embedding_preview = str(embedding[:50])
                 status = f"Embedding exists for item '{item_title}' (ID: {item_id})"
                 return status, f"First 50 elements of embedding:\n{embedding_preview}", json.dumps(metadata, indent=2)
 
             except Exception as e:
-                logging.error(f"Error in check_embedding_status: {str(e)}")
+                logging.error(f"Error in check_embedding_status: {str(e)}", exc_info=True)
                 return f"Error processing item: {selected_item}. Details: {str(e)}", "", ""
 
-        def create_new_embedding_for_item(selected_item, provider, hf_model, openai_model, custom_model, api_url,
-                                          method, max_size, overlap, adaptive,
-                                          item_mapping, use_contextual, contextual_api_choice=None):
+        def refresh_and_update(database_type):
+            choices_update, new_mapping = get_items_with_embedding_status(database_type)
+            return choices_update, new_mapping
+
+        def create_new_embedding_for_item(selected_item, database_type, provider, hf_model, openai_model,
+                                        custom_model, api_url, method, max_size, overlap, adaptive,
+                                        item_mapping, use_contextual, contextual_api_choice=None):
             if not selected_item:
                 return "Please select an item", "", ""
 
@@ -345,8 +523,26 @@ def create_view_embeddings_tab():
                 if item_id is None:
                     return f"Invalid item selected: {selected_item}", "", ""
 
-                items = get_all_content_from_database()
-                item = next((item for item in items if item['id'] == item_id), None)
+                # Get item content based on database type
+                if database_type == "Media DB":
+                    items = get_all_content_from_database()
+                    item = next((item for item in items if item['id'] == item_id), None)
+                elif database_type == "RAG Chat":
+                    item = {
+                        'id': item_id,
+                        'content': get_conversation_text(item_id),
+                        'title': selected_item.rsplit(' (', 1)[0],
+                        'type': 'conversation'
+                    }
+                else:  # Character Chat
+                    note = get_note_by_id(item_id)
+                    item = {
+                        'id': item_id,
+                        'content': f"{note['title']}\n\n{note['content']}",
+                        'title': note['title'],
+                        'type': 'note'
+                    }
+
                 if not item:
                     return f"Item not found: {item_id}", "", ""
 
@@ -359,11 +555,11 @@ def create_view_embeddings_tab():
 
                 logging.info(f"Chunking content for item: {item['title']} (ID: {item_id})")
                 chunks = chunk_for_embedding(item['content'], item['title'], chunk_options)
-                collection_name = "all_content_embeddings"
+                collection_name = f"{database_type.lower().replace(' ', '_')}_embeddings"
                 collection = chroma_client.get_or_create_collection(name=collection_name)
 
                 # Delete existing embeddings for this item
-                existing_ids = [f"doc_{item_id}_chunk_{i}" for i in range(len(chunks))]
+                existing_ids = [f"{database_type.lower()}_{item_id}_chunk_{i}" for i in range(len(chunks))]
                 collection.delete(ids=existing_ids)
                 logging.info(f"Deleted {len(existing_ids)} existing embeddings for item {item_id}")
 
@@ -381,7 +577,7 @@ def create_view_embeddings_tab():
                         contextualized_text = chunk_text
                         context = None
 
-                    chunk_id = f"doc_{item_id}_chunk_{i}"
+                    chunk_id = f"{database_type.lower()}_{item_id}_chunk_{i}"
 
                     # Determine the model to use
                     if provider == "huggingface":
@@ -392,7 +588,7 @@ def create_view_embeddings_tab():
                         model = custom_model
 
                     metadata = {
-                        "media_id": str(item_id),
+                        "content_id": str(item_id),
                         "chunk_index": i,
                         "total_chunks": len(chunks),
                         "chunking_method": method,
@@ -441,15 +637,25 @@ def create_view_embeddings_tab():
                 logging.error(f"Error in create_new_embedding_for_item: {str(e)}", exc_info=True)
                 return f"Error creating embedding: {str(e)}", "", ""
 
+        # Wire up all the event handlers
+        database_selection.change(
+            update_database_path,
+            inputs=[database_selection],
+            outputs=[current_db_path]
+        )
+
         refresh_button.click(
             get_items_with_embedding_status,
+            inputs=[database_selection],
             outputs=[item_dropdown, item_mapping]
         )
+
         item_dropdown.change(
             check_embedding_status,
-            inputs=[item_dropdown, item_mapping],
+            inputs=[item_dropdown, database_selection, item_mapping],
             outputs=[embedding_status, embedding_preview, embedding_metadata]
         )
+
         create_new_embedding_button.click(
             create_new_embedding_for_item,
             inputs=[item_dropdown, embedding_provider, huggingface_model, openai_model, custom_embedding_model, embedding_api_url,
@@ -469,9 +675,10 @@ def create_view_embeddings_tab():
         )
 
     return (item_dropdown, refresh_button, embedding_status, embedding_preview, embedding_metadata,
-            create_new_embedding_button, embedding_provider, huggingface_model, openai_model, custom_embedding_model, embedding_api_url,
-            chunking_method, max_chunk_size, chunk_overlap, adaptive_chunking,
-            use_contextual_embeddings, contextual_api_choice, contextual_api_key)
+            create_new_embedding_button, embedding_provider, huggingface_model, openai_model,
+            custom_embedding_model, embedding_api_url, chunking_method, max_chunk_size,
+            chunk_overlap, adaptive_chunking, use_contextual_embeddings,
+            contextual_api_choice, contextual_api_key)
 
 
 def create_purge_embeddings_tab():

App_Function_Libraries/Gradio_UI/Evaluations_Benchmarks_tab.py

@@ -1,9 +1,12 @@
 ###################################################################################################
 # Evaluations_Benchmarks_tab.py - Gradio code for G-Eval testing
 # We will use the G-Eval API to evaluate the quality of the generated summaries.
+import logging
 
 import gradio as gr
 from App_Function_Libraries.Benchmarks_Evaluations.ms_g_eval import run_geval
+from App_Function_Libraries.Utils.Utils import default_api_endpoint, global_api_endpoints, format_api_name
+
 
 def create_geval_tab():
     with gr.Tab("G-Eval", visible=True):
@@ -31,13 +34,25 @@ def create_geval_tab():
 
 
 def create_infinite_bench_tab():
+    try:
+        default_value = None
+        if default_api_endpoint:
+            if default_api_endpoint in global_api_endpoints:
+                default_value = format_api_name(default_api_endpoint)
+            else:
+                logging.warning(f"Default API endpoint '{default_api_endpoint}' not found in global_api_endpoints")
+    except Exception as e:
+        logging.error(f"Error setting default API endpoint: {str(e)}")
+        default_value = None
     with gr.Tab("Infinite Bench", visible=True):
         gr.Markdown("# Infinite Bench Evaluation (Coming Soon)")
         with gr.Row():
             with gr.Column():
+                # Refactored API selection dropdown
                 api_name_input = gr.Dropdown(
-                    choices=["OpenAI", "Anthropic", "Cohere", "Groq", "OpenRouter", "DeepSeek", "HuggingFace", "Mistral", "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "Local-LLM", "Ollama"],
-                    label="Select API"
+                    choices=["None"] + [format_api_name(api) for api in global_api_endpoints],
+                    value=default_value,
+                    label="API for Summarization (Optional)"
                 )
                 api_key_input = gr.Textbox(label="API Key (if required)", type="password")
                 evaluate_button = gr.Button("Evaluate Summary")

App_Function_Libraries/Gradio_UI/Explain_summarize_tab.py

@@ -7,7 +7,7 @@ import logging
 # External Imports
 import gradio as gr
 
-from App_Function_Libraries.DB.DB_Manager import load_preset_prompts
+from App_Function_Libraries.DB.DB_Manager import list_prompts
 from App_Function_Libraries.Gradio_UI.Gradio_Shared import update_user_prompt
 #
 # Local Imports
@@ -17,6 +17,9 @@ from App_Function_Libraries.Summarization.Local_Summarization_Lib import summari
 from App_Function_Libraries.Summarization.Summarization_General_Lib import summarize_with_openai, summarize_with_anthropic, \
     summarize_with_cohere, summarize_with_groq, summarize_with_openrouter, summarize_with_deepseek, \
     summarize_with_huggingface
+from App_Function_Libraries.Utils.Utils import default_api_endpoint, global_api_endpoints, format_api_name
+
+
 #
 #
 ############################################################################################################
@@ -24,32 +27,62 @@ from App_Function_Libraries.Summarization.Summarization_General_Lib import summa
 # Functions:
 
 def create_summarize_explain_tab():
+    try:
+        default_value = None
+        if default_api_endpoint:
+            if default_api_endpoint in global_api_endpoints:
+                default_value = format_api_name(default_api_endpoint)
+            else:
+                logging.warning(f"Default API endpoint '{default_api_endpoint}' not found in global_api_endpoints")
+    except Exception as e:
+        logging.error(f"Error setting default API endpoint: {str(e)}")
+        default_value = None
+
     with gr.TabItem("Analyze Text", visible=True):
         gr.Markdown("# Analyze / Explain / Summarize Text without ingesting it into the DB")
+
+        # Initialize state variables for pagination
+        current_page_state = gr.State(value=1)
+        total_pages_state = gr.State(value=1)
+
         with gr.Row():
             with gr.Column():
                 with gr.Row():
-                    text_to_work_input = gr.Textbox(label="Text to be Explained or Summarized",
-                                                placeholder="Enter the text you want explained or summarized here",
-                                                lines=20)
+                    text_to_work_input = gr.Textbox(
+                        label="Text to be Explained or Summarized",
+                        placeholder="Enter the text you want explained or summarized here",
+                        lines=20
+                    )
                 with gr.Row():
                     explanation_checkbox = gr.Checkbox(label="Explain Text", value=True)
                     summarization_checkbox = gr.Checkbox(label="Summarize Text", value=True)
-                    custom_prompt_checkbox = gr.Checkbox(label="Use a Custom Prompt",
-                                                         value=False,
-                                                         visible=True)
-                    preset_prompt_checkbox = gr.Checkbox(label="Use a pre-set Prompt",
-                                                         value=False,
-                                                         visible=True)
+                    custom_prompt_checkbox = gr.Checkbox(
+                        label="Use a Custom Prompt",
+                        value=False,
+                        visible=True
+                    )
+                    preset_prompt_checkbox = gr.Checkbox(
+                        label="Use a pre-set Prompt",
+                        value=False,
+                        visible=True
+                    )
                 with gr.Row():
-                    preset_prompt = gr.Dropdown(label="Select Preset Prompt",
-                                                choices=load_preset_prompts(),
-                                                visible=False)
+                    # Add pagination controls
+                    preset_prompt = gr.Dropdown(
+                        label="Select Preset Prompt",
+                        choices=[],
+                        visible=False
+                    )
+                    prev_page_button = gr.Button("Previous Page", visible=False)
+                    page_display = gr.Markdown("Page 1 of X", visible=False)
+                    next_page_button = gr.Button("Next Page", visible=False)
                 with gr.Row():
-                    custom_prompt_input = gr.Textbox(label="Custom Prompt",
-                                                     placeholder="Enter custom prompt here",
-                                                     lines=3,
-                                                     visible=False)
+                    custom_prompt_input = gr.Textbox(
+                        label="Custom Prompt",
+                        placeholder="Enter custom prompt here",
+                        lines=10,
+                        visible=False
+                    )
                 with gr.Row():
                     system_prompt_input = gr.Textbox(label="System Prompt",
                                                      value="""<s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
@@ -69,19 +102,21 @@ def create_summarize_explain_tab():
                     - Ensure adherence to specified format
                     - Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]
                     """,
-                                                     lines=3,
+                                                     lines=10,
                                                      visible=False,
                                                      interactive=True)
+                    # Refactored API selection dropdown
                     api_endpoint = gr.Dropdown(
-                        choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral",
-                                 "OpenRouter",
-                                 "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace", "Custom-OpenAI-API"],
-                        value=None,
-                        label="API to be used for request (Mandatory)"
+                        choices=["None"] + [format_api_name(api) for api in global_api_endpoints],
+                        value=default_value,
+                        label="API for Summarization/Analysis (Optional)"
                     )
                 with gr.Row():
-                    api_key_input = gr.Textbox(label="API Key (if required)", placeholder="Enter your API key here",
-                                               type="password")
+                    api_key_input = gr.Textbox(
+                        label="API Key (if required)",
+                        placeholder="Enter your API key here",
+                        type="password"
+                    )
                 with gr.Row():
                     explain_summarize_button = gr.Button("Explain/Summarize")
 
@@ -90,17 +125,83 @@ def create_summarize_explain_tab():
                 explanation_output = gr.Textbox(label="Explanation:", lines=20)
                 custom_prompt_output = gr.Textbox(label="Custom Prompt:", lines=20, visible=True)
 
+        # Handle custom prompt checkbox change
         custom_prompt_checkbox.change(
             fn=lambda x: (gr.update(visible=x), gr.update(visible=x)),
             inputs=[custom_prompt_checkbox],
             outputs=[custom_prompt_input, system_prompt_input]
         )
+
+        # Handle preset prompt checkbox change
+        def on_preset_prompt_checkbox_change(is_checked):
+            if is_checked:
+                prompts, total_pages, current_page = list_prompts(page=1, per_page=20)
+                page_display_text = f"Page {current_page} of {total_pages}"
+                return (
+                    gr.update(visible=True, interactive=True, choices=prompts),  # preset_prompt
+                    gr.update(visible=True),  # prev_page_button
+                    gr.update(visible=True),  # next_page_button
+                    gr.update(value=page_display_text, visible=True),  # page_display
+                    current_page,  # current_page_state
+                    total_pages    # total_pages_state
+                )
+            else:
+                return (
+                    gr.update(visible=False, interactive=False),  # preset_prompt
+                    gr.update(visible=False),  # prev_page_button
+                    gr.update(visible=False),  # next_page_button
+                    gr.update(visible=False),  # page_display
+                    1,  # current_page_state
+                    1   # total_pages_state
+                )
+
         preset_prompt_checkbox.change(
-            fn=lambda x: gr.update(visible=x),
+            fn=on_preset_prompt_checkbox_change,
             inputs=[preset_prompt_checkbox],
-            outputs=[preset_prompt]
+            outputs=[
+                preset_prompt,
+                prev_page_button,
+                next_page_button,
+                page_display,
+                current_page_state,
+                total_pages_state
+            ]
+        )
+
+        # Pagination button functions
+        def on_prev_page_click(current_page, total_pages):
+            new_page = max(current_page - 1, 1)
+            prompts, total_pages, current_page = list_prompts(page=new_page, per_page=20)
+            page_display_text = f"Page {current_page} of {total_pages}"
+            return (
+                gr.update(choices=prompts),
+                gr.update(value=page_display_text),
+                current_page
+            )
+
+        prev_page_button.click(
+            fn=on_prev_page_click,
+            inputs=[current_page_state, total_pages_state],
+            outputs=[preset_prompt, page_display, current_page_state]
         )
 
+        def on_next_page_click(current_page, total_pages):
+            new_page = min(current_page + 1, total_pages)
+            prompts, total_pages, current_page = list_prompts(page=new_page, per_page=20)
+            page_display_text = f"Page {current_page} of {total_pages}"
+            return (
+                gr.update(choices=prompts),
+                gr.update(value=page_display_text),
+                current_page
+            )
+
+        next_page_button.click(
+            fn=on_next_page_click,
+            inputs=[current_page_state, total_pages_state],
+            outputs=[preset_prompt, page_display, current_page_state]
+        )
+
+        # Update prompts when a preset is selected
         def update_prompts(preset_name):
             prompts = update_user_prompt(preset_name)
             return (
@@ -109,18 +210,27 @@ def create_summarize_explain_tab():
             )
 
         preset_prompt.change(
-            update_prompts,
-            inputs=preset_prompt,
+            fn=update_prompts,
+            inputs=[preset_prompt],
             outputs=[custom_prompt_input, system_prompt_input]
         )
 
         explain_summarize_button.click(
             fn=summarize_explain_text,
-            inputs=[text_to_work_input, api_endpoint, api_key_input, summarization_checkbox, explanation_checkbox, custom_prompt_input, system_prompt_input],
+            inputs=[
+                text_to_work_input,
+                api_endpoint,
+                api_key_input,
+                summarization_checkbox,
+                explanation_checkbox,
+                custom_prompt_input,
+                system_prompt_input
+            ],
             outputs=[summarization_output, explanation_output, custom_prompt_output]
         )
 
 
+
 def summarize_explain_text(message, api_endpoint, api_key, summarization, explanation, custom_prompt, custom_system_prompt,):
     global custom_prompt_output
     summarization_response = None

App_Function_Libraries/Gradio_UI/Export_Functionality.py

@@ -6,9 +6,11 @@ import math
 import logging
 import shutil
 import tempfile
-from typing import List, Dict, Optional, Tuple
+from typing import List, Dict, Optional, Tuple, Any
 import gradio as gr
-from App_Function_Libraries.DB.DB_Manager import DatabaseError
+from App_Function_Libraries.DB.DB_Manager import DatabaseError, fetch_all_notes, fetch_all_conversations, \
+    get_keywords_for_note, fetch_notes_by_ids, fetch_conversations_by_ids
+from App_Function_Libraries.DB.RAG_QA_Chat_DB import get_keywords_for_conversation
 from App_Function_Libraries.Gradio_UI.Gradio_Shared import fetch_item_details, fetch_items_by_keyword, browse_items
 
 logger = logging.getLogger(__name__)
@@ -36,7 +38,7 @@ def export_items_by_keyword(keyword: str) -> str:
         items = fetch_items_by_keyword(keyword)
         if not items:
             logger.warning(f"No items found for keyword: {keyword}")
-            return None
+            return f"No items found for keyword: {keyword}"
 
         # Create a temporary directory to store individual markdown files
         with tempfile.TemporaryDirectory() as temp_dir:
@@ -66,7 +68,7 @@ def export_items_by_keyword(keyword: str) -> str:
         return final_zip_path
     except Exception as e:
         logger.error(f"Error exporting items for keyword '{keyword}': {str(e)}")
-        return None
+        return f"Error exporting items for keyword '{keyword}': {str(e)}"
 
 
 def export_selected_items(selected_items: List[Dict]) -> Tuple[Optional[str], str]:
@@ -146,121 +148,747 @@ def display_search_results_export_tab(search_query: str, search_type: str, page:
         logger.error(error_message)
         return [], error_message, 1, 1
 
+#
+# End of Media DB Export functionality
+################################################################
 
-def create_export_tab():
-    with gr.Tab("Search and Export"):
-        with gr.Row():
-            with gr.Column():
-                gr.Markdown("# Search and Export Items")
-                gr.Markdown("Search for items and export them as markdown files")
-                gr.Markdown("You can also export items by keyword")
-                search_query = gr.Textbox(label="Search Query")
-                search_type = gr.Radio(["Title", "URL", "Keyword", "Content"], label="Search By")
-                search_button = gr.Button("Search")
-
-            with gr.Column():
-                prev_button = gr.Button("Previous Page")
-                next_button = gr.Button("Next Page")
-
-        current_page = gr.State(1)
-        total_pages = gr.State(1)
-
-        search_results = gr.CheckboxGroup(label="Search Results", choices=[])
-        export_selected_button = gr.Button("Export Selected Items")
-
-        keyword_input = gr.Textbox(label="Enter keyword for export")
-        export_by_keyword_button = gr.Button("Export items by keyword")
-
-        export_output = gr.File(label="Download Exported File")
-        error_output = gr.Textbox(label="Status/Error Messages", interactive=False)
-
-    def search_and_update(query, search_type, page):
-        results, message, current, total = display_search_results_export_tab(query, search_type, page)
-        logger.debug(f"search_and_update results: {results}")
-        return results, message, current, total, gr.update(choices=results)
-
-    search_button.click(
-        fn=search_and_update,
-        inputs=[search_query, search_type, current_page],
-        outputs=[search_results, error_output, current_page, total_pages, search_results],
-        show_progress="full"
-    )
-
-
-    def update_page(current, total, direction):
-        new_page = max(1, min(total, current + direction))
-        return new_page
-
-    prev_button.click(
-        fn=update_page,
-        inputs=[current_page, total_pages, gr.State(-1)],
-        outputs=[current_page]
-    ).then(
-        fn=search_and_update,
-        inputs=[search_query, search_type, current_page],
-        outputs=[search_results, error_output, current_page, total_pages],
-        show_progress=True
-    )
-
-    next_button.click(
-        fn=update_page,
-        inputs=[current_page, total_pages, gr.State(1)],
-        outputs=[current_page]
-    ).then(
-        fn=search_and_update,
-        inputs=[search_query, search_type, current_page],
-        outputs=[search_results, error_output, current_page, total_pages],
-        show_progress=True
-    )
-
-    def handle_export_selected(selected_items):
-        logger.debug(f"Exporting selected items: {selected_items}")
-        return export_selected_items(selected_items)
-
-    export_selected_button.click(
-        fn=handle_export_selected,
-        inputs=[search_results],
-        outputs=[export_output, error_output],
-        show_progress="full"
-    )
-
-    export_by_keyword_button.click(
-        fn=export_items_by_keyword,
-        inputs=[keyword_input],
-        outputs=[export_output, error_output],
-        show_progress="full"
-    )
-
-    def handle_item_selection(selected_items):
-        logger.debug(f"Selected items: {selected_items}")
-        if not selected_items:
-            return None, "No item selected"
-
-        try:
-            # Assuming selected_items is a list of dictionaries
-            selected_item = selected_items[0]
-            logger.debug(f"First selected item: {selected_item}")
-
-            # Check if 'value' is a string (JSON) or already a dictionary
-            if isinstance(selected_item['value'], str):
-                item_data = json.loads(selected_item['value'])
-            else:
-                item_data = selected_item['value']
-
-            logger.debug(f"Item data: {item_data}")
-
-            item_id = item_data['id']
-            return export_item_as_markdown(item_id)
-        except Exception as e:
-            error_message = f"Error processing selected item: {str(e)}"
-            logger.error(error_message)
-            return None, error_message
-
-    search_results.select(
-        fn=handle_item_selection,
-        inputs=[search_results],
-        outputs=[export_output, error_output],
-        show_progress="full"
-    )
 
+################################################################
+#
+# Functions for RAG Chat DB Export functionality
+
+
+def export_rag_conversations_as_json(
+    selected_conversations: Optional[List[Dict[str, Any]]] = None
+) -> Tuple[Optional[str], str]:
+    """
+    Export conversations to a JSON file.
+
+    Args:
+        selected_conversations: Optional list of conversation dictionaries
+
+    Returns:
+        Tuple of (filename or None, status message)
+    """
+    try:
+        if selected_conversations:
+            # Extract conversation IDs from selected items
+            conversation_ids = []
+            for item in selected_conversations:
+                if isinstance(item, str):
+                    item_data = json.loads(item)
+                elif isinstance(item, dict) and 'value' in item:
+                    item_data = item['value'] if isinstance(item['value'], dict) else json.loads(item['value'])
+                else:
+                    item_data = item
+                conversation_ids.append(item_data['conversation_id'])
+
+            conversations = fetch_conversations_by_ids(conversation_ids)
+        else:
+            conversations = fetch_all_conversations()
+
+        export_data = []
+        for conversation_id, title, messages in conversations:
+            # Get keywords for the conversation
+            keywords = get_keywords_for_conversation(conversation_id)
+
+            conversation_data = {
+                "conversation_id": conversation_id,
+                "title": title,
+                "keywords": keywords,
+                "messages": [
+                    {"role": role, "content": content}
+                    for role, content in messages
+                ]
+            }
+            export_data.append(conversation_data)
+
+        filename = "rag_conversations_export.json"
+        with open(filename, "w", encoding='utf-8') as f:
+            json.dump(export_data, f, indent=2, ensure_ascii=False)
+
+        logger.info(f"Successfully exported {len(export_data)} conversations to {filename}")
+        return filename, f"Successfully exported {len(export_data)} conversations to {filename}"
+    except Exception as e:
+        error_message = f"Error exporting conversations: {str(e)}"
+        logger.error(error_message)
+        return None, error_message
+
+
+def export_rag_notes_as_json(
+    selected_notes: Optional[List[Dict[str, Any]]] = None
+) -> Tuple[Optional[str], str]:
+    """
+    Export notes to a JSON file.
+
+    Args:
+        selected_notes: Optional list of note dictionaries
+
+    Returns:
+        Tuple of (filename or None, status message)
+    """
+    try:
+        if selected_notes:
+            # Extract note IDs from selected items
+            note_ids = []
+            for item in selected_notes:
+                if isinstance(item, str):
+                    item_data = json.loads(item)
+                elif isinstance(item, dict) and 'value' in item:
+                    item_data = item['value'] if isinstance(item['value'], dict) else json.loads(item['value'])
+                else:
+                    item_data = item
+                note_ids.append(item_data['id'])
+
+            notes = fetch_notes_by_ids(note_ids)
+        else:
+            notes = fetch_all_notes()
+
+        export_data = []
+        for note_id, title, content in notes:
+            # Get keywords for the note
+            keywords = get_keywords_for_note(note_id)
+
+            note_data = {
+                "note_id": note_id,
+                "title": title,
+                "content": content,
+                "keywords": keywords
+            }
+            export_data.append(note_data)
+
+        filename = "rag_notes_export.json"
+        with open(filename, "w", encoding='utf-8') as f:
+            json.dump(export_data, f, indent=2, ensure_ascii=False)
+
+        logger.info(f"Successfully exported {len(export_data)} notes to {filename}")
+        return filename, f"Successfully exported {len(export_data)} notes to {filename}"
+    except Exception as e:
+        error_message = f"Error exporting notes: {str(e)}"
+        logger.error(error_message)
+        return None, error_message
+
+
+def display_rag_conversations(search_query: str = "", page: int = 1, items_per_page: int = 10):
+    """Display conversations for selection in the export tab."""
+    try:
+        conversations = fetch_all_conversations()
+
+        if search_query:
+            # Simple search implementation - can be enhanced based on needs
+            conversations = [
+                conv for conv in conversations
+                if search_query.lower() in conv[1].lower()  # Search in title
+            ]
+
+        # Implement pagination
+        start_idx = (page - 1) * items_per_page
+        end_idx = start_idx + items_per_page
+        paginated_conversations = conversations[start_idx:end_idx]
+        total_pages = (len(conversations) + items_per_page - 1) // items_per_page
+
+        # Format for checkbox group
+        checkbox_data = [
+            {
+                "name": f"Title: {title}\nMessages: {len(messages)}",
+                "value": {"conversation_id": conv_id, "title": title}
+            }
+            for conv_id, title, messages in paginated_conversations
+        ]
+
+        return (
+            checkbox_data,
+            f"Found {len(conversations)} conversations (showing page {page} of {total_pages})",
+            page,
+            total_pages
+        )
+    except Exception as e:
+        error_message = f"Error displaying conversations: {str(e)}"
+        logger.error(error_message)
+        return [], error_message, 1, 1
+
+
+def display_rag_notes(search_query: str = "", page: int = 1, items_per_page: int = 10):
+    """Display notes for selection in the export tab."""
+    try:
+        notes = fetch_all_notes()
+
+        if search_query:
+            # Simple search implementation - can be enhanced based on needs
+            notes = [
+                note for note in notes
+                if search_query.lower() in note[1].lower()  # Search in title
+                   or search_query.lower() in note[2].lower()  # Search in content
+            ]
+
+        # Implement pagination
+        start_idx = (page - 1) * items_per_page
+        end_idx = start_idx + items_per_page
+        paginated_notes = notes[start_idx:end_idx]
+        total_pages = (len(notes) + items_per_page - 1) // items_per_page
+
+        # Format for checkbox group
+        checkbox_data = [
+            {
+                "name": f"Title: {title}\nContent preview: {content[:100]}...",
+                "value": {"id": note_id, "title": title}
+            }
+            for note_id, title, content in paginated_notes
+        ]
+
+        return (
+            checkbox_data,
+            f"Found {len(notes)} notes (showing page {page} of {total_pages})",
+            page,
+            total_pages
+        )
+    except Exception as e:
+        error_message = f"Error displaying notes: {str(e)}"
+        logger.error(error_message)
+        return [], error_message, 1, 1
+
+
+def create_rag_export_tab():
+    """Create the RAG QA Chat export tab interface."""
+    with gr.Tab("RAG QA Chat Export"):
+        with gr.Tabs():
+            # Conversations Export Tab
+            with gr.Tab("Export Conversations"):
+                with gr.Row():
+                    with gr.Column():
+                        gr.Markdown("## Export RAG QA Chat Conversations")
+                        conversation_search = gr.Textbox(label="Search Conversations")
+                        conversation_search_button = gr.Button("Search")
+
+                    with gr.Column():
+                        conversation_prev_button = gr.Button("Previous Page")
+                        conversation_next_button = gr.Button("Next Page")
+
+                conversation_current_page = gr.State(1)
+                conversation_total_pages = gr.State(1)
+
+                conversation_results = gr.CheckboxGroup(label="Select Conversations to Export")
+                export_selected_conversations_button = gr.Button("Export Selected Conversations")
+                export_all_conversations_button = gr.Button("Export All Conversations")
+
+                conversation_export_output = gr.File(label="Download Exported Conversations")
+                conversation_status = gr.Textbox(label="Status", interactive=False)
+
+            # Notes Export Tab
+            with gr.Tab("Export Notes"):
+                with gr.Row():
+                    with gr.Column():
+                        gr.Markdown("## Export RAG QA Chat Notes")
+                        notes_search = gr.Textbox(label="Search Notes")
+                        notes_search_button = gr.Button("Search")
+
+                    with gr.Column():
+                        notes_prev_button = gr.Button("Previous Page")
+                        notes_next_button = gr.Button("Next Page")
+
+                notes_current_page = gr.State(1)
+                notes_total_pages = gr.State(1)
+
+                notes_results = gr.CheckboxGroup(label="Select Notes to Export")
+                export_selected_notes_button = gr.Button("Export Selected Notes")
+                export_all_notes_button = gr.Button("Export All Notes")
+
+                notes_export_output = gr.File(label="Download Exported Notes")
+                notes_status = gr.Textbox(label="Status", interactive=False)
+
+        # Event handlers for conversations
+        def search_conversations(query, page):
+            return display_rag_conversations(query, page)
+
+        conversation_search_button.click(
+            fn=search_conversations,
+            inputs=[conversation_search, conversation_current_page],
+            outputs=[conversation_results, conversation_status, conversation_current_page, conversation_total_pages]
+        )
+
+        def update_conversation_page(current, total, direction):
+            new_page = max(1, min(total, current + direction))
+            return new_page
+
+        conversation_prev_button.click(
+            fn=update_conversation_page,
+            inputs=[conversation_current_page, conversation_total_pages, gr.State(-1)],
+            outputs=[conversation_current_page]
+        ).then(
+            fn=search_conversations,
+            inputs=[conversation_search, conversation_current_page],
+            outputs=[conversation_results, conversation_status, conversation_current_page, conversation_total_pages]
+        )
+
+        conversation_next_button.click(
+            fn=update_conversation_page,
+            inputs=[conversation_current_page, conversation_total_pages, gr.State(1)],
+            outputs=[conversation_current_page]
+        ).then(
+            fn=search_conversations,
+            inputs=[conversation_search, conversation_current_page],
+            outputs=[conversation_results, conversation_status, conversation_current_page, conversation_total_pages]
+        )
+
+        export_selected_conversations_button.click(
+            fn=export_rag_conversations_as_json,
+            inputs=[conversation_results],
+            outputs=[conversation_export_output, conversation_status]
+        )
+
+        export_all_conversations_button.click(
+            fn=lambda: export_rag_conversations_as_json(),
+            outputs=[conversation_export_output, conversation_status]
+        )
+
+        # Event handlers for notes
+        def search_notes(query, page):
+            return display_rag_notes(query, page)
+
+        notes_search_button.click(
+            fn=search_notes,
+            inputs=[notes_search, notes_current_page],
+            outputs=[notes_results, notes_status, notes_current_page, notes_total_pages]
+        )
+
+        def update_notes_page(current, total, direction):
+            new_page = max(1, min(total, current + direction))
+            return new_page
+
+        notes_prev_button.click(
+            fn=update_notes_page,
+            inputs=[notes_current_page, notes_total_pages, gr.State(-1)],
+            outputs=[notes_current_page]
+        ).then(
+            fn=search_notes,
+            inputs=[notes_search, notes_current_page],
+            outputs=[notes_results, notes_status, notes_current_page, notes_total_pages]
+        )
+
+        notes_next_button.click(
+            fn=update_notes_page,
+            inputs=[notes_current_page, notes_total_pages, gr.State(1)],
+            outputs=[notes_current_page]
+        ).then(
+            fn=search_notes,
+            inputs=[notes_search, notes_current_page],
+            outputs=[notes_results, notes_status, notes_current_page, notes_total_pages]
+        )
+
+        export_selected_notes_button.click(
+            fn=export_rag_notes_as_json,
+            inputs=[notes_results],
+            outputs=[notes_export_output, notes_status]
+        )
+
+        export_all_notes_button.click(
+            fn=lambda: export_rag_notes_as_json(),
+            outputs=[notes_export_output, notes_status]
+        )
+
+#
+# End of RAG Chat DB Export functionality
+#####################################################
+
+def create_export_tabs():
+    """Create the unified export interface with all export tabs."""
+    with gr.Tabs():
+        # Media DB Export Tab
+        with gr.Tab("Media DB Export"):
+            with gr.Row():
+                with gr.Column():
+                    gr.Markdown("# Search and Export Items")
+                    gr.Markdown("Search for items and export them as markdown files")
+                    gr.Markdown("You can also export items by keyword")
+                    search_query = gr.Textbox(label="Search Query")
+                    search_type = gr.Radio(["Title", "URL", "Keyword", "Content"], label="Search By")
+                    search_button = gr.Button("Search")
+
+                with gr.Column():
+                    prev_button = gr.Button("Previous Page")
+                    next_button = gr.Button("Next Page")
+
+            current_page = gr.State(1)
+            total_pages = gr.State(1)
+
+            search_results = gr.CheckboxGroup(label="Search Results", choices=[])
+            export_selected_button = gr.Button("Export Selected Items")
+
+            keyword_input = gr.Textbox(label="Enter keyword for export")
+            export_by_keyword_button = gr.Button("Export items by keyword")
+
+            export_output = gr.File(label="Download Exported File")
+            error_output = gr.Textbox(label="Status/Error Messages", interactive=False)
+
+        # Conversations Export Tab
+        with gr.Tab("RAG Conversations Export"):
+            with gr.Row():
+                with gr.Column():
+                    gr.Markdown("## Export RAG QA Chat Conversations")
+                    conversation_search = gr.Textbox(label="Search Conversations")
+                    conversation_search_button = gr.Button("Search")
+
+                with gr.Column():
+                    conversation_prev_button = gr.Button("Previous Page")
+                    conversation_next_button = gr.Button("Next Page")
+
+            conversation_current_page = gr.State(1)
+            conversation_total_pages = gr.State(1)
+
+            conversation_results = gr.CheckboxGroup(label="Select Conversations to Export")
+            export_selected_conversations_button = gr.Button("Export Selected Conversations")
+            export_all_conversations_button = gr.Button("Export All Conversations")
+
+            conversation_export_output = gr.File(label="Download Exported Conversations")
+            conversation_status = gr.Textbox(label="Status", interactive=False)
+
+        # Notes Export Tab
+        with gr.Tab("RAG Notes Export"):
+            with gr.Row():
+                with gr.Column():
+                    gr.Markdown("## Export RAG QA Chat Notes")
+                    notes_search = gr.Textbox(label="Search Notes")
+                    notes_search_button = gr.Button("Search")
+
+                with gr.Column():
+                    notes_prev_button = gr.Button("Previous Page")
+                    notes_next_button = gr.Button("Next Page")
+
+            notes_current_page = gr.State(1)
+            notes_total_pages = gr.State(1)
+
+            notes_results = gr.CheckboxGroup(label="Select Notes to Export")
+            export_selected_notes_button = gr.Button("Export Selected Notes")
+            export_all_notes_button = gr.Button("Export All Notes")
+
+            notes_export_output = gr.File(label="Download Exported Notes")
+            notes_status = gr.Textbox(label="Status", interactive=False)
+
+        # Event handlers for media DB
+        def search_and_update(query, search_type, page):
+            results, message, current, total = display_search_results_export_tab(query, search_type, page)
+            logger.debug(f"search_and_update results: {results}")
+            return results, message, current, total, gr.update(choices=results)
+
+        def update_page(current, total, direction):
+            new_page = max(1, min(total, current + direction))
+            return new_page
+
+        def handle_export_selected(selected_items):
+            logger.debug(f"Exporting selected items: {selected_items}")
+            return export_selected_items(selected_items)
+
+        def handle_item_selection(selected_items):
+            logger.debug(f"Selected items: {selected_items}")
+            if not selected_items:
+                return None, "No item selected"
+
+            try:
+                selected_item = selected_items[0]
+                logger.debug(f"First selected item: {selected_item}")
+
+                if isinstance(selected_item['value'], str):
+                    item_data = json.loads(selected_item['value'])
+                else:
+                    item_data = selected_item['value']
+
+                logger.debug(f"Item data: {item_data}")
+                item_id = item_data['id']
+                return export_item_as_markdown(item_id)
+            except Exception as e:
+                error_message = f"Error processing selected item: {str(e)}"
+                logger.error(error_message)
+                return None, error_message
+
+        search_button.click(
+            fn=search_and_update,
+            inputs=[search_query, search_type, current_page],
+            outputs=[search_results, error_output, current_page, total_pages, search_results],
+            show_progress="full"
+        )
+
+        prev_button.click(
+            fn=update_page,
+            inputs=[current_page, total_pages, gr.State(-1)],
+            outputs=[current_page]
+        ).then(
+            fn=search_and_update,
+            inputs=[search_query, search_type, current_page],
+            outputs=[search_results, error_output, current_page, total_pages],
+            show_progress=True
+        )
+
+        next_button.click(
+            fn=update_page,
+            inputs=[current_page, total_pages, gr.State(1)],
+            outputs=[current_page]
+        ).then(
+            fn=search_and_update,
+            inputs=[search_query, search_type, current_page],
+            outputs=[search_results, error_output, current_page, total_pages],
+            show_progress=True
+        )
+
+        export_selected_button.click(
+            fn=handle_export_selected,
+            inputs=[search_results],
+            outputs=[export_output, error_output],
+            show_progress="full"
+        )
+
+        export_by_keyword_button.click(
+            fn=export_items_by_keyword,
+            inputs=[keyword_input],
+            outputs=[export_output, error_output],
+            show_progress="full"
+        )
+
+        search_results.select(
+            fn=handle_item_selection,
+            inputs=[search_results],
+            outputs=[export_output, error_output],
+            show_progress="full"
+        )
+
+        # Event handlers for conversations
+        def search_conversations(query, page):
+            return display_rag_conversations(query, page)
+
+        def update_conversation_page(current, total, direction):
+            new_page = max(1, min(total, current + direction))
+            return new_page
+
+        conversation_search_button.click(
+            fn=search_conversations,
+            inputs=[conversation_search, conversation_current_page],
+            outputs=[conversation_results, conversation_status, conversation_current_page, conversation_total_pages]
+        )
+
+        conversation_prev_button.click(
+            fn=update_conversation_page,
+            inputs=[conversation_current_page, conversation_total_pages, gr.State(-1)],
+            outputs=[conversation_current_page]
+        ).then(
+            fn=search_conversations,
+            inputs=[conversation_search, conversation_current_page],
+            outputs=[conversation_results, conversation_status, conversation_current_page, conversation_total_pages]
+        )
+
+        conversation_next_button.click(
+            fn=update_conversation_page,
+            inputs=[conversation_current_page, conversation_total_pages, gr.State(1)],
+            outputs=[conversation_current_page]
+        ).then(
+            fn=search_conversations,
+            inputs=[conversation_search, conversation_current_page],
+            outputs=[conversation_results, conversation_status, conversation_current_page, conversation_total_pages]
+        )
+
+        export_selected_conversations_button.click(
+            fn=export_rag_conversations_as_json,
+            inputs=[conversation_results],
+            outputs=[conversation_export_output, conversation_status]
+        )
+
+        export_all_conversations_button.click(
+            fn=lambda: export_rag_conversations_as_json(),
+            outputs=[conversation_export_output, conversation_status]
+        )
+
+        # Event handlers for notes
+        def search_notes(query, page):
+            return display_rag_notes(query, page)
+
+        def update_notes_page(current, total, direction):
+            new_page = max(1, min(total, current + direction))
+            return new_page
+
+        notes_search_button.click(
+            fn=search_notes,
+            inputs=[notes_search, notes_current_page],
+            outputs=[notes_results, notes_status, notes_current_page, notes_total_pages]
+        )
+
+        notes_prev_button.click(
+            fn=update_notes_page,
+            inputs=[notes_current_page, notes_total_pages, gr.State(-1)],
+            outputs=[notes_current_page]
+        ).then(
+            fn=search_notes,
+            inputs=[notes_search, notes_current_page],
+            outputs=[notes_results, notes_status, notes_current_page, notes_total_pages]
+        )
+
+        notes_next_button.click(
+            fn=update_notes_page,
+            inputs=[notes_current_page, notes_total_pages, gr.State(1)],
+            outputs=[notes_current_page]
+        ).then(
+            fn=search_notes,
+            inputs=[notes_search, notes_current_page],
+            outputs=[notes_results, notes_status, notes_current_page, notes_total_pages]
+        )
+
+        export_selected_notes_button.click(
+            fn=export_rag_notes_as_json,
+            inputs=[notes_results],
+            outputs=[notes_export_output, notes_status]
+        )
+
+        export_all_notes_button.click(
+            fn=lambda: export_rag_notes_as_json(),
+            outputs=[notes_export_output, notes_status]
+        )
+
+        with gr.TabItem("Export Prompts", visible=True):
+            gr.Markdown("# Export Prompts Database Content")
+
+            with gr.Row():
+                with gr.Column():
+                    export_type = gr.Radio(
+                        choices=["All Prompts", "Prompts by Keyword"],
+                        label="Export Type",
+                        value="All Prompts"
+                    )
+
+                    # Keyword selection for filtered export
+                    with gr.Column(visible=False) as keyword_col:
+                        keyword_input = gr.Textbox(
+                            label="Enter Keywords (comma-separated)",
+                            placeholder="Enter keywords to filter prompts..."
+                        )
+
+                    # Export format selection
+                    export_format = gr.Radio(
+                        choices=["CSV", "Markdown (ZIP)"],
+                        label="Export Format",
+                        value="CSV"
+                    )
+
+                    # Export options
+                    include_options = gr.CheckboxGroup(
+                        choices=[
+                            "Include System Prompts",
+                            "Include User Prompts",
+                            "Include Details",
+                            "Include Author",
+                            "Include Keywords"
+                        ],
+                        label="Export Options",
+                        value=["Include Keywords", "Include Author"]
+                    )
+
+                    # Markdown-specific options (only visible when Markdown is selected)
+                    with gr.Column(visible=False) as markdown_options_col:
+                        markdown_template = gr.Radio(
+                            choices=[
+                                "Basic Template",
+                                "Detailed Template",
+                                "Custom Template"
+                            ],
+                            label="Markdown Template",
+                            value="Basic Template"
+                        )
+                        custom_template = gr.Textbox(
+                            label="Custom Template",
+                            placeholder="Use {title}, {author}, {details}, {system}, {user}, {keywords} as placeholders",
+                            visible=False
+                        )
+
+                    export_button = gr.Button("Export Prompts")
+
+                with gr.Column():
+                    export_status = gr.Textbox(label="Export Status", interactive=False)
+                    export_file = gr.File(label="Download Export")
+
+            def update_ui_visibility(export_type, format_choice, template_choice):
+                """Update UI elements visibility based on selections"""
+                show_keywords = export_type == "Prompts by Keyword"
+                show_markdown_options = format_choice == "Markdown (ZIP)"
+                show_custom_template = template_choice == "Custom Template" and show_markdown_options
+
+                return [
+                    gr.update(visible=show_keywords),  # keyword_col
+                    gr.update(visible=show_markdown_options),  # markdown_options_col
+                    gr.update(visible=show_custom_template)  # custom_template
+                ]
+
+            def handle_export(export_type, keywords, export_format, options, markdown_template, custom_template):
+                """Handle the export process based on selected options"""
+                try:
+                    # Parse options
+                    include_system = "Include System Prompts" in options
+                    include_user = "Include User Prompts" in options
+                    include_details = "Include Details" in options
+                    include_author = "Include Author" in options
+                    include_keywords = "Include Keywords" in options
+
+                    # Handle keyword filtering
+                    keyword_list = None
+                    if export_type == "Prompts by Keyword" and keywords:
+                        keyword_list = [k.strip() for k in keywords.split(",") if k.strip()]
+
+                    # Get the appropriate template
+                    template = None
+                    if export_format == "Markdown (ZIP)":
+                        if markdown_template == "Custom Template":
+                            template = custom_template
+                        else:
+                            template = markdown_template
+
+                    # Perform export
+                    from App_Function_Libraries.DB.Prompts_DB import export_prompts
+                    status, file_path = export_prompts(
+                        export_format=export_format.split()[0].lower(),  # 'csv' or 'markdown'
+                        filter_keywords=keyword_list,
+                        include_system=include_system,
+                        include_user=include_user,
+                        include_details=include_details,
+                        include_author=include_author,
+                        include_keywords=include_keywords,
+                        markdown_template=template
+                    )
+
+                    return status, file_path
+
+                except Exception as e:
+                    error_msg = f"Export failed: {str(e)}"
+                    logging.error(error_msg)
+                    return error_msg, None
+
+            # Event handlers
+            export_type.change(
+                fn=lambda t, f, m: update_ui_visibility(t, f, m),
+                inputs=[export_type, export_format, markdown_template],
+                outputs=[keyword_col, markdown_options_col, custom_template]
+            )
+
+            export_format.change(
+                fn=lambda t, f, m: update_ui_visibility(t, f, m),
+                inputs=[export_type, export_format, markdown_template],
+                outputs=[keyword_col, markdown_options_col, custom_template]
+            )
+
+            markdown_template.change(
+                fn=lambda t, f, m: update_ui_visibility(t, f, m),
+                inputs=[export_type, export_format, markdown_template],
+                outputs=[keyword_col, markdown_options_col, custom_template]
+            )
+
+            export_button.click(
+                fn=handle_export,
+                inputs=[
+                    export_type,
+                    keyword_input,
+                    export_format,
+                    include_options,
+                    markdown_template,
+                    custom_template
+                ],
+                outputs=[export_status, export_file]
+            )
+
+#
+# End of Export_Functionality.py
+######################################################################################################################
 

App_Function_Libraries/Gradio_UI/Gradio_Shared.py

@@ -216,11 +216,6 @@ def format_content(content):
     return formatted_content
 
 
-def update_prompt_dropdown():
-    prompt_names = list_prompts()
-    return gr.update(choices=prompt_names)
-
-
 def display_prompt_details(selected_prompt):
     if selected_prompt:
         prompts = update_user_prompt(selected_prompt)

App_Function_Libraries/Gradio_UI/Import_Functionality.py

@@ -2,24 +2,31 @@
 # Functionality to import content into the DB
 #
 # Imports
+from datetime import datetime
 from time import sleep
 import logging
 import re
 import shutil
 import tempfile
 import os
+from pathlib import Path
+import sqlite3
 import traceback
+from typing import Optional, List, Dict, Tuple
+import uuid
 import zipfile
 #
 # External Imports
 import gradio as gr
+from chardet import detect
+
 #
 # Local Imports
-from App_Function_Libraries.DB.DB_Manager import insert_prompt_to_db, load_preset_prompts, import_obsidian_note_to_db, \
-    add_media_to_database
+from App_Function_Libraries.DB.DB_Manager import insert_prompt_to_db, import_obsidian_note_to_db, \
+    add_media_to_database, list_prompts
 from App_Function_Libraries.Prompt_Handling import import_prompt_from_file, import_prompts_from_zip#
 from App_Function_Libraries.Summarization.Summarization_General_Lib import perform_summarization
-
+#
 ###################################################################################################################
 #
 # Functions:
@@ -203,15 +210,6 @@ def create_import_single_prompt_tab():
             outputs=save_output
         )
 
-        def update_prompt_dropdown():
-            return gr.update(choices=load_preset_prompts())
-
-        save_button.click(
-            fn=update_prompt_dropdown,
-            inputs=[],
-            outputs=[gr.Dropdown(label="Select Preset Prompt")]
-        )
-
 def create_import_item_tab():
     with gr.TabItem("Import Markdown/Text Files", visible=True):
         gr.Markdown("# Import a markdown file or text file into the database")
@@ -250,11 +248,18 @@ def create_import_multiple_prompts_tab():
         gr.Markdown("# Import multiple prompts into the database")
         gr.Markdown("Upload a zip file containing multiple prompt files (txt or md)")
 
+        # Initialize state variables for pagination
+        current_page_state = gr.State(value=1)
+        total_pages_state = gr.State(value=1)
+
         with gr.Row():
             with gr.Column():
                 zip_file = gr.File(label="Upload zip file for import", file_types=["zip"])
                 import_button = gr.Button("Import Prompts")
                 prompts_dropdown = gr.Dropdown(label="Select Prompt to Edit", choices=[])
+                prev_page_button = gr.Button("Previous Page", visible=False)
+                page_display = gr.Markdown("Page 1 of X", visible=False)
+                next_page_button = gr.Button("Next Page", visible=False)
                 title_input = gr.Textbox(label="Title", placeholder="Enter the title of the content")
                 author_input = gr.Textbox(label="Author", placeholder="Enter the author's name")
                 system_input = gr.Textbox(label="System", placeholder="Enter the system message for the prompt",
@@ -268,6 +273,10 @@ def create_import_multiple_prompts_tab():
                 save_output = gr.Textbox(label="Save Status")
                 prompts_display = gr.Textbox(label="Identified Prompts")
 
+        # State to store imported prompts
+        zip_import_state = gr.State([])
+
+        # Function to handle zip import
         def handle_zip_import(zip_file):
             result = import_prompts_from_zip(zip_file)
             if isinstance(result, list):
@@ -278,6 +287,13 @@ def create_import_multiple_prompts_tab():
             else:
                 return gr.update(value=result), [], gr.update(value=""), []
 
+        import_button.click(
+            fn=handle_zip_import,
+            inputs=[zip_file],
+            outputs=[import_output, prompts_dropdown, prompts_display, zip_import_state]
+        )
+
+        # Function to handle prompt selection from imported prompts
         def handle_prompt_selection(selected_title, prompts):
             selected_prompt = next((prompt for prompt in prompts if prompt['title'] == selected_title), None)
             if selected_prompt:
@@ -305,23 +321,68 @@ def create_import_multiple_prompts_tab():
             outputs=[title_input, author_input, system_input, user_input, keywords_input]
         )
 
+        # Function to save prompt to the database
         def save_prompt_to_db(title, author, system, user, keywords):
             keyword_list = [k.strip() for k in keywords.split(',') if k.strip()]
-            return insert_prompt_to_db(title, author, system, user, keyword_list)
+            result = insert_prompt_to_db(title, author, system, user, keyword_list)
+            return result
 
         save_button.click(
             fn=save_prompt_to_db,
             inputs=[title_input, author_input, system_input, user_input, keywords_input],
-            outputs=save_output
+            outputs=[save_output]
+        )
+
+        # Adding pagination controls to navigate prompts in the database
+        def on_prev_page_click(current_page, total_pages):
+            new_page = max(current_page - 1, 1)
+            prompts, total_pages, current_page = list_prompts(page=new_page, per_page=10)
+            page_display_text = f"Page {current_page} of {total_pages}"
+            return (
+                gr.update(choices=prompts),
+                gr.update(value=page_display_text),
+                current_page
+            )
+
+        def on_next_page_click(current_page, total_pages):
+            new_page = min(current_page + 1, total_pages)
+            prompts, total_pages, current_page = list_prompts(page=new_page, per_page=10)
+            page_display_text = f"Page {current_page} of {total_pages}"
+            return (
+                gr.update(choices=prompts),
+                gr.update(value=page_display_text),
+                current_page
+            )
+
+        prev_page_button.click(
+            fn=on_prev_page_click,
+            inputs=[current_page_state, total_pages_state],
+            outputs=[prompts_dropdown, page_display, current_page_state]
+        )
+
+        next_page_button.click(
+            fn=on_next_page_click,
+            inputs=[current_page_state, total_pages_state],
+            outputs=[prompts_dropdown, page_display, current_page_state]
         )
 
+        # Function to update prompts dropdown after saving to the database
         def update_prompt_dropdown():
-            return gr.update(choices=load_preset_prompts())
+            prompts, total_pages, current_page = list_prompts(page=1, per_page=10)
+            page_display_text = f"Page {current_page} of {total_pages}"
+            return (
+                gr.update(choices=prompts),
+                gr.update(visible=True),
+                gr.update(value=page_display_text, visible=True),
+                current_page,
+                total_pages
+            )
 
+        # Update the dropdown after saving
         save_button.click(
             fn=update_prompt_dropdown,
             inputs=[],
-            outputs=[gr.Dropdown(label="Select Preset Prompt")]
+            outputs=[prompts_dropdown, prev_page_button, page_display, current_page_state, total_pages_state]
         )
 
 
@@ -385,4 +446,392 @@ def import_obsidian_vault(vault_path, progress=gr.Progress()):
     except Exception as e:
         error_msg = f"Error scanning vault: {str(e)}\n{traceback.format_exc()}"
         logger.error(error_msg)
-        return 0, 0, [error_msg]
+        return 0, 0, [error_msg]
+
+
+class RAGQABatchImporter:
+    def __init__(self, db_path: str):
+        self.db_path = Path(db_path)
+        self.setup_logging()
+        self.file_processor = FileProcessor()
+        self.zip_validator = ZipValidator()
+
+    def setup_logging(self):
+        logging.basicConfig(
+            level=logging.INFO,
+            format='%(asctime)s - %(levelname)s - %(message)s',
+            handlers=[
+                logging.FileHandler('rag_qa_import.log'),
+                logging.StreamHandler()
+            ]
+        )
+
+    def process_markdown_content(self, content: str) -> List[Dict[str, str]]:
+        """Process markdown content into a conversation format."""
+        messages = []
+        sections = content.split('\n\n')
+
+        for section in sections:
+            if section.strip():
+                messages.append({
+                    'role': 'user',
+                    'content': section.strip()
+                })
+
+        return messages
+
+    def process_keywords(self, db: sqlite3.Connection, conversation_id: str, keywords: str):
+        """Process and link keywords to a conversation."""
+        if not keywords:
+            return
+
+        keyword_list = [k.strip() for k in keywords.split(',')]
+        for keyword in keyword_list:
+            # Insert keyword if it doesn't exist
+            db.execute("""
+                INSERT OR IGNORE INTO rag_qa_keywords (keyword)
+                VALUES (?)
+            """, (keyword,))
+
+            # Get keyword ID
+            keyword_id = db.execute("""
+                SELECT id FROM rag_qa_keywords WHERE keyword = ?
+            """, (keyword,)).fetchone()[0]
+
+            # Link keyword to conversation
+            db.execute("""
+                INSERT INTO rag_qa_conversation_keywords 
+                (conversation_id, keyword_id)
+                VALUES (?, ?)
+            """, (conversation_id, keyword_id))
+
+    def import_single_file(
+            self,
+            db: sqlite3.Connection,
+            content: str,
+            filename: str,
+            keywords: str,
+            custom_prompt: Optional[str] = None,
+            rating: Optional[int] = None
+    ) -> str:
+        """Import a single file's content into the database"""
+        conversation_id = str(uuid.uuid4())
+        current_time = datetime.now().isoformat()
+
+        # Process filename into title
+        title = FileProcessor.process_filename_to_title(filename)
+        if title.lower().endswith(('.md', '.txt')):
+            title = title[:-3] if title.lower().endswith('.md') else title[:-4]
+
+        # Insert conversation metadata
+        db.execute("""
+            INSERT INTO conversation_metadata 
+            (conversation_id, created_at, last_updated, title, rating)
+            VALUES (?, ?, ?, ?, ?)
+        """, (conversation_id, current_time, current_time, title, rating))
+
+        # Process content and insert messages
+        messages = self.process_markdown_content(content)
+        for msg in messages:
+            db.execute("""
+                INSERT INTO rag_qa_chats 
+                (conversation_id, timestamp, role, content)
+                VALUES (?, ?, ?, ?)
+            """, (conversation_id, current_time, msg['role'], msg['content']))
+
+        # Process keywords
+        self.process_keywords(db, conversation_id, keywords)
+
+        return conversation_id
+
+    def extract_zip(self, zip_path: str) -> List[Tuple[str, str]]:
+        """Extract and validate files from zip"""
+        is_valid, error_msg, valid_files = self.zip_validator.validate_zip_file(zip_path)
+        if not is_valid:
+            raise ValueError(error_msg)
+
+        files = []
+        with zipfile.ZipFile(zip_path, 'r') as zip_ref:
+            for filename in valid_files:
+                with zip_ref.open(filename) as f:
+                    content = f.read()
+                    # Try to decode with detected encoding
+                    try:
+                        detected_encoding = detect(content)['encoding'] or 'utf-8'
+                        content = content.decode(detected_encoding)
+                    except UnicodeDecodeError:
+                        content = content.decode('utf-8', errors='replace')
+
+                    filename = os.path.basename(filename)
+                    files.append((filename, content))
+        return files
+
+    def import_files(
+            self,
+            files: List[str],
+            keywords: str = "",
+            custom_prompt: Optional[str] = None,
+            rating: Optional[int] = None,
+            progress=gr.Progress()
+    ) -> Tuple[bool, str]:
+        """Import multiple files or zip files into the RAG QA database."""
+        try:
+            imported_files = []
+
+            with sqlite3.connect(self.db_path) as db:
+                # Process each file
+                for file_path in progress.tqdm(files, desc="Processing files"):
+                    filename = os.path.basename(file_path)
+
+                    # Handle zip files
+                    if filename.lower().endswith('.zip'):
+                        zip_files = self.extract_zip(file_path)
+                        for zip_filename, content in progress.tqdm(zip_files, desc=f"Processing files from {filename}"):
+                            conv_id = self.import_single_file(
+                                db=db,
+                                content=content,
+                                filename=zip_filename,
+                                keywords=keywords,
+                                custom_prompt=custom_prompt,
+                                rating=rating
+                            )
+                            imported_files.append(zip_filename)
+
+                    # Handle individual markdown/text files
+                    elif filename.lower().endswith(('.md', '.txt')):
+                        with open(file_path, 'r', encoding='utf-8') as f:
+                            content = f.read()
+
+                        conv_id = self.import_single_file(
+                            db=db,
+                            content=content,
+                            filename=filename,
+                            keywords=keywords,
+                            custom_prompt=custom_prompt,
+                            rating=rating
+                        )
+                        imported_files.append(filename)
+
+                db.commit()
+
+            return True, f"Successfully imported {len(imported_files)} files:\n" + "\n".join(imported_files)
+
+        except Exception as e:
+            logging.error(f"Import failed: {str(e)}")
+            return False, f"Import failed: {str(e)}"
+
+
+class FileProcessor:
+    """Handles file reading and name processing"""
+
+    VALID_EXTENSIONS = {'.md', '.txt', '.zip'}
+    ENCODINGS_TO_TRY = [
+        'utf-8',
+        'utf-16',
+        'windows-1252',
+        'iso-8859-1',
+        'ascii'
+    ]
+
+    @staticmethod
+    def detect_encoding(file_path: str) -> str:
+        """Detect the file encoding using chardet"""
+        with open(file_path, 'rb') as file:
+            raw_data = file.read()
+            result = detect(raw_data)
+            return result['encoding'] or 'utf-8'
+
+    @staticmethod
+    def read_file_content(file_path: str) -> str:
+        """Read file content with automatic encoding detection"""
+        detected_encoding = FileProcessor.detect_encoding(file_path)
+
+        # Try detected encoding first
+        try:
+            with open(file_path, 'r', encoding=detected_encoding) as f:
+                return f.read()
+        except UnicodeDecodeError:
+            # If detected encoding fails, try others
+            for encoding in FileProcessor.ENCODINGS_TO_TRY:
+                try:
+                    with open(file_path, 'r', encoding=encoding) as f:
+                        return f.read()
+                except UnicodeDecodeError:
+                    continue
+
+            # If all encodings fail, use utf-8 with error handling
+            with open(file_path, 'r', encoding='utf-8', errors='replace') as f:
+                return f.read()
+
+    @staticmethod
+    def process_filename_to_title(filename: str) -> str:
+        """Convert filename to a readable title"""
+        # Remove extension
+        name = os.path.splitext(filename)[0]
+
+        # Look for date patterns
+        date_pattern = r'(\d{4}[-_]?\d{2}[-_]?\d{2})'
+        date_match = re.search(date_pattern, name)
+        date_str = ""
+        if date_match:
+            try:
+                date = datetime.strptime(date_match.group(1).replace('_', '-'), '%Y-%m-%d')
+                date_str = date.strftime("%b %d, %Y")
+                name = name.replace(date_match.group(1), '').strip('-_')
+            except ValueError:
+                pass
+
+        # Replace separators with spaces
+        name = re.sub(r'[-_]+', ' ', name)
+
+        # Remove redundant spaces
+        name = re.sub(r'\s+', ' ', name).strip()
+
+        # Capitalize words, excluding certain words
+        exclude_words = {'a', 'an', 'the', 'in', 'on', 'at', 'to', 'for', 'of', 'with'}
+        words = name.split()
+        capitalized = []
+        for i, word in enumerate(words):
+            if i == 0 or word not in exclude_words:
+                capitalized.append(word.capitalize())
+            else:
+                capitalized.append(word.lower())
+        name = ' '.join(capitalized)
+
+        # Add date if found
+        if date_str:
+            name = f"{name} - {date_str}"
+
+        return name
+
+
+class ZipValidator:
+    """Validates zip file contents and structure"""
+
+    MAX_ZIP_SIZE = 100 * 1024 * 1024  # 100MB
+    MAX_FILES = 100
+    VALID_EXTENSIONS = {'.md', '.txt'}
+
+    @staticmethod
+    def validate_zip_file(zip_path: str) -> Tuple[bool, str, List[str]]:
+        """
+        Validate zip file and its contents
+        Returns: (is_valid, error_message, valid_files)
+        """
+        try:
+            # Check zip file size
+            if os.path.getsize(zip_path) > ZipValidator.MAX_ZIP_SIZE:
+                return False, "Zip file too large (max 100MB)", []
+
+            valid_files = []
+            with zipfile.ZipFile(zip_path, 'r') as zip_ref:
+                # Check number of files
+                if len(zip_ref.filelist) > ZipValidator.MAX_FILES:
+                    return False, f"Too many files in zip (max {ZipValidator.MAX_FILES})", []
+
+                # Check for directory traversal attempts
+                for file_info in zip_ref.filelist:
+                    if '..' in file_info.filename or file_info.filename.startswith('/'):
+                        return False, "Invalid file paths detected", []
+
+                # Validate each file
+                total_size = 0
+                for file_info in zip_ref.filelist:
+                    # Skip directories
+                    if file_info.filename.endswith('/'):
+                        continue
+
+                    # Check file size
+                    if file_info.file_size > ZipValidator.MAX_ZIP_SIZE:
+                        return False, f"File {file_info.filename} too large", []
+
+                    total_size += file_info.file_size
+                    if total_size > ZipValidator.MAX_ZIP_SIZE:
+                        return False, "Total uncompressed size too large", []
+
+                    # Check file extension
+                    ext = os.path.splitext(file_info.filename)[1].lower()
+                    if ext in ZipValidator.VALID_EXTENSIONS:
+                        valid_files.append(file_info.filename)
+
+            if not valid_files:
+                return False, "No valid markdown or text files found in zip", []
+
+            return True, "", valid_files
+
+        except zipfile.BadZipFile:
+            return False, "Invalid or corrupted zip file", []
+        except Exception as e:
+            return False, f"Error processing zip file: {str(e)}", []
+
+
+def create_conversation_import_tab() -> gr.Tab:
+    """Create the import tab for the Gradio interface"""
+    with gr.Tab("Import RAG Chats") as tab:
+        gr.Markdown("# Import RAG Chats into the Database")
+        gr.Markdown("""
+        Import your RAG Chat markdown/text files individually or as a zip archive
+
+        Supported file types:
+        - Markdown (.md)
+        - Text (.txt)
+        - Zip archives containing .md or .txt files
+
+        Maximum zip file size: 100MB
+        Maximum files per zip: 100
+        """)
+        with gr.Row():
+            with gr.Column():
+                import_files = gr.File(
+                    label="Upload Files",
+                    file_types=["txt", "md", "zip"],
+                    file_count="multiple"
+                )
+
+                keywords_input = gr.Textbox(
+                    label="Keywords",
+                    placeholder="Enter keywords to apply to all imported files (comma-separated)"
+                )
+
+                custom_prompt_input = gr.Textbox(
+                    label="Custom Prompt",
+                    placeholder="Enter a custom prompt for processing (optional)"
+                )
+
+                rating_input = gr.Slider(
+                    minimum=1,
+                    maximum=3,
+                    step=1,
+                    label="Rating (1-3)",
+                    value=None
+                )
+
+            with gr.Column():
+                import_button = gr.Button("Import Files")
+                import_output = gr.Textbox(
+                    label="Import Status",
+                    lines=10
+                )
+
+        def handle_import(files, keywords, custom_prompt, rating):
+            importer = RAGQABatchImporter("rag_qa.db")  # Update with your DB path
+            success, message = importer.import_files(
+                files=[f.name for f in files],
+                keywords=keywords,
+                custom_prompt=custom_prompt,
+                rating=rating
+            )
+            return message
+
+        import_button.click(
+            fn=handle_import,
+            inputs=[
+                import_files,
+                keywords_input,
+                custom_prompt_input,
+                rating_input
+            ],
+            outputs=import_output
+        )
+
+    return tab

App_Function_Libraries/Gradio_UI/Keywords.py

@@ -4,22 +4,29 @@
 # The Keywords tab allows the user to add, delete, view, and export keywords from the database.
 #
 # Imports:
-
 #
 # External Imports
 import gradio as gr
+
+from App_Function_Libraries.DB.Character_Chat_DB import view_char_keywords, add_char_keywords, delete_char_keyword, \
+    export_char_keywords_to_csv
 #
 # Internal Imports
 from App_Function_Libraries.DB.DB_Manager import add_keyword, delete_keyword, keywords_browser_interface, export_keywords_to_csv
-#
+from App_Function_Libraries.DB.Prompts_DB import view_prompt_keywords, delete_prompt_keyword, \
+    export_prompt_keywords_to_csv
+from App_Function_Libraries.DB.RAG_QA_Chat_DB import view_rag_keywords, get_all_collections, \
+    get_keywords_for_collection, create_keyword_collection, add_keyword_to_collection, delete_rag_keyword, \
+    export_rag_keywords_to_csv
+
+
 #
 ######################################################################################################################
 #
 # Functions:
 
-
 def create_export_keywords_tab():
-    with gr.TabItem("Export Keywords", visible=True):
+    with gr.TabItem("Export MediaDB Keywords", visible=True):
         with gr.Row():
             with gr.Column():
                 export_keywords_button = gr.Button("Export Keywords")
@@ -33,8 +40,8 @@ def create_export_keywords_tab():
             )
 
 def create_view_keywords_tab():
-    with gr.TabItem("View Keywords", visible=True):
-        gr.Markdown("# Browse Keywords")
+    with gr.TabItem("View MediaDB Keywords", visible=True):
+        gr.Markdown("# Browse MediaDB Keywords")
         with gr.Column():
             browse_output = gr.Markdown()
             browse_button = gr.Button("View Existing Keywords")
@@ -42,7 +49,7 @@ def create_view_keywords_tab():
 
 
 def create_add_keyword_tab():
-    with gr.TabItem("Add Keywords", visible=True):
+    with gr.TabItem("Add MediaDB Keywords", visible=True):
         with gr.Row():
             with gr.Column():
                 gr.Markdown("# Add Keywords to the Database")
@@ -54,7 +61,7 @@ def create_add_keyword_tab():
 
 
 def create_delete_keyword_tab():
-    with gr.Tab("Delete Keywords", visible=True):
+    with gr.Tab("Delete MediaDB Keywords", visible=True):
         with gr.Row():
             with gr.Column():
                 gr.Markdown("# Delete Keywords from the Database")
@@ -63,3 +70,289 @@ def create_delete_keyword_tab():
             with gr.Row():
                 delete_output = gr.Textbox(label="Result")
                 delete_button.click(fn=delete_keyword, inputs=delete_input, outputs=delete_output)
+
+#
+# End of Media DB Keyword tabs
+##########################################################
+
+
+############################################################
+#
+# Character DB Keyword functions
+
+def create_character_keywords_tab():
+    """Creates the Character Keywords management tab"""
+    with gr.Tab("Character Keywords"):
+        gr.Markdown("# Character Keywords Management")
+
+        with gr.Tabs():
+            # View Character Keywords Tab
+            with gr.TabItem("View Keywords"):
+                with gr.Column():
+                    refresh_char_keywords = gr.Button("Refresh Character Keywords")
+                    char_keywords_output = gr.Markdown()
+                    view_char_keywords()
+                    refresh_char_keywords.click(
+                        fn=view_char_keywords,
+                        outputs=char_keywords_output
+                    )
+
+            # Add Character Keywords Tab
+            with gr.TabItem("Add Keywords"):
+                with gr.Column():
+                    char_name = gr.Textbox(label="Character Name")
+                    new_keywords = gr.Textbox(label="New Keywords (comma-separated)")
+                    add_char_keyword_btn = gr.Button("Add Keywords")
+                    add_char_result = gr.Markdown()
+
+                    add_char_keyword_btn.click(
+                        fn=add_char_keywords,
+                        inputs=[char_name, new_keywords],
+                        outputs=add_char_result
+                    )
+
+            # Delete Character Keywords Tab (New)
+            with gr.TabItem("Delete Keywords"):
+                with gr.Column():
+                    delete_char_name = gr.Textbox(label="Character Name")
+                    delete_char_keyword_input = gr.Textbox(label="Keyword to Delete")
+                    delete_char_keyword_btn = gr.Button("Delete Keyword")
+                    delete_char_result = gr.Markdown()
+
+                    delete_char_keyword_btn.click(
+                        fn=delete_char_keyword,
+                        inputs=[delete_char_name, delete_char_keyword_input],
+                        outputs=delete_char_result
+                    )
+
+            # Export Character Keywords Tab (New)
+            with gr.TabItem("Export Keywords"):
+                with gr.Column():
+                    export_char_keywords_btn = gr.Button("Export Character Keywords")
+                    export_char_file = gr.File(label="Download Exported Keywords")
+                    export_char_status = gr.Textbox(label="Export Status")
+
+                    export_char_keywords_btn.click(
+                        fn=export_char_keywords_to_csv,
+                        outputs=[export_char_status, export_char_file]
+                    )
+
+#
+# End of Character Keywords tab
+##########################################################
+
+############################################################
+#
+# RAG QA Keywords functions
+
+def create_rag_qa_keywords_tab():
+    """Creates the RAG QA Keywords management tab"""
+    with gr.Tab("RAG QA Keywords"):
+        gr.Markdown("# RAG QA Keywords Management")
+
+        with gr.Tabs():
+            # View RAG QA Keywords Tab
+            with gr.TabItem("View Keywords"):
+                with gr.Column():
+                    refresh_rag_keywords = gr.Button("Refresh RAG QA Keywords")
+                    rag_keywords_output = gr.Markdown()
+
+                    view_rag_keywords()
+
+                    refresh_rag_keywords.click(
+                        fn=view_rag_keywords,
+                        outputs=rag_keywords_output
+                    )
+
+            # Add RAG QA Keywords Tab
+            with gr.TabItem("Add Keywords"):
+                with gr.Column():
+                    new_rag_keywords = gr.Textbox(label="New Keywords (comma-separated)")
+                    add_rag_keyword_btn = gr.Button("Add Keywords")
+                    add_rag_result = gr.Markdown()
+
+                    add_rag_keyword_btn.click(
+                        fn=add_keyword,
+                        inputs=new_rag_keywords,
+                        outputs=add_rag_result
+                    )
+
+            # Delete RAG QA Keywords Tab (New)
+            with gr.TabItem("Delete Keywords"):
+                with gr.Column():
+                    delete_rag_keyword_input = gr.Textbox(label="Keyword to Delete")
+                    delete_rag_keyword_btn = gr.Button("Delete Keyword")
+                    delete_rag_result = gr.Markdown()
+
+                    delete_rag_keyword_btn.click(
+                        fn=delete_rag_keyword,
+                        inputs=delete_rag_keyword_input,
+                        outputs=delete_rag_result
+                    )
+
+            # Export RAG QA Keywords Tab (New)
+            with gr.TabItem("Export Keywords"):
+                with gr.Column():
+                    export_rag_keywords_btn = gr.Button("Export RAG QA Keywords")
+                    export_rag_file = gr.File(label="Download Exported Keywords")
+                    export_rag_status = gr.Textbox(label="Export Status")
+
+                    export_rag_keywords_btn.click(
+                        fn=export_rag_keywords_to_csv,
+                        outputs=[export_rag_status, export_rag_file]
+                    )
+
+#
+# End of RAG QA Keywords tab
+##########################################################
+
+
+############################################################
+#
+# Prompt Keywords functions
+
+def create_prompt_keywords_tab():
+    """Creates the Prompt Keywords management tab"""
+    with gr.Tab("Prompt Keywords"):
+        gr.Markdown("# Prompt Keywords Management")
+
+        with gr.Tabs():
+            # View Keywords Tab
+            with gr.TabItem("View Keywords"):
+                with gr.Column():
+                    refresh_prompt_keywords = gr.Button("Refresh Prompt Keywords")
+                    prompt_keywords_output = gr.Markdown()
+
+                    refresh_prompt_keywords.click(
+                        fn=view_prompt_keywords,
+                        outputs=prompt_keywords_output
+                    )
+
+            # Add Keywords Tab (using existing prompt management functions)
+            with gr.TabItem("Add Keywords"):
+                gr.Markdown("""
+                    To add keywords to prompts, please use the Prompt Management interface.
+                    Keywords can be added when creating or editing a prompt.
+                """)
+
+            # Delete Keywords Tab
+            with gr.TabItem("Delete Keywords"):
+                with gr.Column():
+                    delete_prompt_keyword_input = gr.Textbox(label="Keyword to Delete")
+                    delete_prompt_keyword_btn = gr.Button("Delete Keyword")
+                    delete_prompt_result = gr.Markdown()
+
+                    delete_prompt_keyword_btn.click(
+                        fn=delete_prompt_keyword,
+                        inputs=delete_prompt_keyword_input,
+                        outputs=delete_prompt_result
+                    )
+
+            # Export Keywords Tab
+            with gr.TabItem("Export Keywords"):
+                with gr.Column():
+                    export_prompt_keywords_btn = gr.Button("Export Prompt Keywords")
+                    export_prompt_status = gr.Textbox(label="Export Status", interactive=False)
+                    export_prompt_file = gr.File(label="Download Exported Keywords", interactive=False)
+
+                    def handle_export():
+                        status, file_path = export_prompt_keywords_to_csv()
+                        if file_path:
+                            return status, file_path
+                        return status, None
+
+                    export_prompt_keywords_btn.click(
+                        fn=handle_export,
+                        outputs=[export_prompt_status, export_prompt_file]
+                    )
+#
+# End of Prompt Keywords tab
+############################################################
+
+
+############################################################
+#
+# Meta-Keywords functions
+
+def create_meta_keywords_tab():
+    """Creates the Meta-Keywords management tab"""
+    with gr.Tab("Meta-Keywords"):
+        gr.Markdown("# Meta-Keywords Management")
+
+        with gr.Tabs():
+            # View Meta-Keywords Tab
+            with gr.TabItem("View Collections"):
+                with gr.Column():
+                    refresh_collections = gr.Button("Refresh Collections")
+                    collections_output = gr.Markdown()
+
+                    def view_collections():
+                        try:
+                            collections, _, _ = get_all_collections()
+                            if collections:
+                                result = "### Keyword Collections:\n"
+                                for collection in collections:
+                                    keywords = get_keywords_for_collection(collection)
+                                    result += f"\n**{collection}**:\n"
+                                    result += "\n".join([f"- {k}" for k in keywords])
+                                    result += "\n"
+                                return result
+                            return "No collections found."
+                        except Exception as e:
+                            return f"Error retrieving collections: {str(e)}"
+
+                    refresh_collections.click(
+                        fn=view_collections,
+                        outputs=collections_output
+                    )
+
+            # Create Collection Tab
+            with gr.TabItem("Create Collection"):
+                with gr.Column():
+                    collection_name = gr.Textbox(label="Collection Name")
+                    create_collection_btn = gr.Button("Create Collection")
+                    create_result = gr.Markdown()
+
+                    def create_collection(name: str):
+                        try:
+                            create_keyword_collection(name)
+                            return f"Successfully created collection: {name}"
+                        except Exception as e:
+                            return f"Error creating collection: {str(e)}"
+
+                    create_collection_btn.click(
+                        fn=create_collection,
+                        inputs=collection_name,
+                        outputs=create_result
+                    )
+
+            # Add Keywords to Collection Tab
+            with gr.TabItem("Add to Collection"):
+                with gr.Column():
+                    collection_select = gr.Textbox(label="Collection Name")
+                    keywords_to_add = gr.Textbox(label="Keywords to Add (comma-separated)")
+                    add_to_collection_btn = gr.Button("Add Keywords to Collection")
+                    add_to_collection_result = gr.Markdown()
+
+                    def add_keywords_to_collection(collection: str, keywords: str):
+                        try:
+                            keywords_list = [k.strip() for k in keywords.split(",") if k.strip()]
+                            for keyword in keywords_list:
+                                add_keyword_to_collection(collection, keyword)
+                            return f"Successfully added {len(keywords_list)} keywords to collection {collection}"
+                        except Exception as e:
+                            return f"Error adding keywords to collection: {str(e)}"
+
+                    add_to_collection_btn.click(
+                        fn=add_keywords_to_collection,
+                        inputs=[collection_select, keywords_to_add],
+                        outputs=add_to_collection_result
+                    )
+
+#
+# End of Meta-Keywords tab
+##########################################################
+
+#
+# End of Keywords.py
+######################################################################################################################

App_Function_Libraries/Gradio_UI/Live_Recording.py

@@ -13,6 +13,8 @@ from App_Function_Libraries.Audio.Audio_Transcription_Lib import (record_audio,
                                                                   stop_recording)
 from App_Function_Libraries.DB.DB_Manager import add_media_to_database
 from App_Function_Libraries.Metrics.metrics_logger import log_counter, log_histogram
+from App_Function_Libraries.Utils.Utils import default_api_endpoint, global_api_endpoints, format_api_name
+
 #
 #######################################################################################################################
 #
@@ -22,6 +24,16 @@ whisper_models = ["small", "medium", "small.en", "medium.en", "medium", "large",
                   "distil-large-v2", "distil-medium.en", "distil-small.en"]
 
 def create_live_recording_tab():
+    try:
+        default_value = None
+        if default_api_endpoint:
+            if default_api_endpoint in global_api_endpoints:
+                default_value = format_api_name(default_api_endpoint)
+            else:
+                logging.warning(f"Default API endpoint '{default_api_endpoint}' not found in global_api_endpoints")
+    except Exception as e:
+        logging.error(f"Error setting default API endpoint: {str(e)}")
+        default_value = None
     with gr.Tab("Live Recording and Transcription", visible=True):
         gr.Markdown("# Live Audio Recording and Transcription")
         with gr.Row():
@@ -34,6 +46,13 @@ def create_live_recording_tab():
                 custom_title = gr.Textbox(label="Custom Title (for database)", visible=False)
                 record_button = gr.Button("Start Recording")
                 stop_button = gr.Button("Stop Recording")
+                # FIXME - Add a button to perform analysis/summarization on the transcription
+                # Refactored API selection dropdown
+                # api_name_input = gr.Dropdown(
+                #     choices=["None"] + [format_api_name(api) for api in global_api_endpoints],
+                #     value=default_value,
+                #     label="API for Summarization (Optional)"
+                # )
             with gr.Column():
                 output = gr.Textbox(label="Transcription", lines=10)
                 audio_output = gr.Audio(label="Recorded Audio", visible=False)

App_Function_Libraries/Gradio_UI/Llamafile_tab.py

@@ -0,0 +1,327 @@
+# Llamafile_tab.py
+# Description: Gradio interface for configuring and launching Llamafile with Local LLMs
+
+# Imports
+import os
+import logging
+from typing import Tuple, Optional
+import gradio as gr
+
+
+from App_Function_Libraries.Local_LLM.Local_LLM_Inference_Engine_Lib import (
+    download_llm_model,
+    llm_models,
+    start_llamafile,
+    get_gguf_llamafile_files
+)
+#
+#######################################################################################################################
+#
+# Functions:
+
+BASE_DIR = os.path.dirname(os.path.abspath(__file__))
+MODELS_DIR = os.path.join(BASE_DIR, "Models")
+
+def create_chat_with_llamafile_tab():
+    # Function to update model path based on selection
+    def on_local_model_change(selected_model: str, search_directory: str) -> str:
+        if selected_model and isinstance(search_directory, str):
+            model_path = os.path.abspath(os.path.join(search_directory, selected_model))
+            logging.debug(f"Selected model path: {model_path}")  # Debug print for selected model path
+            return model_path
+        return "Invalid selection or directory."
+
+    # Function to update the dropdown with available models
+    def update_dropdowns(search_directory: str) -> Tuple[dict, str]:
+        logging.debug(f"User-entered directory: {search_directory}")  # Debug print for directory
+        if not os.path.isdir(search_directory):
+            logging.debug(f"Directory does not exist: {search_directory}")  # Debug print for non-existing directory
+            return gr.update(choices=[], value=None), "Directory does not exist."
+
+        try:
+            logging.debug(f"Directory exists: {search_directory}, scanning for files...")  # Confirm directory exists
+            model_files = get_gguf_llamafile_files(search_directory)
+            logging.debug("Completed scanning for model files.")
+        except Exception as e:
+            logging.error(f"Error scanning directory: {e}")
+            return gr.update(choices=[], value=None), f"Error scanning directory: {e}"
+
+        if not model_files:
+            logging.debug(f"No model files found in {search_directory}")  # Debug print for no files found
+            return gr.update(choices=[], value=None), "No model files found in the specified directory."
+
+        # Update the dropdown choices with the model files found
+        logging.debug(f"Models loaded from {search_directory}: {model_files}")  # Debug: Print model files loaded
+        return gr.update(choices=model_files, value=None), f"Models loaded from {search_directory}."
+
+
+
+    def download_preset_model(selected_model: str) -> Tuple[str, str]:
+        """
+        Downloads the selected preset model.
+
+        Args:
+            selected_model (str): The key of the selected preset model.
+
+        Returns:
+            Tuple[str, str]: Status message and the path to the downloaded model.
+        """
+        model_info = llm_models.get(selected_model)
+        if not model_info:
+            return "Invalid model selection.", ""
+
+        try:
+            model_path = download_llm_model(
+                model_name=model_info["name"],
+                model_url=model_info["url"],
+                model_filename=model_info["filename"],
+                model_hash=model_info["hash"]
+            )
+            return f"Model '{model_info['name']}' downloaded successfully.", model_path
+        except Exception as e:
+            logging.error(f"Error downloading model: {e}")
+            return f"Failed to download model: {e}", ""
+
+    with gr.TabItem("Local LLM with Llamafile", visible=True):
+        gr.Markdown("# Settings for Llamafile")
+
+        with gr.Row():
+            with gr.Column():
+                am_noob = gr.Checkbox(label="Enable Sane Defaults", value=False, visible=True)
+                advanced_mode_toggle = gr.Checkbox(label="Advanced Mode - Show All Settings", value=False)
+                # Advanced Inputs
+                verbose_checked = gr.Checkbox(label="Enable Verbose Output", value=False, visible=False)
+                threads_checked = gr.Checkbox(label="Set CPU Threads", value=False, visible=False)
+                threads_value = gr.Number(label="Number of CPU Threads", value=None, precision=0, visible=False)
+                threads_batched_checked = gr.Checkbox(label="Enable Batched Inference", value=False, visible=False)
+                threads_batched_value = gr.Number(label="Batch Size for Inference", value=None, precision=0, visible=False)
+                model_alias_checked = gr.Checkbox(label="Set Model Alias", value=False, visible=False)
+                model_alias_value = gr.Textbox(label="Model Alias", value="", visible=False)
+                ctx_size_checked = gr.Checkbox(label="Set Prompt Context Size", value=False, visible=False)
+                ctx_size_value = gr.Number(label="Prompt Context Size", value=8124, precision=0, visible=False)
+                ngl_checked = gr.Checkbox(label="Enable GPU Layers", value=False, visible=True)
+                ngl_value = gr.Number(label="Number of GPU Layers", value=None, precision=0, visible=True)
+                batch_size_checked = gr.Checkbox(label="Set Batch Size", value=False, visible=False)
+                batch_size_value = gr.Number(label="Batch Size", value=512, visible=False)
+                memory_f32_checked = gr.Checkbox(label="Use 32-bit Floating Point", value=False, visible=False)
+                numa_checked = gr.Checkbox(label="Enable NUMA", value=False, visible=False)
+                server_timeout_value = gr.Number(label="Server Timeout", value=600, precision=0, visible=False)
+                host_checked = gr.Checkbox(label="Set IP to Listen On", value=False, visible=False)
+                host_value = gr.Textbox(label="Host IP Address", value="", visible=False)
+                port_checked = gr.Checkbox(label="Set Server Port", value=False, visible=False)
+                port_value = gr.Number(label="Port Number", value=8080, precision=0, visible=False)
+                api_key_checked = gr.Checkbox(label="Set API Key", value=False, visible=False)
+                api_key_value = gr.Textbox(label="API Key", value="", visible=False)
+                http_threads_checked = gr.Checkbox(label="Set HTTP Server Threads", value=False, visible=False)
+                http_threads_value = gr.Number(label="Number of HTTP Server Threads", value=None, precision=0, visible=False)
+                hf_repo_checked = gr.Checkbox(label="Use Huggingface Repo Model", value=False, visible=False)
+                hf_repo_value = gr.Textbox(label="Huggingface Repo Name", value="", visible=False)
+                hf_file_checked = gr.Checkbox(label="Set Huggingface Model File", value=False, visible=False)
+                hf_file_value = gr.Textbox(label="Huggingface Model File", value="", visible=False)
+
+            with gr.Column():
+                # Model Selection Section
+                gr.Markdown("## Model Selection")
+
+                # Option 1: Select from Local Filesystem
+                with gr.Row():
+                    search_directory = gr.Textbox(
+                        label="Model Directory",
+                        placeholder="Enter directory path (currently './Models')",
+                        value=MODELS_DIR,
+                        interactive=True
+                    )
+
+                # Initial population of local models
+                initial_dropdown_update, _ = update_dropdowns(MODELS_DIR)
+                logging.debug(f"Scanning directory: {MODELS_DIR}")
+                refresh_button = gr.Button("Refresh Models")
+                local_model_dropdown = gr.Dropdown(
+                    label="Select Model from Directory",
+                    choices=initial_dropdown_update["choices"],
+                    value=None
+                )
+                # Display selected model path
+                model_value = gr.Textbox(label="Selected Model File Path", value="", interactive=False)
+
+                # Option 2: Download Preset Models
+                gr.Markdown("## Download Preset Models")
+
+                preset_model_dropdown = gr.Dropdown(
+                    label="Select a Preset Model",
+                    choices=list(llm_models.keys()),
+                    value=None,
+                    interactive=True,
+                    info="Choose a preset model to download."
+                )
+                download_preset_button = gr.Button("Download Selected Preset")
+
+        with gr.Row():
+            with gr.Column():
+                start_button = gr.Button("Start Llamafile")
+                stop_button = gr.Button("Stop Llamafile (doesn't work)")
+                output_display = gr.Markdown()
+
+
+        # Show/hide advanced inputs based on toggle
+        def update_visibility(show_advanced: bool):
+            components = [
+                verbose_checked, threads_checked, threads_value,
+                http_threads_checked, http_threads_value,
+                hf_repo_checked, hf_repo_value,
+                hf_file_checked, hf_file_value,
+                ctx_size_checked, ctx_size_value,
+                ngl_checked, ngl_value,
+                host_checked, host_value,
+                port_checked, port_value
+            ]
+            return [gr.update(visible=show_advanced) for _ in components]
+
+        def on_start_button_click(
+                am_noob: bool,
+                verbose_checked: bool,
+                threads_checked: bool,
+                threads_value: Optional[int],
+                threads_batched_checked: bool,
+                threads_batched_value: Optional[int],
+                model_alias_checked: bool,
+                model_alias_value: str,
+                http_threads_checked: bool,
+                http_threads_value: Optional[int],
+                model_value: str,
+                hf_repo_checked: bool,
+                hf_repo_value: str,
+                hf_file_checked: bool,
+                hf_file_value: str,
+                ctx_size_checked: bool,
+                ctx_size_value: Optional[int],
+                ngl_checked: bool,
+                ngl_value: Optional[int],
+                batch_size_checked: bool,
+                batch_size_value: Optional[int],
+                memory_f32_checked: bool,
+                numa_checked: bool,
+                server_timeout_value: Optional[int],
+                host_checked: bool,
+                host_value: str,
+                port_checked: bool,
+                port_value: Optional[int],
+                api_key_checked: bool,
+                api_key_value: str
+        ) -> str:
+            """
+            Event handler for the Start Llamafile button.
+            """
+            try:
+                result = start_llamafile(
+                    am_noob,
+                    verbose_checked,
+                    threads_checked,
+                    threads_value,
+                    threads_batched_checked,
+                    threads_batched_value,
+                    model_alias_checked,
+                    model_alias_value,
+                    http_threads_checked,
+                    http_threads_value,
+                    model_value,
+                    hf_repo_checked,
+                    hf_repo_value,
+                    hf_file_checked,
+                    hf_file_value,
+                    ctx_size_checked,
+                    ctx_size_value,
+                    ngl_checked,
+                    ngl_value,
+                    batch_size_checked,
+                    batch_size_value,
+                    memory_f32_checked,
+                    numa_checked,
+                    server_timeout_value,
+                    host_checked,
+                    host_value,
+                    port_checked,
+                    port_value,
+                    api_key_checked,
+                    api_key_value
+                )
+                return result
+            except Exception as e:
+                logging.error(f"Error starting Llamafile: {e}")
+                return f"Failed to start Llamafile: {e}"
+
+        advanced_mode_toggle.change(
+            fn=update_visibility,
+            inputs=[advanced_mode_toggle],
+            outputs=[
+                verbose_checked, threads_checked, threads_value,
+                http_threads_checked, http_threads_value,
+                hf_repo_checked, hf_repo_value,
+                hf_file_checked, hf_file_value,
+                ctx_size_checked, ctx_size_value,
+                ngl_checked, ngl_value,
+                host_checked, host_value,
+                port_checked, port_value
+            ]
+        )
+
+        start_button.click(
+            fn=on_start_button_click,
+            inputs=[
+                am_noob,
+                verbose_checked,
+                threads_checked,
+                threads_value,
+                threads_batched_checked,
+                threads_batched_value,
+                model_alias_checked,
+                model_alias_value,
+                http_threads_checked,
+                http_threads_value,
+                model_value,
+                hf_repo_checked,
+                hf_repo_value,
+                hf_file_checked,
+                hf_file_value,
+                ctx_size_checked,
+                ctx_size_value,
+                ngl_checked,
+                ngl_value,
+                batch_size_checked,
+                batch_size_value,
+                memory_f32_checked,
+                numa_checked,
+                server_timeout_value,
+                host_checked,
+                host_value,
+                port_checked,
+                port_value,
+                api_key_checked,
+                api_key_value
+            ],
+            outputs=output_display
+        )
+
+        download_preset_button.click(
+            fn=download_preset_model,
+            inputs=[preset_model_dropdown],
+            outputs=[output_display, model_value]
+        )
+
+        # Click event for refreshing models
+        refresh_button.click(
+            fn=update_dropdowns,
+            inputs=[search_directory],  # Ensure that the directory path (string) is passed
+            outputs=[local_model_dropdown, output_display]  # Update dropdown and status
+        )
+
+        # Event to update model_value when a model is selected from the dropdown
+        local_model_dropdown.change(
+            fn=on_local_model_change,  # Function that calculates the model path
+            inputs=[local_model_dropdown, search_directory],  # Inputs: selected model and directory
+            outputs=[model_value]  # Output: Update the model_value textbox with the selected model path
+        )
+
+#
+#
+#######################################################################################################################

App_Function_Libraries/Gradio_UI/Media_edit.py

@@ -10,13 +10,13 @@ import gradio as gr
 #
 # Local Imports
 from App_Function_Libraries.DB.DB_Manager import add_prompt, update_media_content, db, add_or_update_prompt, \
-    load_prompt_details, fetch_keywords_for_media, update_keywords_for_media
-from App_Function_Libraries.Gradio_UI.Gradio_Shared import update_dropdown, update_prompt_dropdown
+    load_prompt_details, fetch_keywords_for_media, update_keywords_for_media, fetch_prompt_details, list_prompts
+from App_Function_Libraries.Gradio_UI.Gradio_Shared import update_dropdown
 from App_Function_Libraries.DB.SQLite_DB import fetch_item_details
 
 
 def create_media_edit_tab():
-    with gr.TabItem("Edit Existing Items", visible=True):
+    with gr.TabItem("Edit Existing Items in the Media DB", visible=True):
         gr.Markdown("# Search and Edit Media Items")
 
         with gr.Row():
@@ -89,7 +89,7 @@ def create_media_edit_tab():
 
 
 def create_media_edit_and_clone_tab():
-    with gr.TabItem("Clone and Edit Existing Items", visible=True):
+    with gr.TabItem("Clone and Edit Existing Items in the Media DB", visible=True):
         gr.Markdown("# Search, Edit, and Clone Existing Items")
 
         with gr.Row():
@@ -199,6 +199,11 @@ def create_media_edit_and_clone_tab():
 
 
 def create_prompt_edit_tab():
+    # Initialize state variables for pagination
+    current_page_state = gr.State(value=1)
+    total_pages_state = gr.State(value=1)
+    per_page = 10  # Number of prompts per page
+
     with gr.TabItem("Add & Edit Prompts", visible=True):
         with gr.Row():
             with gr.Column():
@@ -207,38 +212,145 @@ def create_prompt_edit_tab():
                     choices=[],
                     interactive=True
                 )
+                next_page_button = gr.Button("Next Page", visible=False)
+                page_display = gr.Markdown("Page 1 of X", visible=False)
+                prev_page_button = gr.Button("Previous Page", visible=False)
                 prompt_list_button = gr.Button("List Prompts")
 
             with gr.Column():
                 title_input = gr.Textbox(label="Title", placeholder="Enter the prompt title")
-                author_input = gr.Textbox(label="Author", placeholder="Enter the prompt's author", lines=3)
+                author_input = gr.Textbox(label="Author", placeholder="Enter the prompt's author", lines=1)
                 description_input = gr.Textbox(label="Description", placeholder="Enter the prompt description", lines=3)
                 system_prompt_input = gr.Textbox(label="System Prompt", placeholder="Enter the system prompt", lines=3)
                 user_prompt_input = gr.Textbox(label="User Prompt", placeholder="Enter the user prompt", lines=3)
                 add_prompt_button = gr.Button("Add/Update Prompt")
                 add_prompt_output = gr.HTML()
 
-        # Event handlers
+        # Function to update the prompt dropdown with pagination
+        def update_prompt_dropdown(page=1):
+            prompts, total_pages, current_page = list_prompts(page=page, per_page=per_page)
+            page_display_text = f"Page {current_page} of {total_pages}"
+            prev_button_visible = current_page > 1
+            next_button_visible = current_page < total_pages
+            return (
+                gr.update(choices=prompts),
+                gr.update(value=page_display_text, visible=True),
+                gr.update(visible=prev_button_visible),
+                gr.update(visible=next_button_visible),
+                current_page,
+                total_pages
+            )
+
+        # Event handler for listing prompts
         prompt_list_button.click(
             fn=update_prompt_dropdown,
-            outputs=prompt_dropdown
+            inputs=[],
+            outputs=[
+                prompt_dropdown,
+                page_display,
+                prev_page_button,
+                next_page_button,
+                current_page_state,
+                total_pages_state
+            ]
+        )
+
+        # Functions to handle pagination
+        def on_prev_page_click(current_page):
+            new_page = max(current_page - 1, 1)
+            return update_prompt_dropdown(page=new_page)
+
+        def on_next_page_click(current_page, total_pages):
+            new_page = min(current_page + 1, total_pages)
+            return update_prompt_dropdown(page=new_page)
+
+        # Event handlers for pagination buttons
+        prev_page_button.click(
+            fn=on_prev_page_click,
+            inputs=[current_page_state],
+            outputs=[
+                prompt_dropdown,
+                page_display,
+                prev_page_button,
+                next_page_button,
+                current_page_state,
+                total_pages_state
+            ]
+        )
+
+        next_page_button.click(
+            fn=on_next_page_click,
+            inputs=[current_page_state, total_pages_state],
+            outputs=[
+                prompt_dropdown,
+                page_display,
+                prev_page_button,
+                next_page_button,
+                current_page_state,
+                total_pages_state
+            ]
         )
 
+        # Event handler for adding or updating a prompt
         add_prompt_button.click(
             fn=add_or_update_prompt,
             inputs=[title_input, author_input, description_input, system_prompt_input, user_prompt_input],
-            outputs=add_prompt_output
+            outputs=[add_prompt_output]
+        ).then(
+            fn=update_prompt_dropdown,
+            inputs=[],
+            outputs=[
+                prompt_dropdown,
+                page_display,
+                prev_page_button,
+                next_page_button,
+                current_page_state,
+                total_pages_state
+            ]
         )
 
-        # Load prompt details when selected
+        # Function to load prompt details when a prompt is selected
+        def load_prompt_details(selected_prompt):
+            details = fetch_prompt_details(selected_prompt)
+            if details:
+                title, author, description, system_prompt, user_prompt, keywords = details
+                return (
+                    gr.update(value=title),
+                    gr.update(value=author or ""),
+                    gr.update(value=description or ""),
+                    gr.update(value=system_prompt or ""),
+                    gr.update(value=user_prompt or "")
+                )
+            else:
+                return (
+                    gr.update(value=""),
+                    gr.update(value=""),
+                    gr.update(value=""),
+                    gr.update(value=""),
+                    gr.update(value="")
+                )
+
+        # Event handler for prompt selection change
         prompt_dropdown.change(
             fn=load_prompt_details,
             inputs=[prompt_dropdown],
-            outputs=[title_input, author_input, system_prompt_input, user_prompt_input]
+            outputs=[
+                title_input,
+                author_input,
+                description_input,
+                system_prompt_input,
+                user_prompt_input
+            ]
         )
 
 
+
 def create_prompt_clone_tab():
+    # Initialize state variables for pagination
+    current_page_state = gr.State(value=1)
+    total_pages_state = gr.State(value=1)
+    per_page = 10  # Number of prompts per page
+
     with gr.TabItem("Clone and Edit Prompts", visible=True):
         with gr.Row():
             with gr.Column():
@@ -248,6 +360,9 @@ def create_prompt_clone_tab():
                     choices=[],
                     interactive=True
                 )
+                next_page_button = gr.Button("Next Page", visible=False)
+                page_display = gr.Markdown("Page 1 of X", visible=False)
+                prev_page_button = gr.Button("Previous Page", visible=False)
                 prompt_list_button = gr.Button("List Prompts")
 
             with gr.Column():
@@ -260,19 +375,99 @@ def create_prompt_clone_tab():
                 save_cloned_prompt_button = gr.Button("Save Cloned Prompt", visible=False)
                 add_prompt_output = gr.HTML()
 
-        # Event handlers
+        # Function to update the prompt dropdown with pagination
+        def update_prompt_dropdown(page=1):
+            prompts, total_pages, current_page = list_prompts(page=page, per_page=per_page)
+            page_display_text = f"Page {current_page} of {total_pages}"
+            prev_button_visible = current_page > 1
+            next_button_visible = current_page < total_pages
+            return (
+                gr.update(choices=prompts),
+                gr.update(value=page_display_text, visible=True),
+                gr.update(visible=prev_button_visible),
+                gr.update(visible=next_button_visible),
+                current_page,
+                total_pages
+            )
+
+        # Event handler for listing prompts
         prompt_list_button.click(
             fn=update_prompt_dropdown,
-            outputs=prompt_dropdown
+            inputs=[],
+            outputs=[
+                prompt_dropdown,
+                page_display,
+                prev_page_button,
+                next_page_button,
+                current_page_state,
+                total_pages_state
+            ]
+        )
+
+        # Functions to handle pagination
+        def on_prev_page_click(current_page):
+            new_page = max(current_page - 1, 1)
+            return update_prompt_dropdown(page=new_page)
+
+        def on_next_page_click(current_page, total_pages):
+            new_page = min(current_page + 1, total_pages)
+            return update_prompt_dropdown(page=new_page)
+
+        # Event handlers for pagination buttons
+        prev_page_button.click(
+            fn=on_prev_page_click,
+            inputs=[current_page_state],
+            outputs=[
+                prompt_dropdown,
+                page_display,
+                prev_page_button,
+                next_page_button,
+                current_page_state,
+                total_pages_state
+            ]
+        )
+
+        next_page_button.click(
+            fn=on_next_page_click,
+            inputs=[current_page_state, total_pages_state],
+            outputs=[
+                prompt_dropdown,
+                page_display,
+                prev_page_button,
+                next_page_button,
+                current_page_state,
+                total_pages_state
+            ]
         )
 
         # Load prompt details when selected
+        def load_prompt_details(selected_prompt):
+            if selected_prompt:
+                details = fetch_prompt_details(selected_prompt)
+                if details:
+                    title, author, description, system_prompt, user_prompt, keywords = details
+                    return (
+                        gr.update(value=title),
+                        gr.update(value=author or ""),
+                        gr.update(value=description or ""),
+                        gr.update(value=system_prompt or ""),
+                        gr.update(value=user_prompt or "")
+                    )
+            return (
+                gr.update(value=""),
+                gr.update(value=""),
+                gr.update(value=""),
+                gr.update(value=""),
+                gr.update(value="")
+            )
+
         prompt_dropdown.change(
             fn=load_prompt_details,
             inputs=[prompt_dropdown],
             outputs=[title_input, author_input, description_input, system_prompt_input, user_prompt_input]
         )
 
+        # Prepare for cloning
         def prepare_for_cloning(selected_prompt):
             if selected_prompt:
                 return gr.update(value=f"Copy of {selected_prompt}"), gr.update(visible=True)
@@ -284,18 +479,21 @@ def create_prompt_clone_tab():
             outputs=[title_input, save_cloned_prompt_button]
         )
 
-        def save_cloned_prompt(title, description, system_prompt, user_prompt):
+        # Function to save cloned prompt
+        def save_cloned_prompt(title, author, description, system_prompt, user_prompt, current_page):
             try:
-                result = add_prompt(title, description, system_prompt, user_prompt)
+                result = add_prompt(title, author, description, system_prompt, user_prompt)
                 if result == "Prompt added successfully.":
-                    return result, gr.update(choices=update_prompt_dropdown())
+                    # After adding, refresh the prompt dropdown
+                    prompt_dropdown_update = update_prompt_dropdown(page=current_page)
+                    return (result, *prompt_dropdown_update)
                 else:
-                    return result, gr.update()
+                    return (result, gr.update(), gr.update(), gr.update(), gr.update(), current_page, total_pages_state.value)
             except Exception as e:
-                return f"Error saving cloned prompt: {str(e)}", gr.update()
+                return (f"Error saving cloned prompt: {str(e)}", gr.update(), gr.update(), gr.update(), gr.update(), current_page, total_pages_state.value)
 
         save_cloned_prompt_button.click(
             fn=save_cloned_prompt,
-            inputs=[title_input, description_input, system_prompt_input, user_prompt_input],
-            outputs=[add_prompt_output, prompt_dropdown]
-        )
+            inputs=[title_input, author_input, description_input, system_prompt_input, user_prompt_input, current_page_state],
+            outputs=[add_prompt_output, prompt_dropdown, page_display, prev_page_button, next_page_button, current_page_state, total_pages_state]
+        )

App_Function_Libraries/Gradio_UI/Media_wiki_tab.py

@@ -32,6 +32,13 @@ def create_mediawiki_import_tab():
                     value="sentences",
                     label="Chunking Method"
                 )
+                # FIXME - add API selection dropdown + Analysis/Summarization options
+                # Refactored API selection dropdown
+                # api_name_input = gr.Dropdown(
+                #     choices=["None"] + [format_api_name(api) for api in global_api_endpoints],
+                #     value=default_value,
+                #     label="API for Summarization (Optional)"
+                # )
                 chunk_size = gr.Slider(minimum=100, maximum=2000, value=1000, step=100, label="Chunk Size")
                 chunk_overlap = gr.Slider(minimum=0, maximum=500, value=100, step=10, label="Chunk Overlap")
                 # FIXME - Add checkbox for 'Enable Summarization upon ingestion' for API summarization of chunks

App_Function_Libraries/Gradio_UI/Mind_Map_tab.py

@@ -0,0 +1,128 @@
+# Mind_Map_tab.py
+# Description: File contains functions for generation of PlantUML mindmaps for the gradio tab
+#
+# Imports
+import re
+#
+# External Libraries
+import gradio as gr
+#
+######################################################################################################################
+#
+# Functions:
+
+def parse_plantuml_mindmap(plantuml_text: str) -> dict:
+    """Parse PlantUML mindmap syntax into a nested dictionary structure"""
+    lines = [line.strip() for line in plantuml_text.split('\n')
+             if line.strip() and not line.strip().startswith('@')]
+
+    root = None
+    nodes = []
+    stack = []
+
+    for line in lines:
+        level_match = re.match(r'^([+\-*]+|\*+)', line)
+        if not level_match:
+            continue
+        level = len(level_match.group(0))
+        text = re.sub(r'^([+\-*]+|\*+)\s*', '', line).strip('[]').strip('()')
+        node = {'text': text, 'children': []}
+
+        while stack and stack[-1][0] >= level:
+            stack.pop()
+
+        if stack:
+            stack[-1][1]['children'].append(node)
+        else:
+            root = node
+
+        stack.append((level, node))
+
+    return root
+
+def create_mindmap_html(plantuml_text: str) -> str:
+    """Convert PlantUML mindmap to HTML visualization with collapsible nodes using CSS only"""
+    # Parse the mindmap text into a nested structure
+    root_node = parse_plantuml_mindmap(plantuml_text)
+    if not root_node:
+        return "<p>No valid mindmap content provided.</p>"
+
+    html = "<style>"
+    html += """
+    details {
+        margin-left: 20px;
+    }
+    summary {
+        cursor: pointer;
+        padding: 5px;
+        border: 1px solid #333;
+        border-radius: 3px;
+        background-color: #e6f3ff;
+    }
+    .mindmap-node {
+        margin-left: 20px;
+        padding: 5px;
+        border: 1px solid #333;
+        border-radius: 3px;
+    }
+    """
+    html += "</style>"
+
+    colors = ['#e6f3ff', '#f0f7ff', '#f5f5f5', '#fff0f0', '#f0fff0']
+
+    def create_node_html(node, level):
+        bg_color = colors[(level - 1) % len(colors)]
+        if node['children']:
+            children_html = ''.join(create_node_html(child, level + 1) for child in node['children'])
+            return f"""
+            <details open>
+                <summary style="background-color: {bg_color};">{node['text']}</summary>
+                {children_html}
+            </details>
+            """
+        else:
+            return f"""
+            <div class="mindmap-node" style="background-color: {bg_color}; margin-left: {level * 20}px;">
+                {node['text']}
+            </div>
+            """
+
+    html += create_node_html(root_node, level=1)
+    return html
+
+# Create Gradio interface
+def create_mindmap_tab():
+    with gr.TabItem("PlantUML Mindmap"):
+        gr.Markdown("# Collapsible PlantUML Mindmap Visualizer")
+        gr.Markdown("Convert PlantUML mindmap syntax to a visual mindmap with collapsible nodes.")
+        plantuml_input = gr.Textbox(
+            lines=15,
+            label="Enter PlantUML mindmap",
+            placeholder="""@startmindmap
+    * Project Planning
+    ** Requirements
+    *** Functional Requirements
+    **** User Interface
+    **** Backend Services
+    *** Technical Requirements
+    **** Performance
+    **** Security
+    ** Timeline
+    *** Phase 1
+    *** Phase 2
+    ** Resources
+    *** Team
+    *** Budget
+    @endmindmap"""
+        )
+        submit_btn = gr.Button("Generate Mindmap")
+        mindmap_output = gr.HTML(label="Mindmap Output")
+        submit_btn.click(
+            fn=create_mindmap_html,
+            inputs=plantuml_input,
+            outputs=mindmap_output
+        )
+
+#
+# End of Mind_Map_tab.py
+######################################################################################################################

App_Function_Libraries/Gradio_UI/PDF_ingestion_tab.py

@@ -8,9 +8,12 @@ import tempfile
 #
 # External Imports
 import gradio as gr
+import pymupdf4llm
+from docling.document_converter import DocumentConverter
+
 #
 # Local Imports
-from App_Function_Libraries.DB.DB_Manager import load_preset_prompts
+from App_Function_Libraries.DB.DB_Manager import list_prompts
 from App_Function_Libraries.Gradio_UI.Chat_ui import update_user_prompt
 from App_Function_Libraries.PDF.PDF_Ingestion_Lib import extract_metadata_from_pdf, extract_text_and_format_from_pdf, \
     process_and_cleanup_pdf
@@ -22,92 +25,258 @@ from App_Function_Libraries.PDF.PDF_Ingestion_Lib import extract_metadata_from_p
 
 def create_pdf_ingestion_tab():
     with gr.TabItem("PDF Ingestion", visible=True):
-        # TODO - Add functionality to extract metadata from pdf as part of conversion process in marker
         gr.Markdown("# Ingest PDF Files and Extract Metadata")
         with gr.Row():
             with gr.Column():
-                pdf_file_input = gr.File(label="Uploaded PDF File", file_types=[".pdf"], visible=True)
-                pdf_upload_button = gr.UploadButton("Click to Upload PDF", file_types=[".pdf"])
-                pdf_title_input = gr.Textbox(label="Title (Optional)")
-                pdf_author_input = gr.Textbox(label="Author (Optional)")
-                pdf_keywords_input = gr.Textbox(label="Keywords (Optional, comma-separated)")
-                with gr.Row():
-                    custom_prompt_checkbox = gr.Checkbox(label="Use a Custom Prompt",
-                                                     value=False,
-                                                     visible=True)
-                    preset_prompt_checkbox = gr.Checkbox(label="Use a pre-set Prompt",
-                                                     value=False,
-                                                     visible=True)
-                with gr.Row():
-                    preset_prompt = gr.Dropdown(label="Select Preset Prompt",
-                                                choices=load_preset_prompts(),
-                                                visible=False)
-                with gr.Row():
-                    custom_prompt_input = gr.Textbox(label="Custom Prompt",
-                                                     placeholder="Enter custom prompt here",
-                                                     lines=3,
-                                                     visible=False)
-                with gr.Row():
-                    system_prompt_input = gr.Textbox(label="System Prompt",
-                                                     value="""
-<s>You are a bulleted notes specialist.
-[INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
-**Bulleted Note Creation Guidelines**
-
-**Headings**:
-- Based on referenced topics, not categories like quotes or terms
-- Surrounded by **bold** formatting 
-- Not listed as bullet points
-- No space between headings and list items underneath
-
-**Emphasis**:
-- **Important terms** set in bold font
-- **Text ending in a colon**: also bolded
-
-**Review**:
-- Ensure adherence to specified format
-- Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]""",
-                                                     lines=3,
-                                                     visible=False)
-
-                custom_prompt_checkbox.change(
-                    fn=lambda x: (gr.update(visible=x), gr.update(visible=x)),
-                    inputs=[custom_prompt_checkbox],
-                    outputs=[custom_prompt_input, system_prompt_input]
+                # Changed to support multiple files
+                pdf_file_input = gr.File(
+                    label="Uploaded PDF Files",
+                    file_types=[".pdf"],
+                    visible=True,
+                    file_count="multiple"
                 )
-                preset_prompt_checkbox.change(
-                    fn=lambda x: gr.update(visible=x),
-                    inputs=[preset_prompt_checkbox],
-                    outputs=[preset_prompt]
+                pdf_upload_button = gr.UploadButton(
+                    "Click to Upload PDFs",
+                    file_types=[".pdf"],
+                    file_count="multiple"
                 )
-
-                def update_prompts(preset_name):
-                    prompts = update_user_prompt(preset_name)
-                    return (
-                        gr.update(value=prompts["user_prompt"], visible=True),
-                        gr.update(value=prompts["system_prompt"], visible=True)
-                    )
-
-                preset_prompt.change(
-                    update_prompts,
-                    inputs=preset_prompt,
-                    outputs=[custom_prompt_input, system_prompt_input]
+                parser_selection = gr.Radio(
+                    choices=["pymupdf", "pymupdf4llm", "docling"],
+                    label="Select Parser",
+                    value="pymupdf"  # default value
                 )
+                # Common metadata for all files
+                pdf_keywords_input = gr.Textbox(label="Keywords (Optional, comma-separated)")
+#                 with gr.Row():
+#                     custom_prompt_checkbox = gr.Checkbox(
+#                         label="Use a Custom Prompt",
+#                         value=False,
+#                         visible=True
+#                     )
+#                     preset_prompt_checkbox = gr.Checkbox(
+#                         label="Use a pre-set Prompt",
+#                         value=False,
+#                         visible=True
+#                     )
+#                 # Initialize state variables for pagination
+#                 current_page_state = gr.State(value=1)
+#                 total_pages_state = gr.State(value=1)
+#                 with gr.Row():
+#                     # Add pagination controls
+#                     preset_prompt = gr.Dropdown(
+#                         label="Select Preset Prompt",
+#                         choices=[],
+#                         visible=False
+#                     )
+#                     prev_page_button = gr.Button("Previous Page", visible=False)
+#                     page_display = gr.Markdown("Page 1 of X", visible=False)
+#                     next_page_button = gr.Button("Next Page", visible=False)
+#                 with gr.Row():
+#                     custom_prompt_input = gr.Textbox(
+#                         label="Custom Prompt",
+#                         placeholder="Enter custom prompt here",
+#                         lines=3,
+#                         visible=False
+#                     )
+#                 with gr.Row():
+#                     system_prompt_input = gr.Textbox(
+#                         label="System Prompt",
+#                         value="""
+# <s>You are a bulleted notes specialist.
+# [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
+# **Bulleted Note Creation Guidelines**
+#
+# **Headings**:
+# - Based on referenced topics, not categories like quotes or terms
+# - Surrounded by **bold** formatting
+# - Not listed as bullet points
+# - No space between headings and list items underneath
+#
+# **Emphasis**:
+# - **Important terms** set in bold font
+# - **Text ending in a colon**: also bolded
+#
+# **Review**:
+# - Ensure adherence to specified format
+# - Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]""",
+#                         lines=3,
+#                         visible=False
+#                     )
+#
+#                 custom_prompt_checkbox.change(
+#                     fn=lambda x: (gr.update(visible=x), gr.update(visible=x)),
+#                     inputs=[custom_prompt_checkbox],
+#                     outputs=[custom_prompt_input, system_prompt_input]
+#                 )
+#
+#                 def on_preset_prompt_checkbox_change(is_checked):
+#                     if is_checked:
+#                         prompts, total_pages, current_page = list_prompts(page=1, per_page=10)
+#                         page_display_text = f"Page {current_page} of {total_pages}"
+#                         return (
+#                             gr.update(visible=True, interactive=True, choices=prompts),  # preset_prompt
+#                             gr.update(visible=True),  # prev_page_button
+#                             gr.update(visible=True),  # next_page_button
+#                             gr.update(value=page_display_text, visible=True),  # page_display
+#                             current_page,  # current_page_state
+#                             total_pages  # total_pages_state
+#                         )
+#                     else:
+#                         return (
+#                             gr.update(visible=False, interactive=False),  # preset_prompt
+#                             gr.update(visible=False),  # prev_page_button
+#                             gr.update(visible=False),  # next_page_button
+#                             gr.update(visible=False),  # page_display
+#                             1,  # current_page_state
+#                             1   # total_pages_state
+#                         )
+#
+#                 preset_prompt_checkbox.change(
+#                     fn=on_preset_prompt_checkbox_change,
+#                     inputs=[preset_prompt_checkbox],
+#                     outputs=[preset_prompt, prev_page_button, next_page_button, page_display, current_page_state, total_pages_state]
+#                 )
+#
+#                 def on_prev_page_click(current_page, total_pages):
+#                     new_page = max(current_page - 1, 1)
+#                     prompts, total_pages, current_page = list_prompts(page=new_page, per_page=10)
+#                     page_display_text = f"Page {current_page} of {total_pages}"
+#                     return gr.update(choices=prompts), gr.update(value=page_display_text), current_page
+#
+#                 prev_page_button.click(
+#                     fn=on_prev_page_click,
+#                     inputs=[current_page_state, total_pages_state],
+#                     outputs=[preset_prompt, page_display, current_page_state]
+#                 )
+#
+#                 def on_next_page_click(current_page, total_pages):
+#                     new_page = min(current_page + 1, total_pages)
+#                     prompts, total_pages, current_page = list_prompts(page=new_page, per_page=10)
+#                     page_display_text = f"Page {current_page} of {total_pages}"
+#                     return gr.update(choices=prompts), gr.update(value=page_display_text), current_page
+#
+#                 next_page_button.click(
+#                     fn=on_next_page_click,
+#                     inputs=[current_page_state, total_pages_state],
+#                     outputs=[preset_prompt, page_display, current_page_state]
+#                 )
+#
+#                 def update_prompts(preset_name):
+#                     prompts = update_user_prompt(preset_name)
+#                     return (
+#                         gr.update(value=prompts["user_prompt"], visible=True),
+#                         gr.update(value=prompts["system_prompt"], visible=True)
+#                     )
+#
+#                 preset_prompt.change(
+#                     update_prompts,
+#                     inputs=preset_prompt,
+#                     outputs=[custom_prompt_input, system_prompt_input]
+#                 )
 
-                pdf_ingest_button = gr.Button("Ingest PDF")
+                pdf_ingest_button = gr.Button("Ingest PDFs")
+
+                # Update the upload button handler for multiple files
+                pdf_upload_button.upload(
+                    fn=lambda files: files,
+                    inputs=pdf_upload_button,
+                    outputs=pdf_file_input
+                )
 
-                pdf_upload_button.upload(fn=lambda file: file, inputs=pdf_upload_button, outputs=pdf_file_input)
             with gr.Column():
-                pdf_result_output = gr.Textbox(label="Result")
+                pdf_result_output = gr.DataFrame(
+                    headers=["Filename", "Status", "Message"],
+                    label="Processing Results"
+                )
+
+            # Define a new function to handle multiple PDFs
+            def process_multiple_pdfs(pdf_files, keywords, custom_prompt_checkbox_value, custom_prompt_text, system_prompt_text):
+                results = []
+                if pdf_files is None:
+                    return [["No files", "Error", "No files uploaded"]]
+
+                for pdf_file in pdf_files:
+                    try:
+                        # Extract metadata from PDF
+                        metadata = extract_metadata_from_pdf(pdf_file.name)
 
+                        # Use custom or system prompt if checkbox is checked
+                        if custom_prompt_checkbox_value:
+                            prompt = custom_prompt_text
+                            system_prompt = system_prompt_text
+                        else:
+                            prompt = None
+                            system_prompt = None
+
+                        # Process the PDF with prompts
+                        result = process_and_cleanup_pdf(
+                            pdf_file,
+                            metadata.get('title', os.path.splitext(os.path.basename(pdf_file.name))[0]),
+                            metadata.get('author', 'Unknown'),
+                            keywords,
+                            #prompt=prompt,
+                            #system_prompt=system_prompt
+                        )
+
+                        results.append([
+                            pdf_file.name,
+                            "Success" if "successfully" in result else "Error",
+                            result
+                        ])
+                    except Exception as e:
+                        results.append([
+                            pdf_file.name,
+                            "Error",
+                            str(e)
+                        ])
+
+                return results
+
+            # Update the ingest button click handler
             pdf_ingest_button.click(
-                fn=process_and_cleanup_pdf,
-                inputs=[pdf_file_input, pdf_title_input, pdf_author_input, pdf_keywords_input],
+                fn=process_multiple_pdfs,
+                inputs=[
+                    pdf_file_input,
+                    pdf_keywords_input,
+                    parser_selection,
+                    #custom_prompt_checkbox,
+                    #custom_prompt_input,
+                    #system_prompt_input
+                ],
                 outputs=pdf_result_output
             )
 
 
-def test_pdf_ingestion(pdf_file):
+def test_pymupdf4llm_pdf_ingestion(pdf_file):
+    if pdf_file is None:
+        return "No file uploaded", ""
+
+    try:
+        # Create a temporary directory
+        with tempfile.TemporaryDirectory() as temp_dir:
+            # Create a path for the temporary PDF file
+            temp_path = os.path.join(temp_dir, "temp.pdf")
+
+            # Copy the contents of the uploaded file to the temporary file
+            shutil.copy(pdf_file.name, temp_path)
+
+            # Extract text and convert to Markdown
+            markdown_text = pymupdf4llm.to_markdown(temp_path)
+
+            # Extract metadata from PDF
+            metadata = extract_metadata_from_pdf(temp_path)
+
+            # Use metadata for title and author if not provided
+            title = metadata.get('title', os.path.splitext(os.path.basename(pdf_file.name))[0])
+            author = metadata.get('author', 'Unknown')
+
+        result = f"PDF '{title}' by {author} processed successfully by pymupdf4llm."
+        return result, markdown_text
+    except Exception as e:
+        return f"Error ingesting PDF: {str(e)}", ""
+
+
+def test_pymupdf_pdf_ingestion(pdf_file):
     if pdf_file is None:
         return "No file uploaded", ""
 
@@ -130,7 +299,37 @@ def test_pdf_ingestion(pdf_file):
             title = metadata.get('title', os.path.splitext(os.path.basename(pdf_file.name))[0])
             author = metadata.get('author', 'Unknown')
 
-        result = f"PDF '{title}' by {author} processed successfully."
+        result = f"PDF '{title}' by {author} processed successfully by pymupdf."
+        return result, markdown_text
+    except Exception as e:
+        return f"Error ingesting PDF: {str(e)}", ""
+
+
+def test_docling_pdf_ingestion(pdf_file):
+    if pdf_file is None:
+        return "No file uploaded", ""
+
+    try:
+        # Create a temporary directory
+        with tempfile.TemporaryDirectory() as temp_dir:
+            # Create a path for the temporary PDF file
+            temp_path = os.path.join(temp_dir, "temp.pdf")
+
+            # Copy the contents of the uploaded file to the temporary file
+            shutil.copy(pdf_file.name, temp_path)
+
+            # Extract text and convert to Markdown
+            converter = DocumentConverter()
+            parsed_pdf = converter.convert(temp_path)
+            markdown_text = parsed_pdf.document.export_to_markdown()
+            # Extract metadata from PDF
+            metadata = extract_metadata_from_pdf(temp_path)
+
+            # Use metadata for title and author if not provided
+            title = metadata.get('title', os.path.splitext(os.path.basename(pdf_file.name))[0])
+            author = metadata.get('author', 'Unknown')
+
+        result = f"PDF '{title}' by {author} processed successfully by pymupdf."
         return result, markdown_text
     except Exception as e:
         return f"Error ingesting PDF: {str(e)}", ""
@@ -140,12 +339,24 @@ def create_pdf_ingestion_test_tab():
         with gr.Row():
             with gr.Column():
                 pdf_file_input = gr.File(label="Upload PDF for testing")
-                test_button = gr.Button("Test PDF Ingestion")
+                test_button = gr.Button("Test pymupdf PDF Ingestion")
+                test_button_2 = gr.Button("Test pymupdf4llm PDF Ingestion")
+                test_button_3 = gr.Button("Test Docling PDF Ingestion")
             with gr.Column():
                 test_output = gr.Textbox(label="Test Result")
                 pdf_content_output = gr.Textbox(label="PDF Content", lines=200)
         test_button.click(
-            fn=test_pdf_ingestion,
+            fn=test_pymupdf_pdf_ingestion,
+            inputs=[pdf_file_input],
+            outputs=[test_output, pdf_content_output]
+        )
+        test_button_2.click(
+            fn=test_pymupdf4llm_pdf_ingestion,
+            inputs=[pdf_file_input],
+            outputs=[test_output, pdf_content_output]
+        )
+        test_button_3.click(
+            fn=test_docling_pdf_ingestion,
             inputs=[pdf_file_input],
             outputs=[test_output, pdf_content_output]
         )

App_Function_Libraries/Gradio_UI/Plaintext_tab_import.py

@@ -6,6 +6,7 @@
 #######################################################################################################################
 #
 # Import necessary libraries
+import logging
 import os
 import tempfile
 import zipfile
@@ -16,101 +17,104 @@ from docx2txt import docx2txt
 from pypandoc import convert_file
 #
 # Import Local libraries
-from App_Function_Libraries.Gradio_UI.Import_Functionality import import_data
+from App_Function_Libraries.Plaintext.Plaintext_Files import import_file_handler
+from App_Function_Libraries.Utils.Utils import default_api_endpoint, global_api_endpoints, format_api_name
 #
 #######################################################################################################################
 #
 # Functions:
 
 def create_plain_text_import_tab():
+    try:
+        default_value = None
+        if default_api_endpoint:
+            if default_api_endpoint in global_api_endpoints:
+                default_value = format_api_name(default_api_endpoint)
+            else:
+                logging.warning(f"Default API endpoint '{default_api_endpoint}' not found in global_api_endpoints")
+    except Exception as e:
+        logging.error(f"Error setting default API endpoint: {str(e)}")
+        default_value = None
+
     with gr.TabItem("Import Plain text & .docx Files", visible=True):
         with gr.Row():
             with gr.Column():
-                gr.Markdown("# Import Markdown(`.md`)/Text(`.txt`)/rtf & `.docx` Files")
-                gr.Markdown("Upload a single file or a zip file containing multiple files")
-                import_file = gr.File(label="Upload file for import", file_types=[".md", ".txt", ".rtf", ".docx", ".zip"])
-                title_input = gr.Textbox(label="Title", placeholder="Enter the title of the content (for single files)")
-                author_input = gr.Textbox(label="Author", placeholder="Enter the author's name (for single files)")
-                keywords_input = gr.Textbox(label="Keywords", placeholder="Enter keywords, comma-separated")
-                system_prompt_input = gr.Textbox(label="System Prompt (for Summarization)", lines=3,
-                                                 value="""<s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
-                                                    **Bulleted Note Creation Guidelines**
-
-                                                    **Headings**:
-                                                    - Based on referenced topics, not categories like quotes or terms
-                                                    - Surrounded by **bold** formatting 
-                                                    - Not listed as bullet points
-                                                    - No space between headings and list items underneath
+                gr.Markdown("# Import `.md`/`.txt`/`.rtf`/`.docx`  Files & `.zip` collections of them.")
+                gr.Markdown("Upload multiple files or a zip file containing multiple files")
 
-                                                    **Emphasis**:
-                                                    - **Important terms** set in bold font
-                                                    - **Text ending in a colon**: also bolded
+                # Updated to support multiple files
+                import_files = gr.File(
+                    label="Upload files for import",
+                    file_count="multiple",
+                    file_types=[".md", ".txt", ".rtf", ".docx", ".zip"]
+                )
 
-                                                    **Review**:
-                                                    - Ensure adherence to specified format
-                                                    - Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]""",
-                                                 )
-                custom_prompt_input = gr.Textbox(label="Custom User Prompt", placeholder="Enter a custom user prompt for summarization (optional)")
+                # Optional metadata override fields
+                author_input = gr.Textbox(
+                    label="Author Override (optional)",
+                    placeholder="Enter author name to apply to all files"
+                )
+                keywords_input = gr.Textbox(
+                    label="Keywords",
+                    placeholder="Enter keywords, comma-separated - will be applied to all files"
+                )
+                system_prompt_input = gr.Textbox(
+                    label="System Prompt (for Summarization)",
+                    lines=3,
+                    value="""
+                    <s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
+                    **Bulleted Note Creation Guidelines**
+                    
+                    **Headings**:
+                    - Based on referenced topics, not categories like quotes or terms
+                    - Surrounded by **bold** formatting 
+                    - Not listed as bullet points
+                    - No space between headings and list items underneath
+                    
+                    **Emphasis**:
+                    - **Important terms** set in bold font
+                    - **Text ending in a colon**: also bolded
+                    
+                    **Review**:
+                    - Ensure adherence to specified format
+                    - Do not reference these instructions in your response.</s>[INST]
+                    """
+                )
+                custom_prompt_input = gr.Textbox(
+                    label="Custom User Prompt",
+                    placeholder="Enter a custom user prompt for summarization (optional)"
+                )
                 auto_summarize_checkbox = gr.Checkbox(label="Auto-summarize", value=False)
+
+                # API configuration
                 api_name_input = gr.Dropdown(
-                    choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral",
-                             "OpenRouter", "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace"],
-                    label="API for Auto-summarization"
+                    choices=["None"] + [format_api_name(api) for api in global_api_endpoints],
+                    value=default_value,
+                    label="API for Summarization/Analysis (Optional)"
                 )
                 api_key_input = gr.Textbox(label="API Key", type="password")
                 import_button = gr.Button("Import File(s)")
-            with gr.Column():
-                import_output = gr.Textbox(label="Import Status")
-
 
-        def import_plain_text_file(file_path, title, author, keywords, system_prompt, user_prompt, auto_summarize, api_name, api_key):
-            try:
-                # Determine the file type and convert if necessary
-                file_extension = os.path.splitext(file_path)[1].lower()
-                if file_extension == '.rtf':
-                    with tempfile.NamedTemporaryFile(suffix='.md', delete=False) as temp_file:
-                        convert_file(file_path, 'md', outputfile=temp_file.name)
-                        file_path = temp_file.name
-                elif file_extension == '.docx':
-                    content = docx2txt.process(file_path)
-                else:
-                    with open(file_path, 'r', encoding='utf-8') as file:
-                        content = file.read()
-
-                # Process the content
-                return import_data(content, title, author, keywords, system_prompt,
-                                   user_prompt, auto_summarize, api_name, api_key)
-            except Exception as e:
-                return f"Error processing file: {str(e)}"
-
-        def process_plain_text_zip_file(zip_file, title, author, keywords, system_prompt, user_prompt, auto_summarize, api_name, api_key):
-            results = []
-            with tempfile.TemporaryDirectory() as temp_dir:
-                with zipfile.ZipFile(zip_file.name, 'r') as zip_ref:
-                    zip_ref.extractall(temp_dir)
-
-                for filename in os.listdir(temp_dir):
-                    if filename.lower().endswith(('.md', '.txt', '.rtf', '.docx')):
-                        file_path = os.path.join(temp_dir, filename)
-                        result = import_plain_text_file(file_path, title, author, keywords, system_prompt,
-                                                        user_prompt, auto_summarize, api_name, api_key)
-                        results.append(f"File: {filename} - {result}")
-
-            return "\n".join(results)
-
-        def import_file_handler(file, title, author, keywords, system_prompt, user_prompt, auto_summarize, api_name, api_key):
-            if file.name.lower().endswith(('.md', '.txt', '.rtf', '.docx')):
-                return import_plain_text_file(file.name, title, author, keywords, system_prompt, user_prompt, auto_summarize, api_name, api_key)
-            elif file.name.lower().endswith('.zip'):
-                return process_plain_text_zip_file(file, title, author, keywords, system_prompt, user_prompt, auto_summarize, api_name, api_key)
-            else:
-                return "Unsupported file type. Please upload a .md, .txt, .rtf, .docx file or a .zip file containing these file types."
+            with gr.Column():
+                import_output = gr.Textbox(label="Import Status", lines=10)
 
         import_button.click(
             fn=import_file_handler,
-            inputs=[import_file, title_input, author_input, keywords_input, system_prompt_input,
-                    custom_prompt_input, auto_summarize_checkbox, api_name_input, api_key_input],
+            inputs=[
+                import_files,
+                author_input,
+                keywords_input,
+                system_prompt_input,
+                custom_prompt_input,
+                auto_summarize_checkbox,
+                api_name_input,
+                api_key_input
+            ],
             outputs=import_output
         )
 
-    return import_file, title_input, author_input, keywords_input, system_prompt_input, custom_prompt_input, auto_summarize_checkbox, api_name_input, api_key_input, import_button, import_output
+    return import_files, author_input, keywords_input, system_prompt_input, custom_prompt_input, auto_summarize_checkbox, api_name_input, api_key_input, import_button, import_output
+
+#
+# End of Plain_text_import.py
+#######################################################################################################################

App_Function_Libraries/Gradio_UI/Podcast_tab.py

@@ -2,23 +2,38 @@
 # Description: Gradio UI for ingesting podcasts into the database
 #
 # Imports
+import logging
 #
 # External Imports
 import gradio as gr
 #
 # Local Imports
 from App_Function_Libraries.Audio.Audio_Files import process_podcast
-from App_Function_Libraries.DB.DB_Manager import load_preset_prompts
+from App_Function_Libraries.DB.DB_Manager import list_prompts
 from App_Function_Libraries.Gradio_UI.Gradio_Shared import whisper_models, update_user_prompt
+from App_Function_Libraries.Utils.Utils import default_api_endpoint, global_api_endpoints, format_api_name
 #
 ########################################################################################################################
 #
 # Functions:
 
-
 def create_podcast_tab():
+    try:
+        default_value = None
+        if default_api_endpoint:
+            if default_api_endpoint in global_api_endpoints:
+                default_value = format_api_name(default_api_endpoint)
+            else:
+                logging.warning(f"Default API endpoint '{default_api_endpoint}' not found in global_api_endpoints")
+    except Exception as e:
+        logging.error(f"Error setting default API endpoint: {str(e)}")
+        default_value = None
     with gr.TabItem("Podcast", visible=True):
         gr.Markdown("# Podcast Transcription and Ingestion", visible=True)
+        # Initialize state variables for pagination
+        current_page_state = gr.State(value=1)
+        total_pages_state = gr.State(value=1)
+
         with gr.Row():
             with gr.Column():
                 podcast_url_input = gr.Textbox(label="Podcast URL", placeholder="Enter the podcast URL here")
@@ -35,54 +50,130 @@ def create_podcast_tab():
                 keep_timestamps_input = gr.Checkbox(label="Keep Timestamps", value=True)
 
                 with gr.Row():
-                    podcast_custom_prompt_checkbox = gr.Checkbox(label="Use a Custom Prompt",
-                                                     value=False,
-                                                     visible=True)
-                    preset_prompt_checkbox = gr.Checkbox(label="Use a pre-set Prompt",
-                                                     value=False,
-                                                     visible=True)
+                    podcast_custom_prompt_checkbox = gr.Checkbox(
+                        label="Use a Custom Prompt",
+                        value=False,
+                        visible=True
+                    )
+                    preset_prompt_checkbox = gr.Checkbox(
+                        label="Use a pre-set Prompt",
+                        value=False,
+                        visible=True
+                    )
+
                 with gr.Row():
-                    preset_prompt = gr.Dropdown(label="Select Preset Prompt",
-                                                choices=load_preset_prompts(),
-                                                visible=False)
+                    # Add pagination controls
+                    preset_prompt = gr.Dropdown(
+                        label="Select Preset Prompt",
+                        choices=[],
+                        visible=False
+                    )
                 with gr.Row():
-                    podcast_custom_prompt_input = gr.Textbox(label="Custom Prompt",
-                                                     placeholder="Enter custom prompt here",
-                                                     lines=3,
-                                                     visible=False)
+                    prev_page_button = gr.Button("Previous Page", visible=False)
+                    page_display = gr.Markdown("Page 1 of X", visible=False)
+                    next_page_button = gr.Button("Next Page", visible=False)
+
                 with gr.Row():
-                    system_prompt_input = gr.Textbox(label="System Prompt",
-                                                     value="""<s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
-**Bulleted Note Creation Guidelines**
-
-**Headings**:
-- Based on referenced topics, not categories like quotes or terms
-- Surrounded by **bold** formatting 
-- Not listed as bullet points
-- No space between headings and list items underneath
-
-**Emphasis**:
-- **Important terms** set in bold font
-- **Text ending in a colon**: also bolded
-
-**Review**:
-- Ensure adherence to specified format
-- Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]
-""",
-                                                     lines=3,
-                                                     visible=False)
+                    podcast_custom_prompt_input = gr.Textbox(
+                        label="Custom Prompt",
+                        placeholder="Enter custom prompt here",
+                        lines=10,
+                        visible=False
+                    )
+                with gr.Row():
+                    system_prompt_input = gr.Textbox(
+                        label="System Prompt",
+                        value="""<s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhere to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
+    **Bulleted Note Creation Guidelines**
+
+    **Headings**:
+    - Based on referenced topics, not categories like quotes or terms
+    - Surrounded by **bold** formatting 
+    - Not listed as bullet points
+    - No space between headings and list items underneath
+
+    **Emphasis**:
+    - **Important terms** set in bold font
+    - **Text ending in a colon**: also bolded
+
+    **Review**:
+    - Ensure adherence to specified format
+    - Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]
+    """,
+                        lines=10,
+                        visible=False
+                    )
 
+                # Handle custom prompt checkbox change
                 podcast_custom_prompt_checkbox.change(
                     fn=lambda x: (gr.update(visible=x), gr.update(visible=x)),
                     inputs=[podcast_custom_prompt_checkbox],
                     outputs=[podcast_custom_prompt_input, system_prompt_input]
                 )
+
+                # Handle preset prompt checkbox change
+                def on_preset_prompt_checkbox_change(is_checked):
+                    if is_checked:
+                        prompts, total_pages, current_page = list_prompts(page=1, per_page=20)
+                        page_display_text = f"Page {current_page} of {total_pages}"
+                        return (
+                            gr.update(visible=True, interactive=True, choices=prompts),  # preset_prompt
+                            gr.update(visible=True),  # prev_page_button
+                            gr.update(visible=True),  # next_page_button
+                            gr.update(value=page_display_text, visible=True),  # page_display
+                            current_page,  # current_page_state
+                            total_pages   # total_pages_state
+                        )
+                    else:
+                        return (
+                            gr.update(visible=False, interactive=False),  # preset_prompt
+                            gr.update(visible=False),  # prev_page_button
+                            gr.update(visible=False),  # next_page_button
+                            gr.update(visible=False),  # page_display
+                            1,  # current_page_state
+                            1   # total_pages_state
+                        )
+
                 preset_prompt_checkbox.change(
-                    fn=lambda x: gr.update(visible=x),
+                    fn=on_preset_prompt_checkbox_change,
                     inputs=[preset_prompt_checkbox],
-                    outputs=[preset_prompt]
+                    outputs=[preset_prompt, prev_page_button, next_page_button, page_display, current_page_state, total_pages_state]
+                )
+
+                # Pagination button functions
+                def on_prev_page_click(current_page, total_pages):
+                    new_page = max(current_page - 1, 1)
+                    prompts, total_pages, current_page = list_prompts(page=new_page, per_page=20)
+                    page_display_text = f"Page {current_page} of {total_pages}"
+                    return (
+                        gr.update(choices=prompts),
+                        gr.update(value=page_display_text),
+                        current_page
+                    )
+
+                prev_page_button.click(
+                    fn=on_prev_page_click,
+                    inputs=[current_page_state, total_pages_state],
+                    outputs=[preset_prompt, page_display, current_page_state]
+                )
+
+                def on_next_page_click(current_page, total_pages):
+                    new_page = min(current_page + 1, total_pages)
+                    prompts, total_pages, current_page = list_prompts(page=new_page, per_page=20)
+                    page_display_text = f"Page {current_page} of {total_pages}"
+                    return (
+                        gr.update(choices=prompts),
+                        gr.update(value=page_display_text),
+                        current_page
+                    )
+
+                next_page_button.click(
+                    fn=on_next_page_click,
+                    inputs=[current_page_state, total_pages_state],
+                    outputs=[preset_prompt, page_display, current_page_state]
                 )
 
+                # Update prompts when a preset is selected
                 def update_prompts(preset_name):
                     prompts = update_user_prompt(preset_name)
                     return (
@@ -91,16 +182,16 @@ def create_podcast_tab():
                     )
 
                 preset_prompt.change(
-                    update_prompts,
-                    inputs=preset_prompt,
+                    fn=update_prompts,
+                    inputs=[preset_prompt],
                     outputs=[podcast_custom_prompt_input, system_prompt_input]
                 )
 
+                # Refactored API selection dropdown
                 podcast_api_name_input = gr.Dropdown(
-                    choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral", "OpenRouter", "Llama.cpp",
-                             "Kobold", "Ooba", "Tabbyapi", "VLLM","ollama", "HuggingFace", "Custom-OpenAI-API"],
-                    value=None,
-                    label="API Name for Summarization (Optional)"
+                    choices=["None"] + [format_api_name(api) for api in global_api_endpoints],
+                    value=default_value,
+                    label="API for Summarization/Analysis (Optional)"
                 )
                 podcast_api_key_input = gr.Textbox(label="API Key (if required)", type="password")
                 podcast_whisper_model_input = gr.Dropdown(choices=whisper_models, value="medium", label="Whisper Model")
@@ -151,13 +242,37 @@ def create_podcast_tab():
 
         podcast_process_button.click(
             fn=process_podcast,
-            inputs=[podcast_url_input, podcast_title_input, podcast_author_input,
-                    podcast_keywords_input, podcast_custom_prompt_input, podcast_api_name_input,
-                    podcast_api_key_input, podcast_whisper_model_input, keep_original_input,
-                    enable_diarization_input, use_cookies_input, cookies_input,
-                    chunk_method, max_chunk_size, chunk_overlap, use_adaptive_chunking,
-                    use_multi_level_chunking, chunk_language, keep_timestamps_input],
-            outputs=[podcast_progress_output, podcast_transcription_output, podcast_summary_output,
-                     podcast_title_input, podcast_author_input, podcast_keywords_input, podcast_error_output,
-                     download_transcription, download_summary]
+            inputs=[
+                podcast_url_input,
+                podcast_title_input,
+                podcast_author_input,
+                podcast_keywords_input,
+                podcast_custom_prompt_input,
+                podcast_api_name_input,
+                podcast_api_key_input,
+                podcast_whisper_model_input,
+                keep_original_input,
+                enable_diarization_input,
+                use_cookies_input,
+                cookies_input,
+                chunk_method,
+                max_chunk_size,
+                chunk_overlap,
+                use_adaptive_chunking,
+                use_multi_level_chunking,
+                chunk_language,
+                keep_timestamps_input,
+                system_prompt_input  # Include system prompt input
+            ],
+            outputs=[
+                podcast_progress_output,
+                podcast_transcription_output,
+                podcast_summary_output,
+                podcast_title_input,
+                podcast_author_input,
+                podcast_keywords_input,
+                podcast_error_output,
+                download_transcription,
+                download_summary
+            ]
         )

App_Function_Libraries/Gradio_UI/Prompt_Suggestion_tab.py

@@ -1,11 +1,14 @@
 # Description: Gradio UI for Creating and Testing new Prompts
 #
 # Imports
+import logging
+
 import gradio as gr
 
-from App_Function_Libraries.Chat import chat
-from App_Function_Libraries.DB.SQLite_DB import add_or_update_prompt
+from App_Function_Libraries.Chat.Chat_Functions import chat
+from App_Function_Libraries.DB.DB_Manager import add_or_update_prompt
 from App_Function_Libraries.Prompt_Engineering.Prompt_Engineering import generate_prompt, test_generated_prompt
+from App_Function_Libraries.Utils.Utils import format_api_name, global_api_endpoints, default_api_endpoint
 
 
 #
@@ -18,6 +21,16 @@ from App_Function_Libraries.Prompt_Engineering.Prompt_Engineering import generat
 
 # Gradio tab for prompt suggestion and testing
 def create_prompt_suggestion_tab():
+    try:
+        default_value = None
+        if default_api_endpoint:
+            if default_api_endpoint in global_api_endpoints:
+                default_value = format_api_name(default_api_endpoint)
+            else:
+                logging.warning(f"Default API endpoint '{default_api_endpoint}' not found in global_api_endpoints")
+    except Exception as e:
+        logging.error(f"Error setting default API endpoint: {str(e)}")
+        default_value = None
     with gr.TabItem("Prompt Suggestion/Creation", visible=True):
         gr.Markdown("# Generate and Test AI Prompts with the Metaprompt Approach")
 
@@ -30,11 +43,11 @@ def create_prompt_suggestion_tab():
                                              placeholder="E.g., CUSTOMER_COMPLAINT, COMPANY_NAME")
 
                 # API-related inputs
+                # Refactored API selection dropdown
                 api_name_input = gr.Dropdown(
-                    choices=["OpenAI", "Cohere", "Groq", "DeepSeek", "Mistral", "OpenRouter", "Llama.cpp",
-                             "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace", "Custom-OpenAI-API"],
-                    label="API Provider",
-                    value="OpenAI"  # Default selection
+                    choices=["None"] + [format_api_name(api) for api in global_api_endpoints],
+                    value=default_value,
+                    label="API for Analysis (Optional)"
                 )
 
                 api_key_input = gr.Textbox(label="API Key", placeholder="Enter your API key (if required)",

App_Function_Libraries/Gradio_UI/Prompts_tab.py

@@ -0,0 +1,297 @@
+# Prompts_tab.py
+# Description: This file contains the code for the prompts tab in the Gradio UI
+#
+# Imports
+import html
+import logging
+
+#
+# External Imports
+import gradio as gr
+#
+# Local Imports
+from App_Function_Libraries.DB.DB_Manager import fetch_prompt_details, list_prompts
+#
+####################################################################################################
+#
+# Functions:
+
+def create_prompt_view_tab():
+    with gr.TabItem("View Prompt Database", visible=True):
+        gr.Markdown("# View Prompt Database Entries")
+        with gr.Row():
+            with gr.Column():
+                entries_per_page = gr.Dropdown(choices=[10, 20, 50, 100], label="Entries per Page", value=10)
+                page_number = gr.Number(value=1, label="Page Number", precision=0)
+                view_button = gr.Button("View Page")
+                previous_page_button = gr.Button("Previous Page", visible=True)
+                next_page_button = gr.Button("Next Page", visible=True)
+                pagination_info = gr.Textbox(label="Pagination Info", interactive=False)
+                prompt_selector = gr.Dropdown(label="Select Prompt to View", choices=[])
+            with gr.Column():
+                results_table = gr.HTML()
+                selected_prompt_display = gr.HTML()
+
+        # Function to view database entries
+        def view_database(page, entries_per_page):
+            try:
+                # Use list_prompts to get prompts and total pages
+                prompts, total_pages, current_page = list_prompts(page=int(page), per_page=int(entries_per_page))
+
+                table_html = "<table style='width:100%; border-collapse: collapse;'>"
+                table_html += "<tr><th style='border: 1px solid black; padding: 8px;'>Title</th><th style='border: 1px solid black; padding: 8px;'>Author</th></tr>"
+                prompt_choices = []
+                for prompt_name in prompts:
+                    details = fetch_prompt_details(prompt_name)
+                    if details:
+                        title, author, _, _, _, _ = details
+                        author = author or "Unknown"  # Handle None author
+                        table_html += f"<tr><td style='border: 1px solid black; padding: 8px;'>{html.escape(title)}</td><td style='border: 1px solid black; padding: 8px;'>{html.escape(author)}</td></tr>"
+                        prompt_choices.append(prompt_name)  # Using prompt_name as value
+                table_html += "</table>"
+
+                # Get total prompts if possible
+                total_prompts = total_pages * int(entries_per_page)  # This might overestimate if the last page is not full
+
+                pagination = f"Page {current_page} of {total_pages} (Total prompts: {total_prompts})"
+
+                return table_html, pagination, total_pages, prompt_choices
+            except Exception as e:
+                return f"<p>Error fetching prompts: {e}</p>", "Error", 0, []
+
+        # Function to update page content
+        def update_page(page, entries_per_page):
+            results, pagination, total_pages, prompt_choices = view_database(page, entries_per_page)
+            page = int(page)
+            next_disabled = page >= total_pages
+            prev_disabled = page <= 1
+            return (
+                results,
+                pagination,
+                page,
+                gr.update(visible=True, interactive=not prev_disabled),  # previous_page_button
+                gr.update(visible=True, interactive=not next_disabled),   # next_page_button
+                gr.update(choices=prompt_choices)
+            )
+
+        # Function to go to the next page
+        def go_to_next_page(current_page, entries_per_page):
+            next_page = int(current_page) + 1
+            return update_page(next_page, entries_per_page)
+
+        # Function to go to the previous page
+        def go_to_previous_page(current_page, entries_per_page):
+            previous_page = max(1, int(current_page) - 1)
+            return update_page(previous_page, entries_per_page)
+
+        # Function to display selected prompt details
+        def display_selected_prompt(prompt_name):
+            details = fetch_prompt_details(prompt_name)
+            if details:
+                title, author, description, system_prompt, user_prompt, keywords = details
+                # Handle None values by converting them to empty strings
+                description = description or ""
+                system_prompt = system_prompt or ""
+                user_prompt = user_prompt or ""
+                author = author or "Unknown"
+                keywords = keywords or ""
+
+                html_content = f"""
+                <div style="border: 1px solid #ddd; padding: 10px; margin-bottom: 20px;">
+                    <h3>{html.escape(title)}</h3> <h4>by {html.escape(author)}</h4>
+                    <p><strong>Description:</strong> {html.escape(description)}</p>
+                    <div style="margin-top: 10px;">
+                        <strong>System Prompt:</strong>
+                        <pre style="white-space: pre-wrap; word-wrap: break-word;">{html.escape(system_prompt)}</pre>
+                    </div>
+                    <div style="margin-top: 10px;">
+                        <strong>User Prompt:</strong>
+                        <pre style="white-space: pre-wrap; word-wrap: break-word;">{html.escape(user_prompt)}</pre>
+                    </div>
+                    <p><strong>Keywords:</strong> {html.escape(keywords)}</p>
+                </div>
+                """
+                return html_content
+            else:
+                return "<p>Prompt not found.</p>"
+
+        # Event handlers
+        view_button.click(
+            fn=update_page,
+            inputs=[page_number, entries_per_page],
+            outputs=[results_table, pagination_info, page_number, previous_page_button, next_page_button, prompt_selector]
+        )
+
+        next_page_button.click(
+            fn=go_to_next_page,
+            inputs=[page_number, entries_per_page],
+            outputs=[results_table, pagination_info, page_number, previous_page_button, next_page_button, prompt_selector]
+        )
+
+        previous_page_button.click(
+            fn=go_to_previous_page,
+            inputs=[page_number, entries_per_page],
+            outputs=[results_table, pagination_info, page_number, previous_page_button, next_page_button, prompt_selector]
+        )
+
+        prompt_selector.change(
+            fn=display_selected_prompt,
+            inputs=[prompt_selector],
+            outputs=[selected_prompt_display]
+        )
+
+
+
+def create_prompts_export_tab():
+    """Creates a tab for exporting prompts database content with multiple format options"""
+    with gr.TabItem("Export Prompts", visible=True):
+        gr.Markdown("# Export Prompts Database Content")
+
+        with gr.Row():
+            with gr.Column():
+                export_type = gr.Radio(
+                    choices=["All Prompts", "Prompts by Keyword"],
+                    label="Export Type",
+                    value="All Prompts"
+                )
+
+                # Keyword selection for filtered export
+                with gr.Column(visible=False) as keyword_col:
+                    keyword_input = gr.Textbox(
+                        label="Enter Keywords (comma-separated)",
+                        placeholder="Enter keywords to filter prompts..."
+                    )
+
+                # Export format selection
+                export_format = gr.Radio(
+                    choices=["CSV", "Markdown (ZIP)"],
+                    label="Export Format",
+                    value="CSV"
+                )
+
+                # Export options
+                include_options = gr.CheckboxGroup(
+                    choices=[
+                        "Include System Prompts",
+                        "Include User Prompts",
+                        "Include Details",
+                        "Include Author",
+                        "Include Keywords"
+                    ],
+                    label="Export Options",
+                    value=["Include Keywords", "Include Author"]
+                )
+
+                # Markdown-specific options (only visible when Markdown is selected)
+                with gr.Column(visible=False) as markdown_options_col:
+                    markdown_template = gr.Radio(
+                        choices=[
+                            "Basic Template",
+                            "Detailed Template",
+                            "Custom Template"
+                        ],
+                        label="Markdown Template",
+                        value="Basic Template"
+                    )
+                    custom_template = gr.Textbox(
+                        label="Custom Template",
+                        placeholder="Use {title}, {author}, {details}, {system}, {user}, {keywords} as placeholders",
+                        visible=False
+                    )
+
+                export_button = gr.Button("Export Prompts")
+
+            with gr.Column():
+                export_status = gr.Textbox(label="Export Status", interactive=False)
+                export_file = gr.File(label="Download Export")
+
+        def update_ui_visibility(export_type, format_choice, template_choice):
+            """Update UI elements visibility based on selections"""
+            show_keywords = export_type == "Prompts by Keyword"
+            show_markdown_options = format_choice == "Markdown (ZIP)"
+            show_custom_template = template_choice == "Custom Template" and show_markdown_options
+
+            return [
+                gr.update(visible=show_keywords),  # keyword_col
+                gr.update(visible=show_markdown_options),  # markdown_options_col
+                gr.update(visible=show_custom_template)  # custom_template
+            ]
+
+        def handle_export(export_type, keywords, export_format, options, markdown_template, custom_template):
+            """Handle the export process based on selected options"""
+            try:
+                # Parse options
+                include_system = "Include System Prompts" in options
+                include_user = "Include User Prompts" in options
+                include_details = "Include Details" in options
+                include_author = "Include Author" in options
+                include_keywords = "Include Keywords" in options
+
+                # Handle keyword filtering
+                keyword_list = None
+                if export_type == "Prompts by Keyword" and keywords:
+                    keyword_list = [k.strip() for k in keywords.split(",") if k.strip()]
+
+                # Get the appropriate template
+                template = None
+                if export_format == "Markdown (ZIP)":
+                    if markdown_template == "Custom Template":
+                        template = custom_template
+                    else:
+                        template = markdown_template
+
+                # Perform export
+                from App_Function_Libraries.DB.Prompts_DB import export_prompts
+                status, file_path = export_prompts(
+                    export_format=export_format.split()[0].lower(),  # 'csv' or 'markdown'
+                    filter_keywords=keyword_list,
+                    include_system=include_system,
+                    include_user=include_user,
+                    include_details=include_details,
+                    include_author=include_author,
+                    include_keywords=include_keywords,
+                    markdown_template=template
+                )
+
+                return status, file_path
+
+            except Exception as e:
+                error_msg = f"Export failed: {str(e)}"
+                logging.error(error_msg)
+                return error_msg, None
+
+        # Event handlers
+        export_type.change(
+            fn=lambda t, f, m: update_ui_visibility(t, f, m),
+            inputs=[export_type, export_format, markdown_template],
+            outputs=[keyword_col, markdown_options_col, custom_template]
+        )
+
+        export_format.change(
+            fn=lambda t, f, m: update_ui_visibility(t, f, m),
+            inputs=[export_type, export_format, markdown_template],
+            outputs=[keyword_col, markdown_options_col, custom_template]
+        )
+
+        markdown_template.change(
+            fn=lambda t, f, m: update_ui_visibility(t, f, m),
+            inputs=[export_type, export_format, markdown_template],
+            outputs=[keyword_col, markdown_options_col, custom_template]
+        )
+
+        export_button.click(
+            fn=handle_export,
+            inputs=[
+                export_type,
+                keyword_input,
+                export_format,
+                include_options,
+                markdown_template,
+                custom_template
+            ],
+            outputs=[export_status, export_file]
+        )
+
+#
+# End of Prompts_tab.py
+####################################################################################################

App_Function_Libraries/Gradio_UI/RAG_Chat_tab.py

@@ -10,12 +10,26 @@ import gradio as gr
 # Local Imports
 
 from App_Function_Libraries.RAG.RAG_Library_2 import enhanced_rag_pipeline
+from App_Function_Libraries.Utils.Utils import default_api_endpoint, global_api_endpoints, format_api_name
+
+
 #
 ########################################################################################################################
 #
 # Functions:
 
 def create_rag_tab():
+    try:
+        default_value = None
+        if default_api_endpoint:
+            if default_api_endpoint in global_api_endpoints:
+                default_value = format_api_name(default_api_endpoint)
+            else:
+                logging.warning(f"Default API endpoint '{default_api_endpoint}' not found in global_api_endpoints")
+    except Exception as e:
+        logging.error(f"Error setting default API endpoint: {str(e)}")
+        default_value = None
+
     with gr.TabItem("RAG Search", visible=True):
         gr.Markdown("# Retrieval-Augmented Generation (RAG) Search")
 
@@ -36,10 +50,11 @@ def create_rag_tab():
                     visible=False
                 )
 
+                # Refactored API selection dropdown
                 api_choice = gr.Dropdown(
-                    choices=["Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral", "OpenRouter", "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace"],
-                    label="Select API for RAG",
-                    value="OpenAI"
+                    choices=["None"] + [format_api_name(api) for api in global_api_endpoints],
+                    value=default_value,
+                    label="API for Chat Response (Optional)"
                 )
                 search_button = gr.Button("Search")
 

App_Function_Libraries/Gradio_UI/RAG_QA_Chat_tab.py

@@ -6,6 +6,7 @@ import csv
 import logging
 import json
 import os
+import re
 from datetime import datetime
 #
 # External Imports
@@ -14,32 +15,39 @@ import gradio as gr
 #
 # Local Imports
 from App_Function_Libraries.Books.Book_Ingestion_Lib import read_epub
-from App_Function_Libraries.DB.DB_Manager import DatabaseError, get_paginated_files, add_media_with_keywords
-from App_Function_Libraries.DB.RAG_QA_Chat_DB import (
-    save_notes,
-    add_keywords_to_note,
-    start_new_conversation,
-    save_message,
-    search_conversations_by_keywords,
-    load_chat_history,
-    get_all_conversations,
-    get_note_by_id,
-    get_notes_by_keywords,
-    get_notes_by_keyword_collection,
-    update_note,
-    clear_keywords_from_note, get_notes, get_keywords_for_note, delete_conversation, delete_note, execute_query,
-    add_keywords_to_conversation, fetch_all_notes, fetch_all_conversations, fetch_conversations_by_ids,
-    fetch_notes_by_ids,
-)
+from App_Function_Libraries.DB.Character_Chat_DB import search_character_chat, search_character_cards
+from App_Function_Libraries.DB.DB_Manager import DatabaseError, get_paginated_files, add_media_with_keywords, \
+    get_all_conversations, get_note_by_id, get_notes_by_keywords, start_new_conversation, update_note, save_notes, \
+    clear_keywords_from_note, add_keywords_to_note, load_chat_history, save_message, add_keywords_to_conversation, \
+    get_keywords_for_note, delete_note, search_conversations_by_keywords, get_conversation_title, delete_conversation, \
+    update_conversation_title, fetch_all_conversations, fetch_all_notes, fetch_conversations_by_ids, fetch_notes_by_ids, \
+    search_media_db, search_notes_titles, list_prompts
+from App_Function_Libraries.DB.RAG_QA_Chat_DB import get_notes, delete_messages_in_conversation, search_rag_notes, \
+    search_rag_chat, get_conversation_rating, set_conversation_rating
+from App_Function_Libraries.Gradio_UI.Gradio_Shared import update_user_prompt
 from App_Function_Libraries.PDF.PDF_Ingestion_Lib import extract_text_and_format_from_pdf
 from App_Function_Libraries.RAG.RAG_Library_2 import generate_answer, enhanced_rag_pipeline
 from App_Function_Libraries.RAG.RAG_QA_Chat import search_database, rag_qa_chat
+from App_Function_Libraries.Utils.Utils import default_api_endpoint, global_api_endpoints, format_api_name, \
+    load_comprehensive_config
+
+
 #
 ########################################################################################################################
 #
 # Functions:
 
 def create_rag_qa_chat_tab():
+    try:
+        default_value = None
+        if default_api_endpoint:
+            if default_api_endpoint in global_api_endpoints:
+                default_value = format_api_name(default_api_endpoint)
+            else:
+                logging.warning(f"Default API endpoint '{default_api_endpoint}' not found in global_api_endpoints")
+    except Exception as e:
+        logging.error(f"Error setting default API endpoint: {str(e)}")
+        default_value = None
     with gr.TabItem("RAG QA Chat", visible=True):
         gr.Markdown("# RAG QA Chat")
 
@@ -47,18 +55,53 @@ def create_rag_qa_chat_tab():
             "page": 1,
             "context_source": "Entire Media Database",
             "conversation_messages": [],
+            "conversation_id": None
         })
 
         note_state = gr.State({"note_id": None})
 
+        def auto_save_conversation(message, response, state_value, auto_save_enabled):
+            """Automatically save the conversation if auto-save is enabled"""
+            try:
+                if not auto_save_enabled:
+                    return state_value
+
+                conversation_id = state_value.get("conversation_id")
+                if not conversation_id:
+                    # Create new conversation with default title
+                    title = "Auto-saved Conversation " + datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+                    conversation_id = start_new_conversation(title=title)
+                    state_value = state_value.copy()
+                    state_value["conversation_id"] = conversation_id
+
+                # Save the messages
+                save_message(conversation_id, "user", message)
+                save_message(conversation_id, "assistant", response)
+
+                return state_value
+            except Exception as e:
+                logging.error(f"Error in auto-save: {str(e)}")
+                return state_value
+
         # Update the conversation list function
         def update_conversation_list():
             conversations, total_pages, total_count = get_all_conversations()
-            choices = [f"{title} (ID: {conversation_id})" for conversation_id, title in conversations]
+            choices = [
+                f"{conversation['title']} (ID: {conversation['conversation_id']}) - Rating: {conversation['rating'] or 'Not Rated'}"
+                for conversation in conversations
+            ]
             return choices
 
         with gr.Row():
             with gr.Column(scale=1):
+                # FIXME - Offer the user to search 2+ databases at once
+                database_types = ["Media DB", "RAG Chat", "RAG Notes", "Character Chat", "Character Cards"]
+                db_choice = gr.CheckboxGroup(
+                    label="Select Database(s)",
+                    choices=database_types,
+                    value=["Media DB"],
+                    interactive=True
+                )
                 context_source = gr.Radio(
                     ["All Files in the Database", "Search Database", "Upload File"],
                     label="Context Source",
@@ -71,19 +114,52 @@ def create_rag_qa_chat_tab():
                     next_page_btn = gr.Button("Next Page")
                     page_info = gr.HTML("Page 1")
                 top_k_input = gr.Number(value=10, label="Maximum amount of results to use (Default: 10)", minimum=1, maximum=50, step=1, precision=0, interactive=True)
-                keywords_input = gr.Textbox(label="Keywords (comma-separated) to filter results by)", visible=True)
+                keywords_input = gr.Textbox(label="Keywords (comma-separated) to filter results by)", value="rag_qa_default_keyword" ,visible=True)
                 use_query_rewriting = gr.Checkbox(label="Use Query Rewriting", value=True)
                 use_re_ranking = gr.Checkbox(label="Use Re-ranking", value=True)
-                # with gr.Row():
-                #     page_number = gr.Number(value=1, label="Page", precision=0)
-                #     page_size = gr.Number(value=20, label="Items per page", precision=0)
-                #     total_pages = gr.Number(label="Total Pages", interactive=False)
+                config = load_comprehensive_config()
+                auto_save_value = config.getboolean('auto-save', 'save_character_chats', fallback=False)
+                auto_save_checkbox = gr.Checkbox(
+                    label="Save chats automatically",
+                    value=auto_save_value,
+                    info="When enabled, conversations will be saved automatically after each message"
+                )
+
+                initial_prompts, total_pages, current_page = list_prompts(page=1, per_page=10)
 
+                preset_prompt_checkbox = gr.Checkbox(
+                    label="View Custom Prompts(have to copy/paste them)",
+                    value=False,
+                    visible=True
+                )
+
+                with gr.Row(visible=False) as preset_prompt_controls:
+                    prev_prompt_page = gr.Button("Previous")
+                    current_prompt_page_text = gr.Text(f"Page {current_page} of {total_pages}")
+                    next_prompt_page = gr.Button("Next")
+                    current_prompt_page_state = gr.State(value=1)
+
+                preset_prompt = gr.Dropdown(
+                    label="Select Preset Prompt",
+                    choices=initial_prompts,
+                    visible=False
+                )
+                user_prompt = gr.Textbox(
+                    label="Custom Prompt",
+                    placeholder="Enter custom prompt here",
+                    lines=3,
+                    visible=False
+                )
+
+                system_prompt_input = gr.Textbox(
+                    label="System Prompt",
+                    lines=3,
+                    visible=False
+                )
 
                 search_query = gr.Textbox(label="Search Query", visible=False)
                 search_button = gr.Button("Search", visible=False)
                 search_results = gr.Dropdown(label="Search Results", choices=[], visible=False)
-                # FIXME - Add pages for search results handling
                 file_upload = gr.File(
                     label="Upload File",
                     visible=False,
@@ -95,34 +171,28 @@ def create_rag_qa_chat_tab():
                 load_conversation = gr.Dropdown(
                     label="Load Conversation",
                     choices=update_conversation_list()
-                    )
+                )
                 new_conversation = gr.Button("New Conversation")
                 save_conversation_button = gr.Button("Save Conversation")
                 conversation_title = gr.Textbox(
-                    label="Conversation Title", placeholder="Enter a title for the new conversation"
+                    label="Conversation Title",
+                    placeholder="Enter a title for the new conversation"
                 )
                 keywords = gr.Textbox(label="Keywords (comma-separated)", visible=True)
 
+                # Add the rating display and input
+                rating_display = gr.Markdown(value="", visible=False)
+                rating_input = gr.Radio(
+                    choices=["1", "2", "3"],
+                    label="Rate this Conversation (1-3 stars)",
+                    visible=False
+                )
+
+                # Refactored API selection dropdown
                 api_choice = gr.Dropdown(
-                    choices=[
-                        "Local-LLM",
-                        "OpenAI",
-                        "Anthropic",
-                        "Cohere",
-                        "Groq",
-                        "DeepSeek",
-                        "Mistral",
-                        "OpenRouter",
-                        "Llama.cpp",
-                        "Kobold",
-                        "Ooba",
-                        "Tabbyapi",
-                        "VLLM",
-                        "ollama",
-                        "HuggingFace",
-                    ],
-                    label="Select API for RAG",
-                    value="OpenAI",
+                    choices=["None"] + [format_api_name(api) for api in global_api_endpoints],
+                    value=default_value,
+                    label="API for Chat Response (Optional)"
                 )
 
         with gr.Row():
@@ -145,6 +215,8 @@ def create_rag_qa_chat_tab():
                 clear_notes_btn = gr.Button("Clear Current Note text")
 
                 new_note_btn = gr.Button("New Note")
+                # FIXME - Change from only keywords to generalized search
+                search_notes_title = gr.Textbox(label="Search Notes by Title")
                 search_notes_by_keyword = gr.Textbox(label="Search Notes by Keyword")
                 search_notes_button = gr.Button("Search Notes")
                 note_results = gr.Dropdown(label="Notes", choices=[])
@@ -152,8 +224,58 @@ def create_rag_qa_chat_tab():
 
         loading_indicator = gr.HTML("Loading...", visible=False)
         status_message = gr.HTML()
+        auto_save_status = gr.HTML()
+
+
 
         # Function Definitions
+        def update_prompt_page(direction, current_page_val):
+            new_page = max(1, min(total_pages, current_page_val + direction))
+            prompts, _, _ = list_prompts(page=new_page, per_page=10)
+            return (
+                gr.update(choices=prompts),
+                gr.update(value=f"Page {new_page} of {total_pages}"),
+                new_page
+            )
+
+        def update_prompts(preset_name):
+            prompts = update_user_prompt(preset_name)
+            return (
+                gr.update(value=prompts["user_prompt"], visible=True),
+                gr.update(value=prompts["system_prompt"], visible=True)
+            )
+
+        def toggle_preset_prompt(checkbox_value):
+            return (
+                gr.update(visible=checkbox_value),
+                gr.update(visible=checkbox_value),
+                gr.update(visible=False),
+                gr.update(visible=False)
+            )
+
+        prev_prompt_page.click(
+            lambda x: update_prompt_page(-1, x),
+            inputs=[current_prompt_page_state],
+            outputs=[preset_prompt, current_prompt_page_text, current_prompt_page_state]
+        )
+
+        next_prompt_page.click(
+            lambda x: update_prompt_page(1, x),
+            inputs=[current_prompt_page_state],
+            outputs=[preset_prompt, current_prompt_page_text, current_prompt_page_state]
+        )
+
+        preset_prompt.change(
+            update_prompts,
+            inputs=preset_prompt,
+            outputs=[user_prompt, system_prompt_input]
+        )
+
+        preset_prompt_checkbox.change(
+            toggle_preset_prompt,
+            inputs=[preset_prompt_checkbox],
+            outputs=[preset_prompt, preset_prompt_controls, user_prompt, system_prompt_input]
+        )
 
         def update_state(state, **kwargs):
             new_state = state.copy()
@@ -168,18 +290,28 @@ def create_rag_qa_chat_tab():
             outputs=[note_title, notes, note_state]
         )
 
-        def search_notes(keywords):
+        def search_notes(search_notes_title, keywords):
             if keywords:
                 keywords_list = [kw.strip() for kw in keywords.split(',')]
                 notes_data, total_pages, total_count = get_notes_by_keywords(keywords_list)
-                choices = [f"Note {note_id} ({timestamp})" for note_id, title, content, timestamp in notes_data]
-                return gr.update(choices=choices)
+                choices = [f"Note {note_id} - {title} ({timestamp})" for
+                           note_id, title, content, timestamp, conversation_id in notes_data]
+                return gr.update(choices=choices, label=f"Found {total_count} notes")
+            elif search_notes_title:
+                notes_data, total_pages, total_count = search_notes_titles(search_notes_title)
+                choices = [f"Note {note_id} - {title} ({timestamp})" for
+                           note_id, title, content, timestamp, conversation_id in notes_data]
+                return gr.update(choices=choices, label=f"Found {total_count} notes")
             else:
-                return gr.update(choices=[])
+                # This will now return all notes, ordered by timestamp
+                notes_data, total_pages, total_count = search_notes_titles("")
+                choices = [f"Note {note_id} - {title} ({timestamp})" for
+                           note_id, title, content, timestamp, conversation_id in notes_data]
+                return gr.update(choices=choices, label=f"All notes ({total_count} total)")
 
         search_notes_button.click(
             search_notes,
-            inputs=[search_notes_by_keyword],
+            inputs=[search_notes_title, search_notes_by_keyword],
             outputs=[note_results]
         )
 
@@ -201,31 +333,69 @@ def create_rag_qa_chat_tab():
 
         def save_notes_function(note_title_text, notes_content, keywords_content, note_state_value, state_value):
             """Save the notes and associated keywords to the database."""
-            conversation_id = state_value.get("conversation_id")
-            note_id = note_state_value["note_id"]
-            if conversation_id and notes_content:
+            logging.info(f"Starting save_notes_function with state: {state_value}")
+            logging.info(f"Note title: {note_title_text}")
+            logging.info(f"Notes content length: {len(notes_content) if notes_content else 0}")
+
+            try:
+                # Check current state
+                conversation_id = state_value.get("conversation_id")
+                logging.info(f"Current conversation_id: {conversation_id}")
+
+                # Create new conversation if none exists
+                if not conversation_id:
+                    logging.info("No conversation ID found, creating new conversation")
+                    conversation_title = note_title_text if note_title_text else "Untitled Conversation"
+                    conversation_id = start_new_conversation(title=conversation_title)
+                    state_value = state_value.copy()  # Create a new copy of the state
+                    state_value["conversation_id"] = conversation_id
+                    logging.info(f"Created new conversation with ID: {conversation_id}")
+
+                if not notes_content:
+                    logging.warning("No notes content provided")
+                    return notes_content, note_state_value, state_value, gr.update(
+                        value="<p style='color:red;'>Cannot save empty notes.</p>")
+
+                # Save or update note
+                note_id = note_state_value.get("note_id")
                 if note_id:
-                    # Update existing note
+                    logging.info(f"Updating existing note with ID: {note_id}")
                     update_note(note_id, note_title_text, notes_content)
                 else:
-                    # Save new note
-                    note_id = save_notes(conversation_id, note_title_text, notes_content)
-                    note_state_value["note_id"] = note_id
+                    logging.info(f"Creating new note for conversation: {conversation_id}")
+                    note_id = save_notes(conversation_id, note_title_text or "Untitled Note", notes_content)
+                    note_state_value = {"note_id": note_id}
+                    logging.info(f"Created new note with ID: {note_id}")
+
+                # Handle keywords
                 if keywords_content:
-                    # Clear existing keywords and add new ones
+                    logging.info("Processing keywords")
                     clear_keywords_from_note(note_id)
-                    add_keywords_to_note(note_id, [kw.strip() for kw in keywords_content.split(',')])
+                    keywords = [kw.strip() for kw in keywords_content.split(',')]
+                    add_keywords_to_note(note_id, keywords)
+                    logging.info(f"Added keywords: {keywords}")
 
-                logging.info("Notes and keywords saved successfully!")
-                return notes_content, note_state_value
-            else:
-                logging.warning("No conversation ID or notes to save.")
-                return "", note_state_value
+                logging.info("Notes saved successfully")
+                return (
+                    notes_content,
+                    note_state_value,
+                    state_value,
+                    gr.update(value="<p style='color:green;'>Notes saved successfully!</p>")
+                )
+
+            except Exception as e:
+                logging.error(f"Error in save_notes_function: {str(e)}", exc_info=True)
+                return (
+                    notes_content,
+                    note_state_value,
+                    state_value,
+                    gr.update(value=f"<p style='color:red;'>Error saving notes: {str(e)}</p>")
+                )
 
         save_notes_btn.click(
             save_notes_function,
             inputs=[note_title, notes, keywords_for_notes, note_state, state],
-            outputs=[notes, note_state]
+            outputs=[notes, note_state, state, status_message]
         )
 
         def clear_notes_function():
@@ -237,83 +407,112 @@ def create_rag_qa_chat_tab():
             outputs=[notes, note_state]
         )
 
-        def update_conversation_list():
-            conversations, total_pages, total_count = get_all_conversations()
-            choices = [f"{title} (ID: {conversation_id})" for conversation_id, title in conversations]
-            return choices
-
         # Initialize the conversation list
         load_conversation.choices = update_conversation_list()
 
         def load_conversation_history(selected_conversation, state_value):
-            if selected_conversation:
-                conversation_id = selected_conversation.split('(ID: ')[1][:-1]
+            try:
+                if not selected_conversation:
+                    return [], state_value, "", gr.update(value="", visible=False), gr.update(visible=False)
+                # Extract conversation ID
+                match = re.search(r'\(ID: ([0-9a-fA-F\-]+)\)', selected_conversation)
+                if not match:
+                    logging.error(f"Invalid conversation format: {selected_conversation}")
+                    return [], state_value, "", gr.update(value="", visible=False), gr.update(visible=False)
+                conversation_id = match.group(1)
                 chat_data, total_pages_val, _ = load_chat_history(conversation_id, 1, 50)
-                # Convert chat data to list of tuples (user_message, assistant_response)
+                # Update state with valid conversation id
+                updated_state = state_value.copy()
+                updated_state["conversation_id"] = conversation_id
+                updated_state["conversation_messages"] = chat_data
+                # Format chat history
                 history = []
                 for role, content in chat_data:
                     if role == 'user':
                         history.append((content, ''))
-                    else:
-                        if history:
-                            history[-1] = (history[-1][0], content)
-                        else:
-                            history.append(('', content))
-                # Retrieve notes
+                    elif history:
+                        history[-1] = (history[-1][0], content)
+                # Fetch and display the conversation rating
+                rating = get_conversation_rating(conversation_id)
+                if rating is not None:
+                    rating_text = f"**Current Rating:** {rating} star(s)"
+                    rating_display_update = gr.update(value=rating_text, visible=True)
+                    rating_input_update = gr.update(value=str(rating), visible=True)
+                else:
+                    rating_display_update = gr.update(value="**Current Rating:** Not Rated", visible=True)
+                    rating_input_update = gr.update(value=None, visible=True)
                 notes_content = get_notes(conversation_id)
-                updated_state = update_state(state_value, conversation_id=conversation_id, page=1,
-                                             conversation_messages=[])
-                return history, updated_state, "\n".join(notes_content)
-            return [], state_value, ""
+                return history, updated_state, "\n".join(
+                    notes_content) if notes_content else "", rating_display_update, rating_input_update
+            except Exception as e:
+                logging.error(f"Error loading conversation: {str(e)}")
+                return [], state_value, "", gr.update(value="", visible=False), gr.update(visible=False)
 
         load_conversation.change(
             load_conversation_history,
             inputs=[load_conversation, state],
-            outputs=[chatbot, state, notes]
+            outputs=[chatbot, state, notes, rating_display, rating_input]
         )
 
         # Modify save_conversation_function to use gr.update()
-        def save_conversation_function(conversation_title_text, keywords_text, state_value):
+        def save_conversation_function(conversation_title_text, keywords_text, rating_value, state_value):
             conversation_messages = state_value.get("conversation_messages", [])
+            conversation_id = state_value.get("conversation_id")
             if not conversation_messages:
                 return gr.update(
                     value="<p style='color:red;'>No conversation to save.</p>"
-                ), state_value, gr.update()
-            # Start a new conversation in the database
-            new_conversation_id = start_new_conversation(
-                conversation_title_text if conversation_title_text else "Untitled Conversation"
-            )
+                ), state_value, gr.update(), gr.update(value="", visible=False), gr.update(visible=False)
+            # Start a new conversation in the database if not existing
+            if not conversation_id:
+                conversation_id = start_new_conversation(
+                    conversation_title_text if conversation_title_text else "Untitled Conversation"
+                )
+            else:
+                # Update the conversation title if it has changed
+                update_conversation_title(conversation_id, conversation_title_text)
             # Save the messages
             for role, content in conversation_messages:
-                save_message(new_conversation_id, role, content)
+                save_message(conversation_id, role, content)
             # Save keywords if provided
             if keywords_text:
-                add_keywords_to_conversation(new_conversation_id, [kw.strip() for kw in keywords_text.split(',')])
+                add_keywords_to_conversation(conversation_id, [kw.strip() for kw in keywords_text.split(',')])
+            # Save the rating if provided
+            try:
+                if rating_value:
+                    set_conversation_rating(conversation_id, int(rating_value))
+            except ValueError as ve:
+                logging.error(f"Invalid rating value: {ve}")
+                return gr.update(
+                    value=f"<p style='color:red;'>Invalid rating: {ve}</p>"
+                ), state_value, gr.update(), gr.update(value="", visible=False), gr.update(visible=False)
+
             # Update state
-            updated_state = update_state(state_value, conversation_id=new_conversation_id)
+            updated_state = update_state(state_value, conversation_id=conversation_id)
             # Update the conversation list
             conversation_choices = update_conversation_list()
+            # Reset rating display and input
+            rating_display_update = gr.update(value=f"**Current Rating:** {rating_value} star(s)", visible=True)
+            rating_input_update = gr.update(value=rating_value, visible=True)
             return gr.update(
                 value="<p style='color:green;'>Conversation saved successfully.</p>"
-            ), updated_state, gr.update(choices=conversation_choices)
+            ), updated_state, gr.update(choices=conversation_choices), rating_display_update, rating_input_update
 
         save_conversation_button.click(
             save_conversation_function,
-            inputs=[conversation_title, keywords, state],
-            outputs=[status_message, state, load_conversation]
+            inputs=[conversation_title, keywords, rating_input, state],
+            outputs=[status_message, state, load_conversation, rating_display, rating_input]
         )
 
         def start_new_conversation_wrapper(title, state_value):
-            # Reset the state with no conversation_id
-            updated_state = update_state(state_value, conversation_id=None, page=1,
-                                         conversation_messages=[])
-            # Clear the chat history
-            return [], updated_state
+            # Reset the state with no conversation_id and empty conversation messages
+            updated_state = update_state(state_value, conversation_id=None, page=1, conversation_messages=[])
+            # Clear the chat history and reset rating components
+            return [], updated_state, gr.update(value="", visible=False), gr.update(value=None, visible=False)
 
         new_conversation.click(
             start_new_conversation_wrapper,
             inputs=[conversation_title, state],
-            outputs=[chatbot, state]
+            outputs=[chatbot, state, rating_display, rating_input]
         )
 
         def update_file_list(page):
@@ -328,11 +527,12 @@ def create_rag_qa_chat_tab():
             return update_file_list(max(1, current_page - 1))
 
         def update_context_source(choice):
+            # Update visibility based on context source choice
             return {
                 existing_file: gr.update(visible=choice == "Existing File"),
-                prev_page_btn: gr.update(visible=choice == "Existing File"),
-                next_page_btn: gr.update(visible=choice == "Existing File"),
-                page_info: gr.update(visible=choice == "Existing File"),
+                prev_page_btn: gr.update(visible=choice == "Search Database"),
+                next_page_btn: gr.update(visible=choice == "Search Database"),
+                page_info: gr.update(visible=choice == "Search Database"),
                 search_query: gr.update(visible=choice == "Search Database"),
                 search_button: gr.update(visible=choice == "Search Database"),
                 search_results: gr.update(visible=choice == "Search Database"),
@@ -352,17 +552,36 @@ def create_rag_qa_chat_tab():
         context_source.change(lambda choice: update_file_list(1) if choice == "Existing File" else (gr.update(), gr.update(), 1),
                               inputs=[context_source], outputs=[existing_file, page_info, file_page])
 
-        def perform_search(query):
+        def perform_search(query, selected_databases, keywords):
             try:
-                results = search_database(query)
+                results = []
+
+                # Iterate over selected database types and perform searches accordingly
+                for database_type in selected_databases:
+                    if database_type == "Media DB":
+                        # FIXME - check for existence of keywords before setting as search field
+                        search_fields = ["title", "content", "keywords"]
+                        results += search_media_db(query, search_fields, keywords, page=1, results_per_page=25)
+                    elif database_type == "RAG Chat":
+                        results += search_rag_chat(query)
+                    elif database_type == "RAG Notes":
+                        results += search_rag_notes(query)
+                    elif database_type == "Character Chat":
+                        results += search_character_chat(query)
+                    elif database_type == "Character Cards":
+                        results += search_character_cards(query)
+
+                # Remove duplicate results if necessary
+                results = list(set(results))
                 return gr.update(choices=results)
             except Exception as e:
                 gr.Error(f"Error performing search: {str(e)}")
                 return gr.update(choices=[])
 
+        # Click Event for the DB Search Button
         search_button.click(
             perform_search,
-            inputs=[search_query],
+            inputs=[search_query, db_choice, keywords_input],
             outputs=[search_results]
         )
 
@@ -384,17 +603,22 @@ Rewritten Question:"""
             logging.info(f"Rephrased question: {rephrased_question}")
             return rephrased_question.strip()
 
-        def rag_qa_chat_wrapper(message, history, context_source, existing_file, search_results, file_upload,
-                                convert_to_text, keywords, api_choice, use_query_rewriting, state_value,
-                                keywords_input, top_k_input, use_re_ranking):
+        # FIXME - RAG DB selection
+        def rag_qa_chat_wrapper(
+                message, history, context_source, existing_file, search_results, file_upload,
+                convert_to_text, keywords, api_choice, use_query_rewriting, state_value,
+                keywords_input, top_k_input, use_re_ranking, db_choices, auto_save_enabled
+        ):
             try:
                 logging.info(f"Starting rag_qa_chat_wrapper with message: {message}")
                 logging.info(f"Context source: {context_source}")
                 logging.info(f"API choice: {api_choice}")
                 logging.info(f"Query rewriting: {'enabled' if use_query_rewriting else 'disabled'}")
+                logging.info(f"Selected DB Choices: {db_choices}")
 
                 # Show loading indicator
-                yield history, "", gr.update(visible=True), state_value
+                yield history, "", gr.update(visible=True), state_value, gr.update(visible=False), gr.update(
+                    visible=False)
 
                 conversation_id = state_value.get("conversation_id")
                 conversation_messages = state_value.get("conversation_messages", [])
@@ -408,12 +632,12 @@ Rewritten Question:"""
                     state_value["conversation_messages"] = conversation_messages
 
                 # Ensure api_choice is a string
-                api_choice = api_choice.value if isinstance(api_choice, gr.components.Dropdown) else api_choice
-                logging.info(f"Resolved API choice: {api_choice}")
+                api_choice_str = api_choice.value if isinstance(api_choice, gr.components.Dropdown) else api_choice
+                logging.info(f"Resolved API choice: {api_choice_str}")
 
                 # Only rephrase the question if it's not the first query and query rewriting is enabled
                 if len(history) > 0 and use_query_rewriting:
-                    rephrased_question = rephrase_question(history, message, api_choice)
+                    rephrased_question = rephrase_question(history, message, api_choice_str)
                     logging.info(f"Original question: {message}")
                     logging.info(f"Rephrased question: {rephrased_question}")
                 else:
@@ -421,18 +645,20 @@ Rewritten Question:"""
                     logging.info(f"Using original question: {message}")
 
                 if context_source == "All Files in the Database":
-                    # Use the enhanced_rag_pipeline to search the entire database
-                    context = enhanced_rag_pipeline(rephrased_question, api_choice, keywords_input, top_k_input,
-                                                    use_re_ranking)
+                    # Use the enhanced_rag_pipeline to search the selected databases
+                    context = enhanced_rag_pipeline(
+                        rephrased_question, api_choice_str, keywords_input, top_k_input, use_re_ranking,
+                        database_types=db_choices  # Pass the list of selected databases
+                    )
                     logging.info(f"Using enhanced_rag_pipeline for database search")
                 elif context_source == "Search Database":
                     context = f"media_id:{search_results.split('(ID: ')[1][:-1]}"
                     logging.info(f"Using search result with context: {context}")
-                else:  # Upload File
+                else:
+                    # Upload File
                     logging.info("Processing uploaded file")
                     if file_upload is None:
                         raise ValueError("No file uploaded")
-
                     # Process the uploaded file
                     file_path = file_upload.name
                     file_name = os.path.basename(file_path)
@@ -445,7 +671,6 @@ Rewritten Question:"""
                         logging.info("Reading file content")
                         with open(file_path, 'r', encoding='utf-8') as f:
                             content = f.read()
-
                     logging.info(f"File content length: {len(content)} characters")
 
                     # Process keywords
@@ -467,18 +692,17 @@ Rewritten Question:"""
                         author='Unknown',
                         ingestion_date=datetime.now().strftime('%Y-%m-%d')
                     )
-
                     logging.info(f"Result from add_media_with_keywords: {result}")
                     if isinstance(result, tuple):
                         media_id, _ = result
                     else:
                         media_id = result
-
                     context = f"media_id:{media_id}"
                     logging.info(f"Context for uploaded file: {context}")
 
                 logging.info("Calling rag_qa_chat function")
-                new_history, response = rag_qa_chat(rephrased_question, history, context, api_choice)
+                new_history, response = rag_qa_chat(rephrased_question, history, context, api_choice_str)
+
                 # Log first 100 chars of response
                 logging.info(f"Response received from rag_qa_chat: {response[:100]}...")
 
@@ -490,7 +714,8 @@ Rewritten Question:"""
                     state_value["conversation_messages"] = conversation_messages
 
                 # Update the state
-                state_value["conversation_messages"] = conversation_messages
+                updated_state = auto_save_conversation(message, response, state_value, auto_save_enabled)
+                updated_state["conversation_messages"] = conversation_messages
 
                 # Safely update history
                 if new_history:
@@ -498,24 +723,43 @@ Rewritten Question:"""
                 else:
                     new_history = [(message, response)]
 
+                # Get the current rating and update display
+                conversation_id = updated_state.get("conversation_id")
+                if conversation_id:
+                    rating = get_conversation_rating(conversation_id)
+                    if rating is not None:
+                        rating_display_update = gr.update(value=f"**Current Rating:** {rating} star(s)", visible=True)
+                        rating_input_update = gr.update(value=str(rating), visible=True)
+                    else:
+                        rating_display_update = gr.update(value="**Current Rating:** Not Rated", visible=True)
+                        rating_input_update = gr.update(value=None, visible=True)
+                else:
+                    rating_display_update = gr.update(value="", visible=False)
+                    rating_input_update = gr.update(value=None, visible=False)
+
                 gr.Info("Response generated successfully")
                 logging.info("rag_qa_chat_wrapper completed successfully")
-                yield new_history, "", gr.update(visible=False), state_value  # Include state_value in outputs
+                yield new_history, "", gr.update(
+                    visible=False), updated_state, rating_display_update, rating_input_update
+
             except ValueError as e:
                 logging.error(f"Input error in rag_qa_chat_wrapper: {str(e)}")
                 gr.Error(f"Input error: {str(e)}")
-                yield history, "", gr.update(visible=False), state_value
+                yield history, "", gr.update(visible=False), state_value, gr.update(visible=False), gr.update(
+                    visible=False)
             except DatabaseError as e:
                 logging.error(f"Database error in rag_qa_chat_wrapper: {str(e)}")
                 gr.Error(f"Database error: {str(e)}")
-                yield history, "", gr.update(visible=False), state_value
+                yield history, "", gr.update(visible=False), state_value, gr.update(visible=False), gr.update(
+                    visible=False)
             except Exception as e:
                 logging.error(f"Unexpected error in rag_qa_chat_wrapper: {e}", exc_info=True)
                 gr.Error("An unexpected error occurred. Please try again later.")
-                yield history, "", gr.update(visible=False), state_value
+                yield history, "", gr.update(visible=False), state_value, gr.update(visible=False), gr.update(
+                    visible=False)
 
         def clear_chat_history():
-            return [], ""
+            return [], "", gr.update(value="", visible=False), gr.update(value=None, visible=False)
 
         submit.click(
             rag_qa_chat_wrapper,
@@ -532,14 +776,17 @@ Rewritten Question:"""
                 use_query_rewriting,
                 state,
                 keywords_input,
-                top_k_input
+                top_k_input,
+                use_re_ranking,
+                db_choice,
+                auto_save_checkbox
             ],
-            outputs=[chatbot, msg, loading_indicator, state],
+            outputs=[chatbot, msg, loading_indicator, state, rating_display, rating_input],
         )
 
         clear_chat.click(
             clear_chat_history,
-            outputs=[chatbot, msg]
+            outputs=[chatbot, msg, rating_display, rating_input]
         )
 
     return (
@@ -560,12 +807,10 @@ Rewritten Question:"""
     )
 
 
-
 def create_rag_qa_notes_management_tab():
     # New Management Tab
     with gr.TabItem("Notes Management", visible=True):
         gr.Markdown("# RAG QA Notes Management")
-
         management_state = gr.State({
             "selected_conversation_id": None,
             "selected_note_id": None,
@@ -574,7 +819,8 @@ def create_rag_qa_notes_management_tab():
         with gr.Row():
             with gr.Column(scale=1):
                 # Search Notes
-                search_notes_input = gr.Textbox(label="Search Notes by Keywords")
+                search_notes_title = gr.Textbox(label="Search Notes by Title")
+                search_notes_by_keyword = gr.Textbox(label="Search Notes by Keywords")
                 search_notes_button = gr.Button("Search Notes")
                 notes_list = gr.Dropdown(label="Notes", choices=[])
 
@@ -583,24 +829,34 @@ def create_rag_qa_notes_management_tab():
                 delete_note_button = gr.Button("Delete Note")
                 note_title_input = gr.Textbox(label="Note Title")
                 note_content_input = gr.TextArea(label="Note Content", lines=20)
-                note_keywords_input = gr.Textbox(label="Note Keywords (comma-separated)")
+                note_keywords_input = gr.Textbox(label="Note Keywords (comma-separated)", value="default_note_keyword")
                 save_note_button = gr.Button("Save Note")
                 create_new_note_button = gr.Button("Create New Note")
                 status_message = gr.HTML()
 
         # Function Definitions
-        def search_notes(keywords):
+        def search_notes(search_notes_title, keywords):
             if keywords:
                 keywords_list = [kw.strip() for kw in keywords.split(',')]
                 notes_data, total_pages, total_count = get_notes_by_keywords(keywords_list)
-                choices = [f"{title} (ID: {note_id})" for note_id, title, content, timestamp in notes_data]
-                return gr.update(choices=choices)
+                choices = [f"Note {note_id} - {title} ({timestamp})" for
+                           note_id, title, content, timestamp, conversation_id in notes_data]
+                return gr.update(choices=choices, label=f"Found {total_count} notes")
+            elif search_notes_title:
+                notes_data, total_pages, total_count = search_notes_titles(search_notes_title)
+                choices = [f"Note {note_id} - {title} ({timestamp})" for
+                           note_id, title, content, timestamp, conversation_id in notes_data]
+                return gr.update(choices=choices, label=f"Found {total_count} notes")
             else:
-                return gr.update(choices=[])
+                # This will now return all notes, ordered by timestamp
+                notes_data, total_pages, total_count = search_notes_titles("")
+                choices = [f"Note {note_id} - {title} ({timestamp})" for
+                           note_id, title, content, timestamp, conversation_id in notes_data]
+                return gr.update(choices=choices, label=f"All notes ({total_count} total)")
 
         search_notes_button.click(
             search_notes,
-            inputs=[search_notes_input],
+            inputs=[search_notes_title, search_notes_by_keyword],
             outputs=[notes_list]
         )
 
@@ -664,7 +920,7 @@ def create_rag_qa_notes_management_tab():
                 # Reset state
                 state_value["selected_note_id"] = None
                 # Update notes list
-                updated_notes = search_notes("")
+                updated_notes = search_notes("", "")
                 return updated_notes, gr.update(value="Note deleted successfully."), state_value
             else:
                 return gr.update(), gr.update(value="No note selected."), state_value
@@ -702,7 +958,20 @@ def create_rag_qa_chat_management_tab():
         with gr.Row():
             with gr.Column(scale=1):
                 # Search Conversations
-                search_conversations_input = gr.Textbox(label="Search Conversations by Keywords")
+                with gr.Group():
+                    gr.Markdown("## Search Conversations")
+                    title_search = gr.Textbox(
+                        label="Search by Title",
+                        placeholder="Enter title to search..."
+                    )
+                    content_search = gr.Textbox(
+                        label="Search in Chat Content",
+                        placeholder="Enter text to search in messages..."
+                    )
+                    keyword_search = gr.Textbox(
+                        label="Filter by Keywords (comma-separated)",
+                        placeholder="keyword1, keyword2, ..."
+                    )
                 search_conversations_button = gr.Button("Search Conversations")
                 conversations_list = gr.Dropdown(label="Conversations", choices=[])
                 new_conversation_button = gr.Button("New Conversation")
@@ -716,26 +985,40 @@ def create_rag_qa_chat_management_tab():
                 status_message = gr.HTML()
 
         # Function Definitions
-        def search_conversations(keywords):
-            if keywords:
-                keywords_list = [kw.strip() for kw in keywords.split(',')]
-                conversations, total_pages, total_count = search_conversations_by_keywords(keywords_list)
-            else:
-                conversations, total_pages, total_count = get_all_conversations()
+        def search_conversations(title_query, content_query, keywords):
+            try:
+                # Parse keywords if provided
+                keywords_list = None
+                if keywords and keywords.strip():
+                    keywords_list = [kw.strip() for kw in keywords.split(',')]
+
+                # Search using existing search_conversations_by_keywords function with all criteria
+                results, total_pages, total_count = search_conversations_by_keywords(
+                    keywords=keywords_list,
+                    title_query=title_query if title_query.strip() else None,
+                    content_query=content_query if content_query.strip() else None
+                )
 
-            # Build choices as list of titles (ensure uniqueness)
-            choices = []
-            mapping = {}
-            for conversation_id, title in conversations:
-                display_title = f"{title} (ID: {conversation_id[:8]})"
-                choices.append(display_title)
-                mapping[display_title] = conversation_id
+                # Build choices as list of titles (ensure uniqueness)
+                choices = []
+                mapping = {}
+                for conv in results:
+                    conversation_id = conv['conversation_id']
+                    title = conv['title']
+                    display_title = f"{title} (ID: {conversation_id[:8]})"
+                    choices.append(display_title)
+                    mapping[display_title] = conversation_id
 
-            return gr.update(choices=choices), mapping
+                return gr.update(choices=choices), mapping
 
+            except Exception as e:
+                logging.error(f"Error in search_conversations: {str(e)}")
+                return gr.update(choices=[]), {}
+
+        # Update the search button click event
         search_conversations_button.click(
             search_conversations,
-            inputs=[search_conversations_input],
+            inputs=[title_search, content_search, keyword_search],
             outputs=[conversations_list, conversation_mapping]
         )
 
@@ -892,19 +1175,18 @@ def create_rag_qa_chat_management_tab():
             ]
         )
 
-        def delete_messages_in_conversation(conversation_id):
-            """Helper function to delete all messages in a conversation."""
+        def delete_messages_in_conversation_wrapper(conversation_id):
+            """Wrapper function to delete all messages in a conversation."""
             try:
-                execute_query("DELETE FROM rag_qa_chats WHERE conversation_id = ?", (conversation_id,))
+                delete_messages_in_conversation(conversation_id)
                 logging.info(f"Messages in conversation '{conversation_id}' deleted successfully.")
             except Exception as e:
                 logging.error(f"Error deleting messages in conversation '{conversation_id}': {e}")
                 raise
 
-        def get_conversation_title(conversation_id):
+        def get_conversation_title_wrapper(conversation_id):
             """Helper function to get the conversation title."""
-            query = "SELECT title FROM conversation_metadata WHERE conversation_id = ?"
-            result = execute_query(query, (conversation_id,))
+            result = get_conversation_title(conversation_id)
             if result:
                 return result[0][0]
             else:
@@ -1034,19 +1316,6 @@ def create_export_data_tab():
         )
 
 
-
-
-def update_conversation_title(conversation_id, new_title):
-    """Update the title of a conversation."""
-    try:
-        query = "UPDATE conversation_metadata SET title = ? WHERE conversation_id = ?"
-        execute_query(query, (new_title, conversation_id))
-        logging.info(f"Conversation '{conversation_id}' title updated to '{new_title}'")
-    except Exception as e:
-        logging.error(f"Error updating conversation title: {e}")
-        raise
-
-
 def convert_file_to_text(file_path):
     """Convert various file types to plain text."""
     file_extension = os.path.splitext(file_path)[1].lower()

App_Function_Libraries/Gradio_UI/Re_summarize_tab.py

@@ -10,19 +10,33 @@ import gradio as gr
 #
 # Local Imports
 from App_Function_Libraries.Chunk_Lib import improved_chunking_process
-from App_Function_Libraries.DB.DB_Manager import update_media_content, load_preset_prompts
+from App_Function_Libraries.DB.DB_Manager import update_media_content, list_prompts
 from App_Function_Libraries.Gradio_UI.Chat_ui import update_user_prompt
 from App_Function_Libraries.Gradio_UI.Gradio_Shared import fetch_item_details, fetch_items_by_keyword, \
     fetch_items_by_content, fetch_items_by_title_or_url
 from App_Function_Libraries.Summarization.Summarization_General_Lib import summarize_chunk
-from App_Function_Libraries.Utils.Utils import load_comprehensive_config
-#
+from App_Function_Libraries.Utils.Utils import load_comprehensive_config, default_api_endpoint, global_api_endpoints, \
+    format_api_name
 #
 ######################################################################################################################
 #
 # Functions:
 
 def create_resummary_tab():
+    try:
+        default_value = None
+        if default_api_endpoint:
+            if default_api_endpoint in global_api_endpoints:
+                default_value = format_api_name(default_api_endpoint)
+            else:
+                logging.warning(f"Default API endpoint '{default_api_endpoint}' not found in global_api_endpoints")
+    except Exception as e:
+        logging.error(f"Error setting default API endpoint: {str(e)}")
+        default_value = None
+
+    # Get initial prompts for first page
+    initial_prompts, total_pages, current_page = list_prompts(page=1, per_page=20)
+
     with gr.TabItem("Re-Summarize", visible=True):
         gr.Markdown("# Re-Summarize Existing Content")
         with gr.Row():
@@ -36,9 +50,10 @@ def create_resummary_tab():
 
                 with gr.Row():
                     api_name_input = gr.Dropdown(
-                        choices=["Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral", "OpenRouter",
-                             "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM","ollama", "HuggingFace"],
-                        value="Local-LLM", label="API Name")
+                        choices=["None"] + [format_api_name(api) for api in global_api_endpoints],
+                        value=default_value,
+                        label="API for Summarization/Analysis (Optional)"
+                    )
                     api_key_input = gr.Textbox(label="API Key", placeholder="Enter your API key here", type="password")
 
                 chunking_options_checkbox = gr.Checkbox(label="Use Chunking", value=False)
@@ -55,9 +70,17 @@ def create_resummary_tab():
                     preset_prompt_checkbox = gr.Checkbox(label="Use a pre-set Prompt",
                                                      value=False,
                                                      visible=True)
+
+                # Add pagination controls for preset prompts
+                with gr.Row(visible=False) as preset_prompt_controls:
+                    prev_page = gr.Button("Previous")
+                    current_page_text = gr.Text(f"Page {current_page} of {total_pages}")
+                    next_page = gr.Button("Next")
+                    current_page_state = gr.State(value=1)
+
                 with gr.Row():
                     preset_prompt = gr.Dropdown(label="Select Preset Prompt",
-                                                choices=load_preset_prompts(),
+                                                choices=initial_prompts,
                                                 visible=False)
                 with gr.Row():
                     custom_prompt_input = gr.Textbox(label="Custom Prompt",
@@ -86,6 +109,15 @@ def create_resummary_tab():
                                                      lines=3,
                                                      visible=False)
 
+                def update_prompt_page(direction, current_page_val):
+                    new_page = max(1, min(total_pages, current_page_val + direction))
+                    prompts, _, _ = list_prompts(page=new_page, per_page=10)
+                    return (
+                        gr.update(choices=prompts),
+                        gr.update(value=f"Page {new_page} of {total_pages}"),
+                        new_page
+                    )
+
                 def update_prompts(preset_name):
                     prompts = update_user_prompt(preset_name)
                     return (
@@ -93,6 +125,19 @@ def create_resummary_tab():
                         gr.update(value=prompts["system_prompt"], visible=True)
                     )
 
+                # Connect pagination buttons
+                prev_page.click(
+                    lambda x: update_prompt_page(-1, x),
+                    inputs=[current_page_state],
+                    outputs=[preset_prompt, current_page_text, current_page_state]
+                )
+
+                next_page.click(
+                    lambda x: update_prompt_page(1, x),
+                    inputs=[current_page_state],
+                    outputs=[preset_prompt, current_page_text, current_page_state]
+                )
+
                 preset_prompt.change(
                     update_prompts,
                     inputs=preset_prompt,
@@ -109,9 +154,9 @@ def create_resummary_tab():
             outputs=[custom_prompt_input, system_prompt_input]
         )
         preset_prompt_checkbox.change(
-            fn=lambda x: gr.update(visible=x),
+            fn=lambda x: (gr.update(visible=x), gr.update(visible=x)),
             inputs=[preset_prompt_checkbox],
-            outputs=[preset_prompt]
+            outputs=[preset_prompt, preset_prompt_controls]
         )
 
     # Connect the UI elements
@@ -140,7 +185,12 @@ def create_resummary_tab():
         outputs=result_output
     )
 
-    return search_query_input, search_type_input, search_button, items_output, item_mapping, api_name_input, api_key_input, chunking_options_checkbox, chunking_options_box, chunk_method, max_chunk_size, chunk_overlap, custom_prompt_checkbox, custom_prompt_input, resummarize_button, result_output
+    return (
+        search_query_input, search_type_input, search_button, items_output,
+        item_mapping, api_name_input, api_key_input, chunking_options_checkbox,
+        chunking_options_box, chunk_method, max_chunk_size, chunk_overlap,
+        custom_prompt_checkbox, custom_prompt_input, resummarize_button, result_output
+    )
 
 
 def update_resummarize_dropdown(search_query, search_type):

App_Function_Libraries/Gradio_UI/Search_Tab.py

@@ -11,8 +11,8 @@ import gradio as gr
 #
 # Local Imports
 from App_Function_Libraries.DB.DB_Manager import view_database, search_and_display_items, get_all_document_versions, \
-    fetch_item_details_single, fetch_paginated_data, fetch_item_details, get_latest_transcription
-from App_Function_Libraries.DB.SQLite_DB import search_prompts, get_document_version
+    fetch_item_details_single, fetch_paginated_data, fetch_item_details, get_latest_transcription, search_prompts, \
+    get_document_version
 from App_Function_Libraries.Gradio_UI.Gradio_Shared import update_dropdown, update_detailed_view
 from App_Function_Libraries.Utils.Utils import get_database_path, format_text_with_line_breaks
 #
@@ -80,8 +80,8 @@ def format_as_html(content, title):
     """
 
 def create_search_tab():
-    with gr.TabItem("Search / Detailed View", visible=True):
-        gr.Markdown("# Search across all ingested items in the Database")
+    with gr.TabItem("Media DB Search / Detailed View", visible=True):
+        gr.Markdown("# Search across all ingested items in the Media Database")
         with gr.Row():
             with gr.Column(scale=1):
                 gr.Markdown("by Title / URL / Keyword / or Content via SQLite Full-Text-Search")
@@ -150,8 +150,8 @@ def display_search_results(query):
 
 
 def create_search_summaries_tab():
-    with gr.TabItem("Search/View Title+Summary", visible=True):
-        gr.Markdown("# Search across all ingested items in the Database and review their summaries")
+    with gr.TabItem("Media DB Search/View Title+Summary", visible=True):
+        gr.Markdown("# Search across all ingested items in the Media Database and review their summaries")
         gr.Markdown("Search by Title / URL / Keyword / or Content via SQLite Full-Text-Search")
         with gr.Row():
             with gr.Column():

App_Function_Libraries/Gradio_UI/Semantic_Scholar_tab.py

@@ -0,0 +1,184 @@
+# Sematnic_Scholar_tab.py
+# Description: contains the code to create the Semantic Scholar tab in the Gradio UI.
+#
+# Imports
+#
+# External Libraries
+import gradio as gr
+#
+# Internal Libraries
+from App_Function_Libraries.Third_Party.Semantic_Scholar import search_and_display, FIELDS_OF_STUDY, PUBLICATION_TYPES
+
+
+#
+######################################################################################################################
+# Functions
+def create_semantic_scholar_tab():
+    """Create the Semantic Scholar tab for the Gradio UI"""
+    with gr.Tab("Semantic Scholar Search"):
+        with gr.Row():
+            with gr.Column(scale=2):
+                gr.Markdown("""
+                    ## Semantic Scholar Paper Search
+
+                    This interface allows you to search for academic papers using the Semantic Scholar API with advanced filtering options:
+
+                    ### Search Options
+                    - **Keywords**: Search across titles, abstracts, and other paper content
+                    - **Year Range**: Filter papers by publication year (e.g., "2020-2023" or "2020")
+                    - **Venue**: Filter by publication venue (journal or conference)
+                    - **Minimum Citations**: Filter papers by minimum citation count
+                    - **Fields of Study**: Filter papers by academic field
+                    - **Publication Types**: Filter by type of publication
+                    - **Open Access**: Option to show only papers with free PDF access
+
+                    ### Results Include
+                    - Paper title
+                    - Author list
+                    - Publication year and venue
+                    - Citation count
+                    - Publication types
+                    - Abstract
+                    - Links to PDF (when available) and Semantic Scholar page
+                    """)
+            with gr.Column(scale=2):
+                gr.Markdown("""
+                    ### Pagination
+                    - 10 results per page
+                    - Navigate through results using Previous/Next buttons
+                    - Current page number and total results displayed
+
+                    ### Usage Tips
+                    - Combine multiple filters for more specific results
+                    - Use specific terms for more focused results
+                    - Try different combinations of filters if you don't find what you're looking for
+                    """)
+        with gr.Row():
+            with gr.Column(scale=2):
+                search_input = gr.Textbox(
+                    label="Search Query",
+                    placeholder="Enter keywords to search for papers...",
+                    lines=1
+                )
+
+                # Advanced search options
+                with gr.Row():
+                    year_range = gr.Textbox(
+                        label="Year Range",
+                        placeholder="e.g., 2020-2023 or 2020",
+                        lines=1
+                    )
+                    venue = gr.Textbox(
+                        label="Venue",
+                        placeholder="e.g., Nature, Science",
+                        lines=1
+                    )
+                    min_citations = gr.Number(
+                        label="Minimum Citations",
+                        value=0,
+                        minimum=0,
+                        step=1
+                    )
+
+                with gr.Row():
+                    fields_of_study = gr.Dropdown(
+                        choices=FIELDS_OF_STUDY,
+                        label="Fields of Study",
+                        multiselect=True,
+                        value=[]
+                    )
+                    publication_types = gr.Dropdown(
+                        choices=PUBLICATION_TYPES,
+                        label="Publication Types",
+                        multiselect=True,
+                        value=[]
+                    )
+
+                open_access_only = gr.Checkbox(
+                    label="Open Access Only",
+                    value=False
+                )
+
+            with gr.Column(scale=1):
+                search_button = gr.Button("Search", variant="primary")
+
+                # Pagination controls
+                with gr.Row():
+                    prev_button = gr.Button("← Previous")
+                    current_page = gr.Number(value=0, label="Page", minimum=0, step=1)
+                    max_page = gr.Number(value=0, label="Max Page", visible=False)
+                    next_button = gr.Button("Next →")
+
+                total_results = gr.Textbox(
+                    label="Total Results",
+                    value="0",
+                    interactive=False
+                )
+
+        output_text = gr.Markdown(
+            label="Results",
+            value="Use the search options above to find papers."
+        )
+
+        def update_page(direction, current, maximum):
+            new_page = current + direction
+            if new_page < 0:
+                return 0
+            if new_page > maximum:
+                return maximum
+            return new_page
+
+        # Handle search and pagination
+        def search_from_button(query, fields_of_study, publication_types, year_range, venue, min_citations,
+                               open_access_only):
+            """Wrapper to always search from page 0 when search button is clicked"""
+            return search_and_display(
+                query=query,
+                page=0,  # Force page 0 for new searches
+                fields_of_study=fields_of_study,
+                publication_types=publication_types,
+                year_range=year_range,
+                venue=venue,
+                min_citations=min_citations,
+                open_access_only=open_access_only
+            )
+        normal_search = search_and_display
+
+        search_button.click(
+            fn=search_from_button,
+            inputs=[
+                search_input, fields_of_study, publication_types,
+                year_range, venue, min_citations, open_access_only
+            ],
+            outputs=[output_text, current_page, max_page, total_results]
+        )
+
+        prev_button.click(
+            fn=lambda curr, max_p: update_page(-1, curr, max_p),
+            inputs=[current_page, max_page],
+            outputs=current_page
+        ).then(
+            fn=normal_search,
+            inputs=[
+                search_input, current_page, fields_of_study, publication_types,
+                year_range, venue, min_citations, open_access_only
+            ],
+            outputs=[output_text, current_page, max_page, total_results]
+        )
+
+        next_button.click(
+            fn=lambda curr, max_p: update_page(1, curr, max_p),
+            inputs=[current_page, max_page],
+            outputs=current_page
+        ).then(
+            fn=normal_search,
+            inputs=[
+                search_input, current_page, fields_of_study, publication_types,
+                year_range, venue, min_citations, open_access_only
+            ],
+            outputs=[output_text, current_page, max_page, total_results]
+        )
+
+#
+# End of Semantic_Scholar_tab.py
+######################################################################################################################

App_Function_Libraries/Gradio_UI/Video_transcription_tab.py

@@ -6,22 +6,23 @@ import json
 import logging
 import os
 from datetime import datetime
-from typing import Dict, Any
-
 #
 # External Imports
 import gradio as gr
 import yt_dlp
+
+from App_Function_Libraries.Chunk_Lib import improved_chunking_process
 #
 # Local Imports
-from App_Function_Libraries.DB.DB_Manager import load_preset_prompts, add_media_to_database, \
-    check_media_and_whisper_model, check_existing_media, update_media_content_with_version
+from App_Function_Libraries.DB.DB_Manager import add_media_to_database, \
+    check_media_and_whisper_model, check_existing_media, update_media_content_with_version, list_prompts
 from App_Function_Libraries.Gradio_UI.Gradio_Shared import whisper_models, update_user_prompt
 from App_Function_Libraries.Gradio_UI.Gradio_Shared import error_handler
 from App_Function_Libraries.Summarization.Summarization_General_Lib import perform_transcription, perform_summarization, \
     save_transcription_and_summary
 from App_Function_Libraries.Utils.Utils import convert_to_seconds, safe_read_file, format_transcription, \
-    create_download_directory, generate_unique_identifier, extract_text_from_segments
+    create_download_directory, generate_unique_identifier, extract_text_from_segments, default_api_endpoint, \
+    global_api_endpoints, format_api_name
 from App_Function_Libraries.Video_DL_Ingestion_Lib import parse_and_expand_urls, extract_metadata, download_video
 from App_Function_Libraries.Benchmarks_Evaluations.ms_g_eval import run_geval
 # Import metrics logging
@@ -32,6 +33,16 @@ from App_Function_Libraries.Metrics.metrics_logger import log_counter, log_histo
 # Functions:
 
 def create_video_transcription_tab():
+    try:
+        default_value = None
+        if default_api_endpoint:
+            if default_api_endpoint in global_api_endpoints:
+                default_value = format_api_name(default_api_endpoint)
+            else:
+                logging.warning(f"Default API endpoint '{default_api_endpoint}' not found in global_api_endpoints")
+    except Exception as e:
+        logging.error(f"Error setting default API endpoint: {str(e)}")
+        default_value = None
     with gr.TabItem("Video Transcription + Summarization", visible=True):
         gr.Markdown("# Transcribe & Summarize Videos from URLs")
         with gr.Row():
@@ -56,15 +67,20 @@ def create_video_transcription_tab():
                     preset_prompt_checkbox = gr.Checkbox(label="Use a pre-set Prompt",
                                                          value=False,
                                                          visible=True)
+
+                # Initialize state variables for pagination
+                current_page_state = gr.State(value=1)
+                total_pages_state = gr.State(value=1)
+
                 with gr.Row():
+                    # Add pagination controls
                     preset_prompt = gr.Dropdown(label="Select Preset Prompt",
-                                                choices=load_preset_prompts(),
+                                                choices=[],
                                                 visible=False)
                 with gr.Row():
-                    custom_prompt_input = gr.Textbox(label="Custom Prompt",
-                                                     placeholder="Enter custom prompt here",
-                                                     lines=3,
-                                                     visible=False)
+                    prev_page_button = gr.Button("Previous Page", visible=False)
+                    page_display = gr.Markdown("Page 1 of X", visible=False)
+                    next_page_button = gr.Button("Next Page", visible=False)
                 with gr.Row():
                     system_prompt_input = gr.Textbox(label="System Prompt",
                                                      value="""<s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
@@ -87,22 +103,75 @@ def create_video_transcription_tab():
                                                      lines=3,
                                                      visible=False,
                                                      interactive=True)
+                with gr.Row():
+                    custom_prompt_input = gr.Textbox(label="Custom Prompt",
+                                                     placeholder="Enter custom prompt here",
+                                                     lines=3,
+                                                     visible=False)
+
                 custom_prompt_checkbox.change(
-                    fn=lambda x: (gr.update(visible=x), gr.update(visible=x)),
+                    fn=lambda x: (gr.update(visible=x, interactive=x), gr.update(visible=x, interactive=x)),
                     inputs=[custom_prompt_checkbox],
                     outputs=[custom_prompt_input, system_prompt_input]
                 )
+
+                def on_preset_prompt_checkbox_change(is_checked):
+                    if is_checked:
+                        prompts, total_pages, current_page = list_prompts(page=1, per_page=20)
+                        page_display_text = f"Page {current_page} of {total_pages}"
+                        return (
+                            gr.update(visible=True, interactive=True, choices=prompts),  # preset_prompt
+                            gr.update(visible=True),  # prev_page_button
+                            gr.update(visible=True),  # next_page_button
+                            gr.update(value=page_display_text, visible=True),  # page_display
+                            current_page,  # current_page_state
+                            total_pages  # total_pages_state
+                        )
+                    else:
+                        return (
+                            gr.update(visible=False, interactive=False),  # preset_prompt
+                            gr.update(visible=False),  # prev_page_button
+                            gr.update(visible=False),  # next_page_button
+                            gr.update(visible=False),  # page_display
+                            1,  # current_page_state
+                            1   # total_pages_state
+                        )
+
                 preset_prompt_checkbox.change(
-                    fn=lambda x: gr.update(visible=x),
+                    fn=on_preset_prompt_checkbox_change,
                     inputs=[preset_prompt_checkbox],
-                    outputs=[preset_prompt]
+                    outputs=[preset_prompt, prev_page_button, next_page_button, page_display, current_page_state, total_pages_state]
+                )
+
+                def on_prev_page_click(current_page, total_pages):
+                    new_page = max(current_page - 1, 1)
+                    prompts, total_pages, current_page = list_prompts(page=new_page, per_page=20)
+                    page_display_text = f"Page {current_page} of {total_pages}"
+                    return gr.update(choices=prompts), gr.update(value=page_display_text), current_page
+
+                prev_page_button.click(
+                    fn=on_prev_page_click,
+                    inputs=[current_page_state, total_pages_state],
+                    outputs=[preset_prompt, page_display, current_page_state]
+                )
+
+                def on_next_page_click(current_page, total_pages):
+                    new_page = min(current_page + 1, total_pages)
+                    prompts, total_pages, current_page = list_prompts(page=new_page, per_page=20)
+                    page_display_text = f"Page {current_page} of {total_pages}"
+                    return gr.update(choices=prompts), gr.update(value=page_display_text), current_page
+
+                next_page_button.click(
+                    fn=on_next_page_click,
+                    inputs=[current_page_state, total_pages_state],
+                    outputs=[preset_prompt, page_display, current_page_state]
                 )
 
                 def update_prompts(preset_name):
                     prompts = update_user_prompt(preset_name)
                     return (
-                        gr.update(value=prompts["user_prompt"], visible=True),
-                        gr.update(value=prompts["system_prompt"], visible=True)
+                        gr.update(value=prompts["user_prompt"], visible=True, interactive=True),
+                        gr.update(value=prompts["system_prompt"], visible=True, interactive=True)
                     )
 
                 preset_prompt.change(
@@ -111,11 +180,12 @@ def create_video_transcription_tab():
                     outputs=[custom_prompt_input, system_prompt_input]
                 )
 
+                # Refactored API selection dropdown
                 api_name_input = gr.Dropdown(
-                    choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral",
-                             "OpenRouter",
-                             "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace", "Custom-OpenAI-API"],
-                    value=None, label="API Name (Mandatory)")
+                    choices=["None"] + [format_api_name(api) for api in global_api_endpoints],
+                    value=default_value,
+                    label="API for Summarization/Analysis (Optional)"
+                )
                 api_key_input = gr.Textbox(label="API Key (Optional - Set in Config.txt)", placeholder="Enter your API key here",
                                            type="password")
                 keywords_input = gr.Textbox(label="Keywords", placeholder="Enter keywords here (comma-separated)",
@@ -198,8 +268,7 @@ def create_video_transcription_tab():
                                                    progress: gr.Progress = gr.Progress()) -> tuple:
                 try:
                     # Start overall processing timer
-                    proc_start_time = datetime.utcnow()
-                    # FIXME - summarize_recursively is not being used...
+                    proc_start_time = datetime.now()
                     logging.info("Entering process_videos_with_error_handling")
                     logging.info(f"Received inputs: {inputs}")
 
@@ -251,8 +320,7 @@ def create_video_transcription_tab():
                     all_summaries = ""
 
                     # Start timing
-                    # FIXME - utcnow() is deprecated and scheduled for removal in a future version. Use timezone-aware objects to represent datetimes in UTC: datetime.datetime.now(datetime.UTC).
-                    start_proc = datetime.utcnow()
+                    start_proc = datetime.now()
 
                     for i in range(0, len(all_inputs), batch_size):
                         batch = all_inputs[i:i + batch_size]
@@ -260,7 +328,7 @@ def create_video_transcription_tab():
 
                         for input_item in batch:
                             # Start individual video processing timer
-                            video_start_time = datetime.utcnow()
+                            video_start_time = datetime.now()
                             try:
                                 start_seconds = convert_to_seconds(start_time)
                                 end_seconds = convert_to_seconds(end_time) if end_time else None
@@ -313,7 +381,7 @@ def create_video_transcription_tab():
                                     input_item, 2, whisper_model,
                                     custom_prompt,
                                     start_seconds, api_name, api_key,
-                                    vad_use, False, False, False, 0.01, None, keywords, None, diarize,
+                                    vad_use, False, False, summarize_recursively, 0.01, None, keywords, None, diarize,
                                     end_time=end_seconds,
                                     include_timestamps=timestamp_option,
                                     metadata=video_metadata,
@@ -365,7 +433,7 @@ def create_video_transcription_tab():
                                         )
 
                                         # Calculate processing time
-                                        video_end_time = datetime.utcnow()
+                                        video_end_time = datetime.now()
                                         processing_time = (video_end_time - video_start_time).total_seconds()
                                         log_histogram(
                                             metric_name="video_processing_time_seconds",
@@ -473,7 +541,7 @@ def create_video_transcription_tab():
                     total_inputs = len(all_inputs)
 
                     # End overall processing timer
-                    proc_end_time = datetime.utcnow()
+                    proc_end_time = datetime.now()
                     total_processing_time = (proc_end_time - proc_start_time).total_seconds()
                     log_histogram(
                         metric_name="total_processing_time_seconds",
@@ -702,8 +770,9 @@ def create_video_transcription_tab():
 
                     # Perform transcription
                     logging.info("process_url_with_metadata: Starting transcription...")
+                    logging.info(f"process_url_with_metadata: overwrite existing?: {overwrite_existing}")
                     audio_file_path, segments = perform_transcription(video_file_path, offset, whisper_model,
-                                                                      vad_filter, diarize)
+                                                                      vad_filter, diarize, overwrite_existing)
 
                     if audio_file_path is None or segments is None:
                         logging.error("process_url_with_metadata: Transcription failed or segments not available.")
@@ -771,7 +840,54 @@ def create_video_transcription_tab():
                         # API key resolution handled at base of function if none provided
                         api_key = api_key if api_key else None
                         logging.info(f"process_url_with_metadata: Starting summarization with {api_name}...")
-                        summary_text = perform_summarization(api_name, full_text_with_metadata, custom_prompt, api_key)
+
+                        # Perform Chunking if enabled
+                        # FIXME - Setup a proper prompt for Recursive Summarization
+                        if use_chunking:
+                            logging.info("process_url_with_metadata: Chunking enabled. Starting chunking...")
+                            chunked_texts = improved_chunking_process(full_text_with_metadata, chunk_options)
+
+                            if chunked_texts is None:
+                                logging.warning("Chunking failed, falling back to full text summarization")
+                                summary_text = perform_summarization(api_name, full_text_with_metadata, custom_prompt,
+                                                                     api_key)
+                            else:
+                                logging.debug(
+                                    f"process_url_with_metadata: Chunking completed. Processing {len(chunked_texts)} chunks...")
+                                summaries = []
+
+                                if rolling_summarization:
+                                    # Perform recursive summarization on each chunk
+                                    for chunk in chunked_texts:
+                                        chunk_summary = perform_summarization(api_name, chunk['text'], custom_prompt,
+                                                                              api_key)
+                                        if chunk_summary:
+                                            summaries.append(
+                                                f"Chunk {chunk['metadata']['chunk_index']}/{chunk['metadata']['total_chunks']}: {chunk_summary}")
+                                            summary_text = "\n\n".join(summaries)
+                                        else:
+                                            logging.error("All chunk summarizations failed")
+                                            summary_text = None
+
+                                for chunk in chunked_texts:
+                                    # Perform Non-recursive summarization on each chunk
+                                    chunk_summary = perform_summarization(api_name, chunk['text'], custom_prompt,
+                                                                          api_key)
+                                    if chunk_summary:
+                                        summaries.append(
+                                            f"Chunk {chunk['metadata']['chunk_index']}/{chunk['metadata']['total_chunks']}: {chunk_summary}")
+
+                                    if summaries:
+                                        summary_text = "\n\n".join(summaries)
+                                        logging.info(f"Successfully summarized {len(summaries)} chunks")
+                                    else:
+                                        logging.error("All chunk summarizations failed")
+                                        summary_text = None
+                        else:
+                            # Regular summarization without chunking
+                            summary_text = perform_summarization(api_name, full_text_with_metadata, custom_prompt,
+                                                                 api_key) if api_name else None
+
                         if summary_text is None:
                             logging.error("Summarization failed.")
                             return None, None, None, None, None, None
@@ -859,3 +975,7 @@ def create_video_transcription_tab():
                 ],
                 outputs=[progress_output, error_output, results_output, download_transcription, download_summary, confabulation_output]
             )
+
+#
+# End of Video_transcription_tab.py
+#######################################################################################################################

App_Function_Libraries/Gradio_UI/View_DB_Items_tab.py

@@ -3,131 +3,26 @@
 #
 # Imports
 import html
+import logging
+
 #
 # External Imports
 import gradio as gr
 #
 # Local Imports
 from App_Function_Libraries.DB.DB_Manager import view_database, get_all_document_versions, \
-    fetch_paginated_data, fetch_item_details, get_latest_transcription, list_prompts, fetch_prompt_details, \
-    load_preset_prompts
-from App_Function_Libraries.DB.SQLite_DB import get_document_version
+    fetch_paginated_data, fetch_item_details, get_latest_transcription, list_prompts, fetch_prompt_details
+from App_Function_Libraries.DB.RAG_QA_Chat_DB import get_keywords_for_note, search_conversations_by_keywords, \
+    get_notes_by_keywords, get_keywords_for_conversation, get_db_connection, get_all_conversations, load_chat_history, \
+    get_notes
+from App_Function_Libraries.DB.SQLite_DB import get_document_version, fetch_items_by_keyword, fetch_all_keywords
+
+
 #
 ####################################################################################################
 #
 # Functions
 
-def create_prompt_view_tab():
-    with gr.TabItem("View Prompt Database", visible=True):
-        gr.Markdown("# View Prompt Database Entries")
-        with gr.Row():
-            with gr.Column():
-                entries_per_page = gr.Dropdown(choices=[10, 20, 50, 100], label="Entries per Page", value=10)
-                page_number = gr.Number(value=1, label="Page Number", precision=0)
-                view_button = gr.Button("View Page")
-                next_page_button = gr.Button("Next Page")
-                previous_page_button = gr.Button("Previous Page")
-                pagination_info = gr.Textbox(label="Pagination Info", interactive=False)
-                prompt_selector = gr.Dropdown(label="Select Prompt to View", choices=[])
-            with gr.Column():
-                results_table = gr.HTML()
-                selected_prompt_display = gr.HTML()
-
-        def view_database(page, entries_per_page):
-            try:
-                prompts, total_pages, current_page = list_prompts(page, entries_per_page)
-
-                table_html = "<table style='width:100%; border-collapse: collapse;'>"
-                table_html += "<tr><th style='border: 1px solid black; padding: 8px;'>Title</th><th style='border: 1px solid black; padding: 8px;'>Author</th></tr>"
-                prompt_choices = []
-                for prompt_name in prompts:
-                    details = fetch_prompt_details(prompt_name)
-                    if details:
-                        title, _, _, _, _, _ = details
-                        author = "Unknown"  # Assuming author is not stored in the current schema
-                        table_html += f"<tr><td style='border: 1px solid black; padding: 8px;'>{html.escape(title)}</td><td style='border: 1px solid black; padding: 8px;'>{html.escape(author)}</td></tr>"
-                        prompt_choices.append((title, title))  # Using title as both label and value
-                table_html += "</table>"
-
-                total_prompts = len(load_preset_prompts())  # This might be inefficient for large datasets
-                pagination = f"Page {current_page} of {total_pages} (Total prompts: {total_prompts})"
-
-                return table_html, pagination, total_pages, prompt_choices
-            except Exception as e:
-                return f"<p>Error fetching prompts: {e}</p>", "Error", 0, []
-
-        def update_page(page, entries_per_page):
-            results, pagination, total_pages, prompt_choices = view_database(page, entries_per_page)
-            next_disabled = page >= total_pages
-            prev_disabled = page <= 1
-            return results, pagination, page, gr.update(interactive=not next_disabled), gr.update(
-                interactive=not prev_disabled), gr.update(choices=prompt_choices)
-
-        def go_to_next_page(current_page, entries_per_page):
-            next_page = current_page + 1
-            return update_page(next_page, entries_per_page)
-
-        def go_to_previous_page(current_page, entries_per_page):
-            previous_page = max(1, current_page - 1)
-            return update_page(previous_page, entries_per_page)
-
-        def display_selected_prompt(prompt_name):
-            details = fetch_prompt_details(prompt_name)
-            if details:
-                title, author, description, system_prompt, user_prompt, keywords = details
-                # Handle None values by converting them to empty strings
-                description = description or ""
-                system_prompt = system_prompt or ""
-                user_prompt = user_prompt or ""
-                author = author or "Unknown"
-                keywords = keywords or ""
-
-                html_content = f"""
-                <div style="border: 1px solid #ddd; padding: 10px; margin-bottom: 20px;">
-                    <h3>{html.escape(title)}</h3> <h4>by {html.escape(author)}</h4>
-                    <p><strong>Description:</strong> {html.escape(description)}</p>
-                    <div style="margin-top: 10px;">
-                        <strong>System Prompt:</strong>
-                        <pre style="white-space: pre-wrap; word-wrap: break-word;">{html.escape(system_prompt)}</pre>
-                    </div>
-                    <div style="margin-top: 10px;">
-                        <strong>User Prompt:</strong>
-                        <pre style="white-space: pre-wrap; word-wrap: break-word;">{html.escape(user_prompt)}</pre>
-                    </div>
-                    <p><strong>Keywords:</strong> {html.escape(keywords)}</p>
-                </div>
-                """
-                return html_content
-            else:
-                return "<p>Prompt not found.</p>"
-
-        view_button.click(
-            fn=update_page,
-            inputs=[page_number, entries_per_page],
-            outputs=[results_table, pagination_info, page_number, next_page_button, previous_page_button,
-                     prompt_selector]
-        )
-
-        next_page_button.click(
-            fn=go_to_next_page,
-            inputs=[page_number, entries_per_page],
-            outputs=[results_table, pagination_info, page_number, next_page_button, previous_page_button,
-                     prompt_selector]
-        )
-
-        previous_page_button.click(
-            fn=go_to_previous_page,
-            inputs=[page_number, entries_per_page],
-            outputs=[results_table, pagination_info, page_number, next_page_button, previous_page_button,
-                     prompt_selector]
-        )
-
-        prompt_selector.change(
-            fn=display_selected_prompt,
-            inputs=[prompt_selector],
-            outputs=[selected_prompt_display]
-        )
-
 def format_as_html(content, title):
     escaped_content = html.escape(content)
     formatted_content = escaped_content.replace('\n', '<br>')
@@ -149,9 +44,9 @@ def extract_prompt_and_summary(content: str):
     return prompt, summary
 
 
-def create_view_all_with_versions_tab():
-    with gr.TabItem("View All Items", visible=True):
-        gr.Markdown("# View All Database Entries with Version Selection")
+def create_view_all_mediadb_with_versions_tab():
+    with gr.TabItem("View All MediaDB Items", visible=True):
+        gr.Markdown("# View All Media Database Entries with Version Selection")
         with gr.Row():
             with gr.Column(scale=1):
                 entries_per_page = gr.Dropdown(choices=[10, 20, 50, 100], label="Entries per Page", value=10)
@@ -280,9 +175,143 @@ def create_view_all_with_versions_tab():
         )
 
 
-def create_viewing_tab():
-    with gr.TabItem("View Database Entries", visible=True):
-        gr.Markdown("# View Database Entries")
+def create_mediadb_keyword_search_tab():
+    with gr.TabItem("Search MediaDB by Keyword", visible=True):
+        gr.Markdown("# List Media Database Items by Keyword")
+
+        with gr.Row():
+            with gr.Column(scale=1):
+                # Keyword selection dropdown - initialize with empty list, will be populated on load
+                keyword_dropdown = gr.Dropdown(
+                    label="Select Keyword",
+                    choices=fetch_all_keywords(),  # Initialize with keywords on creation
+                    value=None
+                )
+                entries_per_page = gr.Dropdown(
+                    choices=[10, 20, 50, 100],
+                    label="Entries per Page",
+                    value=10
+                )
+                page_number = gr.Number(
+                    value=1,
+                    label="Page Number",
+                    precision=0
+                )
+
+                # Navigation buttons
+                refresh_keywords_button = gr.Button("Refresh Keywords")
+                view_button = gr.Button("View Results")
+                next_page_button = gr.Button("Next Page")
+                previous_page_button = gr.Button("Previous Page")
+
+                # Pagination information
+                pagination_info = gr.Textbox(
+                    label="Pagination Info",
+                    interactive=False
+                )
+
+            with gr.Column(scale=2):
+                # Results area
+                results_table = gr.HTML(
+                    label="Search Results"
+                )
+                item_details = gr.HTML(
+                    label="Item Details",
+                    visible=True
+                )
+
+        def update_keyword_choices():
+            try:
+                keywords = fetch_all_keywords()
+                return gr.update(choices=keywords)
+            except Exception as e:
+                return gr.update(choices=[], value=None)
+
+        def search_items(keyword, page, entries_per_page):
+            try:
+                # Calculate offset for pagination
+                offset = (page - 1) * entries_per_page
+
+                # Fetch items for the selected keyword
+                items = fetch_items_by_keyword(keyword)
+                total_items = len(items)
+                total_pages = (total_items + entries_per_page - 1) // entries_per_page
+
+                # Paginate results
+                paginated_items = items[offset:offset + entries_per_page]
+
+                # Generate HTML table for results
+                table_html = "<table style='width:100%; border-collapse: collapse;'>"
+                table_html += "<tr><th style='border: 1px solid black; padding: 8px;'>Title</th>"
+                table_html += "<th style='border: 1px solid black; padding: 8px;'>URL</th></tr>"
+
+                for item_id, title, url in paginated_items:
+                    table_html += f"""
+                    <tr>
+                        <td style='border: 1px solid black; padding: 8px;'>{html.escape(title)}</td>
+                        <td style='border: 1px solid black; padding: 8px;'>{html.escape(url)}</td>
+                    </tr>
+                    """
+                table_html += "</table>"
+
+                # Update pagination info
+                pagination = f"Page {page} of {total_pages} (Total items: {total_items})"
+
+                # Determine button states
+                next_disabled = page >= total_pages
+                prev_disabled = page <= 1
+
+                return (
+                    table_html,
+                    pagination,
+                    gr.update(interactive=not next_disabled),
+                    gr.update(interactive=not prev_disabled)
+                )
+            except Exception as e:
+                return (
+                    f"<p>Error: {str(e)}</p>",
+                    "Error in pagination",
+                    gr.update(interactive=False),
+                    gr.update(interactive=False)
+                )
+
+        def go_to_next_page(keyword, current_page, entries_per_page):
+            next_page = current_page + 1
+            return search_items(keyword, next_page, entries_per_page) + (next_page,)
+
+        def go_to_previous_page(keyword, current_page, entries_per_page):
+            previous_page = max(1, current_page - 1)
+            return search_items(keyword, previous_page, entries_per_page) + (previous_page,)
+
+        # Event handlers
+        refresh_keywords_button.click(
+            fn=update_keyword_choices,
+            inputs=[],
+            outputs=[keyword_dropdown]
+        )
+
+        view_button.click(
+            fn=search_items,
+            inputs=[keyword_dropdown, page_number, entries_per_page],
+            outputs=[results_table, pagination_info, next_page_button, previous_page_button]
+        )
+
+        next_page_button.click(
+            fn=go_to_next_page,
+            inputs=[keyword_dropdown, page_number, entries_per_page],
+            outputs=[results_table, pagination_info, next_page_button, previous_page_button, page_number]
+        )
+
+        previous_page_button.click(
+            fn=go_to_previous_page,
+            inputs=[keyword_dropdown, page_number, entries_per_page],
+            outputs=[results_table, pagination_info, next_page_button, previous_page_button, page_number]
+        )
+
+
+def create_viewing_mediadb_tab():
+    with gr.TabItem("View Media Database Entries", visible=True):
+        gr.Markdown("# View Media Database Entries")
         with gr.Row():
             with gr.Column():
                 entries_per_page = gr.Dropdown(choices=[10, 20, 50, 100], label="Entries per Page", value=10)
@@ -327,5 +356,461 @@ def create_viewing_tab():
             outputs=[results_display, pagination_info, page_number, next_page_button, previous_page_button]
         )
 
+#####################################################################
+#
+# RAG DB Viewing Functions:
+
+def create_viewing_ragdb_tab():
+    with gr.TabItem("View RAG Database Entries", visible=True):
+        gr.Markdown("# View RAG Database Entries")
+        with gr.Row():
+            with gr.Column():
+                entries_per_page = gr.Dropdown(choices=[10, 20, 50, 100], label="Entries per Page", value=10)
+                page_number = gr.Number(value=1, label="Page Number", precision=0)
+                view_button = gr.Button("View Page")
+                next_page_button = gr.Button("Next Page")
+                previous_page_button = gr.Button("Previous Page")
+                pagination_info = gr.Textbox(label="Pagination Info", interactive=False)
+            with gr.Column():
+                results_display = gr.HTML()
+
+        def format_conversations_table(conversations):
+            table_html = "<table style='width:100%; border-collapse: collapse;'>"
+            table_html += """
+                <tr>
+                    <th style='border: 1px solid black; padding: 8px;'>Title</th>
+                    <th style='border: 1px solid black; padding: 8px;'>Keywords</th>
+                    <th style='border: 1px solid black; padding: 8px;'>Notes</th>
+                    <th style='border: 1px solid black; padding: 8px;'>Rating</th>
+                </tr>
+            """
+
+            for conversation in conversations:
+                conv_id = conversation['conversation_id']
+                title = conversation['title']
+                rating = conversation.get('rating', '')  # Use get() to handle cases where rating might not exist
+
+                keywords = get_keywords_for_conversation(conv_id)
+                notes = get_notes(conv_id)
+
+                table_html += f"""
+                    <tr>
+                        <td style='border: 1px solid black; padding: 8px;'>{html.escape(str(title))}</td>
+                        <td style='border: 1px solid black; padding: 8px;'>{html.escape(', '.join(keywords))}</td>
+                        <td style='border: 1px solid black; padding: 8px;'>{len(notes)} note(s)</td>
+                        <td style='border: 1px solid black; padding: 8px;'>{html.escape(str(rating))}</td>
+                    </tr>
+                """
+            table_html += "</table>"
+            return table_html
+
+        def update_page(page, entries_per_page):
+            try:
+                conversations, total_pages, total_count = get_all_conversations(page, entries_per_page)
+                results_html = format_conversations_table(conversations)
+                pagination = f"Page {page} of {total_pages} (Total conversations: {total_count})"
+
+                next_disabled = page >= total_pages
+                prev_disabled = page <= 1
+
+                return (
+                    results_html,
+                    pagination,
+                    page,
+                    gr.update(interactive=not next_disabled),
+                    gr.update(interactive=not prev_disabled)
+                )
+            except Exception as e:
+                return (
+                    f"<p>Error: {str(e)}</p>",
+                    "Error in pagination",
+                    page,
+                    gr.update(interactive=False),
+                    gr.update(interactive=False)
+                )
+
+        def go_to_next_page(current_page, entries_per_page):
+            return update_page(current_page + 1, entries_per_page)
+
+        def go_to_previous_page(current_page, entries_per_page):
+            return update_page(max(1, current_page - 1), entries_per_page)
+
+        view_button.click(
+            fn=update_page,
+            inputs=[page_number, entries_per_page],
+            outputs=[results_display, pagination_info, page_number, next_page_button, previous_page_button]
+        )
+
+        next_page_button.click(
+            fn=go_to_next_page,
+            inputs=[page_number, entries_per_page],
+            outputs=[results_display, pagination_info, page_number, next_page_button, previous_page_button]
+        )
+
+        previous_page_button.click(
+            fn=go_to_previous_page,
+            inputs=[page_number, entries_per_page],
+            outputs=[results_display, pagination_info, page_number, next_page_button, previous_page_button]
+        )
+
+
+def create_view_all_rag_notes_tab():
+    with gr.TabItem("View All RAG notes/Conversation Items", visible=True):
+        gr.Markdown("# View All RAG Notes/Conversation Entries")
+        with gr.Row():
+            with gr.Column(scale=1):
+                entries_per_page = gr.Dropdown(choices=[10, 20, 50, 100], label="Entries per Page", value=10)
+                page_number = gr.Number(value=1, label="Page Number", precision=0)
+                view_button = gr.Button("View Page")
+                next_page_button = gr.Button("Next Page")
+                previous_page_button = gr.Button("Previous Page")
+            with gr.Column(scale=2):
+                items_output = gr.Dropdown(label="Select Conversation to View Details", choices=[])
+                conversation_title = gr.Textbox(label="Conversation Title", visible=True)
+        with gr.Row():
+            with gr.Column(scale=1):
+                pagination_info = gr.Textbox(label="Pagination Info", interactive=False)
+            with gr.Column(scale=2):
+                keywords_output = gr.Textbox(label="Keywords", visible=True)
+                chat_history_output = gr.HTML(label="Chat History", visible=True)
+                notes_output = gr.HTML(label="Associated Notes", visible=True)
+
+        item_mapping = gr.State({})
+
+        def update_page(page, entries_per_page):
+            try:
+                conversations, total_pages, total_count = get_all_conversations(page, entries_per_page)
+                pagination = f"Page {page} of {total_pages} (Total conversations: {total_count})"
+
+                # Handle the dictionary structure correctly
+                choices = [f"{conv['title']} (ID: {conv['conversation_id']})" for conv in conversations]
+                new_item_mapping = {
+                    f"{conv['title']} (ID: {conv['conversation_id']})": conv['conversation_id']
+                    for conv in conversations
+                }
+
+                next_disabled = page >= total_pages
+                prev_disabled = page <= 1
+
+                return (
+                    gr.update(choices=choices, value=None),
+                    pagination,
+                    page,
+                    gr.update(interactive=not next_disabled),
+                    gr.update(interactive=not prev_disabled),
+                    "",  # conversation_title
+                    "",  # keywords_output
+                    "",  # chat_history_output
+                    "",  # notes_output
+                    new_item_mapping
+                )
+            except Exception as e:
+                logging.error(f"Error in update_page: {str(e)}", exc_info=True)
+                return (
+                    gr.update(choices=[], value=None),
+                    f"Error: {str(e)}",
+                    page,
+                    gr.update(interactive=False),
+                    gr.update(interactive=False),
+                    "", "", "", "",
+                    {}
+                )
+
+        def format_as_html(content, title):
+            if content is None:
+                content = "No content available."
+            escaped_content = html.escape(str(content))
+            formatted_content = escaped_content.replace('\n', '<br>')
+            return f"""
+            <div style="border: 1px solid #ddd; padding: 10px; margin-bottom: 10px;">
+                <h3>{title}</h3>
+                <div style="max-height: 700px; overflow-y: auto;">
+                    {formatted_content}
+                </div>
+            </div>
+            """
+
+        def format_chat_history(messages):
+            html_content = "<div style='max-height: 500px; overflow-y: auto;'>"
+            for role, content in messages:
+                role_class = "assistant" if role.lower() == "assistant" else "user"
+                html_content += f"""
+                <div class='{role_class}-message' style='margin: 10px 0; padding: 10px; border-radius: 5px; 
+                    background-color: {"#f0f0f0" if role_class == "user" else "#e3f2fd"}'>
+                    <strong>{html.escape(role)}:</strong><br>
+                    {html.escape(content)}
+                </div>
+                """
+            html_content += "</div>"
+            return html_content
+
+        def display_conversation_details(selected_item, item_mapping):
+            if selected_item and item_mapping and selected_item in item_mapping:
+                conv_id = item_mapping[selected_item]
+
+                # Get keywords
+                keywords = get_keywords_for_conversation(conv_id)
+                keywords_text = ", ".join(keywords) if keywords else "No keywords"
+
+                # Get chat history
+                chat_messages, _, _ = load_chat_history(conv_id)
+                chat_html = format_chat_history(chat_messages)
+
+                # Get associated notes
+                notes = get_notes(conv_id)
+                notes_html = ""
+                for note in notes:
+                    notes_html += format_as_html(note, "Note")
+                if not notes:
+                    notes_html = "<p>No notes associated with this conversation.</p>"
+
+                return (
+                    selected_item.split(" (ID:")[0],  # Conversation title
+                    keywords_text,
+                    chat_html,
+                    notes_html
+                )
+            return "", "", "", ""
+
+        view_button.click(
+            fn=update_page,
+            inputs=[page_number, entries_per_page],
+            outputs=[
+                items_output,
+                pagination_info,
+                page_number,
+                next_page_button,
+                previous_page_button,
+                conversation_title,
+                keywords_output,
+                chat_history_output,
+                notes_output,
+                item_mapping
+            ]
+        )
+
+        next_page_button.click(
+            fn=lambda page, entries: update_page(page + 1, entries),
+            inputs=[page_number, entries_per_page],
+            outputs=[items_output, pagination_info, page_number, next_page_button, previous_page_button,
+                     conversation_title, keywords_output, chat_history_output, notes_output, item_mapping]
+        )
+
+        previous_page_button.click(
+            fn=lambda page, entries: update_page(max(1, page - 1), entries),
+            inputs=[page_number, entries_per_page],
+            outputs=[items_output, pagination_info, page_number, next_page_button, previous_page_button,
+                     conversation_title, keywords_output, chat_history_output, notes_output, item_mapping]
+        )
+
+        items_output.change(
+            fn=display_conversation_details,
+            inputs=[items_output, item_mapping],
+            outputs=[conversation_title, keywords_output, chat_history_output, notes_output]
+        )
+
+
+def create_ragdb_keyword_items_tab():
+    with gr.TabItem("View RAG Notes/Conversations by Keyword", visible=True):
+        gr.Markdown("# View RAG Notes and Conversations by Keyword")
+
+        with gr.Row():
+            with gr.Column(scale=1):
+                # Keyword selection
+                keyword_dropdown = gr.Dropdown(
+                    label="Select Keyword",
+                    choices=[],
+                    value=None,
+                    multiselect=True
+                )
+                entries_per_page = gr.Dropdown(
+                    choices=[10, 20, 50, 100],
+                    label="Entries per Page",
+                    value=10
+                )
+                page_number = gr.Number(
+                    value=1,
+                    label="Page Number",
+                    precision=0
+                )
+
+                # Navigation buttons
+                refresh_keywords_button = gr.Button("Refresh Keywords")
+                view_button = gr.Button("View Items")
+                next_page_button = gr.Button("Next Page")
+                previous_page_button = gr.Button("Previous Page")
+                pagination_info = gr.Textbox(
+                    label="Pagination Info",
+                    interactive=False
+                )
+
+            with gr.Column(scale=2):
+                # Results tabs for conversations and notes
+                with gr.Tabs():
+                    with gr.Tab("Notes"):
+                        notes_results = gr.HTML()
+                    with gr.Tab("Conversations"):
+                        conversation_results = gr.HTML()
+
+        def update_keyword_choices():
+            """Fetch all available keywords for the dropdown."""
+            try:
+                query = "SELECT keyword FROM rag_qa_keywords ORDER BY keyword"
+                with get_db_connection() as conn:
+                    cursor = conn.cursor()
+                    cursor.execute(query)
+                    keywords = [row[0] for row in cursor.fetchall()]
+                return gr.update(choices=keywords)
+            except Exception as e:
+                return gr.update(choices=[], value=None)
+
+        def format_conversations_html(conversations_data):
+            """Format conversations data as HTML."""
+            if not conversations_data:
+                return "<p>No conversations found for selected keywords.</p>"
+
+            html_content = "<div class='results-container'>"
+            for conv_id, title in conversations_data:
+                html_content += f"""
+                <div style='border: 1px solid #ddd; padding: 10px; margin-bottom: 10px;'>
+                    <h3>{html.escape(title)}</h3>
+                    <p>Conversation ID: {html.escape(conv_id)}</p>
+                    <p><strong>Keywords:</strong> {', '.join(html.escape(k) for k in get_keywords_for_conversation(conv_id))}</p>
+                </div>
+                """
+            html_content += "</div>"
+            return html_content
+
+        def format_notes_html(notes_data):
+            """Format notes data as HTML."""
+            if not notes_data:
+                return "<p>No notes found for selected keywords.</p>"
+
+            html_content = "<div class='results-container'>"
+            for note_id, title, content, timestamp in notes_data:
+                keywords = get_keywords_for_note(note_id)
+                html_content += f"""
+                <div style='border: 1px solid #ddd; padding: 10px; margin-bottom: 10px;'>
+                    <h3>{html.escape(title)}</h3>
+                    <p><strong>Created:</strong> {timestamp}</p>
+                    <p><strong>Keywords:</strong> {', '.join(html.escape(k) for k in keywords)}</p>
+                    <div style='background: #f5f5f5; padding: 10px; margin-top: 10px;'>
+                        {html.escape(content)}
+                    </div>
+                </div>
+                """
+            html_content += "</div>"
+            return html_content
+
+        def view_items(keywords, page, entries_per_page):
+            if not keywords or (isinstance(keywords, list) and len(keywords) == 0):
+                return (
+                    "<p>Please select at least one keyword.</p>",
+                    "<p>Please select at least one keyword.</p>",
+                    "No results",
+                    gr.update(interactive=False),
+                    gr.update(interactive=False)
+                )
+
+            try:
+                # Ensure keywords is a list
+                keywords_list = keywords if isinstance(keywords, list) else [keywords]
+
+                # Get conversations for selected keywords
+                conversations, conv_total_pages, conv_count = search_conversations_by_keywords(
+                    keywords_list, page, entries_per_page
+                )
+
+                # Get notes for selected keywords
+                notes, notes_total_pages, notes_count = get_notes_by_keywords(
+                    keywords_list, page, entries_per_page
+                )
+
+                # Format results as HTML
+                conv_html = format_conversations_html(conversations)
+                notes_html = format_notes_html(notes)
+
+                # Create pagination info
+                pagination = f"Page {page} of {max(conv_total_pages, notes_total_pages)} "
+                pagination += f"(Conversations: {conv_count}, Notes: {notes_count})"
+
+                # Determine button states
+                max_pages = max(conv_total_pages, notes_total_pages)
+                next_disabled = page >= max_pages
+                prev_disabled = page <= 1
+
+                return (
+                    conv_html,
+                    notes_html,
+                    pagination,
+                    gr.update(interactive=not next_disabled),
+                    gr.update(interactive=not prev_disabled)
+                )
+            except Exception as e:
+                logging.error(f"Error in view_items: {str(e)}")
+                return (
+                    f"<p>Error: {str(e)}</p>",
+                    f"<p>Error: {str(e)}</p>",
+                    "Error in retrieval",
+                    gr.update(interactive=False),
+                    gr.update(interactive=False)
+                )
+
+        def go_to_next_page(keywords, current_page, entries_per_page):
+            return view_items(keywords, current_page + 1, entries_per_page)
+
+        def go_to_previous_page(keywords, current_page, entries_per_page):
+            return view_items(keywords, max(1, current_page - 1), entries_per_page)
+
+        # Event handlers
+        refresh_keywords_button.click(
+            fn=update_keyword_choices,
+            inputs=[],
+            outputs=[keyword_dropdown]
+        )
+
+        view_button.click(
+            fn=view_items,
+            inputs=[keyword_dropdown, page_number, entries_per_page],
+            outputs=[
+                conversation_results,
+                notes_results,
+                pagination_info,
+                next_page_button,
+                previous_page_button
+            ]
+        )
+
+        next_page_button.click(
+            fn=go_to_next_page,
+            inputs=[keyword_dropdown, page_number, entries_per_page],
+            outputs=[
+                conversation_results,
+                notes_results,
+                pagination_info,
+                next_page_button,
+                previous_page_button
+            ]
+        )
+
+        previous_page_button.click(
+            fn=go_to_previous_page,
+            inputs=[keyword_dropdown, page_number, entries_per_page],
+            outputs=[
+                conversation_results,
+                notes_results,
+                pagination_info,
+                next_page_button,
+                previous_page_button
+            ]
+        )
+
+        # Initialize keyword dropdown on page load
+        keyword_dropdown.value = update_keyword_choices()
+
+#
+# End of RAG DB Viewing tabs
+################################################################
+
 #
-####################################################################################################
+#######################################################################################################################

App_Function_Libraries/Gradio_UI/Website_scraping_tab.py

@@ -1,554 +1,754 @@
-# Website_scraping_tab.py
-# Gradio UI for scraping websites
-#
-# Imports
-import asyncio
-import json
-import logging
-import os
-import random
-from concurrent.futures import ThreadPoolExecutor
-from typing import Optional, List, Dict, Any
-from urllib.parse import urlparse, urljoin
-
-#
-# External Imports
-import gradio as gr
-from playwright.async_api import TimeoutError, async_playwright
-from playwright.sync_api import sync_playwright
-
-#
-# Local Imports
-from App_Function_Libraries.Web_Scraping.Article_Extractor_Lib import scrape_from_sitemap, scrape_by_url_level, scrape_article
-from App_Function_Libraries.Web_Scraping.Article_Summarization_Lib import scrape_and_summarize_multiple
-from App_Function_Libraries.DB.DB_Manager import load_preset_prompts
-from App_Function_Libraries.Gradio_UI.Chat_ui import update_user_prompt
-from App_Function_Libraries.Summarization.Summarization_General_Lib import summarize
-
-
-#
-########################################################################################################################
-#
-# Functions:
-
-def get_url_depth(url: str) -> int:
-    return len(urlparse(url).path.strip('/').split('/'))
-
-
-def sync_recursive_scrape(url_input, max_pages, max_depth, progress_callback, delay=1.0):
-    def run_async_scrape():
-        loop = asyncio.new_event_loop()
-        asyncio.set_event_loop(loop)
-        return loop.run_until_complete(
-            recursive_scrape(url_input, max_pages, max_depth, progress_callback, delay)
-        )
-
-    with ThreadPoolExecutor() as executor:
-        future = executor.submit(run_async_scrape)
-        return future.result()
-
-
-async def recursive_scrape(
-        base_url: str,
-        max_pages: int,
-        max_depth: int,
-        progress_callback: callable,
-        delay: float = 1.0,
-        resume_file: str = 'scrape_progress.json',
-        user_agent: str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"
-) -> List[Dict]:
-    async def save_progress():
-        temp_file = resume_file + ".tmp"
-        with open(temp_file, 'w') as f:
-            json.dump({
-                'visited': list(visited),
-                'to_visit': to_visit,
-                'scraped_articles': scraped_articles,
-                'pages_scraped': pages_scraped
-            }, f)
-        os.replace(temp_file, resume_file)  # Atomic replace
-
-    def is_valid_url(url: str) -> bool:
-        return url.startswith("http") and len(url) > 0
-
-    # Load progress if resume file exists
-    if os.path.exists(resume_file):
-        with open(resume_file, 'r') as f:
-            progress_data = json.load(f)
-            visited = set(progress_data['visited'])
-            to_visit = progress_data['to_visit']
-            scraped_articles = progress_data['scraped_articles']
-            pages_scraped = progress_data['pages_scraped']
-    else:
-        visited = set()
-        to_visit = [(base_url, 0)]  # (url, depth)
-        scraped_articles = []
-        pages_scraped = 0
-
-    try:
-        async with async_playwright() as p:
-            browser = await p.chromium.launch(headless=True)
-            context = await browser.new_context(user_agent=user_agent)
-
-            try:
-                while to_visit and pages_scraped < max_pages:
-                    current_url, current_depth = to_visit.pop(0)
-
-                    if current_url in visited or current_depth > max_depth:
-                        continue
-
-                    visited.add(current_url)
-
-                    # Update progress
-                    progress_callback(f"Scraping page {pages_scraped + 1}/{max_pages}: {current_url}")
-
-                    try:
-                        await asyncio.sleep(random.uniform(delay * 0.8, delay * 1.2))
-
-                        # This function should be implemented to handle asynchronous scraping
-                        article_data = await scrape_article_async(context, current_url)
-
-                        if article_data and article_data['extraction_successful']:
-                            scraped_articles.append(article_data)
-                            pages_scraped += 1
-
-                        # If we haven't reached max depth, add child links to to_visit
-                        if current_depth < max_depth:
-                            page = await context.new_page()
-                            await page.goto(current_url)
-                            await page.wait_for_load_state("networkidle")
-
-                            links = await page.eval_on_selector_all('a[href]',
-                                                                    "(elements) => elements.map(el => el.href)")
-                            for link in links:
-                                child_url = urljoin(base_url, link)
-                                if is_valid_url(child_url) and child_url.startswith(
-                                        base_url) and child_url not in visited and should_scrape_url(child_url):
-                                    to_visit.append((child_url, current_depth + 1))
-
-                            await page.close()
-
-                    except Exception as e:
-                        logging.error(f"Error scraping {current_url}: {str(e)}")
-
-                    # Save progress periodically (e.g., every 10 pages)
-                    if pages_scraped % 10 == 0:
-                        await save_progress()
-
-            finally:
-                await browser.close()
-
-    finally:
-        # These statements are now guaranteed to be reached after the scraping is done
-        await save_progress()
-
-        # Remove the progress file when scraping is completed successfully
-        if os.path.exists(resume_file):
-            os.remove(resume_file)
-
-        # Final progress update
-        progress_callback(f"Scraping completed. Total pages scraped: {pages_scraped}")
-
-        return scraped_articles
-
-
-async def scrape_article_async(context, url: str) -> Dict[str, Any]:
-    page = await context.new_page()
-    try:
-        await page.goto(url)
-        await page.wait_for_load_state("networkidle")
-
-        title = await page.title()
-        content = await page.content()
-
-        return {
-            'url': url,
-            'title': title,
-            'content': content,
-            'extraction_successful': True
-        }
-    except Exception as e:
-        logging.error(f"Error scraping article {url}: {str(e)}")
-        return {
-            'url': url,
-            'extraction_successful': False,
-            'error': str(e)
-        }
-    finally:
-        await page.close()
-
-
-def scrape_article_sync(url: str) -> Dict[str, Any]:
-    with sync_playwright() as p:
-        browser = p.chromium.launch(headless=True)
-        page = browser.new_page()
-        try:
-            page.goto(url)
-            page.wait_for_load_state("networkidle")
-
-            title = page.title()
-            content = page.content()
-
-            return {
-                'url': url,
-                'title': title,
-                'content': content,
-                'extraction_successful': True
-            }
-        except Exception as e:
-            logging.error(f"Error scraping article {url}: {str(e)}")
-            return {
-                'url': url,
-                'extraction_successful': False,
-                'error': str(e)
-            }
-        finally:
-            browser.close()
-
-
-def should_scrape_url(url: str) -> bool:
-    parsed_url = urlparse(url)
-    path = parsed_url.path.lower()
-
-    # List of patterns to exclude
-    exclude_patterns = [
-        '/tag/', '/category/', '/author/', '/search/', '/page/',
-        'wp-content', 'wp-includes', 'wp-json', 'wp-admin',
-        'login', 'register', 'cart', 'checkout', 'account',
-        '.jpg', '.png', '.gif', '.pdf', '.zip'
-    ]
-
-    # Check if the URL contains any exclude patterns
-    if any(pattern in path for pattern in exclude_patterns):
-        return False
-
-    # Add more sophisticated checks here
-    # For example, you might want to only include URLs with certain patterns
-    include_patterns = ['/article/', '/post/', '/blog/']
-    if any(pattern in path for pattern in include_patterns):
-        return True
-
-    # By default, return True if no exclusion or inclusion rules matched
-    return True
-
-
-async def scrape_with_retry(url: str, max_retries: int = 3, retry_delay: float = 5.0):
-    for attempt in range(max_retries):
-        try:
-            return await scrape_article(url)
-        except TimeoutError:
-            if attempt < max_retries - 1:
-                logging.warning(f"Timeout error scraping {url}. Retrying in {retry_delay} seconds...")
-                await asyncio.sleep(retry_delay)
-            else:
-                logging.error(f"Failed to scrape {url} after {max_retries} attempts.")
-                return None
-        except Exception as e:
-            logging.error(f"Error scraping {url}: {str(e)}")
-            return None
-
-
-def create_website_scraping_tab():
-    with gr.TabItem("Website Scraping", visible=True):
-        gr.Markdown("# Scrape Websites & Summarize Articles")
-        with gr.Row():
-            with gr.Column():
-                scrape_method = gr.Radio(
-                    ["Individual URLs", "Sitemap", "URL Level", "Recursive Scraping"],
-                    label="Scraping Method",
-                    value="Individual URLs"
-                )
-                url_input = gr.Textbox(
-                    label="Article URLs or Base URL",
-                    placeholder="Enter article URLs here, one per line, or base URL for sitemap/URL level/recursive scraping",
-                    lines=5
-                )
-                url_level = gr.Slider(
-                    minimum=1,
-                    maximum=10,
-                    step=1,
-                    label="URL Level (for URL Level scraping)",
-                    value=2,
-                    visible=False
-                )
-                max_pages = gr.Slider(
-                    minimum=1,
-                    maximum=100,
-                    step=1,
-                    label="Maximum Pages to Scrape (for Recursive Scraping)",
-                    value=10,
-                    visible=False
-                )
-                max_depth = gr.Slider(
-                    minimum=1,
-                    maximum=10,
-                    step=1,
-                    label="Maximum Depth (for Recursive Scraping)",
-                    value=3,
-                    visible=False
-                )
-                custom_article_title_input = gr.Textbox(
-                    label="Custom Article Titles (Optional, one per line)",
-                    placeholder="Enter custom titles for the articles, one per line",
-                    lines=5
-                )
-                with gr.Row():
-                    summarize_checkbox = gr.Checkbox(label="Summarize Articles", value=False)
-                    custom_prompt_checkbox = gr.Checkbox(label="Use a Custom Prompt", value=False, visible=True)
-                    preset_prompt_checkbox = gr.Checkbox(label="Use a pre-set Prompt", value=False, visible=True)
-                with gr.Row():
-                    temp_slider = gr.Slider(0.1, 2.0, 0.7, label="Temperature")
-                with gr.Row():
-                    preset_prompt = gr.Dropdown(
-                        label="Select Preset Prompt",
-                        choices=load_preset_prompts(),
-                        visible=False
-                    )
-                with gr.Row():
-                    website_custom_prompt_input = gr.Textbox(
-                        label="Custom Prompt",
-                        placeholder="Enter custom prompt here",
-                        lines=3,
-                        visible=False
-                    )
-                with gr.Row():
-                    system_prompt_input = gr.Textbox(
-                        label="System Prompt",
-                        value="""<s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
-                                **Bulleted Note Creation Guidelines**
-                                
-                                **Headings**:
-                                - Based on referenced topics, not categories like quotes or terms
-                                - Surrounded by **bold** formatting 
-                                - Not listed as bullet points
-                                - No space between headings and list items underneath
-                                
-                                **Emphasis**:
-                                - **Important terms** set in bold font
-                                - **Text ending in a colon**: also bolded
-                                
-                                **Review**:
-                                - Ensure adherence to specified format
-                                - Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]
-                                """,
-                        lines=3,
-                        visible=False
-                    )
-
-                api_name_input = gr.Dropdown(
-                    choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral",
-                             "OpenRouter",
-                             "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace",
-                             "Custom-OpenAI-API"],
-                    value=None,
-                    label="API Name (Mandatory for Summarization)"
-                )
-                api_key_input = gr.Textbox(
-                    label="API Key (Mandatory if API Name is specified)",
-                    placeholder="Enter your API key here; Ignore if using Local API or Built-in API",
-                    type="password"
-                )
-                keywords_input = gr.Textbox(
-                    label="Keywords",
-                    placeholder="Enter keywords here (comma-separated)",
-                    value="default,no_keyword_set",
-                    visible=True
-                )
-
-                scrape_button = gr.Button("Scrape and Summarize")
-            with gr.Column():
-                progress_output = gr.Textbox(label="Progress", lines=3)
-                result_output = gr.Textbox(label="Result", lines=20)
-
-        def update_ui_for_scrape_method(method):
-            url_level_update = gr.update(visible=(method == "URL Level"))
-            max_pages_update = gr.update(visible=(method == "Recursive Scraping"))
-            max_depth_update = gr.update(visible=(method == "Recursive Scraping"))
-            url_input_update = gr.update(
-                label="Article URLs" if method == "Individual URLs" else "Base URL",
-                placeholder="Enter article URLs here, one per line" if method == "Individual URLs" else "Enter the base URL for scraping"
-            )
-            return url_level_update, max_pages_update, max_depth_update, url_input_update
-
-        scrape_method.change(
-            fn=update_ui_for_scrape_method,
-            inputs=[scrape_method],
-            outputs=[url_level, max_pages, max_depth, url_input]
-        )
-
-        custom_prompt_checkbox.change(
-            fn=lambda x: (gr.update(visible=x), gr.update(visible=x)),
-            inputs=[custom_prompt_checkbox],
-            outputs=[website_custom_prompt_input, system_prompt_input]
-        )
-        preset_prompt_checkbox.change(
-            fn=lambda x: gr.update(visible=x),
-            inputs=[preset_prompt_checkbox],
-            outputs=[preset_prompt]
-        )
-
-        def update_prompts(preset_name):
-            prompts = update_user_prompt(preset_name)
-            return (
-                gr.update(value=prompts["user_prompt"], visible=True),
-                gr.update(value=prompts["system_prompt"], visible=True)
-            )
-
-        preset_prompt.change(
-            update_prompts,
-            inputs=preset_prompt,
-            outputs=[website_custom_prompt_input, system_prompt_input]
-        )
-
-        async def scrape_and_summarize_wrapper(
-                scrape_method: str,
-                url_input: str,
-                url_level: Optional[int],
-                max_pages: int,
-                max_depth: int,
-                summarize_checkbox: bool,
-                custom_prompt: Optional[str],
-                api_name: Optional[str],
-                api_key: Optional[str],
-                keywords: str,
-                custom_titles: Optional[str],
-                system_prompt: Optional[str],
-                temperature: float = 0.7,
-                progress: gr.Progress = gr.Progress()
-        ) -> str:
-            try:
-                result: List[Dict[str, Any]] = []
-
-                if scrape_method == "Individual URLs":
-                    result = await scrape_and_summarize_multiple(url_input, custom_prompt, api_name, api_key, keywords,
-                                                                 custom_titles, system_prompt)
-                elif scrape_method == "Sitemap":
-                    result = await asyncio.to_thread(scrape_from_sitemap, url_input)
-                elif scrape_method == "URL Level":
-                    if url_level is None:
-                        return convert_json_to_markdown(
-                            json.dumps({"error": "URL level is required for URL Level scraping."}))
-                    result = await asyncio.to_thread(scrape_by_url_level, url_input, url_level)
-                elif scrape_method == "Recursive Scraping":
-                    result = await recursive_scrape(url_input, max_pages, max_depth, progress.update, delay=1.0)
-                else:
-                    return convert_json_to_markdown(json.dumps({"error": f"Unknown scraping method: {scrape_method}"}))
-
-                # Ensure result is always a list of dictionaries
-                if isinstance(result, dict):
-                    result = [result]
-                elif isinstance(result, list):
-                    if all(isinstance(item, str) for item in result):
-                        # Convert list of strings to list of dictionaries
-                        result = [{"content": item} for item in result]
-                    elif not all(isinstance(item, dict) for item in result):
-                        raise ValueError("Not all items in result are dictionaries or strings")
-                else:
-                    raise ValueError(f"Unexpected result type: {type(result)}")
-
-                # Ensure all items in result are dictionaries
-                if not all(isinstance(item, dict) for item in result):
-                    raise ValueError("Not all items in result are dictionaries")
-
-                if summarize_checkbox:
-                    total_articles = len(result)
-                    for i, article in enumerate(result):
-                        progress.update(f"Summarizing article {i + 1}/{total_articles}")
-                        content = article.get('content', '')
-                        if content:
-                            summary = await asyncio.to_thread(summarize, content, custom_prompt, api_name, api_key,
-                                                              temperature, system_prompt)
-                            article['summary'] = summary
-                        else:
-                            article['summary'] = "No content available to summarize."
-
-                # Concatenate all content
-                all_content = "\n\n".join(
-                    [f"# {article.get('title', 'Untitled')}\n\n{article.get('content', '')}\n\n" +
-                     (f"Summary: {article.get('summary', '')}" if summarize_checkbox else "")
-                     for article in result])
-
-                # Collect all unique URLs
-                all_urls = list(set(article.get('url', '') for article in result if article.get('url')))
-
-                # Structure the output for the entire website collection
-                website_collection = {
-                    "base_url": url_input,
-                    "scrape_method": scrape_method,
-                    "summarization_performed": summarize_checkbox,
-                    "api_used": api_name if summarize_checkbox else None,
-                    "keywords": keywords if summarize_checkbox else None,
-                    "url_level": url_level if scrape_method == "URL Level" else None,
-                    "max_pages": max_pages if scrape_method == "Recursive Scraping" else None,
-                    "max_depth": max_depth if scrape_method == "Recursive Scraping" else None,
-                    "total_articles_scraped": len(result),
-                    "urls_scraped": all_urls,
-                    "content": all_content
-                }
-
-                # Convert the JSON to markdown and return
-                return convert_json_to_markdown(json.dumps(website_collection, indent=2))
-            except Exception as e:
-                return convert_json_to_markdown(json.dumps({"error": f"An error occurred: {str(e)}"}))
-
-        # Update the scrape_button.click to include the temperature parameter
-        scrape_button.click(
-            fn=lambda *args: asyncio.run(scrape_and_summarize_wrapper(*args)),
-            inputs=[scrape_method, url_input, url_level, max_pages, max_depth, summarize_checkbox,
-                    website_custom_prompt_input, api_name_input, api_key_input, keywords_input,
-                    custom_article_title_input, system_prompt_input, temp_slider],
-            outputs=[result_output]
-        )
-
-
-def convert_json_to_markdown(json_str: str) -> str:
-    """
-    Converts the JSON output from the scraping process into a markdown format.
-
-    Args:
-        json_str (str): JSON-formatted string containing the website collection data
-
-    Returns:
-        str: Markdown-formatted string of the website collection data
-    """
-    try:
-        # Parse the JSON string
-        data = json.loads(json_str)
-
-        # Check if there's an error in the JSON
-        if "error" in data:
-            return f"# Error\n\n{data['error']}"
-
-        # Start building the markdown string
-        markdown = f"# Website Collection: {data['base_url']}\n\n"
-
-        # Add metadata
-        markdown += "## Metadata\n\n"
-        markdown += f"- **Scrape Method:** {data['scrape_method']}\n"
-        markdown += f"- **API Used:** {data['api_used']}\n"
-        markdown += f"- **Keywords:** {data['keywords']}\n"
-        if data['url_level'] is not None:
-            markdown += f"- **URL Level:** {data['url_level']}\n"
-        markdown += f"- **Total Articles Scraped:** {data['total_articles_scraped']}\n\n"
-
-        # Add URLs scraped
-        markdown += "## URLs Scraped\n\n"
-        for url in data['urls_scraped']:
-            markdown += f"- {url}\n"
-        markdown += "\n"
-
-        # Add the content
-        markdown += "## Content\n\n"
-        markdown += data['content']
-
-        return markdown
-
-    except json.JSONDecodeError:
-        return "# Error\n\nInvalid JSON string provided."
-    except KeyError as e:
-        return f"# Error\n\nMissing key in JSON data: {str(e)}"
-    except Exception as e:
-        return f"# Error\n\nAn unexpected error occurred: {str(e)}"
-#
-# End of File
-########################################################################################################################
+# Website_scraping_tab.py
+# Gradio UI for scraping websites
+#
+# Imports
+import asyncio
+import json
+import logging
+import os
+import random
+from concurrent.futures import ThreadPoolExecutor
+from typing import Optional, List, Dict, Any
+from urllib.parse import urlparse, urljoin
+
+#
+# External Imports
+import gradio as gr
+from playwright.async_api import TimeoutError, async_playwright
+from playwright.sync_api import sync_playwright
+
+from App_Function_Libraries.Utils.Utils import default_api_endpoint, global_api_endpoints, format_api_name
+#
+# Local Imports
+from App_Function_Libraries.Web_Scraping.Article_Extractor_Lib import scrape_from_sitemap, scrape_by_url_level, \
+    scrape_article, collect_bookmarks, scrape_and_summarize_multiple, collect_urls_from_file
+from App_Function_Libraries.DB.DB_Manager import list_prompts
+from App_Function_Libraries.Gradio_UI.Chat_ui import update_user_prompt
+from App_Function_Libraries.Summarization.Summarization_General_Lib import summarize
+
+
+#
+########################################################################################################################
+#
+# Functions:
+
+def get_url_depth(url: str) -> int:
+    return len(urlparse(url).path.strip('/').split('/'))
+
+
+def sync_recursive_scrape(url_input, max_pages, max_depth, progress_callback, delay=1.0, custom_cookies=None):
+    def run_async_scrape():
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        return loop.run_until_complete(
+            recursive_scrape(url_input, max_pages, max_depth, progress_callback, delay, custom_cookies=custom_cookies)
+        )
+
+    with ThreadPoolExecutor() as executor:
+        future = executor.submit(run_async_scrape)
+        return future.result()
+
+
+async def recursive_scrape(
+        base_url: str,
+        max_pages: int,
+        max_depth: int,
+        progress_callback: callable,
+        delay: float = 1.0,
+        resume_file: str = 'scrape_progress.json',
+        user_agent: str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3",
+        custom_cookies: Optional[List[Dict[str, Any]]] = None
+) -> List[Dict]:
+    async def save_progress():
+        temp_file = resume_file + ".tmp"
+        with open(temp_file, 'w') as f:
+            json.dump({
+                'visited': list(visited),
+                'to_visit': to_visit,
+                'scraped_articles': scraped_articles,
+                'pages_scraped': pages_scraped
+            }, f)
+        os.replace(temp_file, resume_file)  # Atomic replace
+
+    def is_valid_url(url: str) -> bool:
+        return url.startswith("http") and len(url) > 0
+
+    # Load progress if resume file exists
+    if os.path.exists(resume_file):
+        with open(resume_file, 'r') as f:
+            progress_data = json.load(f)
+            visited = set(progress_data['visited'])
+            to_visit = progress_data['to_visit']
+            scraped_articles = progress_data['scraped_articles']
+            pages_scraped = progress_data['pages_scraped']
+    else:
+        visited = set()
+        to_visit = [(base_url, 0)]  # (url, depth)
+        scraped_articles = []
+        pages_scraped = 0
+
+    try:
+        async with async_playwright() as p:
+            browser = await p.chromium.launch(headless=True)
+            context = await browser.new_context(user_agent=user_agent)
+
+            # Set custom cookies if provided
+            if custom_cookies:
+                await context.add_cookies(custom_cookies)
+
+            try:
+                while to_visit and pages_scraped < max_pages:
+                    current_url, current_depth = to_visit.pop(0)
+
+                    if current_url in visited or current_depth > max_depth:
+                        continue
+
+                    visited.add(current_url)
+
+                    # Update progress
+                    progress_callback(f"Scraping page {pages_scraped + 1}/{max_pages}: {current_url}")
+
+                    try:
+                        await asyncio.sleep(random.uniform(delay * 0.8, delay * 1.2))
+
+                        # This function should be implemented to handle asynchronous scraping
+                        article_data = await scrape_article_async(context, current_url)
+
+                        if article_data and article_data['extraction_successful']:
+                            scraped_articles.append(article_data)
+                            pages_scraped += 1
+
+                        # If we haven't reached max depth, add child links to to_visit
+                        if current_depth < max_depth:
+                            page = await context.new_page()
+                            await page.goto(current_url)
+                            await page.wait_for_load_state("networkidle")
+
+                            links = await page.eval_on_selector_all('a[href]',
+                                                                    "(elements) => elements.map(el => el.href)")
+                            for link in links:
+                                child_url = urljoin(base_url, link)
+                                if is_valid_url(child_url) and child_url.startswith(
+                                        base_url) and child_url not in visited and should_scrape_url(child_url):
+                                    to_visit.append((child_url, current_depth + 1))
+
+                            await page.close()
+
+                    except Exception as e:
+                        logging.error(f"Error scraping {current_url}: {str(e)}")
+
+                    # Save progress periodically (e.g., every 10 pages)
+                    if pages_scraped % 10 == 0:
+                        await save_progress()
+
+            finally:
+                await browser.close()
+
+    finally:
+        # These statements are now guaranteed to be reached after the scraping is done
+        await save_progress()
+
+        # Remove the progress file when scraping is completed successfully
+        if os.path.exists(resume_file):
+            os.remove(resume_file)
+
+        # Final progress update
+        progress_callback(f"Scraping completed. Total pages scraped: {pages_scraped}")
+
+        return scraped_articles
+
+
+async def scrape_article_async(context, url: str) -> Dict[str, Any]:
+    page = await context.new_page()
+    try:
+        await page.goto(url)
+        await page.wait_for_load_state("networkidle")
+
+        title = await page.title()
+        content = await page.content()
+
+        return {
+            'url': url,
+            'title': title,
+            'content': content,
+            'extraction_successful': True
+        }
+    except Exception as e:
+        logging.error(f"Error scraping article {url}: {str(e)}")
+        return {
+            'url': url,
+            'extraction_successful': False,
+            'error': str(e)
+        }
+    finally:
+        await page.close()
+
+
+def scrape_article_sync(url: str) -> Dict[str, Any]:
+    with sync_playwright() as p:
+        browser = p.chromium.launch(headless=True)
+        page = browser.new_page()
+        try:
+            page.goto(url)
+            page.wait_for_load_state("networkidle")
+
+            title = page.title()
+            content = page.content()
+
+            return {
+                'url': url,
+                'title': title,
+                'content': content,
+                'extraction_successful': True
+            }
+        except Exception as e:
+            logging.error(f"Error scraping article {url}: {str(e)}")
+            return {
+                'url': url,
+                'extraction_successful': False,
+                'error': str(e)
+            }
+        finally:
+            browser.close()
+
+
+def should_scrape_url(url: str) -> bool:
+    parsed_url = urlparse(url)
+    path = parsed_url.path.lower()
+
+    # List of patterns to exclude
+    exclude_patterns = [
+        '/tag/', '/category/', '/author/', '/search/', '/page/',
+        'wp-content', 'wp-includes', 'wp-json', 'wp-admin',
+        'login', 'register', 'cart', 'checkout', 'account',
+        '.jpg', '.png', '.gif', '.pdf', '.zip'
+    ]
+
+    # Check if the URL contains any exclude patterns
+    if any(pattern in path for pattern in exclude_patterns):
+        return False
+
+    # Add more sophisticated checks here
+    # For example, you might want to only include URLs with certain patterns
+    include_patterns = ['/article/', '/post/', '/blog/']
+    if any(pattern in path for pattern in include_patterns):
+        return True
+
+    # By default, return True if no exclusion or inclusion rules matched
+    return True
+
+
+async def scrape_with_retry(url: str, max_retries: int = 3, retry_delay: float = 5.0):
+    for attempt in range(max_retries):
+        try:
+            return await scrape_article(url)
+        except TimeoutError:
+            if attempt < max_retries - 1:
+                logging.warning(f"Timeout error scraping {url}. Retrying in {retry_delay} seconds...")
+                await asyncio.sleep(retry_delay)
+            else:
+                logging.error(f"Failed to scrape {url} after {max_retries} attempts.")
+                return None
+        except Exception as e:
+            logging.error(f"Error scraping {url}: {str(e)}")
+            return None
+
+
+def create_website_scraping_tab():
+    try:
+        default_value = None
+        if default_api_endpoint:
+            if default_api_endpoint in global_api_endpoints:
+                default_value = format_api_name(default_api_endpoint)
+            else:
+                logging.warning(f"Default API endpoint '{default_api_endpoint}' not found in global_api_endpoints")
+    except Exception as e:
+        logging.error(f"Error setting default API endpoint: {str(e)}")
+        default_value = None
+    with gr.TabItem("Website Scraping", visible=True):
+        gr.Markdown("# Scrape Websites & Summarize Articles")
+        with gr.Row():
+            with gr.Column():
+                scrape_method = gr.Radio(
+                    ["Individual URLs", "Sitemap", "URL Level", "Recursive Scraping"],
+                    label="Scraping Method",
+                    value="Individual URLs"
+                )
+                url_input = gr.Textbox(
+                    label="Article URLs or Base URL",
+                    placeholder="Enter article URLs here, one per line, or base URL for sitemap/URL level/recursive scraping",
+                    lines=5
+                )
+                url_level = gr.Slider(
+                    minimum=1,
+                    maximum=10,
+                    step=1,
+                    label="URL Level (for URL Level scraping)",
+                    value=2,
+                    visible=False
+                )
+                max_pages = gr.Slider(
+                    minimum=1,
+                    maximum=100,
+                    step=1,
+                    label="Maximum Pages to Scrape (for Recursive Scraping)",
+                    value=10,
+                    visible=False
+                )
+                max_depth = gr.Slider(
+                    minimum=1,
+                    maximum=10,
+                    step=1,
+                    label="Maximum Depth (for Recursive Scraping)",
+                    value=3,
+                    visible=False
+                )
+                custom_article_title_input = gr.Textbox(
+                    label="Custom Article Titles (Optional, one per line)",
+                    placeholder="Enter custom titles for the articles, one per line",
+                    lines=5
+                )
+                with gr.Row():
+                    summarize_checkbox = gr.Checkbox(label="Summarize/Analyze Articles", value=False)
+                    custom_prompt_checkbox = gr.Checkbox(label="Use a Custom Prompt", value=False, visible=True)
+                    preset_prompt_checkbox = gr.Checkbox(label="Use a pre-set Prompt", value=False, visible=True)
+                with gr.Row():
+                    temp_slider = gr.Slider(0.1, 2.0, 0.7, label="Temperature")
+
+                # Initialize state variables for pagination
+                current_page_state = gr.State(value=1)
+                total_pages_state = gr.State(value=1)
+                with gr.Row():
+                    # Add pagination controls
+                    preset_prompt = gr.Dropdown(
+                        label="Select Preset Prompt",
+                        choices=[],
+                        visible=False
+                    )
+                with gr.Row():
+                    prev_page_button = gr.Button("Previous Page", visible=False)
+                    page_display = gr.Markdown("Page 1 of X", visible=False)
+                    next_page_button = gr.Button("Next Page", visible=False)
+
+                with gr.Row():
+                    website_custom_prompt_input = gr.Textbox(
+                        label="Custom Prompt",
+                        placeholder="Enter custom prompt here",
+                        lines=3,
+                        visible=False
+                    )
+                with gr.Row():
+                    system_prompt_input = gr.Textbox(
+                        label="System Prompt",
+                        value="""<s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
+                                **Bulleted Note Creation Guidelines**
+                                
+                                **Headings**:
+                                - Based on referenced topics, not categories like quotes or terms
+                                - Surrounded by **bold** formatting 
+                                - Not listed as bullet points
+                                - No space between headings and list items underneath
+                                
+                                **Emphasis**:
+                                - **Important terms** set in bold font
+                                - **Text ending in a colon**: also bolded
+                                
+                                **Review**:
+                                - Ensure adherence to specified format
+                                - Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]
+                                """,
+                        lines=3,
+                        visible=False
+                    )
+
+                # Refactored API selection dropdown
+                api_name_input = gr.Dropdown(
+                    choices=["None"] + [format_api_name(api) for api in global_api_endpoints],
+                    value=default_value,
+                    label="API for Summarization/Analysis (Optional)"
+                )
+                api_key_input = gr.Textbox(
+                    label="API Key (Mandatory if API Name is specified)",
+                    placeholder="Enter your API key here; Ignore if using Local API or Built-in API",
+                    type="password"
+                )
+                custom_cookies_input = gr.Textbox(
+                    label="Custom Cookies (JSON format)",
+                    placeholder="Enter custom cookies in JSON format",
+                    lines=3,
+                    visible=True
+                )
+                keywords_input = gr.Textbox(
+                    label="Keywords",
+                    placeholder="Enter keywords here (comma-separated)",
+                    value="default,no_keyword_set",
+                    visible=True
+                )
+                bookmarks_file_input = gr.File(
+                    label="Upload Bookmarks File/CSV",
+                    type="filepath",
+                    file_types=[".json", ".html", ".csv"],  # Added .csv
+                    visible=True
+                )
+                gr.Markdown("""
+                Supported file formats:
+                - Chrome/Edge bookmarks (JSON)
+                - Firefox bookmarks (HTML)
+                - CSV file with 'url' column (optionally 'title' or 'name' column)
+                """)
+                parsed_urls_output = gr.Textbox(
+                    label="Parsed URLs",
+                    placeholder="URLs will be displayed here after uploading a file.",
+                    lines=10,
+                    interactive=False,
+                    visible=False
+                )
+
+                scrape_button = gr.Button("Scrape and Summarize")
+
+            with gr.Column():
+                progress_output = gr.Textbox(label="Progress", lines=3)
+                result_output = gr.Textbox(label="Result", lines=20)
+
+        def update_ui_for_scrape_method(method):
+            url_level_update = gr.update(visible=(method == "URL Level"))
+            max_pages_update = gr.update(visible=(method == "Recursive Scraping"))
+            max_depth_update = gr.update(visible=(method == "Recursive Scraping"))
+            url_input_update = gr.update(
+                label="Article URLs" if method == "Individual URLs" else "Base URL",
+                placeholder="Enter article URLs here, one per line" if method == "Individual URLs" else "Enter the base URL for scraping"
+            )
+            return url_level_update, max_pages_update, max_depth_update, url_input_update
+
+        scrape_method.change(
+            fn=update_ui_for_scrape_method,
+            inputs=[scrape_method],
+            outputs=[url_level, max_pages, max_depth, url_input]
+        )
+
+        custom_prompt_checkbox.change(
+            fn=lambda x: (gr.update(visible=x), gr.update(visible=x)),
+            inputs=[custom_prompt_checkbox],
+            outputs=[website_custom_prompt_input, system_prompt_input]
+        )
+
+        def on_preset_prompt_checkbox_change(is_checked):
+            if is_checked:
+                prompts, total_pages, current_page = list_prompts(page=1, per_page=20)
+                page_display_text = f"Page {current_page} of {total_pages}"
+                return (
+                    gr.update(visible=True, interactive=True, choices=prompts),  # preset_prompt
+                    gr.update(visible=True),  # prev_page_button
+                    gr.update(visible=True),  # next_page_button
+                    gr.update(value=page_display_text, visible=True),  # page_display
+                    current_page,  # current_page_state
+                    total_pages   # total_pages_state
+                )
+            else:
+                return (
+                    gr.update(visible=False, interactive=False),  # preset_prompt
+                    gr.update(visible=False),  # prev_page_button
+                    gr.update(visible=False),  # next_page_button
+                    gr.update(visible=False),  # page_display
+                    1,  # current_page_state
+                    1   # total_pages_state
+                )
+
+        preset_prompt_checkbox.change(
+            fn=on_preset_prompt_checkbox_change,
+            inputs=[preset_prompt_checkbox],
+            outputs=[preset_prompt, prev_page_button, next_page_button, page_display, current_page_state, total_pages_state]
+        )
+
+        def on_prev_page_click(current_page, total_pages):
+            new_page = max(current_page - 1, 1)
+            prompts, total_pages, current_page = list_prompts(page=new_page, per_page=20)
+            page_display_text = f"Page {current_page} of {total_pages}"
+            return gr.update(choices=prompts), gr.update(value=page_display_text), current_page
+
+        prev_page_button.click(
+            fn=on_prev_page_click,
+            inputs=[current_page_state, total_pages_state],
+            outputs=[preset_prompt, page_display, current_page_state]
+        )
+
+        def on_next_page_click(current_page, total_pages):
+            new_page = min(current_page + 1, total_pages)
+            prompts, total_pages, current_page = list_prompts(page=new_page, per_page=20)
+            page_display_text = f"Page {current_page} of {total_pages}"
+            return gr.update(choices=prompts), gr.update(value=page_display_text), current_page
+
+        next_page_button.click(
+            fn=on_next_page_click,
+            inputs=[current_page_state, total_pages_state],
+            outputs=[preset_prompt, page_display, current_page_state]
+        )
+
+        def update_prompts(preset_name):
+            prompts = update_user_prompt(preset_name)
+            return (
+                gr.update(value=prompts["user_prompt"], visible=True),
+                gr.update(value=prompts["system_prompt"], visible=True)
+            )
+
+        preset_prompt.change(
+            update_prompts,
+            inputs=[preset_prompt],
+            outputs=[website_custom_prompt_input, system_prompt_input]
+        )
+
+        def parse_bookmarks(file_path):
+            """
+            Parses the uploaded bookmarks file and extracts URLs.
+
+            Args:
+                file_path (str): Path to the uploaded bookmarks file.
+
+            Returns:
+                str: Formatted string of extracted URLs or error message.
+            """
+            try:
+                bookmarks = collect_bookmarks(file_path)
+                # Extract URLs
+                urls = []
+                for value in bookmarks.values():
+                    if isinstance(value, list):
+                        urls.extend(value)
+                    elif isinstance(value, str):
+                        urls.append(value)
+                if not urls:
+                    return "No URLs found in the bookmarks file."
+                # Format URLs for display
+                formatted_urls = "\n".join(urls)
+                return formatted_urls
+            except Exception as e:
+                logging.error(f"Error parsing bookmarks file: {str(e)}")
+                return f"Error parsing bookmarks file: {str(e)}"
+
+        def show_parsed_urls(urls_file):
+            """
+            Determines whether to show the parsed URLs output.
+
+            Args:
+                urls_file: Uploaded file object.
+
+            Returns:
+                Tuple indicating visibility and content of parsed_urls_output.
+            """
+            if urls_file is None:
+                return gr.update(visible=False), ""
+
+            file_path = urls_file.name
+            try:
+                # Use the unified collect_urls_from_file function
+                parsed_urls = collect_urls_from_file(file_path)
+
+                # Format the URLs for display
+                formatted_urls = []
+                for name, urls in parsed_urls.items():
+                    if isinstance(urls, list):
+                        for url in urls:
+                            formatted_urls.append(f"{name}: {url}")
+                    else:
+                        formatted_urls.append(f"{name}: {urls}")
+
+                return gr.update(visible=True), "\n".join(formatted_urls)
+            except Exception as e:
+                return gr.update(visible=True), f"Error parsing file: {str(e)}"
+
+        # Connect the parsing function to the file upload event
+        bookmarks_file_input.change(
+            fn=show_parsed_urls,
+            inputs=[bookmarks_file_input],
+            outputs=[parsed_urls_output, parsed_urls_output]
+        )
+
+        async def scrape_and_summarize_wrapper(
+                scrape_method: str,
+                url_input: str,
+                url_level: Optional[int],
+                max_pages: int,
+                max_depth: int,
+                summarize_checkbox: bool,
+                custom_prompt: Optional[str],
+                api_name: Optional[str],
+                api_key: Optional[str],
+                keywords: str,
+                custom_titles: Optional[str],
+                system_prompt: Optional[str],
+                temperature: float,
+                custom_cookies: Optional[str],
+                bookmarks_file,
+                progress: gr.Progress = gr.Progress()
+        ) -> str:
+            try:
+                result: List[Dict[str, Any]] = []
+
+                # Handle bookmarks file if provided
+                if bookmarks_file is not None:
+                    bookmarks = collect_bookmarks(bookmarks_file.name)
+                    # Extract URLs from bookmarks
+                    urls_from_bookmarks = []
+                    for value in bookmarks.values():
+                        if isinstance(value, list):
+                            urls_from_bookmarks.extend(value)
+                        elif isinstance(value, str):
+                            urls_from_bookmarks.append(value)
+                    if scrape_method == "Individual URLs":
+                        url_input = "\n".join(urls_from_bookmarks)
+                    else:
+                        if urls_from_bookmarks:
+                            url_input = urls_from_bookmarks[0]
+                        else:
+                            return convert_json_to_markdown(json.dumps({"error": "No URLs found in the bookmarks file."}))
+
+                # Handle custom cookies
+                custom_cookies_list = None
+                if custom_cookies:
+                    try:
+                        custom_cookies_list = json.loads(custom_cookies)
+                        if not isinstance(custom_cookies_list, list):
+                            custom_cookies_list = [custom_cookies_list]
+                    except json.JSONDecodeError as e:
+                        return convert_json_to_markdown(json.dumps({"error": f"Invalid JSON format for custom cookies: {e}"}))
+
+                if scrape_method == "Individual URLs":
+                    result = await scrape_and_summarize_multiple(url_input, custom_prompt, api_name, api_key, keywords,
+                                                                 custom_titles, system_prompt, summarize_checkbox, custom_cookies=custom_cookies_list)
+                elif scrape_method == "Sitemap":
+                    result = await asyncio.to_thread(scrape_from_sitemap, url_input)
+                elif scrape_method == "URL Level":
+                    if url_level is None:
+                        return convert_json_to_markdown(
+                            json.dumps({"error": "URL level is required for URL Level scraping."}))
+                    result = await asyncio.to_thread(scrape_by_url_level, url_input, url_level)
+                elif scrape_method == "Recursive Scraping":
+                    result = await recursive_scrape(url_input, max_pages, max_depth, progress.update, delay=1.0,
+                                                    custom_cookies=custom_cookies_list)
+                else:
+                    return convert_json_to_markdown(json.dumps({"error": f"Unknown scraping method: {scrape_method}"}))
+
+                # Ensure result is always a list of dictionaries
+                if isinstance(result, dict):
+                    result = [result]
+                elif isinstance(result, list):
+                    if all(isinstance(item, str) for item in result):
+                        # Convert list of strings to list of dictionaries
+                        result = [{"content": item} for item in result]
+                    elif not all(isinstance(item, dict) for item in result):
+                        raise ValueError("Not all items in result are dictionaries or strings")
+                else:
+                    raise ValueError(f"Unexpected result type: {type(result)}")
+
+                # Ensure all items in result are dictionaries
+                if not all(isinstance(item, dict) for item in result):
+                    raise ValueError("Not all items in result are dictionaries")
+
+                if summarize_checkbox:
+                    total_articles = len(result)
+                    for i, article in enumerate(result):
+                        progress.update(f"Summarizing article {i + 1}/{total_articles}")
+                        content = article.get('content', '')
+                        if content:
+                            summary = await asyncio.to_thread(summarize, content, custom_prompt, api_name, api_key,
+                                                              temperature, system_prompt)
+                            article['summary'] = summary
+                        else:
+                            article['summary'] = "No content available to summarize."
+
+                # Concatenate all content
+                all_content = "\n\n".join(
+                    [f"# {article.get('title', 'Untitled')}\n\n{article.get('content', '')}\n\n" +
+                     (f"Summary: {article.get('summary', '')}" if summarize_checkbox else "")
+                     for article in result])
+
+                # Collect all unique URLs
+                all_urls = list(set(article.get('url', '') for article in result if article.get('url')))
+
+                # Structure the output for the entire website collection
+                website_collection = {
+                    "base_url": url_input,
+                    "scrape_method": scrape_method,
+                    "summarization_performed": summarize_checkbox,
+                    "api_used": api_name if summarize_checkbox else None,
+                    "keywords": keywords if summarize_checkbox else None,
+                    "url_level": url_level if scrape_method == "URL Level" else None,
+                    "max_pages": max_pages if scrape_method == "Recursive Scraping" else None,
+                    "max_depth": max_depth if scrape_method == "Recursive Scraping" else None,
+                    "total_articles_scraped": len(result),
+                    "urls_scraped": all_urls,
+                    "content": all_content
+                }
+
+                # Convert the JSON to markdown and return
+                return convert_json_to_markdown(json.dumps(website_collection, indent=2))
+            except Exception as e:
+                return convert_json_to_markdown(json.dumps({"error": f"An error occurred: {str(e)}"}))
+
+        # Update the scrape_button.click to include the temperature parameter
+        scrape_button.click(
+            fn=lambda *args: asyncio.run(scrape_and_summarize_wrapper(*args)),
+            inputs=[scrape_method, url_input, url_level, max_pages, max_depth, summarize_checkbox,
+                    website_custom_prompt_input, api_name_input, api_key_input, keywords_input,
+                    custom_article_title_input, system_prompt_input, temp_slider,
+                    custom_cookies_input, bookmarks_file_input],
+            outputs=[result_output]
+        )
+
+
+def convert_json_to_markdown(json_str: str) -> str:
+    """
+    Converts the JSON output from the scraping process into a markdown format.
+
+    Args:
+        json_str (str): JSON-formatted string containing the website collection data
+
+    Returns:
+        str: Markdown-formatted string of the website collection data
+    """
+    try:
+        # Parse the JSON string
+        data = json.loads(json_str)
+
+        # Check if there's an error in the JSON
+        if "error" in data:
+            return f"# Error\n\n{data['error']}"
+
+        # Start building the markdown string
+        markdown = f"# Website Collection: {data['base_url']}\n\n"
+
+        # Add metadata
+        markdown += "## Metadata\n\n"
+        markdown += f"- **Scrape Method:** {data['scrape_method']}\n"
+        markdown += f"- **API Used:** {data['api_used']}\n"
+        markdown += f"- **Keywords:** {data['keywords']}\n"
+        if data.get('url_level') is not None:
+            markdown += f"- **URL Level:** {data['url_level']}\n"
+        if data.get('max_pages') is not None:
+            markdown += f"- **Maximum Pages:** {data['max_pages']}\n"
+        if data.get('max_depth') is not None:
+            markdown += f"- **Maximum Depth:** {data['max_depth']}\n"
+        markdown += f"- **Total Articles Scraped:** {data['total_articles_scraped']}\n\n"
+
+        # Add URLs Scraped
+        markdown += "## URLs Scraped\n\n"
+        for url in data['urls_scraped']:
+            markdown += f"- {url}\n"
+        markdown += "\n"
+
+        # Add the content
+        markdown += "## Content\n\n"
+        markdown += data['content']
+
+        return markdown
+
+    except json.JSONDecodeError:
+        return "# Error\n\nInvalid JSON string provided."
+    except KeyError as e:
+        return f"# Error\n\nMissing key in JSON data: {str(e)}"
+    except Exception as e:
+        return f"# Error\n\nAn unexpected error occurred: {str(e)}"
+
+#
+# End of File
+########################################################################################################################

App_Function_Libraries/Gradio_UI/Workflows_tab.py

@@ -0,0 +1,190 @@
+# Chat_Workflows.py
+# Description: Gradio UI for Chat Workflows
+#
+# Imports
+import json
+import logging
+from pathlib import Path
+#
+# External Imports
+import gradio as gr
+#
+# Local Imports
+from App_Function_Libraries.Gradio_UI.Chat_ui import chat_wrapper, search_conversations, \
+    load_conversation
+from App_Function_Libraries.Chat.Chat_Functions import save_chat_history_to_db_wrapper
+from App_Function_Libraries.Utils.Utils import default_api_endpoint, global_api_endpoints, format_api_name
+#
+############################################################################################################
+#
+# Functions:
+
+# Load workflows from a JSON file
+json_path = Path('./Helper_Scripts/Workflows/Workflows.json')
+with json_path.open('r') as f:
+    workflows = json.load(f)
+
+
+def chat_workflows_tab():
+    try:
+        default_value = None
+        if default_api_endpoint:
+            if default_api_endpoint in global_api_endpoints:
+                default_value = format_api_name(default_api_endpoint)
+            else:
+                logging.warning(f"Default API endpoint '{default_api_endpoint}' not found in global_api_endpoints")
+    except Exception as e:
+        logging.error(f"Error setting default API endpoint: {str(e)}")
+        default_value = None
+    with gr.TabItem("Chat Workflows", visible=True):
+        gr.Markdown("# Workflows using LLMs")
+        chat_history = gr.State([])
+        media_content = gr.State({})
+        selected_parts = gr.State([])
+        conversation_id = gr.State(None)
+        workflow_state = gr.State({"current_step": 0, "max_steps": 0, "conversation_id": None})
+
+        with gr.Row():
+            with gr.Column():
+                workflow_selector = gr.Dropdown(label="Select Workflow", choices=[wf['name'] for wf in workflows])
+                # Refactored API selection dropdown
+                api_selector = gr.Dropdown(
+                    choices=["None"] + [format_api_name(api) for api in global_api_endpoints],
+                    value=default_value,
+                    label="API for Interaction (Optional)"
+                )
+                api_key_input = gr.Textbox(label="API Key (optional)", type="password")
+                temperature = gr.Slider(label="Temperature", minimum=0.00, maximum=1.0, step=0.05, value=0.7)
+                save_conversation = gr.Checkbox(label="Save Conversation", value=False)
+            with gr.Column():
+                gr.Markdown("Placeholder")
+        with gr.Row():
+            with gr.Column():
+                conversation_search = gr.Textbox(label="Search Conversations")
+                search_conversations_btn = gr.Button("Search Conversations")
+            with gr.Column():
+                previous_conversations = gr.Dropdown(label="Select Conversation", choices=[], interactive=True)
+                load_conversations_btn = gr.Button("Load Selected Conversation")
+        with gr.Row():
+            with gr.Column():
+                context_input = gr.Textbox(label="Initial Context", lines=5)
+                chatbot = gr.Chatbot(label="Workflow Chat")
+                msg = gr.Textbox(label="Your Input")
+                submit_btn = gr.Button("Submit")
+                clear_btn = gr.Button("Clear Chat")
+                chat_media_name = gr.Textbox(label="Custom Chat Name(optional)")
+                save_btn = gr.Button("Save Chat to Database")
+                save_status = gr.Textbox(label="Save Status", interactive=False)
+
+        def update_workflow_ui(workflow_name):
+            if not workflow_name:
+                return {"current_step": 0, "max_steps": 0, "conversation_id": None}, "", []
+            selected_workflow = next((wf for wf in workflows if wf['name'] == workflow_name), None)
+            if selected_workflow:
+                num_prompts = len(selected_workflow['prompts'])
+                context = selected_workflow.get('context', '')
+                first_prompt = selected_workflow['prompts'][0]
+                initial_chat = [(None, f"{first_prompt}")]
+                logging.info(f"Initializing workflow: {workflow_name} with {num_prompts} steps")
+                return {"current_step": 0, "max_steps": num_prompts, "conversation_id": None}, context, initial_chat
+            else:
+                logging.error(f"Selected workflow not found: {workflow_name}")
+                return {"current_step": 0, "max_steps": 0, "conversation_id": None}, "", []
+
+        def process_workflow_step(message, history, context, workflow_name, api_endpoint, api_key, workflow_state,
+                                  save_conv, temp):
+            logging.info(f"Process workflow step called with message: {message}")
+            logging.info(f"Current workflow state: {workflow_state}")
+            try:
+                selected_workflow = next((wf for wf in workflows if wf['name'] == workflow_name), None)
+                if not selected_workflow:
+                    logging.error(f"Selected workflow not found: {workflow_name}")
+                    return history, workflow_state, gr.update(interactive=True)
+
+                current_step = workflow_state["current_step"]
+                max_steps = workflow_state["max_steps"]
+
+                logging.info(f"Current step: {current_step}, Max steps: {max_steps}")
+
+                if current_step >= max_steps:
+                    logging.info("Workflow completed, disabling input")
+                    return history, workflow_state, gr.update(interactive=False)
+
+                prompt = selected_workflow['prompts'][current_step]
+                full_message = f"{context}\n\nStep {current_step + 1}: {prompt}\nUser: {message}"
+
+                logging.info(f"Calling chat_wrapper with full_message: {full_message[:100]}...")
+                bot_message, new_history, new_conversation_id = chat_wrapper(
+                    full_message, history, media_content.value, selected_parts.value,
+                    api_endpoint, api_key, "", workflow_state["conversation_id"],
+                    save_conv, temp, "You are a helpful assistant guiding through a workflow."
+                )
+
+                logging.info(f"Received bot_message: {bot_message[:100]}...")
+
+                next_step = current_step + 1
+                new_workflow_state = {
+                    "current_step": next_step,
+                    "max_steps": max_steps,
+                    "conversation_id": new_conversation_id
+                }
+
+                if next_step >= max_steps:
+                    logging.info("Workflow completed after this step")
+                    return new_history, new_workflow_state, gr.update(interactive=False)
+                else:
+                    next_prompt = selected_workflow['prompts'][next_step]
+                    new_history.append((None, f"Step {next_step + 1}: {next_prompt}"))
+                    logging.info(f"Moving to next step: {next_step}")
+                    return new_history, new_workflow_state, gr.update(interactive=True)
+            except Exception as e:
+                logging.error(f"Error in process_workflow_step: {str(e)}")
+                return history, workflow_state, gr.update(interactive=True)
+
+        workflow_selector.change(
+            update_workflow_ui,
+            inputs=[workflow_selector],
+            outputs=[workflow_state, context_input, chatbot]
+        )
+
+        submit_btn.click(
+            process_workflow_step,
+            inputs=[msg, chatbot, context_input, workflow_selector, api_selector, api_key_input, workflow_state,
+                    save_conversation, temperature],
+            outputs=[chatbot, workflow_state, msg]
+        ).then(
+            lambda: gr.update(value=""),
+            outputs=[msg]
+        )
+
+        clear_btn.click(
+            lambda: ([], {"current_step": 0, "max_steps": 0, "conversation_id": None}, ""),
+            outputs=[chatbot, workflow_state, context_input]
+        )
+
+        save_btn.click(
+            save_chat_history_to_db_wrapper,
+            inputs=[chatbot, conversation_id, media_content, chat_media_name],
+            outputs=[conversation_id, save_status]
+        )
+
+        search_conversations_btn.click(
+            search_conversations,
+            inputs=[conversation_search],
+            outputs=[previous_conversations]
+        )
+
+        load_conversations_btn.click(
+            lambda: ([], {"current_step": 0, "max_steps": 0, "conversation_id": None}, ""),
+            outputs=[chatbot, workflow_state, context_input]
+        ).then(
+            load_conversation,
+            inputs=[previous_conversations],
+            outputs=[chatbot, conversation_id]
+        )
+
+    return workflow_selector, api_selector, api_key_input, context_input, chatbot, msg, submit_btn, clear_btn, save_btn
+
+#
+# End of script
+############################################################################################################

App_Function_Libraries/Gradio_UI/Writing_tab.py

@@ -4,11 +4,16 @@
 # Imports
 #
 # External Imports
+import logging
+
 import gradio as gr
 import textstat
 #
 # Local Imports
 from App_Function_Libraries.Summarization.Summarization_General_Lib import perform_summarization
+from App_Function_Libraries.Utils.Utils import default_api_endpoint, global_api_endpoints, format_api_name
+
+
 #
 ########################################################################################################################
 #
@@ -42,6 +47,16 @@ def grammar_style_check(input_text, custom_prompt, api_name, api_key, system_pro
 
 def create_grammar_style_check_tab():
     with gr.TabItem("Grammar and Style Check", visible=True):
+        try:
+            default_value = None
+            if default_api_endpoint:
+                if default_api_endpoint in global_api_endpoints:
+                    default_value = format_api_name(default_api_endpoint)
+                else:
+                    logging.warning(f"Default API endpoint '{default_api_endpoint}' not found in global_api_endpoints")
+        except Exception as e:
+            logging.error(f"Error setting default API endpoint: {str(e)}")
+            default_value = None
         with gr.Row():
             with gr.Column():
                 gr.Markdown("# Grammar and Style Check")
@@ -74,11 +89,11 @@ def create_grammar_style_check_tab():
                     inputs=[custom_prompt_checkbox],
                     outputs=[custom_prompt_input, system_prompt_input]
                 )
+                # Refactored API selection dropdown
                 api_name_input = gr.Dropdown(
-                    choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral", "OpenRouter",
-                             "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM","ollama", "HuggingFace", "Custom-OpenAI-API"],
-                    value=None,
-                    label="API for Grammar Check"
+                    choices=["None"] + [format_api_name(api) for api in global_api_endpoints],
+                    value=default_value,
+                    label="API for Analysis (Optional)"
                 )
                 api_key_input = gr.Textbox(label="API Key (if not set in Config_Files/config.txt)", placeholder="Enter your API key here",
                                                type="password")
@@ -302,63 +317,63 @@ def create_document_feedback_tab():
         with gr.Row():
             compare_button = gr.Button("Compare Feedback")
 
-    feedback_history = gr.State([])
-
-    def add_custom_persona(name, description):
-        updated_choices = persona_dropdown.choices + [name]
-        persona_prompts[name] = f"As {name}, {description}, provide feedback on the following text:"
-        return gr.update(choices=updated_choices)
-
-    def update_feedback_history(current_text, persona, feedback):
-        # Ensure feedback_history.value is initialized and is a list
-        if feedback_history.value is None:
-            feedback_history.value = []
-
-        history = feedback_history.value
-
-        # Append the new entry to the history
-        history.append({"text": current_text, "persona": persona, "feedback": feedback})
-
-        # Keep only the last 5 entries in the history
-        feedback_history.value = history[-10:]
-
-        # Generate and return the updated HTML
-        return generate_feedback_history_html(feedback_history.value)
-
-    def compare_feedback(text, selected_personas, api_name, api_key):
-        results = []
-        for persona in selected_personas:
-            feedback = generate_writing_feedback(text, persona, "Overall", api_name, api_key)
-            results.append(f"### {persona}'s Feedback:\n{feedback}\n\n")
-        return "\n".join(results)
-
-    add_custom_persona_button.click(
-        fn=add_custom_persona,
-        inputs=[custom_persona_name, custom_persona_description],
-        outputs=persona_dropdown
-    )
-
-    get_feedback_button.click(
-        fn=lambda text, persona, aspect, api_name, api_key: (
-            generate_writing_feedback(text, persona, aspect, api_name, api_key),
-            calculate_readability(text),
-            update_feedback_history(text, persona, generate_writing_feedback(text, persona, aspect, api_name, api_key))
-        ),
-        inputs=[input_text, persona_dropdown, aspect_dropdown, api_name_input, api_key_input],
-        outputs=[feedback_output, readability_output, feedback_history_display]
-    )
-
-    compare_button.click(
-        fn=compare_feedback,
-        inputs=[input_text, compare_personas, api_name_input, api_key_input],
-        outputs=feedback_output
-    )
-
-    generate_prompt_button.click(
-        fn=generate_writing_prompt,
-        inputs=[persona_dropdown, api_name_input, api_key_input],
-        outputs=input_text
-    )
+        feedback_history = gr.State([])
+
+        def add_custom_persona(name, description):
+            updated_choices = persona_dropdown.choices + [name]
+            persona_prompts[name] = f"As {name}, {description}, provide feedback on the following text:"
+            return gr.update(choices=updated_choices)
+
+        def update_feedback_history(current_text, persona, feedback):
+            # Ensure feedback_history.value is initialized and is a list
+            if feedback_history.value is None:
+                feedback_history.value = []
+
+            history = feedback_history.value
+
+            # Append the new entry to the history
+            history.append({"text": current_text, "persona": persona, "feedback": feedback})
+
+            # Keep only the last 5 entries in the history
+            feedback_history.value = history[-10:]
+
+            # Generate and return the updated HTML
+            return generate_feedback_history_html(feedback_history.value)
+
+        def compare_feedback(text, selected_personas, api_name, api_key):
+            results = []
+            for persona in selected_personas:
+                feedback = generate_writing_feedback(text, persona, "Overall", api_name, api_key)
+                results.append(f"### {persona}'s Feedback:\n{feedback}\n\n")
+            return "\n".join(results)
+
+        add_custom_persona_button.click(
+            fn=add_custom_persona,
+            inputs=[custom_persona_name, custom_persona_description],
+            outputs=persona_dropdown
+        )
+
+        get_feedback_button.click(
+            fn=lambda text, persona, aspect, api_name, api_key: (
+                generate_writing_feedback(text, persona, aspect, api_name, api_key),
+                calculate_readability(text),
+                update_feedback_history(text, persona, generate_writing_feedback(text, persona, aspect, api_name, api_key))
+            ),
+            inputs=[input_text, persona_dropdown, aspect_dropdown, api_name_input, api_key_input],
+            outputs=[feedback_output, readability_output, feedback_history_display]
+        )
+
+        compare_button.click(
+            fn=compare_feedback,
+            inputs=[input_text, compare_personas, api_name_input, api_key_input],
+            outputs=feedback_output
+        )
+
+        generate_prompt_button.click(
+            fn=generate_writing_prompt,
+            inputs=[persona_dropdown, api_name_input, api_key_input],
+            outputs=input_text
+        )
 
     return input_text, feedback_output, readability_output, feedback_history_display