Delete App_Function_Libraries

App_Function_Libraries/Audio/Audio_Files.py

@@ -1,786 +0,0 @@
-# Audio_Files.py
-#########################################
-# Audio Processing Library
-# This library is used to download or load audio files from a local directory.
-#
-####
-#
-# Functions:
-#
-# download_audio_file(url, save_path)
-# process_audio(
-# process_audio_file(audio_url, audio_file, whisper_model="small.en", api_name=None, api_key=None)
-#
-#
-#########################################
-# Imports
-import json
-import logging
-import os
-import subprocess
-import tempfile
-import time
-import uuid
-from datetime import datetime
-from pathlib import Path
-#
-# External Imports
-import requests
-import yt_dlp
-#
-# Local Imports
-from App_Function_Libraries.DB.DB_Manager import add_media_with_keywords, \
-    check_media_and_whisper_model
-from App_Function_Libraries.Metrics.metrics_logger import log_counter, log_histogram
-from App_Function_Libraries.Summarization.Summarization_General_Lib import perform_summarization
-from App_Function_Libraries.Utils.Utils import downloaded_files, \
-    sanitize_filename, generate_unique_id, temp_files
-from App_Function_Libraries.Video_DL_Ingestion_Lib import extract_metadata
-from App_Function_Libraries.Audio.Audio_Transcription_Lib import speech_to_text
-from App_Function_Libraries.Chunk_Lib import improved_chunking_process
-#
-#######################################################################################################################
-# Function Definitions
-#
-
-MAX_FILE_SIZE = 500 * 1024 * 1024
-
-
-def download_audio_file(url, current_whisper_model="", use_cookies=False, cookies=None):
-    try:
-        # Check if media already exists in the database and compare whisper models
-        should_download, reason = check_media_and_whisper_model(
-            url=url,
-            current_whisper_model=current_whisper_model
-        )
-
-        if not should_download:
-            logging.info(f"Skipping audio download: {reason}")
-            return None
-
-        logging.info(f"Proceeding with audio download: {reason}")
-
-        # Set up the request headers
-        headers = {}
-        if use_cookies and cookies:
-            try:
-                cookie_dict = json.loads(cookies)
-                headers['Cookie'] = '; '.join([f'{k}={v}' for k, v in cookie_dict.items()])
-            except json.JSONDecodeError:
-                logging.warning("Invalid cookie format. Proceeding without cookies.")
-
-        # Make the request
-        response = requests.get(url, headers=headers, stream=True)
-        # Raise an exception for bad status codes
-        response.raise_for_status()
-
-        # Get the file size
-        file_size = int(response.headers.get('content-length', 0))
-        if file_size > 500 * 1024 * 1024:  # 500 MB limit
-            raise ValueError("File size exceeds the 500MB limit.")
-
-        # Generate a unique filename
-        file_name = f"audio_{uuid.uuid4().hex[:8]}.mp3"
-        save_path = os.path.join('downloads', file_name)
-
-        # Ensure the downloads directory exists
-        os.makedirs('downloads', exist_ok=True)
-
-
-        # Download the file
-        with open(save_path, 'wb') as f:
-            for chunk in response.iter_content(chunk_size=8192):
-                if chunk:
-                    f.write(chunk)
-
-        logging.info(f"Audio file downloaded successfully: {save_path}")
-        return save_path
-
-    except requests.RequestException as e:
-        logging.error(f"Error downloading audio file: {str(e)}")
-        raise
-    except ValueError as e:
-        logging.error(str(e))
-        raise
-    except Exception as e:
-        logging.error(f"Unexpected error downloading audio file: {str(e)}")
-        raise
-
-def process_audio_files(audio_urls, audio_file, whisper_model, api_name, api_key, use_cookies, cookies, keep_original,
-                        custom_keywords, custom_prompt_input, chunk_method, max_chunk_size, chunk_overlap,
-                        use_adaptive_chunking, use_multi_level_chunking, chunk_language, diarize,
-                        keep_timestamps, custom_title):
-
-    start_time = time.time()  # Start time for processing
-    processed_count = 0
-    failed_count = 0
-    progress = []
-    all_transcriptions = []
-    all_summaries = []
-    #v2
-    def format_transcription_with_timestamps(segments):
-        if keep_timestamps:
-            formatted_segments = []
-            for segment in segments:
-                start = segment.get('Time_Start', 0)
-                end = segment.get('Time_End', 0)
-                text = segment.get('Text', '').strip()  # Ensure text is stripped of leading/trailing spaces
-
-                # Add the formatted timestamp and text to the list, followed by a newline
-                formatted_segments.append(f"[{start:.2f}-{end:.2f}] {text}")
-
-            # Join the segments with a newline to ensure proper formatting
-            return "\n".join(formatted_segments)
-        else:
-            # Join the text without timestamps
-            return "\n".join([segment.get('Text', '').strip() for segment in segments])
-
-    def update_progress(message):
-        progress.append(message)
-        return "\n".join(progress)
-
-    def cleanup_files():
-        for file in temp_files:
-            try:
-                if os.path.exists(file):
-                    os.remove(file)
-                    update_progress(f"Temporary file {file} removed.")
-            except Exception as e:
-                update_progress(f"Failed to remove temporary file {file}: {str(e)}")
-
-    def reencode_mp3(mp3_file_path):
-        try:
-            reencoded_mp3_path = mp3_file_path.replace(".mp3", "_reencoded.mp3")
-            subprocess.run([ffmpeg_cmd, '-i', mp3_file_path, '-codec:a', 'libmp3lame', reencoded_mp3_path], check=True)
-            update_progress(f"Re-encoded {mp3_file_path} to {reencoded_mp3_path}.")
-            return reencoded_mp3_path
-        except subprocess.CalledProcessError as e:
-            update_progress(f"Error re-encoding {mp3_file_path}: {str(e)}")
-            raise
-
-    def convert_mp3_to_wav(mp3_file_path):
-        try:
-            wav_file_path = mp3_file_path.replace(".mp3", ".wav")
-            subprocess.run([ffmpeg_cmd, '-i', mp3_file_path, wav_file_path], check=True)
-            update_progress(f"Converted {mp3_file_path} to {wav_file_path}.")
-            return wav_file_path
-        except subprocess.CalledProcessError as e:
-            update_progress(f"Error converting {mp3_file_path} to WAV: {str(e)}")
-            raise
-
-    try:
-        # Check and set the ffmpeg command
-        global ffmpeg_cmd
-        if os.name == "nt":
-            logging.debug("Running on Windows")
-            ffmpeg_cmd = os.path.join(os.getcwd(), "Bin", "ffmpeg.exe")
-        else:
-            ffmpeg_cmd = 'ffmpeg'  # Assume 'ffmpeg' is in PATH for non-Windows systems
-
-        # Ensure ffmpeg is accessible
-        if not os.path.exists(ffmpeg_cmd) and os.name == "nt":
-            raise FileNotFoundError(f"ffmpeg executable not found at path: {ffmpeg_cmd}")
-
-        # Define chunk options early to avoid undefined errors
-        chunk_options = {
-            'method': chunk_method,
-            'max_size': max_chunk_size,
-            'overlap': chunk_overlap,
-            'adaptive': use_adaptive_chunking,
-            'multi_level': use_multi_level_chunking,
-            'language': chunk_language
-        }
-
-        # Process multiple URLs
-        urls = [url.strip() for url in audio_urls.split('\n') if url.strip()]
-
-        for i, url in enumerate(urls):
-            update_progress(f"Processing URL {i + 1}/{len(urls)}: {url}")
-
-            # Download and process audio file
-            audio_file_path = download_audio_file(url, use_cookies, cookies)
-            if not os.path.exists(audio_file_path):
-                update_progress(f"Downloaded file not found: {audio_file_path}")
-                failed_count += 1
-                log_counter(
-                    metric_name="audio_files_failed_total",
-                    labels={"whisper_model": whisper_model, "api_name": api_name},
-                    value=1
-                )
-                continue
-
-            temp_files.append(audio_file_path)
-            update_progress("Audio file downloaded successfully.")
-
-            # Re-encode MP3 to fix potential issues
-            reencoded_mp3_path = reencode_mp3(audio_file_path)
-            if not os.path.exists(reencoded_mp3_path):
-                update_progress(f"Re-encoded file not found: {reencoded_mp3_path}")
-                failed_count += 1
-                log_counter(
-                    metric_name="audio_files_failed_total",
-                    labels={"whisper_model": whisper_model, "api_name": api_name},
-                    value=1
-                )
-                continue
-
-            temp_files.append(reencoded_mp3_path)
-
-            # Convert re-encoded MP3 to WAV
-            wav_file_path = convert_mp3_to_wav(reencoded_mp3_path)
-            if not os.path.exists(wav_file_path):
-                update_progress(f"Converted WAV file not found: {wav_file_path}")
-                failed_count += 1
-                log_counter(
-                    metric_name="audio_files_failed_total",
-                    labels={"whisper_model": whisper_model, "api_name": api_name},
-                    value=1
-                )
-                continue
-
-            temp_files.append(wav_file_path)
-
-            # Initialize transcription
-            transcription = ""
-
-            # Transcribe audio
-            if diarize:
-                segments = speech_to_text(wav_file_path, whisper_model=whisper_model, diarize=True)
-            else:
-                segments = speech_to_text(wav_file_path, whisper_model=whisper_model)
-
-            # Handle segments nested under 'segments' key
-            if isinstance(segments, dict) and 'segments' in segments:
-                segments = segments['segments']
-
-            if isinstance(segments, list):
-                # Log first 5 segments for debugging
-                logging.debug(f"Segments before formatting: {segments[:5]}")
-                transcription = format_transcription_with_timestamps(segments)
-                logging.debug(f"Formatted transcription (first 500 chars): {transcription[:500]}")
-                update_progress("Audio transcribed successfully.")
-            else:
-                update_progress("Unexpected segments format received from speech_to_text.")
-                logging.error(f"Unexpected segments format: {segments}")
-                failed_count += 1
-                log_counter(
-                    metric_name="audio_files_failed_total",
-                    labels={"whisper_model": whisper_model, "api_name": api_name},
-                    value=1
-                )
-                continue
-
-            if not transcription.strip():
-                update_progress("Transcription is empty.")
-                failed_count += 1
-                log_counter(
-                    metric_name="audio_files_failed_total",
-                    labels={"whisper_model": whisper_model, "api_name": api_name},
-                    value=1
-                )
-            else:
-                # Apply chunking
-                chunked_text = improved_chunking_process(transcription, chunk_options)
-
-                # Summarize
-                logging.debug(f"Audio Transcription API Name: {api_name}")
-                if api_name:
-                    try:
-                        summary = perform_summarization(api_name, chunked_text, custom_prompt_input, api_key)
-                        update_progress("Audio summarized successfully.")
-                    except Exception as e:
-                        logging.error(f"Error during summarization: {str(e)}")
-                        summary = "Summary generation failed"
-                        failed_count += 1
-                        log_counter(
-                            metric_name="audio_files_failed_total",
-                            labels={"whisper_model": whisper_model, "api_name": api_name},
-                            value=1
-                        )
-                else:
-                    summary = "No summary available (API not provided)"
-
-                all_transcriptions.append(transcription)
-                all_summaries.append(summary)
-
-                # Use custom_title if provided, otherwise use the original filename
-                title = custom_title if custom_title else os.path.basename(wav_file_path)
-
-                # Add to database
-                add_media_with_keywords(
-                    url=url,
-                    title=title,
-                    media_type='audio',
-                    content=transcription,
-                    keywords=custom_keywords,
-                    prompt=custom_prompt_input,
-                    summary=summary,
-                    transcription_model=whisper_model,
-                    author="Unknown",
-                    ingestion_date=datetime.now().strftime('%Y-%m-%d')
-                )
-                update_progress("Audio file processed and added to database.")
-                processed_count += 1
-                log_counter(
-                    metric_name="audio_files_processed_total",
-                    labels={"whisper_model": whisper_model, "api_name": api_name},
-                    value=1
-                )
-
-        # Process uploaded file if provided
-        if audio_file:
-            url = generate_unique_id()
-            if os.path.getsize(audio_file.name) > MAX_FILE_SIZE:
-                update_progress(
-                    f"Uploaded file size exceeds the maximum limit of {MAX_FILE_SIZE / (1024 * 1024):.2f}MB. Skipping this file.")
-            else:
-                try:
-                    # Re-encode MP3 to fix potential issues
-                    reencoded_mp3_path = reencode_mp3(audio_file.name)
-                    if not os.path.exists(reencoded_mp3_path):
-                        update_progress(f"Re-encoded file not found: {reencoded_mp3_path}")
-                        return update_progress("Processing failed: Re-encoded file not found"), "", ""
-
-                    temp_files.append(reencoded_mp3_path)
-
-                    # Convert re-encoded MP3 to WAV
-                    wav_file_path = convert_mp3_to_wav(reencoded_mp3_path)
-                    if not os.path.exists(wav_file_path):
-                        update_progress(f"Converted WAV file not found: {wav_file_path}")
-                        return update_progress("Processing failed: Converted WAV file not found"), "", ""
-
-                    temp_files.append(wav_file_path)
-
-                    # Initialize transcription
-                    transcription = ""
-
-                    if diarize:
-                        segments = speech_to_text(wav_file_path, whisper_model=whisper_model, diarize=True)
-                    else:
-                        segments = speech_to_text(wav_file_path, whisper_model=whisper_model)
-
-                    # Handle segments nested under 'segments' key
-                    if isinstance(segments, dict) and 'segments' in segments:
-                        segments = segments['segments']
-
-                    if isinstance(segments, list):
-                        transcription = format_transcription_with_timestamps(segments)
-                    else:
-                        update_progress("Unexpected segments format received from speech_to_text.")
-                        logging.error(f"Unexpected segments format: {segments}")
-
-                    chunked_text = improved_chunking_process(transcription, chunk_options)
-
-                    logging.debug(f"Audio Transcription API Name: {api_name}")
-                    if api_name:
-                        try:
-                            summary = perform_summarization(api_name, chunked_text, custom_prompt_input, api_key)
-                            update_progress("Audio summarized successfully.")
-                        except Exception as e:
-                            logging.error(f"Error during summarization: {str(e)}")
-                            summary = "Summary generation failed"
-                    else:
-                        summary = "No summary available (API not provided)"
-
-                    all_transcriptions.append(transcription)
-                    all_summaries.append(summary)
-
-                    # Use custom_title if provided, otherwise use the original filename
-                    title = custom_title if custom_title else os.path.basename(wav_file_path)
-
-                    add_media_with_keywords(
-                        url="Uploaded File",
-                        title=title,
-                        media_type='audio',
-                        content=transcription,
-                        keywords=custom_keywords,
-                        prompt=custom_prompt_input,
-                        summary=summary,
-                        transcription_model=whisper_model,
-                        author="Unknown",
-                        ingestion_date=datetime.now().strftime('%Y-%m-%d')
-                    )
-                    update_progress("Uploaded file processed and added to database.")
-                    processed_count += 1
-                    log_counter(
-                        metric_name="audio_files_processed_total",
-                        labels={"whisper_model": whisper_model, "api_name": api_name},
-                        value=1
-                    )
-                except Exception as e:
-                    update_progress(f"Error processing uploaded file: {str(e)}")
-                    logging.error(f"Error processing uploaded file: {str(e)}")
-                    failed_count += 1
-                    log_counter(
-                        metric_name="audio_files_failed_total",
-                        labels={"whisper_model": whisper_model, "api_name": api_name},
-                        value=1
-                    )
-                    return update_progress("Processing failed: Error processing uploaded file"), "", ""
-        # Final cleanup
-        if not keep_original:
-            cleanup_files()
-
-        end_time = time.time()
-        processing_time = end_time - start_time
-        # Log processing time
-        log_histogram(
-            metric_name="audio_processing_time_seconds",
-            value=processing_time,
-            labels={"whisper_model": whisper_model, "api_name": api_name}
-        )
-
-        # Optionally, log total counts
-        log_counter(
-            metric_name="total_audio_files_processed",
-            labels={"whisper_model": whisper_model, "api_name": api_name},
-            value=processed_count
-        )
-
-        log_counter(
-            metric_name="total_audio_files_failed",
-            labels={"whisper_model": whisper_model, "api_name": api_name},
-            value=failed_count
-        )
-
-
-        final_progress = update_progress("All processing complete.")
-        final_transcriptions = "\n\n".join(all_transcriptions)
-        final_summaries = "\n\n".join(all_summaries)
-
-        return final_progress, final_transcriptions, final_summaries
-
-    except Exception as e:
-        logging.error(f"Error processing audio files: {str(e)}")
-        log_counter(
-            metric_name="audio_files_failed_total",
-            labels={"whisper_model": whisper_model, "api_name": api_name},
-            value=1
-        )
-        cleanup_files()
-        return update_progress(f"Processing failed: {str(e)}"), "", ""
-
-
-def format_transcription_with_timestamps(segments, keep_timestamps):
-    """
-    Formats the transcription segments with or without timestamps.
-
-    Parameters:
-        segments (list): List of transcription segments.
-        keep_timestamps (bool): Whether to include timestamps.
-
-    Returns:
-        str: Formatted transcription.
-    """
-    if keep_timestamps:
-        formatted_segments = []
-        for segment in segments:
-            start = segment.get('Time_Start', 0)
-            end = segment.get('Time_End', 0)
-            text = segment.get('Text', '').strip()
-
-            formatted_segments.append(f"[{start:.2f}-{end:.2f}] {text}")
-        return "\n".join(formatted_segments)
-    else:
-        return "\n".join([segment.get('Text', '').strip() for segment in segments])
-
-
-def download_youtube_audio(url):
-    try:
-        # Determine ffmpeg path based on the operating system.
-        ffmpeg_path = './Bin/ffmpeg.exe' if os.name == 'nt' else 'ffmpeg'
-
-        # Create a temporary directory
-        with tempfile.TemporaryDirectory() as temp_dir:
-            # Extract information about the video
-            with yt_dlp.YoutubeDL({'quiet': True}) as ydl:
-                info_dict = ydl.extract_info(url, download=False)
-                sanitized_title = sanitize_filename(info_dict['title'])
-
-            # Setup the temporary filenames
-            temp_video_path = Path(temp_dir) / f"{sanitized_title}_temp.mp4"
-            temp_audio_path = Path(temp_dir) / f"{sanitized_title}.mp3"
-
-            # Initialize yt-dlp with options for downloading
-            ydl_opts = {
-                'format': 'bestaudio[ext=m4a]/best[height<=480]',  # Prefer best audio, or video up to 480p
-                'ffmpeg_location': ffmpeg_path,
-                'outtmpl': str(temp_video_path),
-                'noplaylist': True,
-                'quiet': True
-            }
-
-            # Execute yt-dlp to download the video/audio
-            with yt_dlp.YoutubeDL(ydl_opts) as ydl:
-                ydl.download([url])
-
-            # Check if the file exists
-            if not temp_video_path.exists():
-                raise FileNotFoundError(f"Expected file was not found: {temp_video_path}")
-
-            # Use ffmpeg to extract audio
-            ffmpeg_command = [
-                ffmpeg_path,
-                '-i', str(temp_video_path),
-                '-vn',  # No video
-                '-acodec', 'libmp3lame',
-                '-b:a', '192k',
-                str(temp_audio_path)
-            ]
-            subprocess.run(ffmpeg_command, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
-
-            # Check if the audio file was created
-            if not temp_audio_path.exists():
-                raise FileNotFoundError(f"Expected audio file was not found: {temp_audio_path}")
-
-            # Create a persistent directory for the download if it doesn't exist
-            persistent_dir = Path("downloads")
-            persistent_dir.mkdir(exist_ok=True)
-
-            # Move the file from the temporary directory to the persistent directory
-            persistent_file_path = persistent_dir / f"{sanitized_title}.mp3"
-            os.replace(str(temp_audio_path), str(persistent_file_path))
-
-            # Add the file to the list of downloaded files
-            downloaded_files.append(str(persistent_file_path))
-
-            return str(persistent_file_path), f"Audio downloaded successfully: {sanitized_title}.mp3"
-    except Exception as e:
-        return None, f"Error downloading audio: {str(e)}"
-
-
-def process_podcast(url, title, author, keywords, custom_prompt, api_name, api_key, whisper_model,
-                    keep_original=False, enable_diarization=False, use_cookies=False, cookies=None,
-                    chunk_method=None, max_chunk_size=300, chunk_overlap=0, use_adaptive_chunking=False,
-                    use_multi_level_chunking=False, chunk_language='english', keep_timestamps=True):
-    """
-    Processes a podcast by downloading the audio, transcribing it, summarizing the transcription,
-    and adding the results to the database. Metrics are logged throughout the process.
-
-    Parameters:
-        url (str): URL of the podcast.
-        title (str): Title of the podcast.
-        author (str): Author of the podcast.
-        keywords (str): Comma-separated keywords.
-        custom_prompt (str): Custom prompt for summarization.
-        api_name (str): API name for summarization.
-        api_key (str): API key for summarization.
-        whisper_model (str): Whisper model to use for transcription.
-        keep_original (bool): Whether to keep the original audio file.
-        enable_diarization (bool): Whether to enable speaker diarization.
-        use_cookies (bool): Whether to use cookies for authenticated downloads.
-        cookies (str): JSON-formatted cookies string.
-        chunk_method (str): Method for chunking text.
-        max_chunk_size (int): Maximum size for each text chunk.
-        chunk_overlap (int): Overlap size between chunks.
-        use_adaptive_chunking (bool): Whether to use adaptive chunking.
-        use_multi_level_chunking (bool): Whether to use multi-level chunking.
-        chunk_language (str): Language for chunking.
-        keep_timestamps (bool): Whether to keep timestamps in transcription.
-
-    Returns:
-        tuple: (progress_message, transcription, summary, title, author, keywords, error_message)
-    """
-    start_time = time.time()  # Start time for processing
-    error_message = ""
-    temp_files = []
-
-    # Define labels for metrics
-    labels = {
-        "whisper_model": whisper_model,
-        "api_name": api_name if api_name else "None"
-    }
-
-    def update_progress(message):
-        """
-        Updates the progress messages.
-
-        Parameters:
-            message (str): Progress message to append.
-
-        Returns:
-            str: Combined progress messages.
-        """
-        progress.append(message)
-        return "\n".join(progress)
-
-    def cleanup_files():
-        if not keep_original:
-            for file in temp_files:
-                try:
-                    if os.path.exists(file):
-                        os.remove(file)
-                        update_progress(f"Temporary file {file} removed.")
-                except Exception as e:
-                    update_progress(f"Failed to remove temporary file {file}: {str(e)}")
-
-    progress = []  # Initialize progress messages
-
-    try:
-        # Handle cookies if required
-        if use_cookies:
-            cookies = json.loads(cookies)
-
-        # Download the podcast audio file
-        audio_file = download_audio_file(url, whisper_model, use_cookies, cookies)
-        if not audio_file:
-            raise RuntimeError("Failed to download podcast audio.")
-        temp_files.append(audio_file)
-        update_progress("Podcast downloaded successfully.")
-
-        # Extract metadata from the podcast
-        metadata = extract_metadata(url)
-        title = title or metadata.get('title', 'Unknown Podcast')
-        author = author or metadata.get('uploader', 'Unknown Author')
-
-        # Format metadata for storage
-        metadata_text = f"""
-Metadata:
-Title: {title}
-Author: {author}
-Series: {metadata.get('series', 'N/A')}
-Episode: {metadata.get('episode', 'N/A')}
-Season: {metadata.get('season', 'N/A')}
-Upload Date: {metadata.get('upload_date', 'N/A')}
-Duration: {metadata.get('duration', 'N/A')} seconds
-Description: {metadata.get('description', 'N/A')}
-"""
-
-        # Update keywords with metadata information
-        new_keywords = []
-        if metadata.get('series'):
-            new_keywords.append(f"series:{metadata['series']}")
-        if metadata.get('episode'):
-            new_keywords.append(f"episode:{metadata['episode']}")
-        if metadata.get('season'):
-            new_keywords.append(f"season:{metadata['season']}")
-
-        keywords = f"{keywords},{','.join(new_keywords)}" if keywords else ','.join(new_keywords)
-        update_progress(f"Metadata extracted - Title: {title}, Author: {author}, Keywords: {keywords}")
-
-        # Transcribe the podcast audio
-        try:
-            if enable_diarization:
-                segments = speech_to_text(audio_file, whisper_model=whisper_model, diarize=True)
-            else:
-                segments = speech_to_text(audio_file, whisper_model=whisper_model)
-            # SEems like this could be optimized... FIXME
-            def format_segment(segment):
-                start = segment.get('start', 0)
-                end = segment.get('end', 0)
-                text = segment.get('Text', '')
-
-            if isinstance(segments, dict) and 'segments' in segments:
-                segments = segments['segments']
-
-            if isinstance(segments, list):
-                transcription = format_transcription_with_timestamps(segments, keep_timestamps)
-                update_progress("Podcast transcribed successfully.")
-            else:
-                raise ValueError("Unexpected segments format received from speech_to_text.")
-
-            if not transcription.strip():
-                raise ValueError("Transcription is empty.")
-        except Exception as e:
-            error_message = f"Transcription failed: {str(e)}"
-            raise RuntimeError(error_message)
-
-        # Apply chunking to the transcription
-        chunk_options = {
-            'method': chunk_method,
-            'max_size': max_chunk_size,
-            'overlap': chunk_overlap,
-            'adaptive': use_adaptive_chunking,
-            'multi_level': use_multi_level_chunking,
-            'language': chunk_language
-        }
-        chunked_text = improved_chunking_process(transcription, chunk_options)
-
-        # Combine metadata and transcription
-        full_content = metadata_text + "\n\nTranscription:\n" + transcription
-
-        # Summarize the transcription if API is provided
-        summary = None
-        if api_name:
-            try:
-                summary = perform_summarization(api_name, chunked_text, custom_prompt, api_key)
-                update_progress("Podcast summarized successfully.")
-            except Exception as e:
-                error_message = f"Summarization failed: {str(e)}"
-                raise RuntimeError(error_message)
-        else:
-            summary = "No summary available (API not provided)"
-
-        # Add the processed podcast to the database
-        try:
-            add_media_with_keywords(
-                url=url,
-                title=title,
-                media_type='podcast',
-                content=full_content,
-                keywords=keywords,
-                prompt=custom_prompt,
-                summary=summary or "No summary available",
-                transcription_model=whisper_model,
-                author=author,
-                ingestion_date=datetime.now().strftime('%Y-%m-%d')
-            )
-            update_progress("Podcast added to database successfully.")
-        except Exception as e:
-            error_message = f"Error adding podcast to database: {str(e)}"
-            raise RuntimeError(error_message)
-
-        # Cleanup temporary files if required
-        cleanup_files()
-
-        # Calculate processing time
-        end_time = time.time()
-        processing_time = end_time - start_time
-
-        # Log successful processing
-        log_counter(
-            metric_name="podcasts_processed_total",
-            labels=labels,
-            value=1
-        )
-
-        # Log processing time
-        log_histogram(
-            metric_name="podcast_processing_time_seconds",
-            value=processing_time,
-            labels=labels
-        )
-
-        # Return the final outputs
-        final_progress = update_progress("Processing complete.")
-        return (final_progress, full_content, summary or "No summary generated.",
-                title, author, keywords, error_message)
-
-    except Exception as e:
-        # Calculate processing time up to the point of failure
-        end_time = time.time()
-        processing_time = end_time - start_time
-
-        # Log failed processing
-        log_counter(
-            metric_name="podcasts_failed_total",
-            labels=labels,
-            value=1
-        )
-
-        # Log processing time even on failure
-        log_histogram(
-            metric_name="podcast_processing_time_seconds",
-            value=processing_time,
-            labels=labels
-        )
-
-        logging.error(f"Error processing podcast: {str(e)}")
-        cleanup_files()
-        final_progress = update_progress(f"Processing failed: {str(e)}")
-        return (final_progress, "", "", "", "", "", str(e))
-
-
-#
-#
-#######################################################################################################################

App_Function_Libraries/Audio/Audio_Transcription_Lib.py

@@ -1,335 +0,0 @@
-# Audio_Transcription_Lib.py
-#########################################
-# Transcription Library
-# This library is used to perform transcription of audio files.
-# Currently, uses faster_whisper for transcription.
-#
-####################
-# Function List
-#
-# 1. convert_to_wav(video_file_path, offset=0, overwrite=False)
-# 2. speech_to_text(audio_file_path, selected_source_lang='en', whisper_model='small.en', vad_filter=False)
-#
-####################
-#
-# Import necessary libraries to run solo for testing
-import gc
-import json
-import logging
-import multiprocessing
-import os
-import queue
-import sys
-import subprocess
-import tempfile
-import threading
-import time
-# DEBUG Imports
-#from memory_profiler import profile
-import pyaudio
-from faster_whisper import WhisperModel as OriginalWhisperModel
-from typing import Optional, Union, List, Dict, Any
-#
-# Import Local
-from App_Function_Libraries.Utils.Utils import load_comprehensive_config
-from App_Function_Libraries.Metrics.metrics_logger import log_counter, log_histogram
-#
-#######################################################################################################################
-# Function Definitions
-#
-
-# Convert video .m4a into .wav using ffmpeg
-#   ffmpeg -i "example.mp4" -ar 16000 -ac 1 -c:a pcm_s16le "output.wav"
-#       https://www.gyan.dev/ffmpeg/builds/
-#
-
-
-whisper_model_instance = None
-config = load_comprehensive_config()
-processing_choice = config.get('Processing', 'processing_choice', fallback='cpu')
-total_thread_count = multiprocessing.cpu_count()
-
-
-class WhisperModel(OriginalWhisperModel):
-    tldw_dir = os.path.dirname(os.path.dirname(__file__))
-    default_download_root = os.path.join(tldw_dir, 'models', 'Whisper')
-
-    valid_model_sizes = [
-        "tiny.en", "tiny", "base.en", "base", "small.en", "small", "medium.en", "medium",
-        "large-v1", "large-v2", "large-v3", "large", "distil-large-v2", "distil-medium.en",
-        "distil-small.en", "distil-large-v3",
-    ]
-
-    def __init__(
-        self,
-        model_size_or_path: str,
-        device: str = processing_choice,
-        device_index: Union[int, List[int]] = 0,
-        compute_type: str = "default",
-        cpu_threads: int = 0,#total_thread_count, FIXME - I think this should be 0
-        num_workers: int = 1,
-        download_root: Optional[str] = None,
-        local_files_only: bool = False,
-        files: Optional[Dict[str, Any]] = None,
-        **model_kwargs: Any
-    ):
-        if download_root is None:
-            download_root = self.default_download_root
-
-        os.makedirs(download_root, exist_ok=True)
-
-        # FIXME - validate....
-        # Also write an integration test...
-        # Check if model_size_or_path is a valid model size
-        if model_size_or_path in self.valid_model_sizes:
-            # It's a model size, so we'll use the download_root
-            model_path = os.path.join(download_root, model_size_or_path)
-            if not os.path.isdir(model_path):
-                # If it doesn't exist, we'll let the parent class download it
-                model_size_or_path = model_size_or_path  # Keep the original model size
-            else:
-                # If it exists, use the full path
-                model_size_or_path = model_path
-        else:
-            # It's not a valid model size, so assume it's a path
-            model_size_or_path = os.path.abspath(model_size_or_path)
-
-        super().__init__(
-            model_size_or_path,
-            device=device,
-            device_index=device_index,
-            compute_type=compute_type,
-            cpu_threads=cpu_threads,
-            num_workers=num_workers,
-            download_root=download_root,
-            local_files_only=local_files_only,
-# Maybe? idk, FIXME
-#            files=files,
-#            **model_kwargs
-        )
-
-def get_whisper_model(model_name, device):
-    global whisper_model_instance
-    if whisper_model_instance is None:
-        logging.info(f"Initializing new WhisperModel with size {model_name} on device {device}")
-        whisper_model_instance = WhisperModel(model_name, device=device)
-    return whisper_model_instance
-
-# os.system(r'.\Bin\ffmpeg.exe -ss 00:00:00 -i "{video_file_path}" -ar 16000 -ac 1 -c:a pcm_s16le "{out_path}"')
-#DEBUG
-#@profile
-def convert_to_wav(video_file_path, offset=0, overwrite=False):
-    log_counter("convert_to_wav_attempt", labels={"file_path": video_file_path})
-    start_time = time.time()
-
-    out_path = os.path.splitext(video_file_path)[0] + ".wav"
-
-    if os.path.exists(out_path) and not overwrite:
-        print(f"File '{out_path}' already exists. Skipping conversion.")
-        logging.info(f"Skipping conversion as file already exists: {out_path}")
-        log_counter("convert_to_wav_skipped", labels={"file_path": video_file_path})
-        return out_path
-
-    print("Starting conversion process of .m4a to .WAV")
-    out_path = os.path.splitext(video_file_path)[0] + ".wav"
-
-    try:
-        if os.name == "nt":
-            logging.debug("ffmpeg being ran on windows")
-
-            if sys.platform.startswith('win'):
-                ffmpeg_cmd = ".\\Bin\\ffmpeg.exe"
-                logging.debug(f"ffmpeg_cmd: {ffmpeg_cmd}")
-            else:
-                ffmpeg_cmd = 'ffmpeg'  # Assume 'ffmpeg' is in PATH for non-Windows systems
-
-            command = [
-                ffmpeg_cmd,  # Assuming the working directory is correctly set where .\Bin exists
-                "-ss", "00:00:00",  # Start at the beginning of the video
-                "-i", video_file_path,
-                "-ar", "16000",  # Audio sample rate
-                "-ac", "1",  # Number of audio channels
-                "-c:a", "pcm_s16le",  # Audio codec
-                out_path
-            ]
-            try:
-                # Redirect stdin from null device to prevent ffmpeg from waiting for input
-                with open(os.devnull, 'rb') as null_file:
-                    result = subprocess.run(command, stdin=null_file, text=True, capture_output=True)
-                if result.returncode == 0:
-                    logging.info("FFmpeg executed successfully")
-                    logging.debug("FFmpeg output: %s", result.stdout)
-                else:
-                    logging.error("Error in running FFmpeg")
-                    logging.error("FFmpeg stderr: %s", result.stderr)
-                    raise RuntimeError(f"FFmpeg error: {result.stderr}")
-            except Exception as e:
-                logging.error("Error occurred - ffmpeg doesn't like windows")
-                raise RuntimeError("ffmpeg failed")
-        elif os.name == "posix":
-            os.system(f'ffmpeg -ss 00:00:00 -i "{video_file_path}" -ar 16000 -ac 1 -c:a pcm_s16le "{out_path}"')
-        else:
-            raise RuntimeError("Unsupported operating system")
-        logging.info("Conversion to WAV completed: %s", out_path)
-        log_counter("convert_to_wav_success", labels={"file_path": video_file_path})
-    except Exception as e:
-        logging.error("speech-to-text: Error transcribing audio: %s", str(e))
-        log_counter("convert_to_wav_error", labels={"file_path": video_file_path, "error": str(e)})
-        return {"error": str(e)}
-
-    conversion_time = time.time() - start_time
-    log_histogram("convert_to_wav_duration", conversion_time, labels={"file_path": video_file_path})
-
-    gc.collect()
-    return out_path
-
-
-# Transcribe .wav into .segments.json
-#DEBUG
-#@profile
-# FIXME - I feel like the `vad_filter` shoudl be enabled by default....
-def speech_to_text(audio_file_path, selected_source_lang='en', whisper_model='medium.en', vad_filter=False, diarize=False):
-    log_counter("speech_to_text_attempt", labels={"file_path": audio_file_path, "model": whisper_model})
-    time_start = time.time()
-
-    if audio_file_path is None:
-        log_counter("speech_to_text_error", labels={"error": "No audio file provided"})
-        raise ValueError("speech-to-text: No audio file provided")
-    logging.info("speech-to-text: Audio file path: %s", audio_file_path)
-
-    try:
-        _, file_ending = os.path.splitext(audio_file_path)
-        out_file = audio_file_path.replace(file_ending, "-whisper_model-"+whisper_model+".segments.json")
-        prettified_out_file = audio_file_path.replace(file_ending, "-whisper_model-"+whisper_model+".segments_pretty.json")
-        if os.path.exists(out_file):
-            logging.info("speech-to-text: Segments file already exists: %s", out_file)
-            with open(out_file) as f:
-                global segments
-                segments = json.load(f)
-            return segments
-
-        logging.info('speech-to-text: Starting transcription...')
-        # FIXME - revisit this
-        options = dict(language=selected_source_lang, beam_size=10, best_of=10, vad_filter=vad_filter)
-        transcribe_options = dict(task="transcribe", **options)
-        # use function and config at top of file
-        logging.debug("speech-to-text: Using whisper model: %s", whisper_model)
-        whisper_model_instance = get_whisper_model(whisper_model, processing_choice)
-        # faster_whisper transcription right here - FIXME -test batching - ha
-        segments_raw, info = whisper_model_instance.transcribe(audio_file_path, **transcribe_options)
-
-        segments = []
-        for segment_chunk in segments_raw:
-            chunk = {
-                "Time_Start": segment_chunk.start,
-                "Time_End": segment_chunk.end,
-                "Text": segment_chunk.text
-            }
-            logging.debug("Segment: %s", chunk)
-            segments.append(chunk)
-            # Print to verify its working
-            logging.info(f"{segment_chunk.start:.2f}s - {segment_chunk.end:.2f}s | {segment_chunk.text}")
-
-            # Log it as well.
-            logging.debug(
-                f"Transcribed Segment: {segment_chunk.start:.2f}s - {segment_chunk.end:.2f}s | {segment_chunk.text}")
-
-        if segments:
-            segments[0]["Text"] = f"This text was transcribed using whisper model: {whisper_model}\n\n" + segments[0]["Text"]
-
-        if not segments:
-            log_counter("speech_to_text_error", labels={"error": "No transcription produced"})
-            raise RuntimeError("No transcription produced. The audio file may be invalid or empty.")
-
-        transcription_time = time.time() - time_start
-        logging.info("speech-to-text: Transcription completed in %.2f seconds", transcription_time)
-        log_histogram("speech_to_text_duration", transcription_time, labels={"file_path": audio_file_path, "model": whisper_model})
-        log_counter("speech_to_text_success", labels={"file_path": audio_file_path, "model": whisper_model})
-        # Save the segments to a JSON file - prettified and non-prettified
-        # FIXME refactor so this is an optional flag to save either the prettified json file or the normal one
-        save_json = True
-        if save_json:
-            logging.info("speech-to-text: Saving segments to JSON file")
-            output_data = {'segments': segments}
-            logging.info("speech-to-text: Saving prettified JSON to %s", prettified_out_file)
-            with open(prettified_out_file, 'w') as f:
-                json.dump(output_data, f, indent=2)
-
-            logging.info("speech-to-text: Saving JSON to %s", out_file)
-            with open(out_file, 'w') as f:
-                json.dump(output_data, f)
-
-        logging.debug(f"speech-to-text: returning {segments[:500]}")
-        gc.collect()
-        return segments
-
-    except Exception as e:
-        logging.error("speech-to-text: Error transcribing audio: %s", str(e))
-        log_counter("speech_to_text_error", labels={"file_path": audio_file_path, "model": whisper_model, "error": str(e)})
-        raise RuntimeError("speech-to-text: Error transcribing audio")
-
-
-def record_audio(duration, sample_rate=16000, chunk_size=1024):
-    log_counter("record_audio_attempt", labels={"duration": duration})
-    p = pyaudio.PyAudio()
-    stream = p.open(format=pyaudio.paInt16,
-                    channels=1,
-                    rate=sample_rate,
-                    input=True,
-                    frames_per_buffer=chunk_size)
-
-    print("Recording...")
-    frames = []
-    stop_recording = threading.Event()
-    audio_queue = queue.Queue()
-
-    def audio_callback():
-        for _ in range(0, int(sample_rate / chunk_size * duration)):
-            if stop_recording.is_set():
-                break
-            data = stream.read(chunk_size)
-            audio_queue.put(data)
-
-    audio_thread = threading.Thread(target=audio_callback)
-    audio_thread.start()
-
-    return p, stream, audio_queue, stop_recording, audio_thread
-
-
-def stop_recording(p, stream, audio_queue, stop_recording_event, audio_thread):
-    log_counter("stop_recording_attempt")
-    start_time = time.time()
-    stop_recording_event.set()
-    audio_thread.join()
-
-    frames = []
-    while not audio_queue.empty():
-        frames.append(audio_queue.get())
-
-    print("Recording finished.")
-
-    stream.stop_stream()
-    stream.close()
-    p.terminate()
-
-    stop_time = time.time() - start_time
-    log_histogram("stop_recording_duration", stop_time)
-    log_counter("stop_recording_success")
-    return b''.join(frames)
-
-def save_audio_temp(audio_data, sample_rate=16000):
-    log_counter("save_audio_temp_attempt")
-    with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as temp_file:
-        import wave
-        wf = wave.open(temp_file.name, 'wb')
-        wf.setnchannels(1)
-        wf.setsampwidth(2)
-        wf.setframerate(sample_rate)
-        wf.writeframes(audio_data)
-        wf.close()
-        log_counter("save_audio_temp_success")
-        return temp_file.name
-
-#
-#
-#######################################################################################################################

App_Function_Libraries/Audio/Diarization_Lib.py

@@ -1,275 +0,0 @@
-# Diarization_Lib.py
-#########################################
-# Diarization Library
-# This library is used to perform diarization of audio files.
-# Currently, uses FIXME for transcription.
-#
-####################
-####################
-# Function List
-#
-# 1. speaker_diarize(video_file_path, segments, embedding_model = "pyannote/embedding", embedding_size=512, num_speakers=0)
-#
-####################
-# Import necessary libraries
-import logging
-from pathlib import Path
-from typing import Dict, List, Any
-
-#
-# Import Local Libraries
-from App_Function_Libraries.Audio.Audio_Transcription_Lib import speech_to_text
-#
-# Import 3rd Party Libraries
-from pyannote.audio.pipelines.speaker_diarization import SpeakerDiarization
-import yaml
-#
-#######################################################################################################################
-# Function Definitions
-#
-
-def load_pipeline_from_pretrained(path_to_config: str | Path) -> SpeakerDiarization:
-    path_to_config = Path(path_to_config).resolve()
-    logging.debug(f"Loading pyannote pipeline from {path_to_config}...")
-
-    if not path_to_config.exists():
-        raise FileNotFoundError(f"Config file not found: {path_to_config}")
-
-    # Load the YAML configuration
-    with open(path_to_config, 'r') as config_file:
-        config = yaml.safe_load(config_file)
-
-    # Debug: print the entire config
-    logging.debug(f"Loaded config: {config}")
-
-    # Create the SpeakerDiarization pipeline
-    try:
-        pipeline = SpeakerDiarization(
-            segmentation=config['pipeline']['params']['segmentation'],
-            embedding=config['pipeline']['params']['embedding'],
-            clustering=config['pipeline']['params']['clustering'],
-        )
-    except KeyError as e:
-        logging.error(f"Error accessing config key: {e}")
-        raise
-
-    # Set other parameters
-    try:
-        pipeline_params = {
-            "segmentation": {},
-            "clustering": {},
-        }
-
-        if 'params' in config and 'segmentation' in config['params']:
-            if 'min_duration_off' in config['params']['segmentation']:
-                pipeline_params["segmentation"]["min_duration_off"] = config['params']['segmentation']['min_duration_off']
-
-        if 'params' in config and 'clustering' in config['params']:
-            if 'method' in config['params']['clustering']:
-                pipeline_params["clustering"]["method"] = config['params']['clustering']['method']
-            if 'min_cluster_size' in config['params']['clustering']:
-                pipeline_params["clustering"]["min_cluster_size"] = config['params']['clustering']['min_cluster_size']
-            if 'threshold' in config['params']['clustering']:
-                pipeline_params["clustering"]["threshold"] = config['params']['clustering']['threshold']
-
-        if 'pipeline' in config and 'params' in config['pipeline']:
-            if 'embedding_batch_size' in config['pipeline']['params']:
-                pipeline_params["embedding_batch_size"] = config['pipeline']['params']['embedding_batch_size']
-            if 'embedding_exclude_overlap' in config['pipeline']['params']:
-                pipeline_params["embedding_exclude_overlap"] = config['pipeline']['params']['embedding_exclude_overlap']
-            if 'segmentation_batch_size' in config['pipeline']['params']:
-                pipeline_params["segmentation_batch_size"] = config['pipeline']['params']['segmentation_batch_size']
-
-        logging.debug(f"Pipeline params: {pipeline_params}")
-        pipeline.instantiate(pipeline_params)
-    except KeyError as e:
-        logging.error(f"Error accessing config key: {e}")
-        raise
-    except Exception as e:
-        logging.error(f"Error instantiating pipeline: {e}")
-        raise
-
-    return pipeline
-
-
-def audio_diarization(audio_file_path: str) -> list:
-    logging.info('audio-diarization: Loading pyannote pipeline')
-
-    base_dir = Path(__file__).parent.resolve()
-    config_path = base_dir / 'models' / 'pyannote_diarization_config.yaml'
-    logging.info(f"audio-diarization: Loading pipeline from {config_path}")
-
-    try:
-        pipeline = load_pipeline_from_pretrained(config_path)
-    except Exception as e:
-        logging.error(f"Failed to load pipeline: {str(e)}")
-        raise
-
-    logging.info(f"audio-diarization: Audio file path: {audio_file_path}")
-
-    try:
-        logging.info('audio-diarization: Starting diarization...')
-        diarization_result = pipeline(audio_file_path)
-
-        segments = []
-        for turn, _, speaker in diarization_result.itertracks(yield_label=True):
-            segment = {
-                "start": turn.start,
-                "end": turn.end,
-                "speaker": speaker
-            }
-            logging.debug(f"Segment: {segment}")
-            segments.append(segment)
-        logging.info("audio-diarization: Diarization completed with pyannote")
-
-        return segments
-
-    except Exception as e:
-        logging.error(f"audio-diarization: Error performing diarization: {str(e)}")
-        raise RuntimeError("audio-diarization: Error performing diarization") from e
-
-
-# Old
-# def audio_diarization(audio_file_path):
-#     logging.info('audio-diarization: Loading pyannote pipeline')
-#
-#     #config file loading
-#     current_dir = os.path.dirname(os.path.abspath(__file__))
-#     # Construct the path to the config file
-#     config_path = os.path.join(current_dir, 'Config_Files', 'config.txt')
-#     # Read the config file
-#     config = configparser.ConfigParser()
-#     config.read(config_path)
-#     processing_choice = config.get('Processing', 'processing_choice', fallback='cpu')
-#
-#     base_dir = Path(__file__).parent.resolve()
-#     config_path = base_dir / 'models' / 'config.yaml'
-#     pipeline = load_pipeline_from_pretrained(config_path)
-#
-#     time_start = time.time()
-#     if audio_file_path is None:
-#         raise ValueError("audio-diarization: No audio file provided")
-#     logging.info("audio-diarization: Audio file path: %s", audio_file_path)
-#
-#     try:
-#         _, file_ending = os.path.splitext(audio_file_path)
-#         out_file = audio_file_path.replace(file_ending, ".diarization.json")
-#         prettified_out_file = audio_file_path.replace(file_ending, ".diarization_pretty.json")
-#         if os.path.exists(out_file):
-#             logging.info("audio-diarization: Diarization file already exists: %s", out_file)
-#             with open(out_file) as f:
-#                 global diarization_result
-#                 diarization_result = json.load(f)
-#             return diarization_result
-#
-#         logging.info('audio-diarization: Starting diarization...')
-#         diarization_result = pipeline(audio_file_path)
-#
-#         segments = []
-#         for turn, _, speaker in diarization_result.itertracks(yield_label=True):
-#             chunk = {
-#                 "Time_Start": turn.start,
-#                 "Time_End": turn.end,
-#                 "Speaker": speaker
-#             }
-#             logging.debug("Segment: %s", chunk)
-#             segments.append(chunk)
-#         logging.info("audio-diarization: Diarization completed with pyannote")
-#
-#         output_data = {'segments': segments}
-#
-#         logging.info("audio-diarization: Saving prettified JSON to %s", prettified_out_file)
-#         with open(prettified_out_file, 'w') as f:
-#             json.dump(output_data, f, indent=2)
-#
-#         logging.info("audio-diarization: Saving JSON to %s", out_file)
-#         with open(out_file, 'w') as f:
-#             json.dump(output_data, f)
-#
-#     except Exception as e:
-#         logging.error("audio-diarization: Error performing diarization: %s", str(e))
-#         raise RuntimeError("audio-diarization: Error performing diarization")
-#     return segments
-
-def combine_transcription_and_diarization(audio_file_path: str) -> List[Dict[str, Any]]:
-    logging.info('combine-transcription-and-diarization: Starting transcription and diarization...')
-
-    try:
-        logging.info('Performing speech-to-text...')
-        transcription_result = speech_to_text(audio_file_path)
-        logging.info(f"Transcription result type: {type(transcription_result)}")
-        logging.info(f"Transcription result: {transcription_result[:3] if isinstance(transcription_result, list) and len(transcription_result) > 3 else transcription_result}")
-
-        logging.info('Performing audio diarization...')
-        diarization_result = audio_diarization(audio_file_path)
-        logging.info(f"Diarization result type: {type(diarization_result)}")
-        logging.info(f"Diarization result sample: {diarization_result[:3] if isinstance(diarization_result, list) and len(diarization_result) > 3 else diarization_result}")
-
-        if not transcription_result:
-            logging.error("Empty result from transcription")
-            return []
-
-        if not diarization_result:
-            logging.error("Empty result from diarization")
-            return []
-
-        # Handle the case where transcription_result is a dict with a 'segments' key
-        if isinstance(transcription_result, dict) and 'segments' in transcription_result:
-            transcription_segments = transcription_result['segments']
-        elif isinstance(transcription_result, list):
-            transcription_segments = transcription_result
-        else:
-            logging.error(f"Unexpected transcription result format: {type(transcription_result)}")
-            return []
-
-        logging.info(f"Number of transcription segments: {len(transcription_segments)}")
-        logging.info(f"Transcription segments sample: {transcription_segments[:3] if len(transcription_segments) > 3 else transcription_segments}")
-
-        if not isinstance(diarization_result, list):
-            logging.error(f"Unexpected diarization result format: {type(diarization_result)}")
-            return []
-
-        combined_result = []
-        for transcription_segment in transcription_segments:
-            if not isinstance(transcription_segment, dict):
-                logging.warning(f"Unexpected transcription segment format: {transcription_segment}")
-                continue
-
-            for diarization_segment in diarization_result:
-                if not isinstance(diarization_segment, dict):
-                    logging.warning(f"Unexpected diarization segment format: {diarization_segment}")
-                    continue
-
-                try:
-                    trans_start = transcription_segment.get('Time_Start', 0)
-                    trans_end = transcription_segment.get('Time_End', 0)
-                    diar_start = diarization_segment.get('start', 0)
-                    diar_end = diarization_segment.get('end', 0)
-
-                    if trans_start >= diar_start and trans_end <= diar_end:
-                        combined_segment = {
-                            "Time_Start": trans_start,
-                            "Time_End": trans_end,
-                            "Speaker": diarization_segment.get('speaker', 'Unknown'),
-                            "Text": transcription_segment.get('Text', '')
-                        }
-                        combined_result.append(combined_segment)
-                        break
-                except Exception as e:
-                    logging.error(f"Error processing segment: {str(e)}")
-                    logging.error(f"Transcription segment: {transcription_segment}")
-                    logging.error(f"Diarization segment: {diarization_segment}")
-                    continue
-
-        logging.info(f"Combined result length: {len(combined_result)}")
-        logging.info(f"Combined result sample: {combined_result[:3] if len(combined_result) > 3 else combined_result}")
-        return combined_result
-
-    except Exception as e:
-        logging.error(f"Error in combine_transcription_and_diarization: {str(e)}", exc_info=True)
-        return []
-
-
-#
-#
-#######################################################################################################################

App_Function_Libraries/Benchmarks_Evaluations/Confabulation_check.py

@@ -1,81 +0,0 @@
-# Confabulation_check.py
-#
-# This file contains the functions that are used to check the confabulation of the user's input.
-#
-#
-# Imports
-#
-# External Imports
-#
-# Local Imports
-#
-#
-####################################################################################################
-#
-# Functions:
-from App_Function_Libraries.Chat import chat_api_call
-from App_Function_Libraries.Benchmarks_Evaluations.ms_g_eval import validate_inputs, detailed_api_error
-
-
-def simplified_geval(transcript: str, summary: str, api_name: str, api_key: str, temp: float = 0.7) -> str:
-    """
-    Perform a simplified version of G-Eval using a single query to evaluate the summary.
-
-    Args:
-        transcript (str): The original transcript
-        summary (str): The summary to be evaluated
-        api_name (str): The name of the LLM API to use
-        api_key (str): The API key for the chosen LLM
-        temp (float, optional): The temperature parameter for the API call. Defaults to 0.7.
-
-    Returns:
-        str: The evaluation result
-    """
-    try:
-        validate_inputs(transcript, summary, api_name, api_key)
-    except ValueError as e:
-        return str(e)
-
-    prompt = f"""You are an AI assistant tasked with evaluating the quality of a summary. You will be given an original transcript and a summary of that transcript. Your task is to evaluate the summary based on the following criteria:
-
-1. Coherence (1-5): How well-structured and organized is the summary?
-2. Consistency (1-5): How factually aligned is the summary with the original transcript?
-3. Fluency (1-3): How well-written is the summary in terms of grammar, spelling, and readability?
-4. Relevance (1-5): How well does the summary capture the important information from the transcript?
-
-Please provide a score for each criterion and a brief explanation for your scoring. Then, give an overall assessment of the summary's quality.
-
-Original Transcript:
-{transcript}
-
-Summary to Evaluate:
-{summary}
-
-Please provide your evaluation in the following format:
-Coherence: [score] - [brief explanation]
-Consistency: [score] - [brief explanation]
-Fluency: [score] - [brief explanation]
-Relevance: [score] - [brief explanation]
-
-Overall Assessment: [Your overall assessment of the summary's quality]
-"""
-
-    try:
-        result = chat_api_call(
-            api_name,
-            api_key,
-            prompt,
-            "",
-            temp=temp,
-            system_message="You are a helpful AI assistant tasked with evaluating summaries."
-        )
-    except Exception as e:
-        return detailed_api_error(api_name, e)
-
-    formatted_result = f"""
-    Confabulation Check Results:
-
-    {result}
-    """
-
-    return formatted_result

App_Function_Libraries/Benchmarks_Evaluations/ms_g_eval.py

@@ -1,498 +0,0 @@
-#######################################################################################################################
-#
-# Evaluations_Benchmarks_tab.py
-#
-# Description: This file contains the code to evaluate the generated text using G-Eval metric.
-#
-# Scripts taken from https://github.com/microsoft/promptflow/tree/main/examples/flows/evaluation/eval-summarization and modified.
-#
-import configparser
-import inspect
-import json
-import logging
-import os
-import re
-from typing import Dict, Callable, List, Any
-
-import gradio as gr
-from tenacity import (
-    RetryError,
-    Retrying,
-    after_log,
-    before_sleep_log,
-    stop_after_attempt,
-    wait_random_exponential,
-)
-
-from App_Function_Libraries.Chat import chat_api_call
-
-#
-#######################################################################################################################
-#
-# Start of G-Eval.py
-
-logger = logging.getLogger(__name__)
-
-current_dir = os.path.dirname(os.path.abspath(__file__))
-# Construct the path to the config file
-config_path = os.path.join(current_dir, 'Config_Files', 'config.txt')
-# Read the config file
-config = configparser.ConfigParser()
-config.read(config_path)
-
-
-def aggregate(
-    fluency_list: List[float],
-    consistency_list: List[float],
-    relevance_list: List[float],
-    coherence_list: List[float],
-) -> Dict[str, float]:
-    """
-    Takes list of scores for 4 dims and outputs average for them.
-
-    Args:
-        fluency_list (List(float)): list of fluency scores
-        consistency_list (List(float)): list of consistency scores
-        relevance_list (List(float)): list of relevance scores
-        coherence_list (List(float)): list of coherence scores
-
-    Returns:
-        Dict[str, float]: Returns average scores
-    """
-    average_fluency = sum(fluency_list) / len(fluency_list)
-    average_consistency = sum(consistency_list) / len(consistency_list)
-    average_relevance = sum(relevance_list) / len(relevance_list)
-    average_coherence = sum(coherence_list) / len(coherence_list)
-
-    log_metric("average_fluency", average_fluency)
-    log_metric("average_consistency", average_consistency)
-    log_metric("average_relevance", average_relevance)
-    log_metric("average_coherence", average_coherence)
-
-    return {
-        "average_fluency": average_fluency,
-        "average_consistency": average_consistency,
-        "average_relevance": average_relevance,
-        "average_coherence": average_coherence,
-    }
-
-def run_geval(transcript: str, summary: str, api_key: str, api_name: str = None, save: bool = False):
-    try:
-        validate_inputs(transcript, summary, api_name, api_key)
-    except ValueError as e:
-        return str(e)
-
-    prompts = {
-        "coherence": """You will be given one summary written for a source document.
-
-        Your task is to rate the summary on one metric.
-
-        Please make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.
-
-        Evaluation Criteria:
-
-        Coherence (1-5) - the collective quality of all sentences. We align this dimension with the DUC quality question of structure and coherence whereby "the summary should be well-structured and well-organized. The summary should not just be a heap of related information, but should build from sentence to a coherent body of information about a topic."
-
-        Evaluation Steps:
-
-        1. Read the source document carefully and identify the main topic and key points.
-        2. Read the summary and compare it to the source document. Check if the summary covers the main topic and key points of the source document, and if it presents them in a clear and logical order.
-        3. Assign a score for coherence on a scale of 1 to 5, where 1 is the lowest and 5 is the highest based on the Evaluation Criteria.
-
-
-        Example:
-
-
-        Source Document:
-
-        {{Document}}
-
-        Summary:
-
-        {{Summary}}
-
-
-        Evaluation Form (scores ONLY):
-
-        - Coherence:""",
-        "consistency": """You will be given a source document. You will then be given one summary written for this source document.
-
-        Your task is to rate the summary on one metric.
-
-        Please make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.
-
-
-        Evaluation Criteria:
-
-        Consistency (1-5) - the factual alignment between the summary and the summarized source. A factually consistent summary contains only statements that are entailed by the source document. Annotators were also asked to penalize summaries that contained hallucinated facts. 
-
-        Evaluation Steps:
-
-        1. Read the source document carefully and identify the main facts and details it presents.
-        2. Read the summary and compare it to the source document. Check if the summary contains any factual errors that are not supported by the source document.
-        3. Assign a score for consistency based on the Evaluation Criteria.
-
-
-        Example:
-
-
-        Source Document: 
-
-        {{Document}}
-
-        Summary: 
-
-        {{Summary}}
-
-
-        Evaluation Form (scores ONLY):
-
-        - Consistency:""",
-        "fluency": """You will be given one summary written for a source document.
-
-        Your task is to rate the summary on one metric.
-
-        Please make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.
-
-
-        Evaluation Criteria:
-
-        Fluency (1-3): the quality of the summary in terms of grammar, spelling, punctuation, word choice, and sentence structure.
-
-        - 1: Poor. The summary has many errors that make it hard to understand or sound unnatural.
-        - 2: Fair. The summary has some errors that affect the clarity or smoothness of the text, but the main points are still comprehensible.
-        - 3: Good. The summary has few or no errors and is easy to read and follow.
-
-
-        Example:
-
-        Summary:
-
-        {{Summary}}
-
-
-        Evaluation Form (scores ONLY):
-
-        - Fluency (1-3):""",
-        "relevance": """You will be given one summary written for a source document.
-
-        Your task is to rate the summary on one metric.
-
-        Please make sure you read and understand these instructions carefully. Please keep this document open while reviewing, and refer to it as needed.
-
-        Evaluation Criteria:
-
-        Relevance (1-5) - selection of important content from the source. The summary should include only important information from the source document. Annotators were instructed to penalize summaries which contained redundancies and excess information.
-
-        Evaluation Steps:
-
-        1. Read the summary and the source document carefully.
-        2. Compare the summary to the source document and identify the main points of the source document.
-        3. Assess how well the summary covers the main points of the source document, and how much irrelevant or redundant information it contains.
-        4. Assign a relevance score from 1 to 5.
-
-
-        Example:
-
-
-        Source Document:
-
-        {{Document}}
-
-        Summary:
-
-        {{Summary}}
-
-
-        Evaluation Form (scores ONLY):
-
-        - Relevance:"""
-    }
-
-    scores = {}
-    explanations = {}
-    for metric, prompt in prompts.items():
-        full_prompt = prompt.replace("{{Document}}", transcript).replace("{{Summary}}", summary)
-        try:
-            score = geval_summarization(full_prompt, 5 if metric != "fluency" else 3, api_name, api_key)
-            scores[metric] = score
-            explanations[metric] = "Score based on the evaluation criteria."
-        except Exception as e:
-            error_message = detailed_api_error(api_name, e)
-            return error_message
-
-    avg_scores = aggregate([scores['fluency']], [scores['consistency']],
-                           [scores['relevance']], [scores['coherence']])
-
-    results = {
-        "scores": scores,
-        "average_scores": avg_scores
-    }
-    logging.debug("Results: %s", results)
-
-    if save is not None:
-        logging.debug("Saving results to geval_results.json")
-        save_eval_results(results)
-        logging.debug("Results saved to geval_results.json")
-
-    formatted_result = f"""
-    Confabulation Check Results:
-
-    Coherence: {scores['coherence']:.2f} - {explanations['coherence']}
-    Consistency: {scores['consistency']:.2f} - {explanations['consistency']}
-    Fluency: {scores['fluency']:.2f} - {explanations['fluency']}
-    Relevance: {scores['relevance']:.2f} - {explanations['relevance']}
-
-    Overall Assessment: The summary has been evaluated on four key metrics. 
-    The average scores are:
-      Fluency: {avg_scores['average_fluency']:.2f}
-      Consistency: {avg_scores['average_consistency']:.2f}
-      Relevance: {avg_scores['average_relevance']:.2f}
-      Coherence: {avg_scores['average_coherence']:.2f}
-
-    These scores indicate the overall quality of the summary in terms of its 
-    coherence, consistency with the original text, fluency of language, and 
-    relevance of content.
-    """
-
-    return formatted_result
-
-
-def create_geval_tab():
-    with gr.Tab("G-Eval", id="g-eval"):
-        gr.Markdown("# G-Eval Summarization Evaluation")
-        with gr.Row():
-            with gr.Column():
-                document_input = gr.Textbox(label="Source Document", lines=10)
-                summary_input = gr.Textbox(label="Summary", lines=5)
-                api_name_input = gr.Dropdown(
-                    choices=["OpenAI", "Anthropic", "Cohere", "Groq", "OpenRouter", "DeepSeek", "HuggingFace", "Mistral", "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "Local-LLM", "Ollama"],
-                    label="Select API"
-                )
-                api_key_input = gr.Textbox(label="API Key (if required)", type="password")
-                save_value = gr.Checkbox(label="Save Results to a JSON file(geval_results.json)")
-                evaluate_button = gr.Button("Evaluate Summary")
-            with gr.Column():
-                output = gr.Textbox(label="Evaluation Results", lines=10)
-
-        evaluate_button.click(
-            fn=run_geval,
-            inputs=[document_input, summary_input, api_name_input, api_key_input, save_value],
-            outputs=output
-        )
-
-    return document_input, summary_input, api_name_input, api_key_input, evaluate_button, output
-
-
-def parse_output(output: str, max: float) -> float:
-    """
-    Function that extracts numerical score from the beginning of string
-
-    Args:
-        output (str): String to search
-        max (float): Maximum score allowed
-
-    Returns:
-        float: The extracted score
-    """
-    matched: List[str] = re.findall(r"(?<!\S)\d+(?:\.\d+)?", output)
-    if matched:
-        if len(matched) == 1:
-            score = float(matched[0])
-            if score > max:
-                raise ValueError(f"Parsed number: {score} was larger than max score: {max}")
-        else:
-            raise ValueError(f"More than one number detected in input. Input to parser was: {output}")
-    else:
-        raise ValueError(f'No number detected in input. Input to parser was "{output}". ')
-    return score
-
-def geval_summarization(
-    prompt_with_src_and_gen: str,
-    max_score: float,
-    api_endpoint: str,
-    api_key: str,
-) -> float:
-    model = get_model_from_config(api_endpoint)
-
-    try:
-        for attempt in Retrying(
-            reraise=True,
-            before_sleep=before_sleep_log(logger, logging.INFO),
-            after=after_log(logger, logging.INFO),
-            wait=wait_random_exponential(multiplier=1, min=1, max=120),
-            stop=stop_after_attempt(10),
-        ):
-            with attempt:
-                system_message="You are a helpful AI assistant"
-                # TEMP setting for Confabulation check
-                temp = 0.7
-                logging.info(f"Debug - geval_summarization Function - API Endpoint: {api_endpoint}")
-                try:
-                    response = chat_api_call(api_endpoint, api_key, prompt_with_src_and_gen, "", temp, system_message)
-                except Exception as e:
-                    raise ValueError(f"Unsupported API endpoint: {api_endpoint}")
-    except RetryError:
-        logger.exception(f"geval {api_endpoint} call failed\nInput prompt was: {prompt_with_src_and_gen}")
-        raise
-
-    try:
-        score = parse_output(response, max_score)
-    except ValueError as e:
-        logger.warning(f"Error parsing output: {e}")
-        score = 0
-
-    return score
-
-
-def get_model_from_config(api_name: str) -> str:
-    model = config.get('models', api_name)
-    if isinstance(model, dict):
-        # If the model is a dictionary, return a specific key or a default value
-        return model.get('name', str(model))  # Adjust 'name' to the appropriate key if needed
-    return str(model) if model is not None else ""
-
-def aggregate_llm_scores(llm_responses: List[str], max_score: float) -> float:
-    """Parse and average valid scores from the generated responses of
-    the G-Eval LLM call.
-
-    Args:
-        llm_responses (List[str]): List of scores from multiple LLMs
-        max_score (float): The maximum score allowed.
-
-    Returns:
-        float: The average of all the valid scores
-    """
-    all_scores = []
-    error_count = 0
-    for generated in llm_responses:
-        try:
-            parsed = parse_output(generated, max_score)
-            all_scores.append(parsed)
-        except ValueError as e:
-            logger.warning(e)
-            error_count += 1
-    if error_count:
-        logger.warning(f"{error_count} out of 20 scores were discarded due to corrupt g-eval generation")
-    score = sum(all_scores) / len(all_scores)
-    return score
-
-
-def validate_inputs(document: str, summary: str, api_name: str, api_key: str) -> None:
-    """
-    Validate inputs for the G-Eval function.
-
-    Args:
-        document (str): The source document
-        summary (str): The summary to evaluate
-        api_name (str): The name of the API to use
-        api_key (str): The API key
-
-    Raises:
-        ValueError: If any of the inputs are invalid
-    """
-    if not document.strip():
-        raise ValueError("Source document cannot be empty")
-    if not summary.strip():
-        raise ValueError("Summary cannot be empty")
-    if api_name.lower() not in ["openai", "anthropic", "cohere", "groq", "openrouter", "deepseek", "huggingface",
-                                "mistral", "llama.cpp", "kobold", "ooba", "tabbyapi", "vllm", "local-llm", "ollama"]:
-        raise ValueError(f"Unsupported API: {api_name}")
-
-
-def detailed_api_error(api_name: str, error: Exception) -> str:
-    """
-    Generate a detailed error message for API failures.
-
-    Args:
-        api_name (str): The name of the API that failed
-        error (Exception): The exception that was raised
-
-    Returns:
-        str: A detailed error message
-    """
-    error_type = type(error).__name__
-    error_message = str(error)
-    return f"API Failure: {api_name}\nError Type: {error_type}\nError Message: {error_message}\nPlease check your API key and network connection, and try again."
-
-
-def save_eval_results(results: Dict[str, Any], filename: str = "geval_results.json") -> None:
-    """
-    Save evaluation results to a JSON file.
-
-    Args:
-        results (Dict[str, Any]): The evaluation results
-        filename (str): The name of the file to save results to
-    """
-    with open(filename, 'w') as f:
-        json.dump(results, f, indent=2)
-    print(f"Results saved to {filename}")
-
-
-
-
-#
-#
-#######################################################################################################################
-#
-# Taken from: https://github.com/microsoft/promptflow/blob/b5a68f45e4c3818a29e2f79a76f2e73b8ea6be44/src/promptflow-core/promptflow/_core/metric_logger.py
-
-class MetricLoggerManager:
-    _instance = None
-
-    def __init__(self):
-        self._metric_loggers = []
-
-    @staticmethod
-    def get_instance() -> "MetricLoggerManager":
-        if MetricLoggerManager._instance is None:
-            MetricLoggerManager._instance = MetricLoggerManager()
-        return MetricLoggerManager._instance
-
-    def log_metric(self, key, value, variant_id=None):
-        for logger in self._metric_loggers:
-            if len(inspect.signature(logger).parameters) == 2:
-                logger(key, value)  # If the logger only accepts two parameters, we don't pass variant_id
-            else:
-                logger(key, value, variant_id)
-
-    def add_metric_logger(self, logger_func: Callable):
-        existing_logger = next((logger for logger in self._metric_loggers if logger is logger_func), None)
-        if existing_logger:
-            return
-        if not callable(logger_func):
-            return
-        sign = inspect.signature(logger_func)
-        # We accept two kinds of metric loggers:
-        # def log_metric(k, v)
-        # def log_metric(k, v, variant_id)
-        if len(sign.parameters) not in [2, 3]:
-            return
-        self._metric_loggers.append(logger_func)
-
-    def remove_metric_logger(self, logger_func: Callable):
-        self._metric_loggers.remove(logger_func)
-
-
-def log_metric(key, value, variant_id=None):
-    """Log a metric for current promptflow run.
-
-    :param key: Metric name.
-    :type key: str
-    :param value: Metric value.
-    :type value: float
-    :param variant_id: Variant id for the metric.
-    :type variant_id: str
-    """
-    MetricLoggerManager.get_instance().log_metric(key, value, variant_id)
-
-
-def add_metric_logger(logger_func: Callable):
-    MetricLoggerManager.get_instance().add_metric_logger(logger_func)
-
-
-def remove_metric_logger(logger_func: Callable):
-    MetricLoggerManager.get_instance().remove_metric_logger(logger_func)
-#
-# End of G-Eval.py
-#######################################################################################################################

App_Function_Libraries/Books/Book_Ingestion_Lib.py

@@ -1,577 +0,0 @@
-# Book_Ingestion_Lib.py
-#########################################
-# Library to hold functions for ingesting book files.#
-#
-####################
-# Function List
-#
-# 1. ingest_text_file(file_path, title=None, author=None, keywords=None):
-# 2.
-#
-#
-####################
-#
-# Imports
-import os
-import re
-import tempfile
-import zipfile
-from datetime import datetime
-import logging
-#
-# External Imports
-import ebooklib
-from bs4 import BeautifulSoup
-from ebooklib import epub
-#
-# Import Local
-from App_Function_Libraries.DB.DB_Manager import add_media_with_keywords, add_media_to_database
-from App_Function_Libraries.Summarization.Summarization_General_Lib import perform_summarization
-from App_Function_Libraries.Chunk_Lib import chunk_ebook_by_chapters
-from App_Function_Libraries.Metrics.metrics_logger import log_counter, log_histogram
-#
-#######################################################################################################################
-# Function Definitions
-#
-
-def import_epub(file_path,
-                title=None,
-                author=None,
-                keywords=None,
-                custom_prompt=None,
-                system_prompt=None,
-                summary=None,
-                auto_summarize=False,
-                api_name=None,
-                api_key=None,
-                chunk_options=None,
-                custom_chapter_pattern=None
-                ):
-    """
-    Imports an EPUB file, extracts its content, chunks it, optionally summarizes it, and adds it to the database.
-
-    Parameters:
-        - file_path (str): Path to the EPUB file.
-        - title (str, optional): Title of the book.
-        - author (str, optional): Author of the book.
-        - keywords (str, optional): Comma-separated keywords for the book.
-        - custom_prompt (str, optional): Custom user prompt for summarization.
-        - summary (str, optional): Predefined summary of the book.
-        - auto_summarize (bool, optional): Whether to auto-summarize the chunks.
-        - api_name (str, optional): API name for summarization.
-        - api_key (str, optional): API key for summarization.
-        - chunk_options (dict, optional): Options for chunking.
-        - custom_chapter_pattern (str, optional): Custom regex pattern for chapter detection.
-
-    Returns:
-        - str: Status message indicating success or failure.
-    """
-    try:
-        logging.info(f"Importing EPUB file from {file_path}")
-        log_counter("epub_import_attempt", labels={"file_path": file_path})
-
-        start_time = datetime.now()
-
-        # Convert EPUB to Markdown
-        markdown_content = epub_to_markdown(file_path)
-        logging.debug("Converted EPUB to Markdown.")
-
-        # Extract metadata if not provided
-        if not title or not author:
-            extracted_title, extracted_author = extract_epub_metadata(markdown_content)
-            title = title or extracted_title or os.path.splitext(os.path.basename(file_path))[0]
-            author = author or extracted_author or "Unknown"
-            logging.debug(f"Extracted metadata - Title: {title}, Author: {author}")
-
-        # Process keywords
-        keyword_list = [kw.strip() for kw in keywords.split(',')] if keywords else []
-        logging.debug(f"Keywords: {keyword_list}")
-
-        # Set default chunk options if not provided
-        if chunk_options is None:
-            chunk_options = {
-                'method': 'chapter',
-                'max_size': 500,
-                'overlap': 200,
-                'custom_chapter_pattern': custom_chapter_pattern
-            }
-        else:
-            # Ensure 'method' is set to 'chapter' when using chapter chunking
-            chunk_options.setdefault('method', 'chapter')
-            chunk_options.setdefault('custom_chapter_pattern', custom_chapter_pattern)
-
-        # Chunk the content by chapters
-        chunks = chunk_ebook_by_chapters(markdown_content, chunk_options)
-        logging.info(f"Total chunks created: {len(chunks)}")
-        log_histogram("epub_chunks_created", len(chunks), labels={"file_path": file_path})
-
-        if chunks:
-            logging.debug(f"Structure of first chunk: {chunks[0].keys()}")
-
-        # Handle summarization if enabled
-        if auto_summarize and api_name and api_key:
-            logging.info("Auto-summarization is enabled.")
-            summarized_chunks = []
-            for chunk in chunks:
-                chunk_text = chunk.get('text', '')
-                if chunk_text:
-                    summary_text = perform_summarization(api_name, chunk_text, custom_prompt, api_key,
-                                                            recursive_summarization=False, temp=None,
-                                                            system_message=system_prompt
-                                                            )
-                    chunk['metadata']['summary'] = summary_text
-                    summarized_chunks.append(chunk)
-            chunks = summarized_chunks
-            logging.info("Summarization of chunks completed.")
-            log_counter("epub_chunks_summarized", value=len(chunks), labels={"file_path": file_path})
-        else:
-            # If not summarizing, set a default summary or use provided summary
-            if summary:
-                logging.debug("Using provided summary.")
-            else:
-                summary = "No summary provided."
-
-        # Create info_dict
-        info_dict = {
-            'title': title,
-            'uploader': author,
-            'ingestion_date': datetime.now().strftime('%Y-%m-%d')
-        }
-
-        # Prepare segments for database
-        segments = [{'Text': chunk.get('text', chunk.get('content', ''))} for chunk in chunks]
-        logging.debug(f"Prepared segments for database. Number of segments: {len(segments)}")
-
-        # Add to database
-        result = add_media_to_database(
-            url=file_path,
-            info_dict=info_dict,
-            segments=segments,
-            summary=summary,
-            keywords=keyword_list,
-            custom_prompt_input=custom_prompt,
-            whisper_model="Imported",
-            media_type="ebook",
-            overwrite=False
-        )
-
-        end_time = datetime.now()
-        processing_time = (end_time - start_time).total_seconds()
-        log_histogram("epub_import_duration", processing_time, labels={"file_path": file_path})
-
-        logging.info(f"Ebook '{title}' by {author} imported successfully. Database result: {result}")
-        log_counter("epub ingested into the DB successfully", labels={"file_path": file_path})
-        return f"Ebook '{title}' by {author} imported successfully. Database result: {result}"
-
-    except Exception as e:
-        logging.exception(f"Error importing ebook: {str(e)}")
-        log_counter("epub_import_error", labels={"file_path": file_path, "error": str(e)})
-        return f"Error importing ebook: {str(e)}"
-
-
-# FIXME
-def process_zip_file(zip_file,
-                     title,
-                     author,
-                     keywords,
-                     custom_prompt,
-                     system_prompt,
-                     summary,
-                     auto_summarize,
-                     api_name,
-                     api_key,
-                     chunk_options
-                     ):
-    """
-    Processes a ZIP file containing multiple EPUB files and imports each one.
-
-    Parameters:
-        - zip_file (file-like object): The ZIP file to process.
-        - title (str): Title prefix for the books.
-        - author (str): Author name for the books.
-        - keywords (str): Comma-separated keywords.
-        - custom_prompt (str): Custom user prompt for summarization.
-        - summary (str): Predefined summary (not used in this context).
-        - auto_summarize (bool): Whether to auto-summarize the chunks.
-        - api_name (str): API name for summarization.
-        - api_key (str): API key for summarization.
-        - chunk_options (dict): Options for chunking.
-
-    Returns:
-        - str: Combined status messages for all EPUB files in the ZIP.
-    """
-    results = []
-    try:
-        with tempfile.TemporaryDirectory() as temp_dir:
-            zip_path = zip_file.name if hasattr(zip_file, 'name') else zip_file.path
-            logging.info(f"Extracting ZIP file {zip_path} to temporary directory {temp_dir}")
-            log_counter("zip_processing_attempt", labels={"zip_path": zip_path})
-
-            with zipfile.ZipFile(zip_path, 'r') as zip_ref:
-                zip_ref.extractall(temp_dir)
-
-            epub_files = [f for f in os.listdir(temp_dir) if f.lower().endswith('.epub')]
-            log_histogram("epub_files_in_zip", len(epub_files), labels={"zip_path": zip_path})
-
-            for filename in epub_files:
-                file_path = os.path.join(temp_dir, filename)
-                logging.info(f"Processing EPUB file {filename} from ZIP.")
-                result = import_epub(
-                    file_path=file_path,
-                    title=title,
-                    author=author,
-                    keywords=keywords,
-                    custom_prompt=custom_prompt,
-                    summary=summary,
-                    auto_summarize=auto_summarize,
-                    api_name=api_name,
-                    api_key=api_key,
-                    chunk_options=chunk_options,
-                    custom_chapter_pattern=chunk_options.get('custom_chapter_pattern') if chunk_options else None
-                )
-                results.append(f"File: {filename} - {result}")
-
-            logging.info("Completed processing all EPUB files in the ZIP.")
-            log_counter("zip_processing_success", labels={"zip_path": zip_path})
-    except Exception as e:
-        logging.exception(f"Error processing ZIP file: {str(e)}")
-        log_counter("zip_processing_error", labels={"zip_path": zip_path, "error": str(e)})
-        return f"Error processing ZIP file: {str(e)}"
-
-    return "\n".join(results)
-
-
-def import_file_handler(file,
-                        title,
-                        author,
-                        keywords,
-                        system_prompt,
-                        custom_prompt,
-                        auto_summarize,
-                        api_name,
-                        api_key,
-                        max_chunk_size,
-                        chunk_overlap,
-                        custom_chapter_pattern
-                        ):
-    try:
-        log_counter("file_import_attempt", labels={"file_name": file.name})
-
-        # Handle max_chunk_size
-        if isinstance(max_chunk_size, str):
-            max_chunk_size = int(max_chunk_size) if max_chunk_size.strip() else 4000
-        elif not isinstance(max_chunk_size, int):
-            max_chunk_size = 4000  # Default value if not a string or int
-
-        # Handle chunk_overlap
-        if isinstance(chunk_overlap, str):
-            chunk_overlap = int(chunk_overlap) if chunk_overlap.strip() else 0
-        elif not isinstance(chunk_overlap, int):
-            chunk_overlap = 0  # Default value if not a string or int
-
-        chunk_options = {
-            'method': 'chapter',
-            'max_size': max_chunk_size,
-            'overlap': chunk_overlap,
-            'custom_chapter_pattern': custom_chapter_pattern if custom_chapter_pattern else None
-        }
-
-        if file is None:
-            log_counter("file_import_error", labels={"error": "No file uploaded"})
-            return "No file uploaded."
-
-        file_path = file.name
-        if not os.path.exists(file_path):
-            log_counter("file_import_error", labels={"error": "File not found", "file_name": file.name})
-            return "Uploaded file not found."
-
-        start_time = datetime.now()
-
-        if file_path.lower().endswith('.epub'):
-            status = import_epub(
-                file_path,
-                title,
-                author,
-                keywords,
-                custom_prompt=custom_prompt,
-                system_prompt=system_prompt,
-                summary=None,
-                auto_summarize=auto_summarize,
-                api_name=api_name,
-                api_key=api_key,
-                chunk_options=chunk_options,
-                custom_chapter_pattern=custom_chapter_pattern
-            )
-            log_counter("epub_import_success", labels={"file_name": file.name})
-            result = f"📚 EPUB Imported Successfully:\n{status}"
-        elif file.name.lower().endswith('.zip'):
-            status = process_zip_file(
-                zip_file=file,
-                title=title,
-                author=author,
-                keywords=keywords,
-                custom_prompt=custom_prompt,
-                system_prompt=system_prompt,
-                summary=None,
-                auto_summarize=auto_summarize,
-                api_name=api_name,
-                api_key=api_key,
-                chunk_options=chunk_options
-            )
-            log_counter("zip_import_success", labels={"file_name": file.name})
-            result = f"📦 ZIP Processed Successfully:\n{status}"
-        elif file.name.lower().endswith(('.chm', '.html', '.pdf', '.xml', '.opml')):
-            file_type = file.name.split('.')[-1].upper()
-            log_counter("unsupported_file_type", labels={"file_type": file_type})
-            result = f"{file_type} file import is not yet supported."
-        else:
-            log_counter("unsupported_file_type", labels={"file_type": file.name.split('.')[-1]})
-            result = "❌ Unsupported file type. Please upload an `.epub` file or a `.zip` file containing `.epub` files."
-
-        end_time = datetime.now()
-        processing_time = (end_time - start_time).total_seconds()
-        log_histogram("file_import_duration", processing_time, labels={"file_name": file.name})
-
-        return result
-
-    except ValueError as ve:
-        logging.exception(f"Error parsing input values: {str(ve)}")
-        log_counter("file_import_error", labels={"error": "Invalid input", "file_name": file.name})
-        return f"❌ Error: Invalid input for chunk size or overlap. Please enter valid numbers."
-    except Exception as e:
-        logging.exception(f"Error during file import: {str(e)}")
-        log_counter("file_import_error", labels={"error": str(e), "file_name": file.name})
-        return f"❌ Error during import: {str(e)}"
-
-
-def read_epub(file_path):
-    """
-    Reads and extracts text from an EPUB file.
-
-    Parameters:
-        - file_path (str): Path to the EPUB file.
-
-    Returns:
-        - str: Extracted text content from the EPUB.
-    """
-    try:
-        logging.info(f"Reading EPUB file from {file_path}")
-        book = epub.read_epub(file_path)
-        chapters = []
-        for item in book.get_items():
-            if item.get_type() == ebooklib.ITEM_DOCUMENT:
-                chapters.append(item.get_content())
-
-        text = ""
-        for html_content in chapters:
-            soup = BeautifulSoup(html_content, 'html.parser')
-            text += soup.get_text(separator='\n\n') + "\n\n"
-        logging.debug("EPUB content extraction completed.")
-        return text
-    except Exception as e:
-        logging.exception(f"Error reading EPUB file: {str(e)}")
-        raise
-
-
-# Ingest a text file into the database with Title/Author/Keywords
-def extract_epub_metadata(content):
-    title_match = re.search(r'Title:\s*(.*?)\n', content)
-    author_match = re.search(r'Author:\s*(.*?)\n', content)
-
-    title = title_match.group(1) if title_match else None
-    author = author_match.group(1) if author_match else None
-
-    return title, author
-
-
-def ingest_text_file(file_path, title=None, author=None, keywords=None):
-    """
-    Ingests a plain text file into the database with optional metadata.
-
-    Parameters:
-        - file_path (str): Path to the text file.
-        - title (str, optional): Title of the document.
-        - author (str, optional): Author of the document.
-        - keywords (str, optional): Comma-separated keywords.
-
-    Returns:
-        - str: Status message indicating success or failure.
-    """
-    try:
-        with open(file_path, 'r', encoding='utf-8') as file:
-            content = file.read()
-
-        # Check if it's a converted epub and extract metadata if so
-        if 'epub_converted' in (keywords or '').lower():
-            extracted_title, extracted_author = extract_epub_metadata(content)
-            title = title or extracted_title
-            author = author or extracted_author
-            logging.debug(f"Extracted metadata for converted EPUB - Title: {title}, Author: {author}")
-
-        # If title is still not provided, use the filename without extension
-        if not title:
-            title = os.path.splitext(os.path.basename(file_path))[0]
-
-        # If author is still not provided, set it to 'Unknown'
-        if not author:
-            author = 'Unknown'
-
-        # If keywords are not provided, use a default keyword
-        if not keywords:
-            keywords = 'text_file,epub_converted'
-        else:
-            keywords = f'text_file,epub_converted,{keywords}'
-
-        # Add the text file to the database
-        add_media_with_keywords(
-            url=file_path,
-            title=title,
-            media_type='document',
-            content=content,
-            keywords=keywords,
-            prompt='No prompt for text files',
-            summary='No summary for text files',
-            transcription_model='None',
-            author=author,
-            ingestion_date=datetime.now().strftime('%Y-%m-%d')
-        )
-
-        logging.info(f"Text file '{title}' by {author} ingested successfully.")
-        return f"Text file '{title}' by {author} ingested successfully."
-    except Exception as e:
-        logging.error(f"Error ingesting text file: {str(e)}")
-        return f"Error ingesting text file: {str(e)}"
-
-
-def ingest_folder(folder_path, keywords=None):
-    """
-    Ingests all text files within a specified folder.
-
-    Parameters:
-        - folder_path (str): Path to the folder containing text files.
-        - keywords (str, optional): Comma-separated keywords to add to each file.
-
-    Returns:
-        - str: Combined status messages for all ingested text files.
-    """
-    results = []
-    try:
-        logging.info(f"Ingesting all text files from folder {folder_path}")
-        for filename in os.listdir(folder_path):
-            if filename.lower().endswith('.txt'):
-                file_path = os.path.join(folder_path, filename)
-                result = ingest_text_file(file_path, keywords=keywords)
-                results.append(result)
-        logging.info("Completed ingestion of all text files in the folder.")
-    except Exception as e:
-        logging.exception(f"Error ingesting folder: {str(e)}")
-        return f"Error ingesting folder: {str(e)}"
-
-    return "\n".join(results)
-
-
-def epub_to_markdown(epub_path):
-    """
-    Converts an EPUB file to Markdown format, including the table of contents and chapter contents.
-
-    Parameters:
-        - epub_path (str): Path to the EPUB file.
-
-    Returns:
-        - str: Markdown-formatted content of the EPUB.
-    """
-    try:
-        logging.info(f"Converting EPUB to Markdown from {epub_path}")
-        book = epub.read_epub(epub_path)
-        markdown_content = "# Table of Contents\n\n"
-        chapters = []
-
-        # Extract and format the table of contents
-        toc = book.toc
-        for item in toc:
-            if isinstance(item, tuple):
-                section, children = item
-                level = 1
-                markdown_content += format_toc_item(section, level)
-                for child in children:
-                    markdown_content += format_toc_item(child, level + 1)
-            else:
-                markdown_content += format_toc_item(item, 1)
-
-        markdown_content += "\n---\n\n"
-
-        # Process each chapter
-        for item in book.get_items():
-            if item.get_type() == ebooklib.ITEM_DOCUMENT:
-                chapter_content = item.get_content().decode('utf-8')
-                soup = BeautifulSoup(chapter_content, 'html.parser')
-
-                # Extract chapter title
-                title = soup.find(['h1', 'h2', 'h3'])
-                if title:
-                    chapter_title = title.get_text()
-                    markdown_content += f"# {chapter_title}\n\n"
-
-                # Process chapter content
-                for elem in soup.find_all(['p', 'h1', 'h2', 'h3', 'h4', 'h5', 'h6', 'ul', 'ol']):
-                    if elem.name.startswith('h'):
-                        level = int(elem.name[1])
-                        markdown_content += f"{'#' * level} {elem.get_text()}\n\n"
-                    elif elem.name == 'p':
-                        markdown_content += f"{elem.get_text()}\n\n"
-                    elif elem.name in ['ul', 'ol']:
-                        for li in elem.find_all('li'):
-                            prefix = '-' if elem.name == 'ul' else '1.'
-                            markdown_content += f"{prefix} {li.get_text()}\n"
-                        markdown_content += "\n"
-
-                markdown_content += "---\n\n"
-
-        logging.debug("EPUB to Markdown conversion completed.")
-        return markdown_content
-
-    except Exception as e:
-        logging.exception(f"Error converting EPUB to Markdown: {str(e)}")
-        raise
-
-
-def format_toc_item(item, level):
-    """
-    Formats a table of contents item into Markdown list format.
-
-    Parameters:
-        - item (epub.Link or epub.Section): TOC item.
-        - level (int): Heading level for indentation.
-
-    Returns:
-        - str: Markdown-formatted TOC item.
-    """
-    try:
-        if isinstance(item, epub.Link):
-            title = item.title
-        elif isinstance(item, epub.Section):
-            title = item.title
-        else:
-            title = str(item)
-
-        return f"{'  ' * (level - 1)}- [{title}](#{slugify(title)})\n"
-    except Exception as e:
-        logging.exception(f"Error formatting TOC item: {str(e)}")
-        return ""
-
-
-def slugify(text):
-    """
-    Converts a string into a slug suitable for Markdown links.
-
-    Parameters:
-        - text (str): The text to slugify.
-
-    Returns:
-        - str: Slugified text.
-    """
-    return re.sub(r'[\W_]+', '-', text.lower()).strip('-')
-
-#
-# End of Function Definitions
-#######################################################################################################################

App_Function_Libraries/Character_Chat/Character_Chat_Lib.py

@@ -1,607 +0,0 @@
-# Character_Chat_Lib.py
-# Description: Functions for character chat cards.
-#
-# Imports
-import json
-import logging
-import io
-import base64
-import time
-from typing import Dict, Any, Optional, List, Tuple
-#
-# External Imports
-from PIL import Image
-#
-# Local imports
-from App_Function_Libraries.DB.DB_Manager import get_character_card_by_id, get_character_chat_by_id
-from App_Function_Libraries.Metrics.metrics_logger import log_counter, log_histogram
-#
-# Constants
-####################################################################################################
-#
-# Functions
-
-# Using https://github.com/malfoyslastname/character-card-spec-v2 as the standard for v2 character cards
-
-#################################################################################
-#
-# Placeholder functions:
-
-def replace_placeholders(text: str, char_name: str, user_name: str) -> str:
-    """
-    Replace placeholders in the given text with appropriate values.
-
-    Args:
-        text (str): The text containing placeholders.
-        char_name (str): The name of the character.
-        user_name (str): The name of the user.
-
-    Returns:
-        str: The text with placeholders replaced.
-    """
-    replacements = {
-        '{{char}}': char_name,
-        '{{user}}': user_name,
-        '{{random_user}}': user_name  # Assuming random_user is the same as user for simplicity
-    }
-
-    for placeholder, value in replacements.items():
-        text = text.replace(placeholder, value)
-
-    return text
-
-def replace_user_placeholder(history, user_name):
-    """
-    Replaces all instances of '{{user}}' in the chat history with the actual user name.
-
-    Args:
-        history (list): The current chat history as a list of tuples (user_message, bot_message).
-        user_name (str): The name entered by the user.
-
-    Returns:
-        list: Updated chat history with placeholders replaced.
-    """
-    if not user_name:
-        user_name = "User"  # Default name if none provided
-
-    updated_history = []
-    for user_msg, bot_msg in history:
-        # Replace in user message
-        if user_msg:
-            user_msg = user_msg.replace("{{user}}", user_name)
-        # Replace in bot message
-        if bot_msg:
-            bot_msg = bot_msg.replace("{{user}}", user_name)
-        updated_history.append((user_msg, bot_msg))
-    return updated_history
-
-#
-# End of Placeholder functions
-#################################################################################
-
-#################################################################################
-#
-# Functions for character card processing:
-
-def extract_character_id(choice: str) -> int:
-    """Extract the character ID from the dropdown selection string."""
-    log_counter("extract_character_id_attempt")
-    try:
-        character_id = int(choice.split('(ID: ')[1].rstrip(')'))
-        log_counter("extract_character_id_success")
-        return character_id
-    except Exception as e:
-        log_counter("extract_character_id_error", labels={"error": str(e)})
-        raise
-
-def load_character_wrapper(character_id: int, user_name: str) -> Tuple[Dict[str, Any], List[Tuple[Optional[str], str]], Optional[Image.Image]]:
-    """Wrapper function to load character and image using the extracted ID."""
-    log_counter("load_character_wrapper_attempt")
-    start_time = time.time()
-    try:
-        char_data, chat_history, img = load_character_and_image(character_id, user_name)
-        load_duration = time.time() - start_time
-        log_histogram("load_character_wrapper_duration", load_duration)
-        log_counter("load_character_wrapper_success")
-        return char_data, chat_history, img
-    except Exception as e:
-        log_counter("load_character_wrapper_error", labels={"error": str(e)})
-        raise
-
-def parse_character_book(book_data: Dict[str, Any]) -> Dict[str, Any]:
-    """
-    Parse the character book data from a V2 character card.
-
-    Args:
-        book_data (Dict[str, Any]): The raw character book data from the character card.
-
-    Returns:
-        Dict[str, Any]: The parsed and structured character book data.
-    """
-    parsed_book = {
-        'name': book_data.get('name', ''),
-        'description': book_data.get('description', ''),
-        'scan_depth': book_data.get('scan_depth'),
-        'token_budget': book_data.get('token_budget'),
-        'recursive_scanning': book_data.get('recursive_scanning', False),
-        'extensions': book_data.get('extensions', {}),
-        'entries': []
-    }
-
-    for entry in book_data.get('entries', []):
-        parsed_entry = {
-            'keys': entry['keys'],
-            'content': entry['content'],
-            'extensions': entry.get('extensions', {}),
-            'enabled': entry['enabled'],
-            'insertion_order': entry['insertion_order'],
-            'case_sensitive': entry.get('case_sensitive', False),
-            'name': entry.get('name', ''),
-            'priority': entry.get('priority'),
-            'id': entry.get('id'),
-            'comment': entry.get('comment', ''),
-            'selective': entry.get('selective', False),
-            'secondary_keys': entry.get('secondary_keys', []),
-            'constant': entry.get('constant', False),
-            'position': entry.get('position')
-        }
-        parsed_book['entries'].append(parsed_entry)
-
-    return parsed_book
-
-def load_character_and_image(character_id: int, user_name: str) -> Tuple[Optional[Dict[str, Any]], List[Tuple[Optional[str], str]], Optional[Image.Image]]:
-    """
-    Load a character and its associated image based on the character ID.
-
-    Args:
-        character_id (int): The ID of the character to load.
-        user_name (str): The name of the user, used for placeholder replacement.
-
-    Returns:
-        Tuple[Optional[Dict[str, Any]], List[Tuple[Optional[str], str]], Optional[Image.Image]]:
-        A tuple containing the character data, chat history, and character image (if available).
-    """
-    log_counter("load_character_and_image_attempt")
-    start_time = time.time()
-    try:
-        char_data = get_character_card_by_id(character_id)
-        if not char_data:
-            log_counter("load_character_and_image_no_data")
-            logging.warning(f"No character data found for ID: {character_id}")
-            return None, [], None
-
-        # Replace placeholders in character data
-        for field in ['first_mes', 'mes_example', 'scenario', 'description', 'personality']:
-            if field in char_data:
-                char_data[field] = replace_placeholders(char_data[field], char_data['name'], user_name)
-
-        # Replace placeholders in first_mes
-        first_mes = char_data.get('first_mes', "Hello! I'm ready to chat.")
-        first_mes = replace_placeholders(first_mes, char_data['name'], user_name)
-
-        chat_history = [(None, first_mes)] if first_mes else []
-
-        img = None
-        if char_data.get('image'):
-            try:
-                image_data = base64.b64decode(char_data['image'])
-                img = Image.open(io.BytesIO(image_data)).convert("RGBA")
-                log_counter("load_character_image_success")
-            except Exception as e:
-                log_counter("load_character_image_error", labels={"error": str(e)})
-                logging.error(f"Error processing image for character '{char_data['name']}': {e}")
-
-        load_duration = time.time() - start_time
-        log_histogram("load_character_and_image_duration", load_duration)
-        log_counter("load_character_and_image_success")
-        return char_data, chat_history, img
-
-    except Exception as e:
-        log_counter("load_character_and_image_error", labels={"error": str(e)})
-        logging.error(f"Error in load_character_and_image: {e}")
-        return None, [], None
-
-def load_chat_and_character(chat_id: int, user_name: str) -> Tuple[Optional[Dict[str, Any]], List[Tuple[str, str]], Optional[Image.Image]]:
-    """
-    Load a chat and its associated character, including the character image and process templates.
-
-    Args:
-        chat_id (int): The ID of the chat to load.
-        user_name (str): The name of the user.
-
-    Returns:
-        Tuple[Optional[Dict[str, Any]], List[Tuple[str, str]], Optional[Image.Image]]:
-        A tuple containing the character data, processed chat history, and character image (if available).
-    """
-    log_counter("load_chat_and_character_attempt")
-    start_time = time.time()
-    try:
-        # Load the chat
-        chat = get_character_chat_by_id(chat_id)
-        if not chat:
-            log_counter("load_chat_and_character_no_chat")
-            logging.warning(f"No chat found with ID: {chat_id}")
-            return None, [], None
-
-        # Load the associated character
-        character_id = chat['character_id']
-        char_data = get_character_card_by_id(character_id)
-        if not char_data:
-            log_counter("load_chat_and_character_no_character")
-            logging.warning(f"No character found for chat ID: {chat_id}")
-            return None, chat['chat_history'], None
-
-        # Process the chat history
-        processed_history = process_chat_history(chat['chat_history'], char_data['name'], user_name)
-
-        # Load the character image
-        img = None
-        if char_data.get('image'):
-            try:
-                image_data = base64.b64decode(char_data['image'])
-                img = Image.open(io.BytesIO(image_data)).convert("RGBA")
-                log_counter("load_chat_character_image_success")
-            except Exception as e:
-                log_counter("load_chat_character_image_error", labels={"error": str(e)})
-                logging.error(f"Error processing image for character '{char_data['name']}': {e}")
-
-        # Process character data templates
-        for field in ['first_mes', 'mes_example', 'scenario', 'description', 'personality']:
-            if field in char_data:
-                char_data[field] = replace_placeholders(char_data[field], char_data['name'], user_name)
-
-        load_duration = time.time() - start_time
-        log_histogram("load_chat_and_character_duration", load_duration)
-        log_counter("load_chat_and_character_success")
-        return char_data, processed_history, img
-
-    except Exception as e:
-        log_counter("load_chat_and_character_error", labels={"error": str(e)})
-        logging.error(f"Error in load_chat_and_character: {e}")
-        return None, [], None
-
-
-def extract_json_from_image(image_file):
-    logging.debug(f"Attempting to extract JSON from image: {image_file.name}")
-    log_counter("extract_json_from_image_attempt")
-    start_time = time.time()
-    try:
-        with Image.open(image_file) as img:
-            logging.debug("Image opened successfully")
-            metadata = img.info
-            if 'chara' in metadata:
-                logging.debug("Found 'chara' in image metadata")
-                chara_content = metadata['chara']
-                logging.debug(f"Content of 'chara' metadata (first 100 chars): {chara_content[:100]}...")
-                try:
-                    decoded_content = base64.b64decode(chara_content).decode('utf-8')
-                    logging.debug(f"Decoded content (first 100 chars): {decoded_content[:100]}...")
-                    log_counter("extract_json_from_image_metadata_success")
-                    return decoded_content
-                except Exception as e:
-                    logging.error(f"Error decoding base64 content: {e}")
-                    log_counter("extract_json_from_image_decode_error", labels={"error": str(e)})
-
-            logging.warning("'chara' not found in metadata, attempting to find JSON data in image bytes")
-            # Alternative method to extract embedded JSON from image bytes if metadata is not available
-            img_byte_arr = io.BytesIO()
-            img.save(img_byte_arr, format='PNG')
-            img_bytes = img_byte_arr.getvalue()
-            img_str = img_bytes.decode('latin1')
-
-            # Search for JSON-like structures in the image bytes
-            json_start = img_str.find('{')
-            json_end = img_str.rfind('}')
-            if json_start != -1 and json_end != -1 and json_end > json_start:
-                possible_json = img_str[json_start:json_end+1]
-                try:
-                    json.loads(possible_json)
-                    logging.debug("Found JSON data in image bytes")
-                    log_counter("extract_json_from_image_bytes_success")
-                    return possible_json
-                except json.JSONDecodeError:
-                    logging.debug("No valid JSON found in image bytes")
-                    log_counter("extract_json_from_image_invalid_json")
-
-            logging.warning("No JSON data found in the image")
-            log_counter("extract_json_from_image_no_json_found")
-    except Exception as e:
-        log_counter("extract_json_from_image_error", labels={"error": str(e)})
-        logging.error(f"Error extracting JSON from image: {e}")
-
-    extract_duration = time.time() - start_time
-    log_histogram("extract_json_from_image_duration", extract_duration)
-    return None
-
-
-def load_chat_history(file):
-    log_counter("load_chat_history_attempt")
-    start_time = time.time()
-    try:
-        content = file.read().decode('utf-8')
-        chat_data = json.loads(content)
-
-        # Extract history and character name from the loaded data
-        history = chat_data.get('history') or chat_data.get('messages')
-        character_name = chat_data.get('character') or chat_data.get('character_name')
-
-        if not history or not character_name:
-            log_counter("load_chat_history_incomplete_data")
-            logging.error("Chat history or character name missing in the imported file.")
-            return None, None
-
-        load_duration = time.time() - start_time
-        log_histogram("load_chat_history_duration", load_duration)
-        log_counter("load_chat_history_success")
-        return history, character_name
-    except Exception as e:
-        log_counter("load_chat_history_error", labels={"error": str(e)})
-        logging.error(f"Error loading chat history: {e}")
-        return None, None
-
-
-def process_chat_history(chat_history: List[Tuple[str, str]], char_name: str, user_name: str) -> List[Tuple[str, str]]:
-    """
-    Process the chat history to replace placeholders in both user and character messages.
-
-    Args:
-        chat_history (List[Tuple[str, str]]): The chat history.
-        char_name (str): The name of the character.
-        user_name (str): The name of the user.
-
-    Returns:
-        List[Tuple[str, str]]: The processed chat history.
-    """
-    log_counter("process_chat_history_attempt")
-    start_time = time.time()
-    try:
-        processed_history = []
-        for user_msg, char_msg in chat_history:
-            if user_msg:
-                user_msg = replace_placeholders(user_msg, char_name, user_name)
-            if char_msg:
-                char_msg = replace_placeholders(char_msg, char_name, user_name)
-            processed_history.append((user_msg, char_msg))
-
-        process_duration = time.time() - start_time
-        log_histogram("process_chat_history_duration", process_duration)
-        log_counter("process_chat_history_success", labels={"message_count": len(chat_history)})
-        return processed_history
-    except Exception as e:
-        log_counter("process_chat_history_error", labels={"error": str(e)})
-        logging.error(f"Error processing chat history: {e}")
-        raise
-
-def validate_character_book(book_data):
-    """
-    Validate the 'character_book' field in the character card.
-
-    Args:
-        book_data (dict): The character book data.
-
-    Returns:
-        Tuple[bool, List[str]]: A tuple containing a boolean indicating validity and a list of validation messages.
-    """
-    validation_messages = []
-
-    # Optional fields with expected types
-    optional_fields = {
-        'name': str,
-        'description': str,
-        'scan_depth': (int, float),
-        'token_budget': (int, float),
-        'recursive_scanning': bool,
-        'extensions': dict,
-        'entries': list
-    }
-
-    for field, expected_type in optional_fields.items():
-        if field in book_data:
-            if not isinstance(book_data[field], expected_type):
-                validation_messages.append(f"Field 'character_book.{field}' must be of type '{expected_type}'.")
-    # 'entries' is required
-    if 'entries' not in book_data or not isinstance(book_data['entries'], list):
-        validation_messages.append("Field 'character_book.entries' is required and must be a list.")
-        return False, validation_messages
-
-    # Validate each entry in 'entries'
-    entries = book_data.get('entries', [])
-    entry_ids = set()
-    for idx, entry in enumerate(entries):
-        is_valid_entry, entry_messages = validate_character_book_entry(entry, idx, entry_ids)
-        if not is_valid_entry:
-            validation_messages.extend(entry_messages)
-
-    is_valid = len(validation_messages) == 0
-    return is_valid, validation_messages
-
-def validate_character_book_entry(entry, idx, entry_ids):
-    """
-    Validate an entry in the 'character_book.entries' list.
-
-    Args:
-        entry (dict): The entry data.
-        idx (int): The index of the entry in the list.
-        entry_ids (set): A set of existing entry IDs for uniqueness checking.
-
-    Returns:
-        Tuple[bool, List[str]]: A tuple containing a boolean indicating validity and a list of validation messages.
-    """
-    validation_messages = []
-    required_fields = {
-        'keys': list,
-        'content': str,
-        'extensions': dict,
-        'enabled': bool,
-        'insertion_order': (int, float)
-    }
-
-    for field, expected_type in required_fields.items():
-        if field not in entry:
-            validation_messages.append(f"Entry {idx}: Missing required field '{field}'.")
-        elif not isinstance(entry[field], expected_type):
-            validation_messages.append(f"Entry {idx}: Field '{field}' must be of type '{expected_type}'.")
-        elif field == 'content' and not entry[field].strip():
-            validation_messages.append(f"Entry {idx}: Field 'content' cannot be empty.")
-        elif field == 'keys' and not entry[field]:
-            validation_messages.append(f"Entry {idx}: Field 'keys' cannot be empty.")
-
-    # Optional fields
-    optional_fields = {
-        'case_sensitive': bool,
-        'name': str,
-        'priority': (int, float),
-        'id': (int, float),
-        'comment': str,
-        'selective': bool,
-        'secondary_keys': list,
-        'constant': bool,
-        'position': str  # Should be 'before_char' or 'after_char'
-    }
-
-    for field, expected_type in optional_fields.items():
-        if field in entry and not isinstance(entry[field], expected_type):
-            validation_messages.append(f"Entry {idx}: Field '{field}' must be of type '{expected_type}'.")
-
-    # Validate 'position' value if present
-    if 'position' in entry:
-        if entry['position'] not in ['before_char', 'after_char']:
-            validation_messages.append(f"Entry {idx}: Field 'position' must be 'before_char' or 'after_char'.")
-
-    # Validate 'secondary_keys' if 'selective' is True
-    if entry.get('selective', False):
-        if 'secondary_keys' not in entry or not isinstance(entry['secondary_keys'], list):
-            validation_messages.append(f"Entry {idx}: 'secondary_keys' must be a list when 'selective' is True.")
-        elif not entry['secondary_keys']:
-            validation_messages.append(f"Entry {idx}: 'secondary_keys' cannot be empty when 'selective' is True.")
-
-    # Validate 'keys' list elements
-    if 'keys' in entry and isinstance(entry['keys'], list):
-        for i, key in enumerate(entry['keys']):
-            if not isinstance(key, str) or not key.strip():
-                validation_messages.append(f"Entry {idx}: Element {i} in 'keys' must be a non-empty string.")
-
-    # Validate 'secondary_keys' list elements
-    if 'secondary_keys' in entry and isinstance(entry['secondary_keys'], list):
-        for i, key in enumerate(entry['secondary_keys']):
-            if not isinstance(key, str) or not key.strip():
-                validation_messages.append(f"Entry {idx}: Element {i} in 'secondary_keys' must be a non-empty string.")
-
-    # Validate 'id' uniqueness
-    if 'id' in entry:
-        entry_id = entry['id']
-        if entry_id in entry_ids:
-            validation_messages.append \
-                (f"Entry {idx}: Duplicate 'id' value '{entry_id}'. Each entry 'id' must be unique.")
-        else:
-            entry_ids.add(entry_id)
-
-    # Validate 'extensions' keys are namespaced
-    if 'extensions' in entry and isinstance(entry['extensions'], dict):
-        for key in entry['extensions'].keys():
-            if '/' not in key and '_' not in key:
-                validation_messages.append \
-                    (f"Entry {idx}: Extension key '{key}' in 'extensions' should be namespaced to prevent conflicts.")
-
-    is_valid = len(validation_messages) == 0
-    return is_valid, validation_messages
-
-def validate_v2_card(card_data):
-    """
-    Validate a character card according to the V2 specification.
-
-    Args:
-        card_data (dict): The parsed character card data.
-
-    Returns:
-        Tuple[bool, List[str]]: A tuple containing a boolean indicating validity and a list of validation messages.
-    """
-    validation_messages = []
-
-    # Check top-level fields
-    if 'spec' not in card_data:
-        validation_messages.append("Missing 'spec' field.")
-    elif card_data['spec'] != 'chara_card_v2':
-        validation_messages.append(f"Invalid 'spec' value: {card_data['spec']}. Expected 'chara_card_v2'.")
-
-    if 'spec_version' not in card_data:
-        validation_messages.append("Missing 'spec_version' field.")
-    else:
-        # Ensure 'spec_version' is '2.0' or higher
-        try:
-            spec_version = float(card_data['spec_version'])
-            if spec_version < 2.0:
-                validation_messages.append \
-                    (f"'spec_version' must be '2.0' or higher. Found '{card_data['spec_version']}'.")
-        except ValueError:
-            validation_messages.append \
-                (f"Invalid 'spec_version' format: {card_data['spec_version']}. Must be a number as a string.")
-
-    if 'data' not in card_data:
-        validation_messages.append("Missing 'data' field.")
-        return False, validation_messages  # Cannot proceed without 'data' field
-
-    data = card_data['data']
-
-    # Required fields in 'data'
-    required_fields = ['name', 'description', 'personality', 'scenario', 'first_mes', 'mes_example']
-    for field in required_fields:
-        if field not in data:
-            validation_messages.append(f"Missing required field in 'data': '{field}'.")
-        elif not isinstance(data[field], str):
-            validation_messages.append(f"Field '{field}' must be a string.")
-        elif not data[field].strip():
-            validation_messages.append(f"Field '{field}' cannot be empty.")
-
-    # Optional fields with expected types
-    optional_fields = {
-        'creator_notes': str,
-        'system_prompt': str,
-        'post_history_instructions': str,
-        'alternate_greetings': list,
-        'tags': list,
-        'creator': str,
-        'character_version': str,
-        'extensions': dict,
-        'character_book': dict  # If present, should be a dict
-    }
-
-    for field, expected_type in optional_fields.items():
-        if field in data:
-            if not isinstance(data[field], expected_type):
-                validation_messages.append(f"Field '{field}' must be of type '{expected_type.__name__}'.")
-            elif field == 'extensions':
-                # Validate that extensions keys are properly namespaced
-                for key in data[field].keys():
-                    if '/' not in key and '_' not in key:
-                        validation_messages.append \
-                            (f"Extension key '{key}' in 'extensions' should be namespaced to prevent conflicts.")
-
-    # If 'alternate_greetings' is present, check that it's a list of non-empty strings
-    if 'alternate_greetings' in data and isinstance(data['alternate_greetings'], list):
-        for idx, greeting in enumerate(data['alternate_greetings']):
-            if not isinstance(greeting, str) or not greeting.strip():
-                validation_messages.append(f"Element {idx} in 'alternate_greetings' must be a non-empty string.")
-
-    # If 'tags' is present, check that it's a list of non-empty strings
-    if 'tags' in data and isinstance(data['tags'], list):
-        for idx, tag in enumerate(data['tags']):
-            if not isinstance(tag, str) or not tag.strip():
-                validation_messages.append(f"Element {idx} in 'tags' must be a non-empty string.")
-
-    # Validate 'extensions' field
-    if 'extensions' in data and not isinstance(data['extensions'], dict):
-        validation_messages.append("Field 'extensions' must be a dictionary.")
-
-    # Validate 'character_book' if present
-    if 'character_book' in data:
-        is_valid_book, book_messages = validate_character_book(data['character_book'])
-        if not is_valid_book:
-            validation_messages.extend(book_messages)
-
-    is_valid = len(validation_messages) == 0
-    return is_valid, validation_messages
-
-#
-# End of File
-####################################################################################################

App_Function_Libraries/Chat.py

@@ -1,439 +0,0 @@
-# Chat.py
-# Chat functions for interacting with the LLMs as chatbots
-import base64
-# Imports
-import json
-import logging
-import os
-import re
-import tempfile
-import time
-from datetime import datetime
-from pathlib import Path
-#
-# External Imports
-#
-# Local Imports
-from App_Function_Libraries.DB.DB_Manager import get_conversation_name, save_chat_history_to_database
-from App_Function_Libraries.LLM_API_Calls import chat_with_openai, chat_with_anthropic, chat_with_cohere, \
-    chat_with_groq, chat_with_openrouter, chat_with_deepseek, chat_with_mistral, chat_with_huggingface
-from App_Function_Libraries.LLM_API_Calls_Local import chat_with_aphrodite, chat_with_local_llm, chat_with_ollama, \
-    chat_with_kobold, chat_with_llama, chat_with_oobabooga, chat_with_tabbyapi, chat_with_vllm, chat_with_custom_openai
-from App_Function_Libraries.DB.SQLite_DB import load_media_content
-from App_Function_Libraries.Utils.Utils import generate_unique_filename, load_and_log_configs
-from App_Function_Libraries.Metrics.metrics_logger import log_counter, log_histogram
-#
-####################################################################################################
-#
-# Functions:
-
-def chat_api_call(api_endpoint, api_key, input_data, prompt, temp, system_message=None):
-    log_counter("chat_api_call_attempt", labels={"api_endpoint": api_endpoint})
-    start_time = time.time()
-    if not api_key:
-        api_key = None
-    model = None
-    try:
-        logging.info(f"Debug - Chat API Call - API Endpoint: {api_endpoint}")
-        logging.info(f"Debug - Chat API Call - API Key: {api_key}")
-        logging.info(f"Debug - Chat chat_api_call - API Endpoint: {api_endpoint}")
-        if api_endpoint.lower() == 'openai':
-            response = chat_with_openai(api_key, input_data, prompt, temp, system_message)
-
-        elif api_endpoint.lower() == 'anthropic':
-            # Retrieve the model from config
-            loaded_config_data = load_and_log_configs()
-            model = loaded_config_data['models']['anthropic'] if loaded_config_data else None
-            response = chat_with_anthropic(
-                api_key=api_key,
-                input_data=input_data,
-                model=model,
-                custom_prompt_arg=prompt,
-                system_prompt=system_message
-            )
-
-        elif api_endpoint.lower() == "cohere":
-            response = chat_with_cohere(
-                api_key,
-                input_data,
-                model=model,
-                custom_prompt_arg=prompt,
-                system_prompt=system_message,
-                temp=temp
-            )
-
-        elif api_endpoint.lower() == "groq":
-            response = chat_with_groq(api_key, input_data, prompt, temp, system_message)
-
-        elif api_endpoint.lower() == "openrouter":
-            response = chat_with_openrouter(api_key, input_data, prompt, temp, system_message)
-
-        elif api_endpoint.lower() == "deepseek":
-            response = chat_with_deepseek(api_key, input_data, prompt, temp, system_message)
-
-        elif api_endpoint.lower() == "mistral":
-            response = chat_with_mistral(api_key, input_data, prompt, temp, system_message)
-
-        elif api_endpoint.lower() == "llama.cpp":
-            response = chat_with_llama(input_data, prompt, temp, None, api_key, system_message)
-        elif api_endpoint.lower() == "kobold":
-            response = chat_with_kobold(input_data, api_key, prompt, temp, system_message)
-
-        elif api_endpoint.lower() == "ooba":
-            response = chat_with_oobabooga(input_data, api_key, prompt, temp, system_message)
-
-        elif api_endpoint.lower() == "tabbyapi":
-            response = chat_with_tabbyapi(input_data, prompt, temp, system_message)
-
-        elif api_endpoint.lower() == "vllm":
-            response = chat_with_vllm(input_data, prompt, system_message)
-
-        elif api_endpoint.lower() == "local-llm":
-            response = chat_with_local_llm(input_data, prompt, temp, system_message)
-
-        elif api_endpoint.lower() == "huggingface":
-            response = chat_with_huggingface(api_key, input_data, prompt, temp)  # , system_message)
-
-        elif api_endpoint.lower() == "ollama":
-            response = chat_with_ollama(input_data, prompt, None, api_key, temp, system_message)
-
-        elif api_endpoint.lower() == "aphrodite":
-            response = chat_with_aphrodite(input_data, prompt, temp, system_message)
-
-        elif api_endpoint.lower() == "custom-openai-api":
-            response = chat_with_custom_openai(api_key, input_data, prompt, temp, system_message)
-
-        else:
-            raise ValueError(f"Unsupported API endpoint: {api_endpoint}")
-
-        call_duration = time.time() - start_time
-        log_histogram("chat_api_call_duration", call_duration, labels={"api_endpoint": api_endpoint})
-        log_counter("chat_api_call_success", labels={"api_endpoint": api_endpoint})
-        return response
-
-    except Exception as e:
-        log_counter("chat_api_call_error", labels={"api_endpoint": api_endpoint, "error": str(e)})
-        logging.error(f"Error in chat function: {str(e)}")
-        return f"An error occurred: {str(e)}"
-
-
-def chat(message, history, media_content, selected_parts, api_endpoint, api_key, prompt, temperature,
-         system_message=None):
-    log_counter("chat_attempt", labels={"api_endpoint": api_endpoint})
-    start_time = time.time()
-    try:
-        logging.info(f"Debug - Chat Function - Message: {message}")
-        logging.info(f"Debug - Chat Function - Media Content: {media_content}")
-        logging.info(f"Debug - Chat Function - Selected Parts: {selected_parts}")
-        logging.info(f"Debug - Chat Function - API Endpoint: {api_endpoint}")
-        # logging.info(f"Debug - Chat Function - Prompt: {prompt}")
-
-        # Ensure selected_parts is a list
-        if not isinstance(selected_parts, (list, tuple)):
-            selected_parts = [selected_parts] if selected_parts else []
-
-        # logging.debug(f"Debug - Chat Function - Selected Parts (after check): {selected_parts}")
-
-        # Combine the selected parts of the media content
-        combined_content = "\n\n".join(
-            [f"{part.capitalize()}: {media_content.get(part, '')}" for part in selected_parts if part in media_content])
-        # Print first 500 chars
-        # logging.debug(f"Debug - Chat Function - Combined Content: {combined_content[:500]}...")
-
-        # Prepare the input for the API
-        input_data = f"{combined_content}\n\n" if combined_content else ""
-        for old_message, old_response in history:
-            input_data += f"{old_message}\nAssistant: {old_response}\n\n"
-        input_data += f"{message}\n"
-
-        if system_message:
-            print(f"System message: {system_message}")
-            logging.debug(f"Debug - Chat Function - System Message: {system_message}")
-        temperature = float(temperature) if temperature else 0.7
-        temp = temperature
-
-        logging.debug(f"Debug - Chat Function - Temperature: {temperature}")
-        logging.debug(f"Debug - Chat Function - API Key: {api_key[:10]}")
-        logging.debug(f"Debug - Chat Function - Prompt: {prompt}")
-
-        # Use the existing API request code based on the selected endpoint
-        response = chat_api_call(api_endpoint, api_key, input_data, prompt, temp, system_message)
-
-        chat_duration = time.time() - start_time
-        log_histogram("chat_duration", chat_duration, labels={"api_endpoint": api_endpoint})
-        log_counter("chat_success", labels={"api_endpoint": api_endpoint})
-        return response
-    except Exception as e:
-        log_counter("chat_error", labels={"api_endpoint": api_endpoint, "error": str(e)})
-        logging.error(f"Error in chat function: {str(e)}")
-        return f"An error occurred: {str(e)}"
-
-
-def save_chat_history_to_db_wrapper(chatbot, conversation_id, media_content, media_name=None):
-    log_counter("save_chat_history_to_db_attempt")
-    start_time = time.time()
-    logging.info(f"Attempting to save chat history. Media content type: {type(media_content)}")
-    try:
-        # Extract the media_id and media_name from the media_content
-        media_id = None
-        if isinstance(media_content, dict):
-            media_id = None
-            logging.debug(f"Media content keys: {media_content.keys()}")
-            if 'content' in media_content:
-                try:
-                    content = media_content['content']
-                    if isinstance(content, str):
-                        content_json = json.loads(content)
-                    elif isinstance(content, dict):
-                        content_json = content
-                    else:
-                        raise ValueError(f"Unexpected content type: {type(content)}")
-
-                    # Use the webpage_url as the media_id
-                    media_id = content_json.get('webpage_url')
-                    # Use the title as the media_name
-                    media_name = content_json.get('title')
-
-                    logging.info(f"Extracted media_id: {media_id}, media_name: {media_name}")
-                except json.JSONDecodeError:
-                    logging.error("Failed to decode JSON from media_content['content']")
-                except Exception as e:
-                    logging.error(f"Error processing media_content: {str(e)}")
-            else:
-                logging.warning("'content' key not found in media_content")
-        else:
-            logging.warning(f"media_content is not a dictionary. Type: {type(media_content)}")
-
-        if media_id is None:
-            # If we couldn't find a media_id, we'll use a placeholder
-            media_id = "unknown_media"
-            logging.warning(f"Unable to extract media_id from media_content. Using placeholder: {media_id}")
-
-        if media_name is None:
-            media_name = "Unnamed Media"
-            logging.warning(f"Unable to extract media_name from media_content. Using placeholder: {media_name}")
-
-        # Generate a unique conversation name using media_id and current timestamp
-        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-        conversation_name = f"{media_name}_{timestamp}"
-
-        new_conversation_id = save_chat_history_to_database(chatbot, conversation_id, media_id, media_name,
-                                                            conversation_name)
-        save_duration = time.time() - start_time
-        log_histogram("save_chat_history_to_db_duration", save_duration)
-        log_counter("save_chat_history_to_db_success")
-        return new_conversation_id, f"Chat history saved successfully as {conversation_name}!"
-    except Exception as e:
-        log_counter("save_chat_history_to_db_error", labels={"error": str(e)})
-        error_message = f"Failed to save chat history: {str(e)}"
-        logging.error(error_message, exc_info=True)
-        return conversation_id, error_message
-
-
-def save_chat_history(history, conversation_id, media_content):
-    log_counter("save_chat_history_attempt")
-    start_time = time.time()
-    try:
-        content, conversation_name = generate_chat_history_content(history, conversation_id, media_content)
-
-        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-        safe_conversation_name = re.sub(r'[^a-zA-Z0-9_-]', '_', conversation_name)
-        base_filename = f"{safe_conversation_name}_{timestamp}.json"
-
-        # Create a temporary file
-        with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.json') as temp_file:
-            temp_file.write(content)
-            temp_file_path = temp_file.name
-
-        # Generate a unique filename
-        unique_filename = generate_unique_filename(os.path.dirname(temp_file_path), base_filename)
-        final_path = os.path.join(os.path.dirname(temp_file_path), unique_filename)
-
-        # Rename the temporary file to the unique filename
-        os.rename(temp_file_path, final_path)
-
-        save_duration = time.time() - start_time
-        log_histogram("save_chat_history_duration", save_duration)
-        log_counter("save_chat_history_success")
-        return final_path
-    except Exception as e:
-        log_counter("save_chat_history_error", labels={"error": str(e)})
-        logging.error(f"Error saving chat history: {str(e)}")
-        return None
-
-
-def generate_chat_history_content(history, conversation_id, media_content):
-    timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-
-    conversation_name = get_conversation_name(conversation_id)
-
-    if not conversation_name:
-        media_name = extract_media_name(media_content)
-        if media_name:
-            conversation_name = f"{media_name}-chat"
-        else:
-            conversation_name = f"chat-{timestamp}"  # Fallback name
-
-    chat_data = {
-        "conversation_id": conversation_id,
-        "conversation_name": conversation_name,
-        "timestamp": timestamp,
-        "history": [
-            {
-                "role": "user" if i % 2 == 0 else "bot",
-                "content": msg[0] if isinstance(msg, tuple) else msg
-            }
-            for i, msg in enumerate(history)
-        ]
-    }
-
-    return json.dumps(chat_data, indent=2), conversation_name
-
-
-def extract_media_name(media_content):
-    if isinstance(media_content, dict):
-        content = media_content.get('content', {})
-        if isinstance(content, str):
-            try:
-                content = json.loads(content)
-            except json.JSONDecodeError:
-                logging.warning("Failed to parse media_content JSON string")
-                return None
-
-        # Try to extract title from the content
-        if isinstance(content, dict):
-            return content.get('title') or content.get('name')
-
-    logging.warning(f"Unexpected media_content format: {type(media_content)}")
-    return None
-
-
-def update_chat_content(selected_item, use_content, use_summary, use_prompt, item_mapping):
-    log_counter("update_chat_content_attempt")
-    start_time = time.time()
-    logging.debug(f"Debug - Update Chat Content - Selected Item: {selected_item}\n")
-    logging.debug(f"Debug - Update Chat Content - Use Content: {use_content}\n\n\n\n")
-    logging.debug(f"Debug - Update Chat Content - Use Summary: {use_summary}\n\n")
-    logging.debug(f"Debug - Update Chat Content - Use Prompt: {use_prompt}\n\n")
-    logging.debug(f"Debug - Update Chat Content - Item Mapping: {item_mapping}\n\n")
-
-    if selected_item and selected_item in item_mapping:
-        media_id = item_mapping[selected_item]
-        content = load_media_content(media_id)
-        selected_parts = []
-        if use_content and "content" in content:
-            selected_parts.append("content")
-        if use_summary and "summary" in content:
-            selected_parts.append("summary")
-        if use_prompt and "prompt" in content:
-            selected_parts.append("prompt")
-
-        # Modified debug print
-        if isinstance(content, dict):
-            print(f"Debug - Update Chat Content - Content keys: {list(content.keys())}")
-            for key, value in content.items():
-                print(f"Debug - Update Chat Content - {key} (first 500 char): {str(value)[:500]}\n\n\n\n")
-        else:
-            print(f"Debug - Update Chat Content - Content(first 500 char): {str(content)[:500]}\n\n\n\n")
-
-        print(f"Debug - Update Chat Content - Selected Parts: {selected_parts}")
-        update_duration = time.time() - start_time
-        log_histogram("update_chat_content_duration", update_duration)
-        log_counter("update_chat_content_success")
-        return content, selected_parts
-    else:
-        log_counter("update_chat_content_error", labels={"error": str("No item selected or item not in mapping")})
-        print(f"Debug - Update Chat Content - No item selected or item not in mapping")
-        return {}, []
-
-#
-# End of Chat functions
-#######################################################################################################################
-
-
-#######################################################################################################################
-#
-# Character Card Functions
-
-CHARACTERS_FILE = Path('.', 'Helper_Scripts', 'Character_Cards', 'Characters.json')
-
-
-def save_character(character_data):
-    log_counter("save_character_attempt")
-    start_time = time.time()
-    characters_file = os.path.join(os.path.dirname(__file__), '..', 'Helper_Scripts', 'Character_Cards', 'Characters.json')
-    characters_dir = os.path.dirname(characters_file)
-
-    try:
-        if os.path.exists(characters_file):
-            with open(characters_file, 'r') as f:
-                characters = json.load(f)
-        else:
-            characters = {}
-
-        char_name = character_data['name']
-
-        # Save the image separately if it exists
-        if 'image' in character_data:
-            img_data = base64.b64decode(character_data['image'])
-            img_filename = f"{char_name.replace(' ', '_')}.png"
-            img_path = os.path.join(characters_dir, img_filename)
-            with open(img_path, 'wb') as f:
-                f.write(img_data)
-            character_data['image_path'] = os.path.abspath(img_path)
-            del character_data['image']  # Remove the base64 image data from the JSON
-
-        characters[char_name] = character_data
-
-        with open(characters_file, 'w') as f:
-            json.dump(characters, f, indent=2)
-
-        save_duration = time.time() - start_time
-        log_histogram("save_character_duration", save_duration)
-        log_counter("save_character_success")
-        logging.info(f"Character '{char_name}' saved successfully.")
-    except Exception as e:
-        log_counter("save_character_error", labels={"error": str(e)})
-        logging.error(f"Error saving character: {str(e)}")
-
-
-def load_characters():
-    log_counter("load_characters_attempt")
-    start_time = time.time()
-    try:
-        characters_file = os.path.join(os.path.dirname(__file__), '..', 'Helper_Scripts', 'Character_Cards', 'Characters.json')
-        if os.path.exists(characters_file):
-            with open(characters_file, 'r') as f:
-                characters = json.load(f)
-            logging.debug(f"Loaded {len(characters)} characters from {characters_file}")
-            load_duration = time.time() - start_time
-            log_histogram("load_characters_duration", load_duration)
-            log_counter("load_characters_success", labels={"character_count": len(characters)})
-            return characters
-        else:
-            logging.warning(f"Characters file not found: {characters_file}")
-            return {}
-    except Exception as e:
-        log_counter("load_characters_error", labels={"error": str(e)})
-        return {}
-
-
-
-def get_character_names():
-    log_counter("get_character_names_attempt")
-    start_time = time.time()
-    try:
-        characters = load_characters()
-        names = list(characters.keys())
-        get_names_duration = time.time() - start_time
-        log_histogram("get_character_names_duration", get_names_duration)
-        log_counter("get_character_names_success", labels={"name_count": len(names)})
-        return names
-    except Exception as e:
-        log_counter("get_character_names_error", labels={"error": str(e)})
-        logging.error(f"Error getting character names: {str(e)}")
-        return []
-
-#
-# End of Chat.py
-##########################################################################################################################

App_Function_Libraries/Chat_related_functions.py

@@ -1,41 +0,0 @@
-# Chat_related_functions.py
-# Contains functions related to chat
-# WIP.
-#
-# Importing required libraries
-import json
-import os
-from pathlib import Path
-import json
-#
-########################################################################################################################
-# Set globals
-CHARACTERS_FILE = Path('.', 'Helper_Scripts', 'Character_Cards', 'Characters.json')
-
-def save_character(character_data):
-    if CHARACTERS_FILE.exists():
-        with CHARACTERS_FILE.open('r') as f:
-            characters = json.load(f)
-    else:
-        characters = {}
-
-    characters[character_data['name']] = character_data
-
-    with CHARACTERS_FILE.open('w') as f:
-        json.dump(characters, f, indent=2)
-
-
-def load_characters():
-    if os.path.exists(CHARACTERS_FILE):
-        with open(CHARACTERS_FILE, 'r') as f:
-            return json.load(f)
-    return {}
-
-
-def get_character_names():
-    characters = load_characters()
-    return list(characters.keys())
-
-
-
-

App_Function_Libraries/Chunk_Lib.py

@@ -1,1051 +0,0 @@
-# Chunk_Lib.py
-#########################################
-# Chunking Library
-# This library is used to perform chunking of input files.
-# Currently, uses naive approaches. Nothing fancy.
-#
-####
-# Import necessary libraries
-import hashlib
-import json
-import logging
-import re
-from typing import Any, Dict, List, Optional, Tuple
-#
-# Import 3rd party
-from openai import OpenAI
-from tqdm import tqdm
-from langdetect import detect
-from transformers import GPT2Tokenizer
-import nltk
-from nltk.tokenize import sent_tokenize, word_tokenize
-from sklearn.feature_extraction.text import TfidfVectorizer
-from sklearn.metrics.pairwise import cosine_similarity
-#
-# Import Local
-from App_Function_Libraries.Tokenization_Methods_Lib import openai_tokenize
-from App_Function_Libraries.Utils.Utils import load_comprehensive_config
-#
-#######################################################################################################################
-# Config Settings
-#
-#
-# FIXME - Make sure it only downloads if it already exists, and does a check first.
-# Ensure NLTK data is downloaded
-def ensure_nltk_data():
-    try:
-        nltk.data.find('tokenizers/punkt')
-    except LookupError:
-        nltk.download('punkt')
-#ensure_nltk_data()
-
-#
-# Load GPT2 tokenizer
-tokenizer = GPT2Tokenizer.from_pretrained("gpt2")
-#
-# Load configuration
-config = load_comprehensive_config()
-# Embedding Chunking options
-chunk_options = {
-    'method': config.get('Chunking', 'method', fallback='words'),
-    'max_size': config.getint('Chunking', 'max_size', fallback=400),
-    'overlap': config.getint('Chunking', 'overlap', fallback=200),
-    'adaptive': config.getboolean('Chunking', 'adaptive', fallback=False),
-    'multi_level': config.getboolean('Chunking', 'multi_level', fallback=False),
-    'language': config.get('Chunking', 'language', fallback='english')
-}
-
-openai_api_key = config.get('API', 'openai_api_key')
-#
-# End of settings
-#######################################################################################################################
-#
-# Functions:
-
-# Create a chunking class for refactoring FIXME
-# class Chunker:
-#     def __init__(self, tokenizer: GPT2Tokenizer):
-#         self.tokenizer = tokenizer
-#
-#     def detect_language(self, text: str) -> str:
-#         try:
-#             return detect(text)
-#         except:
-#             return 'en'
-#
-#     def chunk_text(self, text: str, method: str, max_size: int, overlap: int, language: str = None) -> List[str]:
-#         if language is None:
-#             language = self.detect_language(text)
-#
-#         if method == 'words':
-#             return self.chunk_text_by_words(text, max_size, overlap, language)
-#         elif method == 'sentences':
-#             return self.chunk_text_by_sentences(text, max_size, overlap, language)
-#         elif method == 'paragraphs':
-#             return self.chunk_text_by_paragraphs(text, max_size, overlap)
-#         elif method == 'tokens':
-#             return self.chunk_text_by_tokens(text, max_size, overlap, language)
-#         elif method == 'semantic':
-#             return self.semantic_chunking(text, max_size)
-#         else:
-#             return [text]
-
-def detect_language(text: str) -> str:
-    try:
-        return detect(text)
-    except:
-        # Default to English if detection fails
-        return 'en'
-
-
-def load_document(file_path: str) -> str:
-    with open(file_path, 'r', encoding='utf-8') as file:
-        text = file.read()
-    return re.sub(r'\s+', ' ', text).strip()
-
-
-def improved_chunking_process(text: str, chunk_options: Dict[str, Any] = None) -> List[Dict[str, Any]]:
-    logging.debug("Improved chunking process started...")
-
-    # Extract JSON metadata if present
-    json_content = {}
-    try:
-        json_end = text.index("}\n") + 1
-        json_content = json.loads(text[:json_end])
-        text = text[json_end:].strip()
-        logging.debug(f"Extracted JSON metadata: {json_content}")
-    except (ValueError, json.JSONDecodeError):
-        logging.debug("No JSON metadata found at the beginning of the text")
-
-    # Extract any additional header text
-    header_match = re.match(r"(This text was transcribed using.*?)\n\n", text, re.DOTALL)
-    header_text = ""
-    if header_match:
-        header_text = header_match.group(1)
-        text = text[len(header_text):].strip()
-        logging.debug(f"Extracted header text: {header_text}")
-
-    options = chunk_options.copy() if chunk_options else {}
-    if chunk_options:
-        options.update(chunk_options)
-
-    chunk_method = options.get('method', 'words')
-    max_size = options.get('max_size', 2000)
-    overlap = options.get('overlap', 0)
-    language = options.get('language', None)
-
-    if language is None:
-        language = detect_language(text)
-
-    if chunk_method == 'json':
-        chunks = chunk_text_by_json(text, max_size=max_size, overlap=overlap)
-    else:
-        chunks = chunk_text(text, chunk_method, max_size, overlap, language)
-
-    chunks_with_metadata = []
-    total_chunks = len(chunks)
-    for i, chunk in enumerate(chunks):
-        metadata = {
-            'chunk_index': i + 1,
-            'total_chunks': total_chunks,
-            'chunk_method': chunk_method,
-            'max_size': max_size,
-            'overlap': overlap,
-            'language': language,
-            'relative_position': (i + 1) / total_chunks
-        }
-        metadata.update(json_content)  # Add the extracted JSON content to metadata
-        metadata['header_text'] = header_text  # Add the header text to metadata
-
-        if chunk_method == 'json':
-            chunk_text_content = json.dumps(chunk['json'], ensure_ascii=False)
-        else:
-            chunk_text_content = chunk
-
-        chunks_with_metadata.append({
-            'text': chunk_text_content,
-            'metadata': metadata
-        })
-
-    return chunks_with_metadata
-
-
-def multi_level_chunking(text: str, method: str, max_size: int, overlap: int, language: str) -> List[str]:
-    logging.debug("Multi-level chunking process started...")
-    # First level: chunk by paragraphs
-    paragraphs = chunk_text_by_paragraphs(text, max_size * 2, overlap)
-
-    # Second level: chunk each paragraph further
-    chunks = []
-    for para in paragraphs:
-        if method == 'words':
-            chunks.extend(chunk_text_by_words(para, max_words=max_size, overlap=overlap, language=language))
-        elif method == 'sentences':
-            chunks.extend(chunk_text_by_sentences(para, max_sentences=max_size, overlap=overlap, language=language))
-        else:
-            chunks.append(para)
-
-    return chunks
-
-
-# FIXME - ensure language detection occurs in each chunk function
-def chunk_text(text: str, method: str, max_size: int, overlap: int, language: str = None) -> List[str]:
-    if method == 'words':
-        logging.debug("Chunking by words...")
-        return chunk_text_by_words(text, max_words=max_size, overlap=overlap, language=language)
-    elif method == 'sentences':
-        logging.debug("Chunking by sentences...")
-        return chunk_text_by_sentences(text, max_sentences=max_size, overlap=overlap, language=language)
-    elif method == 'paragraphs':
-        logging.debug("Chunking by paragraphs...")
-        return chunk_text_by_paragraphs(text, max_paragraphs=max_size, overlap=overlap)
-    elif method == 'tokens':
-        logging.debug("Chunking by tokens...")
-        return chunk_text_by_tokens(text, max_tokens=max_size, overlap=overlap)
-    elif method == 'semantic':
-        logging.debug("Chunking by semantic similarity...")
-        return semantic_chunking(text, max_chunk_size=max_size)
-    else:
-        logging.warning(f"Unknown chunking method '{method}'. Returning full text as a single chunk.")
-        return [text]
-
-def determine_chunk_position(relative_position: float) -> str:
-    if relative_position < 0.33:
-        return "This chunk is from the beginning of the document"
-    elif relative_position < 0.66:
-        return "This chunk is from the middle of the document"
-    else:
-        return "This chunk is from the end of the document"
-
-
-def chunk_text_by_words(text: str, max_words: int = 300, overlap: int = 0, language: str = None) -> List[str]:
-    logging.debug("chunk_text_by_words...")
-    if language is None:
-        language = detect_language(text)
-
-    if language.startswith('zh'):  # Chinese
-        import jieba
-        words = list(jieba.cut(text))
-    elif language == 'ja':  # Japanese
-        import fugashi
-        tagger = fugashi.Tagger()
-        words = [word.surface for word in tagger(text)]
-    else:  # Default to simple splitting for other languages
-        words = text.split()
-
-    chunks = []
-    for i in range(0, len(words), max_words - overlap):
-        chunk = ' '.join(words[i:i + max_words])
-        chunks.append(chunk)
-    return post_process_chunks(chunks)
-
-
-def chunk_text_by_sentences(text: str, max_sentences: int = 10, overlap: int = 0, language: str = None) -> List[str]:
-    logging.debug("chunk_text_by_sentences...")
-    if language is None:
-        language = detect_language(text)
-
-    if language.startswith('zh'):  # Chinese
-        import jieba
-        # Use jieba to perform sentence segmentation
-        # jieba does not support sentence segmentation out of the box
-        # Use punctuation as delimiters
-        sentences = re.split(r'[。！？；]', text)
-        sentences = [s.strip() for s in sentences if s.strip()]
-    elif language == 'ja':  # Japanese
-        import fugashi
-        tagger = fugashi.Tagger()
-        # Simple sentence segmentation based on punctuation
-        sentences = re.split(r'[。！？]', text)
-        sentences = [s.strip() for s in sentences if s.strip()]
-    else:  # Default to NLTK for other languages
-        try:
-            sentences = sent_tokenize(text, language=language)
-        except LookupError:
-            logging.warning(f"Punkt tokenizer not found for language '{language}'. Using default 'english'.")
-            sentences = sent_tokenize(text, language='english')
-
-    chunks = []
-    previous_overlap = []
-
-    for i in range(0, len(sentences), max_sentences - overlap):
-        current_sentences = sentences[i:i + max_sentences]
-        if overlap > 0 and previous_overlap:
-            current_sentences = previous_overlap + current_sentences
-        chunk = ' '.join(current_sentences)
-        chunks.append(chunk)
-        previous_overlap = sentences[i + max_sentences - overlap:i + max_sentences] if overlap > 0 else []
-
-    return post_process_chunks(chunks)
-
-
-def chunk_text_by_paragraphs(text: str, max_paragraphs: int = 5, overlap: int = 0) -> List[str]:
-    logging.debug("chunk_text_by_paragraphs...")
-    paragraphs = re.split(r'\n\s*\n', text)
-    chunks = []
-    for i in range(0, len(paragraphs), max_paragraphs - overlap):
-        chunk = '\n\n'.join(paragraphs[i:i + max_paragraphs])
-        chunks.append(chunk)
-    return post_process_chunks(chunks)
-
-
-def chunk_text_by_tokens(text: str, max_tokens: int = 1000, overlap: int = 0) -> List[str]:
-    logging.debug("chunk_text_by_tokens...")
-    # This is a simplified token-based chunking. For more accurate tokenization,
-    # consider using a proper tokenizer like GPT-2 TokenizerFast
-    words = text.split()
-    chunks = []
-    current_chunk = []
-    current_token_count = 0
-
-    for word in words:
-        word_token_count = len(word) // 4 + 1  # Rough estimate of token count
-        if current_token_count + word_token_count > max_tokens and current_chunk:
-            chunks.append(' '.join(current_chunk))
-            current_chunk = current_chunk[-overlap:] if overlap > 0 else []
-            current_token_count = sum(len(w) // 4 + 1 for w in current_chunk)
-
-        current_chunk.append(word)
-        current_token_count += word_token_count
-
-    if current_chunk:
-        chunks.append(' '.join(current_chunk))
-
-    return post_process_chunks(chunks)
-# def chunk_text_by_tokens(text: str, max_tokens: int = 1000, overlap: int = 0) -> List[str]:
-#     logging.debug("chunk_text_by_tokens...")
-#     # Use GPT2 tokenizer for tokenization
-#     tokens = tokenizer.encode(text)
-#     chunks = []
-#     for i in range(0, len(tokens), max_tokens - overlap):
-#         chunk_tokens = tokens[i:i + max_tokens]
-#         chunk = tokenizer.decode(chunk_tokens)
-#         chunks.append(chunk)
-#     return post_process_chunks(chunks)
-
-
-def post_process_chunks(chunks: List[str]) -> List[str]:
-    return [chunk.strip() for chunk in chunks if chunk.strip()]
-
-
-# FIXME - F
-def get_chunk_metadata(chunk: str, full_text: str, chunk_type: str = "generic",
-                      chapter_number: Optional[int] = None,
-                      chapter_pattern: Optional[str] = None,
-                      language: str = None) -> Dict[str, Any]:
-    """
-    Generate metadata for a chunk based on its position in the full text.
-    """
-    chunk_length = len(chunk)
-    start_index = full_text.find(chunk)
-    end_index = start_index + chunk_length if start_index != -1 else None
-
-    # Calculate a hash for the chunk
-    chunk_hash = hashlib.md5(chunk.encode()).hexdigest()
-
-    metadata = {
-        'start_index': start_index,
-        'end_index': end_index,
-        'word_count': len(chunk.split()),
-        'char_count': chunk_length,
-        'chunk_type': chunk_type,
-        'language': language,
-        'chunk_hash': chunk_hash,
-        'relative_position': start_index / len(full_text) if len(full_text) > 0 and start_index != -1 else 0
-    }
-
-    if chunk_type == "chapter":
-        metadata['chapter_number'] = chapter_number
-        metadata['chapter_pattern'] = chapter_pattern
-
-    return metadata
-
-
-def process_document_with_metadata(text: str, chunk_options: Dict[str, Any],
-                                   document_metadata: Dict[str, Any]) -> Dict[str, Any]:
-    chunks = improved_chunking_process(text, chunk_options)
-
-    return {
-        'document_metadata': document_metadata,
-        'chunks': chunks
-    }
-
-
-# Hybrid approach, chunk each sentence while ensuring total token size does not exceed a maximum number
-def chunk_text_hybrid(text: str, max_tokens: int = 1000, overlap: int = 0) -> List[str]:
-    logging.debug("chunk_text_hybrid...")
-    sentences = sent_tokenize(text)
-    chunks = []
-    current_chunk = []
-    current_length = 0
-
-    for sentence in sentences:
-        tokens = tokenizer.encode(sentence)
-        if current_length + len(tokens) > max_tokens and current_chunk:
-            chunks.append(' '.join(current_chunk))
-            # Handle overlap
-            if overlap > 0:
-                overlap_tokens = tokenizer.encode(' '.join(current_chunk[-overlap:]))
-                current_chunk = current_chunk[-overlap:]
-                current_length = len(overlap_tokens)
-            else:
-                current_chunk = []
-                current_length = 0
-
-        current_chunk.append(sentence)
-        current_length += len(tokens)
-
-    if current_chunk:
-        chunks.append(' '.join(current_chunk))
-
-    return post_process_chunks(chunks)
-
-
-# Thanks openai
-def chunk_on_delimiter(input_string: str,
-                       max_tokens: int,
-                       delimiter: str) -> List[str]:
-    logging.debug("chunk_on_delimiter...")
-    chunks = input_string.split(delimiter)
-    combined_chunks, _, dropped_chunk_count = combine_chunks_with_no_minimum(
-        chunks, max_tokens, chunk_delimiter=delimiter, add_ellipsis_for_overflow=True)
-    if dropped_chunk_count > 0:
-        logging.warning(f"Warning: {dropped_chunk_count} chunks were dropped due to exceeding the token limit.")
-    combined_chunks = [f"{chunk}{delimiter}" for chunk in combined_chunks]
-    return combined_chunks
-
-
-
-
-# FIXME
-def recursive_summarize_chunks(chunks: List[str], summarize_func, custom_prompt: Optional[str] = None,
-                               temp: Optional[float] = None, system_prompt: Optional[str] = None) -> List[str]:
-    logging.debug("recursive_summarize_chunks...")
-    summarized_chunks = []
-    current_summary = ""
-
-    logging.debug(f"Summarizing {len(chunks)} chunks recursively...")
-    logging.debug(f"Temperature is set to {temp}")
-    for i, chunk in enumerate(chunks):
-        if i == 0:
-            current_summary = summarize_func(chunk, custom_prompt, temp, system_prompt)
-        else:
-            combined_text = current_summary + "\n\n" + chunk
-            current_summary = summarize_func(combined_text, custom_prompt, temp, system_prompt)
-
-        summarized_chunks.append(current_summary)
-
-    return summarized_chunks
-
-
-# Sample text for testing
-sample_text = """
-Natural language processing (NLP) is a subfield of linguistics, computer science, and artificial intelligence 
-concerned with the interactions between computers and human language, in particular how to program computers 
-to process and analyze large amounts of natural language data. The result is a computer capable of "understanding" 
-the contents of documents, including the contextual nuances of the language within them. The technology can then 
-accurately extract information and insights contained in the documents as well as categorize and organize the documents themselves.
-
-Challenges in natural language processing frequently involve speech recognition, natural language understanding, 
-and natural language generation.
-
-Natural language processing has its roots in the 1950s. Already in 1950, Alan Turing published an article titled 
-"Computing Machinery and Intelligence" which proposed what is now called the Turing test as a criterion of intelligence.
-"""
-
-# Example usage of different chunking methods
-# print("Chunking by words:")
-# print(chunk_text_by_words(sample_text, max_words=50))
-#
-# print("\nChunking by sentences:")
-# print(chunk_text_by_sentences(sample_text, max_sentences=2))
-#
-# print("\nChunking by paragraphs:")
-# print(chunk_text_by_paragraphs(sample_text, max_paragraphs=1))
-#
-# print("\nChunking by tokens:")
-# print(chunk_text_by_tokens(sample_text, max_tokens=50))
-#
-# print("\nHybrid chunking:")
-# print(chunk_text_hybrid(sample_text, max_tokens=50))
-
-
-
-#######################################################################################################################
-#
-# Experimental Semantic Chunking
-#
-
-# Chunk text into segments based on semantic similarity
-def count_units(text: str, unit: str = 'words') -> int:
-    if unit == 'words':
-        return len(text.split())
-    elif unit == 'tokens':
-        return len(tokenizer.encode(text))
-    elif unit == 'characters':
-        return len(text)
-    else:
-        raise ValueError("Invalid unit. Choose 'words', 'tokens', or 'characters'.")
-
-
-
-def semantic_chunking(text: str, max_chunk_size: int = 2000, unit: str = 'words') -> List[str]:
-    logging.debug("semantic_chunking...")
-    sentences = sent_tokenize(text)
-    vectorizer = TfidfVectorizer()
-    sentence_vectors = vectorizer.fit_transform(sentences)
-
-    chunks = []
-    current_chunk = []
-    current_size = 0
-
-    for i, sentence in enumerate(sentences):
-        sentence_size = count_units(sentence, unit)
-        if current_size + sentence_size > max_chunk_size and current_chunk:
-            chunks.append(' '.join(current_chunk))
-            # Use last 3 sentences for overlap
-            current_chunk = current_chunk[-3:]
-            current_size = count_units(' '.join(current_chunk), unit)
-
-        current_chunk.append(sentence)
-        current_size += sentence_size
-
-        if i + 1 < len(sentences):
-            current_vector = sentence_vectors[i]
-            next_vector = sentence_vectors[i + 1]
-            similarity = cosine_similarity(current_vector, next_vector)[0][0]
-            if similarity < 0.5 and current_size >= max_chunk_size // 2:
-                chunks.append(' '.join(current_chunk))
-                current_chunk = current_chunk[-3:]
-                current_size = count_units(' '.join(current_chunk), unit)
-
-    if current_chunk:
-        chunks.append(' '.join(current_chunk))
-
-    return chunks
-
-
-def semantic_chunk_long_file(file_path: str, max_chunk_size: int = 1000, overlap: int = 100, unit: str = 'words') -> Optional[List[str]]:
-    logging.debug("semantic_chunk_long_file...")
-    try:
-        with open(file_path, 'r', encoding='utf-8') as file:
-            content = file.read()
-
-        chunks = semantic_chunking(content, max_chunk_size, unit)
-        return chunks
-    except Exception as e:
-        logging.error(f"Error chunking text file: {str(e)}")
-        return None
-
-#
-#
-#######################################################################################################################
-
-
-#######################################################################################################################
-#
-#  Embedding Chunking
-
-def chunk_for_embedding(text: str, file_name: str, custom_chunk_options: Dict[str, Any] = None) -> List[Dict[str, Any]]:
-    options = chunk_options.copy()
-    if custom_chunk_options:
-        options.update(custom_chunk_options)
-
-    logging.info(f"Chunking options: {options}")
-    chunks = improved_chunking_process(text, options)
-    total_chunks = len(chunks)
-    logging.info(f"Total chunks created: {total_chunks}")
-
-    chunked_text_with_headers = []
-    for i, chunk in enumerate(chunks, 1):
-        chunk_text = chunk['text']
-        chunk_position = determine_chunk_position(chunk['metadata']['relative_position'])
-        chunk_header = f"""
-        Original Document: {file_name}
-        Chunk: {i} of {total_chunks}
-        Position: {chunk_position}
-
-        --- Chunk Content ---
-        """
-
-        full_chunk_text = chunk_header + chunk_text
-        chunk['text'] = full_chunk_text
-        chunk['metadata']['file_name'] = file_name
-        chunked_text_with_headers.append(chunk)
-
-    return chunked_text_with_headers
-
-#
-# End of Embedding Chunking
-#######################################################################################################################
-
-
-#######################################################################################################################
-#
-# JSON Chunking
-
-# FIXME
-def chunk_text_by_json(text: str, max_size: int = 1000, overlap: int = 0) -> List[Dict[str, Any]]:
-    """
-    Chunk JSON-formatted text into smaller JSON chunks while preserving structure.
-
-    Parameters:
-        - text (str): The JSON-formatted text to be chunked.
-        - max_size (int): Maximum number of items or characters per chunk.
-        - overlap (int): Number of items or characters to overlap between chunks.
-
-    Returns:
-        - List[Dict[str, Any]]: A list of chunks with their metadata.
-    """
-    logging.debug("chunk_text_by_json started...")
-    try:
-        json_data = json.loads(text)
-    except json.JSONDecodeError as e:
-        logging.error(f"Invalid JSON data: {e}")
-        raise ValueError(f"Invalid JSON data: {e}")
-
-    # Determine if JSON data is a list or a dict
-    if isinstance(json_data, list):
-        return chunk_json_list(json_data, max_size, overlap)
-    elif isinstance(json_data, dict):
-        return chunk_json_dict(json_data, max_size, overlap)
-    else:
-        logging.error("Unsupported JSON structure. Only JSON objects and arrays are supported.")
-        raise ValueError("Unsupported JSON structure. Only JSON objects and arrays are supported.")
-
-
-def chunk_json_list(json_list: List[Any], max_size: int, overlap: int) -> List[Dict[str, Any]]:
-    """
-    Chunk a JSON array into smaller chunks.
-
-    Parameters:
-        - json_list (List[Any]): The JSON array to be chunked.
-        - max_size (int): Maximum number of items per chunk.
-        - overlap (int): Number of items to overlap between chunks.
-
-    Returns:
-        - List[Dict[str, Any]]: A list of JSON chunks with metadata.
-    """
-    logging.debug("chunk_json_list started...")
-    chunks = []
-    total_items = len(json_list)
-    step = max_size - overlap
-    if step <= 0:
-        raise ValueError("max_size must be greater than overlap.")
-
-    for i in range(0, total_items, step):
-        chunk = json_list[i:i + max_size]
-        metadata = {
-            'chunk_index': i // step + 1,
-            'total_chunks': (total_items + step - 1) // step,
-            'chunk_method': 'json_list',
-            'max_size': max_size,
-            'overlap': overlap,
-            'relative_position': i / total_items
-        }
-        chunks.append({
-            'json': chunk,
-            'metadata': metadata
-        })
-
-    logging.debug(f"chunk_json_list created {len(chunks)} chunks.")
-    return chunks
-
-
-
-def chunk_json_dict(json_dict: Dict[str, Any], max_size: int, overlap: int) -> List[Dict[str, Any]]:
-    """
-    Chunk a JSON object into smaller chunks based on its 'data' key while preserving other keys like 'metadata'.
-
-    Parameters:
-        - json_dict (Dict[str, Any]): The JSON object to be chunked.
-        - max_size (int): Maximum number of key-value pairs per chunk in the 'data' section.
-        - overlap (int): Number of key-value pairs to overlap between chunks.
-
-    Returns:
-        - List[Dict[str, Any]]: A list of JSON chunks with metadata.
-    """
-    logging.debug("chunk_json_dict started...")
-
-    # Preserve non-chunked sections
-    preserved_keys = ['metadata']
-    preserved_data = {key: value for key, value in json_dict.items() if key in preserved_keys}
-
-    # Identify the chunkable section
-    chunkable_key = 'data'
-    if chunkable_key not in json_dict or not isinstance(json_dict[chunkable_key], dict):
-        logging.error("No chunkable 'data' section found in JSON dictionary.")
-        raise ValueError("No chunkable 'data' section found in JSON dictionary.")
-
-    chunkable_data = json_dict[chunkable_key]
-    data_keys = list(chunkable_data.keys())
-    total_keys = len(data_keys)
-    chunks = []
-    step = max_size - overlap
-    if step <= 0:
-        raise ValueError("max_size must be greater than overlap.")
-
-    # Adjust the loop to prevent creating an extra chunk
-    for i in range(0, total_keys, step):
-        chunk_keys = data_keys[i:i + max_size]
-
-        # Handle overlap
-        if i != 0 and overlap > 0:
-            overlap_keys = data_keys[i - overlap:i]
-            chunk_keys = overlap_keys + chunk_keys
-
-        # Remove duplicate keys caused by overlap
-        unique_chunk_keys = []
-        seen_keys = set()
-        for key in chunk_keys:
-            if key not in seen_keys:
-                unique_chunk_keys.append(key)
-                seen_keys.add(key)
-
-        chunk_data = {key: chunkable_data[key] for key in unique_chunk_keys}
-
-        metadata = {
-            'chunk_index': (i // step) + 1,
-            'total_chunks': (total_keys + step - 1) // step,
-            'chunk_method': 'json_dict',
-            'max_size': max_size,
-            'overlap': overlap,
-            'language': 'english',  # Assuming English; modify as needed
-            'relative_position': (i // step + 1) / ((total_keys + step - 1) // step)
-        }
-
-        # Merge preserved data into metadata
-        metadata.update(preserved_data.get('metadata', {}))
-
-        # Create the chunk with preserved data
-        chunk = {
-            'metadata': preserved_data,
-            'data': chunk_data
-        }
-
-        chunks.append({
-            'json': chunk,
-            'metadata': metadata
-        })
-
-    logging.debug(f"chunk_json_dict created {len(chunks)} chunks.")
-    return chunks
-
-
-#
-# End of JSON Chunking
-#######################################################################################################################
-
-#######################################################################################################################
-#
-# OpenAI Rolling Summarization
-#
-
-client = OpenAI(api_key=openai_api_key)
-def get_chat_completion(messages, model='gpt-4-turbo'):
-    response = client.chat.completions.create(
-        model=model,
-        messages=messages,
-        temperature=0,
-    )
-    return response.choices[0].message.content
-
-
-# This function combines text chunks into larger blocks without exceeding a specified token count.
-#   It returns the combined chunks, their original indices, and the number of dropped chunks due to overflow.
-def combine_chunks_with_no_minimum(
-        chunks: List[str],
-        max_tokens: int,
-        chunk_delimiter: str = "\n\n",
-        header: Optional[str] = None,
-        add_ellipsis_for_overflow: bool = False,
-) -> Tuple[List[str], List[List[int]], int]:
-    dropped_chunk_count = 0
-    output = []  # list to hold the final combined chunks
-    output_indices = []  # list to hold the indices of the final combined chunks
-    candidate = [header] if header else []  # list to hold the current combined chunk candidate
-    candidate_indices = []
-    for chunk_i, chunk in enumerate(chunks):
-        chunk_with_header = [chunk] if not header else [header, chunk]
-        combined_text = chunk_delimiter.join(candidate + chunk_with_header)
-        token_count = len(tokenizer.encode(combined_text))
-        if token_count > max_tokens:
-            if add_ellipsis_for_overflow and len(candidate) > 0:
-                ellipsis_text = chunk_delimiter.join(candidate + ["..."])
-                if len(tokenizer.encode(ellipsis_text)) <= max_tokens:
-                    candidate = candidate + ["..."]
-                    dropped_chunk_count += 1
-            if len(candidate) > 0:
-                output.append(chunk_delimiter.join(candidate))
-                output_indices.append(candidate_indices)
-                candidate = chunk_with_header
-                candidate_indices = [chunk_i]
-            else:
-                logging.warning(f"Single chunk at index {chunk_i} exceeds max_tokens and will be dropped.")
-                dropped_chunk_count += 1
-        else:
-            candidate.extend(chunk_with_header)
-            candidate_indices.append(chunk_i)
-
-    if candidate:
-        output.append(chunk_delimiter.join(candidate))
-        output_indices.append(candidate_indices)
-    return output, output_indices, dropped_chunk_count
-
-
-def rolling_summarize(text: str,
-                      detail: float = 0,
-                      model: str = 'gpt-4o',
-                      additional_instructions: Optional[str] = None,
-                      minimum_chunk_size: Optional[int] = 500,
-                      chunk_delimiter: str = ".",
-                      summarize_recursively: bool = False,
-                      verbose: bool = False) -> str:
-    """
-    Summarizes a given text by splitting it into chunks, each of which is summarized individually.
-    The level of detail in the summary can be adjusted, and the process can optionally be made recursive.
-
-    Parameters:
-        - text (str): The text to be summarized.
-        - detail (float, optional): A value between 0 and 1 indicating the desired level of detail in the summary.
-        - additional_instructions (Optional[str], optional): Additional instructions for the model.
-        - minimum_chunk_size (Optional[int], optional): The minimum size for text chunks.
-        - chunk_delimiter (str, optional): The delimiter used to split the text into chunks.
-        - summarize_recursively (bool, optional): If True, summaries are generated recursively.
-        - verbose (bool, optional): If True, prints detailed information about the chunking process.
-
-    Returns:
-    - str: The final compiled summary of the text.
-
-    The function first determines the number of chunks by interpolating between a minimum and a maximum chunk count
-    based on the `detail` parameter. It then splits the text into chunks and summarizes each chunk. If
-    `summarize_recursively` is True, each summary is based on the previous summaries, adding more context to the
-    summarization process. The function returns a compiled summary of all chunks.
-    """
-
-    # Check detail is set correctly
-    assert 0 <= detail <= 1, "Detail must be between 0 and 1."
-
-    # Interpolate the number of chunks based on the detail parameter
-    text_length = len(tokenizer.encode(text))
-    max_chunks = text_length // minimum_chunk_size if minimum_chunk_size else 10
-    min_chunks = 1
-    num_chunks = int(min_chunks + detail * (max_chunks - min_chunks))
-
-    # Adjust chunk_size based on interpolated number of chunks
-    chunk_size = max(minimum_chunk_size, text_length // num_chunks) if num_chunks else text_length
-    text_chunks = chunk_on_delimiter(text, chunk_size, chunk_delimiter)
-    if verbose:
-        print(f"Splitting the text into {len(text_chunks)} chunks to be summarized.")
-        print(f"Chunk lengths are {[len(tokenizer.encode(x)) for x in text_chunks]} tokens.")
-
-    # Set system message
-    system_message_content = "Rewrite this text in summarized form."
-    if additional_instructions:
-        system_message_content += f"\n\n{additional_instructions}"
-
-    accumulated_summaries = []
-    for i, chunk in enumerate(tqdm(text_chunks, desc="Summarizing chunks")):
-        if summarize_recursively and accumulated_summaries:
-            # Combine previous summary with current chunk for recursive summarization
-            combined_text = accumulated_summaries[-1] + "\n\n" + chunk
-            user_message_content = f"Previous summary and new content to summarize:\n\n{combined_text}"
-        else:
-            user_message_content = chunk
-
-        messages = [
-            {"role": "system", "content": system_message_content},
-            {"role": "user", "content": user_message_content}
-        ]
-
-        response = get_chat_completion(messages, model=model)
-        accumulated_summaries.append(response)
-
-    final_summary = '\n\n'.join(accumulated_summaries)
-    return final_summary
-
-#
-#
-#######################################################################################################################
-#
-# Ebook Chapter Chunking
-
-
-def chunk_ebook_by_chapters(text: str, chunk_options: Dict[str, Any]) -> List[Dict[str, Any]]:
-    logging.debug("chunk_ebook_by_chapters")
-    max_chunk_size = int(chunk_options.get('max_size', 300))
-    overlap = int(chunk_options.get('overlap', 0))
-    custom_pattern = chunk_options.get('custom_chapter_pattern', None)
-
-    # List of chapter heading patterns to try, in order
-    chapter_patterns = [
-        custom_pattern,
-        r'^#{1,2}\s+',  # Markdown style: '# ' or '## '
-        r'^Chapter\s+\d+',  # 'Chapter ' followed by numbers
-        r'^\d+\.\s+',  # Numbered chapters: '1. ', '2. ', etc.
-        r'^[A-Z\s]+$'  # All caps headings
-    ]
-
-    chapter_positions = []
-    used_pattern = None
-
-    for pattern in chapter_patterns:
-        if pattern is None:
-            continue
-        chapter_regex = re.compile(pattern, re.MULTILINE | re.IGNORECASE)
-        chapter_positions = [match.start() for match in chapter_regex.finditer(text)]
-        if chapter_positions:
-            used_pattern = pattern
-            break
-
-    # If no chapters found, return the entire content as one chunk
-    if not chapter_positions:
-        metadata = get_chunk_metadata(
-            chunk=text,
-            full_text=text,
-            chunk_type="whole_document",
-            language=chunk_options.get('language', 'english')
-        )
-        return [{'text': text, 'metadata': metadata}]
-
-    # Split content into chapters
-    chunks = []
-    for i in range(len(chapter_positions)):
-        start = chapter_positions[i]
-        end = chapter_positions[i + 1] if i + 1 < len(chapter_positions) else None
-        chapter = text[start:end]
-
-        # Apply overlap if specified
-        if overlap > 0 and i > 0:
-            overlap_start = max(0, chapter_positions[i] - overlap)
-            chapter = text[overlap_start:end]
-
-        chunks.append(chapter)
-
-    # Post-process chunks
-    processed_chunks = post_process_chunks(chunks)
-
-    # Add metadata to chunks
-    chunks_with_metadata = []
-    for i, chunk in enumerate(processed_chunks):
-        metadata = get_chunk_metadata(
-            chunk=chunk,
-            full_text=text,
-            chunk_type="chapter",
-            chapter_number=i + 1,
-            chapter_pattern=used_pattern,
-            language=chunk_options.get('language', 'english')
-        )
-        chunks_with_metadata.append({'text': chunk, 'metadata': metadata})
-
-    return chunks_with_metadata
-
-#
-# End of ebook chapter chunking
-#######################################################################################################################
-
-#######################################################################################################################
-#
-# Functions for adapative chunking:
-
-# FIXME - punkt
-
-def adaptive_chunk_size(text: str, base_size: int = 1000, min_size: int = 500, max_size: int = 2000) -> int:
-    # Tokenize the text into sentences
-    sentences = sent_tokenize(text)
-
-    if not sentences:
-        return base_size
-
-    # Calculate average sentence length
-    avg_sentence_length = sum(len(s.split()) for s in sentences) / len(sentences)
-
-    # Adjust chunk size based on average sentence length
-    if avg_sentence_length < 10:
-        size_factor = 1.2  # Increase chunk size for short sentences
-    elif avg_sentence_length > 20:
-        size_factor = 0.8  # Decrease chunk size for long sentences
-    else:
-        size_factor = 1.0
-
-    # Calculate adaptive chunk size
-    adaptive_size = int(base_size * size_factor)
-
-    # Ensure chunk size is within bounds
-    return max(min_size, min(adaptive_size, max_size))
-
-
-def adaptive_chunk_size_non_punkt(text: str, base_size: int, min_size: int = 100, max_size: int = 2000) -> int:
-    # Adaptive logic: adjust chunk size based on text complexity
-    words = text.split()
-    if not words:
-        return base_size  # Return base_size if text is empty
-
-    avg_word_length = sum(len(word) for word in words) / len(words)
-
-    if avg_word_length > 6:  # Threshold for "complex" text
-        adjusted_size = int(base_size * 0.8)  # Reduce chunk size for complex text
-    elif avg_word_length < 4:  # Threshold for "simple" text
-        adjusted_size = int(base_size * 1.2)  # Increase chunk size for simple text
-    else:
-        adjusted_size = base_size
-
-    # Ensure the chunk size is within the specified range
-    return max(min_size, min(adjusted_size, max_size))
-
-
-def adaptive_chunking(text: str, base_size: int = 1000, min_size: int = 500, max_size: int = 2000) -> List[str]:
-    logging.debug("adaptive_chunking...")
-    chunk_size = adaptive_chunk_size(text, base_size, min_size, max_size)
-    words = text.split()
-    chunks = []
-    current_chunk = []
-    current_length = 0
-
-    for word in words:
-        if current_length + len(word) > chunk_size and current_chunk:
-            chunks.append(' '.join(current_chunk))
-            current_chunk = []
-            current_length = 0
-        current_chunk.append(word)
-        current_length += len(word) + 1  # +1 for space
-
-    if current_chunk:
-        chunks.append(' '.join(current_chunk))
-
-    return chunks
-
-# FIXME - usage example
-# chunk_options = {
-#     'method': 'words',  # or any other method
-#     'base_size': 1000,
-#     'min_size': 100,
-#     'max_size': 2000,
-#     'adaptive': True,
-#     'language': 'en'
-# }
-#chunks = improved_chunking_process(your_text, chunk_options)
-
-
-# Example of chunking a document with metadata
-# document_metadata = {
-#     'title': 'Example Document',
-#     'author': 'John Doe',
-#     'creation_date': '2023-06-14',
-#     'source': 'https://example.com/document',
-#     'document_type': 'article'
-# }
-#
-# chunk_options = {
-#     'method': 'sentences',
-#     'base_size': 1000,
-#     'adaptive': True,
-#     'language': 'en'
-# }
-#
-# processed_document = process_document_with_metadata(your_text, chunk_options, document_metadata)
-
-
-#
-# End of Chunking Library
-#######################################################################################################################

App_Function_Libraries/DB/Character_Chat_DB.py

@@ -1,702 +0,0 @@
-# character_chat_db.py
-# Database functions for managing character cards and chat histories.
-# #
-# Imports
-import configparser
-import sqlite3
-import json
-import logging
-import os
-import sys
-from typing import List, Dict, Optional, Tuple, Any, Union
-
-from App_Function_Libraries.Utils.Utils import get_database_dir, get_project_relative_path, get_database_path
-#from Tests.Chat_APIs.Chat_APIs_Integration_test import logging
-
-#
-#######################################################################################################################
-#
-#
-
-def ensure_database_directory():
-    os.makedirs(get_database_dir(), exist_ok=True)
-
-ensure_database_directory()
-
-
-# Construct the path to the config file
-config_path = get_project_relative_path('Config_Files/config.txt')
-
-# Read the config file
-config = configparser.ConfigParser()
-config.read(config_path)
-
-# Get the chat db path from the config, or use the default if not specified
-chat_DB_PATH = config.get('Database', 'chatDB_path', fallback=get_database_path('chatDB.db'))
-print(f"Chat Database path: {chat_DB_PATH}")
-
-########################################################################################################
-#
-# Functions
-
-# FIXME - Setup properly and test/add documentation for its existence...
-def initialize_database():
-    """Initialize the SQLite database with required tables and FTS5 virtual tables."""
-    conn = None
-    try:
-        conn = sqlite3.connect(chat_DB_PATH)
-        cursor = conn.cursor()
-
-        # Enable foreign key constraints
-        cursor.execute("PRAGMA foreign_keys = ON;")
-
-        # Create CharacterCards table with V2 fields
-        cursor.execute("""
-        CREATE TABLE IF NOT EXISTS CharacterCards (
-            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            name TEXT UNIQUE NOT NULL,
-            description TEXT,
-            personality TEXT,
-            scenario TEXT,
-            image BLOB,
-            post_history_instructions TEXT,
-            first_mes TEXT,
-            mes_example TEXT,
-            creator_notes TEXT,
-            system_prompt TEXT,
-            alternate_greetings TEXT,
-            tags TEXT,
-            creator TEXT,
-            character_version TEXT,
-            extensions TEXT,
-            created_at DATETIME DEFAULT CURRENT_TIMESTAMP
-        );
-        """)
-
-        # Create CharacterChats table
-        cursor.execute("""
-        CREATE TABLE IF NOT EXISTS CharacterChats (
-            id INTEGER PRIMARY KEY AUTOINCREMENT,
-            character_id INTEGER NOT NULL,
-            conversation_name TEXT,
-            chat_history TEXT,
-            is_snapshot BOOLEAN DEFAULT FALSE,
-            created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
-            FOREIGN KEY (character_id) REFERENCES CharacterCards(id) ON DELETE CASCADE
-        );
-        """)
-
-        # Create FTS5 virtual table for CharacterChats
-        cursor.execute("""
-        CREATE VIRTUAL TABLE IF NOT EXISTS CharacterChats_fts USING fts5(
-            conversation_name,
-            chat_history,
-            content='CharacterChats',
-            content_rowid='id'
-        );
-        """)
-
-        # Create triggers to keep FTS5 table in sync with CharacterChats
-        cursor.executescript("""
-        CREATE TRIGGER IF NOT EXISTS CharacterChats_ai AFTER INSERT ON CharacterChats BEGIN
-            INSERT INTO CharacterChats_fts(rowid, conversation_name, chat_history)
-            VALUES (new.id, new.conversation_name, new.chat_history);
-        END;
-
-        CREATE TRIGGER IF NOT EXISTS CharacterChats_ad AFTER DELETE ON CharacterChats BEGIN
-            DELETE FROM CharacterChats_fts WHERE rowid = old.id;
-        END;
-
-        CREATE TRIGGER IF NOT EXISTS CharacterChats_au AFTER UPDATE ON CharacterChats BEGIN
-            UPDATE CharacterChats_fts SET conversation_name = new.conversation_name, chat_history = new.chat_history
-            WHERE rowid = new.id;
-        END;
-        """)
-
-        # Create ChatKeywords table
-        cursor.execute("""
-        CREATE TABLE IF NOT EXISTS ChatKeywords (
-            chat_id INTEGER NOT NULL,
-            keyword TEXT NOT NULL,
-            FOREIGN KEY (chat_id) REFERENCES CharacterChats(id) ON DELETE CASCADE
-        );
-        """)
-
-        # Create indexes for faster searches
-        cursor.execute("""
-        CREATE INDEX IF NOT EXISTS idx_chatkeywords_keyword ON ChatKeywords(keyword);
-        """)
-        cursor.execute("""
-        CREATE INDEX IF NOT EXISTS idx_chatkeywords_chat_id ON ChatKeywords(chat_id);
-        """)
-
-        conn.commit()
-        logging.info("Database initialized successfully.")
-    except sqlite3.Error as e:
-        logging.error(f"SQLite error occurred during database initialization: {e}")
-        if conn:
-            conn.rollback()
-        raise
-    except Exception as e:
-        logging.error(f"Unexpected error occurred during database initialization: {e}")
-        if conn:
-            conn.rollback()
-        raise
-    finally:
-        if conn:
-            conn.close()
-
-# Call initialize_database() at the start of your application
-def setup_chat_database():
-    try:
-        initialize_database()
-    except Exception as e:
-        logging.critical(f"Failed to initialize database: {e}")
-        sys.exit(1)
-
-setup_chat_database()
-
-########################################################################################################
-#
-# Character Card handling
-
-def parse_character_card(card_data: Dict[str, Any]) -> Dict[str, Any]:
-    """Parse and validate a character card according to V2 specification."""
-    v2_data = {
-        'name': card_data.get('name', ''),
-        'description': card_data.get('description', ''),
-        'personality': card_data.get('personality', ''),
-        'scenario': card_data.get('scenario', ''),
-        'first_mes': card_data.get('first_mes', ''),
-        'mes_example': card_data.get('mes_example', ''),
-        'creator_notes': card_data.get('creator_notes', ''),
-        'system_prompt': card_data.get('system_prompt', ''),
-        'post_history_instructions': card_data.get('post_history_instructions', ''),
-        'alternate_greetings': json.dumps(card_data.get('alternate_greetings', [])),
-        'tags': json.dumps(card_data.get('tags', [])),
-        'creator': card_data.get('creator', ''),
-        'character_version': card_data.get('character_version', ''),
-        'extensions': json.dumps(card_data.get('extensions', {}))
-    }
-
-    # Handle 'image' separately as it might be binary data
-    if 'image' in card_data:
-        v2_data['image'] = card_data['image']
-
-    return v2_data
-
-
-def add_character_card(card_data: Dict[str, Any]) -> Optional[int]:
-    """Add or update a character card in the database."""
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    try:
-        parsed_card = parse_character_card(card_data)
-
-        # Check if character already exists
-        cursor.execute("SELECT id FROM CharacterCards WHERE name = ?", (parsed_card['name'],))
-        row = cursor.fetchone()
-
-        if row:
-            # Update existing character
-            character_id = row[0]
-            update_query = """
-                UPDATE CharacterCards
-                SET description = ?, personality = ?, scenario = ?, image = ?, 
-                    post_history_instructions = ?, first_mes = ?, mes_example = ?,
-                    creator_notes = ?, system_prompt = ?, alternate_greetings = ?,
-                    tags = ?, creator = ?, character_version = ?, extensions = ?
-                WHERE id = ?
-            """
-            cursor.execute(update_query, (
-                parsed_card['description'], parsed_card['personality'], parsed_card['scenario'],
-                parsed_card['image'], parsed_card['post_history_instructions'], parsed_card['first_mes'],
-                parsed_card['mes_example'], parsed_card['creator_notes'], parsed_card['system_prompt'],
-                parsed_card['alternate_greetings'], parsed_card['tags'], parsed_card['creator'],
-                parsed_card['character_version'], parsed_card['extensions'], character_id
-            ))
-        else:
-            # Insert new character
-            insert_query = """
-                INSERT INTO CharacterCards (name, description, personality, scenario, image, 
-                post_history_instructions, first_mes, mes_example, creator_notes, system_prompt, 
-                alternate_greetings, tags, creator, character_version, extensions)
-                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
-            """
-            cursor.execute(insert_query, (
-                parsed_card['name'], parsed_card['description'], parsed_card['personality'],
-                parsed_card['scenario'], parsed_card['image'], parsed_card['post_history_instructions'],
-                parsed_card['first_mes'], parsed_card['mes_example'], parsed_card['creator_notes'],
-                parsed_card['system_prompt'], parsed_card['alternate_greetings'], parsed_card['tags'],
-                parsed_card['creator'], parsed_card['character_version'], parsed_card['extensions']
-            ))
-            character_id = cursor.lastrowid
-
-        conn.commit()
-        return character_id
-    except sqlite3.IntegrityError as e:
-        logging.error(f"Error adding character card: {e}")
-        return None
-    except Exception as e:
-        logging.error(f"Unexpected error adding character card: {e}")
-        return None
-    finally:
-        conn.close()
-
-# def add_character_card(card_data: Dict) -> Optional[int]:
-#     """Add or update a character card in the database.
-#
-#     Returns the ID of the inserted character or None if failed.
-#     """
-#     conn = sqlite3.connect(chat_DB_PATH)
-#     cursor = conn.cursor()
-#     try:
-#         # Ensure all required fields are present
-#         required_fields = ['name', 'description', 'personality', 'scenario', 'image', 'post_history_instructions', 'first_message']
-#         for field in required_fields:
-#             if field not in card_data:
-#                 card_data[field] = ''  # Assign empty string if field is missing
-#
-#         # Check if character already exists
-#         cursor.execute("SELECT id FROM CharacterCards WHERE name = ?", (card_data['name'],))
-#         row = cursor.fetchone()
-#
-#         if row:
-#             # Update existing character
-#             character_id = row[0]
-#             cursor.execute("""
-#                 UPDATE CharacterCards
-#                 SET description = ?, personality = ?, scenario = ?, image = ?, post_history_instructions = ?, first_message = ?
-#                 WHERE id = ?
-#             """, (
-#                 card_data['description'],
-#                 card_data['personality'],
-#                 card_data['scenario'],
-#                 card_data['image'],
-#                 card_data['post_history_instructions'],
-#                 card_data['first_message'],
-#                 character_id
-#             ))
-#         else:
-#             # Insert new character
-#             cursor.execute("""
-#                 INSERT INTO CharacterCards (name, description, personality, scenario, image, post_history_instructions, first_message)
-#                 VALUES (?, ?, ?, ?, ?, ?, ?)
-#             """, (
-#                 card_data['name'],
-#                 card_data['description'],
-#                 card_data['personality'],
-#                 card_data['scenario'],
-#                 card_data['image'],
-#                 card_data['post_history_instructions'],
-#                 card_data['first_message']
-#             ))
-#             character_id = cursor.lastrowid
-#
-#         conn.commit()
-#         return cursor.lastrowid
-#     except sqlite3.IntegrityError as e:
-#         logging.error(f"Error adding character card: {e}")
-#         return None
-#     except Exception as e:
-#         logging.error(f"Unexpected error adding character card: {e}")
-#         return None
-#     finally:
-#         conn.close()
-
-
-def get_character_cards() -> List[Dict]:
-    """Retrieve all character cards from the database."""
-    logging.debug(f"Fetching characters from DB: {chat_DB_PATH}")
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    cursor.execute("SELECT * FROM CharacterCards")
-    rows = cursor.fetchall()
-    columns = [description[0] for description in cursor.description]
-    conn.close()
-    characters = [dict(zip(columns, row)) for row in rows]
-    #logging.debug(f"Characters fetched from DB: {characters}")
-    return characters
-
-
-def get_character_card_by_id(character_id: Union[int, Dict[str, Any]]) -> Optional[Dict[str, Any]]:
-    """
-    Retrieve a single character card by its ID.
-
-    Args:
-        character_id: Can be either an integer ID or a dictionary containing character data.
-
-    Returns:
-        A dictionary containing the character card data, or None if not found.
-    """
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    try:
-        if isinstance(character_id, dict):
-            # If a dictionary is passed, assume it's already a character card
-            return character_id
-        elif isinstance(character_id, int):
-            # If an integer is passed, fetch the character from the database
-            cursor.execute("SELECT * FROM CharacterCards WHERE id = ?", (character_id,))
-            row = cursor.fetchone()
-            if row:
-                columns = [description[0] for description in cursor.description]
-                return dict(zip(columns, row))
-        else:
-            logging.warning(f"Invalid type for character_id: {type(character_id)}")
-        return None
-    except Exception as e:
-        logging.error(f"Error in get_character_card_by_id: {e}")
-        return None
-    finally:
-        conn.close()
-
-
-def update_character_card(character_id: int, card_data: Dict) -> bool:
-    """Update an existing character card."""
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    try:
-        cursor.execute("""
-            UPDATE CharacterCards
-            SET name = ?, description = ?, personality = ?, scenario = ?, image = ?, post_history_instructions = ?, first_message = ?
-            WHERE id = ?
-        """, (
-            card_data.get('name'),
-            card_data.get('description'),
-            card_data.get('personality'),
-            card_data.get('scenario'),
-            card_data.get('image'),
-            card_data.get('post_history_instructions', ''),
-            card_data.get('first_message', "Hello! I'm ready to chat."),
-            character_id
-        ))
-        conn.commit()
-        return cursor.rowcount > 0
-    except sqlite3.IntegrityError as e:
-        logging.error(f"Error updating character card: {e}")
-        return False
-    finally:
-        conn.close()
-
-
-def delete_character_card(character_id: int) -> bool:
-    """Delete a character card and its associated chats."""
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    try:
-        # Delete associated chats first due to foreign key constraint
-        cursor.execute("DELETE FROM CharacterChats WHERE character_id = ?", (character_id,))
-        cursor.execute("DELETE FROM CharacterCards WHERE id = ?", (character_id,))
-        conn.commit()
-        return cursor.rowcount > 0
-    except sqlite3.Error as e:
-        logging.error(f"Error deleting character card: {e}")
-        return False
-    finally:
-        conn.close()
-
-
-def add_character_chat(character_id: int, conversation_name: str, chat_history: List[Tuple[str, str]], keywords: Optional[List[str]] = None, is_snapshot: bool = False) -> Optional[int]:
-    """
-    Add a new chat history for a character, optionally associating keywords.
-
-    Args:
-        character_id (int): The ID of the character.
-        conversation_name (str): Name of the conversation.
-        chat_history (List[Tuple[str, str]]): List of (user, bot) message tuples.
-        keywords (Optional[List[str]]): List of keywords to associate with this chat.
-        is_snapshot (bool, optional): Whether this chat is a snapshot.
-
-    Returns:
-        Optional[int]: The ID of the inserted chat or None if failed.
-    """
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    try:
-        chat_history_json = json.dumps(chat_history)
-        cursor.execute("""
-            INSERT INTO CharacterChats (character_id, conversation_name, chat_history, is_snapshot)
-            VALUES (?, ?, ?, ?)
-        """, (
-            character_id,
-            conversation_name,
-            chat_history_json,
-            is_snapshot
-        ))
-        chat_id = cursor.lastrowid
-
-        if keywords:
-            # Insert keywords into ChatKeywords table
-            keyword_records = [(chat_id, keyword.strip().lower()) for keyword in keywords]
-            cursor.executemany("""
-                INSERT INTO ChatKeywords (chat_id, keyword)
-                VALUES (?, ?)
-            """, keyword_records)
-
-        conn.commit()
-        return chat_id
-    except sqlite3.Error as e:
-        logging.error(f"Error adding character chat: {e}")
-        return None
-    finally:
-        conn.close()
-
-
-def get_character_chats(character_id: Optional[int] = None) -> List[Dict]:
-    """Retrieve all chats, or chats for a specific character if character_id is provided."""
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    if character_id is not None:
-        cursor.execute("SELECT * FROM CharacterChats WHERE character_id = ?", (character_id,))
-    else:
-        cursor.execute("SELECT * FROM CharacterChats")
-    rows = cursor.fetchall()
-    columns = [description[0] for description in cursor.description]
-    conn.close()
-    return [dict(zip(columns, row)) for row in rows]
-
-
-def get_character_chat_by_id(chat_id: int) -> Optional[Dict]:
-    """Retrieve a single chat by its ID."""
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    cursor.execute("SELECT * FROM CharacterChats WHERE id = ?", (chat_id,))
-    row = cursor.fetchone()
-    conn.close()
-    if row:
-        columns = [description[0] for description in cursor.description]
-        chat = dict(zip(columns, row))
-        chat['chat_history'] = json.loads(chat['chat_history'])
-        return chat
-    return None
-
-
-def search_character_chats(query: str, character_id: Optional[int] = None) -> Tuple[List[Dict], str]:
-    """
-    Search for character chats using FTS5, optionally filtered by character_id.
-
-    Args:
-        query (str): The search query.
-        character_id (Optional[int]): The ID of the character to filter chats by.
-
-    Returns:
-        Tuple[List[Dict], str]: A list of matching chats and a status message.
-    """
-    if not query.strip():
-        return [], "Please enter a search query."
-
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    try:
-        if character_id is not None:
-            # Search with character_id filter
-            cursor.execute("""
-                SELECT CharacterChats.id, CharacterChats.conversation_name, CharacterChats.chat_history
-                FROM CharacterChats_fts
-                JOIN CharacterChats ON CharacterChats_fts.rowid = CharacterChats.id
-                WHERE CharacterChats_fts MATCH ? AND CharacterChats.character_id = ?
-                ORDER BY rank
-            """, (query, character_id))
-        else:
-            # Search without character_id filter
-            cursor.execute("""
-                SELECT CharacterChats.id, CharacterChats.conversation_name, CharacterChats.chat_history
-                FROM CharacterChats_fts
-                JOIN CharacterChats ON CharacterChats_fts.rowid = CharacterChats.id
-                WHERE CharacterChats_fts MATCH ?
-                ORDER BY rank
-            """, (query,))
-
-        rows = cursor.fetchall()
-        columns = [description[0] for description in cursor.description]
-        results = [dict(zip(columns, row)) for row in rows]
-
-        if character_id is not None:
-            status_message = f"Found {len(results)} chat(s) matching '{query}' for the selected character."
-        else:
-            status_message = f"Found {len(results)} chat(s) matching '{query}' across all characters."
-
-        return results, status_message
-    except Exception as e:
-        logging.error(f"Error searching chats with FTS5: {e}")
-        return [], f"Error occurred during search: {e}"
-    finally:
-        conn.close()
-
-def update_character_chat(chat_id: int, chat_history: List[Tuple[str, str]]) -> bool:
-    """Update an existing chat history."""
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    try:
-        chat_history_json = json.dumps(chat_history)
-        cursor.execute("""
-            UPDATE CharacterChats
-            SET chat_history = ?
-            WHERE id = ?
-        """, (
-            chat_history_json,
-            chat_id
-        ))
-        conn.commit()
-        return cursor.rowcount > 0
-    except sqlite3.Error as e:
-        logging.error(f"Error updating character chat: {e}")
-        return False
-    finally:
-        conn.close()
-
-
-def delete_character_chat(chat_id: int) -> bool:
-    """Delete a specific chat."""
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    try:
-        cursor.execute("DELETE FROM CharacterChats WHERE id = ?", (chat_id,))
-        conn.commit()
-        return cursor.rowcount > 0
-    except sqlite3.Error as e:
-        logging.error(f"Error deleting character chat: {e}")
-        return False
-    finally:
-        conn.close()
-
-def fetch_keywords_for_chats(keywords: List[str]) -> List[int]:
-    """
-    Fetch chat IDs associated with any of the specified keywords.
-
-    Args:
-        keywords (List[str]): List of keywords to search for.
-
-    Returns:
-        List[int]: List of chat IDs associated with the keywords.
-    """
-    if not keywords:
-        return []
-
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    try:
-        # Construct the WHERE clause to search for each keyword
-        keyword_clauses = " OR ".join(["keyword = ?"] * len(keywords))
-        sql_query = f"SELECT DISTINCT chat_id FROM ChatKeywords WHERE {keyword_clauses}"
-        cursor.execute(sql_query, keywords)
-        rows = cursor.fetchall()
-        chat_ids = [row[0] for row in rows]
-        return chat_ids
-    except Exception as e:
-        logging.error(f"Error in fetch_keywords_for_chats: {e}")
-        return []
-    finally:
-        conn.close()
-
-def save_chat_history_to_character_db(character_id: int, conversation_name: str, chat_history: List[Tuple[str, str]]) -> Optional[int]:
-    """Save chat history to the CharacterChats table.
-
-    Returns the ID of the inserted chat or None if failed.
-    """
-    return add_character_chat(character_id, conversation_name, chat_history)
-
-def migrate_chat_to_media_db():
-    pass
-
-
-def search_db(query: str, fields: List[str], where_clause: str = "", page: int = 1, results_per_page: int = 5) -> List[Dict[str, Any]]:
-    """
-    Perform a full-text search on specified fields with optional filtering and pagination.
-
-    Args:
-        query (str): The search query.
-        fields (List[str]): List of fields to search in.
-        where_clause (str, optional): Additional SQL WHERE clause to filter results.
-        page (int, optional): Page number for pagination.
-        results_per_page (int, optional): Number of results per page.
-
-    Returns:
-        List[Dict[str, Any]]: List of matching chat records with content and metadata.
-    """
-    if not query.strip():
-        return []
-
-    conn = sqlite3.connect(chat_DB_PATH)
-    cursor = conn.cursor()
-    try:
-        # Construct the MATCH query for FTS5
-        match_query = " AND ".join(fields) + f" MATCH ?"
-        # Adjust the query with the fields
-        fts_query = f"""
-            SELECT CharacterChats.id, CharacterChats.conversation_name, CharacterChats.chat_history
-            FROM CharacterChats_fts
-            JOIN CharacterChats ON CharacterChats_fts.rowid = CharacterChats.id
-            WHERE {match_query}
-        """
-        if where_clause:
-            fts_query += f" AND ({where_clause})"
-        fts_query += " ORDER BY rank LIMIT ? OFFSET ?"
-        offset = (page - 1) * results_per_page
-        cursor.execute(fts_query, (query, results_per_page, offset))
-        rows = cursor.fetchall()
-        columns = [description[0] for description in cursor.description]
-        results = [dict(zip(columns, row)) for row in rows]
-        return results
-    except Exception as e:
-        logging.error(f"Error in search_db: {e}")
-        return []
-    finally:
-        conn.close()
-
-
-def perform_full_text_search_chat(query: str, relevant_chat_ids: List[int], page: int = 1, results_per_page: int = 5) -> \
-List[Dict[str, Any]]:
-    """
-    Perform a full-text search within the specified chat IDs using FTS5.
-
-    Args:
-        query (str): The user's query.
-        relevant_chat_ids (List[int]): List of chat IDs to search within.
-        page (int): Pagination page number.
-        results_per_page (int): Number of results per page.
-
-    Returns:
-        List[Dict[str, Any]]: List of search results with content and metadata.
-    """
-    try:
-        # Construct a WHERE clause to limit the search to relevant chat IDs
-        where_clause = " OR ".join([f"media_id = {chat_id}" for chat_id in relevant_chat_ids])
-        if not where_clause:
-            where_clause = "1"  # No restriction if no chat IDs
-
-        # Perform full-text search using FTS5
-        fts_results = search_db(query, ["content"], where_clause, page=page, results_per_page=results_per_page)
-
-        filtered_fts_results = [
-            {
-                "content": result['content'],
-                "metadata": {"media_id": result['id']}
-            }
-            for result in fts_results
-            if result['id'] in relevant_chat_ids
-        ]
-        return filtered_fts_results
-    except Exception as e:
-        logging.error(f"Error in perform_full_text_search_chat: {str(e)}")
-        return []
-
-
-def fetch_all_chats() -> List[Dict[str, Any]]:
-    """
-    Fetch all chat messages from the database.
-
-    Returns:
-        List[Dict[str, Any]]: List of chat messages with relevant metadata.
-    """
-    try:
-        chats = get_character_chats()  # Modify this function to retrieve all chats
-        return chats
-    except Exception as e:
-        logging.error(f"Error fetching all chats: {str(e)}")
-        return []
-
-#
-# End of Character_Chat_DB.py
-#######################################################################################################################

App_Function_Libraries/DB/DB_Manager.py

@@ -1,991 +0,0 @@
-# DB_Manager.py
-# Description: This file contains the DatabaseManager class, which is responsible for managing the database connection, i.e. either SQLite or Elasticsearch.
-#
-# Imports
-import configparser
-import os
-import logging
-import time
-from typing import Tuple, List, Union, Dict
-#
-# 3rd-Party Libraries
-from elasticsearch import Elasticsearch
-#
-# Import your existing SQLite functions
-from App_Function_Libraries.DB.SQLite_DB import DatabaseError
-from App_Function_Libraries.DB.SQLite_DB import (
-    update_media_content as sqlite_update_media_content,
-    list_prompts as sqlite_list_prompts,
-    search_and_display as sqlite_search_and_display,
-    fetch_prompt_details as sqlite_fetch_prompt_details,
-    keywords_browser_interface as sqlite_keywords_browser_interface,
-    add_keyword as sqlite_add_keyword,
-    delete_keyword as sqlite_delete_keyword,
-    export_keywords_to_csv as sqlite_export_keywords_to_csv,
-    ingest_article_to_db as sqlite_ingest_article_to_db,
-    add_media_to_database as sqlite_add_media_to_database,
-    import_obsidian_note_to_db as sqlite_import_obsidian_note_to_db,
-    add_prompt as sqlite_add_prompt,
-    delete_chat_message as sqlite_delete_chat_message,
-    update_chat_message as sqlite_update_chat_message,
-    add_chat_message as sqlite_add_chat_message,
-    get_chat_messages as sqlite_get_chat_messages,
-    search_chat_conversations as sqlite_search_chat_conversations,
-    create_chat_conversation as sqlite_create_chat_conversation,
-    save_chat_history_to_database as sqlite_save_chat_history_to_database,
-    view_database as sqlite_view_database,
-    get_transcripts as sqlite_get_transcripts,
-    get_trashed_items as sqlite_get_trashed_items,
-    user_delete_item as sqlite_user_delete_item,
-    empty_trash as sqlite_empty_trash,
-    create_automated_backup as sqlite_create_automated_backup,
-    add_or_update_prompt as sqlite_add_or_update_prompt,
-    load_prompt_details as sqlite_load_prompt_details,
-    load_preset_prompts as sqlite_load_preset_prompts,
-    insert_prompt_to_db as sqlite_insert_prompt_to_db,
-    delete_prompt as sqlite_delete_prompt,
-    search_and_display_items as sqlite_search_and_display_items,
-    get_conversation_name as sqlite_get_conversation_name,
-    add_media_with_keywords as sqlite_add_media_with_keywords,
-    check_media_and_whisper_model as sqlite_check_media_and_whisper_model, \
-    create_document_version as sqlite_create_document_version,
-    get_document_version as sqlite_get_document_version, sqlite_search_db, add_media_chunk as sqlite_add_media_chunk,
-    sqlite_update_fts_for_media, get_unprocessed_media as sqlite_get_unprocessed_media, fetch_item_details as sqlite_fetch_item_details, \
-    search_media_database as sqlite_search_media_database, mark_as_trash as sqlite_mark_as_trash, \
-    get_media_transcripts as sqlite_get_media_transcripts, get_specific_transcript as sqlite_get_specific_transcript, \
-    get_media_summaries as sqlite_get_media_summaries, get_specific_summary as sqlite_get_specific_summary, \
-    get_media_prompts as sqlite_get_media_prompts, get_specific_prompt as sqlite_get_specific_prompt, \
-    delete_specific_transcript as sqlite_delete_specific_transcript,
-    delete_specific_summary as sqlite_delete_specific_summary, \
-    delete_specific_prompt as sqlite_delete_specific_prompt,
-    fetch_keywords_for_media as sqlite_fetch_keywords_for_media, \
-    update_keywords_for_media as sqlite_update_keywords_for_media, check_media_exists as sqlite_check_media_exists, \
-    search_prompts as sqlite_search_prompts, get_media_content as sqlite_get_media_content, \
-    get_paginated_files as sqlite_get_paginated_files, get_media_title as sqlite_get_media_title, \
-    get_all_content_from_database as sqlite_get_all_content_from_database,
-    get_next_media_id as sqlite_get_next_media_id, \
-    batch_insert_chunks as sqlite_batch_insert_chunks, Database, save_workflow_chat_to_db as sqlite_save_workflow_chat_to_db, \
-    get_workflow_chat as sqlite_get_workflow_chat, update_media_content_with_version as sqlite_update_media_content_with_version, \
-    check_existing_media as sqlite_check_existing_media, get_all_document_versions as sqlite_get_all_document_versions, \
-    fetch_paginated_data as sqlite_fetch_paginated_data, get_latest_transcription as sqlite_get_latest_transcription, \
-    mark_media_as_processed as sqlite_mark_media_as_processed,
-)
-from App_Function_Libraries.DB.Character_Chat_DB import (
-    add_character_card as sqlite_add_character_card, get_character_cards as sqlite_get_character_cards, \
-    get_character_card_by_id as sqlite_get_character_card_by_id, update_character_card as sqlite_update_character_card, \
-    delete_character_card as sqlite_delete_character_card, add_character_chat as sqlite_add_character_chat, \
-    get_character_chats as sqlite_get_character_chats, get_character_chat_by_id as sqlite_get_character_chat_by_id, \
-    update_character_chat as sqlite_update_character_chat, delete_character_chat as sqlite_delete_character_chat, \
-    migrate_chat_to_media_db as sqlite_migrate_chat_to_media_db,
-)
-#
-# Local Imports
-from App_Function_Libraries.Utils.Utils import load_comprehensive_config, get_database_path, get_project_relative_path
-#
-# End of imports
-############################################################################################################
-
-
-############################################################################################################
-#
-# Database Config loading
-
-logger = logging.getLogger(__name__)
-
-config_path = get_project_relative_path('Config_Files/config.txt')
-config = configparser.ConfigParser()
-config.read(config_path)
-
-db_path: str = config.get('Database', 'sqlite_path', fallback='./Databases/media_summary.db')
-backup_path: str = config.get('Database', 'backup_path', fallback='database_backups')
-backup_dir: Union[str, bytes] = os.environ.get('DB_BACKUP_DIR', backup_path)
-
-def get_db_config():
-    try:
-        config = load_comprehensive_config()
-
-        if 'Database' not in config:
-            print("Warning: 'Database' section not found in config. Using default values.")
-            return default_db_config()
-
-        return {
-            'type': config.get('Database', 'type', fallback='sqlite'),
-            'sqlite_path': config.get('Database', 'sqlite_path', fallback='Databases/media_summary.db'),
-            'elasticsearch_host': config.get('Database', 'elasticsearch_host', fallback='localhost'),
-            'elasticsearch_port': config.getint('Database', 'elasticsearch_port', fallback=9200)
-        }
-    except FileNotFoundError:
-        print("Warning: Config file not found. Using default database configuration.")
-        return default_db_config()
-    except Exception as e:
-        print(f"Error reading config: {str(e)}. Using default database configuration.")
-        return default_db_config()
-
-def default_db_config():
-    return {
-        'type': 'sqlite',
-        'sqlite_path': get_database_path('media_summary.db'),
-        'elasticsearch_host': 'localhost',
-        'elasticsearch_port': 9200
-    }
-
-def ensure_directory_exists(file_path):
-    directory = os.path.dirname(file_path)
-    if not os.path.exists(directory):
-        os.makedirs(directory)
-        print(f"Created directory: {directory}")
-
-db_config = get_db_config()
-db_type = db_config['type']
-
-if db_type == 'sqlite':
-    db = Database(os.path.basename(db_config['sqlite_path']))
-elif db_type == 'elasticsearch':
-    raise NotImplementedError("Elasticsearch support not yet implemented")
-else:
-    raise ValueError(f"Unsupported database type: {db_type}")
-
-print(f"Database path: {db.db_path}")
-
-def get_db_config():
-    try:
-        config = load_comprehensive_config()
-
-        if 'Database' not in config:
-            print("Warning: 'Database' section not found in config. Using default values.")
-            return default_db_config()
-
-        return {
-            'type': config.get('Database', 'type', fallback='sqlite'),
-            'sqlite_path': config.get('Database', 'sqlite_path', fallback='Databases/media_summary.db'),
-            'elasticsearch_host': config.get('Database', 'elasticsearch_host', fallback='localhost'),
-            'elasticsearch_port': config.getint('Database', 'elasticsearch_port', fallback=9200)
-        }
-    except FileNotFoundError:
-        print("Warning: Config file not found. Using default database configuration.")
-        return default_db_config()
-    except Exception as e:
-        print(f"Error reading config: {str(e)}. Using default database configuration.")
-        return default_db_config()
-
-
-def default_db_config():
-    """Return the default database configuration with project-relative paths."""
-    return {
-        'type': 'sqlite',
-        'sqlite_path': get_database_path('media_summary.db'),
-        'elasticsearch_host': 'localhost',
-        'elasticsearch_port': 9200
-    }
-
-
-def ensure_directory_exists(file_path):
-    directory = os.path.dirname(file_path)
-    if not os.path.exists(directory):
-        os.makedirs(directory)
-        print(f"Created directory: {directory}")
-
-# Use the config to set up the database
-db_config = get_db_config()
-db_type = db_config['type']
-
-if db_type == 'sqlite':
-    db = Database(os.path.basename(db_config['sqlite_path']))
-elif db_type == 'elasticsearch':
-    # Implement Elasticsearch setup here if needed
-    raise NotImplementedError("Elasticsearch support not yet implemented")
-else:
-    raise ValueError(f"Unsupported database type: {db_type}")
-
-# Print database path for debugging
-print(f"Database path: {db.db_path}")
-
-# Sanity Check for SQLite DB
-# FIXME - Remove this after testing / Writing Unit tests
-# try:
-#     db.execute_query("CREATE TABLE IF NOT EXISTS test_table (id INTEGER PRIMARY KEY)")
-#     logger.info("Successfully created test table")
-# except DatabaseError as e:
-#     logger.error(f"Failed to create test table: {e}")
-
-#
-# End of Database Config loading
-############################################################################################################
-#
-# DB Search functions
-
-def search_db(search_query: str, search_fields: List[str], keywords: str, page: int = 1, results_per_page: int = 10):
-    if db_type == 'sqlite':
-        return sqlite_search_db(search_query, search_fields, keywords, page, results_per_page)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version when available
-        raise NotImplementedError("Elasticsearch version of search_db not yet implemented")
-    else:
-        raise ValueError(f"Unsupported database type: {db_type}")
-
-def view_database(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_view_database(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-def search_and_display_items(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_search_and_display_items(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-def get_all_content_from_database():
-    if db_type == 'sqlite':
-        return sqlite_get_all_content_from_database()
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-def search_and_display(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_search_and_display(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-def check_media_exists(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_check_media_exists(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-def get_paginated_files(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_get_paginated_files(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-def get_media_title(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_get_media_title(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-def get_next_media_id():
-    if db_type == 'sqlite':
-        return sqlite_get_next_media_id()
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-#
-# End of DB-Searching functions
-############################################################################################################
-
-
-############################################################################################################
-#
-# Transcript-related Functions
-
-def get_transcripts(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_get_transcripts(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-#
-# End of Transcript-related Functions
-############################################################################################################
-
-
-############################################################################################################
-#
-# DB-Ingestion functions
-
-def add_media_to_database(*args, **kwargs):
-    if db_type == 'sqlite':
-        result = sqlite_add_media_to_database(*args, **kwargs)
-
-        # Extract content
-        segments = kwargs.get('segments') if 'segments' in kwargs else args[2] if len(args) > 2 else None
-        if segments is None:
-            raise ValueError("Segments not provided in arguments")
-
-        if isinstance(segments, list):
-            content = ' '.join([segment.get('Text', '') for segment in segments if 'Text' in segment])
-        elif isinstance(segments, dict):
-            content = segments.get('text', '') or segments.get('content', '')
-        else:
-            content = str(segments)
-
-        # Extract media_id from the result
-        # Assuming the result is in the format "Media 'Title' added/updated successfully with ID: {media_id}"
-        import re
-        match = re.search(r"with ID: (\d+)", result)
-        if match:
-            media_id = int(match.group(1))
-
-            # Create initial document version
-            sqlite_create_document_version(media_id, content)
-
-        return result
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_to_database not yet implemented")
-
-def check_existing_media(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_check_existing_media(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of check_existing_media not yet implemented")
-
-def update_media_content_with_version(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_update_media_content_with_version(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of update_media_content not yet implemented")
-
-def import_obsidian_note_to_db(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_import_obsidian_note_to_db(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-
-def update_media_content(*args, **kwargs):
-    if db_type == 'sqlite':
-        result = sqlite_update_media_content(*args, **kwargs)
-
-        # Extract media_id and content
-        selected_item = args[0]
-        item_mapping = args[1]
-        content_input = args[2]
-
-        if selected_item and item_mapping and selected_item in item_mapping:
-            media_id = item_mapping[selected_item]
-
-            # Create new document version
-            sqlite_create_document_version(media_id, content_input)
-
-        return result
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of update_media_content not yet implemented")
-
-
-def add_media_with_keywords(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_add_media_with_keywords(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-def check_media_and_whisper_model(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_check_media_and_whisper_model(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        raise NotImplementedError("Elasticsearch version of check_media_and_whisper_model not yet implemented")
-
-def ingest_article_to_db(url, title, author, content, keywords, summary, ingestion_date, custom_prompt):
-    if db_type == 'sqlite':
-        return sqlite_ingest_article_to_db(url, title, author, content, keywords, summary, ingestion_date, custom_prompt)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of ingest_article_to_db not yet implemented")
-    else:
-        raise ValueError(f"Unsupported database type: {db_type}")
-
-
-def add_media_chunk(*args, **kwargs):
-    if db_type == 'sqlite':
-        sqlite_add_media_chunk(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version not yet implemented")
-    else:
-        raise ValueError(f"Unsupported database type: {db_type}")
-
-def batch_insert_chunks(*args, **kwargs):
-    if db_type == 'sqlite':
-        sqlite_batch_insert_chunks(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version not yet implemented")
-    else:
-        raise ValueError(f"Unsupported database type: {db_type}")
-
-def update_fts_for_media(media_id: int):
-    if db_type == 'sqlite':
-        sqlite_update_fts_for_media(db, media_id)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version not yet implemented")
-    else:
-        raise ValueError(f"Unsupported database type: {db_type}")
-
-
-def get_unprocessed_media(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_get_unprocessed_media(db)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of get_unprocessed_media not yet implemented")
-    else:
-        raise ValueError(f"Unsupported database type: {db_type}")
-
-
-def mark_media_as_processed(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_mark_media_as_processed(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of mark_media_as_processed not yet implemented")
-    else:
-        raise ValueError(f"Unsupported database type: {db_type}")
-
-
-#
-# End of DB-Ingestion functions
-############################################################################################################
-
-
-############################################################################################################
-#
-# Prompt-related functions #FIXME rename /resort
-
-def list_prompts(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_list_prompts(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-def search_prompts(query):
-    if db_type == 'sqlite':
-        return sqlite_search_prompts(query)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-def fetch_prompt_details(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_fetch_prompt_details(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-def add_prompt(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_add_prompt(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-
-def add_or_update_prompt(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_add_or_update_prompt(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-def load_prompt_details(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_load_prompt_details(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-def load_preset_prompts(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_load_preset_prompts()
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-def insert_prompt_to_db(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_insert_prompt_to_db(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-def delete_prompt(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_delete_prompt(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-def search_media_database(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_search_media_database(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version when available
-        raise NotImplementedError("Elasticsearch version of search_media_database not yet implemented")
-    else:
-        raise ValueError(f"Unsupported database type: {db_type}")
-
-def mark_as_trash(media_id: int) -> None:
-    if db_type == 'sqlite':
-        return sqlite_mark_as_trash(media_id)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version when available
-        raise NotImplementedError("Elasticsearch version of mark_as_trash not yet implemented")
-    else:
-        raise ValueError(f"Unsupported database type: {db_type}")
-
-
-def get_latest_transcription(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_get_latest_transcription(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of get_latest_transcription not yet implemented")
-
-def fetch_paginated_data(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_fetch_paginated_data(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of fetch_paginated_data not yet implemented")
-    else:
-        raise ValueError(f"Unsupported database type: {db_type}")
-
-        
-def get_media_content(media_id: int) -> str:
-    if db_type == 'sqlite':
-        return sqlite_get_media_content(media_id)
-    elif db_type == 'elasticsearch':
-        raise NotImplementedError("Elasticsearch version of get_media_content not yet implemented")
-    else:
-        raise ValueError(f"Unsupported database type: {db_type}")
-
-def get_media_transcripts(media_id: int) -> List[Dict]:
-    if db_type == 'sqlite':
-        return sqlite_get_media_transcripts(media_id)
-    elif db_type == 'elasticsearch':
-        raise NotImplementedError("Elasticsearch version of get_media_transcripts not yet implemented")
-    else:
-        raise ValueError(f"Unsupported database type: {db_type}")
-
-def get_specific_transcript(transcript_id: int) -> Dict:
-    if db_type == 'sqlite':
-        return sqlite_get_specific_transcript(transcript_id)
-    elif db_type == 'elasticsearch':
-        raise NotImplementedError("Elasticsearch version of get_specific_transcript not yet implemented")
-    else:
-        raise ValueError(f"Unsupported database type: {db_type}")
-
-def get_media_summaries(media_id: int) -> List[Dict]:
-    if db_type == 'sqlite':
-        return sqlite_get_media_summaries(media_id)
-    elif db_type == 'elasticsearch':
-        raise NotImplementedError("Elasticsearch version of get_media_summaries not yet implemented")
-    else:
-        raise ValueError(f"Unsupported database type: {db_type}")
-
-def get_specific_summary(summary_id: int) -> Dict:
-    if db_type == 'sqlite':
-        return sqlite_get_specific_summary(summary_id)
-    elif db_type == 'elasticsearch':
-        raise NotImplementedError("Elasticsearch version of get_specific_summary not yet implemented")
-    else:
-        raise ValueError(f"Unsupported database type: {db_type}")
-
-def fetch_item_details_single(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_fetch_item_details(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of fetch_item_details not yet implemented")
-    else:
-        raise ValueError(f"Unsupported database type: {db_type}")
-
-def get_all_document_versions(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_get_all_document_versions(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of get_all_document_versions not yet implemented")
-    else:
-        raise ValueError(f"Unsupported database type: {db_type}")
-#
-#
-############################################################################################################
-#
-# Prompt Functions:
-
-def get_media_prompts(media_id: int) -> List[Dict]:
-    if db_type == 'sqlite':
-        return sqlite_get_media_prompts(media_id)
-    elif db_type == 'elasticsearch':
-        raise NotImplementedError("Elasticsearch version of get_media_prompts not yet implemented")
-    else:
-        raise ValueError(f"Unsupported database type: {db_type}")
-
-def get_specific_prompt(prompt_id: int) -> Dict:
-    if db_type == 'sqlite':
-        return sqlite_get_specific_prompt(prompt_id)
-    elif db_type == 'elasticsearch':
-        raise NotImplementedError("Elasticsearch version of get_specific_prompt not yet implemented")
-    else:
-        return {'error': f"Unsupported database type: {db_type}"}
-
-def delete_specific_transcript(transcript_id: int) -> str:
-    if db_type == 'sqlite':
-        return sqlite_delete_specific_transcript(transcript_id)
-    elif db_type == 'elasticsearch':
-        raise NotImplementedError("Elasticsearch version of delete_specific_transcript not yet implemented")
-    else:
-        raise ValueError(f"Unsupported database type: {db_type}")
-
-def delete_specific_summary(summary_id: int) -> str:
-    if db_type == 'sqlite':
-        return sqlite_delete_specific_summary(summary_id)
-    elif db_type == 'elasticsearch':
-        raise NotImplementedError("Elasticsearch version of delete_specific_summary not yet implemented")
-    else:
-        raise ValueError(f"Unsupported database type: {db_type}")
-
-def delete_specific_prompt(prompt_id: int) -> str:
-    if db_type == 'sqlite':
-        return sqlite_delete_specific_prompt(prompt_id)
-    elif db_type == 'elasticsearch':
-        raise NotImplementedError("Elasticsearch version of delete_specific_prompt not yet implemented")
-    else:
-        raise ValueError(f"Unsupported database type: {db_type}")
-
-
-#
-# End of Prompt-related functions
-############################################################################################################
-
-############################################################################################################
-#
-# Keywords-related Functions
-
-def keywords_browser_interface(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_keywords_browser_interface()
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-def add_keyword(*args, **kwargs):
-    if db_type == 'sqlite':
-        with db.get_connection() as conn:
-            cursor = conn.cursor()
-        return sqlite_add_keyword(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-def delete_keyword(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_delete_keyword(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-def export_keywords_to_csv(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_export_keywords_to_csv()
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-def update_keywords_for_media(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_update_keywords_for_media(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-def fetch_keywords_for_media(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_fetch_keywords_for_media(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-#
-# End of Keywords-related Functions
-############################################################################################################
-
-############################################################################################################
-#
-# Chat-related Functions
-
-def delete_chat_message(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_delete_chat_message(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-def update_chat_message(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_update_chat_message(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-def add_chat_message(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_add_chat_message(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-def get_chat_messages(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_get_chat_messages(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-def search_chat_conversations(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_search_chat_conversations(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-def create_chat_conversation(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_create_chat_conversation(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-def save_chat_history_to_database(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_save_chat_history_to_database(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-def get_conversation_name(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_get_conversation_name(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-#
-# End of Chat-related Functions
-############################################################################################################
-
-
-############################################################################################################
-#
-# Character Chat-related Functions
-
-def add_character_card(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_add_character_card(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_character_card not yet implemented")
-
-def get_character_cards():
-    if db_type == 'sqlite':
-        return sqlite_get_character_cards()
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of get_character_cards not yet implemented")
-
-def get_character_card_by_id(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_get_character_card_by_id(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of get_character_card_by_id not yet implemented")
-
-def update_character_card(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_update_character_card(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of update_character_card not yet implemented")
-
-def delete_character_card(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_delete_character_card(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of delete_character_card not yet implemented")
-
-def add_character_chat(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_add_character_chat(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_character_chat not yet implemented")
-
-def get_character_chats(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_get_character_chats(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of get_character_chats not yet implemented")
-
-def get_character_chat_by_id(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_get_character_chat_by_id(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of get_character_chat_by_id not yet implemented")
-
-def update_character_chat(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_update_character_chat(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of update_character_chat not yet implemented")
-
-def delete_character_chat(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_delete_character_chat(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of delete_character_chat not yet implemented")
-
-def migrate_chat_to_media_db(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_migrate_chat_to_media_db(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of migrate_chat_to_media_db not yet implemented")
-
-#
-# End of Character Chat-related Functions
-############################################################################################################
-
-
-############################################################################################################
-#
-# Trash-related Functions
-
-def get_trashed_items(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_get_trashed_items()
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-def user_delete_item(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_user_delete_item(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-def empty_trash(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_empty_trash(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-
-def fetch_item_details(media_id: int) -> Tuple[str, str, str]:
-    """
-    Fetch the details of a media item including content, prompt, and summary.
-
-    Args:
-        media_id (int): The ID of the media item.
-
-    Returns:
-        Tuple[str, str, str]: A tuple containing (content, prompt, summary).
-        If an error occurs, it returns empty strings for each field.
-    """
-    if db_type == 'sqlite':
-        return sqlite_fetch_item_details(media_id)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version when available
-        raise NotImplementedError("Elasticsearch version of fetch_item_details not yet implemented")
-    else:
-        raise ValueError(f"Unsupported database type: {db_type}")
-
-#
-# End of Trash-related Functions
-############################################################################################################
-
-
-############################################################################################################
-#
-# DB-Backup Functions
-
-def create_automated_backup(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_create_automated_backup(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of add_media_with_keywords not yet implemented")
-
-#
-# End of DB-Backup Functions
-############################################################################################################
-
-
-############################################################################################################
-#
-# Document Versioning Functions
-
-def create_document_version(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_create_document_version(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of create_document_version not yet implemented")
-
-def get_document_version(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_get_document_version(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of get_document_version not yet implemented")
-
-#
-# End of Document Versioning Functions
-############################################################################################################
-
-
-############################################################################################################
-#
-# Workflow Functions
-
-def get_workflow_chat(*args, **kwargs):
-    if db_type == 'sqlite':
-        return sqlite_get_workflow_chat(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of get_workflow_chat not yet implemented")
-
-
-def save_workflow_chat_to_db(*args, **kwargs):
-    if db_type == 'sqlite':
-        # FIXME
-        return sqlite_save_workflow_chat_to_db(*args, **kwargs)
-    elif db_type == 'elasticsearch':
-        # Implement Elasticsearch version
-        raise NotImplementedError("Elasticsearch version of save_workflow_chat_to_db not yet implemented")
-
-#
-# End of Workflow Functions
-############################################################################################################
-
-# Dead code FIXME
-# def close_connection():
-#     if db_type == 'sqlite':
-#         db.get_connection().close()
-
-#
-# End of file
-############################################################################################################

App_Function_Libraries/DB/RAG_QA_Chat_DB.py

@@ -1,461 +0,0 @@
-# RAG_QA_Chat_DB.py
-# Description: This file contains the database operations for the RAG QA Chat + Notes system.
-#
-# Imports
-import logging
-import re
-import sqlite3
-from contextlib import contextmanager
-from datetime import datetime
-#
-# External Imports
-#
-# Local Imports
-#
-########################################################################################################################
-#
-# Functions:
-
-# Set up logging
-logging.basicConfig(level=logging.INFO)
-logger = logging.getLogger(__name__)
-
-# Database schema
-SCHEMA_SQL = '''
--- Table for storing chat messages
-CREATE TABLE IF NOT EXISTS rag_qa_chats (
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
-    conversation_id TEXT NOT NULL,
-    timestamp DATETIME NOT NULL,
-    role TEXT NOT NULL,
-    content TEXT NOT NULL
-);
-
--- Table for storing conversation metadata
-CREATE TABLE IF NOT EXISTS conversation_metadata (
-    conversation_id TEXT PRIMARY KEY,
-    created_at DATETIME NOT NULL,
-    last_updated DATETIME NOT NULL,
-    title TEXT NOT NULL
-);
-
--- Table for storing keywords
-CREATE TABLE IF NOT EXISTS rag_qa_keywords (
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
-    keyword TEXT NOT NULL UNIQUE
-);
-
--- Table for linking keywords to conversations
-CREATE TABLE IF NOT EXISTS rag_qa_conversation_keywords (
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
-    conversation_id TEXT NOT NULL,
-    keyword_id INTEGER NOT NULL,
-    FOREIGN KEY (conversation_id) REFERENCES conversation_metadata(conversation_id),
-    FOREIGN KEY (keyword_id) REFERENCES rag_qa_keywords(id)
-);
-
--- Table for storing keyword collections
-CREATE TABLE IF NOT EXISTS rag_qa_keyword_collections (
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
-    name TEXT NOT NULL UNIQUE,
-    parent_id INTEGER,
-    FOREIGN KEY (parent_id) REFERENCES rag_qa_keyword_collections(id)
-);
-
--- Table for linking keywords to collections
-CREATE TABLE IF NOT EXISTS rag_qa_collection_keywords (
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
-    collection_id INTEGER NOT NULL,
-    keyword_id INTEGER NOT NULL,
-    FOREIGN KEY (collection_id) REFERENCES rag_qa_keyword_collections(id),
-    FOREIGN KEY (keyword_id) REFERENCES rag_qa_keywords(id)
-);
-
--- Table for storing notes
-CREATE TABLE IF NOT EXISTS rag_qa_notes (
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
-    conversation_id TEXT NOT NULL,
-    content TEXT NOT NULL,
-    timestamp DATETIME NOT NULL,
-    FOREIGN KEY (conversation_id) REFERENCES conversation_metadata(conversation_id)
-);
-
--- Table for linking notes to keywords
-CREATE TABLE IF NOT EXISTS rag_qa_note_keywords (
-    id INTEGER PRIMARY KEY AUTOINCREMENT,
-    note_id INTEGER NOT NULL,
-    keyword_id INTEGER NOT NULL,
-    FOREIGN KEY (note_id) REFERENCES rag_qa_notes(id),
-    FOREIGN KEY (keyword_id) REFERENCES rag_qa_keywords(id)
-);
-
--- Indexes for improved query performance
-CREATE INDEX IF NOT EXISTS idx_rag_qa_chats_conversation_id ON rag_qa_chats(conversation_id);
-CREATE INDEX IF NOT EXISTS idx_rag_qa_chats_timestamp ON rag_qa_chats(timestamp);
-CREATE INDEX IF NOT EXISTS idx_rag_qa_keywords_keyword ON rag_qa_keywords(keyword);
-CREATE INDEX IF NOT EXISTS idx_rag_qa_conversation_keywords_conversation_id ON rag_qa_conversation_keywords(conversation_id);
-CREATE INDEX IF NOT EXISTS idx_rag_qa_conversation_keywords_keyword_id ON rag_qa_conversation_keywords(keyword_id);
-CREATE INDEX IF NOT EXISTS idx_rag_qa_keyword_collections_parent_id ON rag_qa_keyword_collections(parent_id);
-CREATE INDEX IF NOT EXISTS idx_rag_qa_collection_keywords_collection_id ON rag_qa_collection_keywords(collection_id);
-CREATE INDEX IF NOT EXISTS idx_rag_qa_collection_keywords_keyword_id ON rag_qa_collection_keywords(keyword_id);
-
--- Full-text search virtual table for chat content
-CREATE VIRTUAL TABLE IF NOT EXISTS rag_qa_chats_fts USING fts5(conversation_id, timestamp, role, content);
-
--- Trigger to keep the FTS table up to date
-CREATE TRIGGER IF NOT EXISTS rag_qa_chats_ai AFTER INSERT ON rag_qa_chats BEGIN
-  INSERT INTO rag_qa_chats_fts(conversation_id, timestamp, role, content) VALUES (new.conversation_id, new.timestamp, new.role, new.content);
-END;
-'''
-
-# Database connection management
-@contextmanager
-def get_db_connection():
-    conn = sqlite3.connect('rag_qa_chat.db')
-    try:
-        yield conn
-    finally:
-        conn.close()
-
-@contextmanager
-def transaction():
-    with get_db_connection() as conn:
-        try:
-            conn.execute('BEGIN TRANSACTION')
-            yield conn
-            conn.commit()
-        except Exception:
-            conn.rollback()
-            raise
-
-def execute_query(query, params=None, transaction_conn=None):
-    if transaction_conn:
-        cursor = transaction_conn.cursor()
-        if params:
-            cursor.execute(query, params)
-        else:
-            cursor.execute(query)
-        return cursor.fetchall()
-    else:
-        with get_db_connection() as conn:
-            cursor = conn.cursor()
-            if params:
-                cursor.execute(query, params)
-            else:
-                cursor.execute(query)
-            conn.commit()
-            return cursor.fetchall()
-
-def create_tables():
-    with get_db_connection() as conn:
-        conn.executescript(SCHEMA_SQL)
-    logger.info("All RAG QA Chat tables created successfully")
-
-# Initialize the database
-create_tables()
-
-# Input validation
-def validate_keyword(keyword):
-    if not isinstance(keyword, str):
-        raise ValueError("Keyword must be a string")
-    if not keyword.strip():
-        raise ValueError("Keyword cannot be empty or just whitespace")
-    if len(keyword) > 100:
-        raise ValueError("Keyword is too long (max 100 characters)")
-    if not re.match(r'^[a-zA-Z0-9\s\-_]+$', keyword):
-        raise ValueError("Keyword contains invalid characters")
-    return keyword.strip()
-
-def validate_collection_name(name):
-    if not isinstance(name, str):
-        raise ValueError("Collection name must be a string")
-    if not name.strip():
-        raise ValueError("Collection name cannot be empty or just whitespace")
-    if len(name) > 100:
-        raise ValueError("Collection name is too long (max 100 characters)")
-    if not re.match(r'^[a-zA-Z0-9\s\-_]+$', name):
-        raise ValueError("Collection name contains invalid characters")
-    return name.strip()
-
-# Core functions
-def add_keyword(keyword):
-    try:
-        validated_keyword = validate_keyword(keyword)
-        query = "INSERT OR IGNORE INTO rag_qa_keywords (keyword) VALUES (?)"
-        execute_query(query, (validated_keyword,))
-        logger.info(f"Keyword '{validated_keyword}' added successfully")
-    except ValueError as e:
-        logger.error(f"Invalid keyword: {e}")
-        raise
-    except Exception as e:
-        logger.error(f"Error adding keyword '{keyword}': {e}")
-        raise
-
-def create_keyword_collection(name, parent_id=None):
-    try:
-        validated_name = validate_collection_name(name)
-        query = "INSERT INTO rag_qa_keyword_collections (name, parent_id) VALUES (?, ?)"
-        execute_query(query, (validated_name, parent_id))
-        logger.info(f"Keyword collection '{validated_name}' created successfully")
-    except ValueError as e:
-        logger.error(f"Invalid collection name: {e}")
-        raise
-    except Exception as e:
-        logger.error(f"Error creating keyword collection '{name}': {e}")
-        raise
-
-def add_keyword_to_collection(collection_name, keyword):
-    try:
-        validated_collection_name = validate_collection_name(collection_name)
-        validated_keyword = validate_keyword(keyword)
-
-        with transaction() as conn:
-            add_keyword(validated_keyword)
-
-            query = '''
-            INSERT INTO rag_qa_collection_keywords (collection_id, keyword_id)
-            SELECT c.id, k.id
-            FROM rag_qa_keyword_collections c, rag_qa_keywords k
-            WHERE c.name = ? AND k.keyword = ?
-            '''
-            execute_query(query, (validated_collection_name, validated_keyword), conn)
-
-        logger.info(f"Keyword '{validated_keyword}' added to collection '{validated_collection_name}' successfully")
-    except ValueError as e:
-        logger.error(f"Invalid input: {e}")
-        raise
-    except Exception as e:
-        logger.error(f"Error adding keyword '{keyword}' to collection '{collection_name}': {e}")
-        raise
-
-def add_keywords_to_conversation(conversation_id, keywords):
-    if not isinstance(keywords, (list, tuple)):
-        raise ValueError("Keywords must be a list or tuple")
-    try:
-        with transaction() as conn:
-            for keyword in keywords:
-                validated_keyword = validate_keyword(keyword)
-
-                query = "INSERT OR IGNORE INTO rag_qa_keywords (keyword) VALUES (?)"
-                execute_query(query, (validated_keyword,), conn)
-
-                query = '''
-                INSERT INTO rag_qa_conversation_keywords (conversation_id, keyword_id)
-                SELECT ?, id FROM rag_qa_keywords WHERE keyword = ?
-                '''
-                execute_query(query, (conversation_id, validated_keyword), conn)
-
-        logger.info(f"Keywords added to conversation '{conversation_id}' successfully")
-    except ValueError as e:
-        logger.error(f"Invalid keyword: {e}")
-        raise
-    except Exception as e:
-        logger.error(f"Error adding keywords to conversation '{conversation_id}': {e}")
-        raise
-
-def get_keywords_for_conversation(conversation_id):
-    try:
-        query = '''
-        SELECT k.keyword
-        FROM rag_qa_keywords k
-        JOIN rag_qa_conversation_keywords ck ON k.id = ck.keyword_id
-        WHERE ck.conversation_id = ?
-        '''
-        result = execute_query(query, (conversation_id,))
-        keywords = [row[0] for row in result]
-        logger.info(f"Retrieved {len(keywords)} keywords for conversation '{conversation_id}'")
-        return keywords
-    except Exception as e:
-        logger.error(f"Error getting keywords for conversation '{conversation_id}': {e}")
-        raise
-
-def get_keywords_for_collection(collection_name):
-    try:
-        query = '''
-        SELECT k.keyword
-        FROM rag_qa_keywords k
-        JOIN rag_qa_collection_keywords ck ON k.id = ck.keyword_id
-        JOIN rag_qa_keyword_collections c ON ck.collection_id = c.id
-        WHERE c.name = ?
-        '''
-        result = execute_query(query, (collection_name,))
-        keywords = [row[0] for row in result]
-        logger.info(f"Retrieved {len(keywords)} keywords for collection '{collection_name}'")
-        return keywords
-    except Exception as e:
-        logger.error(f"Error getting keywords for collection '{collection_name}': {e}")
-        raise
-
-def save_notes(conversation_id, content):
-    """Save notes to the database."""
-    try:
-        query = "INSERT INTO rag_qa_notes (conversation_id, content, timestamp) VALUES (?, ?, ?)"
-        timestamp = datetime.now().isoformat()
-        execute_query(query, (conversation_id, content, timestamp))
-        logger.info(f"Notes saved for conversation '{conversation_id}'")
-    except Exception as e:
-        logger.error(f"Error saving notes for conversation '{conversation_id}': {e}")
-        raise
-
-def get_notes(conversation_id):
-    """Retrieve notes for a given conversation."""
-    try:
-        query = "SELECT content FROM rag_qa_notes WHERE conversation_id = ?"
-        result = execute_query(query, (conversation_id,))
-        notes = [row[0] for row in result]
-        logger.info(f"Retrieved {len(notes)} notes for conversation '{conversation_id}'")
-        return notes
-    except Exception as e:
-        logger.error(f"Error getting notes for conversation '{conversation_id}': {e}")
-        raise
-
-def clear_notes(conversation_id):
-    """Clear all notes for a given conversation."""
-    try:
-        query = "DELETE FROM rag_qa_notes WHERE conversation_id = ?"
-        execute_query(query, (conversation_id,))
-        logger.info(f"Cleared notes for conversation '{conversation_id}'")
-    except Exception as e:
-        logger.error(f"Error clearing notes for conversation '{conversation_id}': {e}")
-        raise
-
-def add_keywords_to_note(note_id, keywords):
-    """Associate keywords with a note."""
-    try:
-        with transaction() as conn:
-            for keyword in keywords:
-                validated_keyword = validate_keyword(keyword)
-
-                # Insert the keyword into the rag_qa_keywords table if it doesn't exist
-                query = "INSERT OR IGNORE INTO rag_qa_keywords (keyword) VALUES (?)"
-                execute_query(query, (validated_keyword,), conn)
-
-                # Retrieve the keyword ID
-                query = "SELECT id FROM rag_qa_keywords WHERE keyword = ?"
-                keyword_id = execute_query(query, (validated_keyword,), conn)[0][0]
-
-                # Link the note and keyword
-                query = "INSERT INTO rag_qa_note_keywords (note_id, keyword_id) VALUES (?, ?)"
-                execute_query(query, (note_id, keyword_id), conn)
-
-        logger.info(f"Keywords added to note ID '{note_id}' successfully")
-    except Exception as e:
-        logger.error(f"Error adding keywords to note ID '{note_id}': {e}")
-        raise
-
-def get_keywords_for_note(note_id):
-    """Retrieve keywords associated with a given note."""
-    try:
-        query = '''
-        SELECT k.keyword
-        FROM rag_qa_keywords k
-        JOIN rag_qa_note_keywords nk ON k.id = nk.keyword_id
-        WHERE nk.note_id = ?
-        '''
-        result = execute_query(query, (note_id,))
-        keywords = [row[0] for row in result]
-        logger.info(f"Retrieved {len(keywords)} keywords for note ID '{note_id}'")
-        return keywords
-    except Exception as e:
-        logger.error(f"Error getting keywords for note ID '{note_id}': {e}")
-        raise
-
-def clear_keywords_from_note(note_id):
-    """Clear all keywords from a given note."""
-    try:
-        query = "DELETE FROM rag_qa_note_keywords WHERE note_id = ?"
-        execute_query(query, (note_id,))
-        logger.info(f"Cleared keywords for note ID '{note_id}'")
-    except Exception as e:
-        logger.error(f"Error clearing keywords for note ID '{note_id}': {e}")
-        raise
-
-def save_message(conversation_id, role, content):
-    try:
-        query = "INSERT INTO rag_qa_chats (conversation_id, timestamp, role, content) VALUES (?, ?, ?, ?)"
-        timestamp = datetime.now().isoformat()
-        execute_query(query, (conversation_id, timestamp, role, content))
-        logger.info(f"Message saved for conversation '{conversation_id}'")
-    except Exception as e:
-        logger.error(f"Error saving message for conversation '{conversation_id}': {e}")
-        raise
-
-def start_new_conversation(title="Untitled Conversation"):
-    try:
-        conversation_id = datetime.now().isoformat()
-        query = "INSERT INTO conversation_metadata (conversation_id, created_at, last_updated, title) VALUES (?, ?, ?, ?)"
-        now = datetime.now()
-        execute_query(query, (conversation_id, now, now, title))
-        logger.info(f"New conversation '{conversation_id}' started with title '{title}'")
-        return conversation_id
-    except Exception as e:
-        logger.error(f"Error starting new conversation: {e}")
-        raise
-
-# Pagination helper function
-def get_paginated_results(query, params=None, page=1, page_size=20):
-    try:
-        offset = (page - 1) * page_size
-        paginated_query = f"{query} LIMIT ? OFFSET ?"
-        if params:
-            params = tuple(params) + (page_size, offset)
-        else:
-            params = (page_size, offset)
-
-        result = execute_query(paginated_query, params)
-
-        count_query = f"SELECT COUNT(*) FROM ({query})"
-        total_count = execute_query(count_query, params[:-2] if params else None)[0][0]
-
-        total_pages = (total_count + page_size - 1) // page_size
-
-        logger.info(f"Retrieved page {page} of {total_pages} (total items: {total_count})")
-        return result, total_pages, total_count
-    except Exception as e:
-        logger.error(f"Error retrieving paginated results: {e}")
-        raise
-
-def get_all_collections(page=1, page_size=20):
-    try:
-        query = "SELECT name FROM rag_qa_keyword_collections"
-        results, total_pages, total_count = get_paginated_results(query, page=page, page_size=page_size)
-        collections = [row[0] for row in results]
-        logger.info(f"Retrieved {len(collections)} keyword collections (page {page} of {total_pages})")
-        return collections, total_pages, total_count
-    except Exception as e:
-        logger.error(f"Error getting collections: {e}")
-        raise
-
-def search_conversations_by_keywords(keywords, page=1, page_size=20):
-    try:
-        placeholders = ','.join(['?' for _ in keywords])
-        query = f'''
-        SELECT DISTINCT cm.conversation_id, cm.title
-        FROM conversation_metadata cm
-        JOIN rag_qa_conversation_keywords ck ON cm.conversation_id = ck.conversation_id
-        JOIN rag_qa_keywords k ON ck.keyword_id = k.id
-        WHERE k.keyword IN ({placeholders})
-        '''
-        results, total_pages, total_count = get_paginated_results(query, keywords, page, page_size)
-        logger.info(
-            f"Found {total_count} conversations matching keywords: {', '.join(keywords)} (page {page} of {total_pages})")
-        return results, total_pages, total_count
-    except Exception as e:
-        logger.error(f"Error searching conversations by keywords {keywords}: {e}")
-        raise
-
-def load_chat_history(conversation_id, page=1, page_size=50):
-    try:
-        query = "SELECT role, content FROM rag_qa_chats WHERE conversation_id = ? ORDER BY timestamp"
-        results, total_pages, total_count = get_paginated_results(query, (conversation_id,), page, page_size)
-        history = [(msg[1] if msg[0] == 'human' else None, msg[1] if msg[0] == 'ai' else None) for msg in results]
-        logger.info(
-            f"Loaded {len(history)} messages for conversation '{conversation_id}' (page {page} of {total_pages})")
-        return history, total_pages, total_count
-    except Exception as e:
-        logger.error(f"Error loading chat history for conversation '{conversation_id}': {e}")
-        raise
-
-#
-# End of RAG_QA_Chat_DB.py
-####################################################################################################

App_Function_Libraries/DB/Test_SQLite_DB.py

@@ -1,202 +0,0 @@
-# Test_SQLite_DB.py
-# Description:  Test file for SQLite_DB.py
-#
-# Usage: python -m unittest test_sqlite_db.py
-#
-# Imports
-import unittest
-import sqlite3
-import threading
-import time
-from unittest.mock import patch
-#
-# Local Imports
-from App_Function_Libraries.DB.SQLite_DB import Database, add_media_with_keywords, add_media_version, DatabaseError
-#
-#######################################################################################################################
-#
-# Functions:
-
-class TestDatabase(unittest.TestCase):
-    def setUp(self):
-        self.db = Database(':memory:')  # Use in-memory database for testing
-
-    def test_connection_management(self):
-        with self.db.get_connection() as conn:
-            self.assertIsInstance(conn, sqlite3.Connection)
-        self.assertEqual(len(self.db.pool), 1)
-
-    def test_execute_query(self):
-        self.db.execute_query("CREATE TABLE test (id INTEGER PRIMARY KEY, name TEXT)")
-        self.db.execute_query("INSERT INTO test (name) VALUES (?)", ("test_name",))
-        with self.db.get_connection() as conn:
-            cursor = conn.cursor()
-            cursor.execute("SELECT name FROM test")
-            result = cursor.fetchone()
-        self.assertEqual(result[0], "test_name")
-
-    def test_execute_many(self):
-        self.db.execute_query("CREATE TABLE test (id INTEGER PRIMARY KEY, name TEXT)")
-        data = [("name1",), ("name2",), ("name3",)]
-        self.db.execute_many("INSERT INTO test (name) VALUES (?)", data)
-        with self.db.get_connection() as conn:
-            cursor = conn.cursor()
-            cursor.execute("SELECT COUNT(*) FROM test")
-            count = cursor.fetchone()[0]
-        self.assertEqual(count, 3)
-
-    def test_connection_retry(self):
-        def lock_database():
-            with self.db.get_connection() as conn:
-                cursor = conn.cursor()
-                cursor.execute("BEGIN EXCLUSIVE TRANSACTION")
-                time.sleep(2)  # Hold the lock for 2 seconds
-
-        thread = threading.Thread(target=lock_database)
-        thread.start()
-        time.sleep(0.1)  # Give the thread time to acquire the lock
-
-        with self.assertRaises(DatabaseError):
-            self.db.execute_query("SELECT 1")  # This should retry and eventually fail
-
-        thread.join()
-
-class TestAddMediaWithKeywords(unittest.TestCase):
-    def setUp(self):
-        self.db = Database(':memory:')
-        self.db.execute_query("""
-            CREATE TABLE Media (
-                id INTEGER PRIMARY KEY,
-                url TEXT,
-                title TEXT NOT NULL,
-                type TEXT NOT NULL,
-                content TEXT,
-                author TEXT,
-                ingestion_date TEXT,
-                transcription_model TEXT
-            )
-        """)
-        self.db.execute_query("CREATE TABLE Keywords (id INTEGER PRIMARY KEY, keyword TEXT NOT NULL UNIQUE)")
-        self.db.execute_query("""
-            CREATE TABLE MediaKeywords (
-                id INTEGER PRIMARY KEY,
-                media_id INTEGER NOT NULL,
-                keyword_id INTEGER NOT NULL,
-                FOREIGN KEY (media_id) REFERENCES Media(id),
-                FOREIGN KEY (keyword_id) REFERENCES Keywords(id)
-            )
-        """)
-        self.db.execute_query("""
-            CREATE TABLE MediaModifications (
-                id INTEGER PRIMARY KEY,
-                media_id INTEGER NOT NULL,
-                prompt TEXT,
-                summary TEXT,
-                modification_date TEXT,
-                FOREIGN KEY (media_id) REFERENCES Media(id)
-            )
-        """)
-        self.db.execute_query("""
-            CREATE TABLE MediaVersion (
-                id INTEGER PRIMARY KEY,
-                media_id INTEGER NOT NULL,
-                version INTEGER NOT NULL,
-                prompt TEXT,
-                summary TEXT,
-                created_at TEXT NOT NULL,
-                FOREIGN KEY (media_id) REFERENCES Media(id)
-            )
-        """)
-        self.db.execute_query("CREATE VIRTUAL TABLE media_fts USING fts5(title, content)")
-
-    @patch('App_Function_Libraries.DB.SQLite_DB.db', new_callable=lambda: Database(':memory:'))
-    def test_add_new_media(self, mock_db):
-        mock_db.get_connection = self.db.get_connection
-        result = add_media_with_keywords(
-            url="http://example.com",
-            title="Test Title",
-            media_type="article",
-            content="Test content",
-            keywords="test,keyword",
-            prompt="Test prompt",
-            summary="Test summary",
-            transcription_model="Test model",
-            author="Test Author",
-            ingestion_date="2023-01-01"
-        )
-        self.assertIn("added/updated successfully", result)
-
-        with self.db.get_connection() as conn:
-            cursor = conn.cursor()
-            cursor.execute("SELECT COUNT(*) FROM Media")
-            self.assertEqual(cursor.fetchone()[0], 1)
-
-            cursor.execute("SELECT COUNT(*) FROM Keywords")
-            self.assertEqual(cursor.fetchone()[0], 2)
-
-            cursor.execute("SELECT COUNT(*) FROM MediaKeywords")
-            self.assertEqual(cursor.fetchone()[0], 2)
-
-            cursor.execute("SELECT COUNT(*) FROM MediaModifications")
-            self.assertEqual(cursor.fetchone()[0], 1)
-
-            cursor.execute("SELECT COUNT(*) FROM MediaVersion")
-            self.assertEqual(cursor.fetchone()[0], 1)
-
-    @patch('App_Function_Libraries.DB.SQLite_DB.db', new_callable=lambda: Database(':memory:'))
-    def test_update_existing_media(self, mock_db):
-        mock_db.get_connection = self.db.get_connection
-        add_media_with_keywords(
-            url="http://example.com",
-            title="Test Title",
-            media_type="article",
-            content="Test content",
-            keywords="test,keyword",
-            prompt="Test prompt",
-            summary="Test summary",
-            transcription_model="Test model",
-            author="Test Author",
-            ingestion_date="2023-01-01"
-        )
-
-        result = add_media_with_keywords(
-            url="http://example.com",
-            title="Updated Title",
-            media_type="article",
-            content="Updated content",
-            keywords="test,new",
-            prompt="Updated prompt",
-            summary="Updated summary",
-            transcription_model="Updated model",
-            author="Updated Author",
-            ingestion_date="2023-01-02"
-        )
-
-        self.assertIn("added/updated successfully", result)
-
-        with self.db.get_connection() as conn:
-            cursor = conn.cursor()
-            cursor.execute("SELECT COUNT(*) FROM Media")
-            self.assertEqual(cursor.fetchone()[0], 1)
-
-            cursor.execute("SELECT title FROM Media")
-            self.assertEqual(cursor.fetchone()[0], "Updated Title")
-
-            cursor.execute("SELECT COUNT(*) FROM Keywords")
-            self.assertEqual(cursor.fetchone()[0], 3)
-
-            cursor.execute("SELECT COUNT(*) FROM MediaKeywords")
-            self.assertEqual(cursor.fetchone()[0], 3)
-
-            cursor.execute("SELECT COUNT(*) FROM MediaModifications")
-            self.assertEqual(cursor.fetchone()[0], 2)
-
-            cursor.execute("SELECT COUNT(*) FROM MediaVersion")
-            self.assertEqual(cursor.fetchone()[0], 2)
-
-if __name__ == '__main__':
-    unittest.main()
-
-#
-# End of File
-#######################################################################################################################

App_Function_Libraries/Databases/media_summary.db

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:74b972252f8784e4c8446ea921a231f333f76a703575be185d813bc60a6b035d
-size 13217792

App_Function_Libraries/Gradio_Related.py

@@ -1,421 +0,0 @@
-# Gradio_Related.py
-#########################################
-# Gradio UI Functions Library
-# I fucking hate Gradio.
-#
-#########################################
-#
-# Built-In Imports
-import logging
-import os
-import webbrowser
-
-#
-# Import 3rd-Party Libraries
-import gradio as gr
-#
-# Local Imports
-from App_Function_Libraries.DB.DB_Manager import get_db_config
-from App_Function_Libraries.Gradio_UI.Arxiv_tab import create_arxiv_tab
-from App_Function_Libraries.Gradio_UI.Audio_ingestion_tab import create_audio_processing_tab
-from App_Function_Libraries.Gradio_UI.Book_Ingestion_tab import create_import_book_tab
-from App_Function_Libraries.Gradio_UI.Character_Chat_tab import create_character_card_interaction_tab, create_character_chat_mgmt_tab, create_custom_character_card_tab, \
-    create_character_card_validation_tab, create_export_characters_tab
-from App_Function_Libraries.Gradio_UI.Character_interaction_tab import create_narrator_controlled_conversation_tab, \
-    create_multiple_character_chat_tab
-from App_Function_Libraries.Gradio_UI.Chat_ui import create_chat_management_tab, \
-    create_chat_interface_four, create_chat_interface_multi_api, create_chat_interface_stacked, create_chat_interface
-from App_Function_Libraries.Gradio_UI.Config_tab import create_config_editor_tab
-from App_Function_Libraries.Gradio_UI.Explain_summarize_tab import create_summarize_explain_tab
-from App_Function_Libraries.Gradio_UI.Export_Functionality import create_export_tab
-from App_Function_Libraries.Gradio_UI.Backup_Functionality import create_backup_tab, create_view_backups_tab, \
-    create_restore_backup_tab
-from App_Function_Libraries.Gradio_UI.Import_Functionality import create_import_single_prompt_tab, \
-    create_import_obsidian_vault_tab, create_import_item_tab, create_import_multiple_prompts_tab
-from App_Function_Libraries.Gradio_UI.Introduction_tab import create_introduction_tab
-from App_Function_Libraries.Gradio_UI.Keywords import create_view_keywords_tab, create_add_keyword_tab, \
-    create_delete_keyword_tab, create_export_keywords_tab
-from App_Function_Libraries.Gradio_UI.Live_Recording import create_live_recording_tab
-from App_Function_Libraries.Gradio_UI.Llamafile_tab import create_chat_with_llamafile_tab
-#from App_Function_Libraries.Gradio_UI.MMLU_Pro_tab import create_mmlu_pro_tab
-from App_Function_Libraries.Gradio_UI.Media_edit import create_prompt_clone_tab, create_prompt_edit_tab, \
-    create_media_edit_and_clone_tab, create_media_edit_tab
-from App_Function_Libraries.Gradio_UI.Media_wiki_tab import create_mediawiki_import_tab, create_mediawiki_config_tab
-from App_Function_Libraries.Gradio_UI.PDF_ingestion_tab import create_pdf_ingestion_tab, create_pdf_ingestion_test_tab
-from App_Function_Libraries.Gradio_UI.Plaintext_tab_import import create_plain_text_import_tab
-from App_Function_Libraries.Gradio_UI.Podcast_tab import create_podcast_tab
-from App_Function_Libraries.Gradio_UI.Prompt_Suggestion_tab import create_prompt_suggestion_tab
-from App_Function_Libraries.Gradio_UI.RAG_QA_Chat_tab import create_rag_qa_chat_tab
-from App_Function_Libraries.Gradio_UI.Re_summarize_tab import create_resummary_tab
-from App_Function_Libraries.Gradio_UI.Search_Tab import create_prompt_search_tab, \
-    create_search_summaries_tab, create_search_tab
-from App_Function_Libraries.Gradio_UI.RAG_Chat_tab import create_rag_tab
-from App_Function_Libraries.Gradio_UI.Embeddings_tab import create_embeddings_tab, create_view_embeddings_tab, \
-    create_purge_embeddings_tab
-from App_Function_Libraries.Gradio_UI.Trash import create_view_trash_tab, create_empty_trash_tab, \
-    create_delete_trash_tab, create_search_and_mark_trash_tab
-from App_Function_Libraries.Gradio_UI.Utilities import create_utilities_yt_timestamp_tab, create_utilities_yt_audio_tab, \
-    create_utilities_yt_video_tab
-from App_Function_Libraries.Gradio_UI.Video_transcription_tab import create_video_transcription_tab
-from App_Function_Libraries.Gradio_UI.View_tab import create_manage_items_tab
-from App_Function_Libraries.Gradio_UI.Website_scraping_tab import create_website_scraping_tab
-from App_Function_Libraries.Gradio_UI.Chat_Workflows import chat_workflows_tab
-from App_Function_Libraries.Gradio_UI.View_DB_Items_tab import create_prompt_view_tab, \
-    create_view_all_with_versions_tab, create_viewing_tab
-#
-# Gradio UI Imports
-from App_Function_Libraries.Gradio_UI.Evaluations_Benchmarks_tab import create_geval_tab, create_infinite_bench_tab
-#from App_Function_Libraries.Local_LLM.Local_LLM_huggingface import create_huggingface_tab
-from App_Function_Libraries.Gradio_UI.RAG_QA_Chat_Notes import create_rag_qa_chat_notes_tab
-
-#
-#######################################################################################################################
-# Function Definitions
-#
-
-
-# Disable Gradio Analytics
-os.environ['GRADIO_ANALYTICS_ENABLED'] = 'False'
-
-
-custom_prompt_input = None
-server_mode = False
-share_public = False
-custom_prompt_summarize_bulleted_notes = ("""
-                    <s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
-                        **Bulleted Note Creation Guidelines**
-
-                        **Headings**:
-                        - Based on referenced topics, not categories like quotes or terms
-                        - Surrounded by **bold** formatting 
-                        - Not listed as bullet points
-                        - No space between headings and list items underneath
-
-                        **Emphasis**:
-                        - **Important terms** set in bold font
-                        - **Text ending in a colon**: also bolded
-
-                        **Review**:
-                        - Ensure adherence to specified format
-                        - Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]
-                    """)
-#
-# End of globals
-#######################################################################################################################
-#
-# Start of Video/Audio Transcription and Summarization Functions
-#
-# Functions:
-# FIXME
-#
-#
-################################################################################################################
-# Functions for Re-Summarization
-#
-# Functions:
-# FIXME
-# End of Re-Summarization Functions
-#
-############################################################################################################################################################################################################################
-#
-# Explain/Summarize This Tab
-#
-# Functions:
-# FIXME
-#
-#
-############################################################################################################################################################################################################################
-#
-# Transcript Comparison Tab
-#
-# Functions:
-# FIXME
-#
-#
-###########################################################################################################################################################################################################################
-#
-# Search Tab
-#
-# Functions:
-# FIXME
-#
-# End of Search Tab Functions
-#
-##############################################################################################################################################################################################################################
-#
-# Llamafile Tab
-#
-# Functions:
-# FIXME
-#
-# End of Llamafile Tab Functions
-##############################################################################################################################################################################################################################
-#
-# Chat Interface Tab Functions
-#
-# Functions:
-# FIXME
-#
-#
-# End of Chat Interface Tab Functions
-################################################################################################################################################################################################################################
-#
-# Media Edit Tab Functions
-# Functions:
-# Fixme
-# create_media_edit_tab():
-##### Trash Tab
-# FIXME
-# Functions:
-#
-# End of Media Edit Tab Functions
-################################################################################################################
-#
-# Import Items Tab Functions
-#
-# Functions:
-#FIXME
-# End of Import Items Tab Functions
-################################################################################################################
-#
-# Export Items Tab Functions
-#
-# Functions:
-# FIXME
-#
-#
-# End of Export Items Tab Functions
-################################################################################################################
-#
-# Keyword Management Tab Functions
-#
-# Functions:
-#  create_view_keywords_tab():
-# FIXME
-#
-# End of Keyword Management Tab Functions
-################################################################################################################
-#
-# Document Editing Tab Functions
-#
-# Functions:
-#   #FIXME
-#
-#
-################################################################################################################
-#
-# Utilities Tab Functions
-# Functions:
-#   create_utilities_yt_video_tab():
-# #FIXME
-
-#
-# End of Utilities Tab Functions
-################################################################################################################
-
-# FIXME - Prompt sample box
-#
-# # Sample data
-# prompts_category_1 = [
-#     "What are the key points discussed in the video?",
-#     "Summarize the main arguments made by the speaker.",
-#     "Describe the conclusions of the study presented."
-# ]
-#
-# prompts_category_2 = [
-#     "How does the proposed solution address the problem?",
-#     "What are the implications of the findings?",
-#     "Can you explain the theory behind the observed phenomenon?"
-# ]
-#
-# all_prompts2 = prompts_category_1 + prompts_category_2
-
-
-def launch_ui(share_public=None, server_mode=False):
-    webbrowser.open_new_tab('http://127.0.0.1:7860/?__theme=dark')
-    share=share_public
-    css = """
-    .result-box {
-        margin-bottom: 20px;
-        border: 1px solid #ddd;
-        padding: 10px;
-    }
-    .result-box.error {
-        border-color: #ff0000;
-        background-color: #ffeeee;
-    }
-    .transcription, .summary {
-        max-height: 800px;
-        overflow-y: auto;
-        border: 1px solid #eee;
-        padding: 10px;
-        margin-top: 10px;
-    }
-    """
-
-    with gr.Blocks(theme='bethecloud/storj_theme',css=css) as iface:
-        gr.HTML(
-            """
-            <script>
-            document.addEventListener('DOMContentLoaded', (event) => {
-                document.body.classList.add('dark');
-                document.querySelector('gradio-app').style.backgroundColor = 'var(--color-background-primary)';
-            });
-            </script>
-            """
-        )
-        db_config = get_db_config()
-        db_type = db_config['type']
-        gr.Markdown(f"# tl/dw: Your LLM-powered Research Multi-tool")
-        gr.Markdown(f"(Using {db_type.capitalize()} Database)")
-        with gr.Tabs():
-            with gr.TabItem("Transcription / Summarization / Ingestion", id="ingestion-grouping", visible=True):
-                with gr.Tabs():
-                    create_video_transcription_tab()
-                    create_audio_processing_tab()
-                    create_podcast_tab()
-                    create_import_book_tab()
-                    create_plain_text_import_tab()
-                    create_website_scraping_tab()
-                    create_pdf_ingestion_tab()
-                    create_pdf_ingestion_test_tab()
-                    create_resummary_tab()
-                    create_summarize_explain_tab()
-                    create_live_recording_tab()
-                    create_arxiv_tab()
-
-            with gr.TabItem("Text Search", id="text search", visible=True):
-                create_search_tab()
-                create_search_summaries_tab()
-
-            with gr.TabItem("RAG Chat+Notes", id="RAG Chat Notes group", visible=True):
-                create_rag_qa_chat_notes_tab()
-
-            with gr.TabItem("RAG Search", id="RAG Search group", visible=True):
-                create_rag_tab()
-                create_rag_qa_chat_tab()
-
-            with gr.TabItem("Chat with an LLM", id="LLM Chat group", visible=True):
-                create_chat_interface()
-                create_chat_interface_stacked()
-                create_chat_interface_multi_api()
-                create_chat_interface_four()
-                create_chat_with_llamafile_tab()
-                create_chat_management_tab()
-                chat_workflows_tab()
-
-
-            with gr.TabItem("Character Chat", id="character chat group", visible=True):
-                create_character_card_interaction_tab()
-                create_character_chat_mgmt_tab()
-                create_custom_character_card_tab()
-                create_character_card_validation_tab()
-                create_multiple_character_chat_tab()
-                create_narrator_controlled_conversation_tab()
-                create_export_characters_tab()
-
-
-            with gr.TabItem("View DB Items", id="view db items group", visible=True):
-                # This one works
-                create_view_all_with_versions_tab()
-                # This one is WIP
-                create_viewing_tab()
-                create_prompt_view_tab()
-
-
-            with gr.TabItem("Prompts", id='view prompts group', visible=True):
-                create_prompt_view_tab()
-                create_prompt_search_tab()
-                create_prompt_edit_tab()
-                create_prompt_clone_tab()
-                create_prompt_suggestion_tab()
-
-
-            with gr.TabItem("Manage / Edit Existing Items", id="manage group", visible=True):
-                create_media_edit_tab()
-                create_manage_items_tab()
-                create_media_edit_and_clone_tab()
-                # FIXME
-                #create_compare_transcripts_tab()
-
-
-            with gr.TabItem("Embeddings Management", id="embeddings group", visible=True):
-                create_embeddings_tab()
-                create_view_embeddings_tab()
-                create_purge_embeddings_tab()
-
-            with gr.TabItem("Writing Tools", id="writing_tools group", visible=True):
-                from App_Function_Libraries.Gradio_UI.Writing_tab import create_document_feedback_tab
-                create_document_feedback_tab()
-                from App_Function_Libraries.Gradio_UI.Writing_tab import create_grammar_style_check_tab
-                create_grammar_style_check_tab()
-                from App_Function_Libraries.Gradio_UI.Writing_tab import create_tone_adjustment_tab
-                create_tone_adjustment_tab()
-                from App_Function_Libraries.Gradio_UI.Writing_tab import create_creative_writing_tab
-                create_creative_writing_tab()
-                from App_Function_Libraries.Gradio_UI.Writing_tab import create_mikupad_tab
-                create_mikupad_tab()
-
-
-            with gr.TabItem("Keywords", id="keywords group", visible=True):
-                create_view_keywords_tab()
-                create_add_keyword_tab()
-                create_delete_keyword_tab()
-                create_export_keywords_tab()
-
-            with gr.TabItem("Import", id="import group", visible=True):
-                create_import_item_tab()
-                create_import_obsidian_vault_tab()
-                create_import_single_prompt_tab()
-                create_import_multiple_prompts_tab()
-                create_mediawiki_import_tab()
-                create_mediawiki_config_tab()
-
-            with gr.TabItem("Export", id="export group", visible=True):
-                create_export_tab()
-
-            with gr.TabItem("Backup Management", id="backup group", visible=True):
-                create_backup_tab()
-                create_view_backups_tab()
-                create_restore_backup_tab()
-
-            with gr.TabItem("Utilities", id="util group", visible=True):
-                create_utilities_yt_video_tab()
-                create_utilities_yt_audio_tab()
-                create_utilities_yt_timestamp_tab()
-
-            with gr.TabItem("Local LLM", id="local llm group", visible=True):
-                create_chat_with_llamafile_tab()
-                #create_ollama_tab()
-                #create_huggingface_tab()
-
-            with gr.TabItem("Trashcan", id="trashcan group", visible=True):
-                create_search_and_mark_trash_tab()
-                create_view_trash_tab()
-                create_delete_trash_tab()
-                create_empty_trash_tab()
-
-            with gr.TabItem("Evaluations", id="eval", visible=True):
-                create_geval_tab()
-                create_infinite_bench_tab()
-                # FIXME
-                #create_mmlu_pro_tab()
-
-            with gr.TabItem("Introduction/Help", id="introduction group", visible=True):
-                create_introduction_tab()
-
-            with gr.TabItem("Config Editor", id="config group"):
-                create_config_editor_tab()
-
-    # Launch the interface
-    server_port_variable = 7860
-    os.environ['GRADIO_ANALYTICS_ENABLED'] = 'False'
-    if share==True:
-        iface.launch(share=True)
-    elif server_mode and not share_public:
-        iface.launch(share=False, server_name="0.0.0.0", server_port=server_port_variable, )
-    else:
-        try:
-            iface.launch(share=False, server_name="0.0.0.0", server_port=server_port_variable, )
-        except Exception as e:
-            logging.error(f"Error launching interface: {str(e)}")

App_Function_Libraries/Gradio_UI/Arxiv_tab.py

@@ -1,230 +0,0 @@
-# Arxiv_tab.py
-# Description: This file contains the Gradio UI for searching, browsing, and ingesting arXiv papers.
-#
-# Imports
-import tempfile
-from datetime import datetime
-import requests
-
-from App_Function_Libraries.PDF.PDF_Ingestion_Lib import extract_text_and_format_from_pdf
-#
-# Local Imports
-from App_Function_Libraries.Third_Party.Arxiv import convert_xml_to_markdown, fetch_arxiv_xml, parse_arxiv_feed, \
-    build_query_url, ARXIV_PAGE_SIZE, fetch_arxiv_pdf_url
-from App_Function_Libraries.DB.DB_Manager import add_media_with_keywords
-#
-import gradio as gr
-#
-#####################################################################################################
-#
-# Functions:
-
-def create_arxiv_tab():
-    with gr.TabItem("Arxiv Search & Ingest", visible=True):
-        gr.Markdown("# arXiv Search, Browse, Download, and Ingest")
-        gr.Markdown("#### Thank you to arXiv for use of its open access interoperability.")
-        with gr.Row():
-            with gr.Column(scale=1):
-                # Search Inputs
-                with gr.Row():
-                    with gr.Column():
-                        search_query = gr.Textbox(label="Search Query", placeholder="e.g., machine learning")
-                        author_filter = gr.Textbox(label="Author", placeholder="e.g., John Doe")
-                        year_filter = gr.Number(label="Year", precision=0)
-                        search_button = gr.Button("Search")
-
-            with gr.Column(scale=2):
-                # Pagination Controls
-                    paper_selector = gr.Radio(label="Select a Paper", choices=[], interactive=True)
-                    prev_button = gr.Button("Previous Page")
-                    next_button = gr.Button("Next Page")
-                    page_info = gr.Textbox(label="Page", value="1", interactive=False)
-
-        # Ingestion Section
-        with gr.Row():
-            with gr.Column():
-                # Paper Details View
-                paper_view = gr.Markdown(label="Paper Details")
-                arxiv_keywords = gr.Textbox(label="Additional Keywords (comma-separated)",
-                                            placeholder="e.g., AI, Deep Learning")
-                ingest_button = gr.Button("Ingest Selected Paper")
-                ingest_result = gr.Textbox(label="Ingestion Result", interactive=False)
-
-        # Define States for Pagination and Selection
-        state = gr.State(value={"start": 0, "current_page": 1, "last_query": None, "entries": []})
-        selected_paper_id = gr.State(value=None)
-
-        def search_arxiv(query, author, year):
-            start = 0
-            url = build_query_url(query, author, year, start)
-            try:
-                response = requests.get(url)
-                response.raise_for_status()
-            except requests.exceptions.RequestException as e:
-                return gr.update(value=[]), gr.update(value=f"**Error:** {str(e)}"), state.value
-
-            entries = parse_arxiv_feed(response.text)
-            state.value = {"start": start, "current_page": 1, "last_query": (query, author, year), "entries": entries}
-            if not entries:
-                return gr.update(value=[]), "No results found.", state.value
-
-            # Update the dropdown with paper titles for selection
-            titles = [entry['title'] for entry in entries]
-            return gr.update(choices=titles), "1", state.value
-
-        # Dead code? FIXME
-        def handle_pagination(direction):
-            current_state = state.value
-            query, author, year = current_state["last_query"]
-            new_page = current_state["current_page"] + direction
-            if new_page < 1:
-                new_page = 1
-            start = (new_page - 1) * ARXIV_PAGE_SIZE
-            url = build_query_url(query, author, year, start)
-            try:
-                response = requests.get(url)
-                response.raise_for_status()
-            except requests.exceptions.RequestException as e:
-                return gr.update(), gr.update()
-
-            entries = parse_arxiv_feed(response.text)
-            if entries:
-                current_state["start"] = start
-                current_state["current_page"] = new_page
-                current_state["entries"] = entries
-                state.value = current_state
-
-                # Update the dropdown with paper titles for the new page
-                titles = [entry['title'] for entry in entries]
-                return gr.update(choices=titles), str(new_page)
-            else:
-                # If no entries, do not change the page
-                return gr.update(), gr.update()
-
-        def load_selected_paper(selected_title):
-            if not selected_title:
-                return "Please select a paper to view."
-
-            # Find the selected paper from state
-            for entry in state.value["entries"]:
-                if entry['title'] == selected_title:
-                    paper_id = entry['id']
-                    break
-            else:
-                return "Paper not found."
-
-            try:
-                # Fetch the PDF URL and download the full-text
-                pdf_url = fetch_arxiv_pdf_url(paper_id)
-                response = requests.get(pdf_url)
-                response.raise_for_status()
-
-                # Save the PDF temporarily
-                with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as temp_pdf:
-                    temp_pdf.write(response.content)
-                    temp_pdf_path = temp_pdf.name
-
-                # Convert PDF to markdown using your PDF ingestion function
-                full_text_markdown = extract_text_and_format_from_pdf(temp_pdf_path)
-
-                selected_paper_id.value = paper_id
-                return full_text_markdown
-            except Exception as e:
-                return f"Error loading full paper: {str(e)}"
-
-        def process_and_ingest_arxiv_paper(paper_id, additional_keywords):
-            try:
-                if not paper_id:
-                    return "Please select a paper to ingest."
-
-                # Fetch the PDF URL
-                pdf_url = fetch_arxiv_pdf_url(paper_id)
-
-                # Download the PDF
-                response = requests.get(pdf_url)
-                response.raise_for_status()
-
-                # Save the PDF temporarily
-                with tempfile.NamedTemporaryFile(delete=False, suffix=".pdf") as temp_pdf:
-                    temp_pdf.write(response.content)
-                    temp_pdf_path = temp_pdf.name
-
-                # Convert PDF to markdown using your PDF ingestion function
-                markdown_text = extract_text_and_format_from_pdf(temp_pdf_path)
-
-                # Fetch metadata from arXiv to get title, authors, and categories
-                xml_content = fetch_arxiv_xml(paper_id)
-                _, title, authors, categories = convert_xml_to_markdown(xml_content)
-
-                # Prepare the arXiv paper URL for access/download
-                paper_url = f"https://arxiv.org/abs/{paper_id}"
-
-                # Prepare the keywords for ingestion
-                keywords = f"arxiv,{','.join(categories)}"
-                if additional_keywords:
-                    keywords += f",{additional_keywords}"
-
-                # Ingest full paper markdown content
-                add_media_with_keywords(
-                    url=paper_url,
-                    title=title,
-                    media_type='document',
-                    content=markdown_text,  # Full paper content in markdown
-                    keywords=keywords,
-                    prompt='No prompt for arXiv papers',
-                    summary='Full arXiv paper ingested from PDF',
-                    transcription_model='None',
-                    author=', '.join(authors),
-                    ingestion_date=datetime.now().strftime('%Y-%m-%d')
-                )
-
-                # Return success message with paper title and authors
-                return f"arXiv paper '{title}' by {', '.join(authors)} ingested successfully."
-            except Exception as e:
-                # Return error message if anything goes wrong
-                return f"Error processing arXiv paper: {str(e)}"
-
-        # Event Handlers
-        # Connect Search Button
-        search_button.click(
-            fn=search_arxiv,
-            inputs=[search_query, author_filter, year_filter],
-            outputs=[paper_selector, page_info, state],
-            queue=True
-        )
-
-        # Connect Next Button
-        next_button.click(
-            fn=lambda: handle_pagination(1),
-            inputs=None,
-            outputs=[paper_selector, page_info],
-            queue=True
-        )
-
-        # Connect Previous Button
-        prev_button.click(
-            fn=lambda: handle_pagination(-1),
-            inputs=None,
-            outputs=[paper_selector, page_info],
-            queue=True
-        )
-
-        # When the user selects a paper in the Dropdown
-        paper_selector.change(
-            fn=load_selected_paper,
-            inputs=paper_selector,
-            outputs=paper_view,
-            queue=True
-        )
-
-        # Connect Ingest Button
-        ingest_button.click(
-            fn=process_and_ingest_arxiv_paper,
-            inputs=[selected_paper_id, arxiv_keywords],
-            outputs=ingest_result,
-            queue=True
-        )
-
-#
-# End of File
-#####################################################################################################

App_Function_Libraries/Gradio_UI/Audio_ingestion_tab.py

@@ -1,167 +0,0 @@
-# Audio_ingestion_tab.py
-# Description: Gradio UI for ingesting audio files into the database
-#
-# Imports
-#
-# External Imports
-import gradio as gr
-#
-# Local Imports
-from App_Function_Libraries.Audio.Audio_Files import process_audio_files
-from App_Function_Libraries.DB.DB_Manager import load_preset_prompts
-from App_Function_Libraries.Gradio_UI.Chat_ui import update_user_prompt
-from App_Function_Libraries.Gradio_UI.Gradio_Shared import whisper_models
-from App_Function_Libraries.Utils.Utils import cleanup_temp_files
-# Import metrics logging
-from App_Function_Libraries.Metrics.metrics_logger import log_counter, log_histogram
-from App_Function_Libraries.Metrics.logger_config import logger
-#
-#######################################################################################################################
-# Functions:
-
-def create_audio_processing_tab():
-    with gr.TabItem("Audio File Transcription + Summarization", visible=True):
-        gr.Markdown("# Transcribe & Summarize Audio Files from URLs or Local Files!")
-        with gr.Row():
-            with gr.Column():
-                audio_url_input = gr.Textbox(label="Audio File URL(s)", placeholder="Enter the URL(s) of the audio file(s), one per line")
-                audio_file_input = gr.File(label="Upload Audio File", file_types=["audio/*"])
-                custom_title_input = gr.Textbox(label="Custom Title/Name", placeholder="Enter a custom title or name for the audio file")
-                use_cookies_input = gr.Checkbox(label="Use cookies for authenticated download", value=False)
-                cookies_input = gr.Textbox(
-                    label="Audio Download Cookies",
-                    placeholder="Paste your cookies here (JSON format)",
-                    lines=3,
-                    visible=False
-                )
-
-                use_cookies_input.change(
-                    fn=lambda x: gr.update(visible=x),
-                    inputs=[use_cookies_input],
-                    outputs=[cookies_input]
-                )
-
-                diarize_input = gr.Checkbox(label="Enable Speaker Diarization", value=False)
-                whisper_model_input = gr.Dropdown(choices=whisper_models, value="medium", label="Whisper Model")
-                keep_timestamps_input = gr.Checkbox(label="Keep Timestamps", value=True)
-
-                with gr.Row():
-                    custom_prompt_checkbox = gr.Checkbox(label="Use a Custom Prompt",
-                                                     value=False,
-                                                     visible=True)
-                    preset_prompt_checkbox = gr.Checkbox(label="Use a pre-set Prompt",
-                                                     value=False,
-                                                     visible=True)
-                with gr.Row():
-                    preset_prompt = gr.Dropdown(label="Select Preset Prompt",
-                                                choices=load_preset_prompts(),
-                                                visible=False)
-                with gr.Row():
-                    custom_prompt_input = gr.Textbox(label="Custom Prompt",
-                                                     placeholder="Enter custom prompt here",
-                                                     lines=3,
-                                                     visible=False)
-                with gr.Row():
-                    system_prompt_input = gr.Textbox(label="System Prompt",
-                                                     value="""<s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
-**Bulleted Note Creation Guidelines**
-
-**Headings**:
-- Based on referenced topics, not categories like quotes or terms
-- Surrounded by **bold** formatting 
-- Not listed as bullet points
-- No space between headings and list items underneath
-
-**Emphasis**:
-- **Important terms** set in bold font
-- **Text ending in a colon**: also bolded
-
-**Review**:
-- Ensure adherence to specified format
-- Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]
-""",
-                                                     lines=3,
-                                                     visible=False)
-
-                custom_prompt_checkbox.change(
-                    fn=lambda x: (gr.update(visible=x), gr.update(visible=x)),
-                    inputs=[custom_prompt_checkbox],
-                    outputs=[custom_prompt_input, system_prompt_input]
-                )
-                preset_prompt_checkbox.change(
-                    fn=lambda x: gr.update(visible=x),
-                    inputs=[preset_prompt_checkbox],
-                    outputs=[preset_prompt]
-                )
-
-                def update_prompts(preset_name):
-                    prompts = update_user_prompt(preset_name)
-                    return (
-                        gr.update(value=prompts["user_prompt"], visible=True),
-                        gr.update(value=prompts["system_prompt"], visible=True)
-                    )
-
-                preset_prompt.change(
-                    update_prompts,
-                    inputs=preset_prompt,
-                    outputs=[custom_prompt_input, system_prompt_input]
-                )
-
-                api_name_input = gr.Dropdown(
-                    choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral", "OpenRouter",
-                             "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM","ollama", "HuggingFace", "Custom-OpenAI-API"],
-                    value=None,
-                    label="API for Summarization (Optional)"
-                )
-                api_key_input = gr.Textbox(label="API Key (if required)", placeholder="Enter your API key here", type="password")
-                custom_keywords_input = gr.Textbox(label="Custom Keywords", placeholder="Enter custom keywords, comma-separated")
-                keep_original_input = gr.Checkbox(label="Keep original audio file", value=False)
-
-                chunking_options_checkbox = gr.Checkbox(label="Show Chunking Options", value=False)
-                with gr.Row(visible=False) as chunking_options_box:
-                    gr.Markdown("### Chunking Options")
-                    with gr.Column():
-                        chunk_method = gr.Dropdown(choices=['words', 'sentences', 'paragraphs', 'tokens'], label="Chunking Method")
-                        max_chunk_size = gr.Slider(minimum=100, maximum=1000, value=300, step=50, label="Max Chunk Size")
-                        chunk_overlap = gr.Slider(minimum=0, maximum=100, value=0, step=10, label="Chunk Overlap")
-                        use_adaptive_chunking = gr.Checkbox(label="Use Adaptive Chunking")
-                        use_multi_level_chunking = gr.Checkbox(label="Use Multi-level Chunking")
-                        chunk_language = gr.Dropdown(choices=['english', 'french', 'german', 'spanish'], label="Chunking Language")
-
-                chunking_options_checkbox.change(
-                    fn=lambda x: gr.update(visible=x),
-                    inputs=[chunking_options_checkbox],
-                    outputs=[chunking_options_box]
-                )
-
-                process_audio_button = gr.Button("Process Audio File(s)")
-
-            with gr.Column():
-                audio_progress_output = gr.Textbox(label="Progress")
-                audio_transcription_output = gr.Textbox(label="Transcription")
-                audio_summary_output = gr.Textbox(label="Summary")
-                download_transcription = gr.File(label="Download All Transcriptions as JSON")
-                download_summary = gr.File(label="Download All Summaries as Text")
-
-        process_audio_button.click(
-            fn=process_audio_files,
-            inputs=[audio_url_input, audio_file_input, whisper_model_input, api_name_input, api_key_input,
-                    use_cookies_input, cookies_input, keep_original_input, custom_keywords_input, custom_prompt_input,
-                    chunk_method, max_chunk_size, chunk_overlap, use_adaptive_chunking, use_multi_level_chunking,
-                    chunk_language, diarize_input, keep_timestamps_input, custom_title_input],
-            outputs=[audio_progress_output, audio_transcription_output, audio_summary_output]
-        )
-
-        def on_file_clear(file):
-            if file is None:
-                cleanup_temp_files()
-
-        audio_file_input.clear(
-            fn=on_file_clear,
-            inputs=[audio_file_input],
-            outputs=[]
-        )
-
-#
-# End of Audio_ingestion_tab.py
-#######################################################################################################################

App_Function_Libraries/Gradio_UI/Backup_Functionality.py

@@ -1,71 +0,0 @@
-# Backup_Functionality.py
-# Functionality for exporting items as markdown files
-#
-# Imports:
-import os
-import shutil
-import gradio as gr
-#
-# Local Imports:
-from App_Function_Libraries.DB.DB_Manager import create_automated_backup, db_path, backup_dir
-#
-# End of Imports
-#######################################################################################################################
-#
-# Functions:
-
-def create_backup():
-    backup_file = create_automated_backup(db_path, backup_dir)
-    return f"Backup created: {backup_file}"
-
-
-def list_backups():
-    backups = [f for f in os.listdir(backup_dir) if f.endswith('.db')]
-    return "\n".join(backups)
-
-
-def restore_backup(backup_name: str) -> str:
-    backup_path_location: str = os.path.join(str(backup_dir), backup_name)
-    if os.path.exists(backup_path_location):
-        shutil.copy2(str(backup_path_location), str(db_path))
-        return f"Database restored from {backup_name}"
-    else:
-        return "Backup file not found"
-
-
-def create_backup_tab():
-    with gr.Tab("Create Backup", visible=True):
-        gr.Markdown("# Create a backup of the database")
-        gr.Markdown("This will create a backup of the database in the backup directory(the default backup directory is `/tldw_DB_Backups/')")
-        with gr.Row():
-            with gr.Column():
-                create_button = gr.Button("Create Backup")
-                create_output = gr.Textbox(label="Result")
-            with gr.Column():
-                create_button.click(create_backup, inputs=[], outputs=create_output)
-
-
-def create_view_backups_tab():
-    with gr.TabItem("View Backups", visible=True):
-        gr.Markdown("# Browse available backups")
-        with gr.Row():
-            with gr.Column():
-                view_button = gr.Button("View Backups")
-            with gr.Column():
-                backup_list = gr.Textbox(label="Available Backups")
-                view_button.click(list_backups, inputs=[], outputs=backup_list)
-
-
-def create_restore_backup_tab():
-    with gr.TabItem("Restore Backup", visible=True):
-        gr.Markdown("# Restore a backup of the database")
-        with gr.Column():
-            backup_input = gr.Textbox(label="Backup Filename")
-            restore_button = gr.Button("Restore")
-        with gr.Column():
-            restore_output = gr.Textbox(label="Result")
-            restore_button.click(restore_backup, inputs=[backup_input], outputs=restore_output)
-
-#
-# End of Functions
-#######################################################################################################################

App_Function_Libraries/Gradio_UI/Book_Ingestion_tab.py

@@ -1,100 +0,0 @@
-# Book_Ingestion_tab.py
-# Functionality to import epubs/ebooks into the system.
-####################
-# Function List
-#
-# 1. create_import_book_tab()
-# 2. import_epub(epub_file, title, author, keywords, system_prompt, user_prompt, auto_summarize, api_name, api_key)
-#
-####################
-# Imports
-#
-# External Imports
-import gradio as gr
-#
-# Local Imports
-from App_Function_Libraries.Books.Book_Ingestion_Lib import process_zip_file, import_epub, import_file_handler
-#
-########################################################################################################################
-#
-# Functions:
-
-
-
-def create_import_book_tab():
-    with gr.TabItem("Ebook(epub) Files", visible=True):
-        with gr.Row():
-            with gr.Column():
-                gr.Markdown("# Import .epub files")
-                gr.Markdown("Upload a single .epub file or a .zip file containing multiple .epub files")
-                gr.Markdown(
-                    "🔗 **How to remove DRM from your ebooks:** [Reddit Guide](https://www.reddit.com/r/Calibre/comments/1ck4w8e/2024_guide_on_removing_drm_from_kobo_kindle_ebooks/)")
-                import_file = gr.File(label="Upload file for import", file_types=[".epub", ".zip"])
-                title_input = gr.Textbox(label="Title", placeholder="Enter the title of the content (for single files)")
-                author_input = gr.Textbox(label="Author", placeholder="Enter the author's name (for single files)")
-                keywords_input = gr.Textbox(label="Keywords (like genre or publish year)",
-                                            placeholder="Enter keywords, comma-separated")
-                system_prompt_input = gr.Textbox(label="System Prompt", lines=3,
-                                                 value=""""
-                                                    <s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
-                                                    **Bulleted Note Creation Guidelines**
-
-                                                    **Headings**:
-                                                    - Based on referenced topics, not categories like quotes or terms
-                                                    - Surrounded by **bold** formatting 
-                                                    - Not listed as bullet points
-                                                    - No space between headings and list items underneath
-
-                                                    **Emphasis**:
-                                                    - **Important terms** set in bold font
-                                                    - **Text ending in a colon**: also bolded
-
-                                                    **Review**:
-                                                    - Ensure adherence to specified format
-                                                    - Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]
-                                                """, )
-                custom_prompt_input = gr.Textbox(label="Custom User Prompt",
-                                                 placeholder="Enter a custom user prompt for summarization (optional)")
-                auto_summarize_checkbox = gr.Checkbox(label="Auto-summarize", value=False)
-                api_name_input = gr.Dropdown(
-                    choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral",
-                             "OpenRouter", "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace"],
-                    label="API for Auto-summarization"
-                )
-                api_key_input = gr.Textbox(label="API Key", type="password")
-
-                # Chunking options
-                max_chunk_size = gr.Slider(minimum=100, maximum=2000, value=500, step=50, label="Max Chunk Size")
-                chunk_overlap = gr.Slider(minimum=0, maximum=500, value=200, step=10, label="Chunk Overlap")
-                custom_chapter_pattern = gr.Textbox(label="Custom Chapter Pattern (optional)",
-                                                    placeholder="Enter a custom regex pattern for chapter detection")
-
-
-                import_button = gr.Button("Import eBook(s)")
-            with gr.Column():
-                with gr.Row():
-                    import_output = gr.Textbox(label="Import Status", lines=10, interactive=False)
-
-        import_button.click(
-            fn=import_file_handler,
-            inputs=[
-                import_file,
-                title_input,
-                author_input,
-                keywords_input,
-                custom_prompt_input,
-                auto_summarize_checkbox,
-                api_name_input,
-                api_key_input,
-                max_chunk_size,
-                chunk_overlap,
-                custom_chapter_pattern
-            ],
-            outputs=import_output
-        )
-
-    return import_file, title_input, author_input, keywords_input, system_prompt_input, custom_prompt_input, auto_summarize_checkbox, api_name_input, api_key_input, import_button, import_output
-
-#
-# End of File
-########################################################################################################################

App_Function_Libraries/Gradio_UI/Character_Chat_tab.py

@@ -1,1848 +0,0 @@
-# Character_Interaction_Library.py
-# Description: Library for character card import functions
-#
-# Imports
-import os
-import re
-import tempfile
-import uuid
-from datetime import datetime
-import json
-import logging
-import io
-import base64
-from typing import Dict, Any, Optional, List, Tuple, Union, cast
-import zipfile
-#
-# External Imports
-from PIL import Image
-from PIL.PngImagePlugin import PngInfo
-import gradio as gr
-#
-# Local Imports
-from App_Function_Libraries.Character_Chat.Character_Chat_Lib import validate_character_book, validate_v2_card, \
-    replace_placeholders, replace_user_placeholder, extract_json_from_image, parse_character_book, \
-    load_chat_and_character, load_chat_history, load_character_and_image, extract_character_id, load_character_wrapper
-from App_Function_Libraries.Chat import chat
-from App_Function_Libraries.DB.Character_Chat_DB import (
-    add_character_card,
-    get_character_cards,
-    get_character_card_by_id,
-    add_character_chat,
-    get_character_chats,
-    get_character_chat_by_id,
-    update_character_chat,
-    delete_character_chat,
-    delete_character_card,
-    update_character_card, search_character_chats,
-)
-from App_Function_Libraries.Utils.Utils import sanitize_user_input
-#
-############################################################################################################
-#
-# Functions:
-
-#################################################################################
-#
-# Character card import functions:
-
-def import_character_card(file):
-    if file is None:
-        return None, gr.update(), "No file provided for character card import"
-
-    try:
-        if file.name.lower().endswith(('.png', '.webp')):
-            json_data = extract_json_from_image(file)
-            if not json_data:
-                return None, gr.update(), "No character card data found in the image. This might not be a valid character card image."
-        elif file.name.lower().endswith('.json'):
-            with open(file.name, 'r', encoding='utf-8') as f:
-                json_data = f.read()
-        else:
-            return None, gr.update(), "Unsupported file type. Please upload a PNG/WebP image or a JSON file."
-
-        card_data = import_character_card_json(json_data)
-        if not card_data:
-            return None, gr.update(), "Failed to parse character card data. The file might not contain valid character information."
-
-        # Save image data for PNG/WebP files
-        if file.name.lower().endswith(('.png', '.webp')):
-            with Image.open(file) as img:
-                img_byte_arr = io.BytesIO()
-                img.save(img_byte_arr, format='PNG')
-                card_data['image'] = base64.b64encode(img_byte_arr.getvalue()).decode('utf-8')
-
-        # Save character card to database
-        character_id = add_character_card(card_data)
-        if character_id:
-            characters = get_character_cards()
-            character_names = [char['name'] for char in characters]
-            return card_data, gr.update(
-                choices=character_names), f"Character card '{card_data['name']}' imported successfully."
-        else:
-            return None, gr.update(), f"Failed to save character card '{card_data.get('name', 'Unknown')}'. It may already exist."
-    except Exception as e:
-        logging.error(f"Error importing character card: {e}")
-        return None, gr.update(), f"Error importing character card: {e}"
-
-
-def import_character_card_json(json_content: str) -> Optional[Dict[str, Any]]:
-    try:
-        json_content = json_content.strip()
-        card_data = json.loads(json_content)
-
-        if 'spec' in card_data and card_data['spec'] == 'chara_card_v2':
-            logging.info("Detected V2 character card")
-            return parse_v2_card(card_data)
-        else:
-            logging.info("Assuming V1 character card")
-            return parse_v1_card(card_data)
-    except json.JSONDecodeError as e:
-        logging.error(f"JSON decode error: {e}")
-    except Exception as e:
-        logging.error(f"Unexpected error parsing JSON: {e}")
-    return None
-
-
-
-def parse_v2_card(card_data: Dict[str, Any]) -> Optional[Dict[str, Any]]:
-    try:
-        # Validate spec_version
-        if card_data.get('spec_version') != '2.0':
-            logging.warning(f"Unsupported V2 spec version: {card_data.get('spec_version')}")
-            return None
-
-        data = card_data['data']
-
-        # Ensure all required fields are present
-        required_fields = ['name', 'description', 'personality', 'scenario', 'first_mes', 'mes_example']
-        for field in required_fields:
-            if field not in data:
-                logging.error(f"Missing required field in V2 card: {field}")
-                return None
-
-        # Handle new V2 fields
-        parsed_data = {
-            'name': data['name'],
-            'description': data['description'],
-            'personality': data['personality'],
-            'scenario': data['scenario'],
-            'first_mes': data['first_mes'],
-            'mes_example': data['mes_example'],
-            'creator_notes': data.get('creator_notes', ''),
-            'system_prompt': data.get('system_prompt', ''),
-            'post_history_instructions': data.get('post_history_instructions', ''),
-            'alternate_greetings': data.get('alternate_greetings', []),
-            'tags': data.get('tags', []),
-            'creator': data.get('creator', ''),
-            'character_version': data.get('character_version', ''),
-            'extensions': data.get('extensions', {})
-        }
-
-        # Handle character_book if present
-        if 'character_book' in data:
-            parsed_data['character_book'] = parse_character_book(data['character_book'])
-
-        return parsed_data
-    except KeyError as e:
-        logging.error(f"Missing key in V2 card structure: {e}")
-    except Exception as e:
-        logging.error(f"Error parsing V2 card: {e}")
-    return None
-
-def parse_v1_card(card_data: Dict[str, Any]) -> Dict[str, Any]:
-    # Ensure all required V1 fields are present
-    required_fields = ['name', 'description', 'personality', 'scenario', 'first_mes', 'mes_example']
-    for field in required_fields:
-        if field not in card_data:
-            logging.error(f"Missing required field in V1 card: {field}")
-            raise ValueError(f"Missing required field in V1 card: {field}")
-
-    # Convert V1 to V2 format
-    v2_data: Dict[str, Union[str, List[str], Dict[str, Any]]] = {
-        'name': card_data['name'],
-        'description': card_data['description'],
-        'personality': card_data['personality'],
-        'scenario': card_data['scenario'],
-        'first_mes': card_data['first_mes'],
-        'mes_example': card_data['mes_example'],
-        'creator_notes': cast(str, card_data.get('creator_notes', '')),
-        'system_prompt': cast(str, card_data.get('system_prompt', '')),
-        'post_history_instructions': cast(str, card_data.get('post_history_instructions', '')),
-        'alternate_greetings': cast(List[str], card_data.get('alternate_greetings', [])),
-        'tags': cast(List[str], card_data.get('tags', [])),
-        'creator': cast(str, card_data.get('creator', '')),
-        'character_version': cast(str, card_data.get('character_version', '')),
-        'extensions': {}
-    }
-
-    # Move any non-standard V1 fields to extensions
-    for key, value in card_data.items():
-        if key not in v2_data:
-            v2_data['extensions'][key] = value
-
-    return v2_data
-
-#
-# End of Character card import functions
-####################################################
-
-####################################################
-#
-# Character card export functions
-
-def export_character_as_json(character_id):
-    character = get_character_card_by_id(character_id)
-    if character:
-        # Remove the 'id' field from the character data
-        character_data = {k: v for k, v in character.items() if k != 'id'}
-
-        # Convert image to base64 if it exists
-        if 'image' in character_data and character_data['image']:
-            image_data = base64.b64decode(character_data['image'])
-            img = Image.open(io.BytesIO(image_data))
-            buffered = io.BytesIO()
-            img.save(buffered, format="PNG")
-            character_data['image'] = base64.b64encode(buffered.getvalue()).decode('utf-8')
-
-        json_data = json.dumps(character_data, indent=2)
-        return json_data
-    return None
-
-def export_all_characters_as_zip():
-    characters = get_character_cards()
-    with tempfile.NamedTemporaryFile(mode='wb', delete=False, suffix='.zip') as temp_zip:
-        with zipfile.ZipFile(temp_zip, 'w') as zf:
-            for character in characters:
-                character_data = {k: v for k, v in character.items() if k != 'id'}
-
-                # Convert image to base64 if it exists
-                if 'image' in character_data and character_data['image']:
-                    image_data = base64.b64decode(character_data['image'])
-                    img = Image.open(io.BytesIO(image_data))
-                    buffered = io.BytesIO()
-                    img.save(buffered, format="PNG")
-                    character_data['image'] = base64.b64encode(buffered.getvalue()).decode('utf-8')
-                json_data = json.dumps(character_data, indent=2)
-                zf.writestr(f"{character['name']}.json", json_data)
-        return temp_zip.name
-
-def export_single_character(character_selection):
-    if not character_selection:
-        return None, "No character selected."
-
-    character_id = int(character_selection.split('(ID: ')[1].rstrip(')'))
-    json_data = export_character_as_json(character_id)
-
-    if json_data:
-        with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.json', encoding='utf-8') as temp_file:
-            temp_file.write(json_data)
-            return temp_file.name, f"Character '{character_selection.split(' (ID:')[0]}' exported successfully."
-    else:
-        return None, f"Failed to export character '{character_selection.split(' (ID:')[0]}'."
-
-def export_all_characters():
-    zip_path = export_all_characters_as_zip()
-    return zip_path, "All characters exported successfully."
-
-#
-# End of Character card export functions
-####################################################
-
-####################################################
-#
-# Gradio tabs
-
-def create_character_card_interaction_tab():
-    with gr.TabItem("Chat with a Character Card", visible=True):
-        gr.Markdown("# Chat with a Character Card")
-        with gr.Row():
-            with gr.Column(scale=1):
-                character_image = gr.Image(label="Character Image", type="pil")
-                character_card_upload = gr.File(
-                    label="Upload Character Card (PNG, WEBP, JSON)",
-                    file_types=[".png", ".webp", ".json"]
-                )
-                import_card_button = gr.Button("Import Character Card")
-                load_characters_button = gr.Button("Load Existing Characters")
-                character_dropdown = gr.Dropdown(label="Select Character", choices=[])
-                user_name_input = gr.Textbox(label="Your Name", placeholder="Enter your name here")
-                api_name_input = gr.Dropdown(
-                    choices=[
-                        "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral",
-                        "OpenRouter", "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace",
-                        "Custom-OpenAI-API"
-                    ],
-                    value="HuggingFace",
-                    label="API for Interaction (Mandatory)"
-                )
-                api_key_input = gr.Textbox(
-                    label="API Key (if not set in Config_Files/config.txt)",
-                    placeholder="Enter your API key here", type="password"
-                )
-                temperature_slider = gr.Slider(
-                    minimum=0.0, maximum=2.0, value=0.7, step=0.05, label="Temperature"
-                )
-                import_chat_button = gr.Button("Import Chat History")
-                chat_file_upload = gr.File(label="Upload Chat History JSON", visible=True)
-
-                # Chat History Import and Search
-                gr.Markdown("## Search and Load Existing Chats")
-                chat_search_query = gr.Textbox(
-                    label="Search Chats",
-                    placeholder="Enter chat name or keywords to search"
-                )
-                chat_search_button = gr.Button("Search Chats")
-                chat_search_dropdown = gr.Dropdown(label="Search Results", choices=[], visible=False)
-                load_chat_button = gr.Button("Load Selected Chat", visible=False)
-
-                # Checkbox to Decide Whether to Save Chats by Default
-                auto_save_checkbox = gr.Checkbox(label="Save chats automatically", value=True)
-                chat_media_name = gr.Textbox(label="Custom Chat Name (optional)", visible=True)
-                save_chat_history_to_db = gr.Button("Save Chat History to Database")
-                save_status = gr.Textbox(label="Save Status", interactive=False)
-
-            with gr.Column(scale=2):
-                chat_history = gr.Chatbot(label="Conversation", height=800)
-                user_input = gr.Textbox(label="Your message")
-                send_message_button = gr.Button("Send Message")
-                answer_for_me_button = gr.Button("Answer for Me")
-                continue_talking_button = gr.Button("Continue Talking")
-                regenerate_button = gr.Button("Regenerate Last Message")
-                clear_chat_button = gr.Button("Clear Chat")
-                save_snapshot_button = gr.Button("Save Chat Snapshot")
-                update_chat_dropdown = gr.Dropdown(label="Select Chat to Update", choices=[], visible=False)
-                load_selected_chat_button = gr.Button("Load Selected Chat", visible=False)
-                update_chat_button = gr.Button("Update Selected Chat", visible=False)
-
-        # States
-        character_data = gr.State(None)
-        user_name = gr.State("")
-        selected_chat_id = gr.State(None)  # To track the selected chat for updates
-
-        # Callback Functions
-
-        def search_existing_chats(query):
-            results, message = search_character_chats(query)
-            if results:
-                # Format search results for dropdown
-                formatted_results = [
-                    f"{chat['conversation_name']} (ID: {chat['id']})" for chat in results
-                ]
-            else:
-                formatted_results = []
-            return formatted_results, message
-
-        def load_selected_chat_from_search(selected_chat, user_name):
-            if not selected_chat:
-                return None, [], None, "No chat selected."
-
-            try:
-                chat_id_match = re.search(r'\(ID:\s*(\d+)\)', selected_chat)
-                if not chat_id_match:
-                    return None, [], None, "Invalid chat selection format."
-
-                chat_id = int(chat_id_match.group(1))
-
-                # Use the new function to load chat and character data
-                char_data, chat_history, img = load_chat_and_character(chat_id, user_name)
-
-                if not char_data:
-                    return None, [], None, "Failed to load character data for the selected chat."
-
-                return char_data, chat_history, img, f"Chat '{selected_chat}' loaded successfully."
-            except Exception as e:
-                logging.error(f"Error loading selected chat: {e}")
-                return None, [], None, f"Error loading chat: {e}"
-
-
-        def import_chat_history(file, current_history, char_data, user_name_val):
-            """
-            Imports chat history from a file, replacing '{{user}}' with the actual user name.
-
-            Args:
-                file (file): The uploaded chat history file.
-                current_history (list): The current chat history.
-                char_data (dict): The current character data.
-                user_name_val (str): The user's name.
-
-            Returns:
-                tuple: Updated chat history, updated character data, and a status message.
-            """
-            loaded_history, char_name = load_chat_history(file)
-            if loaded_history is None:
-                return current_history, char_data, "Failed to load chat history."
-
-            # Replace '{{user}}' in the loaded chat history
-            loaded_history = replace_user_placeholder(loaded_history, user_name_val)
-
-            # Check if the loaded chat is for the current character
-            if char_data and char_data.get('name') != char_name:
-                return current_history, char_data, (
-                    f"Warning: Loaded chat is for character '{char_name}', "
-                    f"but current character is '{char_data.get('name')}'. Chat not imported."
-                )
-
-            # If no character is selected, try to load the character from the chat
-            if not char_data:
-                characters = get_character_cards()
-                character = next((char for char in characters if char['name'] == char_name), None)
-                if character:
-                    char_data = character
-                    # Replace '{{user}}' in the first_message if necessary
-                    if character.get('first_message'):
-                        character['first_message'] = character['first_message'].replace("{{user}}",
-                                                                                        user_name_val if user_name_val else "User")
-                else:
-                    return current_history, char_data, (
-                        f"Warning: Character '{char_name}' not found. Please select the character manually."
-                    )
-
-            return loaded_history, char_data, f"Chat history for '{char_name}' imported successfully."
-
-        def load_character(name):
-            characters = get_character_cards()
-            character = next((char for char in characters if char['name'] == name), None)
-            if character:
-                first_message = character.get('first_message', "Hello! I'm ready to chat.")
-                return character, [(None, first_message)] if first_message else [], None
-            return None, [], None
-
-        def load_character_image(name):
-            character = next((char for char in get_character_cards() if char['name'] == name), None)
-            if character and 'image' in character and character['image']:
-                try:
-                    # Decode the base64 image
-                    image_data = base64.b64decode(character['image'])
-                    # Load as PIL Image
-                    img = Image.open(io.BytesIO(image_data)).convert("RGBA")
-                    return img
-                except Exception as e:
-                    logging.error(f"Error loading image for character '{name}': {e}")
-                    return None
-            return None
-
-        def character_chat_wrapper(
-                message, history, char_data, api_endpoint, api_key,
-                temperature, user_name_val, auto_save
-        ):
-            if not char_data:
-                return history, "Please select a character first."
-
-            user_name_val = user_name_val or "User"
-            char_name = char_data.get('name', 'AI Assistant')
-
-            # Prepare the character's background information
-            char_background = f"""
-            Name: {char_name}
-            Description: {char_data.get('description', 'N/A')}
-            Personality: {char_data.get('personality', 'N/A')}
-            Scenario: {char_data.get('scenario', 'N/A')}
-            """
-
-            # Prepare the system prompt
-            system_message = f"""You are roleplaying as {char_name}. {char_data.get('system_prompt', '')}"""
-
-            # Prepare chat context
-            media_content = {
-                'id': char_name,
-                'title': char_name,
-                'content': char_background,
-                'description': char_data.get('description', ''),
-                'personality': char_data.get('personality', ''),
-                'scenario': char_data.get('scenario', '')
-            }
-            selected_parts = ['description', 'personality', 'scenario']
-
-            prompt = char_data.get('post_history_instructions', '')
-
-            # Sanitize and format user message
-            user_message = sanitize_user_input(message)
-            user_message = replace_placeholders(user_message, char_name, user_name_val)
-            full_message = f"{user_name_val}: {user_message}"
-
-            # Generate bot response
-            bot_message = chat(
-                full_message,
-                history,
-                media_content,
-                selected_parts,
-                api_endpoint,
-                api_key,
-                prompt,
-                temperature,
-                system_message
-            )
-
-            # Replace placeholders in bot message
-            bot_message = replace_placeholders(bot_message, char_name, user_name_val)
-
-            # Update history
-            history.append((user_message, bot_message))
-
-            # Auto-save if enabled
-            save_status = ""
-            if auto_save:
-                character_id = char_data.get('id')
-                if character_id:
-                    conversation_name = f"Auto-saved chat {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
-                    add_character_chat(character_id, conversation_name, history)
-                    save_status = "Chat auto-saved."
-                else:
-                    save_status = "Character ID not found; chat not saved."
-
-            return history, save_status
-
-        def save_chat_history_to_db_wrapper(
-            chat_history, conversation_id, media_content,
-            chat_media_name, char_data, auto_save
-        ):
-            if not char_data or not chat_history:
-                return "No character or chat history available.", ""
-
-            character_id = char_data.get('id')
-            if not character_id:
-                return "Character ID not found.", ""
-
-            conversation_name = chat_media_name or f"Chat {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
-            chat_id = add_character_chat(character_id, conversation_name, chat_history)
-            if chat_id:
-                return f"Chat saved successfully with ID {chat_id}.", ""
-            else:
-                return "Failed to save chat.", ""
-
-        def update_character_info(name):
-            return load_character_and_image(name, user_name.value)
-
-        def on_character_select(name, user_name_val):
-            logging.debug(f"Character selected: {name}")
-            char_data, chat_history, img = load_character_and_image(name, user_name_val)
-            return char_data, chat_history, img
-
-        def clear_chat_history(char_data, user_name_val):
-            """
-            Clears the chat history and initializes it with the character's first message,
-            replacing the '{{user}}' placeholder with the actual user name.
-
-            Args:
-                char_data (dict): The current character data.
-                user_name_val (str): The user's name.
-
-            Returns:
-                tuple: Updated chat history and the unchanged char_data.
-            """
-            if char_data and 'first_message' in char_data and char_data['first_message']:
-                # Replace '{{user}}' in the first_message
-                first_message = char_data['first_message'].replace("{{user}}",
-                                                                   user_name_val if user_name_val else "User")
-                # Initialize chat history with the updated first_message
-                return [(None, first_message)], char_data
-            else:
-                # If no first_message is defined, simply clear the chat
-                return [], char_data
-
-        def regenerate_last_message(
-                history, char_data, api_endpoint, api_key,
-                temperature, user_name_val, auto_save
-        ):
-            """
-            Regenerates the last bot message by removing it and resending the corresponding user message.
-
-            Args:
-                history (list): The current chat history as a list of tuples (user_message, bot_message).
-                char_data (dict): The current character data.
-                api_endpoint (str): The API endpoint to use for the LLM.
-                api_key (str): The API key for authentication.
-                temperature (float): The temperature setting for the LLM.
-                user_name_val (str): The user's name.
-                auto_save (bool): Flag indicating whether to auto-save the chat.
-
-            Returns:
-                tuple: Updated chat history and a save status message.
-            """
-            if not history:
-                return history, "No messages to regenerate."
-
-            last_entry = history[-1]
-            last_user_message, last_bot_message = last_entry
-
-            # Check if the last bot message exists
-            if last_bot_message is None:
-                return history, "The last message is not from the bot."
-
-            # Remove the last bot message
-            new_history = history[:-1]
-
-            # Resend the last user message to generate a new bot response
-            if not last_user_message:
-                return new_history, "No user message to regenerate the bot response."
-
-            # Prepare the character's background information
-            char_name = char_data.get('name', 'AI Assistant')
-            char_background = f"""
-            Name: {char_name}
-            Description: {char_data.get('description', 'N/A')}
-            Personality: {char_data.get('personality', 'N/A')}
-            Scenario: {char_data.get('scenario', 'N/A')}
-            """
-
-            # Prepare the system prompt for character impersonation
-            system_message = f"""You are roleplaying as {char_name}, the character described below. Respond to the user's messages in character, maintaining the personality and background provided. Do not break character or refer to yourself as an AI. Always refer to yourself as "{char_name}" and refer to the user as "{user_name_val}".
-
-            {char_background}
-
-            Additional instructions: {char_data.get('post_history_instructions', '')}
-            """
-
-            # Prepare media_content and selected_parts
-            media_content = {
-                'id': char_name,
-                'title': char_name,
-                'content': char_background,
-                'description': char_data.get('description', ''),
-                'personality': char_data.get('personality', ''),
-                'scenario': char_data.get('scenario', '')
-            }
-            selected_parts = ['description', 'personality', 'scenario']
-
-            prompt = char_data.get('post_history_instructions', '')
-
-            # Prepare the input for the chat function
-            full_message = f"{user_name_val}: {last_user_message}" if last_user_message else f"{user_name_val}: "
-
-            # Call the chat function to get a new bot message
-            bot_message = chat(
-                full_message,
-                new_history,
-                media_content,
-                selected_parts,
-                api_endpoint,
-                api_key,
-                prompt,
-                temperature,
-                system_message
-            )
-
-            # Append the new bot message to the history
-            new_history.append((last_user_message, bot_message))
-
-            # Auto-save if enabled
-            if auto_save:
-                character_id = char_data.get('id')
-                if character_id:
-                    conversation_name = f"Auto-saved chat {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
-                    add_character_chat(character_id, conversation_name, new_history)
-                    save_status = "Chat auto-saved."
-                else:
-                    save_status = "Character ID not found; chat not saved."
-            else:
-                save_status = ""
-
-            return new_history, save_status
-
-        def toggle_chat_file_upload():
-            return gr.update(visible=True)
-
-        def save_untracked_chat_action(history, char_data):
-            if not char_data or not history:
-                return "No chat to save or character not selected."
-
-            character_id = char_data.get('id')
-            if not character_id:
-                return "Character ID not found."
-
-            conversation_name = f"Snapshot {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
-            chat_id = add_character_chat(character_id, conversation_name, history, is_snapshot=True)
-            if chat_id:
-                return f"Chat snapshot saved successfully with ID {chat_id}."
-            else:
-                return "Failed to save chat snapshot."
-
-        def select_chat_for_update():
-            # Fetch all chats for the selected character
-            if character_data.value:
-                character_id = character_data.value.get('id')
-                if character_id:
-                    chats = get_character_chats(character_id)
-                    chat_choices = [
-                        f"{chat['conversation_name']} (ID: {chat['id']})" for chat in chats
-                    ]
-                    return gr.update(choices=chat_choices), None
-            return gr.update(choices=[]), "No character selected."
-
-        def load_selected_chat(chat_selection):
-            if not chat_selection:
-                return [], "No chat selected."
-
-            try:
-                chat_id = int(chat_selection.split('(ID: ')[1].rstrip(')'))
-                chat = get_character_chat_by_id(chat_id)
-                if chat:
-                    history = chat['chat_history']
-                    selected_chat_id.value = chat_id  # Update the selected_chat_id state
-                    return history, f"Loaded chat '{chat['conversation_name']}' successfully."
-                else:
-                    return [], "Chat not found."
-            except Exception as e:
-                logging.error(f"Error loading selected chat: {e}")
-                return [], f"Error loading chat: {e}"
-
-        def update_chat(chat_id, updated_history):
-            success = update_character_chat(chat_id, updated_history)
-            if success:
-                return "Chat updated successfully."
-            else:
-                return "Failed to update chat."
-
-        def continue_talking(
-                history, char_data, api_endpoint, api_key,
-                temperature, user_name_val, auto_save
-        ):
-            """
-            Causes the character to continue the conversation or think out loud.
-            """
-            if not char_data:
-                return history, "Please select a character first."
-
-            user_name_val = user_name_val or "User"
-            char_name = char_data.get('name', 'AI Assistant')
-
-            # Prepare the character's background information
-            char_background = f"""
-            Name: {char_name}
-            Description: {char_data.get('description', 'N/A')}
-            Personality: {char_data.get('personality', 'N/A')}
-            Scenario: {char_data.get('scenario', 'N/A')}
-            """
-
-            # Prepare the system prompt
-            system_message = f"""You are roleplaying as {char_name}. {char_data.get('system_prompt', '')}
-            If the user does not respond, continue expressing your thoughts or continue the conversation by thinking out loud. If thinking out loud, prefix the message with "Thinking: "."""
-
-            # Prepare chat context
-            media_content = {
-                'id': char_name,
-                'title': char_name,
-                'content': char_background,
-                'description': char_data.get('description', ''),
-                'personality': char_data.get('personality', ''),
-                'scenario': char_data.get('scenario', '')
-            }
-            selected_parts = ['description', 'personality', 'scenario']
-
-            prompt = char_data.get('post_history_instructions', '')
-
-            # Simulate empty user input
-            user_message = ""
-
-            # Generate bot response
-            bot_message = chat(
-                user_message,
-                history,
-                media_content,
-                selected_parts,
-                api_endpoint,
-                api_key,
-                prompt,
-                temperature,
-                system_message
-            )
-
-            # Replace placeholders in bot message
-            bot_message = replace_placeholders(bot_message, char_name, user_name_val)
-
-            # Update history
-            history.append((None, bot_message))
-
-            # Auto-save if enabled
-            save_status = ""
-            if auto_save:
-                character_id = char_data.get('id')
-                if character_id:
-                    conversation_name = f"Auto-saved chat {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
-                    add_character_chat(character_id, conversation_name, history)
-                    save_status = "Chat auto-saved."
-                else:
-                    save_status = "Character ID not found; chat not saved."
-
-            return history, save_status
-
-        def answer_for_me(
-                history, char_data, api_endpoint, api_key,
-                temperature, user_name_val, auto_save
-        ):
-            """
-            Generates a likely user response and continues the conversation.
-            """
-            if not char_data:
-                return history, "Please select a character first."
-
-            user_name_val = user_name_val or "User"
-            char_name = char_data.get('name', 'AI Assistant')
-
-            # Prepare the character's background information
-            char_background = f"""
-            Name: {char_name}
-            Description: {char_data.get('description', 'N/A')}
-            Personality: {char_data.get('personality', 'N/A')}
-            Scenario: {char_data.get('scenario', 'N/A')}
-            """
-
-            # Prepare system message for generating user's response
-            system_message_user = f"""You are simulating the user {user_name_val}. Based on the conversation so far, generate a natural and appropriate response that {user_name_val} might say next. The response should fit the context and flow of the conversation. ONLY SPEAK FOR {user_name_val}."""
-
-            # Prepare chat context
-            media_content = {
-                'id': char_name,
-                'title': char_name,
-                'content': char_background,
-                'description': char_data.get('description', ''),
-                'personality': char_data.get('personality', ''),
-                'scenario': char_data.get('scenario', '')
-            }
-            selected_parts = ['description', 'personality', 'scenario']
-
-            # Generate user response
-            user_response = chat(
-                "",  # No new message
-                history,
-                media_content,
-                selected_parts,
-                api_endpoint,
-                api_key,
-                prompt="",
-                temperature=temperature,
-                system_message=system_message_user
-            )
-
-            # Append the generated user response to history
-            history.append((user_response, None))
-
-            # Now generate the character's response to this user response
-            # Prepare the system message for the character
-            system_message_bot = f"""You are roleplaying as {char_name}. {char_data.get('system_prompt', '')}"""
-
-            bot_message = chat(
-                f"{user_name_val}: {user_response}",
-                history[:-1],
-                media_content,
-                selected_parts,
-                api_endpoint,
-                api_key,
-                prompt=char_data.get('post_history_instructions', ''),
-                temperature=temperature,
-                system_message=system_message_bot
-            )
-
-            # Replace placeholders in bot message
-            bot_message = replace_placeholders(bot_message, char_name, user_name_val)
-
-            # Update history with bot's response
-            history[-1] = (user_response, bot_message)
-
-            # Auto-save if enabled
-            save_status = ""
-            if auto_save:
-                character_id = char_data.get('id')
-                if character_id:
-                    conversation_name = f"Auto-saved chat {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
-                    add_character_chat(character_id, conversation_name, history)
-                    save_status = "Chat auto-saved."
-                else:
-                    save_status = "Character ID not found; chat not saved."
-
-            return history, save_status
-
-
-        # Define States for conversation_id and media_content, which are required for saving chat history
-        conversation_id = gr.State(str(uuid.uuid4()))
-        media_content = gr.State({})
-
-        # Button Callbacks
-
-        # Add the new button callbacks here
-        answer_for_me_button.click(
-            fn=answer_for_me,
-            inputs=[
-                chat_history,
-                character_data,
-                api_name_input,
-                api_key_input,
-                temperature_slider,
-                user_name_input,
-                auto_save_checkbox
-            ],
-            outputs=[chat_history, save_status]
-        )
-
-        continue_talking_button.click(
-            fn=continue_talking,
-            inputs=[
-                chat_history,
-                character_data,
-                api_name_input,
-                api_key_input,
-                temperature_slider,
-                user_name_input,
-                auto_save_checkbox
-            ],
-            outputs=[chat_history, save_status]
-        )
-
-        import_card_button.click(
-            fn=import_character_card,
-            inputs=[character_card_upload],
-            outputs=[character_data, character_dropdown, save_status]
-        )
-
-        load_characters_button.click(
-            fn=lambda: gr.update(choices=[f"{char['name']} (ID: {char['id']})" for char in get_character_cards()]),
-            outputs=character_dropdown
-        )
-
-        # FIXME user_name_val = validate_user_name(user_name_val)
-        clear_chat_button.click(
-            fn=clear_chat_history,
-            inputs=[character_data, user_name_input],
-            outputs=[chat_history, character_data]
-        )
-
-        character_dropdown.change(
-            fn=extract_character_id,
-            inputs=[character_dropdown],
-            outputs=character_data
-        ).then(
-            fn=load_character_wrapper,
-            inputs=[character_data, user_name_input],
-            outputs=[character_data, chat_history, character_image]
-        )
-
-        send_message_button.click(
-            fn=character_chat_wrapper,
-            inputs=[
-                user_input,
-                chat_history,
-                character_data,
-                api_name_input,
-                api_key_input,
-                temperature_slider,
-                user_name_input,
-                auto_save_checkbox
-            ],
-            outputs=[chat_history, save_status]
-        ).then(lambda: "", outputs=user_input)
-
-        regenerate_button.click(
-            fn=regenerate_last_message,
-            inputs=[
-                chat_history,
-                character_data,
-                api_name_input,
-                api_key_input,
-                temperature_slider,
-                user_name_input,
-                auto_save_checkbox
-            ],
-            outputs=[chat_history, save_status]
-        )
-
-        import_chat_button.click(
-            fn=lambda: gr.update(visible=True),
-            outputs=chat_file_upload
-        )
-
-        chat_file_upload.change(
-            fn=import_chat_history,
-            inputs=[chat_file_upload, chat_history, character_data],
-            outputs=[chat_history, character_data, save_status]
-        )
-
-        save_chat_history_to_db.click(
-            fn=save_chat_history_to_db_wrapper,
-            inputs=[
-                chat_history,
-                conversation_id,
-                media_content,
-                chat_media_name,
-                character_data,
-                auto_save_checkbox  # Pass the auto_save state
-            ],
-            outputs=[conversation_id, save_status]
-        )
-
-        # Populate the update_chat_dropdown based on selected character
-        character_dropdown.change(
-            fn=select_chat_for_update,
-            inputs=[],
-            outputs=[update_chat_dropdown, save_status]
-        )
-
-        load_selected_chat_button.click(
-            fn=load_selected_chat,
-            inputs=[update_chat_dropdown],
-            outputs=[chat_history, save_status]
-        )
-
-        save_snapshot_button.click(
-            fn=save_untracked_chat_action,
-            inputs=[chat_history, character_data],
-            outputs=save_status
-        )
-
-        update_chat_button.click(
-            fn=update_chat,
-            inputs=[selected_chat_id, chat_history],
-            outputs=save_status
-        )
-
-        # Search Chats
-        chat_search_button.click(
-            fn=search_existing_chats,
-            inputs=[chat_search_query],
-            outputs=[chat_search_dropdown, save_status]
-        ).then(
-            fn=lambda choices, msg: gr.update(choices=choices, visible=True) if choices else gr.update(visible=False),
-            inputs=[chat_search_dropdown, save_status],
-            outputs=[chat_search_dropdown]
-        )
-
-        # Load Selected Chat from Search
-        load_chat_button.click(
-            fn=load_selected_chat_from_search,
-            inputs=[chat_search_dropdown, user_name_input],
-            outputs=[character_data, chat_history, character_image, save_status]
-        )
-
-        # Show Load Chat Button when a chat is selected
-        chat_search_dropdown.change(
-            fn=lambda selected: gr.update(visible=True) if selected else gr.update(visible=False),
-            inputs=[chat_search_dropdown],
-            outputs=[load_chat_button]
-        )
-
-
-        return character_data, chat_history, user_input, user_name, character_image
-
-
-def create_character_chat_mgmt_tab():
-    with gr.TabItem("Character and Chat Management", visible=True):
-        gr.Markdown("# Character and Chat Management")
-
-        with gr.Row():
-            # Left Column: Character Import and Chat Management
-            with gr.Column(scale=1):
-                gr.Markdown("## Import Characters")
-                character_files = gr.File(
-                    label="Upload Character Files (PNG, WEBP, JSON)",
-                    file_types=[".png", ".webp", ".json"],
-                    file_count="multiple"
-                )
-                import_characters_button = gr.Button("Import Characters")
-                import_status = gr.Markdown("")
-
-            # Right Column: Character Selection and Image Display
-            with gr.Column(scale=2):
-                gr.Markdown("## Select Character")
-                characters = get_character_cards()
-                character_choices = [f"{char['name']} (ID: {char['id']})" for char in characters]
-                load_characters_button = gr.Button("Load Existing Characters")
-                select_character = gr.Dropdown(label="Select Character", choices=character_choices, interactive=True)
-                character_image = gr.Image(label="Character Image", type="pil", interactive=False)
-
-                gr.Markdown("## Search Conversations")
-                search_query = gr.Textbox(label="Search Conversations", placeholder="Enter search keywords")
-                search_button = gr.Button("Search")
-                search_results = gr.Dropdown(label="Search Results", choices=[], visible=False)
-                search_status = gr.Markdown("", visible=True)
-
-        with gr.Row():
-            gr.Markdown("## Chat Management")
-            select_chat = gr.Dropdown(label="Select Chat", choices=[], visible=False, interactive=True)
-            load_chat_button = gr.Button("Load Selected Chat", visible=False)
-            conversation_list = gr.Dropdown(label="Select Conversation or Character", choices=[])
-            conversation_mapping = gr.State({})
-
-        with gr.Tabs():
-            with gr.TabItem("Edit", visible=True):
-                chat_content = gr.TextArea(label="Chat/Character Content (JSON)", lines=20, max_lines=50)
-                save_button = gr.Button("Save Changes")
-                delete_button = gr.Button("Delete Conversation/Character", variant="stop")
-
-            with gr.TabItem("Preview", visible=True):
-                chat_preview = gr.HTML(label="Chat/Character Preview")
-        result_message = gr.Markdown("")
-
-        # Callback Functions
-
-        def load_character_image(character_selection):
-            if not character_selection:
-                return None
-
-            try:
-                character_id = int(character_selection.split('(ID: ')[1].rstrip(')'))
-                character = get_character_card_by_id(character_id)
-                if character and 'image' in character:
-                    image_data = base64.b64decode(character['image'])
-                    img = Image.open(io.BytesIO(image_data))
-                    return img
-            except Exception as e:
-                logging.error(f"Error loading character image: {e}")
-
-            return None
-
-        def search_conversations_or_characters(query, selected_character):
-            if not query.strip():
-                return gr.update(choices=[], visible=False), "Please enter a search query."
-
-            try:
-                # Extract character ID from the selected character
-                character_id = None
-                if selected_character:
-                    character_id = int(selected_character.split('(ID: ')[1].rstrip(')'))
-
-                # Search Chats using FTS5, filtered by character_id if provided
-                chat_results, chat_message = search_character_chats(query, character_id)
-
-                # Format chat results
-                formatted_chat_results = [
-                    f"Chat: {chat['conversation_name']} (ID: {chat['id']})" for chat in chat_results
-                ]
-
-                # If no character is selected, also search for characters
-                if not character_id:
-                    characters = get_character_cards()
-                    filtered_characters = [
-                        char for char in characters
-                        if query.lower() in char['name'].lower()
-                    ]
-                    formatted_character_results = [
-                        f"Character: {char['name']} (ID: {char['id']})" for char in filtered_characters
-                    ]
-                else:
-                    formatted_character_results = []
-
-                # Combine results
-                all_choices = formatted_chat_results + formatted_character_results
-
-                if all_choices:
-                    return gr.update(choices=all_choices, visible=True), chat_message
-                else:
-                    return gr.update(choices=[], visible=False), f"No results found for '{query}'."
-
-            except Exception as e:
-                logging.error(f"Error during search: {e}")
-                return gr.update(choices=[], visible=False), f"Error occurred during search: {e}"
-
-        def load_conversation_or_character(selected, conversation_mapping):
-            if not selected or selected not in conversation_mapping:
-                return "", "<p>No selection made.</p>"
-
-            selected_id = conversation_mapping[selected]
-            if selected.startswith("Chat:"):
-                chat = get_character_chat_by_id(selected_id)
-                if chat:
-                    json_content = json.dumps({
-                        "conversation_id": chat['id'],
-                        "conversation_name": chat['conversation_name'],
-                        "messages": chat['chat_history']
-                    }, indent=2)
-
-                    html_preview = create_chat_preview_html(chat['chat_history'])
-                    return json_content, html_preview
-            elif selected.startswith("Character:"):
-                character = get_character_card_by_id(selected_id)
-                if character:
-                    json_content = json.dumps({
-                        "id": character['id'],
-                        "name": character['name'],
-                        "description": character['description'],
-                        "personality": character['personality'],
-                        "scenario": character['scenario'],
-                        "post_history_instructions": character['post_history_instructions'],
-                        "first_mes": character['first_mes'],
-                        "mes_example": character['mes_example'],
-                        "creator_notes": character.get('creator_notes', ''),
-                        "system_prompt": character.get('system_prompt', ''),
-                        "tags": character.get('tags', []),
-                        "creator": character.get('creator', ''),
-                        "character_version": character.get('character_version', ''),
-                        "extensions": character.get('extensions', {})
-                    }, indent=2)
-
-                    html_preview = create_character_preview_html(character)
-                    return json_content, html_preview
-
-            return "", "<p>Unable to load the selected item.</p>"
-
-        def validate_content(selected, content):
-            try:
-                data = json.loads(content)
-                if selected.startswith("Chat:"):
-                    assert "conversation_id" in data and "messages" in data
-                elif selected.startswith("Character:"):
-                    assert "id" in data and "name" in data
-                return True, data
-            except Exception as e:
-                return False, f"Invalid JSON: {e}"
-
-        def save_conversation_or_character(selected, conversation_mapping, content):
-            if not selected or selected not in conversation_mapping:
-                return "Please select an item to save.", "<p>No changes made.</p>"
-
-            is_valid, result = validate_content(selected, content)
-            if not is_valid:
-                return f"Error: {result}", "<p>No changes made due to validation error.</p>"
-
-            selected_id = conversation_mapping[selected]
-
-            if selected.startswith("Chat:"):
-                success = update_character_chat(selected_id, result['messages'])
-                return ("Chat updated successfully." if success else "Failed to update chat."), ("<p>Chat updated.</p>" if success else "<p>Failed to update chat.</p>")
-            elif selected.startswith("Character:"):
-                success = update_character_card(selected_id, result)
-                return ("Character updated successfully." if success else "Failed to update character."), ("<p>Character updated.</p>" if success else "<p>Failed to update character.</p>")
-
-            return "Unknown item type.", "<p>No changes made.</p>"
-
-        def delete_conversation_or_character(selected, conversation_mapping):
-            if not selected or selected not in conversation_mapping:
-                return "Please select an item to delete.", "<p>No changes made.</p>", gr.update(choices=[])
-
-            selected_id = conversation_mapping[selected]
-
-            if selected.startswith("Chat:"):
-                success = delete_character_chat(selected_id)
-            elif selected.startswith("Character:"):
-                success = delete_character_card(selected_id)
-            else:
-                return "Unknown item type.", "<p>No changes made.</p>", gr.update()
-
-            if success:
-                updated_choices = [choice for choice in conversation_mapping.keys() if choice != selected]
-                conversation_mapping.value.pop(selected, None)
-                return f"{selected.split(':')[0]} deleted successfully.", f"<p>{selected.split(':')[0]} deleted.</p>", gr.update(choices=updated_choices)
-            else:
-                return f"Failed to delete {selected.split(':')[0].lower()}.", f"<p>Failed to delete {selected.split(':')[0].lower()}.</p>", gr.update()
-
-        def populate_chats(character_selection):
-            if not character_selection:
-                return gr.update(choices=[], visible=False), "Please select a character first."
-
-            try:
-                character_id = int(character_selection.split('(ID: ')[1].rstrip(')'))
-                chats = get_character_chats(character_id=character_id)
-
-                if not chats:
-                    return gr.update(choices=[], visible=False), f"No chats found for the selected character."
-
-                formatted_chats = [f"{chat['conversation_name']} (ID: {chat['id']})" for chat in chats]
-                return gr.update(choices=formatted_chats, visible=True), f"Found {len(formatted_chats)} chat(s)."
-            except Exception as e:
-                logging.error(f"Error populating chats: {e}")
-                return gr.update(choices=[], visible=False), f"Error occurred: {e}"
-
-        def load_chat_from_character(selected_chat):
-            if not selected_chat:
-                return "", "<p>No chat selected.</p>"
-
-            try:
-                chat_id = int(selected_chat.split('(ID: ')[1].rstrip(')'))
-                chat = get_character_chat_by_id(chat_id)
-                if not chat:
-                    return "", "<p>Selected chat not found.</p>"
-
-                json_content = json.dumps({
-                    "conversation_id": chat['id'],
-                    "conversation_name": chat['conversation_name'],
-                    "messages": chat['chat_history']
-                }, indent=2)
-
-                html_preview = create_chat_preview_html(chat['chat_history'])
-                return json_content, html_preview
-            except Exception as e:
-                logging.error(f"Error loading chat: {e}")
-                return "", f"<p>Error loading chat: {e}</p>"
-
-        def create_chat_preview_html(chat_history):
-            html_preview = "<div style='max-height: 500px; overflow-y: auto;'>"
-            for user_msg, bot_msg in chat_history:
-                user_style = "background-color: #e6f3ff; padding: 10px; border-radius: 5px; margin-bottom: 5px;"
-                bot_style = "background-color: #f0f0f0; padding: 10px; border-radius: 5px; margin-bottom: 10px;"
-                html_preview += f"<div style='{user_style}'><strong>User:</strong> {user_msg}</div>"
-                html_preview += f"<div style='{bot_style}'><strong>Bot:</strong> {bot_msg}</div>"
-            html_preview += "</div>"
-            return html_preview
-
-        def create_character_preview_html(character):
-            return f"""
-            <div>
-                <h2>{character['name']}</h2>
-                <p><strong>Description:</strong> {character['description']}</p>
-                <p><strong>Personality:</strong> {character['personality']}</p>
-                <p><strong>Scenario:</strong> {character['scenario']}</p>
-                <p><strong>First Message:</strong> {character['first_mes']}</p>
-                <p><strong>Example Message:</strong> {character['mes_example']}</p>
-                <p><strong>Post History Instructions:</strong> {character['post_history_instructions']}</p>
-                <p><strong>System Prompt:</strong> {character.get('system_prompt', 'N/A')}</p>
-                <p><strong>Tags:</strong> {', '.join(character.get('tags', []))}</p>
-                <p><strong>Creator:</strong> {character.get('creator', 'N/A')}</p>
-                <p><strong>Version:</strong> {character.get('character_version', 'N/A')}</p>
-            </div>
-            """
-        def import_multiple_characters(files):
-            if not files:
-                return "No files provided for character import."
-
-            results = []
-            for file in files:
-                result, _, message = import_character_card(file)
-                if result:
-                    results.append(f"Imported: {result['name']}")
-                else:
-                    results.append(f"Failed: {file.name} - {message}")
-
-            # Refresh character choices
-            characters = get_character_cards()
-            character_choices = [f"{char['name']} (ID: {char['id']})" for char in characters]
-            select_character.choices = character_choices
-
-            return "Import results:\n" + "\n".join(results)
-
-        # Register new callback for character import
-        import_characters_button.click(
-            fn=import_multiple_characters,
-            inputs=[character_files],
-            outputs=[import_status]
-        ).then(
-            fn=lambda: gr.update(choices=[f"{char['name']} (ID: {char['id']})" for char in get_character_cards()]),
-            outputs=select_character
-        )
-
-        # Register Callback Functions with Gradio Components
-        search_button.click(
-            fn=search_conversations_or_characters,
-            inputs=[search_query, select_character],
-            outputs=[search_results, search_status]
-        )
-
-        search_results.change(
-            fn=load_conversation_or_character,
-            inputs=[search_results, conversation_mapping],
-            outputs=[chat_content, chat_preview]
-        )
-
-        save_button.click(
-            fn=save_conversation_or_character,
-            inputs=[conversation_list, conversation_mapping, chat_content],
-            outputs=[result_message, chat_preview]
-        )
-
-        delete_button.click(
-            fn=delete_conversation_or_character,
-            inputs=[conversation_list, conversation_mapping],
-            outputs=[result_message, chat_preview, conversation_list]
-        )
-
-        select_character.change(
-            fn=load_character_image,
-            inputs=[select_character],
-            outputs=[character_image]
-        ).then(
-            fn=populate_chats,
-            inputs=[select_character],
-            outputs=[select_chat, search_status]
-        )
-
-        select_chat.change(
-            fn=load_chat_from_character,
-            inputs=[select_chat],
-            outputs=[chat_content, chat_preview]
-        )
-
-        load_chat_button.click(
-            fn=load_chat_from_character,
-            inputs=[select_chat],
-            outputs=[chat_content, chat_preview]
-        )
-
-        load_characters_button.click(
-            fn=lambda: gr.update(choices=[f"{char['name']} (ID: {char['id']})" for char in get_character_cards()]),
-            outputs=select_character
-        )
-
-        return (
-            character_files, import_characters_button, import_status,
-            search_query, search_button, search_results, search_status,
-            select_character, select_chat, load_chat_button,
-            conversation_list, conversation_mapping,
-            chat_content, save_button, delete_button,
-            chat_preview, result_message, character_image
-        )
-
-def create_custom_character_card_tab():
-    with gr.TabItem("Create a New Character Card", visible=True):
-        gr.Markdown("# Create a New Character Card (v2)")
-
-        with gr.Row():
-            with gr.Column():
-                # Input fields for character card data
-                name_input = gr.Textbox(label="Name", placeholder="Enter character name")
-                description_input = gr.TextArea(label="Description", placeholder="Enter character description")
-                personality_input = gr.TextArea(label="Personality", placeholder="Enter character personality")
-                scenario_input = gr.TextArea(label="Scenario", placeholder="Enter character scenario")
-                first_mes_input = gr.TextArea(label="First Message", placeholder="Enter the first message")
-                mes_example_input = gr.TextArea(label="Example Messages", placeholder="Enter example messages")
-                creator_notes_input = gr.TextArea(label="Creator Notes", placeholder="Enter notes for the creator")
-                system_prompt_input = gr.TextArea(label="System Prompt", placeholder="Enter system prompt")
-                post_history_instructions_input = gr.TextArea(label="Post History Instructions", placeholder="Enter post history instructions")
-                alternate_greetings_input = gr.TextArea(
-                    label="Alternate Greetings (one per line)",
-                    placeholder="Enter alternate greetings, one per line"
-                )
-                tags_input = gr.Textbox(label="Tags", placeholder="Enter tags, separated by commas")
-                creator_input = gr.Textbox(label="Creator", placeholder="Enter creator name")
-                character_version_input = gr.Textbox(label="Character Version", placeholder="Enter character version")
-                extensions_input = gr.TextArea(
-                    label="Extensions (JSON)",
-                    placeholder="Enter extensions as JSON (optional)"
-                )
-                image_input = gr.Image(label="Character Image", type="pil")
-
-                # Buttons
-                save_button = gr.Button("Save Character Card")
-                download_button = gr.Button("Download Character Card")
-                download_image_button = gr.Button("Download Character Card as Image")
-
-                # Output status and outputs
-                save_status = gr.Markdown("")
-                download_output = gr.File(label="Download Character Card", interactive=False)
-                download_image_output = gr.File(label="Download Character Card as Image", interactive=False)
-
-        # Import PngInfo
-        from PIL.PngImagePlugin import PngInfo
-
-        # Callback Functions
-        def build_character_card(
-            name, description, personality, scenario, first_mes, mes_example,
-            creator_notes, system_prompt, post_history_instructions,
-            alternate_greetings_str, tags_str, creator, character_version,
-            extensions_str
-        ):
-            # Parse alternate_greetings from multiline string
-            alternate_greetings = [line.strip() for line in alternate_greetings_str.strip().split('\n') if line.strip()]
-
-            # Parse tags from comma-separated string
-            tags = [tag.strip() for tag in tags_str.strip().split(',') if tag.strip()]
-
-            # Parse extensions from JSON string
-            try:
-                extensions = json.loads(extensions_str) if extensions_str.strip() else {}
-            except json.JSONDecodeError as e:
-                extensions = {}
-                logging.error(f"Error parsing extensions JSON: {e}")
-
-            # Build the character card dictionary according to V2 spec
-            character_card = {
-                'spec': 'chara_card_v2',
-                'spec_version': '2.0',
-                'data': {
-                    'name': name,
-                    'description': description,
-                    'personality': personality,
-                    'scenario': scenario,
-                    'first_mes': first_mes,
-                    'mes_example': mes_example,
-                    'creator_notes': creator_notes,
-                    'system_prompt': system_prompt,
-                    'post_history_instructions': post_history_instructions,
-                    'alternate_greetings': alternate_greetings,
-                    'tags': tags,
-                    'creator': creator,
-                    'character_version': character_version,
-                    'extensions': extensions,
-                }
-            }
-            return character_card
-
-        def validate_character_card_data(character_card):
-            """
-            Validates the character card data using the extended validation logic.
-            """
-            is_valid, validation_messages = validate_v2_card(character_card)
-            return is_valid, validation_messages
-
-        def save_character_card(
-            name, description, personality, scenario, first_mes, mes_example,
-            creator_notes, system_prompt, post_history_instructions,
-            alternate_greetings_str, tags_str, creator, character_version,
-            extensions_str, image
-        ):
-            # Build the character card
-            character_card = build_character_card(
-                name, description, personality, scenario, first_mes, mes_example,
-                creator_notes, system_prompt, post_history_instructions,
-                alternate_greetings_str, tags_str, creator, character_version,
-                extensions_str
-            )
-
-            # Validate the character card
-            is_valid, validation_messages = validate_character_card_data(character_card)
-            if not is_valid:
-                # Return validation errors
-                validation_output = "Character card validation failed:\n"
-                validation_output += "\n".join(validation_messages)
-                return validation_output
-
-            # If image is provided, encode it to base64
-            if image:
-                img_byte_arr = io.BytesIO()
-                image.save(img_byte_arr, format='PNG')
-                character_card['data']['image'] = base64.b64encode(img_byte_arr.getvalue()).decode('utf-8')
-
-            # Save character card to database
-            character_id = add_character_card(character_card['data'])
-            if character_id:
-                return f"Character card '{name}' saved successfully."
-            else:
-                return f"Failed to save character card '{name}'. It may already exist."
-
-        def download_character_card(
-            name, description, personality, scenario, first_mes, mes_example,
-            creator_notes, system_prompt, post_history_instructions,
-            alternate_greetings_str, tags_str, creator, character_version,
-            extensions_str, image
-        ):
-            # Build the character card
-            character_card = build_character_card(
-                name, description, personality, scenario, first_mes, mes_example,
-                creator_notes, system_prompt, post_history_instructions,
-                alternate_greetings_str, tags_str, creator, character_version,
-                extensions_str
-            )
-
-            # Validate the character card
-            is_valid, validation_messages = validate_character_card_data(character_card)
-            if not is_valid:
-                # Return validation errors
-                validation_output = "Character card validation failed:\n"
-                validation_output += "\n".join(validation_messages)
-                return gr.update(value=None), validation_output  # Return None for the file output
-
-            # If image is provided, include it as base64
-            if image:
-                img_byte_arr = io.BytesIO()
-                image.save(img_byte_arr, format='PNG')
-                character_card['data']['image'] = base64.b64encode(img_byte_arr.getvalue()).decode('utf-8')
-
-            # Convert to JSON string
-            json_str = json.dumps(character_card, indent=2)
-
-            # Write the JSON to a temporary file
-            with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.json', encoding='utf-8') as temp_file:
-                temp_file.write(json_str)
-                temp_file_path = temp_file.name
-
-            # Return the file path and clear validation output
-            return temp_file_path, ""
-
-        def download_character_card_as_image(
-            name, description, personality, scenario, first_mes, mes_example,
-            creator_notes, system_prompt, post_history_instructions,
-            alternate_greetings_str, tags_str, creator, character_version,
-            extensions_str, image
-        ):
-            # Build the character card
-            character_card = build_character_card(
-                name, description, personality, scenario, first_mes, mes_example,
-                creator_notes, system_prompt, post_history_instructions,
-                alternate_greetings_str, tags_str, creator, character_version,
-                extensions_str
-            )
-
-            # Validate the character card
-            is_valid, validation_messages = validate_character_card_data(character_card)
-            if not is_valid:
-                # Return validation errors
-                validation_output = "Character card validation failed:\n"
-                validation_output += "\n".join(validation_messages)
-                return gr.update(value=None), validation_output  # Return None for the file output
-
-            # Convert the character card JSON to a string
-            json_str = json.dumps(character_card, indent=2)
-
-            # Encode the JSON string to base64
-            chara_content = base64.b64encode(json_str.encode('utf-8')).decode('utf-8')
-
-            # Create PNGInfo object to hold metadata
-            png_info = PngInfo()
-            png_info.add_text('chara', chara_content)
-
-            # If image is provided, use it; otherwise, create a blank image
-            if image:
-                img = image.copy()
-            else:
-                # Create a default blank image
-                img = Image.new('RGB', (512, 512), color='white')
-
-            # Save the image to a temporary file with metadata
-            with tempfile.NamedTemporaryFile(mode='wb', delete=False, suffix='.png') as temp_file:
-                img.save(temp_file, format='PNG', pnginfo=png_info)
-                temp_file_path = temp_file.name
-
-            # Return the file path and clear validation output
-            return temp_file_path, ""
-
-        # Include the validate_v2_card function here (from previous code)
-
-        # Button Callbacks
-        save_button.click(
-            fn=save_character_card,
-            inputs=[
-                name_input, description_input, personality_input, scenario_input,
-                first_mes_input, mes_example_input, creator_notes_input, system_prompt_input,
-                post_history_instructions_input, alternate_greetings_input, tags_input,
-                creator_input, character_version_input, extensions_input, image_input
-            ],
-            outputs=[save_status]
-        )
-
-        download_button.click(
-            fn=download_character_card,
-            inputs=[
-                name_input, description_input, personality_input, scenario_input,
-                first_mes_input, mes_example_input, creator_notes_input, system_prompt_input,
-                post_history_instructions_input, alternate_greetings_input, tags_input,
-                creator_input, character_version_input, extensions_input, image_input
-            ],
-            outputs=[download_output, save_status]
-        )
-
-        download_image_button.click(
-            fn=download_character_card_as_image,
-            inputs=[
-                name_input, description_input, personality_input, scenario_input,
-                first_mes_input, mes_example_input, creator_notes_input, system_prompt_input,
-                post_history_instructions_input, alternate_greetings_input, tags_input,
-                creator_input, character_version_input, extensions_input, image_input
-            ],
-            outputs=[download_image_output, save_status]
-        )
-
-#v1
-def create_character_card_validation_tab():
-    with gr.TabItem("Validate Character Card", visible=True):
-        gr.Markdown("# Validate Character Card (v2)")
-        gr.Markdown("Upload a character card (PNG, WEBP, or JSON) to validate whether it conforms to the Character Card V2 specification.")
-
-        with gr.Row():
-            with gr.Column():
-                # File uploader
-                file_upload = gr.File(
-                    label="Upload Character Card (PNG, WEBP, JSON)",
-                    file_types=[".png", ".webp", ".json"]
-                )
-                # Validation button
-                validate_button = gr.Button("Validate Character Card")
-                # Output area for validation results
-                validation_output = gr.Markdown("")
-
-        # Callback Functions
-        def validate_character_card(file):
-            if file is None:
-                return "No file provided for validation."
-
-            try:
-                if file.name.lower().endswith(('.png', '.webp')):
-                    json_data = extract_json_from_image(file)
-                    if not json_data:
-                        return "Failed to extract JSON data from the image. The image might not contain embedded character card data."
-                elif file.name.lower().endswith('.json'):
-                    with open(file.name, 'r', encoding='utf-8') as f:
-                        json_data = f.read()
-                else:
-                    return "Unsupported file type. Please upload a PNG, WEBP, or JSON file."
-
-                # Parse the JSON content
-                try:
-                    card_data = json.loads(json_data)
-                except json.JSONDecodeError as e:
-                    return f"JSON decoding error: {e}"
-
-                # Validate the character card
-                is_valid, validation_messages = validate_v2_card(card_data)
-
-                # Prepare the validation output
-                if is_valid:
-                    return "Character card is valid according to the V2 specification."
-                else:
-                    # Concatenate all validation error messages
-                    validation_output = "Character card validation failed:\n"
-                    validation_output += "\n".join(validation_messages)
-                    return validation_output
-
-            except Exception as e:
-                logging.error(f"Error validating character card: {e}")
-                return f"An unexpected error occurred during validation: {e}"
-
-        def validate_v2_card(card_data):
-            """
-            Validate a character card according to the V2 specification.
-
-            Args:
-                card_data (dict): The parsed character card data.
-
-            Returns:
-                Tuple[bool, List[str]]: A tuple containing a boolean indicating validity and a list of validation messages.
-            """
-            validation_messages = []
-
-            # Check top-level fields
-            if 'spec' not in card_data:
-                validation_messages.append("Missing 'spec' field.")
-            elif card_data['spec'] != 'chara_card_v2':
-                validation_messages.append(f"Invalid 'spec' value: {card_data['spec']}. Expected 'chara_card_v2'.")
-
-            if 'spec_version' not in card_data:
-                validation_messages.append("Missing 'spec_version' field.")
-            else:
-                # Ensure 'spec_version' is '2.0' or higher
-                try:
-                    spec_version = float(card_data['spec_version'])
-                    if spec_version < 2.0:
-                        validation_messages.append(f"'spec_version' must be '2.0' or higher. Found '{card_data['spec_version']}'.")
-                except ValueError:
-                    validation_messages.append(f"Invalid 'spec_version' format: {card_data['spec_version']}. Must be a number as a string.")
-
-            if 'data' not in card_data:
-                validation_messages.append("Missing 'data' field.")
-                return False, validation_messages  # Cannot proceed without 'data' field
-
-            data = card_data['data']
-
-            # Required fields in 'data'
-            required_fields = ['name', 'description', 'personality', 'scenario', 'first_mes', 'mes_example']
-            for field in required_fields:
-                if field not in data:
-                    validation_messages.append(f"Missing required field in 'data': '{field}'.")
-                elif not isinstance(data[field], str):
-                    validation_messages.append(f"Field '{field}' must be a string.")
-                elif not data[field].strip():
-                    validation_messages.append(f"Field '{field}' cannot be empty.")
-
-            # Optional fields with expected types
-            optional_fields = {
-                'creator_notes': str,
-                'system_prompt': str,
-                'post_history_instructions': str,
-                'alternate_greetings': list,
-                'tags': list,
-                'creator': str,
-                'character_version': str,
-                'extensions': dict,
-                'character_book': dict  # If present, should be a dict
-            }
-
-            for field, expected_type in optional_fields.items():
-                if field in data:
-                    if not isinstance(data[field], expected_type):
-                        validation_messages.append(f"Field '{field}' must be of type '{expected_type.__name__}'.")
-                    elif field == 'extensions':
-                        # Validate that extensions keys are properly namespaced
-                        for key in data[field].keys():
-                            if '/' not in key and '_' not in key:
-                                validation_messages.append(f"Extension key '{key}' in 'extensions' should be namespaced to prevent conflicts.")
-
-            # If 'alternate_greetings' is present, check that it's a list of non-empty strings
-            if 'alternate_greetings' in data and isinstance(data['alternate_greetings'], list):
-                for idx, greeting in enumerate(data['alternate_greetings']):
-                    if not isinstance(greeting, str) or not greeting.strip():
-                        validation_messages.append(f"Element {idx} in 'alternate_greetings' must be a non-empty string.")
-
-            # If 'tags' is present, check that it's a list of non-empty strings
-            if 'tags' in data and isinstance(data['tags'], list):
-                for idx, tag in enumerate(data['tags']):
-                    if not isinstance(tag, str) or not tag.strip():
-                        validation_messages.append(f"Element {idx} in 'tags' must be a non-empty string.")
-
-            # Validate 'extensions' field
-            if 'extensions' in data and not isinstance(data['extensions'], dict):
-                validation_messages.append("Field 'extensions' must be a dictionary.")
-
-            # Validate 'character_book' if present
-            if 'character_book' in data:
-                is_valid_book, book_messages = validate_character_book(data['character_book'])
-                if not is_valid_book:
-                    validation_messages.extend(book_messages)
-
-            is_valid = len(validation_messages) == 0
-            return is_valid, validation_messages
-
-        # Button Callback
-        validate_button.click(
-            fn=validate_character_card,
-            inputs=[file_upload],
-            outputs=[validation_output]
-        )
-
-
-def create_export_characters_tab():
-    with gr.TabItem("Export Characters", visible=True):
-        gr.Markdown("# Export Characters")
-        gr.Markdown("Export character cards individually as JSON files or all together as a ZIP file.")
-
-        with gr.Row():
-            with gr.Column(scale=1):
-                # Dropdown to select a character for individual export
-                characters = get_character_cards()
-                character_choices = [f"{char['name']} (ID: {char['id']})" for char in characters]
-                export_character_dropdown = gr.Dropdown(
-                    label="Select Character to Export",
-                    choices=character_choices
-                )
-                load_characters_button = gr.Button("Load Existing Characters")
-                export_single_button = gr.Button("Export Selected Character")
-                export_all_button = gr.Button("Export All Characters")
-
-            with gr.Column(scale=1):
-                # Output components
-                export_output = gr.File(label="Exported Character(s)", interactive=False)
-                export_status = gr.Markdown("")
-
-# FIXME
-        def export_single_character_wrapper(character_selection):
-            file_path, status_message = export_single_character(character_selection)
-            if file_path:
-                return gr.File.update(value=file_path), status_message
-            else:
-                return gr.File.update(value=None), status_message
-
-        def export_all_characters_wrapper():
-            zip_path = export_all_characters_as_zip()
-            characters = get_character_cards()
-            exported_characters = [char['name'] for char in characters]
-            status_message = f"Exported {len(exported_characters)} characters successfully:\n" + "\n".join(exported_characters)
-            return gr.File.update(value=zip_path), status_message
-
-        # Event listeners
-        load_characters_button.click(
-            fn=lambda: gr.update(choices=[f"{char['name']} (ID: {char['id']})" for char in get_character_cards()]),
-            outputs=export_character_dropdown
-        )
-
-        export_single_button.click(
-            fn=export_single_character_wrapper,
-            inputs=[export_character_dropdown],
-            outputs=[export_output, export_status]
-        )
-
-        export_all_button.click(
-            fn=export_all_characters_wrapper,
-            inputs=[],
-            outputs=[export_output, export_status]
-        )
-
-    return export_character_dropdown, load_characters_button, export_single_button, export_all_button, export_output, export_status
-
-#
-# End of Character_Chat_tab.py
-#######################################################################################################################

App_Function_Libraries/Gradio_UI/Character_Interaction_tab.py

@@ -1,837 +0,0 @@
-# Character_Interaction_tab.py
-# Description: This file contains the functions that are used for Character Interactions in the Gradio UI.
-#
-# Imports
-import base64
-import io
-import uuid
-from datetime import datetime as datetime
-import logging
-import json
-import os
-from typing import List, Dict, Tuple, Union
-
-#
-# External Imports
-import gradio as gr
-from PIL import Image
-#
-# Local Imports
-from App_Function_Libraries.Chat import chat, load_characters, save_chat_history_to_db_wrapper
-from App_Function_Libraries.Gradio_UI.Chat_ui import chat_wrapper
-from App_Function_Libraries.Gradio_UI.Writing_tab import generate_writing_feedback
-#
-########################################################################################################################
-#
-# Single-Character chat Functions:
-
-
-def chat_with_character(user_message, history, char_data, api_name_input, api_key):
-    if char_data is None:
-        return history, "Please import a character card first."
-
-    bot_message = generate_writing_feedback(user_message, char_data['name'], "Overall", api_name_input,
-                                            api_key)
-    history.append((user_message, bot_message))
-    return history, ""
-
-
-def import_character_card(file):
-    if file is None:
-        logging.warning("No file provided for character card import")
-        return None
-    try:
-        if file.name.lower().endswith(('.png', '.webp')):
-            logging.info(f"Attempting to import character card from image: {file.name}")
-            json_data = extract_json_from_image(file)
-            if json_data:
-                logging.info("JSON data extracted from image, attempting to parse")
-                card_data = import_character_card_json(json_data)
-                if card_data:
-                    # Save the image data
-                    with Image.open(file) as img:
-                        img_byte_arr = io.BytesIO()
-                        img.save(img_byte_arr, format='PNG')
-                        card_data['image'] = base64.b64encode(img_byte_arr.getvalue()).decode('utf-8')
-                return card_data
-            else:
-                logging.warning("No JSON data found in the image")
-        else:
-            logging.info(f"Attempting to import character card from JSON file: {file.name}")
-            content = file.read().decode('utf-8')
-            return import_character_card_json(content)
-    except Exception as e:
-        logging.error(f"Error importing character card: {e}")
-    return None
-
-
-def import_character_card_json(json_content):
-    try:
-        # Remove any leading/trailing whitespace
-        json_content = json_content.strip()
-
-        # Log the first 100 characters of the content
-        logging.debug(f"JSON content (first 100 chars): {json_content[:100]}...")
-
-        card_data = json.loads(json_content)
-        logging.debug(f"Parsed JSON data keys: {list(card_data.keys())}")
-        if 'spec' in card_data and card_data['spec'] == 'chara_card_v2':
-            logging.info("Detected V2 character card")
-            return card_data['data']
-        else:
-            logging.info("Assuming V1 character card")
-            return card_data
-    except json.JSONDecodeError as e:
-        logging.error(f"JSON decode error: {e}")
-        logging.error(f"Problematic JSON content: {json_content[:500]}...")
-    except Exception as e:
-        logging.error(f"Unexpected error parsing JSON: {e}")
-    return None
-
-
-def extract_json_from_image(image_file):
-    logging.debug(f"Attempting to extract JSON from image: {image_file.name}")
-    try:
-        with Image.open(image_file) as img:
-            logging.debug("Image opened successfully")
-            metadata = img.info
-            if 'chara' in metadata:
-                logging.debug("Found 'chara' in image metadata")
-                chara_content = metadata['chara']
-                logging.debug(f"Content of 'chara' metadata (first 100 chars): {chara_content[:100]}...")
-                try:
-                    decoded_content = base64.b64decode(chara_content).decode('utf-8')
-                    logging.debug(f"Decoded content (first 100 chars): {decoded_content[:100]}...")
-                    return decoded_content
-                except Exception as e:
-                    logging.error(f"Error decoding base64 content: {e}")
-
-            logging.debug("'chara' not found in metadata, checking for base64 encoded data")
-            raw_data = img.tobytes()
-            possible_json = raw_data.split(b'{', 1)[-1].rsplit(b'}', 1)[0]
-            if possible_json:
-                try:
-                    decoded = base64.b64decode(possible_json).decode('utf-8')
-                    if decoded.startswith('{') and decoded.endswith('}'):
-                        logging.debug("Found and decoded base64 JSON data")
-                        return '{' + decoded + '}'
-                except Exception as e:
-                    logging.error(f"Error decoding base64 data: {e}")
-
-            logging.warning("No JSON data found in the image")
-    except Exception as e:
-        logging.error(f"Error extracting JSON from image: {e}")
-    return None
-
-
-def load_chat_history(file):
-    try:
-        content = file.read().decode('utf-8')
-        chat_data = json.loads(content)
-        return chat_data['history'], chat_data['character']
-    except Exception as e:
-        logging.error(f"Error loading chat history: {e}")
-        return None, None
-
-
-def create_character_card_interaction_tab():
-    with gr.TabItem("Chat with a Character Card"):
-        gr.Markdown("# Chat with a Character Card")
-        with gr.Row():
-            with gr.Column(scale=1):
-                character_image = gr.Image(label="Character Image", type="filepath")
-                character_card_upload = gr.File(label="Upload Character Card")
-                import_card_button = gr.Button("Import Character Card")
-                load_characters_button = gr.Button("Load Existing Characters")
-                from App_Function_Libraries.Chat import get_character_names
-                character_dropdown = gr.Dropdown(label="Select Character", choices=get_character_names())
-                user_name_input = gr.Textbox(label="Your Name", placeholder="Enter your name here")
-                api_name_input = gr.Dropdown(
-                    choices=["Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral",
-                             "OpenRouter", "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace",
-                             "Custom-OpenAI-API"],
-                    value="HuggingFace",
-                    # FIXME - make it so the user cant' click `Send Message` without first setting an API + Chatbot
-                    label="API for Interaction(Mandatory)"
-                )
-                api_key_input = gr.Textbox(label="API Key (if not set in Config_Files/config.txt)",
-                                           placeholder="Enter your API key here", type="password")
-                temperature_slider = gr.Slider(minimum=0.0, maximum=2.0, value=0.7, step=0.05, label="Temperature")
-                import_chat_button = gr.Button("Import Chat History")
-                chat_file_upload = gr.File(label="Upload Chat History JSON", visible=False)
-
-            with gr.Column(scale=2):
-                chat_history = gr.Chatbot(label="Conversation", height=800)
-                user_input = gr.Textbox(label="Your message")
-                send_message_button = gr.Button("Send Message")
-                regenerate_button = gr.Button("Regenerate Last Message")
-                clear_chat_button = gr.Button("Clear Chat")
-                chat_media_name = gr.Textbox(label="Custom Chat Name(optional)", visible=True)
-                save_chat_history_to_db = gr.Button("Save Chat History to DataBase")
-                save_status = gr.Textbox(label="Save Status", interactive=False)
-
-        character_data = gr.State(None)
-        user_name = gr.State("")
-
-    def import_chat_history(file, current_history, char_data):
-        loaded_history, char_name = load_chat_history(file)
-        if loaded_history is None:
-            return current_history, char_data, "Failed to load chat history."
-
-        # Check if the loaded chat is for the current character
-        if char_data and char_data.get('name') != char_name:
-            return current_history, char_data, f"Warning: Loaded chat is for character '{char_name}', but current character is '{char_data.get('name')}'. Chat not imported."
-
-        # If no character is selected, try to load the character from the chat
-        if not char_data:
-            new_char_data = load_character(char_name)[0]
-            if new_char_data:
-                char_data = new_char_data
-            else:
-                return current_history, char_data, f"Warning: Character '{char_name}' not found. Please select the character manually."
-
-        return loaded_history, char_data, f"Chat history for '{char_name}' imported successfully."
-
-    def import_character(file):
-        card_data = import_character_card(file)
-        if card_data:
-            from App_Function_Libraries.Chat import save_character
-            save_character(card_data)
-            return card_data, gr.update(choices=get_character_names())
-        else:
-            return None, gr.update()
-
-    def load_character(name):
-        from App_Function_Libraries.Chat import load_characters
-        characters = load_characters()
-        char_data = characters.get(name)
-        if char_data:
-            first_message = char_data.get('first_mes', "Hello! I'm ready to chat.")
-            return char_data, [(None, first_message)] if first_message else [], None
-        return None, [], None
-
-    def load_character_image(name):
-        from App_Function_Libraries.Chat import load_characters
-        characters = load_characters()
-        char_data = characters.get(name)
-        if char_data and 'image_path' in char_data:
-            image_path = char_data['image_path']
-            if os.path.exists(image_path):
-                return image_path
-            else:
-                logging.warning(f"Image file not found: {image_path}")
-        return None
-
-    def load_character_and_image(name):
-        char_data, chat_history, _ = load_character(name)
-        image_path = load_character_image(name)
-        logging.debug(f"Character: {name}")
-        logging.debug(f"Character data: {char_data}")
-        logging.debug(f"Image path: {image_path}")
-        return char_data, chat_history, image_path
-
-    def character_chat_wrapper(message, history, char_data, api_endpoint, api_key, temperature, user_name):
-        logging.debug("Entered character_chat_wrapper")
-        if char_data is None:
-            return "Please select a character first.", history
-
-        if not user_name:
-            user_name = "User"
-
-        char_name = char_data.get('name', 'AI Assistant')
-
-        # Prepare the character's background information
-        char_background = f"""
-        Name: {char_name}
-        Description: {char_data.get('description', 'N/A')}
-        Personality: {char_data.get('personality', 'N/A')}
-        Scenario: {char_data.get('scenario', 'N/A')}
-        """
-
-        # Prepare the system prompt for character impersonation
-        system_message = f"""You are roleplaying as {char_name}, the character described below. Respond to the user's messages in character, maintaining the personality and background provided. Do not break character or refer to yourself as an AI. Always refer to yourself as "{char_name}" and refer to the user as "{user_name}".
-
-        {char_background}
-
-        Additional instructions: {char_data.get('post_history_instructions', '')}
-        """
-
-        # Prepare media_content and selected_parts
-        media_content = {
-            'id': char_name,
-            'title': char_name,
-            'content': char_background,
-            'description': char_data.get('description', ''),
-            'personality': char_data.get('personality', ''),
-            'scenario': char_data.get('scenario', '')
-        }
-        selected_parts = ['description', 'personality', 'scenario']
-
-        prompt = char_data.get('post_history_instructions', '')
-
-        # Prepare the input for the chat function
-        if not history:
-            full_message = f"{prompt}\n\n{user_name}: {message}" if prompt else f"{user_name}: {message}"
-        else:
-            full_message = f"{user_name}: {message}"
-
-        # Call the chat function
-        bot_message = chat(
-            full_message,
-            history,
-            media_content,
-            selected_parts,
-            api_endpoint,
-            api_key,
-            prompt,
-            temperature,
-            system_message
-        )
-
-        # Update history
-        history.append((message, bot_message))
-        return history
-
-    def save_chat_history(history, character_name):
-        # Create the Saved_Chats folder if it doesn't exist
-        save_directory = "Saved_Chats"
-        os.makedirs(save_directory, exist_ok=True)
-
-        timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-        filename = f"chat_history_{character_name}_{timestamp}.json"
-        filepath = os.path.join(save_directory, filename)
-
-        chat_data = {
-            "character": character_name,
-            "timestamp": timestamp,
-            "history": history
-        }
-
-        try:
-            with open(filepath, 'w', encoding='utf-8') as f:
-                json.dump(chat_data, f, ensure_ascii=False, indent=2)
-            return filepath
-        except Exception as e:
-            return f"Error saving chat: {str(e)}"
-
-    def save_current_chat(history, char_data):
-        if not char_data or not history:
-            return "No chat to save or character not selected."
-
-        character_name = char_data.get('name', 'Unknown')
-        result = save_chat_history(history, character_name)
-        if result.startswith("Error"):
-            return result
-        return f"Chat saved successfully as {result}"
-
-    def regenerate_last_message(history, char_data, api_name, api_key, temperature, user_name):
-        if not history:
-            return history
-
-        last_user_message = history[-1][0]
-        new_history = history[:-1]
-
-        return character_chat_wrapper(last_user_message, new_history, char_data, api_name, api_key, temperature,
-                                      user_name)
-
-    import_chat_button.click(
-        fn=lambda: gr.update(visible=True),
-        outputs=chat_file_upload
-    )
-
-    chat_file_upload.change(
-        fn=import_chat_history,
-        inputs=[chat_file_upload, chat_history, character_data],
-        outputs=[chat_history, character_data, save_status]
-    )
-
-    def update_character_info(name):
-        from App_Function_Libraries.Chat import load_characters
-        characters = load_characters()
-        char_data = characters.get(name)
-
-        image_path = char_data.get('image_path') if char_data else None
-
-        logging.debug(f"Character: {name}")
-        logging.debug(f"Character data: {char_data}")
-        logging.debug(f"Image path: {image_path}")
-
-        if image_path:
-            if os.path.exists(image_path):
-                logging.debug(f"Image file exists at {image_path}")
-                if os.access(image_path, os.R_OK):
-                    logging.debug(f"Image file is readable")
-                else:
-                    logging.warning(f"Image file is not readable: {image_path}")
-                    image_path = None
-            else:
-                logging.warning(f"Image file does not exist: {image_path}")
-                image_path = None
-        else:
-            logging.warning("No image path provided for the character")
-
-        return char_data, None, image_path  # Return None for chat_history
-
-    def on_character_select(name):
-        logging.debug(f"Character selected: {name}")
-        return update_character_info_with_error_handling(name)
-
-    def clear_chat_history():
-        return [], None  # Return empty list for chat_history and None for character_data
-
-    def update_character_info_with_error_handling(name):
-        logging.debug(f"Entering update_character_info_with_error_handling for character: {name}")
-        try:
-            char_data, _, image_path = update_character_info(name)
-            logging.debug(f"Retrieved data: char_data={bool(char_data)}, image_path={image_path}")
-
-            if char_data:
-                first_message = char_data.get('first_mes', "Hello! I'm ready to chat.")
-                chat_history = [(None, first_message)] if first_message else []
-            else:
-                chat_history = []
-
-            logging.debug(f"Created chat_history with length: {len(chat_history)}")
-
-            if image_path and os.path.exists(image_path):
-                logging.debug(f"Image file exists at {image_path}")
-                return char_data, chat_history, image_path
-            else:
-                logging.warning(f"Image not found or invalid path: {image_path}")
-                return char_data, chat_history, None
-        except Exception as e:
-            logging.error(f"Error updating character info: {str(e)}", exc_info=True)
-            return None, [], None
-        finally:
-            logging.debug("Exiting update_character_info_with_error_handling")
-
-    # Define States for conversation_id and media_content, which are required for saving chat history
-    conversation_id = gr.State(str(uuid.uuid4()))
-    media_content = gr.State({})
-
-    import_card_button.click(
-        fn=import_character,
-        inputs=[character_card_upload],
-        outputs=[character_data, character_dropdown]
-    )
-
-    load_characters_button.click(
-        fn=lambda: gr.update(choices=get_character_names()),
-        outputs=character_dropdown
-    )
-
-    clear_chat_button.click(
-        fn=clear_chat_history,
-        inputs=[],
-        outputs=[chat_history, character_data]
-    )
-
-    character_dropdown.change(
-        fn=on_character_select,
-        inputs=[character_dropdown],
-        outputs=[character_data, chat_history, character_image]
-    )
-
-    send_message_button.click(
-        fn=character_chat_wrapper,
-        inputs=[user_input, chat_history, character_data, api_name_input, api_key_input, temperature_slider,
-                user_name_input],
-        outputs=[chat_history]
-    ).then(lambda: "", outputs=user_input)
-
-    regenerate_button.click(
-        fn=regenerate_last_message,
-        inputs=[chat_history, character_data, api_name_input, api_key_input, temperature_slider, user_name_input],
-        outputs=[chat_history]
-    )
-
-    user_name_input.change(
-        fn=lambda name: name,
-        inputs=[user_name_input],
-        outputs=[user_name]
-    )
-
-    # FIXME - Implement saving chat history to database; look at Chat_UI.py for reference
-    save_chat_history_to_db.click(
-        save_chat_history_to_db_wrapper,
-        inputs=[chat_history, conversation_id, media_content, chat_media_name],
-        outputs=[conversation_id, gr.Textbox(label="Save Status")]
-    )
-
-    return character_data, chat_history, user_input, user_name, character_image
-
-
-#
-# End of Character chat tab
-######################################################################################################################
-#
-# Multi-Character Chat Interface
-
-def character_interaction_setup():
-    characters = load_characters()
-    return characters, [], None, None
-
-
-def extract_character_response(response: Union[str, Tuple]) -> str:
-    if isinstance(response, tuple):
-        # If it's a tuple, try to extract the first string element
-        for item in response:
-            if isinstance(item, str):
-                return item.strip()
-        # If no string found, return a default message
-        return "I'm not sure how to respond."
-    elif isinstance(response, str):
-        # If it's already a string, just return it
-        return response.strip()
-    else:
-        # For any other type, return a default message
-        return "I'm having trouble forming a response."
-
-# def process_character_response(response: str) -> str:
-#     # Remove any leading explanatory text before the first '---'
-#     parts = response.split('---')
-#     if len(parts) > 1:
-#         return '---' + '---'.join(parts[1:])
-#     return response.strip()
-def process_character_response(response: Union[str, Tuple]) -> str:
-    if isinstance(response, tuple):
-        response = ' '.join(str(item) for item in response if isinstance(item, str))
-
-    if isinstance(response, str):
-        # Remove any leading explanatory text before the first '---'
-        parts = response.split('---')
-        if len(parts) > 1:
-            return '---' + '---'.join(parts[1:])
-        return response.strip()
-    else:
-        return "I'm having trouble forming a response."
-
-def character_turn(characters: Dict, conversation: List[Tuple[str, str]],
-                   current_character: str, other_characters: List[str],
-                   api_endpoint: str, api_key: str, temperature: float,
-                   scenario: str = "") -> Tuple[List[Tuple[str, str]], str]:
-    if not current_character or current_character not in characters:
-        return conversation, current_character
-
-    if not conversation and scenario:
-        conversation.append(("Scenario", scenario))
-
-    current_char = characters[current_character]
-    other_chars = [characters[char] for char in other_characters if char in characters and char != current_character]
-
-    prompt = f"{current_char['name']}'s personality: {current_char['personality']}\n"
-    for char in other_chars:
-        prompt += f"{char['name']}'s personality: {char['personality']}\n"
-    prompt += "Conversation so far:\n" + "\n".join([f"{sender}: {message}" for sender, message in conversation])
-    prompt += f"\n\nHow would {current_char['name']} respond?"
-
-    try:
-        response = chat_wrapper(prompt, conversation, {}, [], api_endpoint, api_key, "", None, False, temperature, "")
-        processed_response = process_character_response(response)
-        conversation.append((current_char['name'], processed_response))
-    except Exception as e:
-        error_message = f"Error generating response: {str(e)}"
-        conversation.append((current_char['name'], error_message))
-
-    return conversation, current_character
-
-
-def character_interaction(character1: str, character2: str, api_endpoint: str, api_key: str,
-                          num_turns: int, scenario: str, temperature: float,
-                          user_interjection: str = "") -> List[str]:
-    characters = load_characters()
-    char1 = characters[character1]
-    char2 = characters[character2]
-    conversation = []
-    current_speaker = char1
-    other_speaker = char2
-
-    # Add scenario to the conversation start
-    if scenario:
-        conversation.append(f"Scenario: {scenario}")
-
-    for turn in range(num_turns):
-        # Construct the prompt for the current speaker
-        prompt = f"{current_speaker['name']}'s personality: {current_speaker['personality']}\n"
-        prompt += f"{other_speaker['name']}'s personality: {other_speaker['personality']}\n"
-        prompt += f"Conversation so far:\n" + "\n".join(
-            [msg if isinstance(msg, str) else f"{msg[0]}: {msg[1]}" for msg in conversation])
-
-        # Add user interjection if provided
-        if user_interjection and turn == num_turns // 2:
-            prompt += f"\n\nUser interjection: {user_interjection}\n"
-            conversation.append(f"User: {user_interjection}")
-
-        prompt += f"\n\nHow would {current_speaker['name']} respond?"
-
-        # FIXME - figure out why the double print is happening
-        # Get response from the LLM
-        response = chat_wrapper(prompt, conversation, {}, [], api_endpoint, api_key, "", None, False, temperature, "")
-
-        # Add the response to the conversation
-        conversation.append((current_speaker['name'], response))
-
-        # Switch speakers
-        current_speaker, other_speaker = other_speaker, current_speaker
-
-    # Convert the conversation to a list of strings for output
-    return [f"{msg[0]}: {msg[1]}" if isinstance(msg, tuple) else msg for msg in conversation]
-
-
-def create_multiple_character_chat_tab():
-    with gr.TabItem("Multi-Character Chat"):
-        characters, conversation, current_character, other_character = character_interaction_setup()
-
-        with gr.Blocks() as character_interaction:
-            gr.Markdown("# Multi-Character Chat")
-
-            with gr.Row():
-                num_characters = gr.Dropdown(label="Number of Characters", choices=["2", "3", "4"], value="2")
-                character_selectors = [gr.Dropdown(label=f"Character {i + 1}", choices=list(characters.keys())) for i in
-                                       range(4)]
-
-            api_endpoint = gr.Dropdown(label="API Endpoint",
-                                       choices=["Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek",
-                                                "Mistral",
-                                                "OpenRouter", "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM",
-                                                "ollama", "HuggingFace",
-                                                "Custom-OpenAI-API"],
-                                        value="HuggingFace")
-            api_key = gr.Textbox(label="API Key (if required)", type="password")
-            temperature = gr.Slider(label="Temperature", minimum=0.1, maximum=1.0, step=0.1, value=0.7)
-            scenario = gr.Textbox(label="Scenario (optional)", lines=3)
-
-            chat_display = gr.Chatbot(label="Character Interaction")
-            current_index = gr.State(0)
-
-            next_turn_btn = gr.Button("Next Turn")
-            narrator_input = gr.Textbox(label="Narrator Input", placeholder="Add a narration or description...")
-            add_narration_btn = gr.Button("Add Narration")
-            error_box = gr.Textbox(label="Error Messages", visible=False)
-            reset_btn = gr.Button("Reset Conversation")
-            chat_media_name = gr.Textbox(label="Custom Chat Name(optional)", visible=True)
-            save_chat_history_to_db = gr.Button("Save Chat History to DataBase")
-
-            def update_character_selectors(num):
-                return [gr.update(visible=True) if i < int(num) else gr.update(visible=False) for i in range(4)]
-
-            num_characters.change(
-                update_character_selectors,
-                inputs=[num_characters],
-                outputs=character_selectors
-            )
-
-            def reset_conversation():
-                return [], 0, gr.update(value=""), gr.update(value="")
-
-            def take_turn(conversation, current_index, char1, char2, char3, char4, api_endpoint, api_key, temperature,
-                          scenario):
-                char_selectors = [char for char in [char1, char2, char3, char4] if char]  # Remove None values
-                num_chars = len(char_selectors)
-
-                if num_chars == 0:
-                    return conversation, current_index  # No characters selected, return without changes
-
-                if not conversation:
-                    conversation = []
-                    if scenario:
-                        conversation.append(("Scenario", scenario))
-
-                current_character = char_selectors[current_index % num_chars]
-                next_index = (current_index + 1) % num_chars
-
-                prompt = f"Character speaking: {current_character}\nOther characters: {', '.join(char for char in char_selectors if char != current_character)}\n"
-                prompt += "Generate the next part of the conversation, including character dialogues and actions. Characters should speak in first person."
-
-                response, new_conversation, _ = chat_wrapper(prompt, conversation, {}, [], api_endpoint, api_key, "",
-                                                             None, False, temperature, "")
-
-                # Format the response
-                formatted_lines = []
-                for line in response.split('\n'):
-                    if ':' in line:
-                        speaker, text = line.split(':', 1)
-                        formatted_lines.append(f"**{speaker.strip()}**: {text.strip()}")
-                    else:
-                        formatted_lines.append(line)
-
-                formatted_response = '\n'.join(formatted_lines)
-
-                # Update the last message in the conversation with the formatted response
-                if new_conversation:
-                    new_conversation[-1] = (new_conversation[-1][0], formatted_response)
-                else:
-                    new_conversation.append((current_character, formatted_response))
-
-                return new_conversation, next_index
-
-            def add_narration(narration, conversation):
-                if narration:
-                    conversation.append(("Narrator", narration))
-                return conversation, ""
-
-            def take_turn_with_error_handling(conversation, current_index, char1, char2, char3, char4, api_endpoint,
-                                              api_key, temperature, scenario):
-                try:
-                    new_conversation, next_index = take_turn(conversation, current_index, char1, char2, char3, char4,
-                                                             api_endpoint, api_key, temperature, scenario)
-                    return new_conversation, next_index, gr.update(visible=False, value="")
-                except Exception as e:
-                    error_message = f"An error occurred: {str(e)}"
-                    return conversation, current_index, gr.update(visible=True, value=error_message)
-
-            # Define States for conversation_id and media_content, which are required for saving chat history
-            media_content = gr.State({})
-            conversation_id = gr.State(str(uuid.uuid4()))
-
-            next_turn_btn.click(
-                take_turn_with_error_handling,
-                inputs=[chat_display, current_index] + character_selectors + [api_endpoint, api_key, temperature,
-                                                                              scenario],
-                outputs=[chat_display, current_index, error_box]
-            )
-
-            add_narration_btn.click(
-                add_narration,
-                inputs=[narrator_input, chat_display],
-                outputs=[chat_display, narrator_input]
-            )
-
-            reset_btn.click(
-                reset_conversation,
-                outputs=[chat_display, current_index, scenario, narrator_input]
-            )
-
-        # FIXME - Implement saving chat history to database; look at Chat_UI.py for reference
-        save_chat_history_to_db.click(
-            save_chat_history_to_db_wrapper,
-            inputs=[chat_display, conversation_id, media_content, chat_media_name],
-            outputs=[conversation_id, gr.Textbox(label="Save Status")]
-        )
-
-        return character_interaction
-
-#
-# End of Multi-Character chat tab
-########################################################################################################################
-#
-# Narrator-Controlled Conversation Tab
-
-# From `Fuzzlewumper` on Reddit.
-def create_narrator_controlled_conversation_tab():
-    with gr.TabItem("Narrator-Controlled Conversation"):
-        gr.Markdown("# Narrator-Controlled Conversation")
-
-        with gr.Row():
-            with gr.Column(scale=1):
-                api_endpoint = gr.Dropdown(
-                    label="API Endpoint",
-                    choices=["Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral",
-                             "OpenRouter", "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace",
-                             "Custom-OpenAI-API"],
-                    value="HuggingFace"
-                )
-                api_key = gr.Textbox(label="API Key (if required)", type="password")
-                temperature = gr.Slider(label="Temperature", minimum=0.1, maximum=1.0, step=0.1, value=0.7)
-
-            with gr.Column(scale=2):
-                narrator_input = gr.Textbox(
-                    label="Narrator Input",
-                    placeholder="Set the scene or provide context...",
-                    lines=3
-                )
-
-        character_inputs = []
-        for i in range(4):  # Allow up to 4 characters
-            with gr.Row():
-                name = gr.Textbox(label=f"Character {i + 1} Name")
-                description = gr.Textbox(label=f"Character {i + 1} Description", lines=3)
-                character_inputs.append((name, description))
-
-        conversation_display = gr.Chatbot(label="Conversation", height=400)
-        user_input = gr.Textbox(label="Your Input (optional)", placeholder="Add your own dialogue or action...")
-
-        with gr.Row():
-            generate_btn = gr.Button("Generate Next Interaction")
-            reset_btn = gr.Button("Reset Conversation")
-            chat_media_name = gr.Textbox(label="Custom Chat Name(optional)", visible=True)
-            save_chat_history_to_db = gr.Button("Save Chat History to DataBase")
-
-        error_box = gr.Textbox(label="Error Messages", visible=False)
-
-        # Define States for conversation_id and media_content, which are required for saving chat history
-        conversation_id = gr.State(str(uuid.uuid4()))
-        media_content = gr.State({})
-
-        def generate_interaction(conversation, narrator_text, user_text, api_endpoint, api_key, temperature,
-                                 *character_data):
-            try:
-                characters = [{"name": name.strip(), "description": desc.strip()}
-                              for name, desc in zip(character_data[::2], character_data[1::2])
-                              if name.strip() and desc.strip()]
-
-                if not characters:
-                    raise ValueError("At least one character must be defined.")
-
-                prompt = f"Narrator: {narrator_text}\n\n"
-                for char in characters:
-                    prompt += f"Character '{char['name']}': {char['description']}\n"
-                prompt += "\nGenerate the next part of the conversation, including character dialogues and actions. "
-                prompt += "Characters should speak in first person. "
-                if user_text:
-                    prompt += f"\nIncorporate this user input: {user_text}"
-                prompt += "\nResponse:"
-
-                response, conversation, _ = chat_wrapper(prompt, conversation, {}, [], api_endpoint, api_key, "", None,
-                                                         False, temperature, "")
-
-                # Format the response
-                formatted_lines = []
-                for line in response.split('\n'):
-                    if ':' in line:
-                        speaker, text = line.split(':', 1)
-                        formatted_lines.append(f"**{speaker.strip()}**: {text.strip()}")
-                    else:
-                        formatted_lines.append(line)
-
-                formatted_response = '\n'.join(formatted_lines)
-
-                # Update the last message in the conversation with the formatted response
-                if conversation:
-                    conversation[-1] = (conversation[-1][0], formatted_response)
-                else:
-                    conversation.append((None, formatted_response))
-
-                return conversation, gr.update(value=""), gr.update(value=""), gr.update(visible=False, value="")
-            except Exception as e:
-                error_message = f"An error occurred: {str(e)}"
-                return conversation, gr.update(), gr.update(), gr.update(visible=True, value=error_message)
-
-        def reset_conversation():
-            return [], gr.update(value=""), gr.update(value=""), gr.update(visible=False, value="")
-
-        generate_btn.click(
-            generate_interaction,
-            inputs=[conversation_display, narrator_input, user_input, api_endpoint, api_key, temperature] +
-                   [input for char_input in character_inputs for input in char_input],
-            outputs=[conversation_display, narrator_input, user_input, error_box]
-        )
-
-        reset_btn.click(
-            reset_conversation,
-            outputs=[conversation_display, narrator_input, user_input, error_box]
-        )
-
-        # FIXME - Implement saving chat history to database; look at Chat_UI.py for reference
-        save_chat_history_to_db.click(
-            save_chat_history_to_db_wrapper,
-            inputs=[conversation_display, conversation_id, media_content, chat_media_name],
-            outputs=[conversation_id, gr.Textbox(label="Save Status")]
-        )
-
-
-    return api_endpoint, api_key, temperature, narrator_input, conversation_display, user_input, generate_btn, reset_btn, error_box
-
-#
-# End of Narrator-Controlled Conversation tab
-########################################################################################################################

App_Function_Libraries/Gradio_UI/Character_interaction_tab.py

@@ -1,511 +0,0 @@
-# Character_Interaction_tab.py
-# Description: This file contains the functions that are used for Character Interactions in the Gradio UI.
-#
-# Imports
-import base64
-import io
-import uuid
-from datetime import datetime as datetime
-import logging
-import json
-import os
-from typing import List, Dict, Tuple, Union
-
-#
-# External Imports
-import gradio as gr
-from PIL import Image
-#
-# Local Imports
-from App_Function_Libraries.Chat import chat, load_characters, save_chat_history_to_db_wrapper
-from App_Function_Libraries.Gradio_UI.Chat_ui import chat_wrapper
-from App_Function_Libraries.Gradio_UI.Writing_tab import generate_writing_feedback
-#
-########################################################################################################################
-#
-# Single-Character chat Functions:
-# FIXME - add these functions to the Personas library
-
-def chat_with_character(user_message, history, char_data, api_name_input, api_key):
-    if char_data is None:
-        return history, "Please import a character card first."
-
-    bot_message = generate_writing_feedback(user_message, char_data['name'], "Overall", api_name_input,
-                                            api_key)
-    history.append((user_message, bot_message))
-    return history, ""
-
-
-def import_character_card(file):
-    if file is None:
-        logging.warning("No file provided for character card import")
-        return None
-    try:
-        if file.name.lower().endswith(('.png', '.webp')):
-            logging.info(f"Attempting to import character card from image: {file.name}")
-            json_data = extract_json_from_image(file)
-            if json_data:
-                logging.info("JSON data extracted from image, attempting to parse")
-                card_data = import_character_card_json(json_data)
-                if card_data:
-                    # Save the image data
-                    with Image.open(file) as img:
-                        img_byte_arr = io.BytesIO()
-                        img.save(img_byte_arr, format='PNG')
-                        card_data['image'] = base64.b64encode(img_byte_arr.getvalue()).decode('utf-8')
-                return card_data
-            else:
-                logging.warning("No JSON data found in the image")
-        else:
-            logging.info(f"Attempting to import character card from JSON file: {file.name}")
-            content = file.read().decode('utf-8')
-            return import_character_card_json(content)
-    except Exception as e:
-        logging.error(f"Error importing character card: {e}")
-    return None
-
-
-def import_character_card_json(json_content):
-    try:
-        # Remove any leading/trailing whitespace
-        json_content = json_content.strip()
-
-        # Log the first 100 characters of the content
-        logging.debug(f"JSON content (first 100 chars): {json_content[:100]}...")
-
-        card_data = json.loads(json_content)
-        logging.debug(f"Parsed JSON data keys: {list(card_data.keys())}")
-        if 'spec' in card_data and card_data['spec'] == 'chara_card_v2':
-            logging.info("Detected V2 character card")
-            return card_data['data']
-        else:
-            logging.info("Assuming V1 character card")
-            return card_data
-    except json.JSONDecodeError as e:
-        logging.error(f"JSON decode error: {e}")
-        logging.error(f"Problematic JSON content: {json_content[:500]}...")
-    except Exception as e:
-        logging.error(f"Unexpected error parsing JSON: {e}")
-    return None
-
-
-def extract_json_from_image(image_file):
-    logging.debug(f"Attempting to extract JSON from image: {image_file.name}")
-    try:
-        with Image.open(image_file) as img:
-            logging.debug("Image opened successfully")
-            metadata = img.info
-            if 'chara' in metadata:
-                logging.debug("Found 'chara' in image metadata")
-                chara_content = metadata['chara']
-                logging.debug(f"Content of 'chara' metadata (first 100 chars): {chara_content[:100]}...")
-                try:
-                    decoded_content = base64.b64decode(chara_content).decode('utf-8')
-                    logging.debug(f"Decoded content (first 100 chars): {decoded_content[:100]}...")
-                    return decoded_content
-                except Exception as e:
-                    logging.error(f"Error decoding base64 content: {e}")
-
-            logging.debug("'chara' not found in metadata, checking for base64 encoded data")
-            raw_data = img.tobytes()
-            possible_json = raw_data.split(b'{', 1)[-1].rsplit(b'}', 1)[0]
-            if possible_json:
-                try:
-                    decoded = base64.b64decode(possible_json).decode('utf-8')
-                    if decoded.startswith('{') and decoded.endswith('}'):
-                        logging.debug("Found and decoded base64 JSON data")
-                        return '{' + decoded + '}'
-                except Exception as e:
-                    logging.error(f"Error decoding base64 data: {e}")
-
-            logging.warning("No JSON data found in the image")
-    except Exception as e:
-        logging.error(f"Error extracting JSON from image: {e}")
-    return None
-
-
-def load_chat_history(file):
-    try:
-        content = file.read().decode('utf-8')
-        chat_data = json.loads(content)
-        return chat_data['history'], chat_data['character']
-    except Exception as e:
-        logging.error(f"Error loading chat history: {e}")
-        return None, None
-
-
-#
-# End of X
-######################################################################################################################
-#
-# Multi-Character Chat Interface
-
-# FIXME - refactor and move these functions to the Character_Chat library so that it uses the same functions
-def character_interaction_setup():
-    characters = load_characters()
-    return characters, [], None, None
-
-
-def extract_character_response(response: Union[str, Tuple]) -> str:
-    if isinstance(response, tuple):
-        # If it's a tuple, try to extract the first string element
-        for item in response:
-            if isinstance(item, str):
-                return item.strip()
-        # If no string found, return a default message
-        return "I'm not sure how to respond."
-    elif isinstance(response, str):
-        # If it's already a string, just return it
-        return response.strip()
-    else:
-        # For any other type, return a default message
-        return "I'm having trouble forming a response."
-
-# def process_character_response(response: str) -> str:
-#     # Remove any leading explanatory text before the first '---'
-#     parts = response.split('---')
-#     if len(parts) > 1:
-#         return '---' + '---'.join(parts[1:])
-#     return response.strip()
-def process_character_response(response: Union[str, Tuple]) -> str:
-    if isinstance(response, tuple):
-        response = ' '.join(str(item) for item in response if isinstance(item, str))
-
-    if isinstance(response, str):
-        # Remove any leading explanatory text before the first '---'
-        parts = response.split('---')
-        if len(parts) > 1:
-            return '---' + '---'.join(parts[1:])
-        return response.strip()
-    else:
-        return "I'm having trouble forming a response."
-
-def character_turn(characters: Dict, conversation: List[Tuple[str, str]],
-                   current_character: str, other_characters: List[str],
-                   api_endpoint: str, api_key: str, temperature: float,
-                   scenario: str = "") -> Tuple[List[Tuple[str, str]], str]:
-    if not current_character or current_character not in characters:
-        return conversation, current_character
-
-    if not conversation and scenario:
-        conversation.append(("Scenario", scenario))
-
-    current_char = characters[current_character]
-    other_chars = [characters[char] for char in other_characters if char in characters and char != current_character]
-
-    prompt = f"{current_char['name']}'s personality: {current_char['personality']}\n"
-    for char in other_chars:
-        prompt += f"{char['name']}'s personality: {char['personality']}\n"
-    prompt += "Conversation so far:\n" + "\n".join([f"{sender}: {message}" for sender, message in conversation])
-    prompt += f"\n\nHow would {current_char['name']} respond?"
-
-    try:
-        response = chat_wrapper(prompt, conversation, {}, [], api_endpoint, api_key, "", None, False, temperature, "")
-        processed_response = process_character_response(response)
-        conversation.append((current_char['name'], processed_response))
-    except Exception as e:
-        error_message = f"Error generating response: {str(e)}"
-        conversation.append((current_char['name'], error_message))
-
-    return conversation, current_character
-
-
-def character_interaction(character1: str, character2: str, api_endpoint: str, api_key: str,
-                          num_turns: int, scenario: str, temperature: float,
-                          user_interjection: str = "") -> List[str]:
-    characters = load_characters()
-    char1 = characters[character1]
-    char2 = characters[character2]
-    conversation = []
-    current_speaker = char1
-    other_speaker = char2
-
-    # Add scenario to the conversation start
-    if scenario:
-        conversation.append(f"Scenario: {scenario}")
-
-    for turn in range(num_turns):
-        # Construct the prompt for the current speaker
-        prompt = f"{current_speaker['name']}'s personality: {current_speaker['personality']}\n"
-        prompt += f"{other_speaker['name']}'s personality: {other_speaker['personality']}\n"
-        prompt += f"Conversation so far:\n" + "\n".join(
-            [msg if isinstance(msg, str) else f"{msg[0]}: {msg[1]}" for msg in conversation])
-
-        # Add user interjection if provided
-        if user_interjection and turn == num_turns // 2:
-            prompt += f"\n\nUser interjection: {user_interjection}\n"
-            conversation.append(f"User: {user_interjection}")
-
-        prompt += f"\n\nHow would {current_speaker['name']} respond?"
-
-        # FIXME - figure out why the double print is happening
-        # Get response from the LLM
-        response = chat_wrapper(prompt, conversation, {}, [], api_endpoint, api_key, "", None, False, temperature, "")
-
-        # Add the response to the conversation
-        conversation.append((current_speaker['name'], response))
-
-        # Switch speakers
-        current_speaker, other_speaker = other_speaker, current_speaker
-
-    # Convert the conversation to a list of strings for output
-    return [f"{msg[0]}: {msg[1]}" if isinstance(msg, tuple) else msg for msg in conversation]
-
-
-def create_multiple_character_chat_tab():
-    with gr.TabItem("Multi-Character Chat", visible=True):
-        characters, conversation, current_character, other_character = character_interaction_setup()
-
-        with gr.Blocks() as character_interaction:
-            gr.Markdown("# Multi-Character Chat")
-
-            with gr.Row():
-                num_characters = gr.Dropdown(label="Number of Characters", choices=["2", "3", "4"], value="2")
-                character_selectors = [gr.Dropdown(label=f"Character {i + 1}", choices=list(characters.keys())) for i in
-                                       range(4)]
-
-            api_endpoint = gr.Dropdown(label="API Endpoint",
-                                       choices=["Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek",
-                                                "Mistral",
-                                                "OpenRouter", "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM",
-                                                "ollama", "HuggingFace",
-                                                "Custom-OpenAI-API"],
-                                        value="HuggingFace")
-            api_key = gr.Textbox(label="API Key (if required)", type="password")
-            temperature = gr.Slider(label="Temperature", minimum=0.1, maximum=1.0, step=0.1, value=0.7)
-            scenario = gr.Textbox(label="Scenario (optional)", lines=3)
-
-            chat_display = gr.Chatbot(label="Character Interaction")
-            current_index = gr.State(0)
-
-            next_turn_btn = gr.Button("Next Turn")
-            narrator_input = gr.Textbox(label="Narrator Input", placeholder="Add a narration or description...")
-            add_narration_btn = gr.Button("Add Narration")
-            error_box = gr.Textbox(label="Error Messages", visible=False)
-            reset_btn = gr.Button("Reset Conversation")
-            chat_media_name = gr.Textbox(label="Custom Chat Name(optional)", visible=True)
-            save_chat_history_to_db = gr.Button("Save Chat History to DataBase")
-
-            def update_character_selectors(num):
-                return [gr.update(visible=True) if i < int(num) else gr.update(visible=False) for i in range(4)]
-
-            num_characters.change(
-                update_character_selectors,
-                inputs=[num_characters],
-                outputs=character_selectors
-            )
-
-            def reset_conversation():
-                return [], 0, gr.update(value=""), gr.update(value="")
-
-            def take_turn(conversation, current_index, char1, char2, char3, char4, api_endpoint, api_key, temperature,
-                          scenario):
-                char_selectors = [char for char in [char1, char2, char3, char4] if char]  # Remove None values
-                num_chars = len(char_selectors)
-
-                if num_chars == 0:
-                    return conversation, current_index  # No characters selected, return without changes
-
-                if not conversation:
-                    conversation = []
-                    if scenario:
-                        conversation.append(("Scenario", scenario))
-
-                current_character = char_selectors[current_index % num_chars]
-                next_index = (current_index + 1) % num_chars
-
-                prompt = f"Character speaking: {current_character}\nOther characters: {', '.join(char for char in char_selectors if char != current_character)}\n"
-                prompt += "Generate the next part of the conversation, including character dialogues and actions. Characters should speak in first person."
-
-                response, new_conversation, _ = chat_wrapper(prompt, conversation, {}, [], api_endpoint, api_key, "",
-                                                             None, False, temperature, "")
-
-                # Format the response
-                formatted_lines = []
-                for line in response.split('\n'):
-                    if ':' in line:
-                        speaker, text = line.split(':', 1)
-                        formatted_lines.append(f"**{speaker.strip()}**: {text.strip()}")
-                    else:
-                        formatted_lines.append(line)
-
-                formatted_response = '\n'.join(formatted_lines)
-
-                # Update the last message in the conversation with the formatted response
-                if new_conversation:
-                    new_conversation[-1] = (new_conversation[-1][0], formatted_response)
-                else:
-                    new_conversation.append((current_character, formatted_response))
-
-                return new_conversation, next_index
-
-            def add_narration(narration, conversation):
-                if narration:
-                    conversation.append(("Narrator", narration))
-                return conversation, ""
-
-            def take_turn_with_error_handling(conversation, current_index, char1, char2, char3, char4, api_endpoint,
-                                              api_key, temperature, scenario):
-                try:
-                    new_conversation, next_index = take_turn(conversation, current_index, char1, char2, char3, char4,
-                                                             api_endpoint, api_key, temperature, scenario)
-                    return new_conversation, next_index, gr.update(visible=False, value="")
-                except Exception as e:
-                    error_message = f"An error occurred: {str(e)}"
-                    return conversation, current_index, gr.update(visible=True, value=error_message)
-
-            # Define States for conversation_id and media_content, which are required for saving chat history
-            media_content = gr.State({})
-            conversation_id = gr.State(str(uuid.uuid4()))
-
-            next_turn_btn.click(
-                take_turn_with_error_handling,
-                inputs=[chat_display, current_index] + character_selectors + [api_endpoint, api_key, temperature,
-                                                                              scenario],
-                outputs=[chat_display, current_index, error_box]
-            )
-
-            add_narration_btn.click(
-                add_narration,
-                inputs=[narrator_input, chat_display],
-                outputs=[chat_display, narrator_input]
-            )
-
-            reset_btn.click(
-                reset_conversation,
-                outputs=[chat_display, current_index, scenario, narrator_input]
-            )
-
-        # FIXME - Implement saving chat history to database; look at Chat_UI.py for reference
-        save_chat_history_to_db.click(
-            save_chat_history_to_db_wrapper,
-            inputs=[chat_display, conversation_id, media_content, chat_media_name],
-            outputs=[conversation_id, gr.Textbox(label="Save Status")]
-        )
-
-        return character_interaction
-
-#
-# End of Multi-Character chat tab
-########################################################################################################################
-#
-# Narrator-Controlled Conversation Tab
-
-# From `Fuzzlewumper` on Reddit.
-def create_narrator_controlled_conversation_tab():
-    with gr.TabItem("Narrator-Controlled Conversation", visible=True):
-        gr.Markdown("# Narrator-Controlled Conversation")
-
-        with gr.Row():
-            with gr.Column(scale=1):
-                api_endpoint = gr.Dropdown(
-                    label="API Endpoint",
-                    choices=["Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral",
-                             "OpenRouter", "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace",
-                             "Custom-OpenAI-API"],
-                    value="HuggingFace"
-                )
-                api_key = gr.Textbox(label="API Key (if required)", type="password")
-                temperature = gr.Slider(label="Temperature", minimum=0.1, maximum=1.0, step=0.1, value=0.7)
-
-            with gr.Column(scale=2):
-                narrator_input = gr.Textbox(
-                    label="Narrator Input",
-                    placeholder="Set the scene or provide context...",
-                    lines=3
-                )
-
-        character_inputs = []
-        for i in range(4):  # Allow up to 4 characters
-            with gr.Row():
-                name = gr.Textbox(label=f"Character {i + 1} Name")
-                description = gr.Textbox(label=f"Character {i + 1} Description", lines=3)
-                character_inputs.append((name, description))
-
-        conversation_display = gr.Chatbot(label="Conversation", height=400)
-        user_input = gr.Textbox(label="Your Input (optional)", placeholder="Add your own dialogue or action...")
-
-        with gr.Row():
-            generate_btn = gr.Button("Generate Next Interaction")
-            reset_btn = gr.Button("Reset Conversation")
-            chat_media_name = gr.Textbox(label="Custom Chat Name(optional)", visible=True)
-            save_chat_history_to_db = gr.Button("Save Chat History to DataBase")
-
-        error_box = gr.Textbox(label="Error Messages", visible=False)
-
-        # Define States for conversation_id and media_content, which are required for saving chat history
-        conversation_id = gr.State(str(uuid.uuid4()))
-        media_content = gr.State({})
-
-        def generate_interaction(conversation, narrator_text, user_text, api_endpoint, api_key, temperature,
-                                 *character_data):
-            try:
-                characters = [{"name": name.strip(), "description": desc.strip()}
-                              for name, desc in zip(character_data[::2], character_data[1::2])
-                              if name.strip() and desc.strip()]
-
-                if not characters:
-                    raise ValueError("At least one character must be defined.")
-
-                prompt = f"Narrator: {narrator_text}\n\n"
-                for char in characters:
-                    prompt += f"Character '{char['name']}': {char['description']}\n"
-                prompt += "\nGenerate the next part of the conversation, including character dialogues and actions. "
-                prompt += "Characters should speak in first person. "
-                if user_text:
-                    prompt += f"\nIncorporate this user input: {user_text}"
-                prompt += "\nResponse:"
-
-                response, conversation, _ = chat_wrapper(prompt, conversation, {}, [], api_endpoint, api_key, "", None,
-                                                         False, temperature, "")
-
-                # Format the response
-                formatted_lines = []
-                for line in response.split('\n'):
-                    if ':' in line:
-                        speaker, text = line.split(':', 1)
-                        formatted_lines.append(f"**{speaker.strip()}**: {text.strip()}")
-                    else:
-                        formatted_lines.append(line)
-
-                formatted_response = '\n'.join(formatted_lines)
-
-                # Update the last message in the conversation with the formatted response
-                if conversation:
-                    conversation[-1] = (conversation[-1][0], formatted_response)
-                else:
-                    conversation.append((None, formatted_response))
-
-                return conversation, gr.update(value=""), gr.update(value=""), gr.update(visible=False, value="")
-            except Exception as e:
-                error_message = f"An error occurred: {str(e)}"
-                return conversation, gr.update(), gr.update(), gr.update(visible=True, value=error_message)
-
-        def reset_conversation():
-            return [], gr.update(value=""), gr.update(value=""), gr.update(visible=False, value="")
-
-        generate_btn.click(
-            generate_interaction,
-            inputs=[conversation_display, narrator_input, user_input, api_endpoint, api_key, temperature] +
-                   [input for char_input in character_inputs for input in char_input],
-            outputs=[conversation_display, narrator_input, user_input, error_box]
-        )
-
-        reset_btn.click(
-            reset_conversation,
-            outputs=[conversation_display, narrator_input, user_input, error_box]
-        )
-
-        # FIXME - Implement saving chat history to database; look at Chat_UI.py for reference
-        save_chat_history_to_db.click(
-            save_chat_history_to_db_wrapper,
-            inputs=[conversation_display, conversation_id, media_content, chat_media_name],
-            outputs=[conversation_id, gr.Textbox(label="Save Status")]
-        )
-
-
-    return api_endpoint, api_key, temperature, narrator_input, conversation_display, user_input, generate_btn, reset_btn, error_box
-
-#
-# End of Narrator-Controlled Conversation tab
-########################################################################################################################

App_Function_Libraries/Gradio_UI/Chat_Workflows.py

@@ -1,178 +0,0 @@
-# Chat_Workflows.py
-# Description: UI for Chat Workflows
-#
-# Imports
-import json
-import logging
-from pathlib import Path
-#
-# External Imports
-import gradio as gr
-#
-from App_Function_Libraries.Gradio_UI.Chat_ui import chat_wrapper, search_conversations, \
-    load_conversation
-from App_Function_Libraries.Chat import save_chat_history_to_db_wrapper
-#
-############################################################################################################
-#
-# Functions:
-
-# Load workflows from a JSON file
-json_path = Path('./Helper_Scripts/Workflows/Workflows.json')
-with json_path.open('r') as f:
-    workflows = json.load(f)
-
-
-def chat_workflows_tab():
-    with gr.TabItem("Chat Workflows", visible=True):
-        gr.Markdown("# Workflows using LLMs")
-        chat_history = gr.State([])
-        media_content = gr.State({})
-        selected_parts = gr.State([])
-        conversation_id = gr.State(None)
-        workflow_state = gr.State({"current_step": 0, "max_steps": 0, "conversation_id": None})
-
-        with gr.Row():
-            with gr.Column():
-                workflow_selector = gr.Dropdown(label="Select Workflow", choices=[wf['name'] for wf in workflows])
-                api_selector = gr.Dropdown(
-                    label="Select API Endpoint",
-                    choices=["Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral",
-                             "OpenRouter", "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace",
-                             "Custom-OpenAI-API"],
-                    value="HuggingFace"
-                )
-                api_key_input = gr.Textbox(label="API Key (optional)", type="password")
-                temperature = gr.Slider(label="Temperature", minimum=0.00, maximum=1.0, step=0.05, value=0.7)
-                save_conversation = gr.Checkbox(label="Save Conversation", value=False)
-            with gr.Column():
-                gr.Markdown("Placeholder")
-        with gr.Row():
-            with gr.Column():
-                conversation_search = gr.Textbox(label="Search Conversations")
-                search_conversations_btn = gr.Button("Search Conversations")
-            with gr.Column():
-                previous_conversations = gr.Dropdown(label="Select Conversation", choices=[], interactive=True)
-                load_conversations_btn = gr.Button("Load Selected Conversation")
-        with gr.Row():
-            with gr.Column():
-                context_input = gr.Textbox(label="Initial Context", lines=5)
-                chatbot = gr.Chatbot(label="Workflow Chat")
-                msg = gr.Textbox(label="Your Input")
-                submit_btn = gr.Button("Submit")
-                clear_btn = gr.Button("Clear Chat")
-                chat_media_name = gr.Textbox(label="Custom Chat Name(optional)")
-                save_btn = gr.Button("Save Chat to Database")
-
-        def update_workflow_ui(workflow_name):
-            if not workflow_name:
-                return {"current_step": 0, "max_steps": 0, "conversation_id": None}, "", []
-            selected_workflow = next((wf for wf in workflows if wf['name'] == workflow_name), None)
-            if selected_workflow:
-                num_prompts = len(selected_workflow['prompts'])
-                context = selected_workflow.get('context', '')
-                first_prompt = selected_workflow['prompts'][0]
-                initial_chat = [(None, f"{first_prompt}")]
-                logging.info(f"Initializing workflow: {workflow_name} with {num_prompts} steps")
-                return {"current_step": 0, "max_steps": num_prompts, "conversation_id": None}, context, initial_chat
-            else:
-                logging.error(f"Selected workflow not found: {workflow_name}")
-                return {"current_step": 0, "max_steps": 0, "conversation_id": None}, "", []
-
-        def process_workflow_step(message, history, context, workflow_name, api_endpoint, api_key, workflow_state,
-                                  save_conv, temp):
-            logging.info(f"Process workflow step called with message: {message}")
-            logging.info(f"Current workflow state: {workflow_state}")
-            try:
-                selected_workflow = next((wf for wf in workflows if wf['name'] == workflow_name), None)
-                if not selected_workflow:
-                    logging.error(f"Selected workflow not found: {workflow_name}")
-                    return history, workflow_state, gr.update(interactive=True)
-
-                current_step = workflow_state["current_step"]
-                max_steps = workflow_state["max_steps"]
-
-                logging.info(f"Current step: {current_step}, Max steps: {max_steps}")
-
-                if current_step >= max_steps:
-                    logging.info("Workflow completed, disabling input")
-                    return history, workflow_state, gr.update(interactive=False)
-
-                prompt = selected_workflow['prompts'][current_step]
-                full_message = f"{context}\n\nStep {current_step + 1}: {prompt}\nUser: {message}"
-
-                logging.info(f"Calling chat_wrapper with full_message: {full_message[:100]}...")
-                bot_message, new_history, new_conversation_id = chat_wrapper(
-                    full_message, history, media_content.value, selected_parts.value,
-                    api_endpoint, api_key, "", workflow_state["conversation_id"],
-                    save_conv, temp, "You are a helpful assistant guiding through a workflow."
-                )
-
-                logging.info(f"Received bot_message: {bot_message[:100]}...")
-
-                next_step = current_step + 1
-                new_workflow_state = {
-                    "current_step": next_step,
-                    "max_steps": max_steps,
-                    "conversation_id": new_conversation_id
-                }
-
-                if next_step >= max_steps:
-                    logging.info("Workflow completed after this step")
-                    return new_history, new_workflow_state, gr.update(interactive=False)
-                else:
-                    next_prompt = selected_workflow['prompts'][next_step]
-                    new_history.append((None, f"Step {next_step + 1}: {next_prompt}"))
-                    logging.info(f"Moving to next step: {next_step}")
-                    return new_history, new_workflow_state, gr.update(interactive=True)
-            except Exception as e:
-                logging.error(f"Error in process_workflow_step: {str(e)}")
-                return history, workflow_state, gr.update(interactive=True)
-
-        workflow_selector.change(
-            update_workflow_ui,
-            inputs=[workflow_selector],
-            outputs=[workflow_state, context_input, chatbot]
-        )
-
-        submit_btn.click(
-            process_workflow_step,
-            inputs=[msg, chatbot, context_input, workflow_selector, api_selector, api_key_input, workflow_state,
-                    save_conversation, temperature],
-            outputs=[chatbot, workflow_state, msg]
-        ).then(
-            lambda: gr.update(value=""),
-            outputs=[msg]
-        )
-
-        clear_btn.click(
-            lambda: ([], {"current_step": 0, "max_steps": 0, "conversation_id": None}, ""),
-            outputs=[chatbot, workflow_state, context_input]
-        )
-
-        save_btn.click(
-            save_chat_history_to_db_wrapper,
-            inputs=[chatbot, conversation_id, media_content, chat_media_name],
-            outputs=[conversation_id, gr.Textbox(label="Save Status")]
-        )
-
-        search_conversations_btn.click(
-            search_conversations,
-            inputs=[conversation_search],
-            outputs=[previous_conversations]
-        )
-
-        load_conversations_btn.click(
-            lambda: ([], {"current_step": 0, "max_steps": 0, "conversation_id": None}, ""),
-            outputs=[chatbot, workflow_state, context_input]
-        ).then(
-            load_conversation,
-            inputs=[previous_conversations],
-            outputs=[chatbot, conversation_id]
-        )
-
-    return workflow_selector, api_selector, api_key_input, context_input, chatbot, msg, submit_btn, clear_btn, save_btn
-
-#
-# End of script
-############################################################################################################

App_Function_Libraries/Gradio_UI/Chat_ui.py

@@ -1,1185 +0,0 @@
-# Chat_ui.py
-# Description: Chat interface functions for Gradio
-#
-# Imports
-import html
-import json
-import logging
-import os
-import sqlite3
-from datetime import datetime
-#
-# External Imports
-import gradio as gr
-#
-# Local Imports
-from App_Function_Libraries.Chat import chat, save_chat_history, update_chat_content, save_chat_history_to_db_wrapper
-from App_Function_Libraries.DB.DB_Manager import add_chat_message, search_chat_conversations, create_chat_conversation, \
-    get_chat_messages, update_chat_message, delete_chat_message, load_preset_prompts, db
-from App_Function_Libraries.Gradio_UI.Gradio_Shared import update_dropdown, update_user_prompt
-
-
-#
-#
-########################################################################################################################
-#
-# Functions:
-
-
-def show_edit_message(selected):
-    if selected:
-        return gr.update(value=selected[0], visible=True), gr.update(value=selected[1], visible=True), gr.update(
-            visible=True)
-    return gr.update(visible=False), gr.update(visible=False), gr.update(visible=False)
-
-
-def show_delete_message(selected):
-    if selected:
-        return gr.update(value=selected[1], visible=True), gr.update(visible=True)
-    return gr.update(visible=False), gr.update(visible=False)
-
-
-def debug_output(media_content, selected_parts):
-    print(f"Debug - Media Content: {media_content}")
-    print(f"Debug - Selected Parts: {selected_parts}")
-    return ""
-
-
-def update_selected_parts(use_content, use_summary, use_prompt):
-    selected_parts = []
-    if use_content:
-        selected_parts.append("content")
-    if use_summary:
-        selected_parts.append("summary")
-    if use_prompt:
-        selected_parts.append("prompt")
-    print(f"Debug - Update Selected Parts: {selected_parts}")
-    return selected_parts
-
-
-# Old update_user_prompt shim for backwards compatibility
-def get_system_prompt(preset_name):
-    # For backwards compatibility
-    prompts = update_user_prompt(preset_name)
-    return prompts["system_prompt"]
-
-def clear_chat():
-    """
-    Return empty list for chatbot and None for conversation_id
-    @return:
-    """
-    return gr.update(value=[]), None
-
-
-def clear_chat_single():
-    """
-    Clears the chatbot and chat history.
-
-    Returns:
-        list: Empty list for chatbot messages.
-        list: Empty list for chat history.
-    """
-    return [], []
-
-# FIXME - add additional features....
-def chat_wrapper(message, history, media_content, selected_parts, api_endpoint, api_key, custom_prompt, conversation_id,
-                 save_conversation, temperature, system_prompt, max_tokens=None, top_p=None, frequency_penalty=None,
-                 presence_penalty=None, stop_sequence=None):
-    try:
-        if save_conversation:
-            if conversation_id is None:
-                # Create a new conversation
-                media_id = media_content.get('id', None)
-                conversation_name = f"Chat about {media_content.get('title', 'Unknown Media')} - {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
-                conversation_id = create_chat_conversation(media_id, conversation_name)
-
-            # Add user message to the database
-            user_message_id = add_chat_message(conversation_id, "user", message)
-
-        # Include the selected parts and custom_prompt only for the first message
-        if not history and selected_parts:
-            message_body = "\n".join(selected_parts)
-            full_message = f"{custom_prompt}\n\n{message}\n\n{message_body}"
-        elif custom_prompt:
-            full_message = f"{custom_prompt}\n\n{message}"
-        else:
-            full_message = message
-
-        # Generate bot response
-        bot_message = chat(full_message, history, media_content, selected_parts, api_endpoint, api_key, custom_prompt,
-                           temperature, system_prompt)
-
-        logging.debug(f"Bot message being returned: {bot_message}")
-
-        if save_conversation:
-            # Add assistant message to the database
-            add_chat_message(conversation_id, "assistant", bot_message)
-
-        # Update history
-        new_history = history + [(message, bot_message)]
-
-        return bot_message, new_history, conversation_id
-    except Exception as e:
-        logging.error(f"Error in chat wrapper: {str(e)}")
-        return "An error occurred.", history, conversation_id
-
-def search_conversations(query):
-    try:
-        conversations = search_chat_conversations(query)
-        if not conversations:
-            print(f"Debug - Search Conversations - No results found for query: {query}")
-            return gr.update(choices=[])
-
-        conversation_options = [
-            (f"{c['conversation_name']} (Media: {c['media_title']}, ID: {c['id']})", c['id'])
-            for c in conversations
-        ]
-        print(f"Debug - Search Conversations - Options: {conversation_options}")
-        return gr.update(choices=conversation_options)
-    except Exception as e:
-        print(f"Debug - Search Conversations - Error: {str(e)}")
-        return gr.update(choices=[])
-
-
-def load_conversation(conversation_id):
-    if not conversation_id:
-        return [], None
-
-    messages = get_chat_messages(conversation_id)
-    history = [
-        (msg['message'], None) if msg['sender'] == 'user' else (None, msg['message'])
-        for msg in messages
-    ]
-    return history, conversation_id
-
-
-def update_message_in_chat(message_id, new_text, history):
-    update_chat_message(message_id, new_text)
-    updated_history = [(msg1, msg2) if msg1[1] != message_id and msg2[1] != message_id
-                       else ((new_text, msg1[1]) if msg1[1] == message_id else (new_text, msg2[1]))
-                       for msg1, msg2 in history]
-    return updated_history
-
-
-def delete_message_from_chat(message_id, history):
-    delete_chat_message(message_id)
-    updated_history = [(msg1, msg2) for msg1, msg2 in history if msg1[1] != message_id and msg2[1] != message_id]
-    return updated_history
-
-
-def regenerate_last_message(history, media_content, selected_parts, api_endpoint, api_key, custom_prompt, temperature, system_prompt):
-    if not history:
-        return history, "No messages to regenerate."
-
-    last_entry = history[-1]
-    last_user_message, last_bot_message = last_entry
-
-    if last_bot_message is None:
-        return history, "The last message is not from the bot."
-
-    new_history = history[:-1]
-
-    if not last_user_message:
-        return new_history, "No user message to regenerate the bot response."
-
-    full_message = last_user_message
-
-    bot_message = chat(
-        full_message,
-        new_history,
-        media_content,
-        selected_parts,
-        api_endpoint,
-        api_key,
-        custom_prompt,
-        temperature,
-        system_prompt
-    )
-
-    new_history.append((last_user_message, bot_message))
-
-    return new_history, "Last message regenerated successfully."
-
-def create_chat_interface():
-    custom_css = """
-    .chatbot-container .message-wrap .message {
-        font-size: 14px !important;
-    }
-    """
-    with gr.TabItem("Remote LLM Chat (Horizontal)", visible=True):
-        gr.Markdown("# Chat with a designated LLM Endpoint, using your selected item as starting context")
-        chat_history = gr.State([])
-        media_content = gr.State({})
-        selected_parts = gr.State([])
-        conversation_id = gr.State(None)
-
-        with gr.Row():
-            with gr.Column(scale=1):
-                search_query_input = gr.Textbox(label="Search Query", placeholder="Enter your search query here...")
-                search_type_input = gr.Radio(choices=["Title", "URL", "Keyword", "Content"], value="Title",
-                                             label="Search By")
-                search_button = gr.Button("Search")
-                items_output = gr.Dropdown(label="Select Item", choices=[], interactive=True)
-                item_mapping = gr.State({})
-                with gr.Row():
-                    use_content = gr.Checkbox(label="Use Content")
-                    use_summary = gr.Checkbox(label="Use Summary")
-                    use_prompt = gr.Checkbox(label="Use Prompt")
-                    save_conversation = gr.Checkbox(label="Save Conversation", value=False, visible=True)
-                with gr.Row():
-                    temperature = gr.Slider(label="Temperature", minimum=0.00, maximum=1.0, step=0.05, value=0.7)
-                with gr.Row():
-                    conversation_search = gr.Textbox(label="Search Conversations")
-                with gr.Row():
-                    search_conversations_btn = gr.Button("Search Conversations")
-                with gr.Row():
-                    previous_conversations = gr.Dropdown(label="Select Conversation", choices=[], interactive=True)
-                with gr.Row():
-                    load_conversations_btn = gr.Button("Load Selected Conversation")
-
-                api_endpoint = gr.Dropdown(label="Select API Endpoint",
-                                           choices=["Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek",
-                                                    "Mistral", "OpenRouter",
-                                                    "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama",
-                                                    "HuggingFace"])
-                api_key = gr.Textbox(label="API Key (if required)", type="password")
-                custom_prompt_checkbox = gr.Checkbox(label="Use a Custom Prompt",
-                                                     value=False,
-                                                     visible=True)
-                preset_prompt_checkbox = gr.Checkbox(label="Use a pre-set Prompt",
-                                                     value=False,
-                                                     visible=True)
-                preset_prompt = gr.Dropdown(label="Select Preset Prompt",
-                                            choices=load_preset_prompts(),
-                                            visible=False)
-                user_prompt = gr.Textbox(label="Custom Prompt",
-                                         placeholder="Enter custom prompt here",
-                                         lines=3,
-                                         visible=False)
-                system_prompt_input = gr.Textbox(label="System Prompt",
-                                                 value="You are a helpful AI assitant",
-                                                 lines=3,
-                                                 visible=False)
-            with gr.Column(scale=2):
-                chatbot = gr.Chatbot(height=600, elem_classes="chatbot-container")
-                msg = gr.Textbox(label="Enter your message")
-                submit = gr.Button("Submit")
-                regenerate_button = gr.Button("Regenerate Last Message")
-                clear_chat_button = gr.Button("Clear Chat")
-
-                edit_message_id = gr.Number(label="Message ID to Edit", visible=False)
-                edit_message_text = gr.Textbox(label="Edit Message", visible=False)
-                update_message_button = gr.Button("Update Message", visible=False)
-
-                delete_message_id = gr.Number(label="Message ID to Delete", visible=False)
-                delete_message_button = gr.Button("Delete Message", visible=False)
-
-                chat_media_name = gr.Textbox(label="Custom Chat Name(optional)")
-                save_chat_history_to_db = gr.Button("Save Chat History to DataBase")
-                save_chat_history_as_file = gr.Button("Save Chat History as File")
-                download_file = gr.File(label="Download Chat History")
-                save_status = gr.Textbox(label="Save Status", interactive=False)
-
-        # Restore original functionality
-        search_button.click(
-            fn=update_dropdown,
-            inputs=[search_query_input, search_type_input],
-            outputs=[items_output, item_mapping]
-        )
-
-        def save_chat_wrapper(history, conversation_id, media_content):
-            file_path = save_chat_history(history, conversation_id, media_content)
-            if file_path:
-                return file_path, f"Chat history saved successfully as {os.path.basename(file_path)}!"
-            else:
-                return None, "Error saving chat history. Please check the logs and try again."
-
-        save_chat_history_as_file.click(
-            save_chat_wrapper,
-            inputs=[chatbot, conversation_id, media_content],
-            outputs=[download_file, save_status]
-        )
-
-        def update_prompts(preset_name):
-            prompts = update_user_prompt(preset_name)
-            return (
-                gr.update(value=prompts["user_prompt"], visible=True),
-                gr.update(value=prompts["system_prompt"], visible=True)
-            )
-
-        def clear_chat():
-            return [], None  # Return empty list for chatbot and None for conversation_id
-
-        clear_chat_button.click(
-            clear_chat,
-            outputs=[chatbot, conversation_id]
-        )
-        preset_prompt.change(
-            update_prompts,
-            inputs=preset_prompt,
-            outputs=[user_prompt, system_prompt_input]
-        )
-        custom_prompt_checkbox.change(
-            fn=lambda x: (gr.update(visible=x), gr.update(visible=x)),
-            inputs=[custom_prompt_checkbox],
-            outputs=[user_prompt, system_prompt_input]
-        )
-        preset_prompt_checkbox.change(
-            fn=lambda x: gr.update(visible=x),
-            inputs=[preset_prompt_checkbox],
-            outputs=[preset_prompt]
-        )
-        submit.click(
-            chat_wrapper,
-            inputs=[msg, chatbot, media_content, selected_parts, api_endpoint, api_key, user_prompt, conversation_id,
-                    save_conversation, temperature, system_prompt_input],
-            outputs=[msg, chatbot, conversation_id]
-        ).then(  # Clear the message box after submission
-            lambda x: gr.update(value=""),
-            inputs=[chatbot],
-            outputs=[msg]
-        ).then(  # Clear the user prompt after the first message
-            lambda: (gr.update(value=""), gr.update(value="")),
-            outputs=[user_prompt, system_prompt_input]
-        )
-
-        items_output.change(
-            update_chat_content,
-            inputs=[items_output, use_content, use_summary, use_prompt, item_mapping],
-            outputs=[media_content, selected_parts]
-        )
-        use_content.change(update_selected_parts, inputs=[use_content, use_summary, use_prompt],
-                           outputs=[selected_parts])
-        use_summary.change(update_selected_parts, inputs=[use_content, use_summary, use_prompt],
-                           outputs=[selected_parts])
-        use_prompt.change(update_selected_parts, inputs=[use_content, use_summary, use_prompt],
-                          outputs=[selected_parts])
-        items_output.change(debug_output, inputs=[media_content, selected_parts], outputs=[])
-
-        search_conversations_btn.click(
-            search_conversations,
-            inputs=[conversation_search],
-            outputs=[previous_conversations]
-        )
-
-        load_conversations_btn.click(
-            clear_chat,
-            outputs=[chatbot, chat_history]
-        ).then(
-            load_conversation,
-            inputs=[previous_conversations],
-            outputs=[chatbot, conversation_id]
-        )
-
-        previous_conversations.change(
-            load_conversation,
-            inputs=[previous_conversations],
-            outputs=[chat_history]
-        )
-
-        update_message_button.click(
-            update_message_in_chat,
-            inputs=[edit_message_id, edit_message_text, chat_history],
-            outputs=[chatbot]
-        )
-
-        delete_message_button.click(
-            delete_message_from_chat,
-            inputs=[delete_message_id, chat_history],
-            outputs=[chatbot]
-        )
-
-        save_chat_history_as_file.click(
-            save_chat_history,
-            inputs=[chatbot, conversation_id],
-            outputs=[download_file]
-        )
-
-        save_chat_history_to_db.click(
-            save_chat_history_to_db_wrapper,
-            inputs=[chatbot, conversation_id, media_content, chat_media_name],
-            outputs=[conversation_id, gr.Textbox(label="Save Status")]
-        )
-
-        regenerate_button.click(
-            regenerate_last_message,
-            inputs=[chatbot, media_content, selected_parts, api_endpoint, api_key, user_prompt, temperature, system_prompt_input],
-            outputs=[chatbot, save_status]
-        )
-
-        chatbot.select(show_edit_message, None, [edit_message_text, edit_message_id, update_message_button])
-        chatbot.select(show_delete_message, None, [delete_message_id, delete_message_button])
-
-
-def create_chat_interface_stacked():
-    custom_css = """
-    .chatbot-container .message-wrap .message {
-        font-size: 14px !important;
-    }
-    """
-    with gr.TabItem("Remote LLM Chat - Stacked", visible=True):
-        gr.Markdown("# Stacked Chat")
-        chat_history = gr.State([])
-        media_content = gr.State({})
-        selected_parts = gr.State([])
-        conversation_id = gr.State(None)
-
-        with gr.Row():
-            with gr.Column():
-                search_query_input = gr.Textbox(label="Search Query", placeholder="Enter your search query here...")
-                search_type_input = gr.Radio(choices=["Title", "URL", "Keyword", "Content"], value="Title",
-                                             label="Search By")
-                search_button = gr.Button("Search")
-                items_output = gr.Dropdown(label="Select Item", choices=[], interactive=True)
-                item_mapping = gr.State({})
-                with gr.Row():
-                    use_content = gr.Checkbox(label="Use Content")
-                    use_summary = gr.Checkbox(label="Use Summary")
-                    use_prompt = gr.Checkbox(label="Use Prompt")
-                    save_conversation = gr.Checkbox(label="Save Conversation", value=False, visible=True)
-                    temp = gr.Slider(label="Temperature", minimum=0.00, maximum=1.0, step=0.05, value=0.7)
-                with gr.Row():
-                    conversation_search = gr.Textbox(label="Search Conversations")
-                with gr.Row():
-                    previous_conversations = gr.Dropdown(label="Select Conversation", choices=[], interactive=True)
-                with gr.Row():
-                    search_conversations_btn = gr.Button("Search Conversations")
-                    load_conversations_btn = gr.Button("Load Selected Conversation")
-            with gr.Column():
-                api_endpoint = gr.Dropdown(label="Select API Endpoint",
-                                           choices=["Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek",
-                                                    "OpenRouter", "Mistral", "Llama.cpp", "Kobold", "Ooba", "Tabbyapi",
-                                                    "VLLM", "ollama", "HuggingFace"])
-                api_key = gr.Textbox(label="API Key (if required)", type="password")
-                preset_prompt = gr.Dropdown(label="Select Preset Prompt",
-                                            choices=load_preset_prompts(),
-                                            visible=True)
-                system_prompt = gr.Textbox(label="System Prompt",
-                                           value="You are a helpful AI assistant.",
-                                           lines=3,
-                                           visible=True)
-                user_prompt = gr.Textbox(label="Custom User Prompt",
-                                         placeholder="Enter custom prompt here",
-                                         lines=3,
-                                         visible=True)
-                gr.Markdown("Scroll down for the chat window...")
-        with gr.Row():
-            with gr.Column(scale=1):
-                chatbot = gr.Chatbot(height=600, elem_classes="chatbot-container")
-                msg = gr.Textbox(label="Enter your message")
-        with gr.Row():
-            with gr.Column():
-                submit = gr.Button("Submit")
-                regenerate_button = gr.Button("Regenerate Last Message")
-                clear_chat_button = gr.Button("Clear Chat")
-                chat_media_name = gr.Textbox(label="Custom Chat Name(optional)", visible=True)
-                save_chat_history_to_db = gr.Button("Save Chat History to DataBase")
-                save_chat_history_as_file = gr.Button("Save Chat History as File")
-            with gr.Column():
-                download_file = gr.File(label="Download Chat History")
-
-        # Restore original functionality
-        search_button.click(
-            fn=update_dropdown,
-            inputs=[search_query_input, search_type_input],
-            outputs=[items_output, item_mapping]
-        )
-
-        def update_prompts(preset_name):
-            prompts = update_user_prompt(preset_name)
-            return (
-                gr.update(value=prompts["user_prompt"], visible=True),
-                gr.update(value=prompts["system_prompt"], visible=True)
-            )
-
-        clear_chat_button.click(
-            clear_chat,
-            outputs=[chatbot, conversation_id]
-        )
-        preset_prompt.change(
-            update_prompts,
-            inputs=preset_prompt,
-            outputs=[user_prompt, system_prompt]
-        )
-
-        submit.click(
-            chat_wrapper,
-            inputs=[msg, chatbot, media_content, selected_parts, api_endpoint, api_key, user_prompt,
-                    conversation_id, save_conversation, temp, system_prompt],
-            outputs=[msg, chatbot, conversation_id]
-        ).then(  # Clear the message box after submission
-            lambda x: gr.update(value=""),
-            inputs=[chatbot],
-            outputs=[msg]
-        ).then(  # Clear the user prompt after the first message
-            lambda: gr.update(value=""),
-            outputs=[user_prompt, system_prompt]
-        )
-
-        items_output.change(
-            update_chat_content,
-            inputs=[items_output, use_content, use_summary, use_prompt, item_mapping],
-            outputs=[media_content, selected_parts]
-        )
-        use_content.change(update_selected_parts, inputs=[use_content, use_summary, use_prompt],
-                           outputs=[selected_parts])
-        use_summary.change(update_selected_parts, inputs=[use_content, use_summary, use_prompt],
-                           outputs=[selected_parts])
-        use_prompt.change(update_selected_parts, inputs=[use_content, use_summary, use_prompt],
-                          outputs=[selected_parts])
-        items_output.change(debug_output, inputs=[media_content, selected_parts], outputs=[])
-
-        search_conversations_btn.click(
-            search_conversations,
-            inputs=[conversation_search],
-            outputs=[previous_conversations]
-        )
-
-        load_conversations_btn.click(
-            clear_chat,
-            outputs=[chatbot, chat_history]
-        ).then(
-            load_conversation,
-            inputs=[previous_conversations],
-            outputs=[chatbot, conversation_id]
-        )
-
-        previous_conversations.change(
-            load_conversation,
-            inputs=[previous_conversations],
-            outputs=[chat_history]
-        )
-
-        save_chat_history_as_file.click(
-            save_chat_history,
-            inputs=[chatbot, conversation_id],
-            outputs=[download_file]
-        )
-
-        save_chat_history_to_db.click(
-            save_chat_history_to_db_wrapper,
-            inputs=[chatbot, conversation_id, media_content, chat_media_name],
-            outputs=[conversation_id, gr.Textbox(label="Save Status")]
-        )
-
-        regenerate_button.click(
-            regenerate_last_message,
-            inputs=[chatbot, media_content, selected_parts, api_endpoint, api_key, user_prompt, temp, system_prompt],
-            outputs=[chatbot, gr.Textbox(label="Regenerate Status")]
-        )
-
-
-# FIXME - System prompts
-def create_chat_interface_multi_api():
-    custom_css = """
-    .chatbot-container .message-wrap .message {
-        font-size: 14px !important;
-    }
-    .chat-window {
-        height: 400px;
-        overflow-y: auto;
-    }
-    """
-    with gr.TabItem("One Prompt - Multiple APIs", visible=True):
-        gr.Markdown("# One Prompt but Multiple APIs Chat Interface")
-
-        with gr.Row():
-            with gr.Column(scale=1):
-                search_query_input = gr.Textbox(label="Search Query", placeholder="Enter your search query here...")
-                search_type_input = gr.Radio(choices=["Title", "URL", "Keyword", "Content"], value="Title",
-                                             label="Search By")
-                search_button = gr.Button("Search")
-                items_output = gr.Dropdown(label="Select Item", choices=[], interactive=True)
-                item_mapping = gr.State({})
-                with gr.Row():
-                    use_content = gr.Checkbox(label="Use Content")
-                    use_summary = gr.Checkbox(label="Use Summary")
-                    use_prompt = gr.Checkbox(label="Use Prompt")
-            with gr.Column():
-                preset_prompt = gr.Dropdown(label="Select Preset Prompt", choices=load_preset_prompts(), visible=True)
-                system_prompt = gr.Textbox(label="System Prompt", value="You are a helpful AI assistant.", lines=5)
-                user_prompt = gr.Textbox(label="Modify Prompt (Prefixed to your message every time)", lines=5, value="", visible=True)
-
-        with gr.Row():
-            chatbots = []
-            api_endpoints = []
-            api_keys = []
-            temperatures = []
-            regenerate_buttons = []
-            for i in range(3):
-                with gr.Column():
-                    gr.Markdown(f"### Chat Window {i + 1}")
-                    api_endpoint = gr.Dropdown(label=f"API Endpoint {i + 1}",
-                                               choices=["Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq",
-                                                        "DeepSeek", "Mistral", "OpenRouter", "Llama.cpp", "Kobold",
-                                                        "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace"])
-                    api_key = gr.Textbox(label=f"API Key {i + 1} (if required)", type="password")
-                    temperature = gr.Slider(label=f"Temperature {i + 1}", minimum=0.0, maximum=1.0, step=0.05,
-                                            value=0.7)
-                    chatbot = gr.Chatbot(height=800, elem_classes="chat-window")
-                    regenerate_button = gr.Button(f"Regenerate Last Message {i + 1}")
-                    chatbots.append(chatbot)
-                    api_endpoints.append(api_endpoint)
-                    api_keys.append(api_key)
-                    temperatures.append(temperature)
-                    regenerate_buttons.append(regenerate_button)
-
-        with gr.Row():
-            msg = gr.Textbox(label="Enter your message", scale=4)
-            submit = gr.Button("Submit", scale=1)
-            clear_chat_button = gr.Button("Clear All Chats")
-
-        # State variables
-        chat_history = [gr.State([]) for _ in range(3)]
-        media_content = gr.State({})
-        selected_parts = gr.State([])
-        conversation_id = gr.State(None)
-
-        # Event handlers
-        search_button.click(
-            fn=update_dropdown,
-            inputs=[search_query_input, search_type_input],
-            outputs=[items_output, item_mapping]
-        )
-
-        preset_prompt.change(update_user_prompt, inputs=preset_prompt, outputs=user_prompt)
-
-
-        def clear_all_chats():
-            return [[]] * 3 + [[]] * 3
-
-        clear_chat_button.click(
-            clear_all_chats,
-            outputs=chatbots + chat_history
-        )
-        def chat_wrapper_multi(message, custom_prompt, system_prompt, *args):
-            chat_histories = args[:3]
-            chatbots = args[3:6]
-            api_endpoints = args[6:9]
-            api_keys = args[9:12]
-            temperatures = args[12:15]
-            media_content = args[15]
-            selected_parts = args[16]
-
-            new_chat_histories = []
-            new_chatbots = []
-
-            for i in range(3):
-                # Call chat_wrapper with dummy values for conversation_id and save_conversation
-                bot_message, new_history, _ = chat_wrapper(
-                    message, chat_histories[i], media_content, selected_parts,
-                    api_endpoints[i], api_keys[i], custom_prompt, None,  # None for conversation_id
-                    False,  # False for save_conversation
-                    temperature=temperatures[i],
-                    system_prompt=system_prompt
-                )
-
-                new_chatbot = chatbots[i] + [(message, bot_message)]
-
-                new_chat_histories.append(new_history)
-                new_chatbots.append(new_chatbot)
-
-            return [gr.update(value="")] + new_chatbots + new_chat_histories
-
-
-        def regenerate_last_message(chat_history, chatbot, media_content, selected_parts, api_endpoint, api_key, custom_prompt, temperature, system_prompt):
-            if not chat_history:
-                return chatbot, chat_history, "No messages to regenerate."
-
-            last_entry = chat_history[-1]
-            last_user_message, last_bot_message = last_entry
-
-            if last_bot_message is None:
-                return chatbot, chat_history, "The last message is not from the bot."
-
-            new_history = chat_history[:-1]
-
-            if not last_user_message:
-                return chatbot[:-1], new_history, "No user message to regenerate the bot response."
-
-            bot_message = chat(
-                last_user_message,
-                new_history,
-                media_content,
-                selected_parts,
-                api_endpoint,
-                api_key,
-                custom_prompt,
-                temperature,
-                system_prompt
-            )
-
-            new_history.append((last_user_message, bot_message))
-            new_chatbot = chatbot[:-1] + [(last_user_message, bot_message)]
-
-            return new_chatbot, new_history, "Last message regenerated successfully."
-
-        for i in range(3):
-            regenerate_buttons[i].click(
-                regenerate_last_message,
-                inputs=[chat_history[i], chatbots[i], media_content, selected_parts, api_endpoints[i], api_keys[i], user_prompt, temperatures[i], system_prompt],
-                outputs=[chatbots[i], chat_history[i], gr.Textbox(label=f"Regenerate Status {i + 1}")]
-            )
-
-        # In the create_chat_interface_multi_api function:
-        submit.click(
-            chat_wrapper_multi,
-            inputs=[msg, user_prompt,
-                    system_prompt] + chat_history + chatbots + api_endpoints + api_keys + temperatures +
-                   [media_content, selected_parts],
-            outputs=[msg] + chatbots + chat_history
-        ).then(
-            lambda: (gr.update(value=""), gr.update(value="")),
-            outputs=[msg, user_prompt]
-        )
-
-        items_output.change(
-            update_chat_content,
-            inputs=[items_output, use_content, use_summary, use_prompt, item_mapping],
-            outputs=[media_content, selected_parts]
-        )
-
-        for checkbox in [use_content, use_summary, use_prompt]:
-            checkbox.change(
-                update_selected_parts,
-                inputs=[use_content, use_summary, use_prompt],
-                outputs=[selected_parts]
-            )
-
-
-
-def create_chat_interface_four():
-    custom_css = """
-    .chatbot-container .message-wrap .message {
-        font-size: 14px !important;
-    }
-    .chat-window {
-        height: 400px;
-        overflow-y: auto;
-    }
-    """
-
-    with gr.TabItem("Four Independent API Chats", visible=True):
-        gr.Markdown("# Four Independent API Chat Interfaces")
-
-        with gr.Row():
-            with gr.Column():
-                preset_prompt = gr.Dropdown(
-                    label="Select Preset Prompt",
-                    choices=load_preset_prompts(),
-                    visible=True
-                )
-                user_prompt = gr.Textbox(
-                    label="Modify Prompt",
-                    lines=3
-                )
-            with gr.Column():
-                gr.Markdown("Scroll down for the chat windows...")
-
-        chat_interfaces = []
-
-        def create_single_chat_interface(index, user_prompt_component):
-            with gr.Column():
-                gr.Markdown(f"### Chat Window {index + 1}")
-                api_endpoint = gr.Dropdown(
-                    label=f"API Endpoint {index + 1}",
-                    choices=[
-                        "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq",
-                        "DeepSeek", "Mistral", "OpenRouter", "Llama.cpp", "Kobold",
-                        "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace"
-                    ]
-                )
-                api_key = gr.Textbox(
-                    label=f"API Key {index + 1} (if required)",
-                    type="password"
-                )
-                temperature = gr.Slider(
-                    label=f"Temperature {index + 1}",
-                    minimum=0.0,
-                    maximum=1.0,
-                    step=0.05,
-                    value=0.7
-                )
-                chatbot = gr.Chatbot(height=400, elem_classes="chat-window")
-                msg = gr.Textbox(label=f"Enter your message for Chat {index + 1}")
-                submit = gr.Button(f"Submit to Chat {index + 1}")
-                regenerate_button = gr.Button(f"Regenerate Last Message {index + 1}")
-                clear_chat_button = gr.Button(f"Clear Chat {index + 1}")
-
-                # State to maintain chat history
-                chat_history = gr.State([])
-
-                # Append to chat_interfaces list
-                chat_interfaces.append({
-                    'api_endpoint': api_endpoint,
-                    'api_key': api_key,
-                    'temperature': temperature,
-                    'chatbot': chatbot,
-                    'msg': msg,
-                    'submit': submit,
-                    'regenerate_button': regenerate_button,
-                    'clear_chat_button': clear_chat_button,
-                    'chat_history': chat_history
-                })
-
-        # Create four chat interfaces arranged in a 2x2 grid
-        with gr.Row():
-            for i in range(2):
-                with gr.Column():
-                    for j in range(2):
-                        create_single_chat_interface(i * 2 + j, user_prompt)
-
-        # Update user_prompt based on preset_prompt selection
-        preset_prompt.change(
-            fn=update_user_prompt,
-            inputs=preset_prompt,
-            outputs=user_prompt
-        )
-
-        def chat_wrapper_single(message, chat_history, api_endpoint, api_key, temperature, user_prompt):
-            logging.debug(f"Chat Wrapper Single - Message: {message}, Chat History: {chat_history}")
-
-            new_msg, new_history, _ = chat_wrapper(
-                message,
-                chat_history,
-                {},  # Empty media_content
-                [],  # Empty selected_parts
-                api_endpoint,
-                api_key,
-                user_prompt,  # custom_prompt
-                None,  # conversation_id
-                False,  # save_conversation
-                temperature,  # temperature
-                system_prompt="",  # system_prompt
-                max_tokens=None,
-                top_p=None,
-                frequency_penalty=None,
-                presence_penalty=None,
-                stop_sequence=None
-            )
-            if "API request failed" not in new_msg:
-                chat_history.append((message, new_msg))
-            else:
-                logging.error(f"API request failed: {new_msg}")
-
-            return "", chat_history, chat_history
-
-        def regenerate_last_message(chat_history, api_endpoint, api_key, temperature, user_prompt):
-            if not chat_history:
-                return chat_history, chat_history, "No messages to regenerate."
-
-            last_user_message, _ = chat_history[-1]
-
-            new_msg, new_history, _ = chat_wrapper(
-                last_user_message,
-                chat_history[:-1],
-                {},  # Empty media_content
-                [],  # Empty selected_parts
-                api_endpoint,
-                api_key,
-                user_prompt,  # custom_prompt
-                None,  # conversation_id
-                False,  # save_conversation
-                temperature,  # temperature
-                system_prompt="",  # system_prompt
-                max_tokens=None,
-                top_p=None,
-                frequency_penalty=None,
-                presence_penalty=None,
-                stop_sequence=None
-            )
-
-            if "API request failed" not in new_msg:
-                new_history.append((last_user_message, new_msg))
-                return new_history, new_history, "Last message regenerated successfully."
-            else:
-                logging.error(f"API request failed during regeneration: {new_msg}")
-                return chat_history, chat_history, f"Failed to regenerate: {new_msg}"
-
-        # Attach click events for each chat interface
-        for interface in chat_interfaces:
-            interface['submit'].click(
-                chat_wrapper_single,
-                inputs=[
-                    interface['msg'],
-                    interface['chat_history'],
-                    interface['api_endpoint'],
-                    interface['api_key'],
-                    interface['temperature'],
-                    user_prompt
-                ],
-                outputs=[
-                    interface['msg'],
-                    interface['chatbot'],
-                    interface['chat_history']
-                ]
-            )
-
-            interface['regenerate_button'].click(
-                regenerate_last_message,
-                inputs=[
-                    interface['chat_history'],
-                    interface['api_endpoint'],
-                    interface['api_key'],
-                    interface['temperature'],
-                    user_prompt
-                ],
-                outputs=[
-                    interface['chatbot'],
-                    interface['chat_history'],
-                    gr.Textbox(label="Regenerate Status")
-                ]
-            )
-
-            interface['clear_chat_button'].click(
-                clear_chat_single,
-                inputs=[],
-                outputs=[interface['chatbot'], interface['chat_history']]
-            )
-
-
-def chat_wrapper_single(message, chat_history, chatbot, api_endpoint, api_key, temperature, media_content,
-                        selected_parts, conversation_id, save_conversation, user_prompt):
-    new_msg, new_history, new_conv_id = chat_wrapper(
-        message, chat_history, media_content, selected_parts,
-        api_endpoint, api_key, user_prompt, conversation_id,
-        save_conversation, temperature, system_prompt=""
-    )
-
-    if new_msg:
-        updated_chatbot = chatbot + [(message, new_msg)]
-    else:
-        updated_chatbot = chatbot
-
-    return new_msg, updated_chatbot, new_history, new_conv_id
-
-
-# FIXME - Finish implementing functions + testing/valdidation
-def create_chat_management_tab():
-    with gr.TabItem("Chat Management", visible=True):
-        gr.Markdown("# Chat Management")
-
-        with gr.Row():
-            search_query = gr.Textbox(label="Search Conversations")
-            search_button = gr.Button("Search")
-
-        conversation_list = gr.Dropdown(label="Select Conversation", choices=[])
-        conversation_mapping = gr.State({})
-
-        with gr.Tabs():
-            with gr.TabItem("Edit", visible=True):
-                chat_content = gr.TextArea(label="Chat Content (JSON)", lines=20, max_lines=50)
-                save_button = gr.Button("Save Changes")
-                delete_button = gr.Button("Delete Conversation", variant="stop")
-
-            with gr.TabItem("Preview", visible=True):
-                chat_preview = gr.HTML(label="Chat Preview")
-        result_message = gr.Markdown("")
-
-        def search_conversations(query):
-            conversations = search_chat_conversations(query)
-            choices = [f"{conv['conversation_name']} (Media: {conv['media_title']}, ID: {conv['id']})" for conv in
-                       conversations]
-            mapping = {choice: conv['id'] for choice, conv in zip(choices, conversations)}
-            return gr.update(choices=choices), mapping
-
-        def load_conversations(selected, conversation_mapping):
-            logging.info(f"Selected: {selected}")
-            logging.info(f"Conversation mapping: {conversation_mapping}")
-
-            try:
-                if selected and selected in conversation_mapping:
-                    conversation_id = conversation_mapping[selected]
-                    messages = get_chat_messages(conversation_id)
-                    conversation_data = {
-                        "conversation_id": conversation_id,
-                        "messages": messages
-                    }
-                    json_content = json.dumps(conversation_data, indent=2)
-
-                    # Create HTML preview
-                    html_preview = "<div style='max-height: 500px; overflow-y: auto;'>"
-                    for msg in messages:
-                        sender_style = "background-color: #e6f3ff;" if msg[
-                                                                           'sender'] == 'user' else "background-color: #f0f0f0;"
-                        html_preview += f"<div style='margin-bottom: 10px; padding: 10px; border-radius: 5px; {sender_style}'>"
-                        html_preview += f"<strong>{msg['sender']}:</strong> {html.escape(msg['message'])}<br>"
-                        html_preview += f"<small>Timestamp: {msg['timestamp']}</small>"
-                        html_preview += "</div>"
-                    html_preview += "</div>"
-
-                    logging.info("Returning json_content and html_preview")
-                    return json_content, html_preview
-                else:
-                    logging.warning("No conversation selected or not in mapping")
-                    return "", "<p>No conversation selected</p>"
-            except Exception as e:
-                logging.error(f"Error in load_conversations: {str(e)}")
-                return f"Error: {str(e)}", "<p>Error loading conversation</p>"
-
-        def validate_conversation_json(content):
-            try:
-                data = json.loads(content)
-                if not isinstance(data, dict):
-                    return False, "Invalid JSON structure: root should be an object"
-                if "conversation_id" not in data or not isinstance(data["conversation_id"], int):
-                    return False, "Missing or invalid conversation_id"
-                if "messages" not in data or not isinstance(data["messages"], list):
-                    return False, "Missing or invalid messages array"
-                for msg in data["messages"]:
-                    if not all(key in msg for key in ["sender", "message"]):
-                        return False, "Invalid message structure: missing required fields"
-                return True, data
-            except json.JSONDecodeError as e:
-                return False, f"Invalid JSON: {str(e)}"
-
-        def save_conversation(selected, conversation_mapping, content):
-            if not selected or selected not in conversation_mapping:
-                return "Please select a conversation before saving.", "<p>No changes made</p>"
-
-            conversation_id = conversation_mapping[selected]
-            is_valid, result = validate_conversation_json(content)
-
-            if not is_valid:
-                return f"Error: {result}", "<p>No changes made due to error</p>"
-
-            conversation_data = result
-            if conversation_data["conversation_id"] != conversation_id:
-                return "Error: Conversation ID mismatch.", "<p>No changes made due to ID mismatch</p>"
-
-            try:
-                with db.get_connection() as conn:
-                    conn.execute("BEGIN TRANSACTION")
-                    cursor = conn.cursor()
-
-                    # Backup original conversation
-                    cursor.execute("SELECT * FROM ChatMessages WHERE conversation_id = ?", (conversation_id,))
-                    original_messages = cursor.fetchall()
-                    backup_data = json.dumps({"conversation_id": conversation_id, "messages": original_messages})
-
-                    # You might want to save this backup_data somewhere
-
-                    # Delete existing messages
-                    cursor.execute("DELETE FROM ChatMessages WHERE conversation_id = ?", (conversation_id,))
-
-                    # Insert updated messages
-                    for message in conversation_data["messages"]:
-                        cursor.execute('''
-                            INSERT INTO ChatMessages (conversation_id, sender, message, timestamp)
-                            VALUES (?, ?, ?, COALESCE(?, CURRENT_TIMESTAMP))
-                        ''', (conversation_id, message["sender"], message["message"], message.get("timestamp")))
-
-                    conn.commit()
-
-                    # Create updated HTML preview
-                    html_preview = "<div style='max-height: 500px; overflow-y: auto;'>"
-                    for msg in conversation_data["messages"]:
-                        sender_style = "background-color: #e6f3ff;" if msg[
-                                                                           'sender'] == 'user' else "background-color: #f0f0f0;"
-                        html_preview += f"<div style='margin-bottom: 10px; padding: 10px; border-radius: 5px; {sender_style}'>"
-                        html_preview += f"<strong>{msg['sender']}:</strong> {html.escape(msg['message'])}<br>"
-                        html_preview += f"<small>Timestamp: {msg.get('timestamp', 'N/A')}</small>"
-                        html_preview += "</div>"
-                    html_preview += "</div>"
-
-                    return "Conversation updated successfully.", html_preview
-            except sqlite3.Error as e:
-                conn.rollback()
-                logging.error(f"Database error in save_conversation: {e}")
-                return f"Error updating conversation: {str(e)}", "<p>Error occurred while saving</p>"
-            except Exception as e:
-                conn.rollback()
-                logging.error(f"Unexpected error in save_conversation: {e}")
-                return f"Unexpected error: {str(e)}", "<p>Unexpected error occurred</p>"
-
-        def delete_conversation(selected, conversation_mapping):
-            if not selected or selected not in conversation_mapping:
-                return "Please select a conversation before deleting.", "<p>No changes made</p>", gr.update(choices=[])
-
-            conversation_id = conversation_mapping[selected]
-
-            try:
-                with db.get_connection() as conn:
-                    cursor = conn.cursor()
-
-                    # Delete messages associated with the conversation
-                    cursor.execute("DELETE FROM ChatMessages WHERE conversation_id = ?", (conversation_id,))
-
-                    # Delete the conversation itself
-                    cursor.execute("DELETE FROM ChatConversations WHERE id = ?", (conversation_id,))
-
-                    conn.commit()
-
-                # Update the conversation list
-                remaining_conversations = [choice for choice in conversation_mapping.keys() if choice != selected]
-                updated_mapping = {choice: conversation_mapping[choice] for choice in remaining_conversations}
-
-                return "Conversation deleted successfully.", "<p>Conversation deleted</p>", gr.update(choices=remaining_conversations)
-            except sqlite3.Error as e:
-                conn.rollback()
-                logging.error(f"Database error in delete_conversation: {e}")
-                return f"Error deleting conversation: {str(e)}", "<p>Error occurred while deleting</p>", gr.update()
-            except Exception as e:
-                conn.rollback()
-                logging.error(f"Unexpected error in delete_conversation: {e}")
-                return f"Unexpected error: {str(e)}", "<p>Unexpected error occurred</p>", gr.update()
-
-        def parse_formatted_content(formatted_content):
-            lines = formatted_content.split('\n')
-            conversation_id = int(lines[0].split(': ')[1])
-            timestamp = lines[1].split(': ')[1]
-            history = []
-            current_role = None
-            current_content = None
-            for line in lines[3:]:
-                if line.startswith("Role: "):
-                    if current_role is not None:
-                        history.append({"role": current_role, "content": ["", current_content]})
-                    current_role = line.split(': ')[1]
-                elif line.startswith("Content: "):
-                    current_content = line.split(': ', 1)[1]
-            if current_role is not None:
-                history.append({"role": current_role, "content": ["", current_content]})
-            return json.dumps({
-                "conversation_id": conversation_id,
-                "timestamp": timestamp,
-                "history": history
-            }, indent=2)
-
-        search_button.click(
-            search_conversations,
-            inputs=[search_query],
-            outputs=[conversation_list, conversation_mapping]
-        )
-
-        conversation_list.change(
-            load_conversations,
-            inputs=[conversation_list, conversation_mapping],
-            outputs=[chat_content, chat_preview]
-        )
-
-        save_button.click(
-            save_conversation,
-            inputs=[conversation_list, conversation_mapping, chat_content],
-            outputs=[result_message, chat_preview]
-        )
-
-        delete_button.click(
-            delete_conversation,
-            inputs=[conversation_list, conversation_mapping],
-            outputs=[result_message, chat_preview, conversation_list]
-        )
-
-    return search_query, search_button, conversation_list, conversation_mapping, chat_content, save_button, delete_button, result_message, chat_preview
-
-
-
-# Mock function to simulate LLM processing
-def process_with_llm(workflow, context, prompt, api_endpoint, api_key):
-    api_key_snippet = api_key[:5] + "..." if api_key else "Not provided"
-    return f"LLM output using {api_endpoint} (API Key: {api_key_snippet}) for {workflow} with context: {context[:30]}... and prompt: {prompt[:30]}..."
-
-
-#
-# End of Chat_ui.py
-#######################################################################################################################

App_Function_Libraries/Gradio_UI/Config_tab.py

@@ -1,51 +0,0 @@
-import gradio as gr
-import configparser
-
-# FIXME
-CONFIG_PATH = './Config_Files/config.txt'
-
-def load_config():
-    config = configparser.ConfigParser()
-    config.read(CONFIG_PATH)
-    return config
-
-def save_config(config):
-    with open(CONFIG_PATH, 'w') as configfile:
-        config.write(configfile)
-
-def get_config_as_text():
-    with open(CONFIG_PATH, 'r') as file:
-        content = file.read()
-    return content, "Config refreshed successfully"
-
-def save_config_from_text(text):
-    with open(CONFIG_PATH, 'w') as file:
-        file.write(text)
-    return "Config saved successfully"
-
-
-def create_config_editor_tab():
-    with gr.TabItem("Edit Config", visible=True):
-        gr.Markdown("# Edit Configuration File")
-
-        with gr.Row():
-            with gr.Column():
-                refresh_button = gr.Button("Refresh Config")
-
-            with gr.Column():
-                config_text = gr.TextArea(label="Full Config", lines=30)
-                save_text_button = gr.Button("Save Config")
-
-        with gr.Row():
-            output = gr.Textbox(label="Output")
-
-        # Event handlers
-        refresh_button.click(get_config_as_text, inputs=[], outputs=[config_text, output])
-
-        config_text.change(lambda: None, None, None)  # Dummy handler to enable changes
-        save_text_button.click(save_config_from_text, inputs=[config_text], outputs=[output])
-
-        # Initialize the interface
-        config_text.value = get_config_as_text()[0]  # Only set the config text, not the output message
-
-    return refresh_button, config_text, save_text_button, output

App_Function_Libraries/Gradio_UI/Embeddings_tab.py

@@ -1,508 +0,0 @@
-# Embeddings_tabc.py
-# Description: This file contains the code for the RAG Chat tab in the Gradio UI
-#
-# Imports
-import json
-import logging
-#
-# External Imports
-import gradio as gr
-import numpy as np
-from tqdm import tqdm
-#
-# Local Imports
-from App_Function_Libraries.DB.DB_Manager import get_all_content_from_database
-from App_Function_Libraries.RAG.ChromaDB_Library import chroma_client, \
-    store_in_chroma, situate_context
-from App_Function_Libraries.RAG.Embeddings_Create import create_embedding, create_embeddings_batch
-from App_Function_Libraries.Chunk_Lib import improved_chunking_process, chunk_for_embedding
-#
-########################################################################################################################
-#
-# Functions:
-
-def create_embeddings_tab():
-    with gr.TabItem("Create Embeddings", visible=True):
-        gr.Markdown("# Create Embeddings for All Content")
-
-        with gr.Row():
-            with gr.Column():
-                embedding_provider = gr.Radio(
-                    choices=["huggingface", "local", "openai"],
-                    label="Select Embedding Provider",
-                    value="huggingface"
-                )
-                gr.Markdown("Note: Local provider requires a running Llama.cpp/llamafile server.")
-                gr.Markdown("OpenAI provider requires a valid API key.")
-
-                huggingface_model = gr.Dropdown(
-                    choices=[
-                        "jinaai/jina-embeddings-v3",
-                        "Alibaba-NLP/gte-large-en-v1.5",
-                        "dunzhang/setll_en_400M_v5",
-                        "custom"
-                    ],
-                    label="Hugging Face Model",
-                    value="jinaai/jina-embeddings-v3",
-                    visible=True
-                )
-
-                openai_model = gr.Dropdown(
-                    choices=[
-                        "text-embedding-3-small",
-                        "text-embedding-3-large"
-                    ],
-                    label="OpenAI Embedding Model",
-                    value="text-embedding-3-small",
-                    visible=False
-                )
-
-                custom_embedding_model = gr.Textbox(
-                    label="Custom Embedding Model",
-                    placeholder="Enter your custom embedding model name here",
-                    visible=False
-                )
-
-                embedding_api_url = gr.Textbox(
-                    label="API URL (for local provider)",
-                    value="http://localhost:8080/embedding",
-                    visible=False
-                )
-
-                # Add chunking options
-                chunking_method = gr.Dropdown(
-                    choices=["words", "sentences", "paragraphs", "tokens", "semantic"],
-                    label="Chunking Method",
-                    value="words"
-                )
-                max_chunk_size = gr.Slider(
-                    minimum=1, maximum=8000, step=1, value=500,
-                    label="Max Chunk Size"
-                )
-                chunk_overlap = gr.Slider(
-                    minimum=0, maximum=4000, step=1, value=200,
-                    label="Chunk Overlap"
-                )
-                adaptive_chunking = gr.Checkbox(
-                    label="Use Adaptive Chunking",
-                    value=False
-                )
-
-                create_button = gr.Button("Create Embeddings")
-
-            with gr.Column():
-                status_output = gr.Textbox(label="Status", lines=10)
-
-        def update_provider_options(provider):
-            if provider == "huggingface":
-                return gr.update(visible=True), gr.update(visible=False), gr.update(visible=False), gr.update(visible=False)
-            elif provider == "local":
-                return gr.update(visible=False), gr.update(visible=False), gr.update(visible=False), gr.update(visible=True)
-            else:  # OpenAI
-                return gr.update(visible=False), gr.update(visible=True), gr.update(visible=False), gr.update(visible=False)
-
-        def update_huggingface_options(model):
-            if model == "custom":
-                return gr.update(visible=True)
-            else:
-                return gr.update(visible=False)
-
-        embedding_provider.change(
-            fn=update_provider_options,
-            inputs=[embedding_provider],
-            outputs=[huggingface_model, openai_model, custom_embedding_model, embedding_api_url]
-        )
-
-        huggingface_model.change(
-            fn=update_huggingface_options,
-            inputs=[huggingface_model],
-            outputs=[custom_embedding_model]
-        )
-
-        def create_all_embeddings(provider, hf_model, openai_model, custom_model, api_url, method, max_size, overlap, adaptive):
-            try:
-                all_content = get_all_content_from_database()
-                if not all_content:
-                    return "No content found in the database."
-
-                chunk_options = {
-                    'method': method,
-                    'max_size': max_size,
-                    'overlap': overlap,
-                    'adaptive': adaptive
-                }
-
-                collection_name = "all_content_embeddings"
-                collection = chroma_client.get_or_create_collection(name=collection_name)
-
-                # Determine the model to use
-                if provider == "huggingface":
-                    model = custom_model if hf_model == "custom" else hf_model
-                elif provider == "openai":
-                    model = openai_model
-                else:
-                    model = custom_model
-
-                for item in all_content:
-                    media_id = item['id']
-                    text = item['content']
-
-                    chunks = improved_chunking_process(text, chunk_options)
-                    for i, chunk in enumerate(chunks):
-                        chunk_text = chunk['text']
-                        chunk_id = f"doc_{media_id}_chunk_{i}"
-
-                        existing = collection.get(ids=[chunk_id])
-                        if existing['ids']:
-                            continue
-
-                        embedding = create_embedding(chunk_text, provider, model, api_url)
-                        metadata = {
-                            "media_id": str(media_id),
-                            "chunk_index": i,
-                            "total_chunks": len(chunks),
-                            "chunking_method": method,
-                            "max_chunk_size": max_size,
-                            "chunk_overlap": overlap,
-                            "adaptive_chunking": adaptive,
-                            "embedding_model": model,
-                            "embedding_provider": provider,
-                            **chunk['metadata']
-                        }
-                        store_in_chroma(collection_name, [chunk_text], [embedding], [chunk_id], [metadata])
-
-                return "Embeddings created and stored successfully for all content."
-            except Exception as e:
-                logging.error(f"Error during embedding creation: {str(e)}")
-                return f"Error: {str(e)}"
-
-        create_button.click(
-            fn=create_all_embeddings,
-            inputs=[embedding_provider, huggingface_model, openai_model, custom_embedding_model, embedding_api_url,
-                    chunking_method, max_chunk_size, chunk_overlap, adaptive_chunking],
-            outputs=status_output
-        )
-
-
-def create_view_embeddings_tab():
-    with gr.TabItem("View/Update Embeddings", visible=True):
-        gr.Markdown("# View and Update Embeddings")
-        item_mapping = gr.State({})
-        with gr.Row():
-            with gr.Column():
-                item_dropdown = gr.Dropdown(label="Select Item", choices=[], interactive=True)
-                refresh_button = gr.Button("Refresh Item List")
-                embedding_status = gr.Textbox(label="Embedding Status", interactive=False)
-                embedding_preview = gr.Textbox(label="Embedding Preview", interactive=False, lines=5)
-                embedding_metadata = gr.Textbox(label="Embedding Metadata", interactive=False, lines=10)
-
-            with gr.Column():
-                create_new_embedding_button = gr.Button("Create New Embedding")
-                embedding_provider = gr.Radio(
-                    choices=["huggingface", "local", "openai"],
-                    label="Select Embedding Provider",
-                    value="huggingface"
-                )
-                gr.Markdown("Note: Local provider requires a running Llama.cpp/llamafile server.")
-                gr.Markdown("OpenAI provider requires a valid API key.")
-
-                huggingface_model = gr.Dropdown(
-                    choices=[
-                        "jinaai/jina-embeddings-v3",
-                        "Alibaba-NLP/gte-large-en-v1.5",
-                        "dunzhang/stella_en_400M_v5",
-                        "custom"
-                    ],
-                    label="Hugging Face Model",
-                    value="jinaai/jina-embeddings-v3",
-                    visible=True
-                )
-
-                openai_model = gr.Dropdown(
-                    choices=[
-                        "text-embedding-3-small",
-                        "text-embedding-3-large"
-                    ],
-                    label="OpenAI Embedding Model",
-                    value="text-embedding-3-small",
-                    visible=False
-                )
-
-                custom_embedding_model = gr.Textbox(
-                    label="Custom Embedding Model",
-                    placeholder="Enter your custom embedding model name here",
-                    visible=False
-                )
-
-                embedding_api_url = gr.Textbox(
-                    label="API URL (for local provider)",
-                    value="http://localhost:8080/embedding",
-                    visible=False
-                )
-                chunking_method = gr.Dropdown(
-                    choices=["words", "sentences", "paragraphs", "tokens", "semantic"],
-                    label="Chunking Method",
-                    value="words"
-                )
-                max_chunk_size = gr.Slider(
-                    minimum=1, maximum=8000, step=5, value=500,
-                    label="Max Chunk Size"
-                )
-                chunk_overlap = gr.Slider(
-                    minimum=0, maximum=5000, step=5, value=200,
-                    label="Chunk Overlap"
-                )
-                adaptive_chunking = gr.Checkbox(
-                    label="Use Adaptive Chunking",
-                    value=False
-                )
-                contextual_api_choice = gr.Dropdown(
-                    choices=["Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral", "OpenRouter", "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace"],
-                    label="Select API for Contextualized Embeddings",
-                    value="OpenAI"
-                )
-                use_contextual_embeddings = gr.Checkbox(
-                    label="Use Contextual Embeddings",
-                    value=True
-                )
-                contextual_api_key = gr.Textbox(label="API Key", lines=1)
-
-        def get_items_with_embedding_status():
-            try:
-                items = get_all_content_from_database()
-                collection = chroma_client.get_or_create_collection(name="all_content_embeddings")
-                choices = []
-                new_item_mapping = {}
-                for item in items:
-                    try:
-                        result = collection.get(ids=[f"doc_{item['id']}_chunk_0"])
-                        embedding_exists = result is not None and result.get('ids') and len(result['ids']) > 0
-                        status = "Embedding exists" if embedding_exists else "No embedding"
-                    except Exception as e:
-                        print(f"Error checking embedding for item {item['id']}: {str(e)}")
-                        status = "Error checking"
-                    choice = f"{item['title']} ({status})"
-                    choices.append(choice)
-                    new_item_mapping[choice] = item['id']
-                return gr.update(choices=choices), new_item_mapping
-            except Exception as e:
-                print(f"Error in get_items_with_embedding_status: {str(e)}")
-                return gr.update(choices=["Error: Unable to fetch items"]), {}
-
-        def update_provider_options(provider):
-            if provider == "huggingface":
-                return gr.update(visible=True), gr.update(visible=False), gr.update(visible=False), gr.update(visible=False)
-            elif provider == "local":
-                return gr.update(visible=False), gr.update(visible=False), gr.update(visible=False), gr.update(visible=True)
-            else:  # OpenAI
-                return gr.update(visible=False), gr.update(visible=True), gr.update(visible=False), gr.update(visible=False)
-
-        def update_huggingface_options(model):
-            if model == "custom":
-                return gr.update(visible=True)
-            else:
-                return gr.update(visible=False)
-
-        def check_embedding_status(selected_item, item_mapping):
-            if not selected_item:
-                return "Please select an item", "", ""
-
-            try:
-                item_id = item_mapping.get(selected_item)
-                if item_id is None:
-                    return f"Invalid item selected: {selected_item}", "", ""
-
-                item_title = selected_item.rsplit(' (', 1)[0]
-                collection = chroma_client.get_or_create_collection(name="all_content_embeddings")
-
-                result = collection.get(ids=[f"doc_{item_id}_chunk_0"], include=["embeddings", "metadatas"])
-                logging.info(f"ChromaDB result for item '{item_title}' (ID: {item_id}): {result}")
-
-                if not result['ids']:
-                    return f"No embedding found for item '{item_title}' (ID: {item_id})", "", ""
-
-                if not result['embeddings'] or not result['embeddings'][0]:
-                    return f"Embedding data missing for item '{item_title}' (ID: {item_id})", "", ""
-
-                embedding = result['embeddings'][0]
-                metadata = result['metadatas'][0] if result['metadatas'] else {}
-                embedding_preview = str(embedding[:50])
-                status = f"Embedding exists for item '{item_title}' (ID: {item_id})"
-                return status, f"First 50 elements of embedding:\n{embedding_preview}", json.dumps(metadata, indent=2)
-
-            except Exception as e:
-                logging.error(f"Error in check_embedding_status: {str(e)}")
-                return f"Error processing item: {selected_item}. Details: {str(e)}", "", ""
-
-        def create_new_embedding_for_item(selected_item, provider, hf_model, openai_model, custom_model, api_url,
-                                          method, max_size, overlap, adaptive,
-                                          item_mapping, use_contextual, contextual_api_choice=None):
-            if not selected_item:
-                return "Please select an item", "", ""
-
-            try:
-                item_id = item_mapping.get(selected_item)
-                if item_id is None:
-                    return f"Invalid item selected: {selected_item}", "", ""
-
-                items = get_all_content_from_database()
-                item = next((item for item in items if item['id'] == item_id), None)
-                if not item:
-                    return f"Item not found: {item_id}", "", ""
-
-                chunk_options = {
-                    'method': method,
-                    'max_size': max_size,
-                    'overlap': overlap,
-                    'adaptive': adaptive
-                }
-
-                logging.info(f"Chunking content for item: {item['title']} (ID: {item_id})")
-                chunks = chunk_for_embedding(item['content'], item['title'], chunk_options)
-                collection_name = "all_content_embeddings"
-                collection = chroma_client.get_or_create_collection(name=collection_name)
-
-                # Delete existing embeddings for this item
-                existing_ids = [f"doc_{item_id}_chunk_{i}" for i in range(len(chunks))]
-                collection.delete(ids=existing_ids)
-                logging.info(f"Deleted {len(existing_ids)} existing embeddings for item {item_id}")
-
-                texts, ids, metadatas = [], [], []
-                chunk_count = 0
-                logging.info("Generating contextual summaries and preparing chunks for embedding")
-                for i, chunk in enumerate(chunks):
-                    chunk_text = chunk['text']
-                    chunk_metadata = chunk['metadata']
-                    if use_contextual:
-                        logging.debug(f"Generating contextual summary for chunk {chunk_count}")
-                        context = situate_context(contextual_api_choice, item['content'], chunk_text)
-                        contextualized_text = f"{chunk_text}\n\nContextual Summary: {context}"
-                    else:
-                        contextualized_text = chunk_text
-                        context = None
-
-                    chunk_id = f"doc_{item_id}_chunk_{i}"
-
-                    # Determine the model to use
-                    if provider == "huggingface":
-                        model = custom_model if hf_model == "custom" else hf_model
-                    elif provider == "openai":
-                        model = openai_model
-                    else:
-                        model = custom_model
-
-                    metadata = {
-                        "media_id": str(item_id),
-                        "chunk_index": i,
-                        "total_chunks": len(chunks),
-                        "chunking_method": method,
-                        "max_chunk_size": max_size,
-                        "chunk_overlap": overlap,
-                        "adaptive_chunking": adaptive,
-                        "embedding_model": model,
-                        "embedding_provider": provider,
-                        "original_text": chunk_text,
-                        "use_contextual_embeddings": use_contextual,
-                        "contextual_summary": context,
-                        **chunk_metadata
-                    }
-
-                    texts.append(contextualized_text)
-                    ids.append(chunk_id)
-                    metadatas.append(metadata)
-                    chunk_count += 1
-
-                # Create embeddings in batch
-                logging.info(f"Creating embeddings for {len(texts)} chunks")
-                embeddings = create_embeddings_batch(texts, provider, model, api_url)
-
-                # Store in Chroma
-                store_in_chroma(collection_name, texts, embeddings, ids, metadatas)
-
-                # Create a preview of the first embedding
-                if isinstance(embeddings, np.ndarray) and embeddings.size > 0:
-                    embedding_preview = str(embeddings[0][:50])
-                elif isinstance(embeddings, list) and len(embeddings) > 0:
-                    embedding_preview = str(embeddings[0][:50])
-                else:
-                    embedding_preview = "No embeddings created"
-
-                # Return status message
-                status = f"New embeddings created and stored for item: {item['title']} (ID: {item_id})"
-
-                # Add contextual summaries to status message if enabled
-                if use_contextual:
-                    status += " (with contextual summaries)"
-
-                # Return status message, embedding preview, and metadata
-                return status, f"First 50 elements of new embedding:\n{embedding_preview}", json.dumps(metadatas[0],
-                                                                                                       indent=2)
-            except Exception as e:
-                logging.error(f"Error in create_new_embedding_for_item: {str(e)}", exc_info=True)
-                return f"Error creating embedding: {str(e)}", "", ""
-
-        refresh_button.click(
-            get_items_with_embedding_status,
-            outputs=[item_dropdown, item_mapping]
-        )
-        item_dropdown.change(
-            check_embedding_status,
-            inputs=[item_dropdown, item_mapping],
-            outputs=[embedding_status, embedding_preview, embedding_metadata]
-        )
-        create_new_embedding_button.click(
-            create_new_embedding_for_item,
-            inputs=[item_dropdown, embedding_provider, huggingface_model, openai_model, custom_embedding_model, embedding_api_url,
-                    chunking_method, max_chunk_size, chunk_overlap, adaptive_chunking, item_mapping,
-                    use_contextual_embeddings, contextual_api_choice],
-            outputs=[embedding_status, embedding_preview, embedding_metadata]
-        )
-        embedding_provider.change(
-            update_provider_options,
-            inputs=[embedding_provider],
-            outputs=[huggingface_model, openai_model, custom_embedding_model, embedding_api_url]
-        )
-        huggingface_model.change(
-            update_huggingface_options,
-            inputs=[huggingface_model],
-            outputs=[custom_embedding_model]
-        )
-
-    return (item_dropdown, refresh_button, embedding_status, embedding_preview, embedding_metadata,
-            create_new_embedding_button, embedding_provider, huggingface_model, openai_model, custom_embedding_model, embedding_api_url,
-            chunking_method, max_chunk_size, chunk_overlap, adaptive_chunking,
-            use_contextual_embeddings, contextual_api_choice, contextual_api_key)
-
-
-def create_purge_embeddings_tab():
-    with gr.TabItem("Purge Embeddings", visible=True):
-        gr.Markdown("# Purge Embeddings")
-
-        with gr.Row():
-            with gr.Column():
-                purge_button = gr.Button("Purge All Embeddings")
-            with gr.Column():
-                status_output = gr.Textbox(label="Status", lines=10)
-
-    def purge_all_embeddings():
-        try:
-            # It came to me in a dream....I literally don't remember how the fuck this works, cant find documentation...
-            collection_name = "all_content_embeddings"
-            chroma_client.delete_collection(collection_name)
-            chroma_client.create_collection(collection_name)
-            logging.info(f"All embeddings have been purged successfully.")
-            return "All embeddings have been purged successfully."
-        except Exception as e:
-            logging.error(f"Error during embedding purge: {str(e)}")
-            return f"Error: {str(e)}"
-
-    purge_button.click(
-        fn=purge_all_embeddings,
-        outputs=status_output
-    )
-
-
-
-#
-# End of file
-########################################################################################################################

App_Function_Libraries/Gradio_UI/Evaluations_Benchmarks_tab.py

@@ -1,60 +0,0 @@
-###################################################################################################
-# Evaluations_Benchmarks_tab.py - Gradio code for G-Eval testing
-# We will use the G-Eval API to evaluate the quality of the generated summaries.
-
-import gradio as gr
-from App_Function_Libraries.Benchmarks_Evaluations.ms_g_eval import run_geval
-
-def create_geval_tab():
-    with gr.Tab("G-Eval", visible=True):
-        gr.Markdown("# G-Eval Summarization Evaluation")
-        with gr.Row():
-            with gr.Column():
-                document_input = gr.Textbox(label="Source Document", lines=10)
-                summary_input = gr.Textbox(label="Summary", lines=5)
-                api_name_input = gr.Dropdown(
-                    choices=["OpenAI", "Anthropic", "Cohere", "Groq", "OpenRouter", "DeepSeek", "HuggingFace", "Mistral", "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "Local-LLM", "Ollama"],
-                    label="Select API"
-                )
-                api_key_input = gr.Textbox(label="API Key (if required)", type="password")
-                evaluate_button = gr.Button("Evaluate Summary")
-            with gr.Column():
-                output = gr.Textbox(label="Evaluation Results", lines=10)
-
-        evaluate_button.click(
-            fn=run_geval,
-            inputs=[document_input, summary_input, api_name_input, api_key_input],
-            outputs=output
-        )
-
-    return document_input, summary_input, api_name_input, api_key_input, evaluate_button, output
-
-
-def create_infinite_bench_tab():
-    with gr.Tab("Infinite Bench", visible=True):
-        gr.Markdown("# Infinite Bench Evaluation (Coming Soon)")
-        with gr.Row():
-            with gr.Column():
-                api_name_input = gr.Dropdown(
-                    choices=["OpenAI", "Anthropic", "Cohere", "Groq", "OpenRouter", "DeepSeek", "HuggingFace", "Mistral", "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "Local-LLM", "Ollama"],
-                    label="Select API"
-                )
-                api_key_input = gr.Textbox(label="API Key (if required)", type="password")
-                evaluate_button = gr.Button("Evaluate Summary")
-            with gr.Column():
-                output = gr.Textbox(label="Evaluation Results", lines=10)
-
-        # evaluate_button.click(
-        #     fn=run_geval,
-        #     inputs=[api_name_input, api_key_input],
-        #     outputs=output
-        # )
-
-    return api_name_input, api_key_input, evaluate_button, output
-
-
-# If you want to run this as a standalone Gradio app
-if __name__ == "__main__":
-    with gr.Blocks() as demo:
-        create_geval_tab()
-    demo.launch()

App_Function_Libraries/Gradio_UI/Explain_summarize_tab.py

@@ -1,313 +0,0 @@
-# Explain_summarize_tab.py
-# Gradio UI for explaining and summarizing text
-#
-# Imports
-import logging
-#
-# External Imports
-import gradio as gr
-
-from App_Function_Libraries.DB.DB_Manager import load_preset_prompts
-from App_Function_Libraries.Gradio_UI.Gradio_Shared import update_user_prompt
-#
-# Local Imports
-from App_Function_Libraries.Summarization.Local_Summarization_Lib import summarize_with_llama, summarize_with_kobold, \
-    summarize_with_oobabooga, summarize_with_tabbyapi, summarize_with_vllm, summarize_with_local_llm, \
-    summarize_with_ollama
-from App_Function_Libraries.Summarization.Summarization_General_Lib import summarize_with_openai, summarize_with_anthropic, \
-    summarize_with_cohere, summarize_with_groq, summarize_with_openrouter, summarize_with_deepseek, \
-    summarize_with_huggingface
-#
-#
-############################################################################################################
-#
-# Functions:
-
-def create_summarize_explain_tab():
-    with gr.TabItem("Analyze Text", visible=True):
-        gr.Markdown("# Analyze / Explain / Summarize Text without ingesting it into the DB")
-        with gr.Row():
-            with gr.Column():
-                with gr.Row():
-                    text_to_work_input = gr.Textbox(label="Text to be Explained or Summarized",
-                                                placeholder="Enter the text you want explained or summarized here",
-                                                lines=20)
-                with gr.Row():
-                    explanation_checkbox = gr.Checkbox(label="Explain Text", value=True)
-                    summarization_checkbox = gr.Checkbox(label="Summarize Text", value=True)
-                    custom_prompt_checkbox = gr.Checkbox(label="Use a Custom Prompt",
-                                                         value=False,
-                                                         visible=True)
-                    preset_prompt_checkbox = gr.Checkbox(label="Use a pre-set Prompt",
-                                                         value=False,
-                                                         visible=True)
-                with gr.Row():
-                    preset_prompt = gr.Dropdown(label="Select Preset Prompt",
-                                                choices=load_preset_prompts(),
-                                                visible=False)
-                with gr.Row():
-                    custom_prompt_input = gr.Textbox(label="Custom Prompt",
-                                                     placeholder="Enter custom prompt here",
-                                                     lines=3,
-                                                     visible=False)
-                with gr.Row():
-                    system_prompt_input = gr.Textbox(label="System Prompt",
-                                                     value="""<s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
-                    **Bulleted Note Creation Guidelines**
-        
-                    **Headings**:
-                    - Based on referenced topics, not categories like quotes or terms
-                    - Surrounded by **bold** formatting 
-                    - Not listed as bullet points
-                    - No space between headings and list items underneath
-        
-                    **Emphasis**:
-                    - **Important terms** set in bold font
-                    - **Text ending in a colon**: also bolded
-        
-                    **Review**:
-                    - Ensure adherence to specified format
-                    - Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]
-                    """,
-                                                     lines=3,
-                                                     visible=False,
-                                                     interactive=True)
-                    api_endpoint = gr.Dropdown(
-                        choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral",
-                                 "OpenRouter",
-                                 "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace", "Custom-OpenAI-API"],
-                        value=None,
-                        label="API to be used for request (Mandatory)"
-                    )
-                with gr.Row():
-                    api_key_input = gr.Textbox(label="API Key (if required)", placeholder="Enter your API key here",
-                                               type="password")
-                with gr.Row():
-                    explain_summarize_button = gr.Button("Explain/Summarize")
-
-            with gr.Column():
-                summarization_output = gr.Textbox(label="Summary:", lines=20)
-                explanation_output = gr.Textbox(label="Explanation:", lines=20)
-                custom_prompt_output = gr.Textbox(label="Custom Prompt:", lines=20, visible=True)
-
-        custom_prompt_checkbox.change(
-            fn=lambda x: (gr.update(visible=x), gr.update(visible=x)),
-            inputs=[custom_prompt_checkbox],
-            outputs=[custom_prompt_input, system_prompt_input]
-        )
-        preset_prompt_checkbox.change(
-            fn=lambda x: gr.update(visible=x),
-            inputs=[preset_prompt_checkbox],
-            outputs=[preset_prompt]
-        )
-
-        def update_prompts(preset_name):
-            prompts = update_user_prompt(preset_name)
-            return (
-                gr.update(value=prompts["user_prompt"], visible=True),
-                gr.update(value=prompts["system_prompt"], visible=True)
-            )
-
-        preset_prompt.change(
-            update_prompts,
-            inputs=preset_prompt,
-            outputs=[custom_prompt_input, system_prompt_input]
-        )
-
-        explain_summarize_button.click(
-            fn=summarize_explain_text,
-            inputs=[text_to_work_input, api_endpoint, api_key_input, summarization_checkbox, explanation_checkbox, custom_prompt_input, system_prompt_input],
-            outputs=[summarization_output, explanation_output, custom_prompt_output]
-        )
-
-
-def summarize_explain_text(message, api_endpoint, api_key, summarization, explanation, custom_prompt, custom_system_prompt,):
-    global custom_prompt_output
-    summarization_response = None
-    explanation_response = None
-    temp = 0.7
-    try:
-        logging.info(f"Debug - summarize_explain_text Function - Message: {message}")
-        logging.info(f"Debug - summarize_explain_text Function - API Endpoint: {api_endpoint}")
-
-        # Prepare the input for the API
-        input_data = f"User: {message}\n"
-        # Print first 500 chars
-        logging.info(f"Debug - Chat Function - Input Data: {input_data[:500]}...")
-        logging.debug(f"Debug - Chat Function - API Key: {api_key[:10]}")
-        user_prompt = " "
-        if not api_endpoint:
-            return "Please select an API endpoint", "Please select an API endpoint"
-        try:
-            if summarization:
-                system_prompt = """<s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
-                **Bulleted Note Creation Guidelines**
-
-                **Headings**:
-                - Based on referenced topics, not categories like quotes or terms
-                - Surrounded by **bold** formatting 
-                - Not listed as bullet points
-                - No space between headings and list items underneath
-
-                **Emphasis**:
-                - **Important terms** set in bold font
-                - **Text ending in a colon**: also bolded
-
-                **Review**:
-                - Ensure adherence to specified format
-                - Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]"""
-
-                # Use the existing API request code based on the selected endpoint
-                logging.info(f"Debug - Chat Function - API Endpoint: {api_endpoint}")
-                if api_endpoint.lower() == 'openai':
-                    summarization_response = summarize_with_openai(api_key, input_data, user_prompt, temp,
-                                                                   system_prompt)
-                elif api_endpoint.lower() == "anthropic":
-                    summarization_response = summarize_with_anthropic(api_key, input_data, user_prompt, temp,
-                                                                      system_prompt)
-                elif api_endpoint.lower() == "cohere":
-                    summarization_response = summarize_with_cohere(api_key, input_data, user_prompt, temp,
-                                                                   system_prompt)
-                elif api_endpoint.lower() == "groq":
-                    summarization_response = summarize_with_groq(api_key, input_data, user_prompt, temp, system_prompt)
-                elif api_endpoint.lower() == "openrouter":
-                    summarization_response = summarize_with_openrouter(api_key, input_data, user_prompt, temp,
-                                                                       system_prompt)
-                elif api_endpoint.lower() == "deepseek":
-                    summarization_response = summarize_with_deepseek(api_key, input_data, user_prompt, temp,
-                                                                     system_prompt)
-                elif api_endpoint.lower() == "llama.cpp":
-                    summarization_response = summarize_with_llama(input_data, user_prompt, api_key, temp, system_prompt)
-                elif api_endpoint.lower() == "kobold":
-                    summarization_response = summarize_with_kobold(input_data, api_key, user_prompt, temp,
-                                                                   system_prompt)
-                elif api_endpoint.lower() == "ooba":
-                    summarization_response = summarize_with_oobabooga(input_data, api_key, user_prompt, temp,
-                                                                      system_prompt)
-                elif api_endpoint.lower() == "tabbyapi":
-                    summarization_response = summarize_with_tabbyapi(input_data, user_prompt, temp, system_prompt)
-                elif api_endpoint.lower() == "vllm":
-                    summarization_response = summarize_with_vllm(input_data, user_prompt, system_prompt)
-                elif api_endpoint.lower() == "local-llm":
-                    summarization_response = summarize_with_local_llm(input_data, user_prompt, temp, system_prompt)
-                elif api_endpoint.lower() == "huggingface":
-                    summarization_response = summarize_with_huggingface(api_key, input_data, user_prompt,
-                                                                        temp)  # , system_prompt)
-                elif api_endpoint.lower() == "ollama":
-                    summarization_response = summarize_with_ollama(input_data, user_prompt, None, api_key, temp, system_prompt)
-                else:
-                    raise ValueError(f"Unsupported API endpoint: {api_endpoint}")
-        except Exception as e:
-            logging.error(f"Error in summarization: {str(e)}")
-            response1 = f"An error occurred during summarization: {str(e)}"
-
-        try:
-            if explanation:
-                system_prompt = """You are a professional teacher. Please explain the content presented in an easy to digest fashion so that a non-specialist may understand it."""
-                # Use the existing API request code based on the selected endpoint
-                logging.info(f"Debug - Chat Function - API Endpoint: {api_endpoint}")
-                if api_endpoint.lower() == 'openai':
-                    explanation_response = summarize_with_openai(api_key, input_data, user_prompt, temp, system_prompt)
-                elif api_endpoint.lower() == "anthropic":
-                    explanation_response = summarize_with_anthropic(api_key, input_data, user_prompt, temp,
-                                                                    system_prompt)
-                elif api_endpoint.lower() == "cohere":
-                    explanation_response = summarize_with_cohere(api_key, input_data, user_prompt, temp, system_prompt)
-                elif api_endpoint.lower() == "groq":
-                    explanation_response = summarize_with_groq(api_key, input_data, user_prompt, temp, system_prompt)
-                elif api_endpoint.lower() == "openrouter":
-                    explanation_response = summarize_with_openrouter(api_key, input_data, user_prompt, temp,
-                                                                     system_prompt)
-                elif api_endpoint.lower() == "deepseek":
-                    explanation_response = summarize_with_deepseek(api_key, input_data, user_prompt, temp,
-                                                                   system_prompt)
-                elif api_endpoint.lower() == "llama.cpp":
-                    explanation_response = summarize_with_llama(input_data, user_prompt, temp, system_prompt)
-                elif api_endpoint.lower() == "kobold":
-                    explanation_response = summarize_with_kobold(input_data, api_key, user_prompt, temp, system_prompt)
-                elif api_endpoint.lower() == "ooba":
-                    explanation_response = summarize_with_oobabooga(input_data, api_key, user_prompt, temp,
-                                                                    system_prompt)
-                elif api_endpoint.lower() == "tabbyapi":
-                    explanation_response = summarize_with_tabbyapi(input_data, user_prompt, temp, system_prompt)
-                elif api_endpoint.lower() == "vllm":
-                    explanation_response = summarize_with_vllm(input_data, user_prompt, system_prompt)
-                elif api_endpoint.lower() == "local-llm":
-                    explanation_response = summarize_with_local_llm(input_data, user_prompt, temp, system_prompt)
-                elif api_endpoint.lower() == "huggingface":
-                    explanation_response = summarize_with_huggingface(api_key, input_data, user_prompt,
-                                                                      temp)  # , system_prompt)
-                elif api_endpoint.lower() == "ollama":
-                    explanation_response = summarize_with_ollama(input_data, user_prompt, temp, system_prompt)
-                else:
-                    raise ValueError(f"Unsupported API endpoint: {api_endpoint}")
-        except Exception as e:
-            logging.error(f"Error in summarization: {str(e)}")
-            response2 = f"An error occurred during summarization: {str(e)}"
-
-        try:
-            if custom_prompt:
-                system_prompt = custom_system_prompt
-                user_prompt = custom_prompt + input_data
-                # Use the existing API request code based on the selected endpoint
-                logging.info(f"Debug - Chat Function - API Endpoint: {api_endpoint}")
-                if api_endpoint.lower() == 'openai':
-                    custom_prompt_output = summarize_with_openai(api_key, input_data, user_prompt, temp, system_prompt)
-                elif api_endpoint.lower() == "anthropic":
-                    custom_prompt_output = summarize_with_anthropic(api_key, input_data, user_prompt, temp,
-                                                                    system_prompt)
-                elif api_endpoint.lower() == "cohere":
-                    custom_prompt_output = summarize_with_cohere(api_key, input_data, user_prompt, temp, system_prompt)
-                elif api_endpoint.lower() == "groq":
-                    custom_prompt_output = summarize_with_groq(api_key, input_data, user_prompt, temp, system_prompt)
-                elif api_endpoint.lower() == "openrouter":
-                    custom_prompt_output = summarize_with_openrouter(api_key, input_data, user_prompt, temp,
-                                                                     system_prompt)
-                elif api_endpoint.lower() == "deepseek":
-                    custom_prompt_output = summarize_with_deepseek(api_key, input_data, user_prompt, temp,
-                                                                   system_prompt)
-                elif api_endpoint.lower() == "llama.cpp":
-                    custom_prompt_output = summarize_with_llama(input_data, user_prompt, temp, system_prompt)
-                elif api_endpoint.lower() == "kobold":
-                    custom_prompt_output = summarize_with_kobold(input_data, api_key, user_prompt, temp, system_prompt)
-                elif api_endpoint.lower() == "ooba":
-                    custom_prompt_output = summarize_with_oobabooga(input_data, api_key, user_prompt, temp,
-                                                                    system_prompt)
-                elif api_endpoint.lower() == "tabbyapi":
-                    custom_prompt_output = summarize_with_tabbyapi(input_data, user_prompt, temp, system_prompt)
-                elif api_endpoint.lower() == "vllm":
-                    custom_prompt_output = summarize_with_vllm(input_data, user_prompt, system_prompt)
-                elif api_endpoint.lower() == "local-llm":
-                    custom_prompt_output = summarize_with_local_llm(input_data, user_prompt, temp, system_prompt)
-                elif api_endpoint.lower() == "huggingface":
-                    custom_prompt_output = summarize_with_huggingface(api_key, input_data, user_prompt,
-                                                                      temp)  # , system_prompt)
-                elif api_endpoint.lower() == "ollama":
-                    custom_prompt_output = summarize_with_ollama(input_data, user_prompt, temp, system_prompt)
-                else:
-                    raise ValueError(f"Unsupported API endpoint: {api_endpoint}")
-        except Exception as e:
-            logging.error(f"Error in summarization: {str(e)}")
-            response2 = f"An error occurred during summarization: {str(e)}"
-
-
-        if summarization_response:
-            response1 = f"Summary: {summarization_response}"
-        else:
-            response1 = "Summary: No summary requested"
-
-        if explanation_response:
-            response2 = f"Explanation: {explanation_response}"
-        else:
-            response2 = "Explanation: No explanation requested"
-
-        if custom_prompt_output:
-            response3 = f"Custom Prompt: {custom_prompt_output}"
-        else:
-            response3 = "Custom Prompt: No custom prompt requested"
-
-        return response1, response2, response3
-
-    except Exception as e:
-        logging.error(f"Error in chat function: {str(e)}")
-        return f"An error occurred: {str(e)}"

App_Function_Libraries/Gradio_UI/Export_Functionality.py

@@ -1,266 +0,0 @@
-# Export_Functionality.py
-# Functionality for exporting items as markdown files
-import os
-import json
-import math
-import logging
-import shutil
-import tempfile
-from typing import List, Dict, Optional, Tuple
-import gradio as gr
-from App_Function_Libraries.DB.DB_Manager import DatabaseError
-from App_Function_Libraries.Gradio_UI.Gradio_Shared import fetch_item_details, fetch_items_by_keyword, browse_items
-
-logger = logging.getLogger(__name__)
-
-def export_item_as_markdown(media_id: int) -> Tuple[Optional[str], str]:
-    try:
-        content, prompt, summary = fetch_item_details(media_id)
-        title = f"Item {media_id}"  # You might want to fetch the actual title
-        markdown_content = f"# {title}\n\n## Prompt\n{prompt}\n\n## Summary\n{summary}\n\n## Content\n{content}"
-
-        filename = f"export_item_{media_id}.md"
-        with open(filename, "w", encoding='utf-8') as f:
-            f.write(markdown_content)
-
-        logger.info(f"Successfully exported item {media_id} to {filename}")
-        return filename, f"Successfully exported item {media_id} to {filename}"
-    except Exception as e:
-        error_message = f"Error exporting item {media_id}: {str(e)}"
-        logger.error(error_message)
-        return None, error_message
-
-
-def export_items_by_keyword(keyword: str) -> str:
-    try:
-        items = fetch_items_by_keyword(keyword)
-        if not items:
-            logger.warning(f"No items found for keyword: {keyword}")
-            return None
-
-        # Create a temporary directory to store individual markdown files
-        with tempfile.TemporaryDirectory() as temp_dir:
-            folder_name = f"export_keyword_{keyword}"
-            export_folder = os.path.join(temp_dir, folder_name)
-            os.makedirs(export_folder)
-
-            for item in items:
-                content, prompt, summary = fetch_item_details(item['id'])
-                markdown_content = f"# {item['title']}\n\n## Prompt\n{prompt}\n\n## Summary\n{summary}\n\n## Content\n{content}"
-
-                # Create individual markdown file for each item
-                file_name = f"{item['id']}_{item['title'][:50]}.md"  # Limit filename length
-                file_path = os.path.join(export_folder, file_name)
-                with open(file_path, "w", encoding='utf-8') as f:
-                    f.write(markdown_content)
-
-            # Create a zip file containing all markdown files
-            zip_filename = f"{folder_name}.zip"
-            shutil.make_archive(os.path.join(temp_dir, folder_name), 'zip', export_folder)
-
-            # Move the zip file to a location accessible by Gradio
-            final_zip_path = os.path.join(os.getcwd(), zip_filename)
-            shutil.move(os.path.join(temp_dir, zip_filename), final_zip_path)
-
-        logger.info(f"Successfully exported {len(items)} items for keyword '{keyword}' to {zip_filename}")
-        return final_zip_path
-    except Exception as e:
-        logger.error(f"Error exporting items for keyword '{keyword}': {str(e)}")
-        return None
-
-
-def export_selected_items(selected_items: List[Dict]) -> Tuple[Optional[str], str]:
-    try:
-        logger.debug(f"Received selected_items: {selected_items}")
-        if not selected_items:
-            logger.warning("No items selected for export")
-            return None, "No items selected for export"
-
-        markdown_content = "# Selected Items\n\n"
-        for item in selected_items:
-            logger.debug(f"Processing item: {item}")
-            try:
-                # Check if 'value' is a string (JSON) or already a dictionary
-                if isinstance(item, str):
-                    item_data = json.loads(item)
-                elif isinstance(item, dict) and 'value' in item:
-                    item_data = item['value'] if isinstance(item['value'], dict) else json.loads(item['value'])
-                else:
-                    item_data = item
-
-                logger.debug(f"Item data after processing: {item_data}")
-
-                if 'id' not in item_data:
-                    logger.error(f"'id' not found in item data: {item_data}")
-                    continue
-
-                content, prompt, summary = fetch_item_details(item_data['id'])
-                markdown_content += f"## {item_data.get('title', 'Item {}'.format(item_data['id']))}\n\n### Prompt\n{prompt}\n\n### Summary\n{summary}\n\n### Content\n{content}\n\n---\n\n"
-            except Exception as e:
-                logger.error(f"Error processing item {item}: {str(e)}")
-                markdown_content += f"## Error\n\nUnable to process this item.\n\n---\n\n"
-
-        filename = "export_selected_items.md"
-        with open(filename, "w", encoding='utf-8') as f:
-            f.write(markdown_content)
-
-        logger.info(f"Successfully exported {len(selected_items)} selected items to {filename}")
-        return filename, f"Successfully exported {len(selected_items)} items to {filename}"
-    except Exception as e:
-        error_message = f"Error exporting selected items: {str(e)}"
-        logger.error(error_message)
-        return None, error_message
-
-
-def display_search_results_export_tab(search_query: str, search_type: str, page: int = 1, items_per_page: int = 10):
-    logger.info(f"Searching with query: '{search_query}', type: '{search_type}', page: {page}")
-    try:
-        results = browse_items(search_query, search_type)
-        logger.info(f"browse_items returned {len(results)} results")
-
-        if not results:
-            return [], f"No results found for query: '{search_query}'", 1, 1
-
-        total_pages = math.ceil(len(results) / items_per_page)
-        start_index = (page - 1) * items_per_page
-        end_index = start_index + items_per_page
-        paginated_results = results[start_index:end_index]
-
-        checkbox_data = [
-            {
-                "name": f"Name: {item[1]}\nURL: {item[2]}",
-                "value": {"id": item[0], "title": item[1], "url": item[2]}
-            }
-            for item in paginated_results
-        ]
-
-        logger.info(f"Returning {len(checkbox_data)} items for checkbox (page {page} of {total_pages})")
-        return checkbox_data, f"Found {len(results)} results (showing page {page} of {total_pages})", page, total_pages
-
-    except DatabaseError as e:
-        error_message = f"Error in display_search_results_export_tab: {str(e)}"
-        logger.error(error_message)
-        return [], error_message, 1, 1
-    except Exception as e:
-        error_message = f"Unexpected error in display_search_results_export_tab: {str(e)}"
-        logger.error(error_message)
-        return [], error_message, 1, 1
-
-
-def create_export_tab():
-    with gr.Tab("Search and Export"):
-        with gr.Row():
-            with gr.Column():
-                gr.Markdown("# Search and Export Items")
-                gr.Markdown("Search for items and export them as markdown files")
-                gr.Markdown("You can also export items by keyword")
-                search_query = gr.Textbox(label="Search Query")
-                search_type = gr.Radio(["Title", "URL", "Keyword", "Content"], label="Search By")
-                search_button = gr.Button("Search")
-
-            with gr.Column():
-                prev_button = gr.Button("Previous Page")
-                next_button = gr.Button("Next Page")
-
-        current_page = gr.State(1)
-        total_pages = gr.State(1)
-
-        search_results = gr.CheckboxGroup(label="Search Results", choices=[])
-        export_selected_button = gr.Button("Export Selected Items")
-
-        keyword_input = gr.Textbox(label="Enter keyword for export")
-        export_by_keyword_button = gr.Button("Export items by keyword")
-
-        export_output = gr.File(label="Download Exported File")
-        error_output = gr.Textbox(label="Status/Error Messages", interactive=False)
-
-    def search_and_update(query, search_type, page):
-        results, message, current, total = display_search_results_export_tab(query, search_type, page)
-        logger.debug(f"search_and_update results: {results}")
-        return results, message, current, total, gr.update(choices=results)
-
-    search_button.click(
-        fn=search_and_update,
-        inputs=[search_query, search_type, current_page],
-        outputs=[search_results, error_output, current_page, total_pages, search_results],
-        show_progress="full"
-    )
-
-
-    def update_page(current, total, direction):
-        new_page = max(1, min(total, current + direction))
-        return new_page
-
-    prev_button.click(
-        fn=update_page,
-        inputs=[current_page, total_pages, gr.State(-1)],
-        outputs=[current_page]
-    ).then(
-        fn=search_and_update,
-        inputs=[search_query, search_type, current_page],
-        outputs=[search_results, error_output, current_page, total_pages],
-        show_progress=True
-    )
-
-    next_button.click(
-        fn=update_page,
-        inputs=[current_page, total_pages, gr.State(1)],
-        outputs=[current_page]
-    ).then(
-        fn=search_and_update,
-        inputs=[search_query, search_type, current_page],
-        outputs=[search_results, error_output, current_page, total_pages],
-        show_progress=True
-    )
-
-    def handle_export_selected(selected_items):
-        logger.debug(f"Exporting selected items: {selected_items}")
-        return export_selected_items(selected_items)
-
-    export_selected_button.click(
-        fn=handle_export_selected,
-        inputs=[search_results],
-        outputs=[export_output, error_output],
-        show_progress="full"
-    )
-
-    export_by_keyword_button.click(
-        fn=export_items_by_keyword,
-        inputs=[keyword_input],
-        outputs=[export_output, error_output],
-        show_progress="full"
-    )
-
-    def handle_item_selection(selected_items):
-        logger.debug(f"Selected items: {selected_items}")
-        if not selected_items:
-            return None, "No item selected"
-
-        try:
-            # Assuming selected_items is a list of dictionaries
-            selected_item = selected_items[0]
-            logger.debug(f"First selected item: {selected_item}")
-
-            # Check if 'value' is a string (JSON) or already a dictionary
-            if isinstance(selected_item['value'], str):
-                item_data = json.loads(selected_item['value'])
-            else:
-                item_data = selected_item['value']
-
-            logger.debug(f"Item data: {item_data}")
-
-            item_id = item_data['id']
-            return export_item_as_markdown(item_id)
-        except Exception as e:
-            error_message = f"Error processing selected item: {str(e)}"
-            logger.error(error_message)
-            return None, error_message
-
-    search_results.select(
-        fn=handle_item_selection,
-        inputs=[search_results],
-        outputs=[export_output, error_output],
-        show_progress="full"
-    )
-
-

App_Function_Libraries/Gradio_UI/Gradio_Shared.py

@@ -1,285 +0,0 @@
-# Gradio_Shared.py
-# Gradio UI functions that are shared across multiple tabs
-#
-# Imports
-import logging
-import sqlite3
-import traceback
-from functools import wraps
-from typing import List, Tuple
-#
-# External Imports
-import gradio as gr
-#
-# Local Imports
-from App_Function_Libraries.DB.DB_Manager import list_prompts, db, search_and_display, fetch_prompt_details
-from App_Function_Libraries.DB.SQLite_DB import DatabaseError
-from App_Function_Libraries.Utils.Utils import format_transcription
-#
-##############################################################################################################
-#
-# Functions:
-
-whisper_models = ["small", "medium", "small.en", "medium.en", "medium", "large", "large-v1", "large-v2", "large-v3",
-                  "distil-large-v2", "distil-medium.en", "distil-small.en"]
-
-# Sample data
-prompts_category_1 = [
-    "What are the key points discussed in the video?",
-    "Summarize the main arguments made by the speaker.",
-    "Describe the conclusions of the study presented."
-]
-
-prompts_category_2 = [
-    "How does the proposed solution address the problem?",
-    "What are the implications of the findings?",
-    "Can you explain the theory behind the observed phenomenon?"
-]
-
-all_prompts = prompts_category_1 + prompts_category_2
-
-
-
-#FIXME - SQL Functions that need to be addressed/added to DB manager
-def search_media(query, fields, keyword, page):
-    try:
-        results = search_and_display(query, fields, keyword, page)
-        return results
-    except Exception as e:
-        logger = logging.getLogger()
-        logger.error(f"Error searching media: {e}")
-        return str(e)
-
-def fetch_items_by_title_or_url(search_query: str, search_type: str):
-    try:
-        with db.get_connection() as conn:
-            cursor = conn.cursor()
-            if search_type == 'Title':
-                cursor.execute("SELECT id, title, url FROM Media WHERE title LIKE ?", (f'%{search_query}%',))
-            elif search_type == 'URL':
-                cursor.execute("SELECT id, title, url FROM Media WHERE url LIKE ?", (f'%{search_query}%',))
-            results = cursor.fetchall()
-            return results
-    except sqlite3.Error as e:
-        raise DatabaseError(f"Error fetching items by {search_type}: {e}")
-
-def fetch_items_by_keyword(search_query: str):
-    try:
-        with db.get_connection() as conn:
-            cursor = conn.cursor()
-            cursor.execute("""
-                SELECT m.id, m.title, m.url
-                FROM Media m
-                JOIN MediaKeywords mk ON m.id = mk.media_id
-                JOIN Keywords k ON mk.keyword_id = k.id
-                WHERE k.keyword LIKE ?
-            """, (f'%{search_query}%',))
-            results = cursor.fetchall()
-            return results
-    except sqlite3.Error as e:
-        raise DatabaseError(f"Error fetching items by keyword: {e}")
-
-# FIXME - Raw SQL not using DB_Manager...
-def fetch_items_by_content(search_query: str):
-    try:
-        with db.get_connection() as conn:
-            cursor = conn.cursor()
-            cursor.execute("SELECT id, title, url FROM Media WHERE content LIKE ?", (f'%{search_query}%',))
-            results = cursor.fetchall()
-            return results
-    except sqlite3.Error as e:
-        raise DatabaseError(f"Error fetching items by content: {e}")
-
-
-
-# FIXME - RAW SQL not using DB_Manager...
-def fetch_item_details_single(media_id: int):
-    try:
-        with db.get_connection() as conn:
-            cursor = conn.cursor()
-            cursor.execute("""
-                SELECT prompt, summary 
-                FROM MediaModifications 
-                WHERE media_id = ? 
-                ORDER BY modification_date DESC 
-                LIMIT 1
-            """, (media_id,))
-            prompt_summary_result = cursor.fetchone()
-            cursor.execute("SELECT content FROM Media WHERE id = ?", (media_id,))
-            content_result = cursor.fetchone()
-
-            prompt = prompt_summary_result[0] if prompt_summary_result else ""
-            summary = prompt_summary_result[1] if prompt_summary_result else ""
-            content = content_result[0] if content_result else ""
-
-            return prompt, summary, content
-    except sqlite3.Error as e:
-        raise Exception(f"Error fetching item details: {e}")
-
-
-# FIXME - RAW SQL not using DB_Manager...
-def fetch_item_details(media_id: int):
-    try:
-        with db.get_connection() as conn:
-            cursor = conn.cursor()
-            cursor.execute("""
-                SELECT prompt, summary 
-                FROM MediaModifications 
-                WHERE media_id = ? 
-                ORDER BY modification_date DESC 
-                LIMIT 1
-            """, (media_id,))
-            prompt_summary_result = cursor.fetchone()
-            cursor.execute("SELECT content FROM Media WHERE id = ?", (media_id,))
-            content_result = cursor.fetchone()
-
-            prompt = prompt_summary_result[0] if prompt_summary_result else ""
-            summary = prompt_summary_result[1] if prompt_summary_result else ""
-            content = content_result[0] if content_result else ""
-
-            return content, prompt, summary
-    except sqlite3.Error as e:
-        logging.error(f"Error fetching item details: {e}")
-        return "", "", ""  # Return empty strings if there's an error
-
-# Handle prompt selection
-def handle_prompt_selection(prompt):
-    return f"You selected: {prompt}"
-
-
-def update_user_prompt(preset_name):
-    details = fetch_prompt_details(preset_name)
-    if details:
-        # Return a dictionary with all details
-        return {
-            "title": details[0],
-            "author": details[1],
-            "details": details[2],
-            "system_prompt": details[3],
-            "user_prompt": details[4] if len(details) > 3 else "",
-        }
-    return {"title": "", "details": "", "system_prompt": "", "user_prompt": "", "author": ""}
-
-def browse_items(search_query, search_type):
-    if search_type == 'Keyword':
-        results = fetch_items_by_keyword(search_query)
-    elif search_type == 'Content':
-        results = fetch_items_by_content(search_query)
-    else:
-        results = fetch_items_by_title_or_url(search_query, search_type)
-    return results
-
-
-def update_dropdown(search_query, search_type):
-    results = browse_items(search_query, search_type)
-    item_options = [f"{item[1]} ({item[2]})" for item in results]
-    new_item_mapping = {f"{item[1]} ({item[2]})": item[0] for item in results}
-    print(f"Debug - Update Dropdown - New Item Mapping: {new_item_mapping}")
-    return gr.update(choices=item_options), new_item_mapping
-
-
-
-def get_media_id(selected_item, item_mapping):
-    return item_mapping.get(selected_item)
-
-
-def update_detailed_view(item, item_mapping):
-    # Function to update the detailed view based on selected item
-    if item:
-        item_id = item_mapping.get(item)
-        if item_id:
-            content, prompt, summary = fetch_item_details(item_id)
-            if content or prompt or summary:
-                details_html = "<h4>Details:</h4>"
-                if prompt:
-                    formatted_prompt = format_transcription(prompt)
-                    details_html += f"<h4>Prompt:</h4>{formatted_prompt}</p>"
-                if summary:
-                    formatted_summary = format_transcription(summary)
-                    details_html += f"<h4>Summary:</h4>{formatted_summary}</p>"
-                # Format the transcription content for better readability
-                formatted_content = format_transcription(content)
-                #content_html = f"<h4>Transcription:</h4><div style='white-space: pre-wrap;'>{content}</div>"
-                content_html = f"<h4>Transcription:</h4><div style='white-space: pre-wrap;'>{formatted_content}</div>"
-                return details_html, content_html
-            else:
-                return "No details available.", "No details available."
-        else:
-            return "No item selected", "No item selected"
-    else:
-        return "No item selected", "No item selected"
-
-
-def format_content(content):
-    # Format content using markdown
-    formatted_content = f"```\n{content}\n```"
-    return formatted_content
-
-
-def update_prompt_dropdown():
-    prompt_names = list_prompts()
-    return gr.update(choices=prompt_names)
-
-
-def display_prompt_details(selected_prompt):
-    if selected_prompt:
-        prompts = update_user_prompt(selected_prompt)
-        if prompts["title"]:  # Check if we have any details
-            details_str = f"<h4>Details:</h4><p>{prompts['details']}</p>"
-            system_str = f"<h4>System:</h4><p>{prompts['system_prompt']}</p>"
-            user_str = f"<h4>User:</h4><p>{prompts['user_prompt']}</p>" if prompts['user_prompt'] else ""
-            return details_str + system_str + user_str
-    return "No details available."
-
-def search_media_database(query: str) -> List[Tuple[int, str, str]]:
-    return browse_items(query, 'Title')
-
-
-def load_media_content(media_id: int) -> dict:
-    try:
-        print(f"Debug - Load Media Content - Media ID: {media_id}")
-        item_details = fetch_item_details(media_id)
-        print(f"Debug - Load Media Content - Item Details: \n\n{item_details}\n\n\n\n")
-
-        if isinstance(item_details, tuple) and len(item_details) == 3:
-            content, prompt, summary = item_details
-        else:
-            print(f"Debug - Load Media Content - Unexpected item_details format: \n\n{item_details}\n\n\n\n")
-            content, prompt, summary = "", "", ""
-
-        return {
-            "content": content or "No content available",
-            "prompt": prompt or "No prompt available",
-            "summary": summary or "No summary available"
-        }
-    except Exception as e:
-        print(f"Debug - Load Media Content - Error: {str(e)}")
-        return {"content": "", "prompt": "", "summary": ""}
-
-
-def error_handler(func):
-    @wraps(func)
-    def wrapper(*args, **kwargs):
-        try:
-            return func(*args, **kwargs)
-        except Exception as e:
-            error_message = f"Error in {func.__name__}: {str(e)}"
-            logging.error(f"{error_message}\n{traceback.format_exc()}")
-            return {"error": error_message, "details": traceback.format_exc()}
-    return wrapper
-
-
-def create_chunking_inputs():
-    chunk_text_by_words_checkbox = gr.Checkbox(label="Chunk Text by Words", value=False, visible=True)
-    max_words_input = gr.Number(label="Max Words", value=300, precision=0, visible=True)
-    chunk_text_by_sentences_checkbox = gr.Checkbox(label="Chunk Text by Sentences", value=False, visible=True)
-    max_sentences_input = gr.Number(label="Max Sentences", value=10, precision=0, visible=True)
-    chunk_text_by_paragraphs_checkbox = gr.Checkbox(label="Chunk Text by Paragraphs", value=False, visible=True)
-    max_paragraphs_input = gr.Number(label="Max Paragraphs", value=5, precision=0, visible=True)
-    chunk_text_by_tokens_checkbox = gr.Checkbox(label="Chunk Text by Tokens", value=False, visible=True)
-    max_tokens_input = gr.Number(label="Max Tokens", value=1000, precision=0, visible=True)
-    gr_semantic_chunk_long_file = gr.Checkbox(label="Semantic Chunking by Sentence similarity", value=False, visible=True)
-    gr_semantic_chunk_long_file_size = gr.Number(label="Max Chunk Size", value=2000, visible=True)
-    gr_semantic_chunk_long_file_overlap = gr.Number(label="Max Chunk Overlap Size", value=100, visible=True)
-    return [chunk_text_by_words_checkbox, max_words_input, chunk_text_by_sentences_checkbox, max_sentences_input,
-            chunk_text_by_paragraphs_checkbox, max_paragraphs_input, chunk_text_by_tokens_checkbox, max_tokens_input]

App_Function_Libraries/Gradio_UI/Import_Functionality.py

@@ -1,388 +0,0 @@
-# Import_Functionality.py
-# Functionality to import content into the DB
-#
-# Imports
-from time import sleep
-import logging
-import re
-import shutil
-import tempfile
-import os
-import traceback
-import zipfile
-#
-# External Imports
-import gradio as gr
-#
-# Local Imports
-from App_Function_Libraries.DB.DB_Manager import insert_prompt_to_db, load_preset_prompts, import_obsidian_note_to_db, \
-    add_media_to_database
-from App_Function_Libraries.Prompt_Handling import import_prompt_from_file, import_prompts_from_zip#
-from App_Function_Libraries.Summarization.Summarization_General_Lib import perform_summarization
-
-###################################################################################################################
-#
-# Functions:
-
-logger = logging.getLogger()
-
-
-def import_data(file, title, author, keywords, custom_prompt, summary, auto_summarize, api_name, api_key):
-    logging.debug(f"Starting import_data with file: {file} / Title: {title} / Author: {author} / Keywords: {keywords}")
-    if file is None:
-        return "No file uploaded. Please upload a file."
-
-    try:
-        logging.debug(f"File object type: {type(file)}")
-        logging.debug(f"File object attributes: {dir(file)}")
-
-        if hasattr(file, 'name'):
-            file_name = file.name
-        else:
-            file_name = 'unknown_file'
-
-        # Create a temporary file
-        with tempfile.NamedTemporaryFile(mode='w+', delete=False, suffix='.txt', encoding='utf-8') as temp_file:
-            if isinstance(file, str):
-                # If file is a string, it's likely file content
-                temp_file.write(file)
-            elif hasattr(file, 'read'):
-                # If file has a 'read' method, it's likely a file-like object
-                content = file.read()
-                if isinstance(content, bytes):
-                    content = content.decode('utf-8')
-                temp_file.write(content)
-            else:
-                # If it's neither a string nor a file-like object, try converting it to a string
-                temp_file.write(str(file))
-
-            temp_file.seek(0)
-            file_content = temp_file.read()
-
-        logging.debug(f"File name: {file_name}")
-        logging.debug(f"File content (first 100 chars): {file_content[:100]}")
-
-        # Create info_dict
-        info_dict = {
-            'title': title or 'Untitled',
-            'uploader': author or 'Unknown',
-        }
-
-        # FIXME - Add chunking support... I added chapter chunking specifically for this...
-        # Create segments (assuming one segment for the entire content)
-        segments = [{'Text': file_content}]
-
-        # Process keywords
-        keyword_list = [kw.strip() for kw in keywords.split(',') if kw.strip()] if keywords else []
-
-        # Handle summarization
-        if auto_summarize and api_name and api_key:
-            summary = perform_summarization(api_name, file_content, custom_prompt, api_key)
-        elif not summary:
-            summary = "No summary provided"
-
-            # Add to database
-            result = add_media_to_database(
-                url=file_name,  # Using filename as URL
-                info_dict=info_dict,
-                segments=segments,
-                summary=summary,
-                keywords=keyword_list,
-                custom_prompt_input=custom_prompt,
-                whisper_model="Imported",  # Indicating this was an imported file
-                media_type="document",
-                overwrite=False  # Set this to True if you want to overwrite existing entries
-            )
-
-            # Clean up the temporary file
-            os.unlink(temp_file.name)
-
-            return f"File '{file_name}' import attempt complete. Database result: {result}"
-    except Exception as e:
-        logging.exception(f"Error importing file: {str(e)}")
-        return f"Error importing file: {str(e)}"
-
-
-def process_obsidian_zip(zip_file):
-    with tempfile.TemporaryDirectory() as temp_dir:
-        try:
-            with zipfile.ZipFile(zip_file, 'r') as zip_ref:
-                zip_ref.extractall(temp_dir)
-
-            imported_files, total_files, errors = import_obsidian_vault(temp_dir)
-
-            return imported_files, total_files, errors
-        except zipfile.BadZipFile:
-            error_msg = "The uploaded file is not a valid zip file."
-            logger.error(error_msg)
-            return 0, 0, [error_msg]
-        except Exception as e:
-            error_msg = f"Error processing zip file: {str(e)}\n{traceback.format_exc()}"
-            logger.error(error_msg)
-            return 0, 0, [error_msg]
-        finally:
-            shutil.rmtree(temp_dir, ignore_errors=True)
-
-
-
-def scan_obsidian_vault(vault_path):
-    markdown_files = []
-    for root, dirs, files in os.walk(vault_path):
-        for file in files:
-            if file.endswith('.md'):
-                markdown_files.append(os.path.join(root, file))
-    return markdown_files
-
-
-def parse_obsidian_note(file_path):
-    with open(file_path, 'r', encoding='utf-8') as file:
-        content = file.read()
-
-    frontmatter = {}
-    frontmatter_match = re.match(r'^---\s*\n(.*?)\n---\s*\n', content, re.DOTALL)
-    if frontmatter_match:
-        frontmatter_text = frontmatter_match.group(1)
-        import yaml
-        frontmatter = yaml.safe_load(frontmatter_text)
-        content = content[frontmatter_match.end():]
-
-    tags = re.findall(r'#(\w+)', content)
-    links = re.findall(r'\[\[(.*?)\]\]', content)
-
-    return {
-        'title': os.path.basename(file_path).replace('.md', ''),
-        'content': content,
-        'frontmatter': frontmatter,
-        'tags': tags,
-        'links': links,
-        'file_path': file_path  # Add this line
-    }
-
-def create_import_single_prompt_tab():
-    with gr.TabItem("Import a Prompt", visible=True):
-        gr.Markdown("# Import a prompt into the database")
-
-        with gr.Row():
-            with gr.Column():
-                import_file = gr.File(label="Upload file for import", file_types=["txt", "md"])
-                title_input = gr.Textbox(label="Title", placeholder="Enter the title of the content")
-                author_input = gr.Textbox(label="Author", placeholder="Enter the author's name")
-                system_input = gr.Textbox(label="System", placeholder="Enter the system message for the prompt", lines=3)
-                user_input = gr.Textbox(label="User", placeholder="Enter the user message for the prompt", lines=3)
-                keywords_input = gr.Textbox(label="Keywords", placeholder="Enter keywords separated by commas")
-                import_button = gr.Button("Import Prompt")
-
-            with gr.Column():
-                import_output = gr.Textbox(label="Import Status")
-                save_button = gr.Button("Save to Database")
-                save_output = gr.Textbox(label="Save Status")
-
-        def handle_import(file):
-            result = import_prompt_from_file(file)
-            if isinstance(result, tuple) and len(result) == 5:
-                title, author, system, user, keywords = result
-                return gr.update(value="File successfully imported. You can now edit the content before saving."), \
-                       gr.update(value=title), gr.update(value=author), gr.update(value=system), \
-                       gr.update(value=user), gr.update(value=", ".join(keywords))
-            else:
-                return gr.update(value=result), gr.update(), gr.update(), gr.update(), gr.update(), gr.update()
-
-        import_button.click(
-            fn=handle_import,
-            inputs=[import_file],
-            outputs=[import_output, title_input, author_input, system_input, user_input, keywords_input]
-        )
-
-        def save_prompt_to_db(title, author, system, user, keywords):
-            keyword_list = [k.strip() for k in keywords.split(',') if k.strip()]
-            return insert_prompt_to_db(title, author, system, user, keyword_list)
-
-        save_button.click(
-            fn=save_prompt_to_db,
-            inputs=[title_input, author_input, system_input, user_input, keywords_input],
-            outputs=save_output
-        )
-
-        def update_prompt_dropdown():
-            return gr.update(choices=load_preset_prompts())
-
-        save_button.click(
-            fn=update_prompt_dropdown,
-            inputs=[],
-            outputs=[gr.Dropdown(label="Select Preset Prompt")]
-        )
-
-def create_import_item_tab():
-    with gr.TabItem("Import Markdown/Text Files", visible=True):
-        gr.Markdown("# Import a markdown file or text file into the database")
-        gr.Markdown("...and have it tagged + summarized")
-        with gr.Row():
-            with gr.Column():
-                import_file = gr.File(label="Upload file for import", file_types=["txt", "md"])
-                title_input = gr.Textbox(label="Title", placeholder="Enter the title of the content")
-                author_input = gr.Textbox(label="Author", placeholder="Enter the author's name")
-                keywords_input = gr.Textbox(label="Keywords", placeholder="Enter keywords, comma-separated")
-                custom_prompt_input = gr.Textbox(label="Custom Prompt",
-                                             placeholder="Enter a custom prompt for summarization (optional)")
-                summary_input = gr.Textbox(label="Summary",
-                                       placeholder="Enter a summary or leave blank for auto-summarization", lines=3)
-                auto_summarize_checkbox = gr.Checkbox(label="Auto-summarize", value=False)
-                api_name_input = gr.Dropdown(
-                choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral", "OpenRouter",
-                         "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM","ollama", "HuggingFace", "Custom-OpenAI-API"],
-                label="API for Auto-summarization"
-                )
-                api_key_input = gr.Textbox(label="API Key", type="password")
-            with gr.Column():
-                import_button = gr.Button("Import Data")
-                import_output = gr.Textbox(label="Import Status")
-
-        import_button.click(
-            fn=import_data,
-            inputs=[import_file, title_input, author_input, keywords_input, custom_prompt_input,
-                    summary_input, auto_summarize_checkbox, api_name_input, api_key_input],
-            outputs=import_output
-        )
-
-
-def create_import_multiple_prompts_tab():
-    with gr.TabItem("Import Multiple Prompts", visible=True):
-        gr.Markdown("# Import multiple prompts into the database")
-        gr.Markdown("Upload a zip file containing multiple prompt files (txt or md)")
-
-        with gr.Row():
-            with gr.Column():
-                zip_file = gr.File(label="Upload zip file for import", file_types=["zip"])
-                import_button = gr.Button("Import Prompts")
-                prompts_dropdown = gr.Dropdown(label="Select Prompt to Edit", choices=[])
-                title_input = gr.Textbox(label="Title", placeholder="Enter the title of the content")
-                author_input = gr.Textbox(label="Author", placeholder="Enter the author's name")
-                system_input = gr.Textbox(label="System", placeholder="Enter the system message for the prompt",
-                                          lines=3)
-                user_input = gr.Textbox(label="User", placeholder="Enter the user message for the prompt", lines=3)
-                keywords_input = gr.Textbox(label="Keywords", placeholder="Enter keywords separated by commas")
-
-            with gr.Column():
-                import_output = gr.Textbox(label="Import Status")
-                save_button = gr.Button("Save to Database")
-                save_output = gr.Textbox(label="Save Status")
-                prompts_display = gr.Textbox(label="Identified Prompts")
-
-        def handle_zip_import(zip_file):
-            result = import_prompts_from_zip(zip_file)
-            if isinstance(result, list):
-                prompt_titles = [prompt['title'] for prompt in result]
-                return gr.update(
-                    value="Zip file successfully imported. Select a prompt to edit from the dropdown."), prompt_titles, gr.update(
-                    value="\n".join(prompt_titles)), result
-            else:
-                return gr.update(value=result), [], gr.update(value=""), []
-
-        def handle_prompt_selection(selected_title, prompts):
-            selected_prompt = next((prompt for prompt in prompts if prompt['title'] == selected_title), None)
-            if selected_prompt:
-                return (
-                    selected_prompt['title'],
-                    selected_prompt.get('author', ''),
-                    selected_prompt['system'],
-                    selected_prompt.get('user', ''),
-                    ", ".join(selected_prompt.get('keywords', []))
-                )
-            else:
-                return "", "", "", "", ""
-
-        zip_import_state = gr.State([])
-
-        import_button.click(
-            fn=handle_zip_import,
-            inputs=[zip_file],
-            outputs=[import_output, prompts_dropdown, prompts_display, zip_import_state]
-        )
-
-        prompts_dropdown.change(
-            fn=handle_prompt_selection,
-            inputs=[prompts_dropdown, zip_import_state],
-            outputs=[title_input, author_input, system_input, user_input, keywords_input]
-        )
-
-        def save_prompt_to_db(title, author, system, user, keywords):
-            keyword_list = [k.strip() for k in keywords.split(',') if k.strip()]
-            return insert_prompt_to_db(title, author, system, user, keyword_list)
-
-        save_button.click(
-            fn=save_prompt_to_db,
-            inputs=[title_input, author_input, system_input, user_input, keywords_input],
-            outputs=save_output
-        )
-
-        def update_prompt_dropdown():
-            return gr.update(choices=load_preset_prompts())
-
-        save_button.click(
-            fn=update_prompt_dropdown,
-            inputs=[],
-            outputs=[gr.Dropdown(label="Select Preset Prompt")]
-        )
-
-
-def create_import_obsidian_vault_tab():
-    with gr.TabItem("Import Obsidian Vault", visible=True):
-        gr.Markdown("## Import Obsidian Vault")
-        with gr.Row():
-            with gr.Column():
-                vault_path_input = gr.Textbox(label="Obsidian Vault Path (Local)")
-                vault_zip_input = gr.File(label="Upload Obsidian Vault (Zip)")
-            with gr.Column():
-                import_vault_button = gr.Button("Import Obsidian Vault")
-                import_status = gr.Textbox(label="Import Status", interactive=False)
-
-
-    def import_vault(vault_path, vault_zip):
-        if vault_zip:
-            imported, total, errors = process_obsidian_zip(vault_zip.name)
-        elif vault_path:
-            imported, total, errors = import_obsidian_vault(vault_path)
-        else:
-            return "Please provide either a local vault path or upload a zip file."
-
-        status = f"Imported {imported} out of {total} files.\n"
-        if errors:
-            status += f"Encountered {len(errors)} errors:\n" + "\n".join(errors)
-        return status
-
-
-    import_vault_button.click(
-        fn=import_vault,
-        inputs=[vault_path_input, vault_zip_input],
-        outputs=[import_status],
-    )
-
-
-def import_obsidian_vault(vault_path, progress=gr.Progress()):
-    try:
-        markdown_files = scan_obsidian_vault(vault_path)
-        total_files = len(markdown_files)
-        imported_files = 0
-        errors = []
-
-        for i, file_path in enumerate(markdown_files):
-            try:
-                note_data = parse_obsidian_note(file_path)
-                success, error_msg = import_obsidian_note_to_db(note_data)
-                if success:
-                    imported_files += 1
-                else:
-                    errors.append(error_msg)
-            except Exception as e:
-                error_msg = f"Error processing {file_path}: {str(e)}"
-                logger.error(error_msg)
-                errors.append(error_msg)
-
-            progress((i + 1) / total_files, f"Imported {imported_files} of {total_files} files")
-            sleep(0.1)  # Small delay to prevent UI freezing
-
-        return imported_files, total_files, errors
-    except Exception as e:
-        error_msg = f"Error scanning vault: {str(e)}\n{traceback.format_exc()}"
-        logger.error(error_msg)
-        return 0, 0, [error_msg]

App_Function_Libraries/Gradio_UI/Introduction_tab.py

@@ -1,167 +0,0 @@
-# Introduction_tab.py
-# Gradio UI functions for the Introduction tab
-#
-# Imports
-#
-# External Imports
-import gradio as gr
-#
-# Local Imports
-from App_Function_Libraries.DB.DB_Manager import get_db_config
-#
-####################################################################################################
-#
-# Functions:
-
-
-
-def create_introduction_tab():
-    with gr.TabItem("Introduction", visible=True):
-        db_config = get_db_config()
-        db_type = db_config['type']
-        gr.Markdown(f"# tldw: Your LLM-powered Research Multi-tool (Using {db_type.capitalize()} Database)")
-        with gr.Row():
-            with gr.Column():
-                gr.Markdown("""### What can it do?
-                - Transcribe and summarize videos from URLs/Local files
-                - Transcribe and Summarize Audio files/Podcasts (URL/local file)
-                - Summarize articles from URLs/Local notes
-                - Ingest and summarize books(epub/PDF)
-                - Ingest and summarize research papers (PDFs - WIP)
-                - Search and display ingested content + summaries
-                - Create and manage custom prompts
-                - Chat with an LLM of your choice to generate content using the selected item + Prompts
-                - Keyword support for content search and display
-                - Export keywords/items to markdown/CSV(csv is wip)
-                - Import existing notes from Obsidian to the database (Markdown/txt files or a zip containing a collection of files)
-                - View and manage chat history
-                - Writing Tools: Grammar & Style check, Tone Analyzer & Editor, more planned...
-                - RAG (Retrieval-Augmented Generation) support for content generation(think about asking questions about your entire library of items)
-                - More features planned...
-                - All powered by your choice of LLM.
-                    - Currently supports: Local-LLM(llamafile-server), OpenAI, Anthropic, Cohere, Groq, DeepSeek, OpenRouter, Llama.cpp, Kobold, Ooba, Tabbyapi, VLLM and more to come...
-                - All data is stored locally in a SQLite database for easy access and management.
-                - No trackers (Gradio has some analytics but it's disabled here...)
-                - No ads, no tracking, no BS. Just you and your content.
-                - Open-source and free to use. Contributions welcome!
-                - If you have any thoughts or feedback, please let me know on github or via email.
-                """)
-                gr.Markdown(
-                    """Follow this project at [tl/dw: Too Long, Didn't Watch - Your Personal Research Multi-Tool - GitHub](https://github.com/rmusser01/tldw)""")
-            with gr.Column():
-                gr.Markdown("""### How to use:
-                ##### Quick Start: Just click on the appropriate tab for what you're trying to do and fill in the required fields. Click "Process <video/audio/article/etc>" and wait for the results.
-                #### Simple Instructions
-                - Basic Usage:
-                    - If you don't have an API key/don't know what an LLM is/don't know what an API key is, please look further down the page for information on getting started.
-                    - If you want summaries/chat with an LLM, you'll need:
-                        1. An API key for the LLM API service you want to use, or,
-                        2. A local inference server running an LLM (like llamafile-server/llama.cpp - for instructions on how to do so see the projects README or below), or,
-                        3. A "local" inference server you have access to running an LLM.
-                    - If you just want transcriptions you can ignore the above.
-                    - Select the tab for the task you want to perform
-                    - Fill in the required fields
-                    - Click the "Process" button
-                    - Wait for the results to appear
-                    - Download the results if needed
-                    - Repeat as needed
-                    - As of writing this, the UI is still a work in progress.
-                    - That being said, I plan to replace it all eventually. In the meantime, please have patience.
-                    - The UI is divided into tabs for different tasks.
-                    - Each tab has a set of fields that you can fill in to perform the task.
-                    - Some fields are mandatory, some are optional.
-                    - The fields are mostly self-explanatory, but I will try to add more detailed instructions as I go.
-                #### Detailed Usage:
-                - There are 8 Top-level tabs in the UI. Each tab has a specific set of tasks that you can perform by selecting one of the 'sub-tabs' made available by clicking on the top tab.
-                - The tabs are as follows:
-                    1. Transcription / Summarization / Ingestion - This tab is for processing videos, audio files, articles, books, and PDFs/office docs.
-                    2. Search / Detailed View - This tab is for searching and displaying content from the database. You can also view detailed information about the selected item.
-                    3. Chat with an LLM - This tab is for chatting with an LLM to generate content based on the selected item and prompts.
-                    4. Edit Existing Items - This tab is for editing existing items in the database (Prompts + ingested items).
-                    5. Writing Tools - This tab is for using various writing tools like Grammar & Style check, Tone Analyzer & Editor, etc.
-                    6. Keywords - This tab is for managing keywords for content search and display.
-                    7. Import/Export - This tab is for importing notes from Obsidian and exporting keywords/items to markdown/CSV.
-                    8. Utilities - This tab contains some random utilities that I thought might be useful.
-                - Each sub-tab is responsible for that set of functionality. This is reflected in the codebase as well, where I have split the functionality into separate files for each tab/larger goal.
-                """)
-        with gr.Row():
-            gr.Markdown("""### HELP! I don't know what any of this this shit is!
-            ### DON'T PANIC
-            #### Its ok, you're not alone, most people have no clue what any of this stuff is.
-            - So let's try and fix that.
-
-            #### Introduction to LLMs:
-            - Non-Technical introduction to Generative AI and LLMs: https://paruir.medium.com/understanding-generative-ai-and-llms-a-non-technical-overview-part-1-788c0eb0dd64
-            - Google's Intro to LLMs: https://developers.google.com/machine-learning/resources/intro-llms#llm_considerations
-            - LLMs 101(coming from a tech background): https://vinija.ai/models/LLM/
-            - LLM Fundamentals / LLM Scientist / LLM Engineer courses(Free): https://github.com/mlabonne/llm-course
-
-            #### Various Phrases & Terms to know
-            - **LLM** - Large Language Model - A type of neural network that can generate human-like text.
-            - **API** - Application Programming Interface - A set of rules and protocols that allows one software application to communicate with another.
-                * Think of it like a post address for a piece of software. You can send messages to and from it.
-            - **API Key** - A unique identifier that is used to authenticate a user, developer, or calling program to an API.
-                * Like the key to a post office box. You need it to access the contents.
-            - **GUI** - Graphical User Interface - the thing facilitating your interact with this application.
-            - **DB** - Database
-            - **Prompt Engineering** - The process of designing prompts that are used to guide the output of a language model. Is a meme but also very much not.
-            - **Quantization** - The process of converting a continuous range of values into a finite range of discrete values.
-                * https://github.com/ggerganov/llama.cpp/blob/cddae4884c853b1a7ab420458236d666e2e34423/examples/quantize/README.md#L27
-            - **GGUF Files** - GGUF is a binary format that is designed for fast loading and saving of models, and for ease of reading. Models are traditionally developed using PyTorch or another framework, and then converted to GGUF for use in GGML. https://github.com/ggerganov/ggml/blob/master/docs/gguf.md
-            - **Inference Engine** - A software system that is designed to execute a model that has been trained by a machine learning algorithm. Llama.cpp and Kobold.cpp are examples of inference engines.
-            - **Abliteration** - https://huggingface.co/blog/mlabonne/abliteration
-            """)
-        with gr.Row():
-            gr.Markdown("""### Ok cool, but how do I get started? I don't have an API key or a local server running...
-                #### Great, glad you asked! Getting Started:
-                - **Getting an API key for a commercial services provider:
-                    - **OpenAI:**
-                        * https://platform.openai.com/docs/quickstart
-                    - **Anthropic:**
-                        * https://docs.anthropic.com/en/api/getting-started
-                    - **Cohere:**
-                        * https://docs.cohere.com/
-                        * They offer 1k free requests a month(up to 1million tokens total I think?), so you can try it out without paying.
-                    - **Groq:**
-                        * https://console.groq.com/keys
-                        * Offer an account with free credits to try out their service. No idea how much you get.
-                    - **DeepSeek:**
-                        * https://platform.deepseek.com/ (Chinese-hosted/is in english)
-                    - **OpenRouter:**
-                        * https://openrouter.ai/
-                    - **Mistral:**
-                        * https://console.mistral.ai/
-                - **Choosing a Model to download**
-                    - You'll first need to select a model you want to use with the server.
-                        - Keep in mind that the model you select will determine the quality of the output you get, and that models run fastest when offloaded fully to your GPU.
-                        * So this means that you can run a large model (Command-R) on CPU+System RAM, but you're gonna see a massive performance hit. Not saying its unusable, but it's not ideal.
-                        * With that in mind, I would recommend an abliterated version of Meta's Llama3.1 model for most tasks. (Abliterated since it won't refuse requests)
-                        * I say this because of the general quality of the model + it's context size.
-                        * You can find the model here: https://huggingface.co/mlabonne/Meta-Llama-3.1-8B-Instruct-abliterated-GGUF
-                        * And the Q8 quant(total size 8.6GB): https://huggingface.co/mlabonne/Meta-Llama-3.1-8B-Instruct-abliterated-GGUF/resolve/main/meta-llama-3.1-8b-instruct-abliterated.Q8_0.gguf?download=true
-                - **Local Inference Server:**
-                    - **Llamafile-Server (wrapper for llama.cpp):**
-                        * Run this script with the `--local_llm` argument next time, and you'll be walked through setting up a local instance of llamafile-server.
-                    - **Llama.cpp Inference Engine:**
-                        * Download the latest release for your platform here: https://github.com/ggerganov/llama.cpp/releases
-                        * Windows: `llama-<release_number>-bin-win-cuda-cu<11.7.1 or 12.2.0 - version depends on installed cuda>-x64.zip`
-                            * Run it: `llama-server.exe --model <path_to_model> -ctx 8192 -ngl 999`
-                                - `-ctx 8192` sets the context size to 8192 tokens, `-ngl 999` sets the number of layers to offload to the GPU to 999. (essentially ensuring we only use our GPU and not CPU for processing)
-                        * Macos: `llama-<release_number>-bin-macos-arm64.zip - for Apple Silicon / `llama-<release_number>-bin-macos-x64.zip` - for Intel Macs
-                            * Run it: `llama-server --model <path_to_model> -ctx 8192 -ngl 999`
-                                - `-ctx 8192` sets the context size to 8192 tokens, `-ngl 999` sets the number of layers to offload to the GPU to 999. (essentially ensuring we only use our GPU and not CPU for processing)
-                        * Linux: You can probably figure it out.
-                    - **Kobold.cpp Server:**
-                        1. Download from here: https://github.com/LostRuins/koboldcpp/releases/latest
-                        2. `Double click KoboldCPP.exe and select model OR run "KoboldCPP.exe --help" in CMD prompt to get command line arguments for more control.`
-                        3. `Generally you don't have to change much besides the Presets and GPU Layers. Run with CuBLAS or CLBlast for GPU acceleration.`
-                        4. `Select your GGUF or GGML model you downloaded earlier, and connect to the displayed URL once it finishes loading.`
-                    - **Linux**
-                        1. `On Linux, we provide a koboldcpp-linux-x64 PyInstaller prebuilt binary on the releases page for modern systems. Simply download and run the binary.`
-                            * Alternatively, you can also install koboldcpp to the current directory by running the following terminal command: `curl -fLo koboldcpp https://github.com/LostRuins/koboldcpp/releases/latest/download/koboldcpp-linux-x64 && chmod +x koboldcpp`
-                        2. When you can't use the precompiled binary directly, we provide an automated build script which uses conda to obtain all dependencies, and generates (from source) a ready-to-use a pyinstaller binary for linux users. Simply execute the build script with `./koboldcpp.sh dist` and run the generated binary.
-            """)
-
-#
-# End of Introduction_tab.py
-####################################################################################################

App_Function_Libraries/Gradio_UI/Keywords.py

@@ -1,65 +0,0 @@
-# Keywords.py
-# Purpose: This file contains the functions to create the Keywords tab in the Gradio UI.
-#
-# The Keywords tab allows the user to add, delete, view, and export keywords from the database.
-#
-# Imports:
-
-#
-# External Imports
-import gradio as gr
-#
-# Internal Imports
-from App_Function_Libraries.DB.DB_Manager import add_keyword, delete_keyword, keywords_browser_interface, export_keywords_to_csv
-#
-#
-######################################################################################################################
-#
-# Functions:
-
-
-def create_export_keywords_tab():
-    with gr.TabItem("Export Keywords", visible=True):
-        with gr.Row():
-            with gr.Column():
-                export_keywords_button = gr.Button("Export Keywords")
-            with gr.Column():
-                export_keywords_output = gr.File(label="Download Exported Keywords")
-                export_keywords_status = gr.Textbox(label="Export Status")
-
-            export_keywords_button.click(
-                fn=export_keywords_to_csv,
-                outputs=[export_keywords_status, export_keywords_output]
-            )
-
-def create_view_keywords_tab():
-    with gr.TabItem("View Keywords", visible=True):
-        gr.Markdown("# Browse Keywords")
-        with gr.Column():
-            browse_output = gr.Markdown()
-            browse_button = gr.Button("View Existing Keywords")
-            browse_button.click(fn=keywords_browser_interface, outputs=browse_output)
-
-
-def create_add_keyword_tab():
-    with gr.TabItem("Add Keywords", visible=True):
-        with gr.Row():
-            with gr.Column():
-                gr.Markdown("# Add Keywords to the Database")
-                add_input = gr.Textbox(label="Add Keywords (comma-separated)", placeholder="Enter keywords here...")
-                add_button = gr.Button("Add Keywords")
-            with gr.Row():
-                add_output = gr.Textbox(label="Result")
-                add_button.click(fn=add_keyword, inputs=add_input, outputs=add_output)
-
-
-def create_delete_keyword_tab():
-    with gr.Tab("Delete Keywords", visible=True):
-        with gr.Row():
-            with gr.Column():
-                gr.Markdown("# Delete Keywords from the Database")
-                delete_input = gr.Textbox(label="Delete Keyword", placeholder="Enter keyword to delete here...")
-                delete_button = gr.Button("Delete Keyword")
-            with gr.Row():
-                delete_output = gr.Textbox(label="Result")
-                delete_button.click(fn=delete_keyword, inputs=delete_input, outputs=delete_output)

App_Function_Libraries/Gradio_UI/Live_Recording.py

@@ -1,142 +0,0 @@
-# Live_Recording.py
-# Description: Gradio UI for live audio recording and transcription.
-#
-# Import necessary modules and functions
-import logging
-import os
-import time
-
-# External Imports
-import gradio as gr
-# Local Imports
-from App_Function_Libraries.Audio.Audio_Transcription_Lib import (record_audio, speech_to_text, save_audio_temp,
-                                                                  stop_recording)
-from App_Function_Libraries.DB.DB_Manager import add_media_to_database
-from App_Function_Libraries.Metrics.metrics_logger import log_counter, log_histogram
-#
-#######################################################################################################################
-#
-# Functions:
-
-whisper_models = ["small", "medium", "small.en", "medium.en", "medium", "large", "large-v1", "large-v2", "large-v3",
-                  "distil-large-v2", "distil-medium.en", "distil-small.en"]
-
-def create_live_recording_tab():
-    with gr.Tab("Live Recording and Transcription", visible=True):
-        gr.Markdown("# Live Audio Recording and Transcription")
-        with gr.Row():
-            with gr.Column():
-                duration = gr.Slider(minimum=1, maximum=8000, value=15, label="Recording Duration (seconds)")
-                whisper_models_input = gr.Dropdown(choices=whisper_models, value="medium", label="Whisper Model")
-                vad_filter = gr.Checkbox(label="Use VAD Filter")
-                save_recording = gr.Checkbox(label="Save Recording")
-                save_to_db = gr.Checkbox(label="Save Transcription to Database(Must be checked to save - can be checked afer transcription)", value=False)
-                custom_title = gr.Textbox(label="Custom Title (for database)", visible=False)
-                record_button = gr.Button("Start Recording")
-                stop_button = gr.Button("Stop Recording")
-            with gr.Column():
-                output = gr.Textbox(label="Transcription", lines=10)
-                audio_output = gr.Audio(label="Recorded Audio", visible=False)
-
-        recording_state = gr.State(value=None)
-
-        def start_recording(duration):
-            log_counter("live_recording_start_attempt", labels={"duration": duration})
-            p, stream, audio_queue, stop_event, audio_thread = record_audio(duration)
-            log_counter("live_recording_start_success", labels={"duration": duration})
-            return (p, stream, audio_queue, stop_event, audio_thread)
-
-        def end_recording_and_transcribe(recording_state, whisper_model, vad_filter, save_recording, save_to_db, custom_title):
-            log_counter("live_recording_end_attempt", labels={"model": whisper_model})
-            start_time = time.time()
-
-            if recording_state is None:
-                log_counter("live_recording_end_error", labels={"error": "Recording hasn't started yet"})
-                return "Recording hasn't started yet.", None
-
-            p, stream, audio_queue, stop_event, audio_thread = recording_state
-            audio_data = stop_recording(p, stream, audio_queue, stop_event, audio_thread)
-
-            temp_file = save_audio_temp(audio_data)
-            segments = speech_to_text(temp_file, whisper_model=whisper_model, vad_filter=vad_filter)
-            transcription = "\n".join([segment["Text"] for segment in segments])
-
-            if save_recording:
-                log_counter("live_recording_saved", labels={"model": whisper_model})
-            else:
-                os.remove(temp_file)
-
-            end_time = time.time() - start_time
-            log_histogram("live_recording_end_duration", end_time, labels={"model": whisper_model})
-            log_counter("live_recording_end_success", labels={"model": whisper_model})
-            return transcription, temp_file if save_recording else None
-
-        def save_transcription_to_db(transcription, custom_title):
-            log_counter("save_transcription_to_db_attempt")
-            start_time = time.time()
-            if custom_title.strip() == "":
-                custom_title = "Self-recorded Audio"
-
-            try:
-                url = "self_recorded"
-                info_dict = {
-                    "title": custom_title,
-                    "uploader": "self-recorded",
-                    "webpage_url": url
-                }
-                segments = [{"Text": transcription}]
-                summary = ""
-                keywords = ["self-recorded", "audio"]
-                custom_prompt_input = ""
-                whisper_model = "self-recorded"
-                media_type = "audio"
-
-                result = add_media_to_database(
-                    url=url,
-                    info_dict=info_dict,
-                    segments=segments,
-                    summary=summary,
-                    keywords=keywords,
-                    custom_prompt_input=custom_prompt_input,
-                    whisper_model=whisper_model,
-                    media_type=media_type
-                )
-                end_time = time.time() - start_time
-                log_histogram("save_transcription_to_db_duration", end_time)
-                log_counter("save_transcription_to_db_success")
-                return f"Transcription saved to database successfully. {result}"
-            except Exception as e:
-                logging.error(f"Error saving transcription to database: {str(e)}")
-                log_counter("save_transcription_to_db_error", labels={"error": str(e)})
-                return f"Error saving transcription to database: {str(e)}"
-
-        def update_custom_title_visibility(save_to_db):
-            return gr.update(visible=save_to_db)
-
-        record_button.click(
-            fn=start_recording,
-            inputs=[duration],
-            outputs=[recording_state]
-        )
-
-        stop_button.click(
-            fn=end_recording_and_transcribe,
-            inputs=[recording_state, whisper_models_input, vad_filter, save_recording, save_to_db, custom_title],
-            outputs=[output, audio_output]
-        )
-
-        save_to_db.change(
-            fn=update_custom_title_visibility,
-            inputs=[save_to_db],
-            outputs=[custom_title]
-        )
-
-        gr.Button("Save to Database").click(
-            fn=save_transcription_to_db,
-            inputs=[output, custom_title],
-            outputs=gr.Textbox(label="Database Save Status")
-        )
-
-#
-# End of Functions
-########################################################################################################################

App_Function_Libraries/Gradio_UI/Llamafile_tab.py

@@ -1,312 +0,0 @@
-# Llamafile_tab.py
-# Description: Gradio interface for configuring and launching Llamafile with Local LLMs
-
-# Imports
-import os
-import logging
-from typing import Tuple, Optional
-import gradio as gr
-
-
-from App_Function_Libraries.Local_LLM.Local_LLM_Inference_Engine_Lib import (
-    download_llm_model,
-    llm_models,
-    start_llamafile,
-    get_gguf_llamafile_files
-)
-#
-#######################################################################################################################
-#
-# Functions:
-
-def create_chat_with_llamafile_tab():
-    # Function to update model path based on selection
-    def on_local_model_change(selected_model: str, search_directory: str) -> str:
-        if selected_model and isinstance(search_directory, str):
-            model_path = os.path.abspath(os.path.join(search_directory, selected_model))
-            logging.debug(f"Selected model path: {model_path}")  # Debug print for selected model path
-            return model_path
-        return "Invalid selection or directory."
-
-    # Function to update the dropdown with available models
-    def update_dropdowns(search_directory: str) -> Tuple[dict, str]:
-        logging.debug(f"User-entered directory: {search_directory}")  # Debug print for directory
-        if not os.path.isdir(search_directory):
-            logging.debug(f"Directory does not exist: {search_directory}")  # Debug print for non-existing directory
-            return gr.update(choices=[], value=None), "Directory does not exist."
-
-        logging.debug(f"Directory exists: {search_directory}, scanning for files...")  # Confirm directory exists
-        model_files = get_gguf_llamafile_files(search_directory)
-
-        if not model_files:
-            logging.debug(f"No model files found in {search_directory}")  # Debug print for no files found
-            return gr.update(choices=[], value=None), "No model files found in the specified directory."
-
-        # Update the dropdown choices with the model files found
-        logging.debug(f"Models loaded from {search_directory}: {model_files}")  # Debug: Print model files loaded
-        return gr.update(choices=model_files, value=None), f"Models loaded from {search_directory}."
-
-
-
-    def download_preset_model(selected_model: str) -> Tuple[str, str]:
-        """
-        Downloads the selected preset model.
-
-        Args:
-            selected_model (str): The key of the selected preset model.
-
-        Returns:
-            Tuple[str, str]: Status message and the path to the downloaded model.
-        """
-        model_info = llm_models.get(selected_model)
-        if not model_info:
-            return "Invalid model selection.", ""
-
-        try:
-            model_path = download_llm_model(
-                model_name=model_info["name"],
-                model_url=model_info["url"],
-                model_filename=model_info["filename"],
-                model_hash=model_info["hash"]
-            )
-            return f"Model '{model_info['name']}' downloaded successfully.", model_path
-        except Exception as e:
-            logging.error(f"Error downloading model: {e}")
-            return f"Failed to download model: {e}", ""
-
-    with gr.TabItem("Local LLM with Llamafile", visible=True):
-        gr.Markdown("# Settings for Llamafile")
-
-        with gr.Row():
-            with gr.Column():
-                am_noob = gr.Checkbox(label="Enable Sane Defaults", value=False, visible=True)
-                advanced_mode_toggle = gr.Checkbox(label="Advanced Mode - Show All Settings", value=False)
-                # Advanced Inputs
-                verbose_checked = gr.Checkbox(label="Enable Verbose Output", value=False, visible=False)
-                threads_checked = gr.Checkbox(label="Set CPU Threads", value=False, visible=False)
-                threads_value = gr.Number(label="Number of CPU Threads", value=None, precision=0, visible=False)
-                threads_batched_checked = gr.Checkbox(label="Enable Batched Inference", value=False, visible=False)
-                threads_batched_value = gr.Number(label="Batch Size for Inference", value=None, precision=0, visible=False)
-                model_alias_checked = gr.Checkbox(label="Set Model Alias", value=False, visible=False)
-                model_alias_value = gr.Textbox(label="Model Alias", value="", visible=False)
-                ctx_size_checked = gr.Checkbox(label="Set Prompt Context Size", value=False, visible=False)
-                ctx_size_value = gr.Number(label="Prompt Context Size", value=8124, precision=0, visible=False)
-                ngl_checked = gr.Checkbox(label="Enable GPU Layers", value=False, visible=True)
-                ngl_value = gr.Number(label="Number of GPU Layers", value=None, precision=0, visible=True)
-                batch_size_checked = gr.Checkbox(label="Set Batch Size", value=False, visible=False)
-                batch_size_value = gr.Number(label="Batch Size", value=512, visible=False)
-                memory_f32_checked = gr.Checkbox(label="Use 32-bit Floating Point", value=False, visible=False)
-                numa_checked = gr.Checkbox(label="Enable NUMA", value=False, visible=False)
-                server_timeout_value = gr.Number(label="Server Timeout", value=600, precision=0, visible=False)
-                host_checked = gr.Checkbox(label="Set IP to Listen On", value=False, visible=False)
-                host_value = gr.Textbox(label="Host IP Address", value="", visible=False)
-                port_checked = gr.Checkbox(label="Set Server Port", value=False, visible=False)
-                port_value = gr.Number(label="Port Number", value=8080, precision=0, visible=False)
-                api_key_checked = gr.Checkbox(label="Set API Key", value=False, visible=False)
-                api_key_value = gr.Textbox(label="API Key", value="", visible=False)
-                http_threads_checked = gr.Checkbox(label="Set HTTP Server Threads", value=False, visible=False)
-                http_threads_value = gr.Number(label="Number of HTTP Server Threads", value=None, precision=0, visible=False)
-                hf_repo_checked = gr.Checkbox(label="Use Huggingface Repo Model", value=False, visible=False)
-                hf_repo_value = gr.Textbox(label="Huggingface Repo Name", value="", visible=False)
-                hf_file_checked = gr.Checkbox(label="Set Huggingface Model File", value=False, visible=False)
-                hf_file_value = gr.Textbox(label="Huggingface Model File", value="", visible=False)
-
-            with gr.Column():
-                # Model Selection Section
-                gr.Markdown("## Model Selection")
-
-                # Option 1: Select from Local Filesystem
-                with gr.Row():
-                    search_directory = gr.Textbox(label="Model Directory",
-                                                  placeholder="Enter directory path(currently '.\Models')",
-                                                  value=".\Models",
-                                                  interactive=True)
-
-                # Initial population of local models
-                initial_dropdown_update, _ = update_dropdowns(".\Models")
-                refresh_button = gr.Button("Refresh Models")
-                local_model_dropdown = gr.Dropdown(label="Select Model from Directory", choices=[])
-                # Display selected model path
-                model_value = gr.Textbox(label="Selected Model File Path", value="", interactive=False)
-
-                # Option 2: Download Preset Models
-                gr.Markdown("## Download Preset Models")
-
-                preset_model_dropdown = gr.Dropdown(
-                    label="Select a Preset Model",
-                    choices=list(llm_models.keys()),
-                    value=None,
-                    interactive=True,
-                    info="Choose a preset model to download."
-                )
-                download_preset_button = gr.Button("Download Selected Preset")
-
-        with gr.Row():
-            with gr.Column():
-                start_button = gr.Button("Start Llamafile")
-                stop_button = gr.Button("Stop Llamafile (doesn't work)")
-                output_display = gr.Markdown()
-
-
-        # Show/hide advanced inputs based on toggle
-        def update_visibility(show_advanced: bool):
-            components = [
-                verbose_checked, threads_checked, threads_value,
-                http_threads_checked, http_threads_value,
-                hf_repo_checked, hf_repo_value,
-                hf_file_checked, hf_file_value,
-                ctx_size_checked, ctx_size_value,
-                ngl_checked, ngl_value,
-                host_checked, host_value,
-                port_checked, port_value
-            ]
-            return [gr.update(visible=show_advanced) for _ in components]
-
-        def on_start_button_click(
-                am_noob: bool,
-                verbose_checked: bool,
-                threads_checked: bool,
-                threads_value: Optional[int],
-                threads_batched_checked: bool,
-                threads_batched_value: Optional[int],
-                model_alias_checked: bool,
-                model_alias_value: str,
-                http_threads_checked: bool,
-                http_threads_value: Optional[int],
-                model_value: str,
-                hf_repo_checked: bool,
-                hf_repo_value: str,
-                hf_file_checked: bool,
-                hf_file_value: str,
-                ctx_size_checked: bool,
-                ctx_size_value: Optional[int],
-                ngl_checked: bool,
-                ngl_value: Optional[int],
-                batch_size_checked: bool,
-                batch_size_value: Optional[int],
-                memory_f32_checked: bool,
-                numa_checked: bool,
-                server_timeout_value: Optional[int],
-                host_checked: bool,
-                host_value: str,
-                port_checked: bool,
-                port_value: Optional[int],
-                api_key_checked: bool,
-                api_key_value: str
-        ) -> str:
-            """
-            Event handler for the Start Llamafile button.
-            """
-            try:
-                result = start_llamafile(
-                    am_noob,
-                    verbose_checked,
-                    threads_checked,
-                    threads_value,
-                    threads_batched_checked,
-                    threads_batched_value,
-                    model_alias_checked,
-                    model_alias_value,
-                    http_threads_checked,
-                    http_threads_value,
-                    model_value,
-                    hf_repo_checked,
-                    hf_repo_value,
-                    hf_file_checked,
-                    hf_file_value,
-                    ctx_size_checked,
-                    ctx_size_value,
-                    ngl_checked,
-                    ngl_value,
-                    batch_size_checked,
-                    batch_size_value,
-                    memory_f32_checked,
-                    numa_checked,
-                    server_timeout_value,
-                    host_checked,
-                    host_value,
-                    port_checked,
-                    port_value,
-                    api_key_checked,
-                    api_key_value
-                )
-                return result
-            except Exception as e:
-                logging.error(f"Error starting Llamafile: {e}")
-                return f"Failed to start Llamafile: {e}"
-
-        advanced_mode_toggle.change(
-            fn=update_visibility,
-            inputs=[advanced_mode_toggle],
-            outputs=[
-                verbose_checked, threads_checked, threads_value,
-                http_threads_checked, http_threads_value,
-                hf_repo_checked, hf_repo_value,
-                hf_file_checked, hf_file_value,
-                ctx_size_checked, ctx_size_value,
-                ngl_checked, ngl_value,
-                host_checked, host_value,
-                port_checked, port_value
-            ]
-        )
-
-        start_button.click(
-            fn=on_start_button_click,
-            inputs=[
-                am_noob,
-                verbose_checked,
-                threads_checked,
-                threads_value,
-                threads_batched_checked,
-                threads_batched_value,
-                model_alias_checked,
-                model_alias_value,
-                http_threads_checked,
-                http_threads_value,
-                model_value,
-                hf_repo_checked,
-                hf_repo_value,
-                hf_file_checked,
-                hf_file_value,
-                ctx_size_checked,
-                ctx_size_value,
-                ngl_checked,
-                ngl_value,
-                batch_size_checked,
-                batch_size_value,
-                memory_f32_checked,
-                numa_checked,
-                server_timeout_value,
-                host_checked,
-                host_value,
-                port_checked,
-                port_value,
-                api_key_checked,
-                api_key_value
-            ],
-            outputs=output_display
-        )
-
-        download_preset_button.click(
-            fn=download_preset_model,
-            inputs=[preset_model_dropdown],
-            outputs=[output_display, model_value]
-        )
-
-        # Click event for refreshing models
-        refresh_button.click(
-            fn=update_dropdowns,
-            inputs=[search_directory],  # Ensure that the directory path (string) is passed
-            outputs=[local_model_dropdown, output_display]  # Update dropdown and status
-        )
-
-        # Event to update model_value when a model is selected from the dropdown
-        local_model_dropdown.change(
-            fn=on_local_model_change,  # Function that calculates the model path
-            inputs=[local_model_dropdown, search_directory],  # Inputs: selected model and directory
-            outputs=[model_value]  # Output: Update the model_value textbox with the selected model path
-        )
-
-#
-#
-#######################################################################################################################

App_Function_Libraries/Gradio_UI/MMLU_Pro_tab.py

@@ -1,115 +0,0 @@
-# MMLU_Pro_tab.py
-# is a library that contains the Gradio UI code for the MMLU-Pro benchmarking tool.
-#
-##############################################################################################################
-# Imports
-import os
-
-import gradio as gr
-import logging
-#
-# External Imports
-from tqdm import tqdm
-# Local Imports
-from App_Function_Libraries.Benchmarks_Evaluations.MMLU_Pro.MMLU_Pro_rewritten import (
-    load_mmlu_pro, run_mmlu_pro_benchmark, mmlu_pro_main, load_mmlu_pro_config
-)
-#
-##############################################################################################################
-#
-# Functions:
-
-# Set up logging
-logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
-logger = logging.getLogger(__name__)
-
-
-def get_categories():
-    """Fetch categories using the dataset loader from MMLU_Pro_rewritten.py"""
-    try:
-        test_data, _ = load_mmlu_pro()  # Use the function from MMLU_Pro_rewritten.py
-        return list(test_data.keys())  # Return the categories from the test dataset
-    except Exception as e:
-        logger.error(f"Failed to load categories: {e}")
-        return ["Error loading categories"]
-
-
-def load_categories():
-    """Helper function to return the categories for the Gradio dropdown."""
-    categories = get_categories()  # Fetch categories from the dataset
-    if categories:
-        return gr.update(choices=categories, value=categories[0])  # Update dropdown with categories
-    else:
-        return gr.update(choices=["Error loading categories"], value="Error loading categories")
-
-
-def run_benchmark_from_ui(url, api_key, model, timeout, category, parallel, verbosity, log_prompt):
-    """Function to run the benchmark with parameters from the UI."""
-
-    # Override config with UI parameters
-    config = load_mmlu_pro_config(
-        url=url,
-        api_key=api_key,
-        model=model,
-        timeout=timeout,
-        categories=[category] if category else None,
-        parallel=parallel,
-        verbosity=verbosity,
-        log_prompt=log_prompt
-    )
-
-    # Run the benchmarking process
-    try:
-        # Call the main benchmarking function
-        mmlu_pro_main()
-
-        # Assume the final report is generated in "eval_results" folder
-        report_path = os.path.join("eval_results", config["server"]["model"].replace("/", "-"), "final_report.txt")
-
-        # Read the final report
-        with open(report_path, "r") as f:
-            report = f.read()
-
-        return report
-    except Exception as e:
-        logger.error(f"An error occurred during benchmark execution: {e}")
-        return f"An error occurred during benchmark execution. Please check the logs for more information. Error: {str(e)}"
-
-
-def create_mmlu_pro_tab():
-    """Create the Gradio UI tab for MMLU-Pro Benchmark."""
-    with gr.TabItem("MMLU-Pro Benchmark", visible=True):
-        gr.Markdown("## Run MMLU-Pro Benchmark")
-
-        with gr.Row():
-            with gr.Column():
-                # Inputs for the benchmark
-                url = gr.Textbox(label="Server URL")
-                api_key = gr.Textbox(label="API Key", type="password")
-                model = gr.Textbox(label="Model Name")
-                timeout = gr.Number(label="Timeout (seconds)", value=30)
-                category = gr.Dropdown(label="Category", choices=["Load categories..."])
-                load_categories_btn = gr.Button("Load Categories")
-                parallel = gr.Slider(label="Parallel Requests", minimum=1, maximum=10, step=1, value=1)
-                verbosity = gr.Slider(label="Verbosity Level", minimum=0, maximum=2, step=1, value=1)
-                log_prompt = gr.Checkbox(label="Log Prompt")
-
-            with gr.Column():
-                # Run button and output display
-                run_button = gr.Button("Run Benchmark")
-                output = gr.Textbox(label="Benchmark Results", lines=20)
-
-        # When "Load Categories" is clicked, load the categories into the dropdown
-        load_categories_btn.click(
-            load_categories,
-            outputs=category
-        )
-
-        # When "Run Benchmark" is clicked, trigger the run_benchmark_from_ui function
-        run_button.click(
-            run_benchmark_from_ui,  # Use the function defined to run the benchmark
-            inputs=[url, api_key, model, timeout, category, parallel, verbosity, log_prompt],
-            outputs=output
-        )
-
-    return [url, api_key, model, timeout, category, parallel, verbosity, log_prompt, run_button, output]

App_Function_Libraries/Gradio_UI/Media_edit.py

@@ -1,301 +0,0 @@
-# Media_edit.py
-# Functions for Gradio Media_Edit UI
-
-# Imports
-import logging
-import uuid
-
-# External Imports
-import gradio as gr
-#
-# Local Imports
-from App_Function_Libraries.DB.DB_Manager import add_prompt, update_media_content, db, add_or_update_prompt, \
-    load_prompt_details, fetch_keywords_for_media, update_keywords_for_media
-from App_Function_Libraries.Gradio_UI.Gradio_Shared import update_dropdown, update_prompt_dropdown
-from App_Function_Libraries.DB.SQLite_DB import fetch_item_details
-
-
-def create_media_edit_tab():
-    with gr.TabItem("Edit Existing Items", visible=True):
-        gr.Markdown("# Search and Edit Media Items")
-
-        with gr.Row():
-            search_query_input = gr.Textbox(label="Search Query", placeholder="Enter your search query here...")
-            search_type_input = gr.Radio(choices=["Title", "URL", "Keyword", "Content"], value="Title", label="Search By")
-            search_button = gr.Button("Search")
-
-        with gr.Row():
-            items_output = gr.Dropdown(label="Select Item", choices=[], interactive=True)
-            item_mapping = gr.State({})
-
-        content_input = gr.Textbox(label="Edit Content", lines=10)
-        prompt_input = gr.Textbox(label="Edit Prompt", lines=3)
-        summary_input = gr.Textbox(label="Edit Summary", lines=5)
-
-        # Adding keyword input box for editing
-        keywords_input = gr.Textbox(label="Edit Keywords (comma-separated)", placeholder="Enter keywords here...")
-
-        update_button = gr.Button("Update Media Content")
-        status_message = gr.Textbox(label="Status", interactive=False)
-
-        # Function to update the dropdown with search results
-        search_button.click(
-            fn=update_dropdown,
-            inputs=[search_query_input, search_type_input],
-            outputs=[items_output, item_mapping]
-        )
-
-        # Function to load selected media content including keywords
-        def load_selected_media_content(selected_item, item_mapping):
-            if selected_item and item_mapping and selected_item in item_mapping:
-                media_id = item_mapping[selected_item]
-                content, prompt, summary = fetch_item_details(media_id)
-
-                # Fetch keywords for the selected item
-                keywords = fetch_keywords_for_media(media_id)
-                keywords_str = ", ".join(keywords) if keywords else ""
-
-                return content, prompt, summary, keywords_str
-            return "No item selected or invalid selection", "", "", ""
-
-        # Load the selected media content and associated keywords
-        items_output.change(
-            fn=load_selected_media_content,
-            inputs=[items_output, item_mapping],
-            outputs=[content_input, prompt_input, summary_input, keywords_input]
-        )
-
-        # Function to update media content, prompt, summary, and keywords
-        def update_media_with_keywords(selected_item, item_mapping, content, prompt, summary, keywords):
-            if selected_item and item_mapping and selected_item in item_mapping:
-                media_id = item_mapping[selected_item]
-
-                # Split keywords into a list
-                keyword_list = [kw.strip() for kw in keywords.split(",") if kw.strip()]
-
-                # Update content, prompt, summary, and keywords in the database
-                status = update_media_content(media_id, content, prompt, summary)
-                keyword_status = update_keywords_for_media(media_id, keyword_list)
-
-                return f"{status}\nKeywords: {keyword_status}"
-            return "No item selected or invalid selection"
-
-        # Update button click event
-        update_button.click(
-            fn=update_media_with_keywords,
-            inputs=[items_output, item_mapping, content_input, prompt_input, summary_input, keywords_input],
-            outputs=status_message
-        )
-
-
-def create_media_edit_and_clone_tab():
-    with gr.TabItem("Clone and Edit Existing Items", visible=True):
-        gr.Markdown("# Search, Edit, and Clone Existing Items")
-
-        with gr.Row():
-            with gr.Column():
-                search_query_input = gr.Textbox(label="Search Query", placeholder="Enter your search query here...")
-                search_type_input = gr.Radio(choices=["Title", "URL", "Keyword", "Content"], value="Title",
-                                         label="Search By")
-            with gr.Column():
-                search_button = gr.Button("Search")
-                clone_button = gr.Button("Clone Item")
-            save_clone_button = gr.Button("Save Cloned Item", visible=False)
-        with gr.Row():
-            items_output = gr.Dropdown(label="Select Item", choices=[], interactive=True)
-            item_mapping = gr.State({})
-
-        content_input = gr.Textbox(label="Edit Content", lines=10)
-        prompt_input = gr.Textbox(label="Edit Prompt", lines=3)
-        summary_input = gr.Textbox(label="Edit Summary", lines=5)
-        new_title_input = gr.Textbox(label="New Title (for cloning)", visible=False)
-        status_message = gr.Textbox(label="Status", interactive=False)
-
-        search_button.click(
-            fn=update_dropdown,
-            inputs=[search_query_input, search_type_input],
-            outputs=[items_output, item_mapping]
-        )
-
-        def load_selected_media_content(selected_item, item_mapping):
-            if selected_item and item_mapping and selected_item in item_mapping:
-                media_id = item_mapping[selected_item]
-                content, prompt, summary = fetch_item_details(media_id)
-                return content, prompt, summary, gr.update(visible=True), gr.update(visible=False)
-            return "No item selected or invalid selection", "", "", gr.update(visible=False), gr.update(visible=False)
-
-        items_output.change(
-            fn=load_selected_media_content,
-            inputs=[items_output, item_mapping],
-            outputs=[content_input, prompt_input, summary_input, clone_button, save_clone_button]
-        )
-
-        def prepare_for_cloning(selected_item):
-            return gr.update(value=f"Copy of {selected_item}", visible=True), gr.update(visible=True)
-
-        clone_button.click(
-            fn=prepare_for_cloning,
-            inputs=[items_output],
-            outputs=[new_title_input, save_clone_button]
-        )
-
-        def save_cloned_item(selected_item, item_mapping, content, prompt, summary, new_title):
-            if selected_item and item_mapping and selected_item in item_mapping:
-                original_media_id = item_mapping[selected_item]
-                try:
-                    with db.get_connection() as conn:
-                        cursor = conn.cursor()
-
-                        # Fetch the original item's details
-                        cursor.execute("SELECT type, url FROM Media WHERE id = ?", (original_media_id,))
-                        original_type, original_url = cursor.fetchone()
-
-                        # Generate a new unique URL
-                        new_url = f"{original_url}_clone_{uuid.uuid4().hex[:8]}"
-
-                        # Insert new item into Media table
-                        cursor.execute("""
-                            INSERT INTO Media (title, content, url, type)
-                            VALUES (?, ?, ?, ?)
-                        """, (new_title, content, new_url, original_type))
-
-                        new_media_id = cursor.lastrowid
-
-                        # Insert new item into MediaModifications table
-                        cursor.execute("""
-                            INSERT INTO MediaModifications (media_id, prompt, summary, modification_date)
-                            VALUES (?, ?, ?, CURRENT_TIMESTAMP)
-                        """, (new_media_id, prompt, summary))
-
-                        # Copy keywords from the original item
-                        cursor.execute("""
-                            INSERT INTO MediaKeywords (media_id, keyword_id)
-                            SELECT ?, keyword_id
-                            FROM MediaKeywords
-                            WHERE media_id = ?
-                        """, (new_media_id, original_media_id))
-
-                        # Update full-text search index
-                        cursor.execute("""
-                            INSERT INTO media_fts (rowid, title, content)
-                            VALUES (?, ?, ?)
-                        """, (new_media_id, new_title, content))
-
-                        conn.commit()
-
-                    return f"Cloned item saved successfully with ID: {new_media_id}", gr.update(
-                        visible=False), gr.update(visible=False)
-                except Exception as e:
-                    logging.error(f"Error saving cloned item: {e}")
-                    return f"Error saving cloned item: {str(e)}", gr.update(visible=True), gr.update(visible=True)
-            else:
-                return "No item selected or invalid selection", gr.update(visible=True), gr.update(visible=True)
-
-        save_clone_button.click(
-            fn=save_cloned_item,
-            inputs=[items_output, item_mapping, content_input, prompt_input, summary_input, new_title_input],
-            outputs=[status_message, new_title_input, save_clone_button]
-        )
-
-
-def create_prompt_edit_tab():
-    with gr.TabItem("Add & Edit Prompts", visible=True):
-        with gr.Row():
-            with gr.Column():
-                prompt_dropdown = gr.Dropdown(
-                    label="Select Prompt",
-                    choices=[],
-                    interactive=True
-                )
-                prompt_list_button = gr.Button("List Prompts")
-
-            with gr.Column():
-                title_input = gr.Textbox(label="Title", placeholder="Enter the prompt title")
-                author_input = gr.Textbox(label="Author", placeholder="Enter the prompt's author", lines=3)
-                description_input = gr.Textbox(label="Description", placeholder="Enter the prompt description", lines=3)
-                system_prompt_input = gr.Textbox(label="System Prompt", placeholder="Enter the system prompt", lines=3)
-                user_prompt_input = gr.Textbox(label="User Prompt", placeholder="Enter the user prompt", lines=3)
-                add_prompt_button = gr.Button("Add/Update Prompt")
-                add_prompt_output = gr.HTML()
-
-        # Event handlers
-        prompt_list_button.click(
-            fn=update_prompt_dropdown,
-            outputs=prompt_dropdown
-        )
-
-        add_prompt_button.click(
-            fn=add_or_update_prompt,
-            inputs=[title_input, author_input, description_input, system_prompt_input, user_prompt_input],
-            outputs=add_prompt_output
-        )
-
-        # Load prompt details when selected
-        prompt_dropdown.change(
-            fn=load_prompt_details,
-            inputs=[prompt_dropdown],
-            outputs=[title_input, author_input, system_prompt_input, user_prompt_input]
-        )
-
-
-def create_prompt_clone_tab():
-    with gr.TabItem("Clone and Edit Prompts", visible=True):
-        with gr.Row():
-            with gr.Column():
-                gr.Markdown("# Clone and Edit Prompts")
-                prompt_dropdown = gr.Dropdown(
-                    label="Select Prompt",
-                    choices=[],
-                    interactive=True
-                )
-                prompt_list_button = gr.Button("List Prompts")
-
-            with gr.Column():
-                title_input = gr.Textbox(label="Title", placeholder="Enter the prompt title")
-                author_input = gr.Textbox(label="Author", placeholder="Enter the prompt's author", lines=3)
-                description_input = gr.Textbox(label="Description", placeholder="Enter the prompt description", lines=3)
-                system_prompt_input = gr.Textbox(label="System Prompt", placeholder="Enter the system prompt", lines=3)
-                user_prompt_input = gr.Textbox(label="User Prompt", placeholder="Enter the user prompt", lines=3)
-                clone_prompt_button = gr.Button("Clone Selected Prompt")
-                save_cloned_prompt_button = gr.Button("Save Cloned Prompt", visible=False)
-                add_prompt_output = gr.HTML()
-
-        # Event handlers
-        prompt_list_button.click(
-            fn=update_prompt_dropdown,
-            outputs=prompt_dropdown
-        )
-
-        # Load prompt details when selected
-        prompt_dropdown.change(
-            fn=load_prompt_details,
-            inputs=[prompt_dropdown],
-            outputs=[title_input, author_input, description_input, system_prompt_input, user_prompt_input]
-        )
-
-        def prepare_for_cloning(selected_prompt):
-            if selected_prompt:
-                return gr.update(value=f"Copy of {selected_prompt}"), gr.update(visible=True)
-            return gr.update(), gr.update(visible=False)
-
-        clone_prompt_button.click(
-            fn=prepare_for_cloning,
-            inputs=[prompt_dropdown],
-            outputs=[title_input, save_cloned_prompt_button]
-        )
-
-        def save_cloned_prompt(title, description, system_prompt, user_prompt):
-            try:
-                result = add_prompt(title, description, system_prompt, user_prompt)
-                if result == "Prompt added successfully.":
-                    return result, gr.update(choices=update_prompt_dropdown())
-                else:
-                    return result, gr.update()
-            except Exception as e:
-                return f"Error saving cloned prompt: {str(e)}", gr.update()
-
-        save_cloned_prompt_button.click(
-            fn=save_cloned_prompt,
-            inputs=[title_input, description_input, system_prompt_input, user_prompt_input],
-            outputs=[add_prompt_output, prompt_dropdown]
-        )

App_Function_Libraries/Gradio_UI/Media_wiki_tab.py

@@ -1,340 +0,0 @@
-# Media_wiki_tab.py
-# Description: Gradio UI snippet that allows users to import a MediaWiki XML dump file into the application.
-#
-# Imports
-import os
-from threading import Thread
-#
-# 3rd-party Imports
-import gradio as gr
-import yaml
-from ruamel.yaml import YAML
-#
-# Local Imports
-from App_Function_Libraries.MediaWiki.Media_Wiki import import_mediawiki_dump, config
-#
-#######################################################################################################################
-#
-# Create MediaWiki Import Tab
-
-def create_mediawiki_import_tab():
-    with gr.Tab("MediaWiki Import"):
-        gr.Markdown("# Import MediaWiki Dump")
-        with gr.Row():
-            with gr.Column():
-                file_path = gr.File(label="MediaWiki XML Dump File")
-                wiki_name = gr.Textbox(label="Wiki Name", placeholder="Enter a unique name for this wiki")
-                namespaces = gr.Textbox(label="Namespaces (comma-separated integers, leave empty for all)")
-                skip_redirects = gr.Checkbox(label="Skip Redirects", value=True)
-                single_item = gr.Checkbox(label="Import as Single Item", value=False)
-                chunk_method = gr.Dropdown(
-                    choices=["sentences", "words", "paragraphs", "tokens"],
-                    value="sentences",
-                    label="Chunking Method"
-                )
-                chunk_size = gr.Slider(minimum=100, maximum=2000, value=1000, step=100, label="Chunk Size")
-                chunk_overlap = gr.Slider(minimum=0, maximum=500, value=100, step=10, label="Chunk Overlap")
-                # FIXME - Add checkbox for 'Enable Summarization upon ingestion' for API summarization of chunks
-                # api_endpoint = gr.Dropdown(label="Select API Endpoint",
-                #                            choices=["Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek",
-                #                                     "Mistral", "OpenRouter",
-                #                                     "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama",
-                #                                     "HuggingFace"])
-                # api_key = gr.Textbox(label="API Key (if required)", type="password")
-                import_button = gr.Button("Import MediaWiki Dump")
-                cancel_button = gr.Button("Cancel Import", visible=False)
-            with gr.Column():
-                output = gr.Markdown(label="Import Status")
-                progress_bar = gr.Progress()
-
-        def validate_inputs(file_path, wiki_name, namespaces):
-            if not file_path:
-                return "Please select a MediaWiki XML dump file."
-            if not wiki_name:
-                return "Please enter a name for the wiki."
-            if namespaces:
-                try:
-                    [int(ns.strip()) for ns in namespaces.split(',')]
-                except ValueError:
-                    return "Invalid namespaces. Please enter comma-separated integers."
-            return None
-
-        def check_file_size(file_path):
-            max_size_mb = 1000  # 1 GB
-            file_size_mb = os.path.getsize(file_path) / (1024 * 1024)
-            if file_size_mb > max_size_mb:
-                return f"Warning: The selected file is {file_size_mb:.2f} MB. Importing large files may take a long time."
-            return None
-
-        import_thread = None
-        cancel_flag = False
-
-        def run_import(file_path, wiki_name, namespaces, skip_redirects, single_item, chunk_method, chunk_size,
-                       chunk_overlap, progress=gr.Progress()):#, api_endpoint=None, api_key=None):
-            validation_error = validate_inputs(file_path, wiki_name, namespaces)
-            if validation_error:
-                return gr.update(), gr.update(), validation_error
-
-            file_size_warning = check_file_size(file_path.name)
-            status_text = "# MediaWiki Import Process\n\n## Initializing\n- Starting import process...\n"
-            if file_size_warning:
-                status_text += f"- {file_size_warning}\n"
-
-            chunk_options = {
-                'method': chunk_method,
-                'max_size': chunk_size,
-                'overlap': chunk_overlap,
-                'adaptive': True,
-                'language': 'en'
-            }
-            namespaces_list = [int(ns.strip()) for ns in namespaces.split(',')] if namespaces else None
-
-            pages_processed = 0
-
-            try:
-                for progress_info in import_mediawiki_dump(
-                        file_path=file_path.name,
-                        wiki_name=wiki_name,
-                        namespaces=namespaces_list,
-                        skip_redirects=skip_redirects,
-                        chunk_options=chunk_options,
-                        single_item=single_item,
-                        progress_callback=progress,
-#                        api_name=api_endpoint,
-#                        api_key=api_key
-                ):
-                    if progress_info.startswith("Found"):
-                        status_text += f"\n## Parsing\n- {progress_info}\n"
-                    elif progress_info.startswith("Processed page"):
-                        pages_processed += 1
-                        if pages_processed % 10 == 0:  # Update every 10 pages to avoid too frequent updates
-                            status_text += f"- {progress_info}\n"
-                    elif progress_info.startswith("Successfully imported"):
-                        status_text += f"\n## Completed\n- {progress_info}\n- Total pages processed: {pages_processed}"
-                    else:
-                        status_text += f"- {progress_info}\n"
-
-                    yield gr.update(), gr.update(), status_text
-
-                status_text += "\n## Import Process Completed Successfully"
-            except Exception as e:
-                status_text += f"\n## Error\n- An error occurred during the import process: {str(e)}"
-
-            yield gr.update(visible=False), gr.update(visible=True), status_text
-
-        def start_import(*args):
-            nonlocal import_thread
-            import_thread = Thread(target=run_import, args=args)
-            import_thread.start()
-            return gr.update(visible=True), gr.update(visible=False), gr.update(
-                value="Import process started. Please wait...")
-
-        def cancel_import():
-            nonlocal cancel_flag
-            cancel_flag = True
-            return gr.update(visible=False), gr.update(visible=True)
-
-        import_button.click(
-            run_import,
-            inputs=[file_path, wiki_name, namespaces, skip_redirects, single_item, chunk_method, chunk_size,
-                    chunk_overlap],#, api_endpoint, api_key],
-            outputs=[cancel_button, import_button, output]
-        )
-
-        cancel_button.click(
-            cancel_import,
-            outputs=[cancel_button, import_button]
-        )
-
-    return file_path, wiki_name, namespaces, skip_redirects, single_item, chunk_method, chunk_size, chunk_overlap, import_button, output
-
-
-class PreservedTokenSafeDumper(yaml.SafeDumper):
-    def represent_scalar(self, tag, value, style=None):
-        if style is None and isinstance(value, str) and '\n' in value:
-            style = '|'
-        return super().represent_scalar(tag, value, style)
-
-
-def update_yaml_file(file_path, updates):
-    with open(file_path, 'r') as file:
-        lines = file.readlines()
-
-    def format_value(value):
-        if isinstance(value, bool):
-            return str(value).lower()
-        elif isinstance(value, (int, float)):
-            return str(value)
-        elif isinstance(value, list):
-            return '[' + ', '.join(map(str, value)) + ']'
-        else:
-            return f"'{value}'"
-
-    def update_line(line, updates, prefix=''):
-        for key, value in updates.items():
-            full_key = f"{prefix}{key}:" if prefix else f"{key}:"
-            if line.strip().startswith(full_key):
-                indentation = line[:line.index(full_key)]
-                if isinstance(value, dict):
-                    return line  # Keep the line as is for nested structures
-                else:
-                    return f"{indentation}{full_key} {format_value(value)}\n"
-        return line
-
-    updated_lines = []
-    current_prefix = ''
-    for line in lines:
-        stripped = line.strip()
-        if stripped and not stripped.startswith('#'):
-            indent = len(line) - len(line.lstrip())
-            if indent == 0:
-                current_prefix = ''
-            elif ':' in stripped and not stripped.endswith(':'):
-                current_prefix = '.'.join(current_prefix.split('.')[:-1]) + '.' if current_prefix else ''
-
-            updated_line = update_line(line, updates, current_prefix)
-
-            if updated_line == line and ':' in stripped and stripped.endswith(':'):
-                key = stripped[:-1].strip()
-                if current_prefix:
-                    current_prefix += f"{key}."
-                else:
-                    current_prefix = f"{key}."
-
-            updated_lines.append(updated_line)
-        else:
-            updated_lines.append(line)
-
-    with open(file_path, 'w') as file:
-        file.writelines(updated_lines)
-
-#
-#
-#######################################################################################################################
-#
-# Config tab
-
-yaml = YAML()
-yaml.preserve_quotes = True
-yaml.indent(mapping=2, sequence=4, offset=2)
-
-def load_config():
-    config_path = os.path.join('Config_Files', 'mediawiki_import_config.yaml')
-    with open(config_path, 'r') as file:
-        return yaml.load(file)
-
-def save_config(updated_config):
-    config_path = os.path.join('Config_Files', 'mediawiki_import_config.yaml')
-    config = load_config()
-
-
-def create_mediawiki_config_tab():
-    with gr.TabItem("MediaWiki Import Configuration", visible=True):
-        gr.Markdown("# MediaWiki Import Configuration (Broken currently/doesn't work)")
-        with gr.Row():
-            with gr.Column():
-                namespaces = gr.Textbox(label="Default Namespaces (comma-separated integers)",
-                                        value=','.join(map(str, config['import']['default_namespaces'])))
-                skip_redirects = gr.Checkbox(label="Skip Redirects by Default",
-                                             value=config['import']['default_skip_redirects'])
-                single_item = gr.Checkbox(label="Import as Single Item by Default",
-                                          value=config['import']['single_item_default'])
-                batch_size = gr.Number(value=config['import']['batch_size'], label="Batch Size")
-
-                chunk_method = gr.Dropdown(
-                    choices=config['chunking']['methods'],
-                    value=config['chunking']['default_method'],
-                    label="Default Chunking Method"
-                )
-                chunk_size = gr.Slider(minimum=100, maximum=2000, value=config['chunking']['default_size'], step=100,
-                                       label="Default Chunk Size")
-                chunk_overlap = gr.Slider(minimum=0, maximum=500, value=config['chunking']['default_overlap'], step=10,
-                                          label="Default Chunk Overlap")
-
-            with gr.Column():
-                max_workers = gr.Slider(minimum=1, maximum=16, value=config['processing']['max_workers'], step=1,
-                                        label="Max Worker Threads")
-
-                embedding_provider = gr.Dropdown(
-                    choices=['openai', 'local', 'huggingface'],
-                    value=config['embeddings']['provider'],
-                    label="Embedding Provider"
-                )
-                embedding_model = gr.Textbox(label="Embedding Model", value=config['embeddings']['model'])
-                api_key = gr.Textbox(label="API Key (if required)", type="password",
-                                     value=config['embeddings'].get('api_key', ''))
-                local_embedding_url = gr.Textbox(label="Local Embedding URL",
-                                                 value=config['embeddings'].get('local_url', ''))
-
-                checkpoints_enabled = gr.Checkbox(label="Enable Checkpoints", value=config['checkpoints']['enabled'])
-                checkpoint_directory = gr.Textbox(label="Checkpoint Directory", value=config['checkpoints']['directory'])
-
-                max_retries = gr.Number(value=config['error_handling']['max_retries'], label="Max Retries")
-                retry_delay = gr.Number(value=config['error_handling']['retry_delay'], label="Retry Delay (seconds)")
-
-        save_config_button = gr.Button("Save Configuration")
-        config_output = gr.Markdown(label="Configuration Status")
-
-        def update_config_from_ui(namespaces, skip_redirects, single_item, batch_size, chunk_method, chunk_size,
-                                  chunk_overlap, max_workers, embedding_provider, embedding_model, api_key,
-                                  local_embedding_url, checkpoints_enabled, checkpoint_directory, max_retries,
-                                  retry_delay):
-            current_config = load_config()
-            updated_config = {}
-
-            if namespaces != ','.join(map(str, current_config['import']['default_namespaces'])):
-                updated_config.setdefault('import', {})['default_namespaces'] = [int(ns.strip()) for ns in
-                                                                                 namespaces.split(',') if ns.strip()]
-            if skip_redirects != current_config['import']['default_skip_redirects']:
-                updated_config.setdefault('import', {})['default_skip_redirects'] = skip_redirects
-            if single_item != current_config['import']['single_item_default']:
-                updated_config.setdefault('import', {})['single_item_default'] = single_item
-            if int(batch_size) != current_config['import']['batch_size']:
-                updated_config.setdefault('import', {})['batch_size'] = int(batch_size)
-            if chunk_method != current_config['chunking']['default_method']:
-                updated_config.setdefault('chunking', {})['default_method'] = chunk_method
-            if int(chunk_size) != current_config['chunking']['default_size']:
-                updated_config.setdefault('chunking', {})['default_size'] = int(chunk_size)
-            if int(chunk_overlap) != current_config['chunking']['default_overlap']:
-                updated_config.setdefault('chunking', {})['default_overlap'] = int(chunk_overlap)
-            if int(max_workers) != current_config['processing']['max_workers']:
-                updated_config.setdefault('processing', {})['max_workers'] = int(max_workers)
-            if embedding_provider != current_config['embeddings']['provider']:
-                updated_config.setdefault('embeddings', {})['provider'] = embedding_provider
-            if embedding_model != current_config['embeddings']['model']:
-                updated_config.setdefault('embeddings', {})['model'] = embedding_model
-            if api_key != current_config['embeddings'].get('api_key', ''):
-                updated_config.setdefault('embeddings', {})['api_key'] = api_key
-            if local_embedding_url != current_config['embeddings'].get('local_url', ''):
-                updated_config.setdefault('embeddings', {})['local_url'] = local_embedding_url
-            if checkpoints_enabled != current_config['checkpoints']['enabled']:
-                updated_config.setdefault('checkpoints', {})['enabled'] = checkpoints_enabled
-            if checkpoint_directory != current_config['checkpoints']['directory']:
-                updated_config.setdefault('checkpoints', {})['directory'] = checkpoint_directory
-            if int(max_retries) != current_config['error_handling']['max_retries']:
-                updated_config.setdefault('error_handling', {})['max_retries'] = int(max_retries)
-            if int(retry_delay) != current_config['error_handling']['retry_delay']:
-                updated_config.setdefault('error_handling', {})['retry_delay'] = int(retry_delay)
-
-            return updated_config
-
-        def save_config_callback(*args):
-            updated_config = update_config_from_ui(*args)
-            save_config(updated_config)
-            return "Configuration saved successfully."
-
-        save_config_button.click(
-            save_config_callback,
-            inputs=[namespaces, skip_redirects, single_item, batch_size, chunk_method, chunk_size,
-                    chunk_overlap, max_workers, embedding_provider, embedding_model, api_key,
-                    local_embedding_url, checkpoints_enabled, checkpoint_directory, max_retries, retry_delay],
-            outputs=config_output
-        )
-
-    return namespaces, skip_redirects, single_item, batch_size, chunk_method, chunk_size, chunk_overlap, max_workers, \
-           embedding_provider, embedding_model, api_key, local_embedding_url, checkpoints_enabled, checkpoint_directory, \
-           max_retries, retry_delay, save_config_button, config_output
-
-#
-# End of MediaWiki Import Tab
-#######################################################################################################################

App_Function_Libraries/Gradio_UI/PDF_ingestion_tab.py

@@ -1,152 +0,0 @@
-# PDF_ingestion_tab.py
-# Gradio UI for ingesting PDFs into the database
-import os
-import shutil
-import tempfile
-
-# Imports
-#
-# External Imports
-import gradio as gr
-#
-# Local Imports
-from App_Function_Libraries.DB.DB_Manager import load_preset_prompts
-from App_Function_Libraries.Gradio_UI.Chat_ui import update_user_prompt
-from App_Function_Libraries.PDF.PDF_Ingestion_Lib import extract_metadata_from_pdf, extract_text_and_format_from_pdf, \
-    process_and_cleanup_pdf
-#
-#
-########################################################################################################################
-#
-# Functions:
-
-def create_pdf_ingestion_tab():
-    with gr.TabItem("PDF Ingestion", visible=True):
-        # TODO - Add functionality to extract metadata from pdf as part of conversion process in marker
-        gr.Markdown("# Ingest PDF Files and Extract Metadata")
-        with gr.Row():
-            with gr.Column():
-                pdf_file_input = gr.File(label="Uploaded PDF File", file_types=[".pdf"], visible=True)
-                pdf_upload_button = gr.UploadButton("Click to Upload PDF", file_types=[".pdf"])
-                pdf_title_input = gr.Textbox(label="Title (Optional)")
-                pdf_author_input = gr.Textbox(label="Author (Optional)")
-                pdf_keywords_input = gr.Textbox(label="Keywords (Optional, comma-separated)")
-                with gr.Row():
-                    custom_prompt_checkbox = gr.Checkbox(label="Use a Custom Prompt",
-                                                     value=False,
-                                                     visible=True)
-                    preset_prompt_checkbox = gr.Checkbox(label="Use a pre-set Prompt",
-                                                     value=False,
-                                                     visible=True)
-                with gr.Row():
-                    preset_prompt = gr.Dropdown(label="Select Preset Prompt",
-                                                choices=load_preset_prompts(),
-                                                visible=False)
-                with gr.Row():
-                    custom_prompt_input = gr.Textbox(label="Custom Prompt",
-                                                     placeholder="Enter custom prompt here",
-                                                     lines=3,
-                                                     visible=False)
-                with gr.Row():
-                    system_prompt_input = gr.Textbox(label="System Prompt",
-                                                     value="""
-<s>You are a bulleted notes specialist.
-[INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
-**Bulleted Note Creation Guidelines**
-
-**Headings**:
-- Based on referenced topics, not categories like quotes or terms
-- Surrounded by **bold** formatting 
-- Not listed as bullet points
-- No space between headings and list items underneath
-
-**Emphasis**:
-- **Important terms** set in bold font
-- **Text ending in a colon**: also bolded
-
-**Review**:
-- Ensure adherence to specified format
-- Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]""",
-                                                     lines=3,
-                                                     visible=False)
-
-                custom_prompt_checkbox.change(
-                    fn=lambda x: (gr.update(visible=x), gr.update(visible=x)),
-                    inputs=[custom_prompt_checkbox],
-                    outputs=[custom_prompt_input, system_prompt_input]
-                )
-                preset_prompt_checkbox.change(
-                    fn=lambda x: gr.update(visible=x),
-                    inputs=[preset_prompt_checkbox],
-                    outputs=[preset_prompt]
-                )
-
-                def update_prompts(preset_name):
-                    prompts = update_user_prompt(preset_name)
-                    return (
-                        gr.update(value=prompts["user_prompt"], visible=True),
-                        gr.update(value=prompts["system_prompt"], visible=True)
-                    )
-
-                preset_prompt.change(
-                    update_prompts,
-                    inputs=preset_prompt,
-                    outputs=[custom_prompt_input, system_prompt_input]
-                )
-
-                pdf_ingest_button = gr.Button("Ingest PDF")
-
-                pdf_upload_button.upload(fn=lambda file: file, inputs=pdf_upload_button, outputs=pdf_file_input)
-            with gr.Column():
-                pdf_result_output = gr.Textbox(label="Result")
-
-            pdf_ingest_button.click(
-                fn=process_and_cleanup_pdf,
-                inputs=[pdf_file_input, pdf_title_input, pdf_author_input, pdf_keywords_input],
-                outputs=pdf_result_output
-            )
-
-
-def test_pdf_ingestion(pdf_file):
-    if pdf_file is None:
-        return "No file uploaded", ""
-
-    try:
-        # Create a temporary directory
-        with tempfile.TemporaryDirectory() as temp_dir:
-            # Create a path for the temporary PDF file
-            temp_path = os.path.join(temp_dir, "temp.pdf")
-
-            # Copy the contents of the uploaded file to the temporary file
-            shutil.copy(pdf_file.name, temp_path)
-
-            # Extract text and convert to Markdown
-            markdown_text = extract_text_and_format_from_pdf(temp_path)
-
-            # Extract metadata from PDF
-            metadata = extract_metadata_from_pdf(temp_path)
-
-            # Use metadata for title and author if not provided
-            title = metadata.get('title', os.path.splitext(os.path.basename(pdf_file.name))[0])
-            author = metadata.get('author', 'Unknown')
-
-        result = f"PDF '{title}' by {author} processed successfully."
-        return result, markdown_text
-    except Exception as e:
-        return f"Error ingesting PDF: {str(e)}", ""
-
-def create_pdf_ingestion_test_tab():
-    with gr.TabItem("Test PDF Ingestion", visible=True):
-        with gr.Row():
-            with gr.Column():
-                pdf_file_input = gr.File(label="Upload PDF for testing")
-                test_button = gr.Button("Test PDF Ingestion")
-            with gr.Column():
-                test_output = gr.Textbox(label="Test Result")
-                pdf_content_output = gr.Textbox(label="PDF Content", lines=200)
-        test_button.click(
-            fn=test_pdf_ingestion,
-            inputs=[pdf_file_input],
-            outputs=[test_output, pdf_content_output]
-        )
-

App_Function_Libraries/Gradio_UI/Plaintext_tab_import.py

@@ -1,116 +0,0 @@
-# Plaintext_tab_import.py
-# Contains the code for the "Import Plain Text Files" tab in the Gradio UI.
-# This tab allows users to upload plain text files (Markdown, Text, RTF) or a zip file containing multiple files.
-# The user can provide a title, author, keywords, system prompt, custom user prompt, and select an API for auto-summarization.
-#
-#######################################################################################################################
-#
-# Import necessary libraries
-import os
-import tempfile
-import zipfile
-#
-# Import Non-Local
-import gradio as gr
-from docx2txt import docx2txt
-from pypandoc import convert_file
-#
-# Import Local libraries
-from App_Function_Libraries.Gradio_UI.Import_Functionality import import_data
-#
-#######################################################################################################################
-#
-# Functions:
-
-def create_plain_text_import_tab():
-    with gr.TabItem("Import Plain text & .docx Files", visible=True):
-        with gr.Row():
-            with gr.Column():
-                gr.Markdown("# Import Markdown(`.md`)/Text(`.txt`)/rtf & `.docx` Files")
-                gr.Markdown("Upload a single file or a zip file containing multiple files")
-                import_file = gr.File(label="Upload file for import", file_types=[".md", ".txt", ".rtf", ".docx", ".zip"])
-                title_input = gr.Textbox(label="Title", placeholder="Enter the title of the content (for single files)")
-                author_input = gr.Textbox(label="Author", placeholder="Enter the author's name (for single files)")
-                keywords_input = gr.Textbox(label="Keywords", placeholder="Enter keywords, comma-separated")
-                system_prompt_input = gr.Textbox(label="System Prompt (for Summarization)", lines=3,
-                                                 value="""<s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
-                                                    **Bulleted Note Creation Guidelines**
-
-                                                    **Headings**:
-                                                    - Based on referenced topics, not categories like quotes or terms
-                                                    - Surrounded by **bold** formatting 
-                                                    - Not listed as bullet points
-                                                    - No space between headings and list items underneath
-
-                                                    **Emphasis**:
-                                                    - **Important terms** set in bold font
-                                                    - **Text ending in a colon**: also bolded
-
-                                                    **Review**:
-                                                    - Ensure adherence to specified format
-                                                    - Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]""",
-                                                 )
-                custom_prompt_input = gr.Textbox(label="Custom User Prompt", placeholder="Enter a custom user prompt for summarization (optional)")
-                auto_summarize_checkbox = gr.Checkbox(label="Auto-summarize", value=False)
-                api_name_input = gr.Dropdown(
-                    choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral",
-                             "OpenRouter", "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace"],
-                    label="API for Auto-summarization"
-                )
-                api_key_input = gr.Textbox(label="API Key", type="password")
-                import_button = gr.Button("Import File(s)")
-            with gr.Column():
-                import_output = gr.Textbox(label="Import Status")
-
-
-        def import_plain_text_file(file_path, title, author, keywords, system_prompt, user_prompt, auto_summarize, api_name, api_key):
-            try:
-                # Determine the file type and convert if necessary
-                file_extension = os.path.splitext(file_path)[1].lower()
-                if file_extension == '.rtf':
-                    with tempfile.NamedTemporaryFile(suffix='.md', delete=False) as temp_file:
-                        convert_file(file_path, 'md', outputfile=temp_file.name)
-                        file_path = temp_file.name
-                elif file_extension == '.docx':
-                    content = docx2txt.process(file_path)
-                else:
-                    with open(file_path, 'r', encoding='utf-8') as file:
-                        content = file.read()
-
-                # Process the content
-                return import_data(content, title, author, keywords, system_prompt,
-                                   user_prompt, auto_summarize, api_name, api_key)
-            except Exception as e:
-                return f"Error processing file: {str(e)}"
-
-        def process_plain_text_zip_file(zip_file, title, author, keywords, system_prompt, user_prompt, auto_summarize, api_name, api_key):
-            results = []
-            with tempfile.TemporaryDirectory() as temp_dir:
-                with zipfile.ZipFile(zip_file.name, 'r') as zip_ref:
-                    zip_ref.extractall(temp_dir)
-
-                for filename in os.listdir(temp_dir):
-                    if filename.lower().endswith(('.md', '.txt', '.rtf', '.docx')):
-                        file_path = os.path.join(temp_dir, filename)
-                        result = import_plain_text_file(file_path, title, author, keywords, system_prompt,
-                                                        user_prompt, auto_summarize, api_name, api_key)
-                        results.append(f"File: {filename} - {result}")
-
-            return "\n".join(results)
-
-        def import_file_handler(file, title, author, keywords, system_prompt, user_prompt, auto_summarize, api_name, api_key):
-            if file.name.lower().endswith(('.md', '.txt', '.rtf', '.docx')):
-                return import_plain_text_file(file.name, title, author, keywords, system_prompt, user_prompt, auto_summarize, api_name, api_key)
-            elif file.name.lower().endswith('.zip'):
-                return process_plain_text_zip_file(file, title, author, keywords, system_prompt, user_prompt, auto_summarize, api_name, api_key)
-            else:
-                return "Unsupported file type. Please upload a .md, .txt, .rtf, .docx file or a .zip file containing these file types."
-
-        import_button.click(
-            fn=import_file_handler,
-            inputs=[import_file, title_input, author_input, keywords_input, system_prompt_input,
-                    custom_prompt_input, auto_summarize_checkbox, api_name_input, api_key_input],
-            outputs=import_output
-        )
-
-    return import_file, title_input, author_input, keywords_input, system_prompt_input, custom_prompt_input, auto_summarize_checkbox, api_name_input, api_key_input, import_button, import_output

App_Function_Libraries/Gradio_UI/Podcast_tab.py

@@ -1,163 +0,0 @@
-# Podcast_tab.py
-# Description: Gradio UI for ingesting podcasts into the database
-#
-# Imports
-#
-# External Imports
-import gradio as gr
-#
-# Local Imports
-from App_Function_Libraries.Audio.Audio_Files import process_podcast
-from App_Function_Libraries.DB.DB_Manager import load_preset_prompts
-from App_Function_Libraries.Gradio_UI.Gradio_Shared import whisper_models, update_user_prompt
-#
-########################################################################################################################
-#
-# Functions:
-
-
-def create_podcast_tab():
-    with gr.TabItem("Podcast", visible=True):
-        gr.Markdown("# Podcast Transcription and Ingestion", visible=True)
-        with gr.Row():
-            with gr.Column():
-                podcast_url_input = gr.Textbox(label="Podcast URL", placeholder="Enter the podcast URL here")
-                podcast_title_input = gr.Textbox(label="Podcast Title", placeholder="Will be auto-detected if possible")
-                podcast_author_input = gr.Textbox(label="Podcast Author", placeholder="Will be auto-detected if possible")
-
-                podcast_keywords_input = gr.Textbox(
-                    label="Keywords",
-                    placeholder="Enter keywords here (comma-separated, include series name if applicable)",
-                    value="podcast,audio",
-                    elem_id="podcast-keywords-input"
-                )
-
-                keep_timestamps_input = gr.Checkbox(label="Keep Timestamps", value=True)
-
-                with gr.Row():
-                    podcast_custom_prompt_checkbox = gr.Checkbox(label="Use a Custom Prompt",
-                                                     value=False,
-                                                     visible=True)
-                    preset_prompt_checkbox = gr.Checkbox(label="Use a pre-set Prompt",
-                                                     value=False,
-                                                     visible=True)
-                with gr.Row():
-                    preset_prompt = gr.Dropdown(label="Select Preset Prompt",
-                                                choices=load_preset_prompts(),
-                                                visible=False)
-                with gr.Row():
-                    podcast_custom_prompt_input = gr.Textbox(label="Custom Prompt",
-                                                     placeholder="Enter custom prompt here",
-                                                     lines=3,
-                                                     visible=False)
-                with gr.Row():
-                    system_prompt_input = gr.Textbox(label="System Prompt",
-                                                     value="""<s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
-**Bulleted Note Creation Guidelines**
-
-**Headings**:
-- Based on referenced topics, not categories like quotes or terms
-- Surrounded by **bold** formatting 
-- Not listed as bullet points
-- No space between headings and list items underneath
-
-**Emphasis**:
-- **Important terms** set in bold font
-- **Text ending in a colon**: also bolded
-
-**Review**:
-- Ensure adherence to specified format
-- Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]
-""",
-                                                     lines=3,
-                                                     visible=False)
-
-                podcast_custom_prompt_checkbox.change(
-                    fn=lambda x: (gr.update(visible=x), gr.update(visible=x)),
-                    inputs=[podcast_custom_prompt_checkbox],
-                    outputs=[podcast_custom_prompt_input, system_prompt_input]
-                )
-                preset_prompt_checkbox.change(
-                    fn=lambda x: gr.update(visible=x),
-                    inputs=[preset_prompt_checkbox],
-                    outputs=[preset_prompt]
-                )
-
-                def update_prompts(preset_name):
-                    prompts = update_user_prompt(preset_name)
-                    return (
-                        gr.update(value=prompts["user_prompt"], visible=True),
-                        gr.update(value=prompts["system_prompt"], visible=True)
-                    )
-
-                preset_prompt.change(
-                    update_prompts,
-                    inputs=preset_prompt,
-                    outputs=[podcast_custom_prompt_input, system_prompt_input]
-                )
-
-                podcast_api_name_input = gr.Dropdown(
-                    choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral", "OpenRouter", "Llama.cpp",
-                             "Kobold", "Ooba", "Tabbyapi", "VLLM","ollama", "HuggingFace", "Custom-OpenAI-API"],
-                    value=None,
-                    label="API Name for Summarization (Optional)"
-                )
-                podcast_api_key_input = gr.Textbox(label="API Key (if required)", type="password")
-                podcast_whisper_model_input = gr.Dropdown(choices=whisper_models, value="medium", label="Whisper Model")
-
-                keep_original_input = gr.Checkbox(label="Keep original audio file", value=False)
-                enable_diarization_input = gr.Checkbox(label="Enable speaker diarization", value=False)
-
-                use_cookies_input = gr.Checkbox(label="Use cookies for yt-dlp", value=False)
-                cookies_input = gr.Textbox(
-                    label="yt-dlp Cookies",
-                    placeholder="Paste your cookies here (JSON format)",
-                    lines=3,
-                    visible=False
-                )
-
-                use_cookies_input.change(
-                    fn=lambda x: gr.update(visible=x),
-                    inputs=[use_cookies_input],
-                    outputs=[cookies_input]
-                )
-
-                chunking_options_checkbox = gr.Checkbox(label="Show Chunking Options", value=False)
-                with gr.Row(visible=False) as chunking_options_box:
-                    gr.Markdown("### Chunking Options")
-                    with gr.Column():
-                        chunk_method = gr.Dropdown(choices=['words', 'sentences', 'paragraphs', 'tokens'], label="Chunking Method")
-                        max_chunk_size = gr.Slider(minimum=100, maximum=1000, value=300, step=50, label="Max Chunk Size")
-                        chunk_overlap = gr.Slider(minimum=0, maximum=100, value=0, step=10, label="Chunk Overlap")
-                        use_adaptive_chunking = gr.Checkbox(label="Use Adaptive Chunking")
-                        use_multi_level_chunking = gr.Checkbox(label="Use Multi-level Chunking")
-                        chunk_language = gr.Dropdown(choices=['english', 'french', 'german', 'spanish'], label="Chunking Language")
-
-                chunking_options_checkbox.change(
-                    fn=lambda x: gr.update(visible=x),
-                    inputs=[chunking_options_checkbox],
-                    outputs=[chunking_options_box]
-                )
-
-                podcast_process_button = gr.Button("Process Podcast")
-
-            with gr.Column():
-                podcast_progress_output = gr.Textbox(label="Progress")
-                podcast_error_output = gr.Textbox(label="Error Messages")
-                podcast_transcription_output = gr.Textbox(label="Transcription")
-                podcast_summary_output = gr.Textbox(label="Summary")
-                download_transcription = gr.File(label="Download Transcription as JSON")
-                download_summary = gr.File(label="Download Summary as Text")
-
-        podcast_process_button.click(
-            fn=process_podcast,
-            inputs=[podcast_url_input, podcast_title_input, podcast_author_input,
-                    podcast_keywords_input, podcast_custom_prompt_input, podcast_api_name_input,
-                    podcast_api_key_input, podcast_whisper_model_input, keep_original_input,
-                    enable_diarization_input, use_cookies_input, cookies_input,
-                    chunk_method, max_chunk_size, chunk_overlap, use_adaptive_chunking,
-                    use_multi_level_chunking, chunk_language, keep_timestamps_input],
-            outputs=[podcast_progress_output, podcast_transcription_output, podcast_summary_output,
-                     podcast_title_input, podcast_author_input, podcast_keywords_input, podcast_error_output,
-                     download_transcription, download_summary]
-        )