Upload 13 files

App_Function_Libraries/Gradio_UI/Book_Ingestion_tab.py

@@ -8,76 +8,17 @@
 #
 ####################
 # Imports
-import tempfile
-import os
-import zipfile
 #
 # External Imports
 import gradio as gr
 #
 # Local Imports
-from App_Function_Libraries.Gradio_UI.Import_Functionality import import_data
-from App_Function_Libraries.Books.Book_Ingestion_Lib import epub_to_markdown
+from App_Function_Libraries.Books.Book_Ingestion_Lib import process_zip_file, import_epub, import_file_handler
 #
 ########################################################################################################################
 #
 # Functions:
 
-def import_epub(epub_file, title, author, keywords, system_prompt, user_prompt, auto_summarize, api_name, api_key):
-    try:
-        # Create a temporary directory to store the converted file
-        with tempfile.TemporaryDirectory() as temp_dir:
-            # Handle different types of file objects
-            if isinstance(epub_file, (str, os.PathLike)):
-                epub_path = epub_file
-            elif hasattr(epub_file, 'name'):
-                epub_path = epub_file.name
-            elif hasattr(epub_file, 'path'):
-                epub_path = epub_file.path
-            else:
-                raise ValueError("Unsupported file object type")
-
-            md_path = os.path.join(temp_dir, "converted.md")
-
-            # Convert EPUB to Markdown
-            markdown_content = epub_to_markdown(epub_path)
-
-            # Write the markdown content to a file
-            with open(md_path, "w", encoding="utf-8") as md_file:
-                md_file.write(markdown_content)
-
-            # Read the converted markdown content
-            with open(md_path, "r", encoding="utf-8") as md_file:
-                content = md_file.read()
-
-            # Now process the content as you would with a text file
-            return import_data(content, title, author, keywords, system_prompt,
-                               user_prompt, auto_summarize, api_name, api_key)
-    except Exception as e:
-        return f"Error processing EPUB: {str(e)}"
-
-
-def process_zip_file(zip_file, title, author, keywords, system_prompt, user_prompt, auto_summarize, api_name, api_key):
-    results = []
-    with tempfile.TemporaryDirectory() as temp_dir:
-        if hasattr(zip_file, 'name'):
-            zip_path = zip_file.name
-        elif hasattr(zip_file, 'path'):
-            zip_path = zip_file.path
-        else:
-            raise ValueError("Unsupported zip file object type")
-
-        with zipfile.ZipFile(zip_path, 'r') as zip_ref:
-            zip_ref.extractall(temp_dir)
-
-        for filename in os.listdir(temp_dir):
-            if filename.lower().endswith('.epub'):
-                file_path = os.path.join(temp_dir, filename)
-                result = import_epub(file_path, title, author, keywords, system_prompt,
-                                     user_prompt, auto_summarize, api_name, api_key)
-                results.append(f"File: {filename} - {result}")
-
-    return "\n".join(results)
 
 
 def create_import_book_tab():
@@ -87,7 +28,7 @@ def create_import_book_tab():
                 gr.Markdown("# Import .epub files")
                 gr.Markdown("Upload a single .epub file or a .zip file containing multiple .epub files")
                 gr.Markdown(
-                    "How to remove DRM from your ebooks: https://www.reddit.com/r/Calibre/comments/1ck4w8e/2024_guide_on_removing_drm_from_kobo_kindle_ebooks/")
+                    "🔗 **How to remove DRM from your ebooks:** [Reddit Guide](https://www.reddit.com/r/Calibre/comments/1ck4w8e/2024_guide_on_removing_drm_from_kobo_kindle_ebooks/)")
                 import_file = gr.File(label="Upload file for import", file_types=[".epub", ".zip"])
                 title_input = gr.Textbox(label="Title", placeholder="Enter the title of the content (for single files)")
                 author_input = gr.Textbox(label="Author", placeholder="Enter the author's name (for single files)")
@@ -121,23 +62,34 @@ def create_import_book_tab():
                     label="API for Auto-summarization"
                 )
                 api_key_input = gr.Textbox(label="API Key", type="password")
+
+                # Chunking options
+                max_chunk_size = gr.Slider(minimum=100, maximum=2000, value=500, step=50, label="Max Chunk Size")
+                chunk_overlap = gr.Slider(minimum=0, maximum=500, value=200, step=10, label="Chunk Overlap")
+                custom_chapter_pattern = gr.Textbox(label="Custom Chapter Pattern (optional)",
+                                                    placeholder="Enter a custom regex pattern for chapter detection")
+
+
                 import_button = gr.Button("Import eBook(s)")
             with gr.Column():
                 with gr.Row():
-                    import_output = gr.Textbox(label="Import Status")
-
-        def import_file_handler(file, title, author, keywords, system_prompt, user_prompt, auto_summarize, api_name, api_key):
-            if file.name.lower().endswith('.epub'):
-                return import_epub(file, title, author, keywords, system_prompt, user_prompt, auto_summarize, api_name, api_key)
-            elif file.name.lower().endswith('.zip'):
-                return process_zip_file(file, title, author, keywords, system_prompt, user_prompt, auto_summarize, api_name, api_key)
-            else:
-                return "Unsupported file type. Please upload an .epub file or a .zip file containing .epub files."
+                    import_output = gr.Textbox(label="Import Status", lines=10, interactive=False)
 
         import_button.click(
             fn=import_file_handler,
-            inputs=[import_file, title_input, author_input, keywords_input, system_prompt_input,
-                    custom_prompt_input, auto_summarize_checkbox, api_name_input, api_key_input],
+            inputs=[
+                import_file,
+                title_input,
+                author_input,
+                keywords_input,
+                custom_prompt_input,
+                auto_summarize_checkbox,
+                api_name_input,
+                api_key_input,
+                max_chunk_size,
+                chunk_overlap,
+                custom_chapter_pattern
+            ],
             outputs=import_output
         )
 

App_Function_Libraries/Gradio_UI/Character_Chat_tab.py

@@ -1,8 +1,10 @@
-# Character_Interaction_Library_3.py
+# Character_Interaction_Library.py
 # Description: Library for character card import functions
 #
 # Imports
+import os
 import re
+import tempfile
 import uuid
 from datetime import datetime
 import json
@@ -13,9 +15,13 @@ from typing import Dict, Any, Optional, List, Tuple, Union, cast
 #
 # External Imports
 from PIL import Image
+from PIL.PngImagePlugin import PngInfo
 import gradio as gr
 #
 # Local Imports
+from App_Function_Libraries.Character_Chat.Character_Chat_Lib import validate_character_book, validate_v2_card, \
+    replace_placeholders, replace_user_placeholder, extract_json_from_image, parse_character_book, \
+    load_chat_and_character, load_chat_history, load_character_and_image, extract_character_id, load_character_wrapper
 from App_Function_Libraries.Chat import chat
 from App_Function_Libraries.DB.Character_Chat_DB import (
     add_character_card,
@@ -35,62 +41,9 @@ from App_Function_Libraries.Utils.Utils import sanitize_user_input
 #
 # Functions:
 
-
 #################################################################################
 #
-# Placeholder functions:
-
-def replace_placeholders(text: str, char_name: str, user_name: str) -> str:
-    """
-    Replace placeholders in the given text with appropriate values.
-
-    Args:
-        text (str): The text containing placeholders.
-        char_name (str): The name of the character.
-        user_name (str): The name of the user.
-
-    Returns:
-        str: The text with placeholders replaced.
-    """
-    replacements = {
-        '{{char}}': char_name,
-        '{{user}}': user_name,
-        '{{random_user}}': user_name  # Assuming random_user is the same as user for simplicity
-    }
-
-    for placeholder, value in replacements.items():
-        text = text.replace(placeholder, value)
-
-    return text
-
-def replace_user_placeholder(history, user_name):
-    """
-    Replaces all instances of '{{user}}' in the chat history with the actual user name.
-
-    Args:
-        history (list): The current chat history as a list of tuples (user_message, bot_message).
-        user_name (str): The name entered by the user.
-
-    Returns:
-        list: Updated chat history with placeholders replaced.
-    """
-    if not user_name:
-        user_name = "User"  # Default name if none provided
-
-    updated_history = []
-    for user_msg, bot_msg in history:
-        # Replace in user message
-        if user_msg:
-            user_msg = user_msg.replace("{{user}}", user_name)
-        # Replace in bot message
-        if bot_msg:
-            bot_msg = bot_msg.replace("{{user}}", user_name)
-        updated_history.append((user_msg, bot_msg))
-    return updated_history
-
-#
-# End of Placeholder functions
-#################################################################################
+# Character card import functions:
 
 def import_character_card(file):
     if file is None:
@@ -100,7 +53,7 @@ def import_character_card(file):
         if file.name.lower().endswith(('.png', '.webp')):
             json_data = extract_json_from_image(file)
             if not json_data:
-                return None, gr.update(), "No JSON data found in the image."
+                return None, gr.update(), "No character card data found in the image. This might not be a valid character card image."
         elif file.name.lower().endswith('.json'):
             with open(file.name, 'r', encoding='utf-8') as f:
                 json_data = f.read()
@@ -109,7 +62,7 @@ def import_character_card(file):
 
         card_data = import_character_card_json(json_data)
         if not card_data:
-            return None, gr.update(), "Failed to parse character card JSON."
+            return None, gr.update(), "Failed to parse character card data. The file might not contain valid character information."
 
         # Save image data for PNG/WebP files
         if file.name.lower().endswith(('.png', '.webp')):
@@ -131,6 +84,7 @@ def import_character_card(file):
         logging.error(f"Error importing character card: {e}")
         return None, gr.update(), f"Error importing character card: {e}"
 
+
 def import_character_card_json(json_content: str) -> Optional[Dict[str, Any]]:
     try:
         json_content = json_content.strip()
@@ -148,68 +102,7 @@ def import_character_card_json(json_content: str) -> Optional[Dict[str, Any]]:
         logging.error(f"Unexpected error parsing JSON: {e}")
     return None
 
-def extract_json_from_image(image_file):
-    logging.debug(f"Attempting to extract JSON from image: {image_file.name}")
-    try:
-        with Image.open(image_file) as img:
-            logging.debug("Image opened successfully")
-            metadata = img.info
-            if 'chara' in metadata:
-                logging.debug("Found 'chara' in image metadata")
-                chara_content = metadata['chara']
-                logging.debug(f"Content of 'chara' metadata (first 100 chars): {chara_content[:100]}...")
-                try:
-                    decoded_content = base64.b64decode(chara_content).decode('utf-8')
-                    logging.debug(f"Decoded content (first 100 chars): {decoded_content[:100]}...")
-                    return decoded_content
-                except Exception as e:
-                    logging.error(f"Error decoding base64 content: {e}")
-
-            logging.warning("'chara' not found in metadata, attempting to find JSON data in image bytes")
-            # Alternative method to extract embedded JSON from image bytes if metadata is not available
-            img_byte_arr = io.BytesIO()
-            img.save(img_byte_arr, format='PNG')
-            img_bytes = img_byte_arr.getvalue()
-            img_str = img_bytes.decode('latin1')  # Use 'latin1' to preserve byte values
-
-            # Search for JSON-like structures in the image bytes
-            json_start = img_str.find('{')
-            json_end = img_str.rfind('}')
-            if json_start != -1 and json_end != -1 and json_end > json_start:
-                possible_json = img_str[json_start:json_end+1]
-                try:
-                    json.loads(possible_json)
-                    logging.debug("Found JSON data in image bytes")
-                    return possible_json
-                except json.JSONDecodeError:
-                    logging.debug("No valid JSON found in image bytes")
 
-            logging.warning("No JSON data found in the image")
-    except Exception as e:
-        logging.error(f"Error extracting JSON from image: {e}")
-    return None
-
-
-def process_chat_history(chat_history: List[Tuple[str, str]], char_name: str, user_name: str) -> List[Tuple[str, str]]:
-    """
-    Process the chat history to replace placeholders in both user and character messages.
-
-    Args:
-        chat_history (List[Tuple[str, str]]): The chat history.
-        char_name (str): The name of the character.
-        user_name (str): The name of the user.
-
-    Returns:
-        List[Tuple[str, str]]: The processed chat history.
-    """
-    processed_history = []
-    for user_msg, char_msg in chat_history:
-        if user_msg:
-            user_msg = replace_placeholders(user_msg, char_name, user_name)
-        if char_msg:
-            char_msg = replace_placeholders(char_msg, char_name, user_name)
-        processed_history.append((user_msg, char_msg))
-    return processed_history
 
 def parse_v2_card(card_data: Dict[str, Any]) -> Optional[Dict[str, Any]]:
     try:
@@ -289,166 +182,10 @@ def parse_v1_card(card_data: Dict[str, Any]) -> Dict[str, Any]:
 
     return v2_data
 
-def extract_character_id(choice: str) -> int:
-    """Extract the character ID from the dropdown selection string."""
-    return int(choice.split('(ID: ')[1].rstrip(')'))
-
-def load_character_wrapper(character_id: int, user_name: str) -> Tuple[Dict[str, Any], List[Tuple[Optional[str], str]], Optional[Image.Image]]:
-    """Wrapper function to load character and image using the extracted ID."""
-    char_data, chat_history, img = load_character_and_image(character_id, user_name)
-    return char_data, chat_history, img
-
-def parse_character_book(book_data: Dict[str, Any]) -> Dict[str, Any]:
-    """
-    Parse the character book data from a V2 character card.
-
-    Args:
-        book_data (Dict[str, Any]): The raw character book data from the character card.
-
-    Returns:
-        Dict[str, Any]: The parsed and structured character book data.
-    """
-    parsed_book = {
-        'name': book_data.get('name', ''),
-        'description': book_data.get('description', ''),
-        'scan_depth': book_data.get('scan_depth'),
-        'token_budget': book_data.get('token_budget'),
-        'recursive_scanning': book_data.get('recursive_scanning', False),
-        'extensions': book_data.get('extensions', {}),
-        'entries': []
-    }
-
-    for entry in book_data.get('entries', []):
-        parsed_entry = {
-            'keys': entry['keys'],
-            'content': entry['content'],
-            'extensions': entry.get('extensions', {}),
-            'enabled': entry['enabled'],
-            'insertion_order': entry['insertion_order'],
-            'case_sensitive': entry.get('case_sensitive', False),
-            'name': entry.get('name', ''),
-            'priority': entry.get('priority'),
-            'id': entry.get('id'),
-            'comment': entry.get('comment', ''),
-            'selective': entry.get('selective', False),
-            'secondary_keys': entry.get('secondary_keys', []),
-            'constant': entry.get('constant', False),
-            'position': entry.get('position')
-        }
-        parsed_book['entries'].append(parsed_entry)
-
-    return parsed_book
-
-def load_character_and_image(character_id: int, user_name: str) -> Tuple[Optional[Dict[str, Any]], List[Tuple[Optional[str], str]], Optional[Image.Image]]:
-    """
-    Load a character and its associated image based on the character ID.
-
-    Args:
-        character_id (int): The ID of the character to load.
-        user_name (str): The name of the user, used for placeholder replacement.
-
-    Returns:
-        Tuple[Optional[Dict[str, Any]], List[Tuple[Optional[str], str]], Optional[Image.Image]]:
-        A tuple containing the character data, chat history, and character image (if available).
-    """
-    try:
-        char_data = get_character_card_by_id(character_id)
-        if not char_data:
-            logging.warning(f"No character data found for ID: {character_id}")
-            return None, [], None
-
-        # Replace placeholders in character data
-        for field in ['first_mes', 'mes_example', 'scenario', 'description', 'personality']:
-            if field in char_data:
-                char_data[field] = replace_placeholders(char_data[field], char_data['name'], user_name)
-
-        # Replace placeholders in first_mes
-        first_mes = char_data.get('first_mes', "Hello! I'm ready to chat.")
-        first_mes = replace_placeholders(first_mes, char_data['name'], user_name)
-
-        chat_history = [(None, first_mes)] if first_mes else []
-
-        img = None
-        if char_data.get('image'):
-            try:
-                image_data = base64.b64decode(char_data['image'])
-                img = Image.open(io.BytesIO(image_data)).convert("RGBA")
-            except Exception as e:
-                logging.error(f"Error processing image for character '{char_data['name']}': {e}")
-
-        return char_data, chat_history, img
-
-    except Exception as e:
-        logging.error(f"Error in load_character_and_image: {e}")
-        return None, [], None
-
-def load_chat_and_character(chat_id: int, user_name: str) -> Tuple[Optional[Dict[str, Any]], List[Tuple[str, str]], Optional[Image.Image]]:
-    """
-    Load a chat and its associated character, including the character image and process templates.
-
-    Args:
-        chat_id (int): The ID of the chat to load.
-        user_name (str): The name of the user.
-
-    Returns:
-        Tuple[Optional[Dict[str, Any]], List[Tuple[str, str]], Optional[Image.Image]]:
-        A tuple containing the character data, processed chat history, and character image (if available).
-    """
-    try:
-        # Load the chat
-        chat = get_character_chat_by_id(chat_id)
-        if not chat:
-            logging.warning(f"No chat found with ID: {chat_id}")
-            return None, [], None
-
-        # Load the associated character
-        character_id = chat['character_id']
-        char_data = get_character_card_by_id(character_id)
-        if not char_data:
-            logging.warning(f"No character found for chat ID: {chat_id}")
-            return None, chat['chat_history'], None
-
-        # Process the chat history
-        processed_history = process_chat_history(chat['chat_history'], char_data['name'], user_name)
-
-        # Load the character image
-        img = None
-        if char_data.get('image'):
-            try:
-                image_data = base64.b64decode(char_data['image'])
-                img = Image.open(io.BytesIO(image_data)).convert("RGBA")
-            except Exception as e:
-                logging.error(f"Error processing image for character '{char_data['name']}': {e}")
-
-        # Process character data templates
-        for field in ['first_mes', 'mes_example', 'scenario', 'description', 'personality']:
-            if field in char_data:
-                char_data[field] = replace_placeholders(char_data[field], char_data['name'], user_name)
-
-        return char_data, processed_history, img
-
-    except Exception as e:
-        logging.error(f"Error in load_chat_and_character: {e}")
-        return None, [], None
-
-
-def load_chat_history(file):
-    try:
-        content = file.read().decode('utf-8')
-        chat_data = json.loads(content)
-
-        # Extract history and character name from the loaded data
-        history = chat_data.get('history') or chat_data.get('messages')
-        character_name = chat_data.get('character') or chat_data.get('character_name')
-
-        if not history or not character_name:
-            logging.error("Chat history or character name missing in the imported file.")
-            return None, None
+#
+# End of Character card import functions
+####################################################
 
-        return history, character_name
-    except Exception as e:
-        logging.error(f"Error loading chat history: {e}")
-        return None, None
 
 ####################################################
 #
@@ -507,6 +244,8 @@ def create_character_card_interaction_tab():
                 chat_history = gr.Chatbot(label="Conversation", height=800)
                 user_input = gr.Textbox(label="Your message")
                 send_message_button = gr.Button("Send Message")
+                answer_for_me_button = gr.Button("Answer for Me")
+                continue_talking_button = gr.Button("Continue Talking")
                 regenerate_button = gr.Button("Regenerate Last Message")
                 clear_chat_button = gr.Button("Clear Chat")
                 save_snapshot_button = gr.Button("Save Chat Snapshot")
@@ -893,12 +632,201 @@ def create_character_card_interaction_tab():
             else:
                 return "Failed to update chat."
 
+        def continue_talking(
+                history, char_data, api_endpoint, api_key,
+                temperature, user_name_val, auto_save
+        ):
+            """
+            Causes the character to continue the conversation or think out loud.
+            """
+            if not char_data:
+                return history, "Please select a character first."
+
+            user_name_val = user_name_val or "User"
+            char_name = char_data.get('name', 'AI Assistant')
+
+            # Prepare the character's background information
+            char_background = f"""
+            Name: {char_name}
+            Description: {char_data.get('description', 'N/A')}
+            Personality: {char_data.get('personality', 'N/A')}
+            Scenario: {char_data.get('scenario', 'N/A')}
+            """
+
+            # Prepare the system prompt
+            system_message = f"""You are roleplaying as {char_name}. {char_data.get('system_prompt', '')}
+            If the user does not respond, continue expressing your thoughts or continue the conversation by thinking out loud. If thinking out loud, prefix the message with "Thinking: "."""
+
+            # Prepare chat context
+            media_content = {
+                'id': char_name,
+                'title': char_name,
+                'content': char_background,
+                'description': char_data.get('description', ''),
+                'personality': char_data.get('personality', ''),
+                'scenario': char_data.get('scenario', '')
+            }
+            selected_parts = ['description', 'personality', 'scenario']
+
+            prompt = char_data.get('post_history_instructions', '')
+
+            # Simulate empty user input
+            user_message = ""
+
+            # Generate bot response
+            bot_message = chat(
+                user_message,
+                history,
+                media_content,
+                selected_parts,
+                api_endpoint,
+                api_key,
+                prompt,
+                temperature,
+                system_message
+            )
+
+            # Replace placeholders in bot message
+            bot_message = replace_placeholders(bot_message, char_name, user_name_val)
+
+            # Update history
+            history.append((None, bot_message))
+
+            # Auto-save if enabled
+            save_status = ""
+            if auto_save:
+                character_id = char_data.get('id')
+                if character_id:
+                    conversation_name = f"Auto-saved chat {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
+                    add_character_chat(character_id, conversation_name, history)
+                    save_status = "Chat auto-saved."
+                else:
+                    save_status = "Character ID not found; chat not saved."
+
+            return history, save_status
+
+        def answer_for_me(
+                history, char_data, api_endpoint, api_key,
+                temperature, user_name_val, auto_save
+        ):
+            """
+            Generates a likely user response and continues the conversation.
+            """
+            if not char_data:
+                return history, "Please select a character first."
+
+            user_name_val = user_name_val or "User"
+            char_name = char_data.get('name', 'AI Assistant')
+
+            # Prepare the character's background information
+            char_background = f"""
+            Name: {char_name}
+            Description: {char_data.get('description', 'N/A')}
+            Personality: {char_data.get('personality', 'N/A')}
+            Scenario: {char_data.get('scenario', 'N/A')}
+            """
+
+            # Prepare system message for generating user's response
+            system_message_user = f"""You are simulating the user {user_name_val}. Based on the conversation so far, generate a natural and appropriate response that {user_name_val} might say next. The response should fit the context and flow of the conversation. ONLY SPEAK FOR {user_name_val}."""
+
+            # Prepare chat context
+            media_content = {
+                'id': char_name,
+                'title': char_name,
+                'content': char_background,
+                'description': char_data.get('description', ''),
+                'personality': char_data.get('personality', ''),
+                'scenario': char_data.get('scenario', '')
+            }
+            selected_parts = ['description', 'personality', 'scenario']
+
+            # Generate user response
+            user_response = chat(
+                "",  # No new message
+                history,
+                media_content,
+                selected_parts,
+                api_endpoint,
+                api_key,
+                prompt="",
+                temperature=temperature,
+                system_message=system_message_user
+            )
+
+            # Append the generated user response to history
+            history.append((user_response, None))
+
+            # Now generate the character's response to this user response
+            # Prepare the system message for the character
+            system_message_bot = f"""You are roleplaying as {char_name}. {char_data.get('system_prompt', '')}"""
+
+            bot_message = chat(
+                f"{user_name_val}: {user_response}",
+                history[:-1],
+                media_content,
+                selected_parts,
+                api_endpoint,
+                api_key,
+                prompt=char_data.get('post_history_instructions', ''),
+                temperature=temperature,
+                system_message=system_message_bot
+            )
+
+            # Replace placeholders in bot message
+            bot_message = replace_placeholders(bot_message, char_name, user_name_val)
+
+            # Update history with bot's response
+            history[-1] = (user_response, bot_message)
+
+            # Auto-save if enabled
+            save_status = ""
+            if auto_save:
+                character_id = char_data.get('id')
+                if character_id:
+                    conversation_name = f"Auto-saved chat {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}"
+                    add_character_chat(character_id, conversation_name, history)
+                    save_status = "Chat auto-saved."
+                else:
+                    save_status = "Character ID not found; chat not saved."
+
+            return history, save_status
+
+
         # Define States for conversation_id and media_content, which are required for saving chat history
         conversation_id = gr.State(str(uuid.uuid4()))
         media_content = gr.State({})
 
         # Button Callbacks
 
+        # Add the new button callbacks here
+        answer_for_me_button.click(
+            fn=answer_for_me,
+            inputs=[
+                chat_history,
+                character_data,
+                api_name_input,
+                api_key_input,
+                temperature_slider,
+                user_name_input,
+                auto_save_checkbox
+            ],
+            outputs=[chat_history, save_status]
+        )
+
+        continue_talking_button.click(
+            fn=continue_talking,
+            inputs=[
+                chat_history,
+                character_data,
+                api_name_input,
+                api_key_input,
+                temperature_slider,
+                user_name_input,
+                auto_save_checkbox
+            ],
+            outputs=[chat_history, save_status]
+        )
+
         import_card_button.click(
             fn=import_character_card,
             inputs=[character_card_upload],
@@ -1035,28 +963,40 @@ def create_character_card_interaction_tab():
 
 
 def create_character_chat_mgmt_tab():
-    with gr.TabItem("Chat Management"):
-        gr.Markdown("# Chat Management")
+    with gr.TabItem("Character and Chat Management"):
+        gr.Markdown("# Character and Chat Management")
 
         with gr.Row():
-            # Search Section
+            # Left Column: Character Import and Chat Management
             with gr.Column(scale=1):
-                gr.Markdown("## Search Conversations or Characters")
-                search_query = gr.Textbox(label="Search Conversations or Characters", placeholder="Enter search keywords")
-                search_button = gr.Button("Search")
-                search_results = gr.Dropdown(label="Search Results", choices=[], visible=False)
-                search_status = gr.Markdown("", visible=True)
+                gr.Markdown("## Import Characters")
+                character_files = gr.File(
+                    label="Upload Character Files (PNG, WEBP, JSON)",
+                    file_types=[".png", ".webp", ".json"],
+                    file_count="multiple"
+                )
+                import_characters_button = gr.Button("Import Characters")
+                import_status = gr.Markdown("")
 
-            # Select Character and Chat Section
-            with gr.Column(scale=1):
-                gr.Markdown("## Select Character and Associated Chats")
+            # Right Column: Character Selection and Image Display
+            with gr.Column(scale=2):
+                gr.Markdown("## Select Character")
                 characters = get_character_cards()
                 character_choices = [f"{char['name']} (ID: {char['id']})" for char in characters]
+                load_characters_button = gr.Button("Load Existing Characters")
                 select_character = gr.Dropdown(label="Select Character", choices=character_choices, interactive=True)
-                select_chat = gr.Dropdown(label="Select Chat", choices=[], visible=False, interactive=True)
-                load_chat_button = gr.Button("Load Selected Chat", visible=False)
+                character_image = gr.Image(label="Character Image", type="pil", interactive=False)
+
+                gr.Markdown("## Search Conversations")
+                search_query = gr.Textbox(label="Search Conversations", placeholder="Enter search keywords")
+                search_button = gr.Button("Search")
+                search_results = gr.Dropdown(label="Search Results", choices=[], visible=False)
+                search_status = gr.Markdown("", visible=True)
 
         with gr.Row():
+            gr.Markdown("## Chat Management")
+            select_chat = gr.Dropdown(label="Select Chat", choices=[], visible=False, interactive=True)
+            load_chat_button = gr.Button("Load Selected Chat", visible=False)
             conversation_list = gr.Dropdown(label="Select Conversation or Character", choices=[])
             conversation_mapping = gr.State({})
 
@@ -1072,36 +1012,58 @@ def create_character_chat_mgmt_tab():
 
         # Callback Functions
 
-        def search_conversations_or_characters(query):
+        def load_character_image(character_selection):
+            if not character_selection:
+                return None
+
+            try:
+                character_id = int(character_selection.split('(ID: ')[1].rstrip(')'))
+                character = get_character_card_by_id(character_id)
+                if character and 'image' in character:
+                    image_data = base64.b64decode(character['image'])
+                    img = Image.open(io.BytesIO(image_data))
+                    return img
+            except Exception as e:
+                logging.error(f"Error loading character image: {e}")
+
+            return None
+
+        def search_conversations_or_characters(query, selected_character):
             if not query.strip():
                 return gr.update(choices=[], visible=False), "Please enter a search query."
 
             try:
-                # Search Chats using FTS5
-                chat_results, chat_message = search_character_chats(query)
+                # Extract character ID from the selected character
+                character_id = None
+                if selected_character:
+                    character_id = int(selected_character.split('(ID: ')[1].rstrip(')'))
+
+                # Search Chats using FTS5, filtered by character_id if provided
+                chat_results, chat_message = search_character_chats(query, character_id)
 
                 # Format chat results
                 formatted_chat_results = [
                     f"Chat: {chat['conversation_name']} (ID: {chat['id']})" for chat in chat_results
                 ]
 
-                # Search Characters using substring match
-                characters = get_character_cards()
-                filtered_characters = [
-                    char for char in characters
-                    if query.lower() in char['name'].lower()
-                ]
-                formatted_character_results = [
-                    f"Character: {char['name']} (ID: {char['id']})" for char in filtered_characters
-                ]
+                # If no character is selected, also search for characters
+                if not character_id:
+                    characters = get_character_cards()
+                    filtered_characters = [
+                        char for char in characters
+                        if query.lower() in char['name'].lower()
+                    ]
+                    formatted_character_results = [
+                        f"Character: {char['name']} (ID: {char['id']})" for char in filtered_characters
+                    ]
+                else:
+                    formatted_character_results = []
 
                 # Combine results
                 all_choices = formatted_chat_results + formatted_character_results
-                mapping = {choice: conv['id'] for choice, conv in zip(formatted_chat_results, chat_results)}
-                mapping.update({choice: char['id'] for choice, char in zip(formatted_character_results, filtered_characters)})
 
                 if all_choices:
-                    return gr.update(choices=all_choices, visible=True), f"Found {len(all_choices)} result(s) matching '{query}'."
+                    return gr.update(choices=all_choices, visible=True), chat_message
                 else:
                     return gr.update(choices=[], visible=False), f"No results found for '{query}'."
 
@@ -1265,11 +1227,39 @@ def create_character_chat_mgmt_tab():
                 <p><strong>Version:</strong> {character.get('character_version', 'N/A')}</p>
             </div>
             """
+        def import_multiple_characters(files):
+            if not files:
+                return "No files provided for character import."
+
+            results = []
+            for file in files:
+                result, _, message = import_character_card(file)
+                if result:
+                    results.append(f"Imported: {result['name']}")
+                else:
+                    results.append(f"Failed: {file.name} - {message}")
+
+            # Refresh character choices
+            characters = get_character_cards()
+            character_choices = [f"{char['name']} (ID: {char['id']})" for char in characters]
+            select_character.choices = character_choices
+
+            return "Import results:\n" + "\n".join(results)
+
+        # Register new callback for character import
+        import_characters_button.click(
+            fn=import_multiple_characters,
+            inputs=[character_files],
+            outputs=[import_status]
+        ).then(
+            fn=lambda: gr.update(choices=[f"{char['name']} (ID: {char['id']})" for char in get_character_cards()]),
+            outputs=select_character
+        )
 
         # Register Callback Functions with Gradio Components
         search_button.click(
             fn=search_conversations_or_characters,
-            inputs=[search_query],
+            inputs=[search_query, select_character],
             outputs=[search_results, search_status]
         )
 
@@ -1292,6 +1282,10 @@ def create_character_chat_mgmt_tab():
         )
 
         select_character.change(
+            fn=load_character_image,
+            inputs=[select_character],
+            outputs=[character_image]
+        ).then(
             fn=populate_chats,
             inputs=[select_character],
             outputs=[select_chat, search_status]
@@ -1309,10 +1303,875 @@ def create_character_chat_mgmt_tab():
             outputs=[chat_content, chat_preview]
         )
 
+        load_characters_button.click(
+            fn=lambda: gr.update(choices=[f"{char['name']} (ID: {char['id']})" for char in get_character_cards()]),
+            outputs=select_character
+        )
+
         return (
+            character_files, import_characters_button, import_status,
             search_query, search_button, search_results, search_status,
             select_character, select_chat, load_chat_button,
             conversation_list, conversation_mapping,
             chat_content, save_button, delete_button,
-            chat_preview, result_message
+            chat_preview, result_message, character_image
         )
+
+def create_custom_character_card_tab():
+    with gr.TabItem("Create a New Character Card"):
+        gr.Markdown("# Create a New Character Card (v2)")
+
+        with gr.Row():
+            with gr.Column():
+                # Input fields for character card data
+                name_input = gr.Textbox(label="Name", placeholder="Enter character name")
+                description_input = gr.TextArea(label="Description", placeholder="Enter character description")
+                personality_input = gr.TextArea(label="Personality", placeholder="Enter character personality")
+                scenario_input = gr.TextArea(label="Scenario", placeholder="Enter character scenario")
+                first_mes_input = gr.TextArea(label="First Message", placeholder="Enter the first message")
+                mes_example_input = gr.TextArea(label="Example Messages", placeholder="Enter example messages")
+                creator_notes_input = gr.TextArea(label="Creator Notes", placeholder="Enter notes for the creator")
+                system_prompt_input = gr.TextArea(label="System Prompt", placeholder="Enter system prompt")
+                post_history_instructions_input = gr.TextArea(label="Post History Instructions", placeholder="Enter post history instructions")
+                alternate_greetings_input = gr.TextArea(
+                    label="Alternate Greetings (one per line)",
+                    placeholder="Enter alternate greetings, one per line"
+                )
+                tags_input = gr.Textbox(label="Tags", placeholder="Enter tags, separated by commas")
+                creator_input = gr.Textbox(label="Creator", placeholder="Enter creator name")
+                character_version_input = gr.Textbox(label="Character Version", placeholder="Enter character version")
+                extensions_input = gr.TextArea(
+                    label="Extensions (JSON)",
+                    placeholder="Enter extensions as JSON (optional)"
+                )
+                image_input = gr.Image(label="Character Image", type="pil")
+
+                # Buttons
+                save_button = gr.Button("Save Character Card")
+                download_button = gr.Button("Download Character Card")
+                download_image_button = gr.Button("Download Character Card as Image")
+
+                # Output status and outputs
+                save_status = gr.Markdown("")
+                download_output = gr.File(label="Download Character Card", interactive=False)
+                download_image_output = gr.File(label="Download Character Card as Image", interactive=False)
+
+        # Import PngInfo
+        from PIL.PngImagePlugin import PngInfo
+
+        # Callback Functions
+        def build_character_card(
+            name, description, personality, scenario, first_mes, mes_example,
+            creator_notes, system_prompt, post_history_instructions,
+            alternate_greetings_str, tags_str, creator, character_version,
+            extensions_str
+        ):
+            # Parse alternate_greetings from multiline string
+            alternate_greetings = [line.strip() for line in alternate_greetings_str.strip().split('\n') if line.strip()]
+
+            # Parse tags from comma-separated string
+            tags = [tag.strip() for tag in tags_str.strip().split(',') if tag.strip()]
+
+            # Parse extensions from JSON string
+            try:
+                extensions = json.loads(extensions_str) if extensions_str.strip() else {}
+            except json.JSONDecodeError as e:
+                extensions = {}
+                logging.error(f"Error parsing extensions JSON: {e}")
+
+            # Build the character card dictionary according to V2 spec
+            character_card = {
+                'spec': 'chara_card_v2',
+                'spec_version': '2.0',
+                'data': {
+                    'name': name,
+                    'description': description,
+                    'personality': personality,
+                    'scenario': scenario,
+                    'first_mes': first_mes,
+                    'mes_example': mes_example,
+                    'creator_notes': creator_notes,
+                    'system_prompt': system_prompt,
+                    'post_history_instructions': post_history_instructions,
+                    'alternate_greetings': alternate_greetings,
+                    'tags': tags,
+                    'creator': creator,
+                    'character_version': character_version,
+                    'extensions': extensions,
+                }
+            }
+            return character_card
+
+        def validate_character_card_data(character_card):
+            """
+            Validates the character card data using the extended validation logic.
+            """
+            is_valid, validation_messages = validate_v2_card(character_card)
+            return is_valid, validation_messages
+
+        def save_character_card(
+            name, description, personality, scenario, first_mes, mes_example,
+            creator_notes, system_prompt, post_history_instructions,
+            alternate_greetings_str, tags_str, creator, character_version,
+            extensions_str, image
+        ):
+            # Build the character card
+            character_card = build_character_card(
+                name, description, personality, scenario, first_mes, mes_example,
+                creator_notes, system_prompt, post_history_instructions,
+                alternate_greetings_str, tags_str, creator, character_version,
+                extensions_str
+            )
+
+            # Validate the character card
+            is_valid, validation_messages = validate_character_card_data(character_card)
+            if not is_valid:
+                # Return validation errors
+                validation_output = "Character card validation failed:\n"
+                validation_output += "\n".join(validation_messages)
+                return validation_output
+
+            # If image is provided, encode it to base64
+            if image:
+                img_byte_arr = io.BytesIO()
+                image.save(img_byte_arr, format='PNG')
+                character_card['data']['image'] = base64.b64encode(img_byte_arr.getvalue()).decode('utf-8')
+
+            # Save character card to database
+            character_id = add_character_card(character_card['data'])
+            if character_id:
+                return f"Character card '{name}' saved successfully."
+            else:
+                return f"Failed to save character card '{name}'. It may already exist."
+
+        def download_character_card(
+            name, description, personality, scenario, first_mes, mes_example,
+            creator_notes, system_prompt, post_history_instructions,
+            alternate_greetings_str, tags_str, creator, character_version,
+            extensions_str, image
+        ):
+            # Build the character card
+            character_card = build_character_card(
+                name, description, personality, scenario, first_mes, mes_example,
+                creator_notes, system_prompt, post_history_instructions,
+                alternate_greetings_str, tags_str, creator, character_version,
+                extensions_str
+            )
+
+            # Validate the character card
+            is_valid, validation_messages = validate_character_card_data(character_card)
+            if not is_valid:
+                # Return validation errors
+                validation_output = "Character card validation failed:\n"
+                validation_output += "\n".join(validation_messages)
+                return gr.update(value=None), validation_output  # Return None for the file output
+
+            # If image is provided, include it as base64
+            if image:
+                img_byte_arr = io.BytesIO()
+                image.save(img_byte_arr, format='PNG')
+                character_card['data']['image'] = base64.b64encode(img_byte_arr.getvalue()).decode('utf-8')
+
+            # Convert to JSON string
+            json_str = json.dumps(character_card, indent=2)
+
+            # Write the JSON to a temporary file
+            with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.json', encoding='utf-8') as temp_file:
+                temp_file.write(json_str)
+                temp_file_path = temp_file.name
+
+            # Return the file path and clear validation output
+            return temp_file_path, ""
+
+        def download_character_card_as_image(
+            name, description, personality, scenario, first_mes, mes_example,
+            creator_notes, system_prompt, post_history_instructions,
+            alternate_greetings_str, tags_str, creator, character_version,
+            extensions_str, image
+        ):
+            # Build the character card
+            character_card = build_character_card(
+                name, description, personality, scenario, first_mes, mes_example,
+                creator_notes, system_prompt, post_history_instructions,
+                alternate_greetings_str, tags_str, creator, character_version,
+                extensions_str
+            )
+
+            # Validate the character card
+            is_valid, validation_messages = validate_character_card_data(character_card)
+            if not is_valid:
+                # Return validation errors
+                validation_output = "Character card validation failed:\n"
+                validation_output += "\n".join(validation_messages)
+                return gr.update(value=None), validation_output  # Return None for the file output
+
+            # Convert the character card JSON to a string
+            json_str = json.dumps(character_card, indent=2)
+
+            # Encode the JSON string to base64
+            chara_content = base64.b64encode(json_str.encode('utf-8')).decode('utf-8')
+
+            # Create PNGInfo object to hold metadata
+            png_info = PngInfo()
+            png_info.add_text('chara', chara_content)
+
+            # If image is provided, use it; otherwise, create a blank image
+            if image:
+                img = image.copy()
+            else:
+                # Create a default blank image
+                img = Image.new('RGB', (512, 512), color='white')
+
+            # Save the image to a temporary file with metadata
+            with tempfile.NamedTemporaryFile(mode='wb', delete=False, suffix='.png') as temp_file:
+                img.save(temp_file, format='PNG', pnginfo=png_info)
+                temp_file_path = temp_file.name
+
+            # Return the file path and clear validation output
+            return temp_file_path, ""
+
+        # Include the validate_v2_card function here (from previous code)
+
+        # Button Callbacks
+        save_button.click(
+            fn=save_character_card,
+            inputs=[
+                name_input, description_input, personality_input, scenario_input,
+                first_mes_input, mes_example_input, creator_notes_input, system_prompt_input,
+                post_history_instructions_input, alternate_greetings_input, tags_input,
+                creator_input, character_version_input, extensions_input, image_input
+            ],
+            outputs=[save_status]
+        )
+
+        download_button.click(
+            fn=download_character_card,
+            inputs=[
+                name_input, description_input, personality_input, scenario_input,
+                first_mes_input, mes_example_input, creator_notes_input, system_prompt_input,
+                post_history_instructions_input, alternate_greetings_input, tags_input,
+                creator_input, character_version_input, extensions_input, image_input
+            ],
+            outputs=[download_output, save_status]
+        )
+
+        download_image_button.click(
+            fn=download_character_card_as_image,
+            inputs=[
+                name_input, description_input, personality_input, scenario_input,
+                first_mes_input, mes_example_input, creator_notes_input, system_prompt_input,
+                post_history_instructions_input, alternate_greetings_input, tags_input,
+                creator_input, character_version_input, extensions_input, image_input
+            ],
+            outputs=[download_image_output, save_status]
+        )
+
+#v1
+def create_character_card_validation_tab():
+    with gr.TabItem("Validate Character Card"):
+        gr.Markdown("# Validate Character Card (v2)")
+        gr.Markdown("Upload a character card (PNG, WEBP, or JSON) to validate whether it conforms to the Character Card V2 specification.")
+
+        with gr.Row():
+            with gr.Column():
+                # File uploader
+                file_upload = gr.File(
+                    label="Upload Character Card (PNG, WEBP, JSON)",
+                    file_types=[".png", ".webp", ".json"]
+                )
+                # Validation button
+                validate_button = gr.Button("Validate Character Card")
+                # Output area for validation results
+                validation_output = gr.Markdown("")
+
+        # Callback Functions
+        def validate_character_card(file):
+            if file is None:
+                return "No file provided for validation."
+
+            try:
+                if file.name.lower().endswith(('.png', '.webp')):
+                    json_data = extract_json_from_image(file)
+                    if not json_data:
+                        return "Failed to extract JSON data from the image. The image might not contain embedded character card data."
+                elif file.name.lower().endswith('.json'):
+                    with open(file.name, 'r', encoding='utf-8') as f:
+                        json_data = f.read()
+                else:
+                    return "Unsupported file type. Please upload a PNG, WEBP, or JSON file."
+
+                # Parse the JSON content
+                try:
+                    card_data = json.loads(json_data)
+                except json.JSONDecodeError as e:
+                    return f"JSON decoding error: {e}"
+
+                # Validate the character card
+                is_valid, validation_messages = validate_v2_card(card_data)
+
+                # Prepare the validation output
+                if is_valid:
+                    return "Character card is valid according to the V2 specification."
+                else:
+                    # Concatenate all validation error messages
+                    validation_output = "Character card validation failed:\n"
+                    validation_output += "\n".join(validation_messages)
+                    return validation_output
+
+            except Exception as e:
+                logging.error(f"Error validating character card: {e}")
+                return f"An unexpected error occurred during validation: {e}"
+
+        def validate_v2_card(card_data):
+            """
+            Validate a character card according to the V2 specification.
+
+            Args:
+                card_data (dict): The parsed character card data.
+
+            Returns:
+                Tuple[bool, List[str]]: A tuple containing a boolean indicating validity and a list of validation messages.
+            """
+            validation_messages = []
+
+            # Check top-level fields
+            if 'spec' not in card_data:
+                validation_messages.append("Missing 'spec' field.")
+            elif card_data['spec'] != 'chara_card_v2':
+                validation_messages.append(f"Invalid 'spec' value: {card_data['spec']}. Expected 'chara_card_v2'.")
+
+            if 'spec_version' not in card_data:
+                validation_messages.append("Missing 'spec_version' field.")
+            else:
+                # Ensure 'spec_version' is '2.0' or higher
+                try:
+                    spec_version = float(card_data['spec_version'])
+                    if spec_version < 2.0:
+                        validation_messages.append(f"'spec_version' must be '2.0' or higher. Found '{card_data['spec_version']}'.")
+                except ValueError:
+                    validation_messages.append(f"Invalid 'spec_version' format: {card_data['spec_version']}. Must be a number as a string.")
+
+            if 'data' not in card_data:
+                validation_messages.append("Missing 'data' field.")
+                return False, validation_messages  # Cannot proceed without 'data' field
+
+            data = card_data['data']
+
+            # Required fields in 'data'
+            required_fields = ['name', 'description', 'personality', 'scenario', 'first_mes', 'mes_example']
+            for field in required_fields:
+                if field not in data:
+                    validation_messages.append(f"Missing required field in 'data': '{field}'.")
+                elif not isinstance(data[field], str):
+                    validation_messages.append(f"Field '{field}' must be a string.")
+                elif not data[field].strip():
+                    validation_messages.append(f"Field '{field}' cannot be empty.")
+
+            # Optional fields with expected types
+            optional_fields = {
+                'creator_notes': str,
+                'system_prompt': str,
+                'post_history_instructions': str,
+                'alternate_greetings': list,
+                'tags': list,
+                'creator': str,
+                'character_version': str,
+                'extensions': dict,
+                'character_book': dict  # If present, should be a dict
+            }
+
+            for field, expected_type in optional_fields.items():
+                if field in data:
+                    if not isinstance(data[field], expected_type):
+                        validation_messages.append(f"Field '{field}' must be of type '{expected_type.__name__}'.")
+                    elif field == 'extensions':
+                        # Validate that extensions keys are properly namespaced
+                        for key in data[field].keys():
+                            if '/' not in key and '_' not in key:
+                                validation_messages.append(f"Extension key '{key}' in 'extensions' should be namespaced to prevent conflicts.")
+
+            # If 'alternate_greetings' is present, check that it's a list of non-empty strings
+            if 'alternate_greetings' in data and isinstance(data['alternate_greetings'], list):
+                for idx, greeting in enumerate(data['alternate_greetings']):
+                    if not isinstance(greeting, str) or not greeting.strip():
+                        validation_messages.append(f"Element {idx} in 'alternate_greetings' must be a non-empty string.")
+
+            # If 'tags' is present, check that it's a list of non-empty strings
+            if 'tags' in data and isinstance(data['tags'], list):
+                for idx, tag in enumerate(data['tags']):
+                    if not isinstance(tag, str) or not tag.strip():
+                        validation_messages.append(f"Element {idx} in 'tags' must be a non-empty string.")
+
+            # Validate 'extensions' field
+            if 'extensions' in data and not isinstance(data['extensions'], dict):
+                validation_messages.append("Field 'extensions' must be a dictionary.")
+
+            # Validate 'character_book' if present
+            if 'character_book' in data:
+                is_valid_book, book_messages = validate_character_book(data['character_book'])
+                if not is_valid_book:
+                    validation_messages.extend(book_messages)
+
+            is_valid = len(validation_messages) == 0
+            return is_valid, validation_messages
+
+        # Button Callback
+        validate_button.click(
+            fn=validate_character_card,
+            inputs=[file_upload],
+            outputs=[validation_output]
+        )
+# v2-not-working-on-export-def create_character_card_validation_tab():
+#     with gr.TabItem("Validate and Edit Character Card"):
+#         gr.Markdown("# Validate and Edit Character Card (v2)")
+#         gr.Markdown("Upload a character card (PNG, WEBP, or JSON) to validate and modify it.")
+#
+#         with gr.Row():
+#             with gr.Column():
+#                 # File uploader
+#                 file_upload = gr.File(
+#                     label="Upload Character Card (PNG, WEBP, JSON)",
+#                     file_types=[".png", ".webp", ".json"]
+#                 )
+#                 # Validation button
+#                 validate_button = gr.Button("Validate and Load Character Card")
+#                 # Output area for validation results
+#                 validation_output = gr.Markdown("")
+#
+#         # Input fields for character card data (duplicated from the create tab)
+#         with gr.Row():
+#             with gr.Column():
+#                 name_input = gr.Textbox(label="Name", placeholder="Enter character name")
+#                 description_input = gr.TextArea(label="Description", placeholder="Enter character description")
+#                 personality_input = gr.TextArea(label="Personality", placeholder="Enter character personality")
+#                 scenario_input = gr.TextArea(label="Scenario", placeholder="Enter character scenario")
+#                 first_mes_input = gr.TextArea(label="First Message", placeholder="Enter the first message")
+#                 mes_example_input = gr.TextArea(label="Example Messages", placeholder="Enter example messages")
+#                 creator_notes_input = gr.TextArea(label="Creator Notes", placeholder="Enter notes for the creator")
+#                 system_prompt_input = gr.TextArea(label="System Prompt", placeholder="Enter system prompt")
+#                 post_history_instructions_input = gr.TextArea(label="Post History Instructions", placeholder="Enter post history instructions")
+#                 alternate_greetings_input = gr.TextArea(
+#                     label="Alternate Greetings (one per line)",
+#                     placeholder="Enter alternate greetings, one per line"
+#                 )
+#                 tags_input = gr.Textbox(label="Tags", placeholder="Enter tags, separated by commas")
+#                 creator_input = gr.Textbox(label="Creator", placeholder="Enter creator name")
+#                 character_version_input = gr.Textbox(label="Character Version", placeholder="Enter character version")
+#                 extensions_input = gr.TextArea(
+#                     label="Extensions (JSON)",
+#                     placeholder="Enter extensions as JSON (optional)"
+#                 )
+#                 image_input = gr.Image(label="Character Image", type="pil")
+#
+#                 # Buttons
+#                 save_button = gr.Button("Save Character Card")
+#                 download_button = gr.Button("Download Character Card")
+#                 download_image_button = gr.Button("Download Character Card as Image")
+#
+#                 # Output status and outputs
+#                 save_status = gr.Markdown("")
+#                 download_output = gr.File(label="Download Character Card", interactive=False)
+#                 download_image_output = gr.File(label="Download Character Card as Image", interactive=False)
+#
+#         # Callback Functions
+#         def extract_json_from_image(file):
+#             try:
+#                 image = Image.open(file.name)
+#                 if "chara" in image.info:
+#                     json_data = image.info["chara"]
+#                     # Decode base64 if necessary
+#                     try:
+#                         json_data = base64.b64decode(json_data).decode('utf-8')
+#                     except Exception:
+#                         pass  # Assume it's already in plain text
+#                     return json_data
+#                 else:
+#                     return None
+#             except Exception as e:
+#                 logging.error(f"Error extracting JSON from image: {e}")
+#                 return None
+#
+#         def validate_v2_card(card_data):
+#             """
+#             Validate a character card according to the V2 specification.
+#
+#             Args:
+#                 card_data (dict): The parsed character card data.
+#
+#             Returns:
+#                 Tuple[bool, List[str]]: A tuple containing a boolean indicating validity and a list of validation messages.
+#             """
+#             validation_messages = []
+#
+#             # Check top-level fields
+#             if 'spec' not in card_data:
+#                 validation_messages.append("Missing 'spec' field.")
+#             elif card_data['spec'] != 'chara_card_v2':
+#                 validation_messages.append(f"Invalid 'spec' value: {card_data['spec']}. Expected 'chara_card_v2'.")
+#
+#             if 'spec_version' not in card_data:
+#                 validation_messages.append("Missing 'spec_version' field.")
+#             else:
+#                 # Ensure 'spec_version' is '2.0' or higher
+#                 try:
+#                     spec_version = float(card_data['spec_version'])
+#                     if spec_version < 2.0:
+#                         validation_messages.append(
+#                             f"'spec_version' must be '2.0' or higher. Found '{card_data['spec_version']}'.")
+#                 except ValueError:
+#                     validation_messages.append(
+#                         f"Invalid 'spec_version' format: {card_data['spec_version']}. Must be a number as a string.")
+#
+#             if 'data' not in card_data:
+#                 validation_messages.append("Missing 'data' field.")
+#                 return False, validation_messages  # Cannot proceed without 'data' field
+#
+#             data = card_data['data']
+#
+#             # Required fields in 'data'
+#             required_fields = ['name', 'description', 'personality', 'scenario', 'first_mes', 'mes_example']
+#             for field in required_fields:
+#                 if field not in data:
+#                     validation_messages.append(f"Missing required field in 'data': '{field}'.")
+#                 elif not isinstance(data[field], str):
+#                     validation_messages.append(f"Field '{field}' must be a string.")
+#                 elif not data[field].strip():
+#                     validation_messages.append(f"Field '{field}' cannot be empty.")
+#
+#             # Optional fields with expected types
+#             optional_fields = {
+#                 'creator_notes': str,
+#                 'system_prompt': str,
+#                 'post_history_instructions': str,
+#                 'alternate_greetings': list,
+#                 'tags': list,
+#                 'creator': str,
+#                 'character_version': str,
+#                 'extensions': dict,
+#                 'character_book': dict  # If present, should be a dict
+#             }
+#
+#             for field, expected_type in optional_fields.items():
+#                 if field in data:
+#                     if not isinstance(data[field], expected_type):
+#                         validation_messages.append(f"Field '{field}' must be of type '{expected_type.__name__}'.")
+#                     elif field == 'extensions':
+#                         # Validate that extensions keys are properly namespaced
+#                         for key in data[field].keys():
+#                             if '/' not in key and '_' not in key:
+#                                 validation_messages.append(
+#                                     f"Extension key '{key}' in 'extensions' should be namespaced to prevent conflicts.")
+#
+#             # If 'alternate_greetings' is present, check that it's a list of non-empty strings
+#             if 'alternate_greetings' in data and isinstance(data['alternate_greetings'], list):
+#                 for idx, greeting in enumerate(data['alternate_greetings']):
+#                     if not isinstance(greeting, str) or not greeting.strip():
+#                         validation_messages.append(
+#                             f"Element {idx} in 'alternate_greetings' must be a non-empty string.")
+#
+#             # If 'tags' is present, check that it's a list of non-empty strings
+#             if 'tags' in data and isinstance(data['tags'], list):
+#                 for idx, tag in enumerate(data['tags']):
+#                     if not isinstance(tag, str) or not tag.strip():
+#                         validation_messages.append(f"Element {idx} in 'tags' must be a non-empty string.")
+#
+#             # Validate 'extensions' field
+#             if 'extensions' in data and not isinstance(data['extensions'], dict):
+#                 validation_messages.append("Field 'extensions' must be a dictionary.")
+#
+#             # Validate 'character_book' if present
+#             # (Assuming you have a validate_character_book function)
+#             # if 'character_book' in data:
+#             #     is_valid_book, book_messages = validate_character_book(data['character_book'])
+#             #     if not is_valid_book:
+#             #         validation_messages.extend(book_messages)
+#
+#             is_valid = len(validation_messages) == 0
+#             return is_valid, validation_messages
+#
+#         # Include the save_character_card, download_character_card, and download_character_card_as_image functions
+#         def save_character_card(
+#                 name, description, personality, scenario, first_mes, mes_example,
+#                 creator_notes, system_prompt, post_history_instructions,
+#                 alternate_greetings_str, tags_str, creator, character_version,
+#                 extensions_str, image
+#         ):
+#             # Build the character card
+#             character_card = build_character_card(
+#                 name, description, personality, scenario, first_mes, mes_example,
+#                 creator_notes, system_prompt, post_history_instructions,
+#                 alternate_greetings_str, tags_str, creator, character_version,
+#                 extensions_str
+#             )
+#
+#             # Validate the character card
+#             is_valid, validation_messages = validate_v2_card(character_card)
+#             if not is_valid:
+#                 # Return validation errors
+#                 validation_output = "Character card validation failed:\n"
+#                 validation_output += "\n".join(validation_messages)
+#                 return validation_output
+#
+#             # If image is provided, encode it to base64
+#             if image:
+#                 img_byte_arr = io.BytesIO()
+#                 image.save(img_byte_arr, format='PNG')
+#                 character_card['data']['image'] = base64.b64encode(img_byte_arr.getvalue()).decode('utf-8')
+#
+#             # Save character card to database
+#             character_id = add_character_card(character_card['data'])
+#             if character_id:
+#                 return f"Character card '{name}' saved successfully."
+#             else:
+#                 return f"Failed to save character card '{name}'. It may already exist."
+#
+#         def download_character_card(
+#                 name, description, personality, scenario, first_mes, mes_example,
+#                 creator_notes, system_prompt, post_history_instructions,
+#                 alternate_greetings_str, tags_str, creator, character_version,
+#                 extensions_str, image
+#         ):
+#             # Build the character card
+#             character_card = build_character_card(
+#                 name, description, personality, scenario, first_mes, mes_example,
+#                 creator_notes, system_prompt, post_history_instructions,
+#                 alternate_greetings_str, tags_str, creator, character_version,
+#                 extensions_str
+#             )
+#
+#             # Validate the character card
+#             is_valid, validation_messages = validate_v2_card(character_card)
+#             if not is_valid:
+#                 # Return validation errors
+#                 validation_output = "Character card validation failed:\n"
+#                 validation_output += "\n".join(validation_messages)
+#                 return gr.update(value=None), validation_output  # Return None for the file output
+#
+#             # If image is provided, include it as base64
+#             if image:
+#                 img_byte_arr = io.BytesIO()
+#                 image.save(img_byte_arr, format='PNG')
+#                 character_card['data']['image'] = base64.b64encode(img_byte_arr.getvalue()).decode('utf-8')
+#
+#             # Convert to JSON string
+#             json_str = json.dumps(character_card, indent=2)
+#
+#             # Write the JSON to a temporary file
+#             with tempfile.NamedTemporaryFile(mode='w', delete=False, suffix='.json', encoding='utf-8') as temp_file:
+#                 temp_file.write(json_str)
+#                 temp_file_path = temp_file.name
+#
+#             # Return the file path and clear validation output
+#             return temp_file_path, ""
+#
+#         def download_character_card_as_image(
+#                 name, description, personality, scenario, first_mes, mes_example,
+#                 creator_notes, system_prompt, post_history_instructions,
+#                 alternate_greetings_str, tags_str, creator, character_version,
+#                 extensions_str, image
+#         ):
+#             # Build the character card
+#             character_card = build_character_card(
+#                 name, description, personality, scenario, first_mes, mes_example,
+#                 creator_notes, system_prompt, post_history_instructions,
+#                 alternate_greetings_str, tags_str, creator, character_version,
+#                 extensions_str
+#             )
+#
+#             # Validate the character card
+#             is_valid, validation_messages = validate_v2_card(character_card)
+#             if not is_valid:
+#                 # Return validation errors
+#                 validation_output = "Character card validation failed:\n"
+#                 validation_output += "\n".join(validation_messages)
+#                 return gr.update(value=None), validation_output  # Return None for the file output
+#
+#             # Convert the character card JSON to a string
+#             json_str = json.dumps(character_card, indent=2)
+#
+#             # Encode the JSON string to base64
+#             chara_content = base64.b64encode(json_str.encode('utf-8')).decode('utf-8')
+#
+#             # Create PNGInfo object to hold metadata
+#             png_info = PngInfo()
+#             png_info.add_text('chara', chara_content)
+#
+#             # If image is provided, use it; otherwise, create a blank image
+#             if image:
+#                 img = image.copy()
+#             else:
+#                 # Create a default blank image
+#                 img = Image.new('RGB', (512, 512), color='white')
+#
+#             # Save the image to a temporary file with metadata
+#             with tempfile.NamedTemporaryFile(mode='wb', delete=False, suffix='.png') as temp_file:
+#                 img.save(temp_file, format='PNG', pnginfo=png_info)
+#                 temp_file_path = temp_file.name
+#
+#             # Return the file path and clear validation output
+#             return temp_file_path, ""
+#
+#         def build_character_card(
+#                 name, description, personality, scenario, first_mes, mes_example,
+#                 creator_notes, system_prompt, post_history_instructions,
+#                 alternate_greetings_str, tags_str, creator, character_version,
+#                 extensions_str
+#         ):
+#             # Parse alternate_greetings from multiline string
+#             alternate_greetings = [line.strip() for line in alternate_greetings_str.strip().split('\n') if line.strip()]
+#
+#             # Parse tags from comma-separated string
+#             tags = [tag.strip() for tag in tags_str.strip().split(',') if tag.strip()]
+#
+#             # Parse extensions from JSON string
+#             try:
+#                 extensions = json.loads(extensions_str) if extensions_str.strip() else {}
+#             except json.JSONDecodeError as e:
+#                 extensions = {}
+#                 logging.error(f"Error parsing extensions JSON: {e}")
+#
+#             # Build the character card dictionary according to V2 spec
+#             character_card = {
+#                 'spec': 'chara_card_v2',
+#                 'spec_version': '2.0',
+#                 'data': {
+#                     'name': name,
+#                     'description': description,
+#                     'personality': personality,
+#                     'scenario': scenario,
+#                     'first_mes': first_mes,
+#                     'mes_example': mes_example,
+#                     'creator_notes': creator_notes,
+#                     'system_prompt': system_prompt,
+#                     'post_history_instructions': post_history_instructions,
+#                     'alternate_greetings': alternate_greetings,
+#                     'tags': tags,
+#                     'creator': creator,
+#                     'character_version': character_version,
+#                     'extensions': extensions,
+#                 }
+#             }
+#             return character_card
+#
+#         def validate_and_load_character_card(file):
+#             if file is None:
+#                 return ["No file provided for validation."] + [gr.update() for _ in range(15)]
+#
+#             try:
+#                 if file.name.lower().endswith(('.png', '.webp')):
+#                     json_data = extract_json_from_image(file)
+#                     if not json_data:
+#                         return ["Failed to extract JSON data from the image."] + [gr.update() for _ in range(15)]
+#                 elif file.name.lower().endswith('.json'):
+#                     with open(file.name, 'r', encoding='utf-8') as f:
+#                         json_data = f.read()
+#                 else:
+#                     return ["Unsupported file type."] + [gr.update() for _ in range(15)]
+#
+#                 # Parse the JSON content
+#                 try:
+#                     card_data = json.loads(json_data)
+#                 except json.JSONDecodeError as e:
+#                     return [f"JSON decoding error: {e}"] + [gr.update() for _ in range(15)]
+#
+#                 # Validate the character card
+#                 is_valid, validation_messages = validate_v2_card(card_data)
+#
+#                 # Prepare the validation output
+#                 if is_valid:
+#                     validation_output_msg = "Character card is valid according to the V2 specification."
+#                 else:
+#                     validation_output_msg = "Character card validation failed:\n" + "\n".join(validation_messages)
+#
+#                 # Extract data to populate input fields
+#                 data = card_data.get('data', {})
+#
+#                 # Handle image data
+#                 if 'image' in data:
+#                     # Decode base64 image
+#                     image_data = base64.b64decode(data['image'])
+#                     image = Image.open(io.BytesIO(image_data))
+#                 else:
+#                     image = None
+#
+#                 # Prepare values for input fields
+#                 alternate_greetings_str = "\n".join(data.get('alternate_greetings', []))
+#                 tags_str = ", ".join(data.get('tags', []))
+#                 extensions_str = json.dumps(data.get('extensions', {}), indent=2) if data.get('extensions', {}) else ""
+#
+#                 outputs = [
+#                     validation_output_msg,
+#                     data.get('name', ''),
+#                     data.get('description', ''),
+#                     data.get('personality', ''),
+#                     data.get('scenario', ''),
+#                     data.get('first_mes', ''),
+#                     data.get('mes_example', ''),
+#                     data.get('creator_notes', ''),
+#                     data.get('system_prompt', ''),
+#                     data.get('post_history_instructions', ''),
+#                     alternate_greetings_str,
+#                     tags_str,
+#                     data.get('creator', ''),
+#                     data.get('character_version', ''),
+#                     extensions_str,
+#                     image
+#                 ]
+#
+#                 return outputs
+#
+#             except Exception as e:
+#                 logging.error(f"Error validating character card: {e}")
+#                 return [f"An unexpected error occurred: {e}"] + [gr.update() for _ in range(15)]
+#
+#         # Button Callback for validation
+#         validate_button.click(
+#             fn=validate_and_load_character_card,
+#             inputs=[file_upload],
+#             outputs=[
+#                 validation_output,
+#                 name_input, description_input, personality_input, scenario_input,
+#                 first_mes_input, mes_example_input, creator_notes_input, system_prompt_input,
+#                 post_history_instructions_input, alternate_greetings_input, tags_input,
+#                 creator_input, character_version_input, extensions_input, image_input
+#             ]
+#         )
+#
+#         # Button Callbacks for save, download, etc.
+#         save_button.click(
+#             fn=save_character_card,
+#             inputs=[
+#                 name_input, description_input, personality_input, scenario_input,
+#                 first_mes_input, mes_example_input, creator_notes_input, system_prompt_input,
+#                 post_history_instructions_input, alternate_greetings_input, tags_input,
+#                 creator_input, character_version_input, extensions_input, image_input
+#             ],
+#             outputs=[save_status]
+#         )
+#
+#         download_button.click(
+#             fn=download_character_card,
+#             inputs=[
+#                 name_input, description_input, personality_input, scenario_input,
+#                 first_mes_input, mes_example_input, creator_notes_input, system_prompt_input,
+#                 post_history_instructions_input, alternate_greetings_input, tags_input,
+#                 creator_input, character_version_input, extensions_input, image_input
+#             ],
+#             outputs=[download_output, save_status]
+#         )
+#
+#         download_image_button.click(
+#             fn=download_character_card_as_image,
+#             inputs=[
+#                 name_input, description_input, personality_input, scenario_input,
+#                 first_mes_input, mes_example_input, creator_notes_input, system_prompt_input,
+#                 post_history_instructions_input, alternate_greetings_input, tags_input,
+#                 creator_input, character_version_input, extensions_input, image_input
+#             ],
+#             outputs=[download_image_output, save_status]
+#         )
+
+
+#
+# End of Character_Chat_tab.py
+#######################################################################################################################

App_Function_Libraries/Gradio_UI/Character_interaction_tab.py

@@ -24,7 +24,7 @@ from App_Function_Libraries.Gradio_UI.Writing_tab import generate_writing_feedba
 ########################################################################################################################
 #
 # Single-Character chat Functions:
-
+# FIXME - add these functions to the Personas library
 
 def chat_with_character(user_message, history, char_data, api_name_input, api_key):
     if char_data is None:
@@ -134,340 +134,13 @@ def load_chat_history(file):
         return None, None
 
 
-# FIXME - deprecated keeping until sure no longer needed
-# def create_character_card_interaction_tab():
-#     with gr.TabItem("Chat with a Character Card"):
-#         gr.Markdown("# Chat with a Character Card")
-#         with gr.Row():
-#             with gr.Column(scale=1):
-#                 character_image = gr.Image(label="Character Image", type="filepath")
-#                 character_card_upload = gr.File(label="Upload Character Card")
-#                 import_card_button = gr.Button("Import Character Card")
-#                 load_characters_button = gr.Button("Load Existing Characters")
-#                 from App_Function_Libraries.Chat import get_character_names
-#                 character_dropdown = gr.Dropdown(label="Select Character", choices=get_character_names())
-#                 user_name_input = gr.Textbox(label="Your Name", placeholder="Enter your name here")
-#                 api_name_input = gr.Dropdown(
-#                     choices=["Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral",
-#                              "OpenRouter", "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace",
-#                              "Custom-OpenAI-API"],
-#                     value="HuggingFace",
-#                     # FIXME - make it so the user cant' click `Send Message` without first setting an API + Chatbot
-#                     label="API for Interaction(Mandatory)"
-#                 )
-#                 api_key_input = gr.Textbox(label="API Key (if not set in Config_Files/config.txt)",
-#                                            placeholder="Enter your API key here", type="password")
-#                 temperature_slider = gr.Slider(minimum=0.0, maximum=2.0, value=0.7, step=0.05, label="Temperature")
-#                 import_chat_button = gr.Button("Import Chat History")
-#                 chat_file_upload = gr.File(label="Upload Chat History JSON", visible=False)
-#
-#             with gr.Column(scale=2):
-#                 chat_history = gr.Chatbot(label="Conversation", height=800)
-#                 user_input = gr.Textbox(label="Your message")
-#                 send_message_button = gr.Button("Send Message")
-#                 regenerate_button = gr.Button("Regenerate Last Message")
-#                 clear_chat_button = gr.Button("Clear Chat")
-#                 chat_media_name = gr.Textbox(label="Custom Chat Name(optional)", visible=True)
-#                 save_chat_history_to_db = gr.Button("Save Chat History to DataBase")
-#                 save_status = gr.Textbox(label="Save Status", interactive=False)
-#
-#         character_data = gr.State(None)
-#         user_name = gr.State("")
-#
-#     def import_chat_history(file, current_history, char_data):
-#         loaded_history, char_name = load_chat_history(file)
-#         if loaded_history is None:
-#             return current_history, char_data, "Failed to load chat history."
-#
-#         # Check if the loaded chat is for the current character
-#         if char_data and char_data.get('name') != char_name:
-#             return current_history, char_data, f"Warning: Loaded chat is for character '{char_name}', but current character is '{char_data.get('name')}'. Chat not imported."
-#
-#         # If no character is selected, try to load the character from the chat
-#         if not char_data:
-#             new_char_data = load_character(char_name)[0]
-#             if new_char_data:
-#                 char_data = new_char_data
-#             else:
-#                 return current_history, char_data, f"Warning: Character '{char_name}' not found. Please select the character manually."
-#
-#         return loaded_history, char_data, f"Chat history for '{char_name}' imported successfully."
-#
-#     def import_character(file):
-#         card_data = import_character_card(file)
-#         if card_data:
-#             from App_Function_Libraries.Chat import save_character
-#             save_character(card_data)
-#             return card_data, gr.update(choices=get_character_names())
-#         else:
-#             return None, gr.update()
-#
-#     def load_character(name):
-#         from App_Function_Libraries.Chat import load_characters
-#         characters = load_characters()
-#         char_data = characters.get(name)
-#         if char_data:
-#             first_message = char_data.get('first_mes', "Hello! I'm ready to chat.")
-#             return char_data, [(None, first_message)] if first_message else [], None
-#         return None, [], None
-#
-#     def load_character_image(name):
-#         from App_Function_Libraries.Chat import load_characters
-#         characters = load_characters()
-#         char_data = characters.get(name)
-#         if char_data and 'image_path' in char_data:
-#             image_path = char_data['image_path']
-#             if os.path.exists(image_path):
-#                 return image_path
-#             else:
-#                 logging.warning(f"Image file not found: {image_path}")
-#         return None
-#
-#     def load_character_and_image(name):
-#         char_data, chat_history, _ = load_character(name)
-#         image_path = load_character_image(name)
-#         logging.debug(f"Character: {name}")
-#         logging.debug(f"Character data: {char_data}")
-#         logging.debug(f"Image path: {image_path}")
-#         return char_data, chat_history, image_path
-#
-#     def character_chat_wrapper(message, history, char_data, api_endpoint, api_key, temperature, user_name):
-#         logging.debug("Entered character_chat_wrapper")
-#         if char_data is None:
-#             return "Please select a character first.", history
-#
-#         if not user_name:
-#             user_name = "User"
-#
-#         char_name = char_data.get('name', 'AI Assistant')
-#
-#         # Prepare the character's background information
-#         char_background = f"""
-#         Name: {char_name}
-#         Description: {char_data.get('description', 'N/A')}
-#         Personality: {char_data.get('personality', 'N/A')}
-#         Scenario: {char_data.get('scenario', 'N/A')}
-#         """
-#
-#         # Prepare the system prompt for character impersonation
-#         system_message = f"""You are roleplaying as {char_name}, the character described below. Respond to the user's messages in character, maintaining the personality and background provided. Do not break character or refer to yourself as an AI. Always refer to yourself as "{char_name}" and refer to the user as "{user_name}".
-#
-#         {char_background}
-#
-#         Additional instructions: {char_data.get('post_history_instructions', '')}
-#         """
-#
-#         # Prepare media_content and selected_parts
-#         media_content = {
-#             'id': char_name,
-#             'title': char_name,
-#             'content': char_background,
-#             'description': char_data.get('description', ''),
-#             'personality': char_data.get('personality', ''),
-#             'scenario': char_data.get('scenario', '')
-#         }
-#         selected_parts = ['description', 'personality', 'scenario']
-#
-#         prompt = char_data.get('post_history_instructions', '')
-#
-#         # Prepare the input for the chat function
-#         if not history:
-#             full_message = f"{prompt}\n\n{user_name}: {message}" if prompt else f"{user_name}: {message}"
-#         else:
-#             full_message = f"{user_name}: {message}"
-#
-#         # Call the chat function
-#         bot_message = chat(
-#             full_message,
-#             history,
-#             media_content,
-#             selected_parts,
-#             api_endpoint,
-#             api_key,
-#             prompt,
-#             temperature,
-#             system_message
-#         )
-#
-#         # Update history
-#         history.append((message, bot_message))
-#         return history
-#
-#     def save_chat_history(history, character_name):
-#         # Create the Saved_Chats folder if it doesn't exist
-#         save_directory = "Saved_Chats"
-#         os.makedirs(save_directory, exist_ok=True)
-#
-#         timestamp = datetime.now().strftime("%Y%m%d_%H%M%S")
-#         filename = f"chat_history_{character_name}_{timestamp}.json"
-#         filepath = os.path.join(save_directory, filename)
-#
-#         chat_data = {
-#             "character": character_name,
-#             "timestamp": timestamp,
-#             "history": history
-#         }
-#
-#         try:
-#             with open(filepath, 'w', encoding='utf-8') as f:
-#                 json.dump(chat_data, f, ensure_ascii=False, indent=2)
-#             return filepath
-#         except Exception as e:
-#             return f"Error saving chat: {str(e)}"
-#
-#     def save_current_chat(history, char_data):
-#         if not char_data or not history:
-#             return "No chat to save or character not selected."
-#
-#         character_name = char_data.get('name', 'Unknown')
-#         result = save_chat_history(history, character_name)
-#         if result.startswith("Error"):
-#             return result
-#         return f"Chat saved successfully as {result}"
-#
-#     def regenerate_last_message(history, char_data, api_name, api_key, temperature, user_name):
-#         if not history:
-#             return history
-#
-#         last_user_message = history[-1][0]
-#         new_history = history[:-1]
-#
-#         return character_chat_wrapper(last_user_message, new_history, char_data, api_name, api_key, temperature,
-#                                       user_name)
-#
-#     import_chat_button.click(
-#         fn=lambda: gr.update(visible=True),
-#         outputs=chat_file_upload
-#     )
-#
-#     chat_file_upload.change(
-#         fn=import_chat_history,
-#         inputs=[chat_file_upload, chat_history, character_data],
-#         outputs=[chat_history, character_data, save_status]
-#     )
-#
-#     def update_character_info(name):
-#         from App_Function_Libraries.Chat import load_characters
-#         characters = load_characters()
-#         char_data = characters.get(name)
-#
-#         image_path = char_data.get('image_path') if char_data else None
-#
-#         logging.debug(f"Character: {name}")
-#         logging.debug(f"Character data: {char_data}")
-#         logging.debug(f"Image path: {image_path}")
-#
-#         if image_path:
-#             if os.path.exists(image_path):
-#                 logging.debug(f"Image file exists at {image_path}")
-#                 if os.access(image_path, os.R_OK):
-#                     logging.debug(f"Image file is readable")
-#                 else:
-#                     logging.warning(f"Image file is not readable: {image_path}")
-#                     image_path = None
-#             else:
-#                 logging.warning(f"Image file does not exist: {image_path}")
-#                 image_path = None
-#         else:
-#             logging.warning("No image path provided for the character")
-#
-#         return char_data, None, image_path  # Return None for chat_history
-#
-#     def on_character_select(name):
-#         logging.debug(f"Character selected: {name}")
-#         return update_character_info_with_error_handling(name)
-#
-#     def clear_chat_history():
-#         return [], None  # Return empty list for chat_history and None for character_data
-#
-#     def update_character_info_with_error_handling(name):
-#         logging.debug(f"Entering update_character_info_with_error_handling for character: {name}")
-#         try:
-#             char_data, _, image_path = update_character_info(name)
-#             logging.debug(f"Retrieved data: char_data={bool(char_data)}, image_path={image_path}")
-#
-#             if char_data:
-#                 first_message = char_data.get('first_mes', "Hello! I'm ready to chat.")
-#                 chat_history = [(None, first_message)] if first_message else []
-#             else:
-#                 chat_history = []
-#
-#             logging.debug(f"Created chat_history with length: {len(chat_history)}")
-#
-#             if image_path and os.path.exists(image_path):
-#                 logging.debug(f"Image file exists at {image_path}")
-#                 return char_data, chat_history, image_path
-#             else:
-#                 logging.warning(f"Image not found or invalid path: {image_path}")
-#                 return char_data, chat_history, None
-#         except Exception as e:
-#             logging.error(f"Error updating character info: {str(e)}", exc_info=True)
-#             return None, [], None
-#         finally:
-#             logging.debug("Exiting update_character_info_with_error_handling")
-#
-#     # Define States for conversation_id and media_content, which are required for saving chat history
-#     conversation_id = gr.State(str(uuid.uuid4()))
-#     media_content = gr.State({})
-#
-#     import_card_button.click(
-#         fn=import_character,
-#         inputs=[character_card_upload],
-#         outputs=[character_data, character_dropdown]
-#     )
-#
-#     load_characters_button.click(
-#         fn=lambda: gr.update(choices=get_character_names()),
-#         outputs=character_dropdown
-#     )
-#
-#     clear_chat_button.click(
-#         fn=clear_chat_history,
-#         inputs=[],
-#         outputs=[chat_history, character_data]
-#     )
-#
-#     character_dropdown.change(
-#         fn=on_character_select,
-#         inputs=[character_dropdown],
-#         outputs=[character_data, chat_history, character_image]
-#     )
-#
-#     send_message_button.click(
-#         fn=character_chat_wrapper,
-#         inputs=[user_input, chat_history, character_data, api_name_input, api_key_input, temperature_slider,
-#                 user_name_input],
-#         outputs=[chat_history]
-#     ).then(lambda: "", outputs=user_input)
-#
-#     regenerate_button.click(
-#         fn=regenerate_last_message,
-#         inputs=[chat_history, character_data, api_name_input, api_key_input, temperature_slider, user_name_input],
-#         outputs=[chat_history]
-#     )
-#
-#     user_name_input.change(
-#         fn=lambda name: name,
-#         inputs=[user_name_input],
-#         outputs=[user_name]
-#     )
-#
-#     # FIXME - Implement saving chat history to database; look at Chat_UI.py for reference
-#     save_chat_history_to_db.click(
-#         save_chat_history_to_db_wrapper,
-#         inputs=[chat_history, conversation_id, media_content, chat_media_name],
-#         outputs=[conversation_id, gr.Textbox(label="Save Status")]
-#     )
-#
-#     return character_data, chat_history, user_input, user_name, character_image
-
-
 #
-# End of Character chat tab
+# End of X
 ######################################################################################################################
 #
 # Multi-Character Chat Interface
 
+# FIXME - refactor and move these functions to the Character_Chat library so that it uses the same functions
 def character_interaction_setup():
     characters = load_characters()
     return characters, [], None, None

App_Function_Libraries/Gradio_UI/Chat_ui.py

@@ -167,6 +167,39 @@ def delete_message_from_chat(message_id, history):
     return updated_history
 
 
+def regenerate_last_message(history, media_content, selected_parts, api_endpoint, api_key, custom_prompt, temperature, system_prompt):
+    if not history:
+        return history, "No messages to regenerate."
+
+    last_entry = history[-1]
+    last_user_message, last_bot_message = last_entry
+
+    if last_bot_message is None:
+        return history, "The last message is not from the bot."
+
+    new_history = history[:-1]
+
+    if not last_user_message:
+        return new_history, "No user message to regenerate the bot response."
+
+    full_message = last_user_message
+
+    bot_message = chat(
+        full_message,
+        new_history,
+        media_content,
+        selected_parts,
+        api_endpoint,
+        api_key,
+        custom_prompt,
+        temperature,
+        system_prompt
+    )
+
+    new_history.append((last_user_message, bot_message))
+
+    return new_history, "Last message regenerated successfully."
+
 def create_chat_interface():
     custom_css = """
     .chatbot-container .message-wrap .message {
@@ -231,6 +264,7 @@ def create_chat_interface():
                 chatbot = gr.Chatbot(height=600, elem_classes="chatbot-container")
                 msg = gr.Textbox(label="Enter your message")
                 submit = gr.Button("Submit")
+                regenerate_button = gr.Button("Regenerate Last Message")
                 clear_chat_button = gr.Button("Clear Chat")
 
                 edit_message_id = gr.Number(label="Message ID to Edit", visible=False)
@@ -367,6 +401,12 @@ def create_chat_interface():
             outputs=[conversation_id, gr.Textbox(label="Save Status")]
         )
 
+        regenerate_button.click(
+            regenerate_last_message,
+            inputs=[chatbot, media_content, selected_parts, api_endpoint, api_key, user_prompt, temperature, system_prompt_input],
+            outputs=[chatbot, save_status]
+        )
+
         chatbot.select(show_edit_message, None, [edit_message_text, edit_message_id, update_message_button])
         chatbot.select(show_delete_message, None, [delete_message_id, delete_message_button])
 
@@ -430,6 +470,7 @@ def create_chat_interface_stacked():
         with gr.Row():
             with gr.Column():
                 submit = gr.Button("Submit")
+                regenerate_button = gr.Button("Regenerate Last Message")
                 clear_chat_button = gr.Button("Clear Chat")
                 chat_media_name = gr.Textbox(label="Custom Chat Name(optional)", visible=True)
                 save_chat_history_to_db = gr.Button("Save Chat History to DataBase")
@@ -521,6 +562,12 @@ def create_chat_interface_stacked():
             outputs=[conversation_id, gr.Textbox(label="Save Status")]
         )
 
+        regenerate_button.click(
+            regenerate_last_message,
+            inputs=[chatbot, media_content, selected_parts, api_endpoint, api_key, user_prompt, temp, system_prompt],
+            outputs=[chatbot, gr.Textbox(label="Regenerate Status")]
+        )
+
 
 # FIXME - System prompts
 def create_chat_interface_multi_api():
@@ -534,7 +581,7 @@ def create_chat_interface_multi_api():
     }
     """
     with gr.TabItem("One Prompt - Multiple APIs"):
-        gr.Markdown("# One Prompt but Multiple API Chat Interface")
+        gr.Markdown("# One Prompt but Multiple APIs Chat Interface")
 
         with gr.Row():
             with gr.Column(scale=1):
@@ -551,40 +598,36 @@ def create_chat_interface_multi_api():
             with gr.Column():
                 preset_prompt = gr.Dropdown(label="Select Preset Prompt", choices=load_preset_prompts(), visible=True)
                 system_prompt = gr.Textbox(label="System Prompt", value="You are a helpful AI assistant.", lines=5)
-                user_prompt = gr.Textbox(label="Modify Prompt", lines=5, value=".")
+                user_prompt = gr.Textbox(label="Modify Prompt (Prefixed to your message every time)", lines=5, value="", visible=True)
 
         with gr.Row():
             chatbots = []
             api_endpoints = []
             api_keys = []
             temperatures = []
+            regenerate_buttons = []
             for i in range(3):
                 with gr.Column():
                     gr.Markdown(f"### Chat Window {i + 1}")
                     api_endpoint = gr.Dropdown(label=f"API Endpoint {i + 1}",
                                                choices=["Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq",
                                                         "DeepSeek", "Mistral", "OpenRouter", "Llama.cpp", "Kobold",
-                                                        "Ooba",
-                                                        "Tabbyapi", "VLLM", "ollama", "HuggingFace"])
+                                                        "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace"])
                     api_key = gr.Textbox(label=f"API Key {i + 1} (if required)", type="password")
                     temperature = gr.Slider(label=f"Temperature {i + 1}", minimum=0.0, maximum=1.0, step=0.05,
                                             value=0.7)
                     chatbot = gr.Chatbot(height=800, elem_classes="chat-window")
+                    regenerate_button = gr.Button(f"Regenerate Last Message {i + 1}")
                     chatbots.append(chatbot)
                     api_endpoints.append(api_endpoint)
                     api_keys.append(api_key)
                     temperatures.append(temperature)
+                    regenerate_buttons.append(regenerate_button)
 
         with gr.Row():
             msg = gr.Textbox(label="Enter your message", scale=4)
             submit = gr.Button("Submit", scale=1)
-            # FIXME - clear chat
-        #     clear_chat_button = gr.Button("Clear Chat")
-        #
-        # clear_chat_button.click(
-        #     clear_chat,
-        #     outputs=[chatbot]
-        # )
+            clear_chat_button = gr.Button("Clear All Chats")
 
         # State variables
         chat_history = [gr.State([]) for _ in range(3)]
@@ -601,6 +644,14 @@ def create_chat_interface_multi_api():
 
         preset_prompt.change(update_user_prompt, inputs=preset_prompt, outputs=user_prompt)
 
+
+        def clear_all_chats():
+            return [[]] * 3 + [[]] * 3
+
+        clear_chat_button.click(
+            clear_all_chats,
+            outputs=chatbots + chat_history
+        )
         def chat_wrapper_multi(message, custom_prompt, system_prompt, *args):
             chat_histories = args[:3]
             chatbots = args[3:6]
@@ -630,6 +681,46 @@ def create_chat_interface_multi_api():
 
             return [gr.update(value="")] + new_chatbots + new_chat_histories
 
+
+        def regenerate_last_message(chat_history, chatbot, media_content, selected_parts, api_endpoint, api_key, custom_prompt, temperature, system_prompt):
+            if not chat_history:
+                return chatbot, chat_history, "No messages to regenerate."
+
+            last_entry = chat_history[-1]
+            last_user_message, last_bot_message = last_entry
+
+            if last_bot_message is None:
+                return chatbot, chat_history, "The last message is not from the bot."
+
+            new_history = chat_history[:-1]
+
+            if not last_user_message:
+                return chatbot[:-1], new_history, "No user message to regenerate the bot response."
+
+            bot_message = chat(
+                last_user_message,
+                new_history,
+                media_content,
+                selected_parts,
+                api_endpoint,
+                api_key,
+                custom_prompt,
+                temperature,
+                system_prompt
+            )
+
+            new_history.append((last_user_message, bot_message))
+            new_chatbot = chatbot[:-1] + [(last_user_message, bot_message)]
+
+            return new_chatbot, new_history, "Last message regenerated successfully."
+
+        for i in range(3):
+            regenerate_buttons[i].click(
+                regenerate_last_message,
+                inputs=[chat_history[i], chatbots[i], media_content, selected_parts, api_endpoints[i], api_keys[i], user_prompt, temperatures[i], system_prompt],
+                outputs=[chatbots[i], chat_history[i], gr.Textbox(label=f"Regenerate Status {i + 1}")]
+            )
+
         # In the create_chat_interface_multi_api function:
         submit.click(
             chat_wrapper_multi,
@@ -656,6 +747,7 @@ def create_chat_interface_multi_api():
             )
 
 
+
 def create_chat_interface_four():
     custom_css = """
     .chatbot-container .message-wrap .message {
@@ -687,16 +779,6 @@ def create_chat_interface_four():
         chat_interfaces = []
 
         def create_single_chat_interface(index, user_prompt_component):
-            """
-            Creates a single chat interface with its own set of components and event bindings.
-
-            Parameters:
-                index (int): The index of the chat interface.
-                user_prompt_component (gr.Textbox): The user prompt textbox component.
-
-            Returns:
-                dict: A dictionary containing all components of the chat interface.
-            """
             with gr.Column():
                 gr.Markdown(f"### Chat Window {index + 1}")
                 api_endpoint = gr.Dropdown(
@@ -721,6 +803,7 @@ def create_chat_interface_four():
                 chatbot = gr.Chatbot(height=400, elem_classes="chat-window")
                 msg = gr.Textbox(label=f"Enter your message for Chat {index + 1}")
                 submit = gr.Button(f"Submit to Chat {index + 1}")
+                regenerate_button = gr.Button(f"Regenerate Last Message {index + 1}")
                 clear_chat_button = gr.Button(f"Clear Chat {index + 1}")
 
                 # State to maintain chat history
@@ -734,14 +817,11 @@ def create_chat_interface_four():
                     'chatbot': chatbot,
                     'msg': msg,
                     'submit': submit,
+                    'regenerate_button': regenerate_button,
                     'clear_chat_button': clear_chat_button,
                     'chat_history': chat_history
                 })
 
-        # # Create four chat interfaces
-        # for i in range(4):
-        #     create_single_chat_interface(i, user_prompt)
-
         # Create four chat interfaces arranged in a 2x2 grid
         with gr.Row():
             for i in range(2):
@@ -749,7 +829,6 @@ def create_chat_interface_four():
                     for j in range(2):
                         create_single_chat_interface(i * 2 + j, user_prompt)
 
-
         # Update user_prompt based on preset_prompt selection
         preset_prompt.change(
             fn=update_user_prompt,
@@ -760,7 +839,6 @@ def create_chat_interface_four():
         def chat_wrapper_single(message, chat_history, api_endpoint, api_key, temperature, user_prompt):
             logging.debug(f"Chat Wrapper Single - Message: {message}, Chat History: {chat_history}")
 
-            # Call chat_wrapper with the new signature and the additional parameters
             new_msg, new_history, _ = chat_wrapper(
                 message,
                 chat_history,
@@ -773,13 +851,12 @@ def create_chat_interface_four():
                 False,  # save_conversation
                 temperature,  # temperature
                 system_prompt="",  # system_prompt
-                max_tokens=None,  # Additional parameters with default None values
+                max_tokens=None,
                 top_p=None,
                 frequency_penalty=None,
                 presence_penalty=None,
                 stop_sequence=None
             )
-            # Only append to history if the new message was successful (i.e., no error in API response)
             if "API request failed" not in new_msg:
                 chat_history.append((message, new_msg))
             else:
@@ -787,9 +864,40 @@ def create_chat_interface_four():
 
             return "", chat_history, chat_history
 
+        def regenerate_last_message(chat_history, api_endpoint, api_key, temperature, user_prompt):
+            if not chat_history:
+                return chat_history, chat_history, "No messages to regenerate."
+
+            last_user_message, _ = chat_history[-1]
+
+            new_msg, new_history, _ = chat_wrapper(
+                last_user_message,
+                chat_history[:-1],
+                {},  # Empty media_content
+                [],  # Empty selected_parts
+                api_endpoint,
+                api_key,
+                user_prompt,  # custom_prompt
+                None,  # conversation_id
+                False,  # save_conversation
+                temperature,  # temperature
+                system_prompt="",  # system_prompt
+                max_tokens=None,
+                top_p=None,
+                frequency_penalty=None,
+                presence_penalty=None,
+                stop_sequence=None
+            )
+
+            if "API request failed" not in new_msg:
+                new_history.append((last_user_message, new_msg))
+                return new_history, new_history, "Last message regenerated successfully."
+            else:
+                logging.error(f"API request failed during regeneration: {new_msg}")
+                return chat_history, chat_history, f"Failed to regenerate: {new_msg}"
+
         # Attach click events for each chat interface
         for interface in chat_interfaces:
-            logging.debug(f"Chat Interface - Clicked Submit for Chat {interface['chatbot']}"),
             interface['submit'].click(
                 chat_wrapper_single,
                 inputs=[
@@ -807,7 +915,22 @@ def create_chat_interface_four():
                 ]
             )
 
-            # Bind the clear chat button
+            interface['regenerate_button'].click(
+                regenerate_last_message,
+                inputs=[
+                    interface['chat_history'],
+                    interface['api_endpoint'],
+                    interface['api_key'],
+                    interface['temperature'],
+                    user_prompt
+                ],
+                outputs=[
+                    interface['chatbot'],
+                    interface['chat_history'],
+                    gr.Textbox(label="Regenerate Status")
+                ]
+            )
+
             interface['clear_chat_button'].click(
                 clear_chat_single,
                 inputs=[],

App_Function_Libraries/Gradio_UI/Embeddings_tab.py

@@ -7,23 +7,20 @@ import logging
 #
 # External Imports
 import gradio as gr
+import numpy as np
 from tqdm import tqdm
-
-from App_Function_Libraries.Chunk_Lib import improved_chunking_process, chunk_for_embedding
 #
 # Local Imports
 from App_Function_Libraries.DB.DB_Manager import get_all_content_from_database
 from App_Function_Libraries.RAG.ChromaDB_Library import chroma_client, \
     store_in_chroma, situate_context
 from App_Function_Libraries.RAG.Embeddings_Create import create_embedding, create_embeddings_batch
-
-
+from App_Function_Libraries.Chunk_Lib import improved_chunking_process, chunk_for_embedding
 #
 ########################################################################################################################
 #
 # Functions:
 
-# FIXME - under construction
 def create_embeddings_tab():
     with gr.TabItem("Create Embeddings"):
         gr.Markdown("# Create Embeddings for All Content")
@@ -36,13 +33,36 @@ def create_embeddings_tab():
                     value="huggingface"
                 )
                 gr.Markdown("Note: Local provider requires a running Llama.cpp/llamafile server.")
-                gr.Markdown("OpenAI provider requires a valid API key. ")
-                gr.Markdown("OpenAI Embeddings models: `text-embedding-3-small`, `text-embedding-3-large`")
-                gr.Markdown("HuggingFace provider requires a valid model name, i.e. `dunzhang/stella_en_400M_v5`")
-                embedding_model = gr.Textbox(
-                    label="Embedding Model",
-                    value="Enter your embedding model name here", lines=3
+                gr.Markdown("OpenAI provider requires a valid API key.")
+
+                huggingface_model = gr.Dropdown(
+                    choices=[
+                        "jinaai/jina-embeddings-v3",
+                        "Alibaba-NLP/gte-large-en-v1.5",
+                        "dunzhang/setll_en_400M_v5",
+                        "custom"
+                    ],
+                    label="Hugging Face Model",
+                    value="jinaai/jina-embeddings-v3",
+                    visible=True
                 )
+
+                openai_model = gr.Dropdown(
+                    choices=[
+                        "text-embedding-3-small",
+                        "text-embedding-3-large"
+                    ],
+                    label="OpenAI Embedding Model",
+                    value="text-embedding-3-small",
+                    visible=False
+                )
+
+                custom_embedding_model = gr.Textbox(
+                    label="Custom Embedding Model",
+                    placeholder="Enter your custom embedding model name here",
+                    visible=False
+                )
+
                 embedding_api_url = gr.Textbox(
                     label="API URL (for local provider)",
                     value="http://localhost:8080/embedding",
@@ -74,15 +94,32 @@ def create_embeddings_tab():
                 status_output = gr.Textbox(label="Status", lines=10)
 
         def update_provider_options(provider):
-            return gr.update(visible=provider == "local")
+            if provider == "huggingface":
+                return gr.update(visible=True), gr.update(visible=False), gr.update(visible=False), gr.update(visible=False)
+            elif provider == "local":
+                return gr.update(visible=False), gr.update(visible=False), gr.update(visible=False), gr.update(visible=True)
+            else:  # OpenAI
+                return gr.update(visible=False), gr.update(visible=True), gr.update(visible=False), gr.update(visible=False)
+
+        def update_huggingface_options(model):
+            if model == "custom":
+                return gr.update(visible=True)
+            else:
+                return gr.update(visible=False)
 
         embedding_provider.change(
             fn=update_provider_options,
             inputs=[embedding_provider],
-            outputs=[embedding_api_url]
+            outputs=[huggingface_model, openai_model, custom_embedding_model, embedding_api_url]
+        )
+
+        huggingface_model.change(
+            fn=update_huggingface_options,
+            inputs=[huggingface_model],
+            outputs=[custom_embedding_model]
         )
 
-        def create_all_embeddings(provider, model, api_url, method, max_size, overlap, adaptive):
+        def create_all_embeddings(provider, hf_model, openai_model, custom_model, api_url, method, max_size, overlap, adaptive):
             try:
                 all_content = get_all_content_from_database()
                 if not all_content:
@@ -98,6 +135,14 @@ def create_embeddings_tab():
                 collection_name = "all_content_embeddings"
                 collection = chroma_client.get_or_create_collection(name=collection_name)
 
+                # Determine the model to use
+                if provider == "huggingface":
+                    model = custom_model if hf_model == "custom" else hf_model
+                elif provider == "openai":
+                    model = openai_model
+                else:
+                    model = custom_model
+
                 for item in all_content:
                     media_id = item['id']
                     text = item['content']
@@ -133,7 +178,7 @@ def create_embeddings_tab():
 
         create_button.click(
             fn=create_all_embeddings,
-            inputs=[embedding_provider, embedding_model, embedding_api_url,
+            inputs=[embedding_provider, huggingface_model, openai_model, custom_embedding_model, embedding_api_url,
                     chunking_method, max_chunk_size, chunk_overlap, adaptive_chunking],
             outputs=status_output
         )
@@ -159,13 +204,36 @@ def create_view_embeddings_tab():
                     value="huggingface"
                 )
                 gr.Markdown("Note: Local provider requires a running Llama.cpp/llamafile server.")
-                gr.Markdown("OpenAI provider requires a valid API key. ")
-                gr.Markdown("OpenAI Embeddings models: `text-embedding-3-small`, `text-embedding-3-large`")
-                gr.Markdown("HuggingFace provider requires a valid model name, i.e. `dunzhang/stella_en_400M_v5`")
-                embedding_model = gr.Textbox(
-                    label="Embedding Model",
-                    value="Enter your embedding model name here", lines=3
+                gr.Markdown("OpenAI provider requires a valid API key.")
+
+                huggingface_model = gr.Dropdown(
+                    choices=[
+                        "jinaai/jina-embeddings-v3",
+                        "Alibaba-NLP/gte-large-en-v1.5",
+                        "dunzhang/stella_en_400M_v5",
+                        "custom"
+                    ],
+                    label="Hugging Face Model",
+                    value="jinaai/jina-embeddings-v3",
+                    visible=True
                 )
+
+                openai_model = gr.Dropdown(
+                    choices=[
+                        "text-embedding-3-small",
+                        "text-embedding-3-large"
+                    ],
+                    label="OpenAI Embedding Model",
+                    value="text-embedding-3-small",
+                    visible=False
+                )
+
+                custom_embedding_model = gr.Textbox(
+                    label="Custom Embedding Model",
+                    placeholder="Enter your custom embedding model name here",
+                    visible=False
+                )
+
                 embedding_api_url = gr.Textbox(
                     label="API URL (for local provider)",
                     value="http://localhost:8080/embedding",
@@ -222,7 +290,18 @@ def create_view_embeddings_tab():
                 return gr.update(choices=["Error: Unable to fetch items"]), {}
 
         def update_provider_options(provider):
-            return gr.update(visible=provider == "local")
+            if provider == "huggingface":
+                return gr.update(visible=True), gr.update(visible=False), gr.update(visible=False), gr.update(visible=False)
+            elif provider == "local":
+                return gr.update(visible=False), gr.update(visible=False), gr.update(visible=False), gr.update(visible=True)
+            else:  # OpenAI
+                return gr.update(visible=False), gr.update(visible=True), gr.update(visible=False), gr.update(visible=False)
+
+        def update_huggingface_options(model):
+            if model == "custom":
+                return gr.update(visible=True)
+            else:
+                return gr.update(visible=False)
 
         def check_embedding_status(selected_item, item_mapping):
             if not selected_item:
@@ -255,7 +334,8 @@ def create_view_embeddings_tab():
                 logging.error(f"Error in check_embedding_status: {str(e)}")
                 return f"Error processing item: {selected_item}. Details: {str(e)}", "", ""
 
-        def create_new_embedding_for_item(selected_item, provider, model, api_url, method, max_size, overlap, adaptive,
+        def create_new_embedding_for_item(selected_item, provider, hf_model, openai_model, custom_model, api_url,
+                                          method, max_size, overlap, adaptive,
                                           item_mapping, use_contextual, contextual_api_choice=None):
             if not selected_item:
                 return "Please select an item", "", ""
@@ -290,13 +370,10 @@ def create_view_embeddings_tab():
                 texts, ids, metadatas = [], [], []
                 chunk_count = 0
                 logging.info("Generating contextual summaries and preparing chunks for embedding")
-                for i, chunk in tqdm(enumerate(chunks), total=len(chunks), desc="Processing chunks"):
+                for i, chunk in enumerate(chunks):
                     chunk_text = chunk['text']
                     chunk_metadata = chunk['metadata']
-                    if chunk_count == 0:
-                        chunk_count = 1
                     if use_contextual:
-                        # Generate contextual summary
                         logging.debug(f"Generating contextual summary for chunk {chunk_count}")
                         context = situate_context(contextual_api_choice, item['content'], chunk_text)
                         contextualized_text = f"{chunk_text}\n\nContextual Summary: {context}"
@@ -305,6 +382,15 @@ def create_view_embeddings_tab():
                         context = None
 
                     chunk_id = f"doc_{item_id}_chunk_{i}"
+
+                    # Determine the model to use
+                    if provider == "huggingface":
+                        model = custom_model if hf_model == "custom" else hf_model
+                    elif provider == "openai":
+                        model = openai_model
+                    else:
+                        model = custom_model
+
                     metadata = {
                         "media_id": str(item_id),
                         "chunk_index": i,
@@ -324,7 +410,7 @@ def create_view_embeddings_tab():
                     texts.append(contextualized_text)
                     ids.append(chunk_id)
                     metadatas.append(metadata)
-                    chunk_count = chunk_count+1
+                    chunk_count += 1
 
                 # Create embeddings in batch
                 logging.info(f"Creating embeddings for {len(texts)} chunks")
@@ -334,7 +420,12 @@ def create_view_embeddings_tab():
                 store_in_chroma(collection_name, texts, embeddings, ids, metadatas)
 
                 # Create a preview of the first embedding
-                embedding_preview = str(embeddings[0][:50]) if embeddings else "No embeddings created"
+                if isinstance(embeddings, np.ndarray) and embeddings.size > 0:
+                    embedding_preview = str(embeddings[0][:50])
+                elif isinstance(embeddings, list) and len(embeddings) > 0:
+                    embedding_preview = str(embeddings[0][:50])
+                else:
+                    embedding_preview = "No embeddings created"
 
                 # Return status message
                 status = f"New embeddings created and stored for item: {item['title']} (ID: {item_id})"
@@ -344,9 +435,10 @@ def create_view_embeddings_tab():
                     status += " (with contextual summaries)"
 
                 # Return status message, embedding preview, and metadata
-                return status, f"First 50 elements of new embedding:\n{embedding_preview}", json.dumps(metadatas[0], indent=2)
+                return status, f"First 50 elements of new embedding:\n{embedding_preview}", json.dumps(metadatas[0],
+                                                                                                       indent=2)
             except Exception as e:
-                logging.error(f"Error in create_new_embedding_for_item: {str(e)}")
+                logging.error(f"Error in create_new_embedding_for_item: {str(e)}", exc_info=True)
                 return f"Error creating embedding: {str(e)}", "", ""
 
         refresh_button.click(
@@ -360,7 +452,7 @@ def create_view_embeddings_tab():
         )
         create_new_embedding_button.click(
             create_new_embedding_for_item,
-            inputs=[item_dropdown, embedding_provider, embedding_model, embedding_api_url,
+            inputs=[item_dropdown, embedding_provider, huggingface_model, openai_model, custom_embedding_model, embedding_api_url,
                     chunking_method, max_chunk_size, chunk_overlap, adaptive_chunking, item_mapping,
                     use_contextual_embeddings, contextual_api_choice],
             outputs=[embedding_status, embedding_preview, embedding_metadata]
@@ -368,11 +460,16 @@ def create_view_embeddings_tab():
         embedding_provider.change(
             update_provider_options,
             inputs=[embedding_provider],
-            outputs=[embedding_api_url]
+            outputs=[huggingface_model, openai_model, custom_embedding_model, embedding_api_url]
+        )
+        huggingface_model.change(
+            update_huggingface_options,
+            inputs=[huggingface_model],
+            outputs=[custom_embedding_model]
         )
 
     return (item_dropdown, refresh_button, embedding_status, embedding_preview, embedding_metadata,
-            create_new_embedding_button, embedding_provider, embedding_model, embedding_api_url,
+            create_new_embedding_button, embedding_provider, huggingface_model, openai_model, custom_embedding_model, embedding_api_url,
             chunking_method, max_chunk_size, chunk_overlap, adaptive_chunking,
             use_contextual_embeddings, contextual_api_choice, contextual_api_key)
 
@@ -389,9 +486,11 @@ def create_purge_embeddings_tab():
 
     def purge_all_embeddings():
         try:
+            # It came to me in a dream....I literally don't remember how the fuck this works, cant find documentation...
             collection_name = "all_content_embeddings"
             chroma_client.delete_collection(collection_name)
             chroma_client.create_collection(collection_name)
+            logging.info(f"All embeddings have been purged successfully.")
             return "All embeddings have been purged successfully."
         except Exception as e:
             logging.error(f"Error during embedding purge: {str(e)}")

App_Function_Libraries/Gradio_UI/Explain_summarize_tab.py

@@ -177,7 +177,7 @@ def summarize_explain_text(message, api_endpoint, api_key, summarization, explan
                     summarization_response = summarize_with_deepseek(api_key, input_data, user_prompt, temp,
                                                                      system_prompt)
                 elif api_endpoint.lower() == "llama.cpp":
-                    summarization_response = summarize_with_llama(input_data, user_prompt, temp, system_prompt)
+                    summarization_response = summarize_with_llama(input_data, user_prompt, api_key, temp, system_prompt)
                 elif api_endpoint.lower() == "kobold":
                     summarization_response = summarize_with_kobold(input_data, api_key, user_prompt, temp,
                                                                    system_prompt)
@@ -194,7 +194,7 @@ def summarize_explain_text(message, api_endpoint, api_key, summarization, explan
                     summarization_response = summarize_with_huggingface(api_key, input_data, user_prompt,
                                                                         temp)  # , system_prompt)
                 elif api_endpoint.lower() == "ollama":
-                    summarization_response = summarize_with_ollama(input_data, user_prompt, temp, system_prompt)
+                    summarization_response = summarize_with_ollama(input_data, user_prompt, None, api_key, temp, system_prompt)
                 else:
                     raise ValueError(f"Unsupported API endpoint: {api_endpoint}")
         except Exception as e:

App_Function_Libraries/Gradio_UI/Import_Functionality.py

@@ -13,7 +13,6 @@ import zipfile
 #
 # External Imports
 import gradio as gr
-import pypandoc
 #
 # Local Imports
 from App_Function_Libraries.DB.DB_Manager import insert_prompt_to_db, load_preset_prompts, import_obsidian_note_to_db, \
@@ -29,6 +28,7 @@ logger = logging.getLogger()
 
 
 def import_data(file, title, author, keywords, custom_prompt, summary, auto_summarize, api_name, api_key):
+    logging.debug(f"Starting import_data with file: {file} / Title: {title} / Author: {author} / Keywords: {keywords}")
     if file is None:
         return "No file uploaded. Please upload a file."
 
@@ -73,7 +73,7 @@ def import_data(file, title, author, keywords, custom_prompt, summary, auto_summ
         segments = [{'Text': file_content}]
 
         # Process keywords
-        keyword_list = [kw.strip() for kw in keywords.split(',') if kw.strip()]
+        keyword_list = [kw.strip() for kw in keywords.split(',') if kw.strip()] if keywords else []
 
         # Handle summarization
         if auto_summarize and api_name and api_key:
@@ -81,30 +81,28 @@ def import_data(file, title, author, keywords, custom_prompt, summary, auto_summ
         elif not summary:
             summary = "No summary provided"
 
-        # Add to database
-        add_media_to_database(
-            url=file_name,  # Using filename as URL
-            info_dict=info_dict,
-            segments=segments,
-            summary=summary,
-            keywords=keyword_list,
-            custom_prompt_input=custom_prompt,
-            whisper_model="Imported",  # Indicating this was an imported file
-            media_type="document"
-        )
-
-        # Clean up the temporary file
-        os.unlink(temp_file.name)
-
-        return f"File '{file_name}' successfully imported with title '{title}' and author '{author}'."
+            # Add to database
+            result = add_media_to_database(
+                url=file_name,  # Using filename as URL
+                info_dict=info_dict,
+                segments=segments,
+                summary=summary,
+                keywords=keyword_list,
+                custom_prompt_input=custom_prompt,
+                whisper_model="Imported",  # Indicating this was an imported file
+                media_type="document",
+                overwrite=False  # Set this to True if you want to overwrite existing entries
+            )
+
+            # Clean up the temporary file
+            os.unlink(temp_file.name)
+
+            return f"File '{file_name}' import attempt complete. Database result: {result}"
     except Exception as e:
-        logging.error(f"Error importing file: {str(e)}")
+        logging.exception(f"Error importing file: {str(e)}")
         return f"Error importing file: {str(e)}"
 
 
-
-
-
 def process_obsidian_zip(zip_file):
     with tempfile.TemporaryDirectory() as temp_dir:
         try:
@@ -357,13 +355,11 @@ def create_import_obsidian_vault_tab():
         fn=import_vault,
         inputs=[vault_path_input, vault_zip_input],
         outputs=[import_status],
-        show_progress=True
     )
 
 
 def import_obsidian_vault(vault_path, progress=gr.Progress()):
     try:
-        from App_Function_Libraries.Gradio_UI.Export_Functionality import scan_obsidian_vault
         markdown_files = scan_obsidian_vault(vault_path)
         total_files = len(markdown_files)
         imported_files = 0

App_Function_Libraries/Gradio_UI/Search_Tab.py

@@ -73,7 +73,7 @@ def format_as_html(content, title):
     return f"""
     <div style="border: 1px solid #ddd; padding: 10px; margin-bottom: 10px;">
         <h3>{title}</h3>
-        <div style="max-height: 300px; overflow-y: auto;">
+        <div style="max-height: 700px; overflow-y: auto;">
             {formatted_content}
         </div>
     </div>

App_Function_Libraries/Gradio_UI/Video_transcription_tab.py

@@ -43,6 +43,7 @@ def create_video_transcription_tab():
                                        lines=5)
                 video_file_input = gr.File(label="Upload Video File (Optional)", file_types=["video/*"])
                 diarize_input = gr.Checkbox(label="Enable Speaker Diarization", value=False)
+                vad_checkbox = gr.Checkbox(label="Enable Voice-Audio-Detection(VAD)", value=True)
                 whisper_model_input = gr.Dropdown(choices=whisper_models, value="medium", label="Whisper Model")
 
                 with gr.Row():
@@ -185,7 +186,7 @@ def create_video_transcription_tab():
                 download_summary = gr.File(label="Download All Summaries as Text")
 
             @error_handler
-            def process_videos_with_error_handling(inputs, start_time, end_time, diarize, whisper_model,
+            def process_videos_with_error_handling(inputs, start_time, end_time, diarize, vad_use, whisper_model,
                                                    custom_prompt_checkbox, custom_prompt, chunking_options_checkbox,
                                                    chunk_method, max_chunk_size, chunk_overlap, use_adaptive_chunking,
                                                    use_multi_level_chunking, chunk_language, api_name,
@@ -301,7 +302,7 @@ def create_video_transcription_tab():
                                     input_item, 2, whisper_model,
                                     custom_prompt,
                                     start_seconds, api_name, api_key,
-                                    False, False, False, False, 0.01, None, keywords, None, diarize,
+                                    vad_use, False, False, False, 0.01, None, keywords, None, diarize,
                                     end_time=end_seconds,
                                     include_timestamps=timestamp_option,
                                     metadata=video_metadata,
@@ -425,7 +426,7 @@ def create_video_transcription_tab():
                         None
                     )
 
-            def process_videos_wrapper(url_input, video_file, start_time, end_time, diarize, whisper_model,
+            def process_videos_wrapper(url_input, video_file, start_time, end_time, diarize, vad_use, whisper_model,
                                        custom_prompt_checkbox, custom_prompt, chunking_options_checkbox,
                                        chunk_method, max_chunk_size, chunk_overlap, use_adaptive_chunking,
                                        use_multi_level_chunking, chunk_language, summarize_recursively, api_name,
@@ -460,7 +461,7 @@ def create_video_transcription_tab():
                         raise ValueError("No input provided. Please enter URLs or upload a video file.")
 
                     result = process_videos_with_error_handling(
-                        inputs, start_time, end_time, diarize, whisper_model,
+                        inputs, start_time, end_time, diarize, vad_use, whisper_model,
                         custom_prompt_checkbox, custom_prompt, chunking_options_checkbox,
                         chunk_method, max_chunk_size, chunk_overlap, use_adaptive_chunking,
                         use_multi_level_chunking, chunk_language, api_name,
@@ -507,6 +508,7 @@ def create_video_transcription_tab():
                 try:
                     logging.info(f"Starting process_url_metadata for URL: {input_item}")
                     # Create download path
+
                     download_path = create_download_directory("Video_Downloads")
                     logging.info(f"Download path created at: {download_path}")
 
@@ -743,15 +745,37 @@ def create_video_transcription_tab():
                 inputs=[confab_checkbox],
                 outputs=[confabulation_output]
             )
+
             process_button.click(
                 fn=process_videos_wrapper,
                 inputs=[
-                    url_input, video_file_input, start_time_input, end_time_input, diarize_input, whisper_model_input,
-                    custom_prompt_checkbox, custom_prompt_input, chunking_options_checkbox,
-                    chunk_method, max_chunk_size, chunk_overlap, use_adaptive_chunking,
-                    use_multi_level_chunking, chunk_language, summarize_recursively, api_name_input, api_key_input,
-                    keywords_input, use_cookies_input, cookies_input, batch_size_input,
-                    timestamp_option, keep_original_video, confab_checkbox, overwrite_checkbox
+                    url_input,
+                    video_file_input,
+                    start_time_input,
+                    end_time_input,
+                    diarize_input,
+                    vad_checkbox,
+                    whisper_model_input,
+                    custom_prompt_checkbox,
+                    custom_prompt_input,
+                    chunking_options_checkbox,
+                    chunk_method,
+                    max_chunk_size,
+                    chunk_overlap,
+                    use_adaptive_chunking,
+                    use_multi_level_chunking,
+                    chunk_language,
+                    summarize_recursively,
+                    api_name_input,
+                    api_key_input,
+                    keywords_input,
+                    use_cookies_input,
+                    cookies_input,
+                    batch_size_input,
+                    timestamp_option,
+                    keep_original_video,
+                    confab_checkbox,
+                    overwrite_checkbox
                 ],
                 outputs=[progress_output, error_output, results_output, download_transcription, download_summary, confabulation_output]
             )

App_Function_Libraries/Gradio_UI/View_DB_Items_tab.py

@@ -134,7 +134,7 @@ def format_as_html(content, title):
     return f"""
     <div style="border: 1px solid #ddd; padding: 10px; margin-bottom: 10px;">
         <h3>{title}</h3>
-        <div style="max-height: 300px; overflow-y: auto;">
+        <div style="max-height: 700px; overflow-y: auto;">
             {formatted_content}
         </div>
     </div>
@@ -200,7 +200,7 @@ def create_view_all_with_versions_tab():
             return f"""
             <div style="border: 1px solid #ddd; padding: 10px; margin-bottom: 10px;">
                 <h3>{title}</h3>
-                <div style="max-height: 300px; overflow-y: auto;">
+                <div style="max-height: 700px; overflow-y: auto;">
                     {formatted_content}
                 </div>
             </div>