Upload 22 files

App_Function_Libraries/Audio/Audio_Files.py

@@ -0,0 +1,692 @@
+# Audio_Files.py
+#########################################
+# Audio Processing Library
+# This library is used to download or load audio files from a local directory.
+#
+####
+#
+# Functions:
+#
+# download_audio_file(url, save_path)
+# process_audio(
+# process_audio_file(audio_url, audio_file, whisper_model="small.en", api_name=None, api_key=None)
+#
+#
+#########################################
+# Imports
+import json
+import logging
+import os
+import subprocess
+import tempfile
+import uuid
+from datetime import datetime
+from pathlib import Path
+
+import requests
+import yt_dlp
+
+from App_Function_Libraries.Audio.Audio_Transcription_Lib import speech_to_text
+from App_Function_Libraries.Chunk_Lib import improved_chunking_process
+#
+# Local Imports
+from App_Function_Libraries.DB.DB_Manager import add_media_to_database, add_media_with_keywords, \
+    check_media_and_whisper_model
+from App_Function_Libraries.Summarization.Summarization_General_Lib import save_transcription_and_summary, perform_transcription, \
+    perform_summarization
+from App_Function_Libraries.Utils.Utils import create_download_directory, save_segments_to_json, downloaded_files, \
+    sanitize_filename
+from App_Function_Libraries.Video_DL_Ingestion_Lib import extract_metadata
+
+#
+#######################################################################################################################
+# Function Definitions
+#
+
+MAX_FILE_SIZE = 500 * 1024 * 1024
+
+
+def download_audio_file(url, current_whisper_model="", use_cookies=False, cookies=None):
+    try:
+        # Check if media already exists in the database and compare whisper models
+        should_download, reason = check_media_and_whisper_model(
+            url=url,
+            current_whisper_model=current_whisper_model
+        )
+
+        if not should_download:
+            logging.info(f"Skipping audio download: {reason}")
+            return None
+
+        logging.info(f"Proceeding with audio download: {reason}")
+
+        # Set up the request headers
+        headers = {}
+        if use_cookies and cookies:
+            try:
+                cookie_dict = json.loads(cookies)
+                headers['Cookie'] = '; '.join([f'{k}={v}' for k, v in cookie_dict.items()])
+            except json.JSONDecodeError:
+                logging.warning("Invalid cookie format. Proceeding without cookies.")
+
+        # Make the request
+        response = requests.get(url, headers=headers, stream=True)
+        # Raise an exception for bad status codes
+        response.raise_for_status()
+
+        # Get the file size
+        file_size = int(response.headers.get('content-length', 0))
+        if file_size > 500 * 1024 * 1024:  # 500 MB limit
+            raise ValueError("File size exceeds the 500MB limit.")
+
+        # Generate a unique filename
+        file_name = f"audio_{uuid.uuid4().hex[:8]}.mp3"
+        save_path = os.path.join('downloads', file_name)
+
+        # Ensure the downloads directory exists
+        os.makedirs('downloads', exist_ok=True)
+
+
+        # Download the file
+        with open(save_path, 'wb') as f:
+            for chunk in response.iter_content(chunk_size=8192):
+                if chunk:
+                    f.write(chunk)
+
+        logging.info(f"Audio file downloaded successfully: {save_path}")
+        return save_path
+
+    except requests.RequestException as e:
+        logging.error(f"Error downloading audio file: {str(e)}")
+        raise
+    except ValueError as e:
+        logging.error(str(e))
+        raise
+    except Exception as e:
+        logging.error(f"Unexpected error downloading audio file: {str(e)}")
+        raise
+
+
+def process_audio(
+        audio_file_path,
+        num_speakers=2,
+        whisper_model="small.en",
+        custom_prompt_input=None,
+        offset=0,
+        api_name=None,
+        api_key=None,
+        vad_filter=False,
+        rolling_summarization=False,
+        detail_level=0.01,
+        keywords="default,no_keyword_set",
+        chunk_text_by_words=False,
+        max_words=0,
+        chunk_text_by_sentences=False,
+        max_sentences=0,
+        chunk_text_by_paragraphs=False,
+        max_paragraphs=0,
+        chunk_text_by_tokens=False,
+        max_tokens=0
+):
+    try:
+
+        # Perform transcription
+        audio_file_path, segments = perform_transcription(audio_file_path, offset, whisper_model, vad_filter)
+
+        if audio_file_path is None or segments is None:
+            logging.error("Process_Audio: Transcription failed or segments not available.")
+            return "Process_Audio: Transcription failed.", None, None, None, None, None
+
+        logging.debug(f"Process_Audio: Transcription audio_file: {audio_file_path}")
+        logging.debug(f"Process_Audio: Transcription segments: {segments}")
+
+        transcription_text = {'audio_file': audio_file_path, 'transcription': segments}
+        logging.debug(f"Process_Audio: Transcription text: {transcription_text}")
+
+        # Save segments to JSON
+        segments_json_path = save_segments_to_json(segments)
+
+        # Perform summarization
+        summary_text = None
+        if api_name:
+            if rolling_summarization is not None:
+                pass
+                # FIXME rolling summarization
+                # summary_text = rolling_summarize_function(
+                #     transcription_text,
+                #     detail=detail_level,
+                #     api_name=api_name,
+                #     api_key=api_key,
+                #     custom_prompt=custom_prompt_input,
+                #     chunk_by_words=chunk_text_by_words,
+                #     max_words=max_words,
+                #     chunk_by_sentences=chunk_text_by_sentences,
+                #     max_sentences=max_sentences,
+                #     chunk_by_paragraphs=chunk_text_by_paragraphs,
+                #     max_paragraphs=max_paragraphs,
+                #     chunk_by_tokens=chunk_text_by_tokens,
+                #     max_tokens=max_tokens
+                # )
+            else:
+                summary_text = perform_summarization(api_name, segments_json_path, custom_prompt_input, api_key)
+
+            if summary_text is None:
+                logging.error("Summary text is None. Check summarization function.")
+                summary_file_path = None
+        else:
+            summary_text = 'Summary not available'
+            summary_file_path = None
+
+        # Save transcription and summary
+        download_path = create_download_directory("Audio_Processing")
+        json_file_path, summary_file_path = save_transcription_and_summary(transcription_text, summary_text,
+                                                                           download_path)
+
+        # Update function call to add_media_to_database so that it properly applies the title, author and file type
+        # Add to database
+        add_media_to_database(None, {'title': 'Audio File', 'author': 'Unknown'}, segments, summary_text, keywords,
+                              custom_prompt_input, whisper_model)
+
+        return transcription_text, summary_text, json_file_path, summary_file_path, None, None
+
+    except Exception as e:
+        logging.error(f"Error in process_audio: {str(e)}")
+        return str(e), None, None, None, None, None
+
+
+def process_single_audio(audio_file_path, whisper_model, api_name, api_key, keep_original,custom_keywords, source,
+                         custom_prompt_input, chunk_method, max_chunk_size, chunk_overlap, use_adaptive_chunking,
+                         use_multi_level_chunking, chunk_language):
+    progress = []
+    transcription = ""
+    summary = ""
+
+    def update_progress(message):
+        progress.append(message)
+        return "\n".join(progress)
+
+    try:
+        # Check file size before processing
+        file_size = os.path.getsize(audio_file_path)
+        if file_size > MAX_FILE_SIZE:
+            update_progress(f"File size ({file_size / (1024 * 1024):.2f} MB) exceeds the maximum limit of {MAX_FILE_SIZE / (1024 * 1024):.2f} MB. Skipping this file.")
+            return "\n".join(progress), "", ""
+
+        # Perform transcription
+        update_progress("Starting transcription...")
+        segments = speech_to_text(audio_file_path, whisper_model=whisper_model)
+        transcription = " ".join([segment['Text'] for segment in segments])
+        update_progress("Audio transcribed successfully.")
+
+        # Perform summarization if API is provided
+        if api_name and api_key:
+            update_progress("Starting summarization...")
+            summary = perform_summarization(api_name, transcription, "Summarize the following audio transcript",
+                                            api_key)
+            update_progress("Audio summarized successfully.")
+        else:
+            summary = "No summary available"
+
+        # Prepare keywords
+        keywords = "audio,transcription"
+        if custom_keywords:
+            keywords += f",{custom_keywords}"
+
+        # Add to database
+        add_media_with_keywords(
+            url=source,
+            title=os.path.basename(audio_file_path),
+            media_type='audio',
+            content=transcription,
+            keywords=keywords,
+            prompt="Summarize the following audio transcript",
+            summary=summary,
+            transcription_model=whisper_model,
+            author="Unknown",
+            ingestion_date=None  # This will use the current date
+        )
+        update_progress("Audio file added to database successfully.")
+
+        if not keep_original and source != "Uploaded File":
+            os.remove(audio_file_path)
+            update_progress(f"Temporary file {audio_file_path} removed.")
+        elif keep_original and source != "Uploaded File":
+            update_progress(f"Original audio file kept at: {audio_file_path}")
+
+    except Exception as e:
+        update_progress(f"Error processing {source}: {str(e)}")
+        transcription = f"Error: {str(e)}"
+        summary = "No summary due to error"
+
+    return "\n".join(progress), transcription, summary
+
+
+def process_audio_files(audio_urls, audio_file, whisper_model, api_name, api_key, use_cookies, cookies, keep_original,
+                        custom_keywords, custom_prompt_input, chunk_method, max_chunk_size, chunk_overlap,
+                        use_adaptive_chunking, use_multi_level_chunking, chunk_language, diarize):
+    progress = []
+    temp_files = []
+    all_transcriptions = []
+    all_summaries = []
+
+    def update_progress(message):
+        progress.append(message)
+        return "\n".join(progress)
+
+    def cleanup_files():
+        for file in temp_files:
+            try:
+                if os.path.exists(file):
+                    os.remove(file)
+                    update_progress(f"Temporary file {file} removed.")
+            except Exception as e:
+                update_progress(f"Failed to remove temporary file {file}: {str(e)}")
+
+    def reencode_mp3(mp3_file_path):
+        try:
+            reencoded_mp3_path = mp3_file_path.replace(".mp3", "_reencoded.mp3")
+            subprocess.run([ffmpeg_cmd, '-i', mp3_file_path, '-codec:a', 'libmp3lame', reencoded_mp3_path], check=True)
+            update_progress(f"Re-encoded {mp3_file_path} to {reencoded_mp3_path}.")
+            return reencoded_mp3_path
+        except subprocess.CalledProcessError as e:
+            update_progress(f"Error re-encoding {mp3_file_path}: {str(e)}")
+            raise
+
+    def convert_mp3_to_wav(mp3_file_path):
+        try:
+            wav_file_path = mp3_file_path.replace(".mp3", ".wav")
+            subprocess.run([ffmpeg_cmd, '-i', mp3_file_path, wav_file_path], check=True)
+            update_progress(f"Converted {mp3_file_path} to {wav_file_path}.")
+            return wav_file_path
+        except subprocess.CalledProcessError as e:
+            update_progress(f"Error converting {mp3_file_path} to WAV: {str(e)}")
+            raise
+
+    try:
+        # Check and set the ffmpeg command
+        global ffmpeg_cmd
+        if os.name == "nt":
+            logging.debug("Running on Windows")
+            ffmpeg_cmd = os.path.join(os.getcwd(), "Bin", "ffmpeg.exe")
+        else:
+            ffmpeg_cmd = 'ffmpeg'  # Assume 'ffmpeg' is in PATH for non-Windows systems
+
+        # Ensure ffmpeg is accessible
+        if not os.path.exists(ffmpeg_cmd) and os.name == "nt":
+            raise FileNotFoundError(f"ffmpeg executable not found at path: {ffmpeg_cmd}")
+
+        # Define chunk options early to avoid undefined errors
+        chunk_options = {
+            'method': chunk_method,
+            'max_size': max_chunk_size,
+            'overlap': chunk_overlap,
+            'adaptive': use_adaptive_chunking,
+            'multi_level': use_multi_level_chunking,
+            'language': chunk_language
+        }
+
+        # Process multiple URLs
+        urls = [url.strip() for url in audio_urls.split('\n') if url.strip()]
+
+        for i, url in enumerate(urls):
+            update_progress(f"Processing URL {i + 1}/{len(urls)}: {url}")
+
+            # Download and process audio file
+            audio_file_path = download_audio_file(url, use_cookies, cookies)
+            if not os.path.exists(audio_file_path):
+                update_progress(f"Downloaded file not found: {audio_file_path}")
+                continue
+
+            temp_files.append(audio_file_path)
+            update_progress("Audio file downloaded successfully.")
+
+            # Re-encode MP3 to fix potential issues
+            reencoded_mp3_path = reencode_mp3(audio_file_path)
+            if not os.path.exists(reencoded_mp3_path):
+                update_progress(f"Re-encoded file not found: {reencoded_mp3_path}")
+                continue
+
+            temp_files.append(reencoded_mp3_path)
+
+            # Convert re-encoded MP3 to WAV
+            wav_file_path = convert_mp3_to_wav(reencoded_mp3_path)
+            if not os.path.exists(wav_file_path):
+                update_progress(f"Converted WAV file not found: {wav_file_path}")
+                continue
+
+            temp_files.append(wav_file_path)
+
+            # Initialize transcription
+            transcription = ""
+
+            # Transcribe audio
+            if diarize:
+                segments = speech_to_text(wav_file_path, whisper_model=whisper_model, diarize=True)
+            else:
+                segments = speech_to_text(wav_file_path, whisper_model=whisper_model)
+
+            # Handle segments nested under 'segments' key
+            if isinstance(segments, dict) and 'segments' in segments:
+                segments = segments['segments']
+
+            if isinstance(segments, list):
+                transcription = " ".join([segment.get('Text', '') for segment in segments])
+                update_progress("Audio transcribed successfully.")
+            else:
+                update_progress("Unexpected segments format received from speech_to_text.")
+                logging.error(f"Unexpected segments format: {segments}")
+                continue
+
+            if not transcription.strip():
+                update_progress("Transcription is empty.")
+            else:
+                # Apply chunking
+                chunked_text = improved_chunking_process(transcription, chunk_options)
+
+                # Summarize
+                if api_name:
+                    try:
+                        summary = perform_summarization(api_name, chunked_text, custom_prompt_input, api_key)
+                        update_progress("Audio summarized successfully.")
+                    except Exception as e:
+                        logging.error(f"Error during summarization: {str(e)}")
+                        summary = "Summary generation failed"
+                else:
+                    summary = "No summary available (API not provided)"
+
+                all_transcriptions.append(transcription)
+                all_summaries.append(summary)
+
+                # Add to database
+                add_media_with_keywords(
+                    url=url,
+                    title=os.path.basename(wav_file_path),
+                    media_type='audio',
+                    content=transcription,
+                    keywords=custom_keywords,
+                    prompt=custom_prompt_input,
+                    summary=summary,
+                    transcription_model=whisper_model,
+                    author="Unknown",
+                    ingestion_date=datetime.now().strftime('%Y-%m-%d')
+                )
+                update_progress("Audio file processed and added to database.")
+
+        # Process uploaded file if provided
+        if audio_file:
+            if os.path.getsize(audio_file.name) > MAX_FILE_SIZE:
+                update_progress(
+                    f"Uploaded file size exceeds the maximum limit of {MAX_FILE_SIZE / (1024 * 1024):.2f}MB. Skipping this file.")
+            else:
+                # Re-encode MP3 to fix potential issues
+                reencoded_mp3_path = reencode_mp3(audio_file.name)
+                if not os.path.exists(reencoded_mp3_path):
+                    update_progress(f"Re-encoded file not found: {reencoded_mp3_path}")
+                    return update_progress("Processing failed: Re-encoded file not found"), "", ""
+
+                temp_files.append(reencoded_mp3_path)
+
+                # Convert re-encoded MP3 to WAV
+                wav_file_path = convert_mp3_to_wav(reencoded_mp3_path)
+                if not os.path.exists(wav_file_path):
+                    update_progress(f"Converted WAV file not found: {wav_file_path}")
+                    return update_progress("Processing failed: Converted WAV file not found"), "", ""
+
+                temp_files.append(wav_file_path)
+
+                # Initialize transcription
+                transcription = ""
+
+                if diarize:
+                    segments = speech_to_text(wav_file_path, whisper_model=whisper_model, diarize=True)
+                else:
+                    segments = speech_to_text(wav_file_path, whisper_model=whisper_model)
+
+                # Handle segments nested under 'segments' key
+                if isinstance(segments, dict) and 'segments' in segments:
+                    segments = segments['segments']
+
+                if isinstance(segments, list):
+                    transcription = " ".join([segment.get('Text', '') for segment in segments])
+                else:
+                    update_progress("Unexpected segments format received from speech_to_text.")
+                    logging.error(f"Unexpected segments format: {segments}")
+
+                chunked_text = improved_chunking_process(transcription, chunk_options)
+
+                if api_name and api_key:
+                    try:
+                        summary = perform_summarization(api_name, chunked_text, custom_prompt_input, api_key)
+                        update_progress("Audio summarized successfully.")
+                    except Exception as e:
+                        logging.error(f"Error during summarization: {str(e)}")
+                        summary = "Summary generation failed"
+                else:
+                    summary = "No summary available (API not provided)"
+
+                all_transcriptions.append(transcription)
+                all_summaries.append(summary)
+
+                add_media_with_keywords(
+                    url="Uploaded File",
+                    title=os.path.basename(wav_file_path),
+                    media_type='audio',
+                    content=transcription,
+                    keywords=custom_keywords,
+                    prompt=custom_prompt_input,
+                    summary=summary,
+                    transcription_model=whisper_model,
+                    author="Unknown",
+                    ingestion_date=datetime.now().strftime('%Y-%m-%d')
+                )
+                update_progress("Uploaded file processed and added to database.")
+
+        # Final cleanup
+        if not keep_original:
+            cleanup_files()
+
+        final_progress = update_progress("All processing complete.")
+        final_transcriptions = "\n\n".join(all_transcriptions)
+        final_summaries = "\n\n".join(all_summaries)
+
+        return final_progress, final_transcriptions, final_summaries
+
+    except Exception as e:
+        logging.error(f"Error processing audio files: {str(e)}")
+        cleanup_files()
+        return update_progress(f"Processing failed: {str(e)}"), "", ""
+
+
+def download_youtube_audio(url):
+    try:
+        # Determine ffmpeg path based on the operating system.
+        ffmpeg_path = './Bin/ffmpeg.exe' if os.name == 'nt' else 'ffmpeg'
+
+        # Create a temporary directory
+        with tempfile.TemporaryDirectory() as temp_dir:
+            # Extract information about the video
+            with yt_dlp.YoutubeDL({'quiet': True}) as ydl:
+                info_dict = ydl.extract_info(url, download=False)
+                sanitized_title = sanitize_filename(info_dict['title'])
+
+            # Setup the temporary filenames
+            temp_video_path = Path(temp_dir) / f"{sanitized_title}_temp.mp4"
+            temp_audio_path = Path(temp_dir) / f"{sanitized_title}.mp3"
+
+            # Initialize yt-dlp with options for downloading
+            ydl_opts = {
+                'format': 'bestaudio[ext=m4a]/best[height<=480]',  # Prefer best audio, or video up to 480p
+                'ffmpeg_location': ffmpeg_path,
+                'outtmpl': str(temp_video_path),
+                'noplaylist': True,
+                'quiet': True
+            }
+
+            # Execute yt-dlp to download the video/audio
+            with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+                ydl.download([url])
+
+            # Check if the file exists
+            if not temp_video_path.exists():
+                raise FileNotFoundError(f"Expected file was not found: {temp_video_path}")
+
+            # Use ffmpeg to extract audio
+            ffmpeg_command = [
+                ffmpeg_path,
+                '-i', str(temp_video_path),
+                '-vn',  # No video
+                '-acodec', 'libmp3lame',
+                '-b:a', '192k',
+                str(temp_audio_path)
+            ]
+            subprocess.run(ffmpeg_command, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
+
+            # Check if the audio file was created
+            if not temp_audio_path.exists():
+                raise FileNotFoundError(f"Expected audio file was not found: {temp_audio_path}")
+
+            # Create a persistent directory for the download if it doesn't exist
+            persistent_dir = Path("downloads")
+            persistent_dir.mkdir(exist_ok=True)
+
+            # Move the file from the temporary directory to the persistent directory
+            persistent_file_path = persistent_dir / f"{sanitized_title}.mp3"
+            os.replace(str(temp_audio_path), str(persistent_file_path))
+
+            # Add the file to the list of downloaded files
+            downloaded_files.append(str(persistent_file_path))
+
+            return str(persistent_file_path), f"Audio downloaded successfully: {sanitized_title}.mp3"
+    except Exception as e:
+        return None, f"Error downloading audio: {str(e)}"
+
+
+def process_podcast(url, title, author, keywords, custom_prompt, api_name, api_key, whisper_model,
+                    keep_original=False, enable_diarization=False, use_cookies=False, cookies=None,
+                    chunk_method=None, max_chunk_size=300, chunk_overlap=0, use_adaptive_chunking=False,
+                    use_multi_level_chunking=False, chunk_language='english'):
+    progress = []
+    error_message = ""
+    temp_files = []
+
+    def update_progress(message):
+        progress.append(message)
+        return "\n".join(progress)
+
+    def cleanup_files():
+        if not keep_original:
+            for file in temp_files:
+                try:
+                    if os.path.exists(file):
+                        os.remove(file)
+                        update_progress(f"Temporary file {file} removed.")
+                except Exception as e:
+                    update_progress(f"Failed to remove temporary file {file}: {str(e)}")
+
+    try:
+        # Download podcast
+        audio_file = download_audio_file(url, use_cookies, cookies)
+        temp_files.append(audio_file)
+        update_progress("Podcast downloaded successfully.")
+
+        # Extract metadata
+        metadata = extract_metadata(url)
+        title = title or metadata.get('title', 'Unknown Podcast')
+        author = author or metadata.get('uploader', 'Unknown Author')
+
+        # Format metadata for storage
+        metadata_text = f"""
+Metadata:
+Title: {title}
+Author: {author}
+Series: {metadata.get('series', 'N/A')}
+Episode: {metadata.get('episode', 'N/A')}
+Season: {metadata.get('season', 'N/A')}
+Upload Date: {metadata.get('upload_date', 'N/A')}
+Duration: {metadata.get('duration', 'N/A')} seconds
+Description: {metadata.get('description', 'N/A')}
+"""
+
+        # Update keywords
+        new_keywords = []
+        if metadata.get('series'):
+            new_keywords.append(f"series:{metadata['series']}")
+        if metadata.get('episode'):
+            new_keywords.append(f"episode:{metadata['episode']}")
+        if metadata.get('season'):
+            new_keywords.append(f"season:{metadata['season']}")
+
+        keywords = f"{keywords},{','.join(new_keywords)}" if keywords else ','.join(new_keywords)
+
+        update_progress(f"Metadata extracted - Title: {title}, Author: {author}, Keywords: {keywords}")
+
+        # Transcribe the podcast
+        try:
+            if enable_diarization:
+                segments = speech_to_text(audio_file, whisper_model=whisper_model, diarize=True)
+            else:
+                segments = speech_to_text(audio_file, whisper_model=whisper_model)
+            transcription = " ".join([segment['Text'] for segment in segments])
+            update_progress("Podcast transcribed successfully.")
+        except Exception as e:
+            error_message = f"Transcription failed: {str(e)}"
+            raise
+
+        # Apply chunking
+        chunk_options = {
+            'method': chunk_method,
+            'max_size': max_chunk_size,
+            'overlap': chunk_overlap,
+            'adaptive': use_adaptive_chunking,
+            'multi_level': use_multi_level_chunking,
+            'language': chunk_language
+        }
+        chunked_text = improved_chunking_process(transcription, chunk_options)
+
+        # Combine metadata and transcription
+        full_content = metadata_text + "\n\nTranscription:\n" + transcription
+
+        # Summarize if API is provided
+        summary = None
+        if api_name and api_key:
+            try:
+                summary = perform_summarization(api_name, chunked_text, custom_prompt, api_key)
+                update_progress("Podcast summarized successfully.")
+            except Exception as e:
+                error_message = f"Summarization failed: {str(e)}"
+                raise
+
+        # Add to database
+        try:
+            add_media_with_keywords(
+                url=url,
+                title=title,
+                media_type='podcast',
+                content=full_content,
+                keywords=keywords,
+                prompt=custom_prompt,
+                summary=summary or "No summary available",
+                transcription_model=whisper_model,
+                author=author,
+                ingestion_date=datetime.now().strftime('%Y-%m-%d')
+            )
+            update_progress("Podcast added to database successfully.")
+        except Exception as e:
+            error_message = f"Error adding podcast to database: {str(e)}"
+            raise
+
+        # Cleanup
+        cleanup_files()
+
+        return (update_progress("Processing complete."), full_content, summary or "No summary generated.",
+                title, author, keywords, error_message)
+
+    except Exception as e:
+        logging.error(f"Error processing podcast: {str(e)}")
+        cleanup_files()
+        return update_progress(f"Processing failed: {str(e)}"), "", "", "", "", "", str(e)
+
+
+#
+#
+#######################################################################################################################

App_Function_Libraries/Audio/Audio_Transcription_Lib.py

@@ -0,0 +1,329 @@
+# Audio_Transcription_Lib.py
+#########################################
+# Transcription Library
+# This library is used to perform transcription of audio files.
+# Currently, uses faster_whisper for transcription.
+#
+####################
+# Function List
+#
+# 1. convert_to_wav(video_file_path, offset=0, overwrite=False)
+# 2. speech_to_text(audio_file_path, selected_source_lang='en', whisper_model='small.en', vad_filter=False)
+#
+####################
+#
+# Import necessary libraries to run solo for testing
+import gc
+import json
+import logging
+import os
+import queue
+import sys
+import subprocess
+import tempfile
+import threading
+import time
+# DEBUG Imports
+#from memory_profiler import profile
+import pyaudio
+from faster_whisper import WhisperModel as OriginalWhisperModel
+from typing import Optional, Union, List, Dict, Any
+#
+# Import Local
+from App_Function_Libraries.Utils.Utils import load_comprehensive_config
+#
+#######################################################################################################################
+# Function Definitions
+#
+
+# Convert video .m4a into .wav using ffmpeg
+#   ffmpeg -i "example.mp4" -ar 16000 -ac 1 -c:a pcm_s16le "output.wav"
+#       https://www.gyan.dev/ffmpeg/builds/
+#
+
+
+whisper_model_instance = None
+config = load_comprehensive_config()
+processing_choice = config.get('Processing', 'processing_choice', fallback='cpu')
+
+
+
+class WhisperModel(OriginalWhisperModel):
+    tldw_dir = os.path.dirname(os.path.dirname(__file__))
+    default_download_root = os.path.join(tldw_dir, 'App_Function_Libraries', 'models', 'Whisper')
+
+    valid_model_sizes = [
+        "tiny.en", "tiny", "base.en", "base", "small.en", "small", "medium.en", "medium",
+        "large-v1", "large-v2", "large-v3", "large", "distil-large-v2", "distil-medium.en",
+        "distil-small.en", "distil-large-v3"
+    ]
+
+    def __init__(
+        self,
+        model_size_or_path: str,
+        device: str = "auto",
+        device_index: Union[int, List[int]] = 0,
+        compute_type: str = "default",
+        cpu_threads: int = 16,
+        num_workers: int = 1,
+        download_root: Optional[str] = None,
+        local_files_only: bool = False,
+        files: Optional[Dict[str, Any]] = None,
+        **model_kwargs: Any
+    ):
+        if download_root is None:
+            download_root = self.default_download_root
+
+        os.makedirs(download_root, exist_ok=True)
+
+        # FIXME - validate....
+        # Also write an integration test...
+        # Check if model_size_or_path is a valid model size
+        if model_size_or_path in self.valid_model_sizes:
+            # It's a model size, so we'll use the download_root
+            model_path = os.path.join(download_root, model_size_or_path)
+            if not os.path.isdir(model_path):
+                # If it doesn't exist, we'll let the parent class download it
+                model_size_or_path = model_size_or_path  # Keep the original model size
+            else:
+                # If it exists, use the full path
+                model_size_or_path = model_path
+        else:
+            # It's not a valid model size, so assume it's a path
+            model_size_or_path = os.path.abspath(model_size_or_path)
+
+        super().__init__(
+            model_size_or_path,
+            device=device,
+            device_index=device_index,
+            compute_type=compute_type,
+            cpu_threads=cpu_threads,
+            num_workers=num_workers,
+            download_root=download_root,
+            local_files_only=local_files_only,
+# Maybe? idk, FIXME
+#            files=files,
+#            **model_kwargs
+        )
+
+def get_whisper_model(model_name, device):
+    global whisper_model_instance
+    if whisper_model_instance is None:
+        logging.info(f"Initializing new WhisperModel with size {model_name} on device {device}")
+        whisper_model_instance = WhisperModel(model_name, device=device)
+    return whisper_model_instance
+
+# # FIXME: This is a temporary solution.
+# # This doesn't clear older models, which means potentially a lot of memory is being used...
+# def get_whisper_model(model_name, device):
+#     global whisper_model_instance
+#     if whisper_model_instance is None:
+#         from faster_whisper import WhisperModel
+#         logging.info(f"Initializing new WhisperModel with size {model_name} on device {device}")
+#
+#         # FIXME - add logic to detect if the model is already downloaded
+#         # want to first check if the model is already downloaded
+#         # if not, download it using the existing logic in 'WhisperModel'
+#         # https://github.com/SYSTRAN/faster-whisper/blob/d57c5b40b06e59ec44240d93485a95799548af50/faster_whisper/transcribe.py#L584
+#         # Designated path should be `tldw/App_Function_Libraries/models/Whisper/`
+#         WhisperModel.download_root = os.path.join(os.path.dirname(__file__), 'models', 'Whisper')
+#         os.makedirs(WhisperModel.download_root, exist_ok=True)
+#         whisper_model_instance = WhisperModel(model_name, device=device)
+#     return whisper_model_instance
+
+
+# os.system(r'.\Bin\ffmpeg.exe -ss 00:00:00 -i "{video_file_path}" -ar 16000 -ac 1 -c:a pcm_s16le "{out_path}"')
+#DEBUG
+#@profile
+def convert_to_wav(video_file_path, offset=0, overwrite=False):
+    out_path = os.path.splitext(video_file_path)[0] + ".wav"
+
+    if os.path.exists(out_path) and not overwrite:
+        print(f"File '{out_path}' already exists. Skipping conversion.")
+        logging.info(f"Skipping conversion as file already exists: {out_path}")
+        return out_path
+    print("Starting conversion process of .m4a to .WAV")
+    out_path = os.path.splitext(video_file_path)[0] + ".wav"
+
+    try:
+        if os.name == "nt":
+            logging.debug("ffmpeg being ran on windows")
+
+            if sys.platform.startswith('win'):
+                ffmpeg_cmd = ".\\Bin\\ffmpeg.exe"
+                logging.debug(f"ffmpeg_cmd: {ffmpeg_cmd}")
+            else:
+                ffmpeg_cmd = 'ffmpeg'  # Assume 'ffmpeg' is in PATH for non-Windows systems
+
+            command = [
+                ffmpeg_cmd,  # Assuming the working directory is correctly set where .\Bin exists
+                "-ss", "00:00:00",  # Start at the beginning of the video
+                "-i", video_file_path,
+                "-ar", "16000",  # Audio sample rate
+                "-ac", "1",  # Number of audio channels
+                "-c:a", "pcm_s16le",  # Audio codec
+                out_path
+            ]
+            try:
+                # Redirect stdin from null device to prevent ffmpeg from waiting for input
+                with open(os.devnull, 'rb') as null_file:
+                    result = subprocess.run(command, stdin=null_file, text=True, capture_output=True)
+                if result.returncode == 0:
+                    logging.info("FFmpeg executed successfully")
+                    logging.debug("FFmpeg output: %s", result.stdout)
+                else:
+                    logging.error("Error in running FFmpeg")
+                    logging.error("FFmpeg stderr: %s", result.stderr)
+                    raise RuntimeError(f"FFmpeg error: {result.stderr}")
+            except Exception as e:
+                logging.error("Error occurred - ffmpeg doesn't like windows")
+                raise RuntimeError("ffmpeg failed")
+        elif os.name == "posix":
+            os.system(f'ffmpeg -ss 00:00:00 -i "{video_file_path}" -ar 16000 -ac 1 -c:a pcm_s16le "{out_path}"')
+        else:
+            raise RuntimeError("Unsupported operating system")
+        logging.info("Conversion to WAV completed: %s", out_path)
+    except subprocess.CalledProcessError as e:
+        logging.error("Error executing FFmpeg command: %s", str(e))
+        raise RuntimeError("Error converting video file to WAV")
+    except Exception as e:
+        logging.error("speech-to-text: Error transcribing audio: %s", str(e))
+        return {"error": str(e)}
+    gc.collect()
+    return out_path
+
+
+# Transcribe .wav into .segments.json
+#DEBUG
+#@profile
+def speech_to_text(audio_file_path, selected_source_lang='en', whisper_model='medium.en', vad_filter=False, diarize=False):
+    global whisper_model_instance, processing_choice
+    logging.info('speech-to-text: Loading faster_whisper model: %s', whisper_model)
+
+    time_start = time.time()
+    if audio_file_path is None:
+        raise ValueError("speech-to-text: No audio file provided")
+    logging.info("speech-to-text: Audio file path: %s", audio_file_path)
+
+    try:
+        _, file_ending = os.path.splitext(audio_file_path)
+        out_file = audio_file_path.replace(file_ending, ".segments.json")
+        prettified_out_file = audio_file_path.replace(file_ending, ".segments_pretty.json")
+        if os.path.exists(out_file):
+            logging.info("speech-to-text: Segments file already exists: %s", out_file)
+            with open(out_file) as f:
+                global segments
+                segments = json.load(f)
+            return segments
+
+        logging.info('speech-to-text: Starting transcription...')
+        options = dict(language=selected_source_lang, beam_size=5, best_of=5, vad_filter=vad_filter)
+        transcribe_options = dict(task="transcribe", **options)
+        # use function and config at top of file
+        logging.debug("speech-to-text: Using whisper model: %s", whisper_model)
+        whisper_model_instance = get_whisper_model(whisper_model, processing_choice)
+        segments_raw, info = whisper_model_instance.transcribe(audio_file_path, **transcribe_options)
+
+        segments = []
+        for segment_chunk in segments_raw:
+            chunk = {
+                "Time_Start": segment_chunk.start,
+                "Time_End": segment_chunk.end,
+                "Text": segment_chunk.text
+            }
+            logging.debug("Segment: %s", chunk)
+            segments.append(chunk)
+            # Print to verify its working
+            print(f"{segment_chunk.start:.2f}s - {segment_chunk.end:.2f}s | {segment_chunk.text}")
+
+            # Log it as well.
+            logging.debug(
+                f"Transcribed Segment: {segment_chunk.start:.2f}s - {segment_chunk.end:.2f}s | {segment_chunk.text}")
+
+        if segments:
+            segments[0]["Text"] = f"This text was transcribed using whisper model: {whisper_model}\n\n" + segments[0]["Text"]
+
+        if not segments:
+            raise RuntimeError("No transcription produced. The audio file may be invalid or empty.")
+        logging.info("speech-to-text: Transcription completed in %.2f seconds", time.time() - time_start)
+
+        # Save the segments to a JSON file - prettified and non-prettified
+        # FIXME so this is an optional flag to save either the prettified json file or the normal one
+        save_json = True
+        if save_json:
+            logging.info("speech-to-text: Saving segments to JSON file")
+            output_data = {'segments': segments}
+
+            logging.info("speech-to-text: Saving prettified JSON to %s", prettified_out_file)
+            with open(prettified_out_file, 'w') as f:
+                json.dump(output_data, f, indent=2)
+
+            logging.info("speech-to-text: Saving JSON to %s", out_file)
+            with open(out_file, 'w') as f:
+                json.dump(output_data, f)
+
+        logging.debug(f"speech-to-text: returning {segments[:500]}")
+        gc.collect()
+        return segments
+
+    except Exception as e:
+        logging.error("speech-to-text: Error transcribing audio: %s", str(e))
+        raise RuntimeError("speech-to-text: Error transcribing audio")
+
+
+def record_audio(duration, sample_rate=16000, chunk_size=1024):
+    p = pyaudio.PyAudio()
+    stream = p.open(format=pyaudio.paInt16,
+                    channels=1,
+                    rate=sample_rate,
+                    input=True,
+                    frames_per_buffer=chunk_size)
+
+    print("Recording...")
+    frames = []
+    stop_recording = threading.Event()
+    audio_queue = queue.Queue()
+
+    def audio_callback():
+        for _ in range(0, int(sample_rate / chunk_size * duration)):
+            if stop_recording.is_set():
+                break
+            data = stream.read(chunk_size)
+            audio_queue.put(data)
+
+    audio_thread = threading.Thread(target=audio_callback)
+    audio_thread.start()
+
+    return p, stream, audio_queue, stop_recording, audio_thread
+
+
+def stop_recording(p, stream, audio_queue, stop_recording_event, audio_thread):
+    stop_recording_event.set()
+    audio_thread.join()
+
+    frames = []
+    while not audio_queue.empty():
+        frames.append(audio_queue.get())
+
+    print("Recording finished.")
+
+    stream.stop_stream()
+    stream.close()
+    p.terminate()
+
+    return b''.join(frames)
+
+def save_audio_temp(audio_data, sample_rate=16000):
+    with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as temp_file:
+        import wave
+        wf = wave.open(temp_file.name, 'wb')
+        wf.setnchannels(1)
+        wf.setsampwidth(2)
+        wf.setframerate(sample_rate)
+        wf.writeframes(audio_data)
+        wf.close()
+        return temp_file.name
+
+#
+#
+#######################################################################################################################

App_Function_Libraries/Audio/Diarization_Lib.py

@@ -0,0 +1,275 @@
+# Diarization_Lib.py
+#########################################
+# Diarization Library
+# This library is used to perform diarization of audio files.
+# Currently, uses FIXME for transcription.
+#
+####################
+####################
+# Function List
+#
+# 1. speaker_diarize(video_file_path, segments, embedding_model = "pyannote/embedding", embedding_size=512, num_speakers=0)
+#
+####################
+# Import necessary libraries
+import logging
+from pathlib import Path
+from typing import Dict, List, Any
+
+#
+# Import Local Libraries
+from App_Function_Libraries.Audio.Audio_Transcription_Lib import speech_to_text
+#
+# Import 3rd Party Libraries
+from pyannote.audio.pipelines.speaker_diarization import SpeakerDiarization
+import yaml
+#
+#######################################################################################################################
+# Function Definitions
+#
+
+def load_pipeline_from_pretrained(path_to_config: str | Path) -> SpeakerDiarization:
+    path_to_config = Path(path_to_config).resolve()
+    logging.debug(f"Loading pyannote pipeline from {path_to_config}...")
+
+    if not path_to_config.exists():
+        raise FileNotFoundError(f"Config file not found: {path_to_config}")
+
+    # Load the YAML configuration
+    with open(path_to_config, 'r') as config_file:
+        config = yaml.safe_load(config_file)
+
+    # Debug: print the entire config
+    logging.debug(f"Loaded config: {config}")
+
+    # Create the SpeakerDiarization pipeline
+    try:
+        pipeline = SpeakerDiarization(
+            segmentation=config['pipeline']['params']['segmentation'],
+            embedding=config['pipeline']['params']['embedding'],
+            clustering=config['pipeline']['params']['clustering'],
+        )
+    except KeyError as e:
+        logging.error(f"Error accessing config key: {e}")
+        raise
+
+    # Set other parameters
+    try:
+        pipeline_params = {
+            "segmentation": {},
+            "clustering": {},
+        }
+
+        if 'params' in config and 'segmentation' in config['params']:
+            if 'min_duration_off' in config['params']['segmentation']:
+                pipeline_params["segmentation"]["min_duration_off"] = config['params']['segmentation']['min_duration_off']
+
+        if 'params' in config and 'clustering' in config['params']:
+            if 'method' in config['params']['clustering']:
+                pipeline_params["clustering"]["method"] = config['params']['clustering']['method']
+            if 'min_cluster_size' in config['params']['clustering']:
+                pipeline_params["clustering"]["min_cluster_size"] = config['params']['clustering']['min_cluster_size']
+            if 'threshold' in config['params']['clustering']:
+                pipeline_params["clustering"]["threshold"] = config['params']['clustering']['threshold']
+
+        if 'pipeline' in config and 'params' in config['pipeline']:
+            if 'embedding_batch_size' in config['pipeline']['params']:
+                pipeline_params["embedding_batch_size"] = config['pipeline']['params']['embedding_batch_size']
+            if 'embedding_exclude_overlap' in config['pipeline']['params']:
+                pipeline_params["embedding_exclude_overlap"] = config['pipeline']['params']['embedding_exclude_overlap']
+            if 'segmentation_batch_size' in config['pipeline']['params']:
+                pipeline_params["segmentation_batch_size"] = config['pipeline']['params']['segmentation_batch_size']
+
+        logging.debug(f"Pipeline params: {pipeline_params}")
+        pipeline.instantiate(pipeline_params)
+    except KeyError as e:
+        logging.error(f"Error accessing config key: {e}")
+        raise
+    except Exception as e:
+        logging.error(f"Error instantiating pipeline: {e}")
+        raise
+
+    return pipeline
+
+
+def audio_diarization(audio_file_path: str) -> list:
+    logging.info('audio-diarization: Loading pyannote pipeline')
+
+    base_dir = Path(__file__).parent.resolve()
+    config_path = base_dir / 'models' / 'pyannote_diarization_config.yaml'
+    logging.info(f"audio-diarization: Loading pipeline from {config_path}")
+
+    try:
+        pipeline = load_pipeline_from_pretrained(config_path)
+    except Exception as e:
+        logging.error(f"Failed to load pipeline: {str(e)}")
+        raise
+
+    logging.info(f"audio-diarization: Audio file path: {audio_file_path}")
+
+    try:
+        logging.info('audio-diarization: Starting diarization...')
+        diarization_result = pipeline(audio_file_path)
+
+        segments = []
+        for turn, _, speaker in diarization_result.itertracks(yield_label=True):
+            segment = {
+                "start": turn.start,
+                "end": turn.end,
+                "speaker": speaker
+            }
+            logging.debug(f"Segment: {segment}")
+            segments.append(segment)
+        logging.info("audio-diarization: Diarization completed with pyannote")
+
+        return segments
+
+    except Exception as e:
+        logging.error(f"audio-diarization: Error performing diarization: {str(e)}")
+        raise RuntimeError("audio-diarization: Error performing diarization") from e
+
+
+# Old
+# def audio_diarization(audio_file_path):
+#     logging.info('audio-diarization: Loading pyannote pipeline')
+#
+#     #config file loading
+#     current_dir = os.path.dirname(os.path.abspath(__file__))
+#     # Construct the path to the config file
+#     config_path = os.path.join(current_dir, 'Config_Files', 'config.txt')
+#     # Read the config file
+#     config = configparser.ConfigParser()
+#     config.read(config_path)
+#     processing_choice = config.get('Processing', 'processing_choice', fallback='cpu')
+#
+#     base_dir = Path(__file__).parent.resolve()
+#     config_path = base_dir / 'models' / 'config.yaml'
+#     pipeline = load_pipeline_from_pretrained(config_path)
+#
+#     time_start = time.time()
+#     if audio_file_path is None:
+#         raise ValueError("audio-diarization: No audio file provided")
+#     logging.info("audio-diarization: Audio file path: %s", audio_file_path)
+#
+#     try:
+#         _, file_ending = os.path.splitext(audio_file_path)
+#         out_file = audio_file_path.replace(file_ending, ".diarization.json")
+#         prettified_out_file = audio_file_path.replace(file_ending, ".diarization_pretty.json")
+#         if os.path.exists(out_file):
+#             logging.info("audio-diarization: Diarization file already exists: %s", out_file)
+#             with open(out_file) as f:
+#                 global diarization_result
+#                 diarization_result = json.load(f)
+#             return diarization_result
+#
+#         logging.info('audio-diarization: Starting diarization...')
+#         diarization_result = pipeline(audio_file_path)
+#
+#         segments = []
+#         for turn, _, speaker in diarization_result.itertracks(yield_label=True):
+#             chunk = {
+#                 "Time_Start": turn.start,
+#                 "Time_End": turn.end,
+#                 "Speaker": speaker
+#             }
+#             logging.debug("Segment: %s", chunk)
+#             segments.append(chunk)
+#         logging.info("audio-diarization: Diarization completed with pyannote")
+#
+#         output_data = {'segments': segments}
+#
+#         logging.info("audio-diarization: Saving prettified JSON to %s", prettified_out_file)
+#         with open(prettified_out_file, 'w') as f:
+#             json.dump(output_data, f, indent=2)
+#
+#         logging.info("audio-diarization: Saving JSON to %s", out_file)
+#         with open(out_file, 'w') as f:
+#             json.dump(output_data, f)
+#
+#     except Exception as e:
+#         logging.error("audio-diarization: Error performing diarization: %s", str(e))
+#         raise RuntimeError("audio-diarization: Error performing diarization")
+#     return segments
+
+def combine_transcription_and_diarization(audio_file_path: str) -> List[Dict[str, Any]]:
+    logging.info('combine-transcription-and-diarization: Starting transcription and diarization...')
+
+    try:
+        logging.info('Performing speech-to-text...')
+        transcription_result = speech_to_text(audio_file_path)
+        logging.info(f"Transcription result type: {type(transcription_result)}")
+        logging.info(f"Transcription result: {transcription_result[:3] if isinstance(transcription_result, list) and len(transcription_result) > 3 else transcription_result}")
+
+        logging.info('Performing audio diarization...')
+        diarization_result = audio_diarization(audio_file_path)
+        logging.info(f"Diarization result type: {type(diarization_result)}")
+        logging.info(f"Diarization result sample: {diarization_result[:3] if isinstance(diarization_result, list) and len(diarization_result) > 3 else diarization_result}")
+
+        if not transcription_result:
+            logging.error("Empty result from transcription")
+            return []
+
+        if not diarization_result:
+            logging.error("Empty result from diarization")
+            return []
+
+        # Handle the case where transcription_result is a dict with a 'segments' key
+        if isinstance(transcription_result, dict) and 'segments' in transcription_result:
+            transcription_segments = transcription_result['segments']
+        elif isinstance(transcription_result, list):
+            transcription_segments = transcription_result
+        else:
+            logging.error(f"Unexpected transcription result format: {type(transcription_result)}")
+            return []
+
+        logging.info(f"Number of transcription segments: {len(transcription_segments)}")
+        logging.info(f"Transcription segments sample: {transcription_segments[:3] if len(transcription_segments) > 3 else transcription_segments}")
+
+        if not isinstance(diarization_result, list):
+            logging.error(f"Unexpected diarization result format: {type(diarization_result)}")
+            return []
+
+        combined_result = []
+        for transcription_segment in transcription_segments:
+            if not isinstance(transcription_segment, dict):
+                logging.warning(f"Unexpected transcription segment format: {transcription_segment}")
+                continue
+
+            for diarization_segment in diarization_result:
+                if not isinstance(diarization_segment, dict):
+                    logging.warning(f"Unexpected diarization segment format: {diarization_segment}")
+                    continue
+
+                try:
+                    trans_start = transcription_segment.get('Time_Start', 0)
+                    trans_end = transcription_segment.get('Time_End', 0)
+                    diar_start = diarization_segment.get('start', 0)
+                    diar_end = diarization_segment.get('end', 0)
+
+                    if trans_start >= diar_start and trans_end <= diar_end:
+                        combined_segment = {
+                            "Time_Start": trans_start,
+                            "Time_End": trans_end,
+                            "Speaker": diarization_segment.get('speaker', 'Unknown'),
+                            "Text": transcription_segment.get('Text', '')
+                        }
+                        combined_result.append(combined_segment)
+                        break
+                except Exception as e:
+                    logging.error(f"Error processing segment: {str(e)}")
+                    logging.error(f"Transcription segment: {transcription_segment}")
+                    logging.error(f"Diarization segment: {diarization_segment}")
+                    continue
+
+        logging.info(f"Combined result length: {len(combined_result)}")
+        logging.info(f"Combined result sample: {combined_result[:3] if len(combined_result) > 3 else combined_result}")
+        return combined_result
+
+    except Exception as e:
+        logging.error(f"Error in combine_transcription_and_diarization: {str(e)}", exc_info=True)
+        return []
+
+
+#
+#
+#######################################################################################################################

App_Function_Libraries/Chat.py

@@ -1,6 +1,6 @@
 # Chat.py
 # Chat functions for interacting with the LLMs as chatbots
-
+import base64
 # Imports
 import json
 import logging
@@ -15,9 +15,9 @@ from pathlib import Path
 # Local Imports
 from App_Function_Libraries.DB.DB_Manager import get_conversation_name, save_chat_history_to_database
 from App_Function_Libraries.LLM_API_Calls import chat_with_openai, chat_with_anthropic, chat_with_cohere, \
-    chat_with_groq, chat_with_openrouter, chat_with_deepseek, chat_with_mistral, chat_with_huggingface#, chat_with_vllm
+    chat_with_groq, chat_with_openrouter, chat_with_deepseek, chat_with_mistral, chat_with_huggingface  #, chat_with_vllm
 from App_Function_Libraries.LLM_API_Calls_Local import chat_with_aphrodite, chat_with_local_llm, chat_with_ollama, \
-    chat_with_kobold, chat_with_llama, chat_with_oobabooga, chat_with_tabbyapi
+    chat_with_kobold, chat_with_llama, chat_with_oobabooga, chat_with_tabbyapi, chat_with_vllm, chat_with_custom_openai
 from App_Function_Libraries.DB.SQLite_DB import load_media_content
 from App_Function_Libraries.Utils.Utils import generate_unique_filename
 #
@@ -54,8 +54,8 @@ def chat_api_call(api_endpoint, api_key, input_data, prompt, temp, system_messag
             response = chat_with_oobabooga(input_data, api_key, prompt, temp, system_message)
         elif api_endpoint.lower() == "tabbyapi":
             response = chat_with_tabbyapi(input_data, prompt, temp, system_message)
-        #elif api_endpoint.lower() == "vllm":
-        #    response = chat_with_vllm(input_data, prompt, system_message)
+        elif api_endpoint.lower() == "vllm":
+            response = chat_with_vllm(input_data, prompt, system_message)
         elif api_endpoint.lower() == "local-llm":
             response = chat_with_local_llm(input_data, prompt, temp, system_message)
         elif api_endpoint.lower() == "huggingface":
@@ -64,6 +64,8 @@ def chat_api_call(api_endpoint, api_key, input_data, prompt, temp, system_messag
             response = chat_with_ollama(input_data, prompt, temp, system_message)
         elif api_endpoint.lower() == "aphrodite":
             response = chat_with_aphrodite(input_data, prompt, temp, system_message)
+        elif api_endpoint.lower() == "custom-openai-api":
+            response = chat_with_custom_openai(api_key, input_data, prompt, temp, system_message)
         else:
             raise ValueError(f"Unsupported API endpoint: {api_endpoint}")
 
@@ -114,6 +116,8 @@ def chat(message, history, media_content, selected_parts, api_endpoint, api_key,
 
         # Use the existing API request code based on the selected endpoint
         response = chat_api_call(api_endpoint, api_key, input_data, prompt, temp, system_message)
+
+        return response
     except Exception as e:
         logging.error(f"Error in chat function: {str(e)}")
         return f"An error occurred: {str(e)}"
@@ -279,26 +283,60 @@ def update_chat_content(selected_item, use_content, use_summary, use_prompt, ite
         print(f"Debug - Update Chat Content - No item selected or item not in mapping")
         return {}, []
 
+#
+# End of Chat functions
+##########################################################################################################################
+
+
+##########################################################################################################################
+#
+# Character Card Functions
 
 CHARACTERS_FILE = Path('.', 'Helper_Scripts', 'Character_Cards', 'Characters.json')
 
+
 def save_character(character_data):
-    if CHARACTERS_FILE.exists():
-        with CHARACTERS_FILE.open('r') as f:
-            characters = json.load(f)
-    else:
-        characters = {}
+    characters_file = os.path.join(os.path.dirname(__file__), '..', 'Helper_Scripts', 'Character_Cards', 'Characters.json')
+    characters_dir = os.path.dirname(characters_file)
 
-    characters[character_data['name']] = character_data
+    try:
+        if os.path.exists(characters_file):
+            with open(characters_file, 'r') as f:
+                characters = json.load(f)
+        else:
+            characters = {}
+
+        char_name = character_data['name']
+
+        # Save the image separately if it exists
+        if 'image' in character_data:
+            img_data = base64.b64decode(character_data['image'])
+            img_filename = f"{char_name.replace(' ', '_')}.png"
+            img_path = os.path.join(characters_dir, img_filename)
+            with open(img_path, 'wb') as f:
+                f.write(img_data)
+            character_data['image_path'] = os.path.abspath(img_path)
+            del character_data['image']  # Remove the base64 image data from the JSON
+
+        characters[char_name] = character_data
+
+        with open(characters_file, 'w') as f:
+            json.dump(characters, f, indent=2)
+
+        logging.info(f"Character '{char_name}' saved successfully.")
+    except Exception as e:
+        logging.error(f"Error saving character: {str(e)}")
 
-    with CHARACTERS_FILE.open('w') as f:
-        json.dump(characters, f, indent=2)
 
 
 def load_characters():
-    if os.path.exists(CHARACTERS_FILE):
-        with open(CHARACTERS_FILE, 'r') as f:
-            return json.load(f)
+    characters_file = os.path.join(os.path.dirname(__file__), '..', 'Helper_Scripts', 'Character_Cards', 'Characters.json')
+    if os.path.exists(characters_file):
+        with open(characters_file, 'r') as f:
+            characters = json.load(f)
+        logging.debug(f"Loaded {len(characters)} characters from {characters_file}")
+        return characters
+    logging.warning(f"Characters file not found: {characters_file}")
     return {}
 
 

App_Function_Libraries/Chunk_Lib.py

@@ -7,6 +7,7 @@
 ####
 # Import necessary libraries
 import hashlib
+import json
 import logging
 import re
 from typing import Any, Dict, List, Optional, Tuple
@@ -72,42 +73,53 @@ def load_document(file_path):
 
 def improved_chunking_process(text: str, custom_chunk_options: Dict[str, Any] = None) -> List[Dict[str, Any]]:
     logging.debug("Improved chunking process started...")
+
+    # Extract JSON metadata if present
+    json_content = {}
+    try:
+        json_end = text.index("}\n") + 1
+        json_content = json.loads(text[:json_end])
+        text = text[json_end:].strip()
+        logging.debug(f"Extracted JSON metadata: {json_content}")
+    except (ValueError, json.JSONDecodeError):
+        logging.debug("No JSON metadata found at the beginning of the text")
+
+    # Extract any additional header text
+    header_match = re.match(r"(This text was transcribed using.*?)\n\n", text, re.DOTALL)
+    header_text = ""
+    if header_match:
+        header_text = header_match.group(1)
+        text = text[len(header_text):].strip()
+        logging.debug(f"Extracted header text: {header_text}")
+
     options = chunk_options.copy()
     if custom_chunk_options:
         options.update(custom_chunk_options)
 
     chunk_method = options.get('method', 'words')
-    base_size = options.get('base_size', 1000)
-    min_size = options.get('min_size', 100)
     max_size = options.get('max_size', 2000)
     overlap = options.get('overlap', 0)
     language = options.get('language', None)
-    adaptive = options.get('adaptive', False)
-    multi_level = options.get('multi_level', False)
 
     if language is None:
         language = detect_language(text)
 
-    if adaptive:
-        max_chunk_size = adaptive_chunk_size(text, base_size, min_size, max_size)
-    else:
-        max_chunk_size = base_size
-
-    if multi_level:
-        chunks = multi_level_chunking(text, chunk_method, max_chunk_size, overlap, language)
-    else:
-        chunks = chunk_text(text, chunk_method, max_chunk_size, overlap, language)
+    chunks = chunk_text(text, chunk_method, max_size, overlap, language)
 
     chunks_with_metadata = []
+    total_chunks = len(chunks)
     for i, chunk in enumerate(chunks):
-        metadata = get_chunk_metadata(
-            chunk,
-            text,
-            chunk_type=chunk_method,
-            language=language
-        )
-        metadata['chunk_index'] = i
-        metadata['total_chunks'] = len(chunks)
+        metadata = {
+            'chunk_index': i,
+            'total_chunks': total_chunks,
+            'chunk_method': chunk_method,
+            'max_size': max_size,
+            'overlap': overlap,
+            'language': language,
+            'relative_position': i / total_chunks
+        }
+        metadata.update(json_content)  # Add the extracted JSON content to metadata
+        metadata['header_text'] = header_text  # Add the header text to metadata
 
         chunks_with_metadata.append({
             'text': chunk,
@@ -117,6 +129,7 @@ def improved_chunking_process(text: str, custom_chunk_options: Dict[str, Any] =
     return chunks_with_metadata
 
 
+
 def multi_level_chunking(text: str, method: str, max_size: int, overlap: int, language: str) -> List[str]:
     logging.debug("Multi-level chunking process started...")
     # First level: chunk by paragraphs

App_Function_Libraries/Gradio_Related.py

@@ -16,6 +16,8 @@ import gradio as gr
 # Local Imports
 from App_Function_Libraries.DB.DB_Manager import get_db_config
 from App_Function_Libraries.Gradio_UI.Audio_ingestion_tab import create_audio_processing_tab
+from App_Function_Libraries.Gradio_UI.Character_Interaction_tab import create_character_card_interaction_tab, \
+    create_multiple_character_chat_tab, create_narrator_controlled_conversation_tab
 from App_Function_Libraries.Gradio_UI.Chat_ui import create_chat_management_tab, \
     create_chat_interface_four, create_chat_interface_multi_api, create_chat_interface_stacked, create_chat_interface
 from App_Function_Libraries.Gradio_UI.Config_tab import create_config_editor_tab
@@ -39,8 +41,9 @@ from App_Function_Libraries.Gradio_UI.RAG_QA_Chat_tab import create_rag_qa_chat_
 from App_Function_Libraries.Gradio_UI.Re_summarize_tab import create_resummary_tab
 from App_Function_Libraries.Gradio_UI.Search_Tab import create_prompt_view_tab, create_prompt_search_tab, \
     create_search_summaries_tab, create_viewing_tab,  create_search_tab
-from App_Function_Libraries.Gradio_UI.RAG_Chat_tab import create_embeddings_tab, create_rag_tab, \
-    create_view_embeddings_tab
+from App_Function_Libraries.Gradio_UI.RAG_Chat_tab import create_rag_tab
+from App_Function_Libraries.Gradio_UI.Embeddings_tab import create_embeddings_tab, create_view_embeddings_tab, \
+    create_purge_embeddings_tab
 from App_Function_Libraries.Gradio_UI.Trash import create_view_trash_tab, create_empty_trash_tab, \
     create_delete_trash_tab, create_search_and_mark_trash_tab
 from App_Function_Libraries.Gradio_UI.Utilities import create_utilities_yt_timestamp_tab, create_utilities_yt_audio_tab, \
@@ -260,11 +263,9 @@ def launch_ui(share_public=None, server_mode=False):
                 create_search_tab()
                 create_search_summaries_tab()
 
-            with gr.TabItem("RAG Search / Embeddings"):
+            with gr.TabItem("RAG Search"):
                 create_rag_tab()
                 create_rag_qa_chat_tab()
-                create_embeddings_tab()
-                create_view_embeddings_tab()
 
             with gr.TabItem("Chat with an LLM"):
                 create_chat_interface()
@@ -274,9 +275,12 @@ def launch_ui(share_public=None, server_mode=False):
                 create_chat_with_llamafile_tab()
                 create_chat_management_tab()
                 chat_workflows_tab()
-                from App_Function_Libraries.Gradio_UI.Writing_tab import create_character_card_interaction_tab
+                create_multiple_character_chat_tab()
+                create_narrator_controlled_conversation_tab()
                 create_character_card_interaction_tab()
 
+
+
             with gr.TabItem("View DB Items"):
                 create_viewing_tab()
                 create_prompt_view_tab()
@@ -295,6 +299,11 @@ def launch_ui(share_public=None, server_mode=False):
                 # FIXME
                 #create_compare_transcripts_tab()
 
+            with gr.TabItem("Embeddings Management"):
+                create_embeddings_tab()
+                create_view_embeddings_tab()
+                create_purge_embeddings_tab()
+
             with gr.TabItem("Writing Tools"):
                 with gr.Tabs():
                     from App_Function_Libraries.Gradio_UI.Writing_tab import create_document_feedback_tab

App_Function_Libraries/Local_File_Processing_Lib.py

@@ -20,10 +20,10 @@
 
 # Import necessary libraries
 # Import Local
-from App_Function_Libraries.Audio_Transcription_Lib import convert_to_wav
+from App_Function_Libraries.Audio.Audio_Transcription_Lib import convert_to_wav
 from App_Function_Libraries.Video_DL_Ingestion_Lib import *
 from App_Function_Libraries.Video_DL_Ingestion_Lib import get_youtube
-from App_Function_Libraries.Utils import normalize_title, create_download_directory
+from App_Function_Libraries.Utils.Utils import normalize_title, create_download_directory
 
 #######################################################################################################################
 # Function Definitions

App_Function_Libraries/Local_LLM_Inference_Engine_Lib.py

@@ -27,11 +27,11 @@ import subprocess
 import sys
 import time
 
-from App_Function_Libraries.Utils import download_file
+from App_Function_Libraries.Utils.Utils import download_file
 # Import 3rd-pary Libraries
 #
 # Import Local
-from Article_Summarization_Lib import *
+from App_Function_Libraries.Web_Scraping.Article_Summarization_Lib import *
 
 #
 #

App_Function_Libraries/MediaWiki/Media_Wiki.py

@@ -7,6 +7,7 @@ import json
 import logging
 import os
 import re
+import traceback
 from typing import List, Dict, Any, Iterator, Optional
 # 3rd-Party Imports
 import mwparserfromhell
@@ -14,12 +15,19 @@ import mwxml
 import yaml
 #
 # Local Imports
-from App_Function_Libraries.DB.DB_Manager import add_media_with_keywords, check_media_exists
+from App_Function_Libraries.DB.DB_Manager import add_media_with_keywords
 from App_Function_Libraries.RAG.ChromaDB_Library import process_and_store_content
 #
 #######################################################################################################################
 #
 # Functions:
+# Load configuration
+def load_mediawiki_import_config():
+    with open(os.path.join('Config_Files', 'mediawiki_import_config.yaml'), 'r') as f:
+        return yaml.safe_load(f)
+
+config = load_mediawiki_import_config()
+
 
 def setup_logger(name: str, level: int = logging.INFO, log_file: Optional[str] = None) -> logging.Logger:
     """Set up and return a logger with the given name and level."""
@@ -41,11 +49,11 @@ def setup_logger(name: str, level: int = logging.INFO, log_file: Optional[str] =
 # Usage
 logger = setup_logger('mediawiki_import', log_file='mediawiki_import.log')
 
-# Load configuration
-def load_mediawiki_import_config():
-    with open(os.path.join('Config_Files', 'mediawiki_import_config.yaml'), 'r') as f:
-        return yaml.safe_load(f)
-config = load_mediawiki_import_config()
+# End of setup
+#######################################################################################################################
+#
+# Functions:
+
 
 def parse_mediawiki_dump(file_path: str, namespaces: List[int] = None, skip_redirects: bool = False) -> Iterator[
     Dict[str, Any]]:
@@ -57,11 +65,11 @@ def parse_mediawiki_dump(file_path: str, namespaces: List[int] = None, skip_redi
             continue
 
         for revision in page:
-            code = mwparserfromhell.parse(revision.text)
-            text = code.strip_code(normalize=True, collapse=True, keep_template_params=False)
+            wikicode = mwparserfromhell.parse(revision.text)
+            plain_text = wikicode.strip_code()
             yield {
                 "title": page.title,
-                "content": text,
+                "content": plain_text,
                 "namespace": page.namespace,
                 "page_id": page.id,
                 "revision_id": revision.id,
@@ -76,6 +84,7 @@ def optimized_chunking(text: str, chunk_options: Dict[str, Any]) -> List[Dict[st
     current_chunk = ""
     current_size = 0
 
+    logging.debug(f"optimized_chunking: Processing text with {len(sections) // 2} sections")
     for i in range(0, len(sections), 2):
         section_title = sections[i] if i > 0 else "Introduction"
         section_content = sections[i + 1] if i + 1 < len(sections) else ""
@@ -95,33 +104,54 @@ def optimized_chunking(text: str, chunk_options: Dict[str, Any]) -> List[Dict[st
     return chunks
 
 
+
+
+
 def process_single_item(content: str, title: str, wiki_name: str, chunk_options: Dict[str, Any],
-                        is_combined: bool = False, item: Dict[str, Any] = None):
+                        is_combined: bool = False, item: Dict[str, Any] = None, api_name: str = None):
     try:
-        url = f"mediawiki:{wiki_name}" if is_combined else f"mediawiki:{wiki_name}:{title}"
-
-        if not check_media_exists(title, url):
-            media_id = add_media_with_keywords(
-                url=url,
-                title=title,
-                media_type="mediawiki_dump" if is_combined else "mediawiki_article",
-                content=content,
-                keywords=f"mediawiki,{wiki_name}" + (",full_dump" if is_combined else ",article"),
-                prompt="",
-                summary="",
-                transcription_model="",
-                author="MediaWiki",
-                ingestion_date=item['timestamp'].strftime('%Y-%m-%d') if item else None
-            )
-
-            chunks = optimized_chunking(content, chunk_options)
-            for chunk in chunks:
+        logging.debug(f"process_single_item: Processing item: {title}")
+
+        # Create a unique URL using the wiki name and article title
+        encoded_title = title.replace(" ", "_")
+        url = f"mediawiki:{wiki_name}:{encoded_title}"
+        logging.debug(f"Generated URL: {url}")
+
+        result = add_media_with_keywords(
+            url=url,  # Use the generated URL here
+            title=title,
+            media_type="mediawiki_dump" if is_combined else "mediawiki_article",
+            content=content,
+            keywords=f"mediawiki,{wiki_name}" + (",full_dump" if is_combined else ",article"),
+            prompt="",
+            summary="",
+            transcription_model="",
+            author="MediaWiki",
+            ingestion_date=item['timestamp'].strftime('%Y-%m-%d') if item else None
+        )
+        logging.debug(f"Result from add_media_with_keywords: {result}")
+
+        # Unpack the result
+        media_id, message = result
+        logging.info(f"Media item result: {message}")
+        logging.debug(f"Final media_id: {media_id}")
+
+        chunks = optimized_chunking(content, chunk_options)
+        for i, chunk in enumerate(chunks):
+            logging.debug(f"Processing chunk {i + 1}/{len(chunks)} for item: {title}")
+
+            # FIXME
+            # def process_and_store_content(content: str, collection_name: str, media_id: int, file_name: str,
+            #                               create_embeddings: bool = False, create_summary: bool = False,
+            #                               api_name: str = None):
+            if api_name:
+                process_and_store_content(chunk['text'], f"mediawiki_{wiki_name}", media_id, title, True, True, api_name)
+            else:
                 process_and_store_content(chunk['text'], f"mediawiki_{wiki_name}", media_id, title)
-            logger.info(f"Successfully processed item: {title}")
-        else:
-            logger.info(f"Skipping existing article: {title}")
+        logging.info(f"Successfully processed item: {title}")
     except Exception as e:
-        logger.error(f"Error processing item {title}: {str(e)}")
+        logging.error(f"Error processing item {title}: {str(e)}")
+        logging.error(f"Exception details: {traceback.format_exc()}")
 
 
 def load_checkpoint(file_path: str) -> int:
@@ -143,9 +173,12 @@ def import_mediawiki_dump(
         skip_redirects: bool = False,
         chunk_options: Dict[str, Any] = None,
         single_item: bool = False,
-        progress_callback: Any = None
+        progress_callback: Any = None,
+        api_name: str = None,
+        api_key: str = None
 ) -> Iterator[str]:
     try:
+        logging.info(f"Importing MediaWiki dump: {file_path}")
         if chunk_options is None:
             chunk_options = config['chunking']
 
@@ -160,6 +193,10 @@ def import_mediawiki_dump(
         for item in parse_mediawiki_dump(file_path, namespaces, skip_redirects):
             if item['page_id'] <= last_processed_id:
                 continue
+            # FIXME - ensure this works...
+            if api_name is not None:
+                # FIXME - add API key to the call/params
+                process_single_item(item['content'], item['title'], wiki_name, chunk_options, False, item, api_name)
             process_single_item(item['content'], item['title'], wiki_name, chunk_options, False, item)
             save_checkpoint(checkpoint_file, item['page_id'])
             processed_pages += 1

App_Function_Libraries/PDF/PDF_Ingestion_Lib.py

@@ -0,0 +1,318 @@
+# PDF_Ingestion_Lib.py
+#########################################
+# Library to hold functions for ingesting PDF files.#
+#
+####################
+# Function List
+#
+# 1. convert_pdf_to_markdown(pdf_path)
+# 2. ingest_pdf_file(file_path, title=None, author=None, keywords=None):
+# 3.
+#
+#
+####################
+import re
+
+# Import necessary libraries
+
+
+# Import Local
+
+#######################################################################################################################
+# Function Definitions
+#
+
+# Ingest a text file into the database with Title/Author/Keywords
+
+
+# Constants
+MAX_FILE_SIZE_MB = 50
+CONVERSION_TIMEOUT_SECONDS = 300
+
+# Marker PDF solution
+# def convert_pdf_to_markdown(pdf_path):
+#     """
+#     Convert a PDF file to Markdown by calling a script in another virtual environment.
+#     """
+#
+#     logging.debug(f"Marker: Converting PDF file to Markdown: {pdf_path}")
+#     # Check if the file size exceeds the maximum allowed size
+#     file_size_mb = os.path.getsize(pdf_path) / (1024 * 1024)
+#     if file_size_mb > MAX_FILE_SIZE_MB:
+#         raise ValueError(f"File size ({file_size_mb:.2f} MB) exceeds the maximum allowed size of {MAX_FILE_SIZE_MB} MB")
+#
+#     logging.debug("Marker: Converting PDF file to Markdown using Marker virtual environment")
+#     # Path to the Python interpreter in the other virtual environment
+#     other_venv_python = "Helper_Scripts/marker_venv/bin/python"
+#
+#     # Path to the conversion script
+#     converter_script = "Helper_Scripts/PDF_Converter.py"
+#
+#     logging.debug("Marker: Attempting to convert PDF file to Markdown...")
+#     try:
+#         result = subprocess.run(
+#             [other_venv_python, converter_script, pdf_path],
+#             capture_output=True,
+#             text=True,
+#             timeout=CONVERSION_TIMEOUT_SECONDS
+#         )
+#         if result.returncode != 0:
+#             raise Exception(f"Conversion failed: {result.stderr}")
+#         return result.stdout
+#     except subprocess.TimeoutExpired:
+#         raise Exception(f"PDF conversion timed out after {CONVERSION_TIMEOUT_SECONDS} seconds")
+#
+#
+# def process_and_ingest_pdf(file, title, author, keywords):
+#     if file is None:
+#         return "Please select a PDF file to upload."
+#
+#     try:
+#         # Create a temporary directory
+#         with tempfile.TemporaryDirectory() as temp_dir:
+#             # Create a path for the temporary PDF file
+#             temp_path = os.path.join(temp_dir, "temp.pdf")
+#
+#             # Copy the contents of the uploaded file to the temporary file
+#             shutil.copy(file.name, temp_path)
+#
+#             # Call the ingest_pdf_file function with the temporary file path
+#             result = ingest_pdf_file(temp_path, title, author, keywords)
+#
+#         return result
+#     except Exception as e:
+#         return f"Error processing PDF: {str(e)}"
+#
+#
+# def ingest_pdf_file(file_path, title=None, author=None, keywords=None):
+#     try:
+#         # Convert PDF to Markdown
+#         markdown_content = convert_pdf_to_markdown(file_path)
+#
+#         # If title is not provided, use the filename without extension
+#         if not title:
+#             title = os.path.splitext(os.path.basename(file_path))[0]
+#
+#         # If author is not provided, set it to 'Unknown'
+#         if not author:
+#             author = 'Unknown'
+#
+#         # If keywords are not provided, use a default keyword
+#         if not keywords:
+#             keywords = 'pdf_file,markdown_converted'
+#         else:
+#             keywords = f'pdf_file,markdown_converted,{keywords}'
+#
+#         # Add the markdown content to the database
+#         add_media_with_keywords(
+#             url=file_path,
+#             title=title,
+#             media_type='document',
+#             content=markdown_content,
+#             keywords=keywords,
+#             prompt='No prompt for PDF files',
+#             summary='No summary for PDF files',
+#             transcription_model='None',
+#             author=author,
+#             ingestion_date=datetime.now().strftime('%Y-%m-%d')
+#         )
+#
+#         return f"PDF file '{title}' converted to Markdown and ingested successfully.", file_path
+#     except ValueError as e:
+#         logging.error(f"File size error: {str(e)}")
+#         return f"Error: {str(e)}", file_path
+#     except Exception as e:
+#         logging.error(f"Error ingesting PDF file: {str(e)}")
+#         return f"Error ingesting PDF file: {str(e)}", file_path
+#
+#
+# def process_and_cleanup_pdf(file, title, author, keywords):
+#     # FIXME - Update to validate file upload/filetype is pdf....
+#     if file is None:
+#         return "No file uploaded. Please upload a PDF file."
+#
+#     temp_dir = tempfile.mkdtemp()
+#     temp_file_path = os.path.join(temp_dir, "temp.pdf")
+#
+#     try:
+#         # Copy the uploaded file to a temporary location
+#         shutil.copy2(file.name, temp_file_path)
+#
+#         # Process the file
+#         result, _ = ingest_pdf_file(temp_file_path, title, author, keywords)
+#
+#         return result
+#     except Exception as e:
+#         logging.error(f"Error in processing and cleanup: {str(e)}")
+#         return f"Error: {str(e)}"
+#     finally:
+#         # Clean up the temporary directory and its contents
+#         try:
+#             shutil.rmtree(temp_dir)
+#             logging.info(f"Removed temporary directory: {temp_dir}")
+#         except Exception as cleanup_error:
+#             logging.error(f"Error during cleanup: {str(cleanup_error)}")
+#             result += f"\nWarning: Could not remove temporary files: {str(cleanup_error)}"
+
+
+import logging
+#
+#
+#######################################################################################################################
+#
+# Non-Marker implementation
+import os
+import shutil
+import tempfile
+from datetime import datetime
+
+import pymupdf
+
+from App_Function_Libraries.DB.DB_Manager import add_media_with_keywords
+
+
+def extract_text_and_format_from_pdf(pdf_path):
+    """
+    Extract text from a PDF file and convert it to Markdown, preserving formatting.
+    """
+    try:
+        markdown_text = ""
+        with pymupdf.open(pdf_path) as doc:
+            for page_num, page in enumerate(doc, 1):
+                markdown_text += f"## Page {page_num}\n\n"
+                blocks = page.get_text("dict")["blocks"]
+                current_paragraph = ""
+                for block in blocks:
+                    if block["type"] == 0:  # Text block
+                        for line in block["lines"]:
+                            line_text = ""
+                            for span in line["spans"]:
+                                text = span["text"]
+                                font_size = span["size"]
+                                font_flags = span["flags"]
+
+                                # Apply formatting based on font size and flags
+                                if font_size > 20:
+                                    text = f"# {text}"
+                                elif font_size > 16:
+                                    text = f"## {text}"
+                                elif font_size > 14:
+                                    text = f"### {text}"
+
+                                if font_flags & 2 ** 0:  # Bold
+                                    text = f"**{text}**"
+                                if font_flags & 2 ** 1:  # Italic
+                                    text = f"*{text}*"
+
+                                line_text += text + " "
+
+                            # Remove hyphens at the end of lines
+                            line_text = line_text.rstrip()
+                            if line_text.endswith('-'):
+                                line_text = line_text[:-1]
+                            else:
+                                line_text += " "
+
+                            current_paragraph += line_text
+
+                        # End of block, add paragraph
+                        if current_paragraph:
+                            # Remove extra spaces
+                            current_paragraph = re.sub(r'\s+', ' ', current_paragraph).strip()
+                            markdown_text += current_paragraph + "\n\n"
+                            current_paragraph = ""
+                    elif block["type"] == 1:  # Image block
+                        markdown_text += "[Image]\n\n"
+                markdown_text += "\n---\n\n"  # Page separator
+
+        # Clean up hyphenated words
+        markdown_text = re.sub(r'(\w+)-\s*\n(\w+)', r'\1\2', markdown_text)
+
+        return markdown_text
+    except Exception as e:
+        logging.error(f"Error extracting text and formatting from PDF: {str(e)}")
+        raise
+
+
+def extract_metadata_from_pdf(pdf_path):
+    """
+    Extract metadata from a PDF file using PyMuPDF.
+    """
+    try:
+        with pymupdf.open(pdf_path) as doc:
+            metadata = doc.metadata
+        return metadata
+    except Exception as e:
+        logging.error(f"Error extracting metadata from PDF: {str(e)}")
+        return {}
+
+
+def process_and_ingest_pdf(file, title, author, keywords):
+    if file is None:
+        return "Please select a PDF file to upload."
+
+    try:
+        # Create a temporary directory
+        with tempfile.TemporaryDirectory() as temp_dir:
+            # Create a path for the temporary PDF file
+            temp_path = os.path.join(temp_dir, "temp.pdf")
+
+            # Copy the contents of the uploaded file to the temporary file
+            shutil.copy(file.name, temp_path)
+
+            # Extract text and convert to Markdown
+            markdown_text = extract_text_and_format_from_pdf(temp_path)
+
+            # Extract metadata from PDF
+            metadata = extract_metadata_from_pdf(temp_path)
+
+            # Use metadata for title and author if not provided
+            if not title:
+                title = metadata.get('title', os.path.splitext(os.path.basename(file.name))[0])
+            if not author:
+                author = metadata.get('author', 'Unknown')
+
+            # If keywords are not provided, use a default keyword
+            if not keywords:
+                keywords = 'pdf_file,markdown_converted'
+            else:
+                keywords = f'pdf_file,markdown_converted,{keywords}'
+
+            # Add metadata-based keywords
+            if 'subject' in metadata:
+                keywords += f",{metadata['subject']}"
+
+            # Add the PDF content to the database
+            add_media_with_keywords(
+                url=file.name,
+                title=title,
+                media_type='document',
+                content=markdown_text,
+                keywords=keywords,
+                prompt='No prompt for PDF files',
+                summary='No summary for PDF files',
+                transcription_model='None',
+                author=author,
+                ingestion_date=datetime.now().strftime('%Y-%m-%d')
+            )
+
+        return f"PDF file '{title}' by {author} ingested successfully and converted to Markdown."
+    except Exception as e:
+        logging.error(f"Error ingesting PDF file: {str(e)}")
+        return f"Error ingesting PDF file: {str(e)}"
+
+
+def process_and_cleanup_pdf(file, title, author, keywords):
+    if file is None:
+        return "No file uploaded. Please upload a PDF file."
+
+    try:
+        result = process_and_ingest_pdf(file, title, author, keywords)
+        return result
+    except Exception as e:
+        logging.error(f"Error in processing and cleanup: {str(e)}")
+        return f"Error: {str(e)}"
+
+#
+# End of PDF_Ingestion_Lib.py
+#######################################################################################################################

App_Function_Libraries/Summarization/Summarization_General_Lib.py

@@ -25,10 +25,10 @@ from typing import Optional
 import requests
 from requests import RequestException
 
-from App_Function_Libraries.Audio_Transcription_Lib import convert_to_wav, speech_to_text
+from App_Function_Libraries.Audio.Audio_Transcription_Lib import convert_to_wav, speech_to_text
 from App_Function_Libraries.Chunk_Lib import semantic_chunking, rolling_summarize, recursive_summarize_chunks, \
     improved_chunking_process
-from App_Function_Libraries.Diarization_Lib import combine_transcription_and_diarization
+from App_Function_Libraries.Audio.Diarization_Lib import combine_transcription_and_diarization
 from App_Function_Libraries.Summarization.Local_Summarization_Lib import summarize_with_llama, summarize_with_kobold, \
     summarize_with_oobabooga, summarize_with_tabbyapi, summarize_with_vllm, summarize_with_local_llm
 from App_Function_Libraries.DB.DB_Manager import add_media_to_database

App_Function_Libraries/Video_DL_Ingestion_Lib.py

@@ -1,331 +1,332 @@
-# Video_DL_Ingestion_Lib.py
-#########################################
-# Video Downloader and Ingestion Library
-# This library is used to handle downloading videos from YouTube and other platforms.
-# It also handles the ingestion of the videos into the database.
-# It uses yt-dlp to extract video information and download the videos.
-####
-import json
-####################
-# Function List
-#
-# 1. get_video_info(url)
-# 2. create_download_directory(title)
-# 3. sanitize_filename(title)
-# 4. normalize_title(title)
-# 5. get_youtube(video_url)
-# 6. get_playlist_videos(playlist_url)
-# 7. download_video(video_url, download_path, info_dict, download_video_flag)
-# 8. save_to_file(video_urls, filename)
-# 9. save_summary_to_file(summary, file_path)
-# 10. process_url(url, num_speakers, whisper_model, custom_prompt, offset, api_name, api_key, vad_filter, download_video, download_audio, rolling_summarization, detail_level, question_box, keywords, chunk_summarization, chunk_duration_input, words_per_second_input)
-#
-#
-####################
-# Import necessary libraries to run solo for testing
-import logging
-import os
-import re
-import sys
-from urllib.parse import urlparse, parse_qs
-
-import unicodedata
-# 3rd-Party Imports
-import yt_dlp
-
-from App_Function_Libraries.DB.DB_Manager import check_media_and_whisper_model
-
-
-# Import Local
-#
-#######################################################################################################################
-# Function Definitions
-#
-
-def normalize_title(title):
-    # Normalize the string to 'NFKD' form and encode to 'ascii' ignoring non-ascii characters
-    title = unicodedata.normalize('NFKD', title).encode('ascii', 'ignore').decode('ascii')
-    title = title.replace('/', '_').replace('\\', '_').replace(':', '_').replace('"', '').replace('*', '').replace('?',
-                                                                                                                   '').replace(
-        '<', '').replace('>', '').replace('|', '')
-    return title
-
-def get_video_info(url: str) -> dict:
-    ydl_opts = {
-        'quiet': True,
-        'no_warnings': True,
-        'skip_download': True,
-    }
-    with yt_dlp.YoutubeDL(ydl_opts) as ydl:
-        try:
-            info_dict = ydl.extract_info(url, download=False)
-            return info_dict
-        except Exception as e:
-            logging.error(f"Error extracting video info: {e}")
-            return None
-
-
-def get_youtube(video_url):
-    ydl_opts = {
-        'format': 'bestaudio[ext=m4a]',
-        'noplaylist': False,
-        'quiet': True,
-        'extract_flat': True
-    }
-    with yt_dlp.YoutubeDL(ydl_opts) as ydl:
-        logging.debug("About to extract youtube info")
-        info_dict = ydl.extract_info(video_url, download=False)
-        logging.debug("Youtube info successfully extracted")
-    return info_dict
-
-
-def get_playlist_videos(playlist_url):
-    ydl_opts = {
-        'extract_flat': True,
-        'skip_download': True,
-        'quiet': True
-    }
-
-    with yt_dlp.YoutubeDL(ydl_opts) as ydl:
-        info = ydl.extract_info(playlist_url, download=False)
-
-        if 'entries' in info:
-            video_urls = [entry['url'] for entry in info['entries']]
-            playlist_title = info['title']
-            return video_urls, playlist_title
-        else:
-            print("No videos found in the playlist.")
-            return [], None
-
-
-def download_video(video_url, download_path, info_dict, download_video_flag, current_whisper_model):
-    global video_file_path, ffmpeg_path
-    global audio_file_path
-
-    # Normalize Video Title name
-    logging.debug("About to normalize downloaded video title")
-    if 'title' not in info_dict or 'ext' not in info_dict:
-        logging.error("info_dict is missing 'title' or 'ext'")
-        return None
-
-    normalized_video_title = normalize_title(info_dict['title'])
-
-    # Check if media already exists in the database and compare whisper models
-    should_download, reason = check_media_and_whisper_model(
-        title=normalized_video_title,
-        url=video_url,
-        current_whisper_model=current_whisper_model
-    )
-
-    if not should_download:
-        logging.info(f"Skipping download: {reason}")
-        return None
-
-    logging.info(f"Proceeding with download: {reason}")
-
-    video_file_path = os.path.join(download_path, f"{normalized_video_title}.{info_dict['ext']}")
-
-    # Check for existence of video file
-    if os.path.exists(video_file_path):
-        logging.info(f"Video file already exists: {video_file_path}")
-        return video_file_path
-
-    # Setup path handling for ffmpeg on different OSs
-    if sys.platform.startswith('win'):
-        ffmpeg_path = os.path.join(os.getcwd(), 'Bin', 'ffmpeg.exe')
-    elif sys.platform.startswith('linux'):
-        ffmpeg_path = 'ffmpeg'
-    elif sys.platform.startswith('darwin'):
-        ffmpeg_path = 'ffmpeg'
-
-    if download_video_flag:
-        video_file_path = os.path.join(download_path, f"{normalized_video_title}.mp4")
-        ydl_opts_video = {
-            'format': 'bestvideo[ext=mp4]+bestaudio[ext=m4a]',
-            'outtmpl': video_file_path,
-            'ffmpeg_location': ffmpeg_path
-        }
-
-        try:
-            with yt_dlp.YoutubeDL(ydl_opts_video) as ydl:
-                logging.debug("yt_dlp: About to download video with youtube-dl")
-                ydl.download([video_url])
-                logging.debug("yt_dlp: Video successfully downloaded with youtube-dl")
-                if os.path.exists(video_file_path):
-                    return video_file_path
-                else:
-                    logging.error("yt_dlp: Video file not found after download")
-                    return None
-        except Exception as e:
-            logging.error(f"yt_dlp: Error downloading video: {e}")
-            return None
-    elif not download_video_flag:
-        video_file_path = os.path.join(download_path, f"{normalized_video_title}.mp4")
-        # Set options for video and audio
-        ydl_opts = {
-            'format': 'bestaudio[ext=m4a]',
-            'quiet': True,
-            'outtmpl': video_file_path
-        }
-
-        try:
-            with yt_dlp.YoutubeDL(ydl_opts) as ydl:
-                logging.debug("yt_dlp: About to download video with youtube-dl")
-                ydl.download([video_url])
-                logging.debug("yt_dlp: Video successfully downloaded with youtube-dl")
-                if os.path.exists(video_file_path):
-                    return video_file_path
-                else:
-                    logging.error("yt_dlp: Video file not found after download")
-                    return None
-        except Exception as e:
-            logging.error(f"yt_dlp: Error downloading video: {e}")
-            return None
-
-    else:
-        logging.debug("download_video: Download video flag is set to False and video file path is not found")
-        return None
-
-
-def extract_video_info(url):
-    try:
-        with yt_dlp.YoutubeDL({'quiet': True}) as ydl:
-            info = ydl.extract_info(url, download=False)
-
-            # Log only a subset of the info to avoid overwhelming the logs
-            log_info = {
-                'title': info.get('title'),
-                'duration': info.get('duration'),
-                'upload_date': info.get('upload_date')
-            }
-            logging.debug(f"Extracted info for {url}: {log_info}")
-
-            return info
-    except Exception as e:
-        logging.error(f"Error extracting video info for {url}: {str(e)}", exc_info=True)
-        return None
-
-
-def get_youtube_playlist_urls(playlist_id):
-    ydl_opts = {
-        'extract_flat': True,
-        'quiet': True,
-    }
-
-    with yt_dlp.YoutubeDL(ydl_opts) as ydl:
-        result = ydl.extract_info(f'https://www.youtube.com/playlist?list={playlist_id}', download=False)
-        return [entry['url'] for entry in result['entries'] if entry.get('url')]
-
-
-def parse_and_expand_urls(urls):
-    logging.info(f"Starting parse_and_expand_urls with input: {urls}")
-    expanded_urls = []
-
-    for url in urls:
-        try:
-            logging.info(f"Processing URL: {url}")
-            parsed_url = urlparse(url)
-            logging.debug(f"Parsed URL components: {parsed_url}")
-
-            # YouTube playlist handling
-            if 'youtube.com' in parsed_url.netloc and 'list' in parsed_url.query:
-                playlist_id = parse_qs(parsed_url.query)['list'][0]
-                logging.info(f"Detected YouTube playlist with ID: {playlist_id}")
-                playlist_urls = get_youtube_playlist_urls(playlist_id)
-                logging.info(f"Expanded playlist URLs: {playlist_urls}")
-                expanded_urls.extend(playlist_urls)
-
-            # YouTube short URL handling
-            elif 'youtu.be' in parsed_url.netloc:
-                video_id = parsed_url.path.lstrip('/')
-                full_url = f'https://www.youtube.com/watch?v={video_id}'
-                logging.info(f"Expanded YouTube short URL to: {full_url}")
-                expanded_urls.append(full_url)
-
-            # Vimeo handling
-            elif 'vimeo.com' in parsed_url.netloc:
-                video_id = parsed_url.path.lstrip('/')
-                full_url = f'https://vimeo.com/{video_id}'
-                logging.info(f"Processed Vimeo URL: {full_url}")
-                expanded_urls.append(full_url)
-
-            # Add more platform-specific handling here
-
-            else:
-                logging.info(f"URL not recognized as special case, adding as-is: {url}")
-                expanded_urls.append(url)
-
-        except Exception as e:
-            logging.error(f"Error processing URL {url}: {str(e)}", exc_info=True)
-            # Optionally, you might want to add the problematic URL to expanded_urls
-            # expanded_urls.append(url)
-
-    logging.info(f"Final expanded URLs: {expanded_urls}")
-    return expanded_urls
-
-
-def extract_metadata(url, use_cookies=False, cookies=None):
-    ydl_opts = {
-        'quiet': True,
-        'no_warnings': True,
-        'extract_flat': True,
-        'skip_download': True,
-    }
-
-    if use_cookies and cookies:
-        try:
-            cookie_dict = json.loads(cookies)
-            ydl_opts['cookiefile'] = cookie_dict
-        except json.JSONDecodeError:
-            logging.warning("Invalid cookie format. Proceeding without cookies.")
-
-    with yt_dlp.YoutubeDL(ydl_opts) as ydl:
-        try:
-            info = ydl.extract_info(url, download=False)
-            metadata = {
-                'title': info.get('title'),
-                'uploader': info.get('uploader'),
-                'upload_date': info.get('upload_date'),
-                'view_count': info.get('view_count'),
-                'like_count': info.get('like_count'),
-                'duration': info.get('duration'),
-                'tags': info.get('tags'),
-                'description': info.get('description')
-            }
-
-            # Create a safe subset of metadata to log
-            safe_metadata = {
-                'title': metadata.get('title', 'No title'),
-                'duration': metadata.get('duration', 'Unknown duration'),
-                'upload_date': metadata.get('upload_date', 'Unknown upload date'),
-                'uploader': metadata.get('uploader', 'Unknown uploader')
-            }
-
-            logging.info(f"Successfully extracted metadata for {url}: {safe_metadata}")
-            return metadata
-        except Exception as e:
-            logging.error(f"Error extracting metadata for {url}: {str(e)}", exc_info=True)
-            return None
-
-
-def generate_timestamped_url(url, hours, minutes, seconds):
-    # Extract video ID from the URL
-    video_id_match = re.search(r'(?:v=|)([0-9A-Za-z_-]{11}).*', url)
-    if not video_id_match:
-        return "Invalid YouTube URL"
-
-    video_id = video_id_match.group(1)
-
-    # Calculate total seconds
-    total_seconds = int(hours) * 3600 + int(minutes) * 60 + int(seconds)
-
-    # Generate the new URL
-    new_url = f"https://www.youtube.com/watch?v={video_id}&t={total_seconds}s"
-
-    return new_url
-
-
-
-#
-#
-#######################################################################################################################
+# Video_DL_Ingestion_Lib.py
+#########################################
+# Video Downloader and Ingestion Library
+# This library is used to handle downloading videos from YouTube and other platforms.
+# It also handles the ingestion of the videos into the database.
+# It uses yt-dlp to extract video information and download the videos.
+####
+import json
+####################
+# Function List
+#
+# 1. get_video_info(url)
+# 2. create_download_directory(title)
+# 3. sanitize_filename(title)
+# 4. normalize_title(title)
+# 5. get_youtube(video_url)
+# 6. get_playlist_videos(playlist_url)
+# 7. download_video(video_url, download_path, info_dict, download_video_flag)
+# 8. save_to_file(video_urls, filename)
+# 9. save_summary_to_file(summary, file_path)
+# 10. process_url(url, num_speakers, whisper_model, custom_prompt, offset, api_name, api_key, vad_filter, download_video, download_audio, rolling_summarization, detail_level, question_box, keywords, chunk_summarization, chunk_duration_input, words_per_second_input)
+#
+#
+####################
+# Import necessary libraries to run solo for testing
+import logging
+import os
+import re
+import sys
+from urllib.parse import urlparse, parse_qs
+
+import unicodedata
+# 3rd-Party Imports
+import yt_dlp
+
+from App_Function_Libraries.DB.DB_Manager import check_media_and_whisper_model
+
+
+# Import Local
+#
+#######################################################################################################################
+# Function Definitions
+#
+
+def normalize_title(title):
+    # Normalize the string to 'NFKD' form and encode to 'ascii' ignoring non-ascii characters
+    title = unicodedata.normalize('NFKD', title).encode('ascii', 'ignore').decode('ascii')
+    title = title.replace('/', '_').replace('\\', '_').replace(':', '_').replace('"', '').replace('*', '').replace('?',
+                                                                                                                   '').replace(
+        '<', '').replace('>', '').replace('|', '')
+    return title
+
+def get_video_info(url: str) -> dict:
+    ydl_opts = {
+        'quiet': True,
+        'no_warnings': True,
+        'skip_download': True,
+    }
+    with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+        try:
+            info_dict = ydl.extract_info(url, download=False)
+            return info_dict
+        except Exception as e:
+            logging.error(f"Error extracting video info: {e}")
+            return None
+
+
+def get_youtube(video_url):
+    ydl_opts = {
+        'format': 'bestaudio[ext=m4a]',
+        'noplaylist': False,
+        'quiet': True,
+        'extract_flat': True
+    }
+    with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+        logging.debug("About to extract youtube info")
+        info_dict = ydl.extract_info(video_url, download=False)
+        logging.debug("Youtube info successfully extracted")
+    return info_dict
+
+
+def get_playlist_videos(playlist_url):
+    ydl_opts = {
+        'extract_flat': True,
+        'skip_download': True,
+        'quiet': True
+    }
+
+    with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+        info = ydl.extract_info(playlist_url, download=False)
+
+        if 'entries' in info:
+            video_urls = [entry['url'] for entry in info['entries']]
+            playlist_title = info['title']
+            return video_urls, playlist_title
+        else:
+            print("No videos found in the playlist.")
+            return [], None
+
+
+def download_video(video_url, download_path, info_dict, download_video_flag, current_whisper_model):
+    global video_file_path, ffmpeg_path
+    global audio_file_path
+
+    # Normalize Video Title name
+    logging.debug("About to normalize downloaded video title")
+    if 'title' not in info_dict or 'ext' not in info_dict:
+        logging.error("info_dict is missing 'title' or 'ext'")
+        return None
+
+    normalized_video_title = normalize_title(info_dict['title'])
+
+    # FIXME - make sure this works/checks against hte current model
+    # Check if media already exists in the database and compare whisper models
+    should_download, reason = check_media_and_whisper_model(
+        title=normalized_video_title,
+        url=video_url,
+        current_whisper_model=current_whisper_model
+    )
+
+    if not should_download:
+        logging.info(f"Skipping download: {reason}")
+        return None
+
+    logging.info(f"Proceeding with download: {reason}")
+
+    video_file_path = os.path.join(download_path, f"{normalized_video_title}.{info_dict['ext']}")
+
+    # Check for existence of video file
+    if os.path.exists(video_file_path):
+        logging.info(f"Video file already exists: {video_file_path}")
+        return video_file_path
+
+    # Setup path handling for ffmpeg on different OSs
+    if sys.platform.startswith('win'):
+        ffmpeg_path = os.path.join(os.getcwd(), 'Bin', 'ffmpeg.exe')
+    elif sys.platform.startswith('linux'):
+        ffmpeg_path = 'ffmpeg'
+    elif sys.platform.startswith('darwin'):
+        ffmpeg_path = 'ffmpeg'
+
+    if download_video_flag:
+        video_file_path = os.path.join(download_path, f"{normalized_video_title}.mp4")
+        ydl_opts_video = {
+            'format': 'bestvideo[ext=mp4]+bestaudio[ext=m4a]',
+            'outtmpl': video_file_path,
+            'ffmpeg_location': ffmpeg_path
+        }
+
+        try:
+            with yt_dlp.YoutubeDL(ydl_opts_video) as ydl:
+                logging.debug("yt_dlp: About to download video with youtube-dl")
+                ydl.download([video_url])
+                logging.debug("yt_dlp: Video successfully downloaded with youtube-dl")
+                if os.path.exists(video_file_path):
+                    return video_file_path
+                else:
+                    logging.error("yt_dlp: Video file not found after download")
+                    return None
+        except Exception as e:
+            logging.error(f"yt_dlp: Error downloading video: {e}")
+            return None
+    elif not download_video_flag:
+        video_file_path = os.path.join(download_path, f"{normalized_video_title}.mp4")
+        # Set options for video and audio
+        ydl_opts = {
+            'format': 'bestaudio[ext=m4a]',
+            'quiet': True,
+            'outtmpl': video_file_path
+        }
+
+        try:
+            with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+                logging.debug("yt_dlp: About to download video with youtube-dl")
+                ydl.download([video_url])
+                logging.debug("yt_dlp: Video successfully downloaded with youtube-dl")
+                if os.path.exists(video_file_path):
+                    return video_file_path
+                else:
+                    logging.error("yt_dlp: Video file not found after download")
+                    return None
+        except Exception as e:
+            logging.error(f"yt_dlp: Error downloading video: {e}")
+            return None
+
+    else:
+        logging.debug("download_video: Download video flag is set to False and video file path is not found")
+        return None
+
+
+def extract_video_info(url):
+    try:
+        with yt_dlp.YoutubeDL({'quiet': True}) as ydl:
+            info = ydl.extract_info(url, download=False)
+
+            # Log only a subset of the info to avoid overwhelming the logs
+            log_info = {
+                'title': info.get('title'),
+                'duration': info.get('duration'),
+                'upload_date': info.get('upload_date')
+            }
+            logging.debug(f"Extracted info for {url}: {log_info}")
+
+            return info
+    except Exception as e:
+        logging.error(f"Error extracting video info for {url}: {str(e)}", exc_info=True)
+        return None
+
+
+def get_youtube_playlist_urls(playlist_id):
+    ydl_opts = {
+        'extract_flat': True,
+        'quiet': True,
+    }
+
+    with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+        result = ydl.extract_info(f'https://www.youtube.com/playlist?list={playlist_id}', download=False)
+        return [entry['url'] for entry in result['entries'] if entry.get('url')]
+
+
+def parse_and_expand_urls(urls):
+    logging.info(f"Starting parse_and_expand_urls with input: {urls}")
+    expanded_urls = []
+
+    for url in urls:
+        try:
+            logging.info(f"Processing URL: {url}")
+            parsed_url = urlparse(url)
+            logging.debug(f"Parsed URL components: {parsed_url}")
+
+            # YouTube playlist handling
+            if 'youtube.com' in parsed_url.netloc and 'list' in parsed_url.query:
+                playlist_id = parse_qs(parsed_url.query)['list'][0]
+                logging.info(f"Detected YouTube playlist with ID: {playlist_id}")
+                playlist_urls = get_youtube_playlist_urls(playlist_id)
+                logging.info(f"Expanded playlist URLs: {playlist_urls}")
+                expanded_urls.extend(playlist_urls)
+
+            # YouTube short URL handling
+            elif 'youtu.be' in parsed_url.netloc:
+                video_id = parsed_url.path.lstrip('/')
+                full_url = f'https://www.youtube.com/watch?v={video_id}'
+                logging.info(f"Expanded YouTube short URL to: {full_url}")
+                expanded_urls.append(full_url)
+
+            # Vimeo handling
+            elif 'vimeo.com' in parsed_url.netloc:
+                video_id = parsed_url.path.lstrip('/')
+                full_url = f'https://vimeo.com/{video_id}'
+                logging.info(f"Processed Vimeo URL: {full_url}")
+                expanded_urls.append(full_url)
+
+            # Add more platform-specific handling here
+
+            else:
+                logging.info(f"URL not recognized as special case, adding as-is: {url}")
+                expanded_urls.append(url)
+
+        except Exception as e:
+            logging.error(f"Error processing URL {url}: {str(e)}", exc_info=True)
+            # Optionally, you might want to add the problematic URL to expanded_urls
+            # expanded_urls.append(url)
+
+    logging.info(f"Final expanded URLs: {expanded_urls}")
+    return expanded_urls
+
+
+def extract_metadata(url, use_cookies=False, cookies=None):
+    ydl_opts = {
+        'quiet': True,
+        'no_warnings': True,
+        'extract_flat': True,
+        'skip_download': True,
+    }
+
+    if use_cookies and cookies:
+        try:
+            cookie_dict = json.loads(cookies)
+            ydl_opts['cookiefile'] = cookie_dict
+        except json.JSONDecodeError:
+            logging.warning("Invalid cookie format. Proceeding without cookies.")
+
+    with yt_dlp.YoutubeDL(ydl_opts) as ydl:
+        try:
+            info = ydl.extract_info(url, download=False)
+            metadata = {
+                'title': info.get('title'),
+                'uploader': info.get('uploader'),
+                'upload_date': info.get('upload_date'),
+                'view_count': info.get('view_count'),
+                'like_count': info.get('like_count'),
+                'duration': info.get('duration'),
+                'tags': info.get('tags'),
+                'description': info.get('description')
+            }
+
+            # Create a safe subset of metadata to log
+            safe_metadata = {
+                'title': metadata.get('title', 'No title'),
+                'duration': metadata.get('duration', 'Unknown duration'),
+                'upload_date': metadata.get('upload_date', 'Unknown upload date'),
+                'uploader': metadata.get('uploader', 'Unknown uploader')
+            }
+
+            logging.info(f"Successfully extracted metadata for {url}: {safe_metadata}")
+            return metadata
+        except Exception as e:
+            logging.error(f"Error extracting metadata for {url}: {str(e)}", exc_info=True)
+            return None
+
+
+def generate_timestamped_url(url, hours, minutes, seconds):
+    # Extract video ID from the URL
+    video_id_match = re.search(r'(?:v=|)([0-9A-Za-z_-]{11}).*', url)
+    if not video_id_match:
+        return "Invalid YouTube URL"
+
+    video_id = video_id_match.group(1)
+
+    # Calculate total seconds
+    total_seconds = int(hours) * 3600 + int(minutes) * 60 + int(seconds)
+
+    # Generate the new URL
+    new_url = f"https://www.youtube.com/watch?v={video_id}&t={total_seconds}s"
+
+    return new_url
+
+
+
+#
+#
+#######################################################################################################################

App_Function_Libraries/Web_Scraping/Article_Extractor_Lib.py

@@ -0,0 +1,381 @@
+# Article_Extractor_Lib.py
+#########################################
+# Article Extraction Library
+# This library is used to handle scraping and extraction of articles from web pages.
+#
+####################
+# Function List
+#
+# 1. get_page_title(url)
+# 2. get_article_text(url)
+# 3. get_article_title(article_url_arg)
+#
+####################
+#
+# Import necessary libraries
+import logging
+# 3rd-Party Imports
+import asyncio
+import os
+import tempfile
+from datetime import datetime
+from typing import List, Dict
+from urllib.parse import urljoin, urlparse
+from xml.dom import minidom
+from playwright.async_api import async_playwright
+from bs4 import BeautifulSoup
+import requests
+import trafilatura
+import xml.etree.ElementTree as ET
+
+
+# Import Local
+#
+#######################################################################################################################
+# Function Definitions
+#
+
+def get_page_title(url: str) -> str:
+    try:
+        response = requests.get(url)
+        response.raise_for_status()
+        soup = BeautifulSoup(response.text, 'html.parser')
+        title_tag = soup.find('title')
+        return title_tag.string.strip() if title_tag else "Untitled"
+    except requests.RequestException as e:
+        logging.error(f"Error fetching page title: {e}")
+        return "Untitled"
+
+
+async def scrape_article(url):
+    async def fetch_html(url: str) -> str:
+        async with async_playwright() as p:
+            browser = await p.chromium.launch(headless=True)
+            context = await browser.new_context(
+                user_agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3")
+            page = await context.new_page()
+            await page.goto(url)
+            await page.wait_for_load_state("networkidle")  # Wait for the network to be idle
+            content = await page.content()
+            await browser.close()
+            return content
+
+    # FIXME - Add option for extracting comments/tables/images
+    def extract_article_data(html: str, url: str) -> dict:
+        downloaded = trafilatura.extract(html, include_comments=False, include_tables=False, include_images=False)
+        metadata = trafilatura.extract_metadata(html)
+
+        result = {
+            'title': 'N/A',
+            'author': 'N/A',
+            'content': '',
+            'date': 'N/A',
+            'url': url,
+            'extraction_successful': False
+        }
+
+        if downloaded:
+            result['content'] = downloaded
+            result['extraction_successful'] = True
+
+        if metadata:
+            result.update({
+                'title': metadata.title if metadata.title else 'N/A',
+                'author': metadata.author if metadata.author else 'N/A',
+                'date': metadata.date if metadata.date else 'N/A'
+            })
+        else:
+            logging.warning("Metadata extraction failed.")
+
+        if not downloaded:
+            logging.warning("Content extraction failed.")
+
+        return result
+
+    def convert_html_to_markdown(html: str) -> str:
+        soup = BeautifulSoup(html, 'html.parser')
+        for para in soup.find_all('p'):
+            # Add a newline at the end of each paragraph for markdown separation
+            para.append('\n')
+        # Use .get_text() with separator to keep paragraph separation
+        return soup.get_text(separator='\n\n')
+
+    html = await fetch_html(url)
+    article_data = extract_article_data(html, url)
+    if article_data['extraction_successful']:
+        article_data['content'] = convert_html_to_markdown(article_data['content'])
+    return article_data
+
+
+def collect_internal_links(base_url: str) -> set:
+    visited = set()
+    to_visit = {base_url}
+
+    while to_visit:
+        current_url = to_visit.pop()
+        if current_url in visited:
+            continue
+
+        try:
+            response = requests.get(current_url)
+            response.raise_for_status()
+            soup = BeautifulSoup(response.text, 'html.parser')
+
+            # Collect internal links
+            for link in soup.find_all('a', href=True):
+                full_url = urljoin(base_url, link['href'])
+                # Only process links within the same domain
+                if urlparse(full_url).netloc == urlparse(base_url).netloc:
+                    if full_url not in visited:
+                        to_visit.add(full_url)
+
+            visited.add(current_url)
+        except requests.RequestException as e:
+            logging.error(f"Error visiting {current_url}: {e}")
+            continue
+
+    return visited
+
+
+def generate_temp_sitemap_from_links(links: set) -> str:
+    """
+    Generate a temporary sitemap file from collected links and return its path.
+
+    :param links: A set of URLs to include in the sitemap
+    :return: Path to the temporary sitemap file
+    """
+    # Create the root element
+    urlset = ET.Element("urlset")
+    urlset.set("xmlns", "http://www.sitemaps.org/schemas/sitemap/0.9")
+
+    # Add each link to the sitemap
+    for link in links:
+        url = ET.SubElement(urlset, "url")
+        loc = ET.SubElement(url, "loc")
+        loc.text = link
+        lastmod = ET.SubElement(url, "lastmod")
+        lastmod.text = datetime.now().strftime("%Y-%m-%d")
+        changefreq = ET.SubElement(url, "changefreq")
+        changefreq.text = "daily"
+        priority = ET.SubElement(url, "priority")
+        priority.text = "0.5"
+
+    # Create the tree and get it as a string
+    xml_string = ET.tostring(urlset, 'utf-8')
+
+    # Pretty print the XML
+    pretty_xml = minidom.parseString(xml_string).toprettyxml(indent="  ")
+
+    # Create a temporary file
+    with tempfile.NamedTemporaryFile(mode="w", suffix=".xml", delete=False) as temp_file:
+        temp_file.write(pretty_xml)
+        temp_file_path = temp_file.name
+
+    logging.info(f"Temporary sitemap created at: {temp_file_path}")
+    return temp_file_path
+
+
+def generate_sitemap_for_url(url: str) -> List[Dict[str, str]]:
+    """
+    Generate a sitemap for the given URL using the create_filtered_sitemap function.
+
+    Args:
+        url (str): The base URL to generate the sitemap for
+
+    Returns:
+        List[Dict[str, str]]: A list of dictionaries, each containing 'url' and 'title' keys
+    """
+    with tempfile.NamedTemporaryFile(mode="w+", suffix=".xml", delete=False) as temp_file:
+        create_filtered_sitemap(url, temp_file.name, is_content_page)
+        temp_file.seek(0)
+        tree = ET.parse(temp_file.name)
+        root = tree.getroot()
+
+        sitemap = []
+        for url_elem in root.findall(".//{http://www.sitemaps.org/schemas/sitemap/0.9}url"):
+            loc = url_elem.find("{http://www.sitemaps.org/schemas/sitemap/0.9}loc").text
+            sitemap.append({"url": loc, "title": loc.split("/")[-1] or url})  # Use the last part of the URL as a title
+
+    return sitemap
+
+async def scrape_entire_site(base_url: str) -> List[Dict]:
+    """
+    Scrape the entire site by generating a temporary sitemap and extracting content from each page.
+
+    :param base_url: The base URL of the site to scrape
+    :return: A list of dictionaries containing scraped article data
+    """
+    # Step 1: Collect internal links from the site
+    links = collect_internal_links(base_url)
+    logging.info(f"Collected {len(links)} internal links.")
+
+    # Step 2: Generate the temporary sitemap
+    temp_sitemap_path = generate_temp_sitemap_from_links(links)
+
+    # Step 3: Scrape each URL in the sitemap
+    scraped_articles = []
+    try:
+        async def scrape_and_log(link):
+            logging.info(f"Scraping {link} ...")
+            article_data = await scrape_article(link)
+
+            if article_data:
+                logging.info(f"Title: {article_data['title']}")
+                logging.info(f"Author: {article_data['author']}")
+                logging.info(f"Date: {article_data['date']}")
+                logging.info(f"Content: {article_data['content'][:500]}...")
+
+                return article_data
+            return None
+
+        # Use asyncio.gather to scrape multiple articles concurrently
+        scraped_articles = await asyncio.gather(*[scrape_and_log(link) for link in links])
+        # Remove any None values (failed scrapes)
+        scraped_articles = [article for article in scraped_articles if article is not None]
+
+    finally:
+        # Clean up the temporary sitemap file
+        os.unlink(temp_sitemap_path)
+        logging.info("Temporary sitemap file deleted")
+
+    return scraped_articles
+
+
+def scrape_by_url_level(base_url: str, level: int) -> list:
+    """Scrape articles from URLs up to a certain level under the base URL."""
+
+    def get_url_level(url: str) -> int:
+        return len(urlparse(url).path.strip('/').split('/'))
+
+    links = collect_internal_links(base_url)
+    filtered_links = [link for link in links if get_url_level(link) <= level]
+
+    return [article for link in filtered_links if (article := scrape_article(link))]
+
+
+def scrape_from_sitemap(sitemap_url: str) -> list:
+    """Scrape articles from a sitemap URL."""
+    try:
+        response = requests.get(sitemap_url)
+        response.raise_for_status()
+        root = ET.fromstring(response.content)
+
+        return [article for url in root.findall('.//{http://www.sitemaps.org/schemas/sitemap/0.9}loc')
+                if (article := scrape_article(url.text))]
+    except requests.RequestException as e:
+        logging.error(f"Error fetching sitemap: {e}")
+        return []
+
+
+def convert_to_markdown(articles: list) -> str:
+    """Convert a list of article data into a single markdown document."""
+    markdown = ""
+    for article in articles:
+        markdown += f"# {article['title']}\n\n"
+        markdown += f"Author: {article['author']}\n"
+        markdown += f"Date: {article['date']}\n\n"
+        markdown += f"{article['content']}\n\n"
+        markdown += "---\n\n"  # Separator between articles
+    return markdown
+
+
+def is_content_page(url: str) -> bool:
+    """
+    Determine if a URL is likely to be a content page.
+    This is a basic implementation and may need to be adjusted based on the specific website structure.
+
+    :param url: The URL to check
+    :return: True if the URL is likely a content page, False otherwise
+    """
+    #Add more specific checks here based on the website's structure
+    # Exclude common non-content pages
+    exclude_patterns = [
+        '/tag/', '/category/', '/author/', '/search/', '/page/',
+        'wp-content', 'wp-includes', 'wp-json', 'wp-admin',
+        'login', 'register', 'cart', 'checkout', 'account',
+        '.jpg', '.png', '.gif', '.pdf', '.zip'
+    ]
+    return not any(pattern in url.lower() for pattern in exclude_patterns)
+
+
+def create_filtered_sitemap(base_url: str, output_file: str, filter_function):
+    """
+    Create a sitemap from internal links and filter them based on a custom function.
+
+    :param base_url: The base URL of the website
+    :param output_file: The file to save the sitemap to
+    :param filter_function: A function that takes a URL and returns True if it should be included
+    """
+    links = collect_internal_links(base_url)
+    filtered_links = set(filter(filter_function, links))
+
+    root = ET.Element("urlset")
+    root.set("xmlns", "http://www.sitemaps.org/schemas/sitemap/0.9")
+
+    for link in filtered_links:
+        url = ET.SubElement(root, "url")
+        loc = ET.SubElement(url, "loc")
+        loc.text = link
+
+    tree = ET.ElementTree(root)
+    tree.write(output_file, encoding='utf-8', xml_declaration=True)
+    print(f"Filtered sitemap saved to {output_file}")
+
+
+def scrape_from_filtered_sitemap(sitemap_file: str, filter_function) -> list:
+    """
+    Scrape articles from a sitemap file, applying an additional filter function.
+
+    :param sitemap_file: Path to the sitemap file
+    :param filter_function: A function that takes a URL and returns True if it should be scraped
+    :return: List of scraped articles
+    """
+    try:
+        tree = ET.parse(sitemap_file)
+        root = tree.getroot()
+
+        articles = []
+        for url in root.findall('.//{http://www.sitemaps.org/schemas/sitemap/0.9}loc'):
+            if filter_function(url.text):
+                article_data = scrape_article(url.text)
+                if article_data:
+                    articles.append(article_data)
+
+        return articles
+    except ET.ParseError as e:
+        logging.error(f"Error parsing sitemap: {e}")
+        return []
+
+
+def scrape_and_convert_with_filter(source: str, output_file: str, filter_function=is_content_page, level: int = None):
+    """
+    Scrape articles from a sitemap or by URL level, apply filtering, and convert to a single markdown file.
+
+    :param source: URL of the sitemap, base URL for level-based scraping, or path to a local sitemap file
+    :param output_file: Path to save the output markdown file
+    :param filter_function: Function to filter URLs (default is is_content_page)
+    :param level: URL level for scraping (None if using sitemap)
+    """
+    if level is not None:
+        # Scraping by URL level
+        articles = scrape_by_url_level(source, level)
+        articles = [article for article in articles if filter_function(article['url'])]
+    elif source.startswith('http'):
+        # Scraping from online sitemap
+        articles = scrape_from_sitemap(source)
+        articles = [article for article in articles if filter_function(article['url'])]
+    else:
+        # Scraping from local sitemap file
+        articles = scrape_from_filtered_sitemap(source, filter_function)
+
+    articles = [article for article in articles if filter_function(article['url'])]
+    markdown_content = convert_to_markdown(articles)
+
+    with open(output_file, 'w', encoding='utf-8') as f:
+        f.write(markdown_content)
+
+    logging.info(f"Scraped and filtered content saved to {output_file}")
+
+#
+#
+#######################################################################################################################

App_Function_Libraries/Web_Scraping/Article_Summarization_Lib.py

@@ -0,0 +1,249 @@
+# Article_Summarization_Lib.py
+#########################################
+# Article Summarization Library
+# This library is used to handle summarization of articles.
+import asyncio
+# FIXME - this library should be refactored into `Article_Extractor_Lib` and then renamed to `Web_Scraping_Lib`
+
+#
+####
+#
+####################
+# Function List
+#
+# 1.
+#
+####################
+#
+# Import necessary libraries
+import datetime
+from datetime import datetime
+import gradio as gr
+import json
+import os
+import logging
+import requests
+# 3rd-Party Imports
+#
+# Local Imports
+from App_Function_Libraries.Utils.Utils import sanitize_filename
+from App_Function_Libraries.Web_Scraping.Article_Extractor_Lib import scrape_article
+from App_Function_Libraries.Summarization.Local_Summarization_Lib import summarize_with_llama, summarize_with_oobabooga, summarize_with_tabbyapi, \
+    summarize_with_vllm, summarize_with_kobold, save_summary_to_file, summarize_with_local_llm
+from App_Function_Libraries.Summarization.Summarization_General_Lib import summarize_with_openai, summarize_with_anthropic, summarize_with_cohere, \
+    summarize_with_groq, summarize_with_openrouter, summarize_with_deepseek, summarize_with_huggingface, \
+    summarize_with_mistral
+from App_Function_Libraries.DB.DB_Manager import ingest_article_to_db
+#
+#######################################################################################################################
+# Function Definitions
+#
+
+async def scrape_and_summarize_multiple(urls, custom_prompt_arg, api_name, api_key, keywords, custom_article_titles, system_message=None):
+    urls = [url.strip() for url in urls.split('\n') if url.strip()]
+    custom_titles = custom_article_titles.split('\n') if custom_article_titles else []
+
+    results = []
+    errors = []
+
+    # Create a progress bar
+    progress = gr.Progress()
+
+    # FIXME - add progress tracking to the gradio UI
+    for i, url in enumerate(urls):
+        custom_title = custom_titles[i] if i < len(custom_titles) else None
+        try:
+            article = await scrape_article(url)
+            if article and article['extraction_successful']:
+                if custom_title:
+                    article['title'] = custom_title
+                results.append(article)
+        except Exception as e:
+            error_message = f"Error processing URL {i + 1} ({url}): {str(e)}"
+            errors.append(error_message)
+
+        # Update progress
+        progress((i + 1) / len(urls), desc=f"Processed {i + 1}/{len(urls)} URLs")
+
+    if errors:
+        logging.error("\n".join(errors))
+
+    return results
+
+
+
+def scrape_and_summarize(url, custom_prompt_arg, api_name, api_key, keywords, custom_article_title, system_message=None):
+    try:
+        # Step 1: Scrape the article
+        article_data = asyncio.run(scrape_article(url))
+        print(f"Scraped Article Data: {article_data}")  # Debugging statement
+        if not article_data:
+            return "Failed to scrape the article."
+
+        # Use the custom title if provided, otherwise use the scraped title
+        title = custom_article_title.strip() if custom_article_title else article_data.get('title', 'Untitled')
+        author = article_data.get('author', 'Unknown')
+        content = article_data.get('content', '')
+        ingestion_date = datetime.now().strftime('%Y-%m-%d')
+
+        print(f"Title: {title}, Author: {author}, Content Length: {len(content)}")  # Debugging statement
+
+        # Custom system prompt for the article
+        system_message = system_message or "Act as a professional summarizer and summarize this article."
+        # Custom prompt for the article
+        article_custom_prompt = custom_prompt_arg or "Act as a professional summarizer and summarize this article."
+
+        # Step 2: Summarize the article
+        summary = None
+        if api_name:
+            logging.debug(f"Article_Summarizer: Summarization being performed by {api_name}")
+
+            # Sanitize filename for saving the JSON file
+            sanitized_title = sanitize_filename(title)
+            json_file_path = os.path.join("Results", f"{sanitized_title}_segments.json")
+
+            with open(json_file_path, 'w') as json_file:
+                json.dump([{'text': content}], json_file, indent=2)
+
+            try:
+                if api_name.lower() == 'openai':
+                    # def summarize_with_openai(api_key, input_data, custom_prompt_arg)
+                    summary = summarize_with_openai(api_key, json_file_path, article_custom_prompt, system_message)
+
+                elif api_name.lower() == "anthropic":
+                    # def summarize_with_anthropic(api_key, input_data, model, custom_prompt_arg, max_retries=3, retry_delay=5):
+                    summary = summarize_with_anthropic(api_key, json_file_path, article_custom_prompt, system_message)
+                elif api_name.lower() == "cohere":
+                    # def summarize_with_cohere(api_key, input_data, model, custom_prompt_arg)
+                    summary = summarize_with_cohere(api_key, json_file_path, article_custom_prompt, system_message)
+
+                elif api_name.lower() == "groq":
+                    logging.debug(f"MAIN: Trying to summarize with groq")
+                    # def summarize_with_groq(api_key, input_data, model, custom_prompt_arg):
+                    summary = summarize_with_groq(api_key, json_file_path, article_custom_prompt, system_message)
+
+                elif api_name.lower() == "openrouter":
+                    logging.debug(f"MAIN: Trying to summarize with OpenRouter")
+                    # def summarize_with_openrouter(api_key, input_data, custom_prompt_arg):
+                    summary = summarize_with_openrouter(api_key, json_file_path, article_custom_prompt, system_message)
+
+                elif api_name.lower() == "deepseek":
+                    logging.debug(f"MAIN: Trying to summarize with DeepSeek")
+                    # def summarize_with_deepseek(api_key, input_data, custom_prompt_arg):
+                    summary = summarize_with_deepseek(api_key, json_file_path, article_custom_prompt, system_message)
+
+                elif api_name.lower() == "mistral":
+                    summary = summarize_with_mistral(api_key, json_file_path, article_custom_prompt, system_message)
+
+                elif api_name.lower() == "llama.cpp":
+                    logging.debug(f"MAIN: Trying to summarize with Llama.cpp")
+                    # def summarize_with_llama(api_url, file_path, token, custom_prompt)
+                    summary = summarize_with_llama(json_file_path, article_custom_prompt, system_message)
+
+                elif api_name.lower() == "kobold":
+                    logging.debug(f"MAIN: Trying to summarize with Kobold.cpp")
+                    # def summarize_with_kobold(input_data, kobold_api_token, custom_prompt_input, api_url):
+                    summary = summarize_with_kobold(json_file_path, api_key, article_custom_prompt, system_message)
+
+                elif api_name.lower() == "ooba":
+                    # def summarize_with_oobabooga(input_data, api_key, custom_prompt, api_url):
+                    summary = summarize_with_oobabooga(json_file_path, api_key, article_custom_prompt, system_message)
+
+                elif api_name.lower() == "tabbyapi":
+                    # def summarize_with_tabbyapi(input_data, tabby_model, custom_prompt_input, api_key=None, api_IP):
+                    summary = summarize_with_tabbyapi(json_file_path, article_custom_prompt, system_message)
+
+                elif api_name.lower() == "vllm":
+                    logging.debug(f"MAIN: Trying to summarize with VLLM")
+                    # def summarize_with_vllm(api_key, input_data, custom_prompt_input):
+                    summary = summarize_with_vllm(json_file_path, article_custom_prompt, system_message)
+
+                elif api_name.lower() == "local-llm":
+                    logging.debug(f"MAIN: Trying to summarize with Local LLM")
+                    summary = summarize_with_local_llm(json_file_path, article_custom_prompt, system_message)
+
+                elif api_name.lower() == "huggingface":
+                    logging.debug(f"MAIN: Trying to summarize with huggingface")
+                    # def summarize_with_huggingface(api_key, input_data, custom_prompt_arg):
+                    summarize_with_huggingface(api_key, json_file_path, article_custom_prompt, system_message)
+                # Add additional API handlers here...
+
+            except requests.exceptions.ConnectionError as e:
+                logging.error(f"Connection error while trying to summarize with {api_name}: {str(e)}")
+
+            if summary:
+                logging.info(f"Article_Summarizer: Summary generated using {api_name} API")
+                save_summary_to_file(summary, json_file_path)
+            else:
+                summary = "Summary not available"
+                logging.warning(f"Failed to generate summary using {api_name} API")
+
+        else:
+            summary = "Article Summarization: No API provided for summarization."
+
+        print(f"Summary: {summary}")  # Debugging statement
+
+        # Step 3: Ingest the article into the database
+        ingestion_result = ingest_article_to_db(url, title, author, content, keywords, summary, ingestion_date,
+                                                article_custom_prompt)
+
+        return f"Title: {title}\nAuthor: {author}\nIngestion Result: {ingestion_result}\n\nSummary: {summary}\n\nArticle Contents: {content}"
+    except Exception as e:
+        logging.error(f"Error processing URL {url}: {str(e)}")
+        return f"Failed to process URL {url}: {str(e)}"
+
+
+def scrape_and_no_summarize_then_ingest(url, keywords, custom_article_title):
+    try:
+        # Step 1: Scrape the article
+        article_data = asyncio.run(scrape_article(url))
+        print(f"Scraped Article Data: {article_data}")  # Debugging statement
+        if not article_data:
+            return "Failed to scrape the article."
+
+        # Use the custom title if provided, otherwise use the scraped title
+        title = custom_article_title.strip() if custom_article_title else article_data.get('title', 'Untitled')
+        author = article_data.get('author', 'Unknown')
+        content = article_data.get('content', '')
+        ingestion_date = datetime.now().strftime('%Y-%m-%d')
+
+        print(f"Title: {title}, Author: {author}, Content Length: {len(content)}")  # Debugging statement
+
+        # Step 2: Ingest the article into the database
+        ingestion_result = ingest_article_to_db(url, title, author, content, keywords, ingestion_date, None, None)
+
+        return f"Title: {title}\nAuthor: {author}\nIngestion Result: {ingestion_result}\n\nArticle Contents: {content}"
+    except Exception as e:
+        logging.error(f"Error processing URL {url}: {str(e)}")
+        return f"Failed to process URL {url}: {str(e)}"
+
+
+def ingest_unstructured_text(text, custom_prompt, api_name, api_key, keywords, custom_article_title, system_message=None):
+    title = custom_article_title.strip() if custom_article_title else "Unstructured Text"
+    author = "Unknown"
+    ingestion_date = datetime.now().strftime('%Y-%m-%d')
+
+    # Summarize the unstructured text
+    if api_name:
+        json_file_path = f"Results/{title.replace(' ', '_')}_segments.json"
+        with open(json_file_path, 'w') as json_file:
+            json.dump([{'text': text}], json_file, indent=2)
+
+        if api_name.lower() == 'openai':
+            summary = summarize_with_openai(api_key, json_file_path, custom_prompt, system_message)
+        # Add other APIs as needed
+        else:
+            summary = "Unsupported API."
+    else:
+        summary = "No API provided for summarization."
+
+    # Ingest the unstructured text into the database
+    ingestion_result = ingest_article_to_db('Unstructured Text', title, author, text, keywords, summary, ingestion_date,
+                                            custom_prompt)
+    return f"Title: {title}\nSummary: {summary}\nIngestion Result: {ingestion_result}"
+
+
+
+#
+#
+#######################################################################################################################