Update App_Function_Libraries/Gradio_UI/Website_scraping_tab.py

App_Function_Libraries/Gradio_UI/Website_scraping_tab.py

@@ -1,554 +1,554 @@
-# Website_scraping_tab.py
-# Gradio UI for scraping websites
-#
-# Imports
-import asyncio
-import json
-import logging
-import os
-import random
-from concurrent.futures import ThreadPoolExecutor
-from typing import Optional, List, Dict, Any
-from urllib.parse import urlparse, urljoin
-
-#
-# External Imports
-import gradio as gr
-from playwright.async_api import TimeoutError, async_playwright
-from playwright.sync_api import sync_playwright
-
-#
-# Local Imports
-from App_Function_Libraries.Web_Scraping.Article_Extractor_Lib import scrape_from_sitemap, scrape_by_url_level, scrape_article
-from App_Function_Libraries.Web_Scraping.Article_Summarization_Lib import scrape_and_summarize_multiple
-from App_Function_Libraries.DB.DB_Manager import load_preset_prompts
-from App_Function_Libraries.Gradio_UI.Chat_ui import update_user_prompt
-from App_Function_Libraries.Summarization.Summarization_General_Lib import summarize
-
-
-#
-########################################################################################################################
-#
-# Functions:
-
-def get_url_depth(url: str) -> int:
-    return len(urlparse(url).path.strip('/').split('/'))
-
-
-def sync_recursive_scrape(url_input, max_pages, max_depth, progress_callback, delay=1.0):
-    def run_async_scrape():
-        loop = asyncio.new_event_loop()
-        asyncio.set_event_loop(loop)
-        return loop.run_until_complete(
-            recursive_scrape(url_input, max_pages, max_depth, progress_callback, delay)
-        )
-
-    with ThreadPoolExecutor() as executor:
-        future = executor.submit(run_async_scrape)
-        return future.result()
-
-
-async def recursive_scrape(
-        base_url: str,
-        max_pages: int,
-        max_depth: int,
-        progress_callback: callable,
-        delay: float = 1.0,
-        resume_file: str = 'scrape_progress.json',
-        user_agent: str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"
-) -> List[Dict]:
-    async def save_progress():
-        temp_file = resume_file + ".tmp"
-        with open(temp_file, 'w') as f:
-            json.dump({
-                'visited': list(visited),
-                'to_visit': to_visit,
-                'scraped_articles': scraped_articles,
-                'pages_scraped': pages_scraped
-            }, f)
-        os.replace(temp_file, resume_file)  # Atomic replace
-
-    def is_valid_url(url: str) -> bool:
-        return url.startswith("http") and len(url) > 0
-
-    # Load progress if resume file exists
-    if os.path.exists(resume_file):
-        with open(resume_file, 'r') as f:
-            progress_data = json.load(f)
-            visited = set(progress_data['visited'])
-            to_visit = progress_data['to_visit']
-            scraped_articles = progress_data['scraped_articles']
-            pages_scraped = progress_data['pages_scraped']
-    else:
-        visited = set()
-        to_visit = [(base_url, 0)]  # (url, depth)
-        scraped_articles = []
-        pages_scraped = 0
-
-    try:
-        async with async_playwright() as p:
-            browser = await p.chromium.launch(headless=True)
-            context = await browser.new_context(user_agent=user_agent)
-
-            try:
-                while to_visit and pages_scraped < max_pages:
-                    current_url, current_depth = to_visit.pop(0)
-
-                    if current_url in visited or current_depth > max_depth:
-                        continue
-
-                    visited.add(current_url)
-
-                    # Update progress
-                    progress_callback(f"Scraping page {pages_scraped + 1}/{max_pages}: {current_url}")
-
-                    try:
-                        await asyncio.sleep(random.uniform(delay * 0.8, delay * 1.2))
-
-                        # This function should be implemented to handle asynchronous scraping
-                        article_data = await scrape_article_async(context, current_url)
-
-                        if article_data and article_data['extraction_successful']:
-                            scraped_articles.append(article_data)
-                            pages_scraped += 1
-
-                        # If we haven't reached max depth, add child links to to_visit
-                        if current_depth < max_depth:
-                            page = await context.new_page()
-                            await page.goto(current_url)
-                            await page.wait_for_load_state("networkidle")
-
-                            links = await page.eval_on_selector_all('a[href]',
-                                                                    "(elements) => elements.map(el => el.href)")
-                            for link in links:
-                                child_url = urljoin(base_url, link)
-                                if is_valid_url(child_url) and child_url.startswith(
-                                        base_url) and child_url not in visited and should_scrape_url(child_url):
-                                    to_visit.append((child_url, current_depth + 1))
-
-                            await page.close()
-
-                    except Exception as e:
-                        logging.error(f"Error scraping {current_url}: {str(e)}")
-
-                    # Save progress periodically (e.g., every 10 pages)
-                    if pages_scraped % 10 == 0:
-                        await save_progress()
-
-            finally:
-                await browser.close()
-
-    finally:
-        # These statements are now guaranteed to be reached after the scraping is done
-        await save_progress()
-
-        # Remove the progress file when scraping is completed successfully
-        if os.path.exists(resume_file):
-            os.remove(resume_file)
-
-        # Final progress update
-        progress_callback(f"Scraping completed. Total pages scraped: {pages_scraped}")
-
-        return scraped_articles
-
-
-async def scrape_article_async(context, url: str) -> Dict[str, Any]:
-    page = await context.new_page()
-    try:
-        await page.goto(url)
-        await page.wait_for_load_state("networkidle")
-
-        title = await page.title()
-        content = await page.content()
-
-        return {
-            'url': url,
-            'title': title,
-            'content': content,
-            'extraction_successful': True
-        }
-    except Exception as e:
-        logging.error(f"Error scraping article {url}: {str(e)}")
-        return {
-            'url': url,
-            'extraction_successful': False,
-            'error': str(e)
-        }
-    finally:
-        await page.close()
-
-
-def scrape_article_sync(url: str) -> Dict[str, Any]:
-    with sync_playwright() as p:
-        browser = p.chromium.launch(headless=True)
-        page = browser.new_page()
-        try:
-            page.goto(url)
-            page.wait_for_load_state("networkidle")
-
-            title = page.title()
-            content = page.content()
-
-            return {
-                'url': url,
-                'title': title,
-                'content': content,
-                'extraction_successful': True
-            }
-        except Exception as e:
-            logging.error(f"Error scraping article {url}: {str(e)}")
-            return {
-                'url': url,
-                'extraction_successful': False,
-                'error': str(e)
-            }
-        finally:
-            browser.close()
-
-
-def should_scrape_url(url: str) -> bool:
-    parsed_url = urlparse(url)
-    path = parsed_url.path.lower()
-
-    # List of patterns to exclude
-    exclude_patterns = [
-        '/tag/', '/category/', '/author/', '/search/', '/page/',
-        'wp-content', 'wp-includes', 'wp-json', 'wp-admin',
-        'login', 'register', 'cart', 'checkout', 'account',
-        '.jpg', '.png', '.gif', '.pdf', '.zip'
-    ]
-
-    # Check if the URL contains any exclude patterns
-    if any(pattern in path for pattern in exclude_patterns):
-        return False
-
-    # Add more sophisticated checks here
-    # For example, you might want to only include URLs with certain patterns
-    include_patterns = ['/article/', '/post/', '/blog/']
-    if any(pattern in path for pattern in include_patterns):
-        return True
-
-    # By default, return True if no exclusion or inclusion rules matched
-    return True
-
-
-async def scrape_with_retry(url: str, max_retries: int = 3, retry_delay: float = 5.0):
-    for attempt in range(max_retries):
-        try:
-            return await scrape_article(url)
-        except TimeoutError:
-            if attempt < max_retries - 1:
-                logging.warning(f"Timeout error scraping {url}. Retrying in {retry_delay} seconds...")
-                await asyncio.sleep(retry_delay)
-            else:
-                logging.error(f"Failed to scrape {url} after {max_retries} attempts.")
-                return None
-        except Exception as e:
-            logging.error(f"Error scraping {url}: {str(e)}")
-            return None
-
-
-def create_website_scraping_tab():
-        with gr.TabItem("Website Scraping", visible=True):
-            gr.Markdown("# Scrape Websites & Summarize Articles")
-        with gr.Row():
-            with gr.Column():
-                scrape_method = gr.Radio(
-                    ["Individual URLs", "Sitemap", "URL Level", "Recursive Scraping"],
-                    label="Scraping Method",
-                    value="Individual URLs"
-                )
-                url_input = gr.Textbox(
-                    label="Article URLs or Base URL",
-                    placeholder="Enter article URLs here, one per line, or base URL for sitemap/URL level/recursive scraping",
-                    lines=5
-                )
-                url_level = gr.Slider(
-                    minimum=1,
-                    maximum=10,
-                    step=1,
-                    label="URL Level (for URL Level scraping)",
-                    value=2,
-                    visible=False
-                )
-                max_pages = gr.Slider(
-                    minimum=1,
-                    maximum=100,
-                    step=1,
-                    label="Maximum Pages to Scrape (for Recursive Scraping)",
-                    value=10,
-                    visible=False
-                )
-                max_depth = gr.Slider(
-                    minimum=1,
-                    maximum=10,
-                    step=1,
-                    label="Maximum Depth (for Recursive Scraping)",
-                    value=3,
-                    visible=False
-                )
-                custom_article_title_input = gr.Textbox(
-                    label="Custom Article Titles (Optional, one per line)",
-                    placeholder="Enter custom titles for the articles, one per line",
-                    lines=5
-                )
-                with gr.Row():
-                    summarize_checkbox = gr.Checkbox(label="Summarize Articles", value=False)
-                    custom_prompt_checkbox = gr.Checkbox(label="Use a Custom Prompt", value=False, visible=True)
-                    preset_prompt_checkbox = gr.Checkbox(label="Use a pre-set Prompt", value=False, visible=True)
-                with gr.Row():
-                    temp_slider = gr.Slider(0.1, 2.0, 0.7, label="Temperature")
-                with gr.Row():
-                    preset_prompt = gr.Dropdown(
-                        label="Select Preset Prompt",
-                        choices=load_preset_prompts(),
-                        visible=False
-                    )
-                with gr.Row():
-                    website_custom_prompt_input = gr.Textbox(
-                        label="Custom Prompt",
-                        placeholder="Enter custom prompt here",
-                        lines=3,
-                        visible=False
-                    )
-                with gr.Row():
-                    system_prompt_input = gr.Textbox(
-                        label="System Prompt",
-                        value="""<s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
-                                **Bulleted Note Creation Guidelines**
-                                
-                                **Headings**:
-                                - Based on referenced topics, not categories like quotes or terms
-                                - Surrounded by **bold** formatting 
-                                - Not listed as bullet points
-                                - No space between headings and list items underneath
-                                
-                                **Emphasis**:
-                                - **Important terms** set in bold font
-                                - **Text ending in a colon**: also bolded
-                                
-                                **Review**:
-                                - Ensure adherence to specified format
-                                - Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]
-                                """,
-                        lines=3,
-                        visible=False
-                    )
-
-                api_name_input = gr.Dropdown(
-                    choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral",
-                             "OpenRouter",
-                             "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace",
-                             "Custom-OpenAI-API"],
-                    value=None,
-                    label="API Name (Mandatory for Summarization)"
-                )
-                api_key_input = gr.Textbox(
-                    label="API Key (Mandatory if API Name is specified)",
-                    placeholder="Enter your API key here; Ignore if using Local API or Built-in API",
-                    type="password"
-                )
-                keywords_input = gr.Textbox(
-                    label="Keywords",
-                    placeholder="Enter keywords here (comma-separated)",
-                    value="default,no_keyword_set",
-                    visible=True
-                )
-
-                scrape_button = gr.Button("Scrape and Summarize")
-            with gr.Column():
-                progress_output = gr.Textbox(label="Progress", lines=3)
-                result_output = gr.Textbox(label="Result", lines=20)
-
-        def update_ui_for_scrape_method(method):
-            url_level_update = gr.update(visible=(method == "URL Level"))
-            max_pages_update = gr.update(visible=(method == "Recursive Scraping"))
-            max_depth_update = gr.update(visible=(method == "Recursive Scraping"))
-            url_input_update = gr.update(
-                label="Article URLs" if method == "Individual URLs" else "Base URL",
-                placeholder="Enter article URLs here, one per line" if method == "Individual URLs" else "Enter the base URL for scraping"
-            )
-            return url_level_update, max_pages_update, max_depth_update, url_input_update
-
-        scrape_method.change(
-            fn=update_ui_for_scrape_method,
-            inputs=[scrape_method],
-            outputs=[url_level, max_pages, max_depth, url_input]
-        )
-
-        custom_prompt_checkbox.change(
-            fn=lambda x: (gr.update(visible=x), gr.update(visible=x)),
-            inputs=[custom_prompt_checkbox],
-            outputs=[website_custom_prompt_input, system_prompt_input]
-        )
-        preset_prompt_checkbox.change(
-            fn=lambda x: gr.update(visible=x),
-            inputs=[preset_prompt_checkbox],
-            outputs=[preset_prompt]
-        )
-
-        def update_prompts(preset_name):
-            prompts = update_user_prompt(preset_name)
-            return (
-                gr.update(value=prompts["user_prompt"], visible=True),
-                gr.update(value=prompts["system_prompt"], visible=True)
-            )
-
-        preset_prompt.change(
-            update_prompts,
-            inputs=preset_prompt,
-            outputs=[website_custom_prompt_input, system_prompt_input]
-        )
-
-        async def scrape_and_summarize_wrapper(
-                scrape_method: str,
-                url_input: str,
-                url_level: Optional[int],
-                max_pages: int,
-                max_depth: int,
-                summarize_checkbox: bool,
-                custom_prompt: Optional[str],
-                api_name: Optional[str],
-                api_key: Optional[str],
-                keywords: str,
-                custom_titles: Optional[str],
-                system_prompt: Optional[str],
-                temperature: float = 0.7,
-                progress: gr.Progress = gr.Progress()
-        ) -> str:
-            try:
-                result: List[Dict[str, Any]] = []
-
-                if scrape_method == "Individual URLs":
-                    result = await scrape_and_summarize_multiple(url_input, custom_prompt, api_name, api_key, keywords,
-                                                                 custom_titles, system_prompt)
-                elif scrape_method == "Sitemap":
-                    result = await asyncio.to_thread(scrape_from_sitemap, url_input)
-                elif scrape_method == "URL Level":
-                    if url_level is None:
-                        return convert_json_to_markdown(
-                            json.dumps({"error": "URL level is required for URL Level scraping."}))
-                    result = await asyncio.to_thread(scrape_by_url_level, url_input, url_level)
-                elif scrape_method == "Recursive Scraping":
-                    result = await recursive_scrape(url_input, max_pages, max_depth, progress.update, delay=1.0)
-                else:
-                    return convert_json_to_markdown(json.dumps({"error": f"Unknown scraping method: {scrape_method}"}))
-
-                # Ensure result is always a list of dictionaries
-                if isinstance(result, dict):
-                    result = [result]
-                elif isinstance(result, list):
-                    if all(isinstance(item, str) for item in result):
-                        # Convert list of strings to list of dictionaries
-                        result = [{"content": item} for item in result]
-                    elif not all(isinstance(item, dict) for item in result):
-                        raise ValueError("Not all items in result are dictionaries or strings")
-                else:
-                    raise ValueError(f"Unexpected result type: {type(result)}")
-
-                # Ensure all items in result are dictionaries
-                if not all(isinstance(item, dict) for item in result):
-                    raise ValueError("Not all items in result are dictionaries")
-
-                if summarize_checkbox:
-                    total_articles = len(result)
-                    for i, article in enumerate(result):
-                        progress.update(f"Summarizing article {i + 1}/{total_articles}")
-                        content = article.get('content', '')
-                        if content:
-                            summary = await asyncio.to_thread(summarize, content, custom_prompt, api_name, api_key,
-                                                              temperature, system_prompt)
-                            article['summary'] = summary
-                        else:
-                            article['summary'] = "No content available to summarize."
-
-                # Concatenate all content
-                all_content = "\n\n".join(
-                    [f"# {article.get('title', 'Untitled')}\n\n{article.get('content', '')}\n\n" +
-                     (f"Summary: {article.get('summary', '')}" if summarize_checkbox else "")
-                     for article in result])
-
-                # Collect all unique URLs
-                all_urls = list(set(article.get('url', '') for article in result if article.get('url')))
-
-                # Structure the output for the entire website collection
-                website_collection = {
-                    "base_url": url_input,
-                    "scrape_method": scrape_method,
-                    "summarization_performed": summarize_checkbox,
-                    "api_used": api_name if summarize_checkbox else None,
-                    "keywords": keywords if summarize_checkbox else None,
-                    "url_level": url_level if scrape_method == "URL Level" else None,
-                    "max_pages": max_pages if scrape_method == "Recursive Scraping" else None,
-                    "max_depth": max_depth if scrape_method == "Recursive Scraping" else None,
-                    "total_articles_scraped": len(result),
-                    "urls_scraped": all_urls,
-                    "content": all_content
-                }
-
-                # Convert the JSON to markdown and return
-                return convert_json_to_markdown(json.dumps(website_collection, indent=2))
-            except Exception as e:
-                return convert_json_to_markdown(json.dumps({"error": f"An error occurred: {str(e)}"}))
-
-        # Update the scrape_button.click to include the temperature parameter
-        scrape_button.click(
-            fn=lambda *args: asyncio.run(scrape_and_summarize_wrapper(*args)),
-            inputs=[scrape_method, url_input, url_level, max_pages, max_depth, summarize_checkbox,
-                    website_custom_prompt_input, api_name_input, api_key_input, keywords_input,
-                    custom_article_title_input, system_prompt_input, temp_slider],
-            outputs=[result_output]
-        )
-
-
-def convert_json_to_markdown(json_str: str) -> str:
-    """
-    Converts the JSON output from the scraping process into a markdown format.
-
-    Args:
-        json_str (str): JSON-formatted string containing the website collection data
-
-    Returns:
-        str: Markdown-formatted string of the website collection data
-    """
-    try:
-        # Parse the JSON string
-        data = json.loads(json_str)
-
-        # Check if there's an error in the JSON
-        if "error" in data:
-            return f"# Error\n\n{data['error']}"
-
-        # Start building the markdown string
-        markdown = f"# Website Collection: {data['base_url']}\n\n"
-
-        # Add metadata
-        markdown += "## Metadata\n\n"
-        markdown += f"- **Scrape Method:** {data['scrape_method']}\n"
-        markdown += f"- **API Used:** {data['api_used']}\n"
-        markdown += f"- **Keywords:** {data['keywords']}\n"
-        if data['url_level'] is not None:
-            markdown += f"- **URL Level:** {data['url_level']}\n"
-        markdown += f"- **Total Articles Scraped:** {data['total_articles_scraped']}\n\n"
-
-        # Add URLs scraped
-        markdown += "## URLs Scraped\n\n"
-        for url in data['urls_scraped']:
-            markdown += f"- {url}\n"
-        markdown += "\n"
-
-        # Add the content
-        markdown += "## Content\n\n"
-        markdown += data['content']
-
-        return markdown
-
-    except json.JSONDecodeError:
-        return "# Error\n\nInvalid JSON string provided."
-    except KeyError as e:
-        return f"# Error\n\nMissing key in JSON data: {str(e)}"
-    except Exception as e:
-        return f"# Error\n\nAn unexpected error occurred: {str(e)}"
-#
-# End of File
-########################################################################################################################
+# Website_scraping_tab.py
+# Gradio UI for scraping websites
+#
+# Imports
+import asyncio
+import json
+import logging
+import os
+import random
+from concurrent.futures import ThreadPoolExecutor
+from typing import Optional, List, Dict, Any
+from urllib.parse import urlparse, urljoin
+
+#
+# External Imports
+import gradio as gr
+from playwright.async_api import TimeoutError, async_playwright
+from playwright.sync_api import sync_playwright
+
+#
+# Local Imports
+from App_Function_Libraries.Web_Scraping.Article_Extractor_Lib import scrape_from_sitemap, scrape_by_url_level, scrape_article
+from App_Function_Libraries.Web_Scraping.Article_Summarization_Lib import scrape_and_summarize_multiple
+from App_Function_Libraries.DB.DB_Manager import load_preset_prompts
+from App_Function_Libraries.Gradio_UI.Chat_ui import update_user_prompt
+from App_Function_Libraries.Summarization.Summarization_General_Lib import summarize
+
+
+#
+########################################################################################################################
+#
+# Functions:
+
+def get_url_depth(url: str) -> int:
+    return len(urlparse(url).path.strip('/').split('/'))
+
+
+def sync_recursive_scrape(url_input, max_pages, max_depth, progress_callback, delay=1.0):
+    def run_async_scrape():
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        return loop.run_until_complete(
+            recursive_scrape(url_input, max_pages, max_depth, progress_callback, delay)
+        )
+
+    with ThreadPoolExecutor() as executor:
+        future = executor.submit(run_async_scrape)
+        return future.result()
+
+
+async def recursive_scrape(
+        base_url: str,
+        max_pages: int,
+        max_depth: int,
+        progress_callback: callable,
+        delay: float = 1.0,
+        resume_file: str = 'scrape_progress.json',
+        user_agent: str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"
+) -> List[Dict]:
+    async def save_progress():
+        temp_file = resume_file + ".tmp"
+        with open(temp_file, 'w') as f:
+            json.dump({
+                'visited': list(visited),
+                'to_visit': to_visit,
+                'scraped_articles': scraped_articles,
+                'pages_scraped': pages_scraped
+            }, f)
+        os.replace(temp_file, resume_file)  # Atomic replace
+
+    def is_valid_url(url: str) -> bool:
+        return url.startswith("http") and len(url) > 0
+
+    # Load progress if resume file exists
+    if os.path.exists(resume_file):
+        with open(resume_file, 'r') as f:
+            progress_data = json.load(f)
+            visited = set(progress_data['visited'])
+            to_visit = progress_data['to_visit']
+            scraped_articles = progress_data['scraped_articles']
+            pages_scraped = progress_data['pages_scraped']
+    else:
+        visited = set()
+        to_visit = [(base_url, 0)]  # (url, depth)
+        scraped_articles = []
+        pages_scraped = 0
+
+    try:
+        async with async_playwright() as p:
+            browser = await p.chromium.launch(headless=True)
+            context = await browser.new_context(user_agent=user_agent)
+
+            try:
+                while to_visit and pages_scraped < max_pages:
+                    current_url, current_depth = to_visit.pop(0)
+
+                    if current_url in visited or current_depth > max_depth:
+                        continue
+
+                    visited.add(current_url)
+
+                    # Update progress
+                    progress_callback(f"Scraping page {pages_scraped + 1}/{max_pages}: {current_url}")
+
+                    try:
+                        await asyncio.sleep(random.uniform(delay * 0.8, delay * 1.2))
+
+                        # This function should be implemented to handle asynchronous scraping
+                        article_data = await scrape_article_async(context, current_url)
+
+                        if article_data and article_data['extraction_successful']:
+                            scraped_articles.append(article_data)
+                            pages_scraped += 1
+
+                        # If we haven't reached max depth, add child links to to_visit
+                        if current_depth < max_depth:
+                            page = await context.new_page()
+                            await page.goto(current_url)
+                            await page.wait_for_load_state("networkidle")
+
+                            links = await page.eval_on_selector_all('a[href]',
+                                                                    "(elements) => elements.map(el => el.href)")
+                            for link in links:
+                                child_url = urljoin(base_url, link)
+                                if is_valid_url(child_url) and child_url.startswith(
+                                        base_url) and child_url not in visited and should_scrape_url(child_url):
+                                    to_visit.append((child_url, current_depth + 1))
+
+                            await page.close()
+
+                    except Exception as e:
+                        logging.error(f"Error scraping {current_url}: {str(e)}")
+
+                    # Save progress periodically (e.g., every 10 pages)
+                    if pages_scraped % 10 == 0:
+                        await save_progress()
+
+            finally:
+                await browser.close()
+
+    finally:
+        # These statements are now guaranteed to be reached after the scraping is done
+        await save_progress()
+
+        # Remove the progress file when scraping is completed successfully
+        if os.path.exists(resume_file):
+            os.remove(resume_file)
+
+        # Final progress update
+        progress_callback(f"Scraping completed. Total pages scraped: {pages_scraped}")
+
+        return scraped_articles
+
+
+async def scrape_article_async(context, url: str) -> Dict[str, Any]:
+    page = await context.new_page()
+    try:
+        await page.goto(url)
+        await page.wait_for_load_state("networkidle")
+
+        title = await page.title()
+        content = await page.content()
+
+        return {
+            'url': url,
+            'title': title,
+            'content': content,
+            'extraction_successful': True
+        }
+    except Exception as e:
+        logging.error(f"Error scraping article {url}: {str(e)}")
+        return {
+            'url': url,
+            'extraction_successful': False,
+            'error': str(e)
+        }
+    finally:
+        await page.close()
+
+
+def scrape_article_sync(url: str) -> Dict[str, Any]:
+    with sync_playwright() as p:
+        browser = p.chromium.launch(headless=True)
+        page = browser.new_page()
+        try:
+            page.goto(url)
+            page.wait_for_load_state("networkidle")
+
+            title = page.title()
+            content = page.content()
+
+            return {
+                'url': url,
+                'title': title,
+                'content': content,
+                'extraction_successful': True
+            }
+        except Exception as e:
+            logging.error(f"Error scraping article {url}: {str(e)}")
+            return {
+                'url': url,
+                'extraction_successful': False,
+                'error': str(e)
+            }
+        finally:
+            browser.close()
+
+
+def should_scrape_url(url: str) -> bool:
+    parsed_url = urlparse(url)
+    path = parsed_url.path.lower()
+
+    # List of patterns to exclude
+    exclude_patterns = [
+        '/tag/', '/category/', '/author/', '/search/', '/page/',
+        'wp-content', 'wp-includes', 'wp-json', 'wp-admin',
+        'login', 'register', 'cart', 'checkout', 'account',
+        '.jpg', '.png', '.gif', '.pdf', '.zip'
+    ]
+
+    # Check if the URL contains any exclude patterns
+    if any(pattern in path for pattern in exclude_patterns):
+        return False
+
+    # Add more sophisticated checks here
+    # For example, you might want to only include URLs with certain patterns
+    include_patterns = ['/article/', '/post/', '/blog/']
+    if any(pattern in path for pattern in include_patterns):
+        return True
+
+    # By default, return True if no exclusion or inclusion rules matched
+    return True
+
+
+async def scrape_with_retry(url: str, max_retries: int = 3, retry_delay: float = 5.0):
+    for attempt in range(max_retries):
+        try:
+            return await scrape_article(url)
+        except TimeoutError:
+            if attempt < max_retries - 1:
+                logging.warning(f"Timeout error scraping {url}. Retrying in {retry_delay} seconds...")
+                await asyncio.sleep(retry_delay)
+            else:
+                logging.error(f"Failed to scrape {url} after {max_retries} attempts.")
+                return None
+        except Exception as e:
+            logging.error(f"Error scraping {url}: {str(e)}")
+            return None
+
+
+def create_website_scraping_tab():
+    with gr.TabItem("Website Scraping", visible=True):
+        gr.Markdown("# Scrape Websites & Summarize Articles")
+        with gr.Row():
+            with gr.Column():
+                scrape_method = gr.Radio(
+                    ["Individual URLs", "Sitemap", "URL Level", "Recursive Scraping"],
+                    label="Scraping Method",
+                    value="Individual URLs"
+                )
+                url_input = gr.Textbox(
+                    label="Article URLs or Base URL",
+                    placeholder="Enter article URLs here, one per line, or base URL for sitemap/URL level/recursive scraping",
+                    lines=5
+                )
+                url_level = gr.Slider(
+                    minimum=1,
+                    maximum=10,
+                    step=1,
+                    label="URL Level (for URL Level scraping)",
+                    value=2,
+                    visible=False
+                )
+                max_pages = gr.Slider(
+                    minimum=1,
+                    maximum=100,
+                    step=1,
+                    label="Maximum Pages to Scrape (for Recursive Scraping)",
+                    value=10,
+                    visible=False
+                )
+                max_depth = gr.Slider(
+                    minimum=1,
+                    maximum=10,
+                    step=1,
+                    label="Maximum Depth (for Recursive Scraping)",
+                    value=3,
+                    visible=False
+                )
+                custom_article_title_input = gr.Textbox(
+                    label="Custom Article Titles (Optional, one per line)",
+                    placeholder="Enter custom titles for the articles, one per line",
+                    lines=5
+                )
+                with gr.Row():
+                    summarize_checkbox = gr.Checkbox(label="Summarize Articles", value=False)
+                    custom_prompt_checkbox = gr.Checkbox(label="Use a Custom Prompt", value=False, visible=True)
+                    preset_prompt_checkbox = gr.Checkbox(label="Use a pre-set Prompt", value=False, visible=True)
+                with gr.Row():
+                    temp_slider = gr.Slider(0.1, 2.0, 0.7, label="Temperature")
+                with gr.Row():
+                    preset_prompt = gr.Dropdown(
+                        label="Select Preset Prompt",
+                        choices=load_preset_prompts(),
+                        visible=False
+                    )
+                with gr.Row():
+                    website_custom_prompt_input = gr.Textbox(
+                        label="Custom Prompt",
+                        placeholder="Enter custom prompt here",
+                        lines=3,
+                        visible=False
+                    )
+                with gr.Row():
+                    system_prompt_input = gr.Textbox(
+                        label="System Prompt",
+                        value="""<s>You are a bulleted notes specialist. [INST]```When creating comprehensive bulleted notes, you should follow these guidelines: Use multiple headings based on the referenced topics, not categories like quotes or terms. Headings should be surrounded by bold formatting and not be listed as bullet points themselves. Leave no space between headings and their corresponding list items underneath. Important terms within the content should be emphasized by setting them in bold font. Any text that ends with a colon should also be bolded. Before submitting your response, review the instructions, and make any corrections necessary to adhered to the specified format. Do not reference these instructions within the notes.``` \nBased on the content between backticks create comprehensive bulleted notes.[/INST]
+                                **Bulleted Note Creation Guidelines**
+                                
+                                **Headings**:
+                                - Based on referenced topics, not categories like quotes or terms
+                                - Surrounded by **bold** formatting 
+                                - Not listed as bullet points
+                                - No space between headings and list items underneath
+                                
+                                **Emphasis**:
+                                - **Important terms** set in bold font
+                                - **Text ending in a colon**: also bolded
+                                
+                                **Review**:
+                                - Ensure adherence to specified format
+                                - Do not reference these instructions in your response.</s>[INST] {{ .Prompt }} [/INST]
+                                """,
+                        lines=3,
+                        visible=False
+                    )
+
+                api_name_input = gr.Dropdown(
+                    choices=[None, "Local-LLM", "OpenAI", "Anthropic", "Cohere", "Groq", "DeepSeek", "Mistral",
+                             "OpenRouter",
+                             "Llama.cpp", "Kobold", "Ooba", "Tabbyapi", "VLLM", "ollama", "HuggingFace",
+                             "Custom-OpenAI-API"],
+                    value=None,
+                    label="API Name (Mandatory for Summarization)"
+                )
+                api_key_input = gr.Textbox(
+                    label="API Key (Mandatory if API Name is specified)",
+                    placeholder="Enter your API key here; Ignore if using Local API or Built-in API",
+                    type="password"
+                )
+                keywords_input = gr.Textbox(
+                    label="Keywords",
+                    placeholder="Enter keywords here (comma-separated)",
+                    value="default,no_keyword_set",
+                    visible=True
+                )
+
+                scrape_button = gr.Button("Scrape and Summarize")
+            with gr.Column():
+                progress_output = gr.Textbox(label="Progress", lines=3)
+                result_output = gr.Textbox(label="Result", lines=20)
+
+        def update_ui_for_scrape_method(method):
+            url_level_update = gr.update(visible=(method == "URL Level"))
+            max_pages_update = gr.update(visible=(method == "Recursive Scraping"))
+            max_depth_update = gr.update(visible=(method == "Recursive Scraping"))
+            url_input_update = gr.update(
+                label="Article URLs" if method == "Individual URLs" else "Base URL",
+                placeholder="Enter article URLs here, one per line" if method == "Individual URLs" else "Enter the base URL for scraping"
+            )
+            return url_level_update, max_pages_update, max_depth_update, url_input_update
+
+        scrape_method.change(
+            fn=update_ui_for_scrape_method,
+            inputs=[scrape_method],
+            outputs=[url_level, max_pages, max_depth, url_input]
+        )
+
+        custom_prompt_checkbox.change(
+            fn=lambda x: (gr.update(visible=x), gr.update(visible=x)),
+            inputs=[custom_prompt_checkbox],
+            outputs=[website_custom_prompt_input, system_prompt_input]
+        )
+        preset_prompt_checkbox.change(
+            fn=lambda x: gr.update(visible=x),
+            inputs=[preset_prompt_checkbox],
+            outputs=[preset_prompt]
+        )
+
+        def update_prompts(preset_name):
+            prompts = update_user_prompt(preset_name)
+            return (
+                gr.update(value=prompts["user_prompt"], visible=True),
+                gr.update(value=prompts["system_prompt"], visible=True)
+            )
+
+        preset_prompt.change(
+            update_prompts,
+            inputs=preset_prompt,
+            outputs=[website_custom_prompt_input, system_prompt_input]
+        )
+
+        async def scrape_and_summarize_wrapper(
+                scrape_method: str,
+                url_input: str,
+                url_level: Optional[int],
+                max_pages: int,
+                max_depth: int,
+                summarize_checkbox: bool,
+                custom_prompt: Optional[str],
+                api_name: Optional[str],
+                api_key: Optional[str],
+                keywords: str,
+                custom_titles: Optional[str],
+                system_prompt: Optional[str],
+                temperature: float = 0.7,
+                progress: gr.Progress = gr.Progress()
+        ) -> str:
+            try:
+                result: List[Dict[str, Any]] = []
+
+                if scrape_method == "Individual URLs":
+                    result = await scrape_and_summarize_multiple(url_input, custom_prompt, api_name, api_key, keywords,
+                                                                 custom_titles, system_prompt)
+                elif scrape_method == "Sitemap":
+                    result = await asyncio.to_thread(scrape_from_sitemap, url_input)
+                elif scrape_method == "URL Level":
+                    if url_level is None:
+                        return convert_json_to_markdown(
+                            json.dumps({"error": "URL level is required for URL Level scraping."}))
+                    result = await asyncio.to_thread(scrape_by_url_level, url_input, url_level)
+                elif scrape_method == "Recursive Scraping":
+                    result = await recursive_scrape(url_input, max_pages, max_depth, progress.update, delay=1.0)
+                else:
+                    return convert_json_to_markdown(json.dumps({"error": f"Unknown scraping method: {scrape_method}"}))
+
+                # Ensure result is always a list of dictionaries
+                if isinstance(result, dict):
+                    result = [result]
+                elif isinstance(result, list):
+                    if all(isinstance(item, str) for item in result):
+                        # Convert list of strings to list of dictionaries
+                        result = [{"content": item} for item in result]
+                    elif not all(isinstance(item, dict) for item in result):
+                        raise ValueError("Not all items in result are dictionaries or strings")
+                else:
+                    raise ValueError(f"Unexpected result type: {type(result)}")
+
+                # Ensure all items in result are dictionaries
+                if not all(isinstance(item, dict) for item in result):
+                    raise ValueError("Not all items in result are dictionaries")
+
+                if summarize_checkbox:
+                    total_articles = len(result)
+                    for i, article in enumerate(result):
+                        progress.update(f"Summarizing article {i + 1}/{total_articles}")
+                        content = article.get('content', '')
+                        if content:
+                            summary = await asyncio.to_thread(summarize, content, custom_prompt, api_name, api_key,
+                                                              temperature, system_prompt)
+                            article['summary'] = summary
+                        else:
+                            article['summary'] = "No content available to summarize."
+
+                # Concatenate all content
+                all_content = "\n\n".join(
+                    [f"# {article.get('title', 'Untitled')}\n\n{article.get('content', '')}\n\n" +
+                     (f"Summary: {article.get('summary', '')}" if summarize_checkbox else "")
+                     for article in result])
+
+                # Collect all unique URLs
+                all_urls = list(set(article.get('url', '') for article in result if article.get('url')))
+
+                # Structure the output for the entire website collection
+                website_collection = {
+                    "base_url": url_input,
+                    "scrape_method": scrape_method,
+                    "summarization_performed": summarize_checkbox,
+                    "api_used": api_name if summarize_checkbox else None,
+                    "keywords": keywords if summarize_checkbox else None,
+                    "url_level": url_level if scrape_method == "URL Level" else None,
+                    "max_pages": max_pages if scrape_method == "Recursive Scraping" else None,
+                    "max_depth": max_depth if scrape_method == "Recursive Scraping" else None,
+                    "total_articles_scraped": len(result),
+                    "urls_scraped": all_urls,
+                    "content": all_content
+                }
+
+                # Convert the JSON to markdown and return
+                return convert_json_to_markdown(json.dumps(website_collection, indent=2))
+            except Exception as e:
+                return convert_json_to_markdown(json.dumps({"error": f"An error occurred: {str(e)}"}))
+
+        # Update the scrape_button.click to include the temperature parameter
+        scrape_button.click(
+            fn=lambda *args: asyncio.run(scrape_and_summarize_wrapper(*args)),
+            inputs=[scrape_method, url_input, url_level, max_pages, max_depth, summarize_checkbox,
+                    website_custom_prompt_input, api_name_input, api_key_input, keywords_input,
+                    custom_article_title_input, system_prompt_input, temp_slider],
+            outputs=[result_output]
+        )
+
+
+def convert_json_to_markdown(json_str: str) -> str:
+    """
+    Converts the JSON output from the scraping process into a markdown format.
+
+    Args:
+        json_str (str): JSON-formatted string containing the website collection data
+
+    Returns:
+        str: Markdown-formatted string of the website collection data
+    """
+    try:
+        # Parse the JSON string
+        data = json.loads(json_str)
+
+        # Check if there's an error in the JSON
+        if "error" in data:
+            return f"# Error\n\n{data['error']}"
+
+        # Start building the markdown string
+        markdown = f"# Website Collection: {data['base_url']}\n\n"
+
+        # Add metadata
+        markdown += "## Metadata\n\n"
+        markdown += f"- **Scrape Method:** {data['scrape_method']}\n"
+        markdown += f"- **API Used:** {data['api_used']}\n"
+        markdown += f"- **Keywords:** {data['keywords']}\n"
+        if data['url_level'] is not None:
+            markdown += f"- **URL Level:** {data['url_level']}\n"
+        markdown += f"- **Total Articles Scraped:** {data['total_articles_scraped']}\n\n"
+
+        # Add URLs scraped
+        markdown += "## URLs Scraped\n\n"
+        for url in data['urls_scraped']:
+            markdown += f"- {url}\n"
+        markdown += "\n"
+
+        # Add the content
+        markdown += "## Content\n\n"
+        markdown += data['content']
+
+        return markdown
+
+    except json.JSONDecodeError:
+        return "# Error\n\nInvalid JSON string provided."
+    except KeyError as e:
+        return f"# Error\n\nMissing key in JSON data: {str(e)}"
+    except Exception as e:
+        return f"# Error\n\nAn unexpected error occurred: {str(e)}"
+#
+# End of File
+########################################################################################################################