File size: 1,145 Bytes
da7c6fd |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 |
from transformers import Tool
import requests
from markdownify import markdownify as md
from requests.exceptions import RequestException
import re
class VisitWebpageTool(Tool):
name = "visit_webpage"
description = "Visits a webpage at the given URL and returns its content as a markdown string."
inputs = {
"url": {
"type": "text",
"description": "The URL of the webpage to visit.",
}
}
output_type = "text"
def forward(self, url: str) -> str:
try:
# Send a GET request to the URL
response = requests.get(url)
response.raise_for_status() # Raise an exception for bad status codes
# Convert the HTML content to Markdown
markdown_content = md(response.text).strip()
# Remove multiple line breaks
markdown_content = re.sub(r"\n{3,}", "\n\n", markdown_content)
return markdown_content
except RequestException as e:
return f"Error fetching the webpage: {str(e)}"
except Exception as e:
return f"An unexpected error occurred: {str(e)}"
|