From the uDemy course on LLM engineering.
https://www.udemy.com/course/llm-engineering-master-ai-and-large-language-models
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
34 lines
1.1 KiB
34 lines
1.1 KiB
import requests |
|
from bs4 import BeautifulSoup |
|
import logging |
|
import os |
|
|
|
# Logging setup |
|
logging.basicConfig(level=logging.INFO) |
|
logger = logging.getLogger(__name__) |
|
|
|
def fetch_web_content(url): |
|
"""Fetches the webpage content and extracts links.""" |
|
try: |
|
response = requests.get(url, timeout=10) |
|
response.raise_for_status() # Raise error for failed requests |
|
soup = BeautifulSoup(response.text, 'html.parser') |
|
|
|
# Extract all links |
|
links = [a['href'] for a in soup.find_all('a', href=True)] |
|
logger.info(f"Fetched {len(links)} links from {url}") |
|
return links |
|
except requests.RequestException as e: |
|
logger.error(f"Failed to fetch content from {url}: {e}") |
|
return [] |
|
|
|
def format_links(base_url, links): |
|
"""Converts relative links to absolute URLs and filters irrelevant ones.""" |
|
filtered_links = [] |
|
for link in links: |
|
if link.startswith("/"): |
|
link = base_url.rstrip("/") + link |
|
if "contact" not in link.lower() and "privacy" not in link.lower(): |
|
filtered_links.append(link) |
|
|
|
return filtered_links
|
|
|