You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 

34 lines
1.1 KiB

import requests
from bs4 import BeautifulSoup
import logging
import os
# Logging setup
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
def fetch_web_content(url):
"""Fetches the webpage content and extracts links."""
try:
response = requests.get(url, timeout=10)
response.raise_for_status() # Raise error for failed requests
soup = BeautifulSoup(response.text, 'html.parser')
# Extract all links
links = [a['href'] for a in soup.find_all('a', href=True)]
logger.info(f"Fetched {len(links)} links from {url}")
return links
except requests.RequestException as e:
logger.error(f"Failed to fetch content from {url}: {e}")
return []
def format_links(base_url, links):
"""Converts relative links to absolute URLs and filters irrelevant ones."""
filtered_links = []
for link in links:
if link.startswith("/"):
link = base_url.rstrip("/") + link
if "contact" not in link.lower() and "privacy" not in link.lower():
filtered_links.append(link)
return filtered_links