import requests
from bs4 import BeautifulSoup
import logging
import os

# Logging setup
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)

def fetch_web_content(url):
    """Fetches the webpage content and extracts links."""
    try:
        response = requests.get(url, timeout=10)
        response.raise_for_status()  # Raise error for failed requests
        soup = BeautifulSoup(response.text, 'html.parser')

        # Extract all links
        links = [a['href'] for a in soup.find_all('a', href=True)]
        logger.info(f"Fetched {len(links)} links from {url}")
        return links
    except requests.RequestException as e:
        logger.error(f"Failed to fetch content from {url}: {e}")
        return []

def format_links(base_url, links):
    """Converts relative links to absolute URLs and filters irrelevant ones."""
    filtered_links = []
    for link in links:
        if link.startswith("/"):
            link = base_url.rstrip("/") + link
        if "contact" not in link.lower() and "privacy" not in link.lower():
            filtered_links.append(link)
    
    return filtered_links