From the uDemy course on LLM engineering.
https://www.udemy.com/course/llm-engineering-master-ai-and-large-language-models
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
23 lines
809 B
23 lines
809 B
import requests |
|
from bs4 import BeautifulSoup |
|
|
|
def fetch_web_content(url): |
|
try: |
|
response = requests.get(url) |
|
response.raise_for_status() |
|
|
|
# Parse the HTML content |
|
soup = BeautifulSoup(response.text, 'html.parser') |
|
|
|
# Extract readable text from the web page (ignoring scripts, styles, etc.) |
|
page_text = soup.get_text(separator=' ', strip=True) |
|
|
|
return page_text[:5000] # Limit to 5000 chars (API limitation) |
|
except requests.exceptions.RequestException as e: |
|
print(f"Error fetching the webpage: {e}") |
|
return None |
|
|
|
if __name__ == "__main__": |
|
url = "https://en.wikipedia.org/wiki/Natural_language_processing" |
|
content = fetch_web_content(url) |
|
print(content[:500]) # Print a sample of the content
|
|
|