From the uDemy course on LLM engineering.
https://www.udemy.com/course/llm-engineering-master-ai-and-large-language-models
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
176 lines
6.3 KiB
176 lines
6.3 KiB
# imports |
|
# If these fail, please check you're running from an 'activated' environment with (llms) in the command prompt |
|
|
|
import os |
|
import requests |
|
import json |
|
from typing import List |
|
from dotenv import load_dotenv |
|
from bs4 import BeautifulSoup |
|
from IPython.display import Markdown, display, update_display |
|
from openai import OpenAI |
|
|
|
# Initialize and constants |
|
|
|
load_dotenv(override=True) |
|
api_key = os.getenv('OPENAI_API_KEY') |
|
|
|
if api_key and api_key.startswith('sk-proj-') and len(api_key) > 10: |
|
print("API key looks good so far") |
|
else: |
|
print("There might be a problem with your API key? Please visit the troubleshooting notebook!") |
|
|
|
MODEL = 'gpt-4o-mini' |
|
openai = OpenAI() |
|
|
|
# A class to represent a Webpage |
|
|
|
# Some websites need you to use proper headers when fetching them: |
|
headers = { |
|
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36" |
|
} |
|
|
|
|
|
class Website: |
|
""" |
|
A utility class to represent a Website that we have scraped, now with links |
|
""" |
|
|
|
def __init__(self, url): |
|
self.url = url |
|
response = requests.get(url, headers=headers) |
|
self.body = response.content |
|
soup = BeautifulSoup(self.body, 'html.parser') |
|
self.title = soup.title.string if soup.title else "No title found" |
|
if soup.body: |
|
for irrelevant in soup.body(["script", "style", "img", "input"]): |
|
irrelevant.decompose() |
|
self.text = soup.body.get_text(separator="\n", strip=True) |
|
else: |
|
self.text = "" |
|
links = [link.get('href') for link in soup.find_all('a')] |
|
self.links = [link for link in links if link] |
|
|
|
def get_contents(self): |
|
return f"Webpage Title:\n{self.title}\nWebpage Contents:\n{self.text}\n\n" |
|
|
|
|
|
link_system_prompt = "You are provided with a list of links found on a webpage. \ |
|
You are able to decide which of the links would be most relevant to include in a brochure about the company, \ |
|
such as links to an About page, or a Company page, or Careers/Jobs pages.\n" |
|
link_system_prompt += "You should respond in JSON as in this example:" |
|
link_system_prompt += """ |
|
{ |
|
"links": [ |
|
{"type": "about page", "url": "https://full.url/goes/here/about"}, |
|
{"type": "careers page": "url": "https://another.full.url/careers"} |
|
] |
|
} |
|
""" |
|
|
|
|
|
def get_links_user_prompt(website): |
|
user_prompt = f"Here is the list of links on the website of {website.url} - " |
|
user_prompt += "please decide which of these are relevant web links for a brochure about the company, respond with the full https URL in JSON format. \ |
|
Do not include Terms of Service, Privacy, email links.\n" |
|
user_prompt += "Links (some might be relative links):\n" |
|
user_prompt += "\n".join(website.links) |
|
return user_prompt |
|
|
|
|
|
def get_links(url): |
|
website = Website(url) |
|
response = openai.chat.completions.create( |
|
model=MODEL, |
|
messages=[ |
|
{"role": "system", "content": link_system_prompt}, |
|
{"role": "user", "content": get_links_user_prompt(website)} |
|
], |
|
response_format={"type": "json_object"} |
|
) |
|
result = response.choices[0].message.content |
|
return json.loads(result) |
|
|
|
|
|
def get_all_details(url): |
|
result = "Landing page:\n" |
|
result += Website(url).get_contents() |
|
links = get_links(url) |
|
print("Found links:", links) |
|
for link in links["links"]: |
|
result += f"\n\n{link['type']}\n" |
|
result += Website(link["url"]).get_contents() |
|
return result |
|
|
|
|
|
system_prompt_2 = "You are an assistant that analyzes the contents of several relevant pages from a company website \ |
|
and creates a short brochure about the company for prospective customers, investors and recruits. Respond in markdown.\ |
|
Include details of company culture, customers and careers/jobs if you have the information." |
|
|
|
# Or uncomment the lines below for a more humorous brochure - this demonstrates how easy it is to incorporate 'tone': |
|
|
|
# system_prompt = "You are an assistant that analyzes the contents of several relevant pages from a company website \ |
|
# and creates a short humorous, entertaining, jokey brochure about the company for prospective customers, investors and recruits. Respond in markdown.\ |
|
# Include details of company culture, customers and careers/jobs if you have the information." |
|
|
|
def get_brochure_user_prompt(company_name, url): |
|
user_prompt = f"You are looking at a company called: {company_name}\n" |
|
user_prompt += f"Here are the contents of its landing page and other relevant pages; use this information to build a short brochure of the company in markdown.\n" |
|
user_prompt += get_all_details(url) |
|
user_prompt = user_prompt[:5_000] # Truncate if more than 5,000 characters |
|
return user_prompt |
|
|
|
def create_brochure(company_name, url): |
|
response = openai.chat.completions.create( |
|
model=MODEL, |
|
messages=[ |
|
{"role": "system", "content": system_prompt_2}, |
|
{"role": "user", "content": get_brochure_user_prompt(company_name, url)} |
|
], |
|
) |
|
result = response.choices[0].message.content |
|
# Jupytrer |
|
# display(Markdown(result)) |
|
# pycharm |
|
with open("company_brochure.md", "w") as f: |
|
f.write(result) |
|
display(result) |
|
|
|
|
|
def stream_brochure(company_name, url): |
|
stream = openai.chat.completions.create( |
|
model=MODEL, |
|
messages=[ |
|
{"role": "system", "content": system_prompt_2}, |
|
{"role": "user", "content": get_brochure_user_prompt(company_name, url)} |
|
], |
|
stream=True |
|
) |
|
|
|
response = "" |
|
display_handle = display(Markdown(""), display_id=True) |
|
for chunk in stream: |
|
response += chunk.choices[0].delta.content or '' |
|
response = response.replace("```", "").replace("markdown", "") |
|
update_display(Markdown(response), display_id=display_handle.display_id) |
|
|
|
|
|
if __name__ == "__main__": |
|
# ed = Website("https://edwarddonner.com") |
|
# print(ed.links) |
|
|
|
# Anthropic has made their site harder to scrape, so I'm using HuggingFace.. |
|
|
|
# huggingface = Website("https://huggingface.co") |
|
# print(huggingface.links) |
|
# |
|
# print(get_links("https://huggingface.co")) |
|
|
|
# anthropic_page = Website("https://anthropic.com") |
|
# anthropic_page.links |
|
|
|
# print(get_brochure_user_prompt("HuggingFace", "https://huggingface.co")) |
|
|
|
create_brochure("HuggingFace", "https://huggingface.co") |
|
|
|
|
|
|