From the uDemy course on LLM engineering.
https://www.udemy.com/course/llm-engineering-master-ai-and-large-language-models
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
282 lines
12 KiB
282 lines
12 KiB
{ |
|
"cells": [ |
|
{ |
|
"cell_type": "markdown", |
|
"id": "a98030af-fcd1-4d63-a36e-38ba053498fa", |
|
"metadata": {}, |
|
"source": [ |
|
"# A full business solution\n", |
|
"\n", |
|
"Create a product that builds a Brochure for a company to be used for prospective clients, investors and potential recruits.\n", |
|
"\n", |
|
"We will be provided a company name and their primary website." |
|
] |
|
}, |
|
{ |
|
"cell_type": "code", |
|
"execution_count": null, |
|
"id": "0a572211-5fe3-4dd5-9870-849cfb75901f", |
|
"metadata": {}, |
|
"outputs": [], |
|
"source": [ |
|
"# Import necessary libraries\n", |
|
"import os\n", |
|
"import requests\n", |
|
"import json\n", |
|
"from typing import List, Dict\n", |
|
"from dotenv import load_dotenv\n", |
|
"from bs4 import BeautifulSoup\n", |
|
"from IPython.display import Markdown, display, update_display\n", |
|
"from openai import OpenAI\n", |
|
"from urllib.parse import urljoin\n", |
|
"\n", |
|
"# Load environment variables from a .env file\n", |
|
"load_dotenv()\n", |
|
"\n", |
|
"# Define constants\n", |
|
"MODEL = 'gpt-4o-mini' # Specify the OpenAI model to use\n", |
|
"OPENAI_API_KEY = os.getenv('OPENAI_API_KEY', 'your-key-if-not-using-env') # Get API key from environment or use default\n", |
|
"\n", |
|
"# Initialize OpenAI client with the API key\n", |
|
"openai = OpenAI(api_key=OPENAI_API_KEY)\n", |
|
"\n", |
|
"class Website:\n", |
|
" \"\"\"\n", |
|
" A class to represent a website and its contents.\n", |
|
" \"\"\"\n", |
|
" def __init__(self, url: str):\n", |
|
" \"\"\"\n", |
|
" Initialize the Website object with a given URL.\n", |
|
" \n", |
|
" :param url: The URL of the website to scrape\n", |
|
" \"\"\"\n", |
|
" self.url = url\n", |
|
" self.title, self.text, self.links = self._scrape_website()\n", |
|
"\n", |
|
" def _scrape_website(self) -> tuple:\n", |
|
" \"\"\"\n", |
|
" Scrape the website content, extracting title, text, and links.\n", |
|
" \n", |
|
" :return: A tuple containing the title, text content, and links of the website\n", |
|
" \"\"\"\n", |
|
" response = requests.get(self.url)\n", |
|
" soup = BeautifulSoup(response.content, 'html.parser')\n", |
|
" \n", |
|
" # Extract title\n", |
|
" title = soup.title.string if soup.title else \"No title found\"\n", |
|
" \n", |
|
" # Extract text content\n", |
|
" if soup.body:\n", |
|
" for tag in soup.body([\"script\", \"style\", \"img\", \"input\"]):\n", |
|
" tag.decompose() # Remove unwanted tags\n", |
|
" text = soup.body.get_text(separator=\"\\n\", strip=True)\n", |
|
" else:\n", |
|
" text = \"\"\n", |
|
" \n", |
|
" # Extract links\n", |
|
" links = [link.get('href') for link in soup.find_all('a') if link.get('href')]\n", |
|
" \n", |
|
" return title, text, links\n", |
|
"\n", |
|
" def get_contents(self) -> str:\n", |
|
" \"\"\"\n", |
|
" Get a formatted string of the website contents.\n", |
|
" \n", |
|
" :return: A string containing the website title and text content\n", |
|
" \"\"\"\n", |
|
" return f\"Webpage Title:\\n{self.title}\\nWebpage Contents:\\n{self.text}\\n\\n\"\n", |
|
"\n", |
|
"class LinkAnalyzer:\n", |
|
" \"\"\"\n", |
|
" A class to analyze and categorize links from a website.\n", |
|
" \"\"\"\n", |
|
" # System prompt for the OpenAI model to categorize links\n", |
|
" LINK_SYSTEM_PROMPT = \"\"\"\n", |
|
" You are provided with a list of links found on a webpage. Your task is to first categorize each link into one of the following categories:\n", |
|
" - about page\n", |
|
" - careers page\n", |
|
" - terms of service\n", |
|
" - privacy policy\n", |
|
" - contact page\n", |
|
" - other (please specify).\n", |
|
"\n", |
|
" Once the links are categorized, please choose which links are most relevant to include in a brochure about the company. \n", |
|
" The brochure should only include links such as About pages, Careers pages, or Company Overview pages. Exclude any links related to Terms of Service, Privacy Policy, or email addresses.\n", |
|
"\n", |
|
" Respond in the following JSON format:\n", |
|
" {\n", |
|
" \"categorized_links\": [\n", |
|
" {\"category\": \"about page\", \"url\": \"https://full.url/about\"},\n", |
|
" {\"category\": \"careers page\", \"url\": \"https://full.url/careers\"},\n", |
|
" {\"category\": \"terms of service\", \"url\": \"https://full.url/terms\"},\n", |
|
" {\"category\": \"privacy policy\", \"url\": \"https://full.url/privacy\"},\n", |
|
" {\"category\": \"other\", \"specify\": \"contact page\", \"url\": \"https://full.url/contact\"}\n", |
|
" ],\n", |
|
" \"brochure_links\": [\n", |
|
" {\"type\": \"about page\", \"url\": \"https://full.url/about\"},\n", |
|
" {\"type\": \"careers page\", \"url\": \"https://full.url/careers\"}\n", |
|
" ]\n", |
|
" }\n", |
|
"\n", |
|
" Please find the links below and proceed with the task:\n", |
|
"\n", |
|
" Links (some may be relative links):\n", |
|
" [INSERT LINK LIST HERE]\n", |
|
" \"\"\"\n", |
|
"\n", |
|
" @staticmethod\n", |
|
" def get_links(website: Website) -> Dict:\n", |
|
" \"\"\"\n", |
|
" Analyze and categorize links from a given website.\n", |
|
" \n", |
|
" :param website: A Website object containing the links to analyze\n", |
|
" :return: A dictionary containing categorized links and brochure-relevant links\n", |
|
" \"\"\"\n", |
|
" # Prepare the user prompt for the OpenAI model\n", |
|
" user_prompt = f\"Here is the list of links on the website of {website.url} - \"\n", |
|
" user_prompt += \"please decide which of these are relevant web links for a brochure about the company, respond with the full https URL in JSON format. \\\n", |
|
" Do not include Terms of Service, Privacy, email links.\\n\"\n", |
|
" user_prompt += \"Links (some might be relative links):\\n\"\n", |
|
" user_prompt += \"\\n\".join(website.links)\n", |
|
"\n", |
|
" # Make an API call to OpenAI for link analysis\n", |
|
" completion = openai.chat.completions.create(\n", |
|
" model=MODEL,\n", |
|
" messages=[\n", |
|
" {\"role\": \"system\", \"content\": LinkAnalyzer.LINK_SYSTEM_PROMPT},\n", |
|
" {\"role\": \"user\", \"content\": user_prompt}\n", |
|
" ],\n", |
|
" response_format={\"type\": \"json_object\"}\n", |
|
" )\n", |
|
" return json.loads(completion.choices[0].message.content)\n", |
|
"\n", |
|
"class BrochureGenerator:\n", |
|
" \"\"\"\n", |
|
" A class to generate a company brochure based on website content.\n", |
|
" \"\"\"\n", |
|
" # System prompt for the OpenAI model to generate the brochure\n", |
|
" SYSTEM_PROMPT = \"\"\"\n", |
|
" You are an assistant that analyzes the contents of several relevant pages from a company website \n", |
|
" and creates a brochure about the company for prospective customers, investors and recruits. Respond in markdown.\n", |
|
" Include details of company culture, customers and careers/jobs if you have the information.\n", |
|
" Structure the brochure to include specific sections as follows:\n", |
|
" About Us\n", |
|
" What we do\n", |
|
" How We Do It\n", |
|
" Where We Do It\n", |
|
" Our People\n", |
|
" Our Culture\n", |
|
" Connect with Us.\n", |
|
" Please provide two versions of the brochure, the first in English, the second in Spanish. The contents of the brochure are to be the same for both languages.\n", |
|
" \"\"\"\n", |
|
"\n", |
|
" @staticmethod\n", |
|
" def get_all_details(url: str) -> str:\n", |
|
" \"\"\"\n", |
|
" Gather all relevant details from a company's website.\n", |
|
" \n", |
|
" :param url: The URL of the company's main page\n", |
|
" :return: A string containing all relevant website content\n", |
|
" \"\"\"\n", |
|
" result = \"Landing page:\\n\"\n", |
|
" website = Website(url)\n", |
|
" result += website.get_contents()\n", |
|
"\n", |
|
" # Analyze links and get brochure-relevant ones\n", |
|
" links = LinkAnalyzer.get_links(website)\n", |
|
" brochure_links = links.get('brochure_links', [])\n", |
|
" print(\"Found Brochure links:\", brochure_links)\n", |
|
"\n", |
|
" # Gather content from brochure-relevant pages\n", |
|
" for link in brochure_links:\n", |
|
" result += f\"\\n\\n{link['type']}:\\n\"\n", |
|
" full_url = urljoin(url, link[\"url\"])\n", |
|
" result += Website(full_url).get_contents()\n", |
|
"\n", |
|
" return result\n", |
|
"\n", |
|
" @staticmethod\n", |
|
" def get_brochure_user_prompt(company_name: str, url: str) -> str:\n", |
|
" \"\"\"\n", |
|
" Generate a user prompt for the OpenAI model to create a brochure.\n", |
|
" \n", |
|
" :param company_name: The name of the company\n", |
|
" :param url: The URL of the company's main page\n", |
|
" :return: A string containing the user prompt for brochure generation\n", |
|
" \"\"\"\n", |
|
" user_prompt = f\"You are looking at a company called: {company_name}\\n\"\n", |
|
" user_prompt += f\"Here are the contents of its landing page and other relevant pages; use this information to build a short brochure of the company in markdown.\\n\"\n", |
|
" user_prompt += BrochureGenerator.get_all_details(url)\n", |
|
" return user_prompt[:20_000] # Truncate if more than 20,000 characters\n", |
|
"\n", |
|
" @staticmethod\n", |
|
" def stream_brochure(company_name: str, url: str):\n", |
|
" \"\"\"\n", |
|
" Generate and stream a company brochure.\n", |
|
" \n", |
|
" :param company_name: The name of the company\n", |
|
" :param url: The URL of the company's main page\n", |
|
" \"\"\"\n", |
|
" # Make a streaming API call to OpenAI for brochure generation\n", |
|
" stream = openai.chat.completions.create(\n", |
|
" model=MODEL,\n", |
|
" messages=[\n", |
|
" {\"role\": \"system\", \"content\": BrochureGenerator.SYSTEM_PROMPT},\n", |
|
" {\"role\": \"user\", \"content\": BrochureGenerator.get_brochure_user_prompt(company_name, url)}\n", |
|
" ],\n", |
|
" stream=True\n", |
|
" )\n", |
|
"\n", |
|
" # Display the generated brochure in real-time\n", |
|
" response = \"\"\n", |
|
" display_handle = display(Markdown(\"\"), display_id=True)\n", |
|
" for chunk in stream:\n", |
|
" response += chunk.choices[0].delta.content or ''\n", |
|
" response = response.replace(\"```\", \"\").replace(\"markdown\", \"\")\n", |
|
" update_display(Markdown(response), display_id=display_handle.display_id)\n" |
|
] |
|
}, |
|
{ |
|
"cell_type": "code", |
|
"execution_count": null, |
|
"id": "cc4965cf-f704-4d40-8b7d-f8e50913f87c", |
|
"metadata": {}, |
|
"outputs": [], |
|
"source": [ |
|
"# Main execution block\n", |
|
"if __name__ == \"__main__\":\n", |
|
" # Generate a brochure\n", |
|
" BrochureGenerator.stream_brochure(\"Edward Donner\", \"https://edwarddonner.com/\")" |
|
] |
|
}, |
|
{ |
|
"cell_type": "code", |
|
"execution_count": null, |
|
"id": "0acb1194-fe89-40e3-8c3b-a10483315d3f", |
|
"metadata": {}, |
|
"outputs": [], |
|
"source": [] |
|
} |
|
], |
|
"metadata": { |
|
"kernelspec": { |
|
"display_name": "Python 3 (ipykernel)", |
|
"language": "python", |
|
"name": "python3" |
|
}, |
|
"language_info": { |
|
"codemirror_mode": { |
|
"name": "ipython", |
|
"version": 3 |
|
}, |
|
"file_extension": ".py", |
|
"mimetype": "text/x-python", |
|
"name": "python", |
|
"nbconvert_exporter": "python", |
|
"pygments_lexer": "ipython3", |
|
"version": "3.11.11" |
|
} |
|
}, |
|
"nbformat": 4, |
|
"nbformat_minor": 5 |
|
}
|
|
|