You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 

282 lines
12 KiB

{
"cells": [
{
"cell_type": "markdown",
"id": "a98030af-fcd1-4d63-a36e-38ba053498fa",
"metadata": {},
"source": [
"# A full business solution\n",
"\n",
"Create a product that builds a Brochure for a company to be used for prospective clients, investors and potential recruits.\n",
"\n",
"We will be provided a company name and their primary website."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0a572211-5fe3-4dd5-9870-849cfb75901f",
"metadata": {},
"outputs": [],
"source": [
"# Import necessary libraries\n",
"import os\n",
"import requests\n",
"import json\n",
"from typing import List, Dict\n",
"from dotenv import load_dotenv\n",
"from bs4 import BeautifulSoup\n",
"from IPython.display import Markdown, display, update_display\n",
"from openai import OpenAI\n",
"from urllib.parse import urljoin\n",
"\n",
"# Load environment variables from a .env file\n",
"load_dotenv()\n",
"\n",
"# Define constants\n",
"MODEL = 'gpt-4o-mini' # Specify the OpenAI model to use\n",
"OPENAI_API_KEY = os.getenv('OPENAI_API_KEY', 'your-key-if-not-using-env') # Get API key from environment or use default\n",
"\n",
"# Initialize OpenAI client with the API key\n",
"openai = OpenAI(api_key=OPENAI_API_KEY)\n",
"\n",
"class Website:\n",
" \"\"\"\n",
" A class to represent a website and its contents.\n",
" \"\"\"\n",
" def __init__(self, url: str):\n",
" \"\"\"\n",
" Initialize the Website object with a given URL.\n",
" \n",
" :param url: The URL of the website to scrape\n",
" \"\"\"\n",
" self.url = url\n",
" self.title, self.text, self.links = self._scrape_website()\n",
"\n",
" def _scrape_website(self) -> tuple:\n",
" \"\"\"\n",
" Scrape the website content, extracting title, text, and links.\n",
" \n",
" :return: A tuple containing the title, text content, and links of the website\n",
" \"\"\"\n",
" response = requests.get(self.url)\n",
" soup = BeautifulSoup(response.content, 'html.parser')\n",
" \n",
" # Extract title\n",
" title = soup.title.string if soup.title else \"No title found\"\n",
" \n",
" # Extract text content\n",
" if soup.body:\n",
" for tag in soup.body([\"script\", \"style\", \"img\", \"input\"]):\n",
" tag.decompose() # Remove unwanted tags\n",
" text = soup.body.get_text(separator=\"\\n\", strip=True)\n",
" else:\n",
" text = \"\"\n",
" \n",
" # Extract links\n",
" links = [link.get('href') for link in soup.find_all('a') if link.get('href')]\n",
" \n",
" return title, text, links\n",
"\n",
" def get_contents(self) -> str:\n",
" \"\"\"\n",
" Get a formatted string of the website contents.\n",
" \n",
" :return: A string containing the website title and text content\n",
" \"\"\"\n",
" return f\"Webpage Title:\\n{self.title}\\nWebpage Contents:\\n{self.text}\\n\\n\"\n",
"\n",
"class LinkAnalyzer:\n",
" \"\"\"\n",
" A class to analyze and categorize links from a website.\n",
" \"\"\"\n",
" # System prompt for the OpenAI model to categorize links\n",
" LINK_SYSTEM_PROMPT = \"\"\"\n",
" You are provided with a list of links found on a webpage. Your task is to first categorize each link into one of the following categories:\n",
" - about page\n",
" - careers page\n",
" - terms of service\n",
" - privacy policy\n",
" - contact page\n",
" - other (please specify).\n",
"\n",
" Once the links are categorized, please choose which links are most relevant to include in a brochure about the company. \n",
" The brochure should only include links such as About pages, Careers pages, or Company Overview pages. Exclude any links related to Terms of Service, Privacy Policy, or email addresses.\n",
"\n",
" Respond in the following JSON format:\n",
" {\n",
" \"categorized_links\": [\n",
" {\"category\": \"about page\", \"url\": \"https://full.url/about\"},\n",
" {\"category\": \"careers page\", \"url\": \"https://full.url/careers\"},\n",
" {\"category\": \"terms of service\", \"url\": \"https://full.url/terms\"},\n",
" {\"category\": \"privacy policy\", \"url\": \"https://full.url/privacy\"},\n",
" {\"category\": \"other\", \"specify\": \"contact page\", \"url\": \"https://full.url/contact\"}\n",
" ],\n",
" \"brochure_links\": [\n",
" {\"type\": \"about page\", \"url\": \"https://full.url/about\"},\n",
" {\"type\": \"careers page\", \"url\": \"https://full.url/careers\"}\n",
" ]\n",
" }\n",
"\n",
" Please find the links below and proceed with the task:\n",
"\n",
" Links (some may be relative links):\n",
" [INSERT LINK LIST HERE]\n",
" \"\"\"\n",
"\n",
" @staticmethod\n",
" def get_links(website: Website) -> Dict:\n",
" \"\"\"\n",
" Analyze and categorize links from a given website.\n",
" \n",
" :param website: A Website object containing the links to analyze\n",
" :return: A dictionary containing categorized links and brochure-relevant links\n",
" \"\"\"\n",
" # Prepare the user prompt for the OpenAI model\n",
" user_prompt = f\"Here is the list of links on the website of {website.url} - \"\n",
" user_prompt += \"please decide which of these are relevant web links for a brochure about the company, respond with the full https URL in JSON format. \\\n",
" Do not include Terms of Service, Privacy, email links.\\n\"\n",
" user_prompt += \"Links (some might be relative links):\\n\"\n",
" user_prompt += \"\\n\".join(website.links)\n",
"\n",
" # Make an API call to OpenAI for link analysis\n",
" completion = openai.chat.completions.create(\n",
" model=MODEL,\n",
" messages=[\n",
" {\"role\": \"system\", \"content\": LinkAnalyzer.LINK_SYSTEM_PROMPT},\n",
" {\"role\": \"user\", \"content\": user_prompt}\n",
" ],\n",
" response_format={\"type\": \"json_object\"}\n",
" )\n",
" return json.loads(completion.choices[0].message.content)\n",
"\n",
"class BrochureGenerator:\n",
" \"\"\"\n",
" A class to generate a company brochure based on website content.\n",
" \"\"\"\n",
" # System prompt for the OpenAI model to generate the brochure\n",
" SYSTEM_PROMPT = \"\"\"\n",
" You are an assistant that analyzes the contents of several relevant pages from a company website \n",
" and creates a brochure about the company for prospective customers, investors and recruits. Respond in markdown.\n",
" Include details of company culture, customers and careers/jobs if you have the information.\n",
" Structure the brochure to include specific sections as follows:\n",
" About Us\n",
" What we do\n",
" How We Do It\n",
" Where We Do It\n",
" Our People\n",
" Our Culture\n",
" Connect with Us.\n",
" Please provide two versions of the brochure, the first in English, the second in Spanish. The contents of the brochure are to be the same for both languages.\n",
" \"\"\"\n",
"\n",
" @staticmethod\n",
" def get_all_details(url: str) -> str:\n",
" \"\"\"\n",
" Gather all relevant details from a company's website.\n",
" \n",
" :param url: The URL of the company's main page\n",
" :return: A string containing all relevant website content\n",
" \"\"\"\n",
" result = \"Landing page:\\n\"\n",
" website = Website(url)\n",
" result += website.get_contents()\n",
"\n",
" # Analyze links and get brochure-relevant ones\n",
" links = LinkAnalyzer.get_links(website)\n",
" brochure_links = links.get('brochure_links', [])\n",
" print(\"Found Brochure links:\", brochure_links)\n",
"\n",
" # Gather content from brochure-relevant pages\n",
" for link in brochure_links:\n",
" result += f\"\\n\\n{link['type']}:\\n\"\n",
" full_url = urljoin(url, link[\"url\"])\n",
" result += Website(full_url).get_contents()\n",
"\n",
" return result\n",
"\n",
" @staticmethod\n",
" def get_brochure_user_prompt(company_name: str, url: str) -> str:\n",
" \"\"\"\n",
" Generate a user prompt for the OpenAI model to create a brochure.\n",
" \n",
" :param company_name: The name of the company\n",
" :param url: The URL of the company's main page\n",
" :return: A string containing the user prompt for brochure generation\n",
" \"\"\"\n",
" user_prompt = f\"You are looking at a company called: {company_name}\\n\"\n",
" user_prompt += f\"Here are the contents of its landing page and other relevant pages; use this information to build a short brochure of the company in markdown.\\n\"\n",
" user_prompt += BrochureGenerator.get_all_details(url)\n",
" return user_prompt[:20_000] # Truncate if more than 20,000 characters\n",
"\n",
" @staticmethod\n",
" def stream_brochure(company_name: str, url: str):\n",
" \"\"\"\n",
" Generate and stream a company brochure.\n",
" \n",
" :param company_name: The name of the company\n",
" :param url: The URL of the company's main page\n",
" \"\"\"\n",
" # Make a streaming API call to OpenAI for brochure generation\n",
" stream = openai.chat.completions.create(\n",
" model=MODEL,\n",
" messages=[\n",
" {\"role\": \"system\", \"content\": BrochureGenerator.SYSTEM_PROMPT},\n",
" {\"role\": \"user\", \"content\": BrochureGenerator.get_brochure_user_prompt(company_name, url)}\n",
" ],\n",
" stream=True\n",
" )\n",
"\n",
" # Display the generated brochure in real-time\n",
" response = \"\"\n",
" display_handle = display(Markdown(\"\"), display_id=True)\n",
" for chunk in stream:\n",
" response += chunk.choices[0].delta.content or ''\n",
" response = response.replace(\"```\", \"\").replace(\"markdown\", \"\")\n",
" update_display(Markdown(response), display_id=display_handle.display_id)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "cc4965cf-f704-4d40-8b7d-f8e50913f87c",
"metadata": {},
"outputs": [],
"source": [
"# Main execution block\n",
"if __name__ == \"__main__\":\n",
" # Generate a brochure\n",
" BrochureGenerator.stream_brochure(\"Edward Donner\", \"https://edwarddonner.com/\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0acb1194-fe89-40e3-8c3b-a10483315d3f",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.11"
}
},
"nbformat": 4,
"nbformat_minor": 5
}