{ "cells": [ { "cell_type": "markdown", "id": "a98030af-fcd1-4d63-a36e-38ba053498fa", "metadata": {}, "source": [ "# A full business solution\n", "\n", "Create a product that builds a Brochure for a company to be used for prospective clients, investors and potential recruits.\n", "\n", "We will be provided a company name and their primary website." ] }, { "cell_type": "code", "execution_count": null, "id": "0a572211-5fe3-4dd5-9870-849cfb75901f", "metadata": {}, "outputs": [], "source": [ "# Import necessary libraries\n", "import os\n", "import requests\n", "import json\n", "from typing import List, Dict\n", "from dotenv import load_dotenv\n", "from bs4 import BeautifulSoup\n", "from IPython.display import Markdown, display, update_display\n", "from openai import OpenAI\n", "from urllib.parse import urljoin\n", "\n", "# Load environment variables from a .env file\n", "load_dotenv()\n", "\n", "# Define constants\n", "MODEL = 'gpt-4o-mini' # Specify the OpenAI model to use\n", "OPENAI_API_KEY = os.getenv('OPENAI_API_KEY', 'your-key-if-not-using-env') # Get API key from environment or use default\n", "\n", "# Initialize OpenAI client with the API key\n", "openai = OpenAI(api_key=OPENAI_API_KEY)\n", "\n", "class Website:\n", " \"\"\"\n", " A class to represent a website and its contents.\n", " \"\"\"\n", " def __init__(self, url: str):\n", " \"\"\"\n", " Initialize the Website object with a given URL.\n", " \n", " :param url: The URL of the website to scrape\n", " \"\"\"\n", " self.url = url\n", " self.title, self.text, self.links = self._scrape_website()\n", "\n", " def _scrape_website(self) -> tuple:\n", " \"\"\"\n", " Scrape the website content, extracting title, text, and links.\n", " \n", " :return: A tuple containing the title, text content, and links of the website\n", " \"\"\"\n", " response = requests.get(self.url)\n", " soup = BeautifulSoup(response.content, 'html.parser')\n", " \n", " # Extract title\n", " title = soup.title.string if soup.title else \"No title found\"\n", " \n", " # Extract text content\n", " if soup.body:\n", " for tag in soup.body([\"script\", \"style\", \"img\", \"input\"]):\n", " tag.decompose() # Remove unwanted tags\n", " text = soup.body.get_text(separator=\"\\n\", strip=True)\n", " else:\n", " text = \"\"\n", " \n", " # Extract links\n", " links = [link.get('href') for link in soup.find_all('a') if link.get('href')]\n", " \n", " return title, text, links\n", "\n", " def get_contents(self) -> str:\n", " \"\"\"\n", " Get a formatted string of the website contents.\n", " \n", " :return: A string containing the website title and text content\n", " \"\"\"\n", " return f\"Webpage Title:\\n{self.title}\\nWebpage Contents:\\n{self.text}\\n\\n\"\n", "\n", "class LinkAnalyzer:\n", " \"\"\"\n", " A class to analyze and categorize links from a website.\n", " \"\"\"\n", " # System prompt for the OpenAI model to categorize links\n", " LINK_SYSTEM_PROMPT = \"\"\"\n", " You are provided with a list of links found on a webpage. Your task is to first categorize each link into one of the following categories:\n", " - about page\n", " - careers page\n", " - terms of service\n", " - privacy policy\n", " - contact page\n", " - other (please specify).\n", "\n", " Once the links are categorized, please choose which links are most relevant to include in a brochure about the company. \n", " The brochure should only include links such as About pages, Careers pages, or Company Overview pages. Exclude any links related to Terms of Service, Privacy Policy, or email addresses.\n", "\n", " Respond in the following JSON format:\n", " {\n", " \"categorized_links\": [\n", " {\"category\": \"about page\", \"url\": \"https://full.url/about\"},\n", " {\"category\": \"careers page\", \"url\": \"https://full.url/careers\"},\n", " {\"category\": \"terms of service\", \"url\": \"https://full.url/terms\"},\n", " {\"category\": \"privacy policy\", \"url\": \"https://full.url/privacy\"},\n", " {\"category\": \"other\", \"specify\": \"contact page\", \"url\": \"https://full.url/contact\"}\n", " ],\n", " \"brochure_links\": [\n", " {\"type\": \"about page\", \"url\": \"https://full.url/about\"},\n", " {\"type\": \"careers page\", \"url\": \"https://full.url/careers\"}\n", " ]\n", " }\n", "\n", " Please find the links below and proceed with the task:\n", "\n", " Links (some may be relative links):\n", " [INSERT LINK LIST HERE]\n", " \"\"\"\n", "\n", " @staticmethod\n", " def get_links(website: Website) -> Dict:\n", " \"\"\"\n", " Analyze and categorize links from a given website.\n", " \n", " :param website: A Website object containing the links to analyze\n", " :return: A dictionary containing categorized links and brochure-relevant links\n", " \"\"\"\n", " # Prepare the user prompt for the OpenAI model\n", " user_prompt = f\"Here is the list of links on the website of {website.url} - \"\n", " user_prompt += \"please decide which of these are relevant web links for a brochure about the company, respond with the full https URL in JSON format. \\\n", " Do not include Terms of Service, Privacy, email links.\\n\"\n", " user_prompt += \"Links (some might be relative links):\\n\"\n", " user_prompt += \"\\n\".join(website.links)\n", "\n", " # Make an API call to OpenAI for link analysis\n", " completion = openai.chat.completions.create(\n", " model=MODEL,\n", " messages=[\n", " {\"role\": \"system\", \"content\": LinkAnalyzer.LINK_SYSTEM_PROMPT},\n", " {\"role\": \"user\", \"content\": user_prompt}\n", " ],\n", " response_format={\"type\": \"json_object\"}\n", " )\n", " return json.loads(completion.choices[0].message.content)\n", "\n", "class BrochureGenerator:\n", " \"\"\"\n", " A class to generate a company brochure based on website content.\n", " \"\"\"\n", " # System prompt for the OpenAI model to generate the brochure\n", " SYSTEM_PROMPT = \"\"\"\n", " You are an assistant that analyzes the contents of several relevant pages from a company website \n", " and creates a brochure about the company for prospective customers, investors and recruits. Respond in markdown.\n", " Include details of company culture, customers and careers/jobs if you have the information.\n", " Structure the brochure to include specific sections as follows:\n", " About Us\n", " What we do\n", " How We Do It\n", " Where We Do It\n", " Our People\n", " Our Culture\n", " Connect with Us.\n", " Please provide two versions of the brochure, the first in English, the second in Spanish. The contents of the brochure are to be the same for both languages.\n", " \"\"\"\n", "\n", " @staticmethod\n", " def get_all_details(url: str) -> str:\n", " \"\"\"\n", " Gather all relevant details from a company's website.\n", " \n", " :param url: The URL of the company's main page\n", " :return: A string containing all relevant website content\n", " \"\"\"\n", " result = \"Landing page:\\n\"\n", " website = Website(url)\n", " result += website.get_contents()\n", "\n", " # Analyze links and get brochure-relevant ones\n", " links = LinkAnalyzer.get_links(website)\n", " brochure_links = links.get('brochure_links', [])\n", " print(\"Found Brochure links:\", brochure_links)\n", "\n", " # Gather content from brochure-relevant pages\n", " for link in brochure_links:\n", " result += f\"\\n\\n{link['type']}:\\n\"\n", " full_url = urljoin(url, link[\"url\"])\n", " result += Website(full_url).get_contents()\n", "\n", " return result\n", "\n", " @staticmethod\n", " def get_brochure_user_prompt(company_name: str, url: str) -> str:\n", " \"\"\"\n", " Generate a user prompt for the OpenAI model to create a brochure.\n", " \n", " :param company_name: The name of the company\n", " :param url: The URL of the company's main page\n", " :return: A string containing the user prompt for brochure generation\n", " \"\"\"\n", " user_prompt = f\"You are looking at a company called: {company_name}\\n\"\n", " user_prompt += f\"Here are the contents of its landing page and other relevant pages; use this information to build a short brochure of the company in markdown.\\n\"\n", " user_prompt += BrochureGenerator.get_all_details(url)\n", " return user_prompt[:20_000] # Truncate if more than 20,000 characters\n", "\n", " @staticmethod\n", " def stream_brochure(company_name: str, url: str):\n", " \"\"\"\n", " Generate and stream a company brochure.\n", " \n", " :param company_name: The name of the company\n", " :param url: The URL of the company's main page\n", " \"\"\"\n", " # Make a streaming API call to OpenAI for brochure generation\n", " stream = openai.chat.completions.create(\n", " model=MODEL,\n", " messages=[\n", " {\"role\": \"system\", \"content\": BrochureGenerator.SYSTEM_PROMPT},\n", " {\"role\": \"user\", \"content\": BrochureGenerator.get_brochure_user_prompt(company_name, url)}\n", " ],\n", " stream=True\n", " )\n", "\n", " # Display the generated brochure in real-time\n", " response = \"\"\n", " display_handle = display(Markdown(\"\"), display_id=True)\n", " for chunk in stream:\n", " response += chunk.choices[0].delta.content or ''\n", " response = response.replace(\"```\", \"\").replace(\"markdown\", \"\")\n", " update_display(Markdown(response), display_id=display_handle.display_id)\n" ] }, { "cell_type": "code", "execution_count": null, "id": "cc4965cf-f704-4d40-8b7d-f8e50913f87c", "metadata": {}, "outputs": [], "source": [ "# Main execution block\n", "if __name__ == \"__main__\":\n", " # Generate a brochure\n", " BrochureGenerator.stream_brochure(\"Edward Donner\", \"https://edwarddonner.com/\")" ] }, { "cell_type": "code", "execution_count": null, "id": "0acb1194-fe89-40e3-8c3b-a10483315d3f", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.10" } }, "nbformat": 4, "nbformat_minor": 5 }