diff --git a/week1/community-contributions/day1_with_selenium-CB.ipynb b/week1/community-contributions/day1_with_selenium-CB.ipynb new file mode 100644 index 0000000..b7664b6 --- /dev/null +++ b/week1/community-contributions/day1_with_selenium-CB.ipynb @@ -0,0 +1,281 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "0f587c07-9db3-4a29-8373-b2c8d55e51fe", + "metadata": {}, + "outputs": [], + "source": [ + "# imports\n", + "import os\n", + "import requests\n", + "from bs4 import BeautifulSoup\n", + "import time\n", + "import undetected_chromedriver as uc\n", + "from dotenv import load_dotenv\n", + "from selenium.webdriver.common.by import By\n", + "from selenium.webdriver.support.ui import WebDriverWait\n", + "from selenium.webdriver.support import expected_conditions as EC\n", + "from IPython.display import Markdown, display\n", + "from openai import OpenAI" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "91384952-e9ba-42b7-b18b-de899b5dc378", + "metadata": {}, + "outputs": [], + "source": [ + "# Load environment variables in a file called .env\n", + "\n", + "load_dotenv(override=True)\n", + "api_key = os.getenv('OPENAI_API_KEY')\n", + "\n", + "# Check the key\n", + "\n", + "if not api_key:\n", + " print(\"No API key was found - please head over to the troubleshooting notebook in this folder to identify & fix!\")\n", + "elif not api_key.startswith(\"sk-proj-\"):\n", + " print(\"An API key was found, but it doesn't start sk-proj-; please check you're using the right key - see troubleshooting notebook\")\n", + "elif api_key.strip() != api_key:\n", + " print(\"An API key was found, but it looks like it might have space or tab characters at the start or end - please remove them - see troubleshooting notebook\")\n", + "else:\n", + " print(\"API key found and looks good so far!\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e45422ad-884e-44e7-ac1d-c48ff5e83970", + "metadata": {}, + "outputs": [], + "source": [ + "# A class to represent a Webpage\n", + "# If you're not familiar with Classes, check out the \"Intermediate Python\" notebook\n", + "\n", + "# Some websites need you to use proper headers when fetching them:\n", + "\n", + "headers = {\n", + " \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36\"\n", + "}\n", + "\n", + "class Website:\n", + " def __init__(self, url):\n", + " self.url = url\n", + " \n", + " # Set up undetected Chrome options\n", + " options = uc.ChromeOptions()\n", + " options.add_argument(\"--disable-gpu\")\n", + " options.add_argument(\"window-size=1920,1080\")\n", + " # For debugging, you can run in normal mode (i.e., not headless)\n", + " # options.add_argument(\"--headless\")\n", + " \n", + " # Initialize undetected_chromedriver\n", + " driver = uc.Chrome(options=options, version_main=132)\n", + " driver.get(url)\n", + " \n", + " try:\n", + " # Use an explicit wait for an element that confirms the page has fully loaded.\n", + " # Adjust the selector to one that appears after the challenge is complete.\n", + " WebDriverWait(driver, 20).until(\n", + " EC.presence_of_element_located((By.TAG_NAME, \"body\"))\n", + " )\n", + " except Exception as e:\n", + " print(\"Timeout waiting for page to load:\", e)\n", + " \n", + " # Alternatively, you can add a longer sleep if necessary\n", + " time.sleep(10)\n", + " \n", + " page_source = driver.page_source\n", + " soup = BeautifulSoup(page_source, 'html.parser')\n", + " \n", + " # Extract title and clean up unwanted tags\n", + " self.title = driver.title if driver.title else \"No title found\"\n", + " if soup.body:\n", + " for tag in soup.body.find_all([\"script\", \"style\", \"img\", \"input\"]):\n", + " tag.decompose()\n", + " self.text = soup.body.get_text(separator=\"\\n\", strip=True)\n", + " else:\n", + " self.text = \"No body content found.\"\n", + " \n", + " # For debugging, you might want to keep the browser open to inspect:\n", + " # input(\"Press Enter to close the browser...\")\n", + "\n", + " driver.quit()\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b8eb670c-1ca6-48f2-b0c7-8a65fa875548", + "metadata": {}, + "outputs": [], + "source": [ + "# Testing the modified Website class\n", + "#ed = Website(\"https://edwarddonner.com\")\n", + "ed = Website(\"https://openai.com\")\n", + "print(ed.title)\n", + "print(ed.text)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "01d9142b-74c5-460e-8993-7f865052c2ca", + "metadata": {}, + "outputs": [], + "source": [ + "# Define our system prompt - you can experiment with this later, changing the last sentence to 'Respond in markdown in Spanish.\"\n", + "\n", + "system_prompt = \"You are an assistant that analyzes the contents of a website \\\n", + "and provides a short summary, ignoring text that might be navigation related. \\\n", + "Respond in markdown. \"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7155a2c5-315c-4ac0-9d94-70983d8d28e4", + "metadata": {}, + "outputs": [], + "source": [ + "# A function that writes a User Prompt that asks for summaries of websites:\n", + "\n", + "def user_prompt_for(website):\n", + " user_prompt = f\"You are looking at a website titled {website.title}\"\n", + " user_prompt += \"\\nThe contents of this website is as follows; \\\n", + "please provide a short summary of this website in markdown. \\\n", + "If it includes news or announcements, then summarize these too.\\n\\n\"\n", + " user_prompt += website.text\n", + " return user_prompt" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "3564820a-d499-4dd7-91b4-56dd43663b5b", + "metadata": {}, + "outputs": [], + "source": [ + "print(user_prompt_for(ed))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "eff2e8d0-d999-488b-adbe-069c5b2339d3", + "metadata": {}, + "outputs": [], + "source": [ + "# See how this function creates exactly the format above\n", + "\n", + "def messages_for(website):\n", + " return [\n", + " {\"role\": \"system\", \"content\": system_prompt},\n", + " {\"role\": \"user\", \"content\": user_prompt_for(website)}\n", + " ]" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b3cc6a30-fc3b-4a7b-a095-5ff744918cca", + "metadata": {}, + "outputs": [], + "source": [ + "# Try this out, and then try for a few more websites\n", + "\n", + "messages_for(ed)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "1406592c-2156-4c41-921a-e3f15e9b4828", + "metadata": {}, + "outputs": [], + "source": [ + "# And now: call the OpenAI API. You will get very familiar with this!\n", + "\n", + "def summarize(url):\n", + " website = Website(url)\n", + " response = openai.chat.completions.create(\n", + " model = \"gpt-4o-mini\",\n", + " messages = messages_for(website)\n", + " )\n", + " return response.choices[0].message.content\n", + "\n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "134a78a1-119b-4ef5-890b-ceb2d94ab4cc", + "metadata": {}, + "outputs": [], + "source": [ + "#This seems to not working - Need some help\n", + "\n", + "print(summarize(\"https://openai.com\"))\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "930dc9be-5389-4c20-b788-d27cda85389c", + "metadata": {}, + "outputs": [], + "source": [ + "summarize(\"https://openai.com\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "fc77656b-0f1a-4204-944e-e9d04d3072c7", + "metadata": {}, + "outputs": [], + "source": [ + "# A function to display this nicely in the Jupyter output, using markdown\n", + "\n", + "def display_summary(url):\n", + " summary = summarize(url)\n", + " display(Markdown(summary))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "96c03dba-f999-431c-94b7-cbda3732e806", + "metadata": {}, + "outputs": [], + "source": [ + "display_summary(\"https://openai.com\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +}