2 changed files with 360 additions and 0 deletions
@ -0,0 +1,112 @@
|
||||
{ |
||||
"cells": [ |
||||
{ |
||||
"cell_type": "code", |
||||
"execution_count": null, |
||||
"id": "3ba06289-d17a-4ccd-85f5-2b79956d4e59", |
||||
"metadata": {}, |
||||
"outputs": [], |
||||
"source": [ |
||||
"!pip install selenium" |
||||
] |
||||
}, |
||||
{ |
||||
"cell_type": "code", |
||||
"execution_count": null, |
||||
"id": "eabbbc62-1de1-4883-9b3e-9c90145ea6c5", |
||||
"metadata": {}, |
||||
"outputs": [], |
||||
"source": [ |
||||
"from selenium import webdriver\n", |
||||
"from selenium.webdriver.chrome.options import Options\n", |
||||
"from selenium.webdriver.chrome.service import Service\n", |
||||
"from bs4 import BeautifulSoup\n", |
||||
"import time\n", |
||||
"import os \n", |
||||
"\n", |
||||
"class Website:\n", |
||||
" def __init__(self, url, driver_path=None, wait_time=3):\n", |
||||
" self.url = url\n", |
||||
" self.wait_time = wait_time\n", |
||||
"\n", |
||||
" # Headless Chrome settings\n", |
||||
" options = Options()\n", |
||||
" # options.add_argument(\"--headless\") \n", |
||||
" # Headless mode runs the browser in the background (invisible).\n", |
||||
" # However, some websites (like openai.com) block headless browsers.\n", |
||||
" # So if this line is active, the page may not load correctly and you may not get the full content.\n", |
||||
" options.add_argument(\"--disable-gpu\")\n", |
||||
" options.add_argument(\"--no-sandbox\")\n", |
||||
" options.add_argument(\"--window-size=1920x1080\")\n", |
||||
"\n", |
||||
" # Driver path\n", |
||||
" if driver_path:\n", |
||||
" service = Service(executable_path=driver_path)\n", |
||||
" else:\n", |
||||
" service = Service() \n", |
||||
"\n", |
||||
" # Start browser\n", |
||||
" driver = webdriver.Chrome(service=service, options=options)\n", |
||||
" driver.get(url)\n", |
||||
"\n", |
||||
" # Wait for the loading page\n", |
||||
" time.sleep(self.wait_time)\n", |
||||
"\n", |
||||
" # Take page source\n", |
||||
" html = driver.page_source\n", |
||||
" driver.quit()\n", |
||||
"\n", |
||||
" # Analysis with BeautifulSoup \n", |
||||
" soup = BeautifulSoup(html, 'html.parser')\n", |
||||
" self.title = soup.title.string if soup.title else \"No title found\"\n", |
||||
"\n", |
||||
" # Clean irrelevant tags\n", |
||||
" for irrelevant in soup.body([\"script\", \"style\", \"img\", \"input\"]):\n", |
||||
" irrelevant.decompose()\n", |
||||
"\n", |
||||
" self.text = soup.body.get_text(separator=\"\\n\", strip=True)" |
||||
] |
||||
}, |
||||
{ |
||||
"cell_type": "code", |
||||
"execution_count": null, |
||||
"id": "852c52e2-bd4d-4bb9-94ef-e498c33f1a89", |
||||
"metadata": {}, |
||||
"outputs": [], |
||||
"source": [ |
||||
"site = Website(\"https://openai.com\", driver_path=\"/Users/gizemmervedemir/Downloads/chromedriver-mac-arm64/chromedriver\")\n", |
||||
"print(\"Title:\", site.title)\n", |
||||
"print(\"\\nFirst 500 character:\\n\", site.text[:500])" |
||||
] |
||||
}, |
||||
{ |
||||
"cell_type": "code", |
||||
"execution_count": null, |
||||
"id": "7620c685-c35c-4d6b-aaf1-a3da98f19ca7", |
||||
"metadata": {}, |
||||
"outputs": [], |
||||
"source": [] |
||||
} |
||||
], |
||||
"metadata": { |
||||
"kernelspec": { |
||||
"display_name": "Python 3 (ipykernel)", |
||||
"language": "python", |
||||
"name": "python3" |
||||
}, |
||||
"language_info": { |
||||
"codemirror_mode": { |
||||
"name": "ipython", |
||||
"version": 3 |
||||
}, |
||||
"file_extension": ".py", |
||||
"mimetype": "text/x-python", |
||||
"name": "python", |
||||
"nbconvert_exporter": "python", |
||||
"pygments_lexer": "ipython3", |
||||
"version": "3.11.12" |
||||
} |
||||
}, |
||||
"nbformat": 4, |
||||
"nbformat_minor": 5 |
||||
} |
@ -0,0 +1,248 @@
|
||||
{ |
||||
"cells": [ |
||||
{ |
||||
"cell_type": "markdown", |
||||
"id": "d15d8294-3328-4e07-ad16-8a03e9bbfdb9", |
||||
"metadata": {}, |
||||
"source": [ |
||||
"# Welcome to your first assignment!\n", |
||||
"\n", |
||||
"Instructions are below. Please give this a try, and look in the solutions folder if you get stuck (or feel free to ask me!)" |
||||
] |
||||
}, |
||||
{ |
||||
"cell_type": "markdown", |
||||
"id": "ada885d9-4d42-4d9b-97f0-74fbbbfe93a9", |
||||
"metadata": {}, |
||||
"source": [ |
||||
"<table style=\"margin: 0; text-align: left;\">\n", |
||||
" <tr>\n", |
||||
" <td style=\"width: 150px; height: 150px; vertical-align: middle;\">\n", |
||||
" <img src=\"../resources.jpg\" width=\"150\" height=\"150\" style=\"display: block;\" />\n", |
||||
" </td>\n", |
||||
" <td>\n", |
||||
" <h2 style=\"color:#f71;\">Just before we get to the assignment --</h2>\n", |
||||
" <span style=\"color:#f71;\">I thought I'd take a second to point you at this page of useful resources for the course. This includes links to all the slides.<br/>\n", |
||||
" <a href=\"https://edwarddonner.com/2024/11/13/llm-engineering-resources/\">https://edwarddonner.com/2024/11/13/llm-engineering-resources/</a><br/>\n", |
||||
" Please keep this bookmarked, and I'll continue to add more useful links there over time.\n", |
||||
" </span>\n", |
||||
" </td>\n", |
||||
" </tr>\n", |
||||
"</table>" |
||||
] |
||||
}, |
||||
{ |
||||
"cell_type": "code", |
||||
"execution_count": null, |
||||
"id": "23057e00-b6fc-4678-93a9-6b31cb704bff", |
||||
"metadata": {}, |
||||
"outputs": [], |
||||
"source": [ |
||||
"# There's actually an alternative approach that some people might prefer\n", |
||||
"# You can use the OpenAI client python library to call Ollama:\n", |
||||
"\n", |
||||
"from openai import OpenAI\n", |
||||
"ollama_via_openai = OpenAI(base_url='http://localhost:11434/v1', api_key='ollama')\n", |
||||
"\n", |
||||
"response = ollama_via_openai.chat.completions.create(\n", |
||||
" model=MODEL,\n", |
||||
" messages=messages\n", |
||||
")\n", |
||||
"\n", |
||||
"print(response.choices[0].message.content)" |
||||
] |
||||
}, |
||||
{ |
||||
"cell_type": "markdown", |
||||
"id": "1622d9bb-5c68-4d4e-9ca4-b492c751f898", |
||||
"metadata": {}, |
||||
"source": [ |
||||
"# NOW the exercise for you\n", |
||||
"\n", |
||||
"Take the code from day1 and incorporate it here, to build a website summarizer that uses Llama 3.2 running locally instead of OpenAI; use either of the above approaches." |
||||
] |
||||
}, |
||||
{ |
||||
"cell_type": "code", |
||||
"execution_count": 28, |
||||
"id": "37e35a64-7c2a-453d-96fa-9c8119c6618d", |
||||
"metadata": {}, |
||||
"outputs": [], |
||||
"source": [ |
||||
"# imports\n", |
||||
"\n", |
||||
"import os\n", |
||||
"import requests\n", |
||||
"from bs4 import BeautifulSoup\n", |
||||
"from IPython.display import Markdown, display\n", |
||||
"from openai import OpenAI" |
||||
] |
||||
}, |
||||
{ |
||||
"cell_type": "code", |
||||
"execution_count": 29, |
||||
"id": "fc410fe7-7abe-48ab-9206-ec6412278ac5", |
||||
"metadata": {}, |
||||
"outputs": [], |
||||
"source": [ |
||||
"# Constants\n", |
||||
"\n", |
||||
"OLLAMA_API = \"http://localhost:11434/api/chat\"\n", |
||||
"HEADERS = {\"Content-Type\": \"application/json\"}\n", |
||||
"MODEL = \"llama3.2\"" |
||||
] |
||||
}, |
||||
{ |
||||
"cell_type": "code", |
||||
"execution_count": 30, |
||||
"id": "654af616-1ad4-4d28-be41-f3c99b6e8f42", |
||||
"metadata": {}, |
||||
"outputs": [], |
||||
"source": [ |
||||
"# A class to represent a Webpage\n", |
||||
"\n", |
||||
"# Some websites need you to use proper headers when fetching them:\n", |
||||
"headers = {\n", |
||||
" \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36\"\n", |
||||
"}\n", |
||||
"\n", |
||||
"class Website:\n", |
||||
"\n", |
||||
" def __init__(self, url):\n", |
||||
" \"\"\"\n", |
||||
" Create this Website object from the given url using the BeautifulSoup library\n", |
||||
" \"\"\"\n", |
||||
" self.url = url\n", |
||||
" response = requests.get(url, headers=headers)\n", |
||||
" soup = BeautifulSoup(response.content, 'html.parser')\n", |
||||
" self.title = soup.title.string if soup.title else \"No title found\"\n", |
||||
" for irrelevant in soup.body([\"script\", \"style\", \"img\", \"input\"]):\n", |
||||
" irrelevant.decompose()\n", |
||||
" self.text = soup.body.get_text(separator=\"\\n\", strip=True)" |
||||
] |
||||
}, |
||||
{ |
||||
"cell_type": "code", |
||||
"execution_count": 31, |
||||
"id": "f665c051-95a2-4102-8e26-1974bd5c7d3a", |
||||
"metadata": {}, |
||||
"outputs": [], |
||||
"source": [ |
||||
"def user_prompt_for(website):\n", |
||||
" user_prompt = f\"You are looking at a website titled {website.title}\"\n", |
||||
" user_prompt += \"\\nThe contents of this website is as follows; \\\n", |
||||
"please provide a short summary of this website in markdown. \\\n", |
||||
"If it includes news or announcements, then summarize these too.\\n\\n\"\n", |
||||
" user_prompt += website.text\n", |
||||
" return user_prompt" |
||||
] |
||||
}, |
||||
{ |
||||
"cell_type": "code", |
||||
"execution_count": 32, |
||||
"id": "258cf0af-650f-4225-b1c1-8f29e209ebfd", |
||||
"metadata": {}, |
||||
"outputs": [], |
||||
"source": [ |
||||
"def messages_for(website):\n", |
||||
" return [\n", |
||||
" {\"role\": \"user\", \"content\": user_prompt_for(website)}\n", |
||||
" ]" |
||||
] |
||||
}, |
||||
{ |
||||
"cell_type": "code", |
||||
"execution_count": 42, |
||||
"id": "fe5291b0-a2bb-4b60-af77-d33517a7005b", |
||||
"metadata": {}, |
||||
"outputs": [], |
||||
"source": [ |
||||
"def summarize(url):\n", |
||||
" website = Website(url)\n", |
||||
" client = OpenAI(base_url=\"http://localhost:11434/v1\", api_key=\"ollama\")\n", |
||||
" response = client.chat.completions.create(\n", |
||||
" model=MODEL,\n", |
||||
" messages=messages_for(website)\n", |
||||
" )\n", |
||||
" return response.choices[0].message.content" |
||||
] |
||||
}, |
||||
{ |
||||
"cell_type": "code", |
||||
"execution_count": 43, |
||||
"id": "b53f34cd-f8ce-4656-a46a-33e966156e2e", |
||||
"metadata": {}, |
||||
"outputs": [], |
||||
"source": [ |
||||
"# A function to display this nicely in the Jupyter output, using markdown\n", |
||||
"\n", |
||||
"def display_summary(url):\n", |
||||
" summary = summarize(url)\n", |
||||
" display(Markdown(summary))" |
||||
] |
||||
}, |
||||
{ |
||||
"cell_type": "code", |
||||
"execution_count": 44, |
||||
"id": "5b28ccfa-eb27-4154-aeb6-aff439c8a723", |
||||
"metadata": {}, |
||||
"outputs": [ |
||||
{ |
||||
"data": { |
||||
"text/markdown": [ |
||||
"**Website Summary**\n", |
||||
"=====================\n", |
||||
"\n", |
||||
"### About the Website\n", |
||||
"\n", |
||||
"This website is owned by Edward Donner, a co-founder and CTO of Nebula.io, an AI company that applies AI to help people discover their potential. The website provides information about his background, experience, and work with LLMs (Large Language Models).\n", |
||||
"\n", |
||||
"### News and Announcements\n", |
||||
"\n", |
||||
"* **Upcoming Events:**\n", |
||||
" + January 23, 2025: LLM Workshop - Hands-on with Agents - resources\n", |
||||
" + December 21, 2024: Welcome to the SuperDataScientists community!\n", |
||||
" + November 13, 2024: Mastering AI and LLL Engineering - Resources\n", |
||||
" + October 16, 2024: From Software Engineer to AI Data Scientist - resources\n", |
||||
"* **Acquisition:**\n", |
||||
" + In 2021, Edward's previous startup untapt was acquired.\n", |
||||
"\n", |
||||
"### Links\n", |
||||
"\n", |
||||
"The website also provides links to Edward Donner's social media profiles (LinkedIn, Twitter, Facebook), as well as a newsletter signup form." |
||||
], |
||||
"text/plain": [ |
||||
"<IPython.core.display.Markdown object>" |
||||
] |
||||
}, |
||||
"metadata": {}, |
||||
"output_type": "display_data" |
||||
} |
||||
], |
||||
"source": [ |
||||
"display_summary(\"https://edwarddonner.com\")" |
||||
] |
||||
} |
||||
], |
||||
"metadata": { |
||||
"kernelspec": { |
||||
"display_name": "Python 3 (ipykernel)", |
||||
"language": "python", |
||||
"name": "python3" |
||||
}, |
||||
"language_info": { |
||||
"codemirror_mode": { |
||||
"name": "ipython", |
||||
"version": 3 |
||||
}, |
||||
"file_extension": ".py", |
||||
"mimetype": "text/x-python", |
||||
"name": "python", |
||||
"nbconvert_exporter": "python", |
||||
"pygments_lexer": "ipython3", |
||||
"version": "3.11.12" |
||||
} |
||||
}, |
||||
"nbformat": 4, |
||||
"nbformat_minor": 5 |
||||
} |
Loading…
Reference in new issue