{ "cells": [ { "cell_type": "markdown", "id": "c97ad592-c8be-4583-a19c-ac813e56f410", "metadata": {}, "source": [ "## Mac Users\n", "\n", "I find some challenges while setting up this in MAC silicon M1 chip. Execute below commands in MAC terminal.\n", "\n", "1. Download chromedriver.\n", "2. Unzip and add it to the path.\n", "3. Set Extended attributes." ] }, { "cell_type": "markdown", "id": "b635b345-b000-48cc-8a7f-7df279a489a3", "metadata": {}, "source": [ "cd ~/Downloads\n", "wget https://storage.googleapis.com/chrome-for-testing-public/133.0.6943.126/mac-arm64/chromedriver-mac-arm64.zip\n", "unzip chromedriver-mac-arm64.zip\n", "sudo mv chromedriver-mac-arm64/chromedriver /usr/local/bin/\n", "chmod +x /usr/local/bin/chromedriver\n", "cd /usr/local/bin/\n", "xattr -d com.apple.quarantine chromedriver\n", "cd \n", "chromedriver --version" ] }, { "cell_type": "code", "execution_count": null, "id": "17c7c79a-8ae0-4f5d-a7c8-c54aa7ba90fd", "metadata": {}, "outputs": [], "source": [ "!pip install selenium\n", "!pip install undetected-chromedriver\n", "!pip install beautifulsoup4" ] }, { "cell_type": "code", "execution_count": null, "id": "c10bd630-2dfd-4572-8c21-2dc4c6a372ab", "metadata": {}, "outputs": [], "source": [ "from selenium import webdriver\n", "from selenium.webdriver.chrome.service import Service\n", "from selenium.webdriver.common.by import By\n", "from selenium.webdriver.chrome.options import Options\n", "from openai import OpenAI\n", "import os\n", "import requests\n", "from dotenv import load_dotenv\n", "from bs4 import BeautifulSoup\n", "from IPython.display import Markdown, display\n", "from openai import OpenAI" ] }, { "cell_type": "code", "execution_count": null, "id": "6fb3641d-e9f8-4f5b-bb9d-ee0e971cccdb", "metadata": {}, "outputs": [], "source": [ "OLLAMA_API = \"http://localhost:11434/api/chat\"\n", "HEADERS = {\"Content-Type\": \"application/json\"}\n", "MODEL = \"llama3.2\"\n", "PATH_TO_CHROME_DRIVER = '/usr/local/bin/chromedriver'\n", "system_prompt = \"You are an assistant that analyzes the contents of a website \\\n", "and provides a short summary, ignoring text that might be navigation related. \\\n", "Respond in markdown. Highlight all the products this website offered and also find when website is created.\"\n" ] }, { "cell_type": "code", "execution_count": null, "id": "5d57e958", "metadata": {}, "outputs": [], "source": [ "class Website:\n", " url: str\n", " title: str\n", " text: str\n", "\n", " def __init__(self, url):\n", " self.url = url\n", "\n", " options = Options()\n", "\n", " options.add_argument(\"--no-sandbox\")\n", " options.add_argument(\"--disable-dev-shm-usage\")\n", "\n", " service = Service(PATH_TO_CHROME_DRIVER)\n", " driver = webdriver.Chrome(service=service, options=options)\n", " driver.get(url)\n", "\n", " # input(\"Please complete the verification in the browser and press Enter to continue...\")\n", " page_source = driver.page_source\n", " driver.quit()\n", "\n", " soup = BeautifulSoup(page_source, 'html.parser')\n", " self.title = soup.title.string if soup.title else \"No title found\"\n", " for irrelevant in soup([\"script\", \"style\", \"img\", \"input\"]):\n", " irrelevant.decompose()\n", " self.text = soup.get_text(separator=\"\\n\", strip=True)" ] }, { "cell_type": "code", "execution_count": null, "id": "56df8cd2-2707-43f6-a066-3367846929b3", "metadata": {}, "outputs": [], "source": [ "def user_prompt_for(website):\n", " user_prompt = f\"You are looking at a website titled {website.title}\"\n", " user_prompt += \"\\nThe contents of this website is as follows; \\\n", "please provide a short summary of this website in markdown. \\\n", "If it includes news or announcements, then summarize these too.\\n\\n\"\n", " user_prompt += website.text\n", " return user_prompt\n", "\n", "\n", "\n", "def messages_for(website):\n", " return [\n", " {\"role\": \"system\", \"content\": system_prompt},\n", " {\"role\": \"user\", \"content\": user_prompt_for(website)}\n", " ]\n", "\n", "\n", "def summarize(url):\n", " website = Website(url)\n", " ollama_via_openai = OpenAI(base_url='http://localhost:11434/v1', api_key='ollama')\n", " response = ollama_via_openai.chat.completions.create(\n", " model=MODEL,\n", " messages = messages_for(website)\n", " )\n", " return response.choices[0].message.content\n", "\n", "\n", "def display_summary(url):\n", " summary = summarize(url)\n", " display(Markdown(summary))" ] }, { "cell_type": "code", "execution_count": null, "id": "f2eb9599", "metadata": {}, "outputs": [], "source": [ "display_summary(\"https://ae.almosafer.com\")" ] }, { "cell_type": "code", "execution_count": null, "id": "31b66c0f-6b45-4986-b77c-758625945a91", "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "Python 3 (ipykernel)", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.11.11" } }, "nbformat": 4, "nbformat_minor": 5 }