From 5f1a9d9818495821dd152eefeabdd26573daa985 Mon Sep 17 00:00:00 2001 From: nssuu Date: Fri, 7 Feb 2025 00:11:00 +0300 Subject: [PATCH 1/2] Complete web page summarizer using gemini 2.0 flash --- ...eek1_web_page_summarizer_with_gemini.ipynb | 274 ++++++++++++++++++ 1 file changed, 274 insertions(+) create mode 100644 week1/community-contributions/week1_web_page_summarizer_with_gemini.ipynb diff --git a/week1/community-contributions/week1_web_page_summarizer_with_gemini.ipynb b/week1/community-contributions/week1_web_page_summarizer_with_gemini.ipynb new file mode 100644 index 0000000..ebcf6ef --- /dev/null +++ b/week1/community-contributions/week1_web_page_summarizer_with_gemini.ipynb @@ -0,0 +1,274 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "b6a90e2f-9e4e-4db3-ba67-4d57c162d44a", + "metadata": {}, + "source": [ + "\n" + ] + }, + { + "cell_type": "markdown", + "id": "6527632f-5a92-4d8d-b395-181567886832", + "metadata": {}, + "source": [ + "## Extra requirements\n", + "```bash\n", + "pip install -q -U google-genai\n", + "```\n", + "\n", + "## Required environment variable\n", + "GEMINI_API_KEY\n", + "\n", + "### How to get GEMINI API KEY\n", + "\n", + "Use the link: [gemini api key](https://aistudio.google.com/app/apikey) to get yours." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "7a251685-3e61-4e3d-a58f-b560173687d3", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import requests\n", + "from bs4 import BeautifulSoup\n", + "from dotenv import load_dotenv\n", + "from google import genai\n", + "from google.genai import types\n" + ] + }, + { + "cell_type": "code", + "execution_count": 53, + "id": "601dc478-8733-4eca-be1f-23900feea733", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Api key found. Good to go!\n" + ] + } + ], + "source": [ + "load_dotenv()\n", + "api_key = os.getenv(\"GEMINI_API_KEY\")\n", + "\n", + "if not api_key or len(api_key) < 39:\n", + " print(\"No correct api key was found\")\n", + "else:\n", + " print(\"Api key found. Good to go!\")" + ] + }, + { + "cell_type": "code", + "execution_count": 54, + "id": "96c56b5b-2487-4548-94c1-3cd7fb771e2d", + "metadata": {}, + "outputs": [], + "source": [ + "client = genai.Client(api_key=api_key)" + ] + }, + { + "cell_type": "code", + "execution_count": 55, + "id": "552d889c-1699-49b5-a880-60054e2dacee", + "metadata": {}, + "outputs": [], + "source": [ + "class Website:\n", + " url: str\n", + " title: str\n", + " text: str\n", + "\n", + " def __init__(self, url):\n", + " self.url = url\n", + " response = requests.get(url)\n", + " soup = BeautifulSoup(response.content, \"html.parser\")\n", + " self.title = soup.title.string if soup.title else \"No title was found\"\n", + "\n", + " for irr in soup.body([\"script\", \"style\", \"img\", \"input\"]):\n", + " irr.decompose()\n", + " self.text = soup.body.get_text(separator=\"\\n\", strip=True)\n", + " \n", + " " + ] + }, + { + "cell_type": "code", + "execution_count": 56, + "id": "4b2a7585-d6fe-4160-af94-e905eaa13109", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Home - Edward Donner\n", + "Home\n", + "Outsmart\n", + "An arena that pits LLMs against each other in a battle of diplomacy and deviousness\n", + "About\n", + "Posts\n", + "Well, hi there.\n", + "I’m Ed. I like writing code and experimenting with LLMs, and hopefully you’re here because you do too. I also enjoy DJing (but I’m badly out of practice), amateur electronic music production (\n", + "very\n", + "amateur) and losing myself in\n", + "Hacker News\n", + ", nodding my head sagely to things I only half understand.\n", + "I’m the co-founder and CTO of\n", + "Nebula.io\n", + ". We’re applying AI to a field where it can make a massive, positive impact: helping people discover their potential and pursue their reason for being. Recruiters use our product today to source, understand, engage and manage talent. I’m previously the founder and CEO of AI startup untapt,\n", + "acquired in 2021\n", + ".\n", + "We work with groundbreaking, proprietary LLMs verticalized for talent, we’ve\n", + "patented\n", + "our matching model, and our award-winning platform has happy customers and tons of press coverage.\n", + "Connect\n", + "with me for more!\n", + "January 23, 2025\n", + "LLM Workshop – Hands-on with Agents – resources\n", + "December 21, 2024\n", + "Welcome, SuperDataScientists!\n", + "November 13, 2024\n", + "Mastering AI and LLM Engineering – Resources\n", + "October 16, 2024\n", + "From Software Engineer to AI Data Scientist – resources\n", + "Navigation\n", + "Home\n", + "Outsmart\n", + "An arena that pits LLMs against each other in a battle of diplomacy and deviousness\n", + "About\n", + "Posts\n", + "Get in touch\n", + "ed [at] edwarddonner [dot] com\n", + "www.edwarddonner.com\n", + "Follow me\n", + "LinkedIn\n", + "Twitter\n", + "Facebook\n", + "Subscribe to newsletter\n", + "Type your email…\n", + "Subscribe\n" + ] + } + ], + "source": [ + "ed = Website(\"https://edwarddonner.com\")\n", + "print(ed.title)\n", + "print(ed.text)" + ] + }, + { + "cell_type": "code", + "execution_count": 57, + "id": "fd44adf0-9163-4bb6-af2c-a1badffe547f", + "metadata": {}, + "outputs": [], + "source": [ + "system_prompt = \"You are an assistant that analyzes the contents of a website \\\n", + "and provides a short summary, ignoring text that navigation related. Respond \\\n", + "in markdown.\"" + ] + }, + { + "cell_type": "code", + "execution_count": 58, + "id": "94a38a88-bd74-40f5-81f5-9e7169ccc1c3", + "metadata": {}, + "outputs": [], + "source": [ + "def user_prompt_for(website):\n", + " user_prompt = f\"You are looking at a website titled {website.title}\"\n", + " user_prompt += \"\\nThe contents of this website is as follows; \\\n", + "please provide a short summary of this website in markdown. \\\n", + "If it includes news or announcements, then summarize these too.\\n\\n\"\n", + " user_prompt += website.text\n", + " return user_prompt" + ] + }, + { + "cell_type": "code", + "execution_count": 61, + "id": "d783aafb-4636-4909-b1e8-03b216de845a", + "metadata": {}, + "outputs": [], + "source": [ + "def summarize(url):\n", + " website = Website(url)\n", + " response = client.models.generate_content(\n", + " model=\"gemini-2.0-flash\",\n", + " config=types.GenerateContentConfig(\n", + " system_instruction=system_prompt),\n", + " contents=user_prompt_for(website)\n", + " )\n", + " return response.text" + ] + }, + { + "cell_type": "code", + "execution_count": 62, + "id": "d5d45484-dbf1-4d6a-b793-2d50783b59a3", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "\"Edward Donner's website introduces Ed, a coder and LLM enthusiast who enjoys DJing, electronic music, and Hacker News. He is the co-founder and CTO of Nebula.io, an AI company focused on talent discovery. He was also the founder and CEO of AI startup untapt, which was acquired in 2021. The site also has links to LLM resources from past workshops and posts.\\n\"" + ] + }, + "execution_count": 62, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "summarize(\"https://edwarddonner.com\")" + ] + }, + { + "cell_type": "markdown", + "id": "9142555b-d017-46a4-a367-8701384e8b41", + "metadata": {}, + "source": [ + "### Output\n", + "\"Edward Donner's website introduces Ed, a coder and LLM enthusiast who enjoys DJing, electronic music, and Hacker News. He is the co-founder and CTO of Nebula.io, an AI company focused on talent discovery. He was also the founder and CEO of AI startup untapt, which was acquired in 2021. The site also has links to LLM resources from past workshops and posts.\\n\"\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e134e422-79e7-450b-a2fc-96f8c45de321", + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.0" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} From 7c8ad2b7b2732704a81aeb80af8702c15653bf3b Mon Sep 17 00:00:00 2001 From: nssuu Date: Fri, 7 Feb 2025 00:19:02 +0300 Subject: [PATCH 2/2] Complete web page summarizer using gemini 2.0 flash --- ...eek1_web_page_summarizer_with_gemini.ipynb | 57 +------------------ 1 file changed, 2 insertions(+), 55 deletions(-) diff --git a/week1/community-contributions/week1_web_page_summarizer_with_gemini.ipynb b/week1/community-contributions/week1_web_page_summarizer_with_gemini.ipynb index ebcf6ef..a4bc395 100644 --- a/week1/community-contributions/week1_web_page_summarizer_with_gemini.ipynb +++ b/week1/community-contributions/week1_web_page_summarizer_with_gemini.ipynb @@ -102,63 +102,10 @@ }, { "cell_type": "code", - "execution_count": 56, + "execution_count": null, "id": "4b2a7585-d6fe-4160-af94-e905eaa13109", "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Home - Edward Donner\n", - "Home\n", - "Outsmart\n", - "An arena that pits LLMs against each other in a battle of diplomacy and deviousness\n", - "About\n", - "Posts\n", - "Well, hi there.\n", - "I’m Ed. I like writing code and experimenting with LLMs, and hopefully you’re here because you do too. I also enjoy DJing (but I’m badly out of practice), amateur electronic music production (\n", - "very\n", - "amateur) and losing myself in\n", - "Hacker News\n", - ", nodding my head sagely to things I only half understand.\n", - "I’m the co-founder and CTO of\n", - "Nebula.io\n", - ". We’re applying AI to a field where it can make a massive, positive impact: helping people discover their potential and pursue their reason for being. Recruiters use our product today to source, understand, engage and manage talent. I’m previously the founder and CEO of AI startup untapt,\n", - "acquired in 2021\n", - ".\n", - "We work with groundbreaking, proprietary LLMs verticalized for talent, we’ve\n", - "patented\n", - "our matching model, and our award-winning platform has happy customers and tons of press coverage.\n", - "Connect\n", - "with me for more!\n", - "January 23, 2025\n", - "LLM Workshop – Hands-on with Agents – resources\n", - "December 21, 2024\n", - "Welcome, SuperDataScientists!\n", - "November 13, 2024\n", - "Mastering AI and LLM Engineering – Resources\n", - "October 16, 2024\n", - "From Software Engineer to AI Data Scientist – resources\n", - "Navigation\n", - "Home\n", - "Outsmart\n", - "An arena that pits LLMs against each other in a battle of diplomacy and deviousness\n", - "About\n", - "Posts\n", - "Get in touch\n", - "ed [at] edwarddonner [dot] com\n", - "www.edwarddonner.com\n", - "Follow me\n", - "LinkedIn\n", - "Twitter\n", - "Facebook\n", - "Subscribe to newsletter\n", - "Type your email…\n", - "Subscribe\n" - ] - } - ], + "outputs": [], "source": [ "ed = Website(\"https://edwarddonner.com\")\n", "print(ed.title)\n",