From 57d2788241709379f58e8a595436f7866a6e996b Mon Sep 17 00:00:00 2001 From: Petri Alapiessa Date: Fri, 17 Jan 2025 09:46:28 +0200 Subject: [PATCH] working version, class not yet done --- .../day1-webscraping-playwright.ipynb | 503 +++--------------- 1 file changed, 64 insertions(+), 439 deletions(-) diff --git a/week1/community-contributions/day1-webscraping-playwright.ipynb b/week1/community-contributions/day1-webscraping-playwright.ipynb index 8779c48..e9d8902 100644 --- a/week1/community-contributions/day1-webscraping-playwright.ipynb +++ b/week1/community-contributions/day1-webscraping-playwright.ipynb @@ -496,7 +496,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 37, "id": "682eff74-55c4-4d4b-b267-703edbc293c7", "metadata": {}, "outputs": [ @@ -505,435 +505,53 @@ "output_type": "stream", "text": [ "Title: Home - Edward Donner\n", - "Content: \n", - "\t\n", - "\t\n", - "\n", - "\t\n", - "\t\n", - "\t\n", - "\t\n", - "\t\n", - "\t\n", - "\t\n", - "\t\n", - "\t\n", - "\t\n", - "\t\n", - "\t\n", - "\t\n", - "\t\n", - "\t\n", - "\t\n", - "\n", - "\n", - "Home - Edward Donner\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\n", - "\t\n", - "\t\t\n", - "\n", - "\n", - "\n", - "\n", - "\t\t\n", - "\t\t\n", - "\t\t\n", - "\t\t\n", - "\t\t\n", - "\n", - "\n", - "\n", - "Skip to content
\n", - "\n", - "\n", - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "\n", - "\n", - "
\n", - "\n", - "
\n", - "\n", - "\n", - "

Well, hi there.

\n", - "\n", - "\n", - "\n", - "

I’m Ed. I like writing code and experimenting with LLMs, and hopefully you’re here because you do too. I also enjoy DJing (but I’m badly out of practice), amateur electronic music production (very amateur) and losing myself in Hacker News, nodding my head sagely to things I only half understand.

\n", - "\n", - "\n", - "\n", - "

I’m the co-founder and CTO of Nebula.io. We’re applying AI to a field where it can make a massive, positive impact: helping people discover their potential and pursue their reason for being. Recruiters use our product today to source, understand, engage and manage talent. I’m previously the founder and CEO of AI startup untapt, acquired in 2021.

\n", - "\n", - "\n", - "\n", - "

We work with groundbreaking, proprietary LLMs verticalized for talent, we’ve patented our matching model, and our award-winning platform has happy customers and tons of press coverage. Connect with me for more!

\n", - "\n", - "\n", - "\n", - "
\n", - "
\n", - "
\n", - "
\n", - "
\n", - "\n", - "\n", - "\n", - "
\n", - "
\n", - "\n", - "\n", - "\n", - "\n", - "
\n", - "

Get in touch

\n", - "\n", - "\n", - "\n", - "
\n", - "

ed [at] edwarddonner [dot] com

\n", - "\n", - "\n", - "\n", - "

www.edwarddonner.com

\n", - "
\n", - "
\n", - "\n", - "\n", - "\n", - "
\n", - "

Follow me

\n", - "\n", - "\n", - "\n", - "\n", - "
\n", - "\n", - "\n", - "\n", - "
\n", - "

Subscribe to newsletter

\n", - "\n", - "\n", - "\t
\n", - "\t\t
\n", - "\t\t\t\t\t\t\t
\n", - "\t\t\t\t\t
\n", - "\t\t\t\t\t\t\t\t\t\t\t\t

\n", - "\t\t\t\t\t\t\t\n", - "\t\t\t\t\t\t\t\t\t\t\t\t\t

\n", - "\t\t\t\t\t\t\t\t\t\t\t\t

\n", - "\t\t\t\t\t\t\t\n", - "\t\t\t\t\t\t\t\n", - "\t\t\t\t\t\t\t\n", - "\t\t\t\t\t\t\t\n", - "\t\t\t\t\t\t\t\n", - "\t\t\t\t\t\t\t\n", - "\t\t\t\t\t\t\t\n", - "\t\t\t\t\t\t\t\t\t\t\t\t\t\t\n", - "\t\t\t\t\t\t

\n", - "\t\t\t\t\t
\n", - "\t\t\t\t
\n", - "\t\t\t\t\t\t\t\t
\n", - "\t
\n", - "\t
\n", - "
\n", - "
\n", - "
\n", - "\n", - "\n", - "\n", - "\n", - "\"\"\n", - "\n", - "\n", - "\n", - "\n" + "Content: Home - Edward Donner\n", + "Skip to content\n", + "Home\n", + "Outsmart\n", + "An arena that pits LLMs against each other in a battle of diplomacy and deviousness\n", + "About\n", + "Posts\n", + "Well, hi there.\n", + "I’m Ed. I like writing code and experimenting with LLMs, and hopefully you’re here because you do too. I also enjoy DJing (but I’m badly out of practice), amateur electronic music production (\n", + "very\n", + "amateur) and losing myself in\n", + "Hacker News\n", + ", nodding my head sagely to things I only half understand.\n", + "I’m the co-founder and CTO of\n", + "Nebula.io\n", + ". We’re applying AI to a field where it can make a massive, positive impact: helping people discover their potential and pursue their reason for being. Recruiters use our product today to source, understand, engage and manage talent. I’m previously the founder and CEO of AI startup untapt,\n", + "acquired in 2021\n", + ".\n", + "We work with groundbreaking, proprietary LLMs verticalized for talent, we’ve\n", + "patented\n", + "our matching model, and our award-winning platform has happy customers and tons of press coverage.\n", + "Connect\n", + "with me for more!\n", + "December 21, 2024\n", + "Welcome, SuperDataScientists!\n", + "November 13, 2024\n", + "Mastering AI and LLM Engineering – Resources\n", + "October 16, 2024\n", + "From Software Engineer to AI Data Scientist – resources\n", + "August 6, 2024\n", + "Outsmart LLM Arena – a battle of diplomacy and deviousness\n", + "Navigation\n", + "Home\n", + "Outsmart\n", + "An arena that pits LLMs against each other in a battle of diplomacy and deviousness\n", + "About\n", + "Posts\n", + "Get in touch\n", + "ed [at] edwarddonner [dot] com\n", + "www.edwarddonner.com\n", + "Follow me\n", + "LinkedIn\n", + "Twitter\n", + "Facebook\n", + "Subscribe to newsletter\n", + "Type your email…\n", + "Subscribe\n" ] } ], @@ -941,6 +559,8 @@ "import asyncio\n", "from playwright.async_api import async_playwright\n", "import nest_asyncio\n", + "from bs4 import BeautifulSoup\n", + "import time\n", "\n", "nest_asyncio.apply()\n", "\n", @@ -953,30 +573,35 @@ " self.title = title\n", " self.text = text\n", " \n", - "async def run(playwright):\n", - " browser = await playwright.chromium.launch(headless=True)\n", + "async def run(playwright, url):\n", + " browser = await playwright.chromium.launch(headless=False)\n", " page = await browser.new_page()\n", - " await page.goto(\"https://edwarddonner.com\")\n", + " await page.goto(url)\n", + " await page.wait_for_load_state('load')\n", " \n", " # Extract data from the page\n", " title = await page.title()\n", - " \n", " text = await page.content()\n", " await browser.close()\n", "\n", + " soup = BeautifulSoup(text, 'html.parser')\n", + " for irrelevant in soup([\"script\", \"style\", \"img\", \"input\"]):\n", + " irrelevant.decompose()\n", + " text = soup.get_text(separator=\"\\n\", strip=True)\n", + "\n", " return Website(title, text)\n", "\n", " \n", "async def main():\n", " async with async_playwright() as playwright:\n", - " web = await run(playwright)\n", - " print(f\"Title: {web.title}\")\n", - " print(f\"Content: {web.text}\")\n", + " web = await run(playwright, \"https://edwarddonner.com/\")\n", + " return web \n", "\n", "loop = asyncio.get_event_loop()\n", - "loop.run_until_complete(main())\n", + "web = loop.run_until_complete(main())\n", "\n", - " \n", + "print(f\"Title: {web.title}\")\n", + "print(f\"Content: {web.text}\")\n", "\n" ] },