From c16acf028746f0e6081bac2dbf4518ed602b1e1a Mon Sep 17 00:00:00 2001 From: Banasri Saha Date: Thu, 16 Jan 2025 15:57:08 +0000 Subject: [PATCH] added my contributions to community-contributions --- .../day1-selenium-for-javascript-sites.ipynb | 379 ++++++++++++++++-- 1 file changed, 352 insertions(+), 27 deletions(-) diff --git a/week1/community-contributions/day1-selenium-for-javascript-sites.ipynb b/week1/community-contributions/day1-selenium-for-javascript-sites.ipynb index fd3a3ba..d38533e 100644 --- a/week1/community-contributions/day1-selenium-for-javascript-sites.ipynb +++ b/week1/community-contributions/day1-selenium-for-javascript-sites.ipynb @@ -22,7 +22,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 3, "id": "4e2a9393-7767-488e-a8bf-27c12dca35bd", "metadata": {}, "outputs": [], @@ -62,7 +62,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 4, "id": "7b87cadb-d513-4303-baee-a37b6f938e4d", "metadata": {}, "outputs": [], @@ -76,7 +76,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 5, "id": "c5e793b2-6775-426a-a139-4848291d0463", "metadata": {}, "outputs": [], @@ -100,10 +100,63 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 6, "id": "2ef960cf-6dc2-4cda-afb3-b38be12f4c97", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Home - Edward Donner\n", + "Home\n", + "Outsmart\n", + "An arena that pits LLMs against each other in a battle of diplomacy and deviousness\n", + "About\n", + "Posts\n", + "Well, hi there.\n", + "I’m Ed. I like writing code and experimenting with LLMs, and hopefully you’re here because you do too. I also enjoy DJing (but I’m badly out of practice), amateur electronic music production (\n", + "very\n", + "amateur) and losing myself in\n", + "Hacker News\n", + ", nodding my head sagely to things I only half understand.\n", + "I’m the co-founder and CTO of\n", + "Nebula.io\n", + ". We’re applying AI to a field where it can make a massive, positive impact: helping people discover their potential and pursue their reason for being. Recruiters use our product today to source, understand, engage and manage talent. I’m previously the founder and CEO of AI startup untapt,\n", + "acquired in 2021\n", + ".\n", + "We work with groundbreaking, proprietary LLMs verticalized for talent, we’ve\n", + "patented\n", + "our matching model, and our award-winning platform has happy customers and tons of press coverage.\n", + "Connect\n", + "with me for more!\n", + "November 13, 2024\n", + "Mastering AI and LLM Engineering – Resources\n", + "October 16, 2024\n", + "From Software Engineer to AI Data Scientist – resources\n", + "August 6, 2024\n", + "Outsmart LLM Arena – a battle of diplomacy and deviousness\n", + "June 26, 2024\n", + "Choosing the Right LLM: Toolkit and Resources\n", + "Navigation\n", + "Home\n", + "Outsmart\n", + "An arena that pits LLMs against each other in a battle of diplomacy and deviousness\n", + "About\n", + "Posts\n", + "Get in touch\n", + "ed [at] edwarddonner [dot] com\n", + "www.edwarddonner.com\n", + "Follow me\n", + "LinkedIn\n", + "Twitter\n", + "Facebook\n", + "Subscribe to newsletter\n", + "Type your email…\n", + "Subscribe\n" + ] + } + ], "source": [ "# Let's try one out\n", "\n", @@ -132,7 +185,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 7, "id": "abdb8417-c5dc-44bc-9bee-2e059d162699", "metadata": {}, "outputs": [], @@ -144,7 +197,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 8, "id": "f0275b1b-7cfe-4f9d-abfa-7650d378da0c", "metadata": {}, "outputs": [], @@ -177,7 +230,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 9, "id": "0134dfa4-8299-48b5-b444-f2a8c3403c88", "metadata": {}, "outputs": [], @@ -199,7 +252,7 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 10, "id": "905b9919-aba7-45b5-ae65-81b3d1d78e34", "metadata": {}, "outputs": [], @@ -215,17 +268,28 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 11, "id": "05e38d41-dfa4-4b20-9c96-c46ea75d9fb5", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/plain": [ + "'# Summary of Edward Donner\\'s Website\\n\\nThe website is a personal platform for Ed Donner, who is passionate about coding and experimenting with large language models (LLMs). He is the co-founder and CTO of Nebula.io, which focuses on applying AI to enhance talent discovery and management. Prior to this, he founded an AI startup called untapt that was acquired in 2021. \\n\\nEd shares his interests in DJing, amateur electronic music production, and engaging with technical discussions on Hacker News. His work has garnered notable achievements, including patented matching models and positive press coverage.\\n\\n## Recent Posts\\n1. **Mastering AI and LLM Engineering – Resources** (November 13, 2024)\\n2. **From Software Engineer to AI Data Scientist – Resources** (October 16, 2024)\\n3. **Outsmart LLM Arena – A Battle of Diplomacy and Deviousness** (August 6, 2024)\\n4. **Choosing the Right LLM: Toolkit and Resources** (June 26, 2024)\\n\\nThe website also features a section titled \"Outsmart,\" which is an arena designed for LLMs to compete against each other in complex scenarios involving diplomacy and strategy.'" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], "source": [ "summarize(\"https://edwarddonner.com\")" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "id": "3d926d59-450e-4609-92ba-2d6f244f1342", "metadata": {}, "outputs": [], @@ -237,30 +301,110 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "id": "3018853a-445f-41ff-9560-d925d1774b2f", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/markdown": [ + "# Summary of Edward Donner's Website\n", + "\n", + "The website serves as a personal platform for Edward Donner, a software engineer and AI enthusiast. He is the co-founder and CTO of Nebula.io, where he focuses on leveraging AI to improve talent discovery and engagement. The site reflects his interests in coding, LLMs (Large Language Models), electronic music, and tech discussions on platforms like Hacker News.\n", + "\n", + "## Key Announcements and Posts\n", + "- **November 13, 2024**: Post on mastering AI and LLM engineering with resources.\n", + "- **October 16, 2024**: Insights on transitioning from software engineer to AI data scientist.\n", + "- **August 6, 2024**: Introduction to the \"Outsmart LLM Arena,\" focusing on competitive LLM strategies.\n", + "- **June 26, 2024**: Discussion on how to choose the right LLM, with accompanying tools and resources.\n", + "\n", + "Overall, the site serves as a blend of personal branding, technology insights, and professional development advice in the AI field." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "display_summary(\"https://edwarddonner.com\")" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 15, "id": "45d83403-a24c-44b5-84ac-961449b4008f", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/markdown": [ + "# CNN Website Summary\n", + "\n", + "CNN is a major news outlet providing breaking news, latest stories, and video content covering various topics including:\n", + "\n", + "- **US and International News**: Updates on significant events both domestically and worldwide.\n", + "- **Politics**: Current political developments, particularly focusing on elections and legislation.\n", + "- **Health**: Information related to public health, wellness, and medical advancements.\n", + "- **Entertainment**: News on movies, celebrities, and cultural events.\n", + "- **Sports**: Coverage of various sports including football, basketball, and more.\n", + "- **Science and Climate**: Insights into scientific discoveries and environmental issues.\n", + "\n", + "### Recent News Highlights\n", + "\n", + "1. **Christmas Market Attack in Germany**: A driver plowed into a crowd at a Christmas market in Magdeburg, resulting in at least two fatalities and multiple injuries. The driver has been arrested and is reportedly from Saudi Arabia.\n", + "\n", + "2. **Potential US Government Shutdown**: US House Speaker Johnson announced that a government shutdown will be averted, though specific details of the plan remain unclear.\n", + "\n", + "3. **Ukraine-Russia Conflict**: A recent missile strike by Russia in Kyiv resulted in casualties and damage to embassies, as described by Ukrainian officials.\n", + "\n", + "4. **Cultural and Scientific Discoveries**: A notable discovery in Germany could alter the understanding of Christian history in Europe, and 27 new species have been discovered in Peru, showcasing the ongoing exploration of biodiversity.\n", + "\n", + "These summaries provide insight into the type of content available on CNN, highlighting its commitment to delivering timely and relevant news across a broad spectrum of topics." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "display_summary(\"https://cnn.com\")" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 16, "id": "75e9fd40-b354-4341-991e-863ef2e59db7", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/markdown": [ + "# Summary of Anthropic Website\n", + "\n", + "Anthropic is an AI safety and research company based in San Francisco, focusing on developing reliable and beneficial AI systems. The company has introduced its latest AI model, **Claude 3.5 Sonnet**, which is described as their most intelligent AI model to date. They also announced **Claude 3.5 Haiku** as part of their ongoing model updates.\n", + "\n", + "## Recent Announcements\n", + "- **October 22, 2024**: Introduction of new functionalities including computer use and the release of Claude 3.5 Sonnet and Claude 3.5 Haiku.\n", + "- **September 4, 2024**: Announcement related to Claude for Enterprise.\n", + "- **March 8, 2023**: Release of \"Core Views on AI Safety,\" discussing the principles behind AI safety and alignment.\n", + "- **December 15, 2022**: Publication of research on \"Constitutional AI,\" focusing on generating harmlessness from AI feedback.\n", + "\n", + "Overall, the website offers insights into Anthropic’s work in AI research, product development, and its commitment to safety in AI technologies. Users can explore options to build applications using their API and are encouraged to check open career opportunities within the company." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "display_summary(\"https://anthropic.com\")" ] @@ -277,17 +421,67 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 17, "id": "52ae98bb", "metadata": {}, - "outputs": [], + "outputs": [ + { + "data": { + "text/markdown": [ + "# Website Summary\n", + "\n", + "The website currently displays a message indicating that JavaScript and cookies need to be enabled for proper functionality. As there is no additional content available without these features activated, a detailed summary of the site's offerings, including news or announcements, cannot be provided at this time." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "display_summary(\"https://openai.com\")" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, + "id": "9f35f98b-38ba-450a-a779-146de04dbae9", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Requirement already satisfied: selenium in c:\\users\\sahab\\anaconda3\\envs\\llms\\lib\\site-packages (4.27.1)\n", + "Requirement already satisfied: urllib3<3,>=1.26 in c:\\users\\sahab\\anaconda3\\envs\\llms\\lib\\site-packages (from urllib3[socks]<3,>=1.26->selenium) (2.2.3)\n", + "Requirement already satisfied: trio~=0.17 in c:\\users\\sahab\\anaconda3\\envs\\llms\\lib\\site-packages (from selenium) (0.27.0)\n", + "Requirement already satisfied: trio-websocket~=0.9 in c:\\users\\sahab\\anaconda3\\envs\\llms\\lib\\site-packages (from selenium) (0.11.1)\n", + "Requirement already satisfied: certifi>=2021.10.8 in c:\\users\\sahab\\anaconda3\\envs\\llms\\lib\\site-packages (from selenium) (2024.8.30)\n", + "Requirement already satisfied: typing_extensions~=4.9 in c:\\users\\sahab\\anaconda3\\envs\\llms\\lib\\site-packages (from selenium) (4.12.2)\n", + "Requirement already satisfied: websocket-client~=1.8 in c:\\users\\sahab\\anaconda3\\envs\\llms\\lib\\site-packages (from selenium) (1.8.0)\n", + "Requirement already satisfied: attrs>=23.2.0 in c:\\users\\sahab\\anaconda3\\envs\\llms\\lib\\site-packages (from trio~=0.17->selenium) (24.3.0)\n", + "Requirement already satisfied: sortedcontainers in c:\\users\\sahab\\anaconda3\\envs\\llms\\lib\\site-packages (from trio~=0.17->selenium) (2.4.0)\n", + "Requirement already satisfied: idna in c:\\users\\sahab\\anaconda3\\envs\\llms\\lib\\site-packages (from trio~=0.17->selenium) (3.10)\n", + "Requirement already satisfied: outcome in c:\\users\\sahab\\anaconda3\\envs\\llms\\lib\\site-packages (from trio~=0.17->selenium) (1.3.0.post0)\n", + "Requirement already satisfied: sniffio>=1.3.0 in c:\\users\\sahab\\anaconda3\\envs\\llms\\lib\\site-packages (from trio~=0.17->selenium) (1.3.1)\n", + "Requirement already satisfied: cffi>=1.14 in c:\\users\\sahab\\anaconda3\\envs\\llms\\lib\\site-packages (from trio~=0.17->selenium) (1.17.1)\n", + "Requirement already satisfied: wsproto>=0.14 in c:\\users\\sahab\\anaconda3\\envs\\llms\\lib\\site-packages (from trio-websocket~=0.9->selenium) (1.2.0)\n", + "Requirement already satisfied: pysocks!=1.5.7,<2.0,>=1.5.6 in c:\\users\\sahab\\anaconda3\\envs\\llms\\lib\\site-packages (from urllib3[socks]<3,>=1.26->selenium) (1.7.1)\n", + "Requirement already satisfied: pycparser in c:\\users\\sahab\\anaconda3\\envs\\llms\\lib\\site-packages (from cffi>=1.14->trio~=0.17->selenium) (2.22)\n", + "Requirement already satisfied: h11<1,>=0.9.0 in c:\\users\\sahab\\anaconda3\\envs\\llms\\lib\\site-packages (from wsproto>=0.14->trio-websocket~=0.9->selenium) (0.14.0)\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "%pip install -U selenium" + ] + }, + { + "cell_type": "code", + "execution_count": 17, "id": "5d57e958", "metadata": {}, "outputs": [], @@ -299,7 +493,7 @@ "from selenium.webdriver.common.by import By\n", "from selenium.webdriver.chrome.options import Options\n", "\n", - "PATH_TO_CHROME_DRIVER = '..\\\\path\\\\to\\\\chromedriver.exe'\n", + "#PATH_TO_CHROME_DRIVER = 'C:\\\\Users\\\\sahab\\\\Downloads\\\\chromedriver-win64\\\\chromedriver-win64\\\\chromrdriver.exe'\n", "\n", "class Website:\n", " url: str\n", @@ -314,7 +508,8 @@ " options.add_argument(\"--no-sandbox\")\n", " options.add_argument(\"--disable-dev-shm-usage\")\n", "\n", - " service = Service(PATH_TO_CHROME_DRIVER)\n", + " #service = Service(PATH_TO_CHROME_DRIVER)\n", + " service = Service(\"C:/Users/sahab//Downloads/chromedriver-win64/chromedriver-win64/chromedriver.exe\")\n", " driver = webdriver.Chrome(service=service, options=options)\n", " driver.get(url)\n", "\n", @@ -329,6 +524,61 @@ " self.text = soup.get_text(separator=\"\\n\", strip=True)" ] }, + { + "cell_type": "code", + "execution_count": 20, + "id": "eceeb890-eb7f-487b-98d1-cf2c31da8bac", + "metadata": {}, + "outputs": [], + "source": [ + "#Parse webpages which is designed using JavaScript heavely\n", + "# download the chorme driver from here as per your version of chrome - https://developer.chrome.com/docs/chromedriver/downloads\n", + "from selenium import webdriver\n", + "from selenium.webdriver.chrome.service import Service\n", + "from selenium.webdriver.common.by import By\n", + "from selenium.webdriver.chrome.options import Options\n", + "\n", + "#PATH_TO_CHROME_DRIVER = 'C:\\\\Users\\\\sahab\\\\Downloads\\\\chromedriver-win64\\\\chromedriver-win64\\\\chromrdriver.exe'\n", + "\n", + "class Website:\n", + " url: str\n", + " title: str\n", + " text: str\n", + "\n", + " def __init__(self, url):\n", + " self.url = url\n", + "\n", + " options = Options()\n", + "\n", + " options.add_argument(\"--no-sandbox\")\n", + " options.add_argument(\"--disable-dev-shm-usage\")\n", + " \n", + " # Get chromedriver path dynamically\n", + " chromedriver_path = os.getenv(\"CHROMEDRIVER_PATH\", \"C:/Users/sahab//Downloads/chromedriver-win64/chromedriver-win64/chromedriver.exe\")\n", + " service = Service(chromedriver_path)\n", + "\n", + " try:\n", + " # Launch the browser and navigate to the URL\n", + " with webdriver.Chrome(service=service, options=options) as driver:\n", + " driver.get(url)\n", + "\n", + " # Wait for manual CAPTCHA verification\n", + " input(\"Please complete the verification in the browser and press Enter to continue...\")\n", + "\n", + " # Get the page source\n", + " page_source = driver.page_source\n", + "\n", + " except Exception as e:\n", + " print(f\"Error occurred while processing the webpage: {e}\")\n", + " return\n", + "\n", + " soup = BeautifulSoup(page_source, 'html.parser')\n", + " self.title = soup.title.string if soup.title else \"No title found\"\n", + " for irrelevant in soup([\"script\", \"style\", \"img\", \"input\"]):\n", + " irrelevant.decompose()\n", + " self.text = soup.get_text(separator=\"\\n\", strip=True)" + ] + }, { "cell_type": "code", "execution_count": null, @@ -341,23 +591,98 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 22, "id": "f2eb9599", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdin", + "output_type": "stream", + "text": [ + "Please complete the verification in the browser and press Enter to continue... \n" + ] + }, + { + "data": { + "text/markdown": [ + "# Summary of Edward Donner's Website\n", + "\n", + "The website showcases Edward Donner, a co-founder and CTO of Nebula.io, with a focus on AI and LLM (Large Language Model) technologies. Ed shares his passion for coding, LLMs, and amateur music production. He previously led an AI startup, untapt, before its acquisition in 2021.\n", + "\n", + "## Key Sections\n", + "\n", + "- **Outsmart**: Describes an arena where LLMs compete in a \"battle of diplomacy and deviousness.\"\n", + "- **Recent Posts**:\n", + " - **November 13, 2024**: *Mastering AI and LLM Engineering – Resources*\n", + " - **October 16, 2024**: *From Software Engineer to AI Data Scientist – Resources*\n", + " - **August 6, 2024**: *Outsmart LLM Arena – A Battle of Diplomacy and Deviousness*\n", + " - **June 26, 2024**: *Choosing the Right LLM: Toolkit and Resources*\n", + "\n", + "Overall, the website highlights Ed's work and interests in AI, providing resources and insights related to LLMs, technology, and personal development in the field." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "display_summary(\"https://edwarddonner.com\")" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 23, "id": "e7ba56c8", "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdin", + "output_type": "stream", + "text": [ + "Please complete the verification in the browser and press Enter to continue... \n" + ] + }, + { + "data": { + "text/markdown": [ + "# CNN Website Summary\n", + "\n", + "CNN is a news organization providing breaking news, articles, and videos across various topics including US and world news, politics, business, health, entertainment, and more. The website features live updates on significant global events and major incidents, such as ongoing conflicts (e.g., the Ukraine-Russia War and the Israel-Hamas War).\n", + "\n", + "### Key Recent News Highlights:\n", + "- **German Christmas Market Attack**: A driver crashed into a crowd in Magdeburg, resulting in dozens of injuries and at least two fatalities. The suspect has been arrested and is reported to be from Saudi Arabia.\n", + "- **US Government Shutdown**: The House passed a spending package to prevent a government shutdown just before the deadline, pending Senate approval.\n", + "- **Conflict in Kyiv**: Ukraine reports casualties and damage to embassies from a recent Russian missile barrage.\n", + "\n", + "Additional notable stories include:\n", + "- Recent archaeological discoveries that could impact European Christian history.\n", + "- Elon Musk's political endorsements and comments on international affairs.\n", + "- Ongoing labor actions among Amazon drivers amidst the holiday season.\n", + "\n", + "The site emphasizes its commitment to delivering timely and comprehensive news coverage across a range of sectors, allowing users to stay informed on both domestic and international issues." + ], + "text/plain": [ + "" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], "source": [ "display_summary(\"https://cnn.com\")" ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "051f2975-f668-4504-8aa8-183ab3b59053", + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": {