|
|
@ -184,6 +184,11 @@ |
|
|
|
"# A class to represent a Webpage\n", |
|
|
|
"# A class to represent a Webpage\n", |
|
|
|
"# If you're not familiar with Classes, check out the \"Intermediate Python\" notebook\n", |
|
|
|
"# If you're not familiar with Classes, check out the \"Intermediate Python\" notebook\n", |
|
|
|
"\n", |
|
|
|
"\n", |
|
|
|
|
|
|
|
"# Some websites need you to use proper headers when fetching them:\n", |
|
|
|
|
|
|
|
"headers = {\n", |
|
|
|
|
|
|
|
" \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36\"\n", |
|
|
|
|
|
|
|
"}\n", |
|
|
|
|
|
|
|
"\n", |
|
|
|
"class Website:\n", |
|
|
|
"class Website:\n", |
|
|
|
"\n", |
|
|
|
"\n", |
|
|
|
" def __init__(self, url):\n", |
|
|
|
" def __init__(self, url):\n", |
|
|
@ -191,7 +196,7 @@ |
|
|
|
" Create this Website object from the given url using the BeautifulSoup library\n", |
|
|
|
" Create this Website object from the given url using the BeautifulSoup library\n", |
|
|
|
" \"\"\"\n", |
|
|
|
" \"\"\"\n", |
|
|
|
" self.url = url\n", |
|
|
|
" self.url = url\n", |
|
|
|
" response = requests.get(url)\n", |
|
|
|
" response = requests.get(url, headers=headers)\n", |
|
|
|
" soup = BeautifulSoup(response.content, 'html.parser')\n", |
|
|
|
" soup = BeautifulSoup(response.content, 'html.parser')\n", |
|
|
|
" self.title = soup.title.string if soup.title else \"No title found\"\n", |
|
|
|
" self.title = soup.title.string if soup.title else \"No title found\"\n", |
|
|
|
" for irrelevant in soup.body([\"script\", \"style\", \"img\", \"input\"]):\n", |
|
|
|
" for irrelevant in soup.body([\"script\", \"style\", \"img\", \"input\"]):\n", |
|
|
|