From the uDemy course on LLM engineering.
https://www.udemy.com/course/llm-engineering-master-ai-and-large-language-models
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
84 lines
2.4 KiB
84 lines
2.4 KiB
from pydantic import BaseModel |
|
from typing import List |
|
from bs4 import BeautifulSoup |
|
import re |
|
import feedparser |
|
from tqdm import tqdm |
|
import requests |
|
import time |
|
|
|
feeds = [ |
|
"https://www.dealnews.com/c142/Electronics/?rss=1", |
|
"https://www.dealnews.com/c39/Computers/?rss=1", |
|
"https://www.dealnews.com/c238/Automotive/?rss=1", |
|
"https://www.dealnews.com/f1912/Smart-Home/?rss=1", |
|
"https://www.dealnews.com/c196/Home-Garden/?rss=1", |
|
] |
|
|
|
def extract(html_snippet): |
|
soup = BeautifulSoup(html_snippet, 'html.parser') |
|
snippet_div = soup.find('div', class_='snippet summary') |
|
|
|
if snippet_div: |
|
description = snippet_div.get_text(strip=True) |
|
description = BeautifulSoup(description, 'html.parser').get_text() |
|
description = re.sub('<[^<]+?>', '', description) |
|
result = description.strip() |
|
else: |
|
result = html_snippet |
|
return result.replace('\n', ' ') |
|
|
|
class Deal: |
|
category: str |
|
title: str |
|
summary: str |
|
url: str |
|
item_id: int |
|
details: str |
|
features: str |
|
|
|
def __init__(self, entry, id): |
|
self.title = entry['title'] |
|
self.summary = extract(entry['summary']) |
|
self.url = entry['links'][0]['href'] |
|
self.item_id = id |
|
stuff = requests.get(self.url).content |
|
soup = BeautifulSoup(stuff, 'html.parser') |
|
content = soup.find('div', class_='content-section').get_text() |
|
content = content.replace('\nmore', '').replace('\n', ' ') |
|
if "Features" in content: |
|
self.details, self.features = content.split("Features") |
|
else: |
|
self.details = content |
|
self.features = "" |
|
|
|
def __repr__(self): |
|
return f"<{self.title}>" |
|
|
|
def describe(self): |
|
return f"Title: {self.title}\nDetails: {self.details.strip()}\nFeatures: {self.features.strip()}\nURL: {self.url}" |
|
|
|
@classmethod |
|
def fetch(cls): |
|
deals = [] |
|
item_id = 1001 |
|
for feed_url in tqdm(feeds): |
|
feed = feedparser.parse(feed_url) |
|
for entry in feed.entries[:10]: |
|
deals.append(cls(entry, item_id)) |
|
item_id += 1 |
|
time.sleep(1) |
|
return deals |
|
|
|
class QualityDeal(BaseModel): |
|
product_description: str |
|
price: float |
|
url: str |
|
|
|
class QualityDealSelection(BaseModel): |
|
quality_deals: List[QualityDeal] |
|
|
|
class Opportunity(BaseModel): |
|
quality_deal: QualityDeal |
|
estimate: float |
|
discount: float |