From the uDemy course on LLM engineering.
https://www.udemy.com/course/llm-engineering-master-ai-and-large-language-models
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
18 lines
466 B
18 lines
466 B
from exceptions import InvalidCrawlType |
|
from .bs4crawler import BS4Crawler |
|
|
|
|
|
class CrawlerService: |
|
def __init__(self, url, crawl_type): |
|
self.crawler = self.crawl_builder(url, crawl_type) |
|
|
|
async def crawl(self): |
|
await self.crawler.crawl() |
|
return self.crawler.url_contents |
|
|
|
@staticmethod |
|
def crawl_builder(url, crawl_type): |
|
if crawl_type == "normal": |
|
return BS4Crawler(url) |
|
raise InvalidCrawlType() |
|
|
|
|