From the uDemy course on LLM engineering.
https://www.udemy.com/course/llm-engineering-master-ai-and-large-language-models
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
17 lines
387 B
17 lines
387 B
from abc import ABC, abstractmethod |
|
|
|
|
|
class CrawlerBase(ABC): |
|
|
|
@abstractmethod |
|
async def crawl(self): |
|
pass |
|
|
|
@staticmethod |
|
async def _fetch(session, link): |
|
try: |
|
async with session.get(link) as response: |
|
response.raise_for_status() |
|
return await response.text() |
|
except Exception as e: |
|
return e
|
|
|