From the uDemy course on LLM engineering.
https://www.udemy.com/course/llm-engineering-master-ai-and-large-language-models
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
56 lines
2.3 KiB
56 lines
2.3 KiB
from abc import abstractmethod, ABC |
|
|
|
|
|
class BaseSummarizer(ABC): |
|
|
|
@abstractmethod |
|
async def generate(self, *args, **kwargs): |
|
pass |
|
|
|
@abstractmethod |
|
async def remove_unnecessary_link(self, *args, **kwargs): |
|
pass |
|
|
|
def prompts_for_removing_links(self, url, links, description=None, site_type=None): |
|
link_system_prompt = ( |
|
"You are provided with a list of links found on a webpage. " |
|
"Your task is to filter out irrelevant links and retain those that are most\ |
|
relevant for creating a brochure. " |
|
"Consider links that provide valuable information about the site's content,\ |
|
such as main articles, key information pages, or other significant sections.\n" |
|
"Exclude links that are not useful for a brochure, such as Terms of Service,\ |
|
Privacy policies, and email links.\n" |
|
"You should respond in JSON format as shown in the example below:\n" |
|
"{\n" |
|
' "links": [\n' |
|
' {"type": "relevant page type", "url": "https://full.url/goes/here"},\n' |
|
' {"type": "another relevant page type", "url": "https://another.full.url"}\n' |
|
" ]\n" |
|
"}" |
|
) |
|
|
|
user_prompt = self.get_links_user_prompt(url, links, description, site_type) |
|
|
|
return [ |
|
{"role": "system", "content": link_system_prompt}, |
|
{"role": "user", "content": user_prompt}, |
|
] |
|
|
|
@staticmethod |
|
def get_links_user_prompt(url, links, description=None, site_type=None): |
|
user_prompt = f"Here is the list of links found on the {url}:\n" |
|
|
|
if site_type or description: |
|
user_prompt += "Additional context:\n" |
|
if site_type: |
|
user_prompt += f"- Site type: {site_type}\n" |
|
if description: |
|
user_prompt += f"- User description: {description}\n" |
|
|
|
user_prompt += ( |
|
"Please evaluate the following links and select those that are relevant for inclusion in a brochure. " |
|
"Exclude links related to Terms of Service, Privacy policies, and email addresses.\n" |
|
"Links (some might be relative links):\n" |
|
) |
|
user_prompt += "\n".join(links) |
|
return user_prompt
|
|
|