You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 

73 lines
3.1 KiB

import json
import os
from typing import Dict, List
from openai import AsyncOpenAI
from .summarizer_llm import BaseSummarizer
class OpenAISummarize(BaseSummarizer):
def __init__(self, model_name: str = "gpt-4o-mini"):
self.model = model_name
api_key = os.getenv("OPENAI_API_KEY")
self.openai = AsyncOpenAI(api_key=api_key)
async def generate(self, url, content: List[Dict], description,
site_type):
content_dict = {item['url']: item for item in content}
links = list(content_dict.keys())
yield f"Now I Am filtering links that i found on {url}\n"
new_links = await self.remove_unnecessary_link(url=url,
links=links,
description=description,
site_type=site_type)
yield "Links have been filtered. Advancing...\n\n"
new_links = new_links['links']
filtered_content = [content_dict[link_info['url']] for link_info in new_links if
link_info['url'] in content_dict]
yield "It's Almost Done\n"
prompt = self.get_boruchure_prompt(filtered_content)
response = await self.openai.chat.completions.create(model="gpt-4o-mini",
messages=prompt, stream=True)
async for response_chunk in response:
yield response_chunk.choices[0].delta.content
async def remove_unnecessary_link(self, url, links, description,
site_type):
prompt = self.prompts_for_removing_links(url=url,
description=description,
site_type=site_type,
links=links)
links = await self.openai.chat.completions.create(
messages=prompt,
model=self.model,
response_format={"type": "json_object"}
)
result = links.choices[0].message.content
return json.loads(result)
@staticmethod
def get_boruchure_prompt(link_content_list):
system_prompt = "You are an assistant that analyzes \
the contents of several relevant pages from a company website \
and creates a short brochure about the company for prospective\
customers, investors and recruits. Respond in markdown.\
Include details of company culture, customers and careers/jobs if you have the information."
user_prompt = f"Here are the contents of its landing page and other relevant pages; \
use this information to build a short brochure of the company in markdown.\n"
result = "links content are :\n\n"
for item in link_content_list:
link = item['url']
content = item['content']
result += f"url: {link},\t content: {content[:2000]}"
user_prompt += result
return [
{"role": "system", "content": system_prompt},
{"role": "user", "content": user_prompt}
]