10 changed files with 343 additions and 0 deletions
@ -0,0 +1,33 @@ |
|||||||
|
|
||||||
|
# Python |
||||||
|
__pycache__/ |
||||||
|
*.py[cod] |
||||||
|
*.pyo |
||||||
|
*.pyd |
||||||
|
.Python |
||||||
|
env/ |
||||||
|
venv/ |
||||||
|
*.env |
||||||
|
*.ini |
||||||
|
*.log |
||||||
|
|
||||||
|
# VSCode |
||||||
|
.vscode/ |
||||||
|
|
||||||
|
# IDE files |
||||||
|
.idea/ |
||||||
|
|
||||||
|
# System files |
||||||
|
.DS_Store |
||||||
|
Thumbs.db |
||||||
|
|
||||||
|
# Environment variables |
||||||
|
.env |
||||||
|
|
||||||
|
# Jupyter notebook checkpoints |
||||||
|
.ipynb_checkpoints |
||||||
|
|
||||||
|
# Dependencies |
||||||
|
*.egg-info/ |
||||||
|
dist/ |
||||||
|
build/ |
@ -0,0 +1,143 @@ |
|||||||
|
# AI Web Page Summarizer |
||||||
|
|
||||||
|
This project is a simple AI-powered web page summarizer that leverages OpenAI's GPT models and local inference with Ollama to generate concise summaries of given text. The goal is to create a "Reader's Digest of the Internet" by summarizing web content efficiently. |
||||||
|
|
||||||
|
## Features |
||||||
|
|
||||||
|
- Summarize text using OpenAI's GPT models or local Ollama models. |
||||||
|
- Flexible summarization engine selection (OpenAI API, Ollama API, or Ollama library). |
||||||
|
- Simple and modular code structure. |
||||||
|
- Error handling for better reliability. |
||||||
|
|
||||||
|
## Project Structure |
||||||
|
|
||||||
|
``` |
||||||
|
ai-summarizer/ |
||||||
|
│-- summarizer/ |
||||||
|
│ │-- __init__.py |
||||||
|
│ │-- fetcher.py # Web content fetching logic |
||||||
|
│ │-- summarizer.py # Main summarization logic |
||||||
|
│-- utils/ |
||||||
|
│ │-- __init__.py |
||||||
|
│ │-- logger.py # Logging configuration |
||||||
|
│-- main.py # Entry point of the app |
||||||
|
│-- .env # Environment variables |
||||||
|
│-- requirements.txt # Python dependencies |
||||||
|
│-- README.md # Project documentation |
||||||
|
``` |
||||||
|
|
||||||
|
## Prerequisites |
||||||
|
|
||||||
|
- Python 3.8 or higher |
||||||
|
- OpenAI API Key (You can obtain it from [OpenAI](https://platform.openai.com/signup)) |
||||||
|
- Ollama installed locally ([Installation Guide](https://ollama.ai)) |
||||||
|
- `conda` for managing environments (optional) |
||||||
|
|
||||||
|
## Installation |
||||||
|
|
||||||
|
1. **Clone the repository:** |
||||||
|
|
||||||
|
```bash |
||||||
|
git clone https://github.com/your-username/ai-summarizer.git |
||||||
|
cd ai-summarizer |
||||||
|
``` |
||||||
|
|
||||||
|
2. **Create a virtual environment (optional but recommended):** |
||||||
|
|
||||||
|
```bash |
||||||
|
conda create --name summarizer-env python=3.9 |
||||||
|
conda activate summarizer-env |
||||||
|
``` |
||||||
|
|
||||||
|
3. **Install dependencies:** |
||||||
|
|
||||||
|
```bash |
||||||
|
pip install -r requirements.txt |
||||||
|
``` |
||||||
|
|
||||||
|
4. **Set up environment variables:** |
||||||
|
|
||||||
|
Create a `.env` file in the project root and add your OpenAI API key (if using OpenAI): |
||||||
|
|
||||||
|
```env |
||||||
|
OPENAI_API_KEY=your-api-key-here |
||||||
|
``` |
||||||
|
|
||||||
|
## Usage |
||||||
|
|
||||||
|
1. **Run the summarizer:** |
||||||
|
|
||||||
|
```bash |
||||||
|
python main.py |
||||||
|
``` |
||||||
|
|
||||||
|
2. **Sample Output:** |
||||||
|
|
||||||
|
```shell |
||||||
|
Enter a URL to summarize: https://example.com |
||||||
|
Summary of the page: |
||||||
|
AI refers to machines demonstrating intelligence similar to humans and animals. |
||||||
|
``` |
||||||
|
|
||||||
|
3. **Engine Selection:** |
||||||
|
|
||||||
|
The summarizer supports multiple engines. Modify `main.py` to select your preferred model: |
||||||
|
|
||||||
|
```python |
||||||
|
summary = summarize_text(content, 'gpt-4o-mini', engine="openai") |
||||||
|
summary = summarize_text(content, 'deepseek-r1:1.5B', engine="ollama-api") |
||||||
|
summary = summarize_text(content, 'deepseek-r1:1.5B', engine="ollama-lib") |
||||||
|
``` |
||||||
|
|
||||||
|
## Configuration |
||||||
|
|
||||||
|
You can modify the model, max tokens, and temperature in `summarizer/summarizer.py`: |
||||||
|
|
||||||
|
```python |
||||||
|
response = client.chat.completions.create( |
||||||
|
model="gpt-4o-mini", |
||||||
|
messages=[...], |
||||||
|
max_tokens=300, |
||||||
|
temperature=0.7 |
||||||
|
) |
||||||
|
``` |
||||||
|
|
||||||
|
## Error Handling |
||||||
|
|
||||||
|
If any issues occur, the script will print an error message, for example: |
||||||
|
|
||||||
|
``` |
||||||
|
Error during summarization: Invalid API key or Ollama not running. |
||||||
|
``` |
||||||
|
|
||||||
|
## Dependencies |
||||||
|
|
||||||
|
The required dependencies are listed in `requirements.txt`: |
||||||
|
|
||||||
|
``` |
||||||
|
openai |
||||||
|
python-dotenv |
||||||
|
requests |
||||||
|
ollama-api |
||||||
|
``` |
||||||
|
|
||||||
|
Install them using: |
||||||
|
|
||||||
|
```bash |
||||||
|
pip install -r requirements.txt |
||||||
|
``` |
||||||
|
|
||||||
|
## Contributing |
||||||
|
|
||||||
|
Contributions are welcome! Feel free to fork the repository and submit pull requests. |
||||||
|
|
||||||
|
## License |
||||||
|
|
||||||
|
This project is licensed under the MIT License. See the `LICENSE` file for more details. |
||||||
|
|
||||||
|
## Contact |
||||||
|
|
||||||
|
For any inquiries, please reach out to: |
||||||
|
|
||||||
|
- Linkedin: https://www.linkedin.com/in/khanarafat/ |
||||||
|
- GitHub: https://github.com/raoarafat |
@ -0,0 +1,28 @@ |
|||||||
|
from summarizer.fetcher import fetch_web_content |
||||||
|
from summarizer.summarizer import summarize_text |
||||||
|
from utils.logger import logger |
||||||
|
|
||||||
|
def main(): |
||||||
|
url = input("Enter a URL to summarize: ") |
||||||
|
|
||||||
|
logger.info(f"Fetching content from: {url}") |
||||||
|
content = fetch_web_content(url) |
||||||
|
|
||||||
|
if content: |
||||||
|
logger.info("Content fetched successfully. Sending to OpenAI for summarization...") |
||||||
|
# summary = summarize_text(content,'gpt-4o-mini', engine="openai") |
||||||
|
# summary = summarize_text(content, 'deepseek-r1:1.5B', engine="ollama-lib") |
||||||
|
summary = summarize_text(content, 'deepseek-r1:1.5B', engine="ollama-api") |
||||||
|
|
||||||
|
|
||||||
|
if summary: |
||||||
|
logger.info("Summary generated successfully.") |
||||||
|
print("\nSummary of the page:\n") |
||||||
|
print(summary) |
||||||
|
else: |
||||||
|
logger.error("Failed to generate summary.") |
||||||
|
else: |
||||||
|
logger.error("Failed to fetch web content.") |
||||||
|
|
||||||
|
if __name__ == "__main__": |
||||||
|
main() |
@ -0,0 +1,4 @@ |
|||||||
|
openai |
||||||
|
requests |
||||||
|
beautifulsoup4 |
||||||
|
python-dotenv |
@ -0,0 +1,23 @@ |
|||||||
|
import requests |
||||||
|
from bs4 import BeautifulSoup |
||||||
|
|
||||||
|
def fetch_web_content(url): |
||||||
|
try: |
||||||
|
response = requests.get(url) |
||||||
|
response.raise_for_status() |
||||||
|
|
||||||
|
# Parse the HTML content |
||||||
|
soup = BeautifulSoup(response.text, 'html.parser') |
||||||
|
|
||||||
|
# Extract readable text from the web page (ignoring scripts, styles, etc.) |
||||||
|
page_text = soup.get_text(separator=' ', strip=True) |
||||||
|
|
||||||
|
return page_text[:5000] # Limit to 5000 chars (API limitation) |
||||||
|
except requests.exceptions.RequestException as e: |
||||||
|
print(f"Error fetching the webpage: {e}") |
||||||
|
return None |
||||||
|
|
||||||
|
if __name__ == "__main__": |
||||||
|
url = "https://en.wikipedia.org/wiki/Natural_language_processing" |
||||||
|
content = fetch_web_content(url) |
||||||
|
print(content[:500]) # Print a sample of the content |
@ -0,0 +1,85 @@ |
|||||||
|
import openai # type: ignore |
||||||
|
import ollama |
||||||
|
import requests |
||||||
|
from utils.config import Config |
||||||
|
|
||||||
|
# Local Ollama API endpoint |
||||||
|
OLLAMA_API = "http://127.0.0.1:11434/api/chat" |
||||||
|
|
||||||
|
# Initialize OpenAI client with API key |
||||||
|
client = openai.Client(api_key=Config.OPENAI_API_KEY) |
||||||
|
|
||||||
|
def summarize_with_openai(text, model): |
||||||
|
"""Summarize text using OpenAI's GPT model.""" |
||||||
|
try: |
||||||
|
response = client.chat.completions.create( |
||||||
|
model=model, |
||||||
|
messages=[ |
||||||
|
{"role": "system", "content": "You are a helpful assistant that summarizes web pages."}, |
||||||
|
{"role": "user", "content": f"Summarize the following text: {text}"} |
||||||
|
], |
||||||
|
max_tokens=300, |
||||||
|
temperature=0.7 |
||||||
|
) |
||||||
|
return response.choices[0].message.content |
||||||
|
except Exception as e: |
||||||
|
print(f"Error during OpenAI summarization: {e}") |
||||||
|
return None |
||||||
|
|
||||||
|
def summarize_with_ollama_lib(text, model): |
||||||
|
"""Summarize text using Ollama Python library.""" |
||||||
|
try: |
||||||
|
messages = [ |
||||||
|
{"role": "system", "content": "You are a helpful assistant that summarizes web pages."}, |
||||||
|
{"role": "user", "content": f"Summarize the following text: {text}"} |
||||||
|
] |
||||||
|
response = ollama.chat(model=model, messages=messages) |
||||||
|
return response['message']['content'] |
||||||
|
except Exception as e: |
||||||
|
print(f"Error during Ollama summarization: {e}") |
||||||
|
return None |
||||||
|
|
||||||
|
def summarize_with_ollama_api(text, model): |
||||||
|
"""Summarize text using local Ollama API.""" |
||||||
|
try: |
||||||
|
payload = { |
||||||
|
"model": model, |
||||||
|
"messages": [ |
||||||
|
{"role": "system", "content": "You are a helpful assistant that summarizes web pages."}, |
||||||
|
{"role": "user", "content": f"Summarize the following text: {text}"} |
||||||
|
], |
||||||
|
"stream": False # Set to True for streaming responses |
||||||
|
} |
||||||
|
response = requests.post(OLLAMA_API, json=payload) |
||||||
|
response_data = response.json() |
||||||
|
return response_data.get('message', {}).get('content', 'No summary generated') |
||||||
|
except Exception as e: |
||||||
|
print(f"Error during Ollama API summarization: {e}") |
||||||
|
return None |
||||||
|
|
||||||
|
def summarize_text(text, model, engine="openai"): |
||||||
|
"""Generic function to summarize text using the specified engine (openai/ollama-lib/ollama-api).""" |
||||||
|
if engine == "openai": |
||||||
|
return summarize_with_openai(text, model) |
||||||
|
elif engine == "ollama-lib": |
||||||
|
return summarize_with_ollama_lib(text, model) |
||||||
|
elif engine == "ollama-api": |
||||||
|
return summarize_with_ollama_api(text, model) |
||||||
|
else: |
||||||
|
print("Invalid engine specified. Use 'openai', 'ollama-lib', or 'ollama-api'.") |
||||||
|
return None |
||||||
|
|
||||||
|
if __name__ == "__main__": |
||||||
|
sample_text = "Artificial intelligence (AI) is intelligence demonstrated by machines, as opposed to the natural intelligence displayed by animals and humans." |
||||||
|
|
||||||
|
# Summarize using OpenAI |
||||||
|
openai_summary = summarize_text(sample_text, model="gpt-3.5-turbo", engine="openai") |
||||||
|
print("OpenAI Summary:", openai_summary) |
||||||
|
|
||||||
|
# Summarize using Ollama Python library |
||||||
|
ollama_lib_summary = summarize_text(sample_text, model="deepseek-r1:1.5B", engine="ollama-lib") |
||||||
|
print("Ollama Library Summary:", ollama_lib_summary) |
||||||
|
|
||||||
|
# Summarize using local Ollama API |
||||||
|
ollama_api_summary = summarize_text(sample_text, model="deepseek-r1:1.5B", engine="ollama-api") |
||||||
|
print("Ollama API Summary:", ollama_api_summary) |
@ -0,0 +1,11 @@ |
|||||||
|
import os |
||||||
|
from dotenv import load_dotenv |
||||||
|
|
||||||
|
# Load environment variables from .env file |
||||||
|
load_dotenv() |
||||||
|
|
||||||
|
class Config: |
||||||
|
OPENAI_API_KEY = os.getenv("OPENAI_API_KEY") |
||||||
|
|
||||||
|
if __name__ == "__main__": |
||||||
|
print("Your OpenAI Key is:", Config.OPENAI_API_KEY) |
@ -0,0 +1,16 @@ |
|||||||
|
import logging |
||||||
|
|
||||||
|
# Setup logging configuration |
||||||
|
logging.basicConfig( |
||||||
|
level=logging.INFO, |
||||||
|
format="%(asctime)s - %(levelname)s - %(message)s", |
||||||
|
handlers=[ |
||||||
|
logging.FileHandler("app.log"), |
||||||
|
logging.StreamHandler() |
||||||
|
] |
||||||
|
) |
||||||
|
|
||||||
|
logger = logging.getLogger(__name__) |
||||||
|
|
||||||
|
if __name__ == "__main__": |
||||||
|
logger.info("Logger is working correctly.") |
Loading…
Reference in new issue