Browse Source

base of Newspaper4k helper

pull/288/head
zKWolf 1 year ago
parent
commit
b425312b0a
  1. 2
      installer/__init__.py
  2. 1
      installer/client/cli/__init__.py
  3. 136
      installer/client/cli/np4k.py
  4. 2
      pyproject.toml

2
installer/__init__.py

@ -1,4 +1,4 @@
from .client.cli import main as cli, main_save, main_ts, main_yt
from .client.cli import main as cli, main_save, main_ts, main_yt, main_np4k
from .server import (
run_api_server,
run_webui_server,

1
installer/client/cli/__init__.py

@ -2,3 +2,4 @@ from .fabric import main
from .yt import main as main_yt
from .ts import main as main_ts
from .save import cli as main_save
from .np4k import main as main_np4k

136
installer/client/cli/np4k.py

@ -0,0 +1,136 @@
import argparse
from newspaper import Article
import json
import time
class Np4k:
def __init__(self, file_path=None, single_url=None, output_format='stdout'):
self.file_path = file_path
self.single_url = single_url
self.output_format = output_format.lower()
self.articles_data = []
self.urls = self.load_urls()
def load_urls(self):
'''Load URLs from a file or a single URL based on the input provided.'''
urls = []
if self.file_path:
try:
with open(self.file_path, 'r') as file:
urls = [url.strip() for url in file.readlines() if url.strip()]
except FileNotFoundError:
print(f'The file {self.file_path} was not found.')
except Exception as e:
print(f'Error reading from {self.file_path}: {e}')
elif self.single_url:
urls = [self.single_url]
return urls
def process_urls(self):
'''Run newspaper4k against each URL and extract/produce metadata'''
timestamp = int(time.time())
output_filename = f'_output_{timestamp}.{"json" if self.output_format == "json" else "txt"}'
for url in self.urls:
if url: # Check if URL is not empty
try:
article_data = self.newspaper4k(url)
self.articles_data.append(article_data)
# Always print the article text to stdout.
print(article_data.get('text', 'No text extracted'))
except Exception as e:
print(f'Error processing URL {url}: {e}')
continue
# Write the extracted data to a file in the specified format if 'json' or 'kvp' is specified
if self.output_format != 'stdout':
if self.output_format == 'json':
self.write_json(output_filename)
else: # 'kvp' format
self.write_kvp(output_filename)
def format_data(self, article_data, format_type):
'''Formats the article data based on the specified format for terminal output'''
if format_type == 'json':
return json.dumps(article_data, ensure_ascii=False, indent=4)
elif format_type == 'kvp':
formatted_data = ""
for key, value in article_data.items():
if isinstance(value, list):
value = ', '.join(value)
if isinstance(value, str):
value = value.replace('\n', '\\n')
formatted_data += f"{key}: {value}\n"
return formatted_data
elif format_type == 'stdout': # Only print the article text for stdout
return article_data.get('text', 'No text extracted')
def write_json(self, output_filename):
try:
with open(output_filename, 'w', encoding='utf-8') as f:
json.dump(self.articles_data, f, ensure_ascii=False, indent=4)
print(f'Successfully wrote extracted data to {output_filename}')
except Exception as e:
print(f'Error writing data to {output_filename}: {e}')
def write_kvp(self, output_filename):
try:
with open(output_filename, 'w', encoding='utf-8') as f:
for article in self.articles_data:
for key, value in article.items():
if isinstance(value, list):
value = ', '.join(value)
if isinstance(value, str):
value = value.replace('\n', '\\n')
f.write(f"{key}: {value}\n")
f.write("---\n")
print(f'Successfully wrote extracted data to {output_filename}')
except Exception as e:
print(f'Error writing data to {output_filename}: {e}')
def newspaper4k(self, url):
article = Article(url, fetch_images=False)
processed_article = {
"title": "",
"keywords": [],
"tags": [],
"authors": [],
"summary": "",
"text": "",
"publish_date": "",
"url": "",
}
try:
article.download()
article.parse()
article.nlp()
processed_article["title"] = article.title or "Not Found"
processed_article["keywords"] = article.keywords if article.keywords is not None else []
processed_article["tags"] = list(article.tags) if article.tags is not None else []
processed_article["authors"] = article.authors if article.authors is not None else ["Not Found"]
processed_article["summary"] = article.summary or "Not Found"
processed_article["text"] = article.text or "Not Found"
processed_article["publish_date"] = article.publish_date.isoformat() if article.publish_date else "Not Found"
processed_article["url"] = url
except Exception as e:
print(f'Failed to process article from {url}: {e}')
raise e
return processed_article
def parse_arguments():
parser = argparse.ArgumentParser(description='Np4k is a helper to extract information from blogs or articles.')
parser.add_argument('--url', type=str, help='A single URL to process.')
parser.add_argument('--file', type=str, help='A file containing the list of URLs to process.')
parser.add_argument('--output', type=str, choices=['stdout', 'kvp', 'json'], default='stdout', help='The file format to write the extracted data in. Default is stdout.')
return parser.parse_args()
def main():
args = parse_arguments()
np4k = Np4k(file_path=args.file, single_url=args.url, output_format=args.output)
np4k.process_urls()
if __name__ == "__main__":
main()

2
pyproject.toml

@ -29,6 +29,7 @@ python-dotenv = "^1.0.1"
jwt = "^1.3.1"
flask = "^3.0.2"
helpers = "^0.2.0"
newspaper4k = "0.9.3.1"
[tool.poetry.group.cli.dependencies]
pyyaml = "^6.0.1"
@ -69,3 +70,4 @@ fabric-webui = 'installer:run_webui_server'
ts = 'installer:main_ts'
yt = 'installer:main_yt'
save = 'installer:main_save'
np4k = 'installer:main_np4k'

Loading…
Cancel
Save