From c70c6c4f84cae9799679e7655d12cc21d61e08d7 Mon Sep 17 00:00:00 2001 From: Hazperera Date: Thu, 27 Feb 2025 14:18:14 +0000 Subject: [PATCH] add a python script for an automated website content analysis & SEO extraction --- .../day-1-marketing_insights_scraper.py | 176 ++++++++++++++++++ 1 file changed, 176 insertions(+) create mode 100644 week1/community-contributions/day-1-marketing_insights_scraper.py diff --git a/week1/community-contributions/day-1-marketing_insights_scraper.py b/week1/community-contributions/day-1-marketing_insights_scraper.py new file mode 100644 index 0000000..28b8920 --- /dev/null +++ b/week1/community-contributions/day-1-marketing_insights_scraper.py @@ -0,0 +1,176 @@ +import os +import time +import pandas as pd +import re +from dotenv import load_dotenv +from selenium import webdriver +from selenium.webdriver.chrome.service import Service +from selenium.webdriver.chrome.options import Options +from selenium.webdriver.common.by import By +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC +from openai import OpenAI +from openpyxl import load_workbook +from openpyxl.styles import Font, Alignment + +# Load environment variables +load_dotenv(override=True) +api_key = os.getenv('OPENAI_API_KEY') + +# Validate API Key +if not api_key: + raise ValueError("No API key was found - please check your .env file.") + +# Initialize OpenAI client +openai = OpenAI() + +# Set up Selenium WebDriver +chrome_options = Options() +chrome_options.add_argument("--headless") +chrome_options.add_argument("--disable-gpu") +chrome_options.add_argument("--no-sandbox") +chrome_options.add_argument("--disable-dev-shm-usage") + +class Website: + """Scrapes and processes website content using Selenium.""" + + def __init__(self, url: str): + self.url = url + self.text = "No content extracted." + + service = Service(executable_path="/opt/homebrew/bin/chromedriver") + driver = webdriver.Chrome(service=service, options=chrome_options) + + try: + driver.get(url) + WebDriverWait(driver, 10).until( + EC.presence_of_element_located((By.TAG_NAME, "body")) + ) + body_element = driver.find_element(By.TAG_NAME, "body") + self.text = body_element.text.strip() if body_element else "No content extracted." + except Exception as e: + print(f"Error fetching website: {e}") + finally: + driver.quit() + + def summarized_text(self, max_length=1500): + return self.text[:max_length] + ("..." if len(self.text) > max_length else "") + +def clean_text(text): + """ + Cleans extracted text by removing markdown-style formatting. + """ + text = re.sub(r"###*\s*", "", text) + text = re.sub(r"\*\*(.*?)\*\*", r"\1", text) + return text.strip() + +# Aspect-specific prompts for concise output +aspect_prompts = { + "Marketing Strategies": "Summarize the core marketing strategies used on this website in in under 30 words. Do not include a title or introduction.", + "SEO Keywords": "List only the most relevant SEO keywords from this website, separated by commas. Do not include a title or introduction.", + "User Engagement Tactics": "List key engagement tactics used on this website (e.g., interactive features, user incentives, social proof). Keep responses to 3-5 bullet points. Do not include a title or introduction.", + "Call-to-Action Phrases": "List only the most common Call-to-Action phrases used on this website, separated by commas. Do not include a title or introduction.", + "Branding Elements": "Summarize the brand's tone, style, and positioning in under 30 words. Do not include a title or introduction.", + "Competitor Comparison": "Briefly describe how this website differentiates itself from competitors in under 30 words. Do not include a title or introduction.", + "Product Descriptions": "List the most important features or benefits of the products/services described on this website in under 30 words. Do not include a title or introduction.", + "Customer Reviews Sentiment": "Summarize the overall sentiment of customer reviews in oin under 30 words, highlighting common themes. Do not include a title or introduction.", + "Social Media Strategy": "List key social media strategies used on this website, separated by commas. Do not include a title or introduction." +} + + +def summarize(url: str) -> dict: + """ + Fetches a website, extracts relevant content, and generates a separate summary for each aspect. + + :param url: The website URL to analyze. + :return: A dictionary containing extracted information. + """ + website = Website(url) + + if not website.text or website.text == "No content extracted.": + return {"URL": url, "Error": "Failed to extract content"} + + extracted_data = {"URL": url} + + for aspect, prompt in aspect_prompts.items(): + try: + formatted_prompt = f"{prompt} \n\nContent:\n{website.summarized_text()}" + response = openai.chat.completions.create( + model="gpt-4o-mini", + messages=[ + {"role": "system", "content": "You are an expert at extracting structured information from website content."}, + {"role": "user", "content": formatted_prompt} + ] + ) + + extracted_data[aspect] = clean_text(response.choices[0].message.content) + + except Exception as e: + extracted_data[aspect] = f"Error generating summary: {e}" + + return extracted_data + +def save_to_excel(data_list: list, filename="website_analysis.xlsx"): + """ + Saves extracted information to an Excel file with proper formatting. + + :param data_list: A list of dictionaries containing extracted website details. + :param filename: The name of the Excel file to save data. + """ + df = pd.DataFrame(data_list) + + df.to_excel(filename, index=False) + + wb = load_workbook(filename) + ws = wb.active + + # Auto-adjust column widths + for col in ws.columns: + max_length = 0 + col_letter = col[0].column_letter + for cell in col: + try: + if cell.value: + max_length = max(max_length, len(str(cell.value))) + except: + pass + ws.column_dimensions[col_letter].width = min(max_length + 2, 50) + + # Format headers + for cell in ws[1]: + cell.font = Font(bold=True) + cell.alignment = Alignment(horizontal="center", vertical="center") + + # Wrap text for extracted content + for row in ws.iter_rows(min_row=2): + for cell in row: + cell.alignment = Alignment(wrap_text=True, vertical="top") + + wb.save(filename) + print(f"Data saved to {filename} with improved formatting.") + +# 🔹 LIST OF WEBSITES TO PROCESS +websites = [ + "https://www.udacity.com/", + "https://www.coursera.org", + "https://www.udemy.com", + "https://www.edx.org", + "https://www.freecodecamp.org/", + "https://www.datacamp.com/", + "https://www.w3schools.com/", + "https://www.futurelearn.com/", + "https://codefirstgirls.com/", + "https://www.linkedin.com/learning", +] + +if __name__ == "__main__": + print("\nProcessing websites...\n") + extracted_data_list = [] + + for site in websites: + print(f"Extracting data from {site}...") + extracted_data = summarize(site) + extracted_data_list.append(extracted_data) + + save_to_excel(extracted_data_list) + print("\nAll websites processed successfully!")