From ba7455e38ee7ec47111d955d058d95c95aebc8ea Mon Sep 17 00:00:00 2001 From: lakovicb <> Date: Wed, 23 Apr 2025 14:45:58 +0200 Subject: [PATCH] Added Playwright-based scraper solution to community-contributions --- .../playwright-bojan/README.md | 56 ++++ .../openai_scraper_playwright.py | 300 ++++++++++++++++++ 2 files changed, 356 insertions(+) create mode 100644 community-contributions/playwright-bojan/README.md create mode 100644 community-contributions/playwright-bojan/openai_scraper_playwright.py diff --git a/community-contributions/playwright-bojan/README.md b/community-contributions/playwright-bojan/README.md new file mode 100644 index 0000000..f24b91c --- /dev/null +++ b/community-contributions/playwright-bojan/README.md @@ -0,0 +1,56 @@ +# π§ Playwright-Based Web Scraper for openai.com +### π Community Contribution for Ed Donner's "LLM Engineering: Master AI" Course + +> _βAn extra exercise for those who enjoy web scraping... +> In the community-contributions folder, you'll find an example Selenium solution from a student.β_ + +--- + +## π About This Project + +This is a response to Ed Donnerβs bonus exercise to scrape `https://openai.com`, which uses dynamic JavaScript rendering. +A fellow student contributed a Selenium-based solution β this one goes a step further with **Playwright**. + +--- + +## π Why Playwright Over Selenium? + +| Feature | Selenium | Playwright π | +|----------------------|------------------------------|-----------------------------| +| **Installation** | More complex setup | Minimal + faster setup | +| **Speed** | Slower due to architecture | Faster execution (async) | +| **Multi-browser** | Requires config | Built-in Chrome, Firefox, WebKit support | +| **Headless mode** | Less stable | Super stable | +| **Async-friendly** | Not built-in | Native support via asyncio | +| **Interaction APIs** | Limited | Richer simulation (mouse, scroll, etc.) | + +--- + +## βοΈ Features + +- β **Full JavaScript rendering** using Chromium +- β **Human-like behavior simulation** (mouse movement, scrolling, typing) +- β **Caching** with `diskcache` +- β **Prometheus metrics** +- β **Asynchronous scraping logic** +- β **Content summarization via OpenAI GPT API** + +--- + +## π§ Why not in JupyterLab? + +Due to the async nature of Playwright and the use of `asyncio.run()`, running this inside Jupyter causes `RuntimeError` conflicts. + +This solution was developed and tested in: + +- π» WingIDE 10 Pro +- π§ Ubuntu via WSL +- π Conda environment with Anaconda Python 3.12 + +--- + +## π How to Run + +1. Install dependencies: +```bash +pip install -r requirements.txt diff --git a/community-contributions/playwright-bojan/openai_scraper_playwright.py b/community-contributions/playwright-bojan/openai_scraper_playwright.py new file mode 100644 index 0000000..7eac886 --- /dev/null +++ b/community-contributions/playwright-bojan/openai_scraper_playwright.py @@ -0,0 +1,300 @@ +import asyncio +from playwright.async_api import async_playwright +from openai import OpenAI +import logging +import random +import time +import os +from prometheus_client import start_http_server, Counter, Histogram +from diskcache import Cache +from dotenv import load_dotenv + +load_dotenv() + +# Setting up Prometheus metrics +SCRAPE_ATTEMPTS = Counter('scrape_attempts', 'Total scraping attempts') +SCRAPE_DURATION = Histogram( + 'scrape_duration', 'Scraping duration distribution') + +# Setting up cache +cache = Cache('./scraper_cache') + + +class ScrapingError(Exception): + pass + + +class ContentAnalysisError(Exception): + pass + + +class EnhancedOpenAIScraper: + API_KEY = os.getenv("OPENAI_API_KEY") + BROWSER_EXECUTABLE = os.getenv( + "BROWSER_PATH", "/usr/bin/chromium-browser") + MAX_CONTENT_LENGTH = int(os.getenv("MAX_CONTENT_LENGTH", 30000)) + + def __init__(self, headless=True): + self.user_agents = [ + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36", + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36" + ] + self.timeout = 45000 # 45 seconds + self.retry_count = int(os.getenv("RETRY_COUNT", 2)) + self.headless = headless + self.mouse_velocity_range = (100, 500) # px/ms + self.interaction_delays = { + 'scroll': (int(os.getenv("SCROLL_DELAY_MIN", 500)), int(os.getenv("SCROLL_DELAY_MAX", 2000))), + 'click': (int(os.getenv("CLICK_DELAY_MIN", 100)), int(os.getenv("CLICK_DELAY_MAX", 300))), + 'movement': (int(os.getenv("MOVEMENT_DELAY_MIN", 50)), int(os.getenv("MOVEMENT_DELAY_MAX", 200))) + } + self.proxy_servers = [server.strip() for server in os.getenv( + "PROXY_SERVERS", "").split(',') if server.strip()] + + async def human_interaction(self, page): + """Advanced simulation of user behavior""" + # Random mouse movement path + for _ in range(random.randint(2, 5)): + x = random.randint(0, 1366) + y = random.randint(0, 768) + await page.mouse.move(x, y, steps=random.randint(5, 20)) + await page.wait_for_timeout(random.randint(*self.interaction_delays['movement'])) + + # Simulating typing + if random.random() < 0.3: + await page.keyboard.press('Tab') + await page.keyboard.type(' ', delay=random.randint(50, 200)) + + # More realistic scrolling + scroll_distance = random.choice([300, 600, 900]) + await page.mouse.wheel(0, scroll_distance) + await page.wait_for_timeout(random.randint(*self.interaction_delays['scroll'])) + + async def load_page(self, page, url): + """Smarter page loading with dynamic waiting""" + start_time = time.time() + try: + await page.goto(url, wait_until="domcontentloaded", timeout=self.timeout) + + # Smarter content extraction selectors + selectors = [ + 'main article', + '#main-content', + 'section:first-of-type', + 'div[class*="content"]', + 'body' # Fallback + ] + + for selector in selectors: + try: + element = await page.query_selector(selector) + if element: + return True + except Exception: + continue + + # Fallback if no selector is found within a certain time + if time.time() - start_time < 30: # If we haven't used the full timeout + await page.wait_for_timeout(30000 - int(time.time() - start_time)) + + return True # Page likely loaded + except Exception as e: + logging.error(f"Error loading page {url}: {e}") + return False + + @SCRAPE_DURATION.time() + async def scrape_with_retry(self): + """Main function with retry mechanism and browser reuse""" + SCRAPE_ATTEMPTS.inc() + last_error = None + browser = None + context = None + page = None + + try: + async with async_playwright() as p: + launch_args = { + "headless": self.headless, + "args": [ + "--disable-blink-features=AutomationControlled", + "--single-process", + "--no-sandbox", + f"--user-agent={random.choice(self.user_agents)}" + ], + "executable_path": self.BROWSER_EXECUTABLE + } + if self.proxy_servers: + proxy_url = random.choice(self.proxy_servers) + proxy_config = {"server": proxy_url} + proxy_username = os.getenv('PROXY_USER') + proxy_password = os.getenv('PROXY_PASS') + if proxy_username and proxy_password: + proxy_config['username'] = proxy_username + proxy_config['password'] = proxy_password + launch_args['proxy'] = proxy_config + + browser = await p.chromium.launch(**launch_args) + context = await browser.new_context( + user_agent=random.choice(self.user_agents), + viewport={"width": 1366, "height": 768}, + locale=os.getenv("BROWSER_LOCALE", "en-US") + ) + await context.route("**/*", lambda route: route.continue_()) + page = await context.new_page() + await page.add_init_script(""" + Object.defineProperty(navigator, 'webdriver', { get: () => false }); + window.navigator.chrome = { runtime: {}, app: { isInstalled: false } }; + """) + + for attempt in range(self.retry_count): + try: + logging.info( + f"Attempt {attempt + 1}: Loading OpenAI...") + if not await self.load_page(page, "https://openai.com"): + raise ScrapingError( + "Failed to load key content on OpenAI website.") + await self.human_interaction(page) + await page.screenshot(path=f"openai_debug_{attempt}.png") + content = await page.evaluate("""() => { + const selectors = [ + 'main article', + '#main-content', + 'section:first-of-type', + 'div[class*="content"]' + ]; + + let content = ''; + for (const selector of selectors) { + const element = document.querySelector(selector); + if (element) { + content += element.innerText + '\\n\\n'; + } + } + return content.trim() || document.body.innerText; + }""") + if not content.strip(): + raise ContentAnalysisError( + "No content extracted from the page.") + return content[:self.MAX_CONTENT_LENGTH] + + except (ScrapingError, ContentAnalysisError) as e: + last_error = e + logging.warning( + f"Attempt {attempt + 1} failed: {str(e)}") + if attempt < self.retry_count - 1: + await asyncio.sleep(5) + else: + if browser: + await browser.close() + browser = None + raise + except Exception as e: + last_error = e + logging.exception(f"Unexpected error on attempt { + attempt + 1}: {str(e)}") + if attempt < self.retry_count - 1: + await asyncio.sleep(5) + else: + if browser: + await browser.close() + browser = None + raise + + except Exception as e: + last_error = e + finally: + if browser: + await browser.close() + + raise last_error if last_error else Exception( + "All scraping attempts failed.") + + async def get_cached_content(self): + key = 'openai_content_cache_key' + content = cache.get(key) + if content is None: + content = await self.scrape_with_retry() + cache.set(key, content, expire=int( + os.getenv("CACHE_EXPIRY", 3600))) + return content + + +async def analyze_content(headless=True): + try: + scraper = EnhancedOpenAIScraper(headless=headless) + content = await scraper.get_cached_content() + + client = OpenAI(api_key=EnhancedOpenAIScraper.API_KEY) + if not client.api_key: + raise ContentAnalysisError( + "OpenAI API key not configured (check environment variables).") + + prompt_template = """ + Analyze the following website content and extract the following information if present: + + 1. **Overall Summary of the Website:** Provide a concise overview of the website's purpose and the main topics discussed. + 2. **Key Individuals or Entities:** Identify and briefly describe any prominent individuals, companies, or organizations mentioned. + 3. **Recent Announcements or Updates:** List any recent announcements, news, or updates found on the website, including dates if available. + 4. **Main Topics or Themes:** Identify the primary subjects or themes explored on the website. + 5. **Any Noteworthy Features or Projects:** Highlight any significant features, projects, or initiatives mentioned. + + Format the output clearly under each of these headings. If a particular piece of information is not found, indicate that it is not present. + + Content: + {content} + """ + + formatted_prompt = prompt_template.format(content=content) + model_name = os.getenv("OPENAI_MODEL", "gpt-4-turbo") + temperature = float(os.getenv("MODEL_TEMPERATURE", 0.3)) + max_tokens = int(os.getenv("MAX_TOKENS", 1500)) + top_p = float(os.getenv("MODEL_TOP_P", 0.9)) + + response = client.chat.completions.create( + model=model_name, + messages=[ + {"role": "system", "content": "You are a helpful assistant that analyzes website content and extracts key information in a structured format."}, + {"role": "user", "content": formatted_prompt} + ], + temperature=temperature, + max_tokens=max_tokens, + top_p=top_p + ) + + if not response.choices: + raise ContentAnalysisError("Empty response from GPT.") + + return response.choices[0].message.content + + except (ScrapingError, ContentAnalysisError) as e: + logging.error(f"Analysis failed: {str(e)}") + return f"Critical analysis error: {str(e)}" + except Exception as e: + logging.exception("Unexpected error during analysis.") + return f"Unexpected analysis error: {str(e)}" + + +async def main(): + logging.basicConfig( + level=os.getenv("LOG_LEVEL", "INFO").upper(), + format='%(asctime)s - %(levelname)s - %(message)s' + ) + + # Start Prometheus HTTP server for exposing metrics + try: + prometheus_port = int(os.getenv("PROMETHEUS_PORT", 8000)) + start_http_server(prometheus_port) + logging.info(f"Prometheus metrics server started on port { + prometheus_port}") + except Exception as e: + logging.warning(f"Failed to start Prometheus metrics server: {e}") + + start_time = time.time() + result = await analyze_content(headless=True) + end_time = time.time() + + print(f"\nAnalysis completed in {end_time - start_time:.2f} seconds\n") + print(result) + +if __name__ == "__main__": + asyncio.run(main())