1 changed files with 419 additions and 0 deletions
@ -0,0 +1,419 @@
|
||||
{ |
||||
"cells": [ |
||||
{ |
||||
"cell_type": "markdown", |
||||
"id": "c7d95a7f-205a-4262-a1af-4579489025ff", |
||||
"metadata": {}, |
||||
"source": [ |
||||
"# Hello everyone." |
||||
] |
||||
}, |
||||
{ |
||||
"cell_type": "markdown", |
||||
"id": "bc815dbc-acf7-45f9-a043-5767184c44c6", |
||||
"metadata": {}, |
||||
"source": [ |
||||
"I completed the day 1, first LLM Experiment moments ago and found it really awesome. After the challenge was done, I wanted to chip in my two cents by making a PDF summarizer, basing myself on the code for the Website Summarizer. I want to share it in this contribution!\n", |
||||
"### To consider:\n", |
||||
"* To extract the contents of PDF files, I used the PyPDF2 library, which doesn't come with the default configuration of the virtual environment. To remedy the situation, you need to follow the steps:\n", |
||||
" 1. Shut down Anaconda. Running `CTRL-C` in the Anaconda terminal should achieve this.\n", |
||||
" 2. Run the following command, `pip install PyPDF2 --user`\n", |
||||
" 3. Restart Jupyter lab with `jupyter lab`\n", |
||||
"* To find PDF files online, you can add `filetype:url` on your browser query, i.e. searching the following can give you PDF files to add as input: `AI Engineering prompts filetype:pdf`!\n", |
||||
"\n", |
||||
"Without further ado, here's the PDF Summarizer!" |
||||
] |
||||
}, |
||||
{ |
||||
"cell_type": "code", |
||||
"execution_count": null, |
||||
"id": "14e98361-f302-423d-87a5-f2f5d570f4ac", |
||||
"metadata": {}, |
||||
"outputs": [], |
||||
"source": [ |
||||
"### Prerequisite - Install below package\n", |
||||
"#--> !pip install PyPDF2 --user" |
||||
] |
||||
}, |
||||
{ |
||||
"cell_type": "code", |
||||
"execution_count": null, |
||||
"id": "06b63787-c6c8-4868-8a71-eb56b7618626", |
||||
"metadata": {}, |
||||
"outputs": [], |
||||
"source": [ |
||||
"# Import statements\n", |
||||
"import os\n", |
||||
"import requests\n", |
||||
"from dotenv import load_dotenv\n", |
||||
"from IPython.display import Markdown, display\n", |
||||
"from openai import OpenAI\n", |
||||
"from io import BytesIO\n", |
||||
"from PyPDF2 import PdfReader\n", |
||||
"import logging" |
||||
] |
||||
}, |
||||
{ |
||||
"cell_type": "code", |
||||
"execution_count": null, |
||||
"id": "284ca770-5da4-495c-b1cf-637727a8609f", |
||||
"metadata": {}, |
||||
"outputs": [], |
||||
"source": [ |
||||
"# Load environment variables in a file called .env\n", |
||||
"\n", |
||||
"load_dotenv()\n", |
||||
"api_key = os.getenv('OPENAI_API_KEY')\n", |
||||
"\n", |
||||
"# Check the key\n", |
||||
"\n", |
||||
"if not api_key:\n", |
||||
" print(\"No API key was found - please head over to the troubleshooting notebook in this folder to identify & fix!\")\n", |
||||
"elif not api_key.startswith(\"sk-proj-\"):\n", |
||||
" print(\"An API key was found, but it doesn't start sk-proj-; please check you're using the right key - see troubleshooting notebook\")\n", |
||||
"elif api_key.strip() != api_key:\n", |
||||
" print(\"An API key was found, but it looks like it might have space or tab characters at the start or end - please remove them - see troubleshooting notebook\")\n", |
||||
"else:\n", |
||||
" print(\"API key found and looks good so far!\")" |
||||
] |
||||
}, |
||||
{ |
||||
"cell_type": "code", |
||||
"execution_count": null, |
||||
"id": "d4c316d7-d9c9-4400-b03e-1dd629c6b2ad", |
||||
"metadata": {}, |
||||
"outputs": [], |
||||
"source": [ |
||||
"openai = OpenAI()\n", |
||||
"\n", |
||||
"# If this doesn't work, try Kernel menu >> Restart Kernel and Clear Outputs Of All Cells, then run the cells from the top of this notebook down.\n", |
||||
"# If it STILL doesn't work (horrors!) then please see the troubleshooting notebook, or try the below line instead:\n", |
||||
"# openai = OpenAI(api_key=\"your-key-here-starting-sk-proj-\")" |
||||
] |
||||
}, |
||||
{ |
||||
"cell_type": "code", |
||||
"execution_count": 60, |
||||
"id": "b6f2967e-4779-458f-aea8-c47147528f77", |
||||
"metadata": {}, |
||||
"outputs": [], |
||||
"source": [ |
||||
"# Step 1: Defince calss for article pdf document\n", |
||||
"\n", |
||||
"class Article:\n", |
||||
" def __init__(self, url):\n", |
||||
" # Configure logging\n", |
||||
" logging.basicConfig(level=logging.ERROR)\n", |
||||
"\n", |
||||
" # Comprehensive headers to mimic browser\n", |
||||
" headers = {\n", |
||||
" 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',\n", |
||||
" 'Accept': 'application/pdf,text/html,application/xhtml+xml,application/xml',\n", |
||||
" 'Accept-Language': 'en-US,en;q=0.5',\n", |
||||
" 'Referer': self._extract_base_url(url)\n", |
||||
" }\n", |
||||
"\n", |
||||
" try:\n", |
||||
" # Enhanced download attempt with multiple strategies\n", |
||||
" response = self._download_pdf(url, headers)\n", |
||||
" \n", |
||||
" # Process PDF\n", |
||||
" pdf_bytes = BytesIO(response.content)\n", |
||||
" reader = PdfReader(pdf_bytes)\n", |
||||
" \n", |
||||
" # Efficient text extraction\n", |
||||
" self.text = \" \".join(page.extract_text() for page in reader.pages)\n", |
||||
" \n", |
||||
" # Safe metadata extraction\n", |
||||
" self.title = (reader.metadata or {}).get(\"/Title\", \"No title found\") or \"No title found\"\n", |
||||
"\n", |
||||
" except Exception as error:\n", |
||||
" logging.error(f\"PDF Processing Error: {error}\")\n", |
||||
" self.text = \"No text found\"\n", |
||||
" self.title = \"No title found\"\n", |
||||
"\n", |
||||
" def _download_pdf(self, url, headers):\n", |
||||
" \"\"\"\n", |
||||
" Multiple download strategies to handle different access scenarios\n", |
||||
" \"\"\"\n", |
||||
" try:\n", |
||||
" # Primary download attempt\n", |
||||
" response = requests.get(url, headers=headers, timeout=10)\n", |
||||
" response.raise_for_status()\n", |
||||
" return response\n", |
||||
"\n", |
||||
" except requests.RequestException as primary_error:\n", |
||||
" logging.warning(f\"Primary download failed: {primary_error}\")\n", |
||||
" \n", |
||||
" try:\n", |
||||
" # Fallback strategy: Disable SSL verification\n", |
||||
" response = requests.get(url, headers=headers, verify=False, timeout=10)\n", |
||||
" response.raise_for_status()\n", |
||||
" return response\n", |
||||
" \n", |
||||
" except requests.RequestException as fallback_error:\n", |
||||
" logging.error(f\"All download attempts failed: {fallback_error}\")\n", |
||||
" raise\n", |
||||
"\n", |
||||
" def _extract_base_url(self, url):\n", |
||||
" \"\"\"\n", |
||||
" Extract base URL for Referer header\n", |
||||
" \"\"\"\n", |
||||
" from urllib.parse import urlparse\n", |
||||
" parsed_url = urlparse(url)\n", |
||||
" return f\"{parsed_url.scheme}://{parsed_url.netloc}\"" |
||||
] |
||||
}, |
||||
{ |
||||
"cell_type": "markdown", |
||||
"id": "95933aab-54df-4afc-a19c-46033f51378f", |
||||
"metadata": {}, |
||||
"source": [ |
||||
"#####user_prompt_input = input(\"Explain how you would like me to summarise the document? \\n\")" |
||||
] |
||||
}, |
||||
{ |
||||
"cell_type": "code", |
||||
"execution_count": 61, |
||||
"id": "821acbfa-ad06-4afd-9ac1-b9f68b7a418e", |
||||
"metadata": {}, |
||||
"outputs": [], |
||||
"source": [ |
||||
"# General Research Analyst\n", |
||||
"#### Step 1: Create your prompts\n", |
||||
"\n", |
||||
"def craft_user_prompt(article):\n", |
||||
" user_prompt = f\"You are looking at a document titled {article.title}\\n Based on the body of the document, provide an impactful summary. The body of the article is as follows.\"\n", |
||||
" user_prompt += article.text\n", |
||||
" return user_prompt\n", |
||||
"\n", |
||||
"### Step 2: Make the messages list\n", |
||||
"def craft_messages(article):\n", |
||||
" system_prompt = '''You are a research assistant specialising in processing documents. You have been tasked with analysing the contents of the articel and call out the important points as bullets and highlights. \\\n", |
||||
" Ignore text that doesn't belong to the article, like headers or navigation related text. Respond in markdown. Structure your text in the form of question/answer.'''\n", |
||||
" return [\n", |
||||
" {\"role\": \"system\", \"content\": system_prompt},\n", |
||||
" {\"role\": \"user\", \"content\": craft_user_prompt(article)}\n", |
||||
" ]" |
||||
] |
||||
}, |
||||
{ |
||||
"cell_type": "code", |
||||
"execution_count": 62, |
||||
"id": "81ab896e-1ba9-4964-a477-2a0608b7036c", |
||||
"metadata": {}, |
||||
"outputs": [], |
||||
"source": [ |
||||
"# Step 3: Call OpenAI\n", |
||||
"def summarize(url):\n", |
||||
" article = Article(url)\n", |
||||
" response = openai.chat.completions.create(\n", |
||||
" model = \"gpt-4o-mini\",\n", |
||||
" messages = craft_messages(article)\n", |
||||
" )\n", |
||||
" return response.choices[0].message.content" |
||||
] |
||||
}, |
||||
{ |
||||
"cell_type": "markdown", |
||||
"id": "ccdfdd33-652f-4ecf-9a0d-355197aba9d5", |
||||
"metadata": {}, |
||||
"source": [ |
||||
"# Results below:" |
||||
] |
||||
}, |
||||
{ |
||||
"cell_type": "code", |
||||
"execution_count": 63, |
||||
"id": "008c03fd-9cc5-4449-a054-7bbef1863b01", |
||||
"metadata": {}, |
||||
"outputs": [ |
||||
{ |
||||
"name": "stdin", |
||||
"output_type": "stream", |
||||
"text": [ |
||||
"Please input the pdf URL: \n", |
||||
" https://www.binasss.sa.cr/bibliotecas/bhm/ago22/32.pdf\n" |
||||
] |
||||
}, |
||||
{ |
||||
"data": { |
||||
"text/markdown": [ |
||||
"## Summary of **Bronchiectasis — A Clinical Review**\n", |
||||
"\n", |
||||
"### What is bronchiectasis?\n", |
||||
"- Bronchiectasis is a clinical syndrome characterized by:\n", |
||||
" - Chronic cough \n", |
||||
" - Sputum production\n", |
||||
" - Abnormal thickening and dilation of bronchial walls, visible on lung imaging.\n", |
||||
"\n", |
||||
"### Historical Background\n", |
||||
"- First reported by René Laënnec in 1819.\n", |
||||
"- Radiographic characteristics understood more clearly since the 1950s.\n", |
||||
"\n", |
||||
"### Current Trends\n", |
||||
"- Significantly increasing incidence and prevalence over the last 20 years.\n", |
||||
"- Diagnosed across a wide age range, with geographic variability in prevalence (as high as 1.5% in some populations).\n", |
||||
"- Improved recognition, partly due to advanced imaging technologies like CT scans.\n", |
||||
" \n", |
||||
"### Clinical Presentation\n", |
||||
"- Symptoms include chronic productive cough, chest pain, and shortness of breath.\n", |
||||
"- Often misdiagnosed as chronic bronchitis or asthma, leading to delayed diagnosis.\n", |
||||
"- Intermittent exacerbations are common, defined by a worsening of cough and sputum characteristics for 48 hours or more.\n", |
||||
"\n", |
||||
"### Diagnostic Approaches\n", |
||||
"- High-resolution CT scans are essential for diagnosis, seeking specific airway abnormalities.\n", |
||||
"- A systematic evaluation to identify underlying causes (such as congenital disorders or autoimmune diseases) is critical.\n", |
||||
"\n", |
||||
"### Pathobiology\n", |
||||
"- Multiple factors lead to airway dysfunction, inflammatory responses, and cyclical deterioration.\n", |
||||
"- Neutrophils and neutrophil elastase play significant roles in disease progression and exacerbations.\n", |
||||
"\n", |
||||
"### Microbiological Features\n", |
||||
"- **Common pathogens**:\n", |
||||
" - Pseudomonas aeruginosa (notably linked to exacerbations and worse prognosis)\n", |
||||
" - Staphylococcus aureus and other bacteria.\n", |
||||
"- Non-tuberculous mycobacterial infections are reported increasingly.\n", |
||||
"\n", |
||||
"### Treatment Strategies\n", |
||||
"- Emphasizes education about the disease and its management.\n", |
||||
"- Goals include symptom management, quality of life improvement, and reduction of exacerbation frequency.\n", |
||||
"- Options include:\n", |
||||
" - Airway-clearance therapies\n", |
||||
" - Antibiotics (macrolides or inhaled antibiotics)\n", |
||||
" - Addressing any treatable underlying conditions.\n", |
||||
"\n", |
||||
"### Prognostic Indicators\n", |
||||
"- Disease severity scores like the Bronchiectasis Severity Index and FACED scale are used for predicting outcomes.\n", |
||||
"\n", |
||||
"### Future Directions\n", |
||||
"- Emphasis on research into novel therapies targeting inflamatory pathways and innovative diagnostic tools.\n", |
||||
"\n", |
||||
"### Conclusion\n", |
||||
"- Bronchiectasis represents a complex clinical challenge that requires a multidisciplinary approach for effective management and treatment improvements. Further understanding of the disease mechanisms, enhancing diagnostic processes, and tailored therapies are vital for future advancements in the care of patients." |
||||
], |
||||
"text/plain": [ |
||||
"<IPython.core.display.Markdown object>" |
||||
] |
||||
}, |
||||
"metadata": {}, |
||||
"output_type": "display_data" |
||||
} |
||||
], |
||||
"source": [ |
||||
"# Step 4: Print the result of an example pdf\n", |
||||
"article_url=input('Please input the pdf URL: \\n')\n", |
||||
"summary = summarize(article_url)\n", |
||||
"display(Markdown(summary))" |
||||
] |
||||
}, |
||||
{ |
||||
"cell_type": "markdown", |
||||
"id": "fd2cb395-9e29-438e-9388-5555673b4689", |
||||
"metadata": {}, |
||||
"source": [ |
||||
"# Financial Report Analyst\n", |
||||
"#### Step 1: Create your prompts\n", |
||||
"\n", |
||||
"def craft_user_prompt(article):\n", |
||||
" user_prompt = f\"You are looking at a document titled {article.title}\\n Based on the body of the document, provide an impactful summary. Call out important points as bullets and highlughts . The body of the article is as follows.\"\n", |
||||
" user_prompt += article.text\n", |
||||
" return user_prompt\n", |
||||
"\n", |
||||
"#### Step 2: Make the messages list\n", |
||||
"def craft_messages(article):\n", |
||||
" system_prompt = '''You are a financial report research assistant that specialises in analysing the contents of a financial report and provide summary for investors to \\ \n", |
||||
" help adjust investment strategies. Call out important points as bullets and highlights. \\\n", |
||||
" Ignore text that doesn't belong to the article, like headers or navigation related text. Respond in markdown. Structure your text in the form of question/answer.'''\n", |
||||
" return [\n", |
||||
" {\"role\": \"system\", \"content\": system_prompt},\n", |
||||
" {\"role\": \"user\", \"content\": craft_user_prompt(article)}\n", |
||||
" ]" |
||||
] |
||||
}, |
||||
{ |
||||
"cell_type": "markdown", |
||||
"id": "70cbbffd-bf70-47d1-b5d6-5f929bdd89eb", |
||||
"metadata": {}, |
||||
"source": [ |
||||
"# Medical Report Analyst\n", |
||||
"#### Step 1: Create your prompts\n", |
||||
"\n", |
||||
"def craft_user_prompt(article):\n", |
||||
" user_prompt = f\"You are looking at a research article titled {article.title}\\n Based on the body of the article, Summarise the research article while highliting, Introduction, the problem, possible solution and conclusion. The body of the article is as follows.\"\n", |
||||
" user_prompt += article.text\n", |
||||
" return user_prompt\n", |
||||
"\n", |
||||
"#### Step 2: Make the messages list\n", |
||||
"def craft_messages(article):\n", |
||||
" system_prompt = \"You are a medical research assistant that analyses the contents of a research article and provides a summary while calling out important points as bullets and quotes. \\\n", |
||||
" Ignore text that doesn't belong to the article, like headers or navigation related text. Respond in markdown. Structure your text in the form of question/answer.\"\n", |
||||
" return [\n", |
||||
" {\"role\": \"system\", \"content\": system_prompt},\n", |
||||
" {\"role\": \"user\", \"content\": craft_user_prompt(article)}\n", |
||||
" ]" |
||||
] |
||||
}, |
||||
{ |
||||
"cell_type": "markdown", |
||||
"id": "c0bb9579-686a-4277-a139-7dff2c64cc9f", |
||||
"metadata": {}, |
||||
"source": [ |
||||
"# QA Format\n", |
||||
"#### Step 1: Create your prompts\n", |
||||
"\n", |
||||
"def craft_user_prompt(article):\n", |
||||
" user_prompt = f\"You are looking at a research article titled {article.title}\\n Based on the body of the article, how are micro RNAs produced in the cell? State the function of the proteins \\\n", |
||||
" involved. The body of the article is as follows.\"\n", |
||||
" user_prompt += article.text\n", |
||||
" return user_prompt\n", |
||||
"\n", |
||||
"#### Step 2: Make the messages list\n", |
||||
"def craft_messages(article):\n", |
||||
" system_prompt = \"You are an assistant that analyses the contents of a research article and provide answers to the question asked by the user in 250 words or less. \\\n", |
||||
" Ignore text that doesn't belong to the article, like headers or navigation related text. Respond in markdown. Structure your text in the form of question/answer.\"\n", |
||||
" return [\n", |
||||
" {\"role\": \"system\", \"content\": system_prompt},\n", |
||||
" {\"role\": \"user\", \"content\": craft_user_prompt(article)}\n", |
||||
" ]" |
||||
] |
||||
}, |
||||
{ |
||||
"cell_type": "code", |
||||
"execution_count": null, |
||||
"id": "680b45ae-6f3d-43b5-b69a-4e7f4fcca268", |
||||
"metadata": {}, |
||||
"outputs": [], |
||||
"source": [] |
||||
}, |
||||
{ |
||||
"cell_type": "code", |
||||
"execution_count": null, |
||||
"id": "988864cc-08a2-4cfb-a09b-524504ee3803", |
||||
"metadata": {}, |
||||
"outputs": [], |
||||
"source": [] |
||||
} |
||||
], |
||||
"metadata": { |
||||
"kernelspec": { |
||||
"display_name": "Python 3 (ipykernel)", |
||||
"language": "python", |
||||
"name": "python3" |
||||
}, |
||||
"language_info": { |
||||
"codemirror_mode": { |
||||
"name": "ipython", |
||||
"version": 3 |
||||
}, |
||||
"file_extension": ".py", |
||||
"mimetype": "text/x-python", |
||||
"name": "python", |
||||
"nbconvert_exporter": "python", |
||||
"pygments_lexer": "ipython3", |
||||
"version": "3.11.11" |
||||
} |
||||
}, |
||||
"nbformat": 4, |
||||
"nbformat_minor": 5 |
||||
} |
Loading…
Reference in new issue