Browse Source

Merge branch 'main' of github.com:ed-donner/llm_engineering

pull/275/head
Edward Donner 2 months ago
parent
commit
9ba3a99f84
  1. 273
      week1/community-contributions/Week1-Day2-Ollama-Exercise.ipynb
  2. 279
      week1/community-contributions/Week_1-Day 2-Article_Title_Generator.ipynb
  3. 472
      week1/community-contributions/Week_1-Day 5-Article_Title_Generator-V2.ipynb
  4. 4
      week1/community-contributions/day-1-ollama-app.ipynb
  5. 229
      week1/community-contributions/website-summarizer-by-tithi.ipynb
  6. 180
      week1/community-contributions/week1_Ollama_generate_streams.ipynb
  7. 129
      week2/community-contributions/week2_day2_gradio/gradio_ui.py
  8. 60
      week2/community-contributions/week2_day2_gradio/json_handlers.py
  9. 6
      week2/community-contributions/week2_day2_gradio/languages.json
  10. 15
      week2/community-contributions/week2_day2_gradio/main.py
  11. 28
      week2/community-contributions/week2_day2_gradio/ollama_utils.py
  12. 1
      week2/community-contributions/week2_day2_gradio/readme.txt
  13. 1
      week2/community-contributions/week2_day2_gradio/settings.json
  14. 17
      week2/community-contributions/week2_day2_gradio/system_prompt.txt
  15. 2
      week2/day1.ipynb
  16. 186
      week3/community-contributions/day5_qwen2_whisper.ipynb
  17. 394
      week5/community-contributions/day 4 no_langchain/RAG_chat_no_LangChain.ipynb
  18. 42
      week5/community-contributions/day 4 no_langchain/knowledge_collection/languages/alsacien.md
  19. 31
      week5/community-contributions/day 4 no_langchain/knowledge_collection/languages/bourguignon.md
  20. 33
      week5/community-contributions/day 4 no_langchain/knowledge_collection/languages/breton.md
  21. 34
      week5/community-contributions/day 4 no_langchain/knowledge_collection/languages/gascon.md
  22. 30
      week5/community-contributions/day 4 no_langchain/knowledge_collection/languages/languedocien.md
  23. 26
      week5/community-contributions/day 4 no_langchain/knowledge_collection/languages/lorrain.md
  24. 34
      week5/community-contributions/day 4 no_langchain/knowledge_collection/languages/normand.md
  25. 27
      week5/community-contributions/day 4 no_langchain/knowledge_collection/languages/picard.md
  26. 27
      week5/community-contributions/day 4 no_langchain/knowledge_collection/languages/provencal.md
  27. 37
      week5/community-contributions/day 4 no_langchain/knowledge_collection/mountains/alpes.md
  28. 36
      week5/community-contributions/day 4 no_langchain/knowledge_collection/mountains/ardennes.md
  29. 37
      week5/community-contributions/day 4 no_langchain/knowledge_collection/mountains/jura.md
  30. 35
      week5/community-contributions/day 4 no_langchain/knowledge_collection/mountains/massif_armorican.md
  31. 34
      week5/community-contributions/day 4 no_langchain/knowledge_collection/mountains/massif_central.md
  32. 44
      week5/community-contributions/day 4 no_langchain/knowledge_collection/mountains/morvan.md
  33. 40
      week5/community-contributions/day 4 no_langchain/knowledge_collection/mountains/pyrenees.md
  34. 33
      week5/community-contributions/day 4 no_langchain/knowledge_collection/mountains/vosges.md
  35. 47
      week5/community-contributions/day 4 no_langchain/knowledge_collection/regions/alsace_lorraine.md
  36. 47
      week5/community-contributions/day 4 no_langchain/knowledge_collection/regions/bourgogne.md
  37. 45
      week5/community-contributions/day 4 no_langchain/knowledge_collection/regions/bretagne.md
  38. 47
      week5/community-contributions/day 4 no_langchain/knowledge_collection/regions/gascogne.md
  39. 47
      week5/community-contributions/day 4 no_langchain/knowledge_collection/regions/ile_de_france.md
  40. 46
      week5/community-contributions/day 4 no_langchain/knowledge_collection/regions/languedoc.md
  41. 48
      week5/community-contributions/day 4 no_langchain/knowledge_collection/regions/normandie.md
  42. 48
      week5/community-contributions/day 4 no_langchain/knowledge_collection/regions/poitou.md
  43. 50
      week5/community-contributions/day 4 no_langchain/knowledge_collection/regions/provence.md
  44. 636
      week6/community-contributions/week6_day2_add_validation_set.ipynb

273
week1/community-contributions/Week1-Day2-Ollama-Exercise.ipynb

@ -0,0 +1,273 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "fad31e32-2e42-42ae-ae63-c15d90292839",
"metadata": {},
"source": [
"# First Project\n",
"Ollama -> Summary\n",
"huggingface_hub -> \"facebook/m2m100_418M\" for translation"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5fb79a20-a455-4d27-91a1-91958af786c1",
"metadata": {},
"outputs": [],
"source": [
"!pip install transformers datasets torch\n",
"!pip install huggingface_hub"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e95ac7f2-5192-4f83-acf3-61df30cd3109",
"metadata": {},
"outputs": [],
"source": [
"# imports\n",
"import requests\n",
"from bs4 import BeautifulSoup\n",
"import json\n",
"import ollama"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "12276d74-0e79-4e66-9135-1c9d1a80b943",
"metadata": {},
"outputs": [],
"source": [
"class Website:\n",
" def __init__(self, url):\n",
" self.url = url\n",
" response = requests.get(url)\n",
" soup = BeautifulSoup(response.content, 'html.parser')\n",
" self.title = soup.title.string if soup.title else \"No title found\"\n",
" for irrelevant in soup.body([\"script\", \"style\", \"img\", \"input\"]):\n",
" irrelevant.decompose()\n",
" self.text = soup.body.get_text(separator=\"\\n\", strip=True)\n",
"\n",
"huggingface_url = \"https://huggingface.co/learn/ml-for-3d-course\"\n",
"huggingface_website = Website(huggingface_url)\n",
"\n",
"huggingface_data = {\n",
" \"title\": huggingface_website.title,\n",
" \"text\": huggingface_website.text\n",
"}\n",
"print(huggingface_data)\n",
"\n",
"with open('ml_for_3d_course_data.json', 'w') as f:\n",
" json.dump(huggingface_data, f)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7d74c85c-3e09-4514-bde4-4cafc4910c52",
"metadata": {},
"outputs": [],
"source": [
"# huggingface_data 'text' value\n",
"huggingface_text = huggingface_data['text']\n",
"\n",
"# Summary\n",
"response_summary = ollama.chat(model=\"llama3.2:latest\", messages=[{\"role\": \"user\", \"content\": f\"Summarize the following text: {huggingface_text}\"}])\n",
"print(response_summary)\n",
"\n",
"# print summary\n",
"summary_huggingface_text = response_summary.message['content']\n",
"print(\"Summary Text:\", summary_huggingface_text)\n",
"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d13764d5-cb76-46c5-bbe6-d132b31a9ea6",
"metadata": {},
"outputs": [],
"source": [
"# HuggingFace Translation"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "08405038-4115-487f-9efc-de58572453c1",
"metadata": {},
"outputs": [],
"source": [
"class Website:\n",
" url: str\n",
" title: str\n",
" text: str\n",
"\n",
" def __init__(self, url):\n",
" self.url = url\n",
" response = requests.get(url)\n",
" soup = BeautifulSoup(response.content, 'html.parser')\n",
" self.title = soup.title.string if soup.title else \"No title found\"\n",
" for irrelevant in soup.body([\"script\", \"style\", \"img\", \"input\"]):\n",
" irrelevant.decompose()\n",
" self.text = soup.body.get_text(separator=\"\\n\", strip=True)\n",
"\n",
"url = \"https://huggingface.co/learn/ml-for-3d-course\"\n",
"website = Website(url)\n",
"print(website.title) \n",
"print(website.text[:1000])\n",
"\n",
"data = {\n",
" \"title\": website.title,\n",
" \"text\": website.text\n",
"}\n",
"\n",
"with open('ml_for_3d_course_data.json', 'w') as f:\n",
" json.dump(data, f)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0632352f-4b16-4125-83bf-f3cc3aabd659",
"metadata": {},
"outputs": [],
"source": [
"print(data)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a85f8625-725d-4d7f-8cb7-8da4276f81cf",
"metadata": {},
"outputs": [],
"source": [
"!pip install sacremoses"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c800cea4-f4a4-4e41-9637-31ff11afb256",
"metadata": {},
"outputs": [],
"source": [
"import json\n",
"from transformers import M2M100ForConditionalGeneration, M2M100Tokenizer\n",
"\n",
"# Load the M2M100 model and tokenizer\n",
"model_name = \"facebook/m2m100_418M\"\n",
"model = M2M100ForConditionalGeneration.from_pretrained(model_name)\n",
"tokenizer = M2M100Tokenizer.from_pretrained(model_name)\n",
"\n",
"# Load the saved JSON file\n",
"with open('ml_for_3d_course_data.json', 'r') as f:\n",
" data = json.load(f)\n",
"\n",
"# Extract text from the loaded data\n",
"text = data[\"text\"]\n",
"\n",
"# Set the source language to English and target language to Korean\n",
"source_lang = \"en\"\n",
"target_lang = \"ko\"\n",
"\n",
"# Set the language for tokenizer (important for M2M100)\n",
"tokenizer.src_lang = source_lang\n",
"tokenizer.tgt_lang = target_lang\n",
"\n",
"# Split text into smaller chunks if it's too large\n",
"# This step ensures we don't exceed the model's maximum length (512 tokens)\n",
"max_input_length = 512\n",
"chunks = [text[i:i+max_input_length] for i in range(0, len(text), max_input_length)]\n",
"\n",
"print(chunks)\n",
"# Initialize a list to hold the translated text\n",
"translated_chunks = []\n",
"\n",
"# Iterate through each chunk and translate it\n",
"for chunk in chunks:\n",
" # Tokenize the chunk\n",
" encoded = tokenizer(chunk, return_tensors=\"pt\", padding=True, truncation=True, max_length=512)\n",
"\n",
" # Generate translation from the model, forcing the output to be in Korean\n",
" generated_tokens = model.generate(**encoded, forced_bos_token_id=tokenizer.get_lang_id(target_lang), max_length=512)\n",
"\n",
" # Decode the translated tokens to text\n",
" translated_text = tokenizer.batch_decode(generated_tokens, skip_special_tokens=True)[0]\n",
" translated_chunks.append(translated_text)\n",
"\n",
"# Combine all translated chunks back together\n",
"final_translated_text = ' '.join(translated_chunks)\n",
"print(\"Translated Text:\", final_translated_text)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ffe0f264-a588-422f-a6e1-b60504d1e02c",
"metadata": {},
"outputs": [],
"source": [
"import json\n",
"import requests\n",
"\n",
"# Ollama API URL 설정\n",
"ollama_url = \"http://localhost:11411/v1/models/facebook/m2m100_418M/generate\"\n",
"\n",
"# 저장된 JSON 파일 로드\n",
"with open('ml_for_3d_course_data.json', 'r') as f:\n",
" data = json.load(f)\n",
"\n",
"# 텍스트 추출\n",
"course_text = data[\"text\"]\n",
"\n",
"# 번역할 소스 언어 및 타겟 언어 설정\n",
"source_language = \"en\"\n",
"target_language = \"ko\"\n",
"\n",
"# 데이터 준비\n",
"payload = {\n",
" \"input_text\": course_text,\n",
" \"src_lang\": source_language,\n",
" \"tgt_lang\": target_language\n",
"}\n",
"\n",
"# API 호출\n",
"response = requests.post(ollama_url, json=payload)\n",
"\n",
"# 응답 확인\n",
"if response.status_code == 200:\n",
" translated_course_text = response.json().get(\"translated_text\", \"Translation failed\")\n",
" print(\"Translated Course Text:\", translated_course_text)\n",
"else:\n",
" print(f\"Error {response.status_code}: {response.text}\")\n"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.11"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

279
week1/community-contributions/Week_1-Day 2-Article_Title_Generator.ipynb

@ -0,0 +1,279 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "603cd418-504a-4b4d-b1c3-be04febf3e79",
"metadata": {},
"source": [
"# Article Title Generator\n",
"\n",
"Summarization use-case in which the user provides an article, which the LLM will analyze to suggest an SEO-optimized title.\n",
"\n",
"**NOTES**:\n",
"\n",
"1. This version does NOT support website scrapping. You must copy and paste the required article.\n",
"2. The following models were configured:\n",
" a. OpenAI gpt-4o-mini\n",
" b. Llama llama3.2\n",
" c. Deepseek deepseek-r1:1.5b\n",
" It is possible to configure additional models by adding the new model to the MODELS dictionary and its\n",
" initialization to the CLIENTS dictionary. Then, call the model with --> ***answer =\n",
" get_answer('NEW_MODEL')***.\n",
"3. Users are encouraged to assess and rank the suggested titles using any headline analyzer tool online.\n",
" Example: https://www.isitwp.com/headline-analyzer/. "
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e773daa6-d05e-49bf-ad8e-a8ed4882b77e",
"metadata": {},
"outputs": [],
"source": [
"# Confirming Llama is loaded\n",
"!ollama pull llama3.2"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "279b0c00-9bb0-4c7f-9c6d-aa0b108274b9",
"metadata": {},
"outputs": [],
"source": [
"# imports\n",
"import os\n",
"from dotenv import load_dotenv\n",
"from IPython.display import Markdown, display\n",
"from openai import OpenAI"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d4730d8d-3e20-4f3c-a4ff-ed2ac0a8aa27",
"metadata": {},
"outputs": [],
"source": [
"# set environment variables for OpenAi\n",
"load_dotenv(override=True)\n",
"api_key = os.getenv('OPENAI_API_KEY')\n",
"\n",
"# validate API Key\n",
"if not api_key:\n",
" raise ValueError(\"No API key was found! Please check the .env file.\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1abbb826-de66-498c-94d8-33369ad01885",
"metadata": {},
"outputs": [],
"source": [
"# constants\n",
"MODELS = { 'GPT': 'gpt-4o-mini', \n",
" 'LLAMA': 'llama3.2', \n",
" 'DEEPSEEK': 'deepseek-r1:1.5b'\n",
" }\n",
"\n",
"CLIENTS = { 'GPT': OpenAI(), \n",
" 'LLAMA': OpenAI(base_url='http://localhost:11434/v1', api_key='ollama'),\n",
" 'DEEPSEEK': OpenAI(base_url='http://localhost:11434/v1', api_key='ollama') \n",
" }"
]
},
{
"cell_type": "markdown",
"id": "6f490fe4-32d5-41f3-890d-ecf4e5e01dd4",
"metadata": {},
"source": [
"### Copy & paste your article (without a title)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ddd76319-13ce-480b-baa7-cab6a5c88168",
"metadata": {},
"outputs": [],
"source": [
"# article - copy & paste your article\n",
"article = \"\"\"\n",
" REPLACE WITH YOUR ARTICLE CONTENT\n",
" \"\"\""
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1914afad-dbd8-4c1f-8e68-80b0e5d743a9",
"metadata": {},
"outputs": [],
"source": [
"# system prompt\n",
"system_prompt = \"\"\"\n",
" You are an experienced SEO-focused copywriter. The user will provide an article, and your task is to analyze its content and generate the most effective, keyword-optimized title to maximize SEO performance.Respond in Markdown format.\n",
" \"\"\""
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "176cfac7-5e6d-4d4a-a1c4-1b63b60de1f7",
"metadata": {},
"outputs": [],
"source": [
"# user prompt\n",
"user_prompt = f\"Following the article to be analyzed. Respond in Markdown format./n/n{article}\"\n",
" "
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c45fc7d7-08c9-4e34-b427-b928a219bb94",
"metadata": {},
"outputs": [],
"source": [
"# message list\n",
"messages = [\n",
" {\"role\": \"system\", \"content\": system_prompt},\n",
" {\"role\": \"user\", \"content\": user_prompt}\n",
" ]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f67b881f-1040-4cf7-82c5-e85f4c0bd252",
"metadata": {},
"outputs": [],
"source": [
"# call model and get answer\n",
"def get_answer(model):\n",
" # set required client\n",
" client = CLIENTS[model]\n",
"\n",
" # call model\n",
" response = client.chat.completions.create(\n",
" model=MODELS[model],\n",
" messages=messages\n",
" )\n",
" \n",
" # return answer\n",
" return response.choices[0].message.content\n",
" "
]
},
{
"cell_type": "markdown",
"id": "947b42ed-5b43-486d-8af3-e5b671c1fd0e",
"metadata": {},
"source": [
"### Get OpenAI Suggested Title"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "eb6f66e3-ab99-4f76-9358-896cb43c1fa1",
"metadata": {},
"outputs": [],
"source": [
"# get openAi answer\n",
"answer = get_answer('GPT')\n",
"\n",
"# display openAi answer\n",
"display(Markdown(f\"### {MODELS['GPT']} Answer\\n\\n{answer}\" ))"
]
},
{
"cell_type": "markdown",
"id": "70073ebf-a00a-416b-854d-642d450cd99b",
"metadata": {},
"source": [
"### Get Llama Suggested Title"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "caa190bb-de5f-45cc-b671-5d62688f7b25",
"metadata": {},
"outputs": [],
"source": [
"# get Llama answer\n",
"answer = get_answer('LLAMA')\n",
"\n",
"# display Llama answer\n",
"display(Markdown(f\"### {MODELS['LLAMA']} Answer\\n\\n{answer}\" ))"
]
},
{
"cell_type": "markdown",
"id": "811edc4f-20e2-482d-ac89-fae9d1b70bed",
"metadata": {},
"source": [
"### Get Deepseek Suggested Title"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "082628e4-ff4c-46dd-ae5f-76578eb017ad",
"metadata": {},
"outputs": [],
"source": [
"# get Deepseek answer\n",
"answer = get_answer('DEEPSEEK')\n",
"\n",
"# display Deepseek answer\n",
"display(Markdown(f\"### {MODELS['DEEPSEEK']} Answer\\n\\n{answer}\" ))"
]
},
{
"cell_type": "markdown",
"id": "7fc404a6-3a91-4c09-89de-867d3d69b4b2",
"metadata": {},
"source": [
"### Suggested future improvements\n",
"\n",
"1. Add website scrapping support to replace copy/pasting of articles.\n",
"2. Improve the system_prompt to provide specific SEO best practices to adopt during the title generation.\n",
"3. Rephrase the system_prompt to ensure the model provides a single Title (not a list of suggestions). \n",
"4. Add the logic that would allow each model to assess the recommendations from the different models and \n",
" select the best among these. "
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "cf7403ac-d43b-4493-98bb-6fee94950cb0",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.11"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

472
week1/community-contributions/Week_1-Day 5-Article_Title_Generator-V2.ipynb

@ -0,0 +1,472 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "603cd418-504a-4b4d-b1c3-be04febf3e79",
"metadata": {},
"source": [
"# Article Title Generator (V2)\n",
"\n",
"Summarization use-case in which the user provides an article, which the LLM will analyze to suggest an SEO-optimized title.\n",
"\n",
"**NOTES**:\n",
"\n",
"1. This version supports website scrapping using Selenium (based on the code from **/week1/community-\n",
" contributions/day1-webscraping-selenium-for-javascript.ipynb** - Thanks for the contribution!)\n",
"2. Leverage streaming (OpenAI only).\n",
"3. The following models were configured:\\\n",
" \n",
" a. OpenAI gpt-4o-mini\\\n",
" b. Llama llama3.2\\\n",
" c. Deepseek deepseek-r1:1.5b\\\n",
"\n",
" It is possible to configure additional models by adding the new model to the MODELS dictionary and its\n",
" initialization to the CLIENTS dictionary. Then, call the model with --> ***answer =\n",
" get_answer('NEW_MODEL')***.\n",
"5. Improved system_prompt to provide specific SEO best practices to adopt during the title generation.\n",
"6. Rephrased the system_prompt to ensure the model provides a single Title (not a list of suggestions).\n",
"7. Includes function to remove unrequired thinking/reasoning verbose from the model response (Deepseek). \n",
"8. Users are encouraged to assess and rank the suggested titles using any headline analyzer tool online.\n",
" Example: https://www.isitwp.com/headline-analyzer/. "
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "115004a8-747a-4954-9580-1ed548f80336",
"metadata": {},
"outputs": [],
"source": [
"# install required libraries if they were not part of the requirements.txt\n",
"!pip install selenium\n",
"!pip install undetected-chromedriver"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e773daa6-d05e-49bf-ad8e-a8ed4882b77e",
"metadata": {},
"outputs": [],
"source": [
"# confirming Llama is loaded\n",
"!ollama pull llama3.2"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "279b0c00-9bb0-4c7f-9c6d-aa0b108274b9",
"metadata": {},
"outputs": [],
"source": [
"# imports\n",
"import os\n",
"from dotenv import load_dotenv\n",
"from IPython.display import Markdown, display, update_display\n",
"from openai import OpenAI\n",
"import undetected_chromedriver as uc\n",
"from selenium.webdriver.common.by import By\n",
"from selenium.webdriver.support.ui import WebDriverWait\n",
"from selenium.webdriver.support import expected_conditions as EC\n",
"import time\n",
"from bs4 import BeautifulSoup"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d4730d8d-3e20-4f3c-a4ff-ed2ac0a8aa27",
"metadata": {},
"outputs": [],
"source": [
"# set environment variables for OpenAi\n",
"load_dotenv(override=True)\n",
"api_key = os.getenv('OPENAI_API_KEY')\n",
"\n",
"# validate API Key\n",
"if not api_key:\n",
" raise ValueError(\"No API key was found! Please check the .env file.\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1abbb826-de66-498c-94d8-33369ad01885",
"metadata": {},
"outputs": [],
"source": [
"# constants\n",
"MODELS = { 'GPT': 'gpt-4o-mini', \n",
" 'LLAMA': 'llama3.2', \n",
" 'DEEPSEEK': 'deepseek-r1:1.5b'\n",
" }\n",
"\n",
"CLIENTS = { 'GPT': OpenAI(), \n",
" 'LLAMA': OpenAI(base_url='http://localhost:11434/v1', api_key='ollama'),\n",
" 'DEEPSEEK': OpenAI(base_url='http://localhost:11434/v1', api_key='ollama') \n",
" }\n",
"\n",
"# path to Chrome\n",
"CHROME_PATH = \"C:/Program Files/Google/Chrome/Application/chrome.exe\""
]
},
{
"cell_type": "markdown",
"id": "6f490fe4-32d5-41f3-890d-ecf4e5e01dd4",
"metadata": {},
"source": [
"**Webcrawler** (based on the code from __/week1/community-contributions/day1-webscraping-selenium-for-javascript.ipynb__)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c2a1cf7a-044f-4a9c-b76e-8f112d384550",
"metadata": {},
"outputs": [],
"source": [
"class WebsiteCrawler:\n",
" def __init__(self, url, wait_time=20, chrome_path=None):\n",
" \"\"\"\n",
" Initialize the WebsiteCrawler using Selenium to scrape JavaScript-rendered content.\n",
" \"\"\"\n",
" self.url = url\n",
" self.wait_time = wait_time\n",
"\n",
" options = uc.ChromeOptions()\n",
" options.add_argument(\"--disable-gpu\")\n",
" options.add_argument(\"--no-sandbox\")\n",
" options.add_argument(\"--disable-dev-shm-usage\")\n",
" options.add_argument(\"--disable-blink-features=AutomationControlled\")\n",
" # options.add_argument(\"--headless=new\") # For Chrome >= 109 - unreliable on my end!\n",
" options.add_argument(\"start-maximized\")\n",
" options.add_argument(\n",
" \"user-agent=Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36\"\n",
" )\n",
" if chrome_path:\n",
" options.binary_location = chrome_path\n",
"\n",
" self.driver = uc.Chrome(options=options)\n",
"\n",
" try:\n",
" # Load the URL\n",
" self.driver.get(url)\n",
"\n",
" # Wait for Cloudflare or similar checks\n",
" time.sleep(10)\n",
"\n",
" # Ensure the main content is loaded\n",
" WebDriverWait(self.driver, self.wait_time).until(\n",
" EC.presence_of_element_located((By.TAG_NAME, \"main\"))\n",
" )\n",
"\n",
" # Extract the main content\n",
" main_content = self.driver.find_element(By.CSS_SELECTOR, \"main\").get_attribute(\"outerHTML\")\n",
"\n",
" # Parse with BeautifulSoup\n",
" soup = BeautifulSoup(main_content, \"html.parser\")\n",
" self.title = self.driver.title if self.driver.title else \"No title found\"\n",
" self.text = soup.get_text(separator=\"\\n\", strip=True)\n",
"\n",
" except Exception as e:\n",
" print(f\"Error occurred: {e}\")\n",
" self.title = \"Error occurred\"\n",
" self.text = \"\"\n",
"\n",
" finally:\n",
" self.driver.quit()\n"
]
},
{
"cell_type": "markdown",
"id": "592d8f86-fbf7-4b16-a69d-468030d72dc4",
"metadata": {},
"source": [
"### Prompts"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1914afad-dbd8-4c1f-8e68-80b0e5d743a9",
"metadata": {},
"outputs": [],
"source": [
"# system prompt\n",
"system_prompt = \"\"\"\n",
" You are an experienced SEO-focused copywriter. The user will provide an article, and your task is to analyze its content and generate a single, most effective, keyword-optimized title to maximize SEO performance.\n",
"\n",
"Instructions:\n",
"Ignore irrelevant content, such as the current title (if any), navigation menus, advertisements, or unrelated text.\n",
"Prioritize SEO best practices, considering:\n",
"Keyword relevance and search intent (informational, transactional, etc.).\n",
"Readability and engagement.\n",
"Avoiding keyword stuffing.\n",
"Ensure conciseness and clarity, keeping the title under 60 characters when possible for optimal SERP display.\n",
"Use a compelling structure that balances informativeness and engagement, leveraging formats like:\n",
"Listicles (\"10 Best Strategies for…\")\n",
"How-to guides (\"How to Boost…\")\n",
"Questions (\"What Is the Best Way to…\")\n",
"Power words to enhance click-through rates (e.g., \"Proven,\" \"Ultimate,\" \"Essential\").\n",
"Provide only one single, best title—do not suggest multiple options.\n",
"Limit the answer to the following Response Format (Markdown):\n",
"Optimized Title: [Provide only one title here]\n",
"Justification: [Explain why this title is effective for SEO]\n",
"\n",
" \"\"\""
]
},
{
"cell_type": "markdown",
"id": "b0486867-6d38-4cb5-91d4-fb60952c3a9b",
"metadata": {},
"source": [
"**Provide the article URL and get its content for analysis**"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ddd76319-13ce-480b-baa7-cab6a5c88168",
"metadata": {},
"outputs": [],
"source": [
"# article url - change to any other article URL\n",
"article_url = \"https://searchengineland.com/seo-trends-2025-447745\"\n",
"\n",
"# get article content\n",
"article = WebsiteCrawler(url=article_url, chrome_path=CHROME_PATH)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "176cfac7-5e6d-4d4a-a1c4-1b63b60de1f7",
"metadata": {},
"outputs": [],
"source": [
"# user prompt\n",
"user_prompt = \"\"\"\n",
"Following the article to be analyzed to suggest a title. Limit the answer to the following Response Format (Markdown): \n",
"Optimized Title: [Provide only one title here]\n",
"Justification: [Explain why this title is effective for SEO].\n",
"\"\"\"\n",
"\n",
"user_prompt = f\"{user_prompt} {article}\"\n",
" "
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c45fc7d7-08c9-4e34-b427-b928a219bb94",
"metadata": {},
"outputs": [],
"source": [
"# message list\n",
"messages = [\n",
" {\"role\": \"system\", \"content\": system_prompt},\n",
" {\"role\": \"user\", \"content\": user_prompt}\n",
" ]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f67b881f-1040-4cf7-82c5-e85f4c0bd252",
"metadata": {},
"outputs": [],
"source": [
"# get suggested title\n",
"def get_title(model, **kwargs):\n",
" # stream if GPT\n",
" if 'stream' in kwargs:\n",
" response = CLIENTS[model].chat.completions.create(\n",
" model=MODELS[model],\n",
" messages=messages,\n",
" stream=kwargs['stream']\n",
" )\n",
" else:\n",
" response = CLIENTS[model].chat.completions.create(\n",
" model=MODELS[model],\n",
" messages=messages,\n",
" )\n",
"\n",
" return response\n",
" "
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8988d6ff-076a-4eae-baf4-26a8d6a2bc44",
"metadata": {},
"outputs": [],
"source": [
"# filter response from model verbose - like Deepseek reasoning/thinking verbose\n",
"def filter_response(response):\n",
" # Find last occurrence of 'Optimized Title:' to avoid displaying reasoning verbose\n",
" substring = 'Optimized Title:'\n",
" start = response.rfind('Optimized Title:')\n",
" if start > -1:\n",
" filtered_response = response[start:]\n",
"\n",
" # insert line break to preserve format\n",
" filtered_response = filtered_response.replace(\"**Justification:**\", \"\\n**Justification:**\")\n",
" \n",
" return filtered_response"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0e9e99cf-5e25-4a1f-ab11-a2255e318671",
"metadata": {},
"outputs": [],
"source": [
"# display suggested title\n",
"def display_title(model):\n",
" # get model-suggested title\n",
" title = get_title(model)\n",
" \n",
" display(Markdown(f\"### {model} (___{MODELS[model]}___) Answer\\n\\n_______\")) \n",
"\n",
" response = \"\"\n",
"\n",
" if model == 'GPT':\n",
" display_handle = display(Markdown(\"\"), display_id=True)\n",
" # for chunk in stream:\n",
" for chunk in get_title(model=model, stream=True):\n",
" response += chunk.choices[0].delta.content or ''\n",
" response = (\n",
" response.replace(\"```\",\"\")\n",
" .replace(\"markdown\", \"\")\n",
" .replace(\"Optimized Title:\", \"**Optimized Title:**\")\n",
" .replace(\"Justification:\", \"**Justification:**\")\n",
" )\n",
" update_display(Markdown(response), display_id=display_handle.display_id)\n",
" else:\n",
" response = get_title(model=model)\n",
" response = response.choices[0].message.content\n",
" response = filter_response(response)\n",
" response = (\n",
" response.replace(\"Optimized Title:\", \"**Optimized Title:**\")\n",
" .replace(\"Justification:\", \"**Justification:**\")\n",
" )\n",
" display(Markdown(response))"
]
},
{
"cell_type": "markdown",
"id": "947b42ed-5b43-486d-8af3-e5b671c1fd0e",
"metadata": {},
"source": [
"### Get OpenAI Suggested Title"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "eb6f66e3-ab99-4f76-9358-896cb43c1fa1",
"metadata": {},
"outputs": [],
"source": [
"# get and display openAi suggested title\n",
"display_title(model='GPT')"
]
},
{
"cell_type": "markdown",
"id": "70073ebf-a00a-416b-854d-642d450cd99b",
"metadata": {},
"source": [
"### Get Llama Suggested Title"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "caa190bb-de5f-45cc-b671-5d62688f7b25",
"metadata": {},
"outputs": [],
"source": [
"# get and display Llama suggested title\n",
"display_title(model='LLAMA')"
]
},
{
"cell_type": "markdown",
"id": "811edc4f-20e2-482d-ac89-fae9d1b70bed",
"metadata": {},
"source": [
"### Get Deepseek Suggested Title"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "082628e4-ff4c-46dd-ae5f-76578eb017ad",
"metadata": {},
"outputs": [],
"source": [
"# get and display Deepseek title\n",
"display_title(model='DEEPSEEK')"
]
},
{
"cell_type": "markdown",
"id": "7fc404a6-3a91-4c09-89de-867d3d69b4b2",
"metadata": {
"jp-MarkdownHeadingCollapsed": true
},
"source": [
"### Observations\n",
"\n",
"1. **Selenium:** The headless option (__options.add_argument(\"--headless=new\")__), while ideal to speed up the scanning process, presented problems while scanning several websites (including openai.com and canva.com).\n",
"2. **Deepseek challenges:**\\\n",
" a.It always returns its thinking/reasoning verbose, which, while helpful to understand how it works, is not always\n",
" required, such as in this example code. A new function (**filter_response**) was created to remove the additional verbose.\\\n",
" b. It is unreliable with the response, sometimes returning the required format for the response instead of the\n",
" actual response. For example, for the title, it may sometimes return:\n",
" \n",
" **Optimized Title:** \\[The user wants the suggested title here]\n",
" \n",
"### Suggested future improvements\n",
"\n",
"1. Add the logic that would allow each model to assess the recommendations from the different models and \n",
" select the best among these.\n",
"2. Add the logic to leverage an API (if available) that automatically assesses the suggested titles."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1af8260b-5ba1-4eeb-acd0-02de537b1bf4",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.11"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

4
week1/community-contributions/day-1-ollama-app.ipynb

@ -234,7 +234,7 @@
],
"metadata": {
"kernelspec": {
"display_name": "llms",
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
@ -252,5 +252,5 @@
}
},
"nbformat": 4,
"nbformat_minor": 2
"nbformat_minor": 4
}

229
week1/community-contributions/website-summarizer-by-tithi.ipynb

@ -0,0 +1,229 @@
{
"cells": [
{
"cell_type": "code",
"execution_count": 10,
"id": "29ddd15d-a3c5-4f4e-a678-873f56162724",
"metadata": {},
"outputs": [],
"source": [
"import requests\n",
"from bs4 import BeautifulSoup\n",
"from IPython.display import Markdown, display\n",
"import ollama"
]
},
{
"cell_type": "code",
"execution_count": 11,
"id": "479ff514-e8bd-4985-a572-2ea28bb4fa40",
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"\u001b[?2026h\u001b[?25l\u001b[1Gpulling manifest â ‹ \u001b[K\u001b[?25h\u001b[?2026l\u001b[?2026h\u001b[?25l\u001b[1Gpulling manifest â ™ \u001b[K\u001b[?25h\u001b[?2026l\u001b[?2026h\u001b[?25l\u001b[1Gpulling manifest â ¹ \u001b[K\u001b[?25h\u001b[?2026l\u001b[?2026h\u001b[?25l\u001b[1Gpulling manifest â ¸ \u001b[K\u001b[?25h\u001b[?2026l\u001b[?2026h\u001b[?25l\u001b[1Gpulling manifest â ¼ \u001b[K\u001b[?25h\u001b[?2026l\u001b[?2026h\u001b[?25l\u001b[1Gpulling manifest â ´ \u001b[K\u001b[?25h\u001b[?2026l\u001b[?2026h\u001b[?25l\u001b[1Gpulling manifest â ¦ \u001b[K\u001b[?25h\u001b[?2026l\u001b[?2026h\u001b[?25l\u001b[1Gpulling manifest â § \u001b[K\u001b[?25h\u001b[?2026l\u001b[?2026h\u001b[?25l\u001b[1Gpulling manifest â ‡ \u001b[K\u001b[?25h\u001b[?2026l\u001b[?2026h\u001b[?25l\u001b[1Gpulling manifest â <EFBFBD> \u001b[K\u001b[?25h\u001b[?2026l\u001b[?2026h\u001b[?25l\u001b[1Gpulling manifest \u001b[K\n",
"pulling 2bada8a74506... 100% ▕████████████████â–<EFBFBD> 4.7 GB \u001b[K\n",
"pulling 66b9ea09bd5b... 100% ▕████████████████â–<EFBFBD> 68 B \u001b[K\n",
"pulling eb4402837c78... 100% ▕████████████████â–<EFBFBD> 1.5 KB \u001b[K\n",
"pulling 832dd9e00a68... 100% ▕████████████████â–<EFBFBD> 11 KB \u001b[K\n",
"pulling 2f15b3218f05... 100% ▕████████████████â–<EFBFBD> 487 B \u001b[K\n",
"verifying sha256 digest \u001b[K\n",
"writing manifest \u001b[K\n",
"success \u001b[K\u001b[?25h\u001b[?2026l\n"
]
}
],
"source": [
"# Let's just make sure the model is loaded\n",
"\n",
"!ollama pull qwen2.5\n",
"MODEL = \"qwen2.5\""
]
},
{
"cell_type": "code",
"execution_count": 12,
"id": "6de38216-6d1c-48c4-877b-86d403f4e0f8",
"metadata": {},
"outputs": [],
"source": [
"headers = {\n",
" \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/117.0.0.0 Safari/537.36\"\n",
"}\n",
"\n",
"class Website:\n",
"\n",
" def __init__(self, url):\n",
" \"\"\"\n",
" Create this Website object from the given url using the BeautifulSoup library\n",
" \"\"\"\n",
" self.url = url\n",
" response = requests.get(url, headers=headers)\n",
" soup = BeautifulSoup(response.content, 'html.parser')\n",
" self.title = soup.title.string if soup.title else \"No title found\"\n",
" for irrelevant in soup.body([\"script\", \"style\", \"img\", \"input\"]):\n",
" irrelevant.decompose()\n",
" self.text = soup.body.get_text(separator=\"\\n\", strip=True)"
]
},
{
"cell_type": "code",
"execution_count": 13,
"id": "a531b8f6-d4f8-4140-b54d-bcf280bd7a99",
"metadata": {},
"outputs": [],
"source": [
"system_prompt = \"You are an assistant that analyzes the contents of a website \\\n",
"and provides a short summary, ignoring text that might be navigation related. \\\n",
"Respond in markdown.\""
]
},
{
"cell_type": "code",
"execution_count": 14,
"id": "6b46ff43-4817-431e-8335-8d2cc9957910",
"metadata": {},
"outputs": [],
"source": [
"def user_prompt_for(website):\n",
" user_prompt = f\"You are looking at a website titled {website.title}\"\n",
" user_prompt += \"\\nThe contents of this website is as follows; \\\n",
"please provide a summary of this website in markdown. \\\n",
"If it includes news or announcements, then summarize these too.(only if they are present)\\n\\n\"\n",
" user_prompt += website.text\n",
" return user_prompt"
]
},
{
"cell_type": "code",
"execution_count": 15,
"id": "13a3a001-5d91-4269-ab60-493bbf35bda4",
"metadata": {},
"outputs": [],
"source": [
"def messages_for(website):\n",
" return [\n",
" {\"role\": \"system\", \"content\": system_prompt},\n",
" {\"role\": \"user\", \"content\": user_prompt_for(website)}\n",
" ]"
]
},
{
"cell_type": "code",
"execution_count": 16,
"id": "c61ad738-9395-415d-b88b-d4a70d4331aa",
"metadata": {},
"outputs": [],
"source": [
"def summarize(url):\n",
" website = Website(url)\n",
" response = ollama.chat(model=MODEL, messages=messages_for(website))\n",
" return response['message']['content']"
]
},
{
"cell_type": "code",
"execution_count": 17,
"id": "bdbcfa75-980b-4542-872d-af8b20546b5d",
"metadata": {},
"outputs": [
{
"data": {
"text/plain": [
"'```markdown\\n# Tailwind CSS Cheat Sheet Summary\\n\\nThis website serves as a comprehensive guide for developers using Tailwind CSS, providing quick access to commonly used utility classes and configurations. The content is organized into sections such as typography, layout, colors, shadows, and more, making it easy for users to find specific styles or settings.\\n\\n- **Typography**: Includes various font sizes, weights, line heights, and other typographic utilities.\\n- **Layout**: Features columns, grid, flexbox, spacing, and responsive design utilities.\\n- **Colors**: Lists predefined color palettes and utility classes for color manipulation.\\n- **Shadows**: Provides options to add depth and dimension to elements through shadow effects.\\n- **Other Sections**: Covers forms, animations, and more, with concise descriptions and examples.\\n\\nThe site is designed to be a one-stop reference tool, allowing developers to quickly apply Tailwind CSS styles without having to consult the official documentation every time.\\n```'"
]
},
"execution_count": 17,
"metadata": {},
"output_type": "execute_result"
}
],
"source": [
"summarize(\"https://www.creative-tim.com/twcomponents/cheatsheet/\")"
]
},
{
"cell_type": "code",
"execution_count": 18,
"id": "817e6f73-1abe-4f79-9010-f4264e0f324a",
"metadata": {},
"outputs": [],
"source": [
"def display_summary(url):\n",
" summary = summarize(url)\n",
" display(Markdown(summary))"
]
},
{
"cell_type": "code",
"execution_count": 19,
"id": "504c19cf-9add-4a78-a028-fe2710e0604d",
"metadata": {},
"outputs": [
{
"data": {
"text/markdown": [
"# Summary\n",
"\n",
"**Home Page:**\n",
"- The website is titled \"Home - Edward Donner\" and introduces Ed, who enjoys coding, experimenting with large language models (LLMs), DJing, and engaging in Hacker News.\n",
"- He co-founded Nebula.io, an AI company focusing on helping people discover their potential. The platform uses proprietary LLMs for talent discovery and has been patented.\n",
"\n",
"**News/Announcements:**\n",
"- **January 23, 2025:** LLM Workshop – Hands-on with Agents\n",
"- **December 21, 2024:** Welcome, SuperDataScientists!\n",
"- **November 13, 2024:** Mastering AI and LLM Engineering – Resources\n",
"- **October 16, 2024:** From Software Engineer to AI Data Scientist – resources\n",
"\n",
"**Connect Section:**\n",
"- Provides ways to get in touch with Ed, including email, LinkedIn, Twitter, Facebook, and a newsletter subscription form.\n",
"\n",
"**Additional Content:**\n",
"- **Connect Four:** Describes it as an arena where LLMs compete against each other.\n",
"- **About Page:** Further details about Ed's background and Nebula.io."
],
"text/plain": [
"<IPython.core.display.Markdown object>"
]
},
"metadata": {},
"output_type": "display_data"
}
],
"source": [
"display_summary('https://edwarddonner.com')"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "20d621cb-6bfb-41a6-bd98-a51ef0a8b158",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.11"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

180
week1/community-contributions/week1_Ollama_generate_streams.ipynb

@ -0,0 +1,180 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "fe12c203-e6a6-452c-a655-afb8a03a4ff5",
"metadata": {},
"source": [
"# End of week 1 exercise\n",
"\n",
"To demonstrate your familiarity with OpenAI API, and also Ollama, build a tool that takes a technical question, \n",
"and responds with an explanation. This is a tool that you will be able to use yourself during the course!"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c1070317-3ed9-4659-abe3-828943230e03",
"metadata": {},
"outputs": [],
"source": [
"# imports\n",
"import os\n",
"import requests\n",
"import json\n",
"from typing import List\n",
"from dotenv import load_dotenv\n",
"from bs4 import BeautifulSoup\n",
"from IPython.display import Markdown, display, update_display\n",
"from openai import OpenAI\n",
"import ollama"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "4a456906-915a-4bfd-bb9d-57e505c5093f",
"metadata": {},
"outputs": [],
"source": [
"# constants\n",
"MODEL_GPT = 'gpt-4o-mini'\n",
"MODEL_LLAMA = 'llama3.2'"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a8d7923c-5f28-4c30-8556-342d7c8497c1",
"metadata": {},
"outputs": [],
"source": [
"# set up environment\n",
"load_dotenv(override=True)\n",
"api_key = os.getenv('OPENAI_API_KEY')\n",
"\n",
"if api_key and api_key.startswith('sk-proj-') and len(api_key)>10:\n",
" print(\"API key looks good so far\")\n",
"else:\n",
" print(\"There might be a problem with your API key? Please visit the troubleshooting notebook!\")\n",
"\n",
"openai = OpenAI()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3f0d0137-52b0-47a8-81a8-11a90a010798",
"metadata": {},
"outputs": [],
"source": [
"system_prompt = \"You are provided with a technical question. \\\n",
"You are answering by providing a quick explanation and giving some examples.\\n\"\n",
"\n",
"# here is the question; type over this to ask something new\n",
"question = \"\"\"\n",
"Please explain what this code does and why:\n",
"yield from {book.get(\"author\") for book in books if book.get(\"author\")}\n",
"\"\"\""
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "60ce7000-a4a5-4cce-a261-e75ef45063b4",
"metadata": {},
"outputs": [],
"source": [
"# Get gpt-4o-mini to answer, with streaming\n",
"def get_answer_gpt():\n",
" stream = openai.chat.completions.create(\n",
" model=MODEL_GPT,\n",
" messages=[\n",
" {\"role\": \"system\", \"content\": system_prompt},\n",
" {\"role\": \"user\", \"content\": question}\n",
" ],\n",
" stream=True\n",
" )\n",
"\n",
" response = \"\"\n",
" display_handle = display(Markdown(\"\"), display_id=True)\n",
" for chunk in stream:\n",
" response += chunk.choices[0].delta.content or ''\n",
" response = response.replace(\"```\",\"\").replace(\"markdown\", \"\")\n",
" update_display(Markdown(response), display_id=display_handle.display_id)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8f7c8ea8-4082-4ad0-8751-3301adcf6538",
"metadata": {},
"outputs": [],
"source": [
"# Get Llama 3.2 to answer\n",
"def get_answer_ollama():\n",
" stream = ollama.generate(\n",
" MODEL_LLAMA,\n",
" question,\n",
" stream=True\n",
" )\n",
" \n",
" response = \"\"\n",
" display_handle = display(Markdown(\"\"), display_id=True)\n",
" for chunk in stream:\n",
" response += chunk['response'] or ''\n",
" response = response.replace(\"```\",\"\").replace(\"markdown\", \"\")\n",
" update_display(Markdown(response), display_id=display_handle.display_id)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "4a859eb1-23fa-40dd-ba91-b35084433a00",
"metadata": {},
"outputs": [],
"source": [
"get_answer_gpt()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1c73f046-da3a-49a5-8a74-4b8a86a9032a",
"metadata": {},
"outputs": [],
"source": [
"get_answer_ollama()"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "bea20f33-a710-44ab-9a4d-856db05e4201",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.11"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

129
week2/community-contributions/week2_day2_gradio/gradio_ui.py

@ -0,0 +1,129 @@
import gradio as gr
import requests
import json
from json_handlers import SettingsHandler, LanguagesHandler
from ollama_utils import get_ollama_response
class GradioUI:
def __init__(self, models: list, settings: SettingsHandler, languages: LanguagesHandler):
self.models = models
self.settings = settings
self.languages = languages
self.langs = self.languages.get_supported_languages()
def _translate_callback(self, text, model, translte_from, translte_to):
model_options = self.settings.get_advanced_settings()
full_response = ""
chunck_response = get_ollama_response(model, text, translte_from, translte_to, model_options)
for chunck in chunck_response:
full_response += chunck
yield full_response
def _temp_setting_callback(self, temp_dropdown_val):
self.settings.update_advanced_settings_param("temperature", temp_dropdown_val)
def _top_k_setting_callback(self, top_k_dropdown_val):
self.settings.update_advanced_settings_param("top_k", top_k_dropdown_val)
def _top_p_setting_callback(self, top_p_dropdown_val):
self.settings.update_advanced_settings_param("top_p", top_p_dropdown_val)
def _reset_to_default_callback(self):
temperature = 0.0
top_k = 40.0
top_p = 0.9
default_settings = {
"temperature": temperature,
"top_k": top_k,
"top_p": top_p
}
self.settings.update_advanced_settings(default_settings)
return temperature, top_k, top_p
def build_and_launch(self):
with gr.Blocks() as gui:
gr.Markdown("# LLM Translator")
with gr.Tab("Translate"):
with gr.Row():
model_dropdown = gr.Dropdown(
label="Model",
info="Choose LLM Model",
choices=self.models
)
with gr.Group():
with gr.Row():
translte_from = gr.Dropdown(
value=self.langs[0],
show_label=False,
choices=self.langs,
interactive=True
)
translte_to = gr.Dropdown(
value=self.langs[1],
show_label=False,
choices=self.langs,
interactive=True
)
with gr.Row():
translate_input = gr.Textbox(label="Your Input", lines=15, max_lines=15)
translate_output = gr.Textbox(label="Translated", lines=15, max_lines=15)
btn = gr.Button("Translate", variant="primary")
btn.click(
fn=self._translate_callback,
inputs=[translate_input, model_dropdown, translte_from, translte_to],
outputs=translate_output
)
with gr.Tab("Advanced Settings"):
temp_dropdown = gr.Number(
value=self.settings.get_advanced_setting_param("temperature"),
label="Temperature",
info="This parameter control how creative the model is\n0 means no creativity\n1 means very creative",
minimum=0,
maximum=1,
step=0.1,
interactive=True
)
gr.Markdown() # Used only for spacing
top_k_dropdown = gr.Number(
value=self.settings.get_advanced_setting_param("top_k"),
label="Top K",
info="A higher value (e.g. 100) will give more diverse answers\nwhile a lower value (e.g. 10) will be more conservative.",
minimum=1,
maximum=200,
step=1,
interactive=True
)
gr.Markdown() # Used only for spacing
top_p_dropdown = gr.Number(
value=self.settings.get_advanced_setting_param("top_p"),
label="Top P",
info="A higher value (e.g., 0.95) will lead to more diverse answers\nwhile a lower value (e.g., 0.5) will be more conservative",
minimum=0.1,
maximum=1.0,
step=0.1,
interactive=True
)
gr.Markdown() # Used only for spacing
reset_btn = gr.Button("Reset to Default")
reset_btn.click(
fn=self._reset_to_default_callback,
outputs=[temp_dropdown, top_k_dropdown, top_p_dropdown]
)
temp_dropdown.change(self._temp_setting_callback, temp_dropdown)
top_k_dropdown.change(self._top_k_setting_callback, top_k_dropdown)
top_p_dropdown.change(self._top_p_setting_callback, top_p_dropdown)
gui.launch()

60
week2/community-contributions/week2_day2_gradio/json_handlers.py

@ -0,0 +1,60 @@
import json
class SettingsHandler:
def __init__(self, json_filename):
self.json_filename = json_filename
self.advanced_settings = self.load_current_settings()
def load_current_settings(self) -> dict:
with open(self.json_filename, "r") as file:
settings_dict = json.load(file)
advanced_settings = settings_dict["Advanced Settings"]
return advanced_settings
def update_advanced_settings(self, updated_advanced_settings: dict):
new_dict = {
"Advanced Settings": updated_advanced_settings
}
print(new_dict)
with open(self.json_filename, "w") as file:
json.dump(new_dict, file)
self.advanced_settings = updated_advanced_settings
def update_advanced_settings_param(self, key: str, new_val):
if self.get_advanced_setting_param(key) is not None:
update_advanced_settings_dict = self.advanced_settings
update_advanced_settings_dict[key] = new_val
self.update_advanced_settings(update_advanced_settings_dict)
def get_advanced_settings(self):
return self.advanced_settings
def get_advanced_setting_param(self, key: str):
return self.advanced_settings.get(key)
class LanguagesHandler:
def __init__(self, json_filename):
self.json_filename = json_filename
self.langs = self.load_languages()
def load_languages(self) -> list:
with open(self.json_filename, "r") as file:
langs = json.load(file)
if type(langs) != list:
raise RuntimeError("Languages must be provided as lists")
if len(langs) < 2:
raise RuntimeError("At least 2 languages must be supported")
return langs
def get_supported_languages(self):
return self.langs

6
week2/community-contributions/week2_day2_gradio/languages.json

@ -0,0 +1,6 @@
[
"German",
"English",
"Spanish",
"French"
]

15
week2/community-contributions/week2_day2_gradio/main.py

@ -0,0 +1,15 @@
from json_handlers import SettingsHandler, LanguagesHandler
from ollama_utils import get_downloaded_models
from gradio_ui import GradioUI
settings_json = "settings.json"
languages_json = "languages.json"
if __name__ == "__main__":
settings = SettingsHandler(settings_json)
languages = LanguagesHandler(languages_json)
models = get_downloaded_models()
gradio_ui = GradioUI(models, settings, languages)
gradio_ui.build_and_launch()

28
week2/community-contributions/week2_day2_gradio/ollama_utils.py

@ -0,0 +1,28 @@
import requests
import json
import ollama
def get_downloaded_models():
models_raw = requests.get("http://localhost:11434/api/tags").content
models_dict = json.loads(models_raw)
models = [model["name"] for model in models_dict["models"]]
return models
def get_ollama_response(model, prompt, translte_from, translte_to, options):
def get_system_prompt():
with open('system_prompt.txt', 'r') as file:
system_prompt = file.read()
return system_prompt
system_prompt = get_system_prompt()
user_prompt = f"Translate from {translte_from} to {translte_to}: {prompt}"
messages = [
{"role": "system", "content": system_prompt},
{"role": "user", "content": user_prompt}
]
response = ollama.chat(model, messages, options=options, stream=True)
for chunck in response:
yield chunck["message"]["content"]

1
week2/community-contributions/week2_day2_gradio/readme.txt

@ -0,0 +1 @@
Just run the main.py script after activating conda environment 'llms'

1
week2/community-contributions/week2_day2_gradio/settings.json

@ -0,0 +1 @@
{"Advanced Settings": {"temperature": 0.0, "top_k": 40.0, "top_p": 0.9}}

17
week2/community-contributions/week2_day2_gradio/system_prompt.txt

@ -0,0 +1,17 @@
You are a translator.
You should translate the prompts according to the following criteria:
- You should respond in a clear and straight to the point responses.
- Your response should have a good structure and good linguistic features.
- You should translate the sentence as it is. Do not add extra sentences or phrases on your own.
- Do not answer questions even if the prompt is a question, you should translate the question and do not anwer it.
- If you do not understand the prompt, do not say that you do not understand, just echo the prompt.
- Do not include in the response phrases like 'here is the translation' or any phrases like that
Here are some examples for good responses:
<
Prompt: 'Translate from French to English: Hier, j'ai passé toute la journée à explorer la ville avec mes amis, et nous avons visité plusieurs musées avant de nous arrêter pour un délicieux dîner dans un restaurant local.'
Response: 'Yesterday, I spent the whole day exploring the city with my friends, and we visited several museums before stopping for a delicious dinner at a local restaurant.'
>
<
Prompt: 'Translate from Spanish to English: vdaiughadvlkj'
Response: 'vdaiughadvlkj'
>

2
week2/day1.ipynb

@ -485,7 +485,7 @@
"\n",
"print(reasoning_content)\n",
"print(content)\n",
"print(\"Number of words:\", len(reply.split(\" \")))"
"print(\"Number of words:\", len(content.split(\" \")))"
]
},
{

186
week3/community-contributions/day5_qwen2_whisper.ipynb

@ -0,0 +1,186 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "6fb7858c-8ea7-4dea-95ea-f5d7d5210b9a",
"metadata": {},
"source": [
"The following is **Meeting minutes Generator** by using **QWEN2** and **Openai Opensource model whisper for transcription**, check the following colab link to see the outputs\n",
"\n",
"https://colab.research.google.com/drive/1_pqFmQXjOYG9Se4Zov4blIGeoYX6ViTJ?usp=sharing\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "2103adb0-51f3-4240-bc5d-e27b6103cd8a",
"metadata": {},
"outputs": [],
"source": [
"import torch\n",
"from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor, pipeline\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "47dba08d-5829-417c-9c6c-bdb35ca846a6",
"metadata": {},
"outputs": [],
"source": [
"AUDIO_MODEL = \"openai/whisper-medium\"\n",
"speech_model = AutoModelForSpeechSeq2Seq.from_pretrained(AUDIO_MODEL, torch_dtype=torch.float16, low_cpu_mem_usage=True, use_safetensors=True)\n",
"speech_model.to('cuda')\n",
"processor = AutoProcessor.from_pretrained(AUDIO_MODEL)\n",
"\n",
"pipe = pipeline(\n",
" \"automatic-speech-recognition\",\n",
" model=speech_model,\n",
" tokenizer=processor.tokenizer,\n",
" feature_extractor=processor.feature_extractor,\n",
" torch_dtype=torch.float16,\n",
" device='cuda',\n",
" return_timestamps=True #important if audio is more than 30sec\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c35d6c76-01a9-495f-ad4e-84c98e320750",
"metadata": {},
"outputs": [],
"source": [
"result = pipe(\"your-audio.mp3\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8fba2d46-b806-4bb3-b02d-e628343db986",
"metadata": {},
"outputs": [],
"source": [
"transcription = result[\"text\"]\n",
"print(transcription)"
]
},
{
"cell_type": "markdown",
"id": "1778c4db-d003-4fb9-a0d0-6cfa71e6208d",
"metadata": {},
"source": [
"## MODEL"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "9eb579a7-b5de-4537-8ad9-e3117b24c2ff",
"metadata": {},
"outputs": [],
"source": [
"from transformers import AutoTokenizer, AutoModelForCausalLM, TextStreamer, BitsAndBytesConfig"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "4c632023-9b37-4c0d-b43a-190aacbbd80d",
"metadata": {},
"outputs": [],
"source": [
"QWEN2 = \"Qwen/Qwen2-7B-Instruct\""
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "175814b9-81b2-4f75-bf40-9ef7cac492cd",
"metadata": {},
"outputs": [],
"source": [
"quant_config = BitsAndBytesConfig(\n",
" load_in_4bit=True,\n",
" bnb_4bit_use_double_quant=True,\n",
" bnb_4bit_compute_dtype=torch.bfloat16,\n",
" bnb_4bit_quant_type=\"nf4\"\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8aaa160e-7c2b-4080-b24a-995df4469edd",
"metadata": {},
"outputs": [],
"source": [
"tokenizer = AutoTokenizer.from_pretrained(QWEN2)\n",
"#tokenizer.pad_token = tokenizer.oes_token\n",
"inputs = tokenizer.apply_chat_template(messages, return_tensors=\"pt\", add_generation_ptrompt=True).to(\"cuda\")\n",
"streamer = TextStreamer(tokenizer)\n",
"model = AutoModelForCausalLM.from_pretrained(QWEN2 , device_map=\"auto\", quantization_config=quant_config)\n",
"outputs = model.generate(inputs, max_new_tokens=2000, streamer=streamer)\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "517443aa-d230-4248-88aa-b06efd8ee3cd",
"metadata": {},
"outputs": [],
"source": [
"response = tokenizer.decode(outputs[0])"
]
},
{
"cell_type": "markdown",
"id": "47562f76-fd35-4eb0-a399-8e8f1fa054c3",
"metadata": {},
"source": [
"## **For Markdown display**"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1f77fea1-0920-46e5-9230-d0e8b9f69353",
"metadata": {},
"outputs": [],
"source": [
"from IPython.display import Markdown, display, update_display"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "35ac81e2-f960-4705-aaca-2385d8aa12d6",
"metadata": {},
"outputs": [],
"source": [
"display(Markdown(response))"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.13.2"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

394
week5/community-contributions/day 4 no_langchain/RAG_chat_no_LangChain.ipynb

@ -0,0 +1,394 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "e9025a4a-b8ef-4901-b98e-753b756b028a",
"metadata": {},
"source": [
"# Building a RAG chat without the langchain framework\n",
"## To understand more in detail what's going on\n",
"\n",
"The technical know-how comes from Ed Donner, obviously, as well as from Sakalya Mitra & Pradip Nichite on [this gem of a blog post](https://blog.futuresmart.ai/building-rag-applications-without-langchain-or-llamaindex) I found on futuresmart.ai"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1b7acfb5-8bf9-48b5-a219-46f1e3bfafc3",
"metadata": {},
"outputs": [],
"source": [
"import os\n",
"from dotenv import load_dotenv\n",
"import gradio as gr\n",
"import re\n",
"from openai import OpenAI"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "19af6b8b-be29-4086-a69f-5e2cdb867ede",
"metadata": {},
"outputs": [],
"source": [
"# imports for Chroma and plotly\n",
"\n",
"import chromadb\n",
"from chromadb.utils import embedding_functions\n",
"import numpy as np\n",
"from sklearn.manifold import TSNE\n",
"import plotly.graph_objects as go"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "bc6d9ab4-816a-498c-a04c-c3838770d848",
"metadata": {},
"outputs": [],
"source": [
"MODEL = \"gpt-4o-mini\"\n",
"db_name = \"chroma_db\"\n",
"client = chromadb.PersistentClient(path=\"chroma_db\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a3715b81-eed0-4412-8c01-0623ed113657",
"metadata": {},
"outputs": [],
"source": [
"load_dotenv()\n",
"openai_api_key = os.getenv('OPENAI_API_KEY')\n",
"openai = OpenAI()"
]
},
{
"cell_type": "markdown",
"id": "3017e1dd-d0d5-4ef4-8c72-84517a927793",
"metadata": {},
"source": [
"### Making stuff at home: documents"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e83480a5-927b-4756-a978-520a56ceed85",
"metadata": {},
"outputs": [],
"source": [
"# items in documents are actually objects: Documents(metadata={...}, page_content=\"...\"), so we need a \"Document\" class\n",
"# btw all the quadruple-backslash madness here is due to Windows (there might be a more efficient way, still)\n",
"\n",
"class Document:\n",
" def __init__(self, metadata, page_content):\n",
" self.metadata = metadata\n",
" self.page_content = page_content\n",
"\n",
" def __repr__(self):\n",
" return f\"Document(metadata={self.metadata}, page_content={repr(self.page_content)})\"\n",
"\n",
"\n",
"documents = []\n",
"\n",
"def get_documents(path='.'):\n",
" for entry in os.listdir(path):\n",
" if len(re.findall(\"^\\.\", entry)) == 0:\n",
" full_path = os.path.join(path, entry)\n",
" if os.path.isdir(full_path):\n",
" get_documents(full_path)\n",
" else:\n",
" parent = re.sub(\"^\\.[\\\\\\\\].*[\\\\\\\\]\", \"\", os.path.dirname(full_path))\n",
" self = os.path.basename(full_path)\n",
" content = \"\"\n",
"\n",
" with open(full_path, mode=\"r\", encoding=\"utf-8\") as f:\n",
" content = f.read()\n",
" \n",
" doc = Document(metadata={\"source\": full_path, \"doc_type\": parent, \"self\": self}, page_content=content)\n",
" documents.append(doc)\n",
"\n",
"# where the knowledge collection lives\n",
"directory_path = r'.\\knowledge_collection'\n",
"get_documents(directory_path)"
]
},
{
"cell_type": "markdown",
"id": "fd846bc0-54d0-4802-a18b-196c396a241c",
"metadata": {},
"source": [
"### Making stuff at home: chunks"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "202b33e2-c3fe-424c-9c8e-a90e517add42",
"metadata": {},
"outputs": [],
"source": [
"eos_pattern = re.compile(r\"((?<=[.!?;])[\\s]+)|([\\n\\r]+)\")\n",
"chunk_size = 1000\n",
"chunks = []"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a19a61ec-d204-4b87-9f05-88832d03fad6",
"metadata": {},
"outputs": [],
"source": [
"for doc in documents:\n",
"\n",
" sentence_ends = [end.start() for end in list(re.finditer(eos_pattern, doc.page_content)) if end.start() > chunk_size - 50]\n",
" start = 0\n",
" \n",
" if len(sentence_ends) == 0 and len(doc.page_content) > 5:\n",
" chunk = Document(metadata=doc.metadata, page_content=doc.page_content)\n",
" chunk.metadata['id'] = f\"{doc.metadata['source']}_chunk_\"\n",
" chunks.append(chunk)\n",
"\n",
" else: \n",
" for point in sentence_ends:\n",
" if point - start >= chunk_size - 50:\n",
" text = doc.page_content[start:point]\n",
" chunk = Document(metadata=doc.metadata, page_content=text)\n",
" chunk.metadata['id'] = f\"{doc.metadata['source']}_chunk_\"\n",
" chunks.append(chunk)\n",
" start = point\n",
" \n",
" # Add the remaining part of the text as the last chunk if it's big enough\n",
" if len(doc.page_content) - start > 5:\n",
" text = doc.page_content[start:]\n",
" chunk = Document(metadata=doc.metadata, page_content=text)\n",
" chunk.metadata['id'] = f\"{doc.metadata['source']}_chunk_\"\n",
" chunks.append(chunk)"
]
},
{
"cell_type": "markdown",
"id": "966ae50c-e0e5-403a-9465-8f26967f8922",
"metadata": {},
"source": [
"### Making stuff without a framework: embeddings"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b97391c0-e55f-4e08-b0cb-5e62fb119ae6",
"metadata": {},
"outputs": [],
"source": [
"# Configure sentence transformer embeddings\n",
"embeddings = embedding_functions.SentenceTransformerEmbeddingFunction(\n",
" model_name=\"all-MiniLM-L6-v2\"\n",
")\n",
"\n",
"collection_name = \"documents_collection\"\n",
"\n",
"try:\n",
" client.delete_collection(collection_name)\n",
"except ValueError:\n",
" print(f\"{collection_name} doesn't exist yet\")\n",
"\n",
"# Create collection\n",
"collection = client.get_or_create_collection(\n",
" name=collection_name,\n",
" embedding_function=embeddings\n",
")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5222dfec-8cf4-4e87-aeb8-33d0f3b3b5cb",
"metadata": {},
"outputs": [],
"source": [
"# adding our chunks to the \"collection\"\n",
"\n",
"for chunk in chunks:\n",
" index = chunks.index(chunk)\n",
" collection.add(\n",
" documents=chunk.page_content,\n",
" metadatas=chunk.metadata,\n",
" ids=chunk.metadata['id'] + f\"{index}\"\n",
" )"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "5effcada-ee5f-4207-9fa6-1fc5604b068b",
"metadata": {},
"outputs": [],
"source": [
"def semantic_search(collection, query: str, n_results: int = 4):\n",
" results = collection.query(\n",
" query_texts=[query],\n",
" n_results=n_results\n",
" )\n",
" return results"
]
},
{
"cell_type": "markdown",
"id": "99f0a366-3dcb-4824-9f33-70e07af984d8",
"metadata": {},
"source": [
"## Visualizing the Vector Store\n",
"\n",
"The results actually look just as good with `all-MiniLM-L6-v2`"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e12751ab-f102-4dc6-9c0f-313e5832b75f",
"metadata": {},
"outputs": [],
"source": [
"# Prework\n",
"\n",
"result = collection.get(include=['embeddings', 'documents', 'metadatas'])\n",
"vectors = np.array(result['embeddings'])\n",
"documents = result['documents']\n",
"doc_types = [metadata['doc_type'] for metadata in result['metadatas']]\n",
"colors = [['blue', 'red', 'orange'][['languages', 'mountains', 'regions'].index(t)] for t in doc_types]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "422e3247-2de0-44ba-82bc-30b4f739da7e",
"metadata": {},
"outputs": [],
"source": [
"# Reduce the dimensionality of the vectors to 2D using t-SNE\n",
"# (t-distributed stochastic neighbor embedding)\n",
"\n",
"tsne = TSNE(n_components=2, random_state=42)\n",
"reduced_vectors = tsne.fit_transform(vectors)\n",
"\n",
"# Create the 2D scatter plot\n",
"fig = go.Figure(data=[go.Scatter(\n",
" x=reduced_vectors[:, 0],\n",
" y=reduced_vectors[:, 1],\n",
" mode='markers',\n",
" marker=dict(size=5, color=colors, opacity=0.8),\n",
" text=[f\"Type: {t}<br>Text: {d[:100]}...\" for t, d in zip(doc_types, documents)],\n",
" hoverinfo='text'\n",
")])\n",
"\n",
"fig.update_layout(\n",
" title='2D Chroma Vector Store Visualization',\n",
" scene=dict(xaxis_title='x',yaxis_title='y'),\n",
" width=800,\n",
" height=600,\n",
" margin=dict(r=20, b=10, l=10, t=40)\n",
")\n",
"\n",
"fig.show()"
]
},
{
"cell_type": "markdown",
"id": "2cff9065-de3d-4e91-8aff-c7ad750a4334",
"metadata": {},
"source": [
"#### Comment: Relying on Gradio's history handling seems to be memory enough\n",
"##### If all you need is your favorite LLM with expertise in your knowlege collection"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "aebb676f-883e-4b2b-8420-13f2a8399e77",
"metadata": {},
"outputs": [],
"source": [
"system_prompt = \"You are a helpful assistant for everything French. Give brief, accurate answers. \\\n",
"Do not provide any information that you haven't been asked for, even if you have lots of context. \\\n",
"If you haven't been provided with relevant context, say you don't know. Do not make anything up, only \\\n",
"provide answers that are based in the context you have been given. Do not comment on the provided context. \\\n",
"If the user doesn't ask for any information, engage in brief niceties and offer your expertise regarding France.\"\n",
"\n",
"history = [{\"role\": \"system\", \"content\": system_prompt}]\n",
"\n",
"def get_user_prompt(prompt):\n",
" # semantic search!!\n",
" context = semantic_search(collection, prompt)['documents'][0]\n",
"\n",
" if len(context) > 0:\n",
" prompt += f\"\\n\\n[AUTOMATIC SYSTEM CONTEXT ADDITION] Here is some context that might be useful for answering the question:\"\n",
"\n",
" for doc in context:\n",
" prompt += f\"\\n\\n{doc}\"\n",
" \n",
" user_prompt = {\"role\": \"user\", \"content\": prompt}\n",
"\n",
" return user_prompt"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "23b70162-2c4f-443e-97c8-3e675304d307",
"metadata": {},
"outputs": [],
"source": [
"def stream_gpt(message, history):\n",
" messages = [{\"role\": \"system\", \"content\": system_prompt}] + history\n",
" messages.append(get_user_prompt(message))\n",
" stream = openai.chat.completions.create(\n",
" model=MODEL,\n",
" messages=messages,\n",
" stream=True\n",
" )\n",
" result = \"\"\n",
" for chunk in stream:\n",
" result += chunk.choices[0].delta.content or \"\"\n",
" yield result"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "4ecf4a30-452d-4d41-aa60-fa62c8e2559b",
"metadata": {},
"outputs": [],
"source": [
"# Gradio\n",
"\n",
"gr.ChatInterface(fn=stream_gpt, type=\"messages\").launch(inbrowser=True)"
]
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.11"
}
},
"nbformat": 4,
"nbformat_minor": 5
}

42
week5/community-contributions/day 4 no_langchain/knowledge_collection/languages/alsacien.md

@ -0,0 +1,42 @@
# Overview of Alsacien Language
## Definition
Alsacien, also known as Alsatian or Alsatian German, is a variety of the Alemannic branch of the Germanic languages spoken predominantly in Alsace, France.
## Geographic Distribution
- Primarily spoken in Alsace, a region in northeastern France.
- Communities of Alsacien speakers can also be found in neighboring regions of Germany and Switzerland.
## Linguistic Classification
- **Language Family**: Indo-European
- **Subfamily**: Germanic
- **Group**: West Germanic
- **Branch**: High German
## Speakers
- Estimates of native speakers range from 500,000 to 1 million, though use has declined due to factors like urbanization and language shift towards French.
## Dialectal Variations
- Alsacien includes multiple dialects, which may vary significantly from one locality to another.
- Two main dialects:
- **Haut-Rhin** (Upper Rhine)
- **Bas-Rhin** (Lower Rhine)
## Characteristics
- Strongly influenced by both French and standard German, leading to unique vocabulary and pronunciation.
- Grammar and syntax retain features of Middle High German.
## Cultural Significance
- Acts as a marker of regional identity for the people of Alsace.
- Extensively used in local media, literature, and music, particularly folk traditions.
## Status
- Considered a vulnerable language by UNESCO.
- Efforts are ongoing for revitalization, including teaching in schools and cultural associations promoting its use.
## Related Languages
- Closely related to Swiss German and other Alemannic dialects.
- Influenced by and influences neighboring languages, particularly French.
## Conclusion
Alsacien is a vital part of the cultural heritage of the Alsace region, with ongoing efforts aimed at preserving and promoting its use among younger generations.

31
week5/community-contributions/day 4 no_langchain/knowledge_collection/languages/bourguignon.md

@ -0,0 +1,31 @@
# Overview of the Bourguignon Language
## General Information
- **Name**: Bourguignon
- **Region**: Primarily spoken in the Burgundy region of France
- **Language Family**: Romance languages
- **Classification**: It is part of the Langue d'oïl group, which also includes languages like French, Norman, and Picard.
## Historical Context
- **Origin**: Derived from Vulgar Latin, Bourguignon developed in the medieval period and reflects the linguistic evolution of the region.
- **Influence**: Historically influenced by Old French, as well as regional dialects and neighboring languages.
## Features
- **Dialects**: Bourguignon comprises several dialects, often differing significantly from one another.
- **Phonetics**: The phonetic system exhibits distinct sounds not found in Standard French.
- **Vocabulary**: Contains unique vocabulary and expressions that may not be understood by standard French speakers.
## Current Status
- **Speaker Population**: The number of speakers has declined over the years, with estimates suggesting only a few thousand fluent speakers today.
- **Recognition**: Bourguignon is not an official language in France, but there are efforts to preserve and promote its use among local communities.
## Cultural Significance
- **Folklore and Literature**: Bourguignon has a rich tradition of oral literature, including folk tales and songs that reflect the cultural heritage of Burgundy.
- **Festivals and Events**: Local festivals often include performances in Bourguignon, celebrating the language's place in regional identity.
## Modern Efforts
- **Revitalization**: Initiatives to teach Bourguignon in schools and promote its use in cultural activities aim to preserve the language for future generations.
- **Media Presence**: Some local media, including radio stations and publications, feature Bourguignon, fostering a sense of community among speakers.
## Conclusion
Bourguignon remains an important part of the cultural identity of the Burgundy region, reflecting the historical and linguistic diversity of France. Efforts to revive and sustain the language highlight its significance within the local heritage.

33
week5/community-contributions/day 4 no_langchain/knowledge_collection/languages/breton.md

@ -0,0 +1,33 @@
# Overview of the Breton Language
## General Information
- **Name**: Breton (Brezhoneg)
- **Language Family**: Celtic, part of the Brythonic branch
- **Region**: Brittany (Breizh), France
## Historical Background
- **Origins**: Breton is derived from the Brythonic Celtic languages that were spoken in Great Britain. It arrived in Brittany with settlers from Britain during the early medieval period.
- **First Documented Evidence**: The earliest written examples of Breton date back to the 8th century.
## Linguistic Features
- **Dialects**: There are three main dialects of Breton:
- **Gouèze** (Western)
- **Kerne** (Central)
- **Leoneg** (Eastern)
- **Alphabet**: The modern Breton alphabet uses the Latin script with some diacritics.
## Current Status
- **Speakers**: Approximately 200,000 to 300,000 speakers as of recent estimates.
- **Recognition**: Breton is recognized as a regional language in France, but it does not hold official status.
- **Revitalization Efforts**: There are ongoing initiatives to promote the language, including bilingual education and media in Breton.
## Cultural Significance
- **Literature and Music**: Breton has a rich oral tradition, including folklore, songs, and poetry. Contemporary literature and music often embrace the language.
- **Festivals**: Events like Fest-Noz (night festivals) celebrate Breton culture and often feature music and dance in the Breton language.
## Challenges
- **Decline**: The number of native speakers has declined significantly due to historical policies and the dominance of French.
- **Education**: Breton is not widely taught in schools, although there are some bilingual programs and immersion schools.
## Conclusion
Breton is a vibrant Celtic language with a rich history and cultural heritage, facing challenges in the modern age but supported by revitalization efforts and community engagement.

34
week5/community-contributions/day 4 no_langchain/knowledge_collection/languages/gascon.md

@ -0,0 +1,34 @@
# Overview of the Gascon Language
## General Information
- **Language Family**: Occitan branch of the Romance languages.
- **Region**: Primarily spoken in the Gascony region of southwestern France, which includes parts of the departments of Gers, Landes, and Pyrénées-Atlantiques.
## Historical Context
- **Origins**: Gascon evolved from Vulgar Latin and has influences from the Visigoths and various other historical invaders.
- **Status**: Once a widely spoken language, Gascon has seen a decline in the number of speakers, particularly in urban areas, due to the rise of French as the dominant language.
## Dialects
- **Varieties**: Gascon includes several dialects, most notably:
- **Bigourdan**: Spoken in the region of Bigorre.
- **Armanac**: Found in Armagnac.
- **Languedocien**: This influences some Gascon speakers, particularly those in mixed-language areas.
## Linguistic Features
- **Phonetics**: Gascon has unique phonetic characteristics, such as the preservation of the Latin 'u' sound and certain nasal vowels.
- **Vocabulary**: Contains a wealth of regional vocabulary, along with borrowings from French, Occitan, and Basque.
## Cultural Significance
- **Literature**: Historically, Gascon has been used in regional literature and songs, contributing richly to the cultural heritage of the area.
- **Folklore and Traditions**: Gascon is an important vehicle for local folklore, traditions, and customs in Gascony.
## Current Status
- **Revitalization Efforts**: There are ongoing efforts to promote and teach Gascon in schools, cultural organizations, and through local media.
- **Number of Speakers**: As of recent estimates, the number of fluent speakers is declining, with efforts being made to preserve the language among younger generations.
## Related Languages
- **Occitan**: Gascon is one of the major dialects of the Occitan language, which also includes Provençal and Languedocien.
- **Comparison to French**: While Gascon shares some similarities with French, it retains distinct grammatical structures and vocabulary.
## Conclusion
Gascon is not only a language but a crucial component of the cultural identity of the Gascon people, reflecting their history, traditions, and regional pride. Efforts for revitalization continue to be important in preserving this unique linguistic heritage.

30
week5/community-contributions/day 4 no_langchain/knowledge_collection/languages/languedocien.md

@ -0,0 +1,30 @@
# Overview of Languedocien Language
## General Information
- **Language Family**: Occitan
- **Region**: Primarily spoken in the Languedoc region of southern France.
- **ISO Code**: Not officially assigned, but sometimes referred to as "oc" for Occitan.
## Linguistic Features
- **Dialects**: Languedocien is one of the major dialects of the Occitan language, which also includes Provençal, Gascon, and Auvergnat.
- **Phonetics**: Characterized by the presence of certain vowel sounds and the use of diphthongs that may differ from other dialects.
- **Grammar**: Similar to other Occitan dialects, it features a subject-verb-object structure, but with unique local variations.
## Vocabulary
- **Lexical Influence**: Languedocien vocabulary is heavily influenced by Latin, with a significant number of words also derived from Provençal and other regional languages.
- **Regionalisms**: Contains unique words and expressions that are specific to local culture and traditions.
## Cultural Context
- **Recognition**: While part of the Occitan language family, Languedocien does not have official status in France and is considered a regional language.
- **Literature**: Historically used in medieval literature; notable authors include Frédéric Mistral and others who contributed to the revival of Occitan literature.
## Current Status
- **Speakers**: There are an estimated few hundred thousand speakers, with numbers decreasing due to the dominance of French.
- **Revitalization Efforts**: Various cultural organizations and schools aim to preserve and promote the use of Languedocien through courses, workshops, and public events.
## Geographic Distribution
- **Primary Areas**: Predominantly spoken in the departments of Hérault, Aude, Gard, and parts of Lozère and Pyrénées-Orientales.
- **Urban vs. Rural**: More commonly spoken in rural areas, with younger generations tending to use it less in urban settings.
## Conclusion
Languedocien remains an essential part of the cultural heritage of southern France, reflecting the region's history, traditions, and linguistic diversity. Efforts to sustain and promote the language continue amidst challenges posed by modernization and globalization.

26
week5/community-contributions/day 4 no_langchain/knowledge_collection/languages/lorrain.md

@ -0,0 +1,26 @@
# Overview of the Lorrain Language
## General Information
- **Language Family**: Lorrain is part of the Langue d'Oïl languages, which are a subgroup of the Romance languages.
- **Region**: Primarily spoken in the Lorraine region of northeastern France.
- **Dialects**: There are various dialects of Lorrain, including certain variations influenced by local languages and cultures.
## Historical Context
- **Origins**: The language has roots dating back to the medieval period and was influenced by the historical presence of the Duchy of Lorraine.
- **Language Shift**: Over the 19th and 20th centuries, Lorrain saw a decline in usage due to the dominance of French, leading many speakers to shift to French.
## Linguistic Features
- **Phonology**: Lorrain phonetics include distinct sounds that differentiate it from standard French and other Langue d'Oïl languages.
- **Vocabulary**: The lexicon of Lorrain retains several archaic words and expressions that have disappeared from modern French.
- **Grammar**: Similar to French but with unique grammatical structures and conjugations, reflecting its distinct identity.
## Cultural Significance
- **Traditions**: Lorrain is often associated with local folklore, songs, and literature, which contribute to the cultural identity of Lorraine.
- **Preservation Efforts**: Various initiatives have been undertaken to promote and preserve the Lorrain language, including cultural festivals and educational programs.
## Current Status
- **Speaker Population**: The number of active speakers has significantly decreased, with many older speakers and limited transmission to younger generations.
- **Revitalization**: Recent efforts are being made to revive interest in Lorrain among younger populations through workshops, classes, and media.
## Conclusion
Lorrain is a unique language that embodies the rich cultural heritage of the Lorraine region. While it faces challenges, ongoing efforts aim to preserve and revitalize this historical language for future generations.

34
week5/community-contributions/day 4 no_langchain/knowledge_collection/languages/normand.md

@ -0,0 +1,34 @@
# Overview of the Normand Language
## What is Normand?
Normand is a regional language of France, part of the Oïl language group. It originates from the Normandy region and is historically linked to Old Norman, which developed from the Old Norman dialect of Old French.
## Geographic Distribution
- Predominantly spoken in Normandy, particularly in the departments of Seine-Maritime and Calvados.
- Some dialects extend into the Channel Islands (like Jersey and Guernsey), where it is closely related to Jèrriais and Guernésiais.
## Dialects
Normand has several dialects, which can vary significantly in terms of vocabulary, pronunciation, and grammar. Key dialects include:
- **Bocage**: Spoken in the rural areas of western Normandy.
- **Mélée**: Found in the northeastern part.
- **Sèvres**: A dialect with influences from the urban centers.
## Linguistic Features
- Normand retains many archaic French features that have evolved in Standard French.
- The pronunciation of vowels and some consonant sounds can be quite distinct from Standard French.
- There are notable differences in use of articles and noun endings compared to Standard French.
## Historical Context
- Norman was historically influential due to the Viking settlement of Normandy in the 9th century and subsequent Norman Conquest of England in 1066.
- It was widely used by the nobility and in administrative contexts until French became more dominant post-16th century.
## Current Status
- Normand is considered a minority language and has seen a decline in speakers over the years.
- Efforts for revitalization are ongoing, with various cultural associations promoting the language through education and media.
## Cultural Aspects
- Normand has a rich oral tradition, with folk tales, songs, and proverbs integral to the culture of Normandy.
- Festivals and events celebrating Normand language and culture are held in various communities.
## Conclusion
While facing challenges due to globalization and the dominance of Standard French, Normand remains an important part of the cultural heritage of Normandy. Efforts to preserve and promote the language continue, aiming to maintain its presence for future generations.

27
week5/community-contributions/day 4 no_langchain/knowledge_collection/languages/picard.md

@ -0,0 +1,27 @@
# Overview of the Picard Language
## General Information
- **Language Family**: Romance, specifically a part of the West Oïl languages, which also includes French.
- **Region**: Primarily spoken in the historic region of Picardy in northern France, as well as in parts of Belgium and historically in the areas of the nearby Nord-Pas-de-Calais.
## Linguistic Characteristics
- **Dialects**: There are several dialects of Picard, including Amiénois, Beauvaisis, and Hesdinois.
- **Vocabulary**: Shares many lexical items with French but also retains unique words and expressions. Some vocabulary is influenced by local historical interactions with Dutch and German.
## Historical Context
- **Origins**: Evolved from Latin, like other Romance languages. Roots trace back to the Vulgar Latin spoken in the region during the Roman Empire.
- **Literary Tradition**: Has a rich but lesser-known literary tradition, with poetry and prose dating back to the Middle Ages.
## Current Status
- **Speakers**: The number of speakers has declined significantly over the 20th century due to the dominance of standard French and the 1999 ban on the usage of Picard in all of France.
- **Revitalization Efforts**: Recent efforts outside of France include community classes, cultural organizations, and media in Picard to promote the language. It is rumored that there is an underground movement in France to keep Picard alive in spite of the language being banned and illegal to use since 1999.
## Cultural Significance
- **Identity**: Picard is an important part of regional identity and cultural heritage for many people in northern France.
- **Festivals and Events**: Regional festivals celebrate Picard culture, featuring traditional songs, dances, and cuisine.
## Legal Status
- **Recognition**: Picard has no official status in France, but it is recognized as a regional language. Efforts have been made to include it in educational curricula and local government documents in some areas.
## Conclusion
Picard is a unique language that reflects the cultural and historical tapestry of northern France. Despite challenges, there are active efforts to preserve and promote its usage among future generations.

27
week5/community-contributions/day 4 no_langchain/knowledge_collection/languages/provencal.md

@ -0,0 +1,27 @@
# Overview of Provençal Language
## Definition
Provençal is a Romance language that belongs to the Occitan language family, which is spoken primarily in the Provence region of southern France.
## Historical Background
- **Origins**: Provençal has its roots in Vulgar Latin and has been influenced by various languages and cultures throughout history, including Celtic, Germanic, and Arabic.
- **Literary Tradition**: It has a rich literary tradition dating back to the 11th century, with notable poets such as Frédéric Mistral contributing to its revival in the 19th century.
## Geographic Distribution
- **Regions**: Primarily spoken in Provence, it also has speakers in parts of Italy and Spain, particularly in the Val d'Aran valley in Catalonia, known as Aranese.
- **Dialectal Variations**: Provençal encompasses several dialects, such as Alémanique, Boulégue, and Languedocien, reflecting the linguistic diversity within the Occitan language.
## Current Status
- **Recognition**: Provençal is recognized as a cultural language in France but has a minority status and faces challenges due to the dominance of French.
- **Revitalization Efforts**: There are ongoing efforts to promote and teach Provençal, including in schools and cultural institutions.
## Linguistic Features
- **Grammar and Syntax**: Provençal has distinct grammatical structures that differentiate it from standard French, including the use of gendered nouns and specific verb conjugations.
- **Vocabulary**: It retains many words and expressions derived from Latin, along with unique local terms and influences from neighboring languages.
## Cultural Significance
- **Folklore and Traditions**: Provençal is an important part of the cultural identity in Provence, associated with local traditions, music, festivals, and cuisine.
- **Media and Literature**: There are books, newspapers, and online resources available in Provençal, contributing to its presence in modern media.
## Conclusion
Provençal is a vibrant language with a deep historical and cultural significance in southern France. While it faces challenges, ongoing efforts for its preservation continue to foster interest and engagement in this unique linguistic heritage.

37
week5/community-contributions/day 4 no_langchain/knowledge_collection/mountains/alpes.md

@ -0,0 +1,37 @@
# Overview of the French Alps
## General Information
- **Location:** Southeastern France, extending into Switzerland and Italy.
- **Length:** Approximately 1,200 kilometers (750 miles).
- **Highest Peak:** Mont Blanc, standing at 4,808 meters (15,774 feet).
- **Mountain Chain:** Part of the larger Alpine range that spans across several European countries.
## Geography
- **Geological Composition:** Primarily composed of limestone and granite.
- **Major Valleys:** Includes the Rhône and Isère valleys.
- **Natural Parks:** Home to several national parks, including Écrins National Park and Vanoise National Park.
## Climate
- **Variety:** Alpine climate with large variations; cold winters and mild summers.
- **Snowfall:** Heavy snowfall in winter makes it a prime destination for winter sports.
## Flora and Fauna
- **Biodiversity:** Rich diversity of species; includes both alpine and Mediterranean flora.
- **Wildlife:** Encounters with species such as chamois, ibex, and golden eagles.
## Activities
- **Winter Sports:** Skiing and snowboarding are popular, with famous resorts like Chamonix, Courchevel, and Val d’Isère.
- **Summer Activities:** Hiking, mountaineering, and mountain biking attract visitors during the warmer months.
- **Paragliding:** Known as a hotspot for paragliding due to favorable winds and stunning views.
## Cultural Significance
- **Local Communities:** Home to various Alpine villages and cultures, each with unique traditions and languages.
- **Gastronomy:** Famous for local cheeses (like Beaufort and Reblochon), charcuterie, and dishes such as fondue and raclette.
## Historical Aspects
- **Cultural Heritage:** Influenced by Roman and medieval settlements, with significant archaeological sites.
- **Tourism:** Became a major tourist destination in the 19th century.
## Importance
- **Economic Significance:** Tourism is a vital part of the local economy, alongside agriculture and forestry.
- **Sustainability Focus:** Growing emphasis on sustainable tourism practices to protect the fragile alpine ecosystem.

36
week5/community-contributions/day 4 no_langchain/knowledge_collection/mountains/ardennes.md

@ -0,0 +1,36 @@
# Overview of the Ardennes Mountain Range
## Location
- The Ardennes is a region located in the northeastern part of France, extending into Belgium and Luxembourg.
## Geography
- The Ardennes is characterized by dense forests, deep valleys, and rolling hills.
- The highest peak in the French Ardennes is Le Signal de Botrange, which reaches an elevation of about 2,277 feet (694 meters), although it is situated in Belgium.
## Geology
- The area is known for its rugged terrain and is primarily composed of sedimentary rocks such as limestone and sandstone.
- The landscape has been shaped by glacial and river erosion over millennia.
## Climate
- The Ardennes has a temperate maritime climate, with cool summers and mild winters.
- Precipitation is relatively high, leading to lush vegetation.
## Flora and Fauna
- The region is home to diverse wildlife, including deer, wild boar, and various bird species.
- Dense forests are dominated by beech and fir trees, and many areas are protected as nature reserves.
## Human Activity
- The Ardennes has a rich history, having been inhabited since prehistoric times.
- It has significance in World War I and II, particularly during the Battle of the Bulge.
- The region is known for outdoor activities such as hiking, cycling, and kayaking.
## Cultural Aspects
- The Ardennes is dotted with picturesque villages and towns, showcasing traditional architecture.
- The area is known for its beer production, particularly in Belgium, with many breweries operating in the region.
## Tourism
- Key attractions include the Semois River, the fortress of Bouillon, and the expansive forests of the Ardennes.
- The region offers several trails and parks, attracting nature lovers and adventure enthusiasts.
## Conclusion
The Ardennes is a unique blend of natural beauty, historical significance, and cultural richness, making it an important region in France and beyond.

37
week5/community-contributions/day 4 no_langchain/knowledge_collection/mountains/jura.md

@ -0,0 +1,37 @@
# Overview of the Jura Mountain Range in France
## Location
- The Jura Mountains are located along the border between France and Switzerland.
- They stretch approximately 365 kilometers (227 miles) from the Rhône River in the south to the Rhine River in the north.
## Geography
- The Jura is characterized by its rugged terrain, with numerous peaks, plateaus, and deep valleys.
- The highest peak in the French Jura is Crêt de la Neige, which rises to an elevation of 1,720 meters (5,643 feet).
## Geology
- The range is primarily composed of limestone, which has been shaped by erosion, creating unique karst formations, caves, and cliffs.
- The Jura Mountains were formed during the Jurassic period, which is reflected in their name.
## Climate
- The climate in the Jura varies from humid in the west to drier conditions in the east.
- The area experiences significant snowfall in winter, making it popular for winter sports.
## Flora and Fauna
- The Jura is home to diverse ecosystems, including forests, alpine meadows, and wetlands.
- Wildlife includes species such as deer, chamois, marmots, and a variety of bird species.
## Activities
- The Jura Mountains offer various outdoor activities, including hiking, skiing, and mountain biking.
- The region is known for its beautiful landscapes and natural parks, attracting tourists and nature enthusiasts.
## Cultural Significance
- The Jura region is also known for its traditional cheese production, particularly Comté cheese.
- Numerous charming villages and towns, such as Arbois and Clairvaux-les-Lacs, showcase the cultural heritage of the area.
## History
- The Jura Mountains have historical significance, having served as a natural barrier and route for trade and exploration.
- The region has witnessed various historical events, including battles during the French Revolutionary Wars and the Napoleonic Wars.
## Accessibility
- The Jura is accessible from major cities like Geneva, Lyon, and Besançon, making it a popular destination for both locals and tourists.
- Several scenic routes and parks are maintained to facilitate exploration and enjoyment of the natural beauty.

35
week5/community-contributions/day 4 no_langchain/knowledge_collection/mountains/massif_armorican.md

@ -0,0 +1,35 @@
# Overview of the Massif Armorican
## Location
- **Region**: Brittany, France
- **Coordinates**: Approximately 47° N latitude and 2° W longitude
## Geography
- **Type**: Mountain range and geological massif
- **Area**: Covers parts of the departments of Ille-et-Vilaine, Morbihan, and Finistère
- **Elevation**: The highest peak, **Montagnes Noires**, reaches around 600 meters (1,969 feet)
## Geology
- **Formation**: Primarily composed of ancient metamorphic rocks and granite formations, dating back to the Precambrian and Paleozoic eras
- **Tectonic Activity**: Influenced by the Variscan orogeny, which caused significant geological changes
## Flora and Fauna
- **Biodiversity**: Home to diverse ecosystems, including heathlands, forests, and wetlands
- **Protected Areas**: Parts of the massif are designated as natural parks and reserves, promoting conservation efforts
## Culture and History
- **Historical Significance**: The area is rich in megalithic structures and archaeological sites, reflecting ancient Celtic culture
- **Tourism**: Popular for hiking, cycling, and exploring its historical sites, contributing to local economies
## Climate
- **Climate Type**: Maritime temperate climate, characterized by mild winters and cool summers
- **Precipitation**: Receives a significant amount of rainfall throughout the year, supporting its lush vegetation
## Attractions
- **Sites of Interest**: Includes historic towns, châteaux, and picturesque landscapes, attracting visitors for both natural beauty and cultural heritage
- **Outdoor Activities**: Offers opportunities for outdoor sports such as hiking, horseback riding, and nature observation
## Transportation
- **Accessibility**: Well-connected by road and rail, making it easily accessible from major urban centers in Brittany
This overview encapsulates the essential aspects of the Massif Armorican, highlighting its geographical, geological, and cultural significance in France.

34
week5/community-contributions/day 4 no_langchain/knowledge_collection/mountains/massif_central.md

@ -0,0 +1,34 @@
# Overview of Massif Central
## General Information
- **Location**: South-central France
- **Area**: Approximately 85,000 km²
- **Highest Peak**: Puy de Sancy (1,885 meters)
- **Geological Composition**: Primarily volcanic and sedimentary rocks
## Geography
- **Regions Covered**: Spans across several French departments including Cantal, Puy-de-Dôme, Haute-Loire, and Lozère.
- **Landscape**: Characterized by plateaus, volcanic cones, deep valleys, and rivers.
## Climate
- **Type**: Predominantly oceanic climate with a continental influence.
- **Precipitation**: Higher rainfall in the western regions, often resulting in lush landscapes.
## Flora and Fauna
- **Biodiversity**: Home to various ecosystems, including grasslands, forests, and wetlands.
- **Protected Areas**: Includes several national parks and nature reserves, such as the Parc Naturel Régional des Volcans d'Auvergne.
## Cultural Significance
- **History**: Affected by various historical events and populations, including the Gauls and the Roman Empire.
- **Heritage**: Rich cultural heritage with medieval towns, castles, and traditional practices.
## Economic Importance
- **Agriculture**: Known for agriculture, particularly cheese production (e.g., Saint-Nectaire, Cantal).
- **Tourism**: Popular destination for outdoor activities such as hiking, skiing, and exploring natural parks.
## Notable Features
- **Volcanic Activity**: The region contains many extinct volcanoes, with some still showing geothermal activity.
- **Natural Attractions**: Features stunning sites like the Gorges de la Loire and the Chaîne des Puys, a UNESCO World Heritage site.
## Accessibility
- **Transport**: Well-connected by road and rail, with several towns providing access points for visitors.

44
week5/community-contributions/day 4 no_langchain/knowledge_collection/mountains/morvan.md

@ -0,0 +1,44 @@
# Overview of the Morvan Mountain Range
## Location
- **Country**: France
- **Region**: Burgundy (Bourgogne)
- **Department**: Nièvre, Saône-et-Loire, Côte-d'Or
## Geography
- **Coordinates**: Approximately 47°10′N 3°55′E
- **Highest Peak**: Mont Beuvray
- **Elevation**: 821 meters (2,700 feet)
- **Area**: Approximately 3,500 square kilometers
- **Major Rivers**: Cure, Yonne, and Loing flow through the region.
## Geology
- Composed primarily of granitic and metamorphic rocks.
- The landscape features rolling hills, valleys, and plateaus.
- Known for its rich biodiversity and varied ecosystems.
## Climate
- **Type**: Temperate continental climate.
- **Weather**: Mild summers and cold winters with occasional snowfall.
## History
- The Morvan area has a rich history dating back to prehistoric times.
- Notable archaeological sites include the remnants of the Gallic tribe of the Aedui in Mont Beuvray.
- The region was significant during the Roman conquest of Gaul.
## Culture and Economy
- The Morvan is known for its traditional rural lifestyle and local crafts.
- Main industries include agriculture, forestry, and tourism.
- Famous for Morvan cheese and wines from the surrounding Burgundy region.
## Tourism
- Offers a variety of outdoor activities such as hiking, cycling, and fishing.
- Home to the Morvan Regional Natural Park, established in 1970, which promotes conservation and sustainable tourism.
- Attractions include ancient ruins, beautiful landscapes, and charming villages.
## Wildlife
- Habitat for various species, including deer, wild boars, and numerous bird species.
- Rich flora with many endemic plant species.
## Conservation
- The region emphasizes environmental protection and sustainability in its natural park initiatives.

40
week5/community-contributions/day 4 no_langchain/knowledge_collection/mountains/pyrenees.md

@ -0,0 +1,40 @@
# Overview of the Pyrenees Mountain Range
## Geographic Location
- The Pyrenees mountain range forms a natural border between **France** and **Spain**.
- It extends approximately **430 kilometers (267 miles)** from the Atlantic Ocean (Bay of Biscay) in the west to the Mediterranean Sea in the east.
## Major Peaks
- **Aneto** is the highest peak, with an elevation of **3,404 meters (11,168 feet)**.
- Other notable peaks include **Monte Perdido**, **Vignemale**, and **Pic du Midi d'Ossau**.
## Geography and Geology
- The Pyrenees are divided into three sections:
- **Western Pyrenees**: Characterized by rugged terrain and steep valleys.
- **Central Pyrenees**: Known for its glacial landscapes and high peaks.
- **Eastern Pyrenees**: Features more rounded hills and a transition to the Mediterranean landscape.
- The range is primarily composed of granite, limestone, and schist rock formations.
## Climate
- The climate varies from oceanic in the west to Mediterranean in the east.
- Snowfall is common during the winter months, making it a popular destination for skiing and winter sports.
## Flora and Fauna
- The region is home to diverse ecosystems, featuring forests, meadows, and alpine tundra.
- Wildlife includes species such as the **Pyrenean ibex**, **brown bear**, **vultures**, and various endemic plants.
## Cultural Significance
- The Pyrenees have a rich history, with numerous prehistoric caves, Roman ruins, and medieval castles.
- The region is culturally significant for both France and Spain, with unique traditions, languages (such as **Occitan** and **Catalan**), and gastronomy.
## Outdoor Activities
- The Pyrenees are a popular destination for various activities including:
- **Hiking**: Numerous trails cater to different skill levels.
- **Skiing and Snowboarding**: Several ski resorts like **Saint-Lary-Soulan** and **Baqueira Beret**.
- **Climbing and Mountaineering**: Challenging routes attract climbers from around the world.
## National Parks
- Several national parks, including **Pyrenees National Park** in France and **Ordesa y Monte Perdido National Park** in Spain, protect this stunning natural environment and its biodiversity.
## Accessibility
- The Pyrenees can be accessed from various cities, including **Toulouse** and **Barcelona**, with numerous roads and hiking paths connecting different areas of the mountains.

33
week5/community-contributions/day 4 no_langchain/knowledge_collection/mountains/vosges.md

@ -0,0 +1,33 @@
# Vosges Mountains Overview
## Geography
- **Location**: Northeastern France, bordering Germany to the east.
- **Length**: Approximately 150 kilometers (93 miles) from north to south.
- **Elevation**: The highest peak is **Haut du Tôt**, which reaches an elevation of **1,424 meters** (4,672 feet).
## Natural Features
- **Landscape**: Characterized by rolling hills, dense forests, and numerous lakes and streams.
- **Geology**: Composed mainly of granite and sandstone, along with some limestone.
- **Flora and Fauna**: Home to diverse ecosystems, including coniferous and deciduous forests, and various wildlife such as deer, wild boar, and a range of bird species.
## Climate
- **Influence**: The Vosges mountains create a rainshadow effect, leading to varied climates on either side of the range.
- **Weather**: Generally humid, with abundant rainfall, particularly in the western slopes.
## Culture and History
- **Human Settlement**: Historically inhabited by Celtic tribes, later significant in both the Roman Empire and medieval periods.
- **Tourism**: Popular for hiking, skiing, and outdoor activities, with many marked trails and ski resorts.
- **Cultural Heritage**: Known for traditional villages, local cuisine, and the Alsace wine route.
## Notable Locations
- **Ballons des Vosges Regional Nature Park**: A protected area showcasing the natural beauty of the mountains.
- **Colmar and Gérardmer**: Prominent towns known for their cultural significance and as tourist destinations.
- **Route des Crêtes**: A scenic road that offers breathtaking views of the Vosges and surrounding regions.
## Activities
- **Hiking**: Numerous trails, including the famous GR5 long-distance path.
- **Skiing**: Various ski resorts, particularly in the higher altitudes.
- **Cycling**: The region is cyclist-friendly with several bike routes.
## Accessibility
- **Transport**: Well-connected by road and rail, making it accessible from major French cities and neighboring countries.

47
week5/community-contributions/day 4 no_langchain/knowledge_collection/regions/alsace_lorraine.md

@ -0,0 +1,47 @@
# Overview of Alsace-Lorraine Region in France
Alsace-Lorraine is a historically significant and culturally diverse region located in northeastern France. Known for its unique blend of French and German influences, the region has a fascinating history, charming towns, and beautiful landscapes.
## Geography
- **Location**: Situated along the Rhine River, Alsace-Lorraine borders Germany to the east and Luxembourg to the north. The region is part of the Grand Est administrative region of France.
- **Area**: Covers approximately 14,524 square kilometers.
- **Major Cities**: Strasbourg (capital of Alsace), Metz (capital of Lorraine), Mulhouse, Nancy, Colmar, and Epinal.
## History
- **German and French Control**: The region has alternated between French and German control multiple times, particularly during the 19th and 20th centuries. It was part of the German Empire from 1871 to 1918, and again during World War II, before returning to France after the war.
- **Franco-Prussian War (1870-1871)**: Alsace and most of Lorraine were ceded to Germany after France's defeat in the war. This period marked significant German cultural and linguistic influence.
- **Post-World War II**: After World War II, Alsace-Lorraine was definitively integrated into France, with the region's mixed identity still influencing its culture and language.
## Culture
- **Bilingualism**: The region has strong Germanic roots, and many people speak both French and a variety of regional dialects, such as Alsatian (a dialect of German). This bilingual heritage is reflected in the local culture, architecture, and cuisine.
- **Festivals**: Alsace-Lorraine is known for its rich tradition of festivals, especially those celebrating wine and food. The Strasbourg Christmas Market is one of the oldest and most famous in Europe.
- **Cuisine**: The region is renowned for its hearty and flavorful cuisine, which blends French and German influences. Notable dishes include choucroute (sauerkraut with sausages), tarte flambée (a type of pizza), and kugelhopf (a traditional cake).
- **Wine**: Alsace is one of the premier wine-producing regions in France, known for its white wines, particularly Riesling, Gewürztraminer, and Pinot Gris. The Alsace Wine Route is a popular tourist attraction.
## Natural Beauty
- **Vosges Mountains**: Located in Lorraine, the Vosges Mountains offer scenic landscapes, hiking trails, and ski resorts.
- **The Alsace Wine Route**: Stretching over 170 kilometers, this picturesque route offers breathtaking views of vineyards and charming villages.
- **Regional Parks**: The region is home to several natural parks, including the Ballons des Vosges Regional Nature Park, which features forests, lakes, and wildlife.
## Landmarks and Attractions
- **Strasbourg Cathedral**: The Cathedral of Notre-Dame in Strasbourg is a masterpiece of Gothic architecture and a UNESCO World Heritage site. Its astronomical clock and panoramic views from the tower are major attractions.
- **Château de Haut-Koenigsbourg**: A stunning medieval castle located in the Vosges Mountains, offering panoramic views of the Alsace plain.
- **Metz’s Cathedral**: The Cathedral of Saint-Étienne in Metz is a notable example of Gothic architecture, with some of the largest stained-glass windows in France.
- **Colmar**: Known for its well-preserved old town, Colmar is a charming medieval town with colorful half-timbered houses and canals that resemble a fairytale village.
## Economy
- **Industry**: Alsace-Lorraine has a diverse economy that includes manufacturing, automotive, chemicals, and electronics. The region is home to several large industrial companies, particularly in Strasbourg and Mulhouse.
- **Agriculture**: The region is known for its agricultural output, particularly in wine production, as well as fruit and vegetable farming.
- **Tourism**: With its rich history, picturesque landscapes, and cultural festivals, Alsace-Lorraine attracts millions of tourists each year.
## Climate
- **Continental Climate**: Alsace-Lorraine experiences a continental climate with cold winters and hot, often humid summers. The region’s proximity to the Vosges Mountains means it can also experience significant rainfall, particularly in Lorraine.
- **Average Temperatures**: Winters can see temperatures drop to around 0°C (32°F), while summer temperatures typically range from 18°C to 25°C (64°F to 77°F).
## Notable People
- **Jean-Jacques Rousseau**: The famous philosopher and writer was born in Geneva but spent much of his life in the region, influencing its intellectual culture.
- **Gérard Depardieu**: The internationally acclaimed French actor hails from Châteauroux but has connections to the region through his career and projects.
- **François Rabelais**: The influential Renaissance writer, known for his work *Gargantua and Pantagruel*, was born in the region.
## Conclusion
Alsace-Lorraine is a region with a rich, multifaceted history and culture, shaped by its unique position between France and Germany. Its charming towns, breathtaking landscapes, and exceptional food and wine make it a significant part of French heritage and a beloved destination for travelers.

47
week5/community-contributions/day 4 no_langchain/knowledge_collection/regions/bourgogne.md

@ -0,0 +1,47 @@
# Overview of Bourgogne (Burgundy) Region in France
Bourgogne, or Burgundy, is a historic and picturesque region located in eastern France. Known for its rich wine heritage, medieval towns, and stunning landscapes, Burgundy is a symbol of French culture and tradition.
## Geography
- **Location**: Bourgogne is located in central-eastern France, bordered by the regions of Franche-Comté, Rhône-Alpes, Auvergne, and Champagne-Ardenne.
- **Area**: Covers approximately 31,000 square kilometers.
- **Major Cities**: Dijon (capital), Auxerre, Beaune, Chalon-sur-Saône, Nevers.
## History
- **Duchy of Burgundy**: Burgundy was once an independent duchy, and during the Middle Ages, it was one of the most powerful and influential regions in France. It played a key role in European politics.
- **Unification with France**: In the 15th century, the Duchy of Burgundy became part of France after the death of the last Duke, Charles the Bold, in 1477. The region’s autonomy was gradually absorbed into the French crown.
- **Historical Significance**: Burgundy has a deep historical legacy, with numerous medieval abbeys, castles, and battlefields that have shaped the region’s identity.
## Culture
- **Wine Culture**: Burgundy is one of the world’s most famous wine-producing regions, renowned for its Pinot Noir and Chardonnay wines. The region’s vineyards produce some of the finest wines, especially in areas like Côte de Nuits, Côte de Beaune, and Chablis.
- **Cuisine**: Burgundy cuisine is rich and hearty, with dishes like boeuf bourguignon (beef stew in red wine), coq au vin (chicken cooked in wine), and escargots de Bourgogne (snails cooked in garlic and parsley butter). The region is also known for its mustard, particularly Dijon mustard.
- **Art and Architecture**: Burgundy is home to several historical and architectural landmarks, including Romanesque churches, medieval towns, and Renaissance palaces. The region has a long-standing tradition of art, with influences from both French and Flemish masters.
## Natural Beauty
- **Burgundy Canal**: The Burgundy Canal offers scenic views and is a popular spot for boaters and cyclists. It connects the Yonne River to the Saône River and passes through charming villages.
- **Morvan Regional Natural Park**: Located in the heart of Burgundy, the Morvan Park is known for its forests, lakes, and wildlife, making it a haven for outdoor enthusiasts.
- **Vineyards**: The rolling hills of the Burgundy vineyards are a UNESCO World Heritage site and are dotted with charming wine villages like Beaune and Meursault.
## Landmarks and Attractions
- **Dijon**: The capital of Burgundy, known for its well-preserved medieval architecture, the Palace of the Dukes of Burgundy, and the famous Dijon mustard.
- **Chablis**: Famous for its world-renowned white wines, Chablis is a picturesque village surrounded by vineyards and stunning views.
- **Abbey of Fontenay**: A UNESCO World Heritage site, this Cistercian abbey dates back to the 12th century and is an example of Romanesque architecture at its best.
- **Basilica of Vézelay**: Another UNESCO site, this basilica is a key pilgrimage site and an important example of Romanesque architecture in France.
- **Clos de Vougeot**: A historic wine estate and château in the Côte de Nuits, Clos de Vougeot is at the heart of Burgundy's wine heritage.
## Economy
- **Wine Industry**: Burgundy’s wine industry is the cornerstone of the region’s economy. The vineyards produce some of the world’s most sought-after wines, and the region is home to prestigious wine estates.
- **Agriculture**: In addition to wine production, Burgundy is also known for its agricultural output, including grain, dairy products, and livestock, especially cattle.
- **Tourism**: Burgundy attracts tourists for its wine tourism, beautiful landscapes, medieval towns, and rich history. The region is a popular destination for wine lovers, history buffs, and outdoor adventurers.
## Climate
- **Continental Climate**: Burgundy has a continental climate with hot summers and cold winters. The region’s climate is ideal for viticulture, with warm days during the growing season and cool nights that help preserve the flavors of the grapes.
- **Average Temperatures**: Summers typically range from 20°C to 28°C (68°F to 82°F), while winters can dip to around 0°C (32°F).
## Notable People
- **Gustave Eiffel**: Born in Dijon, Eiffel is famous for designing the Eiffel Tower in Paris.
- **Bernard Loiseau**: A renowned French chef from Burgundy, Loiseau was known for his exceptional culinary skills and Michelin-starred restaurants.
- **Romain Rolland**: The Nobel Prize-winning writer, known for his works such as *Jean-Christophe*, was born in Clamecy, Burgundy.
## Conclusion
Bourgogne is a region that embodies the essence of French culture, combining rich history, world-class wine, exceptional cuisine, and beautiful landscapes. Whether you’re savoring a glass of Burgundy wine, exploring its medieval towns, or hiking through its scenic parks, Burgundy offers a timeless experience for travelers and connoisseurs alike.

45
week5/community-contributions/day 4 no_langchain/knowledge_collection/regions/bretagne.md

@ -0,0 +1,45 @@
# Overview of Bretagne (Brittany) Region in France
Bretagne, or Brittany, is a culturally distinct region located in the northwest of France. Known for its rugged coastline, rich history, and unique cultural heritage, Bretagne offers a fascinating blend of natural beauty and ancient traditions.
## Geography
- **Location**: Situated on the Brittany Peninsula, bordered by the English Channel to the north, the Atlantic Ocean to the west and south, and the Normandy and Pays de la Loire regions to the east.
- **Area**: Covers approximately 27,208 square kilometers.
- **Major Cities**: Rennes (capital), Brest, Nantes, Saint-Malo, Quimper, Lorient.
## History
- **Celtic Origins**: Originally inhabited by the Celts, who brought their language, traditions, and culture to the region. Bretagne still maintains a strong Celtic identity.
- **Duchy of Brittany**: From the 9th to the 16th century, Brittany was an independent duchy before joining France in 1532.
- **Breton Language**: Breton (Brezhoneg) is a Celtic language still spoken by a small population, especially in rural areas and in cultural events.
## Culture
- **Music**: Bretagne is known for its traditional Celtic music, including bagpipes, fiddles, and the bombard. The region hosts festivals like the Festival Interceltique de Lorient, which celebrates Celtic culture.
- **Cuisine**: The local cuisine includes specialties like crêpes, galettes (buckwheat pancakes), seafood, and cider (known as "cidre"). The region is famous for its oysters and mussels.
- **Festivals**: Brittany hosts several cultural festivals, such as the Fest Noz, a traditional Breton dance event, and the Breizh Festival, which celebrates Breton culture.
## Natural Beauty
- **Coastline**: Bretagne is known for its stunning coastline with dramatic cliffs, sandy beaches, and picturesque coves. The region has more than 2,700 kilometers of coastline.
- **Mont Saint-Michel**: While technically in Normandy, it is often associated with Brittany due to its proximity. This island commune with a striking abbey is a UNESCO World Heritage site.
- **Regional Parks**: Brittany is home to several regional natural parks, such as the Armorique Regional Nature Park, known for its varied landscapes, including moors, forests, and hills.
## Landmarks and Attractions
- **Carnac Stones**: Prehistoric standing stones dating back to the Neolithic period, located in the town of Carnac. They are among the most famous megalithic sites in the world.
- **Fort La Latte**: A medieval fortress on the north coast of Brittany, offering incredible views of the sea.
- **Saint-Malo**: A walled port city, famous for its cobblestone streets, stunning beaches, and historical significance as a center of piracy.
## Economy
- **Agriculture**: The region is known for its dairy farming, particularly in the production of butter and cheese. Bretagne is also famous for its apple orchards, which are used to make cider.
- **Fishing**: Historically, Brittany has been one of the most important fishing regions in France, especially for shellfish, sardines, and tuna.
- **Tourism**: The natural beauty, history, and culture make Bretagne a popular destination for tourists, with significant income coming from visitors.
## Climate
- **Mild Climate**: Brittany experiences a temperate maritime climate, characterized by mild winters and cool summers. The region is known for frequent rainfall and variable weather.
- **Average Temperatures**: Winters rarely drop below 5°C (41°F), while summers range from 15°C to 20°C (59°F to 68°F).
## Notable People
- **Bertrand Du Guesclin**: A famous medieval French knight and national hero.
- **Jacques Cartier**: The explorer credited with claiming Canada for France in the 16th century.
- **Yann Tiersen**: A modern musician and composer, best known for his soundtrack for the film *Amélie*.
## Conclusion
Bretagne is a region of deep cultural significance, rich history, and extraordinary natural landscapes. Whether you’re drawn to its Celtic roots, its rugged coastline, or its historical landmarks, Brittany offers something for everyone.

47
week5/community-contributions/day 4 no_langchain/knowledge_collection/regions/gascogne.md

@ -0,0 +1,47 @@
# Overview of Gascogne Region in France
Gascogne is a historical and cultural region in southwestern France, known for its rolling hills, vineyards, charming villages, and rich heritage. It is often associated with the rustic lifestyle, gastronomy, and the famed Musketeers of Dumas’ novels.
## Geography
- **Location**: Situated in the southwest of France, Gascogne is bordered by the regions of Aquitaine to the west, Midi-Pyrénées to the south, and the Auvergne-Rhône-Alpes region to the east. It also touches the Pyrenees mountains to the south.
- **Area**: The region encompasses parts of the modern-day regions of Occitanie and Nouvelle-Aquitaine.
- **Major Cities**: Auch (historical capital), Agen, Condom, Lectoure, and Eauze.
## History
- **Roman Influence**: Gascogne was known as part of the ancient Roman province of Novempopulania. The region’s rich history is reflected in its architecture and ancient ruins.
- **Visigoths and Franks**: The region saw control by the Visigoths and later the Franks, whose influence shaped local customs and governance.
- **Duchy of Gascogne**: During the Middle Ages, Gascogne was an independent duchy before becoming part of the Kingdom of France in the 13th century.
- **The Musketeers**: Gascogne is famously associated with the “Three Musketeers” of Alexandre Dumas’ novels. The fictional characters D'Artagnan, Athos, Porthos, and Aramis are portrayed as hailing from this region.
## Culture
- **Gascon Language**: The Gascon language, a variety of Occitan, was historically spoken in the region. Though it has declined in use, it still carries cultural significance and is a symbol of regional identity.
- **Folk Traditions**: Gascogne is known for its folk traditions, including traditional music, dances, and festivals. The region is famous for its rural festivals, celebrating everything from local history to agricultural practices.
- **Cuisine**: Gascon cuisine is renowned for its hearty and flavorful dishes. Notable dishes include *foie gras*, *confit de canard* (duck confit), and *garbure* (a rich vegetable and meat soup). The region is also famous for its Armagnac, a brandy that is produced using traditional methods.
## Natural Beauty
- **Rolling Hills and Vineyards**: Gascogne is known for its picturesque landscapes, featuring rolling hills, vast forests, and scenic vineyards. The region is ideal for hiking, cycling, and exploring the rural countryside.
- **The Pyrenees**: The southern border of Gascogne is defined by the Pyrenees mountains, which offer opportunities for outdoor activities like hiking and skiing.
- **Rivers and Lakes**: Gascogne is crisscrossed by rivers such as the Garonne and the Adour, making the region fertile for agriculture and creating stunning natural scenery.
## Landmarks and Attractions
- **Auch Cathedral**: A UNESCO World Heritage site, the Cathedral of Sainte-Marie in Auch is an impressive Gothic structure with a magnificent staircase leading to the church.
- **D’Artagnan’s Birthplace**: The town of Lupiac, where D'Artagnan, the hero of Alexandre Dumas’ *The Three Musketeers*, was born, attracts fans of the novels and history alike.
- **Château de Larressingle**: Often referred to as one of the most beautiful fortified villages in France, this medieval castle offers a glimpse into the region's past.
- **Armagnac Distilleries**: Visitors can tour the distilleries that produce the famous Armagnac brandy, with opportunities to taste and learn about the traditional distilling process.
## Economy
- **Agriculture**: Gascogne is an important agricultural region, known for its production of ducks, geese (for foie gras), and pigs. The fertile soil supports the cultivation of corn, sunflowers, and grapes.
- **Wine and Brandy**: The region is famous for its vineyards and the production of Armagnac, a type of brandy. The wines of the region, especially those from the Côtes de Gascogne, are increasingly recognized for their quality.
- **Tourism**: With its rich history, natural beauty, and culinary traditions, Gascogne attracts tourists who are looking to experience authentic French rural life, enjoy local food and wine, and explore historical landmarks.
## Climate
- **Mediterranean Climate**: Gascogne enjoys a temperate climate, with warm summers and mild winters. The southern part of the region, near the Pyrenees, has a more Mediterranean climate, while the northern part experiences a more oceanic influence.
- **Average Temperatures**: Summer temperatures typically range from 20°C to 30°C (68°F to 86°F), while winters are generally mild with temperatures ranging from 5°C to 10°C (41°F to 50°F).
## Notable People
- **D'Artagnan**: The fictional hero of *The Three Musketeers*, D'Artagnan is one of the most famous characters associated with Gascogne, although based on a real person.
- **Charles de Batz-Castelmore d'Armanac**: The historical figure who inspired D'Artagnan, born in Gascogne, was a nobleman and soldier.
- **Henri IV**: The King of France, born in Pau (near Gascogne), famously said, “Paris is worth a Mass” and was instrumental in uniting France after years of religious conflict.
## Conclusion
Gascogne is a region that offers a unique blend of history, culture, and natural beauty. From its medieval villages and legendary connections to the Musketeers, to its rich culinary traditions and scenic landscapes, Gascogne provides a true taste of southwestern France. Whether exploring its vineyards, tasting Armagnac, or immersing yourself in its rural charm, Gascogne is a region full of life and tradition.

47
week5/community-contributions/day 4 no_langchain/knowledge_collection/regions/ile_de_france.md

@ -0,0 +1,47 @@
# Overview of Île-de-France Region in France
Île-de-France is the central region of France, encompassing the nation’s capital, Paris. As the political, economic, and cultural heart of France, this region is not only historically significant but also a global center for art, fashion, and business.
## Geography
- **Location**: Situated in the north-central part of France, Île-de-France is surrounded by the regions of Normandy, Hauts-de-France, Grand Est, Bourgogne-Franche-Comté, Centre-Val de Loire, and Provence-Alpes-Côte d'Azur.
- **Area**: Covers approximately 12,012 square kilometers.
- **Major Cities**: Paris (capital of both the region and France), Versailles, Créteil, Nanterre, and Montreuil.
## History
- **Royal Legacy**: Île-de-France has historically been the core of the French monarchy. It was the heart of the Capetian Dynasty, beginning in the 10th century. The region is home to many royal palaces and historic sites.
- **French Revolution**: Paris, located in Île-de-France, was the focal point of the French Revolution in the late 18th century. Important revolutionary events, such as the storming of the Bastille, took place here.
- **World War II**: During WWII, Paris was occupied by Nazi forces from 1940 to 1944. The city was liberated in August 1944 by Allied forces.
## Culture
- **Capital of Culture**: Paris is widely recognized as one of the world’s greatest cultural capitals. It is home to numerous world-class museums, theaters, and art galleries, including the Louvre, Musée d'Orsay, and the Centre Pompidou.
- **Fashion and Art**: Paris is the global capital of fashion, known for haute couture, and hosts prestigious fashion events like Paris Fashion Week. The city has also been the center of the art world for centuries, influencing movements such as Impressionism and Surrealism.
- **Gastronomy**: Île-de-France is known for its fine dining, with Michelin-starred restaurants, cafés, and bistros. The region is also famous for pâtisseries, including macarons and éclairs, and its traditional French dishes such as coq au vin and escargot.
## Natural Beauty
- **Seine River**: The Seine River flows through Paris and the Île-de-France region, providing beautiful riverbanks and parks, perfect for leisure activities like boat tours, picnicking, and walking along its iconic bridges.
- **Bois de Boulogne & Bois de Vincennes**: These expansive public parks on the outskirts of Paris offer lush green spaces for recreation, hiking, and cycling.
- **Versailles Gardens**: The Gardens of the Palace of Versailles, with their meticulously designed lawns, fountains, and sculptures, are a UNESCO World Heritage site and one of the most famous gardens in the world.
## Landmarks and Attractions
- **Eiffel Tower**: The most iconic landmark in Paris, the Eiffel Tower attracts millions of visitors every year. It stands as a symbol of France and offers stunning panoramic views of the city.
- **Notre-Dame Cathedral**: A masterpiece of Gothic architecture, the Notre-Dame Cathedral is one of the most famous religious sites in the world, located on the Île de la Cité in the Seine.
- **Palace of Versailles**: A short trip from Paris, the Palace of Versailles is one of the grandest royal palaces in Europe, famous for its opulent architecture and the Hall of Mirrors.
- **Sainte-Chapelle**: Known for its stunning stained-glass windows, this Gothic chapel in Paris is one of the most beautiful examples of medieval architecture.
- **The Louvre**: The world’s largest art museum, the Louvre in Paris, is home to thousands of works of art, including Leonardo da Vinci's *Mona Lisa* and the *Venus de Milo*.
## Economy
- **Economic Powerhouse**: Île-de-France is the economic center of France, contributing a significant portion to the country’s GDP. It is home to many multinational companies and is the main business hub in France.
- **Finance and Technology**: The region has a thriving financial sector centered in La Défense, Paris’s business district. It also hosts tech startups and innovations, particularly in areas like AI, fintech, and digital media.
- **Tourism**: Paris is one of the world’s top tourist destinations, attracting millions of visitors each year. The region’s tourism is a key driver of the economy, with tourists coming for the history, culture, and attractions.
## Climate
- **Oceanic Climate**: Île-de-France experiences a temperate oceanic climate with mild winters and warm summers. Paris typically has rainy weather in the autumn and spring, with summer temperatures ranging from 18°C to 25°C (64°F to 77°F).
- **Average Temperatures**: Winter temperatures can hover around 3°C to 7°C (37°F to 45°F), while summer highs can range from 25°C to 30°C (77°F to 86°F).
## Notable People
- **Napoleon Bonaparte**: Born on the island of Corsica, Napoleon became the Emperor of France and played a pivotal role in shaping the history of France and Europe. His influence is still felt throughout Île-de-France.
- **Marcel Proust**: The famous French writer, best known for his work *In Search of Lost Time*, lived and wrote in Paris during the late 19th and early 20th centuries.
- **Édith Piaf**: One of France’s most beloved singers, Piaf was born and raised in Paris and became an international icon of French music.
## Conclusion
Île-de-France is the heart of France, blending rich history, cultural innovation, and economic power. With Paris at its center, the region is a global leader in fashion, art, and business. From historic landmarks like the Eiffel Tower and Versailles to its world-class museums and gastronomic delights, Île-de-France is a region that offers something for every visitor, making it a must-see destination for travelers.

46
week5/community-contributions/day 4 no_langchain/knowledge_collection/regions/languedoc.md

@ -0,0 +1,46 @@
# Overview of Languedoc Region in France
Languedoc is a historic and culturally rich region located in the southern part of France, known for its Mediterranean coastline, picturesque villages, and deep-rooted traditions. It is often celebrated for its wines, beaches, and beautiful landscapes.
## Geography
- **Location**: Languedoc is situated in the southernmost part of France, bordered by the Mediterranean Sea to the east, the regions of Provence-Alpes-Côte d'Azur, Rhône-Alpes, and Auvergne-Rhône-Alpes to the north, and Midi-Pyrénées to the west.
- **Area**: Covers approximately 27,000 square kilometers.
- **Major Cities**: Montpellier (capital), Nîmes, Perpignan, Carcassonne, Béziers, and Sète.
## History
- **Roman Influence**: Languedoc has a strong Roman heritage, with many ancient ruins, including the well-preserved Roman aqueduct, Pont du Gard, and the ancient city of Nîmes.
- **Cathar History**: In the Middle Ages, Languedoc was the center of the Cathar religious movement. The region was the focus of the Albigensian Crusade (1209-1229), a military campaign aimed at eradicating Catharism.
- **Rural Culture**: Historically, the region was a center of agriculture and viticulture, and it remains deeply connected to farming traditions, particularly wine production.
## Culture
- **Language**: The Occitan language, historically spoken in the region, was once widely used, and it still carries cultural significance today. Languedoc’s name itself derives from the Occitan phrase *"langue d'oc,"* meaning “language of yes.”
- **Cuisine**: Languedoc cuisine is characterized by its Mediterranean influence, with seafood, olive oil, and fresh produce playing a central role. Famous dishes include *cassoulet* (a rich stew made with beans and meats), *brandade de morue* (a cod and garlic dish), and *tapenade* (olive spread).
- **Festivals**: The region is known for its vibrant festivals, such as the Feria de Nîmes, which celebrates bullfighting and the culture of southern France, and the Carcassonne Festival, which features music, theater, and other arts.
## Natural Beauty
- **Mediterranean Coast**: The region boasts a stunning coastline along the Mediterranean Sea, with beautiful beaches like those in Cap d'Agde and the scenic Étang de Thau.
- **Languedoc-Roussillon Wine Route**: The Languedoc region is one of the largest wine-producing areas in France, and its wine route takes visitors through vineyards, picturesque villages, and wine estates.
- **Cévennes National Park**: This UNESCO-listed park is part of the Massif Central and offers stunning mountain landscapes, gorges, and wildlife, ideal for hiking and nature lovers.
## Landmarks and Attractions
- **Carcassonne**: A UNESCO World Heritage site, the medieval fortress of Carcassonne is one of France’s most iconic landmarks. The double-walled citadel offers a glimpse into the past with its preserved medieval architecture.
- **Pont du Gard**: A well-preserved Roman aqueduct, the Pont du Gard is a UNESCO World Heritage site and an engineering marvel of antiquity, offering scenic views of the surrounding landscape.
- **Nîmes**: Known as the "French Rome," Nîmes is home to remarkable Roman monuments, including the Arena of Nîmes (a Roman amphitheater), the Temple of Diana, and the Maison Carrée.
- **Sète**: A picturesque coastal town known for its canals, seafood, and vibrant cultural scene, Sète is often referred to as the "Venice of Languedoc."
- **Abbey of Saint-Guilhem-le-Désert**: This UNESCO World Heritage site is a well-preserved medieval abbey located in the stunning Hérault Valley.
## Economy
- **Wine Production**: Languedoc is one of the largest wine-producing regions in France, known for producing a wide variety of wines, including reds, whites, and rosés. The region is famous for its *AOC* (Appellation d'Origine Contrôlée) wines, such as those from the Minervois, Faugères, and Corbières appellations.
- **Agriculture**: In addition to wine, Languedoc is known for producing fruits (particularly melons, peaches, and cherries), olives, and lavender. It is also a significant producer of sheep and goat cheese.
- **Tourism**: With its Mediterranean coastline, historic cities, and scenic landscapes, Languedoc is a popular tourist destination. The region’s vineyards and charming towns attract visitors for wine tourism, cultural exploration, and outdoor activities.
## Climate
- **Mediterranean Climate**: Languedoc enjoys a Mediterranean climate, characterized by hot, dry summers and mild, wet winters. The region’s climate is perfect for vineyards and outdoor activities.
- **Average Temperatures**: Summer temperatures typically range from 25°C to 35°C (77°F to 95°F), while winters are mild, with temperatures ranging from 8°C to 15°C (46°F to 59°F).
## Notable People
- **Georges Brassens**: The famous French singer-songwriter and poet was born in Sète, and his legacy is celebrated in the town with a museum and annual festivals.
- **Pierre-Paul Riquet**: The engineer who designed the Canal du Midi, which connects the Garonne River to the Mediterranean, greatly impacting the region’s agriculture and trade during the 17th century.
## Conclusion
Languedoc is a region rich in history, culture, and natural beauty. From its Roman heritage and medieval fortresses to its beautiful beaches and vineyards, Languedoc offers a unique blend of ancient traditions and modern charm. Whether you’re enjoying a glass of wine, exploring historic towns, or relaxing by the sea, Languedoc provides an unforgettable experience for travelers.

48
week5/community-contributions/day 4 no_langchain/knowledge_collection/regions/normandie.md

@ -0,0 +1,48 @@
# Overview of Normandie Region in France
Normandie (Normandy) is a historic and picturesque region located in the northern part of France. Known for its dramatic coastline, rich history, and cultural heritage, Normandy plays a central role in both French and world history.
## Geography
- **Location**: Situated in the northernmost part of France, Normandy is bordered by the English Channel to the north, the regions of Île-de-France, Centre-Val de Loire, and Pays de la Loire to the south, and Brittany to the west.
- **Area**: Covers approximately 29,907 square kilometers.
- **Major Cities**: Rouen (capital), Caen, Le Havre, Cherbourg, and Dieppe.
## History
- **Viking Heritage**: Normandy gets its name from the Norsemen (Vikings), who settled in the region in the 9th and 10th centuries. The region became known as "Normandy" after the Vikings (Normans) were granted land by the King of France.
- **William the Conqueror**: One of the most famous historical figures associated with Normandy is William the Conqueror, who, as Duke of Normandy, successfully invaded England in 1066 and became the King of England.
- **D-Day and WWII**: Normandy is internationally known for the D-Day landings on June 6, 1944, during World War II. The Allied invasion of Normandy was a pivotal event in the liberation of Western Europe from Nazi occupation. The beaches, such as Omaha Beach and Utah Beach, are significant historical sites.
## Culture
- **Language**: The regional language of Normandy is Norman, a variety of the Old French language with influences from Old Norse. However, French is the primary language spoken today.
- **Cuisine**: Normandy cuisine is influenced by its coastal location, featuring seafood like oysters, mussels, and scallops. The region is also famous for its apples, which are used to make cider (cidre) and the famous apple brandy, Calvados. Dishes such as *coquilles Saint-Jacques* (scallops) and *camembert cheese* are iconic.
- **Folk Traditions**: The region is known for its folk traditions, including festivals, music, and dances that celebrate its Viking and maritime heritage.
## Natural Beauty
- **Dramatic Coastline**: Normandy is known for its stunning coastline, including cliffs, sandy beaches, and small coves. The cliffs at Etretat are among the most photographed natural sites in France.
- **Normandy Beaches**: Famous for their historical significance, Normandy’s beaches are also a popular destination for travelers. The beaches of Omaha, Utah, and Juno were sites of the D-Day landings.
- **Countryside and Farming**: Normandy is also known for its green countryside, dotted with rolling hills, fields, and traditional farmhouses. The region's fertile land is perfect for the production of dairy products, apples, and crops.
## Landmarks and Attractions
- **Mont Saint-Michel**: A UNESCO World Heritage site, Mont Saint-Michel is one of France’s most iconic landmarks. This island commune features a medieval abbey perched atop a rocky hill, surrounded by tidal waters, creating a stunning visual.
- **D-Day Landing Beaches**: The beaches where the D-Day landings took place, such as Utah Beach, Omaha Beach, and Sword Beach, are significant historical sites and are home to several museums, memorials, and cemeteries dedicated to the soldiers who fought there.
- **Rouen Cathedral**: A masterpiece of Gothic architecture, the Rouen Cathedral is famous for its stunning facade and for being the subject of a series of paintings by Claude Monet.
- **Château de Caen**: Built by William the Conqueror in the 11th century, this castle in Caen is one of the largest medieval fortresses in Europe.
- **Jardin des Plantes de Rouen**: A botanical garden in Rouen that showcases a variety of plant species, it is a great place to explore nature and relax.
## Economy
- **Agriculture**: Normandy is a major agricultural region, known for dairy farming, particularly the production of butter and cheese. The region is famous for its dairy products, with cheeses like Camembert, Livarot, and Pont-l’Évêque being integral to the local economy.
- **Cider Production**: Normandy is one of the primary cider-producing regions in France, with a long tradition of apple orchards. The region’s cider is often made from a variety of apples, resulting in dry, sweet, or sparkling ciders.
- **Fishing and Maritime**: The region’s location along the English Channel makes it a significant player in France’s fishing industry. Ports like Le Havre and Cherbourg are vital to the French maritime economy.
- **Tourism**: With its rich historical sites, picturesque countryside, and seaside attractions, Normandy is a popular tourist destination, drawing visitors to its beaches, memorials, and unique landmarks.
## Climate
- **Oceanic Climate**: Normandy enjoys an oceanic climate, with mild winters and cool summers. The weather is influenced by the proximity to the English Channel, often resulting in cloudy, rainy days.
- **Average Temperatures**: Summers generally range from 18°C to 22°C (64°F to 72°F), while winters are mild, with temperatures ranging from 3°C to 7°C (37°F to 45°F).
## Notable People
- **William the Conqueror**: Born in Falaise, Normandy, William the Conqueror is one of the most famous figures in history, known for his conquest of England in 1066.
- **Joan of Arc**: A national heroine of France, Joan of Arc was born in Domrémy, which was then part of Normandy, and played a significant role in the Hundred Years' War.
- **Gustave Flaubert**: The renowned French writer, best known for his novel *Madame Bovary*, was born in Rouen, Normandy.
## Conclusion
Normandy is a region rich in history, culture, and natural beauty. From the stunning Mont Saint-Michel and the beaches of the D-Day landings to the pastoral landscapes and delicious cuisine, Normandy offers a mix of historical depth and natural charm. Whether exploring its historic towns, enjoying fresh seafood and cider, or paying tribute to its WWII heritage, Normandy provides a unique and unforgettable experience.

48
week5/community-contributions/day 4 no_langchain/knowledge_collection/regions/poitou.md

@ -0,0 +1,48 @@
# Overview of Poitou Region in France
Poitou is a historic region located in the western part of France, known for its rich cultural heritage, beautiful landscapes, and historical significance. Today, it forms part of the Nouvelle-Aquitaine region, but it retains its unique identity through its history, architecture, and traditions.
## Geography
- **Location**: Poitou is situated in the western part of France, bordered by the Atlantic Ocean to the west, the regions of Pays de la Loire to the north, Aquitaine to the south, and Centre-Val de Loire to the east.
- **Area**: Covers approximately 10,000 square kilometers.
- **Major Cities**: Poitiers (capital), La Rochelle, Niort, and Châtellerault.
## History
- **Medieval Influence**: Poitou was an important region during the medieval period, especially known for its connection to the powerful counts of Poitou and the Dukes of Aquitaine. The region was also the birthplace of Eleanor of Aquitaine, one of the most influential women of the medieval period.
- **Anglo-French Conflict**: Poitou played a significant role during the Hundred Years' War, with both the English and the French vying for control of the region. It was once part of the Angevin Empire, which included large parts of modern-day France and England.
- **Renaissance and Religious Wars**: During the Renaissance, Poitou became a center for intellectual and cultural development. It also saw significant involvement in the Wars of Religion between Catholics and Protestants in the 16th century.
## Culture
- **Language**: The traditional language of Poitou is Poitevin, a variety of the Occitan language, which was widely spoken in the region in medieval times. However, French is predominantly spoken today.
- **Cuisine**: Poitou cuisine is characterized by its use of fresh local ingredients, with specialties such as *mogettes* (white beans), *salmis* (a stew of game), and the region’s famous cheeses, including *Chabichou du Poitou*, a soft, creamy goat cheese. The region is also known for its seafood, particularly oysters from the Marennes-Oléron area.
- **Folk Traditions**: Poitou has a rich tradition of folk music and dance, with regional festivals celebrating the local culture. The region’s craft heritage, including pottery, woodwork, and textiles, continues to be celebrated.
## Natural Beauty
- **Atlantic Coast**: Poitou has a beautiful coastline along the Atlantic Ocean, with scenic beaches and coastal landscapes. The island of Île de Ré, accessible by bridge from La Rochelle, is a popular destination for its charming villages, vineyards, and sandy beaches.
- **Marais Poitevin**: Also known as the “Green Venice,” the Marais Poitevin is a vast marshland and wetland area that is crisscrossed with canals. It is a paradise for nature lovers, offering opportunities for boating, birdwatching, and hiking.
- **Countryside**: The region also features gentle rolling hills, vineyards, and forests. The Poitou-Charentes region is known for its peaceful, rural landscapes, making it ideal for outdoor activities like cycling, hiking, and nature walks.
## Landmarks and Attractions
- **Poitiers**: The historic city of Poitiers is famous for its medieval architecture, including the Church of Saint-Hilaire-le-Grand, a UNESCO World Heritage site, and the Palais des Ducs d'Aquitaine, a former royal palace.
- **La Rochelle**: Known for its well-preserved Old Port, La Rochelle is a charming coastal town with a rich maritime history. The city's landmarks include the iconic La Rochelle Towers and the Maritime Museum.
- **Futuroscope**: Located near Poitiers, Futuroscope is one of France’s most popular theme parks, offering futuristic attractions, multimedia shows, and cutting-edge technology exhibitions.
- **Île de Ré**: This picturesque island is known for its beautiful beaches, historic lighthouses, and charming villages. It is a popular vacation spot for tourists seeking relaxation and outdoor activities.
- **Château de Niort**: This medieval fortress in Niort dates back to the 12th century and offers visitors a glimpse into the region’s medieval history.
## Economy
- **Agriculture**: Poitou is traditionally an agricultural region, known for its livestock farming, particularly the production of Charolais cattle, as well as the cultivation of cereals, potatoes, and sunflowers. The region also produces a variety of fruits, including apples and grapes.
- **Wine Production**: The region is part of the larger wine-growing area of Charentes, which is famous for producing Cognac, a renowned brandy. The vineyards of the Charente and Charente-Maritime departments are integral to the local economy.
- **Tourism**: Poitou’s rich history, natural beauty, and charming cities attract many tourists. La Rochelle, Poitiers, and Île de Ré are major tourist destinations, while the Marais Poitevin and the coastal areas draw those interested in nature and outdoor activities.
- **Cognac Production**: Poitou is at the heart of the Cognac-producing region, with many distilleries located around the Charente River, where the famous spirit is made from grapes and aged for years in oak barrels.
## Climate
- **Oceanic Climate**: Poitou enjoys an oceanic climate with mild winters and warm summers, influenced by the Atlantic Ocean. Coastal areas experience more moderate temperatures, while inland regions can have slightly warmer summers.
- **Average Temperatures**: Summer temperatures typically range from 18°C to 25°C (64°F to 77°F), while winters are generally mild, with temperatures ranging from 5°C to 10°C (41°F to 50°F).
## Notable People
- **Eleanor of Aquitaine**: Born in Poitou, Eleanor was one of the most powerful and influential women in medieval Europe. She was Queen of France and later Queen of England and played a key role in the politics of both kingdoms.
- **François Rabelais**: The famous Renaissance writer, best known for his satirical work *Gargantua and Pantagruel*, was born in the Poitou region, and his works remain an important part of French literature.
- **René Descartes**: One of the most influential philosophers of the 17th century, Descartes spent much of his early life in Poitou, and his legacy continues to shape modern philosophy.
## Conclusion
Poitou is a region rich in history, culture, and natural beauty. From its medieval towns and historic landmarks to its picturesque countryside and coastal beauty, Poitou offers a unique blend of traditions and modern attractions. Whether exploring the city of Poitiers, enjoying the fresh produce and local wine, or relaxing on the beaches of Île de Ré, Poitou provides an unforgettable experience for visitors.

50
week5/community-contributions/day 4 no_langchain/knowledge_collection/regions/provence.md

@ -0,0 +1,50 @@
# Overview of Provence Region in France
Provence is a stunning region in the southeastern part of France, renowned for its breathtaking landscapes, rich history, vibrant culture, and Mediterranean climate. It is one of the most beloved regions in France, known for its lavender fields, vineyards, ancient Roman ruins, and charming villages.
## Geography
- **Location**: Provence is located in the southeastern part of France, bordered by the Mediterranean Sea to the south, the Rhône River to the west, the Alps to the north, and the region of Côte d'Azur to the east.
- **Area**: Covers approximately 31,400 square kilometers.
- **Major Cities**: Marseille (capital), Aix-en-Provence, Avignon, Arles, and Toulon.
## History
- **Roman Heritage**: Provence has a rich Roman history, with the city of Arles serving as a significant Roman settlement. The region is home to some of the best-preserved Roman monuments in France, including the Arena of Nîmes and the Pont du Gard.
- **Medieval Influence**: Provence was part of the Kingdom of Arles in the Middle Ages and later became a major part of the Comtat Venaissin. It was also home to the Papacy for a time, with the popes residing in Avignon from 1309 to 1377.
- **Renaissance and Revolution**: Provence was a key region during the Renaissance, flourishing in the arts and culture. During the French Revolution, Provence played a significant role, with several uprisings and political changes.
## Culture
- **Language**: The traditional language of Provence is Provençal, a variety of the Occitan language. While French is predominantly spoken today, Provençal still has cultural significance and is used in regional poetry, music, and literature.
- **Cuisine**: Provence is famous for its Mediterranean cuisine, emphasizing fresh vegetables, olive oil, herbs, seafood, and wine. Popular dishes include *bouillabaisse* (a fish stew), *ratatouille* (vegetable medley), *tapenade* (olive paste), and *pissaladière* (onion tart).
- **Wine**: The region is renowned for its wine production, particularly rosé wines from the Côtes de Provence, as well as reds and whites. The vineyards of Provence benefit from the Mediterranean climate, producing wines with distinctive flavors.
- **Folk Traditions**: Provence is known for its rich folk traditions, including festivals, music, dance, and crafts. The region celebrates a variety of traditional events, such as the Festival of the Calissons in Aix-en-Provence, and the Fête de la Lavande (Lavender Festival) in Sault.
## Natural Beauty
- **Mediterranean Coast**: Provence boasts a beautiful coastline along the Mediterranean, with stunning beaches, rocky coves, and picturesque seaside towns such as Cassis, Sainte-Maxime, and Bandol.
- **Lavender Fields**: The lavender fields of Provence are one of the region's most iconic features. The fields bloom in vibrant purple hues during the summer months and are a major tourist attraction.
- **Alps and Vineyards**: To the north of Provence, the landscape rises into the Alps, offering spectacular mountain scenery, hiking, and skiing opportunities. The rolling hills and vineyards of the region produce some of the finest wines in France.
- **Gorges du Verdon**: Known as the "Grand Canyon of Europe," the Gorges du Verdon is a breathtaking river canyon with turquoise waters, cliffs, and stunning landscapes. It is a popular destination for outdoor activities like hiking, kayaking, and rock climbing.
## Landmarks and Attractions
- **Palace of the Popes (Palais des Papes)**: Located in Avignon, this UNESCO World Heritage site is one of the largest and most important medieval Gothic buildings in Europe. It was the residence of popes during the 14th century.
- **Pont du Gard**: An ancient Roman aqueduct bridge located near Nîmes, the Pont du Gard is a UNESCO World Heritage site and an engineering marvel.
- **Roman Arena of Nîmes**: One of the best-preserved Roman amphitheaters, the Arena of Nîmes in Nîmes is still used for events today, including bullfights and concerts.
- **Château des Baux-de-Provence**: A ruined medieval castle perched atop the hills of Les Baux-de-Provence, offering panoramic views of the surrounding landscape.
- **Cassis and Calanques National Park**: The seaside town of Cassis is famous for its beautiful harbor and access to the Calanques National Park, a stunning area of limestone cliffs, turquoise waters, and hidden coves.
## Economy
- **Agriculture**: Provence is known for its agricultural production, including the cultivation of olives, lavender, tomatoes, and herbs such as thyme and rosemary. Olive oil production is a key industry, and the region’s lavender fields are famous worldwide.
- **Wine Production**: Provence is one of the most important wine regions in France, especially known for its rosé wines. Vineyards are spread throughout the region, including areas like Côtes de Provence, Bandol, and Cassis.
- **Tourism**: Tourism is a major part of Provence's economy, with millions of visitors flocking to the region for its beaches, lavender fields, Roman ruins, and charming towns. The region’s Mediterranean climate and picturesque landscapes make it a year-round destination.
- **Crafts and Industry**: Provence is known for its artisanal crafts, such as pottery, textiles, and perfume making, particularly in the town of Grasse, which is renowned as the perfume capital of the world.
## Climate
- **Mediterranean Climate**: Provence enjoys a Mediterranean climate, characterized by hot, dry summers and mild, wet winters. This climate is ideal for growing grapes, olives, and lavender, and contributes to the region’s appeal as a tourist destination.
- **Average Temperatures**: Summers are typically hot, with temperatures ranging from 25°C to 35°C (77°F to 95°F), while winters are mild, with temperatures ranging from 5°C to 15°C (41°F to 59°F).
## Notable People
- **Paul Cézanne**: A famous Post-Impressionist painter, Cézanne was born in Aix-en-Provence and is closely associated with the landscapes of the region. His works, particularly those depicting the Mont Sainte-Victoire mountain, are iconic in the art world.
- **Marcel Pagnol**: A renowned writer, playwright, and filmmaker, Pagnol was born in Aubagne and is known for his works about Provençal life, including *Marius*, *Fanny*, and *César*, as well as his memoirs.
- **Vincent van Gogh**: The Dutch painter spent a year in the town of Saint-Rémy-de-Provence, where he produced some of his most famous works, including *Starry Night* and *Irises*.
## Conclusion
Provence is a region that captivates with its stunning landscapes, rich history, and vibrant culture. From the lavender fields and Mediterranean beaches to the Roman ruins and charming villages, Provence offers something for everyone. Whether you're visiting for the cuisine, the wine, the history, or simply to relax in its beautiful surroundings, Provence is a timeless and unforgettable destination.

636
week6/community-contributions/week6_day2_add_validation_set.ipynb

@ -0,0 +1,636 @@
{
"cells": [
{
"cell_type": "markdown",
"id": "28a0673e-96b5-43f2-8a8b-bd033bf851b0",
"metadata": {},
"source": [
"# Add a Validation Set\n",
"\n",
"In the lecture, we created a curated dataset with **400,000 training items** and **2,000 test items**, but we did not include a validation (dev) set. This notebook demonstrates how to take Ed Donner’s dataset, [ed-donner/pricer-data](https://huggingface.co/datasets/ed-donner/pricer-data), and add a dev set to it.\n",
"\n",
"> **Note**: This notebook heavily uses snippets from the lectures’ `day2.ipynb` of Week 6.\n",
"\n",
"**Download the Updated Dataset**: \n",
"You can find the resulting dataset here: [antonawinkler/pricer-data](https://huggingface.co/datasets/antonawinkler/pricer-data)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "67cedf85-8125-4322-998e-9375fe745597",
"metadata": {},
"outputs": [],
"source": [
"# imports\n",
"\n",
"# Standard libraries\n",
"import os\n",
"import random\n",
"from itertools import chain\n",
"from collections import Counter, defaultdict\n",
"\n",
"# Third-party libraries\n",
"from dotenv import load_dotenv\n",
"from huggingface_hub import login\n",
"from datasets import concatenate_datasets, load_dataset, Dataset, DatasetDict\n",
"import matplotlib.pyplot as plt\n",
"import numpy as np\n",
"\n",
"# Local modules\n",
"from items import Item\n",
"from loaders import ItemLoader\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "7390a6aa-79cb-4dea-b6d7-de7e4b13e472",
"metadata": {},
"outputs": [],
"source": [
"# environment\n",
"\n",
"load_dotenv()\n",
"os.environ['HF_TOKEN'] = os.getenv('HF_TOKEN', 'your-key-if-not-using-env')"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0732274a-aa6a-44fc-aee2-40dc8a8e4451",
"metadata": {},
"outputs": [],
"source": [
"# Log in to HuggingFace\n",
"\n",
"hf_token = os.environ['HF_TOKEN']\n",
"login(hf_token, add_to_git_credential=True)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1adcf323-de9d-4c24-a9c3-d7ae554d06ca",
"metadata": {},
"outputs": [],
"source": [
"%matplotlib inline"
]
},
{
"cell_type": "markdown",
"id": "e2b6dc50-ac5c-4cf2-af2e-968ed8ef86d7",
"metadata": {},
"source": [
"## Load the Original Dataset\n",
"\n",
"Load the original data from McAuley-Lab/Amazon-Reviews-2023."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d1d06cd3-f3c2-44f0-a9f2-13b54ff8be5c",
"metadata": {},
"outputs": [],
"source": [
"dataset_names = [\n",
" \"Automotive\",\n",
" \"Electronics\",\n",
" \"Office_Products\",\n",
" \"Tools_and_Home_Improvement\",\n",
" \"Cell_Phones_and_Accessories\",\n",
" \"Toys_and_Games\",\n",
" \"Appliances\",\n",
" \"Musical_Instruments\",\n",
"]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "aa8fd0f0-509a-4298-8fcc-e499a061e1be",
"metadata": {},
"outputs": [],
"source": [
"items = []\n",
"for dataset_name in dataset_names:\n",
" loader = ItemLoader(dataset_name)\n",
" items.extend(loader.load())"
]
},
{
"cell_type": "markdown",
"id": "bf6b6b66-4a4b-41c2-b366-1f598cf18351",
"metadata": {},
"source": [
"# Create Balanced Dataset\n",
"\n",
"We apply the balancing algorithm from the course."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "549a4bad-abe7-4d36-ad77-fc70ba0f151c",
"metadata": {},
"outputs": [],
"source": [
"slots = defaultdict(list)\n",
"for item in items:\n",
" slots[round(item.price)].append(item)\n",
"\n",
"np.random.seed(42)\n",
"random.seed(42)\n",
"sample = []\n",
"for i in range(1, 1000):\n",
" slot = slots[i]\n",
" if i>=240:\n",
" sample.extend(slot)\n",
" elif len(slot) <= 1200:\n",
" sample.extend(slot)\n",
" else:\n",
" weights = np.array([1 if item.category=='Automotive' else 5 for item in slot])\n",
" weights = weights / np.sum(weights)\n",
" selected_indices = np.random.choice(len(slot), size=1200, replace=False, p=weights)\n",
" selected = [slot[i] for i in selected_indices]\n",
" sample.extend(selected)\n",
"\n",
"print(f\"There are {len(sample):,} items in the sample\")"
]
},
{
"cell_type": "markdown",
"id": "04280d2b-210a-4fad-9163-1b32a87fb990",
"metadata": {},
"source": [
"The output I get is `There are 408,635 items in the sample`\n",
"\n",
"Since there are 400,000 items in the train set of ed-donner/pricer-data, we can aim for a 98/1/1 split."
]
},
{
"cell_type": "markdown",
"id": "0d1e2836-0cae-4496-a5d4-d80bc14d566b",
"metadata": {},
"source": [
"## Load Ed Donner's Pricer Data Set"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a84e5a71-fc44-4cdf-9bc2-c69f80b8ee94",
"metadata": {},
"outputs": [],
"source": [
"dataset_ori = load_dataset(\"ed-donner/pricer-data\")\n",
"train_ori = dataset_ori['train']\n",
"test_ori = dataset_ori['test']"
]
},
{
"cell_type": "markdown",
"id": "e9c5c877-3d30-4013-9d0f-1e490755afeb",
"metadata": {},
"source": [
"## Observation 1: Order of the Data Has Changed\n",
"\n",
"`dataset_without_devset` should be a subset of `sample`. The order however can be different. Let us check this.\n",
"\n",
"I see different results for the following two cells below, indicating that the order has changed."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "56ad8682-4d7f-4aad-9976-96eb6d9b4a5a",
"metadata": {},
"outputs": [],
"source": [
"sample[0].prompt"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3e29a5ab-ca61-41cc-9b33-22d374681b85",
"metadata": {},
"outputs": [],
"source": [
"train_ori[0]['text']"
]
},
{
"cell_type": "markdown",
"id": "469a5b3c-c1a2-461d-a88d-27aa08905b31",
"metadata": {},
"source": [
"## Observation 2: Duplicate Items\n",
"\n",
"As an further challenge, the dataset shows duplicates with identical scrubbed descriptions. For some of these duplicates the prices are identical too (I see 1774), for others they differ (I see 6747).\n",
"\n",
"> **Note**: Below we use `defaultdict(list)` instead of `set` because it allows to inspect duplicates easily."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "94adffe8-edf6-4503-9f8f-34e4dfd29da9",
"metadata": {},
"outputs": [],
"source": [
"PRICE_IS = \"\\n\\nPrice is $\"\n",
"def get_key(text, price):\n",
" prefix, price_is, _price_nearest_dollar = text.partition(PRICE_IS)\n",
" return f\"{prefix}{price_is}{price}\"\n",
"def get_key_without_price(text):\n",
" prefix, price_is, _price_nearest_dollar = text.partition(PRICE_IS)\n",
" return f\"{prefix}\"\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a015ba1b-69e0-4651-850f-d93d3f078d16",
"metadata": {},
"outputs": [],
"source": [
"# Identify duplicates by text+price\n",
"train_ori_dict = defaultdict(list)\n",
"for datapoint in train_ori:\n",
" # Creates a key from the text and price (scrubbed)\n",
" key = get_key(datapoint[\"text\"], datapoint[\"price\"])\n",
" train_ori_dict[key].append(datapoint)\n",
"\n",
"# Number of exact duplicates (same text AND same price)\n",
"exact_duplicates = len(train_ori) - len(train_ori_dict)\n",
"print(f\"There are {exact_duplicates} duplicates with the same description and price.\")\n",
"\n",
"# Identify duplicates by text alone (ignoring price)\n",
"train_ori_dict_no_price = defaultdict(list)\n",
"for datapoint in train_ori:\n",
" key_no_price = get_key_without_price(datapoint[\"text\"])\n",
" train_ori_dict_no_price[key_no_price].append(datapoint)\n",
"\n",
"# Number of duplicates that differ in price but share the same text\n",
"different_price_duplicates = len(train_ori_dict) - len(train_ori_dict_no_price)\n",
"print(f\"In addition, there are {different_price_duplicates} data points where the description is duplicated but the price is different.\")\n",
"\n",
"# Total number of duplicates if we consider text alone\n",
"overall_duplicates = len(train_ori) - len(train_ori_dict_no_price)\n",
"print(f\"Overall number of duplicates: {overall_duplicates}\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e577dd8b-be0f-4ab0-b45f-9d3459b1286a",
"metadata": {},
"outputs": [],
"source": [
"test_ori_dict = defaultdict(list)\n",
"for datapoint in test_ori:\n",
" key = get_key(datapoint['text'], datapoint['price'])\n",
" test_ori_dict[key].append(datapoint)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0198fc23-0825-4ce1-a961-1d390d86cbdc",
"metadata": {},
"outputs": [],
"source": [
"sample_dict = defaultdict(list)\n",
"for datapoint in sample:\n",
" key = get_key(datapoint.prompt, datapoint.price)\n",
" sample_dict[key].append(datapoint)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "37f24d22-51ef-472b-8c73-e969637fa925",
"metadata": {},
"outputs": [],
"source": [
"# Check if all data points in train_ori/test_ori are included in the new sample_dict.\n",
"missing = []\n",
"count_found = 0\n",
"\n",
"for datapoint in chain(train_ori, test_ori):\n",
" key = get_key(datapoint[\"text\"], datapoint[\"price\"])\n",
" if key not in sample_dict:\n",
" missing.append(datapoint)\n",
" else:\n",
" count_found += 1\n",
"\n",
"print(f\"We found {count_found} datapoints in sample_dict.\")\n",
"print(f\"We are missing {len(missing)} datapoints that are not present in sample_dict.\")"
]
},
{
"cell_type": "markdown",
"id": "60c9d186-c688-4559-9b51-f0045d16829b",
"metadata": {},
"source": [
"Expected output of the previous cell\n",
"```\n",
"We found 402000 datapoints in sample_dict.\n",
"We are missing 0 datapoints that are not present in sample_dict.\n",
"```"
]
},
{
"cell_type": "markdown",
"id": "3b05e22d-a755-4ee5-a18b-620f7ab1df8f",
"metadata": {},
"source": [
"## Add Data Points to the Test and Validation Sets\n",
"\n",
"Since we can match all data points in the original train and test sets from `ed-donner/pricer-data`, we’ll now incorporate any *unused* items from our balanced sample into the test set and create a new validation (dev) set. Our goal is to achieve a **98/1/1** split for train, validation, and test."
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "16638cf9-03c3-46bc-8116-cafdd9e23ac9",
"metadata": {},
"outputs": [],
"source": [
"sample_not_used_yet = [datapoint for key in sample_dict.keys() - train_ori_dict.keys() - test_ori_dict.keys() for datapoint in sample_dict[key]]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "58a593ad-29a1-4b35-9753-45db75e09666",
"metadata": {},
"outputs": [],
"source": [
"# As a santity check, let us visually verify that the distribution of sample_still_available is in line with the complete sample.\n",
"\n",
"# Plot the distribution of prices in sample\n",
"def plot_price_distribution(items, name):\n",
" prices = [float(item.price) for item in items]\n",
" plt.figure(figsize=(15, 10))\n",
" plt.title(f\"{name} - Avg {sum(prices)/len(prices):.2f} and highest {max(prices):,.2f}\\n\")\n",
" plt.xlabel('Price ($)')\n",
" plt.ylabel('Count')\n",
" # see https://stackoverflow.com/questions/57026223/how-to-re-scale-the-counts-in-a-matplotlib-histogram\n",
" (counts, bins) = np.histogram(prices, bins=range(0, 1000, 10))\n",
" plt.hist(bins[:-1], color=\"darkblue\", bins=bins, weights=counts/len(prices))\n",
" plt.show() \n",
"\n",
"\n",
"def plot_category_distribution(items, name):\n",
" category_counts = Counter()\n",
" for item in items:\n",
" category_counts[item.category]+=1\n",
" categories = sorted(category_counts.keys())\n",
" counts = [category_counts[category] for category in categories]\n",
"\n",
" # plot a pie chart\n",
" plt.figure(figsize=(12, 10))\n",
" plt.pie(counts, labels=categories, autopct='%1.0f%%', startangle=90)\n",
" \n",
" # Add a circle at the center to create a donut chart (optional)\n",
" centre_circle = plt.Circle((0,0), 0.70, fc='white')\n",
" fig = plt.gcf()\n",
" fig.gca().add_artist(centre_circle)\n",
" plt.title(f'{name} - Categories')\n",
" \n",
" # Equal aspect ratio ensures that pie is drawn as a circle\n",
" plt.axis('equal') \n",
"\n",
" plt.show()\n",
"plot_price_distribution(sample, 'Complete set')\n",
"plot_price_distribution(sample_not_used_yet, 'Not used yet')\n",
"plot_category_distribution(sample, 'Complete set')\n",
"plot_category_distribution(sample_not_used_yet, 'Not used yet')"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ba252265-b976-426a-aefc-ebc93b153fd4",
"metadata": {},
"outputs": [],
"source": [
"# now add the unused items to the validation and test set\n",
"random.seed(42)\n",
"random.shuffle(sample_not_used_yet)\n",
"validation_items = sample_not_used_yet[:4000]\n",
"added_test_items = sample_not_used_yet[4000:]\n",
"\n",
"# create Huggingface dataset\n",
"validation_dataset = Dataset.from_dict({\"text\": [item.prompt for item in validation_items], \"price\": [item.price for item in validation_items]})\n",
"added_test_dataset = Dataset.from_dict({\"text\": [item.prompt for item in added_test_items], \"price\": [item.price for item in added_test_items]})\n",
"\n",
"dataset = DatasetDict({\n",
" \"train\": train_ori,\n",
" \"test\": concatenate_datasets([test_ori, added_test_dataset]),\n",
" \"validation\": validation_dataset,\n",
"})\n",
"\n",
"print(f\"Divided into a training set of {dataset['train'].num_rows:,} items, a validation set of {dataset['validation'].num_rows:,} items, and a test set of {dataset['test'].num_rows:,} items\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c39ac5d7-84f8-4f7d-98e1-d24651ba3a80",
"metadata": {},
"outputs": [],
"source": [
"# If you're ready to push to the hub, and fill in the dots with your HF username\n",
"\n",
"HF_USER = ...\n",
"DATASET_NAME = f\"{HF_USER}/pricer-data\"\n",
"dataset.push_to_hub(DATASET_NAME, private=True)"
]
},
{
"cell_type": "markdown",
"id": "3fcb2492-ef2a-468e-8bf1-deb18eef4d9c",
"metadata": {},
"source": [
"## Use of Validation Sets\n",
"\n",
"When you train your model in Week 7.\n",
"\n",
"```python\n",
"# load the train and validation set\n",
"train = load_dataset(DATASET_NAME, split='train[:100%]') # or less than 100%\n",
"validation = load_dataset(DATASET_NAME, split='validation[:100%]') # or less than 100% \n",
"\n",
"# Define training parameters\n",
"train_parameters = SFTConfig(\n",
" eval_strategy=\"steps\", # or \"epoch\"\n",
" eval_steps=EVAL_STEPS,\n",
" ...\n",
")\n",
"\n",
"# Initialize fine-tuning with validation set\n",
"fine_tuning = SFTTrainer(\n",
" eval_dataset=validation,\n",
" ...\n",
")\n",
"```"
]
},
{
"cell_type": "markdown",
"id": "bceb4407-d91d-4731-9e96-189f6f953cbc",
"metadata": {},
"source": [
"## A Closer Look at the Duplicates\n",
"\n",
"We have now created a dataset that includes a validation set and additional test data. During this process, we observed that **2% of the data contains duplicates**, where the scrubbed descriptions are identical.\n",
"\n",
"Duplicates can contribute to model overfitting. However, since only **2% of the dataset is duplicated**, the impact is likely minimal. Moreover, many of these duplicates actually refer to different physical objects rather than being true duplicates.\n",
"\n",
"### False Duplicates\n",
"\n",
"The “duplicates” we observe are often not duplicates in the original dataset. Minor differences in product descriptions may be removed by the scrubbing process, leading to items that *appear* identical but aren’t. For example:\n",
"\n",
"```\n",
"<RinoGear Screen Protector Designed for Sony Xperia XZ Screen Protector Case Friendly Accessories Flexible Full Coverage Clear TPU Film = $0.95>\n",
"<RinoGear (2-Pack) Screen Protector Designed for Sony Xperia XZ Screen Protector Case Friendly Accessories Flexible Full Coverage Clear TPU Film = $2.95>\n",
"```\n",
"\"(2-Pack)\" is removed in the scrub method.\n",
"\n",
"Similarly:\n",
"```\n",
"[<EBC Brakes USR7115 USR Series Sport Slotted Rotor = $31.22>,\n",
" <EBC Brakes USR7314 USR Series Sport Slotted Rotor = $71.46>,\n",
" <EBC Brakes USR7409 USR Series Sport Slotted Rotor = $88.67>,\n",
"...\n",
" <EBC Brakes USR7305 USR Series Sport Slotted Rotor = $406.55>,\n",
" <EBC Brakes USR7384 USR Series Sport Slotted Rotor = $413.61>,\n",
" <EBC Brakes USR1602 USR Series Sport Slotted Rotor = $615.1>]\n",
"```\n",
"These all represent different rotor models. \n",
"\n",
"**Even when both the scrubbed text and the price are identical**, the items may still refer to distinct products. For instance:\n",
"```\n",
"<5304486359 Refrigerator Door Handles Set Replacement for Frigidaire FFTR1821QW5A Refrigerator - Compatible with 5304486359 White Door Handles - UpStart Components Brand = $17.99>\n",
"<5304486359 Refrigerator Door Handles Set Replacement for Frigidaire FFTR1831QP1 Refrigerator - Compatible with 5304486359 White Door Handles - UpStart Components Brand = $17.99>\n",
"```\n",
"\n",
"### True Duplicates\n",
"Finding *true* duplicates—where the scrubbed text, price, and underlying real-world product match—seems relatively rare. The following items in the **Appliances** set, for instance, likely refer to the same physical product:\n",
"```python\n",
"{'main_category': 'Tools & Home Improvement',\n",
" 'title': 'Whirlpool 8318084 Lid Switch for Washer',\n",
" 'average_rating': 4.6,\n",
" 'rating_number': 511,\n",
" 'features': ['Works with the following models: Whirlpool 1CLBR5432PQ0, Whirlpool 1CLBR5432PQ1, Whirlpool 1CLSQ9549PG0',\n",
" 'This products adds a great value',\n",
" 'This product is manufactured in United States',\n",
" 'Works with the following models: Whirlpool 1CLBR5432PQ0, Whirlpool 1CLBR5432PQ1, Whirlpool 1CLSQ9549PG0',\n",
" 'Whirlpool 1CLSQ9549PG1, Whirlpool 1CLSQ9549PW0',\n",
" 'Whirlpool 1CLSQ9549PW1, Whirlpool 1CLSR7010PQ0',\n",
" 'Whirlpool 1CLSR7010PQ1, Whirlpool 1CLSR7300PQ0',\n",
" 'Genuine Replacement Part'],\n",
" 'description': ['Product Description',\n",
" 'Part Number 8318084 (AP3180933) replaces 1018522, AH886960, EA886960, PS886960., Easy to use and handle. This products adds a great value This product is manufactured in United States.',\n",
" 'From the Manufacturer',\n",
" 'Whirlpool 8318084 Lid Switch for Washer. Works with the following models: Whirlpool 1CLBR5432PQ0, Whirlpool 1CLBR5432PQ1, Whirlpool 1CLSQ9549PG0, Whirlpool 1CLSQ9549PG1, Whirlpool 1CLSQ9549PW0, Whirlpool 1CLSQ9549PW1, Whirlpool 1CLSR7010PQ0, Whirlpool 1CLSR7010PQ1, Whirlpool 1CLSR7300PQ0. Genuine Replacement Part.'],\n",
" 'price': '25.55',\n",
" 'images': {'hi_res': [None],\n",
" 'large': ['https://m.media-amazon.com/images/I/31QE91zX0mL._AC_.jpg'],\n",
" 'thumb': ['https://m.media-amazon.com/images/I/31QE91zX0mL._AC_US75_.jpg'],\n",
" 'variant': ['MAIN']},\n",
" 'videos': {'title': [\"Your Washer Won't Spin?\", '8318084 Washer Lid Switch'],\n",
" 'url': ['https://www.amazon.com/vdp/09c00a975b4b46198b5703483f424981?ref=dp_vse_rvc_0',\n",
" 'https://www.amazon.com/vdp/3c9b3dc3c93444978d542af3fab13c49?ref=dp_vse_rvc_1'],\n",
" 'user_id': ['', '']},\n",
" 'store': 'Whirlpool',\n",
" 'categories': ['Appliances',\n",
" 'Parts & Accessories',\n",
" 'Washer Parts & Accessories'],\n",
" 'details': '{\"Manufacturer\": \"Whirlpool\", \"Part Number\": \"8318084\", \"Item Weight\": \"1.34 ounces\", \"Product Dimensions\": \"3 x 2 x 2 inches\", \"Item model number\": \"8318084\", \"Is Discontinued By Manufacturer\": \"No\", \"Item Package Quantity\": \"1\", \"Included Components\": \"Kkk\", \"Batteries Included?\": \"No\", \"Batteries Required?\": \"No\", \"Warranty Description\": \"Kk\", \"Best Sellers Rank\": {\"Tools & Home Improvement\": 231142, \"Washer Parts & Accessories\": 1074}, \"Date First Available\": \"August 7, 2008\"}',\n",
" 'parent_asin': 'B01CT25N26',\n",
" 'bought_together': None,\n",
" 'subtitle': None,\n",
" 'author': None}\n",
"\n",
"{'main_category': 'Tools & Home Improvement',\n",
" 'title': 'Whirlpool 8318084 Lid Switch for Washer',\n",
" 'average_rating': 4.6,\n",
" 'rating_number': 514,\n",
" 'features': ['Works with the following models: Whirlpool 1CLBR5432PQ0, Whirlpool 1CLBR5432PQ1, Whirlpool 1CLSQ9549PG0',\n",
" 'This products adds a great value',\n",
" 'This product is manufactured in United States',\n",
" 'Works with the following models: Whirlpool 1CLBR5432PQ0, Whirlpool 1CLBR5432PQ1, Whirlpool 1CLSQ9549PG0',\n",
" 'Whirlpool 1CLSQ9549PG1, Whirlpool 1CLSQ9549PW0',\n",
" 'Whirlpool 1CLSQ9549PW1, Whirlpool 1CLSR7010PQ0',\n",
" 'Whirlpool 1CLSR7010PQ1, Whirlpool 1CLSR7300PQ0',\n",
" 'Genuine Replacement Part'],\n",
" 'description': ['Product Description',\n",
" 'Part Number 8318084 (AP3180933) replaces 1018522, AH886960, EA886960, PS886960., Easy to use and handle. This products adds a great value This product is manufactured in United States.',\n",
" 'From the Manufacturer',\n",
" 'Whirlpool 8318084 Lid Switch for Washer. Works with the following models: Whirlpool 1CLBR5432PQ0, Whirlpool 1CLBR5432PQ1, Whirlpool 1CLSQ9549PG0, Whirlpool 1CLSQ9549PG1, Whirlpool 1CLSQ9549PW0, Whirlpool 1CLSQ9549PW1, Whirlpool 1CLSR7010PQ0, Whirlpool 1CLSR7010PQ1, Whirlpool 1CLSR7300PQ0. Genuine Replacement Part.'],\n",
" 'price': '25.55',\n",
" 'images': {'hi_res': [None],\n",
" 'large': ['https://m.media-amazon.com/images/I/31QE91zX0mL._AC_.jpg'],\n",
" 'thumb': ['https://m.media-amazon.com/images/I/31QE91zX0mL._AC_US75_.jpg'],\n",
" 'variant': ['MAIN']},\n",
" 'videos': {'title': ['AMI PARTS,Parts Specialist'],\n",
" 'url': ['https://www.amazon.com/vdp/09a12ea79b1a4081a18909825437760b?ref=dp_vse_rvc_0'],\n",
" 'user_id': ['']},\n",
" 'store': 'Whirlpool',\n",
" 'categories': ['Appliances',\n",
" 'Parts & Accessories',\n",
" 'Washer Parts & Accessories'],\n",
" 'details': '{\"Manufacturer\": \"Whirlpool\", \"Part Number\": \"8318084\", \"Item Weight\": \"1.34 ounces\", \"Product Dimensions\": \"3 x 2 x 2 inches\", \"Item model number\": \"8318084\", \"Is Discontinued By Manufacturer\": \"No\", \"Item Package Quantity\": \"1\", \"Included Components\": \"kkk\", \"Batteries Included?\": \"No\", \"Batteries Required?\": \"No\", \"Warranty Description\": \"kk\", \"Best Sellers Rank\": {\"Tools & Home Improvement\": 166821, \"Washer Parts & Accessories\": 684}, \"Date First Available\": \"August 7, 2008\"}',\n",
" 'parent_asin': 'B0050O1UR8',\n",
" 'bought_together': None,\n",
" 'subtitle': None,\n",
" 'author': None}\n",
"```\n",
"\n",
"### Takeaway\n",
"2% of the dataset contains duplicates, but most of these represent different physical objects. It does not appear to be worthwhile to remove them from the dataset. In fact it can be better the keep them to have representative data.\n"
]
},
{
"cell_type": "markdown",
"id": "0a1d7b72-a1ab-4fc4-9065-738bd11f8058",
"metadata": {},
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "403a42a2-3913-4905-9475-97509fe86c5e",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.9"
}
},
"nbformat": 4,
"nbformat_minor": 5
}
Loading…
Cancel
Save