1 changed files with 405 additions and 0 deletions
@ -0,0 +1,405 @@ |
|||||||
|
{ |
||||||
|
"cells": [ |
||||||
|
{ |
||||||
|
"cell_type": "markdown", |
||||||
|
"id": "dfe37963-1af6-44fc-a841-8e462443f5e6", |
||||||
|
"metadata": {}, |
||||||
|
"source": [ |
||||||
|
"## This notebook compares the embeddings generated by OpenAIEmbeddings.\n", |
||||||
|
"\n", |
||||||
|
"It shows that OpenAIEmbeddings embeddings can differ slightly (typically at 4 the decimal place).\n", |
||||||
|
"\n", |
||||||
|
"### Results from OpenAIEmbeddings:\n", |
||||||
|
"encodings are NOT identical on each run.\n", |
||||||
|
"\n", |
||||||
|
"### Repeating with sentence-transformers/all-MiniLM-L6-v2:\n", |
||||||
|
"encodings ARE identical on each run.\n", |
||||||
|
"\n", |
||||||
|
"Tests verify simple numerical comparisons.\n", |
||||||
|
"\n", |
||||||
|
"### Advanced Comparison\n", |
||||||
|
"A more advanced euclidean and cosine comparison is also included.\n", |
||||||
|
"\n", |
||||||
|
"## NOTES: Tests run on local Jupiter Notebook| Anaconda setup for the course." |
||||||
|
] |
||||||
|
}, |
||||||
|
{ |
||||||
|
"cell_type": "code", |
||||||
|
"execution_count": null, |
||||||
|
"id": "ba2779af-84ef-4227-9e9e-6eaf0df87e77", |
||||||
|
"metadata": {}, |
||||||
|
"outputs": [], |
||||||
|
"source": [ |
||||||
|
"# imports\n", |
||||||
|
"\n", |
||||||
|
"import os\n", |
||||||
|
"import glob\n", |
||||||
|
"from dotenv import load_dotenv\n", |
||||||
|
"import gradio as gr" |
||||||
|
] |
||||||
|
}, |
||||||
|
{ |
||||||
|
"cell_type": "code", |
||||||
|
"execution_count": null, |
||||||
|
"id": "802137aa-8a74-45e0-a487-d1974927d7ca", |
||||||
|
"metadata": {}, |
||||||
|
"outputs": [], |
||||||
|
"source": [ |
||||||
|
"# imports for langchain\n", |
||||||
|
"\n", |
||||||
|
"from langchain.document_loaders import DirectoryLoader, TextLoader\n", |
||||||
|
"from langchain.text_splitter import CharacterTextSplitter\n", |
||||||
|
"from langchain.schema import Document\n", |
||||||
|
"from langchain_openai import OpenAIEmbeddings, ChatOpenAI\n", |
||||||
|
"from langchain_chroma import Chroma\n", |
||||||
|
"import numpy as np\n", |
||||||
|
"from sklearn.manifold import TSNE\n", |
||||||
|
"import plotly.graph_objects as go\n", |
||||||
|
"from langchain.memory import ConversationBufferMemory\n", |
||||||
|
"from langchain.chains import ConversationalRetrievalChain" |
||||||
|
] |
||||||
|
}, |
||||||
|
{ |
||||||
|
"cell_type": "code", |
||||||
|
"execution_count": null, |
||||||
|
"id": "58c85082-e417-4708-9efe-81a5d55d1424", |
||||||
|
"metadata": {}, |
||||||
|
"outputs": [], |
||||||
|
"source": [ |
||||||
|
"# price is a factor for our company, so we're going to use a low cost model\n", |
||||||
|
"\n", |
||||||
|
"MODEL = \"gpt-4o-mini\"\n", |
||||||
|
"db_name = \"vector_db\"" |
||||||
|
] |
||||||
|
}, |
||||||
|
{ |
||||||
|
"cell_type": "code", |
||||||
|
"execution_count": null, |
||||||
|
"id": "ee78efcb-60fe-449e-a944-40bab26261af", |
||||||
|
"metadata": {}, |
||||||
|
"outputs": [], |
||||||
|
"source": [ |
||||||
|
"# Load environment variables in a file called .env\n", |
||||||
|
"\n", |
||||||
|
"load_dotenv()\n", |
||||||
|
"os.environ['OPENAI_API_KEY'] = os.getenv('OPENAI_API_KEY', 'your-key-if-not-using-env')" |
||||||
|
] |
||||||
|
}, |
||||||
|
{ |
||||||
|
"cell_type": "code", |
||||||
|
"execution_count": null, |
||||||
|
"id": "730711a9-6ffe-4eee-8f48-d6cfb7314905", |
||||||
|
"metadata": {}, |
||||||
|
"outputs": [], |
||||||
|
"source": [ |
||||||
|
"# Read in documents using LangChain's loaders\n", |
||||||
|
"# Take everything in all the sub-folders of our knowledgebase\n", |
||||||
|
"\n", |
||||||
|
"folders = glob.glob(\"knowledge-base/*\")\n", |
||||||
|
"\n", |
||||||
|
"# With thanks to CG and Jon R, students on the course, for this fix needed for some users \n", |
||||||
|
"text_loader_kwargs = {'encoding': 'utf-8'}\n", |
||||||
|
"# If that doesn't work, some Windows users might need to uncomment the next line instead\n", |
||||||
|
"# text_loader_kwargs={'autodetect_encoding': True}\n", |
||||||
|
"\n", |
||||||
|
"documents = []\n", |
||||||
|
"for folder in folders:\n", |
||||||
|
" doc_type = os.path.basename(folder)\n", |
||||||
|
" loader = DirectoryLoader(folder, glob=\"**/*.md\", loader_cls=TextLoader, loader_kwargs=text_loader_kwargs)\n", |
||||||
|
" folder_docs = loader.load()\n", |
||||||
|
" for doc in folder_docs:\n", |
||||||
|
" doc.metadata[\"doc_type\"] = doc_type\n", |
||||||
|
" documents.append(doc)" |
||||||
|
] |
||||||
|
}, |
||||||
|
{ |
||||||
|
"cell_type": "code", |
||||||
|
"execution_count": null, |
||||||
|
"id": "7310c9c8-03c1-4efc-a104-5e89aec6db1a", |
||||||
|
"metadata": {}, |
||||||
|
"outputs": [], |
||||||
|
"source": [ |
||||||
|
"text_splitter = CharacterTextSplitter(chunk_size=1000, chunk_overlap=200)\n", |
||||||
|
"chunks = text_splitter.split_documents(documents)" |
||||||
|
] |
||||||
|
}, |
||||||
|
{ |
||||||
|
"cell_type": "code", |
||||||
|
"execution_count": null, |
||||||
|
"id": "cd06e02f-6d9b-44cc-a43d-e1faa8acc7bb", |
||||||
|
"metadata": {}, |
||||||
|
"outputs": [], |
||||||
|
"source": [ |
||||||
|
"len(chunks)" |
||||||
|
] |
||||||
|
}, |
||||||
|
{ |
||||||
|
"cell_type": "code", |
||||||
|
"execution_count": null, |
||||||
|
"id": "2c54b4b6-06da-463d-bee7-4dd456c2b887", |
||||||
|
"metadata": {}, |
||||||
|
"outputs": [], |
||||||
|
"source": [ |
||||||
|
"doc_types = set(chunk.metadata['doc_type'] for chunk in chunks)\n", |
||||||
|
"print(f\"Document types found: {', '.join(doc_types)}\")" |
||||||
|
] |
||||||
|
}, |
||||||
|
{ |
||||||
|
"cell_type": "code", |
||||||
|
"execution_count": null, |
||||||
|
"id": "a8b5ef27-70c2-4111-bce7-854bc1ebd02a", |
||||||
|
"metadata": {}, |
||||||
|
"outputs": [], |
||||||
|
"source": [ |
||||||
|
"# Use a where filter to specify the metadata condition\n", |
||||||
|
"# Get the 3 company vectors (corresponds to our 3 yellow dots)\n", |
||||||
|
"\n", |
||||||
|
"def get_company_vectors(collection):\n", |
||||||
|
" company_vectors = collection.get(\n", |
||||||
|
" where={\"doc_type\": \"company\"}, # Filter for documents where source = \"XXXX\"\n", |
||||||
|
" limit=10,\n", |
||||||
|
" include=[\"embeddings\", \"metadatas\", \"documents\"]\n", |
||||||
|
" )\n", |
||||||
|
" print(f\"Found {len(company_vectors)} company vectors\")\n", |
||||||
|
" return company_vectors\n" |
||||||
|
] |
||||||
|
}, |
||||||
|
{ |
||||||
|
"cell_type": "code", |
||||||
|
"execution_count": null, |
||||||
|
"id": "d688b873-b52b-4d80-9df2-f70b389f5dc7", |
||||||
|
"metadata": {}, |
||||||
|
"outputs": [], |
||||||
|
"source": [ |
||||||
|
"\n", |
||||||
|
"def print_vectors_summary(vectors):\n", |
||||||
|
" for i in range(len(vectors[\"documents\"])):\n", |
||||||
|
" print(f\"\\n--- Chunk {i+1} ---\")\n", |
||||||
|
" \n", |
||||||
|
" # Print document content (first 100 chars)\n", |
||||||
|
" print(f\"Content: {vectors['documents'][i][:100]}...\")\n", |
||||||
|
" \n", |
||||||
|
" # Print metadata\n", |
||||||
|
" print(f\"Metadata: {vectors['metadatas'][i]}\")\n", |
||||||
|
" \n", |
||||||
|
" # Print embedding info (not the full vector as it would be too long)\n", |
||||||
|
" embedding = vectors[\"embeddings\"][i]\n", |
||||||
|
" print(f\"Embedding: Vector of length {len(embedding)}, first 5 values: {embedding[:5]}\")\n", |
||||||
|
"\n", |
||||||
|
"\n", |
||||||
|
"def get_dimensions_for_vectors(vectors):\n", |
||||||
|
" dimensions = []\n", |
||||||
|
"\n", |
||||||
|
" for i in range(len(vectors[\"documents\"])):\n", |
||||||
|
" embedding = vectors[\"embeddings\"][i]\n", |
||||||
|
" dimensions.append(embedding)\n", |
||||||
|
"\n", |
||||||
|
" return dimensions\n" |
||||||
|
] |
||||||
|
}, |
||||||
|
{ |
||||||
|
"cell_type": "code", |
||||||
|
"execution_count": null, |
||||||
|
"id": "0b195184-4920-404a-9bfa-0231f1dbe276", |
||||||
|
"metadata": {}, |
||||||
|
"outputs": [], |
||||||
|
"source": [ |
||||||
|
"# Quick check if any single value is different\n", |
||||||
|
"def quick_diff_check(emb1, emb2):\n", |
||||||
|
" result = \"Embeddings are identical\"\n", |
||||||
|
" print(\"\\n\\nComparing two embeddings:\\n\\n\")\n", |
||||||
|
" print(emb1)\n", |
||||||
|
" print(emb2)\n", |
||||||
|
" for i, (v1, v2) in enumerate(zip(emb1, emb2)):\n", |
||||||
|
" if v1 != v2:\n", |
||||||
|
" result = f\"Different at dimension {i}: {v1} vs {v2}\"\n", |
||||||
|
" break\n", |
||||||
|
" print(result)\n", |
||||||
|
" return result\n", |
||||||
|
"\n", |
||||||
|
"#quick_diff_check(dimensions[0], dimensions[1])" |
||||||
|
] |
||||||
|
}, |
||||||
|
{ |
||||||
|
"cell_type": "code", |
||||||
|
"execution_count": null, |
||||||
|
"id": "06ba838d-d179-4e2d-b208-dd9cc1fd0097", |
||||||
|
"metadata": {}, |
||||||
|
"outputs": [], |
||||||
|
"source": [ |
||||||
|
"\n", |
||||||
|
"embeddings = OpenAIEmbeddings()\n", |
||||||
|
"\n", |
||||||
|
"def create_vectorstores(embeddings):\n", |
||||||
|
"\n", |
||||||
|
" if os.path.exists(\"vectorstore1\"):\n", |
||||||
|
" Chroma(persist_directory=\"vectorstore1\", embedding_function=embeddings).delete_collection()\n", |
||||||
|
" if os.path.exists(\"vectorstore2\"):\n", |
||||||
|
" Chroma(persist_directory=\"vectorstore2\", embedding_function=embeddings).delete_collection()\n", |
||||||
|
" \n", |
||||||
|
" \n", |
||||||
|
" # Create vectorstore 1\n", |
||||||
|
" vectorstore1 = Chroma.from_documents(documents=chunks, embedding=embeddings, persist_directory=\"vectorstore1\")\n", |
||||||
|
" print(f\"Vectorstore 1 created with {vectorstore1._collection.count()} documents\")\n", |
||||||
|
" \n", |
||||||
|
" # Create vectorstore 2\n", |
||||||
|
" vectorstore2 = Chroma.from_documents(documents=chunks, embedding=embeddings, persist_directory=\"vectorstore2\")\n", |
||||||
|
" print(f\"Vectorstore 2 created with {vectorstore2._collection.count()} documents\")\n", |
||||||
|
"\n", |
||||||
|
" return vectorstore1, vectorstore2\n", |
||||||
|
"\n", |
||||||
|
"vectorstore1, vectorstore2 = create_vectorstores(embeddings)\n", |
||||||
|
"\n" |
||||||
|
] |
||||||
|
}, |
||||||
|
{ |
||||||
|
"cell_type": "code", |
||||||
|
"execution_count": null, |
||||||
|
"id": "e24242eb-613a-4edb-a081-6b8937f106a7", |
||||||
|
"metadata": {}, |
||||||
|
"outputs": [], |
||||||
|
"source": [ |
||||||
|
"## Uncomment this and rerun cells below, \n", |
||||||
|
"## to see that HuggingFaceEmbeddings is idential\n", |
||||||
|
"\n", |
||||||
|
"#from langchain.embeddings import HuggingFaceEmbeddings\n", |
||||||
|
"#embeddings = HuggingFaceEmbeddings(model_name=\"sentence-transformers/all-MiniLM-L6-v2\")\n", |
||||||
|
"#vectorstore1, vectorstore2 = create_vectorstores(embeddings)\n" |
||||||
|
] |
||||||
|
}, |
||||||
|
{ |
||||||
|
"cell_type": "code", |
||||||
|
"execution_count": null, |
||||||
|
"id": "000b9e70-2958-40db-bbed-56a00e4249ce", |
||||||
|
"metadata": {}, |
||||||
|
"outputs": [], |
||||||
|
"source": [ |
||||||
|
"# Get the 3 company doc_type vectors\n", |
||||||
|
"collection1 = vectorstore1._collection\n", |
||||||
|
"collection2 = vectorstore2._collection\n", |
||||||
|
"\n", |
||||||
|
"company_vectors1=get_company_vectors(collection1)\n", |
||||||
|
"company_vectors2=get_company_vectors(collection2)\n" |
||||||
|
] |
||||||
|
}, |
||||||
|
{ |
||||||
|
"cell_type": "code", |
||||||
|
"execution_count": null, |
||||||
|
"id": "63cd63e4-9d3e-405a-8ef9-dac16fe2570e", |
||||||
|
"metadata": {}, |
||||||
|
"outputs": [], |
||||||
|
"source": [ |
||||||
|
"# Lets print out summary info just to see we have the same chunks.\n", |
||||||
|
"\n", |
||||||
|
"def print_summary_info (vectors):\n", |
||||||
|
" print(\"VECTORS SUMMARY\\n\")\n", |
||||||
|
" print_vectors_summary(vectors)\n", |
||||||
|
"\n", |
||||||
|
"\n", |
||||||
|
"print(\"\\n\\n\\n========= VECTORS 1 =========\\n\\n\")\n", |
||||||
|
"print_summary_info(company_vectors1)\n", |
||||||
|
"\n", |
||||||
|
"print(\"\\n\\n\\n========= VECTORS 2 =========\\n\\n\")\n", |
||||||
|
"print_summary_info(company_vectors2)\n", |
||||||
|
"\n", |
||||||
|
"\n" |
||||||
|
] |
||||||
|
}, |
||||||
|
{ |
||||||
|
"cell_type": "code", |
||||||
|
"execution_count": null, |
||||||
|
"id": "bc085a35-f0ec-4ddb-955c-244cb2d3eb2a", |
||||||
|
"metadata": {}, |
||||||
|
"outputs": [], |
||||||
|
"source": [ |
||||||
|
"dimensions1 = get_dimensions_for_vectors(company_vectors1)\n", |
||||||
|
"dimensions2 = get_dimensions_for_vectors(company_vectors2)\n", |
||||||
|
"\n", |
||||||
|
"result1 = quick_diff_check(dimensions1[0], dimensions2[0]) \n", |
||||||
|
"result2 = quick_diff_check(dimensions1[1], dimensions2[1]) \n", |
||||||
|
"result3 = quick_diff_check(dimensions1[2], dimensions2[2]) \n", |
||||||
|
"\n", |
||||||
|
"print(\"\\n\\nSUMMARY RESULTS:\")\n", |
||||||
|
"print(\"================\\n\\n\")\n", |
||||||
|
"print(result1) \n", |
||||||
|
"print(result2)\n", |
||||||
|
"print(result3)\n" |
||||||
|
] |
||||||
|
}, |
||||||
|
{ |
||||||
|
"cell_type": "code", |
||||||
|
"execution_count": null, |
||||||
|
"id": "164cf94d-9d63-4bae-91f9-4b02da1537ae", |
||||||
|
"metadata": {}, |
||||||
|
"outputs": [], |
||||||
|
"source": [ |
||||||
|
"## ADVANCED COMPARISONS:\n", |
||||||
|
"# More advanced comparisons (from Claude 3.7 Sonnet):\n", |
||||||
|
"\n", |
||||||
|
"\n", |
||||||
|
"## !IMPORTANT *** Uncomment final line to execute ***\n", |
||||||
|
"\n", |
||||||
|
"\n", |
||||||
|
"import numpy as np\n", |
||||||
|
"from scipy.spatial.distance import cosine\n", |
||||||
|
"\n", |
||||||
|
"# Method 1: Euclidean distance (L2 norm)\n", |
||||||
|
"def compare_embeddings_euclidean(emb1, emb2):\n", |
||||||
|
" emb1_array = np.array(emb1)\n", |
||||||
|
" emb2_array = np.array(emb2)\n", |
||||||
|
" distance = np.linalg.norm(emb1_array - emb2_array)\n", |
||||||
|
" return {\n", |
||||||
|
" \"different\": distance > 0,\n", |
||||||
|
" \"distance\": distance,\n", |
||||||
|
" \"similarity\": 1/(1+distance) # Converts distance to similarity score\n", |
||||||
|
" }\n", |
||||||
|
"\n", |
||||||
|
"# Method 2: Cosine similarity (common for embeddings)\n", |
||||||
|
"def compare_embeddings_cosine(emb1, emb2):\n", |
||||||
|
" emb1_array = np.array(emb1)\n", |
||||||
|
" emb2_array = np.array(emb2)\n", |
||||||
|
" similarity = 1 - cosine(emb1_array, emb2_array) # Cosine returns distance, so subtract from 1\n", |
||||||
|
" return {\n", |
||||||
|
" \"different\": similarity < 0.9999, # Almost identical if > 0.9999\n", |
||||||
|
" \"similarity\": similarity\n", |
||||||
|
" }\n", |
||||||
|
"\n", |
||||||
|
"# Method 3: Simple exact equality check\n", |
||||||
|
"def are_embeddings_identical(emb1, emb2):\n", |
||||||
|
" return np.array_equal(np.array(emb1), np.array(emb2))\n", |
||||||
|
"\n", |
||||||
|
"\n", |
||||||
|
"def run_advanced_comparisons():\n", |
||||||
|
" for i in range(0, 3):\n", |
||||||
|
" print(f\"\\n\\nComparing vector dimensions for dimension[{i}]....\\n\")\n", |
||||||
|
" print(\"Exactly identical? ---> \", are_embeddings_identical(dimensions1[i], dimensions2[i]))\n", |
||||||
|
" print(\"Cosine comparison: ---> \", compare_embeddings_cosine(dimensions1[i], dimensions2[i]))\n", |
||||||
|
" print(\"Euclidean comparison: ---> \", compare_embeddings_euclidean(dimensions1[i], dimensions2[i]))\n", |
||||||
|
"\n", |
||||||
|
"\n", |
||||||
|
"#run_advanced_comparisons()" |
||||||
|
] |
||||||
|
} |
||||||
|
], |
||||||
|
"metadata": { |
||||||
|
"kernelspec": { |
||||||
|
"display_name": "Python 3 (ipykernel)", |
||||||
|
"language": "python", |
||||||
|
"name": "python3" |
||||||
|
}, |
||||||
|
"language_info": { |
||||||
|
"codemirror_mode": { |
||||||
|
"name": "ipython", |
||||||
|
"version": 3 |
||||||
|
}, |
||||||
|
"file_extension": ".py", |
||||||
|
"mimetype": "text/x-python", |
||||||
|
"name": "python", |
||||||
|
"nbconvert_exporter": "python", |
||||||
|
"pygments_lexer": "ipython3", |
||||||
|
"version": "3.11.11" |
||||||
|
} |
||||||
|
}, |
||||||
|
"nbformat": 4, |
||||||
|
"nbformat_minor": 5 |
||||||
|
} |
Loading…
Reference in new issue