You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 

566 lines
14 KiB

{
"cells": [
{
"cell_type": "code",
"execution_count": 1,
"id": "fbcdfea8-7241-46d7-a771-c0381a3e7063",
"metadata": {},
"outputs": [],
"source": [
"# imports\n",
"\n",
"import os\n",
"import re\n",
"import math\n",
"import json\n",
"from tqdm import tqdm\n",
"import random\n",
"from dotenv import load_dotenv\n",
"from huggingface_hub import login\n",
"import numpy as np\n",
"import pickle\n",
"from openai import OpenAI\n",
"from sentence_transformers import SentenceTransformer\n",
"from datasets import load_dataset\n",
"import chromadb\n",
"from items import Item\n",
"from testing import Tester\n",
"from agents.pricer_agent import price\n",
"import pandas as pd\n",
"import numpy as np\n",
"from sklearn.linear_model import LinearRegression\n",
"from sklearn.metrics import mean_squared_error, r2_score"
]
},
{
"cell_type": "code",
"execution_count": 2,
"id": "e6e88bd1-f89c-4b98-92fa-aa4bc1575bca",
"metadata": {},
"outputs": [],
"source": [
"# CONSTANTS\n",
"\n",
"QUESTION = \"How much does this cost to the nearest dollar?\\n\\n\"\n",
"DB = \"products_vectorstore\""
]
},
{
"cell_type": "code",
"execution_count": 3,
"id": "98666e73-938e-469d-8987-e6e55ba5e034",
"metadata": {},
"outputs": [],
"source": [
"# environment\n",
"\n",
"load_dotenv()\n",
"os.environ['OPENAI_API_KEY'] = os.getenv('OPENAI_API_KEY', 'your-key-if-not-using-env')\n",
"os.environ['HF_TOKEN'] = os.getenv('HF_TOKEN', 'your-key-if-not-using-env')"
]
},
{
"cell_type": "code",
"execution_count": 4,
"id": "9a25a5cf-8f6c-4b5d-ad98-fdd096f5adf8",
"metadata": {},
"outputs": [],
"source": [
"openai = OpenAI()"
]
},
{
"cell_type": "code",
"execution_count": 5,
"id": "dc696493-0b6f-48aa-9fa8-b1ae0ecaf3cd",
"metadata": {},
"outputs": [],
"source": [
"# Load in the test pickle file:\n",
"\n",
"with open('test.pkl', 'rb') as file:\n",
" test = pickle.load(file)"
]
},
{
"cell_type": "code",
"execution_count": 6,
"id": "33d38a06-0c0d-4e96-94d1-35ee183416ce",
"metadata": {},
"outputs": [],
"source": [
"def make_context(similars, prices):\n",
" message = \"To provide some context, here are some other items that might be similar to the item you need to estimate.\\n\\n\"\n",
" for similar, price in zip(similars, prices):\n",
" message += f\"Potentially related product:\\n{similar}\\nPrice is ${price:.2f}\\n\\n\"\n",
" return message"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "61f203b7-63b6-48ed-869b-e393b5bfcad3",
"metadata": {},
"outputs": [],
"source": [
"def messages_for(item, similars, prices):\n",
" system_message = \"You estimate prices of items. Reply only with the price, no explanation\"\n",
" user_prompt = make_context(similars, prices)\n",
" user_prompt += \"And now the question for you:\\n\\n\"\n",
" user_prompt += item.test_prompt().replace(\" to the nearest dollar\",\"\").replace(\"\\n\\nPrice is $\",\"\")\n",
" return [\n",
" {\"role\": \"system\", \"content\": system_message},\n",
" {\"role\": \"user\", \"content\": user_prompt},\n",
" {\"role\": \"assistant\", \"content\": \"Price is $\"}\n",
" ]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b26f405d-6e1f-4caa-b97f-1f62cd9d1ebc",
"metadata": {},
"outputs": [],
"source": []
},
{
"cell_type": "code",
"execution_count": null,
"id": "d26a1104-cd11-4361-ab25-85fb576e0582",
"metadata": {},
"outputs": [],
"source": [
"client = chromadb.PersistentClient(path=DB)\n",
"collection = client.get_or_create_collection('products')"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1e339760-96d8-4485-bec7-43fadcd30c4d",
"metadata": {},
"outputs": [],
"source": [
"def description(item):\n",
" text = item.prompt.replace(\"How much does this cost to the nearest dollar?\\n\\n\", \"\")\n",
" return text.split(\"\\n\\nPrice is $\")[0]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "9f759bd2-7a7e-4c1a-80a0-e12470feca89",
"metadata": {},
"outputs": [],
"source": [
"model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e44dbd25-fb95-4b6b-bbbb-8da5fc817105",
"metadata": {},
"outputs": [],
"source": [
"def vector(item):\n",
" return model.encode([description(item)])"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "ffd5ee47-db5d-4263-b0d9-80d568c91341",
"metadata": {},
"outputs": [],
"source": [
"def find_similars(item):\n",
" results = collection.query(query_embeddings=vector(item).astype(float).tolist(), n_results=5)\n",
" documents = results['documents'][0][:]\n",
" prices = [m['price'] for m in results['metadatas'][0][:]]\n",
" return documents, prices"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d11f1c8d-7480-4d64-a274-b030d701f1b8",
"metadata": {},
"outputs": [],
"source": [
"def get_price(s):\n",
" s = s.replace('$','').replace(',','')\n",
" match = re.search(r\"[-+]?\\d*\\.\\d+|\\d+\", s)\n",
" return float(match.group()) if match else 0"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a919cf7d-b3d3-4968-8c96-54a0da0b0219",
"metadata": {},
"outputs": [],
"source": [
"# The function for gpt-4o-mini\n",
"\n",
"def gpt_4o_mini_rag(item):\n",
" documents, prices = find_similars(item)\n",
" response = openai.chat.completions.create(\n",
" model=\"gpt-4o-mini\", \n",
" messages=messages_for(item, documents, prices),\n",
" seed=42,\n",
" max_tokens=5\n",
" )\n",
" reply = response.choices[0].message.content\n",
" return get_price(reply)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8b918cfc-76c1-442a-8caa-bec500cd504b",
"metadata": {},
"outputs": [],
"source": [
"gpt_4o_mini_rag(test[1000])"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "c92cfc0b-b36d-456f-94cc-fe3f315cc25e",
"metadata": {},
"outputs": [],
"source": [
"test[1000]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e6d5deb3-6a2a-4484-872c-37176c5e1f07",
"metadata": {},
"outputs": [],
"source": [
"def proprietary(item):\n",
" text = item.prompt.split(\"to the nearest dollar?\\n\\n\")[1].split(\"\\n\\nPrice is $\")[0]\n",
" return price(text)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "bacdf607-37b9-4997-adb1-d63abfb645b1",
"metadata": {},
"outputs": [],
"source": [
"print(proprietary(test[1]))\n",
"print(gpt_4o_mini_rag(test[1]))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b35532e7-098a-4ab9-a8f7-8f101b437181",
"metadata": {},
"outputs": [],
"source": [
"truths = []\n",
"proprietaries = []\n",
"rags = []\n",
"for i in tqdm(range(1000,1250)):\n",
" item = test[i]\n",
" truths.append(item.price)\n",
" proprietaries.append(proprietary(item))\n",
" rags.append(gpt_4o_mini_rag(item))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e6ae54c7-6e8e-4333-b075-b59978fed560",
"metadata": {},
"outputs": [],
"source": [
"mins = [min(p,r) for p,r in zip(proprietaries, rags)]\n",
"maxes = [max(p,r) for p,r in zip(proprietaries, rags)]\n",
"\n",
"X = pd.DataFrame({\n",
" 'Proprietary': proprietaries,\n",
" 'RAG': rags,\n",
" 'Min': mins,\n",
" 'Max': maxes,\n",
"})\n",
"\n",
"# Convert y to a Series\n",
"y = pd.Series(truths)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e68684ed-d029-4d95-bb13-eead19b20e49",
"metadata": {},
"outputs": [],
"source": [
"# Train a Linear Regression\n",
"np.random.seed(42)\n",
"\n",
"lr = LinearRegression()\n",
"lr.fit(X, y)\n",
"\n",
"feature_columns = [\"Proprietary\", \"RAG\", \"Min\", \"Max\"]\n",
"\n",
"for feature, coef in zip(feature_columns, lr.coef_):\n",
" print(f\"{feature}: {coef:.2f}\")\n",
"print(f\"Intercept={lr.intercept_:.2f}\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "28530362-97b8-42a0-bf89-967539b6f170",
"metadata": {},
"outputs": [],
"source": [
"def ensemble(item):\n",
" prop = proprietary(item)\n",
" rag = gpt_4o_mini_rag(item)\n",
" Xt = pd.DataFrame({\n",
" 'Proprietary': [prop],\n",
" 'RAG': [rag],\n",
" 'Min': [min(prop,rag)],\n",
" 'Max': [max(prop,rag)],\n",
" })\n",
" yt = lr.predict(Xt)\n",
" return yt[0]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "08021c05-340b-4ee2-9d11-4b280766976f",
"metadata": {},
"outputs": [],
"source": [
"ensemble(test[0])"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d8308c74-546f-4fc0-ada4-1974addacfd1",
"metadata": {},
"outputs": [],
"source": [
"test[0].price"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "80792910-c59f-4d96-aa53-683464a8e60c",
"metadata": {},
"outputs": [],
"source": [
"Tester.test(ensemble, test)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "d0c41043-2049-4883-947f-2aad2f6954c2",
"metadata": {},
"outputs": [],
"source": [
"from sklearn.ensemble import RandomForestRegressor\n",
"\n",
"result = collection.get(include=['embeddings', 'documents', 'metadatas'])\n",
"vectors = np.array(result['embeddings'])\n",
"documents = result['documents']\n",
"prices = [metadata['price'] for metadata in result['metadatas']]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e9c3276f-ae01-478d-bb27-dc73b567b41a",
"metadata": {},
"outputs": [],
"source": [
"rf_model = RandomForestRegressor(n_estimators=100, random_state=42, n_jobs=8)\n",
"rf_model.fit(vectors, prices)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3e8f70cd-4147-40c6-9861-a3513b7e5499",
"metadata": {},
"outputs": [],
"source": [
"def new_rf(item):\n",
" text = item.prompt.split(\"to the nearest dollar?\\n\\n\")[1].split(\"\\n\\nPrice is $\")[0]\n",
" vector = model.encode([text])\n",
" return max(0, rf_model.predict(vector)[0])"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "a2e3340f-7ed4-47eb-a5a9-dff4c0353f58",
"metadata": {},
"outputs": [],
"source": [
"new_rf(test[0])"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "f91c903b-8db1-4374-807e-3a8ce282ef30",
"metadata": {},
"outputs": [],
"source": [
"Tester.test(new_rf, test)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "3c8e23c5-1ed3-4bd1-a3c0-129d4712c93a",
"metadata": {},
"outputs": [],
"source": [
"forests = []\n",
"for i in tqdm(range(1000,1250)):\n",
" item = test[i]\n",
" forests.append(new_rf(item))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "8e2eca63-8230-4904-9a79-7e779747479e",
"metadata": {},
"outputs": [],
"source": [
"truths2 = []\n",
"proprietaries2 = []\n",
"rags2 = []\n",
"forests2 = []\n",
"for i in tqdm(range(1000,2000)):\n",
" item = test[i]\n",
" truths2.append(item.price)\n",
" proprietaries2.append(proprietary(item))\n",
" rags2.append(gpt_4o_mini_rag(item))\n",
" forests2.append(new_rf(item))"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "0a3e057f-05c5-4f8f-8b3b-0afdfccc1412",
"metadata": {},
"outputs": [],
"source": [
"mins2 = [min(p,r,f) for p,r,f in zip(proprietaries2, rags2, forests2)]\n",
"maxes2 = [max(p,r,f) for p,r,f in zip(proprietaries2, rags2, forests2)]\n",
"\n",
"\n",
"\n",
"X2 = pd.DataFrame({\n",
" 'Proprietary': proprietaries2,\n",
" 'RAG': rags2,\n",
" 'Forest': forests2,\n",
" 'Min': mins2,\n",
" 'Max': maxes2,\n",
"})\n",
"\n",
"# Convert y to a Series\n",
"y2 = pd.Series(truths2)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "1ae62175-b955-428e-b077-705c49ee71bd",
"metadata": {},
"outputs": [],
"source": [
"# Train a Linear Regression\n",
"np.random.seed(42)\n",
"\n",
"lr2 = LinearRegression()\n",
"lr2.fit(X2, y2)\n",
"\n",
"feature_columns = X2.columns.tolist()\n",
"\n",
"for feature, coef in zip(feature_columns, lr2.coef_):\n",
" print(f\"{feature}: {coef:.2f}\")\n",
"print(f\"Intercept={lr.intercept_:.2f}\")"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "214a3831-c464-4218-a349-534b6bda7f12",
"metadata": {},
"outputs": [],
"source": [
"def ensemble2(item):\n",
" prop = proprietary(item)\n",
" rag = gpt_4o_mini_rag(item)\n",
" r_f = new_rf(item)\n",
" Xt2 = pd.DataFrame({\n",
" 'Proprietary': [prop],\n",
" 'RAG': [rag],\n",
" 'Forest': [r_f],\n",
" 'Min': [min(prop,rag, r_f)],\n",
" 'Max': [max(prop,rag, r_f)],\n",
" })\n",
" yt2 = lr.predict(Xt2)\n",
" return yt2[0]"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "b234cb68-af68-4475-ae18-8892aac6b74e",
"metadata": {},
"outputs": [],
"source": [
"Tester.test(ensemble2, test)"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "10a7275f-1aa9-4446-9100-a7a0ba0215f2",
"metadata": {},
"outputs": [],
"source": []
}
],
"metadata": {
"kernelspec": {
"display_name": "Python 3 (ipykernel)",
"language": "python",
"name": "python3"
},
"language_info": {
"codemirror_mode": {
"name": "ipython",
"version": 3
},
"file_extension": ".py",
"mimetype": "text/x-python",
"name": "python",
"nbconvert_exporter": "python",
"pygments_lexer": "ipython3",
"version": "3.11.10"
}
},
"nbformat": 4,
"nbformat_minor": 5
}