{ "cells": [ { "cell_type": "markdown", "id": "d006b2ea-9dfe-49c7-88a9-a5a0775185fd", "metadata": {}, "source": [ "# Additional End of week Exercise - week 2\n", "\n", "Now use everything you've learned from Week 2 to build a full prototype for the technical question/answerer you built in Week 1 Exercise.\n", "\n", "This should include a Gradio UI, streaming, use of the system prompt to add expertise, and the ability to switch between models. Bonus points if you can demonstrate use of a tool!\n", "\n", "If you feel bold, see if you can add audio input so you can talk to it, and have it respond with audio. ChatGPT or Claude can help you, or email me if you have questions.\n", "\n", "I will publish a full solution here soon - unless someone beats me to it...\n", "\n", "There are so many commercial applications for this, from a language tutor, to a company onboarding solution, to a companion AI to a course (like this one!) I can't wait to see your results." ] }, { "cell_type": "code", "execution_count": 1, "id": "a07e7793-b8f5-44f4-aded-5562f633271a", "metadata": {}, "outputs": [], "source": [ "# imports\n", "\n", "import os\n", "import json\n", "from dotenv import load_dotenv\n", "from openai import OpenAI\n", "import gradio as gr\n", "import base64\n", "from io import BytesIO\n", "import tempfile\n", "import subprocess\n", "from pydub import AudioSegment\n", "import time\n", "import anthropic" ] }, { "cell_type": "code", "execution_count": 2, "id": "717ea9d4-1e72-4035-b7c5-5d61da5b8ea3", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "OpenAI API Key exists and begins sk-proj-\n" ] } ], "source": [ "# Initialization\n", "\n", "load_dotenv(override=True)\n", "\n", "openai_api_key = os.getenv('OPENAI_API_KEY')\n", "if openai_api_key:\n", " print(f\"OpenAI API Key exists and begins {openai_api_key[:8]}\")\n", "else:\n", " print(\"OpenAI API Key not set\")\n", " \n", "gpt_model = \"gpt-4o-mini\"\n", "claude_model = \"claude-3-haiku-20240307\"\n", "\n", "openai = OpenAI()\n", "claude = anthropic.Anthropic()" ] }, { "cell_type": "code", "execution_count": 3, "id": "cc78f4fd-9920-4872-9117-90cd2aeb2a06", "metadata": {}, "outputs": [], "source": [ "system_message = \"You are a helpful assistant. \"\n", "system_message += \"Give short, courteous answers. You can check ticket price, availability, and reserve tickets for users. \"\n", "system_message += \"Always be accurate. If you don't know the answer, say so.\"" ] }, { "cell_type": "code", "execution_count": 4, "id": "b2701cc0-6403-4880-9b31-e6e39e89feb4", "metadata": {}, "outputs": [], "source": [ "# Let's start by making a useful function\n", "\n", "ticket_prices = {\"london\": \"$799\", \"paris\": \"$899\", \"tokyo\": \"$1400\", \"berlin\": \"$499\"}\n", "\n", "def get_ticket_price(destination_city):\n", " print(f\"Tool get_ticket_price called for {destination_city}\")\n", " city = destination_city.lower()\n", " return ticket_prices.get(city, \"Unknown\")" ] }, { "cell_type": "code", "execution_count": 5, "id": "5e33902f-c2c3-4fb0-b01d-a346a4dff811", "metadata": {}, "outputs": [], "source": [ "ticket_availability = {\"london\": \"20\", \"paris\": \"90\", \"tokyo\": \"100\", \"berlin\": \"2\"}" ] }, { "cell_type": "code", "execution_count": 6, "id": "27dfca47-2a38-49f3-8905-f583d98710a5", "metadata": {}, "outputs": [], "source": [ "# Let's start by making a useful function\n", "def get_ticket_availability(destination_city):\n", " print(f\"Tool get_ticket_availability called for {destination_city}\")\n", " available = destination_city.lower()\n", " return ticket_availability.get(available, \"Unknown\")" ] }, { "cell_type": "code", "execution_count": 7, "id": "6ae7371b-031e-47d7-afaf-42d6758ccd92", "metadata": {}, "outputs": [], "source": [ "def get_ticket_price_availability(destination_city):\n", " print(f\"Tool get_ticket_price_availability called for {destination_city}\")\n", " available = destination_city.lower()\n", " price = destination_city.lower()\n", " return ticket_price.get(price, \"Unknown\"), ticket_availability.get(available, \"Unknown\")" ] }, { "cell_type": "code", "execution_count": 8, "id": "c919b13a-50b6-4510-8e9d-02cdfd95cb98", "metadata": {}, "outputs": [], "source": [ "def book_ticket(destination_city,price,availability):\n", " status=\"\"\n", " if availability == 0:\n", " status=\"Cannot book a ticket, no seat available\\n\"\n", " else:\n", " print(f\"Tool book_function called for {destination_city}\")\n", " f = open(\"C:/Users/aruna/Desktop/book_status.txt\", \"a\")\n", " f.write(f\"Ticket to {destination_city} booked for {price}, currently available - {int(availability)-1}\")\n", " f.write(\"\\n\")\n", " f.close()\n", " ticket_availability[destination_city.lower()]=str(int(availability)-1)\n", " \n", " status=\"Ticket reservation is a success\\n\"\n", " return status" ] }, { "cell_type": "code", "execution_count": 9, "id": "d2628781-6f5e-4ac1-bbe3-2e08aa0aae0d", "metadata": {}, "outputs": [], "source": [ "book_function = {\n", " \"name\": \"book_ticket\",\n", " \"description\": \"Book the ticket based on the ticket price and availability as confirmed by the user. For example, when a \\\n", " customer confirms to purchase the ticket for Tokyo after getting to know the ticket price and/or the availability, then \\\n", " proceed with this tool call. Please help the customer in booking the ticket if tickets are available\",\n", " \"parameters\": {\n", " \"type\": \"object\",\n", " \"properties\": {\n", " \"destination_city\": {\n", " \"type\": \"string\",\n", " \"description\": \"The city that the customer wants to travel to\",\n", " },\n", " \"price\": {\n", " \"type\": \"string\",\n", " \"description\": \"The price of the ticket to the city\",\n", " },\n", " \"availability\": {\n", " \"type\": \"string\",\n", " \"description\": \"ticket availability to the city the customer wants to travel to\",\n", " },\n", " },\n", " \"required\": [\"destination_city\",\"price\",\"availability\"],\n", " \"additionalProperties\": False\n", " }\n", "}" ] }, { "cell_type": "code", "execution_count": 10, "id": "480de296-4a36-4ec4-a5f6-149fc198c7a8", "metadata": {}, "outputs": [], "source": [ "# There's a particular dictionary structure that's required to describe our function:\n", "\n", "price_function = {\n", " \"name\": \"get_ticket_price\",\n", " \"description\": \"Get the price of a return ticket to the destination city. Call this whenever you need to know the ticket price, for example when a customer asks 'How much is a ticket to this city'\",\n", " \"parameters\": {\n", " \"type\": \"object\",\n", " \"properties\": {\n", " \"destination_city\": {\n", " \"type\": \"string\",\n", " \"description\": \"The city that the customer wants to travel to\",\n", " },\n", " },\n", " \"required\": [\"destination_city\"],\n", " \"additionalProperties\": False\n", " }\n", "}" ] }, { "cell_type": "code", "execution_count": 11, "id": "cf1b3e35-08ee-478e-aa1c-534418d78daf", "metadata": {}, "outputs": [], "source": [ "availability_function = {\n", " \"name\": \"get_ticket_availability\",\n", " \"description\": \"Get the availability of a one-way ticket to the destination city. Call this whenever you need to know the ticket availability, for example when a customer asks 'What is the ticket availability to this city'\",\n", " \"parameters\": {\n", " \"type\": \"object\",\n", " \"properties\": {\n", " \"destination_city\": {\n", " \"type\": \"string\",\n", " \"description\": \"The city that the customer wants to travel to\",\n", " },\n", " },\n", " \"required\": [\"destination_city\"],\n", " \"additionalProperties\": False\n", " }\n", "}" ] }, { "cell_type": "code", "execution_count": 12, "id": "73e4c8a2-c034-41a4-9b97-7b2aa4aca504", "metadata": {}, "outputs": [], "source": [ "ticket_price_availability_function = {\n", " \"name\": \"get_ticket_price_availability\",\n", " \"description\": \"Get the price or availability of a one-way ticket to the destination city. Call this whenever you need to know the ticket price and availability, for example when a customer asks 'What is the ticket availability and price to this city'\\\n", " or 'what is the price and ticket for the city?'\",\n", " \"parameters\": {\n", " \"type\": \"object\",\n", " \"properties\": {\n", " \"destination_city\": {\n", " \"type\": \"string\",\n", " \"description\": \"The city that the customer wants to travel to\",\n", " },\n", " },\n", " \"required\": [\"destination_city\"],\n", " \"additionalProperties\": False\n", " }\n", "}" ] }, { "cell_type": "code", "execution_count": 13, "id": "1d5d74a0-9c25-46a4-84ee-1f700bd55fa7", "metadata": {}, "outputs": [], "source": [ "# And this is included in a list of tools:\n", "\n", "tools = [{\"type\": \"function\", \"function\": price_function},\n", " {\"type\": \"function\", \"function\": availability_function},\n", " {\"type\": \"function\", \"function\": ticket_price_availability_function},\n", " {\"type\": \"function\", \"function\": book_function}]" ] }, { "cell_type": "code", "execution_count": 14, "id": "fa18f535-f8a7-4386-b39a-df0f84d23406", "metadata": {}, "outputs": [], "source": [ "def play_audio(audio_segment):\n", " temp_dir = tempfile.gettempdir()\n", " temp_path = os.path.join(temp_dir, \"temp_audio.wav\")\n", " try:\n", " audio_segment.export(temp_path, format=\"wav\")\n", " time.sleep(3) # Student Dominic found that this was needed. You could also try commenting out to see if not needed on your PC\n", " subprocess.call([\n", " \"ffplay\",\n", " \"-nodisp\",\n", " \"-autoexit\",\n", " \"-hide_banner\",\n", " temp_path\n", " ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)\n", " finally:\n", " try:\n", " os.remove(temp_path)\n", " except Exception:\n", " pass\n", " \n", "def talker(message):\n", " response = openai.audio.speech.create(\n", " model=\"tts-1\",\n", " voice=\"onyx\", # Also, try replacing onyx with alloy\n", " input=message\n", " )\n", " audio_stream = BytesIO(response.content)\n", " audio = AudioSegment.from_file(audio_stream, format=\"mp3\")\n", " play_audio(audio)\n" ] }, { "cell_type": "code", "execution_count": 15, "id": "b588d711-5f20-4a3a-9422-81a1fda8d5b0", "metadata": {}, "outputs": [], "source": [ "# We have to write that function handle_tool_call:\n", "\n", "def handle_tool_call1(message):\n", " tool_call = message.tool_calls[0]\n", " arguments = json.loads(tool_call.function.arguments)\n", " name = json.dumps(tool_call.function.name)\n", " city = arguments.get('destination_city')\n", " \n", " if name.replace('\"','') == \"get_ticket_price\":\n", " price = get_ticket_price(city)\n", " response = {\n", " \"role\": \"tool\",\n", " \"content\": json.dumps({\"destination_city\": city,\"price\": price}),\n", " \"tool_call_id\": tool_call.id\n", " }\n", " elif name.replace('\"','') == \"book_ticket\":\n", " price = get_ticket_price(city)\n", " availability = get_ticket_availability(city)\n", " booked=book_ticket(city,price,availability)\n", " response = {\n", " \"role\": \"tool\",\n", " \"content\": json.dumps({\"destination_city\": city,\"booking_status\": booked}),\n", " \"tool_call_id\": tool_call.id\n", " }\n", " else :\n", " availability = get_ticket_availability(city)\n", " response = {\n", " \"role\": \"tool\",\n", " \"content\": json.dumps({\"destination_city\": city,\"availability\": availability},),\n", " \"tool_call_id\": tool_call.id\n", " }\n", " \n", " return response, city" ] }, { "cell_type": "code", "execution_count": 16, "id": "e74eee70-f89e-4c03-922c-74f9ab567a4c", "metadata": {}, "outputs": [], "source": [ "def chat_open_ai(history):\n", " messages = [{\"role\": \"system\", \"content\": system_message}] + history \n", " response = openai.chat.completions.create(model=gpt_model, messages=messages, tools=tools)\n", " if response.choices[0].finish_reason==\"tool_calls\":\n", " message = response.choices[0].message\n", " print(message)\n", " tool_call = message.tool_calls[0]\n", " arguments = json.loads(tool_call.function.arguments)\n", " name = json.dumps(tool_call.function.name)\n", " city = arguments.get('destination_city')\n", " \n", " if name.replace('\"','') == \"book_ticket\":\n", " response, city = handle_tool_call1(message)\n", " messages.append(message)\n", " messages.append(response)\n", " # image = artist(city)\n", " response = openai.chat.completions.create(model=gpt_model, messages=messages)\n", " elif name.replace('\"','') == \"get_ticket_price_availability\":\n", " price = get_ticket_price(city)\n", " availability = get_ticket_availability(city)\n", " response = {\n", " \"role\": \"tool\",\n", " \"content\": json.dumps({\"destination_city\": city,\"price\": price,\"availability\": availability}),\n", " \"tool_call_id\": tool_call.id\n", " }\n", " messages.append(message)\n", " messages.append(response)\n", " print(messages)\n", " response = openai.chat.completions.create(model=gpt_model, messages=messages) \n", " else: \n", " response, city = handle_tool_call1(message)\n", " messages.append(message)\n", " messages.append(response)\n", " # image = artist(city)\n", " response = openai.chat.completions.create(model=gpt_model, messages=messages)\n", " \n", " reply = response.choices[0].message.content\n", " history += [{\"role\":\"assistant\", \"content\":reply}]\n", "\n", " # Comment out or delete the next line if you'd rather skip Audio for now..\n", " # talker(reply)\n", " \n", " # return history, image\n", " return history" ] }, { "cell_type": "code", "execution_count": 17, "id": "b8f25812-2609-4e26-b929-9cee2d1e4467", "metadata": {}, "outputs": [], "source": [ "tools_claude=[\n", " {\n", " \"name\": \"get_ticket_price_availability\",\n", " \"description\": \"Get the availability of a one-way ticket to the destination city or the price. Call this whenever you need to know the ticket price or availability or both, for example, when a customer asks 'What is the ticket availability and/ or price to this city'\",\n", " \"input_schema\": {\n", " \"type\": \"object\",\n", " \"properties\": {\n", " \"destination_city\": {\n", " \"type\": \"string\",\n", " \"description\": \"The city that the customer wants to travel to\",\n", " },\n", " },\n", " \"required\": [\"destination_city\"]\n", " ,\"additionalProperties\": False\n", " }\n", " }\n", " ]\n", "tool_choice = [{\"type\": \"tool\", \"name\": \"get_ticket_price_availability\"}]" ] }, { "cell_type": "code", "execution_count": 18, "id": "1728e70b-596c-4048-8c02-ac3c26756470", "metadata": {}, "outputs": [], "source": [ "def chat_claude(history):\n", " for element in history:\n", " del element[\"metadata\"]\n", " del element[\"options\"]\n", "\n", " messages = history\n", " response = claude.messages.create(\n", " model=claude_model,\n", " system=system_message,\n", " messages=messages,\n", " max_tokens=500,\n", " tools=tools_claude\n", " )\n", " print(response.content[0])\n", " if response.stop_reason==\"tool_use\": \n", " if \"text=\" in str(response.content[0]):\n", " # if response.content[0].text is None:\n", " tool_name = response.content[1].name\n", " tool_input = response.content[1].input\n", " tool_id = response.content[1].id\n", " tool_use=response.content[1]\n", " else:\n", " tool_name = response.content[0].name\n", " tool_input = response.content[0].input\n", " tool_id = response.content[0].id\n", " tool_use=response.content[0]\n", " \n", " \n", " city = tool_input.get('destination_city') \n", " if tool_name == \"get_ticket_price_availability\":\n", " price = get_ticket_price(city)\n", " availability = get_ticket_availability(city)\n", " result_dict = {\n", " 'destination_city': city,\n", " 'price': price,\n", " 'availability': availability\n", " }\n", " messages += [{\"role\": \"user\",\"content\": json.dumps(result_dict)}]\n", " response = claude.messages.create(\n", " model=claude_model,\n", " system=system_message,\n", " messages=messages,\n", " max_tokens=500,\n", " # tools=tools_claude\n", " ) \n", " history.pop(len(history)-1)\n", " print(history)\n", " reply = response.content[0].text\n", " history += [{\"role\":\"assistant\", \"content\":reply}]\n", " # Comment out or delete the next line if you'd rather skip Audio for now..\n", " # talker(reply)\n", " \n", " # return history, image\n", " return history" ] }, { "cell_type": "code", "execution_count": 19, "id": "a2547bb0-43a5-4b1d-8b9a-95da15a11040", "metadata": {}, "outputs": [], "source": [ "def chat1(history, Model):\n", " # + [{\"role\": \"user\", \"content\": message}]\n", " if Model==\"Open AI\":\n", " history = chat_open_ai(history)\n", " else:\n", " history = chat_claude(history)\n", " return history" ] }, { "cell_type": "code", "execution_count": 20, "id": "07f72649-9d2f-4bf5-b76f-97e52e2f01aa", "metadata": {}, "outputs": [], "source": [ "# gr.ChatInterface(fn=chat1, type=\"messages\").launch(inbrowser=True)" ] }, { "cell_type": "code", "execution_count": 21, "id": "23b102a4-e544-4560-acc8-a15620478582", "metadata": {}, "outputs": [], "source": [ "import speech_recognition as sr\n", "from pydub import AudioSegment\n", "import simpleaudio as sa\n", "\n", "def listener():\n", " recognizer = sr.Recognizer()\n", " \n", " with sr.Microphone() as source:\n", " print(\"Listening... Speak now!\")\n", " recognizer.adjust_for_ambient_noise(source) # Adjust for background noise\n", " audio = recognizer.listen(source)\n", " \n", " try:\n", " print(\"Processing speech...\")\n", " text = recognizer.recognize_google(audio) # Use Google Speech-to-Text\n", " print(f\"You said: {text}\")\n", " return text\n", " except sr.UnknownValueError:\n", " print(\"Sorry, I could not understand what you said.\")\n", " return None\n", " except sr.RequestError:\n", " print(\"Could not request results, please check your internet connection.\")\n", " return None\n", "\n", "# Example usage:\n", "# text = listener() # Listen for speech\n", "# if text:\n", "# print(f\"You just said: {text}\") " ] }, { "cell_type": "code", "execution_count": 22, "id": "133904cf-4d72-4552-84a8-76650f334857", "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "* Running on local URL: http://127.0.0.1:7860\n", "\n", "To create a public link, set `share=True` in `launch()`.\n" ] }, { "data": { "text/html": [ "
" ], "text/plain": [ "