diff --git a/week2/community-contributions/day5-voice-to-text-converter-for-hearing-impaired-people.ipynb b/week2/community-contributions/day5-voice-to-text-converter-for-hearing-impaired-people.ipynb new file mode 100644 index 0000000..fc49d9c --- /dev/null +++ b/week2/community-contributions/day5-voice-to-text-converter-for-hearing-impaired-people.ipynb @@ -0,0 +1,375 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "4c3c6553-daa4-4a03-8017-15d0cad8f280", + "metadata": {}, + "source": [ + "# About Mini Project\n", + "\n", + "Mini project for hearing impaired people, using tools, suggesting songs according to a certain genre and in sign language. Speech to text converter with multiple language support." + ] + }, + { + "cell_type": "markdown", + "id": "a32a79cb-3d16-4b3b-a029-a059bd0b1c0b", + "metadata": {}, + "source": [ + "## Extra requirements\n", + "- pip install pydub simpleaudio speechrecognition pipwin pyaudio\n" + ] + }, + { + "cell_type": "code", + "execution_count": 80, + "id": "3e214aa3-a977-434f-a436-90a89b81a5ee", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import json\n", + "from dotenv import load_dotenv\n", + "from openai import OpenAI\n", + "import gradio as gr" + ] + }, + { + "cell_type": "code", + "execution_count": 81, + "id": "d654cb96-9bcd-4b64-bd79-2d27fa6a62d0", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "OpenAI API Key exists and begins sk-proj-\n" + ] + } + ], + "source": [ + "load_dotenv(override=True)\n", + "\n", + "openai_api_key = os.getenv('OPENAI_API_KEY')\n", + "if openai_api_key:\n", + " print(f\"OpenAI API Key exists and begins {openai_api_key[:8]}\")\n", + "else:\n", + " print(\"OpenAI API Key not set\")\n", + " \n", + "MODEL = \"gpt-4o-mini\"\n", + "openai = OpenAI()" + ] + }, + { + "cell_type": "code", + "execution_count": 82, + "id": "b2d9214f-25d0-4f09-ba88-641beeaa20db", + "metadata": {}, + "outputs": [], + "source": [ + "system_message = \"You are a helpful assistant for hearing impaired people. \"\n", + "system_message += \"Your mission is convert text to speech and speech to text. \"\n", + "system_message += \"Always be accurate. If you don't know the answer, say so.\"" + ] + }, + { + "cell_type": "code", + "execution_count": 83, + "id": "3d9a1478-08bf-4195-8f38-34c29757012f", + "metadata": {}, + "outputs": [], + "source": [ + "songs_with_signs = {\n", + " \"electronic\": (\"God is a dj\", \"https://www.youtube.com/watch?v=bhSB8EEnCAM\", \"Faithless\"), \n", + " \"pop\": (\"Yitirmeden\", \"https://www.youtube.com/watch?v=aObdAXq1ZIo\", \"Pinhani\"), \n", + " \"rock\": (\"Bohemian Rhapsody\", \"https://www.youtube.com/watch?v=sjln9OMOw-0\", \"Queen\")\n", + "}\n", + "\n", + "def get_songs_with_sign_language(genre):\n", + " print(f\"Tool get_songs_with_sign_language called for {genre}\")\n", + " city = genre.lower()\n", + " return songs_with_signs.get(genre, \"Unknown\")" + ] + }, + { + "cell_type": "code", + "execution_count": 84, + "id": "93a3d7ee-78c2-4e19-b7e4-8239b07aaecc", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Tool get_songs_with_sign_language called for rock\n" + ] + }, + { + "data": { + "text/plain": [ + "('Bohemian Rhapsody', 'https://www.youtube.com/watch?v=sjln9OMOw-0', 'Queen')" + ] + }, + "execution_count": 84, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "get_songs_with_sign_language(\"rock\")" + ] + }, + { + "cell_type": "code", + "execution_count": 85, + "id": "7307aa61-86fe-4c46-9f9d-faa3d1fb1eb7", + "metadata": {}, + "outputs": [], + "source": [ + "song_function = {\n", + " \"name\": \"get_songs_with_sign_language\",\n", + " \"description\": \"Get the corresponding song information for the specified given music genre. Call this whenever you need to know the songs with specific genre and in sign language, for example when a customer asks 'Suggest me sign language supported songs'\",\n", + " \"parameters\": {\n", + " \"type\": \"object\",\n", + " \"properties\": {\n", + " \"genre\": {\n", + " \"type\": \"string\",\n", + " \"description\": \"The music genre that the customer wants to listen-watch to\",\n", + " },\n", + " },\n", + " \"required\": [\"genre\"],\n", + " \"additionalProperties\": False\n", + " }\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": 86, + "id": "160d790c-dda6-4c6e-b814-8be64ca7086b", + "metadata": {}, + "outputs": [], + "source": [ + "tools = [{\"type\": \"function\", \"function\": song_function}]" + ] + }, + { + "cell_type": "code", + "execution_count": 87, + "id": "96cdf319-11cd-4be2-8830-097225047d65", + "metadata": {}, + "outputs": [], + "source": [ + "def handle_tool_call(message):\n", + " tool_call = message.tool_calls[0]\n", + " arguments = json.loads(tool_call.function.arguments)\n", + " genre = arguments.get('genre')\n", + " song = get_songs_with_sign_language(genre)\n", + " song_info = song[2] + \": \" + song[1]\n", + " response = {\n", + " \"role\": \"tool\",\n", + " \"content\": json.dumps({\"genre\": genre,\"song\": song_info}),\n", + " \"tool_call_id\": tool_call.id\n", + " }\n", + " return response, song[1]" + ] + }, + { + "cell_type": "code", + "execution_count": 88, + "id": "bbd8ad0c-135b-406f-8ab9-0e1f9b58538d", + "metadata": {}, + "outputs": [], + "source": [ + "def chat(history):\n", + " messages = [{\"role\": \"system\", \"content\": system_message}] + history\n", + " response = openai.chat.completions.create(model=MODEL, messages=messages, tools=tools)\n", + " genre = None\n", + " \n", + " if response.choices[0].finish_reason==\"tool_calls\":\n", + " message = response.choices[0].message\n", + " response, genre = handle_tool_call(message)\n", + " messages.append(message)\n", + " messages.append(response)\n", + " response = openai.chat.completions.create(model=MODEL, messages=messages)\n", + " \n", + " reply = response.choices[0].message.content\n", + " history += [{\"role\":\"assistant\", \"content\":reply}]\n", + " \n", + " return history, genre" + ] + }, + { + "cell_type": "code", + "execution_count": 89, + "id": "69f43096-3557-4218-b0de-bd286237fdeb", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Listening... Speak now!\n", + "Processing speech...\n", + "Sorry, I could not understand what you said.\n" + ] + } + ], + "source": [ + "import speech_recognition as sr\n", + "from pydub import AudioSegment\n", + "import simpleaudio as sa\n", + "\n", + "def listener():\n", + " recognizer = sr.Recognizer()\n", + " \n", + " with sr.Microphone() as source:\n", + " print(\"Listening... Speak now!\")\n", + " recognizer.adjust_for_ambient_noise(source) # Adjust for background noise\n", + " audio = recognizer.listen(source)\n", + " \n", + " try:\n", + " print(\"Processing speech...\")\n", + " text = recognizer.recognize_google(audio) # Use Google Speech-to-Text\n", + " print(f\"You said: {text}\")\n", + " return text\n", + " except sr.UnknownValueError:\n", + " print(\"Sorry, I could not understand what you said.\")\n", + " return None\n", + " except sr.RequestError:\n", + " print(\"Could not request results, please check your internet connection.\")\n", + " return None\n", + "\n", + "# Example usage:\n", + "text = listener() # Listen for speech\n", + "if text:\n", + " print(f\"You just said: {text}\") " + ] + }, + { + "cell_type": "code", + "execution_count": 90, + "id": "23c9deeb-d9ad-439a-a39d-7eac9553bd5e", + "metadata": {}, + "outputs": [], + "source": [ + "import gradio as gr\n", + "\n", + "convert = gr.State(False)\n", + "def toggle_convert(current_value):\n", + " return not current_value" + ] + }, + { + "cell_type": "code", + "execution_count": 78, + "id": "32d3ea9f-fe3c-4cc5-9902-550c63c58a69", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "* Running on local URL: http://127.0.0.1:7892\n", + "* Running on public URL: https://f9549e6cf738e17cc7.gradio.live\n", + "\n", + "This share link expires in 72 hours. For free permanent hosting and GPU upgrades, run `gradio deploy` from the terminal in the working directory to deploy to Hugging Face Spaces (https://huggingface.co/spaces)\n" + ] + }, + { + "data": { + "text/html": [ + "
" + ], + "text/plain": [ + "