From 13354c02a6f6417c662eecf6c15d64c4bf0c5c69 Mon Sep 17 00:00:00 2001 From: Kishore Kumar R Date: Sun, 5 Jan 2025 19:44:09 +0530 Subject: [PATCH] feat(week2): llama3.1-instruct tools usecase --- .../day5_llama3.1_tools_usecase.ipynb | 444 ++++++++++++++++++ 1 file changed, 444 insertions(+) create mode 100644 week2/community-contributions/day5_llama3.1_tools_usecase.ipynb diff --git a/week2/community-contributions/day5_llama3.1_tools_usecase.ipynb b/week2/community-contributions/day5_llama3.1_tools_usecase.ipynb new file mode 100644 index 0000000..2114d53 --- /dev/null +++ b/week2/community-contributions/day5_llama3.1_tools_usecase.ipynb @@ -0,0 +1,444 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Code tested in google colab with T4 GPU" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "bK-WkZmI_L4S" + }, + "outputs": [], + "source": [ + "!pip install -q requests torch bitsandbytes transformers sentencepiece accelerate openai httpx==0.27.2" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Anm0jUBC_dQF" + }, + "outputs": [], + "source": [ + "import os\n", + "import requests\n", + "from IPython.display import Markdown, display, update_display\n", + "from openai import OpenAI\n", + "from google.colab import drive\n", + "from huggingface_hub import login\n", + "from google.colab import userdata\n", + "from transformers import AutoTokenizer, AutoModelForCausalLM, TextStreamer, BitsAndBytesConfig\n", + "import torch\n", + "import json" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "GsKyLRam_hpi" + }, + "outputs": [], + "source": [ + "LLAMA = \"meta-llama/Meta-Llama-3.1-8B-Instruct\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "AWo-zjJJAIIF" + }, + "outputs": [], + "source": [ + "hf_token = userdata.get('HF_TOKEN')\n", + "login(hf_token, add_to_git_credential=True)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "TSeMKC4S7Ip6" + }, + "outputs": [], + "source": [ + "# Utility Functions ::\n", + "\n", + "def get_tokenizer(model_name):\n", + " \"\"\"\n", + " Args:\n", + " model_name: LLM model name\n", + "\n", + " Returns:\n", + " Tokenizer\n", + " \"\"\"\n", + " tokenizer = AutoTokenizer.from_pretrained(model_name)\n", + " tokenizer.pad_token = tokenizer.eos_token\n", + "\n", + " return tokenizer\n", + "\n", + "\n", + "def get_model(model_name):\n", + " \"\"\"\n", + " Args:\n", + " model_name: LLM model name\n", + "\n", + " Returns:\n", + " Model\n", + " \"\"\"\n", + " quant_config = BitsAndBytesConfig(\n", + " load_in_4bit=True,\n", + " bnb_4bit_use_double_quant=True,\n", + " bnb_4bit_compute_dtype=torch.bfloat16,\n", + " bnb_4bit_quant_type=\"nf4\"\n", + " )\n", + "\n", + " model = AutoModelForCausalLM.from_pretrained(model_name, device_map=\"auto\", quantization_config=quant_config)\n", + "\n", + " return model\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "O_mUtibSJFWt" + }, + "outputs": [], + "source": [ + "tokenizer = get_tokenizer(LLAMA)\n", + "model = get_model(LLAMA)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "Mc0RE4XM_2hW" + }, + "outputs": [], + "source": [ + "\n", + "def parse_response(inputs, outputs):\n", + " \"\"\"\n", + " Args:\n", + " inputs: Input Tokens, pt tensors\n", + " outputs: Model generated output token\n", + "\n", + " Returns:\n", + " response\n", + " \"\"\"\n", + " full_input =(tokenizer.batch_decode(inputs, skip_special_tokens=True)[0])\n", + " full_outputs=tokenizer.batch_decode(outputs, skip_special_tokens=True)\n", + " response = full_outputs[0][len(full_input):]\n", + "\n", + " return response\n", + "\n", + "# Tools Functions:\n", + "\n", + "def get_tools_response(tools_params):\n", + " function_name = tools_params['name']\n", + " arguments = tools_params['arguments']\n", + " location = arguments['location']\n", + "\n", + " if function_name == 'get_current_temperature':\n", + " response = get_current_temperature(location)\n", + " return response\n", + " elif function_name == 'get_current_wind_speed':\n", + " response = get_current_wind_speed(location)\n", + " return response\n", + "\n", + "\n", + "def get_current_temperature(location: str) -> float:\n", + " \"\"\"\n", + " Get the current temperature at a location.\n", + "\n", + " Args:\n", + " location: The location to get the temperature for, in the format \"City, Country\"\n", + " Returns:\n", + " The current temperature at the specified location in the specified units, as a float.\n", + " \"\"\"\n", + " if location == \"Bangalore, India\": \n", + " return 22 # for testing purpose, please replace with your logic\n", + "\n", + "\n", + "def get_current_wind_speed(location: str) -> str:\n", + " \"\"\"\n", + " Get the current wind speed at a given location.\n", + "\n", + " Args:\n", + " location: The location to get the wind speed for, in the format \"City, Country\"\n", + " Returns:\n", + " The current wind speed at the given location in the specified units, as a string.\n", + " \"\"\"\n", + " return 6 # for testing purpose, please replace with your logic\n", + "\n", + "\n", + "def get_llm_response(message):\n", + "\n", + " messages = [\n", + " {\"role\": \"system\", \"content\": \"\"\"\n", + " You are an helpful assistant that responds to weather and wind queries. Please provide accurate answers.\n", + " Follow instructions:\n", + " 1. Please provide courteous answers.\n", + " 2. If you do not know the answer say so.\n", + " 3. Do not provide any explanations or suggestions.\n", + " 4. Provide the response in no more than 1 sentence.\n", + " \"\"\"\n", + " },\n", + " ]\n", + " message = {\"role\": \"user\", \"content\": message}\n", + "\n", + " messages.append(message)\n", + "\n", + "\n", + " inputs = tokenizer.apply_chat_template(messages, tools=[get_current_temperature, get_current_wind_speed], add_generation_prompt=True, return_tensors=\"pt\").to(\"cuda\")\n", + "\n", + " #streamer = TextStreamer(tokenizer) # include to check the output of model, only for debug\n", + " outputs = model.generate(inputs, max_new_tokens=2000 )#, streamer=streamer ) # , include to check the output of model, only for debug\n", + "\n", + " model_response = parse_response(inputs, outputs)\n", + "\n", + " tool_call = json.loads(model_response.replace(\"parameters\", \"arguments\"))\n", + " messages.append({\"role\": \"assistant\", \"tool_calls\": [{\"type\": \"function\", \"function\": tool_call}]})\n", + "\n", + " response = get_tools_response(tool_call)\n", + "\n", + " messages.append({\"role\": \"tool\", \"name\": tool_call[\"name\"], \"content\": response})\n", + "\n", + " inputs = tokenizer.apply_chat_template(messages, add_generation_prompt=True, return_tensors=\"pt\").to(\"cuda\")\n", + " outputs = model.generate(inputs, max_new_tokens=200)\n", + "\n", + " response = parse_response(inputs, outputs)\n", + "\n", + " return response\n", + "\n", + "\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "id": "AKGLvmnsCrnk", + "outputId": "f3e5448a-76a5-47a6-a5e8-491ef25b27c2" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n", + "Setting `pad_token_id` to `eos_token_id`:128001 for open-end generation.\n", + "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n", + "Setting `pad_token_id` to `eos_token_id`:128001 for open-end generation.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\"name\": \"get_current_wind_speed\", \"parameters\": {\"location\": \"Bangalore, India\"}}\n", + "6\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n", + "Setting `pad_token_id` to `eos_token_id`:128001 for open-end generation.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The current wind speed in Bangalore is approximately 6 km/h.\n", + "####################################################################################################\n", + "Query: what is the wind speed in Bangalore?\n", + "Response: The current wind speed in Bangalore is approximately 6 km/h.\n", + "\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n", + "Setting `pad_token_id` to `eos_token_id`:128001 for open-end generation.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\"name\": \"get_current_temperature\", \"parameters\": {\"location\": \"Bangalore, India\"}}\n", + "22\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n", + "Setting `pad_token_id` to `eos_token_id`:128001 for open-end generation.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "I don't have real-time information, but Bangalore's average temperature is around 22 degrees Celsius.\n", + "####################################################################################################\n", + "Query: what is the temperature of Bangalore?\n", + "Response: I don't have real-time information, but Bangalore's average temperature is around 22 degrees Celsius.\n", + "\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n", + "Setting `pad_token_id` to `eos_token_id`:128001 for open-end generation.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\"name\": \"get_current_temperature\", \"parameters\": {\"location\": \"Delhi, India\"}}\n", + "None\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n", + "Setting `pad_token_id` to `eos_token_id`:128001 for open-end generation.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "I don't have the current temperature in Delhi.\n", + "####################################################################################################\n", + "Query: temperature in Delhi?\n", + "Response: I don't have the current temperature in Delhi.\n", + "\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n", + "Setting `pad_token_id` to `eos_token_id`:128001 for open-end generation.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\"name\": \"get_current_wind_speed\", \"parameters\": {\"location\": \"Goa\"}}\n", + "6\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n", + "Setting `pad_token_id` to `eos_token_id`:128001 for open-end generation.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The wind speed in Goa is approximately 6 km/h.\n", + "####################################################################################################\n", + "Query: wind speed in Goa?\n", + "Response: The wind speed in Goa is approximately 6 km/h.\n", + "\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.\n", + "Setting `pad_token_id` to `eos_token_id`:128001 for open-end generation.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\"name\": \"get_current_temperature\", \"parameters\": {\"location\": \"Chennai, India\"}}\n", + "None\n", + "I don't have the current temperature of Chennai.\n", + "####################################################################################################\n", + "Query: Chennai's temperature?\n", + "Response: I don't have the current temperature of Chennai.\n", + "\n", + "\n" + ] + } + ], + "source": [ + "\n", + "\n", + "data = [\n", + " \"what is the wind speed in Bangalore?\",\n", + " \"what is the temperature of Bangalore?\",\n", + " \"temperature in Delhi?\",\n", + " \"wind speed in Goa?\",\n", + " \"Chennai's temperature?\",\n", + "]\n", + "\n", + "for query in data:\n", + " response = get_llm_response(query)\n", + " print(\"#\"*100)\n", + " print(f\"Query: {query}\\nResponse: {response}\\n\\n\")\n", + "\n" + ] + } + ], + "metadata": { + "accelerator": "GPU", + "colab": { + "gpuType": "T4", + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +}