From the uDemy course on LLM engineering.
https://www.udemy.com/course/llm-engineering-master-ai-and-large-language-models
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
33 lines
984 B
33 lines
984 B
import ollama |
|
import requests |
|
import json |
|
import gradio as gr |
|
|
|
from bs4 import BeautifulSoup |
|
|
|
MODEL = "llama3.1-Q8-8b:latest" |
|
client = ollama.Client() |
|
system_message = ''' |
|
You are a sophisitaced advanced artificial intelligence like Data from Star Trek. You respond in a dry or matter-of-fact tone like Data. |
|
You are helfpul, but not extranious with your words.''' |
|
|
|
def chat(message, history): |
|
messages = [{"role": "system", "content": system_message}] + history + [{"role": "user", "content": message}] |
|
|
|
stream = client.chat( |
|
model=MODEL, |
|
messages=messages, |
|
stream=True, |
|
options={ |
|
"temperature": 0.5 |
|
} |
|
) |
|
|
|
result = "" |
|
for chunk in stream: |
|
# have to build the response, otherwise each word gets written that overwritten by next in the response |
|
result += chunk['message']['content'] or "" |
|
yield result |
|
|
|
if __name__ == '__main__': |
|
gr.ChatInterface(fn=chat, type='messages').launch() |