|
|
@ -67,12 +67,11 @@ class Standalone: |
|
|
|
response = None |
|
|
|
response = None |
|
|
|
if host: |
|
|
|
if host: |
|
|
|
response = await AsyncClient(host=host).chat(model=self.model, messages=messages, |
|
|
|
response = await AsyncClient(host=host).chat(model=self.model, messages=messages, |
|
|
|
options=Options(temperature=self.args.temp, |
|
|
|
options=Options(temperature=self.args.temp, |
|
|
|
top_p=self.args.top_p)) |
|
|
|
top_p=self.args.top_p)) |
|
|
|
else: |
|
|
|
else: |
|
|
|
response = await AsyncClient().chat(model=self.model, messages=messages, |
|
|
|
response = await AsyncClient().chat(model=self.model, messages=messages, |
|
|
|
options=Options(temperature=self.args.temp, |
|
|
|
options=Options(temperature=self.args.temp, top_p=self.args.top_p)) |
|
|
|
top_p=self.args.top_p)) |
|
|
|
|
|
|
|
print(response['message']['content']) |
|
|
|
print(response['message']['content']) |
|
|
|
copy = self.args.copy |
|
|
|
copy = self.args.copy |
|
|
|
if copy: |
|
|
|
if copy: |
|
|
@ -87,8 +86,8 @@ class Standalone: |
|
|
|
if host: |
|
|
|
if host: |
|
|
|
# local ollama with stream and spec host |
|
|
|
# local ollama with stream and spec host |
|
|
|
async for part in await AsyncClient(host=host).chat(model=self.model, messages=messages, stream=True, |
|
|
|
async for part in await AsyncClient(host=host).chat(model=self.model, messages=messages, stream=True, |
|
|
|
options=Options(temperature=self.args.temp, |
|
|
|
options=Options(temperature=self.args.temp, |
|
|
|
top_p=self.args.top_p)): |
|
|
|
top_p=self.args.top_p)): |
|
|
|
buffer += part['message']['content'] |
|
|
|
buffer += part['message']['content'] |
|
|
|
print(part['message']['content'], end='', flush=True) |
|
|
|
print(part['message']['content'], end='', flush=True) |
|
|
|
else: |
|
|
|
else: |
|
|
|