Skip to content
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 24 additions & 3 deletions benchmarks/benchmark-local.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,10 @@
import time
import urllib.request
import urllib.parse
import urllib.error
from pathlib import Path


ROOT = Path(__file__).parent.parent

TASKS = [
Expand Down Expand Up @@ -73,10 +75,29 @@ def call_ollama(model, system_prompt, user_prompt, ollama_url):
method="POST",
)
t0 = time.time()
with urllib.request.urlopen(req, timeout=180) as resp:
data = json.loads(resp.read())
try:
with urllib.request.urlopen(req, timeout=180) as resp:
data = json.loads(resp.read())
except urllib.error.HTTPError as e:
body = e.read().decode(errors="replace")
raise RuntimeError(
f"Ollama returned HTTP {e.code} for model '{model}': {body}. "
"Check that the model name is correct and has been pulled."
) from e
except urllib.error.URLError as e:
Comment thread
Anuj04432 marked this conversation as resolved.
raise RuntimeError(
f"Could not reach Ollama at {ollama_url} (model '{model}'): {e}. "
"Is Ollama running and is the model pulled?"
) from e
except json.JSONDecodeError as e:
raise RuntimeError(f"Ollama returned invalid JSON for model '{model}': {e}") from e
elapsed = time.time() - t0
return data["message"]["content"], round(elapsed, 1)
try:
return data["message"]["content"], round(elapsed, 1)
except KeyError as e:
raise RuntimeError(
f"Unexpected response shape from Ollama for model '{model}': {data}"
) from e


def run(model, repeat, ollama_url):
Expand Down