From cf986b5097c49ec17c7736958f0f6ea5acb1bd5e Mon Sep 17 00:00:00 2001 From: Bryan Van Deusen Date: Thu, 21 May 2026 13:34:20 -0400 Subject: [PATCH] bench_ollama: add --think on|off|auto for cross-family comparison MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The curator scenario hardcoded think=true, which is qwen3-family-specific. Non-qwen3 models silently ignore the field, so cross-family curator comparisons were apples-to-oranges (qwen thinks, others don't). New --think flag: - auto (default): scenario-driven — chat=off, curator=on. Matches the prior behaviour and the most common case. - off: force disabled across all runs. Use for fair cross-family comparison; aligns behaviour explicitly even though non-qwen models would ignore think anyway. - on: force enabled across all runs. Use to measure what think contributes on the same model (paired runs: --think off then on). Output markdown table now records the think mode used, so saved results are self-documenting when you diff cross-server or cross-config. Docstring + usage examples updated to reflect the qwen3 candidate set the bench was originally tuned for. Co-Authored-By: Claude Opus 4.7 (1M context) --- scripts/bench_ollama.py | 89 ++++++++++++++++++++++++++++++++++------- 1 file changed, 74 insertions(+), 15 deletions(-) diff --git a/scripts/bench_ollama.py b/scripts/bench_ollama.py index b045864..a698cdf 100755 --- a/scripts/bench_ollama.py +++ b/scripts/bench_ollama.py @@ -13,18 +13,26 @@ Scenarios: chat — small input, small output, no thinking. Mirrors the chat-only journal companion's expected load (short user message → curious follow-up). - curator — longer transcript input, structured-output extraction, with - thinking enabled. Mirrors the curator's expected load - (read recent conversation → emit captures). + curator — longer transcript input, structured-output extraction. + Thinking defaults ON (qwen3 family) but is overridable via + --think for cross-family comparisons. + +Think modes: + auto — chat=off, curator=on (default; matches scenario intent). + off — force think disabled for all runs (fair cross-family + comparison; non-qwen3 models silently ignore the field + anyway, so this aligns behaviour explicitly). + on — force think enabled for all runs (measures what think + contributes vs `off` on the same model). Prerequisites: - Ollama running and reachable at --server (default http://localhost:11434). - The models named in --models must already be pulled - (`ollama pull qwen2.5:32b` etc). + (`ollama pull qwen3:30b-a3b` etc). Usage examples: # Curator candidate on CPU, 3 runs each (default), one model: - python scripts/bench_ollama.py --models qwen2.5:32b --scenario curator + python scripts/bench_ollama.py --models qwen3:30b-a3b --scenario curator # Chat candidate on GPU, against a remote server: python scripts/bench_ollama.py \\ @@ -32,10 +40,16 @@ Usage examples: --models llama3.2:3b \\ --scenario chat --num-gpu 99 - # Compare three curator candidates on CPU, 5 runs each, write markdown: + # Compare two qwen curator candidates on CPU, 5 runs each: python scripts/bench_ollama.py \\ - --models qwen2.5:32b,qwen3:14b,gemma2:27b \\ - --scenario curator --runs 5 --out bench-cpu.md + --models qwen3:30b-a3b,qwen3:32b \\ + --scenario curator --runs 5 --out bench-qwen-curator.md + + # Cross-family curator comparison with think forced OFF (apples-to-apples): + python scripts/bench_ollama.py \\ + --models qwen3:30b-a3b,gemma2:27b,mistral-small:22b \\ + --scenario curator --think off --runs 3 \\ + --out bench-curator-no-think.md The first request to a (model, num_gpu) pair triggers a model load and is excluded from the timing as a warm-up. Subsequent runs reflect warm-cache @@ -145,23 +159,36 @@ class ScenarioResult: } +def _resolve_think(scenario: str, think_mode: str) -> bool: + """Map (scenario, --think mode) → boolean think flag.""" + if think_mode == "on": + return True + if think_mode == "off": + return False + # auto: scenario-driven + return scenario == "curator" + + def build_request( - scenario: str, model: str, num_gpu: int, keep_alive: str + scenario: str, + model: str, + num_gpu: int, + keep_alive: str, + think_mode: str = "auto", ) -> dict: if scenario == "chat": messages = [ {"role": "system", "content": SYSTEM_PROMPT_CHAT}, {"role": "user", "content": USER_MESSAGE_CHAT}, ] - think = False elif scenario == "curator": messages = [ {"role": "system", "content": SYSTEM_PROMPT_CURATOR}, {"role": "user", "content": USER_TRANSCRIPT_CURATOR}, ] - think = True else: raise ValueError(f"unknown scenario: {scenario}") + think = _resolve_think(scenario, think_mode) return { "model": model, "messages": messages, @@ -226,12 +253,15 @@ def benchmark( runs: int, num_gpu: int, keep_alive: str, + think_mode: str, ) -> list[ScenarioResult]: results: list[ScenarioResult] = [] for model in models: for scenario in scenarios: sr = ScenarioResult(model=model, scenario=scenario) - payload = build_request(scenario, model, num_gpu, keep_alive) + payload = build_request( + scenario, model, num_gpu, keep_alive, think_mode + ) # Warm-up run loads the model into RAM/VRAM with the requested # num_gpu setting. Excluded from the measured runs because it # otherwise dominates TTFT with model-load time. @@ -268,15 +298,27 @@ def benchmark( return results -def format_markdown(results: list[ScenarioResult], *, server: str, num_gpu: int) -> str: +def format_markdown( + results: list[ScenarioResult], + *, + server: str, + num_gpu: int, + think_mode: str, +) -> str: mode = "CPU only" if num_gpu == 0 else ( f"GPU offload ({num_gpu} layers)" if num_gpu > 0 else "Ollama default" ) + think_label = { + "auto": "auto (chat=off, curator=on)", + "on": "forced ON", + "off": "forced OFF", + }.get(think_mode, think_mode) lines = [ "# Ollama benchmark", "", f"- Server: `{server}`", - f"- Mode: {mode} (`num_gpu={num_gpu}`)", + f"- Hardware mode: {mode} (`num_gpu={num_gpu}`)", + f"- Think: {think_label}", "", "| Model | Scenario | Runs | Prompt tok | TTFT p50 (ms) " "| Total p50 (ms) | tok/s p50 | Output tok (mean) |", @@ -337,6 +379,17 @@ def main(): default="10m", help="Ollama keep_alive (default %(default)s)", ) + parser.add_argument( + "--think", + choices=["auto", "on", "off"], + default="auto", + help=( + "Think-mode control. auto = chat off / curator on (default). " + "off = force disabled for fair cross-family comparison " + "(non-qwen3 models silently ignore think anyway). " + "on = force enabled to measure what think contributes." + ), + ) parser.add_argument( "--out", help="Write markdown table to this file (also prints to stdout)", @@ -355,9 +408,15 @@ def main(): runs=args.runs, num_gpu=args.num_gpu, keep_alive=args.keep_alive, + think_mode=args.think, ) - md = format_markdown(results, server=args.server, num_gpu=args.num_gpu) + md = format_markdown( + results, + server=args.server, + num_gpu=args.num_gpu, + think_mode=args.think, + ) print("\n" + md) if args.out: with open(args.out, "w") as f: