from flask import Flask, request, Response
import requests, os, json, time

app = Flask(__name__)
OLLAMA_API = 'http://ollama:11434'
LOG_PATH = '/logs/bench.log'

@app.route('/api/generate', methods=['POST'])
def proxy_generate():
    start_time = time.time()
    resp = requests.post(f"{OLLAMA_API}/api/generate", json=request.get_json())
    duration = time.time() - start_time
    data = resp.json()

    eval_count = data.get("eval_count")
    eval_duration_ns = data.get("eval_duration")

    if eval_count and eval_duration_ns:
        token_sec = eval_count / (eval_duration_ns / 1_000_000_000)
        log_entry = {
            "timestamp": time.strftime("%Y-%m-%d %H:%M:%S"),
            "token_per_sec": token_sec
        }
        os.makedirs(os.path.dirname(LOG_PATH), exist_ok=True)
        with open(LOG_PATH, "a") as f:
            f.write(json.dumps(log_entry, ensure_ascii=False) + "\n")

    return Response(resp.content, status=resp.status_code, content_type=resp.headers.get('Content-Type'))

flask
requests

「記事は見たけど、自分の環境で動かせないと意味がない」そんな方のために、今回私が検証したフォルダ構成の全体イメージを載せておきます。（フォルダ構成内のコメント先頭に (*) があるフォルダは、新規で作成する必要があります）

./ollama
  ├ compose.yaml
  ├ ollama                  # ollama用のdockerマウントフォルダ
  │    └ ：
  ├ open-webui              # open webui用のdockerマウントフォルダ
  │    └ ：
  ├ proxy_server            # (*) 自作プロキシ用のフォルダ
  │    ├ proxy_server.py
  │    └ requirements.txt
  └ logs                    # (*) 自作プロキシが出力するログ用フォルダ
        └ bench.log         # ログファイル

実際に取れたログとその考察

リレー処理を挟んだ結果、以下のようなログが取れました。（ログは抜粋して表示してます）

{"timestamp": "2025-06-28 12:24:22", "model": "gemma3:27b", "total_duration_ns": 94632160664, "load_duration_ns": 19994323564, "prompt_eval_count": 1031, "prompt_eval_duration_ns": 60875012677, "eval_count": 58, "eval_duration_ns": 13735608879, "token_per_sec": 4.222601306642811}
{"timestamp": "2025-06-28 12:30:58", "model": "gemma3:27b", "total_duration_ns": 394963073595, "load_duration_ns": 53579880, "prompt_eval_count": 3546, "prompt_eval_duration_ns": 195046011140, "eval_count": 760, "eval_duration_ns": 199816400248, "token_per_sec": 3.8034916005729964}
{"timestamp": "2025-06-28 13:59:10", "model": "qwen2.5vl:7b", "total_duration_ns": 759561136, "load_duration_ns": 13410971, "prompt_eval_count": 667, "prompt_eval_duration_ns": 408703073, "eval_count": 13, "eval_duration_ns": 334954117, "token_per_sec": 38.81128590516772}
{"timestamp": "2025-06-28 13:59:10", "model": "qwen2.5vl:7b", "total_duration_ns": 875906258, "load_duration_ns": 11146979, "prompt_eval_count": 558, "prompt_eval_duration_ns": 335325142, "eval_count": 20, "eval_duration_ns": 526305136, "token_per_sec": 38.00076919636977}
{"timestamp": "2025-06-28 13:59:37", "model": "qwen2.5vl:7b", "total_duration_ns": 3153000322, "load_duration_ns": 11314367, "prompt_eval_count": 721, "prompt_eval_duration_ns": 754606991, "eval_count": 72, "eval_duration_ns": 2383351049, "token_per_sec": 30.20956565765231}
{"timestamp": "2025-06-28 14:05:37", "model": "gemma3:12b", "total_duration_ns": 5428418296, "load_duration_ns": 22034384, "prompt_eval_count": 448, "prompt_eval_duration_ns": 4024403009, "eval_count": 21, "eval_duration_ns": 1380531832, "token_per_sec": 15.21152900152758}
{"timestamp": "2025-06-28 14:05:42", "model": "gemma3:12b", "total_duration_ns": 4395797252, "load_duration_ns": 24517951, "prompt_eval_count": 337, "prompt_eval_duration_ns": 2879844775, "eval_count": 23, "eval_duration_ns": 1490088937, "token_per_sec": 15.435320287865476}
{"timestamp": "2025-06-28 14:06:01", "model": "gemma3:12b", "total_duration_ns": 8595806105, "load_duration_ns": 30188016, "prompt_eval_count": 497, "prompt_eval_duration_ns": 4741797711, "eval_count": 51, "eval_duration_ns": 3822096375, "token_per_sec": 13.343462591259227}
{"timestamp": "2025-06-28 16:06:21", "model": "gemma3n:e4b", "total_duration_ns": 23945618955, "load_duration_ns": 13359735942, "prompt_eval_count": 16, "prompt_eval_duration_ns": 800558935, "eval_count": 200, "eval_duration_ns": 9782352502, "token_per_sec": 20.444979871570773}
{"timestamp": "2025-06-28 16:06:25", "model": "gemma3n:e4b", "total_duration_ns": 3821787614, "load_duration_ns": 62551020, "prompt_eval_count": 477, "prompt_eval_duration_ns": 593423184, "eval_count": 67, "eval_duration_ns": 3164968738, "token_per_sec": 21.169245432211913}
{"timestamp": "2025-06-28 16:06:27", "model": "gemma3n:e4b", "total_duration_ns": 1725808520, "load_duration_ns": 49401411, "prompt_eval_count": 556, "prompt_eval_duration_ns": 768251988, "eval_count": 20, "eval_duration_ns": 907228651, "token_per_sec": 22.045159153599084}
{"timestamp": "2025-06-28 16:06:30", "model": "gemma3n:e4b", "total_duration_ns": 2859492266, "load_duration_ns": 43687267, "prompt_eval_count": 432, "prompt_eval_duration_ns": 542335379, "eval_count": 46, "eval_duration_ns": 2272627867, "token_per_sec": 20.24088530636679}