diff --git a/llm-proxy.py b/llm-proxy.py index 5030cd5..dd67468 100644 --- a/llm-proxy.py +++ b/llm-proxy.py @@ -75,6 +75,25 @@ def _load_friendly_names(): FRIENDLY_NAMES = _load_friendly_names() +# Optional list of local model-id substrings whose thinking should be split into a +# separate `reasoning` stream (via the chat-template `enable_thinking` kwarg), so +# clients can surface the model's reasoning instead of having it inlined or dropped. +# From $LLM_PROXY_CONFIG_DIR/reasoning-models.json (a JSON list of substrings). +# Absent / empty = feature off, so this changes nothing unless configured. +def _load_reasoning_models(): + try: + with open(os.path.join(CONFIG_DIR, "reasoning-models.json")) as f: + data = json.load(f) + return [s for s in data if isinstance(s, str)] if isinstance(data, list) else [] + except Exception: + return [] +REASONING_LOCAL_MODELS = _load_reasoning_models() + + +def _is_reasoning_local(model): + return bool(model) and any(sub in model for sub in REASONING_LOCAL_MODELS) + + CONNECT_TIMEOUT = 3 READ_TIMEOUT = 1800 DISCOVERY_TTL = 30 @@ -992,6 +1011,16 @@ class Handler(http.server.BaseHTTPRequestHandler): total_chars = sum(len(str(m.get("content", ""))) for m in j.get("messages", [])) stream = j.get("stream", False) hint = f" model={model} msgs={n_msgs} in_chars={total_chars} max_tok={max_t} stream={stream}" + # reasoning passthrough: ask reasoning-capable local models to split + # their thinking into a separate `reasoning` stream (chat-template kwarg), + # so clients can surface it. The byte-relay forwards the field untouched. + if _is_reasoning_local(model): + ck = j.get("chat_template_kwargs") + ck = ck if isinstance(ck, dict) else {} + if "enable_thinking" not in ck: + ck["enable_thinking"] = True + j["chat_template_kwargs"] = ck + body = json.dumps(j).encode() except Exception: pass diff --git a/reasoning-models.example.json b/reasoning-models.example.json new file mode 100644 index 0000000..b9f72f2 --- /dev/null +++ b/reasoning-models.example.json @@ -0,0 +1 @@ +["dspark"]