From cf4d6ca610114a3afa1520b4020f6a7b503e800a Mon Sep 17 00:00:00 2001 From: Markus Fritsche Date: Fri, 24 Jul 2026 15:35:03 +0200 Subject: [PATCH] =?UTF-8?q?stream=20fix:=20read1()=20not=20read(65536)=20?= =?UTF-8?q?=E2=80=94=20flush=20SSE=20as=20it=20arrives?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _stream_response did resp.read(65536), which blocks until 64KB or EOF. A whole chat response is only a few KB, so it never filled the buffer and only released at EOF — reasoning + content arrived in one burst at the end instead of streaming. Upstream (bosch sglang) streams fine (reasoning 0.2->6.5s); the proxy was the buffer. read1() returns on first available data. Verified reasoning now streams through :8082. Also fixes streaming for OWUI. Co-Authored-By: Claude Opus 4.8 --- llm-proxy.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/llm-proxy.py b/llm-proxy.py index dd67468..f6f24e1 100644 --- a/llm-proxy.py +++ b/llm-proxy.py @@ -659,7 +659,11 @@ class Handler(http.server.BaseHTTPRequestHandler): bytes_out = 0 try: while True: - chunk = resp.read(65536) + # read1(): return as soon as ANY data is available (one underlying read), + # instead of read()'s "block until 64KB or EOF" -- otherwise a small + # streaming response (a few KB of SSE) only flushes at EOF, so reasoning + # + content land in one burst at the end instead of streaming through. + chunk = resp.read1(65536) if not chunk: break self.wfile.write(chunk)