Initial commit: Boltzmann LLM proxy with compression + classifier middleware

- llm-proxy.py v4: aggregator with classify/compress endpoints, gemma4 backend
- classifier-server.py: Llama 3.2 1B query complexity classifier
- compress-server.py: token-budget compression middleware
- start-proxy.sh / start.sh: launcher scripts
This commit is contained in:
Markus Fritsche
2026-06-15 15:12:25 +02:00
commit c121428c58
8 changed files with 712 additions and 0 deletions
+12
View File
@@ -0,0 +1,12 @@
#!/bin/bash
# Manual launcher for the NPU Qwen 3.5 9B llama-server backend.
# Used for ad-hoc testing; the systemd user unit
# llama-server-qwen35-npu.service is the production path.
set -e
ulimit -n 65536
exec /home/mfritsche/src/rk-llama.cpp/build/bin/llama-server \
-m /home/mfritsche/models/Qwen3.5-9B-Q8_0.gguf \
--alias qwen3.5-9b-npu \
--reasoning off \
-c 4096 -t 4 \
--host 0.0.0.0 --port 8086