Initial commit: Boltzmann LLM proxy with compression + classifier middleware
- llm-proxy.py v4: aggregator with classify/compress endpoints, gemma4 backend - classifier-server.py: Llama 3.2 1B query complexity classifier - compress-server.py: token-budget compression middleware - start-proxy.sh / start.sh: launcher scripts
This commit is contained in:
Executable
+12
@@ -0,0 +1,12 @@
|
||||
#!/bin/bash
|
||||
# Manual launcher for the NPU Qwen 3.5 9B llama-server backend.
|
||||
# Used for ad-hoc testing; the systemd user unit
|
||||
# llama-server-qwen35-npu.service is the production path.
|
||||
set -e
|
||||
ulimit -n 65536
|
||||
exec /home/mfritsche/src/rk-llama.cpp/build/bin/llama-server \
|
||||
-m /home/mfritsche/models/Qwen3.5-9B-Q8_0.gguf \
|
||||
--alias qwen3.5-9b-npu \
|
||||
--reasoning off \
|
||||
-c 4096 -t 4 \
|
||||
--host 0.0.0.0 --port 8086
|
||||
Reference in New Issue
Block a user