Markus Fritsche
c121428c58
- llm-proxy.py v4: aggregator with classify/compress endpoints, gemma4 backend - classifier-server.py: Llama 3.2 1B query complexity classifier - compress-server.py: token-budget compression middleware - start-proxy.sh / start.sh: launcher scripts
13 lines
413 B
Bash
Executable File
13 lines
413 B
Bash
Executable File
#!/bin/bash
|
|
# Manual launcher for the NPU Qwen 3.5 9B llama-server backend.
|
|
# Used for ad-hoc testing; the systemd user unit
|
|
# llama-server-qwen35-npu.service is the production path.
|
|
set -e
|
|
ulimit -n 65536
|
|
exec /home/mfritsche/src/rk-llama.cpp/build/bin/llama-server \
|
|
-m /home/mfritsche/models/Qwen3.5-9B-Q8_0.gguf \
|
|
--alias qwen3.5-9b-npu \
|
|
--reasoning off \
|
|
-c 4096 -t 4 \
|
|
--host 0.0.0.0 --port 8086
|