llama: limit max outputs of llama_context (#23861)

* llama: save more VRAM by reserving n_outputs == n_seqs when possible * add n_outputs_per_seq * move n_outputs_max to server-context * change ubatch to batch everywhere
2026-06-01 23:01:38 +08:00
parent 95b8b8ec1a
commit de6f727aae
6 changed files with 71 additions and 11 deletions
@@ -1563,6 +1563,7 @@ struct llama_context_params common_context_params_to_llama(const common_params &
    cparams.n_ctx             = params.n_ctx;
    cparams.n_seq_max         = params.n_parallel;
    cparams.n_rs_seq          = params.speculative.need_n_rs_seq();
+    cparams.n_outputs_max     = std::max(params.n_outputs_max, 0);
    cparams.n_batch           = params.n_batch;
    cparams.n_ubatch          = params.n_ubatch;
    cparams.n_threads         = params.cpuparams.n_threads;