Claude (noether)
dbf97a0fc1
Device-shared TLS scratch + grid-independent WLS reuse collapse per-cmdbuf 304MB TLS and per-dispatch WLS into ~1 BO each. Peak priv-VA 3985MB->908MB, no OOM at any ubatch (was crash at >=72). Verified on panthor. Real upstreamable Mesa panvk bug fix (mesa-panvk-0010). BUT prefill speedup NOT achieved: larger ubatch is 3x SLOWER (ub64=21, ub128=5.8, ub256=7.4 t/s) because llama.cpp's L-variant matmul shader spills ~10x more registers (the 304MB stack) -> poor Mali occupancy. ub64 @ 21 t/s stays the ceiling, ties CPU. Remaining lever = ggml-vulkan L-shader Mali tuning (separate effort). See docs/RESULTS.md. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
47 lines
2.6 KiB
Diff
47 lines
2.6 KiB
Diff
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
|
|
index 0a79310..7905c84 100644
|
|
--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp
|
|
+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
|
|
@@ -2612,8 +2612,14 @@ static void ggml_vk_command_pool_cleanup(vk_device& device, vk_command_pool& p)
|
|
static void ggml_vk_queue_command_pools_cleanup(vk_device& device) {
|
|
VK_LOG_DEBUG("ggml_vk_queue_command_pools_cleanup()");
|
|
|
|
- // Arbitrary frequency to cleanup/reuse command buffers
|
|
- static constexpr uint32_t cleanup_frequency = 10;
|
|
+ // Arbitrary frequency to cleanup/reuse command buffers.
|
|
+ // rocky-vulkan-llama: env-tunable. panvk's small (~4GB) priv VA heap OOMs when many
|
|
+ // cmdbufs (each retaining a large per-cmdbuf TLS/WLS BO) accumulate before reset;
|
|
+ // GGML_VK_CMD_CLEANUP_FREQ lowers the accumulation bound on Mali/panvk.
|
|
+ static const uint32_t cleanup_frequency = []() {
|
|
+ const char *e = getenv("GGML_VK_CMD_CLEANUP_FREQ");
|
|
+ return e ? (uint32_t)atoi(e) : 10u;
|
|
+ }();
|
|
|
|
if (device->compute_queue.cmd_pool.buffers_in_use() >= cleanup_frequency) {
|
|
ggml_vk_command_pool_cleanup(device, device->compute_queue.cmd_pool);
|
|
@@ -14684,11 +14690,14 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg
|
|
// (and scaled down based on model size, so smaller models submit earlier).
|
|
// Also submit at least every 100 nodes, in case there are workloads without as much matmul.
|
|
int nodes_per_submit = 100;
|
|
+ if (const char* e = getenv("GGML_VK_NODES_PER_SUBMIT")) { nodes_per_submit = atoi(e); }
|
|
int submitted_nodes = 0;
|
|
int submit_count = 0;
|
|
uint64_t mul_mat_bytes = 0;
|
|
uint64_t total_mul_mat_bytes = 0;
|
|
- uint64_t mul_mat_bytes_per_submit = std::min(uint64_t(100*1000*1000), ctx->last_total_mul_mat_bytes / 40u);
|
|
+ uint64_t mul_mat_bytes_cap = 100*1000*1000;
|
|
+ if (const char* e = getenv("GGML_VK_MAX_MUL_MAT_BYTES_PER_SUBMIT")) { mul_mat_bytes_cap = strtoull(e, nullptr, 10); }
|
|
+ uint64_t mul_mat_bytes_per_submit = std::min(mul_mat_bytes_cap, ctx->last_total_mul_mat_bytes / 40u);
|
|
for (int i = 0; i < cgraph->n_nodes; i++) {
|
|
if (first_node_in_batch) {
|
|
submit_node_idx = i;
|
|
@@ -14925,7 +14934,7 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg
|
|
submitted_nodes = 0;
|
|
mul_mat_bytes = 0;
|
|
if (submit_count < 3) {
|
|
- mul_mat_bytes_per_submit *= 2;
|
|
+ mul_mat_bytes_per_submit = std::min(mul_mat_bytes_per_submit * 2, mul_mat_bytes_cap);
|
|
}
|
|
submit_count++;
|
|
}
|