diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 0a79310..7905c84 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -2612,8 +2612,14 @@ static void ggml_vk_command_pool_cleanup(vk_device& device, vk_command_pool& p) static void ggml_vk_queue_command_pools_cleanup(vk_device& device) { VK_LOG_DEBUG("ggml_vk_queue_command_pools_cleanup()"); - // Arbitrary frequency to cleanup/reuse command buffers - static constexpr uint32_t cleanup_frequency = 10; + // Arbitrary frequency to cleanup/reuse command buffers. + // rocky-vulkan-llama: env-tunable. panvk's small (~4GB) priv VA heap OOMs when many + // cmdbufs (each retaining a large per-cmdbuf TLS/WLS BO) accumulate before reset; + // GGML_VK_CMD_CLEANUP_FREQ lowers the accumulation bound on Mali/panvk. + static const uint32_t cleanup_frequency = []() { + const char *e = getenv("GGML_VK_CMD_CLEANUP_FREQ"); + return e ? (uint32_t)atoi(e) : 10u; + }(); if (device->compute_queue.cmd_pool.buffers_in_use() >= cleanup_frequency) { ggml_vk_command_pool_cleanup(device, device->compute_queue.cmd_pool); @@ -14684,11 +14690,14 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg // (and scaled down based on model size, so smaller models submit earlier). // Also submit at least every 100 nodes, in case there are workloads without as much matmul. int nodes_per_submit = 100; + if (const char* e = getenv("GGML_VK_NODES_PER_SUBMIT")) { nodes_per_submit = atoi(e); } int submitted_nodes = 0; int submit_count = 0; uint64_t mul_mat_bytes = 0; uint64_t total_mul_mat_bytes = 0; - uint64_t mul_mat_bytes_per_submit = std::min(uint64_t(100*1000*1000), ctx->last_total_mul_mat_bytes / 40u); + uint64_t mul_mat_bytes_cap = 100*1000*1000; + if (const char* e = getenv("GGML_VK_MAX_MUL_MAT_BYTES_PER_SUBMIT")) { mul_mat_bytes_cap = strtoull(e, nullptr, 10); } + uint64_t mul_mat_bytes_per_submit = std::min(mul_mat_bytes_cap, ctx->last_total_mul_mat_bytes / 40u); for (int i = 0; i < cgraph->n_nodes; i++) { if (first_node_in_batch) { submit_node_idx = i; @@ -14925,7 +14934,7 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg submitted_nodes = 0; mul_mat_bytes = 0; if (submit_count < 3) { - mul_mat_bytes_per_submit *= 2; + mul_mat_bytes_per_submit = std::min(mul_mat_bytes_per_submit * 2, mul_mat_bytes_cap); } submit_count++; }