diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 0a79310..369f388 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -108,6 +108,7 @@ static bool is_pow2(uint32_t x) { return x > 1 && (x & (x-1)) == 0; } #define VK_VENDOR_ID_INTEL 0x8086 #define VK_VENDOR_ID_NVIDIA 0x10de #define VK_VENDOR_ID_QUALCOMM 0x5143 +#define VK_VENDOR_ID_ARM 0x13B5 #define VK_DEVICE_DESCRIPTOR_POOL_SIZE 256 @@ -2612,8 +2613,14 @@ static void ggml_vk_command_pool_cleanup(vk_device& device, vk_command_pool& p) static void ggml_vk_queue_command_pools_cleanup(vk_device& device) { VK_LOG_DEBUG("ggml_vk_queue_command_pools_cleanup()"); - // Arbitrary frequency to cleanup/reuse command buffers - static constexpr uint32_t cleanup_frequency = 10; + // Arbitrary frequency to cleanup/reuse command buffers. + // rocky-vulkan-llama: env-tunable. panvk's small (~4GB) priv VA heap OOMs when many + // cmdbufs (each retaining a large per-cmdbuf TLS/WLS BO) accumulate before reset; + // GGML_VK_CMD_CLEANUP_FREQ lowers the accumulation bound on Mali/panvk. + static const uint32_t cleanup_frequency = []() { + const char *e = getenv("GGML_VK_CMD_CLEANUP_FREQ"); + return e ? (uint32_t)atoi(e) : 10u; + }(); if (device->compute_queue.cmd_pool.buffers_in_use() >= cleanup_frequency) { ggml_vk_command_pool_cleanup(device, device->compute_queue.cmd_pool); @@ -3411,6 +3418,36 @@ static void ggml_vk_load_shaders(vk_device& device) { l_warptile_mmq = { 512, 128, 128, 32, subgroup_size_8, 32, 2, tm_m, tn_m, tk_m, subgroup_size_8 }; } + // rocky-vulkan-llama: Mali/panvk (Mali-G610, subgroup=16, no coopmat). + // The stock L tile computes WM*WN/WARP accumulators/thread (L: + // 32*64/16 = 128), spilling ~5328 B/thread to the TLS stack on Mali's + // small register file -> occupancy collapse at ub>64. We shrink ONLY + // the L-variant; M/S stay stock so ub<=64 remains the ~21 t/s control. + // + // Iter1 (WM=WN=16, TM=TN=2 -> 16 acc, BM=64,BN=32, BLOCK 128): spill + // 5328->96 B, 5.8->23.4 t/s. CHAMPION. + // Iter2 (WM=32 -> 32 acc, AI 1.6->2.67): 624 B spill, 23.4->9.7 t/s. + // REGRESSED 2.4x -> we are OCCUPANCY-bound, not AI-bound; the extra + // registers/spill cost more than the intensity gain. + // Iter3 (this): max occupancy at the CHAMPION register profile. acc = + // WM*WN/WARP is independent of BM/BN, so hold WM=WN=16/TM=TN=2 (16 + // acc, ~96 B, no spill) and enlarge the workgroup: BM=64, BN=64 -> + // 16 warps -> BLOCK 256. Shared grows ~BM+BN but threads grow ~BM*BN, + // so shared-per-thread FALLS -> more resident threads under the + // 32KB/core shared budget (~640 -> ~768). Zero spill-cliff risk. + // shared(int_k) = (64+64)*17*4 = 8704 B < 32768. + const bool is_mali = + (device->vendor_id == VK_VENDOR_ID_ARM) || + (device->subgroup_size == 16 && !device->coopmat_support && + !device->coopmat2 && device->vendor_id != VK_VENDOR_ID_INTEL); + if (is_mali) { + // BLK BM BN BK WM WN WMI TM TN TK WARP + l_warptile = { 256, 64, 64, 16, 16, 16, 1, 2, 2, 1, 16 }; + l_warptile_mmq = { 256, 64, 64, 32, 16, 16, 1, 2, 2, 1, 16 }; + l_warptile_mmq_int = { 256, 64, 64, 32, 16, 16, 1, 2, 2, 1, 16 }; + l_warptile_mmq_int_k = { 256, 64, 64, 32, 16, 16, 1, 2, 2, 1, 16 }; + } + l_mmq_wg_denoms = l_wg_denoms = {128, 128, 1 }; m_mmq_wg_denoms = m_wg_denoms = { 64, 64, 1 }; s_mmq_wg_denoms = s_wg_denoms = { 32, 32, 1 }; @@ -3418,6 +3455,13 @@ static void ggml_vk_load_shaders(vk_device& device) { m_align = 64; s_align = 32; + if (is_mali) { + // Match the shrunk L tile (BM=64, BN=64). l_warptile[_mmq/_mmq_int/ + // _mmq_int_k] all consume l_[mmq_]wg_denoms; align tracks BN. + l_wg_denoms = l_mmq_wg_denoms = { 64, 64, 1 }; + l_align = 64; + } + for (uint32_t i = 0; i < GGML_TYPE_COUNT; ++i) { ggml_type t = (ggml_type)i; // Disable medium and large matrix multiplication if not enough shared memory is available @@ -14684,11 +14728,14 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg // (and scaled down based on model size, so smaller models submit earlier). // Also submit at least every 100 nodes, in case there are workloads without as much matmul. int nodes_per_submit = 100; + if (const char* e = getenv("GGML_VK_NODES_PER_SUBMIT")) { nodes_per_submit = atoi(e); } int submitted_nodes = 0; int submit_count = 0; uint64_t mul_mat_bytes = 0; uint64_t total_mul_mat_bytes = 0; - uint64_t mul_mat_bytes_per_submit = std::min(uint64_t(100*1000*1000), ctx->last_total_mul_mat_bytes / 40u); + uint64_t mul_mat_bytes_cap = 100*1000*1000; + if (const char* e = getenv("GGML_VK_MAX_MUL_MAT_BYTES_PER_SUBMIT")) { mul_mat_bytes_cap = strtoull(e, nullptr, 10); } + uint64_t mul_mat_bytes_per_submit = std::min(mul_mat_bytes_cap, ctx->last_total_mul_mat_bytes / 40u); for (int i = 0; i < cgraph->n_nodes; i++) { if (first_node_in_batch) { submit_node_idx = i; @@ -14925,7 +14972,7 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg submitted_nodes = 0; mul_mat_bytes = 0; if (submit_count < 3) { - mul_mat_bytes_per_submit *= 2; + mul_mat_bytes_per_submit = std::min(mul_mat_bytes_per_submit * 2, mul_mat_bytes_cap); } submit_count++; }