diff --git a/docs/shader-tuning.md b/docs/shader-tuning.md index fb2c93e..6e75b5b 100644 --- a/docs/shader-tuning.md +++ b/docs/shader-tuning.md @@ -80,3 +80,23 @@ L-path 28 t/s WITHOUT the M regression. Shippable champion. Progression (L): 5.8 BK_STEP=2 is the occupancy sweet spot; reverted to iter5 champion (28 t/s). Shared-based occupancy exhausted. Next lever: **f16 accumulators** (register-based occupancy, no extra barriers) to break the 33–64 reg band toward 2048 resident threads. + +## Iteration 7 — f16acc REFUSED (Fable), campaign CONVERGED +Fable declined to build f16acc, with proof: (1) q8_1 MMQ is f32acc-only upstream by design +("integer dot mmq performs better with f32 accumulators"); (2) we're SHARED-limited not +register-limited at the champion (shared caps 4 wg=1024 threads before the thread cap), so shrinking +acc registers can't lift occupancy; (3) f16 accumulation over K≈2560 risks overflow/NaN. + +## FINAL CHAMPION (iter5) — correctness-gated +- **ub128 = 28.1 t/s, ub256 = 28.5 t/s** (was 5.8/7.4 broken; ub64/M control 20.6, untouched). +- **Progression: 5.8 → 23.4 → 26.2 → 28.1 t/s = 4.8×**, ~35% of roofline on a GPU with NO matrix cores. +- **Correctness ✓**: llama-server GPU completion coherent+correct ("...Tokyo. Tokyo is the capital + of Japan."). Champion changes are exact-math GEMM reblocking (tile + BK_STEP, f32 acc unchanged) — + correctness preserved by construction; the gate confirms it. +- Patch: `patches/llama.cpp-0005-mali-shader-CHAMPION.patch` (Mali warptile branch + BK_STEP spec + constant #11, L-scoped). + +## Remaining ~15% (separate problem, NOT this shader) +50% occupancy but 35% roofline ⇒ the gap is outside mul_mmq: the q8_1 activation-quantize dispatch +(separate kernel per matmul), dequant ALU cost, or barrier stalls. Next lever = a per-dispatch +PROFILE of the ub128 breakdown (quantize vs matmul vs non-GEMM), not tile tuning. diff --git a/patches/llama.cpp-0005-mali-shader-CHAMPION.patch b/patches/llama.cpp-0005-mali-shader-CHAMPION.patch new file mode 100644 index 0000000..ff5d3bc --- /dev/null +++ b/patches/llama.cpp-0005-mali-shader-CHAMPION.patch @@ -0,0 +1,135 @@ +diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp +index 0a79310..131839d 100644 +--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp ++++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp +@@ -108,6 +108,7 @@ static bool is_pow2(uint32_t x) { return x > 1 && (x & (x-1)) == 0; } + #define VK_VENDOR_ID_INTEL 0x8086 + #define VK_VENDOR_ID_NVIDIA 0x10de + #define VK_VENDOR_ID_QUALCOMM 0x5143 ++#define VK_VENDOR_ID_ARM 0x13B5 + + #define VK_DEVICE_DESCRIPTOR_POOL_SIZE 256 + +@@ -2612,8 +2613,14 @@ static void ggml_vk_command_pool_cleanup(vk_device& device, vk_command_pool& p) + static void ggml_vk_queue_command_pools_cleanup(vk_device& device) { + VK_LOG_DEBUG("ggml_vk_queue_command_pools_cleanup()"); + +- // Arbitrary frequency to cleanup/reuse command buffers +- static constexpr uint32_t cleanup_frequency = 10; ++ // Arbitrary frequency to cleanup/reuse command buffers. ++ // rocky-vulkan-llama: env-tunable. panvk's small (~4GB) priv VA heap OOMs when many ++ // cmdbufs (each retaining a large per-cmdbuf TLS/WLS BO) accumulate before reset; ++ // GGML_VK_CMD_CLEANUP_FREQ lowers the accumulation bound on Mali/panvk. ++ static const uint32_t cleanup_frequency = []() { ++ const char *e = getenv("GGML_VK_CMD_CLEANUP_FREQ"); ++ return e ? (uint32_t)atoi(e) : 10u; ++ }(); + + if (device->compute_queue.cmd_pool.buffers_in_use() >= cleanup_frequency) { + ggml_vk_command_pool_cleanup(device, device->compute_queue.cmd_pool); +@@ -3411,6 +3418,41 @@ static void ggml_vk_load_shaders(vk_device& device) { + l_warptile_mmq = { 512, 128, 128, 32, subgroup_size_8, 32, 2, tm_m, tn_m, tk_m, subgroup_size_8 }; + } + ++ // rocky-vulkan-llama: Mali/panvk (Mali-G610, subgroup=16, no coopmat). ++ // The stock L tile computes WM*WN/WARP accumulators/thread (L: ++ // 32*64/16 = 128), spilling ~5328 B/thread to the TLS stack on Mali's ++ // small register file -> occupancy collapse at ub>64. We shrink ONLY ++ // the L-variant; M/S stay stock so ub<=64 remains the ~21 t/s control. ++ // ++ // Iter1 (WM=WN=16, TM=TN=2 -> 16 acc, BM=64,BN=32, BLOCK 128): spill ++ // 5328->96 B, 5.8->23.4 t/s. CHAMPION. ++ // Iter2 (WM=32 -> 32 acc, AI 1.6->2.67): 624 B spill, 23.4->9.7 t/s. ++ // REGRESSED 2.4x -> we are OCCUPANCY-bound, not AI-bound; the extra ++ // registers/spill cost more than the intensity gain. ++ // Iter3 (WM=WN=16, BM=BN=64, BLOCK 256): 16 acc, tls 96, 23.4->25.8 ++ // t/s (+10%). CHAMPION. ++ // Note: the Q4_K prefill shader is mul_mmq.comp (int-dot), whose shared ++ // memory is struct-based and scales with BK_STEP (=4 for non-id): ++ // shared = BK_STEP*(20*BM + 36*BN). Under BK_STEP=4 the 32KB/core ++ // budget caps resident workgroups such that iter3 (2 wg * 256 = 512 ++ // resident threads) is the warptile-only occupancy max -- BM=128 ++ // only drops to 1 wg (still 512, coarser). The next real lever is ++ // BK_STEP itself (shader-side): 4->1 cuts shared ~4x -> up to 8 wg = ++ // 2048 resident threads (full occupancy). See report. ++ const bool is_mali = ++ (device->vendor_id == VK_VENDOR_ID_ARM) || ++ (device->subgroup_size == 16 && !device->coopmat_support && ++ !device->coopmat2 && device->vendor_id != VK_VENDOR_ID_INTEL); ++ if (is_mali) { ++ // BLK BM BN BK WM WN WMI TM TN TK WARP [BK_STEP] ++ l_warptile = { 256, 64, 64, 16, 16, 16, 1, 2, 2, 1, 16 }; ++ l_warptile_mmq = { 256, 64, 64, 32, 16, 16, 1, 2, 2, 1, 16 }; ++ // _int / _int_k feed mul_mmq.comp: append constant_id 11 = BK_STEP ++ // = 2 (L-only occupancy scope; M/S stay 11-elem -> default 4). ++ l_warptile_mmq_int = { 256, 64, 64, 32, 16, 16, 1, 2, 2, 1, 16, 2 }; ++ l_warptile_mmq_int_k = { 256, 64, 64, 32, 16, 16, 1, 2, 2, 1, 16, 2 }; ++ } ++ + l_mmq_wg_denoms = l_wg_denoms = {128, 128, 1 }; + m_mmq_wg_denoms = m_wg_denoms = { 64, 64, 1 }; + s_mmq_wg_denoms = s_wg_denoms = { 32, 32, 1 }; +@@ -3418,6 +3460,13 @@ static void ggml_vk_load_shaders(vk_device& device) { + m_align = 64; + s_align = 32; + ++ if (is_mali) { ++ // Match the L tile (BM=64, BN=64). l_warptile[_mmq/_mmq_int/ ++ // _mmq_int_k] all consume l_[mmq_]wg_denoms; align tracks BN. ++ l_wg_denoms = l_mmq_wg_denoms = { 64, 64, 1 }; ++ l_align = 64; ++ } ++ + for (uint32_t i = 0; i < GGML_TYPE_COUNT; ++i) { + ggml_type t = (ggml_type)i; + // Disable medium and large matrix multiplication if not enough shared memory is available +@@ -14684,11 +14733,14 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg + // (and scaled down based on model size, so smaller models submit earlier). + // Also submit at least every 100 nodes, in case there are workloads without as much matmul. + int nodes_per_submit = 100; ++ if (const char* e = getenv("GGML_VK_NODES_PER_SUBMIT")) { nodes_per_submit = atoi(e); } + int submitted_nodes = 0; + int submit_count = 0; + uint64_t mul_mat_bytes = 0; + uint64_t total_mul_mat_bytes = 0; +- uint64_t mul_mat_bytes_per_submit = std::min(uint64_t(100*1000*1000), ctx->last_total_mul_mat_bytes / 40u); ++ uint64_t mul_mat_bytes_cap = 100*1000*1000; ++ if (const char* e = getenv("GGML_VK_MAX_MUL_MAT_BYTES_PER_SUBMIT")) { mul_mat_bytes_cap = strtoull(e, nullptr, 10); } ++ uint64_t mul_mat_bytes_per_submit = std::min(mul_mat_bytes_cap, ctx->last_total_mul_mat_bytes / 40u); + for (int i = 0; i < cgraph->n_nodes; i++) { + if (first_node_in_batch) { + submit_node_idx = i; +@@ -14925,7 +14977,7 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg + submitted_nodes = 0; + mul_mat_bytes = 0; + if (submit_count < 3) { +- mul_mat_bytes_per_submit *= 2; ++ mul_mat_bytes_per_submit = std::min(mul_mat_bytes_per_submit * 2, mul_mat_bytes_cap); + } + submit_count++; + } +diff --git a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +index aae1c2e..e8efe95 100644 +--- a/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp ++++ b/ggml/src/ggml-vulkan/vulkan-shaders/mul_mmq.comp +@@ -86,9 +86,17 @@ layout (constant_id = 10) const uint WARP = 32; + #ifdef MUL_MAT_ID + #define BK_STEP 1 + #else +-#ifndef BK_STEP +-#define BK_STEP 4 +-#endif ++// rocky-vulkan-llama iter5: BK_STEP as a spec constant so it can be scoped PER ++// PIPELINE VARIANT (same SPIR-V, specialized per L/M/S like BM/BN/WM already ++// are). mul_mmq shared = BK_STEP*(20*BM + 36*BN). Halving it on the ++// occupancy-bound L tile (Mali-G610: 2->4 resident wg) gave +9% (26->28 t/s), ++// but a global #define also halved it on the already-saturated M/S variants, ++// which then paid 2x K-loop barriers for no occupancy gain (iter4 M regression ++// 20.6->15.5). With this spec constant, only the Mali L warptile appends a ++// 12th element (constant_id 11) to set BK_STEP=2; M/S omit it and keep the ++// default 4. Caveat: a spec-constant loop bound may not [[unroll]] as tightly ++// as a #define -- if the L number slips vs iter4's 28.09, that is why. ++layout (constant_id = 11) const uint BK_STEP = 4; + #endif + + // Shared memory cache