Shader iter3 CHAMPION: BLOCK 256 same-registers -> ub128 25.8 t/s (+10%), spill-free

Shared-occupancy is the lever. Progression 5.8->23.4->26.2 t/s (~30% roofline). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-11 17:58:10 +02:00
parent 2600465c3c
commit 05f240ef6c
2 changed files with 118 additions and 0 deletions
@@ -34,3 +34,16 @@ Bigger tile improved arithmetic intensity (AI 1.6→2.67) but tanked throughput.
 OCCUPANCY-bound, not AI-bound**: 8704 B shared + 624 B spill cut resident workgroups per core.
 "Smaller is better" wins the head-to-head — **iter1 (16 acc) stays champion at ~23.5 t/s**.
 Iteration 3: go smaller / maximize concurrent workgroups, not bigger.
+
+## Iteration 3 — bigger workgroup, SAME registers (BLOCK 256, still 16 acc) — WINS
+| ubatch | iter1 | iter3 | tls_size |
+|--------|-------|-------|----------|
+| 128 | 23.4 | **25.8 t/s (+10%)** | 96 (held) |
+| 256 | 23.8 | **26.2 t/s (+10%)** | 96 (held) |
+
+Confirms shared-occupancy is the lever (not registers, not AI). Fable back-derived
+max_threads_per_core=2048 and Valhall reg-bands from our TLS data: at iter1's shape shared-mem/wg
+capped occupancy ~640 threads; bigger tiles lower shared-per-thread (shared∝BM+BN, threads∝BM·BN)
+→ ~768 threads → +10%. Progression so far: **5.8 (broken) → 23.4 (i1) → 26.2 (i3)**, ~30% of roofline.
+Iter4: aggressive BM=128 (BLOCK 512, ~1024 threads) to saturate the register ceiling; then the only
+remaining lever is a mul_mm.comp register-pressure cut (f16 acc / fewer dequant temporaries).
@@ -0,0 +1,105 @@
+diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
+index 0a79310..369f388 100644
+--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp
+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
+@@ -108,6 +108,7 @@ static bool is_pow2(uint32_t x) { return x > 1 && (x & (x-1)) == 0; }
+ #define VK_VENDOR_ID_INTEL 0x8086
+ #define VK_VENDOR_ID_NVIDIA 0x10de
+ #define VK_VENDOR_ID_QUALCOMM 0x5143
+#define VK_VENDOR_ID_ARM 0x13B5
+ 
+ #define VK_DEVICE_DESCRIPTOR_POOL_SIZE 256
+ 
+@@ -2612,8 +2613,14 @@ static void ggml_vk_command_pool_cleanup(vk_device& device, vk_command_pool& p)
+ static void ggml_vk_queue_command_pools_cleanup(vk_device& device) {
+     VK_LOG_DEBUG("ggml_vk_queue_command_pools_cleanup()");
+ 
+-    // Arbitrary frequency to cleanup/reuse command buffers
+-    static constexpr uint32_t cleanup_frequency = 10;
+    // Arbitrary frequency to cleanup/reuse command buffers.
+    // rocky-vulkan-llama: env-tunable. panvk's small (~4GB) priv VA heap OOMs when many
+    // cmdbufs (each retaining a large per-cmdbuf TLS/WLS BO) accumulate before reset;
+    // GGML_VK_CMD_CLEANUP_FREQ lowers the accumulation bound on Mali/panvk.
+    static const uint32_t cleanup_frequency = []() {
+        const char *e = getenv("GGML_VK_CMD_CLEANUP_FREQ");
+        return e ? (uint32_t)atoi(e) : 10u;
+    }();
+ 
+     if (device->compute_queue.cmd_pool.buffers_in_use() >= cleanup_frequency) {
+         ggml_vk_command_pool_cleanup(device, device->compute_queue.cmd_pool);
+@@ -3411,6 +3418,36 @@ static void ggml_vk_load_shaders(vk_device& device) {
+             l_warptile_mmq = { 512, 128, 128, 32, subgroup_size_8, 32, 2, tm_m, tn_m, tk_m, subgroup_size_8 };
+         }
+ 
+        // rocky-vulkan-llama: Mali/panvk (Mali-G610, subgroup=16, no coopmat).
+        // The stock L tile computes WM*WN/WARP accumulators/thread (L:
+        // 32*64/16 = 128), spilling ~5328 B/thread to the TLS stack on Mali's
+        // small register file -> occupancy collapse at ub>64. We shrink ONLY
+        // the L-variant; M/S stay stock so ub<=64 remains the ~21 t/s control.
+        //
+        // Iter1 (WM=WN=16, TM=TN=2 -> 16 acc, BM=64,BN=32, BLOCK 128): spill
+        //   5328->96 B, 5.8->23.4 t/s. CHAMPION.
+        // Iter2 (WM=32 -> 32 acc, AI 1.6->2.67): 624 B spill, 23.4->9.7 t/s.
+        //   REGRESSED 2.4x -> we are OCCUPANCY-bound, not AI-bound; the extra
+        //   registers/spill cost more than the intensity gain.
+        // Iter3 (this): max occupancy at the CHAMPION register profile. acc =
+        //   WM*WN/WARP is independent of BM/BN, so hold WM=WN=16/TM=TN=2 (16
+        //   acc, ~96 B, no spill) and enlarge the workgroup: BM=64, BN=64 ->
+        //   16 warps -> BLOCK 256. Shared grows ~BM+BN but threads grow ~BM*BN,
+        //   so shared-per-thread FALLS -> more resident threads under the
+        //   32KB/core shared budget (~640 -> ~768). Zero spill-cliff risk.
+        //   shared(int_k) = (64+64)*17*4 = 8704 B < 32768.
+        const bool is_mali =
+            (device->vendor_id == VK_VENDOR_ID_ARM) ||
+            (device->subgroup_size == 16 && !device->coopmat_support &&
+             !device->coopmat2 && device->vendor_id != VK_VENDOR_ID_INTEL);
+        if (is_mali) {
+            //                       BLK  BM  BN  BK  WM  WN  WMI TM TN TK WARP
+            l_warptile           = { 256, 64, 64, 16, 16, 16, 1, 2, 2, 1, 16 };
+            l_warptile_mmq       = { 256, 64, 64, 32, 16, 16, 1, 2, 2, 1, 16 };
+            l_warptile_mmq_int   = { 256, 64, 64, 32, 16, 16, 1, 2, 2, 1, 16 };
+            l_warptile_mmq_int_k = { 256, 64, 64, 32, 16, 16, 1, 2, 2, 1, 16 };
+        }
+
+         l_mmq_wg_denoms = l_wg_denoms = {128, 128, 1 };
+         m_mmq_wg_denoms = m_wg_denoms = { 64,  64, 1 };
+         s_mmq_wg_denoms = s_wg_denoms = { 32,  32, 1 };
+@@ -3418,6 +3455,13 @@ static void ggml_vk_load_shaders(vk_device& device) {
+         m_align =  64;
+         s_align =  32;
+ 
+        if (is_mali) {
+            // Match the shrunk L tile (BM=64, BN=64). l_warptile[_mmq/_mmq_int/
+            // _mmq_int_k] all consume l_[mmq_]wg_denoms; align tracks BN.
+            l_wg_denoms = l_mmq_wg_denoms = { 64, 64, 1 };
+            l_align = 64;
+        }
+
+         for (uint32_t i = 0; i < GGML_TYPE_COUNT; ++i) {
+             ggml_type t = (ggml_type)i;
+             // Disable medium and large matrix multiplication if not enough shared memory is available
+@@ -14684,11 +14728,14 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg
+     // (and scaled down based on model size, so smaller models submit earlier).
+     // Also submit at least every 100 nodes, in case there are workloads without as much matmul.
+     int nodes_per_submit = 100;
+    if (const char* e = getenv("GGML_VK_NODES_PER_SUBMIT")) { nodes_per_submit = atoi(e); }
+     int submitted_nodes = 0;
+     int submit_count = 0;
+     uint64_t mul_mat_bytes = 0;
+     uint64_t total_mul_mat_bytes = 0;
+-    uint64_t mul_mat_bytes_per_submit = std::min(uint64_t(100*1000*1000), ctx->last_total_mul_mat_bytes / 40u);
+    uint64_t mul_mat_bytes_cap = 100*1000*1000;
+    if (const char* e = getenv("GGML_VK_MAX_MUL_MAT_BYTES_PER_SUBMIT")) { mul_mat_bytes_cap = strtoull(e, nullptr, 10); }
+    uint64_t mul_mat_bytes_per_submit = std::min(mul_mat_bytes_cap, ctx->last_total_mul_mat_bytes / 40u);
+     for (int i = 0; i < cgraph->n_nodes; i++) {
+         if (first_node_in_batch) {
+             submit_node_idx = i;
+@@ -14925,7 +14972,7 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg
+             submitted_nodes = 0;
+             mul_mat_bytes = 0;
+             if (submit_count < 3) {
+-                mul_mat_bytes_per_submit *= 2;
+                mul_mat_bytes_per_submit = std::min(mul_mat_bytes_per_submit * 2, mul_mat_bytes_cap);
+             }
+             submit_count++;
+         }