Shader iter1: Mali L-warptile (WM=WN=16) -> ub128 4x (5.8->23.4 t/s), spill 5328->96
Confirms the ceiling was the desktop-tuned shader, not the hardware. Per-thread acc = WM*WN/WARP (matches our tls_size instrument to the byte). Large ubatch now beats the ub64 ceiling + CPU. ~25-30% of roofline; iter2 grows the tile. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Claude (noether)
@@ -0,0 +1,25 @@
|
|||||||
|
# Shader tuning — Mali-tuned mul_mm warptile (the throughput unlock)
|
||||||
|
|
||||||
|
After the driver fix (no OOM) revealed the real ceiling was llama.cpp's desktop-tuned matmul
|
||||||
|
shader, we tune it for Mali. Fable implements the tile; hardware-in-the-loop measures on the G610.
|
||||||
|
|
||||||
|
## The model (Fable, verified against our instrument)
|
||||||
|
Per-thread matmul accumulators = **WM·WN/WARP** (mul_mm.comp: sums[WMITER·TM·WNITER·TN/2],
|
||||||
|
WNITER=WM·WN/(WARP·TM·TN·WMITER)). That count IS the register-spill driver, measured directly by
|
||||||
|
our panvk `[panvk-tls] tls_size` instrument. Stock on G610 (WARP=16):
|
||||||
|
- L: 32·64/16 = 128 acc → 5328 B spill (matches measured L exactly)
|
||||||
|
- M: 16·32/16 = 32 acc → 544 B spill (matches measured M exactly)
|
||||||
|
|
||||||
|
Fix = shrink WM·WN/WARP so the tile fits Mali's register file (no spill → occupancy recovers).
|
||||||
|
|
||||||
|
## Iteration 1 — L-variant WM=WN=16 (16 acc). Qwen2.5-3B-Q4_K_M, pp512
|
||||||
|
| ubatch | shader | before | after it1 | tls_size |
|
||||||
|
|--------|--------|--------|-----------|----------|
|
||||||
|
| 64 | M (control, untouched) | 21.0 | 20.6 t/s | 544 (unchanged) |
|
||||||
|
| 128 | L (new tile) | 5.8 | **23.4 t/s (4.0×)** | 5328 → **96** |
|
||||||
|
| 256 | L (new tile) | 7.4 | **23.8 t/s (3.2×)** | 5328 → **96** |
|
||||||
|
|
||||||
|
Spill collapsed 55×; large ubatch now beats the ub64/M ceiling AND the CPU. Confirms: the ceiling
|
||||||
|
was the shader, NOT the hardware. ~24 t/s ≈ 25–30% of roofline (~80–150 t/s ideal) → headroom left.
|
||||||
|
Patch: `patches/llama.cpp-0003-mali-warptile-iter1.patch` (adds VK_VENDOR_ID_ARM + is_mali warptile
|
||||||
|
branch, L tuples only). Iteration 2: grow the tile toward the register ceiling for arithmetic reuse.
|
||||||
@@ -0,0 +1,95 @@
|
|||||||
|
diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
|
||||||
|
index 0a79310..9980031 100644
|
||||||
|
--- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp
|
||||||
|
+++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp
|
||||||
|
@@ -108,6 +108,7 @@ static bool is_pow2(uint32_t x) { return x > 1 && (x & (x-1)) == 0; }
|
||||||
|
#define VK_VENDOR_ID_INTEL 0x8086
|
||||||
|
#define VK_VENDOR_ID_NVIDIA 0x10de
|
||||||
|
#define VK_VENDOR_ID_QUALCOMM 0x5143
|
||||||
|
+#define VK_VENDOR_ID_ARM 0x13B5
|
||||||
|
|
||||||
|
#define VK_DEVICE_DESCRIPTOR_POOL_SIZE 256
|
||||||
|
|
||||||
|
@@ -2612,8 +2613,14 @@ static void ggml_vk_command_pool_cleanup(vk_device& device, vk_command_pool& p)
|
||||||
|
static void ggml_vk_queue_command_pools_cleanup(vk_device& device) {
|
||||||
|
VK_LOG_DEBUG("ggml_vk_queue_command_pools_cleanup()");
|
||||||
|
|
||||||
|
- // Arbitrary frequency to cleanup/reuse command buffers
|
||||||
|
- static constexpr uint32_t cleanup_frequency = 10;
|
||||||
|
+ // Arbitrary frequency to cleanup/reuse command buffers.
|
||||||
|
+ // rocky-vulkan-llama: env-tunable. panvk's small (~4GB) priv VA heap OOMs when many
|
||||||
|
+ // cmdbufs (each retaining a large per-cmdbuf TLS/WLS BO) accumulate before reset;
|
||||||
|
+ // GGML_VK_CMD_CLEANUP_FREQ lowers the accumulation bound on Mali/panvk.
|
||||||
|
+ static const uint32_t cleanup_frequency = []() {
|
||||||
|
+ const char *e = getenv("GGML_VK_CMD_CLEANUP_FREQ");
|
||||||
|
+ return e ? (uint32_t)atoi(e) : 10u;
|
||||||
|
+ }();
|
||||||
|
|
||||||
|
if (device->compute_queue.cmd_pool.buffers_in_use() >= cleanup_frequency) {
|
||||||
|
ggml_vk_command_pool_cleanup(device, device->compute_queue.cmd_pool);
|
||||||
|
@@ -3411,6 +3418,26 @@ static void ggml_vk_load_shaders(vk_device& device) {
|
||||||
|
l_warptile_mmq = { 512, 128, 128, 32, subgroup_size_8, 32, 2, tm_m, tn_m, tk_m, subgroup_size_8 };
|
||||||
|
}
|
||||||
|
|
||||||
|
+ // rocky-vulkan-llama Iteration 1: Mali/panvk (Mali-G610, subgroup=16,
|
||||||
|
+ // no coopmat). The stock L tile computes WM*WN/WARP accumulators/thread
|
||||||
|
+ // (L: 32*64/16 = 128) which spill ~5328 B/thread to the TLS stack on
|
||||||
|
+ // Mali's small register file -> occupancy collapse at ub>64 (5.8 t/s vs
|
||||||
|
+ // 21 @ ub64). Shrink ONLY the L-variant to a spill-free 16-accumulator
|
||||||
|
+ // tile (WM=WN=16, WARP=16 -> 256/16 = 16 acc). M/S are left stock, so
|
||||||
|
+ // ub<=64 keeps the M path as the 21 t/s control. BM=64, BN=32,
|
||||||
|
+ // BK=32 (quant) / 16 (base); 8 warps * 16 = BLOCK 128; WNITER=4.
|
||||||
|
+ const bool is_mali =
|
||||||
|
+ (device->vendor_id == VK_VENDOR_ID_ARM) ||
|
||||||
|
+ (device->subgroup_size == 16 && !device->coopmat_support &&
|
||||||
|
+ !device->coopmat2 && device->vendor_id != VK_VENDOR_ID_INTEL);
|
||||||
|
+ if (is_mali) {
|
||||||
|
+ // BLK BM BN BK WM WN WMI TM TN TK WARP
|
||||||
|
+ l_warptile = { 128, 64, 32, 16, 16, 16, 1, 2, 2, 1, 16 };
|
||||||
|
+ l_warptile_mmq = { 128, 64, 32, 32, 16, 16, 1, 2, 2, 1, 16 };
|
||||||
|
+ l_warptile_mmq_int = { 128, 64, 32, 32, 16, 16, 1, 2, 2, 1, 16 };
|
||||||
|
+ l_warptile_mmq_int_k = { 128, 64, 32, 32, 16, 16, 1, 2, 2, 1, 16 };
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
l_mmq_wg_denoms = l_wg_denoms = {128, 128, 1 };
|
||||||
|
m_mmq_wg_denoms = m_wg_denoms = { 64, 64, 1 };
|
||||||
|
s_mmq_wg_denoms = s_wg_denoms = { 32, 32, 1 };
|
||||||
|
@@ -3418,6 +3445,13 @@ static void ggml_vk_load_shaders(vk_device& device) {
|
||||||
|
m_align = 64;
|
||||||
|
s_align = 32;
|
||||||
|
|
||||||
|
+ if (is_mali) {
|
||||||
|
+ // Match the shrunk L tile (BM=64, BN=32). l_warptile[_mmq/_mmq_int/
|
||||||
|
+ // _mmq_int_k] all consume l_[mmq_]wg_denoms; align tracks BN.
|
||||||
|
+ l_wg_denoms = l_mmq_wg_denoms = { 64, 32, 1 };
|
||||||
|
+ l_align = 32;
|
||||||
|
+ }
|
||||||
|
+
|
||||||
|
for (uint32_t i = 0; i < GGML_TYPE_COUNT; ++i) {
|
||||||
|
ggml_type t = (ggml_type)i;
|
||||||
|
// Disable medium and large matrix multiplication if not enough shared memory is available
|
||||||
|
@@ -14684,11 +14718,14 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg
|
||||||
|
// (and scaled down based on model size, so smaller models submit earlier).
|
||||||
|
// Also submit at least every 100 nodes, in case there are workloads without as much matmul.
|
||||||
|
int nodes_per_submit = 100;
|
||||||
|
+ if (const char* e = getenv("GGML_VK_NODES_PER_SUBMIT")) { nodes_per_submit = atoi(e); }
|
||||||
|
int submitted_nodes = 0;
|
||||||
|
int submit_count = 0;
|
||||||
|
uint64_t mul_mat_bytes = 0;
|
||||||
|
uint64_t total_mul_mat_bytes = 0;
|
||||||
|
- uint64_t mul_mat_bytes_per_submit = std::min(uint64_t(100*1000*1000), ctx->last_total_mul_mat_bytes / 40u);
|
||||||
|
+ uint64_t mul_mat_bytes_cap = 100*1000*1000;
|
||||||
|
+ if (const char* e = getenv("GGML_VK_MAX_MUL_MAT_BYTES_PER_SUBMIT")) { mul_mat_bytes_cap = strtoull(e, nullptr, 10); }
|
||||||
|
+ uint64_t mul_mat_bytes_per_submit = std::min(mul_mat_bytes_cap, ctx->last_total_mul_mat_bytes / 40u);
|
||||||
|
for (int i = 0; i < cgraph->n_nodes; i++) {
|
||||||
|
if (first_node_in_batch) {
|
||||||
|
submit_node_idx = i;
|
||||||
|
@@ -14925,7 +14962,7 @@ static ggml_status ggml_backend_vk_graph_compute(ggml_backend_t backend, ggml_cg
|
||||||
|
submitted_nodes = 0;
|
||||||
|
mul_mat_bytes = 0;
|
||||||
|
if (submit_count < 3) {
|
||||||
|
- mul_mat_bytes_per_submit *= 2;
|
||||||
|
+ mul_mat_bytes_per_submit = std::min(mul_mat_bytes_per_submit * 2, mul_mat_bytes_cap);
|
||||||
|
}
|
||||||
|
submit_count++;
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user