diff --git a/docs/root-cause-final.md b/docs/root-cause-final.md new file mode 100644 index 0000000..3893b08 --- /dev/null +++ b/docs/root-cause-final.md @@ -0,0 +1,43 @@ +# Root cause — FULLY characterized (2026-07-11, live panthor instrumentation) + +The ub64 crash cliff is **priv-VA-heap exhaustion from large per-command-buffer `tls`-pool +allocations accumulating across ~12 command buffers**, with TWO consumers: + +## 1. WLS (workgroup local storage) — FIXED (partial contribution) +- Per-dispatch, grid-dependent `instances` → sizes climb (155 KB … 19 MB). +- **Fix applied & verified:** grid-independent instances (`pan_calc_wls_instances(...,NULL)`) + + grow-only reuse. Instrumentation confirms it works: **644 REUSE / 46 ALLOC** at ub128 (vs 326 + without) — collapses to ~4 allocs per cmdbuf. Cut WLS VA roughly in half (326→690 survivable + dispatches). See `patches/mesa-panvk-0001-*`, `mesa-panvk-0002-*`. + +## 2. TLS stack (register spill) — the DOMINANT remaining consumer +- `emit_tls()` (once per cmdbuf, at `vkEndCommandBuffer` — exactly where the OOM fires) allocates + `pan_get_total_stack_size(tls_size, thread_tls_alloc, core_id_range)` from the `tls` pool. +- **Measured `tls_size` per thread: ub64 (M shader) = 544 B → 38 MB stack; ub128 (L shader) = + 5328 B → 304 MB stack.** The L-variant matmul shader spills ~10× more registers. +- `core_id_range = 19` (sparse core mask `0x50005`, real cores = 4) → **~5× overallocation** baked + into every WLS *and* TLS size. +- ~12 command buffers per pp512 run, each retaining a 150–304 MB TLS stack BO in the pool (freed + only at `panvk_reset_cmdbuf`) → **~3 GB of TLS alone**, + ~1 GB WLS → over the 4064 MB priv heap. + +## Why it's ub64-sharp +llama.cpp switches M→L matmul shader at n=64 (`ggml_vk_guess_matmul_pipeline`). L's register +pressure raises `tls_size` 10× → the per-cmdbuf TLS stack jumps 38 MB→304 MB. WLS doubling (16→32 KB +shared mem) is the smaller half. Together they cross the heap ceiling at ub>64. + +## Remaining fix options (for the TLS-stack term) +1. **Reuse the TLS stack BO** the same way as WLS (grow-only, in `emit_tls`) — but it's already + one-per-cmdbuf; the real accumulation is CROSS-cmdbuf (the pool isn't freed between the ~12 + cmdbufs). So within-cmdbuf reuse won't fix this term. +2. **Free `tls`-pool big BOs after submit** (not just at reset) — Fable's constraint is the CS holds + WLS/TLS VA until *submit*, so freeing is safe post-submit. This is the structural fix: recycle + the pool per submit so cross-cmdbuf accumulation stops. Needs care in `panvk_pool`/submit path. +3. **Cut the ×19 → ×4 core overallocation** (sparse core-id remap) — ~5× smaller WLS+TLS. Biggest + single lever but HW-indexed by core id; needs a remap table. Would bring 304 MB→64 MB, ~3 GB→ + ~640 MB, likely under the ceiling by itself. +4. Reduce L-shader register spill (llama.cpp side) — out of scope for the driver. + +## Status +Mechanism fully characterized and instrumented. WLS half fixed & verified. The TLS-stack half needs +a pool-lifecycle change (option 2) and/or the core-overallocation fix (option 3) — a larger, +upstream-worthy change. Not a guess anymore: every number above is measured on the live GPU. diff --git a/patches/mesa-panvk-0001-csf-reuse-wls-across-dispatches.patch b/patches/mesa-panvk-0002-csf-wls-reuse-and-grid-independent-instances.patch similarity index 56% rename from patches/mesa-panvk-0001-csf-reuse-wls-across-dispatches.patch rename to patches/mesa-panvk-0002-csf-wls-reuse-and-grid-independent-instances.patch index 2a717b5..d3d392c 100644 --- a/patches/mesa-panvk-0001-csf-reuse-wls-across-dispatches.patch +++ b/patches/mesa-panvk-0002-csf-wls-reuse-and-grid-independent-instances.patch @@ -1,5 +1,5 @@ ---- a/b/src/panfrost/vulkan/csf/panvk_cmd_buffer.h 2026-06-18 08:44:14.000000000 +0200 -+++ b/src/panfrost/vulkan/csf/panvk_cmd_buffer.h 2026-07-11 13:50:15.332900002 +0200 +--- a/src/panfrost/vulkan/csf/panvk_cmd_buffer.h ++++ src/panfrost/vulkan/csf/panvk_cmd_buffer.h @@ -409,6 +409,11 @@ struct pan_ptr desc; struct pan_tls_info info; @@ -12,8 +12,8 @@ }; struct panvk_cond_render_state { ---- a/b/src/panfrost/vulkan/csf/panvk_vX_cmd_dispatch.c 2026-06-18 08:44:14.000000000 +0200 -+++ b/src/panfrost/vulkan/csf/panvk_vX_cmd_dispatch.c 2026-07-11 13:50:15.336233209 +0200 +--- a/src/panfrost/vulkan/csf/panvk_vX_cmd_dispatch.c ++++ src/panfrost/vulkan/csf/panvk_vX_cmd_dispatch.c @@ -13,6 +13,7 @@ #include "genxml/gen_macros.h" @@ -22,7 +22,23 @@ #include "panvk_cmd_alloc.h" #include "panvk_cmd_buffer.h" #include "panvk_cmd_desc_state.h" -@@ -100,10 +101,26 @@ +@@ -89,8 +90,14 @@ + unsigned core_id_range; + pan_query_core_count(&phys_dev->kmod.dev->props, &core_id_range); + ++ /* rocky-vulkan-llama: size WLS with the grid-INDEPENDENT upper bound ++ * on instances (max_instances_per_core), not the per-dispatch grid. ++ * Makes wls_total_size constant per pipeline so the grow-only reuse ++ * below collapses to ~1 BO per distinct shader wls_size for the whole ++ * command buffer instead of one-per-new-high-water. Mirrors the ++ * indirect path and the JM/gallium single-batch-WLS model. */ + tlsinfo.wls.instances = pan_calc_wls_instances( +- &cs->cs.local_size, &phys_dev->kmod.dev->props, indirect ? NULL : dim); ++ &cs->cs.local_size, &phys_dev->kmod.dev->props, NULL); + + unsigned wls_total_size = pan_calc_total_wls_size( + tlsinfo.wls.size, tlsinfo.wls.instances, core_id_range); +@@ -100,10 +107,29 @@ * instance count) might differ significantly between dispatch commands, * rather than track a single maximum size, we might want to consider * multiple allocations for different size buckets. */ @@ -35,8 +51,10 @@ + * dispatch, exhausting the sub-4GB priv VA heap over a ~300-dispatch + * LLM prefill graph -> OOM at ub>64. Gated by PANVK_WLS_REUSE (A/B). */ + const bool wls_reuse = getenv("PANVK_WLS_REUSE") != NULL; ++ bool wls_did_reuse = false; + if (wls_reuse && wls_total_size <= cmdbuf->state.tls.wls_size) { + tlsinfo.wls.ptr = cmdbuf->state.tls.wls_ptr; ++ wls_did_reuse = true; + } else { + tlsinfo.wls.ptr = + panvk_cmd_alloc_dev_mem(cmdbuf, tls, wls_total_size, 4096).gpu; @@ -48,8 +66,9 @@ + } + } + if (getenv("PANVK_WLS_LOG")) -+ fprintf(stderr, "[panvk-wls] total_size=%u reuse=%d\n", -+ wls_total_size, wls_reuse); ++ fprintf(stderr, "[panvk-wls] total_size=%u %s prev_max=%u\n", ++ wls_total_size, wls_did_reuse ? "REUSE" : "ALLOC", ++ cmdbuf->state.tls.wls_size); } cmdbuf->state.tls.info.tls.size = diff --git a/patches/mesa-panvk-0003-tls-stack-instrumentation.patch b/patches/mesa-panvk-0003-tls-stack-instrumentation.patch new file mode 100644 index 0000000..d6e20b0 --- /dev/null +++ b/patches/mesa-panvk-0003-tls-stack-instrumentation.patch @@ -0,0 +1,20 @@ +--- a/src/panfrost/vulkan/csf/panvk_vX_cmd_buffer.c ++++ src/panfrost/vulkan/csf/panvk_vX_cmd_buffer.c +@@ -35,6 +35,7 @@ + #include "pan_props.h" + #include "pan_samples.h" + ++#include + #include "util/bitscan.h" + #include "vk_descriptor_update_template.h" + #include "vk_format.h" +@@ -55,6 +56,9 @@ + unsigned size = pan_get_total_stack_size(cmdbuf->state.tls.info.tls.size, + thread_tls_alloc, core_id_range); + ++ if (getenv("PANVK_WLS_LOG")) ++ fprintf(stderr, "[panvk-tls] stack_size=%u (%.1f MB) tls_size=%u cores=%u\n", ++ size, size/1048576.0, cmdbuf->state.tls.info.tls.size, core_id_range); + cmdbuf->state.tls.info.tls.ptr = + panvk_cmd_alloc_dev_mem(cmdbuf, tls, size, 4096).gpu; + }