marfrit/rocky-vulkan-llama
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Fully characterize the ub64 cliff: TLS-stack register spill is the dominant term

Browse Source 
Live instrumentation on panthor: WLS reuse verified working (644 REUSE/46 ALLOC) but the
crash is dominated by the TLS *stack* alloc in emit_tls() at EndCommandBuffer — L-variant
matmul shader spills 10x more registers (tls_size 544->5328 B/thread => 38MB->304MB stack),
x~12 cmdbufs retained in the tls pool => ~3GB, over the 4GB priv heap. Remaining fix is
structural (free tls-pool BOs post-submit and/or fix the x19 sparse-core overallocation).
See docs/root-cause-final.md.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Claude (noether)
2026-07-11 14:31:27 +02:00
parent e74bae69fe
commit 97629e935c
3 changed files with 89 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
patches/mesa-panvk-0001-csf-reuse-wls-across-dispatches.patch → patches/mesa-panvk-0002-csf-wls-reuse-and-grid-independent-instances.patch
+26 -7
View File
@@ -1,5 +1,5 @@
--- a/b/src/panfrost/vulkan/csf/panvk_cmd_buffer.h 2026-06-18 08:44:14.000000000 +0200
+++ b/src/panfrost/vulkan/csf/panvk_cmd_buffer.h 2026-07-11 13:50:15.332900002 +0200
--- a/src/panfrost/vulkan/csf/panvk_cmd_buffer.h
+++ src/panfrost/vulkan/csf/panvk_cmd_buffer.h
@@ -409,6 +409,11 @@
struct pan_ptr desc;
struct pan_tls_info info;
@@ -12,8 +12,8 @@
};
struct panvk_cond_render_state {
--- a/b/src/panfrost/vulkan/csf/panvk_vX_cmd_dispatch.c 2026-06-18 08:44:14.000000000 +0200
+++ b/src/panfrost/vulkan/csf/panvk_vX_cmd_dispatch.c 2026-07-11 13:50:15.336233209 +0200
--- a/src/panfrost/vulkan/csf/panvk_vX_cmd_dispatch.c
+++ src/panfrost/vulkan/csf/panvk_vX_cmd_dispatch.c
@@ -13,6 +13,7 @@
#include "genxml/gen_macros.h"
@@ -22,7 +22,23 @@
#include "panvk_cmd_alloc.h"
#include "panvk_cmd_buffer.h"
#include "panvk_cmd_desc_state.h"
@@ -100,10 +101,26 @@
@@ -89,8 +90,14 @@
unsigned core_id_range;
pan_query_core_count(&phys_dev->kmod.dev->props, &core_id_range);
+ /* rocky-vulkan-llama: size WLS with the grid-INDEPENDENT upper bound
+ * on instances (max_instances_per_core), not the per-dispatch grid.
+ * Makes wls_total_size constant per pipeline so the grow-only reuse
+ * below collapses to ~1 BO per distinct shader wls_size for the whole
+ * command buffer instead of one-per-new-high-water. Mirrors the
+ * indirect path and the JM/gallium single-batch-WLS model. */
tlsinfo.wls.instances = pan_calc_wls_instances(
- &cs->cs.local_size, &phys_dev->kmod.dev->props, indirect ? NULL : dim);
+ &cs->cs.local_size, &phys_dev->kmod.dev->props, NULL);
unsigned wls_total_size = pan_calc_total_wls_size(
tlsinfo.wls.size, tlsinfo.wls.instances, core_id_range);
@@ -100,10 +107,29 @@
* instance count) might differ significantly between dispatch commands,
* rather than track a single maximum size, we might want to consider
* multiple allocations for different size buckets. */
@@ -35,8 +51,10 @@
+ * dispatch, exhausting the sub-4GB priv VA heap over a ~300-dispatch
+ * LLM prefill graph -> OOM at ub>64. Gated by PANVK_WLS_REUSE (A/B). */
+ const bool wls_reuse = getenv("PANVK_WLS_REUSE") != NULL;
+ bool wls_did_reuse = false;
+ if (wls_reuse && wls_total_size <= cmdbuf->state.tls.wls_size) {
+ tlsinfo.wls.ptr = cmdbuf->state.tls.wls_ptr;
+ wls_did_reuse = true;
+ } else {
+ tlsinfo.wls.ptr =
+ panvk_cmd_alloc_dev_mem(cmdbuf, tls, wls_total_size, 4096).gpu;
@@ -48,8 +66,9 @@
+ }
+ }
+ if (getenv("PANVK_WLS_LOG"))
+ fprintf(stderr, "[panvk-wls] total_size=%u reuse=%d\n",
+ wls_total_size, wls_reuse);
+ fprintf(stderr, "[panvk-wls] total_size=%u %s prev_max=%u\n",
+ wls_total_size, wls_did_reuse ? "REUSE" : "ALLOC",
+ cmdbuf->state.tls.wls_size);
}
cmdbuf->state.tls.info.tls.size =
patches/mesa-panvk-0003-tls-stack-instrumentation.patch
+20
View File
@@ -0,0 +1,20 @@
--- a/src/panfrost/vulkan/csf/panvk_vX_cmd_buffer.c
+++ src/panfrost/vulkan/csf/panvk_vX_cmd_buffer.c
@@ -35,6 +35,7 @@
#include "pan_props.h"
#include "pan_samples.h"
+#include <stdio.h>
#include "util/bitscan.h"
#include "vk_descriptor_update_template.h"
#include "vk_format.h"
@@ -55,6 +56,9 @@
unsigned size = pan_get_total_stack_size(cmdbuf->state.tls.info.tls.size,
thread_tls_alloc, core_id_range);
+ if (getenv("PANVK_WLS_LOG"))
+ fprintf(stderr, "[panvk-tls] stack_size=%u (%.1f MB) tls_size=%u cores=%u\n",
+ size, size/1048576.0, cmdbuf->state.tls.info.tls.size, core_id_range);
cmdbuf->state.tls.info.tls.ptr =
panvk_cmd_alloc_dev_mem(cmdbuf, tls, size, 4096).gpu;
}