panvk WLS fix v1 + confirmed mechanism (partial: reuse halves but doesn't eliminate)

Instrument-first panthor boot confirms Fable's diagnosis: ~326 per-dispatch WLS allocs exhaust the sub-4GB priv VA heap at ub>=128. Grow-only single-max reuse (fix v1) halves the pressure (326->715 dispatches) but WLS sizes vary 128x (155KB-19MB) so a single buffer is insufficient. Refined fix pending 2nd review. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-07-11 14:06:47 +02:00
parent 16b96537fd
commit e74bae69fe
3 changed files with 134 additions and 0 deletions
@@ -0,0 +1,55 @@
+--- a/b/src/panfrost/vulkan/csf/panvk_cmd_buffer.h	2026-06-18 08:44:14.000000000 +0200
+++ b/src/panfrost/vulkan/csf/panvk_cmd_buffer.h	2026-07-11 13:50:15.332900002 +0200
+@@ -409,6 +409,11 @@
+    struct pan_ptr desc;
+    struct pan_tls_info info;
+    unsigned max_wg_count;
+
+   /* rocky-vulkan-llama: grow-only WLS buffer reused across dispatch
+    * commands (PANVK_WLS_REUSE) instead of one BO per dispatch. */
+   uint64_t wls_ptr;
+   unsigned wls_size;
+ };
+ 
+ struct panvk_cond_render_state {
+--- a/b/src/panfrost/vulkan/csf/panvk_vX_cmd_dispatch.c	2026-06-18 08:44:14.000000000 +0200
+++ b/src/panfrost/vulkan/csf/panvk_vX_cmd_dispatch.c	2026-07-11 13:50:15.336233209 +0200
+@@ -13,6 +13,7 @@
+ #include "genxml/gen_macros.h"
+ 
+ #include "panvk_buffer.h"
+#include <stdio.h>
+ #include "panvk_cmd_alloc.h"
+ #include "panvk_cmd_buffer.h"
+ #include "panvk_cmd_desc_state.h"
+@@ -100,10 +101,26 @@
+        * instance count) might differ significantly between dispatch commands,
+        * rather than track a single maximum size, we might want to consider
+        * multiple allocations for different size buckets. */
+-      tlsinfo.wls.ptr =
+-         panvk_cmd_alloc_dev_mem(cmdbuf, tls, wls_total_size, 4096).gpu;
+-      if (!tlsinfo.wls.ptr)
+-         return 0;
+      /* rocky-vulkan-llama fix (addresses the TODO above): reuse a grow-only
+       * WLS buffer across dispatches. CSF allocated a fresh WLS BO per
+       * dispatch, exhausting the sub-4GB priv VA heap over a ~300-dispatch
+       * LLM prefill graph -> OOM at ub>64. Gated by PANVK_WLS_REUSE (A/B). */
+      const bool wls_reuse = getenv("PANVK_WLS_REUSE") != NULL;
+      if (wls_reuse && wls_total_size <= cmdbuf->state.tls.wls_size) {
+         tlsinfo.wls.ptr = cmdbuf->state.tls.wls_ptr;
+      } else {
+         tlsinfo.wls.ptr =
+            panvk_cmd_alloc_dev_mem(cmdbuf, tls, wls_total_size, 4096).gpu;
+         if (!tlsinfo.wls.ptr)
+            return 0;
+         if (wls_reuse) {
+            cmdbuf->state.tls.wls_ptr = tlsinfo.wls.ptr;
+            cmdbuf->state.tls.wls_size = wls_total_size;
+         }
+      }
+      if (getenv("PANVK_WLS_LOG"))
+         fprintf(stderr, "[panvk-wls] total_size=%u reuse=%d\n",
+                 wls_total_size, wls_reuse);
+    }
+ 
+    cmdbuf->state.tls.info.tls.size =