From 396159c9d57391f20cd5f1ad329adc659684e3cb Mon Sep 17 00:00:00 2001 From: mfritsche Date: Mon, 20 Jul 2026 04:21:49 +0200 Subject: [PATCH] ggml-rknpu2: graceful abort with W8A8 hint on NPU IOVA exhaustion rknn_create_mem returns null when the RK3588 NPU's ~4 GiB IOVA aperture is full -- this backend maps every weight tensor into a single 4 GiB IOMMU window (rkopnu currently ignores iommu_domain_id, so the IOMMUDomainManager's 2 GiB domain-spreading is a no-op and everything lands in one 4 GiB window). The old GGML_ASSERT reported a cryptic "Failed to allocate tensor memory via RKNN API" with no guidance. Replace it with a GGML_ABORT that names the aperture and points at the fix: use a quantized GGUF, or set RKNPU_HYBRID=W8A8_STANDARD for on-the-fly INT8 weights (which fits >4 GiB dense f16 models -- e.g. Qwen2.5-3B-f16 runs at ~130 t/s pp512 that way). Also null-check the grow re-alloc path, which was a latent copy of the same crash. No behaviour change for models that fit; only the failure message improves. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01EWpfhDgYNA21tETDP9ueBE --- ggml/src/ggml-rknpu2/ggml-rknpu2.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ggml/src/ggml-rknpu2/ggml-rknpu2.cpp b/ggml/src/ggml-rknpu2/ggml-rknpu2.cpp index 108446104..5752a7b1c 100644 --- a/ggml/src/ggml-rknpu2/ggml-rknpu2.cpp +++ b/ggml/src/ggml-rknpu2/ggml-rknpu2.cpp @@ -296,6 +296,9 @@ struct ggml_backend_rknpu_buffer_context { it->second.iommu_domain_id = g_domain_manager.assign_domain_memory(size); rknn_matmul_ctx new_ctx = g_domain_manager.get_allocator_context(it->second.iommu_domain_id); it->second.mem = rknn_create_mem(new_ctx, size); + if (it->second.mem == nullptr) { + GGML_ABORT("rknpu2: rknn_create_mem failed for %zu bytes (grow re-alloc) -- the NPU's ~4 GiB IOVA aperture is exhausted; this backend maps all weights into a single 4 GiB IOMMU window. Use a quantized GGUF, or set RKNPU_HYBRID=W8A8_STANDARD for on-the-fly INT8 weights.", size); + } it->second.size = size; } return it->second; @@ -311,7 +314,9 @@ struct ggml_backend_rknpu_buffer_context { alloc.size = size; alloc.iommu_domain_id = domain_id; - GGML_ASSERT(alloc.mem != nullptr && "Failed to allocate tensor memory via RKNN API"); + if (alloc.mem == nullptr) { + GGML_ABORT("rknpu2: rknn_create_mem failed for %zu bytes -- the NPU's ~4 GiB IOVA aperture is exhausted; this backend maps all weights into a single 4 GiB IOMMU window. Use a quantized GGUF, or set RKNPU_HYBRID=W8A8_STANDARD for on-the-fly INT8 weights.", size); + } tensor_allocs[tensor_offset] = alloc; return alloc;