ggml-rknpu2: graceful abort with W8A8 hint on NPU IOVA exhaustion
rknn_create_mem returns null when the RK3588 NPU's ~4 GiB IOVA aperture is full -- this backend maps every weight tensor into a single 4 GiB IOMMU window (rkopnu currently ignores iommu_domain_id, so the IOMMUDomainManager's 2 GiB domain-spreading is a no-op and everything lands in one 4 GiB window). The old GGML_ASSERT reported a cryptic "Failed to allocate tensor memory via RKNN API" with no guidance. Replace it with a GGML_ABORT that names the aperture and points at the fix: use a quantized GGUF, or set RKNPU_HYBRID=W8A8_STANDARD for on-the-fly INT8 weights (which fits >4 GiB dense f16 models -- e.g. Qwen2.5-3B-f16 runs at ~130 t/s pp512 that way). Also null-check the grow re-alloc path, which was a latent copy of the same crash. No behaviour change for models that fit; only the failure message improves. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EWpfhDgYNA21tETDP9ueBE
This commit is contained in:
@@ -296,6 +296,9 @@ struct ggml_backend_rknpu_buffer_context {
|
||||
it->second.iommu_domain_id = g_domain_manager.assign_domain_memory(size);
|
||||
rknn_matmul_ctx new_ctx = g_domain_manager.get_allocator_context(it->second.iommu_domain_id);
|
||||
it->second.mem = rknn_create_mem(new_ctx, size);
|
||||
if (it->second.mem == nullptr) {
|
||||
GGML_ABORT("rknpu2: rknn_create_mem failed for %zu bytes (grow re-alloc) -- the NPU's ~4 GiB IOVA aperture is exhausted; this backend maps all weights into a single 4 GiB IOMMU window. Use a quantized GGUF, or set RKNPU_HYBRID=W8A8_STANDARD for on-the-fly INT8 weights.", size);
|
||||
}
|
||||
it->second.size = size;
|
||||
}
|
||||
return it->second;
|
||||
@@ -311,7 +314,9 @@ struct ggml_backend_rknpu_buffer_context {
|
||||
alloc.size = size;
|
||||
alloc.iommu_domain_id = domain_id;
|
||||
|
||||
GGML_ASSERT(alloc.mem != nullptr && "Failed to allocate tensor memory via RKNN API");
|
||||
if (alloc.mem == nullptr) {
|
||||
GGML_ABORT("rknpu2: rknn_create_mem failed for %zu bytes -- the NPU's ~4 GiB IOVA aperture is exhausted; this backend maps all weights into a single 4 GiB IOMMU window. Use a quantized GGUF, or set RKNPU_HYBRID=W8A8_STANDARD for on-the-fly INT8 weights.", size);
|
||||
}
|
||||
tensor_allocs[tensor_offset] = alloc;
|
||||
|
||||
return alloc;
|
||||
|
||||
Reference in New Issue
Block a user