marfrit/rk-llama.cpp
1
0
Fork 0
Code Issues Pull Requests Actions 30 Packages Projects Releases Wiki Activity

ggml-rocket: implement offload_op; add GGML_NO_REPACK measurement guard

Browse Source 
offload_op: the sched only offloads a host-resident weight-matmul to an ACCEL
backend when ggml_backend_offload_op() (device->offload_op) returns true; ours
was NULL so almost nothing reached the NPU via the intended path. Return true
for MUL_MAT (supports_op still gates each op; the sched's is_host check keeps
CPU_REPACK-resident quantized weights on CPU, so this is safe and correct).

Diagnosis of the coverage ceiling: llama.cpp keeps quantized weights in the
CPU_REPACK buffer type (is_host=0, ARM-optimized NEON layout) which we cannot
read with ggml to_float, so those matmuls stay on CPU. Forcing full offload
(GGML_NO_REPACK, added here as an env guard in ggml-cpu extra-bufts) is host-
submit-bound and slower on our single-core-per-op path. Net best remains the
hybrid: offload the largest (host-buffer) matmuls, keep the rest on CPU-repack.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EWpfhDgYNA21tETDP9ueBE
This commit is contained in:
mfritsche
2026-07-14 12:36:39 +02:00
parent 3d395c1612
commit 367143edf3
2 changed files with 27 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/src/ggml-cpu/ggml-cpu.cpp
+3 -1
View File
@@ -75,7 +75,9 @@ std::vector<ggml_backend_buffer_type_t> & ggml_backend_cpu_get_extra_buffer_type
static ggml_backend_buffer_type_t * ggml_backend_cpu_device_get_extra_buffers_type(ggml_backend_dev_t device) {
static std::vector<ggml_backend_buffer_type_t> extra_bufts = [] {
std::vector<ggml_backend_buffer_type_t> bufts = ggml_backend_cpu_get_extra_buffer_types();
std::vector<ggml_backend_buffer_type_t> bufts;
if (getenv("GGML_NO_REPACK") == nullptr) // disable CPU repack bufts for measurement
bufts = ggml_backend_cpu_get_extra_buffer_types();
bufts.push_back(nullptr);
return bufts;
}();
ggml/src/ggml-rocket/ggml-rocket.cpp
+24 -15
View File
@@ -352,20 +352,20 @@ static bool ggml_backend_rocket_device_supports_op(ggml_backend_dev_t dev, const
{
const int64_t K = src1->ne[0]; // == src0->ne[0]
const int64_t M = op->ne[1]; // batch rows (tokens)
// Offload only when the NPU can amortise its per-tile submit
// overhead: decode (M=1) is bandwidth-bound and strictly faster on
// CPU, so keep small batches there (mirrors ggml-blas min_batch).
if (M < ROCKET_MIN_BATCH) return false;
if (src1->type != GGML_TYPE_F32) return false;
if (!ggml_is_contiguous(src0)) return false;
if (!ggml_is_contiguous(src1)) return false;
if (K <= 0 || K > ROCKET_K_MAX) return false;
if (K % 16 != 0) return false; // feature-atomic align
// weight must be dequantizable to F32 (F32 src0 has no to_float
// trait -> leave those on the CPU backend)
if (ggml_get_type_traits(src0->type)->to_float == NULL) return false;
return true;
const char *why = NULL;
if (M < ROCKET_MIN_BATCH) why="M<min_batch";
else if (src1->type != GGML_TYPE_F32) why="src1!=F32";
else if (!ggml_is_contiguous(src0)) why="src0 noncontig";
else if (!ggml_is_contiguous(src1)) why="src1 noncontig";
else if (K <= 0 || K > ROCKET_K_MAX) why="K>Kmax";
else if (K % 16 != 0) why="K%16";
else if (ggml_get_type_traits(src0->type)->to_float == NULL) why="src0 no to_float";
if (getenv("GGML_ROCKET_DEBUG"))
fprintf(stderr, "[rocket-supp] MUL_MAT M=%lld N=%lld K=%lld src0=%s src1=%s -> %s\n",
(long long)M, (long long)op->ne[0], (long long)K,
ggml_type_name(src0->type), ggml_type_name(src1->type),
why ? why : "OFFLOAD");
return why == NULL;
}
default:
@@ -375,6 +375,15 @@ static bool ggml_backend_rocket_device_supports_op(ggml_backend_dev_t dev, const
GGML_UNUSED(dev);
}
static bool ggml_backend_rocket_device_offload_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) {
// Tell the scheduler we want MUL_MATs even when their weights live in CPU
// host buffers (supports_op still does the real per-op gating). Without
// this the sched's op-offload path (ggml_backend_offload_op) returns false
// and almost nothing reaches the NPU.
GGML_UNUSED(dev);
return op->op == GGML_OP_MUL_MAT;
}
static bool ggml_backend_rocket_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) {
return ggml_backend_buft_is_host(buft);
GGML_UNUSED(dev);
@@ -392,7 +401,7 @@ static const struct ggml_backend_device_i ggml_backend_rocket_device_i = {
/* .buffer_from_host_ptr = */ ggml_backend_rocket_device_buffer_from_host_ptr,
/* .supports_op = */ ggml_backend_rocket_device_supports_op,
/* .supports_buft = */ ggml_backend_rocket_device_supports_buft,
/* .offload_op = */ NULL,
/* .offload_op = */ ggml_backend_rocket_device_offload_op,
/* .event_new = */ NULL,
/* .event_free = */ NULL,
/* .event_synchronize = */ NULL,