From 367143edf34d21da89a837bb94e762180178f88b Mon Sep 17 00:00:00 2001 From: mfritsche Date: Tue, 14 Jul 2026 12:36:39 +0200 Subject: [PATCH] ggml-rocket: implement offload_op; add GGML_NO_REPACK measurement guard offload_op: the sched only offloads a host-resident weight-matmul to an ACCEL backend when ggml_backend_offload_op() (device->offload_op) returns true; ours was NULL so almost nothing reached the NPU via the intended path. Return true for MUL_MAT (supports_op still gates each op; the sched's is_host check keeps CPU_REPACK-resident quantized weights on CPU, so this is safe and correct). Diagnosis of the coverage ceiling: llama.cpp keeps quantized weights in the CPU_REPACK buffer type (is_host=0, ARM-optimized NEON layout) which we cannot read with ggml to_float, so those matmuls stay on CPU. Forcing full offload (GGML_NO_REPACK, added here as an env guard in ggml-cpu extra-bufts) is host- submit-bound and slower on our single-core-per-op path. Net best remains the hybrid: offload the largest (host-buffer) matmuls, keep the rest on CPU-repack. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01EWpfhDgYNA21tETDP9ueBE --- ggml/src/ggml-cpu/ggml-cpu.cpp | 4 ++- ggml/src/ggml-rocket/ggml-rocket.cpp | 39 +++++++++++++++++----------- 2 files changed, 27 insertions(+), 16 deletions(-) diff --git a/ggml/src/ggml-cpu/ggml-cpu.cpp b/ggml/src/ggml-cpu/ggml-cpu.cpp index 128883b41..407faeeeb 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.cpp +++ b/ggml/src/ggml-cpu/ggml-cpu.cpp @@ -75,7 +75,9 @@ std::vector & ggml_backend_cpu_get_extra_buffer_type static ggml_backend_buffer_type_t * ggml_backend_cpu_device_get_extra_buffers_type(ggml_backend_dev_t device) { static std::vector extra_bufts = [] { - std::vector bufts = ggml_backend_cpu_get_extra_buffer_types(); + std::vector bufts; + if (getenv("GGML_NO_REPACK") == nullptr) // disable CPU repack bufts for measurement + bufts = ggml_backend_cpu_get_extra_buffer_types(); bufts.push_back(nullptr); return bufts; }(); diff --git a/ggml/src/ggml-rocket/ggml-rocket.cpp b/ggml/src/ggml-rocket/ggml-rocket.cpp index 1a868afc6..6c18aa97a 100644 --- a/ggml/src/ggml-rocket/ggml-rocket.cpp +++ b/ggml/src/ggml-rocket/ggml-rocket.cpp @@ -352,20 +352,20 @@ static bool ggml_backend_rocket_device_supports_op(ggml_backend_dev_t dev, const { const int64_t K = src1->ne[0]; // == src0->ne[0] const int64_t M = op->ne[1]; // batch rows (tokens) - - // Offload only when the NPU can amortise its per-tile submit - // overhead: decode (M=1) is bandwidth-bound and strictly faster on - // CPU, so keep small batches there (mirrors ggml-blas min_batch). - if (M < ROCKET_MIN_BATCH) return false; - if (src1->type != GGML_TYPE_F32) return false; - if (!ggml_is_contiguous(src0)) return false; - if (!ggml_is_contiguous(src1)) return false; - if (K <= 0 || K > ROCKET_K_MAX) return false; - if (K % 16 != 0) return false; // feature-atomic align - // weight must be dequantizable to F32 (F32 src0 has no to_float - // trait -> leave those on the CPU backend) - if (ggml_get_type_traits(src0->type)->to_float == NULL) return false; - return true; + const char *why = NULL; + if (M < ROCKET_MIN_BATCH) why="Mtype != GGML_TYPE_F32) why="src1!=F32"; + else if (!ggml_is_contiguous(src0)) why="src0 noncontig"; + else if (!ggml_is_contiguous(src1)) why="src1 noncontig"; + else if (K <= 0 || K > ROCKET_K_MAX) why="K>Kmax"; + else if (K % 16 != 0) why="K%16"; + else if (ggml_get_type_traits(src0->type)->to_float == NULL) why="src0 no to_float"; + if (getenv("GGML_ROCKET_DEBUG")) + fprintf(stderr, "[rocket-supp] MUL_MAT M=%lld N=%lld K=%lld src0=%s src1=%s -> %s\n", + (long long)M, (long long)op->ne[0], (long long)K, + ggml_type_name(src0->type), ggml_type_name(src1->type), + why ? why : "OFFLOAD"); + return why == NULL; } default: @@ -375,6 +375,15 @@ static bool ggml_backend_rocket_device_supports_op(ggml_backend_dev_t dev, const GGML_UNUSED(dev); } +static bool ggml_backend_rocket_device_offload_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) { + // Tell the scheduler we want MUL_MATs even when their weights live in CPU + // host buffers (supports_op still does the real per-op gating). Without + // this the sched's op-offload path (ggml_backend_offload_op) returns false + // and almost nothing reaches the NPU. + GGML_UNUSED(dev); + return op->op == GGML_OP_MUL_MAT; +} + static bool ggml_backend_rocket_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) { return ggml_backend_buft_is_host(buft); GGML_UNUSED(dev); @@ -392,7 +401,7 @@ static const struct ggml_backend_device_i ggml_backend_rocket_device_i = { /* .buffer_from_host_ptr = */ ggml_backend_rocket_device_buffer_from_host_ptr, /* .supports_op = */ ggml_backend_rocket_device_supports_op, /* .supports_buft = */ ggml_backend_rocket_device_supports_buft, - /* .offload_op = */ NULL, + /* .offload_op = */ ggml_backend_rocket_device_offload_op, /* .event_new = */ NULL, /* .event_free = */ NULL, /* .event_synchronize = */ NULL,