From 04d2c6e548d11eb0de0a851744cccf7e316d236d Mon Sep 17 00:00:00 2001 From: mfritsche Date: Thu, 9 Jul 2026 00:38:02 +0200 Subject: [PATCH] =?UTF-8?q?rknpu2:=20M-bucketing=20=E2=80=94=20ceil-to-128?= =?UTF-8?q?=20for=20M>128=20instead=20of=20next=5Fpower=5Fof=5Ftwo?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Experiment: next_power_of_two pads prefill M up to the next pow2 (300->512), wasting matmul work. Switch to ceil-to-128 for M>128 (300->384); keep pow2 for M<=128. Byte-identical output (padding only). Baseline pp300/pp512/pp700/tg128: 30.66/40.42/40.17/2.58 t/s. Measured: 37.01/40.25/40.03/2.61 t/s. pp300 +20.7%; pp512/pp700 flat (ubatch<=512 caps M; 700=512+188, both bucketings map 188->256); tg unchanged. --- ggml/src/ggml-rknpu2/ggml-rknpu2.cpp | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ggml/src/ggml-rknpu2/ggml-rknpu2.cpp b/ggml/src/ggml-rknpu2/ggml-rknpu2.cpp index 67fa4562b..9abc4b2cb 100644 --- a/ggml/src/ggml-rknpu2/ggml-rknpu2.cpp +++ b/ggml/src/ggml-rknpu2/ggml-rknpu2.cpp @@ -488,9 +488,12 @@ static enum ggml_status ggml_backend_rknpu_graph_compute(ggml_backend_t backend, continue; } - // Using next power of two for M for efficient caching + // M bucketing: pow2 up to 128, then ceil-to-128 for larger M. + // Avoids next_power_of_two doubling waste at large prefill (300->512, 700->1024). int M_op = M; - if (M > 1) { + if (M > 128) { + M_op = ((M + 127) / 128) * 128; + } else if (M > 1) { M_op = rknpu2_calibration::next_power_of_two(M); }