From 4a0bfcde1568c450bb3e1b654339c4ccd413e69f Mon Sep 17 00:00:00 2001 From: Markus Fritsche Date: Tue, 14 Jul 2026 09:13:48 +0200 Subject: [PATCH] Rosenblatt U6: per-output-channel requant in rkt_npu_matmul Adds optional out_scales[N] (per-output-channel requant scale) to rkt_npu_matmul; per N-tile it uses the tile's channel scale (exact per-channel with tile_n=1). This is what gemma4's per-channel weight quantization needs. Verified on NPU: per-channel test (out_scales[oc]=1/(oc+1)) -> row 144 160 176 192 208 224 240 255 matches CPU exactly. All 5 gemmtest cases pass. Next U6: ggml glue (Q8_0 weights -> per-oc int8 + per-oc out_scale array; F32 activations -> int8 + activation scale; call this primitive). Co-Authored-By: Claude Opus 4.8 --- userspace/npu-probe/gemmtest.c | 57 ++++++++++++++++++++-------- userspace/npu-probe/rkt_npu_matmul.c | 4 +- userspace/npu-probe/rkt_npu_matmul.h | 2 +- 3 files changed, 44 insertions(+), 19 deletions(-) diff --git a/userspace/npu-probe/gemmtest.c b/userspace/npu-probe/gemmtest.c index dc11daf..3b428a3 100644 --- a/userspace/npu-probe/gemmtest.c +++ b/userspace/npu-probe/gemmtest.c @@ -1,13 +1,15 @@ /* SPDX-License-Identifier: MIT * * gemmtest.c — test driver for rkt_npu_matmul() (the U6 core primitive). - * Runs arbitrary (M,N,K) INT8 matmuls on the rocket NPU and checks vs CPU. + * Runs arbitrary (M,N,K) INT8 matmuls on the rocket NPU and checks vs CPU, + * incl. per-output-channel requant scales (gemma-style). */ #define _GNU_SOURCE #include #include #include #include +#include #include "librocket.h" #include "rkt_npu_matmul.h" @@ -16,7 +18,9 @@ #define WZP 128 #define OZP 128 -/* uniform +1 operands: acc = K -> saturate int8 -> +128. returns #mismatches. */ +static int sat8(int v) { return v > 127 ? 127 : (v < -128 ? -128 : v); } + +/* uniform +1 operands, single global scale=1: acc=K -> sat -> +128 */ static long run(int fd, uint32_t M, uint32_t N, uint32_t K, uint32_t tm, uint32_t tn, const char *tag) { @@ -24,25 +28,45 @@ static long run(int fd, uint32_t M, uint32_t N, uint32_t K, uint8_t *Y = calloc((size_t)M, N); memset(X, IZP + 1, (size_t)M * K); memset(Wc, WZP + 1, (size_t)N * K); - int r = rkt_npu_matmul(fd, X, Wc, NULL, M, N, K, IZP, WZP, OZP, - 1.0f, 1.0f, 1.0f, tm, tn, Y); - int expect = (int)K > 127 ? 127 : (int)K; - expect += 128; - long bad = 0; - if (r) { - bad = (long)M * N; - } else { - for (long i = 0; i < (long)M * N; i++) - if (Y[i] != (uint8_t)expect) bad++; - } - printf("%-28s M=%-5u N=%-4u K=%-5u tile=%ux%u Y[0]=%u exp=%u %s (%ld/%ld)\n", - tag, M, N, K, tm, tn, Y[0], (unsigned)expect, - bad ? "FAIL" : "OK", bad, (long)M * N); + 1.0f, 1.0f, 1.0f, NULL, tm, tn, Y); + int expect = sat8((int)K) + 128; + long bad = r ? (long)M * N : 0; + if (!r) for (long i = 0; i < (long)M * N; i++) + if (Y[i] != (uint8_t)expect) bad++; + printf("%-26s M=%-5u N=%-4u K=%-5u tile=%ux%u Y[0]=%u exp=%u %s (%ld/%ld)\n", + tag, M, N, K, tm, tn, Y[0], (unsigned)expect, bad ? "FAIL" : "OK", + bad, (long)M * N); free(X); free(Wc); free(Y); return bad; } +/* per-output-channel scales: out_scales[oc]=1/(oc+1) => conv_scale=oc+1; + * X,Wc uniform +1 => acc=K => out[oc]=sat(K*(oc+1))+128. tile_n=1 = exact. */ +static long pc_run(int fd, uint32_t M, uint32_t N, uint32_t K, const char *tag) +{ + uint8_t *X = malloc((size_t)M * K), *Wc = malloc((size_t)N * K); + uint8_t *Y = calloc((size_t)M, N); + float *os = malloc(N * sizeof(float)); + memset(X, IZP + 1, (size_t)M * K); + memset(Wc, WZP + 1, (size_t)N * K); + for (unsigned oc = 0; oc < N; oc++) os[oc] = 1.0f / (float)(oc + 1); + int r = rkt_npu_matmul(fd, X, Wc, NULL, M, N, K, IZP, WZP, OZP, + 1.0f, 1.0f, 1.0f, os, M, 1, Y); /* tile_n=1 */ + long bad = r ? (long)M * N : 0; + if (!r) for (unsigned m = 0; m < M; m++) + for (unsigned oc = 0; oc < N; oc++) { + int exp = sat8((int)(K * (oc + 1))) + 128; + int g = Y[(size_t)m * N + oc]; + if (g > exp + 1 || g < exp - 1) bad++; + } + printf("%-26s M=%-5u N=%-4u K=%-5u per-chan tile_n=1 row0:", tag, M, N, K); + for (unsigned oc = 0; oc < N && oc < 8; oc++) printf(" %u", Y[oc]); + printf(" %s (%ld/%ld)\n", bad ? "FAIL" : "OK", bad, (long)M * N); + free(X); free(Wc); free(Y); free(os); + return bad; +} + int main(void) { int fd = rocket_open("/dev/accel/accel0"); @@ -52,6 +76,7 @@ int main(void) bad += run(fd, 4096, 8, 16, 1024, 8, "large-M (tiled)"); bad += run(fd, 2048, 256, 16, 1024, 128, "2D-tiled"); bad += run(fd, 12, 256, 2048, 6, 128, "gemma-scale K=2048"); + bad += pc_run(fd, 4, 8, 16, "per-channel scales"); printf(bad ? "\nSOME FAILED\n" : "\nALL rkt_npu_matmul TESTS PASSED\n"); return bad ? 1 : 0; } diff --git a/userspace/npu-probe/rkt_npu_matmul.c b/userspace/npu-probe/rkt_npu_matmul.c index c755698..84759a3 100644 --- a/userspace/npu-probe/rkt_npu_matmul.c +++ b/userspace/npu-probe/rkt_npu_matmul.c @@ -48,7 +48,7 @@ static void bo_write(int fd, struct rkt_bo *b, const void *src, unsigned n) int rkt_npu_matmul(int fd, const uint8_t *X, const uint8_t *Wc, const int32_t *bias, uint32_t M, uint32_t N, uint32_t K, uint8_t izp, uint8_t wzp, uint8_t ozp, - float in_scale, float wt_scale, float out_scale, + float in_scale, float wt_scale, float out_scale, const float *out_scales, uint32_t tile_m, uint32_t tile_n, uint8_t *Y) { int ret = -1; @@ -106,7 +106,7 @@ int rkt_npu_matmul(int fd, const uint8_t *X, const uint8_t *Wc, int nw = rkt_build_matmul_regcmd_scaled( rc, 4096, m, n, K, in.dma, w.dma, o.dma, izp, wzp, ozp, - in_scale, wt_scale, out_scale, b.dma); + in_scale, wt_scale, out_scales ? out_scales[cc] : out_scale, b.dma); if (nw < 0) { rocket_close_bo(fd, in.h); rocket_close_bo(fd, w.h); rocket_close_bo(fd, b.h); rocket_close_bo(fd, o.h); diff --git a/userspace/npu-probe/rkt_npu_matmul.h b/userspace/npu-probe/rkt_npu_matmul.h index 0b022f5..705237b 100644 --- a/userspace/npu-probe/rkt_npu_matmul.h +++ b/userspace/npu-probe/rkt_npu_matmul.h @@ -29,7 +29,7 @@ int rkt_npu_matmul(int fd, const uint8_t *X, const uint8_t *Wc, const int32_t *bias, uint32_t M, uint32_t N, uint32_t K, uint8_t izp, uint8_t wzp, uint8_t ozp, - float in_scale, float wt_scale, float out_scale, + float in_scale, float wt_scale, float out_scale, const float *out_scales, uint32_t tile_m, uint32_t tile_n, uint8_t *Y); #endif /* RKT_NPU_MATMUL_H */