diff --git a/userspace/npu-probe/hwtest.c b/userspace/npu-probe/hwtest.c index 812e78e..78d3386 100644 --- a/userspace/npu-probe/hwtest.c +++ b/userspace/npu-probe/hwtest.c @@ -1,29 +1,30 @@ /* SPDX-License-Identifier: MIT * - * hwtest.c — first real matmul submission to the mainline rocket NPU. + * hwtest.c — run a real INT8 matmul on the mainline rocket NPU and verify. * - * Allocates BOs, builds our golden regcmd with the BOs' real NPU DMA - * addresses, submits the job to /dev/accel/accel0, and waits for completion. - * - * GOAL of this test: prove the SUBMIT PATH — that a job built from our regcmd - * executes on the NPU and completes (no IOMMU fault / no timeout). Numeric - * correctness needs the NPU feature/weight tiling layout (next unit); here the - * operands are zeroed, so the output value is not yet meaningful. + * Y[M][N] = X[M][K] * W[K][N], expressed as a 1x1 conv (input_height=M, + * input_channels=K, output_channels=N). Operands are laid out in the NPU's + * tiled format (rkt_operands.c), submitted via our golden regcmd, and the + * de-tiled output is compared against a CPU reference. */ #define _GNU_SOURCE #include #include +#include #include #include #include "librocket.h" #include "rocket_accel.h" #include "rkt_matmul.h" +#include "rkt_operands.h" #include "coredpu_defs.h" -#define DEV "/dev/accel/accel0" -#define BOSZ 0x10000u /* 64 KiB data BOs — far above this tiny op's strides */ -#define REGSZ 0x1000u +#define DEV "/dev/accel/accel0" + +/* dims: M rows, K contraction, N outputs; 1x1 conv (iw=ow=1) */ +enum { M = 4, K = 16, N = 8, IW = 1, OW = 1, WW = 1, WH = 1 }; +enum { IZP = 128, WZP = 128, OZP = 128 }; struct bo { uint32_t handle; @@ -33,100 +34,130 @@ struct bo { uint32_t size; }; -static int mkbo(int fd, struct bo *b, uint32_t size, const char *name) +static int mkbo(int fd, struct bo *b, uint32_t size) { - b->size = size; - int r = rocket_create_bo(fd, size, &b->handle, &b->dma, &b->off); - if (r) { - printf("create_bo(%s,%u) failed: %d\n", name, size, r); + b->size = (size + 0xfff) & ~0xfffu; + int r = rocket_create_bo(fd, b->size, &b->handle, &b->dma, &b->off); + if (r) return r; - } - b->map = rocket_mmap_bo(fd, b->off, size); - if (b->map == MAP_FAILED) { - printf("mmap(%s) failed\n", name); - return -1; - } - return 0; + b->map = rocket_mmap_bo(fd, b->off, b->size); + return b->map == MAP_FAILED ? -1 : 0; +} +static void put(int fd, struct bo *b, const void *src, unsigned n) +{ + rocket_prep_bo(fd, b->handle, -1); + memset(b->map, 0, b->size); + if (src) + memcpy(b->map, src, n); + rocket_fini_bo(fd, b->handle); } int main(void) { + /* ---- host operands ---- */ + uint8_t X[IW * M * K]; /* [iw][ih=M][ic=K] */ + uint8_t Wt[N * WW * WH * K]; /* [oc=N][ww][wh][ic=K] */ + int32_t Bin[N]; + for (int i = 0; i < IW * M * K; i++) + X[i] = 128; + for (int m = 0; m < M; m++) + X[m * K + 0] = 128 + 1; /* signed +1 at channel 0 */ + memset(Wt, 128, sizeof Wt); + for (int n = 0; n < N; n++) + Wt[n * K + 0] = 128 + (n + 1); /* signed +(n+1) at channel 0 */ + for (int n = 0; n < N; n++) + Bin[n] = 0; + + /* ---- CPU reference (identity requant; izp=128 => no bias correction) ---- */ + uint8_t ref[M * OW * N]; + for (int m = 0; m < M; m++) + for (int n = 0; n < N; n++) { + int32_t acc = 0; + for (int k = 0; k < K; k++) + acc += ((int)X[m * K + k] - IZP) * + ((int)Wt[n * K + k] - WZP); + acc += Bin[n]; + if (acc > 127) acc = 127; + if (acc < -128) acc = -128; + ref[m * N + n] = (uint8_t)(acc + 128); + } + + /* ---- pack into NPU layout ---- */ + unsigned wsz = rkt_packed_weights_size(WW, WH, K, N); + unsigned isz = rkt_raw_input_size(IW, M, K); + unsigned osz = rkt_raw_output_size(OW, M, N); + uint8_t *wpk = calloc(1, wsz), *ipk = calloc(1, isz); + int32_t bpk[N]; + rkt_pack_weights(Wt, WW, WH, K, N, WZP, wpk); + rkt_pack_input(X, IW, M, K, IZP, ipk); + rkt_compute_biases(Wt, Bin, WW, WH, K, N, WZP, IZP, bpk); + + /* ---- open + allocate (guard first so nothing sits at IOVA 0) ---- */ int fd = rocket_open(DEV); - if (fd < 0) { - printf("open %s failed: %d\n", DEV, fd); - return 1; + if (fd < 0) { printf("open failed %d\n", fd); return 1; } + struct bo guard = {0}, in = {0}, wt = {0}, bias = {0}, out = {0}, reg = {0}; + if (mkbo(fd, &guard, 0x1000) || mkbo(fd, &wt, wsz) || mkbo(fd, &bias, sizeof bpk) || + mkbo(fd, &in, isz) || mkbo(fd, &out, osz) || mkbo(fd, ®, 0x1000)) { + printf("bo alloc failed\n"); return 1; } - printf("opened %s fd=%d\n", DEV, fd); - - struct bo in = {0}, wt = {0}, bias = {0}, out = {0}, reg = {0}; - if (mkbo(fd, &in, BOSZ, "input") || mkbo(fd, &wt, BOSZ, "weights") || - mkbo(fd, &bias, BOSZ, "bias") || mkbo(fd, &out, BOSZ, "output") || - mkbo(fd, ®, REGSZ, "regcmd")) - return 1; - printf("BO dmas: in=%#x wt=%#x bias=%#x out=%#x reg=%#x\n", + printf("dmas: in=%#x wt=%#x bias=%#x out=%#x reg=%#x (guard=%#x)\n", (uint32_t)in.dma, (uint32_t)wt.dma, (uint32_t)bias.dma, - (uint32_t)out.dma, (uint32_t)reg.dma); + (uint32_t)out.dma, (uint32_t)reg.dma, (uint32_t)guard.dma); - /* zero operands (prep = take CPU ownership / sync, then release) */ - struct bo *z[] = { &in, &wt, &bias, &out }; - for (int i = 0; i < 4; i++) { - rocket_prep_bo(fd, z[i]->handle, -1); - memset(z[i]->map, 0, z[i]->size); - rocket_fini_bo(fd, z[i]->handle); - } + put(fd, &wt, wpk, wsz); + put(fd, &in, ipk, isz); + put(fd, &bias, bpk, sizeof bpk); + put(fd, &out, NULL, 0); - /* golden regcmd with the REAL bo dma addresses */ + /* ---- golden regcmd with real addresses; patch bias addr ---- */ uint64_t rc[512]; - int n = rkt_build_matmul_regcmd(rc, 512, 4, 8, 16, in.dma, wt.dma, - out.dma, 128, 128, 128); - if (n < 0) { - printf("build regcmd failed\n"); - return 1; - } - /* builder hardcodes biases_addr=0; point it at the real bias BO */ - int patched = 0; - for (int i = 0; i < n; i++) { - if ((uint32_t)(rc[i] & 0xffff) == REG_DPU_RDMA_RDMA_BS_BASE_ADDR) { + int n = rkt_build_matmul_regcmd(rc, 512, M, N, K, in.dma, wt.dma, out.dma, + IZP, WZP, OZP); + if (n < 0) { printf("regcmd build failed\n"); return 1; } + for (int i = 0; i < n; i++) + if ((uint32_t)(rc[i] & 0xffff) == REG_DPU_RDMA_RDMA_BS_BASE_ADDR) rc[i] = (rc[i] & ~(0xffffffffULL << 16)) | ((uint64_t)(uint32_t)bias.dma << 16); - patched++; - } - } - printf("regcmd: %d words, patched %d bias-addr word(s)\n", n, patched); + put(fd, ®, rc, (unsigned)n * sizeof(uint64_t)); - rocket_prep_bo(fd, reg.handle, -1); - memcpy(reg.map, rc, (size_t)n * sizeof(uint64_t)); - rocket_fini_bo(fd, reg.handle); - - struct drm_rocket_task task = { - .regcmd = (uint32_t)reg.dma, - .regcmd_count = (uint32_t)n, - }; - uint32_t in_h[] = { in.handle, wt.handle, bias.handle, reg.handle }; - uint32_t out_h[] = { out.handle }; + /* ---- submit ---- */ + struct drm_rocket_task task = { .regcmd = (uint32_t)reg.dma, + .regcmd_count = (uint32_t)n }; + uint32_t inh[] = { in.handle, wt.handle, bias.handle, reg.handle }; + uint32_t outh[] = { out.handle }; struct drm_rocket_job job; memset(&job, 0, sizeof job); job.tasks = (uintptr_t)&task; job.task_count = 1; job.task_struct_size = sizeof(struct drm_rocket_task); - job.in_bo_handles = (uintptr_t)in_h; + job.in_bo_handles = (uintptr_t)inh; job.in_bo_handle_count = 4; - job.out_bo_handles = (uintptr_t)out_h; + job.out_bo_handles = (uintptr_t)outh; job.out_bo_handle_count = 1; int sr = rocket_submit(fd, &job, 1); - printf("SUBMIT rc=%d %s\n", sr, sr == 0 ? "(accepted)" : "(FAILED)"); - if (sr) - return 1; + printf("SUBMIT rc=%d\n", sr); + if (sr) return 1; + int wr = rocket_prep_bo(fd, out.handle, 3000000000LL); + printf("WAIT rc=%d %s\n", wr, wr == 0 ? "== COMPLETED ==" : "(timeout/err)"); + if (wr) return 2; - int wr = rocket_prep_bo(fd, out.handle, 2000000000LL); - printf("WAIT rc=%d %s\n", wr, wr == 0 ? "== JOB COMPLETED ==" : "(timeout/err)"); - - unsigned char *o = out.map; - printf("output[0..15]:"); - for (int i = 0; i < 16; i++) - printf(" %02x", o[i]); - printf("\n"); + /* ---- de-tile + verify ---- */ + uint8_t got[M * OW * N]; + rkt_unpack_output(out.map, OW, M, N, got); + int bad = 0; + for (int m = 0; m < M; m++) { + printf("row %d ref:", m); + for (int j = 0; j < N; j++) printf(" %3u", ref[m * N + j]); + printf(" npu:"); + for (int j = 0; j < N; j++) { + unsigned g = got[m * N + j], r = ref[m * N + j]; + printf(" %3u", g); + if (g > r + 1 || r > g + 1) bad++; + } + printf("\n"); + } + printf(bad ? "\nMISMATCH: %d elems off by >1\n" : "\nMATCH (within +/-1) — NPU MATMUL CORRECT\n", bad); rocket_fini_bo(fd, out.handle); - return wr == 0 ? 0 : 2; + return bad ? 3 : 0; } diff --git a/userspace/npu-probe/rkt_operands.c b/userspace/npu-probe/rkt_operands.c new file mode 100644 index 0000000..89afc0d --- /dev/null +++ b/userspace/npu-probe/rkt_operands.c @@ -0,0 +1,127 @@ +/* SPDX-License-Identifier: MIT + * + * rkt_operands.c — NPU operand layout (faithful ports of Mesa rocket). + * + * Non-depthwise conv/matmul. Row-major host tensors: + * weights_in : [oc_real][ww][wh][ic_real] (conv weights) + * input_in : [iw][ih][ic_real] + * output_out : [oh][ow][oc_real] + * + * Ported from Mesa rkt_coefs.c (rkt_fill_weights, rkt_fill_biases) and + * rkt_ml.c (feature tiling in rkt_ml_subgraph_invoke, output de-tile in + * rkt_ml_subgraph_read_outputs). Assumes input_zero_point handling as Mesa. + */ +#include "rkt_operands.h" + +#define FEATURE_ATOMIC_SIZE 16 +#define WEIGHT_ATOMIC_SIZE 32 +#define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d)) +#define MAX2(a, b) ((a) > (b) ? (a) : (b)) +#define MIN2(a, b) ((a) < (b) ? (a) : (b)) +static unsigned alignu(unsigned v, unsigned a) { return ((v + a - 1) / a) * a; } + +unsigned rkt_raw_input_size(unsigned iw, unsigned ih, unsigned ic_real) +{ + return iw * ih * (DIV_ROUND_UP(ic_real, FEATURE_ATOMIC_SIZE) * 2) * + FEATURE_ATOMIC_SIZE; +} + +unsigned rkt_raw_output_size(unsigned ow, unsigned oh, unsigned oc_real) +{ + return ow * oh * (DIV_ROUND_UP(oc_real, FEATURE_ATOMIC_SIZE) * 2) * + FEATURE_ATOMIC_SIZE; +} + +unsigned rkt_packed_weights_size(unsigned ww, unsigned wh, unsigned ic_real, + unsigned oc_real) +{ + unsigned ic = alignu(MAX2(ic_real, FEATURE_ATOMIC_SIZE), FEATURE_ATOMIC_SIZE); + unsigned oc = alignu(oc_real, 2); + return ww * wh * oc * alignu(ic, WEIGHT_ATOMIC_SIZE) * 2; +} + +/* Mesa rkt_fill_weights (non-depthwise). */ +unsigned rkt_pack_weights(const uint8_t *win, unsigned ww, unsigned wh, + unsigned ic_real, unsigned oc_real, uint8_t wzp, + uint8_t *out) +{ + unsigned ic = alignu(MAX2(ic_real, FEATURE_ATOMIC_SIZE), FEATURE_ATOMIC_SIZE); + unsigned oc = alignu(oc_real, 2); + unsigned groups = WEIGHT_ATOMIC_SIZE; + unsigned ic1n = DIV_ROUND_UP(ic, groups); + unsigned ic2n = MIN2(ic, groups); + unsigned n = 0; + + for (unsigned oc1 = 0; oc1 < DIV_ROUND_UP(oc, WEIGHT_ATOMIC_SIZE); oc1++) + for (unsigned ic1 = 0; ic1 < ic1n; ic1++) + for (unsigned x = 0; x < ww; x++) + for (unsigned y = 0; y < wh; y++) + for (unsigned oc2 = 0; + oc2 < MIN2(oc, WEIGHT_ATOMIC_SIZE); oc2++) + for (unsigned ic2 = 0; ic2 < ic2n; ic2++) { + unsigned occ = oc1 * WEIGHT_ATOMIC_SIZE + oc2; + unsigned icc = ic1 * groups + ic2; + if (oc_real > 2 && occ >= alignu(oc_real, 2)) + continue; + if (occ >= oc_real) { + out[n++] = 0x0; + } else if (icc >= ic_real) { + if (ic2 < 16 || (ic_real % 32) > 16) + out[n++] = (uint8_t)(wzp - 0x80); + } else { + unsigned idx = ((occ * ww + x) * wh + y) * ic_real + icc; + out[n++] = (uint8_t)(win[idx] - 0x80); + } + } + return n; +} + +/* Mesa rkt_fill_biases (truncate_bits=0 path) + calculate_bias_correction. */ +void rkt_compute_biases(const uint8_t *win, const int32_t *bin, unsigned ww, + unsigned wh, unsigned ic_real, unsigned oc_real, + uint8_t wzp, uint8_t izp, int32_t *bout) +{ + for (unsigned oc = 0; oc < oc_real; oc++) { + int32_t corr = 0; + for (unsigned x = 0; x < ww; x++) + for (unsigned y = 0; y < wh; y++) + for (unsigned ic = 0; ic < ic_real; ic++) { + unsigned idx = ((oc * ww + x) * wh + y) * ic_real + ic; + corr += ((int32_t)win[idx] - wzp) * + ((int32_t)izp - 0x80); + } + bout[oc] = bin[oc] - corr; + } +} + +/* Mesa feature tiling (rkt_ml_subgraph_invoke, input_channels>1 path). */ +unsigned rkt_pack_input(const uint8_t *iin, unsigned iw, unsigned ih, + unsigned ic_real, uint8_t izp, uint8_t *out) +{ + unsigned n = 0; + for (unsigned u = 0; u < DIV_ROUND_UP(ic_real, FEATURE_ATOMIC_SIZE); u++) + for (unsigned x = 0; x < iw; x++) + for (unsigned y = 0; y < ih; y++) + for (unsigned c = 0; c < FEATURE_ATOMIC_SIZE; c++) { + unsigned ich = c + u * FEATURE_ATOMIC_SIZE; + if (ich < ic_real) + out[n++] = (uint8_t)(iin[(x * ih + y) * ic_real + ich] - 0x80); + else + out[n++] = (uint8_t)(izp - 0x80); + } + return n; +} + +/* Mesa output de-tile (rkt_ml_subgraph_read_outputs). */ +void rkt_unpack_output(const uint8_t *raw, unsigned ow, unsigned oh, + unsigned oc_real, uint8_t *out) +{ + for (unsigned oc = 0; oc < oc_real; oc++) + for (unsigned x = 0; x < ow; x++) + for (unsigned y = 0; y < oh; y++) { + unsigned c = oc % FEATURE_ATOMIC_SIZE; + unsigned g = oc / FEATURE_ATOMIC_SIZE; + out[(y * ow + x) * oc_real + oc] = + (uint8_t)(raw[((g * oh + y) * ow + x) * FEATURE_ATOMIC_SIZE + c] + 0x80); + } +} diff --git a/userspace/npu-probe/rkt_operands.h b/userspace/npu-probe/rkt_operands.h new file mode 100644 index 0000000..fc76876 --- /dev/null +++ b/userspace/npu-probe/rkt_operands.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: MIT + * + * rkt_operands.h — NPU operand layout ports (see rkt_operands.c). + * Non-depthwise. Row-major host tensors: + * weights_in [oc_real][ww][wh][ic_real], input_in [iw][ih][ic_real], + * output_out [oh][ow][oc_real]. + */ +#ifndef RKT_OPERANDS_H +#define RKT_OPERANDS_H + +#include + +unsigned rkt_raw_input_size(unsigned iw, unsigned ih, unsigned ic_real); +unsigned rkt_raw_output_size(unsigned ow, unsigned oh, unsigned oc_real); +unsigned rkt_packed_weights_size(unsigned ww, unsigned wh, unsigned ic_real, + unsigned oc_real); + +unsigned rkt_pack_weights(const uint8_t *weights_in, unsigned ww, unsigned wh, + unsigned ic_real, unsigned oc_real, uint8_t wzp, + uint8_t *out); +void rkt_compute_biases(const uint8_t *weights_in, const int32_t *biases_in, + unsigned ww, unsigned wh, unsigned ic_real, + unsigned oc_real, uint8_t wzp, uint8_t izp, + int32_t *biases_out); +unsigned rkt_pack_input(const uint8_t *input_in, unsigned iw, unsigned ih, + unsigned ic_real, uint8_t izp, uint8_t *out); +void rkt_unpack_output(const uint8_t *raw, unsigned ow, unsigned oh, + unsigned oc_real, uint8_t *out); + +#endif /* RKT_OPERANDS_H */