Rosenblatt: NPU operand layout (weights/features/bias/output tiling) + hwtest v2

rkt_operands.{c,h}: faithful ports of Mesa's operand layout — rkt_pack_weights
(rkt_coefs.c tiling+sign), rkt_compute_biases (zero-point correction),
rkt_pack_input (feature tiling from rkt_ml_subgraph_invoke), rkt_unpack_output
(de-tile from read_outputs). hwtest.c v2 lays out a real 4x8x16 INT8 matmul,
submits our golden regcmd with real BO addresses, and checks vs a CPU ref.

State: SUBMIT accepted; job still does NOT complete. Root cause located in the
kernel driver (rocket_job.c): completion is a DPU interrupt (INTERRUPT_MASK
DPU_0|DPU_1) and JOB_TIMEOUT_MS=500 — our job runs but the DPU never raises
done within 500ms, so drm_sched resets the core and prep_bo returns -EBUSY.
Interface/regcmd/operands all match Mesa; the stall is at hardware execution.
Next: known-good Mesa/Teflon reference run on this NPU to disambiguate operand
tiling vs early-driver/DTB corner, or NPU status-register readback post-timeout.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Markus Fritsche
2026-07-13 23:56:11 +02:00
parent 673a36c6db
commit 2b34125760
3 changed files with 269 additions and 81 deletions
+112 -81
View File
@@ -1,29 +1,30 @@
/* SPDX-License-Identifier: MIT
*
* hwtest.c — first real matmul submission to the mainline rocket NPU.
* hwtest.c — run a real INT8 matmul on the mainline rocket NPU and verify.
*
* Allocates BOs, builds our golden regcmd with the BOs' real NPU DMA
* addresses, submits the job to /dev/accel/accel0, and waits for completion.
*
* GOAL of this test: prove the SUBMIT PATH — that a job built from our regcmd
* executes on the NPU and completes (no IOMMU fault / no timeout). Numeric
* correctness needs the NPU feature/weight tiling layout (next unit); here the
* operands are zeroed, so the output value is not yet meaningful.
* Y[M][N] = X[M][K] * W[K][N], expressed as a 1x1 conv (input_height=M,
* input_channels=K, output_channels=N). Operands are laid out in the NPU's
* tiled format (rkt_operands.c), submitted via our golden regcmd, and the
* de-tiled output is compared against a CPU reference.
*/
#define _GNU_SOURCE
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include "librocket.h"
#include "rocket_accel.h"
#include "rkt_matmul.h"
#include "rkt_operands.h"
#include "coredpu_defs.h"
#define DEV "/dev/accel/accel0"
#define BOSZ 0x10000u /* 64 KiB data BOs — far above this tiny op's strides */
#define REGSZ 0x1000u
#define DEV "/dev/accel/accel0"
/* dims: M rows, K contraction, N outputs; 1x1 conv (iw=ow=1) */
enum { M = 4, K = 16, N = 8, IW = 1, OW = 1, WW = 1, WH = 1 };
enum { IZP = 128, WZP = 128, OZP = 128 };
struct bo {
uint32_t handle;
@@ -33,100 +34,130 @@ struct bo {
uint32_t size;
};
static int mkbo(int fd, struct bo *b, uint32_t size, const char *name)
static int mkbo(int fd, struct bo *b, uint32_t size)
{
b->size = size;
int r = rocket_create_bo(fd, size, &b->handle, &b->dma, &b->off);
if (r) {
printf("create_bo(%s,%u) failed: %d\n", name, size, r);
b->size = (size + 0xfff) & ~0xfffu;
int r = rocket_create_bo(fd, b->size, &b->handle, &b->dma, &b->off);
if (r)
return r;
}
b->map = rocket_mmap_bo(fd, b->off, size);
if (b->map == MAP_FAILED) {
printf("mmap(%s) failed\n", name);
return -1;
}
return 0;
b->map = rocket_mmap_bo(fd, b->off, b->size);
return b->map == MAP_FAILED ? -1 : 0;
}
static void put(int fd, struct bo *b, const void *src, unsigned n)
{
rocket_prep_bo(fd, b->handle, -1);
memset(b->map, 0, b->size);
if (src)
memcpy(b->map, src, n);
rocket_fini_bo(fd, b->handle);
}
int main(void)
{
/* ---- host operands ---- */
uint8_t X[IW * M * K]; /* [iw][ih=M][ic=K] */
uint8_t Wt[N * WW * WH * K]; /* [oc=N][ww][wh][ic=K] */
int32_t Bin[N];
for (int i = 0; i < IW * M * K; i++)
X[i] = 128;
for (int m = 0; m < M; m++)
X[m * K + 0] = 128 + 1; /* signed +1 at channel 0 */
memset(Wt, 128, sizeof Wt);
for (int n = 0; n < N; n++)
Wt[n * K + 0] = 128 + (n + 1); /* signed +(n+1) at channel 0 */
for (int n = 0; n < N; n++)
Bin[n] = 0;
/* ---- CPU reference (identity requant; izp=128 => no bias correction) ---- */
uint8_t ref[M * OW * N];
for (int m = 0; m < M; m++)
for (int n = 0; n < N; n++) {
int32_t acc = 0;
for (int k = 0; k < K; k++)
acc += ((int)X[m * K + k] - IZP) *
((int)Wt[n * K + k] - WZP);
acc += Bin[n];
if (acc > 127) acc = 127;
if (acc < -128) acc = -128;
ref[m * N + n] = (uint8_t)(acc + 128);
}
/* ---- pack into NPU layout ---- */
unsigned wsz = rkt_packed_weights_size(WW, WH, K, N);
unsigned isz = rkt_raw_input_size(IW, M, K);
unsigned osz = rkt_raw_output_size(OW, M, N);
uint8_t *wpk = calloc(1, wsz), *ipk = calloc(1, isz);
int32_t bpk[N];
rkt_pack_weights(Wt, WW, WH, K, N, WZP, wpk);
rkt_pack_input(X, IW, M, K, IZP, ipk);
rkt_compute_biases(Wt, Bin, WW, WH, K, N, WZP, IZP, bpk);
/* ---- open + allocate (guard first so nothing sits at IOVA 0) ---- */
int fd = rocket_open(DEV);
if (fd < 0) {
printf("open %s failed: %d\n", DEV, fd);
return 1;
if (fd < 0) { printf("open failed %d\n", fd); return 1; }
struct bo guard = {0}, in = {0}, wt = {0}, bias = {0}, out = {0}, reg = {0};
if (mkbo(fd, &guard, 0x1000) || mkbo(fd, &wt, wsz) || mkbo(fd, &bias, sizeof bpk) ||
mkbo(fd, &in, isz) || mkbo(fd, &out, osz) || mkbo(fd, &reg, 0x1000)) {
printf("bo alloc failed\n"); return 1;
}
printf("opened %s fd=%d\n", DEV, fd);
struct bo in = {0}, wt = {0}, bias = {0}, out = {0}, reg = {0};
if (mkbo(fd, &in, BOSZ, "input") || mkbo(fd, &wt, BOSZ, "weights") ||
mkbo(fd, &bias, BOSZ, "bias") || mkbo(fd, &out, BOSZ, "output") ||
mkbo(fd, &reg, REGSZ, "regcmd"))
return 1;
printf("BO dmas: in=%#x wt=%#x bias=%#x out=%#x reg=%#x\n",
printf("dmas: in=%#x wt=%#x bias=%#x out=%#x reg=%#x (guard=%#x)\n",
(uint32_t)in.dma, (uint32_t)wt.dma, (uint32_t)bias.dma,
(uint32_t)out.dma, (uint32_t)reg.dma);
(uint32_t)out.dma, (uint32_t)reg.dma, (uint32_t)guard.dma);
/* zero operands (prep = take CPU ownership / sync, then release) */
struct bo *z[] = { &in, &wt, &bias, &out };
for (int i = 0; i < 4; i++) {
rocket_prep_bo(fd, z[i]->handle, -1);
memset(z[i]->map, 0, z[i]->size);
rocket_fini_bo(fd, z[i]->handle);
}
put(fd, &wt, wpk, wsz);
put(fd, &in, ipk, isz);
put(fd, &bias, bpk, sizeof bpk);
put(fd, &out, NULL, 0);
/* golden regcmd with the REAL bo dma addresses */
/* ---- golden regcmd with real addresses; patch bias addr ---- */
uint64_t rc[512];
int n = rkt_build_matmul_regcmd(rc, 512, 4, 8, 16, in.dma, wt.dma,
out.dma, 128, 128, 128);
if (n < 0) {
printf("build regcmd failed\n");
return 1;
}
/* builder hardcodes biases_addr=0; point it at the real bias BO */
int patched = 0;
for (int i = 0; i < n; i++) {
if ((uint32_t)(rc[i] & 0xffff) == REG_DPU_RDMA_RDMA_BS_BASE_ADDR) {
int n = rkt_build_matmul_regcmd(rc, 512, M, N, K, in.dma, wt.dma, out.dma,
IZP, WZP, OZP);
if (n < 0) { printf("regcmd build failed\n"); return 1; }
for (int i = 0; i < n; i++)
if ((uint32_t)(rc[i] & 0xffff) == REG_DPU_RDMA_RDMA_BS_BASE_ADDR)
rc[i] = (rc[i] & ~(0xffffffffULL << 16)) |
((uint64_t)(uint32_t)bias.dma << 16);
patched++;
}
}
printf("regcmd: %d words, patched %d bias-addr word(s)\n", n, patched);
put(fd, &reg, rc, (unsigned)n * sizeof(uint64_t));
rocket_prep_bo(fd, reg.handle, -1);
memcpy(reg.map, rc, (size_t)n * sizeof(uint64_t));
rocket_fini_bo(fd, reg.handle);
struct drm_rocket_task task = {
.regcmd = (uint32_t)reg.dma,
.regcmd_count = (uint32_t)n,
};
uint32_t in_h[] = { in.handle, wt.handle, bias.handle, reg.handle };
uint32_t out_h[] = { out.handle };
/* ---- submit ---- */
struct drm_rocket_task task = { .regcmd = (uint32_t)reg.dma,
.regcmd_count = (uint32_t)n };
uint32_t inh[] = { in.handle, wt.handle, bias.handle, reg.handle };
uint32_t outh[] = { out.handle };
struct drm_rocket_job job;
memset(&job, 0, sizeof job);
job.tasks = (uintptr_t)&task;
job.task_count = 1;
job.task_struct_size = sizeof(struct drm_rocket_task);
job.in_bo_handles = (uintptr_t)in_h;
job.in_bo_handles = (uintptr_t)inh;
job.in_bo_handle_count = 4;
job.out_bo_handles = (uintptr_t)out_h;
job.out_bo_handles = (uintptr_t)outh;
job.out_bo_handle_count = 1;
int sr = rocket_submit(fd, &job, 1);
printf("SUBMIT rc=%d %s\n", sr, sr == 0 ? "(accepted)" : "(FAILED)");
if (sr)
return 1;
printf("SUBMIT rc=%d\n", sr);
if (sr) return 1;
int wr = rocket_prep_bo(fd, out.handle, 3000000000LL);
printf("WAIT rc=%d %s\n", wr, wr == 0 ? "== COMPLETED ==" : "(timeout/err)");
if (wr) return 2;
int wr = rocket_prep_bo(fd, out.handle, 2000000000LL);
printf("WAIT rc=%d %s\n", wr, wr == 0 ? "== JOB COMPLETED ==" : "(timeout/err)");
unsigned char *o = out.map;
printf("output[0..15]:");
for (int i = 0; i < 16; i++)
printf(" %02x", o[i]);
printf("\n");
/* ---- de-tile + verify ---- */
uint8_t got[M * OW * N];
rkt_unpack_output(out.map, OW, M, N, got);
int bad = 0;
for (int m = 0; m < M; m++) {
printf("row %d ref:", m);
for (int j = 0; j < N; j++) printf(" %3u", ref[m * N + j]);
printf(" npu:");
for (int j = 0; j < N; j++) {
unsigned g = got[m * N + j], r = ref[m * N + j];
printf(" %3u", g);
if (g > r + 1 || r > g + 1) bad++;
}
printf("\n");
}
printf(bad ? "\nMISMATCH: %d elems off by >1\n" : "\nMATCH (within +/-1) — NPU MATMUL CORRECT\n", bad);
rocket_fini_bo(fd, out.handle);
return wr == 0 ? 0 : 2;
return bad ? 3 : 0;
}
+127
View File
@@ -0,0 +1,127 @@
/* SPDX-License-Identifier: MIT
*
* rkt_operands.c — NPU operand layout (faithful ports of Mesa rocket).
*
* Non-depthwise conv/matmul. Row-major host tensors:
* weights_in : [oc_real][ww][wh][ic_real] (conv weights)
* input_in : [iw][ih][ic_real]
* output_out : [oh][ow][oc_real]
*
* Ported from Mesa rkt_coefs.c (rkt_fill_weights, rkt_fill_biases) and
* rkt_ml.c (feature tiling in rkt_ml_subgraph_invoke, output de-tile in
* rkt_ml_subgraph_read_outputs). Assumes input_zero_point handling as Mesa.
*/
#include "rkt_operands.h"
#define FEATURE_ATOMIC_SIZE 16
#define WEIGHT_ATOMIC_SIZE 32
#define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
#define MAX2(a, b) ((a) > (b) ? (a) : (b))
#define MIN2(a, b) ((a) < (b) ? (a) : (b))
static unsigned alignu(unsigned v, unsigned a) { return ((v + a - 1) / a) * a; }
unsigned rkt_raw_input_size(unsigned iw, unsigned ih, unsigned ic_real)
{
return iw * ih * (DIV_ROUND_UP(ic_real, FEATURE_ATOMIC_SIZE) * 2) *
FEATURE_ATOMIC_SIZE;
}
unsigned rkt_raw_output_size(unsigned ow, unsigned oh, unsigned oc_real)
{
return ow * oh * (DIV_ROUND_UP(oc_real, FEATURE_ATOMIC_SIZE) * 2) *
FEATURE_ATOMIC_SIZE;
}
unsigned rkt_packed_weights_size(unsigned ww, unsigned wh, unsigned ic_real,
unsigned oc_real)
{
unsigned ic = alignu(MAX2(ic_real, FEATURE_ATOMIC_SIZE), FEATURE_ATOMIC_SIZE);
unsigned oc = alignu(oc_real, 2);
return ww * wh * oc * alignu(ic, WEIGHT_ATOMIC_SIZE) * 2;
}
/* Mesa rkt_fill_weights (non-depthwise). */
unsigned rkt_pack_weights(const uint8_t *win, unsigned ww, unsigned wh,
unsigned ic_real, unsigned oc_real, uint8_t wzp,
uint8_t *out)
{
unsigned ic = alignu(MAX2(ic_real, FEATURE_ATOMIC_SIZE), FEATURE_ATOMIC_SIZE);
unsigned oc = alignu(oc_real, 2);
unsigned groups = WEIGHT_ATOMIC_SIZE;
unsigned ic1n = DIV_ROUND_UP(ic, groups);
unsigned ic2n = MIN2(ic, groups);
unsigned n = 0;
for (unsigned oc1 = 0; oc1 < DIV_ROUND_UP(oc, WEIGHT_ATOMIC_SIZE); oc1++)
for (unsigned ic1 = 0; ic1 < ic1n; ic1++)
for (unsigned x = 0; x < ww; x++)
for (unsigned y = 0; y < wh; y++)
for (unsigned oc2 = 0;
oc2 < MIN2(oc, WEIGHT_ATOMIC_SIZE); oc2++)
for (unsigned ic2 = 0; ic2 < ic2n; ic2++) {
unsigned occ = oc1 * WEIGHT_ATOMIC_SIZE + oc2;
unsigned icc = ic1 * groups + ic2;
if (oc_real > 2 && occ >= alignu(oc_real, 2))
continue;
if (occ >= oc_real) {
out[n++] = 0x0;
} else if (icc >= ic_real) {
if (ic2 < 16 || (ic_real % 32) > 16)
out[n++] = (uint8_t)(wzp - 0x80);
} else {
unsigned idx = ((occ * ww + x) * wh + y) * ic_real + icc;
out[n++] = (uint8_t)(win[idx] - 0x80);
}
}
return n;
}
/* Mesa rkt_fill_biases (truncate_bits=0 path) + calculate_bias_correction. */
void rkt_compute_biases(const uint8_t *win, const int32_t *bin, unsigned ww,
unsigned wh, unsigned ic_real, unsigned oc_real,
uint8_t wzp, uint8_t izp, int32_t *bout)
{
for (unsigned oc = 0; oc < oc_real; oc++) {
int32_t corr = 0;
for (unsigned x = 0; x < ww; x++)
for (unsigned y = 0; y < wh; y++)
for (unsigned ic = 0; ic < ic_real; ic++) {
unsigned idx = ((oc * ww + x) * wh + y) * ic_real + ic;
corr += ((int32_t)win[idx] - wzp) *
((int32_t)izp - 0x80);
}
bout[oc] = bin[oc] - corr;
}
}
/* Mesa feature tiling (rkt_ml_subgraph_invoke, input_channels>1 path). */
unsigned rkt_pack_input(const uint8_t *iin, unsigned iw, unsigned ih,
unsigned ic_real, uint8_t izp, uint8_t *out)
{
unsigned n = 0;
for (unsigned u = 0; u < DIV_ROUND_UP(ic_real, FEATURE_ATOMIC_SIZE); u++)
for (unsigned x = 0; x < iw; x++)
for (unsigned y = 0; y < ih; y++)
for (unsigned c = 0; c < FEATURE_ATOMIC_SIZE; c++) {
unsigned ich = c + u * FEATURE_ATOMIC_SIZE;
if (ich < ic_real)
out[n++] = (uint8_t)(iin[(x * ih + y) * ic_real + ich] - 0x80);
else
out[n++] = (uint8_t)(izp - 0x80);
}
return n;
}
/* Mesa output de-tile (rkt_ml_subgraph_read_outputs). */
void rkt_unpack_output(const uint8_t *raw, unsigned ow, unsigned oh,
unsigned oc_real, uint8_t *out)
{
for (unsigned oc = 0; oc < oc_real; oc++)
for (unsigned x = 0; x < ow; x++)
for (unsigned y = 0; y < oh; y++) {
unsigned c = oc % FEATURE_ATOMIC_SIZE;
unsigned g = oc / FEATURE_ATOMIC_SIZE;
out[(y * ow + x) * oc_real + oc] =
(uint8_t)(raw[((g * oh + y) * ow + x) * FEATURE_ATOMIC_SIZE + c] + 0x80);
}
}
+30
View File
@@ -0,0 +1,30 @@
/* SPDX-License-Identifier: MIT
*
* rkt_operands.h — NPU operand layout ports (see rkt_operands.c).
* Non-depthwise. Row-major host tensors:
* weights_in [oc_real][ww][wh][ic_real], input_in [iw][ih][ic_real],
* output_out [oh][ow][oc_real].
*/
#ifndef RKT_OPERANDS_H
#define RKT_OPERANDS_H
#include <stdint.h>
unsigned rkt_raw_input_size(unsigned iw, unsigned ih, unsigned ic_real);
unsigned rkt_raw_output_size(unsigned ow, unsigned oh, unsigned oc_real);
unsigned rkt_packed_weights_size(unsigned ww, unsigned wh, unsigned ic_real,
unsigned oc_real);
unsigned rkt_pack_weights(const uint8_t *weights_in, unsigned ww, unsigned wh,
unsigned ic_real, unsigned oc_real, uint8_t wzp,
uint8_t *out);
void rkt_compute_biases(const uint8_t *weights_in, const int32_t *biases_in,
unsigned ww, unsigned wh, unsigned ic_real,
unsigned oc_real, uint8_t wzp, uint8_t izp,
int32_t *biases_out);
unsigned rkt_pack_input(const uint8_t *input_in, unsigned iw, unsigned ih,
unsigned ic_real, uint8_t izp, uint8_t *out);
void rkt_unpack_output(const uint8_t *raw, unsigned ow, unsigned oh,
unsigned oc_real, uint8_t *out);
#endif /* RKT_OPERANDS_H */