marfrit/rosenblatt
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Rosenblatt: golden-byte verification — port matches Mesa exactly

Browse Source 
Adds userspace/npu-probe/verify/: a reference oracle built from Mesa's
UNMODIFIED rkt_task.c + rkt_regcmd.c (vendored, MIT) linked against a thin
shim (rkt_ml.h), which emits the ground-truth regcmd buffer for our test
matmul. 'make diff' byte-compares it against rkt_build_matmul_regcmd.

First run caught three real bugs, now fixed:
  1. REG_DPU_RDMA_RDMA_S_POINTER emitted at target 0x1001, must be 0x2001
  2. an 18-word CNA register run (CONV_CON2..FC_CON1) emitted twice
  3. the whole DPU_RDMA block at 0x1001 instead of 0x2001 (EMIT_RDMA added)
Values were correct throughout; only block-targets and a copy-paste were
wrong. CNA stage rewritten to Mesa's single-pass order; coredpu RDMA regs
repointed. selftest histogram now counts the RDMA (0x2001) block.

Result: 130 words, IDENTICAL — GOLDEN MATCH. The builder is now proven, not
just faithful.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
claude-noether
2026-07-13 20:56:16 +00:00
parent b39d23153d
commit d783dd1b28
14 changed files with 6140 additions and 164 deletions
Download Patch File Download Diff File Expand all files Collapse all files
userspace/npu-probe/.gitignore
+5
View File
@@ -1 +1,6 @@
*.o
selftest
verify/oracle
verify/ourdump
verify/*.txt
verify/golden.diff
userspace/npu-probe/rkt_matmul_cna.c
+72 -138
View File
@@ -4,7 +4,10 @@
/*
* Packed register command: (target << 48) | (value << 16) | reg
* target = block_enum + 1. CNA block = 0x200 -> target 0x201; DPU = 0x1000 -> 0x1001.
* target = block_enum + 1. CNA=0x200->0x201, DPU=0x1000->0x1001,
* DPU_RDMA=0x2000->0x2001. The target is a property of the register's block
* (Mesa derives it via rkt_get_target(reg)+1), NOT of the emit site — so the
* DPU_RDMA registers that appear inside this CNA sequence still carry 0x2001.
*/
static int rkt_push(uint64_t *out, int *n, int cap, uint32_t target,
uint32_t reg, uint32_t value)
@@ -16,8 +19,9 @@ static int rkt_push(uint64_t *out, int *n, int cap, uint32_t target,
return 0;
}
#define EMIT_CNA(reg, val) do { if (rkt_push(out, &n, cap, 0x201, (reg), (val)) < 0) return -1; } while (0)
#define EMIT_DPU(reg, val) do { if (rkt_push(out, &n, cap, 0x1001, (reg), (val)) < 0) return -1; } while (0)
#define EMIT_CNA(reg, val) do { if (rkt_push(out, &n, cap, 0x201, (reg), (val)) < 0) return -1; } while (0)
#define EMIT_DPU(reg, val) do { if (rkt_push(out, &n, cap, 0x1001, (reg), (val)) < 0) return -1; } while (0)
#define EMIT_RDMA(reg, val) do { if (rkt_push(out, &n, cap, 0x2001, (reg), (val)) < 0) return -1; } while (0)
/* Field-set: (value << FIELD__SHIFT) & FIELD__MASK */
#define F(v, name) (((uint32_t)(v) << name##__SHIFT) & name##__MASK)
@@ -26,171 +30,104 @@ int rkt_emit_cna(uint64_t *out, int cap, const struct cna_params *p)
{
int n = 0;
/* ==== CNA emit sequence (Montessori, chunk by chunk) ==== */
uint32_t con0 = F(p->weights_banks, CNA_CBUF_CON0_WEIGHT_BANK) |
F(p->input_banks, CNA_CBUF_CON0_DATA_BANK);
F(p->input_banks, CNA_CBUF_CON0_DATA_BANK);
if (p->task_num > 0 && p->reuse_weights_cbuf)
con0 |= F(1, CNA_CBUF_CON0_WEIGHT_REUSE);
con0 |= F(1, CNA_CBUF_CON0_WEIGHT_REUSE);
EMIT_CNA(REG_CNA_CBUF_CON0, con0);
EMIT_CNA(REG_CNA_DCOMP_REGNUM, 0);
EMIT_CNA(REG_CNA_DCOMP_CTRL, 0);
uint32_t con1 = 0x0;
if (p->input_channels_real == 1) {
con1 |= F(1, CNA_CONV_CON1_NONALIGN_DMA) | F(1, CNA_CONV_CON1_GROUP_LINE_OFF) |
F(8, CNA_CONV_CON1_ARGB_IN);
con1 |= F(1, CNA_CONV_CON1_NONALIGN_DMA) | F(1, CNA_CONV_CON1_GROUP_LINE_OFF) |
F(8, CNA_CONV_CON1_ARGB_IN);
}
if (p->depthwise)
con1 |= F(3, CNA_CONV_CON1_CONV_MODE);
con1 |= F(3, CNA_CONV_CON1_CONV_MODE);
EMIT_CNA(REG_CNA_CONV_CON1, con1);
EMIT_DPU(REG_DPU_S_POINTER, F(1, DPU_S_POINTER_POINTER_PP_MODE) |
F(1, DPU_S_POINTER_EXECUTER_PP_EN) |
F(1, DPU_S_POINTER_POINTER_PP_EN));
EMIT_DPU(REG_DPU_RDMA_RDMA_S_POINTER,
F(1, DPU_RDMA_RDMA_S_POINTER_POINTER_PP_MODE) |
F(1, DPU_RDMA_RDMA_S_POINTER_EXECUTER_PP_EN) |
F(1, DPU_RDMA_RDMA_S_POINTER_POINTER_PP_EN));
F(1, DPU_S_POINTER_EXECUTER_PP_EN) |
F(1, DPU_S_POINTER_POINTER_PP_EN));
EMIT_RDMA(REG_DPU_RDMA_RDMA_S_POINTER,
F(1, DPU_RDMA_RDMA_S_POINTER_POINTER_PP_MODE) |
F(1, DPU_RDMA_RDMA_S_POINTER_EXECUTER_PP_EN) |
F(1, DPU_RDMA_RDMA_S_POINTER_POINTER_PP_EN));
EMIT_CNA(REG_CNA_CONV_CON1, con1);
EMIT_CNA(REG_CNA_CONV_CON2,
F(50 + p->stride_y + 1, CNA_CONV_CON2_FEATURE_GRAINS)); /* Magic: Seems to pass the most tests */
F(50 + p->stride_y + 1, CNA_CONV_CON2_FEATURE_GRAINS)); /* Magic: passes the most tests */
EMIT_CNA(REG_CNA_CONV_CON3, F(p->stride_x, CNA_CONV_CON3_CONV_X_STRIDE) |
F(p->stride_y, CNA_CONV_CON3_CONV_Y_STRIDE));
F(p->stride_y, CNA_CONV_CON3_CONV_Y_STRIDE));
EMIT_CNA(REG_CNA_DATA_SIZE0,
F(p->input_width, CNA_DATA_SIZE0_DATAIN_WIDTH) |
F(p->input_height, CNA_DATA_SIZE0_DATAIN_HEIGHT));
F(p->input_width, CNA_DATA_SIZE0_DATAIN_WIDTH) |
F(p->input_height, CNA_DATA_SIZE0_DATAIN_HEIGHT));
EMIT_CNA(REG_CNA_DATA_SIZE1,
F(p->input_channels_real - 1, CNA_DATA_SIZE1_DATAIN_CHANNEL_REAL) |
F(p->input_channels, CNA_DATA_SIZE1_DATAIN_CHANNEL));
F(p->input_channels_real - 1, CNA_DATA_SIZE1_DATAIN_CHANNEL_REAL) |
F(p->input_channels, CNA_DATA_SIZE1_DATAIN_CHANNEL));
EMIT_CNA(REG_CNA_DATA_SIZE2, F(p->output_width, CNA_DATA_SIZE2_DATAOUT_WIDTH));
EMIT_CNA(REG_CNA_DATA_SIZE3, F(p->atomic_count, CNA_DATA_SIZE3_DATAOUT_ATOMICS));
EMIT_CNA(REG_CNA_WEIGHT_SIZE0, p->weights_width * p->weights_height *
p->input_channels * p->weights_kernels);
p->input_channels * p->weights_kernels);
EMIT_CNA(REG_CNA_WEIGHT_SIZE1,
p->weights_width * p->weights_height * p->input_channels);
p->weights_width * p->weights_height * p->input_channels);
EMIT_CNA(REG_CNA_WEIGHT_SIZE2,
F(p->weights_width, CNA_WEIGHT_SIZE2_WEIGHT_WIDTH) |
F(p->weights_height, CNA_WEIGHT_SIZE2_WEIGHT_HEIGHT) |
F(p->weights_kernels, CNA_WEIGHT_SIZE2_WEIGHT_KERNELS));
F(p->weights_width, CNA_WEIGHT_SIZE2_WEIGHT_WIDTH) |
F(p->weights_height, CNA_WEIGHT_SIZE2_WEIGHT_HEIGHT) |
F(p->weights_kernels, CNA_WEIGHT_SIZE2_WEIGHT_KERNELS));
EMIT_CNA(REG_CNA_CBUF_CON0, con0);
EMIT_CNA(REG_CNA_CBUF_CON1, F(p->input_data_entries, CNA_CBUF_CON1_DATA_ENTRIES));
if (p->input_channels_real == 1) {
unsigned truncate = 14;
unsigned scale = 16384;
unsigned offset = 65408;
unsigned truncate = 14;
unsigned scale = 16384;
unsigned offset = 65408;
if (p->addition_input || p->add_tensor != -1) {
truncate = 15;
scale = 32388;
}
if (p->addition_input || p->add_tensor != -1) {
truncate = 15;
scale = 32388;
}
EMIT_CNA(REG_CNA_CVT_CON0, F(truncate, CNA_CVT_CON0_CVT_TRUNCATE_3) |
F(truncate, CNA_CVT_CON0_CVT_TRUNCATE_2) |
F(truncate, CNA_CVT_CON0_CVT_TRUNCATE_1) |
F(truncate, CNA_CVT_CON0_CVT_TRUNCATE_0));
EMIT_CNA(REG_CNA_CVT_CON1,
F(scale, CNA_CVT_CON1_CVT_SCALE0) | F(offset, CNA_CVT_CON1_CVT_OFFSET0));
EMIT_CNA(REG_CNA_CVT_CON2,
F(scale, CNA_CVT_CON2_CVT_SCALE1) | F(offset, CNA_CVT_CON2_CVT_OFFSET1));
EMIT_CNA(REG_CNA_CVT_CON3,
F(scale, CNA_CVT_CON3_CVT_SCALE2) | F(offset, CNA_CVT_CON3_CVT_OFFSET2));
EMIT_CNA(REG_CNA_CVT_CON4,
F(scale, CNA_CVT_CON4_CVT_SCALE3) | F(offset, CNA_CVT_CON4_CVT_OFFSET3));
EMIT_CNA(REG_CNA_CVT_CON0, F(truncate, CNA_CVT_CON0_CVT_TRUNCATE_3) |
F(truncate, CNA_CVT_CON0_CVT_TRUNCATE_2) |
F(truncate, CNA_CVT_CON0_CVT_TRUNCATE_1) |
F(truncate, CNA_CVT_CON0_CVT_TRUNCATE_0));
EMIT_CNA(REG_CNA_CVT_CON1,
F(scale, CNA_CVT_CON1_CVT_SCALE0) | F(offset, CNA_CVT_CON1_CVT_OFFSET0));
EMIT_CNA(REG_CNA_CVT_CON2,
F(scale, CNA_CVT_CON2_CVT_SCALE1) | F(offset, CNA_CVT_CON2_CVT_OFFSET1));
EMIT_CNA(REG_CNA_CVT_CON3,
F(scale, CNA_CVT_CON3_CVT_SCALE2) | F(offset, CNA_CVT_CON3_CVT_OFFSET2));
EMIT_CNA(REG_CNA_CVT_CON4,
F(scale, CNA_CVT_CON4_CVT_SCALE3) | F(offset, CNA_CVT_CON4_CVT_OFFSET3));
} else {
EMIT_CNA(REG_CNA_CVT_CON0, F(1, CNA_CVT_CON0_DATA_SIGN) |
F(1, CNA_CVT_CON0_CVT_TYPE) |
F(1, CNA_CVT_CON0_CVT_BYPASS));
EMIT_CNA(REG_CNA_CVT_CON1, F(1, CNA_CVT_CON1_CVT_SCALE0));
EMIT_CNA(REG_CNA_CVT_CON2, F(1, CNA_CVT_CON2_CVT_SCALE1));
EMIT_CNA(REG_CNA_CVT_CON3, F(1, CNA_CVT_CON3_CVT_SCALE2));
EMIT_CNA(REG_CNA_CVT_CON4, F(1, CNA_CVT_CON4_CVT_SCALE3));
EMIT_CNA(REG_CNA_CVT_CON0, F(1, CNA_CVT_CON0_DATA_SIGN) |
F(1, CNA_CVT_CON0_CVT_TYPE) |
F(1, CNA_CVT_CON0_CVT_BYPASS));
EMIT_CNA(REG_CNA_CVT_CON1, F(1, CNA_CVT_CON1_CVT_SCALE0));
EMIT_CNA(REG_CNA_CVT_CON2, F(1, CNA_CVT_CON2_CVT_SCALE1));
EMIT_CNA(REG_CNA_CVT_CON3, F(1, CNA_CVT_CON3_CVT_SCALE2));
EMIT_CNA(REG_CNA_CVT_CON4, F(1, CNA_CVT_CON4_CVT_SCALE3));
}
EMIT_CNA(REG_CNA_FC_CON0, 0);
EMIT_CNA(REG_CNA_FC_CON1, 0);
EMIT_CNA(REG_CNA_CONV_CON2,
F(50 + p->stride_y + 1, CNA_CONV_CON2_FEATURE_GRAINS)); /* Magic: Seems to pass the most tests */
EMIT_CNA(REG_CNA_CONV_CON3, F(p->stride_x, CNA_CONV_CON3_CONV_X_STRIDE) |
F(p->stride_y, CNA_CONV_CON3_CONV_Y_STRIDE));
EMIT_CNA(REG_CNA_DATA_SIZE0,
F(p->input_width, CNA_DATA_SIZE0_DATAIN_WIDTH) |
F(p->input_height, CNA_DATA_SIZE0_DATAIN_HEIGHT));
EMIT_CNA(REG_CNA_DATA_SIZE1,
F(p->input_channels_real - 1, CNA_DATA_SIZE1_DATAIN_CHANNEL_REAL) |
F(p->input_channels, CNA_DATA_SIZE1_DATAIN_CHANNEL));
EMIT_CNA(REG_CNA_DATA_SIZE2, F(p->output_width, CNA_DATA_SIZE2_DATAOUT_WIDTH));
EMIT_CNA(REG_CNA_DATA_SIZE3, F(p->atomic_count, CNA_DATA_SIZE3_DATAOUT_ATOMICS));
EMIT_CNA(REG_CNA_WEIGHT_SIZE0, p->weights_width * p->weights_height *
p->input_channels * p->weights_kernels);
EMIT_CNA(REG_CNA_WEIGHT_SIZE1,
p->weights_width * p->weights_height * p->input_channels);
EMIT_CNA(REG_CNA_WEIGHT_SIZE2,
F(p->weights_width, CNA_WEIGHT_SIZE2_WEIGHT_WIDTH) |
F(p->weights_height, CNA_WEIGHT_SIZE2_WEIGHT_HEIGHT) |
F(p->weights_kernels, CNA_WEIGHT_SIZE2_WEIGHT_KERNELS));
EMIT_CNA(REG_CNA_CBUF_CON0, con0);
EMIT_CNA(REG_CNA_CBUF_CON1, F(p->input_data_entries, CNA_CBUF_CON1_DATA_ENTRIES));
if (p->input_channels_real == 1) {
unsigned truncate = 14;
unsigned scale = 16384;
unsigned offset = 65408;
if (p->addition_input || p->add_tensor != -1) {
truncate = 15;
scale = 32388;
}
EMIT_CNA(REG_CNA_CVT_CON0, F(truncate, CNA_CVT_CON0_CVT_TRUNCATE_3) |
F(truncate, CNA_CVT_CON0_CVT_TRUNCATE_2) |
F(truncate, CNA_CVT_CON0_CVT_TRUNCATE_1) |
F(truncate, CNA_CVT_CON0_CVT_TRUNCATE_0));
EMIT_CNA(REG_CNA_CVT_CON1,
F(scale, CNA_CVT_CON1_CVT_SCALE0) | F(offset, CNA_CVT_CON1_CVT_OFFSET0));
EMIT_CNA(REG_CNA_CVT_CON2,
F(scale, CNA_CVT_CON2_CVT_SCALE1) | F(offset, CNA_CVT_CON2_CVT_OFFSET1));
EMIT_CNA(REG_CNA_CVT_CON3,
F(scale, CNA_CVT_CON3_CVT_SCALE2) | F(offset, CNA_CVT_CON3_CVT_OFFSET2));
EMIT_CNA(REG_CNA_CVT_CON4,
F(scale, CNA_CVT_CON4_CVT_SCALE3) | F(offset, CNA_CVT_CON4_CVT_OFFSET3));
} else {
EMIT_CNA(REG_CNA_CVT_CON0, F(1, CNA_CVT_CON0_DATA_SIGN) |
F(1, CNA_CVT_CON0_CVT_TYPE) |
F(1, CNA_CVT_CON0_CVT_BYPASS));
EMIT_CNA(REG_CNA_CVT_CON1, F(1, CNA_CVT_CON1_CVT_SCALE0));
EMIT_CNA(REG_CNA_CVT_CON2, F(1, CNA_CVT_CON2_CVT_SCALE1));
EMIT_CNA(REG_CNA_CVT_CON3, F(1, CNA_CVT_CON3_CVT_SCALE2));
EMIT_CNA(REG_CNA_CVT_CON4, F(1, CNA_CVT_CON4_CVT_SCALE3));
}
EMIT_CNA(REG_CNA_FC_CON0, 0);
EMIT_CNA(REG_CNA_FC_CON1, 0);
EMIT_CNA(REG_CNA_PAD_CON0, F(p->pad_left, CNA_PAD_CON0_PAD_LEFT) |
F(p->pad_top, CNA_PAD_CON0_PAD_TOP));
EMIT_CNA(REG_CNA_FEATURE_DATA_ADDR,
(uint32_t)p->input_dma);
F(p->pad_top, CNA_PAD_CON0_PAD_TOP));
EMIT_CNA(REG_CNA_FEATURE_DATA_ADDR, (uint32_t)p->input_dma);
EMIT_CNA(REG_CNA_FC_CON2, 0);
EMIT_CNA(REG_CNA_DMA_CON0,
F(15, CNA_DMA_CON0_WEIGHT_BURST_LEN) | F(15, CNA_DMA_CON0_DATA_BURST_LEN));
F(15, CNA_DMA_CON0_WEIGHT_BURST_LEN) | F(15, CNA_DMA_CON0_DATA_BURST_LEN));
EMIT_CNA(REG_CNA_DMA_CON1, F(p->input_line_stride, CNA_DMA_CON1_LINE_STRIDE));
EMIT_CNA(REG_CNA_DMA_CON2, F(p->input_surface_stride, CNA_DMA_CON2_SURF_STRIDE));
EMIT_CNA(REG_CNA_FC_DATA_SIZE0,
F(p->input_width, CNA_FC_DATA_SIZE0_DMA_WIDTH) |
F(p->input_height, CNA_FC_DATA_SIZE0_DMA_HEIGHT));
F(p->input_width, CNA_FC_DATA_SIZE0_DMA_WIDTH) |
F(p->input_height, CNA_FC_DATA_SIZE0_DMA_HEIGHT));
EMIT_CNA(REG_CNA_FC_DATA_SIZE1,
F(p->input_channels, CNA_FC_DATA_SIZE1_DMA_CHANNEL));
F(p->input_channels, CNA_FC_DATA_SIZE1_DMA_CHANNEL));
EMIT_CNA(REG_CNA_DCOMP_CTRL, 0);
EMIT_CNA(REG_CNA_DCOMP_REGNUM, 0);
EMIT_CNA(REG_CNA_DCOMP_ADDR0, (uint32_t)p->weights_dma);
@@ -211,27 +148,24 @@ int rkt_emit_cna(uint64_t *out, int cap, const struct cna_params *p)
EMIT_CNA(REG_CNA_DCOMP_AMOUNT14, 0);
EMIT_CNA(REG_CNA_DCOMP_AMOUNT15, 0);
if (p->input_channels_real == 1) {
EMIT_CNA(REG_CNA_CVT_CON5, 65535);
} else {
EMIT_CNA(REG_CNA_CVT_CON5, 0);
}
if (p->input_channels_real == 1)
EMIT_CNA(REG_CNA_CVT_CON5, 65535);
else
EMIT_CNA(REG_CNA_CVT_CON5, 0);
int32_t pad_con1;
if (p->weights_width >= 3 && p->input_zero_point == 0x0)
pad_con1 = 0xffff8080;
pad_con1 = 0xffff8080;
else
pad_con1 = p->input_zero_point - 0x80;
pad_con1 = p->input_zero_point - 0x80;
if (p->addition_input || p->add_tensor != -1)
pad_con1 = 0xffffff80;
pad_con1 = 0xffffff80;
if (p->depthwise && p->input_zero_point == 0x8b)
pad_con1 = 0x0b0b;
pad_con1 = 0x0b0b;
EMIT_CNA(REG_CNA_PAD_CON1, pad_con1);
/* ==== end CNA stage ==== */
return n;
}
userspace/npu-probe/rkt_matmul_coredpu.c
+24 -23
View File
@@ -16,6 +16,7 @@ static int rkt_push(uint64_t *out, int *n, int cap, uint32_t target,
#define EMIT_CORE(reg, val) do { if (rkt_push(out, &n, cap, 0x801, (reg), (val)) < 0) return -1; } while (0)
#define EMIT_DPU(reg, val) do { if (rkt_push(out, &n, cap, 0x1001, (reg), (val)) < 0) return -1; } while (0)
#define EMIT_RDMA(reg, val) do { if (rkt_push(out, &n, cap, 0x2001, (reg), (val)) < 0) return -1; } while (0)
#define EMIT_PC(reg, val) do { if (rkt_push(out, &n, cap, 0x101, (reg), (val)) < 0) return -1; } while (0)
#define EMIT_RAW(tgt, reg, val) do { if (rkt_push(out, &n, cap, (tgt), (reg), (val)) < 0) return -1; } while (0)
@@ -194,45 +195,45 @@ int rkt_emit_coredpu(uint64_t *out, int cap, const struct coredpu_params *p)
EMIT_DPU(REG_DPU_LUT_LE_SLOPE_SHIFT, 0);
EMIT_DPU(REG_DPU_LUT_LO_SLOPE_SCALE, 0);
EMIT_DPU(REG_DPU_LUT_LO_SLOPE_SHIFT, 0);
EMIT_DPU(REG_DPU_RDMA_RDMA_DATA_CUBE_WIDTH,
EMIT_RDMA(REG_DPU_RDMA_RDMA_DATA_CUBE_WIDTH,
F(p->output_width - 1, DPU_RDMA_RDMA_DATA_CUBE_WIDTH_WIDTH));
EMIT_DPU(REG_DPU_RDMA_RDMA_DATA_CUBE_HEIGHT,
EMIT_RDMA(REG_DPU_RDMA_RDMA_DATA_CUBE_HEIGHT,
F(p->output_height - 1, DPU_RDMA_RDMA_DATA_CUBE_HEIGHT_HEIGHT));
EMIT_DPU(REG_DPU_RDMA_RDMA_DATA_CUBE_CHANNEL,
EMIT_RDMA(REG_DPU_RDMA_RDMA_DATA_CUBE_CHANNEL,
F(p->output_channels - 1, DPU_RDMA_RDMA_DATA_CUBE_CHANNEL_CHANNEL));
if (p->add_tensor != -1) {
EMIT_DPU(REG_DPU_RDMA_RDMA_SRC_BASE_ADDR,
EMIT_RDMA(REG_DPU_RDMA_RDMA_SRC_BASE_ADDR,
p->add_tensor_addr +
p->output_offset);
} else {
EMIT_DPU(REG_DPU_RDMA_RDMA_SRC_BASE_ADDR, 0);
EMIT_RDMA(REG_DPU_RDMA_RDMA_SRC_BASE_ADDR, 0);
}
EMIT_DPU(REG_DPU_RDMA_RDMA_BRDMA_CFG, F(1, DPU_RDMA_RDMA_BRDMA_CFG_BRDMA_DATA_USE));
EMIT_DPU(REG_DPU_RDMA_RDMA_BS_BASE_ADDR,
EMIT_RDMA(REG_DPU_RDMA_RDMA_BRDMA_CFG, F(1, DPU_RDMA_RDMA_BRDMA_CFG_BRDMA_DATA_USE));
EMIT_RDMA(REG_DPU_RDMA_RDMA_BS_BASE_ADDR,
p->biases_addr);
EMIT_DPU(REG_DPU_RDMA_RDMA_NRDMA_CFG, 0);
EMIT_DPU(REG_DPU_RDMA_RDMA_BN_BASE_ADDR, 0);
EMIT_RDMA(REG_DPU_RDMA_RDMA_NRDMA_CFG, 0);
EMIT_RDMA(REG_DPU_RDMA_RDMA_BN_BASE_ADDR, 0);
unsigned ew_stride =
MAX2(p->output_width * p->output_height, 12);
if (p->add_tensor != -1) {
EMIT_DPU(REG_DPU_RDMA_RDMA_ERDMA_CFG,
EMIT_RDMA(REG_DPU_RDMA_RDMA_ERDMA_CFG,
F(1, DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_DATA_MODE) |
F(1, DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_DATA_SIZE));
unsigned ew_base_offset =
p->output_width * p->output_height * ATOMIC_K_SIZE;
EMIT_DPU(REG_DPU_RDMA_RDMA_EW_BASE_ADDR,
EMIT_RDMA(REG_DPU_RDMA_RDMA_EW_BASE_ADDR,
p->add_tensor_addr +
p->output_offset + ew_base_offset);
EMIT_DPU(REG_DPU_RDMA_RDMA_EW_SURF_STRIDE,
EMIT_RDMA(REG_DPU_RDMA_RDMA_EW_SURF_STRIDE,
F(ew_stride, DPU_RDMA_RDMA_EW_SURF_STRIDE_EW_SURF_STRIDE));
} else {
EMIT_DPU(REG_DPU_RDMA_RDMA_ERDMA_CFG, F(1, DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_DISABLE));
EMIT_DPU(REG_DPU_RDMA_RDMA_EW_BASE_ADDR, 0);
EMIT_DPU(REG_DPU_RDMA_RDMA_EW_SURF_STRIDE, 0);
EMIT_RDMA(REG_DPU_RDMA_RDMA_ERDMA_CFG, F(1, DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_DISABLE));
EMIT_RDMA(REG_DPU_RDMA_RDMA_EW_BASE_ADDR, 0);
EMIT_RDMA(REG_DPU_RDMA_RDMA_EW_SURF_STRIDE, 0);
}
uint32_t rdma_feat_mode_cfg = 0x0;
@@ -248,8 +249,8 @@ int rkt_emit_coredpu(uint64_t *out, int cap, const struct coredpu_params *p)
if (p->depthwise)
rdma_feat_mode_cfg |= F(3, DPU_RDMA_RDMA_FEATURE_MODE_CFG_CONV_MODE);
EMIT_DPU(REG_DPU_RDMA_RDMA_FEATURE_MODE_CFG, rdma_feat_mode_cfg);
EMIT_DPU(REG_DPU_RDMA_RDMA_SRC_DMA_CFG, 0);
EMIT_RDMA(REG_DPU_RDMA_RDMA_FEATURE_MODE_CFG, rdma_feat_mode_cfg);
EMIT_RDMA(REG_DPU_RDMA_RDMA_SRC_DMA_CFG, 0);
unsigned surf_notch =
ew_stride +
@@ -260,22 +261,22 @@ int rkt_emit_coredpu(uint64_t *out, int cap, const struct coredpu_params *p)
}
if (p->add_tensor != -1) {
EMIT_DPU(REG_DPU_RDMA_RDMA_SURF_NOTCH,
EMIT_RDMA(REG_DPU_RDMA_RDMA_SURF_NOTCH,
F(surf_notch, DPU_RDMA_RDMA_SURF_NOTCH_SURF_NOTCH_ADDR));
} else {
EMIT_DPU(REG_DPU_RDMA_RDMA_SURF_NOTCH, 0);
EMIT_RDMA(REG_DPU_RDMA_RDMA_SURF_NOTCH, 0);
}
EMIT_DPU(REG_DPU_RDMA_RDMA_PAD_CFG, 0);
EMIT_DPU(REG_DPU_RDMA_RDMA_WEIGHT,
EMIT_RDMA(REG_DPU_RDMA_RDMA_PAD_CFG, 0);
EMIT_RDMA(REG_DPU_RDMA_RDMA_WEIGHT,
F(1, DPU_RDMA_RDMA_WEIGHT_E_WEIGHT) | F(1, DPU_RDMA_RDMA_WEIGHT_N_WEIGHT) |
F(1, DPU_RDMA_RDMA_WEIGHT_B_WEIGHT) | F(1, DPU_RDMA_RDMA_WEIGHT_M_WEIGHT));
if (p->add_tensor != -1) {
EMIT_DPU(REG_DPU_RDMA_RDMA_EW_SURF_NOTCH,
EMIT_RDMA(REG_DPU_RDMA_RDMA_EW_SURF_NOTCH,
F(surf_notch, DPU_RDMA_RDMA_EW_SURF_NOTCH_EW_SURF_NOTCH));
} else {
EMIT_DPU(REG_DPU_RDMA_RDMA_EW_SURF_NOTCH, 0x0);
EMIT_RDMA(REG_DPU_RDMA_RDMA_EW_SURF_NOTCH, 0x0);
}
if (num_tasks == 1)
userspace/npu-probe/selftest.c
+4 -3
View File
@@ -25,7 +25,7 @@
* through to raw/other and are expected. Returns fail count. */
static int histogram(const char *tag, const uint64_t *buf, int n)
{
int pc = 0, cna = 0, core = 0, dpu = 0, raw = 0;
int pc = 0, cna = 0, core = 0, dpu = 0, rdma = 0, raw = 0;
for (int i = 0; i < n; i++) {
uint32_t tgt = (uint32_t)(buf[i] >> 48) & 0xffff;
switch (tgt) {
@@ -33,11 +33,12 @@ static int histogram(const char *tag, const uint64_t *buf, int n)
case 0x201: cna++; break;
case 0x801: core++; break;
case 0x1001: dpu++; break;
case 0x2001: rdma++; break;
default: raw++; break;
}
}
printf(" %s: %d words | PC=%d CNA=%d CORE=%d DPU=%d raw/other=%d\n",
tag, n, pc, cna, core, dpu, raw);
printf(" %s: %d words | PC=%d CNA=%d CORE=%d DPU=%d RDMA=%d raw/other=%d\n",
tag, n, pc, cna, core, dpu, rdma, raw);
if (cna == 0 || core == 0 || dpu == 0) {
printf(" FAIL: %s — a stage emitted zero commands\n", tag);
return 1;
userspace/npu-probe/verify/Makefile
+32
View File
@@ -0,0 +1,32 @@
# Golden-verification: Mesa oracle vs our builder.
# The oracle compiles VENDORED Mesa source (rkt_task.c / rkt_regcmd.c) against
# shims; warnings there are Mesa's, so build it with -w. ourdump uses our own
# objects from the parent dir.
CC ?= gcc
OFLAGS ?= -O2 -w
CFLAGS ?= -Wall -Wextra -O2
PARENT_OBJS = ../rkt_matmul.o ../rkt_matmul_cna.o ../rkt_matmul_coredpu.o
.PHONY: all diff clean
all: oracle ourdump
oracle: oracle.c rkt_task.c rkt_regcmd.c rkt_ml.h rkt_task.h rkt_regcmd.h rkt_registers.h
$(CC) $(OFLAGS) -o $@ oracle.c rkt_task.c rkt_regcmd.c
ourdump: ourdump.c
$(MAKE) -C .. selftest >/dev/null
$(CC) $(CFLAGS) -I.. -o $@ ourdump.c $(PARENT_OBJS)
diff: oracle ourdump
@./oracle > oracle.txt
@./ourdump > ours.txt
@echo "oracle=$$(wc -l < oracle.txt) words ours=$$(wc -l < ours.txt) words"
@if diff -u oracle.txt ours.txt > golden.diff; then \
echo "IDENTICAL — GOLDEN MATCH"; \
else \
echo "DIVERGENCE (see golden.diff):"; cat golden.diff; \
fi
clean:
rm -f oracle ourdump oracle.txt ours.txt golden.diff
userspace/npu-probe/verify/README.md
+37
View File
@@ -0,0 +1,37 @@
# Golden-byte verification
Proves our `rkt_build_matmul_regcmd()` emits the **byte-identical** regcmd
buffer that upstream Mesa's `rocket` driver would, for the same op.
## How it works
- `rkt_task.c`, `rkt_regcmd.c`, `rkt_registers.h`, `rkt_task.h`, `rkt_regcmd.h`
are **vendored unmodified** from the Mesa `rocket` Gallium driver
(`src/gallium/drivers/rocket/`, © Tomeu Vizoso, **MIT**). Do not edit them —
they are the ground truth.
- `rkt_ml.h` is a **shim** (ours) that supplies just enough of Mesa's util +
struct surface (`util_dynarray`, `struct split_task` / `rkt_operation` copied
verbatim, `align`/`fui`/CBUF geometry) for the vendored code to compile and
run standalone — no full Mesa build needed.
- `oracle.c` builds the same matmul op our builder targets (Y[4][8] =
X[4][16]·W[16][8] as a 1×1 conv), runs Mesa's real `rkt_split_tasks` +
`rkt_fill_regcmd`, and dumps the buffer.
- `ourdump.c` dumps our builder's buffer for the same op.
## Run
```
make diff
```
`IDENTICAL — GOLDEN MATCH` means our port reproduces Mesa exactly. Any
divergence is printed as a unified diff of packed `(target<<48)|(value<<16)|reg`
words (see `golden.diff`).
## History
The first run caught three real bugs in our port: a mis-targeted
`REG_DPU_RDMA_RDMA_S_POINTER`, an 18-word duplicated CNA register run, and the
entire DPU_RDMA block emitted at target `0x1001` instead of `0x2001`. All
fixed; the values had been correct all along — only block-targets and one
copy-paste were wrong.
userspace/npu-probe/verify/oracle.c
+83
View File
@@ -0,0 +1,83 @@
/* SPDX-License-Identifier: MIT
*
* oracle.c — golden-reference harness.
*
* Drives Mesa's UNMODIFIED rkt_split_tasks + rkt_fill_regcmd for the same
* matmul op our builder targets (Y[4][8] = X[4][16] * W[16][8], as a 1x1
* conv), then dumps the resulting regcmd buffer as one hex uint64 per line.
* Diff this against ourdump to golden-verify rkt_build_matmul_regcmd.
*
* Addresses are chosen to match ourdump: input=0x1000, weights=0x2000,
* output=0x3000, biases=0 (our builder assumes no bias), zero points 128.
*/
#include <stdio.h>
#include "rkt_ml.h"
#include "rkt_task.h"
#include "rkt_regcmd.h"
/* The subgraph just holds tensor resources indexed by input/output index. */
struct rkt_ml_subgraph {
struct rkt_resource tensors[8];
};
struct rkt_resource *rkt_get_tensor(struct rkt_ml_subgraph *sg, unsigned idx)
{
return &sg->tensors[idx];
}
int main(void)
{
const unsigned M = 4, N = 8, K = 16;
const unsigned IN = 0, OUT = 1;
struct rkt_ml_subgraph sg;
memset(&sg, 0, sizeof sg);
sg.tensors[IN].phys_addr = 0x1000;
sg.tensors[OUT].phys_addr = 0x3000;
struct rkt_resource weights_res = { .phys_addr = 0x2000 };
struct rkt_resource biases_res = { .phys_addr = 0x0 };
struct rkt_operation op;
memset(&op, 0, sizeof op);
op.depthwise = false;
op.reuse_weights_cbuf = false;
op.truncate_bits = 0;
op.padding_top = op.padding_bottom = op.padding_left = op.padding_right = 0;
op.stride = 1;
op.addition_input = false;
op.addition_offset = 0;
op.addition_scale = 1.0f;
op.input_index = IN;
op.input_width = 1;
op.input_height = M;
op.input_channels = K;
op.input_zero_point = 128;
op.input_scale = 1.0f;
op.output_index = OUT;
op.output_width = 1;
op.output_height = M;
op.output_channels = N;
op.output_zero_point = 128;
op.output_scale = 1.0f;
op.weights_width = 1;
op.weights_height = 1;
op.weights_zero_point = 128;
op.weights_scale = 1.0f;
op.add_tensor = -1;
op.weights = (struct pipe_resource *)&weights_res;
op.biases = (struct pipe_resource *)&biases_res;
rkt_split_tasks(&sg, &op);
struct util_dynarray regs;
memset(&regs, 0, sizeof regs);
rkt_fill_regcmd(&sg, &op, &regs, 0);
unsigned n = regs.size / sizeof(uint64_t);
const uint64_t *w = (const uint64_t *)regs.data;
for (unsigned i = 0; i < n; i++)
printf("%016llx\n", (unsigned long long)w[i]);
return 0;
}
userspace/npu-probe/verify/ourdump.c
+23
View File
@@ -0,0 +1,23 @@
/* SPDX-License-Identifier: MIT
*
* ourdump.c — dump OUR builder's regcmd buffer in the oracle's hex format,
* for the same op, so `make diff` can byte-compare the two.
*/
#include <stdint.h>
#include <stdio.h>
#include "rkt_matmul.h"
int main(void)
{
uint64_t buf[512];
int n = rkt_build_matmul_regcmd(buf, 512, 4, 8, 16,
0x1000, 0x2000, 0x3000, 128, 128, 128);
if (n < 0) {
fprintf(stderr, "rkt_build_matmul_regcmd failed\n");
return 1;
}
for (int i = 0; i < n; i++)
printf("%016llx\n", (unsigned long long)buf[i]);
return 0;
}
userspace/npu-probe/verify/rkt_ml.h
+190
View File
@@ -0,0 +1,190 @@
/* SPDX-License-Identifier: MIT
*
* rkt_ml.h — SHIM for the golden-verification oracle only.
*
* Provides just enough of Mesa's util + rkt_ml surface for the UNMODIFIED
* vendored rkt_task.c and rkt_regcmd.c to compile and run standalone, so we
* can capture the ground-truth regcmd buffer and byte-diff it against our
* own builder. The struct layouts are copied verbatim from Mesa's rkt_ml.h.
* This is NOT used by the real npu-probe build.
*/
#ifndef RKT_ML_H
#define RKT_ML_H
#include <stdint.h>
#include <stdbool.h>
#include <stdlib.h>
#include <string.h>
#ifndef DIV_ROUND_UP
#define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
#endif
#ifndef MIN2
#define MIN2(a, b) ((a) < (b) ? (a) : (b))
#endif
#ifndef MAX2
#define MAX2(a, b) ((a) > (b) ? (a) : (b))
#endif
static inline unsigned _rkt_align(unsigned v, unsigned a)
{
return ((v + a - 1) / a) * a;
}
#ifndef align
#define align(v, a) _rkt_align((unsigned)(v), (unsigned)(a))
#endif
static inline uint32_t fui(float f)
{
uint32_t u;
memcpy(&u, &f, sizeof u);
return u;
}
/* CBUF geometry — verbatim from Mesa rkt_ml.h */
#define CBUF_BANK_SIZE 32768
#define CBUF_BANKS 12
#define CBUF_ENTRIES_PER_BANK 256
#define CBUF_ENTRY_SIZE (CBUF_BANK_SIZE / CBUF_ENTRIES_PER_BANK)
#define FEATURE_ATOMIC_SIZE 16
#define WEIGHT_ATOMIC_SIZE 32
#define ATOMIC_K_SIZE 16
/* ---- minimal util_dynarray (byte-buffer append) ---- */
struct util_dynarray {
void *data;
unsigned size;
unsigned capacity;
};
static inline void _dyn_grow(struct util_dynarray *a, unsigned need)
{
if (a->size + need <= a->capacity)
return;
unsigned nc = a->capacity ? a->capacity * 2 : 4096;
while (nc < a->size + need)
nc *= 2;
a->data = realloc(a->data, nc);
a->capacity = nc;
}
static inline void _dyn_push(struct util_dynarray *a, const void *v, unsigned sz)
{
_dyn_grow(a, sz);
memcpy((char *)a->data + a->size, v, sz);
a->size += sz;
}
static inline void *_dyn_pop(struct util_dynarray *a, unsigned sz)
{
a->size -= sz;
return (char *)a->data + a->size;
}
#define util_dynarray_append(a, v) \
do { \
__typeof__(v) _t = (v); \
_dyn_push((a), &_t, sizeof(_t)); \
} while (0)
#define util_dynarray_append_typed(a, type, v) \
do { \
type _t = (type)(v); \
_dyn_push((a), &_t, sizeof(type)); \
} while (0)
#define util_dynarray_element(a, type, i) (&((type *)(a)->data)[i])
#define util_dynarray_num_elements(a, type) ((a)->size / sizeof(type))
#define util_dynarray_pop_ptr(a, type) ((type *)_dyn_pop((a), sizeof(type)))
/* ---- resource / tensor accessors ---- */
struct pipe_resource; /* opaque */
struct rkt_resource {
uint64_t phys_addr;
};
static inline struct rkt_resource *rkt_resource(struct pipe_resource *p)
{
return (struct rkt_resource *)p;
}
/* ---- struct split_task / rkt_operation — verbatim from Mesa rkt_ml.h ---- */
struct split_task {
unsigned num;
unsigned top_slice;
unsigned bottom_slice;
unsigned num_overlap_slices;
unsigned num_retain_slices;
unsigned convolutions;
unsigned pad_top;
unsigned pad_bottom;
unsigned pad_left;
unsigned pad_right;
unsigned stride_x;
unsigned stride_y;
unsigned input_width;
unsigned input_height;
unsigned input_channels;
unsigned input_channels_real;
unsigned input_zero_point;
float input_scale;
unsigned input_data_entries;
int input_line_stride;
int input_surface_stride;
unsigned input_offset;
unsigned output_width;
unsigned output_height;
unsigned output_channels;
unsigned output_channels_real;
unsigned output_zero_point;
float output_scale;
int output_surface_stride;
unsigned output_offset;
unsigned weights_width;
unsigned weights_height;
unsigned weights_kernels;
unsigned weights_zero_point;
float weights_scale;
unsigned input_banks;
unsigned weights_banks;
unsigned atomic_count;
unsigned surfaces_per_row;
unsigned regcfg_amount;
uint32_t regcfg_addr;
};
struct rkt_operation {
struct pipe_resource *regcmd;
struct pipe_resource *weights;
struct pipe_resource *biases;
bool depthwise;
bool reuse_weights_cbuf;
unsigned truncate_bits;
unsigned padding_top;
unsigned padding_bottom;
unsigned padding_left;
unsigned padding_right;
unsigned stride;
bool addition_input;
int addition_offset;
float addition_scale;
unsigned input_index;
unsigned input_width;
unsigned input_height;
unsigned input_channels;
uint8_t input_zero_point;
float input_scale;
unsigned output_index;
unsigned output_width;
unsigned output_height;
unsigned output_channels;
uint8_t output_zero_point;
float output_scale;
unsigned weights_width;
unsigned weights_height;
uint8_t weights_zero_point;
float weights_scale;
int add_tensor;
struct util_dynarray tasks;
};
/* opaque to the vendored code; the oracle harness defines the real one */
struct rkt_ml_subgraph;
struct rkt_resource *rkt_get_tensor(struct rkt_ml_subgraph *subgraph,
unsigned idx);
#endif /* RKT_ML_H */
userspace/npu-probe/verify/rkt_regcmd.c
+462
View File
@@ -0,0 +1,462 @@
/*
* Copyright (c) 2024 Tomeu Vizoso <tomeu@tomeuvizoso.net>
* SPDX-License-Identifier: MIT
*/
#include "rkt_regcmd.h"
#include "rkt_ml.h"
#include "rkt_registers.h"
static void
emit_raw(struct util_dynarray *regs, uint32_t target, uint32_t reg,
uint32_t value)
{
uint64_t packed_value = 0;
packed_value = ((uint64_t)target) << 48;
packed_value |= ((uint64_t)value) << 16;
packed_value |= (uint64_t)reg;
util_dynarray_append(regs, packed_value);
}
static void
emit(struct util_dynarray *regs, uint32_t reg, uint32_t value)
{
uint32_t target = rkt_get_target(reg) + 0x1;
emit_raw(regs, target, reg, value);
}
#define EMIT(offset, value) emit(regs, offset, value);
static void
fill_first_regcmd(struct rkt_ml_subgraph *subgraph,
const struct rkt_operation *operation,
struct util_dynarray *regs, unsigned task_num)
{
struct split_task *task =
util_dynarray_element(&operation->tasks, struct split_task, task_num);
unsigned num_tasks =
util_dynarray_num_elements(&operation->tasks, struct split_task);
unsigned output_zero_point = task->output_zero_point;
unsigned weights_zero_point = task->weights_zero_point;
unsigned offset = output_zero_point - 0x80;
uint32_t con0 = CNA_CBUF_CON0_WEIGHT_BANK(task->weights_banks) |
CNA_CBUF_CON0_DATA_BANK(task->input_banks);
if (task_num > 0 && operation->reuse_weights_cbuf)
con0 |= CNA_CBUF_CON0_WEIGHT_REUSE(1);
EMIT(REG_CNA_CBUF_CON0, con0);
EMIT(REG_CNA_DCOMP_REGNUM, 0);
EMIT(REG_CNA_DCOMP_CTRL, 0);
uint32_t con1 = 0x0;
if (task->input_channels_real == 1) {
con1 |= CNA_CONV_CON1_NONALIGN_DMA(1) | CNA_CONV_CON1_GROUP_LINE_OFF(1) |
CNA_CONV_CON1_ARGB_IN(8);
}
if (operation->depthwise)
con1 |= CNA_CONV_CON1_CONV_MODE(3);
EMIT(REG_CNA_CONV_CON1, con1);
EMIT(REG_DPU_S_POINTER, DPU_S_POINTER_POINTER_PP_MODE(1) |
DPU_S_POINTER_EXECUTER_PP_EN(1) |
DPU_S_POINTER_POINTER_PP_EN(1));
EMIT(REG_DPU_RDMA_RDMA_S_POINTER,
DPU_RDMA_RDMA_S_POINTER_POINTER_PP_MODE(1) |
DPU_RDMA_RDMA_S_POINTER_EXECUTER_PP_EN(1) |
DPU_RDMA_RDMA_S_POINTER_POINTER_PP_EN(1));
EMIT(REG_CNA_CONV_CON1, con1);
EMIT(REG_CNA_CONV_CON2,
CNA_CONV_CON2_FEATURE_GRAINS(
50 + task->stride_y + 1)); /* Magic: Seems to pass the most tests */
EMIT(REG_CNA_CONV_CON3, CNA_CONV_CON3_CONV_X_STRIDE(task->stride_x) |
CNA_CONV_CON3_CONV_Y_STRIDE(task->stride_y));
EMIT(REG_CNA_DATA_SIZE0,
CNA_DATA_SIZE0_DATAIN_WIDTH(task->input_width) |
CNA_DATA_SIZE0_DATAIN_HEIGHT(task->input_height));
EMIT(REG_CNA_DATA_SIZE1,
CNA_DATA_SIZE1_DATAIN_CHANNEL_REAL(task->input_channels_real - 1) |
CNA_DATA_SIZE1_DATAIN_CHANNEL(task->input_channels));
EMIT(REG_CNA_DATA_SIZE2, CNA_DATA_SIZE2_DATAOUT_WIDTH(task->output_width));
EMIT(REG_CNA_DATA_SIZE3, CNA_DATA_SIZE3_DATAOUT_ATOMICS(task->atomic_count));
EMIT(REG_CNA_WEIGHT_SIZE0, task->weights_width * task->weights_height *
task->input_channels * task->weights_kernels);
EMIT(REG_CNA_WEIGHT_SIZE1,
task->weights_width * task->weights_height * task->input_channels);
EMIT(REG_CNA_WEIGHT_SIZE2,
CNA_WEIGHT_SIZE2_WEIGHT_WIDTH(task->weights_width) |
CNA_WEIGHT_SIZE2_WEIGHT_HEIGHT(task->weights_height) |
CNA_WEIGHT_SIZE2_WEIGHT_KERNELS(task->weights_kernels));
EMIT(REG_CNA_CBUF_CON0, con0);
EMIT(REG_CNA_CBUF_CON1, CNA_CBUF_CON1_DATA_ENTRIES(task->input_data_entries));
if (task->input_channels_real == 1) {
unsigned truncate = 14;
unsigned scale = 16384;
unsigned offset = 65408;
if (operation->addition_input || operation->add_tensor != -1) {
truncate = 15;
scale = 32388;
}
EMIT(REG_CNA_CVT_CON0, CNA_CVT_CON0_CVT_TRUNCATE_3(truncate) |
CNA_CVT_CON0_CVT_TRUNCATE_2(truncate) |
CNA_CVT_CON0_CVT_TRUNCATE_1(truncate) |
CNA_CVT_CON0_CVT_TRUNCATE_0(truncate));
EMIT(REG_CNA_CVT_CON1,
CNA_CVT_CON1_CVT_SCALE0(scale) | CNA_CVT_CON1_CVT_OFFSET0(offset));
EMIT(REG_CNA_CVT_CON2,
CNA_CVT_CON2_CVT_SCALE1(scale) | CNA_CVT_CON2_CVT_OFFSET1(offset));
EMIT(REG_CNA_CVT_CON3,
CNA_CVT_CON3_CVT_SCALE2(scale) | CNA_CVT_CON3_CVT_OFFSET2(offset));
EMIT(REG_CNA_CVT_CON4,
CNA_CVT_CON4_CVT_SCALE3(scale) | CNA_CVT_CON4_CVT_OFFSET3(offset));
} else {
EMIT(REG_CNA_CVT_CON0, CNA_CVT_CON0_DATA_SIGN(1) |
CNA_CVT_CON0_CVT_TYPE(1) |
CNA_CVT_CON0_CVT_BYPASS(1));
EMIT(REG_CNA_CVT_CON1, CNA_CVT_CON1_CVT_SCALE0(1));
EMIT(REG_CNA_CVT_CON2, CNA_CVT_CON2_CVT_SCALE1(1));
EMIT(REG_CNA_CVT_CON3, CNA_CVT_CON3_CVT_SCALE2(1));
EMIT(REG_CNA_CVT_CON4, CNA_CVT_CON4_CVT_SCALE3(1));
}
EMIT(REG_CNA_FC_CON0, 0);
EMIT(REG_CNA_FC_CON1, 0);
EMIT(REG_CNA_PAD_CON0, CNA_PAD_CON0_PAD_LEFT(task->pad_left) |
CNA_PAD_CON0_PAD_TOP(task->pad_top));
EMIT(REG_CNA_FEATURE_DATA_ADDR,
rkt_get_tensor(subgraph, operation->input_index)->phys_addr +
task->input_offset);
EMIT(REG_CNA_FC_CON2, 0);
EMIT(REG_CNA_DMA_CON0,
CNA_DMA_CON0_WEIGHT_BURST_LEN(15) | CNA_DMA_CON0_DATA_BURST_LEN(15));
EMIT(REG_CNA_DMA_CON1, CNA_DMA_CON1_LINE_STRIDE(task->input_line_stride));
EMIT(REG_CNA_DMA_CON2, CNA_DMA_CON2_SURF_STRIDE(task->input_surface_stride));
EMIT(REG_CNA_FC_DATA_SIZE0,
CNA_FC_DATA_SIZE0_DMA_WIDTH(operation->input_width) |
CNA_FC_DATA_SIZE0_DMA_HEIGHT(task->input_height));
EMIT(REG_CNA_FC_DATA_SIZE1,
CNA_FC_DATA_SIZE1_DMA_CHANNEL(task->input_channels));
EMIT(REG_CNA_DCOMP_CTRL, 0);
EMIT(REG_CNA_DCOMP_REGNUM, 0);
EMIT(REG_CNA_DCOMP_ADDR0, rkt_resource(operation->weights)->phys_addr);
EMIT(REG_CNA_DCOMP_AMOUNT0, 0);
EMIT(REG_CNA_DCOMP_AMOUNT1, 0);
EMIT(REG_CNA_DCOMP_AMOUNT2, 0);
EMIT(REG_CNA_DCOMP_AMOUNT3, 0);
EMIT(REG_CNA_DCOMP_AMOUNT4, 0);
EMIT(REG_CNA_DCOMP_AMOUNT5, 0);
EMIT(REG_CNA_DCOMP_AMOUNT6, 0);
EMIT(REG_CNA_DCOMP_AMOUNT7, 0);
EMIT(REG_CNA_DCOMP_AMOUNT8, 0);
EMIT(REG_CNA_DCOMP_AMOUNT9, 0);
EMIT(REG_CNA_DCOMP_AMOUNT10, 0);
EMIT(REG_CNA_DCOMP_AMOUNT11, 0);
EMIT(REG_CNA_DCOMP_AMOUNT12, 0);
EMIT(REG_CNA_DCOMP_AMOUNT13, 0);
EMIT(REG_CNA_DCOMP_AMOUNT14, 0);
EMIT(REG_CNA_DCOMP_AMOUNT15, 0);
if (task->input_channels_real == 1) {
EMIT(REG_CNA_CVT_CON5, 65535);
} else {
EMIT(REG_CNA_CVT_CON5, 0);
}
int32_t pad_con1;
if (task->weights_width >= 3 && task->input_zero_point == 0x0)
pad_con1 = 0xffff8080;
else
pad_con1 = task->input_zero_point - 0x80;
if (operation->addition_input || operation->add_tensor != -1)
pad_con1 = 0xffffff80;
if (operation->depthwise && task->input_zero_point == 0x8b)
pad_con1 = 0x0b0b;
EMIT(REG_CNA_PAD_CON1, pad_con1);
uint32_t misc_cfg = CORE_MISC_CFG_QD_EN(1);
if (operation->depthwise)
misc_cfg |= CORE_MISC_CFG_DW_EN(1);
EMIT(REG_CORE_MISC_CFG, misc_cfg);
EMIT(REG_CORE_DATAOUT_SIZE_0,
CORE_DATAOUT_SIZE_0_DATAOUT_HEIGHT(task->output_height - 1) |
CORE_DATAOUT_SIZE_0_DATAOUT_WIDTH(task->output_width - 1));
EMIT(REG_CORE_DATAOUT_SIZE_1,
CORE_DATAOUT_SIZE_1_DATAOUT_CHANNEL(task->output_channels - 1));
EMIT(REG_CORE_CLIP_TRUNCATE,
CORE_CLIP_TRUNCATE_CLIP_TRUNCATE(operation->truncate_bits));
emit_raw(regs, CORE | 0x1, 0x3030, 0);
uint32_t feat_mode_cfg =
DPU_FEATURE_MODE_CFG_BURST_LEN(15) | DPU_FEATURE_MODE_CFG_OUTPUT_MODE(2);
if (operation->depthwise)
feat_mode_cfg |= DPU_FEATURE_MODE_CFG_CONV_MODE(3);
EMIT(REG_DPU_FEATURE_MODE_CFG, feat_mode_cfg);
EMIT(REG_DPU_DATA_FORMAT, 0);
EMIT(REG_DPU_OFFSET_PEND, 0);
EMIT(REG_DPU_DST_BASE_ADDR,
rkt_get_tensor(subgraph, operation->output_index)->phys_addr +
task->output_offset);
EMIT(REG_DPU_DST_SURF_STRIDE,
DPU_DST_SURF_STRIDE_DST_SURF_STRIDE(task->output_surface_stride));
EMIT(REG_DPU_DATA_CUBE_WIDTH,
DPU_DATA_CUBE_WIDTH_WIDTH(task->output_width - 1));
EMIT(REG_DPU_DATA_CUBE_HEIGHT,
DPU_DATA_CUBE_HEIGHT_HEIGHT(task->output_height - 1));
EMIT(REG_DPU_DATA_CUBE_NOTCH_ADDR, 0);
EMIT(REG_DPU_DATA_CUBE_CHANNEL,
DPU_DATA_CUBE_CHANNEL_ORIG_CHANNEL(task->output_channels_real - 1) |
DPU_DATA_CUBE_CHANNEL_CHANNEL(task->output_channels - 1));
EMIT(REG_DPU_BS_CFG, DPU_BS_CFG_BS_ALU_ALGO(2) | DPU_BS_CFG_BS_ALU_SRC(1) |
DPU_BS_CFG_BS_RELU_BYPASS(1) |
DPU_BS_CFG_BS_MUL_BYPASS(1));
EMIT(REG_DPU_BS_ALU_CFG, 0);
EMIT(REG_DPU_BS_MUL_CFG, 0);
EMIT(REG_DPU_BS_RELUX_CMP_VALUE, 0);
if (operation->depthwise) {
EMIT(REG_DPU_BS_OW_CFG, DPU_BS_OW_CFG_SIZE_E_2(3) |
DPU_BS_OW_CFG_SIZE_E_1(3) |
DPU_BS_OW_CFG_SIZE_E_0(3));
} else {
EMIT(REG_DPU_BS_OW_CFG, DPU_BS_OW_CFG_SIZE_E_2(1) |
DPU_BS_OW_CFG_SIZE_E_1(1) |
DPU_BS_OW_CFG_SIZE_E_0(1));
}
EMIT(REG_DPU_BS_OW_OP, DPU_BS_OW_OP_OW_OP(0x80 - weights_zero_point));
EMIT(REG_DPU_WDMA_SIZE_0,
DPU_WDMA_SIZE_0_CHANNEL_WDMA(task->output_channels - 1));
EMIT(REG_DPU_WDMA_SIZE_1,
DPU_WDMA_SIZE_1_HEIGHT_WDMA(task->output_height - 1) |
DPU_WDMA_SIZE_1_WIDTH_WDMA(task->output_width - 1));
EMIT(REG_DPU_BN_CFG,
DPU_BN_CFG_BN_RELU_BYPASS(1) | DPU_BN_CFG_BN_MUL_BYPASS(1) |
DPU_BN_CFG_BN_ALU_BYPASS(1) | DPU_BN_CFG_BN_BYPASS(1));
EMIT(REG_DPU_BN_ALU_CFG, 0);
EMIT(REG_DPU_BN_MUL_CFG, 0);
EMIT(REG_DPU_BN_RELUX_CMP_VALUE, 0);
if (operation->add_tensor != -1) {
EMIT(REG_DPU_EW_CFG,
DPU_EW_CFG_EW_CVT_TYPE(1) | DPU_EW_CFG_EW_DATA_MODE(1) |
DPU_EW_CFG_EDATA_SIZE(1) | DPU_EW_CFG_EW_ALU_ALGO(2) |
DPU_EW_CFG_EW_RELU_BYPASS(1) | DPU_EW_CFG_EW_LUT_BYPASS(1) |
DPU_EW_CFG_EW_OP_SRC(1));
/* See http://nvdla.org/hw/v1/ias/precision.html#element-wise */
EMIT(REG_DPU_EW_CVT_OFFSET_VALUE, operation->addition_offset);
float add_scale =
operation->addition_scale / (task->input_scale * task->weights_scale);
uint32_t add_scale_bits = fui(add_scale);
/* Taken from
* https://github.com/pytorch/QNNPACK/blob/master/src/qnnpack/requantization.h#L130
*/
unsigned add_shift = 127 + 31 - 32 - (add_scale_bits >> 23) + 16;
unsigned scale = ((add_scale_bits >> 9) & 0x7fff);
if (scale < 1 << 14)
scale |= 1 << 14;
EMIT(REG_DPU_EW_CVT_SCALE_VALUE,
DPU_EW_CVT_SCALE_VALUE_EW_OP_CVT_SHIFT(add_shift - 1) |
DPU_EW_CVT_SCALE_VALUE_EW_OP_CVT_SCALE(scale));
EMIT(REG_DPU_EW_RELUX_CMP_VALUE, 0x0);
float out_conv_scale =
(task->input_scale * task->weights_scale) / task->output_scale;
uint32_t out_scale_bits = fui(out_conv_scale);
unsigned out_shift = 127 + 31 - 32 - (out_scale_bits >> 23) + 16;
if (operation->truncate_bits > 0)
out_shift--;
unsigned out_scale = ((out_scale_bits >> 9) & 0x7fff) + 1;
if (out_scale < 1 << 14)
out_scale |= 1 << 14;
EMIT(REG_DPU_OUT_CVT_OFFSET, offset);
EMIT(REG_DPU_OUT_CVT_SCALE, DPU_OUT_CVT_SCALE_OUT_CVT_SCALE(out_scale));
EMIT(REG_DPU_OUT_CVT_SHIFT, DPU_OUT_CVT_SHIFT_OUT_CVT_SHIFT(out_shift - 1));
} else {
EMIT(REG_DPU_EW_CFG,
DPU_EW_CFG_EW_RELU_BYPASS(1) | DPU_EW_CFG_EW_OP_CVT_BYPASS(1) |
DPU_EW_CFG_EW_LUT_BYPASS(1) | DPU_EW_CFG_EW_OP_BYPASS(1) |
DPU_EW_CFG_EW_BYPASS(1));
EMIT(REG_DPU_EW_CVT_OFFSET_VALUE, 0);
EMIT(REG_DPU_EW_CVT_SCALE_VALUE, DPU_EW_CVT_SCALE_VALUE_EW_OP_CVT_SCALE(1));
EMIT(REG_DPU_EW_RELUX_CMP_VALUE, 0);
EMIT(REG_DPU_OUT_CVT_OFFSET, offset);
float conv_scale =
(task->input_scale * task->weights_scale) / task->output_scale;
// DBG("conv_scale %f\n", conv_scale);
uint32_t scale_bits = fui(conv_scale);
/* Taken from
* https://github.com/pytorch/QNNPACK/blob/master/src/qnnpack/requantization.h#L130
*/
unsigned shift = 127 + 31 - 32 - (scale_bits >> 23) + 16;
if (operation->truncate_bits > 0)
shift--;
unsigned scale = ((scale_bits >> 9) & 0x7fff) + 1;
if (scale < 1 << 14)
scale |= 1 << 14;
EMIT(REG_DPU_OUT_CVT_SCALE, DPU_OUT_CVT_SCALE_OUT_CVT_SCALE(scale));
EMIT(REG_DPU_OUT_CVT_SHIFT, DPU_OUT_CVT_SHIFT_OUT_CVT_SHIFT(shift - 1));
}
EMIT(REG_DPU_EW_OP_VALUE_0, 0);
EMIT(REG_DPU_EW_OP_VALUE_1, 0);
EMIT(REG_DPU_EW_OP_VALUE_2, 0);
EMIT(REG_DPU_EW_OP_VALUE_3, 0);
EMIT(REG_DPU_EW_OP_VALUE_4, 0);
EMIT(REG_DPU_EW_OP_VALUE_5, 0);
EMIT(REG_DPU_EW_OP_VALUE_6, 0);
EMIT(REG_DPU_EW_OP_VALUE_7, 0);
EMIT(REG_DPU_SURFACE_ADD, DPU_SURFACE_ADD_SURF_ADD(task->surfaces_per_row));
emit_raw(regs, DPU | 0x1, 0x40c4, 0);
EMIT(REG_DPU_LUT_ACCESS_CFG, 0);
EMIT(REG_DPU_LUT_ACCESS_DATA, 0);
EMIT(REG_DPU_LUT_CFG, 0);
EMIT(REG_DPU_LUT_INFO, 0);
EMIT(REG_DPU_LUT_LE_START, 0);
EMIT(REG_DPU_LUT_LE_END, 0);
EMIT(REG_DPU_LUT_LO_START, 0);
EMIT(REG_DPU_LUT_LO_END, 0);
EMIT(REG_DPU_LUT_LE_SLOPE_SCALE, 0);
EMIT(REG_DPU_LUT_LE_SLOPE_SHIFT, 0);
EMIT(REG_DPU_LUT_LO_SLOPE_SCALE, 0);
EMIT(REG_DPU_LUT_LO_SLOPE_SHIFT, 0);
EMIT(REG_DPU_RDMA_RDMA_DATA_CUBE_WIDTH,
DPU_RDMA_RDMA_DATA_CUBE_WIDTH_WIDTH(task->output_width - 1));
EMIT(REG_DPU_RDMA_RDMA_DATA_CUBE_HEIGHT,
DPU_RDMA_RDMA_DATA_CUBE_HEIGHT_HEIGHT(task->output_height - 1));
EMIT(REG_DPU_RDMA_RDMA_DATA_CUBE_CHANNEL,
DPU_RDMA_RDMA_DATA_CUBE_CHANNEL_CHANNEL(task->output_channels - 1));
if (operation->add_tensor != -1) {
EMIT(REG_DPU_RDMA_RDMA_SRC_BASE_ADDR,
rkt_get_tensor(subgraph, operation->add_tensor)->phys_addr +
task->output_offset);
} else {
EMIT(REG_DPU_RDMA_RDMA_SRC_BASE_ADDR, 0);
}
EMIT(REG_DPU_RDMA_RDMA_BRDMA_CFG, DPU_RDMA_RDMA_BRDMA_CFG_BRDMA_DATA_USE(1));
EMIT(REG_DPU_RDMA_RDMA_BS_BASE_ADDR,
rkt_resource(operation->biases)->phys_addr);
EMIT(REG_DPU_RDMA_RDMA_NRDMA_CFG, 0);
EMIT(REG_DPU_RDMA_RDMA_BN_BASE_ADDR, 0);
unsigned ew_stride =
MAX2(operation->output_width * operation->output_height, 12);
if (operation->add_tensor != -1) {
EMIT(REG_DPU_RDMA_RDMA_ERDMA_CFG,
DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_DATA_MODE(1) |
DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_DATA_SIZE(1));
unsigned ew_base_offset =
operation->output_width * operation->output_height * ATOMIC_K_SIZE;
EMIT(REG_DPU_RDMA_RDMA_EW_BASE_ADDR,
rkt_get_tensor(subgraph, operation->add_tensor)->phys_addr +
task->output_offset + ew_base_offset);
EMIT(REG_DPU_RDMA_RDMA_EW_SURF_STRIDE,
DPU_RDMA_RDMA_EW_SURF_STRIDE_EW_SURF_STRIDE(ew_stride));
} else {
EMIT(REG_DPU_RDMA_RDMA_ERDMA_CFG, DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_DISABLE(1));
EMIT(REG_DPU_RDMA_RDMA_EW_BASE_ADDR, 0);
EMIT(REG_DPU_RDMA_RDMA_EW_SURF_STRIDE, 0);
}
uint32_t rdma_feat_mode_cfg = 0x0;
if (operation->add_tensor != -1) {
rdma_feat_mode_cfg |= DPU_RDMA_RDMA_FEATURE_MODE_CFG_BURST_LEN(15) |
DPU_RDMA_RDMA_FEATURE_MODE_CFG_COMB_USE(5);
} else {
rdma_feat_mode_cfg |= DPU_RDMA_RDMA_FEATURE_MODE_CFG_BURST_LEN(15) |
DPU_RDMA_RDMA_FEATURE_MODE_CFG_MRDMA_DISABLE(1);
}
if (operation->depthwise)
rdma_feat_mode_cfg |= DPU_RDMA_RDMA_FEATURE_MODE_CFG_CONV_MODE(3);
EMIT(REG_DPU_RDMA_RDMA_FEATURE_MODE_CFG, rdma_feat_mode_cfg);
EMIT(REG_DPU_RDMA_RDMA_SRC_DMA_CFG, 0);
unsigned surf_notch =
ew_stride +
task->output_width * (operation->output_height - task->output_height);
if (operation->input_width == 3) {
surf_notch = 15;
}
if (operation->add_tensor != -1) {
EMIT(REG_DPU_RDMA_RDMA_SURF_NOTCH,
DPU_RDMA_RDMA_SURF_NOTCH_SURF_NOTCH_ADDR(surf_notch));
} else {
EMIT(REG_DPU_RDMA_RDMA_SURF_NOTCH, 0);
}
EMIT(REG_DPU_RDMA_RDMA_PAD_CFG, 0);
EMIT(REG_DPU_RDMA_RDMA_WEIGHT,
DPU_RDMA_RDMA_WEIGHT_E_WEIGHT(1) | DPU_RDMA_RDMA_WEIGHT_N_WEIGHT(1) |
DPU_RDMA_RDMA_WEIGHT_B_WEIGHT(1) | DPU_RDMA_RDMA_WEIGHT_M_WEIGHT(1));
if (operation->add_tensor != -1) {
EMIT(REG_DPU_RDMA_RDMA_EW_SURF_NOTCH,
DPU_RDMA_RDMA_EW_SURF_NOTCH_EW_SURF_NOTCH(surf_notch));
} else {
EMIT(REG_DPU_RDMA_RDMA_EW_SURF_NOTCH, 0x0);
}
if (num_tasks == 1)
util_dynarray_append_typed(regs, uint64_t, 0x0);
else
EMIT(REG_PC_BASE_ADDRESS, 0);
EMIT(REG_PC_REGISTER_AMOUNTS, 0);
/* TRM: before op_en, 64'h0041_xxxx_xxxx_xxxx must be set. */
util_dynarray_append_typed(regs, uint64_t, 0x0041000000000000);
/* TRM: 64'h0081_0000_007f_0008 will set each block's op_en(CNA, CORE, ...,
* PPU_RDMA). */
emit_raw(regs, 0x81, REG_PC_OPERATION_ENABLE,
PC_OPERATION_ENABLE_RESERVED_0(14) | PC_OPERATION_ENABLE_OP_EN(1));
}
void
rkt_fill_regcmd(struct rkt_ml_subgraph *subgraph,
const struct rkt_operation *operation,
struct util_dynarray *regs, unsigned task_num)
{
/*
* TODO: We should only need to set all the registers on the regcmd for the first
* task in an operation, but for now set them all to be sure.
*/
fill_first_regcmd(subgraph, operation, regs, task_num);
}
userspace/npu-probe/verify/rkt_regcmd.h
+15
View File
@@ -0,0 +1,15 @@
/*
* Copyright (c) 2024 Tomeu Vizoso <tomeu@tomeuvizoso.net>
* SPDX-License-Identifier: MIT
*/
#ifndef RKT_REGCMD_H
#define RKT_REGCMD_H
#include "rkt_ml.h"
void rkt_fill_regcmd(struct rkt_ml_subgraph *subgraph,
const struct rkt_operation *operation,
struct util_dynarray *regs, unsigned task_num);
#endif /* RKT_REGCMD_H */
userspace/npu-probe/verify/rkt_registers.h
+4852
View File
File diff suppressed because it is too large Load Diff
userspace/npu-probe/verify/rkt_task.c
+327
View File
@@ -0,0 +1,327 @@
/*
* Copyright (c) 2024 Tomeu Vizoso <tomeu@tomeuvizoso.net>
* SPDX-License-Identifier: MIT
*/
#include "rkt_task.h"
#include "rkt_ml.h"
static unsigned
calc_entries_per_slice(struct rkt_operation *operation)
{
unsigned bpe = sizeof(uint8_t);
unsigned atomics_per_entry = CBUF_ENTRY_SIZE / FEATURE_ATOMIC_SIZE;
unsigned total_c_atomics =
DIV_ROUND_UP(operation->input_channels * bpe, FEATURE_ATOMIC_SIZE);
unsigned last_c_atomics = total_c_atomics % atomics_per_entry;
unsigned int_c_entries =
(total_c_atomics / atomics_per_entry) * operation->input_width;
unsigned frac_c_entries =
(last_c_atomics == 3)
? operation->input_width
: DIV_ROUND_UP(last_c_atomics * operation->input_width,
atomics_per_entry);
return int_c_entries + frac_c_entries;
}
static unsigned
calc_input_banks(struct rkt_operation *operation)
{
unsigned entries_per_slice = calc_entries_per_slice(operation);
return DIV_ROUND_UP(entries_per_slice * operation->input_height,
CBUF_ENTRIES_PER_BANK);
}
static unsigned
calc_weights_banks(struct rkt_operation *operation)
{
unsigned bpe = sizeof(uint8_t);
unsigned bytes = operation->weights_width * operation->weights_height *
operation->input_channels * bpe;
unsigned entries;
unsigned banks;
if (!operation->depthwise)
bytes *= operation->output_channels;
entries = DIV_ROUND_UP(bytes, CBUF_ENTRY_SIZE);
banks = DIV_ROUND_UP(entries, CBUF_ENTRIES_PER_BANK);
/* Why do we need an extra bank? The calc above might be wrong on this HW */
banks++;
return banks;
}
static unsigned
calc_line_stride(unsigned width)
{
return width * ATOMIC_K_SIZE * sizeof(uint8_t);
}
static void
fill_task(struct rkt_ml_subgraph *subgraph,
struct rkt_operation *operation,
struct split_task *task)
{
task->stride_x = operation->stride;
task->stride_y = operation->stride;
task->input_width = operation->input_width;
if (task->input_width == 8 &&
(operation->addition_input || operation->add_tensor != -1))
task->input_width *= 2;
task->input_height = operation->input_height;
task->input_channels =
align(MAX2(operation->input_channels, FEATURE_ATOMIC_SIZE),
FEATURE_ATOMIC_SIZE);
task->input_channels_real = operation->input_channels;
task->input_zero_point = operation->input_zero_point;
task->input_scale = operation->input_scale;
task->output_width = operation->output_width;
task->output_height = operation->output_height;
task->output_channels_real = operation->output_channels;
task->output_channels = align(MAX2(operation->output_channels, 32), 32);
if (operation->depthwise) {
if (task->output_channels_real <= 32)
task->output_channels *= 2;
task->output_channels = align(task->output_channels, 64);
}
task->output_zero_point = operation->output_zero_point;
task->output_scale = operation->output_scale;
if (task->input_channels_real == 1 &&
(task->output_channels_real > 1 ||
(operation->addition_input || operation->add_tensor != -1))) {
task->input_width = MAX2(task->input_width, FEATURE_ATOMIC_SIZE);
task->input_line_stride =
MAX2(calc_line_stride(operation->input_width) / FEATURE_ATOMIC_SIZE,
FEATURE_ATOMIC_SIZE);
if (operation->input_channels == 32 && operation->input_width == 80) {
task->input_line_stride *= 4;
task->input_surface_stride = (float)task->input_line_stride *
(((float)task->input_height / 4) - 1);
} else
task->input_surface_stride =
(float)task->input_line_stride * (((float)task->input_height) - 1);
} else {
task->input_line_stride = calc_line_stride(operation->input_width) / 4;
task->input_surface_stride =
(float)task->input_line_stride * (((float)task->input_height / 4) - 1);
}
if (task->input_width == 8 &&
(operation->addition_input || operation->add_tensor != -1)) {
task->input_line_stride /= 2;
task->input_surface_stride = 112;
}
int output_line_stride = calc_line_stride(operation->output_width);
task->output_surface_stride = output_line_stride * task->output_height;
task->output_surface_stride /= FEATURE_ATOMIC_SIZE;
if (task->input_channels_real == 1)
task->input_data_entries = task->input_width * task->input_height;
else if (task->input_width == 40 && task->input_channels_real == 40)
task->input_data_entries = 40;
else
task->input_data_entries = DIV_ROUND_UP(
task->input_width * 2 *
DIV_ROUND_UP(task->input_channels_real, FEATURE_ATOMIC_SIZE),
8);
task->weights_width = operation->weights_width;
task->weights_height = operation->weights_height;
task->weights_zero_point = operation->weights_zero_point;
task->weights_scale = operation->weights_scale;
if (operation->depthwise)
task->weights_kernels = 1;
else
task->weights_kernels = align(operation->output_channels, 2);
task->surfaces_per_row = task->output_width * task->output_height * 2;
if (operation->depthwise)
task->surfaces_per_row *= 2;
}
void
rkt_split_tasks(struct rkt_ml_subgraph *subgraph,
struct rkt_operation *operation)
{
/* Function mostly taken from NVDLA */
unsigned entries_per_slice = calc_entries_per_slice(operation);
unsigned input_banks_required = calc_input_banks(operation);
unsigned weights_banks_required = calc_weights_banks(operation);
unsigned available_weights_banks = weights_banks_required;
unsigned available_input_banks = CBUF_BANKS - weights_banks_required;
unsigned pad_top;
unsigned pad_bottom;
unsigned pad_left;
unsigned pad_right;
pad_top = operation->padding_top;
pad_bottom = operation->padding_bottom;
pad_left = operation->padding_left;
pad_right = operation->padding_right;
if (weights_banks_required + 1 < CBUF_BANKS) {
/* Full weights, partial input */
operation->reuse_weights_cbuf = true;
} else {
/* Partial weights, partial input */
operation->reuse_weights_cbuf = false;
available_input_banks = 7;
available_weights_banks = CBUF_BANKS - available_input_banks;
}
if (input_banks_required <= available_input_banks) {
/* Full weights, full input */
struct split_task task = {0};
task.num = 0;
fill_task(subgraph, operation, &task);
task.input_banks = input_banks_required;
task.weights_banks = CBUF_BANKS - task.input_banks;
task.input_height = operation->input_height;
task.pad_top = pad_top;
task.pad_bottom = pad_bottom;
task.pad_left = pad_left;
task.pad_right = pad_right;
task.atomic_count = task.output_width * task.output_height;
util_dynarray_append(&operation->tasks, task);
return;
}
struct split_task task = {0};
unsigned available_slices =
(CBUF_ENTRIES_PER_BANK * available_input_banks) / entries_per_slice;
task.num = 0;
fill_task(subgraph, operation, &task);
task.input_banks = available_input_banks;
task.weights_banks = available_weights_banks;
task.top_slice = 0;
task.bottom_slice = available_slices - 1;
task.pad_top = pad_top;
task.pad_left = pad_left;
task.pad_right = pad_right;
util_dynarray_append(&operation->tasks, task);
for (unsigned slice = operation->weights_height - pad_top - 1;
slice < operation->input_height;) {
memset(&task, 0, sizeof(task));
struct split_task *prev_task = util_dynarray_element(
&operation->tasks, struct split_task,
util_dynarray_num_elements(&operation->tasks, struct split_task) - 1);
while (slice <= prev_task->bottom_slice) {
slice += operation->stride;
}
if (slice > prev_task->bottom_slice) {
slice -= operation->stride;
}
task.num = util_dynarray_num_elements(&operation->tasks, struct split_task);
fill_task(subgraph, operation, &task);
task.top_slice = MIN2(slice, prev_task->bottom_slice) -
(operation->weights_height - 1) + operation->stride;
task.bottom_slice = task.top_slice + available_slices - 1;
task.pad_left = pad_left;
task.pad_right = pad_right;
// check if current task is the last one
if (task.bottom_slice >= operation->input_height - 1) {
task.bottom_slice = operation->input_height - 1;
task.pad_bottom = pad_bottom;
util_dynarray_append(&operation->tasks, task);
break;
}
slice = task.top_slice + operation->weights_height - 1;
util_dynarray_append(&operation->tasks, task);
}
struct split_task *last_task = util_dynarray_element(
&operation->tasks, struct split_task,
util_dynarray_num_elements(&operation->tasks, struct split_task) - 1);
if (last_task->top_slice >= operation->input_height ||
last_task->bottom_slice >= (operation->input_height + pad_bottom)) {
(void)util_dynarray_pop_ptr(&operation->tasks, struct split_task);
}
// determine overlap slices between 2 split chunks
for (int i = 1;
i < util_dynarray_num_elements(&operation->tasks, struct split_task);
i++) {
struct split_task *prev_task =
util_dynarray_element(&operation->tasks, struct split_task, i - 1);
struct split_task *cur_task =
util_dynarray_element(&operation->tasks, struct split_task, i);
if (prev_task->bottom_slice >= cur_task->top_slice) {
cur_task->num_overlap_slices =
prev_task->bottom_slice - cur_task->top_slice + 1;
prev_task->num_retain_slices = cur_task->num_overlap_slices;
} else {
cur_task->num_overlap_slices = 0;
prev_task->num_retain_slices = 0;
}
}
unsigned output_height_processed = 0;
for (int i = 0;
i < util_dynarray_num_elements(&operation->tasks, struct split_task);
i++) {
struct split_task *cur_task =
util_dynarray_element(&operation->tasks, struct split_task, i);
unsigned slice = cur_task->top_slice + (operation->weights_height - 1) -
cur_task->pad_top;
while (slice <= cur_task->bottom_slice + cur_task->pad_bottom) {
slice += operation->stride;
cur_task->convolutions++;
}
cur_task->bottom_slice =
MIN2(cur_task->bottom_slice, operation->input_height - 1);
cur_task->input_height = cur_task->bottom_slice - cur_task->top_slice + 1;
cur_task->output_width = (cur_task->input_width + cur_task->pad_left +
cur_task->pad_right - operation->weights_width) /
operation->stride +
1;
cur_task->output_height =
(cur_task->input_height + cur_task->pad_top + cur_task->pad_bottom -
operation->weights_height) /
operation->stride +
1;
cur_task->atomic_count = cur_task->output_width * cur_task->output_height;
cur_task->input_offset =
calc_line_stride(operation->input_width) * cur_task->top_slice;
cur_task->output_offset =
calc_line_stride(operation->output_width) * output_height_processed;
cur_task->input_banks = available_input_banks;
cur_task->weights_banks = available_weights_banks;
output_height_processed += cur_task->output_height;
}
}
userspace/npu-probe/verify/rkt_task.h
+14
View File
@@ -0,0 +1,14 @@
/*
* Copyright (c) 2024 Tomeu Vizoso <tomeu@tomeuvizoso.net>
* SPDX-License-Identifier: MIT
*/
#ifndef RKT_TASK_H
#define RKT_TASK_H
#include "rkt_ml.h"
void rkt_split_tasks(struct rkt_ml_subgraph *subgraph,
struct rkt_operation *operation);
#endif /* RKT_TASK_H */