marfrit/rk-llama.cpp
1
0
Fork 0
Code Issues Pull Requests Actions 30 Packages Projects Releases Wiki Activity

ggml: add rocket NPU backend (RK3588 mainline accel driver)

Browse Source 
New MUL_MAT backend for the RK3588 NPU via the mainline 'rocket' DRM-accel
driver (/dev/accel/accel0), with no vendor RKNPU2 userspace. Modeled on
ggml-blas: an ACCEL device with CPU-resident weights; the scheduler offloads
supported MUL_MAT nodes and leaves the rest on CPU.

Per plane: dequantize weight to F32 (ggml to_float), requantize per-output-
channel to int8, quantize F32 activation per-tensor to int8, run the tiled
int8 GEMM on the NPU (rkt_npu_matmul, ported from the rosenblatt npu-probe:
Mesa-rocket regcmd builder + NVDLA CBUF tiling + librocket DRM ioctls), then
dequantize the int8 result. out_scale is a Cauchy-Schwarz bound on |output|
so the int8 output never clips.

Empirically-verified HW limits gate supports_op: TILE_M=16, TILE_N=128,
K<=8192 (int8 K-limit), K%16==0, F32 activations, dequantizable weight.
Job-completion wait uses a finite 6s absolute deadline so a hung NPU job
errors instead of deadlocking.

Enable with -DGGML_ROCKET=ON (and -DLLAMA_RKNPU2=OFF so the vendor backend
does not claim the weight buffers). Verified end-to-end: gemma-4-E2B-it-Q8_0
generates coherent text on boltzmann's mainline rocket kernel, matmuls
offloaded to the NPU, ~3.9 tok/s eval.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01EWpfhDgYNA21tETDP9ueBE
This commit is contained in:
mfritsche
2026-07-14 10:52:06 +02:00
parent 0ebb855635
commit 8e49f43486
23 changed files with 3421 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/CMakeLists.txt
+1
View File
@@ -267,6 +267,7 @@ set (GGML_OPENCL_TARGET_VERSION "300" CACHE STRING
option(GGML_HEXAGON "ggml: enable Hexagon backend" OFF)
option(GGML_RKNPU2 "ggml: use RKNPU2" OFF)
option(GGML_ROCKET "ggml: use RK3588 NPU (rocket accel)" OFF)
set(GGML_HEXAGON_FP32_QUANTIZE_GROUP_SIZE 128 CACHE STRING "ggml: quantize group size (32, 64, or 128)")
# toolchain for vulkan-shaders-gen
ggml/src/CMakeLists.txt
+1
View File
@@ -480,6 +480,7 @@ ggml_add_backend(Hexagon)
ggml_add_backend(ZenDNN)
ggml_add_backend(OPENVINO)
ggml_add_backend(RKNPU2)
ggml_add_backend(ROCKET)
foreach (target ggml-base ggml)
target_include_directories(${target} PUBLIC $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../include> $<INSTALL_INTERFACE:include>)
ggml/src/ggml-backend-reg.cpp
+7
View File
@@ -90,6 +90,10 @@
#include "ggml-rknpu2.h"
#endif
#ifdef GGML_USE_ROCKET
#include "ggml-rocket.h"
#endif
// disable C++17 deprecation warning for std::codecvt_utf8
#if defined(__clang__)
# pragma clang diagnostic push
@@ -179,6 +183,9 @@ struct ggml_backend_registry {
#endif
#ifdef GGML_USE_RKNPU2
register_backend(ggml_backend_rknpu2_reg());
#endif
#ifdef GGML_USE_ROCKET
register_backend(ggml_backend_rocket_reg());
#endif
}
ggml/src/ggml-rocket/CMakeLists.txt
+21
View File
@@ -0,0 +1,21 @@
# ggml-rocket: RK3588 NPU backend via mainline "rocket" DRM-accel driver.
# No vendor library — raw DRM ioctls against /dev/accel/accel0.
ggml_add_backend_library(ggml-rocket
ggml-rocket.cpp
librocket.c
rkt_npu_matmul.c
rkt_operands.c
rkt_matmul.c
rkt_matmul_cna.c
rkt_matmul_coredpu.c
rkt_gemm.c
)
target_include_directories(ggml-rocket
PRIVATE
/usr/include/drm
/usr/include/libdrm
PUBLIC
${CMAKE_CURRENT_SOURCE_DIR}
)
ggml/src/ggml-rocket/cna_defs.h
+532
View File
@@ -0,0 +1,532 @@
#ifndef CNA_DEFS_H
#define CNA_DEFS_H
#define REG_CNA_S_STATUS 0x00001000
#define CNA_S_STATUS_RESERVED_0__MASK 0xfffc0000
#define CNA_S_STATUS_RESERVED_0__SHIFT 18
#define CNA_S_STATUS_STATUS_1__MASK 0x00030000
#define CNA_S_STATUS_STATUS_1__SHIFT 16
#define CNA_S_STATUS_RESERVED_1__MASK 0x0000fffc
#define CNA_S_STATUS_RESERVED_1__SHIFT 2
#define CNA_S_STATUS_STATUS_0__MASK 0x00000003
#define CNA_S_STATUS_STATUS_0__SHIFT 0
#define REG_CNA_S_POINTER 0x00001004
#define CNA_S_POINTER_RESERVED_0__MASK 0xfffe0000
#define CNA_S_POINTER_RESERVED_0__SHIFT 17
#define CNA_S_POINTER_EXECUTER__MASK 0x00010000
#define CNA_S_POINTER_EXECUTER__SHIFT 16
#define CNA_S_POINTER_RESERVED_1__MASK 0x0000ffc0
#define CNA_S_POINTER_RESERVED_1__SHIFT 6
#define CNA_S_POINTER_EXECUTER_PP_CLEAR__MASK 0x00000020
#define CNA_S_POINTER_EXECUTER_PP_CLEAR__SHIFT 5
#define CNA_S_POINTER_POINTER_PP_CLEAR__MASK 0x00000010
#define CNA_S_POINTER_POINTER_PP_CLEAR__SHIFT 4
#define CNA_S_POINTER_POINTER_PP_MODE__MASK 0x00000008
#define CNA_S_POINTER_POINTER_PP_MODE__SHIFT 3
#define CNA_S_POINTER_EXECUTER_PP_EN__MASK 0x00000004
#define CNA_S_POINTER_EXECUTER_PP_EN__SHIFT 2
#define CNA_S_POINTER_POINTER_PP_EN__MASK 0x00000002
#define CNA_S_POINTER_POINTER_PP_EN__SHIFT 1
#define CNA_S_POINTER_POINTER__MASK 0x00000001
#define CNA_S_POINTER_POINTER__SHIFT 0
#define REG_CNA_OPERATION_ENABLE 0x00001008
#define CNA_OPERATION_ENABLE_RESERVED_0__MASK 0xfffffffe
#define CNA_OPERATION_ENABLE_RESERVED_0__SHIFT 1
#define CNA_OPERATION_ENABLE_OP_EN__MASK 0x00000001
#define CNA_OPERATION_ENABLE_OP_EN__SHIFT 0
#define REG_CNA_CONV_CON1 0x0000100c
#define CNA_CONV_CON1_RESERVED_0__MASK 0x80000000
#define CNA_CONV_CON1_RESERVED_0__SHIFT 31
#define CNA_CONV_CON1_NONALIGN_DMA__MASK 0x40000000
#define CNA_CONV_CON1_NONALIGN_DMA__SHIFT 30
#define CNA_CONV_CON1_GROUP_LINE_OFF__MASK 0x20000000
#define CNA_CONV_CON1_GROUP_LINE_OFF__SHIFT 29
#define CNA_CONV_CON1_RESERVED_1__MASK 0x1ffe0000
#define CNA_CONV_CON1_RESERVED_1__SHIFT 17
#define CNA_CONV_CON1_DECONV__MASK 0x00010000
#define CNA_CONV_CON1_DECONV__SHIFT 16
#define CNA_CONV_CON1_ARGB_IN__MASK 0x0000f000
#define CNA_CONV_CON1_ARGB_IN__SHIFT 12
#define CNA_CONV_CON1_RESERVED_2__MASK 0x00000c00
#define CNA_CONV_CON1_RESERVED_2__SHIFT 10
#define CNA_CONV_CON1_PROC_PRECISION__MASK 0x00000380
#define CNA_CONV_CON1_PROC_PRECISION__SHIFT 7
#define CNA_CONV_CON1_IN_PRECISION__MASK 0x00000070
#define CNA_CONV_CON1_IN_PRECISION__SHIFT 4
#define CNA_CONV_CON1_CONV_MODE__MASK 0x0000000f
#define CNA_CONV_CON1_CONV_MODE__SHIFT 0
#define REG_CNA_CONV_CON2 0x00001010
#define CNA_CONV_CON2_RESERVED_0__MASK 0xff000000
#define CNA_CONV_CON2_RESERVED_0__SHIFT 24
#define CNA_CONV_CON2_KERNEL_GROUP__MASK 0x00ff0000
#define CNA_CONV_CON2_KERNEL_GROUP__SHIFT 16
#define CNA_CONV_CON2_RESERVED_1__MASK 0x0000c000
#define CNA_CONV_CON2_RESERVED_1__SHIFT 14
#define CNA_CONV_CON2_FEATURE_GRAINS__MASK 0x00003ff0
#define CNA_CONV_CON2_FEATURE_GRAINS__SHIFT 4
#define CNA_CONV_CON2_RESERVED_2__MASK 0x00000008
#define CNA_CONV_CON2_RESERVED_2__SHIFT 3
#define CNA_CONV_CON2_CSC_WO_EN__MASK 0x00000004
#define CNA_CONV_CON2_CSC_WO_EN__SHIFT 2
#define CNA_CONV_CON2_CSC_DO_EN__MASK 0x00000002
#define CNA_CONV_CON2_CSC_DO_EN__SHIFT 1
#define CNA_CONV_CON2_CMD_FIFO_SRST__MASK 0x00000001
#define CNA_CONV_CON2_CMD_FIFO_SRST__SHIFT 0
#define REG_CNA_CONV_CON3 0x00001014
#define CNA_CONV_CON3_RESERVED_0__MASK 0x80000000
#define CNA_CONV_CON3_RESERVED_0__SHIFT 31
#define CNA_CONV_CON3_NN_MODE__MASK 0x70000000
#define CNA_CONV_CON3_NN_MODE__SHIFT 28
#define CNA_CONV_CON3_RESERVED_1__MASK 0x0c000000
#define CNA_CONV_CON3_RESERVED_1__SHIFT 26
#define CNA_CONV_CON3_ATROUS_Y_DILATION__MASK 0x03e00000
#define CNA_CONV_CON3_ATROUS_Y_DILATION__SHIFT 21
#define CNA_CONV_CON3_ATROUS_X_DILATION__MASK 0x001f0000
#define CNA_CONV_CON3_ATROUS_X_DILATION__SHIFT 16
#define CNA_CONV_CON3_RESERVED_2__MASK 0x0000c000
#define CNA_CONV_CON3_RESERVED_2__SHIFT 14
#define CNA_CONV_CON3_DECONV_Y_STRIDE__MASK 0x00003800
#define CNA_CONV_CON3_DECONV_Y_STRIDE__SHIFT 11
#define CNA_CONV_CON3_DECONV_X_STRIDE__MASK 0x00000700
#define CNA_CONV_CON3_DECONV_X_STRIDE__SHIFT 8
#define CNA_CONV_CON3_RESERVED_3__MASK 0x000000c0
#define CNA_CONV_CON3_RESERVED_3__SHIFT 6
#define CNA_CONV_CON3_CONV_Y_STRIDE__MASK 0x00000038
#define CNA_CONV_CON3_CONV_Y_STRIDE__SHIFT 3
#define CNA_CONV_CON3_CONV_X_STRIDE__MASK 0x00000007
#define CNA_CONV_CON3_CONV_X_STRIDE__SHIFT 0
#define REG_CNA_DATA_SIZE0 0x00001020
#define CNA_DATA_SIZE0_RESERVED_0__MASK 0xf8000000
#define CNA_DATA_SIZE0_RESERVED_0__SHIFT 27
#define CNA_DATA_SIZE0_DATAIN_WIDTH__MASK 0x07ff0000
#define CNA_DATA_SIZE0_DATAIN_WIDTH__SHIFT 16
#define CNA_DATA_SIZE0_RESERVED_1__MASK 0x0000f800
#define CNA_DATA_SIZE0_RESERVED_1__SHIFT 11
#define CNA_DATA_SIZE0_DATAIN_HEIGHT__MASK 0x000007ff
#define CNA_DATA_SIZE0_DATAIN_HEIGHT__SHIFT 0
#define REG_CNA_DATA_SIZE1 0x00001024
#define CNA_DATA_SIZE1_RESERVED_0__MASK 0xc0000000
#define CNA_DATA_SIZE1_RESERVED_0__SHIFT 30
#define CNA_DATA_SIZE1_DATAIN_CHANNEL_REAL__MASK 0x3fff0000
#define CNA_DATA_SIZE1_DATAIN_CHANNEL_REAL__SHIFT 16
#define CNA_DATA_SIZE1_DATAIN_CHANNEL__MASK 0x0000ffff
#define CNA_DATA_SIZE1_DATAIN_CHANNEL__SHIFT 0
#define REG_CNA_DATA_SIZE2 0x00001028
#define CNA_DATA_SIZE2_RESERVED_0__MASK 0xfffff800
#define CNA_DATA_SIZE2_RESERVED_0__SHIFT 11
#define CNA_DATA_SIZE2_DATAOUT_WIDTH__MASK 0x000007ff
#define CNA_DATA_SIZE2_DATAOUT_WIDTH__SHIFT 0
#define REG_CNA_DATA_SIZE3 0x0000102c
#define CNA_DATA_SIZE3_RESERVED_0__MASK 0xff000000
#define CNA_DATA_SIZE3_RESERVED_0__SHIFT 24
#define CNA_DATA_SIZE3_SURF_MODE__MASK 0x00c00000
#define CNA_DATA_SIZE3_SURF_MODE__SHIFT 22
#define CNA_DATA_SIZE3_DATAOUT_ATOMICS__MASK 0x003fffff
#define CNA_DATA_SIZE3_DATAOUT_ATOMICS__SHIFT 0
#define REG_CNA_WEIGHT_SIZE0 0x00001030
#define CNA_WEIGHT_SIZE0_WEIGHT_BYTES__MASK 0xffffffff
#define CNA_WEIGHT_SIZE0_WEIGHT_BYTES__SHIFT 0
#define REG_CNA_WEIGHT_SIZE1 0x00001034
#define CNA_WEIGHT_SIZE1_RESERVED_0__MASK 0xfff80000
#define CNA_WEIGHT_SIZE1_RESERVED_0__SHIFT 19
#define CNA_WEIGHT_SIZE1_WEIGHT_BYTES_PER_KERNEL__MASK 0x0007ffff
#define CNA_WEIGHT_SIZE1_WEIGHT_BYTES_PER_KERNEL__SHIFT 0
#define REG_CNA_WEIGHT_SIZE2 0x00001038
#define CNA_WEIGHT_SIZE2_RESERVED_0__MASK 0xe0000000
#define CNA_WEIGHT_SIZE2_RESERVED_0__SHIFT 29
#define CNA_WEIGHT_SIZE2_WEIGHT_WIDTH__MASK 0x1f000000
#define CNA_WEIGHT_SIZE2_WEIGHT_WIDTH__SHIFT 24
#define CNA_WEIGHT_SIZE2_RESERVED_1__MASK 0x00e00000
#define CNA_WEIGHT_SIZE2_RESERVED_1__SHIFT 21
#define CNA_WEIGHT_SIZE2_WEIGHT_HEIGHT__MASK 0x001f0000
#define CNA_WEIGHT_SIZE2_WEIGHT_HEIGHT__SHIFT 16
#define CNA_WEIGHT_SIZE2_RESERVED_2__MASK 0x0000c000
#define CNA_WEIGHT_SIZE2_RESERVED_2__SHIFT 14
#define CNA_WEIGHT_SIZE2_WEIGHT_KERNELS__MASK 0x00003fff
#define CNA_WEIGHT_SIZE2_WEIGHT_KERNELS__SHIFT 0
#define REG_CNA_CBUF_CON0 0x00001040
#define CNA_CBUF_CON0_RESERVED_0__MASK 0xffffc000
#define CNA_CBUF_CON0_RESERVED_0__SHIFT 14
#define CNA_CBUF_CON0_WEIGHT_REUSE__MASK 0x00002000
#define CNA_CBUF_CON0_WEIGHT_REUSE__SHIFT 13
#define CNA_CBUF_CON0_DATA_REUSE__MASK 0x00001000
#define CNA_CBUF_CON0_DATA_REUSE__SHIFT 12
#define CNA_CBUF_CON0_RESERVED_1__MASK 0x00000800
#define CNA_CBUF_CON0_RESERVED_1__SHIFT 11
#define CNA_CBUF_CON0_FC_DATA_BANK__MASK 0x00000700
#define CNA_CBUF_CON0_FC_DATA_BANK__SHIFT 8
#define CNA_CBUF_CON0_WEIGHT_BANK__MASK 0x000000f0
#define CNA_CBUF_CON0_WEIGHT_BANK__SHIFT 4
#define CNA_CBUF_CON0_DATA_BANK__MASK 0x0000000f
#define CNA_CBUF_CON0_DATA_BANK__SHIFT 0
#define REG_CNA_CBUF_CON1 0x00001044
#define CNA_CBUF_CON1_RESERVED_0__MASK 0xffffc000
#define CNA_CBUF_CON1_RESERVED_0__SHIFT 14
#define CNA_CBUF_CON1_DATA_ENTRIES__MASK 0x00003fff
#define CNA_CBUF_CON1_DATA_ENTRIES__SHIFT 0
#define REG_CNA_CVT_CON0 0x0000104c
#define CNA_CVT_CON0_RESERVED_0__MASK 0xf0000000
#define CNA_CVT_CON0_RESERVED_0__SHIFT 28
#define CNA_CVT_CON0_CVT_TRUNCATE_3__MASK 0x0fc00000
#define CNA_CVT_CON0_CVT_TRUNCATE_3__SHIFT 22
#define CNA_CVT_CON0_CVT_TRUNCATE_2__MASK 0x003f0000
#define CNA_CVT_CON0_CVT_TRUNCATE_2__SHIFT 16
#define CNA_CVT_CON0_CVT_TRUNCATE_1__MASK 0x0000fc00
#define CNA_CVT_CON0_CVT_TRUNCATE_1__SHIFT 10
#define CNA_CVT_CON0_CVT_TRUNCATE_0__MASK 0x000003f0
#define CNA_CVT_CON0_CVT_TRUNCATE_0__SHIFT 4
#define CNA_CVT_CON0_DATA_SIGN__MASK 0x00000008
#define CNA_CVT_CON0_DATA_SIGN__SHIFT 3
#define CNA_CVT_CON0_ROUND_TYPE__MASK 0x00000004
#define CNA_CVT_CON0_ROUND_TYPE__SHIFT 2
#define CNA_CVT_CON0_CVT_TYPE__MASK 0x00000002
#define CNA_CVT_CON0_CVT_TYPE__SHIFT 1
#define CNA_CVT_CON0_CVT_BYPASS__MASK 0x00000001
#define CNA_CVT_CON0_CVT_BYPASS__SHIFT 0
#define REG_CNA_CVT_CON1 0x00001050
#define CNA_CVT_CON1_CVT_SCALE0__MASK 0xffff0000
#define CNA_CVT_CON1_CVT_SCALE0__SHIFT 16
#define CNA_CVT_CON1_CVT_OFFSET0__MASK 0x0000ffff
#define CNA_CVT_CON1_CVT_OFFSET0__SHIFT 0
#define REG_CNA_CVT_CON2 0x00001054
#define CNA_CVT_CON2_CVT_SCALE1__MASK 0xffff0000
#define CNA_CVT_CON2_CVT_SCALE1__SHIFT 16
#define CNA_CVT_CON2_CVT_OFFSET1__MASK 0x0000ffff
#define CNA_CVT_CON2_CVT_OFFSET1__SHIFT 0
#define REG_CNA_CVT_CON3 0x00001058
#define CNA_CVT_CON3_CVT_SCALE2__MASK 0xffff0000
#define CNA_CVT_CON3_CVT_SCALE2__SHIFT 16
#define CNA_CVT_CON3_CVT_OFFSET2__MASK 0x0000ffff
#define CNA_CVT_CON3_CVT_OFFSET2__SHIFT 0
#define REG_CNA_CVT_CON4 0x0000105c
#define CNA_CVT_CON4_CVT_SCALE3__MASK 0xffff0000
#define CNA_CVT_CON4_CVT_SCALE3__SHIFT 16
#define CNA_CVT_CON4_CVT_OFFSET3__MASK 0x0000ffff
#define CNA_CVT_CON4_CVT_OFFSET3__SHIFT 0
#define REG_CNA_FC_CON0 0x00001060
#define CNA_FC_CON0_FC_SKIP_DATA__MASK 0xffff0000
#define CNA_FC_CON0_FC_SKIP_DATA__SHIFT 16
#define CNA_FC_CON0_RESERVED_0__MASK 0x0000fffe
#define CNA_FC_CON0_RESERVED_0__SHIFT 1
#define CNA_FC_CON0_FC_SKIP_EN__MASK 0x00000001
#define CNA_FC_CON0_FC_SKIP_EN__SHIFT 0
#define REG_CNA_FC_CON1 0x00001064
#define CNA_FC_CON1_RESERVED_0__MASK 0xfffe0000
#define CNA_FC_CON1_RESERVED_0__SHIFT 17
#define CNA_FC_CON1_DATA_OFFSET__MASK 0x0001ffff
#define CNA_FC_CON1_DATA_OFFSET__SHIFT 0
#define REG_CNA_PAD_CON0 0x00001068
#define CNA_PAD_CON0_RESERVED_0__MASK 0xffffff00
#define CNA_PAD_CON0_RESERVED_0__SHIFT 8
#define CNA_PAD_CON0_PAD_LEFT__MASK 0x000000f0
#define CNA_PAD_CON0_PAD_LEFT__SHIFT 4
#define CNA_PAD_CON0_PAD_TOP__MASK 0x0000000f
#define CNA_PAD_CON0_PAD_TOP__SHIFT 0
#define REG_CNA_FEATURE_DATA_ADDR 0x00001070
#define CNA_FEATURE_DATA_ADDR_FEATURE_BASE_ADDR__MASK 0xffffffff
#define CNA_FEATURE_DATA_ADDR_FEATURE_BASE_ADDR__SHIFT 0
#define REG_CNA_FC_CON2 0x00001074
#define CNA_FC_CON2_RESERVED_0__MASK 0xfffe0000
#define CNA_FC_CON2_RESERVED_0__SHIFT 17
#define CNA_FC_CON2_WEIGHT_OFFSET__MASK 0x0001ffff
#define CNA_FC_CON2_WEIGHT_OFFSET__SHIFT 0
#define REG_CNA_DMA_CON0 0x00001078
#define CNA_DMA_CON0_OV4K_BYPASS__MASK 0x80000000
#define CNA_DMA_CON0_OV4K_BYPASS__SHIFT 31
#define CNA_DMA_CON0_RESERVED_0__MASK 0x7ff00000
#define CNA_DMA_CON0_RESERVED_0__SHIFT 20
#define CNA_DMA_CON0_WEIGHT_BURST_LEN__MASK 0x000f0000
#define CNA_DMA_CON0_WEIGHT_BURST_LEN__SHIFT 16
#define CNA_DMA_CON0_RESERVED_1__MASK 0x0000fff0
#define CNA_DMA_CON0_RESERVED_1__SHIFT 4
#define CNA_DMA_CON0_DATA_BURST_LEN__MASK 0x0000000f
#define CNA_DMA_CON0_DATA_BURST_LEN__SHIFT 0
#define REG_CNA_DMA_CON1 0x0000107c
#define CNA_DMA_CON1_RESERVED_0__MASK 0xf0000000
#define CNA_DMA_CON1_RESERVED_0__SHIFT 28
#define CNA_DMA_CON1_LINE_STRIDE__MASK 0x0fffffff
#define CNA_DMA_CON1_LINE_STRIDE__SHIFT 0
#define REG_CNA_DMA_CON2 0x00001080
#define CNA_DMA_CON2_RESERVED_0__MASK 0xf0000000
#define CNA_DMA_CON2_RESERVED_0__SHIFT 28
#define CNA_DMA_CON2_SURF_STRIDE__MASK 0x0fffffff
#define CNA_DMA_CON2_SURF_STRIDE__SHIFT 0
#define REG_CNA_FC_DATA_SIZE0 0x00001084
#define CNA_FC_DATA_SIZE0_RESERVED_0__MASK 0xc0000000
#define CNA_FC_DATA_SIZE0_RESERVED_0__SHIFT 30
#define CNA_FC_DATA_SIZE0_DMA_WIDTH__MASK 0x3fff0000
#define CNA_FC_DATA_SIZE0_DMA_WIDTH__SHIFT 16
#define CNA_FC_DATA_SIZE0_RESERVED_1__MASK 0x0000f800
#define CNA_FC_DATA_SIZE0_RESERVED_1__SHIFT 11
#define CNA_FC_DATA_SIZE0_DMA_HEIGHT__MASK 0x000007ff
#define CNA_FC_DATA_SIZE0_DMA_HEIGHT__SHIFT 0
#define REG_CNA_FC_DATA_SIZE1 0x00001088
#define CNA_FC_DATA_SIZE1_RESERVED_0__MASK 0xffff0000
#define CNA_FC_DATA_SIZE1_RESERVED_0__SHIFT 16
#define CNA_FC_DATA_SIZE1_DMA_CHANNEL__MASK 0x0000ffff
#define CNA_FC_DATA_SIZE1_DMA_CHANNEL__SHIFT 0
#define REG_CNA_CLK_GATE 0x00001090
#define CNA_CLK_GATE_RESERVED_0__MASK 0xffffffe0
#define CNA_CLK_GATE_RESERVED_0__SHIFT 5
#define CNA_CLK_GATE_CBUF_CS_DISABLE_CLKGATE__MASK 0x00000010
#define CNA_CLK_GATE_CBUF_CS_DISABLE_CLKGATE__SHIFT 4
#define CNA_CLK_GATE_RESERVED_1__MASK 0x00000008
#define CNA_CLK_GATE_RESERVED_1__SHIFT 3
#define CNA_CLK_GATE_CSC_DISABLE_CLKGATE__MASK 0x00000004
#define CNA_CLK_GATE_CSC_DISABLE_CLKGATE__SHIFT 2
#define CNA_CLK_GATE_CNA_WEIGHT_DISABLE_CLKGATE__MASK 0x00000002
#define CNA_CLK_GATE_CNA_WEIGHT_DISABLE_CLKGATE__SHIFT 1
#define CNA_CLK_GATE_CNA_FEATURE_DISABLE_CLKGATE__MASK 0x00000001
#define CNA_CLK_GATE_CNA_FEATURE_DISABLE_CLKGATE__SHIFT 0
#define REG_CNA_DCOMP_CTRL 0x00001100
#define CNA_DCOMP_CTRL_RESERVED_0__MASK 0xfffffff0
#define CNA_DCOMP_CTRL_RESERVED_0__SHIFT 4
#define CNA_DCOMP_CTRL_WT_DEC_BYPASS__MASK 0x00000008
#define CNA_DCOMP_CTRL_WT_DEC_BYPASS__SHIFT 3
#define CNA_DCOMP_CTRL_DECOMP_CONTROL__MASK 0x00000007
#define CNA_DCOMP_CTRL_DECOMP_CONTROL__SHIFT 0
#define REG_CNA_DCOMP_REGNUM 0x00001104
#define CNA_DCOMP_REGNUM_DCOMP_REGNUM__MASK 0xffffffff
#define CNA_DCOMP_REGNUM_DCOMP_REGNUM__SHIFT 0
#define REG_CNA_DCOMP_ADDR0 0x00001110
#define CNA_DCOMP_ADDR0_DECOMPRESS_ADDR0__MASK 0xffffffff
#define CNA_DCOMP_ADDR0_DECOMPRESS_ADDR0__SHIFT 0
#define REG_CNA_DCOMP_AMOUNT0 0x00001140
#define CNA_DCOMP_AMOUNT0_DCOMP_AMOUNT0__MASK 0xffffffff
#define CNA_DCOMP_AMOUNT0_DCOMP_AMOUNT0__SHIFT 0
#define REG_CNA_DCOMP_AMOUNT1 0x00001144
#define CNA_DCOMP_AMOUNT1_DCOMP_AMOUNT1__MASK 0xffffffff
#define CNA_DCOMP_AMOUNT1_DCOMP_AMOUNT1__SHIFT 0
#define REG_CNA_DCOMP_AMOUNT2 0x00001148
#define CNA_DCOMP_AMOUNT2_DCOMP_AMOUNT2__MASK 0xffffffff
#define CNA_DCOMP_AMOUNT2_DCOMP_AMOUNT2__SHIFT 0
#define REG_CNA_DCOMP_AMOUNT3 0x0000114c
#define CNA_DCOMP_AMOUNT3_DCOMP_AMOUNT3__MASK 0xffffffff
#define CNA_DCOMP_AMOUNT3_DCOMP_AMOUNT3__SHIFT 0
#define REG_CNA_DCOMP_AMOUNT4 0x00001150
#define CNA_DCOMP_AMOUNT4_DCOMP_AMOUNT4__MASK 0xffffffff
#define CNA_DCOMP_AMOUNT4_DCOMP_AMOUNT4__SHIFT 0
#define REG_CNA_DCOMP_AMOUNT5 0x00001154
#define CNA_DCOMP_AMOUNT5_DCOMP_AMOUNT5__MASK 0xffffffff
#define CNA_DCOMP_AMOUNT5_DCOMP_AMOUNT5__SHIFT 0
#define REG_CNA_DCOMP_AMOUNT6 0x00001158
#define CNA_DCOMP_AMOUNT6_DCOMP_AMOUNT6__MASK 0xffffffff
#define CNA_DCOMP_AMOUNT6_DCOMP_AMOUNT6__SHIFT 0
#define REG_CNA_DCOMP_AMOUNT7 0x0000115c
#define CNA_DCOMP_AMOUNT7_DCOMP_AMOUNT7__MASK 0xffffffff
#define CNA_DCOMP_AMOUNT7_DCOMP_AMOUNT7__SHIFT 0
#define REG_CNA_DCOMP_AMOUNT8 0x00001160
#define CNA_DCOMP_AMOUNT8_DCOMP_AMOUNT8__MASK 0xffffffff
#define CNA_DCOMP_AMOUNT8_DCOMP_AMOUNT8__SHIFT 0
#define REG_CNA_DCOMP_AMOUNT9 0x00001164
#define CNA_DCOMP_AMOUNT9_DCOMP_AMOUNT9__MASK 0xffffffff
#define CNA_DCOMP_AMOUNT9_DCOMP_AMOUNT9__SHIFT 0
#define REG_CNA_DCOMP_AMOUNT10 0x00001168
#define CNA_DCOMP_AMOUNT10_DCOMP_AMOUNT10__MASK 0xffffffff
#define CNA_DCOMP_AMOUNT10_DCOMP_AMOUNT10__SHIFT 0
#define REG_CNA_DCOMP_AMOUNT11 0x0000116c
#define CNA_DCOMP_AMOUNT11_DCOMP_AMOUNT11__MASK 0xffffffff
#define CNA_DCOMP_AMOUNT11_DCOMP_AMOUNT11__SHIFT 0
#define REG_CNA_DCOMP_AMOUNT12 0x00001170
#define CNA_DCOMP_AMOUNT12_DCOMP_AMOUNT12__MASK 0xffffffff
#define CNA_DCOMP_AMOUNT12_DCOMP_AMOUNT12__SHIFT 0
#define REG_CNA_DCOMP_AMOUNT13 0x00001174
#define CNA_DCOMP_AMOUNT13_DCOMP_AMOUNT13__MASK 0xffffffff
#define CNA_DCOMP_AMOUNT13_DCOMP_AMOUNT13__SHIFT 0
#define REG_CNA_DCOMP_AMOUNT14 0x00001178
#define CNA_DCOMP_AMOUNT14_DCOMP_AMOUNT14__MASK 0xffffffff
#define CNA_DCOMP_AMOUNT14_DCOMP_AMOUNT14__SHIFT 0
#define REG_CNA_DCOMP_AMOUNT15 0x0000117c
#define CNA_DCOMP_AMOUNT15_DCOMP_AMOUNT15__MASK 0xffffffff
#define CNA_DCOMP_AMOUNT15_DCOMP_AMOUNT15__SHIFT 0
#define REG_CNA_CVT_CON5 0x00001180
#define CNA_CVT_CON5_PER_CHANNEL_CVT_EN__MASK 0xffffffff
#define CNA_CVT_CON5_PER_CHANNEL_CVT_EN__SHIFT 0
#define REG_CNA_PAD_CON1 0x00001184
#define CNA_PAD_CON1_PAD_VALUE__MASK 0xffffffff
#define CNA_PAD_CON1_PAD_VALUE__SHIFT 0
#define REG_DPU_S_POINTER 0x00004004
#define DPU_S_POINTER_RESERVED_0__MASK 0xfffe0000
#define DPU_S_POINTER_RESERVED_0__SHIFT 17
#define DPU_S_POINTER_EXECUTER__MASK 0x00010000
#define DPU_S_POINTER_EXECUTER__SHIFT 16
#define DPU_S_POINTER_RESERVED_1__MASK 0x0000ffc0
#define DPU_S_POINTER_RESERVED_1__SHIFT 6
#define DPU_S_POINTER_EXECUTER_PP_CLEAR__MASK 0x00000020
#define DPU_S_POINTER_EXECUTER_PP_CLEAR__SHIFT 5
#define DPU_S_POINTER_POINTER_PP_CLEAR__MASK 0x00000010
#define DPU_S_POINTER_POINTER_PP_CLEAR__SHIFT 4
#define DPU_S_POINTER_POINTER_PP_MODE__MASK 0x00000008
#define DPU_S_POINTER_POINTER_PP_MODE__SHIFT 3
#define DPU_S_POINTER_EXECUTER_PP_EN__MASK 0x00000004
#define DPU_S_POINTER_EXECUTER_PP_EN__SHIFT 2
#define DPU_S_POINTER_POINTER_PP_EN__MASK 0x00000002
#define DPU_S_POINTER_POINTER_PP_EN__SHIFT 1
#define DPU_S_POINTER_POINTER__MASK 0x00000001
#define DPU_S_POINTER_POINTER__SHIFT 0
#define REG_DPU_RDMA_RDMA_S_STATUS 0x00005000
#define DPU_RDMA_RDMA_S_STATUS_RESERVED_0__MASK 0xfffc0000
#define DPU_RDMA_RDMA_S_STATUS_RESERVED_0__SHIFT 18
#define DPU_RDMA_RDMA_S_STATUS_STATUS_1__MASK 0x00030000
#define DPU_RDMA_RDMA_S_STATUS_STATUS_1__SHIFT 16
#define DPU_RDMA_RDMA_S_STATUS_RESERVED_1__MASK 0x0000fffc
#define DPU_RDMA_RDMA_S_STATUS_RESERVED_1__SHIFT 2
#define DPU_RDMA_RDMA_S_STATUS_STATUS_0__MASK 0x00000003
#define DPU_RDMA_RDMA_S_STATUS_STATUS_0__SHIFT 0
#define REG_DPU_RDMA_RDMA_S_POINTER 0x00005004
#define DPU_RDMA_RDMA_S_POINTER_RESERVED_0__MASK 0xfffe0000
#define DPU_RDMA_RDMA_S_POINTER_RESERVED_0__SHIFT 17
#define DPU_RDMA_RDMA_S_POINTER_EXECUTER__MASK 0x00010000
#define DPU_RDMA_RDMA_S_POINTER_EXECUTER__SHIFT 16
#define DPU_RDMA_RDMA_S_POINTER_RESERVED_1__MASK 0x0000ffc0
#define DPU_RDMA_RDMA_S_POINTER_RESERVED_1__SHIFT 6
#define DPU_RDMA_RDMA_S_POINTER_EXECUTER_PP_CLEAR__MASK 0x00000020
#define DPU_RDMA_RDMA_S_POINTER_EXECUTER_PP_CLEAR__SHIFT 5
#define DPU_RDMA_RDMA_S_POINTER_POINTER_PP_CLEAR__MASK 0x00000010
#define DPU_RDMA_RDMA_S_POINTER_POINTER_PP_CLEAR__SHIFT 4
#define DPU_RDMA_RDMA_S_POINTER_POINTER_PP_MODE__MASK 0x00000008
#define DPU_RDMA_RDMA_S_POINTER_POINTER_PP_MODE__SHIFT 3
#define DPU_RDMA_RDMA_S_POINTER_EXECUTER_PP_EN__MASK 0x00000004
#define DPU_RDMA_RDMA_S_POINTER_EXECUTER_PP_EN__SHIFT 2
#define DPU_RDMA_RDMA_S_POINTER_POINTER_PP_EN__MASK 0x00000002
#define DPU_RDMA_RDMA_S_POINTER_POINTER_PP_EN__SHIFT 1
#define DPU_RDMA_RDMA_S_POINTER_POINTER__MASK 0x00000001
#define DPU_RDMA_RDMA_S_POINTER_POINTER__SHIFT 0
#define REG_DPU_RDMA_RDMA_OPERATION_ENABLE 0x00005008
#define DPU_RDMA_RDMA_OPERATION_ENABLE_RESERVED_0__MASK 0xfffffffe
#define DPU_RDMA_RDMA_OPERATION_ENABLE_RESERVED_0__SHIFT 1
#define DPU_RDMA_RDMA_OPERATION_ENABLE_OP_EN__MASK 0x00000001
#define DPU_RDMA_RDMA_OPERATION_ENABLE_OP_EN__SHIFT 0
#define REG_DPU_RDMA_RDMA_DATA_CUBE_WIDTH 0x0000500c
#define DPU_RDMA_RDMA_DATA_CUBE_WIDTH_RESERVED_0__MASK 0xffffe000
#define DPU_RDMA_RDMA_DATA_CUBE_WIDTH_RESERVED_0__SHIFT 13
#define DPU_RDMA_RDMA_DATA_CUBE_WIDTH_WIDTH__MASK 0x00001fff
#define DPU_RDMA_RDMA_DATA_CUBE_WIDTH_WIDTH__SHIFT 0
#define REG_DPU_RDMA_RDMA_DATA_CUBE_HEIGHT 0x00005010
#define DPU_RDMA_RDMA_DATA_CUBE_HEIGHT_RESERVED_0__MASK 0xe0000000
#define DPU_RDMA_RDMA_DATA_CUBE_HEIGHT_RESERVED_0__SHIFT 29
#define DPU_RDMA_RDMA_DATA_CUBE_HEIGHT_EW_LINE_NOTCH_ADDR__MASK 0x1fff0000
#define DPU_RDMA_RDMA_DATA_CUBE_HEIGHT_EW_LINE_NOTCH_ADDR__SHIFT 16
#define DPU_RDMA_RDMA_DATA_CUBE_HEIGHT_RESERVED_1__MASK 0x0000e000
#define DPU_RDMA_RDMA_DATA_CUBE_HEIGHT_RESERVED_1__SHIFT 13
#define DPU_RDMA_RDMA_DATA_CUBE_HEIGHT_HEIGHT__MASK 0x00001fff
#define DPU_RDMA_RDMA_DATA_CUBE_HEIGHT_HEIGHT__SHIFT 0
#define REG_DPU_RDMA_RDMA_DATA_CUBE_CHANNEL 0x00005014
#define DPU_RDMA_RDMA_DATA_CUBE_CHANNEL_RESERVED_0__MASK 0xffffe000
#define DPU_RDMA_RDMA_DATA_CUBE_CHANNEL_RESERVED_0__SHIFT 13
#define DPU_RDMA_RDMA_DATA_CUBE_CHANNEL_CHANNEL__MASK 0x00001fff
#define DPU_RDMA_RDMA_DATA_CUBE_CHANNEL_CHANNEL__SHIFT 0
#define REG_DPU_RDMA_RDMA_SRC_BASE_ADDR 0x00005018
#define DPU_RDMA_RDMA_SRC_BASE_ADDR_SRC_BASE_ADDR__MASK 0xffffffff
#define DPU_RDMA_RDMA_SRC_BASE_ADDR_SRC_BASE_ADDR__SHIFT 0
#define REG_DPU_RDMA_RDMA_BRDMA_CFG 0x0000501c
#define DPU_RDMA_RDMA_BRDMA_CFG_RESERVED_0__MASK 0xffffffe0
#define DPU_RDMA_RDMA_BRDMA_CFG_RESERVED_0__SHIFT 5
#define DPU_RDMA_RDMA_BRDMA_CFG_BRDMA_DATA_USE__MASK 0x0000001e
#define DPU_RDMA_RDMA_BRDMA_CFG_BRDMA_DATA_USE__SHIFT 1
#define DPU_RDMA_RDMA_BRDMA_CFG_RESERVED_1__MASK 0x00000001
#define DPU_RDMA_RDMA_BRDMA_CFG_RESERVED_1__SHIFT 0
#define REG_DPU_RDMA_RDMA_BS_BASE_ADDR 0x00005020
#define DPU_RDMA_RDMA_BS_BASE_ADDR_BS_BASE_ADDR__MASK 0xffffffff
#define DPU_RDMA_RDMA_BS_BASE_ADDR_BS_BASE_ADDR__SHIFT 0
#define REG_DPU_RDMA_RDMA_NRDMA_CFG 0x00005028
#define DPU_RDMA_RDMA_NRDMA_CFG_RESERVED_0__MASK 0xffffffe0
#define DPU_RDMA_RDMA_NRDMA_CFG_RESERVED_0__SHIFT 5
#define DPU_RDMA_RDMA_NRDMA_CFG_NRDMA_DATA_USE__MASK 0x0000001e
#define DPU_RDMA_RDMA_NRDMA_CFG_NRDMA_DATA_USE__SHIFT 1
#define DPU_RDMA_RDMA_NRDMA_CFG_RESERVED_1__MASK 0x00000001
#define DPU_RDMA_RDMA_NRDMA_CFG_RESERVED_1__SHIFT 0
#define REG_DPU_RDMA_RDMA_BN_BASE_ADDR 0x0000502c
#define DPU_RDMA_RDMA_BN_BASE_ADDR_BN_BASE_ADDR__MASK 0xffffffff
#define DPU_RDMA_RDMA_BN_BASE_ADDR_BN_BASE_ADDR__SHIFT 0
#define REG_DPU_RDMA_RDMA_ERDMA_CFG 0x00005034
#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_DATA_MODE__MASK 0xc0000000
#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_DATA_MODE__SHIFT 30
#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_SURF_MODE__MASK 0x20000000
#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_SURF_MODE__SHIFT 29
#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_NONALIGN__MASK 0x10000000
#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_NONALIGN__SHIFT 28
#define DPU_RDMA_RDMA_ERDMA_CFG_RESERVED_0__MASK 0x0ffffff0
#define DPU_RDMA_RDMA_ERDMA_CFG_RESERVED_0__SHIFT 4
#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_DATA_SIZE__MASK 0x0000000c
#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_DATA_SIZE__SHIFT 2
#define DPU_RDMA_RDMA_ERDMA_CFG_OV4K_BYPASS__MASK 0x00000002
#define DPU_RDMA_RDMA_ERDMA_CFG_OV4K_BYPASS__SHIFT 1
#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_DISABLE__MASK 0x00000001
#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_DISABLE__SHIFT 0
#define REG_DPU_RDMA_RDMA_EW_BASE_ADDR 0x00005038
#define DPU_RDMA_RDMA_EW_BASE_ADDR_EW_BASE_ADDR__MASK 0xffffffff
#define DPU_RDMA_RDMA_EW_BASE_ADDR_EW_BASE_ADDR__SHIFT 0
#define REG_DPU_RDMA_RDMA_EW_SURF_STRIDE 0x00005040
#define DPU_RDMA_RDMA_EW_SURF_STRIDE_EW_SURF_STRIDE__MASK 0xfffffff0
#define DPU_RDMA_RDMA_EW_SURF_STRIDE_EW_SURF_STRIDE__SHIFT 4
#define DPU_RDMA_RDMA_EW_SURF_STRIDE_RESERVED_0__MASK 0x0000000f
#define DPU_RDMA_RDMA_EW_SURF_STRIDE_RESERVED_0__SHIFT 0
#define REG_DPU_RDMA_RDMA_FEATURE_MODE_CFG 0x00005044
#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_RESERVED_0__MASK 0xfffc0000
#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_RESERVED_0__SHIFT 18
#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_IN_PRECISION__MASK 0x00038000
#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_IN_PRECISION__SHIFT 15
#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_BURST_LEN__MASK 0x00007800
#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_BURST_LEN__SHIFT 11
#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_COMB_USE__MASK 0x00000700
#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_COMB_USE__SHIFT 8
#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_PROC_PRECISION__MASK 0x000000e0
#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_PROC_PRECISION__SHIFT 5
#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_MRDMA_DISABLE__MASK 0x00000010
#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_MRDMA_DISABLE__SHIFT 4
#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_MRDMA_FP16TOFP32_EN__MASK 0x00000008
#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_MRDMA_FP16TOFP32_EN__SHIFT 3
#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_CONV_MODE__MASK 0x00000006
#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_CONV_MODE__SHIFT 1
#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_FLYING_MODE__MASK 0x00000001
#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_FLYING_MODE__SHIFT 0
#define REG_DPU_RDMA_RDMA_SRC_DMA_CFG 0x00005048
#define DPU_RDMA_RDMA_SRC_DMA_CFG_LINE_NOTCH_ADDR__MASK 0xfff80000
#define DPU_RDMA_RDMA_SRC_DMA_CFG_LINE_NOTCH_ADDR__SHIFT 19
#define DPU_RDMA_RDMA_SRC_DMA_CFG_RESERVED_0__MASK 0x0007c000
#define DPU_RDMA_RDMA_SRC_DMA_CFG_RESERVED_0__SHIFT 14
#define DPU_RDMA_RDMA_SRC_DMA_CFG_POOLING_METHOD__MASK 0x00002000
#define DPU_RDMA_RDMA_SRC_DMA_CFG_POOLING_METHOD__SHIFT 13
#define DPU_RDMA_RDMA_SRC_DMA_CFG_UNPOOLING_EN__MASK 0x00001000
#define DPU_RDMA_RDMA_SRC_DMA_CFG_UNPOOLING_EN__SHIFT 12
#define DPU_RDMA_RDMA_SRC_DMA_CFG_KERNEL_STRIDE_HEIGHT__MASK 0x00000e00
#define DPU_RDMA_RDMA_SRC_DMA_CFG_KERNEL_STRIDE_HEIGHT__SHIFT 9
#define DPU_RDMA_RDMA_SRC_DMA_CFG_KERNEL_STRIDE_WIDTH__MASK 0x000001c0
#define DPU_RDMA_RDMA_SRC_DMA_CFG_KERNEL_STRIDE_WIDTH__SHIFT 6
#define DPU_RDMA_RDMA_SRC_DMA_CFG_KERNEL_HEIGHT__MASK 0x00000038
#define DPU_RDMA_RDMA_SRC_DMA_CFG_KERNEL_HEIGHT__SHIFT 3
#define DPU_RDMA_RDMA_SRC_DMA_CFG_KERNEL_WIDTH__MASK 0x00000007
#define DPU_RDMA_RDMA_SRC_DMA_CFG_KERNEL_WIDTH__SHIFT 0
#define REG_DPU_RDMA_RDMA_SURF_NOTCH 0x0000504c
#define DPU_RDMA_RDMA_SURF_NOTCH_SURF_NOTCH_ADDR__MASK 0xfffffff0
#define DPU_RDMA_RDMA_SURF_NOTCH_SURF_NOTCH_ADDR__SHIFT 4
#define DPU_RDMA_RDMA_SURF_NOTCH_RESERVED_0__MASK 0x0000000f
#define DPU_RDMA_RDMA_SURF_NOTCH_RESERVED_0__SHIFT 0
#define REG_DPU_RDMA_RDMA_PAD_CFG 0x00005064
#define DPU_RDMA_RDMA_PAD_CFG_PAD_VALUE__MASK 0xffff0000
#define DPU_RDMA_RDMA_PAD_CFG_PAD_VALUE__SHIFT 16
#define DPU_RDMA_RDMA_PAD_CFG_RESERVED_0__MASK 0x0000ff80
#define DPU_RDMA_RDMA_PAD_CFG_RESERVED_0__SHIFT 7
#define DPU_RDMA_RDMA_PAD_CFG_PAD_TOP__MASK 0x00000070
#define DPU_RDMA_RDMA_PAD_CFG_PAD_TOP__SHIFT 4
#define DPU_RDMA_RDMA_PAD_CFG_RESERVED_1__MASK 0x00000008
#define DPU_RDMA_RDMA_PAD_CFG_RESERVED_1__SHIFT 3
#define DPU_RDMA_RDMA_PAD_CFG_PAD_LEFT__MASK 0x00000007
#define DPU_RDMA_RDMA_PAD_CFG_PAD_LEFT__SHIFT 0
#define REG_DPU_RDMA_RDMA_WEIGHT 0x00005068
#define DPU_RDMA_RDMA_WEIGHT_E_WEIGHT__MASK 0xff000000
#define DPU_RDMA_RDMA_WEIGHT_E_WEIGHT__SHIFT 24
#define DPU_RDMA_RDMA_WEIGHT_N_WEIGHT__MASK 0x00ff0000
#define DPU_RDMA_RDMA_WEIGHT_N_WEIGHT__SHIFT 16
#define DPU_RDMA_RDMA_WEIGHT_B_WEIGHT__MASK 0x0000ff00
#define DPU_RDMA_RDMA_WEIGHT_B_WEIGHT__SHIFT 8
#define DPU_RDMA_RDMA_WEIGHT_M_WEIGHT__MASK 0x000000ff
#define DPU_RDMA_RDMA_WEIGHT_M_WEIGHT__SHIFT 0
#define REG_DPU_RDMA_RDMA_EW_SURF_NOTCH 0x0000506c
#define DPU_RDMA_RDMA_EW_SURF_NOTCH_EW_SURF_NOTCH__MASK 0xfffffff0
#define DPU_RDMA_RDMA_EW_SURF_NOTCH_EW_SURF_NOTCH__SHIFT 4
#define DPU_RDMA_RDMA_EW_SURF_NOTCH_RESERVED_0__MASK 0x0000000f
#define DPU_RDMA_RDMA_EW_SURF_NOTCH_RESERVED_0__SHIFT 0
#endif
ggml/src/ggml-rocket/coredpu_defs.h
+769
View File
@@ -0,0 +1,769 @@
#ifndef COREDPU_DEFS_H
#define COREDPU_DEFS_H
#define REG_PC_VERSION 0x00000000
#define PC_VERSION_VERSION__MASK 0xffffffff
#define PC_VERSION_VERSION__SHIFT 0
#define REG_PC_VERSION_NUM 0x00000004
#define PC_VERSION_NUM_VERSION_NUM__MASK 0xffffffff
#define PC_VERSION_NUM_VERSION_NUM__SHIFT 0
#define REG_PC_OPERATION_ENABLE 0x00000008
#define PC_OPERATION_ENABLE_RESERVED_0__MASK 0xfffffffe
#define PC_OPERATION_ENABLE_RESERVED_0__SHIFT 1
#define PC_OPERATION_ENABLE_OP_EN__MASK 0x00000001
#define PC_OPERATION_ENABLE_OP_EN__SHIFT 0
#define REG_PC_BASE_ADDRESS 0x00000010
#define PC_BASE_ADDRESS_PC_SOURCE_ADDR__MASK 0xfffffff0
#define PC_BASE_ADDRESS_PC_SOURCE_ADDR__SHIFT 4
#define PC_BASE_ADDRESS_RESERVED_0__MASK 0x0000000e
#define PC_BASE_ADDRESS_RESERVED_0__SHIFT 1
#define PC_BASE_ADDRESS_PC_SEL__MASK 0x00000001
#define PC_BASE_ADDRESS_PC_SEL__SHIFT 0
#define REG_PC_REGISTER_AMOUNTS 0x00000014
#define PC_REGISTER_AMOUNTS_RESERVED_0__MASK 0xffff0000
#define PC_REGISTER_AMOUNTS_RESERVED_0__SHIFT 16
#define PC_REGISTER_AMOUNTS_PC_DATA_AMOUNT__MASK 0x0000ffff
#define PC_REGISTER_AMOUNTS_PC_DATA_AMOUNT__SHIFT 0
#define REG_PC_INTERRUPT_MASK 0x00000020
#define PC_INTERRUPT_MASK_RESERVED_0__MASK 0xffffc000
#define PC_INTERRUPT_MASK_RESERVED_0__SHIFT 14
#define PC_INTERRUPT_MASK_DMA_WRITE_ERROR 0x00002000
#define PC_INTERRUPT_MASK_DMA_READ_ERROR 0x00001000
#define PC_INTERRUPT_MASK_PPU_1 0x00000800
#define PC_INTERRUPT_MASK_PPU_0 0x00000400
#define PC_INTERRUPT_MASK_DPU_1 0x00000200
#define PC_INTERRUPT_MASK_DPU_0 0x00000100
#define PC_INTERRUPT_MASK_CORE_1 0x00000080
#define PC_INTERRUPT_MASK_CORE_0 0x00000040
#define PC_INTERRUPT_MASK_CNA_CSC_1 0x00000020
#define PC_INTERRUPT_MASK_CNA_CSC_0 0x00000010
#define PC_INTERRUPT_MASK_CNA_WEIGHT_1 0x00000008
#define PC_INTERRUPT_MASK_CNA_WEIGHT_0 0x00000004
#define PC_INTERRUPT_MASK_CNA_FEATURE_1 0x00000002
#define PC_INTERRUPT_MASK_CNA_FEATURE_0 0x00000001
#define REG_PC_INTERRUPT_CLEAR 0x00000024
#define PC_INTERRUPT_CLEAR_RESERVED_0__MASK 0xffffc000
#define PC_INTERRUPT_CLEAR_RESERVED_0__SHIFT 14
#define PC_INTERRUPT_CLEAR_DMA_WRITE_ERROR 0x00002000
#define PC_INTERRUPT_CLEAR_DMA_READ_ERROR 0x00001000
#define PC_INTERRUPT_CLEAR_PPU_1 0x00000800
#define PC_INTERRUPT_CLEAR_PPU_0 0x00000400
#define PC_INTERRUPT_CLEAR_DPU_1 0x00000200
#define PC_INTERRUPT_CLEAR_DPU_0 0x00000100
#define PC_INTERRUPT_CLEAR_CORE_1 0x00000080
#define PC_INTERRUPT_CLEAR_CORE_0 0x00000040
#define PC_INTERRUPT_CLEAR_CNA_CSC_1 0x00000020
#define PC_INTERRUPT_CLEAR_CNA_CSC_0 0x00000010
#define PC_INTERRUPT_CLEAR_CNA_WEIGHT_1 0x00000008
#define PC_INTERRUPT_CLEAR_CNA_WEIGHT_0 0x00000004
#define PC_INTERRUPT_CLEAR_CNA_FEATURE_1 0x00000002
#define PC_INTERRUPT_CLEAR_CNA_FEATURE_0 0x00000001
#define REG_PC_INTERRUPT_STATUS 0x00000028
#define PC_INTERRUPT_STATUS_RESERVED_0__MASK 0xffffc000
#define PC_INTERRUPT_STATUS_RESERVED_0__SHIFT 14
#define PC_INTERRUPT_STATUS_DMA_WRITE_ERROR 0x00002000
#define PC_INTERRUPT_STATUS_DMA_READ_ERROR 0x00001000
#define PC_INTERRUPT_STATUS_PPU_1 0x00000800
#define PC_INTERRUPT_STATUS_PPU_0 0x00000400
#define PC_INTERRUPT_STATUS_DPU_1 0x00000200
#define PC_INTERRUPT_STATUS_DPU_0 0x00000100
#define PC_INTERRUPT_STATUS_CORE_1 0x00000080
#define PC_INTERRUPT_STATUS_CORE_0 0x00000040
#define PC_INTERRUPT_STATUS_CNA_CSC_1 0x00000020
#define PC_INTERRUPT_STATUS_CNA_CSC_0 0x00000010
#define PC_INTERRUPT_STATUS_CNA_WEIGHT_1 0x00000008
#define PC_INTERRUPT_STATUS_CNA_WEIGHT_0 0x00000004
#define PC_INTERRUPT_STATUS_CNA_FEATURE_1 0x00000002
#define PC_INTERRUPT_STATUS_CNA_FEATURE_0 0x00000001
#define REG_PC_INTERRUPT_RAW_STATUS 0x0000002c
#define PC_INTERRUPT_RAW_STATUS_RESERVED_0__MASK 0xffffc000
#define PC_INTERRUPT_RAW_STATUS_RESERVED_0__SHIFT 14
#define PC_INTERRUPT_RAW_STATUS_DMA_WRITE_ERROR 0x00002000
#define PC_INTERRUPT_RAW_STATUS_DMA_READ_ERROR 0x00001000
#define PC_INTERRUPT_RAW_STATUS_PPU_1 0x00000800
#define PC_INTERRUPT_RAW_STATUS_PPU_0 0x00000400
#define PC_INTERRUPT_RAW_STATUS_DPU_1 0x00000200
#define PC_INTERRUPT_RAW_STATUS_DPU_0 0x00000100
#define PC_INTERRUPT_RAW_STATUS_CORE_1 0x00000080
#define PC_INTERRUPT_RAW_STATUS_CORE_0 0x00000040
#define PC_INTERRUPT_RAW_STATUS_CNA_CSC_1 0x00000020
#define PC_INTERRUPT_RAW_STATUS_CNA_CSC_0 0x00000010
#define PC_INTERRUPT_RAW_STATUS_CNA_WEIGHT_1 0x00000008
#define PC_INTERRUPT_RAW_STATUS_CNA_WEIGHT_0 0x00000004
#define PC_INTERRUPT_RAW_STATUS_CNA_FEATURE_1 0x00000002
#define PC_INTERRUPT_RAW_STATUS_CNA_FEATURE_0 0x00000001
#define REG_PC_TASK_CON 0x00000030
#define PC_TASK_CON_RESERVED_0__MASK 0xffffc000
#define PC_TASK_CON_RESERVED_0__SHIFT 14
#define PC_TASK_CON_TASK_COUNT_CLEAR__MASK 0x00002000
#define PC_TASK_CON_TASK_COUNT_CLEAR__SHIFT 13
#define PC_TASK_CON_TASK_PP_EN__MASK 0x00001000
#define PC_TASK_CON_TASK_PP_EN__SHIFT 12
#define PC_TASK_CON_TASK_NUMBER__MASK 0x00000fff
#define PC_TASK_CON_TASK_NUMBER__SHIFT 0
#define REG_PC_TASK_DMA_BASE_ADDR 0x00000034
#define PC_TASK_DMA_BASE_ADDR_DMA_BASE_ADDR__MASK 0xfffffff0
#define PC_TASK_DMA_BASE_ADDR_DMA_BASE_ADDR__SHIFT 4
#define PC_TASK_DMA_BASE_ADDR_RESERVED_0__MASK 0x0000000f
#define PC_TASK_DMA_BASE_ADDR_RESERVED_0__SHIFT 0
#define REG_PC_TASK_STATUS 0x0000003c
#define PC_TASK_STATUS_RESERVED_0__MASK 0xf0000000
#define PC_TASK_STATUS_RESERVED_0__SHIFT 28
#define PC_TASK_STATUS_TASK_STATUS__MASK 0x0fffffff
#define PC_TASK_STATUS_TASK_STATUS__SHIFT 0
#define REG_CORE_S_STATUS 0x00003000
#define CORE_S_STATUS_RESERVED_0__MASK 0xfffc0000
#define CORE_S_STATUS_RESERVED_0__SHIFT 18
#define CORE_S_STATUS_STATUS_1__MASK 0x00030000
#define CORE_S_STATUS_STATUS_1__SHIFT 16
#define CORE_S_STATUS_RESERVED_1__MASK 0x0000fffc
#define CORE_S_STATUS_RESERVED_1__SHIFT 2
#define CORE_S_STATUS_STATUS_0__MASK 0x00000003
#define CORE_S_STATUS_STATUS_0__SHIFT 0
#define REG_CORE_S_POINTER 0x00003004
#define CORE_S_POINTER_RESERVED_0__MASK 0xfffe0000
#define CORE_S_POINTER_RESERVED_0__SHIFT 17
#define CORE_S_POINTER_EXECUTER__MASK 0x00010000
#define CORE_S_POINTER_EXECUTER__SHIFT 16
#define CORE_S_POINTER_RESERVED_1__MASK 0x0000ffc0
#define CORE_S_POINTER_RESERVED_1__SHIFT 6
#define CORE_S_POINTER_EXECUTER_PP_CLEAR__MASK 0x00000020
#define CORE_S_POINTER_EXECUTER_PP_CLEAR__SHIFT 5
#define CORE_S_POINTER_POINTER_PP_CLEAR__MASK 0x00000010
#define CORE_S_POINTER_POINTER_PP_CLEAR__SHIFT 4
#define CORE_S_POINTER_POINTER_PP_MODE__MASK 0x00000008
#define CORE_S_POINTER_POINTER_PP_MODE__SHIFT 3
#define CORE_S_POINTER_EXECUTER_PP_EN__MASK 0x00000004
#define CORE_S_POINTER_EXECUTER_PP_EN__SHIFT 2
#define CORE_S_POINTER_POINTER_PP_EN__MASK 0x00000002
#define CORE_S_POINTER_POINTER_PP_EN__SHIFT 1
#define CORE_S_POINTER_POINTER__MASK 0x00000001
#define CORE_S_POINTER_POINTER__SHIFT 0
#define REG_CORE_OPERATION_ENABLE 0x00003008
#define CORE_OPERATION_ENABLE_RESERVED_0__MASK 0xfffffffe
#define CORE_OPERATION_ENABLE_RESERVED_0__SHIFT 1
#define CORE_OPERATION_ENABLE_OP_EN__MASK 0x00000001
#define CORE_OPERATION_ENABLE_OP_EN__SHIFT 0
#define REG_CORE_MAC_GATING 0x0000300c
#define CORE_MAC_GATING_RESERVED_0__MASK 0xf8000000
#define CORE_MAC_GATING_RESERVED_0__SHIFT 27
#define CORE_MAC_GATING_SLCG_OP_EN__MASK 0x07ffffff
#define CORE_MAC_GATING_SLCG_OP_EN__SHIFT 0
#define REG_CORE_MISC_CFG 0x00003010
#define CORE_MISC_CFG_RESERVED_0__MASK 0xfff00000
#define CORE_MISC_CFG_RESERVED_0__SHIFT 20
#define CORE_MISC_CFG_SOFT_GATING__MASK 0x000fc000
#define CORE_MISC_CFG_SOFT_GATING__SHIFT 14
#define CORE_MISC_CFG_RESERVED_1__MASK 0x00003800
#define CORE_MISC_CFG_RESERVED_1__SHIFT 11
#define CORE_MISC_CFG_PROC_PRECISION__MASK 0x00000700
#define CORE_MISC_CFG_PROC_PRECISION__SHIFT 8
#define CORE_MISC_CFG_RESERVED_2__MASK 0x000000fc
#define CORE_MISC_CFG_RESERVED_2__SHIFT 2
#define CORE_MISC_CFG_DW_EN__MASK 0x00000002
#define CORE_MISC_CFG_DW_EN__SHIFT 1
#define CORE_MISC_CFG_QD_EN__MASK 0x00000001
#define CORE_MISC_CFG_QD_EN__SHIFT 0
#define REG_CORE_DATAOUT_SIZE_0 0x00003014
#define CORE_DATAOUT_SIZE_0_DATAOUT_HEIGHT__MASK 0xffff0000
#define CORE_DATAOUT_SIZE_0_DATAOUT_HEIGHT__SHIFT 16
#define CORE_DATAOUT_SIZE_0_DATAOUT_WIDTH__MASK 0x0000ffff
#define CORE_DATAOUT_SIZE_0_DATAOUT_WIDTH__SHIFT 0
#define REG_CORE_DATAOUT_SIZE_1 0x00003018
#define CORE_DATAOUT_SIZE_1_RESERVED_0__MASK 0xffff0000
#define CORE_DATAOUT_SIZE_1_RESERVED_0__SHIFT 16
#define CORE_DATAOUT_SIZE_1_DATAOUT_CHANNEL__MASK 0x0000ffff
#define CORE_DATAOUT_SIZE_1_DATAOUT_CHANNEL__SHIFT 0
#define REG_CORE_CLIP_TRUNCATE 0x0000301c
#define CORE_CLIP_TRUNCATE_RESERVED_0__MASK 0xffffff80
#define CORE_CLIP_TRUNCATE_RESERVED_0__SHIFT 7
#define CORE_CLIP_TRUNCATE_ROUND_TYPE__MASK 0x00000040
#define CORE_CLIP_TRUNCATE_ROUND_TYPE__SHIFT 6
#define CORE_CLIP_TRUNCATE_RESERVED_1__MASK 0x00000020
#define CORE_CLIP_TRUNCATE_RESERVED_1__SHIFT 5
#define CORE_CLIP_TRUNCATE_CLIP_TRUNCATE__MASK 0x0000001f
#define CORE_CLIP_TRUNCATE_CLIP_TRUNCATE__SHIFT 0
#define REG_DPU_S_STATUS 0x00004000
#define DPU_S_STATUS_RESERVED_0__MASK 0xfffc0000
#define DPU_S_STATUS_RESERVED_0__SHIFT 18
#define DPU_S_STATUS_STATUS_1__MASK 0x00030000
#define DPU_S_STATUS_STATUS_1__SHIFT 16
#define DPU_S_STATUS_RESERVED_1__MASK 0x0000fffc
#define DPU_S_STATUS_RESERVED_1__SHIFT 2
#define DPU_S_STATUS_STATUS_0__MASK 0x00000003
#define DPU_S_STATUS_STATUS_0__SHIFT 0
#define REG_DPU_S_POINTER 0x00004004
#define DPU_S_POINTER_RESERVED_0__MASK 0xfffe0000
#define DPU_S_POINTER_RESERVED_0__SHIFT 17
#define DPU_S_POINTER_EXECUTER__MASK 0x00010000
#define DPU_S_POINTER_EXECUTER__SHIFT 16
#define DPU_S_POINTER_RESERVED_1__MASK 0x0000ffc0
#define DPU_S_POINTER_RESERVED_1__SHIFT 6
#define DPU_S_POINTER_EXECUTER_PP_CLEAR__MASK 0x00000020
#define DPU_S_POINTER_EXECUTER_PP_CLEAR__SHIFT 5
#define DPU_S_POINTER_POINTER_PP_CLEAR__MASK 0x00000010
#define DPU_S_POINTER_POINTER_PP_CLEAR__SHIFT 4
#define DPU_S_POINTER_POINTER_PP_MODE__MASK 0x00000008
#define DPU_S_POINTER_POINTER_PP_MODE__SHIFT 3
#define DPU_S_POINTER_EXECUTER_PP_EN__MASK 0x00000004
#define DPU_S_POINTER_EXECUTER_PP_EN__SHIFT 2
#define DPU_S_POINTER_POINTER_PP_EN__MASK 0x00000002
#define DPU_S_POINTER_POINTER_PP_EN__SHIFT 1
#define DPU_S_POINTER_POINTER__MASK 0x00000001
#define DPU_S_POINTER_POINTER__SHIFT 0
#define REG_DPU_OPERATION_ENABLE 0x00004008
#define DPU_OPERATION_ENABLE_RESERVED_0__MASK 0xfffffffe
#define DPU_OPERATION_ENABLE_RESERVED_0__SHIFT 1
#define DPU_OPERATION_ENABLE_OP_EN__MASK 0x00000001
#define DPU_OPERATION_ENABLE_OP_EN__SHIFT 0
#define REG_DPU_FEATURE_MODE_CFG 0x0000400c
#define DPU_FEATURE_MODE_CFG_COMB_USE__MASK 0x80000000
#define DPU_FEATURE_MODE_CFG_COMB_USE__SHIFT 31
#define DPU_FEATURE_MODE_CFG_TP_EN__MASK 0x40000000
#define DPU_FEATURE_MODE_CFG_TP_EN__SHIFT 30
#define DPU_FEATURE_MODE_CFG_RGP_TYPE__MASK 0x3c000000
#define DPU_FEATURE_MODE_CFG_RGP_TYPE__SHIFT 26
#define DPU_FEATURE_MODE_CFG_NONALIGN__MASK 0x02000000
#define DPU_FEATURE_MODE_CFG_NONALIGN__SHIFT 25
#define DPU_FEATURE_MODE_CFG_SURF_LEN__MASK 0x01fffe00
#define DPU_FEATURE_MODE_CFG_SURF_LEN__SHIFT 9
#define DPU_FEATURE_MODE_CFG_BURST_LEN__MASK 0x000001e0
#define DPU_FEATURE_MODE_CFG_BURST_LEN__SHIFT 5
#define DPU_FEATURE_MODE_CFG_CONV_MODE__MASK 0x00000018
#define DPU_FEATURE_MODE_CFG_CONV_MODE__SHIFT 3
#define DPU_FEATURE_MODE_CFG_OUTPUT_MODE__MASK 0x00000006
#define DPU_FEATURE_MODE_CFG_OUTPUT_MODE__SHIFT 1
#define DPU_FEATURE_MODE_CFG_FLYING_MODE__MASK 0x00000001
#define DPU_FEATURE_MODE_CFG_FLYING_MODE__SHIFT 0
#define REG_DPU_DATA_FORMAT 0x00004010
#define DPU_DATA_FORMAT_OUT_PRECISION__MASK 0xe0000000
#define DPU_DATA_FORMAT_OUT_PRECISION__SHIFT 29
#define DPU_DATA_FORMAT_IN_PRECISION__MASK 0x1c000000
#define DPU_DATA_FORMAT_IN_PRECISION__SHIFT 26
#define DPU_DATA_FORMAT_EW_TRUNCATE_NEG__MASK 0x03ff0000
#define DPU_DATA_FORMAT_EW_TRUNCATE_NEG__SHIFT 16
#define DPU_DATA_FORMAT_BN_MUL_SHIFT_VALUE_NEG__MASK 0x0000fc00
#define DPU_DATA_FORMAT_BN_MUL_SHIFT_VALUE_NEG__SHIFT 10
#define DPU_DATA_FORMAT_BS_MUL_SHIFT_VALUE_NEG__MASK 0x000003f0
#define DPU_DATA_FORMAT_BS_MUL_SHIFT_VALUE_NEG__SHIFT 4
#define DPU_DATA_FORMAT_MC_SURF_OUT__MASK 0x00000008
#define DPU_DATA_FORMAT_MC_SURF_OUT__SHIFT 3
#define DPU_DATA_FORMAT_PROC_PRECISION__MASK 0x00000007
#define DPU_DATA_FORMAT_PROC_PRECISION__SHIFT 0
#define REG_DPU_OFFSET_PEND 0x00004014
#define DPU_OFFSET_PEND_RESERVED_0__MASK 0xffff0000
#define DPU_OFFSET_PEND_RESERVED_0__SHIFT 16
#define DPU_OFFSET_PEND_OFFSET_PEND__MASK 0x0000ffff
#define DPU_OFFSET_PEND_OFFSET_PEND__SHIFT 0
#define REG_DPU_DST_BASE_ADDR 0x00004020
#define DPU_DST_BASE_ADDR_DST_BASE_ADDR__MASK 0xffffffff
#define DPU_DST_BASE_ADDR_DST_BASE_ADDR__SHIFT 0
#define REG_DPU_DST_SURF_STRIDE 0x00004024
#define DPU_DST_SURF_STRIDE_DST_SURF_STRIDE__MASK 0xfffffff0
#define DPU_DST_SURF_STRIDE_DST_SURF_STRIDE__SHIFT 4
#define DPU_DST_SURF_STRIDE_RESERVED_0__MASK 0x0000000f
#define DPU_DST_SURF_STRIDE_RESERVED_0__SHIFT 0
#define REG_DPU_DATA_CUBE_WIDTH 0x00004030
#define DPU_DATA_CUBE_WIDTH_RESERVED_0__MASK 0xffffe000
#define DPU_DATA_CUBE_WIDTH_RESERVED_0__SHIFT 13
#define DPU_DATA_CUBE_WIDTH_WIDTH__MASK 0x00001fff
#define DPU_DATA_CUBE_WIDTH_WIDTH__SHIFT 0
#define REG_DPU_DATA_CUBE_HEIGHT 0x00004034
#define DPU_DATA_CUBE_HEIGHT_RESERVED_0__MASK 0xfe000000
#define DPU_DATA_CUBE_HEIGHT_RESERVED_0__SHIFT 25
#define DPU_DATA_CUBE_HEIGHT_MINMAX_CTL__MASK 0x01c00000
#define DPU_DATA_CUBE_HEIGHT_MINMAX_CTL__SHIFT 22
#define DPU_DATA_CUBE_HEIGHT_RESERVED_1__MASK 0x003fe000
#define DPU_DATA_CUBE_HEIGHT_RESERVED_1__SHIFT 13
#define DPU_DATA_CUBE_HEIGHT_HEIGHT__MASK 0x00001fff
#define DPU_DATA_CUBE_HEIGHT_HEIGHT__SHIFT 0
#define REG_DPU_DATA_CUBE_NOTCH_ADDR 0x00004038
#define DPU_DATA_CUBE_NOTCH_ADDR_RESERVED_0__MASK 0xe0000000
#define DPU_DATA_CUBE_NOTCH_ADDR_RESERVED_0__SHIFT 29
#define DPU_DATA_CUBE_NOTCH_ADDR_NOTCH_ADDR_1__MASK 0x1fff0000
#define DPU_DATA_CUBE_NOTCH_ADDR_NOTCH_ADDR_1__SHIFT 16
#define DPU_DATA_CUBE_NOTCH_ADDR_RESERVED_1__MASK 0x0000e000
#define DPU_DATA_CUBE_NOTCH_ADDR_RESERVED_1__SHIFT 13
#define DPU_DATA_CUBE_NOTCH_ADDR_NOTCH_ADDR_0__MASK 0x00001fff
#define DPU_DATA_CUBE_NOTCH_ADDR_NOTCH_ADDR_0__SHIFT 0
#define REG_DPU_DATA_CUBE_CHANNEL 0x0000403c
#define DPU_DATA_CUBE_CHANNEL_RESERVED_0__MASK 0xe0000000
#define DPU_DATA_CUBE_CHANNEL_RESERVED_0__SHIFT 29
#define DPU_DATA_CUBE_CHANNEL_ORIG_CHANNEL__MASK 0x1fff0000
#define DPU_DATA_CUBE_CHANNEL_ORIG_CHANNEL__SHIFT 16
#define DPU_DATA_CUBE_CHANNEL_RESERVED_1__MASK 0x0000e000
#define DPU_DATA_CUBE_CHANNEL_RESERVED_1__SHIFT 13
#define DPU_DATA_CUBE_CHANNEL_CHANNEL__MASK 0x00001fff
#define DPU_DATA_CUBE_CHANNEL_CHANNEL__SHIFT 0
#define REG_DPU_BS_CFG 0x00004040
#define DPU_BS_CFG_RESERVED_0__MASK 0xfff00000
#define DPU_BS_CFG_RESERVED_0__SHIFT 20
#define DPU_BS_CFG_BS_ALU_ALGO__MASK 0x000f0000
#define DPU_BS_CFG_BS_ALU_ALGO__SHIFT 16
#define DPU_BS_CFG_RESERVED_1__MASK 0x0000fe00
#define DPU_BS_CFG_RESERVED_1__SHIFT 9
#define DPU_BS_CFG_BS_ALU_SRC__MASK 0x00000100
#define DPU_BS_CFG_BS_ALU_SRC__SHIFT 8
#define DPU_BS_CFG_BS_RELUX_EN__MASK 0x00000080
#define DPU_BS_CFG_BS_RELUX_EN__SHIFT 7
#define DPU_BS_CFG_BS_RELU_BYPASS__MASK 0x00000040
#define DPU_BS_CFG_BS_RELU_BYPASS__SHIFT 6
#define DPU_BS_CFG_BS_MUL_PRELU__MASK 0x00000020
#define DPU_BS_CFG_BS_MUL_PRELU__SHIFT 5
#define DPU_BS_CFG_BS_MUL_BYPASS__MASK 0x00000010
#define DPU_BS_CFG_BS_MUL_BYPASS__SHIFT 4
#define DPU_BS_CFG_RESERVED_2__MASK 0x0000000c
#define DPU_BS_CFG_RESERVED_2__SHIFT 2
#define DPU_BS_CFG_BS_ALU_BYPASS__MASK 0x00000002
#define DPU_BS_CFG_BS_ALU_BYPASS__SHIFT 1
#define DPU_BS_CFG_BS_BYPASS__MASK 0x00000001
#define DPU_BS_CFG_BS_BYPASS__SHIFT 0
#define REG_DPU_BS_ALU_CFG 0x00004044
#define DPU_BS_ALU_CFG_BS_ALU_OPERAND__MASK 0xffffffff
#define DPU_BS_ALU_CFG_BS_ALU_OPERAND__SHIFT 0
#define REG_DPU_BS_MUL_CFG 0x00004048
#define DPU_BS_MUL_CFG_BS_MUL_OPERAND__MASK 0xffff0000
#define DPU_BS_MUL_CFG_BS_MUL_OPERAND__SHIFT 16
#define DPU_BS_MUL_CFG_RESERVED_0__MASK 0x0000c000
#define DPU_BS_MUL_CFG_RESERVED_0__SHIFT 14
#define DPU_BS_MUL_CFG_BS_MUL_SHIFT_VALUE__MASK 0x00003f00
#define DPU_BS_MUL_CFG_BS_MUL_SHIFT_VALUE__SHIFT 8
#define DPU_BS_MUL_CFG_RESERVED_1__MASK 0x000000fc
#define DPU_BS_MUL_CFG_RESERVED_1__SHIFT 2
#define DPU_BS_MUL_CFG_BS_TRUNCATE_SRC__MASK 0x00000002
#define DPU_BS_MUL_CFG_BS_TRUNCATE_SRC__SHIFT 1
#define DPU_BS_MUL_CFG_BS_MUL_SRC__MASK 0x00000001
#define DPU_BS_MUL_CFG_BS_MUL_SRC__SHIFT 0
#define REG_DPU_BS_RELUX_CMP_VALUE 0x0000404c
#define DPU_BS_RELUX_CMP_VALUE_BS_RELUX_CMP_DAT__MASK 0xffffffff
#define DPU_BS_RELUX_CMP_VALUE_BS_RELUX_CMP_DAT__SHIFT 0
#define REG_DPU_BS_OW_CFG 0x00004050
#define DPU_BS_OW_CFG_RGP_CNTER__MASK 0xf0000000
#define DPU_BS_OW_CFG_RGP_CNTER__SHIFT 28
#define DPU_BS_OW_CFG_TP_ORG_EN__MASK 0x08000000
#define DPU_BS_OW_CFG_TP_ORG_EN__SHIFT 27
#define DPU_BS_OW_CFG_RESERVED_0__MASK 0x07fff800
#define DPU_BS_OW_CFG_RESERVED_0__SHIFT 11
#define DPU_BS_OW_CFG_SIZE_E_2__MASK 0x00000700
#define DPU_BS_OW_CFG_SIZE_E_2__SHIFT 8
#define DPU_BS_OW_CFG_SIZE_E_1__MASK 0x000000e0
#define DPU_BS_OW_CFG_SIZE_E_1__SHIFT 5
#define DPU_BS_OW_CFG_SIZE_E_0__MASK 0x0000001c
#define DPU_BS_OW_CFG_SIZE_E_0__SHIFT 2
#define DPU_BS_OW_CFG_OD_BYPASS__MASK 0x00000002
#define DPU_BS_OW_CFG_OD_BYPASS__SHIFT 1
#define DPU_BS_OW_CFG_OW_SRC__MASK 0x00000001
#define DPU_BS_OW_CFG_OW_SRC__SHIFT 0
#define REG_DPU_BS_OW_OP 0x00004054
#define DPU_BS_OW_OP_RESERVED_0__MASK 0xffff0000
#define DPU_BS_OW_OP_RESERVED_0__SHIFT 16
#define DPU_BS_OW_OP_OW_OP__MASK 0x0000ffff
#define DPU_BS_OW_OP_OW_OP__SHIFT 0
#define REG_DPU_WDMA_SIZE_0 0x00004058
#define DPU_WDMA_SIZE_0_RESERVED_0__MASK 0xf0000000
#define DPU_WDMA_SIZE_0_RESERVED_0__SHIFT 28
#define DPU_WDMA_SIZE_0_TP_PRECISION__MASK 0x08000000
#define DPU_WDMA_SIZE_0_TP_PRECISION__SHIFT 27
#define DPU_WDMA_SIZE_0_SIZE_C_WDMA__MASK 0x07ff0000
#define DPU_WDMA_SIZE_0_SIZE_C_WDMA__SHIFT 16
#define DPU_WDMA_SIZE_0_RESERVED_1__MASK 0x0000e000
#define DPU_WDMA_SIZE_0_RESERVED_1__SHIFT 13
#define DPU_WDMA_SIZE_0_CHANNEL_WDMA__MASK 0x00001fff
#define DPU_WDMA_SIZE_0_CHANNEL_WDMA__SHIFT 0
#define REG_DPU_WDMA_SIZE_1 0x0000405c
#define DPU_WDMA_SIZE_1_RESERVED_0__MASK 0xe0000000
#define DPU_WDMA_SIZE_1_RESERVED_0__SHIFT 29
#define DPU_WDMA_SIZE_1_HEIGHT_WDMA__MASK 0x1fff0000
#define DPU_WDMA_SIZE_1_HEIGHT_WDMA__SHIFT 16
#define DPU_WDMA_SIZE_1_RESERVED_1__MASK 0x0000e000
#define DPU_WDMA_SIZE_1_RESERVED_1__SHIFT 13
#define DPU_WDMA_SIZE_1_WIDTH_WDMA__MASK 0x00001fff
#define DPU_WDMA_SIZE_1_WIDTH_WDMA__SHIFT 0
#define REG_DPU_BN_CFG 0x00004060
#define DPU_BN_CFG_RESERVED_0__MASK 0xfff00000
#define DPU_BN_CFG_RESERVED_0__SHIFT 20
#define DPU_BN_CFG_BN_ALU_ALGO__MASK 0x000f0000
#define DPU_BN_CFG_BN_ALU_ALGO__SHIFT 16
#define DPU_BN_CFG_RESERVED_1__MASK 0x0000fe00
#define DPU_BN_CFG_RESERVED_1__SHIFT 9
#define DPU_BN_CFG_BN_ALU_SRC__MASK 0x00000100
#define DPU_BN_CFG_BN_ALU_SRC__SHIFT 8
#define DPU_BN_CFG_BN_RELUX_EN__MASK 0x00000080
#define DPU_BN_CFG_BN_RELUX_EN__SHIFT 7
#define DPU_BN_CFG_BN_RELU_BYPASS__MASK 0x00000040
#define DPU_BN_CFG_BN_RELU_BYPASS__SHIFT 6
#define DPU_BN_CFG_BN_MUL_PRELU__MASK 0x00000020
#define DPU_BN_CFG_BN_MUL_PRELU__SHIFT 5
#define DPU_BN_CFG_BN_MUL_BYPASS__MASK 0x00000010
#define DPU_BN_CFG_BN_MUL_BYPASS__SHIFT 4
#define DPU_BN_CFG_RESERVED_2__MASK 0x0000000c
#define DPU_BN_CFG_RESERVED_2__SHIFT 2
#define DPU_BN_CFG_BN_ALU_BYPASS__MASK 0x00000002
#define DPU_BN_CFG_BN_ALU_BYPASS__SHIFT 1
#define DPU_BN_CFG_BN_BYPASS__MASK 0x00000001
#define DPU_BN_CFG_BN_BYPASS__SHIFT 0
#define REG_DPU_BN_ALU_CFG 0x00004064
#define DPU_BN_ALU_CFG_BN_ALU_OPERAND__MASK 0xffffffff
#define DPU_BN_ALU_CFG_BN_ALU_OPERAND__SHIFT 0
#define REG_DPU_BN_MUL_CFG 0x00004068
#define DPU_BN_MUL_CFG_BN_MUL_OPERAND__MASK 0xffff0000
#define DPU_BN_MUL_CFG_BN_MUL_OPERAND__SHIFT 16
#define DPU_BN_MUL_CFG_RESERVED_0__MASK 0x0000c000
#define DPU_BN_MUL_CFG_RESERVED_0__SHIFT 14
#define DPU_BN_MUL_CFG_BN_MUL_SHIFT_VALUE__MASK 0x00003f00
#define DPU_BN_MUL_CFG_BN_MUL_SHIFT_VALUE__SHIFT 8
#define DPU_BN_MUL_CFG_RESERVED_1__MASK 0x000000fc
#define DPU_BN_MUL_CFG_RESERVED_1__SHIFT 2
#define DPU_BN_MUL_CFG_BN_TRUNCATE_SRC__MASK 0x00000002
#define DPU_BN_MUL_CFG_BN_TRUNCATE_SRC__SHIFT 1
#define DPU_BN_MUL_CFG_BN_MUL_SRC__MASK 0x00000001
#define DPU_BN_MUL_CFG_BN_MUL_SRC__SHIFT 0
#define REG_DPU_BN_RELUX_CMP_VALUE 0x0000406c
#define DPU_BN_RELUX_CMP_VALUE_BN_RELUX_CMP_DAT__MASK 0xffffffff
#define DPU_BN_RELUX_CMP_VALUE_BN_RELUX_CMP_DAT__SHIFT 0
#define REG_DPU_EW_CFG 0x00004070
#define DPU_EW_CFG_EW_CVT_TYPE__MASK 0x80000000
#define DPU_EW_CFG_EW_CVT_TYPE__SHIFT 31
#define DPU_EW_CFG_EW_CVT_ROUND__MASK 0x40000000
#define DPU_EW_CFG_EW_CVT_ROUND__SHIFT 30
#define DPU_EW_CFG_EW_DATA_MODE__MASK 0x30000000
#define DPU_EW_CFG_EW_DATA_MODE__SHIFT 28
#define DPU_EW_CFG_RESERVED_0__MASK 0x0f000000
#define DPU_EW_CFG_RESERVED_0__SHIFT 24
#define DPU_EW_CFG_EDATA_SIZE__MASK 0x00c00000
#define DPU_EW_CFG_EDATA_SIZE__SHIFT 22
#define DPU_EW_CFG_EW_EQUAL_EN__MASK 0x00200000
#define DPU_EW_CFG_EW_EQUAL_EN__SHIFT 21
#define DPU_EW_CFG_EW_BINARY_EN__MASK 0x00100000
#define DPU_EW_CFG_EW_BINARY_EN__SHIFT 20
#define DPU_EW_CFG_EW_ALU_ALGO__MASK 0x000f0000
#define DPU_EW_CFG_EW_ALU_ALGO__SHIFT 16
#define DPU_EW_CFG_RESERVED_1__MASK 0x0000f800
#define DPU_EW_CFG_RESERVED_1__SHIFT 11
#define DPU_EW_CFG_EW_RELUX_EN__MASK 0x00000400
#define DPU_EW_CFG_EW_RELUX_EN__SHIFT 10
#define DPU_EW_CFG_EW_RELU_BYPASS__MASK 0x00000200
#define DPU_EW_CFG_EW_RELU_BYPASS__SHIFT 9
#define DPU_EW_CFG_EW_OP_CVT_BYPASS__MASK 0x00000100
#define DPU_EW_CFG_EW_OP_CVT_BYPASS__SHIFT 8
#define DPU_EW_CFG_EW_LUT_BYPASS__MASK 0x00000080
#define DPU_EW_CFG_EW_LUT_BYPASS__SHIFT 7
#define DPU_EW_CFG_EW_OP_SRC__MASK 0x00000040
#define DPU_EW_CFG_EW_OP_SRC__SHIFT 6
#define DPU_EW_CFG_EW_MUL_PRELU__MASK 0x00000020
#define DPU_EW_CFG_EW_MUL_PRELU__SHIFT 5
#define DPU_EW_CFG_RESERVED_2__MASK 0x00000018
#define DPU_EW_CFG_RESERVED_2__SHIFT 3
#define DPU_EW_CFG_EW_OP_TYPE__MASK 0x00000004
#define DPU_EW_CFG_EW_OP_TYPE__SHIFT 2
#define DPU_EW_CFG_EW_OP_BYPASS__MASK 0x00000002
#define DPU_EW_CFG_EW_OP_BYPASS__SHIFT 1
#define DPU_EW_CFG_EW_BYPASS__MASK 0x00000001
#define DPU_EW_CFG_EW_BYPASS__SHIFT 0
#define REG_DPU_EW_CVT_OFFSET_VALUE 0x00004074
#define DPU_EW_CVT_OFFSET_VALUE_EW_OP_CVT_OFFSET__MASK 0xffffffff
#define DPU_EW_CVT_OFFSET_VALUE_EW_OP_CVT_OFFSET__SHIFT 0
#define REG_DPU_EW_CVT_SCALE_VALUE 0x00004078
#define DPU_EW_CVT_SCALE_VALUE_EW_TRUNCATE__MASK 0xffc00000
#define DPU_EW_CVT_SCALE_VALUE_EW_TRUNCATE__SHIFT 22
#define DPU_EW_CVT_SCALE_VALUE_EW_OP_CVT_SHIFT__MASK 0x003f0000
#define DPU_EW_CVT_SCALE_VALUE_EW_OP_CVT_SHIFT__SHIFT 16
#define DPU_EW_CVT_SCALE_VALUE_EW_OP_CVT_SCALE__MASK 0x0000ffff
#define DPU_EW_CVT_SCALE_VALUE_EW_OP_CVT_SCALE__SHIFT 0
#define REG_DPU_EW_RELUX_CMP_VALUE 0x0000407c
#define DPU_EW_RELUX_CMP_VALUE_EW_RELUX_CMP_DAT__MASK 0xffffffff
#define DPU_EW_RELUX_CMP_VALUE_EW_RELUX_CMP_DAT__SHIFT 0
#define REG_DPU_OUT_CVT_OFFSET 0x00004080
#define DPU_OUT_CVT_OFFSET_OUT_CVT_OFFSET__MASK 0xffffffff
#define DPU_OUT_CVT_OFFSET_OUT_CVT_OFFSET__SHIFT 0
#define REG_DPU_OUT_CVT_SCALE 0x00004084
#define DPU_OUT_CVT_SCALE_RESERVED_0__MASK 0xfffe0000
#define DPU_OUT_CVT_SCALE_RESERVED_0__SHIFT 17
#define DPU_OUT_CVT_SCALE_FP32TOFP16_EN__MASK 0x00010000
#define DPU_OUT_CVT_SCALE_FP32TOFP16_EN__SHIFT 16
#define DPU_OUT_CVT_SCALE_OUT_CVT_SCALE__MASK 0x0000ffff
#define DPU_OUT_CVT_SCALE_OUT_CVT_SCALE__SHIFT 0
#define REG_DPU_OUT_CVT_SHIFT 0x00004088
#define DPU_OUT_CVT_SHIFT_CVT_TYPE__MASK 0x80000000
#define DPU_OUT_CVT_SHIFT_CVT_TYPE__SHIFT 31
#define DPU_OUT_CVT_SHIFT_CVT_ROUND__MASK 0x40000000
#define DPU_OUT_CVT_SHIFT_CVT_ROUND__SHIFT 30
#define DPU_OUT_CVT_SHIFT_RESERVED_0__MASK 0x3ff00000
#define DPU_OUT_CVT_SHIFT_RESERVED_0__SHIFT 20
#define DPU_OUT_CVT_SHIFT_MINUS_EXP__MASK 0x000ff000
#define DPU_OUT_CVT_SHIFT_MINUS_EXP__SHIFT 12
#define DPU_OUT_CVT_SHIFT_OUT_CVT_SHIFT__MASK 0x00000fff
#define DPU_OUT_CVT_SHIFT_OUT_CVT_SHIFT__SHIFT 0
#define REG_DPU_EW_OP_VALUE_0 0x00004090
#define DPU_EW_OP_VALUE_0_EW_OPERAND_0__MASK 0xffffffff
#define DPU_EW_OP_VALUE_0_EW_OPERAND_0__SHIFT 0
#define REG_DPU_EW_OP_VALUE_1 0x00004094
#define DPU_EW_OP_VALUE_1_EW_OPERAND_1__MASK 0xffffffff
#define DPU_EW_OP_VALUE_1_EW_OPERAND_1__SHIFT 0
#define REG_DPU_EW_OP_VALUE_2 0x00004098
#define DPU_EW_OP_VALUE_2_EW_OPERAND_2__MASK 0xffffffff
#define DPU_EW_OP_VALUE_2_EW_OPERAND_2__SHIFT 0
#define REG_DPU_EW_OP_VALUE_3 0x0000409c
#define DPU_EW_OP_VALUE_3_EW_OPERAND_3__MASK 0xffffffff
#define DPU_EW_OP_VALUE_3_EW_OPERAND_3__SHIFT 0
#define REG_DPU_EW_OP_VALUE_4 0x000040a0
#define DPU_EW_OP_VALUE_4_EW_OPERAND_4__MASK 0xffffffff
#define DPU_EW_OP_VALUE_4_EW_OPERAND_4__SHIFT 0
#define REG_DPU_EW_OP_VALUE_5 0x000040a4
#define DPU_EW_OP_VALUE_5_EW_OPERAND_5__MASK 0xffffffff
#define DPU_EW_OP_VALUE_5_EW_OPERAND_5__SHIFT 0
#define REG_DPU_EW_OP_VALUE_6 0x000040a8
#define DPU_EW_OP_VALUE_6_EW_OPERAND_6__MASK 0xffffffff
#define DPU_EW_OP_VALUE_6_EW_OPERAND_6__SHIFT 0
#define REG_DPU_EW_OP_VALUE_7 0x000040ac
#define DPU_EW_OP_VALUE_7_EW_OPERAND_7__MASK 0xffffffff
#define DPU_EW_OP_VALUE_7_EW_OPERAND_7__SHIFT 0
#define REG_DPU_SURFACE_ADD 0x000040c0
#define DPU_SURFACE_ADD_SURF_ADD__MASK 0xfffffff0
#define DPU_SURFACE_ADD_SURF_ADD__SHIFT 4
#define DPU_SURFACE_ADD_RESERVED_0__MASK 0x0000000f
#define DPU_SURFACE_ADD_RESERVED_0__SHIFT 0
#define REG_DPU_LUT_ACCESS_CFG 0x00004100
#define DPU_LUT_ACCESS_CFG_RESERVED_0__MASK 0xfffc0000
#define DPU_LUT_ACCESS_CFG_RESERVED_0__SHIFT 18
#define DPU_LUT_ACCESS_CFG_LUT_ACCESS_TYPE__MASK 0x00020000
#define DPU_LUT_ACCESS_CFG_LUT_ACCESS_TYPE__SHIFT 17
#define DPU_LUT_ACCESS_CFG_LUT_TABLE_ID__MASK 0x00010000
#define DPU_LUT_ACCESS_CFG_LUT_TABLE_ID__SHIFT 16
#define DPU_LUT_ACCESS_CFG_RESERVED_1__MASK 0x0000fc00
#define DPU_LUT_ACCESS_CFG_RESERVED_1__SHIFT 10
#define DPU_LUT_ACCESS_CFG_LUT_ADDR__MASK 0x000003ff
#define DPU_LUT_ACCESS_CFG_LUT_ADDR__SHIFT 0
#define REG_DPU_LUT_ACCESS_DATA 0x00004104
#define DPU_LUT_ACCESS_DATA_RESERVED_0__MASK 0xffff0000
#define DPU_LUT_ACCESS_DATA_RESERVED_0__SHIFT 16
#define DPU_LUT_ACCESS_DATA_LUT_ACCESS_DATA__MASK 0x0000ffff
#define DPU_LUT_ACCESS_DATA_LUT_ACCESS_DATA__SHIFT 0
#define REG_DPU_LUT_CFG 0x00004108
#define DPU_LUT_CFG_RESERVED_0__MASK 0xffffff00
#define DPU_LUT_CFG_RESERVED_0__SHIFT 8
#define DPU_LUT_CFG_LUT_CAL_SEL__MASK 0x00000080
#define DPU_LUT_CFG_LUT_CAL_SEL__SHIFT 7
#define DPU_LUT_CFG_LUT_HYBRID_PRIORITY__MASK 0x00000040
#define DPU_LUT_CFG_LUT_HYBRID_PRIORITY__SHIFT 6
#define DPU_LUT_CFG_LUT_OFLOW_PRIORITY__MASK 0x00000020
#define DPU_LUT_CFG_LUT_OFLOW_PRIORITY__SHIFT 5
#define DPU_LUT_CFG_LUT_UFLOW_PRIORITY__MASK 0x00000010
#define DPU_LUT_CFG_LUT_UFLOW_PRIORITY__SHIFT 4
#define DPU_LUT_CFG_LUT_LO_LE_MUX__MASK 0x0000000c
#define DPU_LUT_CFG_LUT_LO_LE_MUX__SHIFT 2
#define DPU_LUT_CFG_LUT_EXPAND_EN__MASK 0x00000002
#define DPU_LUT_CFG_LUT_EXPAND_EN__SHIFT 1
#define DPU_LUT_CFG_LUT_ROAD_SEL__MASK 0x00000001
#define DPU_LUT_CFG_LUT_ROAD_SEL__SHIFT 0
#define REG_DPU_LUT_INFO 0x0000410c
#define DPU_LUT_INFO_RESERVED_0__MASK 0xff000000
#define DPU_LUT_INFO_RESERVED_0__SHIFT 24
#define DPU_LUT_INFO_LUT_LO_INDEX_SELECT__MASK 0x00ff0000
#define DPU_LUT_INFO_LUT_LO_INDEX_SELECT__SHIFT 16
#define DPU_LUT_INFO_LUT_LE_INDEX_SELECT__MASK 0x0000ff00
#define DPU_LUT_INFO_LUT_LE_INDEX_SELECT__SHIFT 8
#define DPU_LUT_INFO_RESERVED_1__MASK 0x000000ff
#define DPU_LUT_INFO_RESERVED_1__SHIFT 0
#define REG_DPU_LUT_LE_START 0x00004110
#define DPU_LUT_LE_START_LUT_LE_START__MASK 0xffffffff
#define DPU_LUT_LE_START_LUT_LE_START__SHIFT 0
#define REG_DPU_LUT_LE_END 0x00004114
#define DPU_LUT_LE_END_LUT_LE_END__MASK 0xffffffff
#define DPU_LUT_LE_END_LUT_LE_END__SHIFT 0
#define REG_DPU_LUT_LO_START 0x00004118
#define DPU_LUT_LO_START_LUT_LO_START__MASK 0xffffffff
#define DPU_LUT_LO_START_LUT_LO_START__SHIFT 0
#define REG_DPU_LUT_LO_END 0x0000411c
#define DPU_LUT_LO_END_LUT_LO_END__MASK 0xffffffff
#define DPU_LUT_LO_END_LUT_LO_END__SHIFT 0
#define REG_DPU_LUT_LE_SLOPE_SCALE 0x00004120
#define DPU_LUT_LE_SLOPE_SCALE_LUT_LE_SLOPE_OFLOW_SCALE__MASK 0xffff0000
#define DPU_LUT_LE_SLOPE_SCALE_LUT_LE_SLOPE_OFLOW_SCALE__SHIFT 16
#define DPU_LUT_LE_SLOPE_SCALE_LUT_LE_SLOPE_UFLOW_SCALE__MASK 0x0000ffff
#define DPU_LUT_LE_SLOPE_SCALE_LUT_LE_SLOPE_UFLOW_SCALE__SHIFT 0
#define REG_DPU_LUT_LE_SLOPE_SHIFT 0x00004124
#define DPU_LUT_LE_SLOPE_SHIFT_RESERVED_0__MASK 0xfffffc00
#define DPU_LUT_LE_SLOPE_SHIFT_RESERVED_0__SHIFT 10
#define DPU_LUT_LE_SLOPE_SHIFT_LUT_LE_SLOPE_OFLOW_SHIFT__MASK 0x000003e0
#define DPU_LUT_LE_SLOPE_SHIFT_LUT_LE_SLOPE_OFLOW_SHIFT__SHIFT 5
#define DPU_LUT_LE_SLOPE_SHIFT_LUT_LE_SLOPE_UFLOW_SHIFT__MASK 0x0000001f
#define DPU_LUT_LE_SLOPE_SHIFT_LUT_LE_SLOPE_UFLOW_SHIFT__SHIFT 0
#define REG_DPU_LUT_LO_SLOPE_SCALE 0x00004128
#define DPU_LUT_LO_SLOPE_SCALE_LUT_LO_SLOPE_OFLOW_SCALE__MASK 0xffff0000
#define DPU_LUT_LO_SLOPE_SCALE_LUT_LO_SLOPE_OFLOW_SCALE__SHIFT 16
#define DPU_LUT_LO_SLOPE_SCALE_LUT_LO_SLOPE_UFLOW_SCALE__MASK 0x0000ffff
#define DPU_LUT_LO_SLOPE_SCALE_LUT_LO_SLOPE_UFLOW_SCALE__SHIFT 0
#define REG_DPU_LUT_LO_SLOPE_SHIFT 0x0000412c
#define DPU_LUT_LO_SLOPE_SHIFT_RESERVED_0__MASK 0xfffffc00
#define DPU_LUT_LO_SLOPE_SHIFT_RESERVED_0__SHIFT 10
#define DPU_LUT_LO_SLOPE_SHIFT_LUT_LO_SLOPE_OFLOW_SHIFT__MASK 0x000003e0
#define DPU_LUT_LO_SLOPE_SHIFT_LUT_LO_SLOPE_OFLOW_SHIFT__SHIFT 5
#define DPU_LUT_LO_SLOPE_SHIFT_LUT_LO_SLOPE_UFLOW_SHIFT__MASK 0x0000001f
#define DPU_LUT_LO_SLOPE_SHIFT_LUT_LO_SLOPE_UFLOW_SHIFT__SHIFT 0
#define REG_DPU_RDMA_RDMA_S_STATUS 0x00005000
#define DPU_RDMA_RDMA_S_STATUS_RESERVED_0__MASK 0xfffc0000
#define DPU_RDMA_RDMA_S_STATUS_RESERVED_0__SHIFT 18
#define DPU_RDMA_RDMA_S_STATUS_STATUS_1__MASK 0x00030000
#define DPU_RDMA_RDMA_S_STATUS_STATUS_1__SHIFT 16
#define DPU_RDMA_RDMA_S_STATUS_RESERVED_1__MASK 0x0000fffc
#define DPU_RDMA_RDMA_S_STATUS_RESERVED_1__SHIFT 2
#define DPU_RDMA_RDMA_S_STATUS_STATUS_0__MASK 0x00000003
#define DPU_RDMA_RDMA_S_STATUS_STATUS_0__SHIFT 0
#define REG_DPU_RDMA_RDMA_S_POINTER 0x00005004
#define DPU_RDMA_RDMA_S_POINTER_RESERVED_0__MASK 0xfffe0000
#define DPU_RDMA_RDMA_S_POINTER_RESERVED_0__SHIFT 17
#define DPU_RDMA_RDMA_S_POINTER_EXECUTER__MASK 0x00010000
#define DPU_RDMA_RDMA_S_POINTER_EXECUTER__SHIFT 16
#define DPU_RDMA_RDMA_S_POINTER_RESERVED_1__MASK 0x0000ffc0
#define DPU_RDMA_RDMA_S_POINTER_RESERVED_1__SHIFT 6
#define DPU_RDMA_RDMA_S_POINTER_EXECUTER_PP_CLEAR__MASK 0x00000020
#define DPU_RDMA_RDMA_S_POINTER_EXECUTER_PP_CLEAR__SHIFT 5
#define DPU_RDMA_RDMA_S_POINTER_POINTER_PP_CLEAR__MASK 0x00000010
#define DPU_RDMA_RDMA_S_POINTER_POINTER_PP_CLEAR__SHIFT 4
#define DPU_RDMA_RDMA_S_POINTER_POINTER_PP_MODE__MASK 0x00000008
#define DPU_RDMA_RDMA_S_POINTER_POINTER_PP_MODE__SHIFT 3
#define DPU_RDMA_RDMA_S_POINTER_EXECUTER_PP_EN__MASK 0x00000004
#define DPU_RDMA_RDMA_S_POINTER_EXECUTER_PP_EN__SHIFT 2
#define DPU_RDMA_RDMA_S_POINTER_POINTER_PP_EN__MASK 0x00000002
#define DPU_RDMA_RDMA_S_POINTER_POINTER_PP_EN__SHIFT 1
#define DPU_RDMA_RDMA_S_POINTER_POINTER__MASK 0x00000001
#define DPU_RDMA_RDMA_S_POINTER_POINTER__SHIFT 0
#define REG_DPU_RDMA_RDMA_OPERATION_ENABLE 0x00005008
#define DPU_RDMA_RDMA_OPERATION_ENABLE_RESERVED_0__MASK 0xfffffffe
#define DPU_RDMA_RDMA_OPERATION_ENABLE_RESERVED_0__SHIFT 1
#define DPU_RDMA_RDMA_OPERATION_ENABLE_OP_EN__MASK 0x00000001
#define DPU_RDMA_RDMA_OPERATION_ENABLE_OP_EN__SHIFT 0
#define REG_DPU_RDMA_RDMA_DATA_CUBE_WIDTH 0x0000500c
#define DPU_RDMA_RDMA_DATA_CUBE_WIDTH_RESERVED_0__MASK 0xffffe000
#define DPU_RDMA_RDMA_DATA_CUBE_WIDTH_RESERVED_0__SHIFT 13
#define DPU_RDMA_RDMA_DATA_CUBE_WIDTH_WIDTH__MASK 0x00001fff
#define DPU_RDMA_RDMA_DATA_CUBE_WIDTH_WIDTH__SHIFT 0
#define REG_DPU_RDMA_RDMA_DATA_CUBE_HEIGHT 0x00005010
#define DPU_RDMA_RDMA_DATA_CUBE_HEIGHT_RESERVED_0__MASK 0xe0000000
#define DPU_RDMA_RDMA_DATA_CUBE_HEIGHT_RESERVED_0__SHIFT 29
#define DPU_RDMA_RDMA_DATA_CUBE_HEIGHT_EW_LINE_NOTCH_ADDR__MASK 0x1fff0000
#define DPU_RDMA_RDMA_DATA_CUBE_HEIGHT_EW_LINE_NOTCH_ADDR__SHIFT 16
#define DPU_RDMA_RDMA_DATA_CUBE_HEIGHT_RESERVED_1__MASK 0x0000e000
#define DPU_RDMA_RDMA_DATA_CUBE_HEIGHT_RESERVED_1__SHIFT 13
#define DPU_RDMA_RDMA_DATA_CUBE_HEIGHT_HEIGHT__MASK 0x00001fff
#define DPU_RDMA_RDMA_DATA_CUBE_HEIGHT_HEIGHT__SHIFT 0
#define REG_DPU_RDMA_RDMA_DATA_CUBE_CHANNEL 0x00005014
#define DPU_RDMA_RDMA_DATA_CUBE_CHANNEL_RESERVED_0__MASK 0xffffe000
#define DPU_RDMA_RDMA_DATA_CUBE_CHANNEL_RESERVED_0__SHIFT 13
#define DPU_RDMA_RDMA_DATA_CUBE_CHANNEL_CHANNEL__MASK 0x00001fff
#define DPU_RDMA_RDMA_DATA_CUBE_CHANNEL_CHANNEL__SHIFT 0
#define REG_DPU_RDMA_RDMA_SRC_BASE_ADDR 0x00005018
#define DPU_RDMA_RDMA_SRC_BASE_ADDR_SRC_BASE_ADDR__MASK 0xffffffff
#define DPU_RDMA_RDMA_SRC_BASE_ADDR_SRC_BASE_ADDR__SHIFT 0
#define REG_DPU_RDMA_RDMA_BRDMA_CFG 0x0000501c
#define DPU_RDMA_RDMA_BRDMA_CFG_RESERVED_0__MASK 0xffffffe0
#define DPU_RDMA_RDMA_BRDMA_CFG_RESERVED_0__SHIFT 5
#define DPU_RDMA_RDMA_BRDMA_CFG_BRDMA_DATA_USE__MASK 0x0000001e
#define DPU_RDMA_RDMA_BRDMA_CFG_BRDMA_DATA_USE__SHIFT 1
#define DPU_RDMA_RDMA_BRDMA_CFG_RESERVED_1__MASK 0x00000001
#define DPU_RDMA_RDMA_BRDMA_CFG_RESERVED_1__SHIFT 0
#define REG_DPU_RDMA_RDMA_BS_BASE_ADDR 0x00005020
#define DPU_RDMA_RDMA_BS_BASE_ADDR_BS_BASE_ADDR__MASK 0xffffffff
#define DPU_RDMA_RDMA_BS_BASE_ADDR_BS_BASE_ADDR__SHIFT 0
#define REG_DPU_RDMA_RDMA_NRDMA_CFG 0x00005028
#define DPU_RDMA_RDMA_NRDMA_CFG_RESERVED_0__MASK 0xffffffe0
#define DPU_RDMA_RDMA_NRDMA_CFG_RESERVED_0__SHIFT 5
#define DPU_RDMA_RDMA_NRDMA_CFG_NRDMA_DATA_USE__MASK 0x0000001e
#define DPU_RDMA_RDMA_NRDMA_CFG_NRDMA_DATA_USE__SHIFT 1
#define DPU_RDMA_RDMA_NRDMA_CFG_RESERVED_1__MASK 0x00000001
#define DPU_RDMA_RDMA_NRDMA_CFG_RESERVED_1__SHIFT 0
#define REG_DPU_RDMA_RDMA_BN_BASE_ADDR 0x0000502c
#define DPU_RDMA_RDMA_BN_BASE_ADDR_BN_BASE_ADDR__MASK 0xffffffff
#define DPU_RDMA_RDMA_BN_BASE_ADDR_BN_BASE_ADDR__SHIFT 0
#define REG_DPU_RDMA_RDMA_ERDMA_CFG 0x00005034
#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_DATA_MODE__MASK 0xc0000000
#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_DATA_MODE__SHIFT 30
#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_SURF_MODE__MASK 0x20000000
#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_SURF_MODE__SHIFT 29
#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_NONALIGN__MASK 0x10000000
#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_NONALIGN__SHIFT 28
#define DPU_RDMA_RDMA_ERDMA_CFG_RESERVED_0__MASK 0x0ffffff0
#define DPU_RDMA_RDMA_ERDMA_CFG_RESERVED_0__SHIFT 4
#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_DATA_SIZE__MASK 0x0000000c
#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_DATA_SIZE__SHIFT 2
#define DPU_RDMA_RDMA_ERDMA_CFG_OV4K_BYPASS__MASK 0x00000002
#define DPU_RDMA_RDMA_ERDMA_CFG_OV4K_BYPASS__SHIFT 1
#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_DISABLE__MASK 0x00000001
#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_DISABLE__SHIFT 0
#define REG_DPU_RDMA_RDMA_EW_BASE_ADDR 0x00005038
#define DPU_RDMA_RDMA_EW_BASE_ADDR_EW_BASE_ADDR__MASK 0xffffffff
#define DPU_RDMA_RDMA_EW_BASE_ADDR_EW_BASE_ADDR__SHIFT 0
#define REG_DPU_RDMA_RDMA_EW_SURF_STRIDE 0x00005040
#define DPU_RDMA_RDMA_EW_SURF_STRIDE_EW_SURF_STRIDE__MASK 0xfffffff0
#define DPU_RDMA_RDMA_EW_SURF_STRIDE_EW_SURF_STRIDE__SHIFT 4
#define DPU_RDMA_RDMA_EW_SURF_STRIDE_RESERVED_0__MASK 0x0000000f
#define DPU_RDMA_RDMA_EW_SURF_STRIDE_RESERVED_0__SHIFT 0
#define REG_DPU_RDMA_RDMA_FEATURE_MODE_CFG 0x00005044
#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_RESERVED_0__MASK 0xfffc0000
#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_RESERVED_0__SHIFT 18
#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_IN_PRECISION__MASK 0x00038000
#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_IN_PRECISION__SHIFT 15
#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_BURST_LEN__MASK 0x00007800
#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_BURST_LEN__SHIFT 11
#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_COMB_USE__MASK 0x00000700
#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_COMB_USE__SHIFT 8
#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_PROC_PRECISION__MASK 0x000000e0
#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_PROC_PRECISION__SHIFT 5
#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_MRDMA_DISABLE__MASK 0x00000010
#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_MRDMA_DISABLE__SHIFT 4
#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_MRDMA_FP16TOFP32_EN__MASK 0x00000008
#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_MRDMA_FP16TOFP32_EN__SHIFT 3
#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_CONV_MODE__MASK 0x00000006
#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_CONV_MODE__SHIFT 1
#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_FLYING_MODE__MASK 0x00000001
#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_FLYING_MODE__SHIFT 0
#define REG_DPU_RDMA_RDMA_SRC_DMA_CFG 0x00005048
#define DPU_RDMA_RDMA_SRC_DMA_CFG_LINE_NOTCH_ADDR__MASK 0xfff80000
#define DPU_RDMA_RDMA_SRC_DMA_CFG_LINE_NOTCH_ADDR__SHIFT 19
#define DPU_RDMA_RDMA_SRC_DMA_CFG_RESERVED_0__MASK 0x0007c000
#define DPU_RDMA_RDMA_SRC_DMA_CFG_RESERVED_0__SHIFT 14
#define DPU_RDMA_RDMA_SRC_DMA_CFG_POOLING_METHOD__MASK 0x00002000
#define DPU_RDMA_RDMA_SRC_DMA_CFG_POOLING_METHOD__SHIFT 13
#define DPU_RDMA_RDMA_SRC_DMA_CFG_UNPOOLING_EN__MASK 0x00001000
#define DPU_RDMA_RDMA_SRC_DMA_CFG_UNPOOLING_EN__SHIFT 12
#define DPU_RDMA_RDMA_SRC_DMA_CFG_KERNEL_STRIDE_HEIGHT__MASK 0x00000e00
#define DPU_RDMA_RDMA_SRC_DMA_CFG_KERNEL_STRIDE_HEIGHT__SHIFT 9
#define DPU_RDMA_RDMA_SRC_DMA_CFG_KERNEL_STRIDE_WIDTH__MASK 0x000001c0
#define DPU_RDMA_RDMA_SRC_DMA_CFG_KERNEL_STRIDE_WIDTH__SHIFT 6
#define DPU_RDMA_RDMA_SRC_DMA_CFG_KERNEL_HEIGHT__MASK 0x00000038
#define DPU_RDMA_RDMA_SRC_DMA_CFG_KERNEL_HEIGHT__SHIFT 3
#define DPU_RDMA_RDMA_SRC_DMA_CFG_KERNEL_WIDTH__MASK 0x00000007
#define DPU_RDMA_RDMA_SRC_DMA_CFG_KERNEL_WIDTH__SHIFT 0
#define REG_DPU_RDMA_RDMA_SURF_NOTCH 0x0000504c
#define DPU_RDMA_RDMA_SURF_NOTCH_SURF_NOTCH_ADDR__MASK 0xfffffff0
#define DPU_RDMA_RDMA_SURF_NOTCH_SURF_NOTCH_ADDR__SHIFT 4
#define DPU_RDMA_RDMA_SURF_NOTCH_RESERVED_0__MASK 0x0000000f
#define DPU_RDMA_RDMA_SURF_NOTCH_RESERVED_0__SHIFT 0
#define REG_DPU_RDMA_RDMA_PAD_CFG 0x00005064
#define DPU_RDMA_RDMA_PAD_CFG_PAD_VALUE__MASK 0xffff0000
#define DPU_RDMA_RDMA_PAD_CFG_PAD_VALUE__SHIFT 16
#define DPU_RDMA_RDMA_PAD_CFG_RESERVED_0__MASK 0x0000ff80
#define DPU_RDMA_RDMA_PAD_CFG_RESERVED_0__SHIFT 7
#define DPU_RDMA_RDMA_PAD_CFG_PAD_TOP__MASK 0x00000070
#define DPU_RDMA_RDMA_PAD_CFG_PAD_TOP__SHIFT 4
#define DPU_RDMA_RDMA_PAD_CFG_RESERVED_1__MASK 0x00000008
#define DPU_RDMA_RDMA_PAD_CFG_RESERVED_1__SHIFT 3
#define DPU_RDMA_RDMA_PAD_CFG_PAD_LEFT__MASK 0x00000007
#define DPU_RDMA_RDMA_PAD_CFG_PAD_LEFT__SHIFT 0
#define REG_DPU_RDMA_RDMA_WEIGHT 0x00005068
#define DPU_RDMA_RDMA_WEIGHT_E_WEIGHT__MASK 0xff000000
#define DPU_RDMA_RDMA_WEIGHT_E_WEIGHT__SHIFT 24
#define DPU_RDMA_RDMA_WEIGHT_N_WEIGHT__MASK 0x00ff0000
#define DPU_RDMA_RDMA_WEIGHT_N_WEIGHT__SHIFT 16
#define DPU_RDMA_RDMA_WEIGHT_B_WEIGHT__MASK 0x0000ff00
#define DPU_RDMA_RDMA_WEIGHT_B_WEIGHT__SHIFT 8
#define DPU_RDMA_RDMA_WEIGHT_M_WEIGHT__MASK 0x000000ff
#define DPU_RDMA_RDMA_WEIGHT_M_WEIGHT__SHIFT 0
#define REG_DPU_RDMA_RDMA_EW_SURF_NOTCH 0x0000506c
#define DPU_RDMA_RDMA_EW_SURF_NOTCH_EW_SURF_NOTCH__MASK 0xfffffff0
#define DPU_RDMA_RDMA_EW_SURF_NOTCH_EW_SURF_NOTCH__SHIFT 4
#define DPU_RDMA_RDMA_EW_SURF_NOTCH_RESERVED_0__MASK 0x0000000f
#define DPU_RDMA_RDMA_EW_SURF_NOTCH_RESERVED_0__SHIFT 0
#endif
ggml/src/ggml-rocket/ggml-rocket.cpp
+425
View File
@@ -0,0 +1,425 @@
// SPDX-License-Identifier: MIT
//
// ggml-rocket: MUL_MAT backend for the RK3588 NPU via the mainline "rocket"
// DRM-accel driver (/dev/accel/accel0), no vendor RKNPU2 userspace.
//
// Modeled on ggml-blas: an ACCEL device with no weight buffers of its own
// (weights stay in CPU memory); the scheduler offloads MUL_MAT nodes it
// supports and leaves everything else on the CPU backend.
//
// Per MUL_MAT plane: dequantize the weight to F32 (ggml to_float trait),
// requantize per-output-channel to int8, quantize the F32 activation
// per-tensor to int8, run rkt_npu_matmul (int8 GEMM on the NPU with a
// per-channel out_scale array), then dequantize the int8 result to F32.
//
// out_scale is chosen from a Cauchy-Schwarz upper bound on |output| so the
// int8 output never clips: |<w_n, x_m>| <= ||w_n||2 * ||x_m||2. This is the
// pragmatic "int8-first" path; a lossless int32-accumulator readout is the
// planned follow-on (needs an OUT_CVT-bypass builder mode).
#include "ggml-impl.h"
#include "ggml-rocket.h"
#include "ggml-backend-impl.h"
#include "ggml-cpu.h"
#include <vector>
#include <cstring>
#include <cstdio>
#include <cstdlib>
#include <cmath>
extern "C" {
#include "librocket.h"
#include "rkt_npu_matmul.h"
}
// Empirically verified single-op limits on boltzmann's rocket NPU (int8):
// rows per tile up to 32 correct at K=2560 -> use 16 for margin
// N per tile up to 256 correct -> use 128
// full K correct through 8192, catastrophic at 11008 (vendor int8 K-limit)
#define ROCKET_TILE_M 16
#define ROCKET_TILE_N 128
#define ROCKET_K_MAX 8192
struct ggml_backend_rocket_context {
int fd = -1;
// reusable scratch (grown as needed, never shrunk)
std::vector<float> wf; // dequantized weight plane [N*K]
std::vector<uint8_t> aq; // quantized activations [M*K]
std::vector<uint8_t> wq; // quantized weights [N*K]
std::vector<uint8_t> yq; // int8 result [M*N]
std::vector<float> ws; // per-channel weight scale [N]
std::vector<float> os; // per-channel out scale [N]
};
static inline uint8_t rocket_q8(float v, float inv_s) {
int q = (int)lrintf(v * inv_s);
if (q > 127) q = 127;
if (q < -127) q = -127;
return (uint8_t)(q + 128); // symmetric int8 stored as uint8, zp=128
}
static void ggml_backend_rocket_mul_mat(ggml_backend_rocket_context * ctx, struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0]; // weight [K, N]
const struct ggml_tensor * src1 = dst->src[1]; // activ [K, M]
GGML_TENSOR_BINARY_OP_LOCALS
const enum ggml_type type = src0->type;
const auto * tt = ggml_get_type_traits(type);
ggml_to_float_t const to_float = tt->to_float;
const int64_t K = ne00; // == ne10
const int64_t N = ne01; // output channels
const int64_t M = ne11; // rows (tokens in batch)
static const bool dbg = getenv("GGML_ROCKET_DEBUG") != NULL;
if (dbg) fprintf(stderr, "[rocket] enter mul_mat M=%lld N=%lld K=%lld type=%s to_float=%p\n",
(long long)M, (long long)N, (long long)K,
ggml_type_name(type), (void*)to_float), fflush(stderr);
if (to_float == NULL || M == 0 || N == 0 || K == 0) {
GGML_ABORT("rocket mul_mat: unexpected op (to_float=%p M=%lld N=%lld K=%lld)",
(void*)to_float, (long long)M, (long long)N, (long long)K);
}
// broadcast of src0 over src1's higher dims (as in ggml-blas)
const int64_t r2 = ne12 / ne02;
const int64_t r3 = ne13 / ne03;
ctx->wf.resize((size_t)N * K);
ctx->aq.resize((size_t)M * K);
ctx->wq.resize((size_t)N * K);
ctx->yq.resize((size_t)M * N);
ctx->ws.resize((size_t)N);
ctx->os.resize((size_t)N);
for (int64_t i13 = 0; i13 < ne13; i13++) {
for (int64_t i12 = 0; i12 < ne12; i12++) {
const int64_t i03 = i13 / r3;
const int64_t i02 = i12 / r2;
const char * w_plane = (const char *) src0->data + i02*nb02 + i03*nb03;
const char * a_plane = (const char *) src1->data + i12*nb12 + i13*nb13;
char * d_plane = ( char *) dst->data + i12*nb2 + i13*nb3;
// --- dequantize weight plane to F32, per-channel scale + L2 norm ---
float wn2max = 0.0f; // max_n ||w_n||_2
for (int64_t n = 0; n < N; n++) {
float * wrow = ctx->wf.data() + (size_t)n * K;
to_float((const char *) w_plane + n*nb01, wrow, K);
float amax = 0.0f, l2 = 0.0f;
for (int64_t k = 0; k < K; k++) {
float v = wrow[k];
float a = std::fabs(v);
if (a > amax) amax = a;
l2 += v * v;
}
ctx->ws[n] = (amax > 0.0f ? amax : 1e-6f) / 127.0f;
l2 = std::sqrt(l2);
if (l2 > wn2max) wn2max = l2;
}
// --- quantize activation plane, per-tensor scale + max L2 row norm ---
float amax_a = 0.0f, ym2max = 0.0f;
for (int64_t m = 0; m < M; m++) {
const float * arow = (const float *) (a_plane + m*nb11);
float l2 = 0.0f;
for (int64_t k = 0; k < K; k++) {
float v = arow[k];
float a = std::fabs(v);
if (a > amax_a) amax_a = a;
l2 += v * v;
}
l2 = std::sqrt(l2);
if (l2 > ym2max) ym2max = l2;
}
const float a_scale = (amax_a > 0.0f ? amax_a : 1e-6f) / 127.0f;
const float inv_a = 1.0f / a_scale;
for (int64_t m = 0; m < M; m++) {
const float * arow = (const float *) (a_plane + m*nb11);
uint8_t * qrow = ctx->aq.data() + (size_t)m * K;
for (int64_t k = 0; k < K; k++) qrow[k] = rocket_q8(arow[k], inv_a);
}
// Cauchy-Schwarz bound on |output| -> never clips int8 output.
float out_scale = (wn2max * ym2max) / 127.0f;
if (!(out_scale > 0.0f)) out_scale = 1e-6f;
for (int64_t n = 0; n < N; n++) {
const float inv_w = 1.0f / ctx->ws[n];
const float * wrow = ctx->wf.data() + (size_t)n * K;
uint8_t * qrow = ctx->wq.data() + (size_t)n * K;
for (int64_t k = 0; k < K; k++) qrow[k] = rocket_q8(wrow[k], inv_w);
ctx->os[n] = out_scale / ctx->ws[n];
}
static const bool dbg = getenv("GGML_ROCKET_DEBUG") != NULL;
if (dbg) fprintf(stderr, "[rocket] mul_mat M=%lld N=%lld K=%lld plane(%lld,%lld) type=%s ...",
(long long)M, (long long)N, (long long)K,
(long long)i12, (long long)i13, ggml_type_name(type)), fflush(stderr);
int rc = rkt_npu_matmul(ctx->fd, ctx->aq.data(), ctx->wq.data(), NULL,
(uint32_t)M, (uint32_t)N, (uint32_t)K,
128, 128, 128,
a_scale, 1.0f, out_scale, ctx->os.data(),
ROCKET_TILE_M, ROCKET_TILE_N, ctx->yq.data());
if (dbg) fprintf(stderr, " rc=%d\n", rc);
GGML_ASSERT(rc == 0 && "rkt_npu_matmul failed");
// --- dequantize int8 result into dst (F32) ---
for (int64_t m = 0; m < M; m++) {
float * drow = (float *) (d_plane + m*nb1);
const uint8_t * yrow = ctx->yq.data() + (size_t)m * N;
for (int64_t n = 0; n < N; n++)
drow[n] = ((int)yrow[n] - 128) * out_scale;
}
}
}
}
// backend interface
static const char * ggml_backend_rocket_get_name(ggml_backend_t backend) {
return "Rocket";
GGML_UNUSED(backend);
}
static void ggml_backend_rocket_free(ggml_backend_t backend) {
ggml_backend_rocket_context * ctx = (ggml_backend_rocket_context *)backend->context;
delete ctx;
delete backend;
}
static enum ggml_status ggml_backend_rocket_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) {
ggml_backend_rocket_context * ctx = (ggml_backend_rocket_context *)backend->context;
for (int i = 0; i < cgraph->n_nodes; i++) {
struct ggml_tensor * node = cgraph->nodes[i];
if ((node->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) {
continue;
}
switch (node->op) {
case GGML_OP_MUL_MAT:
ggml_backend_rocket_mul_mat(ctx, node);
break;
case GGML_OP_NONE:
case GGML_OP_RESHAPE:
case GGML_OP_VIEW:
case GGML_OP_PERMUTE:
case GGML_OP_TRANSPOSE:
break;
default:
GGML_ABORT("%s: unsupported op %s\n", __func__, ggml_op_desc(node));
}
}
return GGML_STATUS_SUCCESS;
GGML_UNUSED(backend);
}
static struct ggml_backend_i rocket_backend_i = {
/* .get_name = */ ggml_backend_rocket_get_name,
/* .free = */ ggml_backend_rocket_free,
/* .set_tensor_async = */ NULL,
/* .get_tensor_async = */ NULL,
/* .set_tensor_2d_async = */ NULL,
/* .get_tensor_2d_async = */ NULL,
/* .cpy_tensor_async = */ NULL,
/* .synchronize = */ NULL,
/* .graph_plan_create = */ NULL,
/* .graph_plan_free = */ NULL,
/* .graph_plan_update = */ NULL,
/* .graph_plan_compute = */ NULL,
/* .graph_compute = */ ggml_backend_rocket_graph_compute,
/* .event_record = */ NULL,
/* .event_wait = */ NULL,
/* .graph_optimize = */ NULL,
};
static ggml_guid_t ggml_backend_rocket_guid(void) {
static ggml_guid guid = { 0x52, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x4e, 0x50, 0x55, 0x33, 0x35, 0x38, 0x38, 0x01, 0x00, 0x01 };
return &guid;
}
ggml_backend_t ggml_backend_rocket_init(void) {
int fd = rocket_open("/dev/accel/accel0");
if (fd < 0) {
GGML_LOG_ERROR("%s: failed to open /dev/accel/accel0 (%d)\n", __func__, fd);
return NULL;
}
ggml_backend_rocket_context * ctx = new ggml_backend_rocket_context;
ctx->fd = fd;
ggml_backend_t backend = new ggml_backend {
/* .guid = */ ggml_backend_rocket_guid(),
/* .iface = */ rocket_backend_i,
/* .device = */ ggml_backend_reg_dev_get(ggml_backend_rocket_reg(), 0),
/* .context = */ ctx,
};
return backend;
}
bool ggml_backend_is_rocket(ggml_backend_t backend) {
return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_rocket_guid());
}
// device interface
static const char * ggml_backend_rocket_device_get_name(ggml_backend_dev_t dev) {
return "Rocket";
GGML_UNUSED(dev);
}
static const char * ggml_backend_rocket_device_get_description(ggml_backend_dev_t dev) {
return "RK3588 NPU (mainline rocket accel)";
GGML_UNUSED(dev);
}
static void ggml_backend_rocket_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) {
*free = 0;
*total = 0;
GGML_UNUSED(dev);
}
static enum ggml_backend_dev_type ggml_backend_rocket_device_get_type(ggml_backend_dev_t dev) {
return GGML_BACKEND_DEVICE_TYPE_ACCEL;
GGML_UNUSED(dev);
}
static void ggml_backend_rocket_device_get_props(ggml_backend_dev_t dev, struct ggml_backend_dev_props * props) {
props->name = ggml_backend_rocket_device_get_name(dev);
props->description = ggml_backend_rocket_device_get_description(dev);
props->type = ggml_backend_rocket_device_get_type(dev);
ggml_backend_rocket_device_get_memory(dev, &props->memory_free, &props->memory_total);
props->caps = {
/* .async = */ false,
/* .host_buffer = */ false,
/* .buffer_from_host_ptr = */ true,
/* .events = */ false,
};
}
static ggml_backend_t ggml_backend_rocket_device_init_backend(ggml_backend_dev_t dev, const char * params) {
return ggml_backend_rocket_init();
GGML_UNUSED(dev);
GGML_UNUSED(params);
}
static ggml_backend_buffer_type_t ggml_backend_rocket_device_get_buffer_type(ggml_backend_dev_t dev) {
return ggml_backend_cpu_buffer_type();
GGML_UNUSED(dev);
}
static ggml_backend_buffer_t ggml_backend_rocket_device_buffer_from_host_ptr(ggml_backend_dev_t dev, void * ptr, size_t size, size_t max_tensor_size) {
return ggml_backend_cpu_buffer_from_ptr(ptr, size);
GGML_UNUSED(dev);
GGML_UNUSED(max_tensor_size);
}
static bool ggml_backend_rocket_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) {
const struct ggml_tensor * src0 = op->src[0];
const struct ggml_tensor * src1 = op->src[1];
switch (op->op) {
case GGML_OP_NONE:
case GGML_OP_RESHAPE:
case GGML_OP_VIEW:
case GGML_OP_PERMUTE:
case GGML_OP_TRANSPOSE:
return true;
case GGML_OP_MUL_MAT:
{
const int64_t K = src1->ne[0]; // == src0->ne[0]
if (src1->type != GGML_TYPE_F32) return false;
if (!ggml_is_contiguous(src0)) return false;
if (!ggml_is_contiguous(src1)) return false;
if (K <= 0 || K > ROCKET_K_MAX) return false;
if (K % 16 != 0) return false; // feature-atomic align
// weight must be dequantizable to F32 (F32 src0 has no to_float
// trait -> leave those on the CPU backend)
if (ggml_get_type_traits(src0->type)->to_float == NULL) return false;
return true;
}
default:
return false;
}
GGML_UNUSED(dev);
}
static bool ggml_backend_rocket_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) {
return ggml_backend_buft_is_host(buft);
GGML_UNUSED(dev);
}
static const struct ggml_backend_device_i ggml_backend_rocket_device_i = {
/* .get_name = */ ggml_backend_rocket_device_get_name,
/* .get_description = */ ggml_backend_rocket_device_get_description,
/* .get_memory = */ ggml_backend_rocket_device_get_memory,
/* .get_type = */ ggml_backend_rocket_device_get_type,
/* .get_props = */ ggml_backend_rocket_device_get_props,
/* .init_backend = */ ggml_backend_rocket_device_init_backend,
/* .get_buffer_type = */ ggml_backend_rocket_device_get_buffer_type,
/* .get_host_buffer_type = */ NULL,
/* .buffer_from_host_ptr = */ ggml_backend_rocket_device_buffer_from_host_ptr,
/* .supports_op = */ ggml_backend_rocket_device_supports_op,
/* .supports_buft = */ ggml_backend_rocket_device_supports_buft,
/* .offload_op = */ NULL,
/* .event_new = */ NULL,
/* .event_free = */ NULL,
/* .event_synchronize = */ NULL,
};
// backend reg interface
static const char * ggml_backend_rocket_reg_get_name(ggml_backend_reg_t reg) {
return "Rocket";
GGML_UNUSED(reg);
}
static size_t ggml_backend_rocket_reg_get_device_count(ggml_backend_reg_t reg) {
return 1;
GGML_UNUSED(reg);
}
static ggml_backend_dev_t ggml_backend_rocket_reg_get_device(ggml_backend_reg_t reg, size_t index) {
GGML_ASSERT(index == 0);
static ggml_backend_device ggml_backend_rocket_device = {
/* .iface = */ ggml_backend_rocket_device_i,
/* .reg = */ reg,
/* .context = */ nullptr,
};
return &ggml_backend_rocket_device;
GGML_UNUSED(reg);
GGML_UNUSED(index);
}
static const struct ggml_backend_reg_i ggml_backend_rocket_reg_i = {
/* .get_name = */ ggml_backend_rocket_reg_get_name,
/* .get_device_count = */ ggml_backend_rocket_reg_get_device_count,
/* .get_device = */ ggml_backend_rocket_reg_get_device,
/* .get_proc_address = */ NULL,
};
ggml_backend_reg_t ggml_backend_rocket_reg(void) {
static struct ggml_backend_reg ggml_backend_rocket_reg = {
/* .api_version = */ GGML_BACKEND_API_VERSION,
/* .iface = */ ggml_backend_rocket_reg_i,
/* .context = */ NULL,
};
return &ggml_backend_rocket_reg;
}
GGML_BACKEND_DL_IMPL(ggml_backend_rocket_reg)
ggml/src/ggml-rocket/ggml-rocket.h
+13
View File
@@ -0,0 +1,13 @@
#pragma once
#include "ggml-backend.h"
#ifdef __cplusplus
extern "C" {
#endif
GGML_API ggml_backend_reg_t ggml_backend_rocket_reg(void);
#ifdef __cplusplus
}
#endif
ggml/src/ggml-rocket/librocket.c
+106
View File
@@ -0,0 +1,106 @@
/* SPDX-License-Identifier: MIT */
/*
* librocket — thin C wrapper over the Rockchip "rocket" NPU DRM-accel uAPI.
*
* Implementation — pure syscall-surface plumbing.
*/
#include "librocket.h"
#include <errno.h>
#include <fcntl.h>
#include <string.h>
#include <sys/ioctl.h>
#include <sys/mman.h>
#include <unistd.h>
int rocket_open(const char *path)
{
int fd = open(path, O_RDWR | O_CLOEXEC);
if (fd < 0)
return -errno;
return fd;
}
int rocket_create_bo(int fd, uint32_t size, uint32_t *handle_out,
uint64_t *dma_address_out, uint64_t *mmap_offset_out)
{
struct drm_rocket_create_bo create_bo;
int ret;
memset(&create_bo, 0, sizeof(create_bo));
create_bo.size = size;
ret = ioctl(fd, DRM_IOCTL_ROCKET_CREATE_BO, &create_bo);
if (ret < 0)
return -errno;
if (handle_out)
*handle_out = create_bo.handle;
if (dma_address_out)
*dma_address_out = create_bo.dma_address;
if (mmap_offset_out)
*mmap_offset_out = create_bo.offset;
return 0;
}
void *rocket_mmap_bo(int fd, uint64_t mmap_offset, uint32_t size)
{
return mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd,
mmap_offset);
}
int rocket_prep_bo(int fd, uint32_t handle, int64_t timeout_ns)
{
struct drm_rocket_prep_bo prep_bo;
int ret;
memset(&prep_bo, 0, sizeof(prep_bo));
prep_bo.handle = handle;
prep_bo.timeout_ns = timeout_ns;
ret = ioctl(fd, DRM_IOCTL_ROCKET_PREP_BO, &prep_bo);
if (ret < 0)
return -errno;
return 0;
}
int rocket_fini_bo(int fd, uint32_t handle)
{
struct drm_rocket_fini_bo fini_bo;
int ret;
memset(&fini_bo, 0, sizeof(fini_bo));
fini_bo.handle = handle;
ret = ioctl(fd, DRM_IOCTL_ROCKET_FINI_BO, &fini_bo);
if (ret < 0)
return -errno;
return 0;
}
int rocket_submit(int fd, struct drm_rocket_job *jobs, uint32_t job_count)
{
struct drm_rocket_submit submit;
int ret;
memset(&submit, 0, sizeof(submit));
submit.jobs = (uintptr_t)jobs;
submit.job_count = job_count;
submit.job_struct_size = sizeof(struct drm_rocket_job);
ret = ioctl(fd, DRM_IOCTL_ROCKET_SUBMIT, &submit);
if (ret < 0)
return -errno;
return 0;
}
int rocket_close_bo(int fd, uint32_t handle)
{
struct drm_gem_close c;
memset(&c, 0, sizeof c);
c.handle = handle;
if (ioctl(fd, DRM_IOCTL_GEM_CLOSE, &c) < 0)
return -errno;
return 0;
}
ggml/src/ggml-rocket/librocket.h
+87
View File
@@ -0,0 +1,87 @@
/* SPDX-License-Identifier: MIT */
/*
* librocket — thin C wrapper over the Rockchip "rocket" NPU DRM-accel uAPI.
*
* This is pure syscall-surface plumbing. No NPU/tensor semantics, no register
* commands. Every function returns 0 on success or -errno on failure (except
* rocket_mmap_bo which returns the mmap'd pointer or MAP_FAILED).
*/
#ifndef LIBROCKET_H
#define LIBROCKET_H
#include "rocket_accel.h"
#include <stdint.h>
#include <sys/types.h>
#ifdef __cplusplus
extern "C" {
#endif
/**
* rocket_open - open a rocket NPU accel device
* @path: device node path (e.g. "/dev/accel/accel0")
*
* Returns the file descriptor on success, or -errno on failure.
*/
int rocket_open(const char *path);
/**
* rocket_create_bo - allocate a buffer object
* @fd: rocket device fd
* @size: desired BO size in bytes
* @handle_out: on success, written with the GEM handle
* @dma_address_out: on success, written with the NPU DMA address
* @mmap_offset_out: on success, written with the mmap offset
*
* Returns 0 on success, -errno on failure.
*/
int rocket_create_bo(int fd, uint32_t size, uint32_t *handle_out,
uint64_t *dma_address_out, uint64_t *mmap_offset_out);
/**
* rocket_mmap_bo - mmap a BO into userspace
* @fd: rocket device fd
* @mmap_offset: offset returned by rocket_create_bo
* @size: number of bytes to map
*
* Returns the mapped pointer, or MAP_FAILED on failure.
*/
void *rocket_mmap_bo(int fd, uint64_t mmap_offset, uint32_t size);
/**
* rocket_prep_bo - take CPU ownership of a BO (wait for NPU, sync caches)
* @fd: rocket device fd
* @handle: GEM handle of the BO
* @timeout_ns: ABSOLUTE CLOCK_MONOTONIC deadline (ns); INT64_MAX = block until signalled
*
* Returns 0 on success, -errno on failure.
*/
int rocket_prep_bo(int fd, uint32_t handle, int64_t timeout_ns);
/**
* rocket_fini_bo - hand a BO back to the device (sync caches)
* @fd: rocket device fd
* @handle: GEM handle of the BO
*
* Returns 0 on success, -errno on failure.
*/
int rocket_fini_bo(int fd, uint32_t handle);
/**
* rocket_submit - submit NPU jobs
* @fd: rocket device fd
* @jobs: pointer to an array of struct drm_rocket_job
* @job_count: number of jobs in the array
*
* Returns 0 on success, -errno on failure.
*/
int rocket_submit(int fd, struct drm_rocket_job *jobs, uint32_t job_count);
/* Release a BO handle (DRM_IOCTL_GEM_CLOSE). Returns 0 or -errno. */
int rocket_close_bo(int fd, uint32_t handle);
#ifdef __cplusplus
}
#endif
#endif /* LIBROCKET_H */
ggml/src/ggml-rocket/rkt_gemm.c
+36
View File
@@ -0,0 +1,36 @@
/* SPDX-License-Identifier: MIT
*
* rkt_gemm.c — U6 GEMM tiler (see rkt_gemm.h).
*/
#include "rkt_gemm.h"
#include "rkt_matmul.h"
int rkt_gemm_plan(uint32_t M, uint32_t N, uint32_t tile_m, uint32_t tile_n,
struct rkt_gemm_tile *tiles, int max_tiles)
{
if (M == 0 || N == 0 || tile_m == 0 || tile_n == 0)
return -1;
int t = 0;
for (uint32_t r = 0; r < M; r += tile_m) {
for (uint32_t c = 0; c < N; c += tile_n) {
if (t >= max_tiles)
return -1;
tiles[t].row = r;
tiles[t].col = c;
tiles[t].m = (r + tile_m <= M) ? tile_m : (M - r);
tiles[t].n = (c + tile_n <= N) ? tile_n : (N - c);
t++;
}
}
return t;
}
int rkt_gemm_op_fits(uint32_t m, uint32_t n, uint32_t k)
{
uint64_t scratch[512];
/* Addresses/zero-points are placeholders — fit is shape-only. */
return rkt_build_matmul_regcmd(scratch, 512, m, n, k,
0x1000, 0x2000, 0x3000,
128, 128, 128) >= 0;
}
ggml/src/ggml-rocket/rkt_gemm.h
+36
View File
@@ -0,0 +1,36 @@
/* SPDX-License-Identifier: MIT
*
* rkt_gemm.h — U6 backend: tile a large INT8 GEMM into rocket-op-sized
* sub-matmuls, each buildable by rkt_build_matmul_regcmd.
*
* A single rocket op has a fixed CBUF budget (12 banks), so a gemma-scale
* GEMM must be blocked. This layer tiles the M (rows/tokens) and N (output
* channels) dimensions. K (contraction / input channels) is NOT tiled here:
* the full K must fit one op — K-splitting needs partial-sum accumulation via
* the DPU add path and is deferred (see rkt_matmul_coredpu add_tensor path).
*/
#ifndef RKT_GEMM_H
#define RKT_GEMM_H
#include <stdint.h>
struct rkt_gemm_tile {
uint32_t row; /* first row in M this tile covers */
uint32_t col; /* first col in N this tile covers */
uint32_t m; /* number of rows */
uint32_t n; /* number of cols */
};
/*
* Partition an M x N output (full K) into tiles of at most tile_m x tile_n,
* row-major. Writes tiles[] and returns the tile count, or -1 on bad args or
* if more than max_tiles would be produced. Pure geometry — call
* rkt_gemm_op_fits() to confirm a tile actually fits one rocket op.
*/
int rkt_gemm_plan(uint32_t M, uint32_t N, uint32_t tile_m, uint32_t tile_n,
struct rkt_gemm_tile *tiles, int max_tiles);
/* True (1) if an m x n, full-k matmul fits a single rocket op. */
int rkt_gemm_op_fits(uint32_t m, uint32_t n, uint32_t k);
#endif /* RKT_GEMM_H */
ggml/src/ggml-rocket/rkt_matmul.c
+255
View File
@@ -0,0 +1,255 @@
/* SPDX-License-Identifier: MIT
*
* rkt_matmul.c — unified INT8 matmul regcmd builder.
*
* Implements rkt_build_matmul_regcmd(): given a plain GEMM shape it derives
* every low-level CNA / CORE / DPU field via the NVDLA-style CBUF tiling math
* (ported from Mesa rocket rkt_task.c: fill_task + the single-tile branch of
* rkt_split_tasks) and drives the two emit stages.
*
* The matmul Y[M][N] = X[M][K] * W[K][N] is expressed as a 1x1 convolution:
* input_width = 1, input_height = M, input_channels = K,
* output_width = 1, output_height = M, output_channels = N,
* weights 1x1, stride 1, no padding, no bias, identity requantisation.
*
* SCOPE: single-tile only. If the operation would need CBUF task-splitting
* (input feature map too tall to fit the available input banks) this returns
* -1 rather than emit a wrong single task. The multi-task splitter and real
* scale/bias handling are separate follow-on work.
*/
#include "rkt_matmul.h"
#include "rkt_matmul_cna.h"
#include "rkt_matmul_coredpu.h"
#include <string.h>
/* CBUF geometry — verbatim from Mesa rkt_ml.h (NVDLA v1 convolution buffer). */
#define CBUF_BANK_SIZE 32768
#define CBUF_BANKS 12
#define CBUF_ENTRIES_PER_BANK 256
#define CBUF_ENTRY_SIZE (CBUF_BANK_SIZE / CBUF_ENTRIES_PER_BANK) /* 128 */
#define FEATURE_ATOMIC_SIZE 16
#define ATOMIC_K_SIZE 16
#define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
#define MAX2(a, b) ((a) > (b) ? (a) : (b))
static unsigned alignu(unsigned v, unsigned a) { return ((v + a - 1) / a) * a; }
/* Fields Mesa's tiling math reads off rkt_operation, specialised to a matmul. */
struct op {
unsigned stride;
unsigned input_width, input_height, input_channels;
unsigned output_width, output_height, output_channels;
unsigned weights_width, weights_height;
int depthwise, addition_input, add_tensor;
};
static unsigned calc_line_stride(unsigned width)
{
return width * ATOMIC_K_SIZE * sizeof(uint8_t);
}
static unsigned calc_entries_per_slice(const struct op *o)
{
unsigned atomics_per_entry = CBUF_ENTRY_SIZE / FEATURE_ATOMIC_SIZE;
unsigned total_c_atomics =
DIV_ROUND_UP(o->input_channels * sizeof(uint8_t), FEATURE_ATOMIC_SIZE);
unsigned last_c_atomics = total_c_atomics % atomics_per_entry;
unsigned int_c_entries =
(total_c_atomics / atomics_per_entry) * o->input_width;
unsigned frac_c_entries =
(last_c_atomics == 3)
? o->input_width
: DIV_ROUND_UP(last_c_atomics * o->input_width, atomics_per_entry);
return int_c_entries + frac_c_entries;
}
static unsigned calc_input_banks(const struct op *o)
{
return DIV_ROUND_UP(calc_entries_per_slice(o) * o->input_height,
CBUF_ENTRIES_PER_BANK);
}
static unsigned calc_weights_banks(const struct op *o)
{
unsigned bytes = o->weights_width * o->weights_height *
o->input_channels * sizeof(uint8_t);
if (!o->depthwise)
bytes *= o->output_channels;
unsigned entries = DIV_ROUND_UP(bytes, CBUF_ENTRY_SIZE);
unsigned banks = DIV_ROUND_UP(entries, CBUF_ENTRIES_PER_BANK);
banks++; /* Mesa: "the calc above might be wrong on this HW" */
return banks;
}
int rkt_build_matmul_regcmd_scaled(uint64_t *out, int out_capacity,
uint32_t M, uint32_t N, uint32_t K,
uint64_t input_dma, uint64_t weights_dma,
uint64_t output_dma,
int32_t input_zero_point,
int32_t weight_zero_point,
int32_t output_zero_point,
float input_scale, float weights_scale,
float output_scale, uint64_t bias_dma)
{
if (M == 0 || N == 0 || K == 0)
return -1;
struct op o = {
.stride = 1,
.input_width = 1, .input_height = M, .input_channels = K,
.output_width = 1, .output_height = M, .output_channels = N,
.weights_width = 1, .weights_height = 1,
.depthwise = 0, .addition_input = 0, .add_tensor = -1,
};
/* ---- fill_task (Mesa rkt_task.c), stride 1 / no pad / no add ---- */
unsigned t_input_width = o.input_width; /* width != 8, not doubled */
unsigned t_input_height = o.input_height;
unsigned t_input_channels =
alignu(MAX2(o.input_channels, FEATURE_ATOMIC_SIZE), FEATURE_ATOMIC_SIZE);
unsigned t_input_channels_real = o.input_channels;
unsigned t_output_width = o.output_width;
unsigned t_output_height = o.output_height;
unsigned t_output_channels_real = o.output_channels;
unsigned t_output_channels = alignu(MAX2(o.output_channels, 32), 32);
/* depthwise output_channels fix-ups skipped (depthwise == 0) */
int t_input_line_stride, t_input_surface_stride;
if (t_input_channels_real == 1 &&
(t_output_channels_real > 1 || o.addition_input || o.add_tensor != -1)) {
t_input_width = MAX2(t_input_width, FEATURE_ATOMIC_SIZE);
t_input_line_stride =
MAX2((int)(calc_line_stride(o.input_width) / FEATURE_ATOMIC_SIZE),
(int)FEATURE_ATOMIC_SIZE);
t_input_surface_stride =
(int)((float)t_input_line_stride * ((float)t_input_height - 1.0f));
} else {
t_input_line_stride = (int)(calc_line_stride(o.input_width) / 4);
t_input_surface_stride = (int)((float)t_input_line_stride *
(((float)t_input_height / 4.0f) - 1.0f));
}
/* input_width == 8 && addition branch skipped */
int output_line_stride = (int)calc_line_stride(o.output_width);
int t_output_surface_stride = output_line_stride * (int)t_output_height;
t_output_surface_stride /= FEATURE_ATOMIC_SIZE;
unsigned t_input_data_entries;
if (t_input_channels_real == 1)
t_input_data_entries = t_input_width * t_input_height;
else if (t_input_width == 40 && t_input_channels_real == 40)
t_input_data_entries = 40;
else
t_input_data_entries = DIV_ROUND_UP(
t_input_width * 2 *
DIV_ROUND_UP(t_input_channels_real, FEATURE_ATOMIC_SIZE),
8);
unsigned t_weights_kernels =
o.depthwise ? 1 : alignu(o.output_channels, 2);
unsigned t_surfaces_per_row = t_output_width * t_output_height * 2;
/* depthwise surfaces_per_row *= 2 skipped */
/* ---- rkt_split_tasks, single-tile branch (full weights, full input) ---- */
unsigned weights_banks_required = calc_weights_banks(&o);
unsigned input_banks_required = calc_input_banks(&o);
int reuse_weights_cbuf;
unsigned available_input_banks;
if (weights_banks_required + 1 < CBUF_BANKS) {
reuse_weights_cbuf = 1;
available_input_banks = CBUF_BANKS - weights_banks_required;
} else {
reuse_weights_cbuf = 0;
available_input_banks = 7;
}
if (input_banks_required > available_input_banks)
return -1; /* needs CBUF task-splitting — unsupported here */
unsigned t_input_banks = input_banks_required;
unsigned t_weights_banks = CBUF_BANKS - t_input_banks;
unsigned t_atomic_count = t_output_width * t_output_height;
/* ---- CNA params ---- */
struct cna_params c;
memset(&c, 0, sizeof c);
c.input_width = t_input_width;
c.input_height = t_input_height;
c.input_channels = t_input_channels;
c.input_channels_real = t_input_channels_real;
c.output_width = t_output_width;
c.atomic_count = t_atomic_count;
c.weights_width = o.weights_width;
c.weights_height = o.weights_height;
c.weights_kernels = t_weights_kernels;
c.weights_banks = t_weights_banks;
c.input_banks = t_input_banks;
c.input_data_entries = t_input_data_entries;
c.input_line_stride = (uint32_t)t_input_line_stride;
c.input_surface_stride = (uint32_t)t_input_surface_stride;
c.input_dma = input_dma;
c.weights_dma = weights_dma;
c.stride_x = o.stride;
c.stride_y = o.stride;
c.output_zero_point = output_zero_point;
c.weights_zero_point = weight_zero_point;
c.input_zero_point = input_zero_point;
c.depthwise = o.depthwise;
c.reuse_weights_cbuf = reuse_weights_cbuf;
c.addition_input = o.addition_input;
c.add_tensor = o.add_tensor;
c.task_num = 0;
/* ---- CORE/DPU/PC params ---- */
struct coredpu_params d;
memset(&d, 0, sizeof d);
d.input_width = t_input_width;
d.output_width = t_output_width;
d.output_height = t_output_height;
d.output_channels = t_output_channels;
d.output_channels_real = t_output_channels_real;
d.output_offset = 0;
d.output_surface_stride = (uint32_t)t_output_surface_stride;
d.surfaces_per_row = t_surfaces_per_row;
d.truncate_bits = 0;
d.output_zero_point = output_zero_point;
d.weights_zero_point = weight_zero_point;
d.depthwise = o.depthwise;
d.addition_offset = 0;
d.add_tensor = o.add_tensor;
d.input_scale = input_scale;
d.weights_scale = weights_scale;
d.output_scale = output_scale;
d.addition_scale = 1.0f;
d.output_addr = output_dma;
d.add_tensor_addr = 0;
d.biases_addr = bias_dma;
/* ---- emit both stages into one contiguous regcmd buffer ---- */
int n1 = rkt_emit_cna(out, out_capacity, &c);
if (n1 < 0)
return -1;
int n2 = rkt_emit_coredpu(out + n1, out_capacity - n1, &d);
if (n2 < 0)
return -1;
return n1 + n2;
}
/* Back-compat: identity requant, no bias. */
int rkt_build_matmul_regcmd(uint64_t *out, int out_capacity,
uint32_t M, uint32_t N, uint32_t K,
uint64_t input_dma, uint64_t weights_dma,
uint64_t output_dma,
int32_t input_zero_point,
int32_t weight_zero_point,
int32_t output_zero_point)
{
return rkt_build_matmul_regcmd_scaled(out, out_capacity, M, N, K,
input_dma, weights_dma, output_dma,
input_zero_point, weight_zero_point,
output_zero_point,
1.0f, 1.0f, 1.0f, 0);
}
ggml/src/ggml-rocket/rkt_matmul.h
+64
View File
@@ -0,0 +1,64 @@
/*
* rkt_matmul.h — Rockchip "rocket" NPU matmul regcmd generator
*
* Builds a register-command stream for a single INT8 matmul
* Y[M][N] = X[M][K] * W[K][N] expressed as a 1×1 convolution.
*
* Faithfully adapted from the upstream Mesa "rocket" Gallium driver
* (rkt_regcmd.c, rkt_task.c, rkt_ml.c, rkt_registers.h).
*
* SPDX-License-Identifier: MIT
*/
#ifndef RKT_MATMUL_H
#define RKT_MATMUL_H
#include <stdint.h>
/**
* rkt_build_matmul_regcmd - build a single-task NPU regcmd for an INT8 matmul
*
* The matmul is emitted as a 1×1 convolution on the NPU:
* input_channels = K
* kernels = N
* height = M (conv spatial rows)
* width = 1 (conv spatial columns)
*
* @param out caller-provided output buffer
* @param out_capacity max uint64_t entries in @out
* @param M number of rows of X (and rows of Y)
* @param N number of columns of W (and columns of Y)
* @param K number of columns of X / rows of W
* @param input_dma bus address of X[M][K] (must be 256-byte aligned)
* @param weights_dma bus address of W[K][N] (must be 256-byte aligned)
* @param output_dma bus address of Y[M][N] (must be 256-byte aligned)
* @param input_zero_point int8 zero-point for input (0-255, usually 128)
* @param weight_zero_point int8 zero-point for weights (0-255, usually 128)
* @param output_zero_point int8 zero-point for output (0-255, usually 128)
*
* @return number of uint64_t commands written, or -1 if capacity exceeded.
*
* NOTE: Rate/scale parameters are not exposed. This generator assumes
* input_scale = weights_scale = output_scale = 1.0 (identity requantisation).
* Callers requiring proper quantised output must patch REG_DPU_OUT_CVT_SCALE
* and REG_DPU_OUT_CVT_SHIFT after construction, or provide pre-computed
* scale/shift values via an extended API.
*/
int rkt_build_matmul_regcmd_scaled(uint64_t *out, int out_capacity,
uint32_t M, uint32_t N, uint32_t K,
uint64_t input_dma, uint64_t weights_dma,
uint64_t output_dma,
int32_t input_zero_point, int32_t weight_zero_point,
int32_t output_zero_point,
float input_scale, float weights_scale,
float output_scale, uint64_t bias_dma);
int rkt_build_matmul_regcmd(uint64_t *out, int out_capacity,
uint32_t M, uint32_t N, uint32_t K,
uint64_t input_dma, uint64_t weights_dma,
uint64_t output_dma,
int32_t input_zero_point,
int32_t weight_zero_point,
int32_t output_zero_point);
#endif /* RKT_MATMUL_H */
ggml/src/ggml-rocket/rkt_matmul_cna.c
+171
View File
@@ -0,0 +1,171 @@
#include "rkt_matmul_cna.h"
#include "cna_defs.h"
#include <stdint.h>
/*
* Packed register command: (target << 48) | (value << 16) | reg
* target = block_enum + 1. CNA=0x200->0x201, DPU=0x1000->0x1001,
* DPU_RDMA=0x2000->0x2001. The target is a property of the register's block
* (Mesa derives it via rkt_get_target(reg)+1), NOT of the emit site — so the
* DPU_RDMA registers that appear inside this CNA sequence still carry 0x2001.
*/
static int rkt_push(uint64_t *out, int *n, int cap, uint32_t target,
uint32_t reg, uint32_t value)
{
if (*n >= cap)
return -1;
out[*n] = ((uint64_t)target << 48) | ((uint64_t)value << 16) | (uint64_t)reg;
(*n)++;
return 0;
}
#define EMIT_CNA(reg, val) do { if (rkt_push(out, &n, cap, 0x201, (reg), (val)) < 0) return -1; } while (0)
#define EMIT_DPU(reg, val) do { if (rkt_push(out, &n, cap, 0x1001, (reg), (val)) < 0) return -1; } while (0)
#define EMIT_RDMA(reg, val) do { if (rkt_push(out, &n, cap, 0x2001, (reg), (val)) < 0) return -1; } while (0)
/* Field-set: (value << FIELD__SHIFT) & FIELD__MASK */
#define F(v, name) (((uint32_t)(v) << name##__SHIFT) & name##__MASK)
int rkt_emit_cna(uint64_t *out, int cap, const struct cna_params *p)
{
int n = 0;
uint32_t con0 = F(p->weights_banks, CNA_CBUF_CON0_WEIGHT_BANK) |
F(p->input_banks, CNA_CBUF_CON0_DATA_BANK);
if (p->task_num > 0 && p->reuse_weights_cbuf)
con0 |= F(1, CNA_CBUF_CON0_WEIGHT_REUSE);
EMIT_CNA(REG_CNA_CBUF_CON0, con0);
EMIT_CNA(REG_CNA_DCOMP_REGNUM, 0);
EMIT_CNA(REG_CNA_DCOMP_CTRL, 0);
uint32_t con1 = 0x0;
if (p->input_channels_real == 1) {
con1 |= F(1, CNA_CONV_CON1_NONALIGN_DMA) | F(1, CNA_CONV_CON1_GROUP_LINE_OFF) |
F(8, CNA_CONV_CON1_ARGB_IN);
}
if (p->depthwise)
con1 |= F(3, CNA_CONV_CON1_CONV_MODE);
EMIT_CNA(REG_CNA_CONV_CON1, con1);
EMIT_DPU(REG_DPU_S_POINTER, F(1, DPU_S_POINTER_POINTER_PP_MODE) |
F(1, DPU_S_POINTER_EXECUTER_PP_EN) |
F(1, DPU_S_POINTER_POINTER_PP_EN));
EMIT_RDMA(REG_DPU_RDMA_RDMA_S_POINTER,
F(1, DPU_RDMA_RDMA_S_POINTER_POINTER_PP_MODE) |
F(1, DPU_RDMA_RDMA_S_POINTER_EXECUTER_PP_EN) |
F(1, DPU_RDMA_RDMA_S_POINTER_POINTER_PP_EN));
EMIT_CNA(REG_CNA_CONV_CON1, con1);
EMIT_CNA(REG_CNA_CONV_CON2,
F(50 + p->stride_y + 1, CNA_CONV_CON2_FEATURE_GRAINS)); /* Magic: passes the most tests */
EMIT_CNA(REG_CNA_CONV_CON3, F(p->stride_x, CNA_CONV_CON3_CONV_X_STRIDE) |
F(p->stride_y, CNA_CONV_CON3_CONV_Y_STRIDE));
EMIT_CNA(REG_CNA_DATA_SIZE0,
F(p->input_width, CNA_DATA_SIZE0_DATAIN_WIDTH) |
F(p->input_height, CNA_DATA_SIZE0_DATAIN_HEIGHT));
EMIT_CNA(REG_CNA_DATA_SIZE1,
F(p->input_channels_real - 1, CNA_DATA_SIZE1_DATAIN_CHANNEL_REAL) |
F(p->input_channels, CNA_DATA_SIZE1_DATAIN_CHANNEL));
EMIT_CNA(REG_CNA_DATA_SIZE2, F(p->output_width, CNA_DATA_SIZE2_DATAOUT_WIDTH));
EMIT_CNA(REG_CNA_DATA_SIZE3, F(p->atomic_count, CNA_DATA_SIZE3_DATAOUT_ATOMICS));
EMIT_CNA(REG_CNA_WEIGHT_SIZE0, p->weights_width * p->weights_height *
p->input_channels * p->weights_kernels);
EMIT_CNA(REG_CNA_WEIGHT_SIZE1,
p->weights_width * p->weights_height * p->input_channels);
EMIT_CNA(REG_CNA_WEIGHT_SIZE2,
F(p->weights_width, CNA_WEIGHT_SIZE2_WEIGHT_WIDTH) |
F(p->weights_height, CNA_WEIGHT_SIZE2_WEIGHT_HEIGHT) |
F(p->weights_kernels, CNA_WEIGHT_SIZE2_WEIGHT_KERNELS));
EMIT_CNA(REG_CNA_CBUF_CON0, con0);
EMIT_CNA(REG_CNA_CBUF_CON1, F(p->input_data_entries, CNA_CBUF_CON1_DATA_ENTRIES));
if (p->input_channels_real == 1) {
unsigned truncate = 14;
unsigned scale = 16384;
unsigned offset = 65408;
if (p->addition_input || p->add_tensor != -1) {
truncate = 15;
scale = 32388;
}
EMIT_CNA(REG_CNA_CVT_CON0, F(truncate, CNA_CVT_CON0_CVT_TRUNCATE_3) |
F(truncate, CNA_CVT_CON0_CVT_TRUNCATE_2) |
F(truncate, CNA_CVT_CON0_CVT_TRUNCATE_1) |
F(truncate, CNA_CVT_CON0_CVT_TRUNCATE_0));
EMIT_CNA(REG_CNA_CVT_CON1,
F(scale, CNA_CVT_CON1_CVT_SCALE0) | F(offset, CNA_CVT_CON1_CVT_OFFSET0));
EMIT_CNA(REG_CNA_CVT_CON2,
F(scale, CNA_CVT_CON2_CVT_SCALE1) | F(offset, CNA_CVT_CON2_CVT_OFFSET1));
EMIT_CNA(REG_CNA_CVT_CON3,
F(scale, CNA_CVT_CON3_CVT_SCALE2) | F(offset, CNA_CVT_CON3_CVT_OFFSET2));
EMIT_CNA(REG_CNA_CVT_CON4,
F(scale, CNA_CVT_CON4_CVT_SCALE3) | F(offset, CNA_CVT_CON4_CVT_OFFSET3));
} else {
EMIT_CNA(REG_CNA_CVT_CON0, F(1, CNA_CVT_CON0_DATA_SIGN) |
F(1, CNA_CVT_CON0_CVT_TYPE) |
F(1, CNA_CVT_CON0_CVT_BYPASS));
EMIT_CNA(REG_CNA_CVT_CON1, F(1, CNA_CVT_CON1_CVT_SCALE0));
EMIT_CNA(REG_CNA_CVT_CON2, F(1, CNA_CVT_CON2_CVT_SCALE1));
EMIT_CNA(REG_CNA_CVT_CON3, F(1, CNA_CVT_CON3_CVT_SCALE2));
EMIT_CNA(REG_CNA_CVT_CON4, F(1, CNA_CVT_CON4_CVT_SCALE3));
}
EMIT_CNA(REG_CNA_FC_CON0, 0);
EMIT_CNA(REG_CNA_FC_CON1, 0);
EMIT_CNA(REG_CNA_PAD_CON0, F(p->pad_left, CNA_PAD_CON0_PAD_LEFT) |
F(p->pad_top, CNA_PAD_CON0_PAD_TOP));
EMIT_CNA(REG_CNA_FEATURE_DATA_ADDR, (uint32_t)p->input_dma);
EMIT_CNA(REG_CNA_FC_CON2, 0);
EMIT_CNA(REG_CNA_DMA_CON0,
F(15, CNA_DMA_CON0_WEIGHT_BURST_LEN) | F(15, CNA_DMA_CON0_DATA_BURST_LEN));
EMIT_CNA(REG_CNA_DMA_CON1, F(p->input_line_stride, CNA_DMA_CON1_LINE_STRIDE));
EMIT_CNA(REG_CNA_DMA_CON2, F(p->input_surface_stride, CNA_DMA_CON2_SURF_STRIDE));
EMIT_CNA(REG_CNA_FC_DATA_SIZE0,
F(p->input_width, CNA_FC_DATA_SIZE0_DMA_WIDTH) |
F(p->input_height, CNA_FC_DATA_SIZE0_DMA_HEIGHT));
EMIT_CNA(REG_CNA_FC_DATA_SIZE1,
F(p->input_channels, CNA_FC_DATA_SIZE1_DMA_CHANNEL));
EMIT_CNA(REG_CNA_DCOMP_CTRL, 0);
EMIT_CNA(REG_CNA_DCOMP_REGNUM, 0);
EMIT_CNA(REG_CNA_DCOMP_ADDR0, (uint32_t)p->weights_dma);
EMIT_CNA(REG_CNA_DCOMP_AMOUNT0, 0);
EMIT_CNA(REG_CNA_DCOMP_AMOUNT1, 0);
EMIT_CNA(REG_CNA_DCOMP_AMOUNT2, 0);
EMIT_CNA(REG_CNA_DCOMP_AMOUNT3, 0);
EMIT_CNA(REG_CNA_DCOMP_AMOUNT4, 0);
EMIT_CNA(REG_CNA_DCOMP_AMOUNT5, 0);
EMIT_CNA(REG_CNA_DCOMP_AMOUNT6, 0);
EMIT_CNA(REG_CNA_DCOMP_AMOUNT7, 0);
EMIT_CNA(REG_CNA_DCOMP_AMOUNT8, 0);
EMIT_CNA(REG_CNA_DCOMP_AMOUNT9, 0);
EMIT_CNA(REG_CNA_DCOMP_AMOUNT10, 0);
EMIT_CNA(REG_CNA_DCOMP_AMOUNT11, 0);
EMIT_CNA(REG_CNA_DCOMP_AMOUNT12, 0);
EMIT_CNA(REG_CNA_DCOMP_AMOUNT13, 0);
EMIT_CNA(REG_CNA_DCOMP_AMOUNT14, 0);
EMIT_CNA(REG_CNA_DCOMP_AMOUNT15, 0);
if (p->input_channels_real == 1)
EMIT_CNA(REG_CNA_CVT_CON5, 65535);
else
EMIT_CNA(REG_CNA_CVT_CON5, 0);
int32_t pad_con1;
if (p->weights_width >= 3 && p->input_zero_point == 0x0)
pad_con1 = 0xffff8080;
else
pad_con1 = p->input_zero_point - 0x80;
if (p->addition_input || p->add_tensor != -1)
pad_con1 = 0xffffff80;
if (p->depthwise && p->input_zero_point == 0x8b)
pad_con1 = 0x0b0b;
EMIT_CNA(REG_CNA_PAD_CON1, pad_con1);
return n;
}
ggml/src/ggml-rocket/rkt_matmul_cna.h
+25
View File
@@ -0,0 +1,25 @@
#ifndef RKT_MATMUL_CNA_H
#define RKT_MATMUL_CNA_H
#include <stdint.h>
/* All the task-/operation-level fields the Mesa CNA emit reads, flattened. */
struct cna_params {
uint32_t input_width, input_height, input_channels, input_channels_real;
uint32_t output_width, atomic_count;
uint32_t weights_width, weights_height, weights_kernels;
uint32_t weights_banks, input_banks, input_data_entries;
uint32_t input_line_stride, input_surface_stride;
uint64_t input_dma, weights_dma; /* NPU DMA addresses of the input/weight BOs */
uint32_t stride_x, stride_y;
uint32_t pad_left, pad_right, pad_top, pad_bottom;
int32_t output_zero_point, weights_zero_point, input_zero_point;
int depthwise, reuse_weights_cbuf, addition_input, add_tensor;
unsigned task_num; /* 0 for the first/only task */
};
/* Append the CNA-stage register commands (packed uint64_t) into out[] (capacity cap).
Returns the number of commands written, or -1 if capacity would be exceeded. */
int rkt_emit_cna(uint64_t *out, int cap, const struct cna_params *p);
#endif /* RKT_MATMUL_CNA_H */
ggml/src/ggml-rocket/rkt_matmul_coredpu.c
+298
View File
@@ -0,0 +1,298 @@
#include "rkt_matmul_coredpu.h"
#include "coredpu_defs.h"
#include <stdint.h>
/* Packed register command: (target<<48) | (value<<16) | reg. target = block_enum + 1.
CORE=0x800->0x801, DPU=0x1000->0x1001, PC=0x100->0x101. */
static int rkt_push(uint64_t *out, int *n, int cap, uint32_t target,
uint32_t reg, uint32_t value)
{
if (*n >= cap)
return -1;
out[*n] = ((uint64_t)target << 48) | ((uint64_t)value << 16) | (uint64_t)reg;
(*n)++;
return 0;
}
#define EMIT_CORE(reg, val) do { if (rkt_push(out, &n, cap, 0x801, (reg), (val)) < 0) return -1; } while (0)
#define EMIT_DPU(reg, val) do { if (rkt_push(out, &n, cap, 0x1001, (reg), (val)) < 0) return -1; } while (0)
#define EMIT_RDMA(reg, val) do { if (rkt_push(out, &n, cap, 0x2001, (reg), (val)) < 0) return -1; } while (0)
#define EMIT_PC(reg, val) do { if (rkt_push(out, &n, cap, 0x101, (reg), (val)) < 0) return -1; } while (0)
#define EMIT_RAW(tgt, reg, val) do { if (rkt_push(out, &n, cap, (tgt), (reg), (val)) < 0) return -1; } while (0)
#define F(v, name) (((uint32_t)(v) << name##__SHIFT) & name##__MASK)
/* float -> raw IEEE-754 bits (Mesa fui()) */
static uint32_t fui(float f) { uint32_t u; __builtin_memcpy(&u, &f, sizeof u); return u; }
#define MAX2(a, b) ((a) > (b) ? (a) : (b))
#define ATOMIC_K_SIZE 16
/* raw 64-bit command append (Mesa util_dynarray_append_typed(regs, uint64_t, v)) */
#define RAW64(v) do { if (n >= cap) return -1; out[n++] = (uint64_t)(v); } while (0)
int rkt_emit_coredpu(uint64_t *out, int cap, const struct coredpu_params *p)
{
int n = 0;
int num_tasks = 1; /* single-task port */
int offset = (int)p->output_zero_point - 0x80; /* Mesa CNA local */
/* ==== CORE + DPU + PC emit (filled by slice) ==== */
uint32_t misc_cfg = F(1, CORE_MISC_CFG_QD_EN);
if (p->depthwise)
misc_cfg |= F(1, CORE_MISC_CFG_DW_EN);
EMIT_CORE(REG_CORE_MISC_CFG, misc_cfg);
EMIT_CORE(REG_CORE_DATAOUT_SIZE_0,
F(p->output_height - 1, CORE_DATAOUT_SIZE_0_DATAOUT_HEIGHT) |
F(p->output_width - 1, CORE_DATAOUT_SIZE_0_DATAOUT_WIDTH));
EMIT_CORE(REG_CORE_DATAOUT_SIZE_1,
F(p->output_channels - 1, CORE_DATAOUT_SIZE_1_DATAOUT_CHANNEL));
EMIT_CORE(REG_CORE_CLIP_TRUNCATE,
F(p->truncate_bits, CORE_CLIP_TRUNCATE_CLIP_TRUNCATE));
EMIT_RAW(0x801, 0x3030, 0);
uint32_t feat_mode_cfg =
F(15, DPU_FEATURE_MODE_CFG_BURST_LEN) | F(2, DPU_FEATURE_MODE_CFG_OUTPUT_MODE);
if (p->depthwise)
feat_mode_cfg |= F(3, DPU_FEATURE_MODE_CFG_CONV_MODE);
EMIT_DPU(REG_DPU_FEATURE_MODE_CFG, feat_mode_cfg);
EMIT_DPU(REG_DPU_DATA_FORMAT, 0);
EMIT_DPU(REG_DPU_OFFSET_PEND, 0);
EMIT_DPU(REG_DPU_DST_BASE_ADDR,
p->output_addr +
p->output_offset);
EMIT_DPU(REG_DPU_DST_SURF_STRIDE,
F(p->output_surface_stride, DPU_DST_SURF_STRIDE_DST_SURF_STRIDE));
EMIT_DPU(REG_DPU_DATA_CUBE_WIDTH,
F(p->output_width - 1, DPU_DATA_CUBE_WIDTH_WIDTH));
EMIT_DPU(REG_DPU_DATA_CUBE_HEIGHT,
F(p->output_height - 1, DPU_DATA_CUBE_HEIGHT_HEIGHT));
EMIT_DPU(REG_DPU_DATA_CUBE_NOTCH_ADDR, 0);
EMIT_DPU(REG_DPU_DATA_CUBE_CHANNEL,
F(p->output_channels_real - 1, DPU_DATA_CUBE_CHANNEL_ORIG_CHANNEL) |
F(p->output_channels - 1, DPU_DATA_CUBE_CHANNEL_CHANNEL));
EMIT_DPU(REG_DPU_BS_CFG, F(2, DPU_BS_CFG_BS_ALU_ALGO) | F(1, DPU_BS_CFG_BS_ALU_SRC) |
F(1, DPU_BS_CFG_BS_RELU_BYPASS) |
F(1, DPU_BS_CFG_BS_MUL_BYPASS));
EMIT_DPU(REG_DPU_BS_ALU_CFG, 0);
EMIT_DPU(REG_DPU_BS_MUL_CFG, 0);
EMIT_DPU(REG_DPU_BS_RELUX_CMP_VALUE, 0);
if (p->depthwise) {
EMIT_DPU(REG_DPU_BS_OW_CFG, F(3, DPU_BS_OW_CFG_SIZE_E_2) |
F(3, DPU_BS_OW_CFG_SIZE_E_1) |
F(3, DPU_BS_OW_CFG_SIZE_E_0));
} else {
EMIT_DPU(REG_DPU_BS_OW_CFG, F(1, DPU_BS_OW_CFG_SIZE_E_2) |
F(1, DPU_BS_OW_CFG_SIZE_E_1) |
F(1, DPU_BS_OW_CFG_SIZE_E_0));
}
EMIT_DPU(REG_DPU_BS_OW_OP, F(0x80 - p->weights_zero_point, DPU_BS_OW_OP_OW_OP));
EMIT_DPU(REG_DPU_WDMA_SIZE_0,
F(p->output_channels - 1, DPU_WDMA_SIZE_0_CHANNEL_WDMA));
EMIT_DPU(REG_DPU_WDMA_SIZE_1,
F(p->output_height - 1, DPU_WDMA_SIZE_1_HEIGHT_WDMA) |
F(p->output_width - 1, DPU_WDMA_SIZE_1_WIDTH_WDMA));
EMIT_DPU(REG_DPU_BN_CFG,
F(1, DPU_BN_CFG_BN_RELU_BYPASS) | F(1, DPU_BN_CFG_BN_MUL_BYPASS) |
F(1, DPU_BN_CFG_BN_ALU_BYPASS) | F(1, DPU_BN_CFG_BN_BYPASS));
EMIT_DPU(REG_DPU_BN_ALU_CFG, 0);
EMIT_DPU(REG_DPU_BN_MUL_CFG, 0);
EMIT_DPU(REG_DPU_BN_RELUX_CMP_VALUE, 0);
if (p->add_tensor != -1) {
EMIT_DPU(REG_DPU_EW_CFG,
F(1, DPU_EW_CFG_EW_CVT_TYPE) | F(1, DPU_EW_CFG_EW_DATA_MODE) |
F(1, DPU_EW_CFG_EDATA_SIZE) | F(2, DPU_EW_CFG_EW_ALU_ALGO) |
F(1, DPU_EW_CFG_EW_RELU_BYPASS) | F(1, DPU_EW_CFG_EW_LUT_BYPASS) |
F(1, DPU_EW_CFG_EW_OP_SRC));
/* See http://nvdla.org/hw/v1/ias/precision.html#element-wise */
EMIT_DPU(REG_DPU_EW_CVT_OFFSET_VALUE, p->addition_offset);
float add_scale =
p->addition_scale / (p->input_scale * p->weights_scale);
uint32_t add_scale_bits = fui(add_scale);
/* Taken from
* https://github.com/pytorch/QNNPACK/blob/master/src/qnnpack/requantization.h#L130
*/
unsigned add_shift = 127 + 31 - 32 - (add_scale_bits >> 23) + 16;
unsigned scale = ((add_scale_bits >> 9) & 0x7fff);
if (scale < 1 << 14)
scale |= 1 << 14;
EMIT_DPU(REG_DPU_EW_CVT_SCALE_VALUE,
F(add_shift - 1, DPU_EW_CVT_SCALE_VALUE_EW_OP_CVT_SHIFT) |
F(scale, DPU_EW_CVT_SCALE_VALUE_EW_OP_CVT_SCALE));
EMIT_DPU(REG_DPU_EW_RELUX_CMP_VALUE, 0x0);
float out_conv_scale =
(p->input_scale * p->weights_scale) / p->output_scale;
uint32_t out_scale_bits = fui(out_conv_scale);
unsigned out_shift = 127 + 31 - 32 - (out_scale_bits >> 23) + 16;
if (p->truncate_bits > 0)
out_shift--;
unsigned out_scale = ((out_scale_bits >> 9) & 0x7fff) + 1;
if (out_scale < 1 << 14)
out_scale |= 1 << 14;
EMIT_DPU(REG_DPU_OUT_CVT_OFFSET, offset);
EMIT_DPU(REG_DPU_OUT_CVT_SCALE, F(out_scale, DPU_OUT_CVT_SCALE_OUT_CVT_SCALE));
EMIT_DPU(REG_DPU_OUT_CVT_SHIFT, F(out_shift - 1, DPU_OUT_CVT_SHIFT_OUT_CVT_SHIFT));
} else {
EMIT_DPU(REG_DPU_EW_CFG,
F(1, DPU_EW_CFG_EW_RELU_BYPASS) | F(1, DPU_EW_CFG_EW_OP_CVT_BYPASS) |
F(1, DPU_EW_CFG_EW_LUT_BYPASS) | F(1, DPU_EW_CFG_EW_OP_BYPASS) |
F(1, DPU_EW_CFG_EW_BYPASS));
EMIT_DPU(REG_DPU_EW_CVT_OFFSET_VALUE, 0);
EMIT_DPU(REG_DPU_EW_CVT_SCALE_VALUE, F(1, DPU_EW_CVT_SCALE_VALUE_EW_OP_CVT_SCALE));
EMIT_DPU(REG_DPU_EW_RELUX_CMP_VALUE, 0);
EMIT_DPU(REG_DPU_OUT_CVT_OFFSET, offset);
float conv_scale =
(p->input_scale * p->weights_scale) / p->output_scale;
// DBG("conv_scale %f\n", conv_scale);
uint32_t scale_bits = fui(conv_scale);
/* Taken from
* https://github.com/pytorch/QNNPACK/blob/master/src/qnnpack/requantization.h#L130
*/
unsigned shift = 127 + 31 - 32 - (scale_bits >> 23) + 16;
if (p->truncate_bits > 0)
shift--;
unsigned scale = ((scale_bits >> 9) & 0x7fff) + 1;
if (scale < 1 << 14)
scale |= 1 << 14;
EMIT_DPU(REG_DPU_OUT_CVT_SCALE, F(scale, DPU_OUT_CVT_SCALE_OUT_CVT_SCALE));
EMIT_DPU(REG_DPU_OUT_CVT_SHIFT, F(shift - 1, DPU_OUT_CVT_SHIFT_OUT_CVT_SHIFT));
}
EMIT_DPU(REG_DPU_EW_OP_VALUE_0, 0);
EMIT_DPU(REG_DPU_EW_OP_VALUE_1, 0);
EMIT_DPU(REG_DPU_EW_OP_VALUE_2, 0);
EMIT_DPU(REG_DPU_EW_OP_VALUE_3, 0);
EMIT_DPU(REG_DPU_EW_OP_VALUE_4, 0);
EMIT_DPU(REG_DPU_EW_OP_VALUE_5, 0);
EMIT_DPU(REG_DPU_EW_OP_VALUE_6, 0);
EMIT_DPU(REG_DPU_EW_OP_VALUE_7, 0);
EMIT_DPU(REG_DPU_SURFACE_ADD, F(p->surfaces_per_row, DPU_SURFACE_ADD_SURF_ADD));
EMIT_RAW(0x1001, 0x40c4, 0);
EMIT_DPU(REG_DPU_LUT_ACCESS_CFG, 0);
EMIT_DPU(REG_DPU_LUT_ACCESS_DATA, 0);
EMIT_DPU(REG_DPU_LUT_CFG, 0);
EMIT_DPU(REG_DPU_LUT_INFO, 0);
EMIT_DPU(REG_DPU_LUT_LE_START, 0);
EMIT_DPU(REG_DPU_LUT_LE_END, 0);
EMIT_DPU(REG_DPU_LUT_LO_START, 0);
EMIT_DPU(REG_DPU_LUT_LO_END, 0);
EMIT_DPU(REG_DPU_LUT_LE_SLOPE_SCALE, 0);
EMIT_DPU(REG_DPU_LUT_LE_SLOPE_SHIFT, 0);
EMIT_DPU(REG_DPU_LUT_LO_SLOPE_SCALE, 0);
EMIT_DPU(REG_DPU_LUT_LO_SLOPE_SHIFT, 0);
EMIT_RDMA(REG_DPU_RDMA_RDMA_DATA_CUBE_WIDTH,
F(p->output_width - 1, DPU_RDMA_RDMA_DATA_CUBE_WIDTH_WIDTH));
EMIT_RDMA(REG_DPU_RDMA_RDMA_DATA_CUBE_HEIGHT,
F(p->output_height - 1, DPU_RDMA_RDMA_DATA_CUBE_HEIGHT_HEIGHT));
EMIT_RDMA(REG_DPU_RDMA_RDMA_DATA_CUBE_CHANNEL,
F(p->output_channels - 1, DPU_RDMA_RDMA_DATA_CUBE_CHANNEL_CHANNEL));
if (p->add_tensor != -1) {
EMIT_RDMA(REG_DPU_RDMA_RDMA_SRC_BASE_ADDR,
p->add_tensor_addr +
p->output_offset);
} else {
EMIT_RDMA(REG_DPU_RDMA_RDMA_SRC_BASE_ADDR, 0);
}
EMIT_RDMA(REG_DPU_RDMA_RDMA_BRDMA_CFG, F(1, DPU_RDMA_RDMA_BRDMA_CFG_BRDMA_DATA_USE));
EMIT_RDMA(REG_DPU_RDMA_RDMA_BS_BASE_ADDR,
p->biases_addr);
EMIT_RDMA(REG_DPU_RDMA_RDMA_NRDMA_CFG, 0);
EMIT_RDMA(REG_DPU_RDMA_RDMA_BN_BASE_ADDR, 0);
unsigned ew_stride =
MAX2(p->output_width * p->output_height, 12);
if (p->add_tensor != -1) {
EMIT_RDMA(REG_DPU_RDMA_RDMA_ERDMA_CFG,
F(1, DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_DATA_MODE) |
F(1, DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_DATA_SIZE));
unsigned ew_base_offset =
p->output_width * p->output_height * ATOMIC_K_SIZE;
EMIT_RDMA(REG_DPU_RDMA_RDMA_EW_BASE_ADDR,
p->add_tensor_addr +
p->output_offset + ew_base_offset);
EMIT_RDMA(REG_DPU_RDMA_RDMA_EW_SURF_STRIDE,
F(ew_stride, DPU_RDMA_RDMA_EW_SURF_STRIDE_EW_SURF_STRIDE));
} else {
EMIT_RDMA(REG_DPU_RDMA_RDMA_ERDMA_CFG, F(1, DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_DISABLE));
EMIT_RDMA(REG_DPU_RDMA_RDMA_EW_BASE_ADDR, 0);
EMIT_RDMA(REG_DPU_RDMA_RDMA_EW_SURF_STRIDE, 0);
}
uint32_t rdma_feat_mode_cfg = 0x0;
if (p->add_tensor != -1) {
rdma_feat_mode_cfg |= F(15, DPU_RDMA_RDMA_FEATURE_MODE_CFG_BURST_LEN) |
F(5, DPU_RDMA_RDMA_FEATURE_MODE_CFG_COMB_USE);
} else {
rdma_feat_mode_cfg |= F(15, DPU_RDMA_RDMA_FEATURE_MODE_CFG_BURST_LEN) |
F(1, DPU_RDMA_RDMA_FEATURE_MODE_CFG_MRDMA_DISABLE);
}
if (p->depthwise)
rdma_feat_mode_cfg |= F(3, DPU_RDMA_RDMA_FEATURE_MODE_CFG_CONV_MODE);
EMIT_RDMA(REG_DPU_RDMA_RDMA_FEATURE_MODE_CFG, rdma_feat_mode_cfg);
EMIT_RDMA(REG_DPU_RDMA_RDMA_SRC_DMA_CFG, 0);
unsigned surf_notch =
ew_stride +
p->output_width * (p->output_height - p->output_height);
if (p->input_width == 3) {
surf_notch = 15;
}
if (p->add_tensor != -1) {
EMIT_RDMA(REG_DPU_RDMA_RDMA_SURF_NOTCH,
F(surf_notch, DPU_RDMA_RDMA_SURF_NOTCH_SURF_NOTCH_ADDR));
} else {
EMIT_RDMA(REG_DPU_RDMA_RDMA_SURF_NOTCH, 0);
}
EMIT_RDMA(REG_DPU_RDMA_RDMA_PAD_CFG, 0);
EMIT_RDMA(REG_DPU_RDMA_RDMA_WEIGHT,
F(1, DPU_RDMA_RDMA_WEIGHT_E_WEIGHT) | F(1, DPU_RDMA_RDMA_WEIGHT_N_WEIGHT) |
F(1, DPU_RDMA_RDMA_WEIGHT_B_WEIGHT) | F(1, DPU_RDMA_RDMA_WEIGHT_M_WEIGHT));
if (p->add_tensor != -1) {
EMIT_RDMA(REG_DPU_RDMA_RDMA_EW_SURF_NOTCH,
F(surf_notch, DPU_RDMA_RDMA_EW_SURF_NOTCH_EW_SURF_NOTCH));
} else {
EMIT_RDMA(REG_DPU_RDMA_RDMA_EW_SURF_NOTCH, 0x0);
}
if (num_tasks == 1)
RAW64(0x0);
else
EMIT_PC(REG_PC_BASE_ADDRESS, 0);
EMIT_PC(REG_PC_REGISTER_AMOUNTS, 0);
/* TRM: before op_en, 64'h0041_xxxx_xxxx_xxxx must be set. */
RAW64(0x0041000000000000);
/* TRM: 64'h0081_0000_007f_0008 will set each block's op_en(CNA, CORE, ...,
* PPU_RDMA). */
EMIT_RAW(0x81, REG_PC_OPERATION_ENABLE,
F(14, PC_OPERATION_ENABLE_RESERVED_0) | F(1, PC_OPERATION_ENABLE_OP_EN));
return n;
}
ggml/src/ggml-rocket/rkt_matmul_coredpu.h
+25
View File
@@ -0,0 +1,25 @@
#ifndef RKT_MATMUL_COREDPU_H
#define RKT_MATMUL_COREDPU_H
#include <stdint.h>
/* Fields the Mesa CORE/DPU/PC emit reads (task-/operation-level flattened). */
struct coredpu_params {
uint32_t input_width;
uint32_t output_width, output_height;
uint32_t output_channels, output_channels_real;
uint32_t output_offset, output_surface_stride, surfaces_per_row;
uint32_t truncate_bits;
int32_t output_zero_point, weights_zero_point;
int depthwise;
int32_t addition_offset;
int32_t add_tensor; /* index, or -1 if none */
float input_scale, weights_scale, output_scale, addition_scale;
uint64_t output_addr, add_tensor_addr, biases_addr; /* pre-computed DMA addresses */
};
/* Append CORE+DPU+PC register commands (packed uint64_t) into out[] (capacity cap).
Returns number written, or -1 on overflow. */
int rkt_emit_coredpu(uint64_t *out, int cap, const struct coredpu_params *p);
#endif /* RKT_MATMUL_COREDPU_H */
ggml/src/ggml-rocket/rkt_npu_matmul.c
+165
View File
@@ -0,0 +1,165 @@
/* SPDX-License-Identifier: MIT
*
* rkt_npu_matmul.c — reusable 2D-tiled INT8 matmul on the rocket NPU (see .h).
*/
#define _GNU_SOURCE
#include "rkt_npu_matmul.h"
#include "librocket.h"
#include "rocket_accel.h"
#include "rkt_matmul.h"
#include "rkt_operands.h"
#include "rkt_gemm.h"
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <sys/mman.h>
#include <time.h>
/* rocket_prep_bo timeout is an ABSOLUTE CLOCK_MONOTONIC deadline. Build one
* rel_ns in the future, for the job-completion wait so a hung NPU job returns
* an error instead of blocking forever (the driver watchdog is ~5s). */
static int64_t rkt_deadline_ns(int64_t rel_ns)
{
struct timespec t;
clock_gettime(CLOCK_MONOTONIC, &t);
return (int64_t)t.tv_sec * 1000000000LL + (int64_t)t.tv_nsec + rel_ns;
}
int rkt_build_matmul_regcmd_scaled(uint64_t *out, int out_capacity,
uint32_t M, uint32_t N, uint32_t K,
uint64_t input_dma, uint64_t weights_dma,
uint64_t output_dma,
int32_t input_zero_point, int32_t weight_zero_point,
int32_t output_zero_point,
float input_scale, float weights_scale,
float output_scale, uint64_t bias_dma);
struct rkt_bo { uint32_t h; uint64_t dma, off; void *map; uint32_t sz; };
static int bo_alloc(int fd, struct rkt_bo *b, uint32_t size)
{
b->sz = (size + 0xfffu) & ~0xfffu;
if (rocket_create_bo(fd, b->sz, &b->h, &b->dma, &b->off))
return -1;
b->map = rocket_mmap_bo(fd, b->off, b->sz);
return b->map == MAP_FAILED ? -1 : 0;
}
/* prep(CPU) -> zero -> copy -> fini(device); prep uses INT64_MAX = block. */
static void bo_write(int fd, struct rkt_bo *b, const void *src, unsigned n)
{
rocket_prep_bo(fd, b->h, INT64_MAX);
memset(b->map, 0, b->sz);
if (src)
memcpy(b->map, src, n);
rocket_fini_bo(fd, b->h);
}
int rkt_npu_matmul(int fd, const uint8_t *X, const uint8_t *Wc,
const int32_t *bias, uint32_t M, uint32_t N, uint32_t K,
uint8_t izp, uint8_t wzp, uint8_t ozp,
float in_scale, float wt_scale, float out_scale, const float *out_scales,
uint32_t tile_m, uint32_t tile_n, uint8_t *Y)
{
int ret = -1;
int32_t *bias0 = NULL;
if (!bias) {
bias0 = calloc(N, sizeof(int32_t));
if (!bias0)
return -1;
bias = bias0;
}
unsigned max_isz = rkt_raw_input_size(1, tile_m, K);
unsigned max_wsz = rkt_packed_weights_size(1, 1, K, tile_n);
uint8_t *Xt = malloc((size_t)tile_m * K);
uint8_t *Wt = malloc((size_t)tile_n * K);
uint8_t *ipk = calloc(1, max_isz);
uint8_t *wpk = calloc(1, max_wsz);
int32_t *bpk = malloc((size_t)tile_n * sizeof(int32_t));
struct rkt_gemm_tile *tiles = malloc(
((M + tile_m - 1) / tile_m) * ((N + tile_n - 1) / tile_n) *
sizeof(struct rkt_gemm_tile));
uint64_t *rc = malloc(4096 * sizeof(uint64_t));
if (!Xt || !Wt || !ipk || !wpk || !bpk || !tiles || !rc)
goto out;
int nt = rkt_gemm_plan(M, N, tile_m, tile_n, tiles,
(int)(((M + tile_m - 1) / tile_m) *
((N + tile_n - 1) / tile_n)));
if (nt <= 0)
goto out;
for (int t = 0; t < nt; t++) {
unsigned r = tiles[t].row, cc = tiles[t].col;
unsigned m = tiles[t].m, n = tiles[t].n;
for (unsigned y = 0; y < m; y++)
memcpy(Xt + (size_t)y * K, X + (size_t)(r + y) * K, K);
for (unsigned j = 0; j < n; j++)
memcpy(Wt + (size_t)j * K, Wc + (size_t)(cc + j) * K, K);
rkt_pack_input(Xt, 1, m, K, izp, ipk);
rkt_pack_weights(Wt, 1, 1, K, n, wzp, wpk);
rkt_compute_biases(Wt, bias + cc, 1, 1, K, n, wzp, izp, bpk);
struct rkt_bo in = {0}, w = {0}, b = {0}, o = {0}, reg = {0};
if (bo_alloc(fd, &in, rkt_raw_input_size(1, m, K)) ||
bo_alloc(fd, &w, rkt_packed_weights_size(1, 1, K, n)) ||
bo_alloc(fd, &b, n * sizeof(int32_t)) ||
bo_alloc(fd, &o, rkt_raw_output_size(1, m, n)) ||
bo_alloc(fd, &reg, 0x1000))
goto out;
bo_write(fd, &in, ipk, rkt_raw_input_size(1, m, K));
bo_write(fd, &w, wpk, rkt_packed_weights_size(1, 1, K, n));
bo_write(fd, &b, bpk, n * sizeof(int32_t));
bo_write(fd, &o, NULL, 0);
int nw = rkt_build_matmul_regcmd_scaled(
rc, 4096, m, n, K, in.dma, w.dma, o.dma, izp, wzp, ozp,
in_scale, wt_scale, out_scales ? out_scales[cc] : out_scale, b.dma);
if (nw < 0) {
rocket_close_bo(fd, in.h); rocket_close_bo(fd, w.h);
rocket_close_bo(fd, b.h); rocket_close_bo(fd, o.h);
rocket_close_bo(fd, reg.h);
goto out;
}
bo_write(fd, &reg, rc, (unsigned)nw * sizeof(uint64_t));
struct drm_rocket_task task = { .regcmd = (uint32_t)reg.dma,
.regcmd_count = (uint32_t)nw };
uint32_t ih[] = { in.h, w.h, b.h, reg.h }, oh[] = { o.h };
struct drm_rocket_job job;
memset(&job, 0, sizeof job);
job.tasks = (uintptr_t)&task;
job.task_count = 1;
job.task_struct_size = sizeof(struct drm_rocket_task);
job.in_bo_handles = (uintptr_t)ih;
job.in_bo_handle_count = 4;
job.out_bo_handles = (uintptr_t)oh;
job.out_bo_handle_count = 1;
int sr = rocket_submit(fd, &job, 1);
int wr = sr ? -1 : rocket_prep_bo(fd, o.h, rkt_deadline_ns(6000000000LL));
if (!sr && !wr) {
uint8_t *got = malloc((size_t)m * n);
if (got) {
rkt_unpack_output(o.map, 1, m, n, got);
for (unsigned y = 0; y < m; y++)
memcpy(Y + (size_t)(r + y) * N + cc,
got + (size_t)y * n, n);
free(got);
}
}
rocket_close_bo(fd, in.h); rocket_close_bo(fd, w.h);
rocket_close_bo(fd, b.h); rocket_close_bo(fd, o.h);
rocket_close_bo(fd, reg.h);
if (sr || wr)
goto out;
}
ret = 0;
out:
free(Xt); free(Wt); free(ipk); free(wpk); free(bpk);
free(tiles); free(rc); free(bias0);
return ret;
}
ggml/src/ggml-rocket/rkt_npu_matmul.h
+35
View File
@@ -0,0 +1,35 @@
/* SPDX-License-Identifier: MIT
*
* rkt_npu_matmul.h — reusable arbitrary-size INT8 matmul on the rocket NPU.
*
* The U6 core primitive: Y[M][N] = X[M][K] * W[K][N] computed by 2D-tiling
* into single-op rocket matmuls (rkt_gemm_plan + rkt_build_matmul_regcmd +
* librocket). Intended to be called from a ggml MUL_MAT backend.
*/
#ifndef RKT_NPU_MATMUL_H
#define RKT_NPU_MATMUL_H
#include <stdint.h>
/*
* fd open /dev/accel/accel0 (rocket_open).
* X [M][K] uint8 row-major activations.
* Wc [N][K] uint8 conv-order weights (Wc[out_ch][in_ch]).
* bias [N] int32 per-output-channel bias, or NULL for none.
* izp/wzp/ozp int8 zero-points (0-255).
* in/wt/out_scale per-tensor requant scales.
* tile_m/tile_n desired tile sizes; each (m x n, full-K) tile must satisfy
* rkt_gemm_op_fits (m limited by CBUF input banks, K*n <= ~327k).
* Y [M][N] uint8 output (caller-allocated).
*
* Returns 0 on success, -1 on error (bad args / tile too big / submit fail).
* Allocates weights/bias/input/output/regcmd BOs per tile (correctness-first;
* weight-reuse across M-tiles is a later perf optimization).
*/
int rkt_npu_matmul(int fd, const uint8_t *X, const uint8_t *Wc,
const int32_t *bias, uint32_t M, uint32_t N, uint32_t K,
uint8_t izp, uint8_t wzp, uint8_t ozp,
float in_scale, float wt_scale, float out_scale, const float *out_scales,
uint32_t tile_m, uint32_t tile_n, uint8_t *Y);
#endif /* RKT_NPU_MATMUL_H */
ggml/src/ggml-rocket/rkt_operands.c
+127
View File
@@ -0,0 +1,127 @@
/* SPDX-License-Identifier: MIT
*
* rkt_operands.c — NPU operand layout (faithful ports of Mesa rocket).
*
* Non-depthwise conv/matmul. Row-major host tensors:
* weights_in : [oc_real][ww][wh][ic_real] (conv weights)
* input_in : [iw][ih][ic_real]
* output_out : [oh][ow][oc_real]
*
* Ported from Mesa rkt_coefs.c (rkt_fill_weights, rkt_fill_biases) and
* rkt_ml.c (feature tiling in rkt_ml_subgraph_invoke, output de-tile in
* rkt_ml_subgraph_read_outputs). Assumes input_zero_point handling as Mesa.
*/
#include "rkt_operands.h"
#define FEATURE_ATOMIC_SIZE 16
#define WEIGHT_ATOMIC_SIZE 32
#define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d))
#define MAX2(a, b) ((a) > (b) ? (a) : (b))
#define MIN2(a, b) ((a) < (b) ? (a) : (b))
static unsigned alignu(unsigned v, unsigned a) { return ((v + a - 1) / a) * a; }
unsigned rkt_raw_input_size(unsigned iw, unsigned ih, unsigned ic_real)
{
return iw * ih * (DIV_ROUND_UP(ic_real, FEATURE_ATOMIC_SIZE) * 2) *
FEATURE_ATOMIC_SIZE;
}
unsigned rkt_raw_output_size(unsigned ow, unsigned oh, unsigned oc_real)
{
return ow * oh * (DIV_ROUND_UP(oc_real, FEATURE_ATOMIC_SIZE) * 2) *
FEATURE_ATOMIC_SIZE;
}
unsigned rkt_packed_weights_size(unsigned ww, unsigned wh, unsigned ic_real,
unsigned oc_real)
{
unsigned ic = alignu(MAX2(ic_real, FEATURE_ATOMIC_SIZE), FEATURE_ATOMIC_SIZE);
unsigned oc = alignu(oc_real, 2);
return ww * wh * oc * alignu(ic, WEIGHT_ATOMIC_SIZE) * 2;
}
/* Mesa rkt_fill_weights (non-depthwise). */
unsigned rkt_pack_weights(const uint8_t *win, unsigned ww, unsigned wh,
unsigned ic_real, unsigned oc_real, uint8_t wzp,
uint8_t *out)
{
unsigned ic = alignu(MAX2(ic_real, FEATURE_ATOMIC_SIZE), FEATURE_ATOMIC_SIZE);
unsigned oc = alignu(oc_real, 2);
unsigned groups = WEIGHT_ATOMIC_SIZE;
unsigned ic1n = DIV_ROUND_UP(ic, groups);
unsigned ic2n = MIN2(ic, groups);
unsigned n = 0;
for (unsigned oc1 = 0; oc1 < DIV_ROUND_UP(oc, WEIGHT_ATOMIC_SIZE); oc1++)
for (unsigned ic1 = 0; ic1 < ic1n; ic1++)
for (unsigned x = 0; x < ww; x++)
for (unsigned y = 0; y < wh; y++)
for (unsigned oc2 = 0;
oc2 < MIN2(oc, WEIGHT_ATOMIC_SIZE); oc2++)
for (unsigned ic2 = 0; ic2 < ic2n; ic2++) {
unsigned occ = oc1 * WEIGHT_ATOMIC_SIZE + oc2;
unsigned icc = ic1 * groups + ic2;
if (oc_real > 2 && occ >= alignu(oc_real, 2))
continue;
if (occ >= oc_real) {
out[n++] = 0x0;
} else if (icc >= ic_real) {
if (ic2 < 16 || (ic_real % 32) > 16)
out[n++] = (uint8_t)(wzp - 0x80);
} else {
unsigned idx = ((occ * ww + x) * wh + y) * ic_real + icc;
out[n++] = (uint8_t)(win[idx] - 0x80);
}
}
return n;
}
/* Mesa rkt_fill_biases (truncate_bits=0 path) + calculate_bias_correction. */
void rkt_compute_biases(const uint8_t *win, const int32_t *bin, unsigned ww,
unsigned wh, unsigned ic_real, unsigned oc_real,
uint8_t wzp, uint8_t izp, int32_t *bout)
{
for (unsigned oc = 0; oc < oc_real; oc++) {
int32_t corr = 0;
for (unsigned x = 0; x < ww; x++)
for (unsigned y = 0; y < wh; y++)
for (unsigned ic = 0; ic < ic_real; ic++) {
unsigned idx = ((oc * ww + x) * wh + y) * ic_real + ic;
corr += ((int32_t)win[idx] - wzp) *
((int32_t)izp - 0x80);
}
bout[oc] = bin[oc] - corr;
}
}
/* Mesa feature tiling (rkt_ml_subgraph_invoke, input_channels>1 path). */
unsigned rkt_pack_input(const uint8_t *iin, unsigned iw, unsigned ih,
unsigned ic_real, uint8_t izp, uint8_t *out)
{
unsigned n = 0;
for (unsigned u = 0; u < DIV_ROUND_UP(ic_real, FEATURE_ATOMIC_SIZE); u++)
for (unsigned x = 0; x < iw; x++)
for (unsigned y = 0; y < ih; y++)
for (unsigned c = 0; c < FEATURE_ATOMIC_SIZE; c++) {
unsigned ich = c + u * FEATURE_ATOMIC_SIZE;
if (ich < ic_real)
out[n++] = (uint8_t)(iin[(x * ih + y) * ic_real + ich] - 0x80);
else
out[n++] = (uint8_t)(izp - 0x80);
}
return n;
}
/* Mesa output de-tile (rkt_ml_subgraph_read_outputs). */
void rkt_unpack_output(const uint8_t *raw, unsigned ow, unsigned oh,
unsigned oc_real, uint8_t *out)
{
for (unsigned oc = 0; oc < oc_real; oc++)
for (unsigned x = 0; x < ow; x++)
for (unsigned y = 0; y < oh; y++) {
unsigned c = oc % FEATURE_ATOMIC_SIZE;
unsigned g = oc / FEATURE_ATOMIC_SIZE;
out[(y * ow + x) * oc_real + oc] =
(uint8_t)(raw[((g * oh + y) * ow + x) * FEATURE_ATOMIC_SIZE + c] + 0x80);
}
}
ggml/src/ggml-rocket/rkt_operands.h
+30
View File
@@ -0,0 +1,30 @@
/* SPDX-License-Identifier: MIT
*
* rkt_operands.h — NPU operand layout ports (see rkt_operands.c).
* Non-depthwise. Row-major host tensors:
* weights_in [oc_real][ww][wh][ic_real], input_in [iw][ih][ic_real],
* output_out [oh][ow][oc_real].
*/
#ifndef RKT_OPERANDS_H
#define RKT_OPERANDS_H
#include <stdint.h>
unsigned rkt_raw_input_size(unsigned iw, unsigned ih, unsigned ic_real);
unsigned rkt_raw_output_size(unsigned ow, unsigned oh, unsigned oc_real);
unsigned rkt_packed_weights_size(unsigned ww, unsigned wh, unsigned ic_real,
unsigned oc_real);
unsigned rkt_pack_weights(const uint8_t *weights_in, unsigned ww, unsigned wh,
unsigned ic_real, unsigned oc_real, uint8_t wzp,
uint8_t *out);
void rkt_compute_biases(const uint8_t *weights_in, const int32_t *biases_in,
unsigned ww, unsigned wh, unsigned ic_real,
unsigned oc_real, uint8_t wzp, uint8_t izp,
int32_t *biases_out);
unsigned rkt_pack_input(const uint8_t *input_in, unsigned iw, unsigned ih,
unsigned ic_real, uint8_t izp, uint8_t *out);
void rkt_unpack_output(const uint8_t *raw, unsigned ow, unsigned oh,
unsigned oc_real, uint8_t *out);
#endif /* RKT_OPERANDS_H */
ggml/src/ggml-rocket/rocket_accel.h
+192
View File
@@ -0,0 +1,192 @@
/* SPDX-License-Identifier: MIT */
/*
* Copyright © 2024 Tomeu Vizoso
*/
#ifndef __DRM_UAPI_ROCKET_ACCEL_H__
#define __DRM_UAPI_ROCKET_ACCEL_H__
#include "drm.h"
#if defined(__cplusplus)
extern "C" {
#endif
#define DRM_ROCKET_CREATE_BO 0x00
#define DRM_ROCKET_SUBMIT 0x01
#define DRM_ROCKET_PREP_BO 0x02
#define DRM_ROCKET_FINI_BO 0x03
#define DRM_IOCTL_ROCKET_CREATE_BO DRM_IOWR(DRM_COMMAND_BASE + DRM_ROCKET_CREATE_BO, struct drm_rocket_create_bo)
#define DRM_IOCTL_ROCKET_SUBMIT DRM_IOW(DRM_COMMAND_BASE + DRM_ROCKET_SUBMIT, struct drm_rocket_submit)
#define DRM_IOCTL_ROCKET_PREP_BO DRM_IOW(DRM_COMMAND_BASE + DRM_ROCKET_PREP_BO, struct drm_rocket_prep_bo)
#define DRM_IOCTL_ROCKET_FINI_BO DRM_IOW(DRM_COMMAND_BASE + DRM_ROCKET_FINI_BO, struct drm_rocket_fini_bo)
/**
* struct drm_rocket_create_bo - ioctl argument for creating Rocket BOs.
*
*/
struct drm_rocket_create_bo {
/**
* @size: Input: Size of the requested BO.
*/
__u32 size;
/**
* @handle: Output: GEM handle for the BO.
*/
__u32 handle;
/**
* @dma_address: Output: DMA address for the BO in the NPU address
* space. This address is private to the DRM fd and is valid for
* the lifetime of the GEM handle.
*/
__u64 dma_address;
/**
* @offset: Output: Offset into the drm node to use for subsequent
* mmap call.
*/
__u64 offset;
};
/**
* struct drm_rocket_prep_bo - ioctl argument for starting CPU ownership of the BO.
*
* Takes care of waiting for any NPU jobs that might still use the NPU and performs cache
* synchronization.
*/
struct drm_rocket_prep_bo {
/**
* @handle: Input: GEM handle of the buffer object.
*/
__u32 handle;
/**
* @reserved: Reserved, must be zero.
*/
__u32 reserved;
/**
* @timeout_ns: Input: Amount of time to wait for NPU jobs.
*/
__s64 timeout_ns;
};
/**
* struct drm_rocket_fini_bo - ioctl argument for finishing CPU ownership of the BO.
*
* Synchronize caches for NPU access.
*/
struct drm_rocket_fini_bo {
/**
* @handle: Input: GEM handle of the buffer object.
*/
__u32 handle;
/**
* @reserved: Reserved, must be zero.
*/
__u32 reserved;
};
/**
* struct drm_rocket_task - A task to be run on the NPU
*
* A task is the smallest unit of work that can be run on the NPU.
*/
struct drm_rocket_task {
/**
* @regcmd: Input: DMA address to NPU mapping of register command buffer
*/
__u32 regcmd;
/**
* @regcmd_count: Input: Number of commands in the register command
* buffer
*/
__u32 regcmd_count;
};
/**
* struct drm_rocket_job - A job to be run on the NPU
*
* The kernel will schedule the execution of this job taking into account its
* dependencies with other jobs. All tasks in the same job will be executed
* sequentially on the same core, to benefit from memory residency in SRAM.
*/
struct drm_rocket_job {
/**
* @tasks: Input: Pointer to an array of struct drm_rocket_task.
*/
__u64 tasks;
/**
* @in_bo_handles: Input: Pointer to a u32 array of the BOs that
* are read by the job.
*/
__u64 in_bo_handles;
/**
* @out_bo_handles: Input: Pointer to a u32 array of the BOs that
* are written to by the job.
*/
__u64 out_bo_handles;
/**
* @task_count: Input: Number of tasks passed in.
*/
__u32 task_count;
/**
* @task_struct_size: Input: Size in bytes of the structs in the
* @tasks field.
*/
__u32 task_struct_size;
/**
* @in_bo_handle_count: Input: Number of input BO handles passed in
* (size is that times 4).
*/
__u32 in_bo_handle_count;
/**
* @out_bo_handle_count: Input: Number of output BO handles passed in
* (size is that times 4).
*/
__u32 out_bo_handle_count;
};
/**
* struct drm_rocket_submit - ioctl argument for submitting commands to the NPU.
*
* The kernel will schedule the execution of these jobs in dependency order.
*/
struct drm_rocket_submit {
/**
* @jobs: Input: Pointer to an array of struct drm_rocket_job.
*/
__u64 jobs;
/**
* @job_count: Input: Number of jobs passed in.
*/
__u32 job_count;
/**
* @job_struct_size: Input: Size in bytes of the structs in the
* @jobs field.
*/
__u32 job_struct_size;
/**
* @reserved: Reserved, must be zero.
*/
__u64 reserved;
};
#if defined(__cplusplus)
}
#endif
#endif /* __DRM_UAPI_ROCKET_ACCEL_H__ */