From 8e49f43486bb663523d4328867ed24d824d4e12a Mon Sep 17 00:00:00 2001 From: mfritsche Date: Tue, 14 Jul 2026 10:52:06 +0200 Subject: [PATCH] ggml: add rocket NPU backend (RK3588 mainline accel driver) New MUL_MAT backend for the RK3588 NPU via the mainline 'rocket' DRM-accel driver (/dev/accel/accel0), with no vendor RKNPU2 userspace. Modeled on ggml-blas: an ACCEL device with CPU-resident weights; the scheduler offloads supported MUL_MAT nodes and leaves the rest on CPU. Per plane: dequantize weight to F32 (ggml to_float), requantize per-output- channel to int8, quantize F32 activation per-tensor to int8, run the tiled int8 GEMM on the NPU (rkt_npu_matmul, ported from the rosenblatt npu-probe: Mesa-rocket regcmd builder + NVDLA CBUF tiling + librocket DRM ioctls), then dequantize the int8 result. out_scale is a Cauchy-Schwarz bound on |output| so the int8 output never clips. Empirically-verified HW limits gate supports_op: TILE_M=16, TILE_N=128, K<=8192 (int8 K-limit), K%16==0, F32 activations, dequantizable weight. Job-completion wait uses a finite 6s absolute deadline so a hung NPU job errors instead of deadlocking. Enable with -DGGML_ROCKET=ON (and -DLLAMA_RKNPU2=OFF so the vendor backend does not claim the weight buffers). Verified end-to-end: gemma-4-E2B-it-Q8_0 generates coherent text on boltzmann's mainline rocket kernel, matmuls offloaded to the NPU, ~3.9 tok/s eval. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01EWpfhDgYNA21tETDP9ueBE --- ggml/CMakeLists.txt | 1 + ggml/src/CMakeLists.txt | 1 + ggml/src/ggml-backend-reg.cpp | 7 + ggml/src/ggml-rocket/CMakeLists.txt | 21 + ggml/src/ggml-rocket/cna_defs.h | 532 +++++++++++++++ ggml/src/ggml-rocket/coredpu_defs.h | 769 ++++++++++++++++++++++ ggml/src/ggml-rocket/ggml-rocket.cpp | 425 ++++++++++++ ggml/src/ggml-rocket/ggml-rocket.h | 13 + ggml/src/ggml-rocket/librocket.c | 106 +++ ggml/src/ggml-rocket/librocket.h | 87 +++ ggml/src/ggml-rocket/rkt_gemm.c | 36 + ggml/src/ggml-rocket/rkt_gemm.h | 36 + ggml/src/ggml-rocket/rkt_matmul.c | 255 +++++++ ggml/src/ggml-rocket/rkt_matmul.h | 64 ++ ggml/src/ggml-rocket/rkt_matmul_cna.c | 171 +++++ ggml/src/ggml-rocket/rkt_matmul_cna.h | 25 + ggml/src/ggml-rocket/rkt_matmul_coredpu.c | 298 +++++++++ ggml/src/ggml-rocket/rkt_matmul_coredpu.h | 25 + ggml/src/ggml-rocket/rkt_npu_matmul.c | 165 +++++ ggml/src/ggml-rocket/rkt_npu_matmul.h | 35 + ggml/src/ggml-rocket/rkt_operands.c | 127 ++++ ggml/src/ggml-rocket/rkt_operands.h | 30 + ggml/src/ggml-rocket/rocket_accel.h | 192 ++++++ 23 files changed, 3421 insertions(+) create mode 100644 ggml/src/ggml-rocket/CMakeLists.txt create mode 100644 ggml/src/ggml-rocket/cna_defs.h create mode 100644 ggml/src/ggml-rocket/coredpu_defs.h create mode 100644 ggml/src/ggml-rocket/ggml-rocket.cpp create mode 100644 ggml/src/ggml-rocket/ggml-rocket.h create mode 100644 ggml/src/ggml-rocket/librocket.c create mode 100644 ggml/src/ggml-rocket/librocket.h create mode 100644 ggml/src/ggml-rocket/rkt_gemm.c create mode 100644 ggml/src/ggml-rocket/rkt_gemm.h create mode 100644 ggml/src/ggml-rocket/rkt_matmul.c create mode 100644 ggml/src/ggml-rocket/rkt_matmul.h create mode 100644 ggml/src/ggml-rocket/rkt_matmul_cna.c create mode 100644 ggml/src/ggml-rocket/rkt_matmul_cna.h create mode 100644 ggml/src/ggml-rocket/rkt_matmul_coredpu.c create mode 100644 ggml/src/ggml-rocket/rkt_matmul_coredpu.h create mode 100644 ggml/src/ggml-rocket/rkt_npu_matmul.c create mode 100644 ggml/src/ggml-rocket/rkt_npu_matmul.h create mode 100644 ggml/src/ggml-rocket/rkt_operands.c create mode 100644 ggml/src/ggml-rocket/rkt_operands.h create mode 100644 ggml/src/ggml-rocket/rocket_accel.h diff --git a/ggml/CMakeLists.txt b/ggml/CMakeLists.txt index c94a40f94..4e4efbcd6 100644 --- a/ggml/CMakeLists.txt +++ b/ggml/CMakeLists.txt @@ -267,6 +267,7 @@ set (GGML_OPENCL_TARGET_VERSION "300" CACHE STRING option(GGML_HEXAGON "ggml: enable Hexagon backend" OFF) option(GGML_RKNPU2 "ggml: use RKNPU2" OFF) +option(GGML_ROCKET "ggml: use RK3588 NPU (rocket accel)" OFF) set(GGML_HEXAGON_FP32_QUANTIZE_GROUP_SIZE 128 CACHE STRING "ggml: quantize group size (32, 64, or 128)") # toolchain for vulkan-shaders-gen diff --git a/ggml/src/CMakeLists.txt b/ggml/src/CMakeLists.txt index 2c93a1de9..da75c8d4e 100644 --- a/ggml/src/CMakeLists.txt +++ b/ggml/src/CMakeLists.txt @@ -480,6 +480,7 @@ ggml_add_backend(Hexagon) ggml_add_backend(ZenDNN) ggml_add_backend(OPENVINO) ggml_add_backend(RKNPU2) +ggml_add_backend(ROCKET) foreach (target ggml-base ggml) target_include_directories(${target} PUBLIC $ $) diff --git a/ggml/src/ggml-backend-reg.cpp b/ggml/src/ggml-backend-reg.cpp index 3aa8ed6c0..b29f1e4f0 100644 --- a/ggml/src/ggml-backend-reg.cpp +++ b/ggml/src/ggml-backend-reg.cpp @@ -90,6 +90,10 @@ #include "ggml-rknpu2.h" #endif +#ifdef GGML_USE_ROCKET +#include "ggml-rocket.h" +#endif + // disable C++17 deprecation warning for std::codecvt_utf8 #if defined(__clang__) # pragma clang diagnostic push @@ -179,6 +183,9 @@ struct ggml_backend_registry { #endif #ifdef GGML_USE_RKNPU2 register_backend(ggml_backend_rknpu2_reg()); +#endif +#ifdef GGML_USE_ROCKET + register_backend(ggml_backend_rocket_reg()); #endif } diff --git a/ggml/src/ggml-rocket/CMakeLists.txt b/ggml/src/ggml-rocket/CMakeLists.txt new file mode 100644 index 000000000..19ec91a2e --- /dev/null +++ b/ggml/src/ggml-rocket/CMakeLists.txt @@ -0,0 +1,21 @@ +# ggml-rocket: RK3588 NPU backend via mainline "rocket" DRM-accel driver. +# No vendor library — raw DRM ioctls against /dev/accel/accel0. + +ggml_add_backend_library(ggml-rocket + ggml-rocket.cpp + librocket.c + rkt_npu_matmul.c + rkt_operands.c + rkt_matmul.c + rkt_matmul_cna.c + rkt_matmul_coredpu.c + rkt_gemm.c +) + +target_include_directories(ggml-rocket + PRIVATE + /usr/include/drm + /usr/include/libdrm + PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR} +) diff --git a/ggml/src/ggml-rocket/cna_defs.h b/ggml/src/ggml-rocket/cna_defs.h new file mode 100644 index 000000000..d957d2f9d --- /dev/null +++ b/ggml/src/ggml-rocket/cna_defs.h @@ -0,0 +1,532 @@ +#ifndef CNA_DEFS_H +#define CNA_DEFS_H + +#define REG_CNA_S_STATUS 0x00001000 +#define CNA_S_STATUS_RESERVED_0__MASK 0xfffc0000 +#define CNA_S_STATUS_RESERVED_0__SHIFT 18 +#define CNA_S_STATUS_STATUS_1__MASK 0x00030000 +#define CNA_S_STATUS_STATUS_1__SHIFT 16 +#define CNA_S_STATUS_RESERVED_1__MASK 0x0000fffc +#define CNA_S_STATUS_RESERVED_1__SHIFT 2 +#define CNA_S_STATUS_STATUS_0__MASK 0x00000003 +#define CNA_S_STATUS_STATUS_0__SHIFT 0 +#define REG_CNA_S_POINTER 0x00001004 +#define CNA_S_POINTER_RESERVED_0__MASK 0xfffe0000 +#define CNA_S_POINTER_RESERVED_0__SHIFT 17 +#define CNA_S_POINTER_EXECUTER__MASK 0x00010000 +#define CNA_S_POINTER_EXECUTER__SHIFT 16 +#define CNA_S_POINTER_RESERVED_1__MASK 0x0000ffc0 +#define CNA_S_POINTER_RESERVED_1__SHIFT 6 +#define CNA_S_POINTER_EXECUTER_PP_CLEAR__MASK 0x00000020 +#define CNA_S_POINTER_EXECUTER_PP_CLEAR__SHIFT 5 +#define CNA_S_POINTER_POINTER_PP_CLEAR__MASK 0x00000010 +#define CNA_S_POINTER_POINTER_PP_CLEAR__SHIFT 4 +#define CNA_S_POINTER_POINTER_PP_MODE__MASK 0x00000008 +#define CNA_S_POINTER_POINTER_PP_MODE__SHIFT 3 +#define CNA_S_POINTER_EXECUTER_PP_EN__MASK 0x00000004 +#define CNA_S_POINTER_EXECUTER_PP_EN__SHIFT 2 +#define CNA_S_POINTER_POINTER_PP_EN__MASK 0x00000002 +#define CNA_S_POINTER_POINTER_PP_EN__SHIFT 1 +#define CNA_S_POINTER_POINTER__MASK 0x00000001 +#define CNA_S_POINTER_POINTER__SHIFT 0 +#define REG_CNA_OPERATION_ENABLE 0x00001008 +#define CNA_OPERATION_ENABLE_RESERVED_0__MASK 0xfffffffe +#define CNA_OPERATION_ENABLE_RESERVED_0__SHIFT 1 +#define CNA_OPERATION_ENABLE_OP_EN__MASK 0x00000001 +#define CNA_OPERATION_ENABLE_OP_EN__SHIFT 0 +#define REG_CNA_CONV_CON1 0x0000100c +#define CNA_CONV_CON1_RESERVED_0__MASK 0x80000000 +#define CNA_CONV_CON1_RESERVED_0__SHIFT 31 +#define CNA_CONV_CON1_NONALIGN_DMA__MASK 0x40000000 +#define CNA_CONV_CON1_NONALIGN_DMA__SHIFT 30 +#define CNA_CONV_CON1_GROUP_LINE_OFF__MASK 0x20000000 +#define CNA_CONV_CON1_GROUP_LINE_OFF__SHIFT 29 +#define CNA_CONV_CON1_RESERVED_1__MASK 0x1ffe0000 +#define CNA_CONV_CON1_RESERVED_1__SHIFT 17 +#define CNA_CONV_CON1_DECONV__MASK 0x00010000 +#define CNA_CONV_CON1_DECONV__SHIFT 16 +#define CNA_CONV_CON1_ARGB_IN__MASK 0x0000f000 +#define CNA_CONV_CON1_ARGB_IN__SHIFT 12 +#define CNA_CONV_CON1_RESERVED_2__MASK 0x00000c00 +#define CNA_CONV_CON1_RESERVED_2__SHIFT 10 +#define CNA_CONV_CON1_PROC_PRECISION__MASK 0x00000380 +#define CNA_CONV_CON1_PROC_PRECISION__SHIFT 7 +#define CNA_CONV_CON1_IN_PRECISION__MASK 0x00000070 +#define CNA_CONV_CON1_IN_PRECISION__SHIFT 4 +#define CNA_CONV_CON1_CONV_MODE__MASK 0x0000000f +#define CNA_CONV_CON1_CONV_MODE__SHIFT 0 +#define REG_CNA_CONV_CON2 0x00001010 +#define CNA_CONV_CON2_RESERVED_0__MASK 0xff000000 +#define CNA_CONV_CON2_RESERVED_0__SHIFT 24 +#define CNA_CONV_CON2_KERNEL_GROUP__MASK 0x00ff0000 +#define CNA_CONV_CON2_KERNEL_GROUP__SHIFT 16 +#define CNA_CONV_CON2_RESERVED_1__MASK 0x0000c000 +#define CNA_CONV_CON2_RESERVED_1__SHIFT 14 +#define CNA_CONV_CON2_FEATURE_GRAINS__MASK 0x00003ff0 +#define CNA_CONV_CON2_FEATURE_GRAINS__SHIFT 4 +#define CNA_CONV_CON2_RESERVED_2__MASK 0x00000008 +#define CNA_CONV_CON2_RESERVED_2__SHIFT 3 +#define CNA_CONV_CON2_CSC_WO_EN__MASK 0x00000004 +#define CNA_CONV_CON2_CSC_WO_EN__SHIFT 2 +#define CNA_CONV_CON2_CSC_DO_EN__MASK 0x00000002 +#define CNA_CONV_CON2_CSC_DO_EN__SHIFT 1 +#define CNA_CONV_CON2_CMD_FIFO_SRST__MASK 0x00000001 +#define CNA_CONV_CON2_CMD_FIFO_SRST__SHIFT 0 +#define REG_CNA_CONV_CON3 0x00001014 +#define CNA_CONV_CON3_RESERVED_0__MASK 0x80000000 +#define CNA_CONV_CON3_RESERVED_0__SHIFT 31 +#define CNA_CONV_CON3_NN_MODE__MASK 0x70000000 +#define CNA_CONV_CON3_NN_MODE__SHIFT 28 +#define CNA_CONV_CON3_RESERVED_1__MASK 0x0c000000 +#define CNA_CONV_CON3_RESERVED_1__SHIFT 26 +#define CNA_CONV_CON3_ATROUS_Y_DILATION__MASK 0x03e00000 +#define CNA_CONV_CON3_ATROUS_Y_DILATION__SHIFT 21 +#define CNA_CONV_CON3_ATROUS_X_DILATION__MASK 0x001f0000 +#define CNA_CONV_CON3_ATROUS_X_DILATION__SHIFT 16 +#define CNA_CONV_CON3_RESERVED_2__MASK 0x0000c000 +#define CNA_CONV_CON3_RESERVED_2__SHIFT 14 +#define CNA_CONV_CON3_DECONV_Y_STRIDE__MASK 0x00003800 +#define CNA_CONV_CON3_DECONV_Y_STRIDE__SHIFT 11 +#define CNA_CONV_CON3_DECONV_X_STRIDE__MASK 0x00000700 +#define CNA_CONV_CON3_DECONV_X_STRIDE__SHIFT 8 +#define CNA_CONV_CON3_RESERVED_3__MASK 0x000000c0 +#define CNA_CONV_CON3_RESERVED_3__SHIFT 6 +#define CNA_CONV_CON3_CONV_Y_STRIDE__MASK 0x00000038 +#define CNA_CONV_CON3_CONV_Y_STRIDE__SHIFT 3 +#define CNA_CONV_CON3_CONV_X_STRIDE__MASK 0x00000007 +#define CNA_CONV_CON3_CONV_X_STRIDE__SHIFT 0 +#define REG_CNA_DATA_SIZE0 0x00001020 +#define CNA_DATA_SIZE0_RESERVED_0__MASK 0xf8000000 +#define CNA_DATA_SIZE0_RESERVED_0__SHIFT 27 +#define CNA_DATA_SIZE0_DATAIN_WIDTH__MASK 0x07ff0000 +#define CNA_DATA_SIZE0_DATAIN_WIDTH__SHIFT 16 +#define CNA_DATA_SIZE0_RESERVED_1__MASK 0x0000f800 +#define CNA_DATA_SIZE0_RESERVED_1__SHIFT 11 +#define CNA_DATA_SIZE0_DATAIN_HEIGHT__MASK 0x000007ff +#define CNA_DATA_SIZE0_DATAIN_HEIGHT__SHIFT 0 +#define REG_CNA_DATA_SIZE1 0x00001024 +#define CNA_DATA_SIZE1_RESERVED_0__MASK 0xc0000000 +#define CNA_DATA_SIZE1_RESERVED_0__SHIFT 30 +#define CNA_DATA_SIZE1_DATAIN_CHANNEL_REAL__MASK 0x3fff0000 +#define CNA_DATA_SIZE1_DATAIN_CHANNEL_REAL__SHIFT 16 +#define CNA_DATA_SIZE1_DATAIN_CHANNEL__MASK 0x0000ffff +#define CNA_DATA_SIZE1_DATAIN_CHANNEL__SHIFT 0 +#define REG_CNA_DATA_SIZE2 0x00001028 +#define CNA_DATA_SIZE2_RESERVED_0__MASK 0xfffff800 +#define CNA_DATA_SIZE2_RESERVED_0__SHIFT 11 +#define CNA_DATA_SIZE2_DATAOUT_WIDTH__MASK 0x000007ff +#define CNA_DATA_SIZE2_DATAOUT_WIDTH__SHIFT 0 +#define REG_CNA_DATA_SIZE3 0x0000102c +#define CNA_DATA_SIZE3_RESERVED_0__MASK 0xff000000 +#define CNA_DATA_SIZE3_RESERVED_0__SHIFT 24 +#define CNA_DATA_SIZE3_SURF_MODE__MASK 0x00c00000 +#define CNA_DATA_SIZE3_SURF_MODE__SHIFT 22 +#define CNA_DATA_SIZE3_DATAOUT_ATOMICS__MASK 0x003fffff +#define CNA_DATA_SIZE3_DATAOUT_ATOMICS__SHIFT 0 +#define REG_CNA_WEIGHT_SIZE0 0x00001030 +#define CNA_WEIGHT_SIZE0_WEIGHT_BYTES__MASK 0xffffffff +#define CNA_WEIGHT_SIZE0_WEIGHT_BYTES__SHIFT 0 +#define REG_CNA_WEIGHT_SIZE1 0x00001034 +#define CNA_WEIGHT_SIZE1_RESERVED_0__MASK 0xfff80000 +#define CNA_WEIGHT_SIZE1_RESERVED_0__SHIFT 19 +#define CNA_WEIGHT_SIZE1_WEIGHT_BYTES_PER_KERNEL__MASK 0x0007ffff +#define CNA_WEIGHT_SIZE1_WEIGHT_BYTES_PER_KERNEL__SHIFT 0 +#define REG_CNA_WEIGHT_SIZE2 0x00001038 +#define CNA_WEIGHT_SIZE2_RESERVED_0__MASK 0xe0000000 +#define CNA_WEIGHT_SIZE2_RESERVED_0__SHIFT 29 +#define CNA_WEIGHT_SIZE2_WEIGHT_WIDTH__MASK 0x1f000000 +#define CNA_WEIGHT_SIZE2_WEIGHT_WIDTH__SHIFT 24 +#define CNA_WEIGHT_SIZE2_RESERVED_1__MASK 0x00e00000 +#define CNA_WEIGHT_SIZE2_RESERVED_1__SHIFT 21 +#define CNA_WEIGHT_SIZE2_WEIGHT_HEIGHT__MASK 0x001f0000 +#define CNA_WEIGHT_SIZE2_WEIGHT_HEIGHT__SHIFT 16 +#define CNA_WEIGHT_SIZE2_RESERVED_2__MASK 0x0000c000 +#define CNA_WEIGHT_SIZE2_RESERVED_2__SHIFT 14 +#define CNA_WEIGHT_SIZE2_WEIGHT_KERNELS__MASK 0x00003fff +#define CNA_WEIGHT_SIZE2_WEIGHT_KERNELS__SHIFT 0 +#define REG_CNA_CBUF_CON0 0x00001040 +#define CNA_CBUF_CON0_RESERVED_0__MASK 0xffffc000 +#define CNA_CBUF_CON0_RESERVED_0__SHIFT 14 +#define CNA_CBUF_CON0_WEIGHT_REUSE__MASK 0x00002000 +#define CNA_CBUF_CON0_WEIGHT_REUSE__SHIFT 13 +#define CNA_CBUF_CON0_DATA_REUSE__MASK 0x00001000 +#define CNA_CBUF_CON0_DATA_REUSE__SHIFT 12 +#define CNA_CBUF_CON0_RESERVED_1__MASK 0x00000800 +#define CNA_CBUF_CON0_RESERVED_1__SHIFT 11 +#define CNA_CBUF_CON0_FC_DATA_BANK__MASK 0x00000700 +#define CNA_CBUF_CON0_FC_DATA_BANK__SHIFT 8 +#define CNA_CBUF_CON0_WEIGHT_BANK__MASK 0x000000f0 +#define CNA_CBUF_CON0_WEIGHT_BANK__SHIFT 4 +#define CNA_CBUF_CON0_DATA_BANK__MASK 0x0000000f +#define CNA_CBUF_CON0_DATA_BANK__SHIFT 0 +#define REG_CNA_CBUF_CON1 0x00001044 +#define CNA_CBUF_CON1_RESERVED_0__MASK 0xffffc000 +#define CNA_CBUF_CON1_RESERVED_0__SHIFT 14 +#define CNA_CBUF_CON1_DATA_ENTRIES__MASK 0x00003fff +#define CNA_CBUF_CON1_DATA_ENTRIES__SHIFT 0 +#define REG_CNA_CVT_CON0 0x0000104c +#define CNA_CVT_CON0_RESERVED_0__MASK 0xf0000000 +#define CNA_CVT_CON0_RESERVED_0__SHIFT 28 +#define CNA_CVT_CON0_CVT_TRUNCATE_3__MASK 0x0fc00000 +#define CNA_CVT_CON0_CVT_TRUNCATE_3__SHIFT 22 +#define CNA_CVT_CON0_CVT_TRUNCATE_2__MASK 0x003f0000 +#define CNA_CVT_CON0_CVT_TRUNCATE_2__SHIFT 16 +#define CNA_CVT_CON0_CVT_TRUNCATE_1__MASK 0x0000fc00 +#define CNA_CVT_CON0_CVT_TRUNCATE_1__SHIFT 10 +#define CNA_CVT_CON0_CVT_TRUNCATE_0__MASK 0x000003f0 +#define CNA_CVT_CON0_CVT_TRUNCATE_0__SHIFT 4 +#define CNA_CVT_CON0_DATA_SIGN__MASK 0x00000008 +#define CNA_CVT_CON0_DATA_SIGN__SHIFT 3 +#define CNA_CVT_CON0_ROUND_TYPE__MASK 0x00000004 +#define CNA_CVT_CON0_ROUND_TYPE__SHIFT 2 +#define CNA_CVT_CON0_CVT_TYPE__MASK 0x00000002 +#define CNA_CVT_CON0_CVT_TYPE__SHIFT 1 +#define CNA_CVT_CON0_CVT_BYPASS__MASK 0x00000001 +#define CNA_CVT_CON0_CVT_BYPASS__SHIFT 0 +#define REG_CNA_CVT_CON1 0x00001050 +#define CNA_CVT_CON1_CVT_SCALE0__MASK 0xffff0000 +#define CNA_CVT_CON1_CVT_SCALE0__SHIFT 16 +#define CNA_CVT_CON1_CVT_OFFSET0__MASK 0x0000ffff +#define CNA_CVT_CON1_CVT_OFFSET0__SHIFT 0 +#define REG_CNA_CVT_CON2 0x00001054 +#define CNA_CVT_CON2_CVT_SCALE1__MASK 0xffff0000 +#define CNA_CVT_CON2_CVT_SCALE1__SHIFT 16 +#define CNA_CVT_CON2_CVT_OFFSET1__MASK 0x0000ffff +#define CNA_CVT_CON2_CVT_OFFSET1__SHIFT 0 +#define REG_CNA_CVT_CON3 0x00001058 +#define CNA_CVT_CON3_CVT_SCALE2__MASK 0xffff0000 +#define CNA_CVT_CON3_CVT_SCALE2__SHIFT 16 +#define CNA_CVT_CON3_CVT_OFFSET2__MASK 0x0000ffff +#define CNA_CVT_CON3_CVT_OFFSET2__SHIFT 0 +#define REG_CNA_CVT_CON4 0x0000105c +#define CNA_CVT_CON4_CVT_SCALE3__MASK 0xffff0000 +#define CNA_CVT_CON4_CVT_SCALE3__SHIFT 16 +#define CNA_CVT_CON4_CVT_OFFSET3__MASK 0x0000ffff +#define CNA_CVT_CON4_CVT_OFFSET3__SHIFT 0 +#define REG_CNA_FC_CON0 0x00001060 +#define CNA_FC_CON0_FC_SKIP_DATA__MASK 0xffff0000 +#define CNA_FC_CON0_FC_SKIP_DATA__SHIFT 16 +#define CNA_FC_CON0_RESERVED_0__MASK 0x0000fffe +#define CNA_FC_CON0_RESERVED_0__SHIFT 1 +#define CNA_FC_CON0_FC_SKIP_EN__MASK 0x00000001 +#define CNA_FC_CON0_FC_SKIP_EN__SHIFT 0 +#define REG_CNA_FC_CON1 0x00001064 +#define CNA_FC_CON1_RESERVED_0__MASK 0xfffe0000 +#define CNA_FC_CON1_RESERVED_0__SHIFT 17 +#define CNA_FC_CON1_DATA_OFFSET__MASK 0x0001ffff +#define CNA_FC_CON1_DATA_OFFSET__SHIFT 0 +#define REG_CNA_PAD_CON0 0x00001068 +#define CNA_PAD_CON0_RESERVED_0__MASK 0xffffff00 +#define CNA_PAD_CON0_RESERVED_0__SHIFT 8 +#define CNA_PAD_CON0_PAD_LEFT__MASK 0x000000f0 +#define CNA_PAD_CON0_PAD_LEFT__SHIFT 4 +#define CNA_PAD_CON0_PAD_TOP__MASK 0x0000000f +#define CNA_PAD_CON0_PAD_TOP__SHIFT 0 +#define REG_CNA_FEATURE_DATA_ADDR 0x00001070 +#define CNA_FEATURE_DATA_ADDR_FEATURE_BASE_ADDR__MASK 0xffffffff +#define CNA_FEATURE_DATA_ADDR_FEATURE_BASE_ADDR__SHIFT 0 +#define REG_CNA_FC_CON2 0x00001074 +#define CNA_FC_CON2_RESERVED_0__MASK 0xfffe0000 +#define CNA_FC_CON2_RESERVED_0__SHIFT 17 +#define CNA_FC_CON2_WEIGHT_OFFSET__MASK 0x0001ffff +#define CNA_FC_CON2_WEIGHT_OFFSET__SHIFT 0 +#define REG_CNA_DMA_CON0 0x00001078 +#define CNA_DMA_CON0_OV4K_BYPASS__MASK 0x80000000 +#define CNA_DMA_CON0_OV4K_BYPASS__SHIFT 31 +#define CNA_DMA_CON0_RESERVED_0__MASK 0x7ff00000 +#define CNA_DMA_CON0_RESERVED_0__SHIFT 20 +#define CNA_DMA_CON0_WEIGHT_BURST_LEN__MASK 0x000f0000 +#define CNA_DMA_CON0_WEIGHT_BURST_LEN__SHIFT 16 +#define CNA_DMA_CON0_RESERVED_1__MASK 0x0000fff0 +#define CNA_DMA_CON0_RESERVED_1__SHIFT 4 +#define CNA_DMA_CON0_DATA_BURST_LEN__MASK 0x0000000f +#define CNA_DMA_CON0_DATA_BURST_LEN__SHIFT 0 +#define REG_CNA_DMA_CON1 0x0000107c +#define CNA_DMA_CON1_RESERVED_0__MASK 0xf0000000 +#define CNA_DMA_CON1_RESERVED_0__SHIFT 28 +#define CNA_DMA_CON1_LINE_STRIDE__MASK 0x0fffffff +#define CNA_DMA_CON1_LINE_STRIDE__SHIFT 0 +#define REG_CNA_DMA_CON2 0x00001080 +#define CNA_DMA_CON2_RESERVED_0__MASK 0xf0000000 +#define CNA_DMA_CON2_RESERVED_0__SHIFT 28 +#define CNA_DMA_CON2_SURF_STRIDE__MASK 0x0fffffff +#define CNA_DMA_CON2_SURF_STRIDE__SHIFT 0 +#define REG_CNA_FC_DATA_SIZE0 0x00001084 +#define CNA_FC_DATA_SIZE0_RESERVED_0__MASK 0xc0000000 +#define CNA_FC_DATA_SIZE0_RESERVED_0__SHIFT 30 +#define CNA_FC_DATA_SIZE0_DMA_WIDTH__MASK 0x3fff0000 +#define CNA_FC_DATA_SIZE0_DMA_WIDTH__SHIFT 16 +#define CNA_FC_DATA_SIZE0_RESERVED_1__MASK 0x0000f800 +#define CNA_FC_DATA_SIZE0_RESERVED_1__SHIFT 11 +#define CNA_FC_DATA_SIZE0_DMA_HEIGHT__MASK 0x000007ff +#define CNA_FC_DATA_SIZE0_DMA_HEIGHT__SHIFT 0 +#define REG_CNA_FC_DATA_SIZE1 0x00001088 +#define CNA_FC_DATA_SIZE1_RESERVED_0__MASK 0xffff0000 +#define CNA_FC_DATA_SIZE1_RESERVED_0__SHIFT 16 +#define CNA_FC_DATA_SIZE1_DMA_CHANNEL__MASK 0x0000ffff +#define CNA_FC_DATA_SIZE1_DMA_CHANNEL__SHIFT 0 +#define REG_CNA_CLK_GATE 0x00001090 +#define CNA_CLK_GATE_RESERVED_0__MASK 0xffffffe0 +#define CNA_CLK_GATE_RESERVED_0__SHIFT 5 +#define CNA_CLK_GATE_CBUF_CS_DISABLE_CLKGATE__MASK 0x00000010 +#define CNA_CLK_GATE_CBUF_CS_DISABLE_CLKGATE__SHIFT 4 +#define CNA_CLK_GATE_RESERVED_1__MASK 0x00000008 +#define CNA_CLK_GATE_RESERVED_1__SHIFT 3 +#define CNA_CLK_GATE_CSC_DISABLE_CLKGATE__MASK 0x00000004 +#define CNA_CLK_GATE_CSC_DISABLE_CLKGATE__SHIFT 2 +#define CNA_CLK_GATE_CNA_WEIGHT_DISABLE_CLKGATE__MASK 0x00000002 +#define CNA_CLK_GATE_CNA_WEIGHT_DISABLE_CLKGATE__SHIFT 1 +#define CNA_CLK_GATE_CNA_FEATURE_DISABLE_CLKGATE__MASK 0x00000001 +#define CNA_CLK_GATE_CNA_FEATURE_DISABLE_CLKGATE__SHIFT 0 +#define REG_CNA_DCOMP_CTRL 0x00001100 +#define CNA_DCOMP_CTRL_RESERVED_0__MASK 0xfffffff0 +#define CNA_DCOMP_CTRL_RESERVED_0__SHIFT 4 +#define CNA_DCOMP_CTRL_WT_DEC_BYPASS__MASK 0x00000008 +#define CNA_DCOMP_CTRL_WT_DEC_BYPASS__SHIFT 3 +#define CNA_DCOMP_CTRL_DECOMP_CONTROL__MASK 0x00000007 +#define CNA_DCOMP_CTRL_DECOMP_CONTROL__SHIFT 0 +#define REG_CNA_DCOMP_REGNUM 0x00001104 +#define CNA_DCOMP_REGNUM_DCOMP_REGNUM__MASK 0xffffffff +#define CNA_DCOMP_REGNUM_DCOMP_REGNUM__SHIFT 0 +#define REG_CNA_DCOMP_ADDR0 0x00001110 +#define CNA_DCOMP_ADDR0_DECOMPRESS_ADDR0__MASK 0xffffffff +#define CNA_DCOMP_ADDR0_DECOMPRESS_ADDR0__SHIFT 0 +#define REG_CNA_DCOMP_AMOUNT0 0x00001140 +#define CNA_DCOMP_AMOUNT0_DCOMP_AMOUNT0__MASK 0xffffffff +#define CNA_DCOMP_AMOUNT0_DCOMP_AMOUNT0__SHIFT 0 +#define REG_CNA_DCOMP_AMOUNT1 0x00001144 +#define CNA_DCOMP_AMOUNT1_DCOMP_AMOUNT1__MASK 0xffffffff +#define CNA_DCOMP_AMOUNT1_DCOMP_AMOUNT1__SHIFT 0 +#define REG_CNA_DCOMP_AMOUNT2 0x00001148 +#define CNA_DCOMP_AMOUNT2_DCOMP_AMOUNT2__MASK 0xffffffff +#define CNA_DCOMP_AMOUNT2_DCOMP_AMOUNT2__SHIFT 0 +#define REG_CNA_DCOMP_AMOUNT3 0x0000114c +#define CNA_DCOMP_AMOUNT3_DCOMP_AMOUNT3__MASK 0xffffffff +#define CNA_DCOMP_AMOUNT3_DCOMP_AMOUNT3__SHIFT 0 +#define REG_CNA_DCOMP_AMOUNT4 0x00001150 +#define CNA_DCOMP_AMOUNT4_DCOMP_AMOUNT4__MASK 0xffffffff +#define CNA_DCOMP_AMOUNT4_DCOMP_AMOUNT4__SHIFT 0 +#define REG_CNA_DCOMP_AMOUNT5 0x00001154 +#define CNA_DCOMP_AMOUNT5_DCOMP_AMOUNT5__MASK 0xffffffff +#define CNA_DCOMP_AMOUNT5_DCOMP_AMOUNT5__SHIFT 0 +#define REG_CNA_DCOMP_AMOUNT6 0x00001158 +#define CNA_DCOMP_AMOUNT6_DCOMP_AMOUNT6__MASK 0xffffffff +#define CNA_DCOMP_AMOUNT6_DCOMP_AMOUNT6__SHIFT 0 +#define REG_CNA_DCOMP_AMOUNT7 0x0000115c +#define CNA_DCOMP_AMOUNT7_DCOMP_AMOUNT7__MASK 0xffffffff +#define CNA_DCOMP_AMOUNT7_DCOMP_AMOUNT7__SHIFT 0 +#define REG_CNA_DCOMP_AMOUNT8 0x00001160 +#define CNA_DCOMP_AMOUNT8_DCOMP_AMOUNT8__MASK 0xffffffff +#define CNA_DCOMP_AMOUNT8_DCOMP_AMOUNT8__SHIFT 0 +#define REG_CNA_DCOMP_AMOUNT9 0x00001164 +#define CNA_DCOMP_AMOUNT9_DCOMP_AMOUNT9__MASK 0xffffffff +#define CNA_DCOMP_AMOUNT9_DCOMP_AMOUNT9__SHIFT 0 +#define REG_CNA_DCOMP_AMOUNT10 0x00001168 +#define CNA_DCOMP_AMOUNT10_DCOMP_AMOUNT10__MASK 0xffffffff +#define CNA_DCOMP_AMOUNT10_DCOMP_AMOUNT10__SHIFT 0 +#define REG_CNA_DCOMP_AMOUNT11 0x0000116c +#define CNA_DCOMP_AMOUNT11_DCOMP_AMOUNT11__MASK 0xffffffff +#define CNA_DCOMP_AMOUNT11_DCOMP_AMOUNT11__SHIFT 0 +#define REG_CNA_DCOMP_AMOUNT12 0x00001170 +#define CNA_DCOMP_AMOUNT12_DCOMP_AMOUNT12__MASK 0xffffffff +#define CNA_DCOMP_AMOUNT12_DCOMP_AMOUNT12__SHIFT 0 +#define REG_CNA_DCOMP_AMOUNT13 0x00001174 +#define CNA_DCOMP_AMOUNT13_DCOMP_AMOUNT13__MASK 0xffffffff +#define CNA_DCOMP_AMOUNT13_DCOMP_AMOUNT13__SHIFT 0 +#define REG_CNA_DCOMP_AMOUNT14 0x00001178 +#define CNA_DCOMP_AMOUNT14_DCOMP_AMOUNT14__MASK 0xffffffff +#define CNA_DCOMP_AMOUNT14_DCOMP_AMOUNT14__SHIFT 0 +#define REG_CNA_DCOMP_AMOUNT15 0x0000117c +#define CNA_DCOMP_AMOUNT15_DCOMP_AMOUNT15__MASK 0xffffffff +#define CNA_DCOMP_AMOUNT15_DCOMP_AMOUNT15__SHIFT 0 +#define REG_CNA_CVT_CON5 0x00001180 +#define CNA_CVT_CON5_PER_CHANNEL_CVT_EN__MASK 0xffffffff +#define CNA_CVT_CON5_PER_CHANNEL_CVT_EN__SHIFT 0 +#define REG_CNA_PAD_CON1 0x00001184 +#define CNA_PAD_CON1_PAD_VALUE__MASK 0xffffffff +#define CNA_PAD_CON1_PAD_VALUE__SHIFT 0 +#define REG_DPU_S_POINTER 0x00004004 +#define DPU_S_POINTER_RESERVED_0__MASK 0xfffe0000 +#define DPU_S_POINTER_RESERVED_0__SHIFT 17 +#define DPU_S_POINTER_EXECUTER__MASK 0x00010000 +#define DPU_S_POINTER_EXECUTER__SHIFT 16 +#define DPU_S_POINTER_RESERVED_1__MASK 0x0000ffc0 +#define DPU_S_POINTER_RESERVED_1__SHIFT 6 +#define DPU_S_POINTER_EXECUTER_PP_CLEAR__MASK 0x00000020 +#define DPU_S_POINTER_EXECUTER_PP_CLEAR__SHIFT 5 +#define DPU_S_POINTER_POINTER_PP_CLEAR__MASK 0x00000010 +#define DPU_S_POINTER_POINTER_PP_CLEAR__SHIFT 4 +#define DPU_S_POINTER_POINTER_PP_MODE__MASK 0x00000008 +#define DPU_S_POINTER_POINTER_PP_MODE__SHIFT 3 +#define DPU_S_POINTER_EXECUTER_PP_EN__MASK 0x00000004 +#define DPU_S_POINTER_EXECUTER_PP_EN__SHIFT 2 +#define DPU_S_POINTER_POINTER_PP_EN__MASK 0x00000002 +#define DPU_S_POINTER_POINTER_PP_EN__SHIFT 1 +#define DPU_S_POINTER_POINTER__MASK 0x00000001 +#define DPU_S_POINTER_POINTER__SHIFT 0 +#define REG_DPU_RDMA_RDMA_S_STATUS 0x00005000 +#define DPU_RDMA_RDMA_S_STATUS_RESERVED_0__MASK 0xfffc0000 +#define DPU_RDMA_RDMA_S_STATUS_RESERVED_0__SHIFT 18 +#define DPU_RDMA_RDMA_S_STATUS_STATUS_1__MASK 0x00030000 +#define DPU_RDMA_RDMA_S_STATUS_STATUS_1__SHIFT 16 +#define DPU_RDMA_RDMA_S_STATUS_RESERVED_1__MASK 0x0000fffc +#define DPU_RDMA_RDMA_S_STATUS_RESERVED_1__SHIFT 2 +#define DPU_RDMA_RDMA_S_STATUS_STATUS_0__MASK 0x00000003 +#define DPU_RDMA_RDMA_S_STATUS_STATUS_0__SHIFT 0 +#define REG_DPU_RDMA_RDMA_S_POINTER 0x00005004 +#define DPU_RDMA_RDMA_S_POINTER_RESERVED_0__MASK 0xfffe0000 +#define DPU_RDMA_RDMA_S_POINTER_RESERVED_0__SHIFT 17 +#define DPU_RDMA_RDMA_S_POINTER_EXECUTER__MASK 0x00010000 +#define DPU_RDMA_RDMA_S_POINTER_EXECUTER__SHIFT 16 +#define DPU_RDMA_RDMA_S_POINTER_RESERVED_1__MASK 0x0000ffc0 +#define DPU_RDMA_RDMA_S_POINTER_RESERVED_1__SHIFT 6 +#define DPU_RDMA_RDMA_S_POINTER_EXECUTER_PP_CLEAR__MASK 0x00000020 +#define DPU_RDMA_RDMA_S_POINTER_EXECUTER_PP_CLEAR__SHIFT 5 +#define DPU_RDMA_RDMA_S_POINTER_POINTER_PP_CLEAR__MASK 0x00000010 +#define DPU_RDMA_RDMA_S_POINTER_POINTER_PP_CLEAR__SHIFT 4 +#define DPU_RDMA_RDMA_S_POINTER_POINTER_PP_MODE__MASK 0x00000008 +#define DPU_RDMA_RDMA_S_POINTER_POINTER_PP_MODE__SHIFT 3 +#define DPU_RDMA_RDMA_S_POINTER_EXECUTER_PP_EN__MASK 0x00000004 +#define DPU_RDMA_RDMA_S_POINTER_EXECUTER_PP_EN__SHIFT 2 +#define DPU_RDMA_RDMA_S_POINTER_POINTER_PP_EN__MASK 0x00000002 +#define DPU_RDMA_RDMA_S_POINTER_POINTER_PP_EN__SHIFT 1 +#define DPU_RDMA_RDMA_S_POINTER_POINTER__MASK 0x00000001 +#define DPU_RDMA_RDMA_S_POINTER_POINTER__SHIFT 0 +#define REG_DPU_RDMA_RDMA_OPERATION_ENABLE 0x00005008 +#define DPU_RDMA_RDMA_OPERATION_ENABLE_RESERVED_0__MASK 0xfffffffe +#define DPU_RDMA_RDMA_OPERATION_ENABLE_RESERVED_0__SHIFT 1 +#define DPU_RDMA_RDMA_OPERATION_ENABLE_OP_EN__MASK 0x00000001 +#define DPU_RDMA_RDMA_OPERATION_ENABLE_OP_EN__SHIFT 0 +#define REG_DPU_RDMA_RDMA_DATA_CUBE_WIDTH 0x0000500c +#define DPU_RDMA_RDMA_DATA_CUBE_WIDTH_RESERVED_0__MASK 0xffffe000 +#define DPU_RDMA_RDMA_DATA_CUBE_WIDTH_RESERVED_0__SHIFT 13 +#define DPU_RDMA_RDMA_DATA_CUBE_WIDTH_WIDTH__MASK 0x00001fff +#define DPU_RDMA_RDMA_DATA_CUBE_WIDTH_WIDTH__SHIFT 0 +#define REG_DPU_RDMA_RDMA_DATA_CUBE_HEIGHT 0x00005010 +#define DPU_RDMA_RDMA_DATA_CUBE_HEIGHT_RESERVED_0__MASK 0xe0000000 +#define DPU_RDMA_RDMA_DATA_CUBE_HEIGHT_RESERVED_0__SHIFT 29 +#define DPU_RDMA_RDMA_DATA_CUBE_HEIGHT_EW_LINE_NOTCH_ADDR__MASK 0x1fff0000 +#define DPU_RDMA_RDMA_DATA_CUBE_HEIGHT_EW_LINE_NOTCH_ADDR__SHIFT 16 +#define DPU_RDMA_RDMA_DATA_CUBE_HEIGHT_RESERVED_1__MASK 0x0000e000 +#define DPU_RDMA_RDMA_DATA_CUBE_HEIGHT_RESERVED_1__SHIFT 13 +#define DPU_RDMA_RDMA_DATA_CUBE_HEIGHT_HEIGHT__MASK 0x00001fff +#define DPU_RDMA_RDMA_DATA_CUBE_HEIGHT_HEIGHT__SHIFT 0 +#define REG_DPU_RDMA_RDMA_DATA_CUBE_CHANNEL 0x00005014 +#define DPU_RDMA_RDMA_DATA_CUBE_CHANNEL_RESERVED_0__MASK 0xffffe000 +#define DPU_RDMA_RDMA_DATA_CUBE_CHANNEL_RESERVED_0__SHIFT 13 +#define DPU_RDMA_RDMA_DATA_CUBE_CHANNEL_CHANNEL__MASK 0x00001fff +#define DPU_RDMA_RDMA_DATA_CUBE_CHANNEL_CHANNEL__SHIFT 0 +#define REG_DPU_RDMA_RDMA_SRC_BASE_ADDR 0x00005018 +#define DPU_RDMA_RDMA_SRC_BASE_ADDR_SRC_BASE_ADDR__MASK 0xffffffff +#define DPU_RDMA_RDMA_SRC_BASE_ADDR_SRC_BASE_ADDR__SHIFT 0 +#define REG_DPU_RDMA_RDMA_BRDMA_CFG 0x0000501c +#define DPU_RDMA_RDMA_BRDMA_CFG_RESERVED_0__MASK 0xffffffe0 +#define DPU_RDMA_RDMA_BRDMA_CFG_RESERVED_0__SHIFT 5 +#define DPU_RDMA_RDMA_BRDMA_CFG_BRDMA_DATA_USE__MASK 0x0000001e +#define DPU_RDMA_RDMA_BRDMA_CFG_BRDMA_DATA_USE__SHIFT 1 +#define DPU_RDMA_RDMA_BRDMA_CFG_RESERVED_1__MASK 0x00000001 +#define DPU_RDMA_RDMA_BRDMA_CFG_RESERVED_1__SHIFT 0 +#define REG_DPU_RDMA_RDMA_BS_BASE_ADDR 0x00005020 +#define DPU_RDMA_RDMA_BS_BASE_ADDR_BS_BASE_ADDR__MASK 0xffffffff +#define DPU_RDMA_RDMA_BS_BASE_ADDR_BS_BASE_ADDR__SHIFT 0 +#define REG_DPU_RDMA_RDMA_NRDMA_CFG 0x00005028 +#define DPU_RDMA_RDMA_NRDMA_CFG_RESERVED_0__MASK 0xffffffe0 +#define DPU_RDMA_RDMA_NRDMA_CFG_RESERVED_0__SHIFT 5 +#define DPU_RDMA_RDMA_NRDMA_CFG_NRDMA_DATA_USE__MASK 0x0000001e +#define DPU_RDMA_RDMA_NRDMA_CFG_NRDMA_DATA_USE__SHIFT 1 +#define DPU_RDMA_RDMA_NRDMA_CFG_RESERVED_1__MASK 0x00000001 +#define DPU_RDMA_RDMA_NRDMA_CFG_RESERVED_1__SHIFT 0 +#define REG_DPU_RDMA_RDMA_BN_BASE_ADDR 0x0000502c +#define DPU_RDMA_RDMA_BN_BASE_ADDR_BN_BASE_ADDR__MASK 0xffffffff +#define DPU_RDMA_RDMA_BN_BASE_ADDR_BN_BASE_ADDR__SHIFT 0 +#define REG_DPU_RDMA_RDMA_ERDMA_CFG 0x00005034 +#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_DATA_MODE__MASK 0xc0000000 +#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_DATA_MODE__SHIFT 30 +#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_SURF_MODE__MASK 0x20000000 +#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_SURF_MODE__SHIFT 29 +#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_NONALIGN__MASK 0x10000000 +#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_NONALIGN__SHIFT 28 +#define DPU_RDMA_RDMA_ERDMA_CFG_RESERVED_0__MASK 0x0ffffff0 +#define DPU_RDMA_RDMA_ERDMA_CFG_RESERVED_0__SHIFT 4 +#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_DATA_SIZE__MASK 0x0000000c +#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_DATA_SIZE__SHIFT 2 +#define DPU_RDMA_RDMA_ERDMA_CFG_OV4K_BYPASS__MASK 0x00000002 +#define DPU_RDMA_RDMA_ERDMA_CFG_OV4K_BYPASS__SHIFT 1 +#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_DISABLE__MASK 0x00000001 +#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_DISABLE__SHIFT 0 +#define REG_DPU_RDMA_RDMA_EW_BASE_ADDR 0x00005038 +#define DPU_RDMA_RDMA_EW_BASE_ADDR_EW_BASE_ADDR__MASK 0xffffffff +#define DPU_RDMA_RDMA_EW_BASE_ADDR_EW_BASE_ADDR__SHIFT 0 +#define REG_DPU_RDMA_RDMA_EW_SURF_STRIDE 0x00005040 +#define DPU_RDMA_RDMA_EW_SURF_STRIDE_EW_SURF_STRIDE__MASK 0xfffffff0 +#define DPU_RDMA_RDMA_EW_SURF_STRIDE_EW_SURF_STRIDE__SHIFT 4 +#define DPU_RDMA_RDMA_EW_SURF_STRIDE_RESERVED_0__MASK 0x0000000f +#define DPU_RDMA_RDMA_EW_SURF_STRIDE_RESERVED_0__SHIFT 0 +#define REG_DPU_RDMA_RDMA_FEATURE_MODE_CFG 0x00005044 +#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_RESERVED_0__MASK 0xfffc0000 +#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_RESERVED_0__SHIFT 18 +#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_IN_PRECISION__MASK 0x00038000 +#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_IN_PRECISION__SHIFT 15 +#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_BURST_LEN__MASK 0x00007800 +#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_BURST_LEN__SHIFT 11 +#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_COMB_USE__MASK 0x00000700 +#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_COMB_USE__SHIFT 8 +#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_PROC_PRECISION__MASK 0x000000e0 +#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_PROC_PRECISION__SHIFT 5 +#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_MRDMA_DISABLE__MASK 0x00000010 +#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_MRDMA_DISABLE__SHIFT 4 +#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_MRDMA_FP16TOFP32_EN__MASK 0x00000008 +#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_MRDMA_FP16TOFP32_EN__SHIFT 3 +#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_CONV_MODE__MASK 0x00000006 +#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_CONV_MODE__SHIFT 1 +#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_FLYING_MODE__MASK 0x00000001 +#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_FLYING_MODE__SHIFT 0 +#define REG_DPU_RDMA_RDMA_SRC_DMA_CFG 0x00005048 +#define DPU_RDMA_RDMA_SRC_DMA_CFG_LINE_NOTCH_ADDR__MASK 0xfff80000 +#define DPU_RDMA_RDMA_SRC_DMA_CFG_LINE_NOTCH_ADDR__SHIFT 19 +#define DPU_RDMA_RDMA_SRC_DMA_CFG_RESERVED_0__MASK 0x0007c000 +#define DPU_RDMA_RDMA_SRC_DMA_CFG_RESERVED_0__SHIFT 14 +#define DPU_RDMA_RDMA_SRC_DMA_CFG_POOLING_METHOD__MASK 0x00002000 +#define DPU_RDMA_RDMA_SRC_DMA_CFG_POOLING_METHOD__SHIFT 13 +#define DPU_RDMA_RDMA_SRC_DMA_CFG_UNPOOLING_EN__MASK 0x00001000 +#define DPU_RDMA_RDMA_SRC_DMA_CFG_UNPOOLING_EN__SHIFT 12 +#define DPU_RDMA_RDMA_SRC_DMA_CFG_KERNEL_STRIDE_HEIGHT__MASK 0x00000e00 +#define DPU_RDMA_RDMA_SRC_DMA_CFG_KERNEL_STRIDE_HEIGHT__SHIFT 9 +#define DPU_RDMA_RDMA_SRC_DMA_CFG_KERNEL_STRIDE_WIDTH__MASK 0x000001c0 +#define DPU_RDMA_RDMA_SRC_DMA_CFG_KERNEL_STRIDE_WIDTH__SHIFT 6 +#define DPU_RDMA_RDMA_SRC_DMA_CFG_KERNEL_HEIGHT__MASK 0x00000038 +#define DPU_RDMA_RDMA_SRC_DMA_CFG_KERNEL_HEIGHT__SHIFT 3 +#define DPU_RDMA_RDMA_SRC_DMA_CFG_KERNEL_WIDTH__MASK 0x00000007 +#define DPU_RDMA_RDMA_SRC_DMA_CFG_KERNEL_WIDTH__SHIFT 0 +#define REG_DPU_RDMA_RDMA_SURF_NOTCH 0x0000504c +#define DPU_RDMA_RDMA_SURF_NOTCH_SURF_NOTCH_ADDR__MASK 0xfffffff0 +#define DPU_RDMA_RDMA_SURF_NOTCH_SURF_NOTCH_ADDR__SHIFT 4 +#define DPU_RDMA_RDMA_SURF_NOTCH_RESERVED_0__MASK 0x0000000f +#define DPU_RDMA_RDMA_SURF_NOTCH_RESERVED_0__SHIFT 0 +#define REG_DPU_RDMA_RDMA_PAD_CFG 0x00005064 +#define DPU_RDMA_RDMA_PAD_CFG_PAD_VALUE__MASK 0xffff0000 +#define DPU_RDMA_RDMA_PAD_CFG_PAD_VALUE__SHIFT 16 +#define DPU_RDMA_RDMA_PAD_CFG_RESERVED_0__MASK 0x0000ff80 +#define DPU_RDMA_RDMA_PAD_CFG_RESERVED_0__SHIFT 7 +#define DPU_RDMA_RDMA_PAD_CFG_PAD_TOP__MASK 0x00000070 +#define DPU_RDMA_RDMA_PAD_CFG_PAD_TOP__SHIFT 4 +#define DPU_RDMA_RDMA_PAD_CFG_RESERVED_1__MASK 0x00000008 +#define DPU_RDMA_RDMA_PAD_CFG_RESERVED_1__SHIFT 3 +#define DPU_RDMA_RDMA_PAD_CFG_PAD_LEFT__MASK 0x00000007 +#define DPU_RDMA_RDMA_PAD_CFG_PAD_LEFT__SHIFT 0 +#define REG_DPU_RDMA_RDMA_WEIGHT 0x00005068 +#define DPU_RDMA_RDMA_WEIGHT_E_WEIGHT__MASK 0xff000000 +#define DPU_RDMA_RDMA_WEIGHT_E_WEIGHT__SHIFT 24 +#define DPU_RDMA_RDMA_WEIGHT_N_WEIGHT__MASK 0x00ff0000 +#define DPU_RDMA_RDMA_WEIGHT_N_WEIGHT__SHIFT 16 +#define DPU_RDMA_RDMA_WEIGHT_B_WEIGHT__MASK 0x0000ff00 +#define DPU_RDMA_RDMA_WEIGHT_B_WEIGHT__SHIFT 8 +#define DPU_RDMA_RDMA_WEIGHT_M_WEIGHT__MASK 0x000000ff +#define DPU_RDMA_RDMA_WEIGHT_M_WEIGHT__SHIFT 0 +#define REG_DPU_RDMA_RDMA_EW_SURF_NOTCH 0x0000506c +#define DPU_RDMA_RDMA_EW_SURF_NOTCH_EW_SURF_NOTCH__MASK 0xfffffff0 +#define DPU_RDMA_RDMA_EW_SURF_NOTCH_EW_SURF_NOTCH__SHIFT 4 +#define DPU_RDMA_RDMA_EW_SURF_NOTCH_RESERVED_0__MASK 0x0000000f +#define DPU_RDMA_RDMA_EW_SURF_NOTCH_RESERVED_0__SHIFT 0 + +#endif diff --git a/ggml/src/ggml-rocket/coredpu_defs.h b/ggml/src/ggml-rocket/coredpu_defs.h new file mode 100644 index 000000000..d9d8b087a --- /dev/null +++ b/ggml/src/ggml-rocket/coredpu_defs.h @@ -0,0 +1,769 @@ +#ifndef COREDPU_DEFS_H +#define COREDPU_DEFS_H + +#define REG_PC_VERSION 0x00000000 +#define PC_VERSION_VERSION__MASK 0xffffffff +#define PC_VERSION_VERSION__SHIFT 0 +#define REG_PC_VERSION_NUM 0x00000004 +#define PC_VERSION_NUM_VERSION_NUM__MASK 0xffffffff +#define PC_VERSION_NUM_VERSION_NUM__SHIFT 0 +#define REG_PC_OPERATION_ENABLE 0x00000008 +#define PC_OPERATION_ENABLE_RESERVED_0__MASK 0xfffffffe +#define PC_OPERATION_ENABLE_RESERVED_0__SHIFT 1 +#define PC_OPERATION_ENABLE_OP_EN__MASK 0x00000001 +#define PC_OPERATION_ENABLE_OP_EN__SHIFT 0 +#define REG_PC_BASE_ADDRESS 0x00000010 +#define PC_BASE_ADDRESS_PC_SOURCE_ADDR__MASK 0xfffffff0 +#define PC_BASE_ADDRESS_PC_SOURCE_ADDR__SHIFT 4 +#define PC_BASE_ADDRESS_RESERVED_0__MASK 0x0000000e +#define PC_BASE_ADDRESS_RESERVED_0__SHIFT 1 +#define PC_BASE_ADDRESS_PC_SEL__MASK 0x00000001 +#define PC_BASE_ADDRESS_PC_SEL__SHIFT 0 +#define REG_PC_REGISTER_AMOUNTS 0x00000014 +#define PC_REGISTER_AMOUNTS_RESERVED_0__MASK 0xffff0000 +#define PC_REGISTER_AMOUNTS_RESERVED_0__SHIFT 16 +#define PC_REGISTER_AMOUNTS_PC_DATA_AMOUNT__MASK 0x0000ffff +#define PC_REGISTER_AMOUNTS_PC_DATA_AMOUNT__SHIFT 0 +#define REG_PC_INTERRUPT_MASK 0x00000020 +#define PC_INTERRUPT_MASK_RESERVED_0__MASK 0xffffc000 +#define PC_INTERRUPT_MASK_RESERVED_0__SHIFT 14 +#define PC_INTERRUPT_MASK_DMA_WRITE_ERROR 0x00002000 +#define PC_INTERRUPT_MASK_DMA_READ_ERROR 0x00001000 +#define PC_INTERRUPT_MASK_PPU_1 0x00000800 +#define PC_INTERRUPT_MASK_PPU_0 0x00000400 +#define PC_INTERRUPT_MASK_DPU_1 0x00000200 +#define PC_INTERRUPT_MASK_DPU_0 0x00000100 +#define PC_INTERRUPT_MASK_CORE_1 0x00000080 +#define PC_INTERRUPT_MASK_CORE_0 0x00000040 +#define PC_INTERRUPT_MASK_CNA_CSC_1 0x00000020 +#define PC_INTERRUPT_MASK_CNA_CSC_0 0x00000010 +#define PC_INTERRUPT_MASK_CNA_WEIGHT_1 0x00000008 +#define PC_INTERRUPT_MASK_CNA_WEIGHT_0 0x00000004 +#define PC_INTERRUPT_MASK_CNA_FEATURE_1 0x00000002 +#define PC_INTERRUPT_MASK_CNA_FEATURE_0 0x00000001 +#define REG_PC_INTERRUPT_CLEAR 0x00000024 +#define PC_INTERRUPT_CLEAR_RESERVED_0__MASK 0xffffc000 +#define PC_INTERRUPT_CLEAR_RESERVED_0__SHIFT 14 +#define PC_INTERRUPT_CLEAR_DMA_WRITE_ERROR 0x00002000 +#define PC_INTERRUPT_CLEAR_DMA_READ_ERROR 0x00001000 +#define PC_INTERRUPT_CLEAR_PPU_1 0x00000800 +#define PC_INTERRUPT_CLEAR_PPU_0 0x00000400 +#define PC_INTERRUPT_CLEAR_DPU_1 0x00000200 +#define PC_INTERRUPT_CLEAR_DPU_0 0x00000100 +#define PC_INTERRUPT_CLEAR_CORE_1 0x00000080 +#define PC_INTERRUPT_CLEAR_CORE_0 0x00000040 +#define PC_INTERRUPT_CLEAR_CNA_CSC_1 0x00000020 +#define PC_INTERRUPT_CLEAR_CNA_CSC_0 0x00000010 +#define PC_INTERRUPT_CLEAR_CNA_WEIGHT_1 0x00000008 +#define PC_INTERRUPT_CLEAR_CNA_WEIGHT_0 0x00000004 +#define PC_INTERRUPT_CLEAR_CNA_FEATURE_1 0x00000002 +#define PC_INTERRUPT_CLEAR_CNA_FEATURE_0 0x00000001 +#define REG_PC_INTERRUPT_STATUS 0x00000028 +#define PC_INTERRUPT_STATUS_RESERVED_0__MASK 0xffffc000 +#define PC_INTERRUPT_STATUS_RESERVED_0__SHIFT 14 +#define PC_INTERRUPT_STATUS_DMA_WRITE_ERROR 0x00002000 +#define PC_INTERRUPT_STATUS_DMA_READ_ERROR 0x00001000 +#define PC_INTERRUPT_STATUS_PPU_1 0x00000800 +#define PC_INTERRUPT_STATUS_PPU_0 0x00000400 +#define PC_INTERRUPT_STATUS_DPU_1 0x00000200 +#define PC_INTERRUPT_STATUS_DPU_0 0x00000100 +#define PC_INTERRUPT_STATUS_CORE_1 0x00000080 +#define PC_INTERRUPT_STATUS_CORE_0 0x00000040 +#define PC_INTERRUPT_STATUS_CNA_CSC_1 0x00000020 +#define PC_INTERRUPT_STATUS_CNA_CSC_0 0x00000010 +#define PC_INTERRUPT_STATUS_CNA_WEIGHT_1 0x00000008 +#define PC_INTERRUPT_STATUS_CNA_WEIGHT_0 0x00000004 +#define PC_INTERRUPT_STATUS_CNA_FEATURE_1 0x00000002 +#define PC_INTERRUPT_STATUS_CNA_FEATURE_0 0x00000001 +#define REG_PC_INTERRUPT_RAW_STATUS 0x0000002c +#define PC_INTERRUPT_RAW_STATUS_RESERVED_0__MASK 0xffffc000 +#define PC_INTERRUPT_RAW_STATUS_RESERVED_0__SHIFT 14 +#define PC_INTERRUPT_RAW_STATUS_DMA_WRITE_ERROR 0x00002000 +#define PC_INTERRUPT_RAW_STATUS_DMA_READ_ERROR 0x00001000 +#define PC_INTERRUPT_RAW_STATUS_PPU_1 0x00000800 +#define PC_INTERRUPT_RAW_STATUS_PPU_0 0x00000400 +#define PC_INTERRUPT_RAW_STATUS_DPU_1 0x00000200 +#define PC_INTERRUPT_RAW_STATUS_DPU_0 0x00000100 +#define PC_INTERRUPT_RAW_STATUS_CORE_1 0x00000080 +#define PC_INTERRUPT_RAW_STATUS_CORE_0 0x00000040 +#define PC_INTERRUPT_RAW_STATUS_CNA_CSC_1 0x00000020 +#define PC_INTERRUPT_RAW_STATUS_CNA_CSC_0 0x00000010 +#define PC_INTERRUPT_RAW_STATUS_CNA_WEIGHT_1 0x00000008 +#define PC_INTERRUPT_RAW_STATUS_CNA_WEIGHT_0 0x00000004 +#define PC_INTERRUPT_RAW_STATUS_CNA_FEATURE_1 0x00000002 +#define PC_INTERRUPT_RAW_STATUS_CNA_FEATURE_0 0x00000001 +#define REG_PC_TASK_CON 0x00000030 +#define PC_TASK_CON_RESERVED_0__MASK 0xffffc000 +#define PC_TASK_CON_RESERVED_0__SHIFT 14 +#define PC_TASK_CON_TASK_COUNT_CLEAR__MASK 0x00002000 +#define PC_TASK_CON_TASK_COUNT_CLEAR__SHIFT 13 +#define PC_TASK_CON_TASK_PP_EN__MASK 0x00001000 +#define PC_TASK_CON_TASK_PP_EN__SHIFT 12 +#define PC_TASK_CON_TASK_NUMBER__MASK 0x00000fff +#define PC_TASK_CON_TASK_NUMBER__SHIFT 0 +#define REG_PC_TASK_DMA_BASE_ADDR 0x00000034 +#define PC_TASK_DMA_BASE_ADDR_DMA_BASE_ADDR__MASK 0xfffffff0 +#define PC_TASK_DMA_BASE_ADDR_DMA_BASE_ADDR__SHIFT 4 +#define PC_TASK_DMA_BASE_ADDR_RESERVED_0__MASK 0x0000000f +#define PC_TASK_DMA_BASE_ADDR_RESERVED_0__SHIFT 0 +#define REG_PC_TASK_STATUS 0x0000003c +#define PC_TASK_STATUS_RESERVED_0__MASK 0xf0000000 +#define PC_TASK_STATUS_RESERVED_0__SHIFT 28 +#define PC_TASK_STATUS_TASK_STATUS__MASK 0x0fffffff +#define PC_TASK_STATUS_TASK_STATUS__SHIFT 0 +#define REG_CORE_S_STATUS 0x00003000 +#define CORE_S_STATUS_RESERVED_0__MASK 0xfffc0000 +#define CORE_S_STATUS_RESERVED_0__SHIFT 18 +#define CORE_S_STATUS_STATUS_1__MASK 0x00030000 +#define CORE_S_STATUS_STATUS_1__SHIFT 16 +#define CORE_S_STATUS_RESERVED_1__MASK 0x0000fffc +#define CORE_S_STATUS_RESERVED_1__SHIFT 2 +#define CORE_S_STATUS_STATUS_0__MASK 0x00000003 +#define CORE_S_STATUS_STATUS_0__SHIFT 0 +#define REG_CORE_S_POINTER 0x00003004 +#define CORE_S_POINTER_RESERVED_0__MASK 0xfffe0000 +#define CORE_S_POINTER_RESERVED_0__SHIFT 17 +#define CORE_S_POINTER_EXECUTER__MASK 0x00010000 +#define CORE_S_POINTER_EXECUTER__SHIFT 16 +#define CORE_S_POINTER_RESERVED_1__MASK 0x0000ffc0 +#define CORE_S_POINTER_RESERVED_1__SHIFT 6 +#define CORE_S_POINTER_EXECUTER_PP_CLEAR__MASK 0x00000020 +#define CORE_S_POINTER_EXECUTER_PP_CLEAR__SHIFT 5 +#define CORE_S_POINTER_POINTER_PP_CLEAR__MASK 0x00000010 +#define CORE_S_POINTER_POINTER_PP_CLEAR__SHIFT 4 +#define CORE_S_POINTER_POINTER_PP_MODE__MASK 0x00000008 +#define CORE_S_POINTER_POINTER_PP_MODE__SHIFT 3 +#define CORE_S_POINTER_EXECUTER_PP_EN__MASK 0x00000004 +#define CORE_S_POINTER_EXECUTER_PP_EN__SHIFT 2 +#define CORE_S_POINTER_POINTER_PP_EN__MASK 0x00000002 +#define CORE_S_POINTER_POINTER_PP_EN__SHIFT 1 +#define CORE_S_POINTER_POINTER__MASK 0x00000001 +#define CORE_S_POINTER_POINTER__SHIFT 0 +#define REG_CORE_OPERATION_ENABLE 0x00003008 +#define CORE_OPERATION_ENABLE_RESERVED_0__MASK 0xfffffffe +#define CORE_OPERATION_ENABLE_RESERVED_0__SHIFT 1 +#define CORE_OPERATION_ENABLE_OP_EN__MASK 0x00000001 +#define CORE_OPERATION_ENABLE_OP_EN__SHIFT 0 +#define REG_CORE_MAC_GATING 0x0000300c +#define CORE_MAC_GATING_RESERVED_0__MASK 0xf8000000 +#define CORE_MAC_GATING_RESERVED_0__SHIFT 27 +#define CORE_MAC_GATING_SLCG_OP_EN__MASK 0x07ffffff +#define CORE_MAC_GATING_SLCG_OP_EN__SHIFT 0 +#define REG_CORE_MISC_CFG 0x00003010 +#define CORE_MISC_CFG_RESERVED_0__MASK 0xfff00000 +#define CORE_MISC_CFG_RESERVED_0__SHIFT 20 +#define CORE_MISC_CFG_SOFT_GATING__MASK 0x000fc000 +#define CORE_MISC_CFG_SOFT_GATING__SHIFT 14 +#define CORE_MISC_CFG_RESERVED_1__MASK 0x00003800 +#define CORE_MISC_CFG_RESERVED_1__SHIFT 11 +#define CORE_MISC_CFG_PROC_PRECISION__MASK 0x00000700 +#define CORE_MISC_CFG_PROC_PRECISION__SHIFT 8 +#define CORE_MISC_CFG_RESERVED_2__MASK 0x000000fc +#define CORE_MISC_CFG_RESERVED_2__SHIFT 2 +#define CORE_MISC_CFG_DW_EN__MASK 0x00000002 +#define CORE_MISC_CFG_DW_EN__SHIFT 1 +#define CORE_MISC_CFG_QD_EN__MASK 0x00000001 +#define CORE_MISC_CFG_QD_EN__SHIFT 0 +#define REG_CORE_DATAOUT_SIZE_0 0x00003014 +#define CORE_DATAOUT_SIZE_0_DATAOUT_HEIGHT__MASK 0xffff0000 +#define CORE_DATAOUT_SIZE_0_DATAOUT_HEIGHT__SHIFT 16 +#define CORE_DATAOUT_SIZE_0_DATAOUT_WIDTH__MASK 0x0000ffff +#define CORE_DATAOUT_SIZE_0_DATAOUT_WIDTH__SHIFT 0 +#define REG_CORE_DATAOUT_SIZE_1 0x00003018 +#define CORE_DATAOUT_SIZE_1_RESERVED_0__MASK 0xffff0000 +#define CORE_DATAOUT_SIZE_1_RESERVED_0__SHIFT 16 +#define CORE_DATAOUT_SIZE_1_DATAOUT_CHANNEL__MASK 0x0000ffff +#define CORE_DATAOUT_SIZE_1_DATAOUT_CHANNEL__SHIFT 0 +#define REG_CORE_CLIP_TRUNCATE 0x0000301c +#define CORE_CLIP_TRUNCATE_RESERVED_0__MASK 0xffffff80 +#define CORE_CLIP_TRUNCATE_RESERVED_0__SHIFT 7 +#define CORE_CLIP_TRUNCATE_ROUND_TYPE__MASK 0x00000040 +#define CORE_CLIP_TRUNCATE_ROUND_TYPE__SHIFT 6 +#define CORE_CLIP_TRUNCATE_RESERVED_1__MASK 0x00000020 +#define CORE_CLIP_TRUNCATE_RESERVED_1__SHIFT 5 +#define CORE_CLIP_TRUNCATE_CLIP_TRUNCATE__MASK 0x0000001f +#define CORE_CLIP_TRUNCATE_CLIP_TRUNCATE__SHIFT 0 +#define REG_DPU_S_STATUS 0x00004000 +#define DPU_S_STATUS_RESERVED_0__MASK 0xfffc0000 +#define DPU_S_STATUS_RESERVED_0__SHIFT 18 +#define DPU_S_STATUS_STATUS_1__MASK 0x00030000 +#define DPU_S_STATUS_STATUS_1__SHIFT 16 +#define DPU_S_STATUS_RESERVED_1__MASK 0x0000fffc +#define DPU_S_STATUS_RESERVED_1__SHIFT 2 +#define DPU_S_STATUS_STATUS_0__MASK 0x00000003 +#define DPU_S_STATUS_STATUS_0__SHIFT 0 +#define REG_DPU_S_POINTER 0x00004004 +#define DPU_S_POINTER_RESERVED_0__MASK 0xfffe0000 +#define DPU_S_POINTER_RESERVED_0__SHIFT 17 +#define DPU_S_POINTER_EXECUTER__MASK 0x00010000 +#define DPU_S_POINTER_EXECUTER__SHIFT 16 +#define DPU_S_POINTER_RESERVED_1__MASK 0x0000ffc0 +#define DPU_S_POINTER_RESERVED_1__SHIFT 6 +#define DPU_S_POINTER_EXECUTER_PP_CLEAR__MASK 0x00000020 +#define DPU_S_POINTER_EXECUTER_PP_CLEAR__SHIFT 5 +#define DPU_S_POINTER_POINTER_PP_CLEAR__MASK 0x00000010 +#define DPU_S_POINTER_POINTER_PP_CLEAR__SHIFT 4 +#define DPU_S_POINTER_POINTER_PP_MODE__MASK 0x00000008 +#define DPU_S_POINTER_POINTER_PP_MODE__SHIFT 3 +#define DPU_S_POINTER_EXECUTER_PP_EN__MASK 0x00000004 +#define DPU_S_POINTER_EXECUTER_PP_EN__SHIFT 2 +#define DPU_S_POINTER_POINTER_PP_EN__MASK 0x00000002 +#define DPU_S_POINTER_POINTER_PP_EN__SHIFT 1 +#define DPU_S_POINTER_POINTER__MASK 0x00000001 +#define DPU_S_POINTER_POINTER__SHIFT 0 +#define REG_DPU_OPERATION_ENABLE 0x00004008 +#define DPU_OPERATION_ENABLE_RESERVED_0__MASK 0xfffffffe +#define DPU_OPERATION_ENABLE_RESERVED_0__SHIFT 1 +#define DPU_OPERATION_ENABLE_OP_EN__MASK 0x00000001 +#define DPU_OPERATION_ENABLE_OP_EN__SHIFT 0 +#define REG_DPU_FEATURE_MODE_CFG 0x0000400c +#define DPU_FEATURE_MODE_CFG_COMB_USE__MASK 0x80000000 +#define DPU_FEATURE_MODE_CFG_COMB_USE__SHIFT 31 +#define DPU_FEATURE_MODE_CFG_TP_EN__MASK 0x40000000 +#define DPU_FEATURE_MODE_CFG_TP_EN__SHIFT 30 +#define DPU_FEATURE_MODE_CFG_RGP_TYPE__MASK 0x3c000000 +#define DPU_FEATURE_MODE_CFG_RGP_TYPE__SHIFT 26 +#define DPU_FEATURE_MODE_CFG_NONALIGN__MASK 0x02000000 +#define DPU_FEATURE_MODE_CFG_NONALIGN__SHIFT 25 +#define DPU_FEATURE_MODE_CFG_SURF_LEN__MASK 0x01fffe00 +#define DPU_FEATURE_MODE_CFG_SURF_LEN__SHIFT 9 +#define DPU_FEATURE_MODE_CFG_BURST_LEN__MASK 0x000001e0 +#define DPU_FEATURE_MODE_CFG_BURST_LEN__SHIFT 5 +#define DPU_FEATURE_MODE_CFG_CONV_MODE__MASK 0x00000018 +#define DPU_FEATURE_MODE_CFG_CONV_MODE__SHIFT 3 +#define DPU_FEATURE_MODE_CFG_OUTPUT_MODE__MASK 0x00000006 +#define DPU_FEATURE_MODE_CFG_OUTPUT_MODE__SHIFT 1 +#define DPU_FEATURE_MODE_CFG_FLYING_MODE__MASK 0x00000001 +#define DPU_FEATURE_MODE_CFG_FLYING_MODE__SHIFT 0 +#define REG_DPU_DATA_FORMAT 0x00004010 +#define DPU_DATA_FORMAT_OUT_PRECISION__MASK 0xe0000000 +#define DPU_DATA_FORMAT_OUT_PRECISION__SHIFT 29 +#define DPU_DATA_FORMAT_IN_PRECISION__MASK 0x1c000000 +#define DPU_DATA_FORMAT_IN_PRECISION__SHIFT 26 +#define DPU_DATA_FORMAT_EW_TRUNCATE_NEG__MASK 0x03ff0000 +#define DPU_DATA_FORMAT_EW_TRUNCATE_NEG__SHIFT 16 +#define DPU_DATA_FORMAT_BN_MUL_SHIFT_VALUE_NEG__MASK 0x0000fc00 +#define DPU_DATA_FORMAT_BN_MUL_SHIFT_VALUE_NEG__SHIFT 10 +#define DPU_DATA_FORMAT_BS_MUL_SHIFT_VALUE_NEG__MASK 0x000003f0 +#define DPU_DATA_FORMAT_BS_MUL_SHIFT_VALUE_NEG__SHIFT 4 +#define DPU_DATA_FORMAT_MC_SURF_OUT__MASK 0x00000008 +#define DPU_DATA_FORMAT_MC_SURF_OUT__SHIFT 3 +#define DPU_DATA_FORMAT_PROC_PRECISION__MASK 0x00000007 +#define DPU_DATA_FORMAT_PROC_PRECISION__SHIFT 0 +#define REG_DPU_OFFSET_PEND 0x00004014 +#define DPU_OFFSET_PEND_RESERVED_0__MASK 0xffff0000 +#define DPU_OFFSET_PEND_RESERVED_0__SHIFT 16 +#define DPU_OFFSET_PEND_OFFSET_PEND__MASK 0x0000ffff +#define DPU_OFFSET_PEND_OFFSET_PEND__SHIFT 0 +#define REG_DPU_DST_BASE_ADDR 0x00004020 +#define DPU_DST_BASE_ADDR_DST_BASE_ADDR__MASK 0xffffffff +#define DPU_DST_BASE_ADDR_DST_BASE_ADDR__SHIFT 0 +#define REG_DPU_DST_SURF_STRIDE 0x00004024 +#define DPU_DST_SURF_STRIDE_DST_SURF_STRIDE__MASK 0xfffffff0 +#define DPU_DST_SURF_STRIDE_DST_SURF_STRIDE__SHIFT 4 +#define DPU_DST_SURF_STRIDE_RESERVED_0__MASK 0x0000000f +#define DPU_DST_SURF_STRIDE_RESERVED_0__SHIFT 0 +#define REG_DPU_DATA_CUBE_WIDTH 0x00004030 +#define DPU_DATA_CUBE_WIDTH_RESERVED_0__MASK 0xffffe000 +#define DPU_DATA_CUBE_WIDTH_RESERVED_0__SHIFT 13 +#define DPU_DATA_CUBE_WIDTH_WIDTH__MASK 0x00001fff +#define DPU_DATA_CUBE_WIDTH_WIDTH__SHIFT 0 +#define REG_DPU_DATA_CUBE_HEIGHT 0x00004034 +#define DPU_DATA_CUBE_HEIGHT_RESERVED_0__MASK 0xfe000000 +#define DPU_DATA_CUBE_HEIGHT_RESERVED_0__SHIFT 25 +#define DPU_DATA_CUBE_HEIGHT_MINMAX_CTL__MASK 0x01c00000 +#define DPU_DATA_CUBE_HEIGHT_MINMAX_CTL__SHIFT 22 +#define DPU_DATA_CUBE_HEIGHT_RESERVED_1__MASK 0x003fe000 +#define DPU_DATA_CUBE_HEIGHT_RESERVED_1__SHIFT 13 +#define DPU_DATA_CUBE_HEIGHT_HEIGHT__MASK 0x00001fff +#define DPU_DATA_CUBE_HEIGHT_HEIGHT__SHIFT 0 +#define REG_DPU_DATA_CUBE_NOTCH_ADDR 0x00004038 +#define DPU_DATA_CUBE_NOTCH_ADDR_RESERVED_0__MASK 0xe0000000 +#define DPU_DATA_CUBE_NOTCH_ADDR_RESERVED_0__SHIFT 29 +#define DPU_DATA_CUBE_NOTCH_ADDR_NOTCH_ADDR_1__MASK 0x1fff0000 +#define DPU_DATA_CUBE_NOTCH_ADDR_NOTCH_ADDR_1__SHIFT 16 +#define DPU_DATA_CUBE_NOTCH_ADDR_RESERVED_1__MASK 0x0000e000 +#define DPU_DATA_CUBE_NOTCH_ADDR_RESERVED_1__SHIFT 13 +#define DPU_DATA_CUBE_NOTCH_ADDR_NOTCH_ADDR_0__MASK 0x00001fff +#define DPU_DATA_CUBE_NOTCH_ADDR_NOTCH_ADDR_0__SHIFT 0 +#define REG_DPU_DATA_CUBE_CHANNEL 0x0000403c +#define DPU_DATA_CUBE_CHANNEL_RESERVED_0__MASK 0xe0000000 +#define DPU_DATA_CUBE_CHANNEL_RESERVED_0__SHIFT 29 +#define DPU_DATA_CUBE_CHANNEL_ORIG_CHANNEL__MASK 0x1fff0000 +#define DPU_DATA_CUBE_CHANNEL_ORIG_CHANNEL__SHIFT 16 +#define DPU_DATA_CUBE_CHANNEL_RESERVED_1__MASK 0x0000e000 +#define DPU_DATA_CUBE_CHANNEL_RESERVED_1__SHIFT 13 +#define DPU_DATA_CUBE_CHANNEL_CHANNEL__MASK 0x00001fff +#define DPU_DATA_CUBE_CHANNEL_CHANNEL__SHIFT 0 +#define REG_DPU_BS_CFG 0x00004040 +#define DPU_BS_CFG_RESERVED_0__MASK 0xfff00000 +#define DPU_BS_CFG_RESERVED_0__SHIFT 20 +#define DPU_BS_CFG_BS_ALU_ALGO__MASK 0x000f0000 +#define DPU_BS_CFG_BS_ALU_ALGO__SHIFT 16 +#define DPU_BS_CFG_RESERVED_1__MASK 0x0000fe00 +#define DPU_BS_CFG_RESERVED_1__SHIFT 9 +#define DPU_BS_CFG_BS_ALU_SRC__MASK 0x00000100 +#define DPU_BS_CFG_BS_ALU_SRC__SHIFT 8 +#define DPU_BS_CFG_BS_RELUX_EN__MASK 0x00000080 +#define DPU_BS_CFG_BS_RELUX_EN__SHIFT 7 +#define DPU_BS_CFG_BS_RELU_BYPASS__MASK 0x00000040 +#define DPU_BS_CFG_BS_RELU_BYPASS__SHIFT 6 +#define DPU_BS_CFG_BS_MUL_PRELU__MASK 0x00000020 +#define DPU_BS_CFG_BS_MUL_PRELU__SHIFT 5 +#define DPU_BS_CFG_BS_MUL_BYPASS__MASK 0x00000010 +#define DPU_BS_CFG_BS_MUL_BYPASS__SHIFT 4 +#define DPU_BS_CFG_RESERVED_2__MASK 0x0000000c +#define DPU_BS_CFG_RESERVED_2__SHIFT 2 +#define DPU_BS_CFG_BS_ALU_BYPASS__MASK 0x00000002 +#define DPU_BS_CFG_BS_ALU_BYPASS__SHIFT 1 +#define DPU_BS_CFG_BS_BYPASS__MASK 0x00000001 +#define DPU_BS_CFG_BS_BYPASS__SHIFT 0 +#define REG_DPU_BS_ALU_CFG 0x00004044 +#define DPU_BS_ALU_CFG_BS_ALU_OPERAND__MASK 0xffffffff +#define DPU_BS_ALU_CFG_BS_ALU_OPERAND__SHIFT 0 +#define REG_DPU_BS_MUL_CFG 0x00004048 +#define DPU_BS_MUL_CFG_BS_MUL_OPERAND__MASK 0xffff0000 +#define DPU_BS_MUL_CFG_BS_MUL_OPERAND__SHIFT 16 +#define DPU_BS_MUL_CFG_RESERVED_0__MASK 0x0000c000 +#define DPU_BS_MUL_CFG_RESERVED_0__SHIFT 14 +#define DPU_BS_MUL_CFG_BS_MUL_SHIFT_VALUE__MASK 0x00003f00 +#define DPU_BS_MUL_CFG_BS_MUL_SHIFT_VALUE__SHIFT 8 +#define DPU_BS_MUL_CFG_RESERVED_1__MASK 0x000000fc +#define DPU_BS_MUL_CFG_RESERVED_1__SHIFT 2 +#define DPU_BS_MUL_CFG_BS_TRUNCATE_SRC__MASK 0x00000002 +#define DPU_BS_MUL_CFG_BS_TRUNCATE_SRC__SHIFT 1 +#define DPU_BS_MUL_CFG_BS_MUL_SRC__MASK 0x00000001 +#define DPU_BS_MUL_CFG_BS_MUL_SRC__SHIFT 0 +#define REG_DPU_BS_RELUX_CMP_VALUE 0x0000404c +#define DPU_BS_RELUX_CMP_VALUE_BS_RELUX_CMP_DAT__MASK 0xffffffff +#define DPU_BS_RELUX_CMP_VALUE_BS_RELUX_CMP_DAT__SHIFT 0 +#define REG_DPU_BS_OW_CFG 0x00004050 +#define DPU_BS_OW_CFG_RGP_CNTER__MASK 0xf0000000 +#define DPU_BS_OW_CFG_RGP_CNTER__SHIFT 28 +#define DPU_BS_OW_CFG_TP_ORG_EN__MASK 0x08000000 +#define DPU_BS_OW_CFG_TP_ORG_EN__SHIFT 27 +#define DPU_BS_OW_CFG_RESERVED_0__MASK 0x07fff800 +#define DPU_BS_OW_CFG_RESERVED_0__SHIFT 11 +#define DPU_BS_OW_CFG_SIZE_E_2__MASK 0x00000700 +#define DPU_BS_OW_CFG_SIZE_E_2__SHIFT 8 +#define DPU_BS_OW_CFG_SIZE_E_1__MASK 0x000000e0 +#define DPU_BS_OW_CFG_SIZE_E_1__SHIFT 5 +#define DPU_BS_OW_CFG_SIZE_E_0__MASK 0x0000001c +#define DPU_BS_OW_CFG_SIZE_E_0__SHIFT 2 +#define DPU_BS_OW_CFG_OD_BYPASS__MASK 0x00000002 +#define DPU_BS_OW_CFG_OD_BYPASS__SHIFT 1 +#define DPU_BS_OW_CFG_OW_SRC__MASK 0x00000001 +#define DPU_BS_OW_CFG_OW_SRC__SHIFT 0 +#define REG_DPU_BS_OW_OP 0x00004054 +#define DPU_BS_OW_OP_RESERVED_0__MASK 0xffff0000 +#define DPU_BS_OW_OP_RESERVED_0__SHIFT 16 +#define DPU_BS_OW_OP_OW_OP__MASK 0x0000ffff +#define DPU_BS_OW_OP_OW_OP__SHIFT 0 +#define REG_DPU_WDMA_SIZE_0 0x00004058 +#define DPU_WDMA_SIZE_0_RESERVED_0__MASK 0xf0000000 +#define DPU_WDMA_SIZE_0_RESERVED_0__SHIFT 28 +#define DPU_WDMA_SIZE_0_TP_PRECISION__MASK 0x08000000 +#define DPU_WDMA_SIZE_0_TP_PRECISION__SHIFT 27 +#define DPU_WDMA_SIZE_0_SIZE_C_WDMA__MASK 0x07ff0000 +#define DPU_WDMA_SIZE_0_SIZE_C_WDMA__SHIFT 16 +#define DPU_WDMA_SIZE_0_RESERVED_1__MASK 0x0000e000 +#define DPU_WDMA_SIZE_0_RESERVED_1__SHIFT 13 +#define DPU_WDMA_SIZE_0_CHANNEL_WDMA__MASK 0x00001fff +#define DPU_WDMA_SIZE_0_CHANNEL_WDMA__SHIFT 0 +#define REG_DPU_WDMA_SIZE_1 0x0000405c +#define DPU_WDMA_SIZE_1_RESERVED_0__MASK 0xe0000000 +#define DPU_WDMA_SIZE_1_RESERVED_0__SHIFT 29 +#define DPU_WDMA_SIZE_1_HEIGHT_WDMA__MASK 0x1fff0000 +#define DPU_WDMA_SIZE_1_HEIGHT_WDMA__SHIFT 16 +#define DPU_WDMA_SIZE_1_RESERVED_1__MASK 0x0000e000 +#define DPU_WDMA_SIZE_1_RESERVED_1__SHIFT 13 +#define DPU_WDMA_SIZE_1_WIDTH_WDMA__MASK 0x00001fff +#define DPU_WDMA_SIZE_1_WIDTH_WDMA__SHIFT 0 +#define REG_DPU_BN_CFG 0x00004060 +#define DPU_BN_CFG_RESERVED_0__MASK 0xfff00000 +#define DPU_BN_CFG_RESERVED_0__SHIFT 20 +#define DPU_BN_CFG_BN_ALU_ALGO__MASK 0x000f0000 +#define DPU_BN_CFG_BN_ALU_ALGO__SHIFT 16 +#define DPU_BN_CFG_RESERVED_1__MASK 0x0000fe00 +#define DPU_BN_CFG_RESERVED_1__SHIFT 9 +#define DPU_BN_CFG_BN_ALU_SRC__MASK 0x00000100 +#define DPU_BN_CFG_BN_ALU_SRC__SHIFT 8 +#define DPU_BN_CFG_BN_RELUX_EN__MASK 0x00000080 +#define DPU_BN_CFG_BN_RELUX_EN__SHIFT 7 +#define DPU_BN_CFG_BN_RELU_BYPASS__MASK 0x00000040 +#define DPU_BN_CFG_BN_RELU_BYPASS__SHIFT 6 +#define DPU_BN_CFG_BN_MUL_PRELU__MASK 0x00000020 +#define DPU_BN_CFG_BN_MUL_PRELU__SHIFT 5 +#define DPU_BN_CFG_BN_MUL_BYPASS__MASK 0x00000010 +#define DPU_BN_CFG_BN_MUL_BYPASS__SHIFT 4 +#define DPU_BN_CFG_RESERVED_2__MASK 0x0000000c +#define DPU_BN_CFG_RESERVED_2__SHIFT 2 +#define DPU_BN_CFG_BN_ALU_BYPASS__MASK 0x00000002 +#define DPU_BN_CFG_BN_ALU_BYPASS__SHIFT 1 +#define DPU_BN_CFG_BN_BYPASS__MASK 0x00000001 +#define DPU_BN_CFG_BN_BYPASS__SHIFT 0 +#define REG_DPU_BN_ALU_CFG 0x00004064 +#define DPU_BN_ALU_CFG_BN_ALU_OPERAND__MASK 0xffffffff +#define DPU_BN_ALU_CFG_BN_ALU_OPERAND__SHIFT 0 +#define REG_DPU_BN_MUL_CFG 0x00004068 +#define DPU_BN_MUL_CFG_BN_MUL_OPERAND__MASK 0xffff0000 +#define DPU_BN_MUL_CFG_BN_MUL_OPERAND__SHIFT 16 +#define DPU_BN_MUL_CFG_RESERVED_0__MASK 0x0000c000 +#define DPU_BN_MUL_CFG_RESERVED_0__SHIFT 14 +#define DPU_BN_MUL_CFG_BN_MUL_SHIFT_VALUE__MASK 0x00003f00 +#define DPU_BN_MUL_CFG_BN_MUL_SHIFT_VALUE__SHIFT 8 +#define DPU_BN_MUL_CFG_RESERVED_1__MASK 0x000000fc +#define DPU_BN_MUL_CFG_RESERVED_1__SHIFT 2 +#define DPU_BN_MUL_CFG_BN_TRUNCATE_SRC__MASK 0x00000002 +#define DPU_BN_MUL_CFG_BN_TRUNCATE_SRC__SHIFT 1 +#define DPU_BN_MUL_CFG_BN_MUL_SRC__MASK 0x00000001 +#define DPU_BN_MUL_CFG_BN_MUL_SRC__SHIFT 0 +#define REG_DPU_BN_RELUX_CMP_VALUE 0x0000406c +#define DPU_BN_RELUX_CMP_VALUE_BN_RELUX_CMP_DAT__MASK 0xffffffff +#define DPU_BN_RELUX_CMP_VALUE_BN_RELUX_CMP_DAT__SHIFT 0 +#define REG_DPU_EW_CFG 0x00004070 +#define DPU_EW_CFG_EW_CVT_TYPE__MASK 0x80000000 +#define DPU_EW_CFG_EW_CVT_TYPE__SHIFT 31 +#define DPU_EW_CFG_EW_CVT_ROUND__MASK 0x40000000 +#define DPU_EW_CFG_EW_CVT_ROUND__SHIFT 30 +#define DPU_EW_CFG_EW_DATA_MODE__MASK 0x30000000 +#define DPU_EW_CFG_EW_DATA_MODE__SHIFT 28 +#define DPU_EW_CFG_RESERVED_0__MASK 0x0f000000 +#define DPU_EW_CFG_RESERVED_0__SHIFT 24 +#define DPU_EW_CFG_EDATA_SIZE__MASK 0x00c00000 +#define DPU_EW_CFG_EDATA_SIZE__SHIFT 22 +#define DPU_EW_CFG_EW_EQUAL_EN__MASK 0x00200000 +#define DPU_EW_CFG_EW_EQUAL_EN__SHIFT 21 +#define DPU_EW_CFG_EW_BINARY_EN__MASK 0x00100000 +#define DPU_EW_CFG_EW_BINARY_EN__SHIFT 20 +#define DPU_EW_CFG_EW_ALU_ALGO__MASK 0x000f0000 +#define DPU_EW_CFG_EW_ALU_ALGO__SHIFT 16 +#define DPU_EW_CFG_RESERVED_1__MASK 0x0000f800 +#define DPU_EW_CFG_RESERVED_1__SHIFT 11 +#define DPU_EW_CFG_EW_RELUX_EN__MASK 0x00000400 +#define DPU_EW_CFG_EW_RELUX_EN__SHIFT 10 +#define DPU_EW_CFG_EW_RELU_BYPASS__MASK 0x00000200 +#define DPU_EW_CFG_EW_RELU_BYPASS__SHIFT 9 +#define DPU_EW_CFG_EW_OP_CVT_BYPASS__MASK 0x00000100 +#define DPU_EW_CFG_EW_OP_CVT_BYPASS__SHIFT 8 +#define DPU_EW_CFG_EW_LUT_BYPASS__MASK 0x00000080 +#define DPU_EW_CFG_EW_LUT_BYPASS__SHIFT 7 +#define DPU_EW_CFG_EW_OP_SRC__MASK 0x00000040 +#define DPU_EW_CFG_EW_OP_SRC__SHIFT 6 +#define DPU_EW_CFG_EW_MUL_PRELU__MASK 0x00000020 +#define DPU_EW_CFG_EW_MUL_PRELU__SHIFT 5 +#define DPU_EW_CFG_RESERVED_2__MASK 0x00000018 +#define DPU_EW_CFG_RESERVED_2__SHIFT 3 +#define DPU_EW_CFG_EW_OP_TYPE__MASK 0x00000004 +#define DPU_EW_CFG_EW_OP_TYPE__SHIFT 2 +#define DPU_EW_CFG_EW_OP_BYPASS__MASK 0x00000002 +#define DPU_EW_CFG_EW_OP_BYPASS__SHIFT 1 +#define DPU_EW_CFG_EW_BYPASS__MASK 0x00000001 +#define DPU_EW_CFG_EW_BYPASS__SHIFT 0 +#define REG_DPU_EW_CVT_OFFSET_VALUE 0x00004074 +#define DPU_EW_CVT_OFFSET_VALUE_EW_OP_CVT_OFFSET__MASK 0xffffffff +#define DPU_EW_CVT_OFFSET_VALUE_EW_OP_CVT_OFFSET__SHIFT 0 +#define REG_DPU_EW_CVT_SCALE_VALUE 0x00004078 +#define DPU_EW_CVT_SCALE_VALUE_EW_TRUNCATE__MASK 0xffc00000 +#define DPU_EW_CVT_SCALE_VALUE_EW_TRUNCATE__SHIFT 22 +#define DPU_EW_CVT_SCALE_VALUE_EW_OP_CVT_SHIFT__MASK 0x003f0000 +#define DPU_EW_CVT_SCALE_VALUE_EW_OP_CVT_SHIFT__SHIFT 16 +#define DPU_EW_CVT_SCALE_VALUE_EW_OP_CVT_SCALE__MASK 0x0000ffff +#define DPU_EW_CVT_SCALE_VALUE_EW_OP_CVT_SCALE__SHIFT 0 +#define REG_DPU_EW_RELUX_CMP_VALUE 0x0000407c +#define DPU_EW_RELUX_CMP_VALUE_EW_RELUX_CMP_DAT__MASK 0xffffffff +#define DPU_EW_RELUX_CMP_VALUE_EW_RELUX_CMP_DAT__SHIFT 0 +#define REG_DPU_OUT_CVT_OFFSET 0x00004080 +#define DPU_OUT_CVT_OFFSET_OUT_CVT_OFFSET__MASK 0xffffffff +#define DPU_OUT_CVT_OFFSET_OUT_CVT_OFFSET__SHIFT 0 +#define REG_DPU_OUT_CVT_SCALE 0x00004084 +#define DPU_OUT_CVT_SCALE_RESERVED_0__MASK 0xfffe0000 +#define DPU_OUT_CVT_SCALE_RESERVED_0__SHIFT 17 +#define DPU_OUT_CVT_SCALE_FP32TOFP16_EN__MASK 0x00010000 +#define DPU_OUT_CVT_SCALE_FP32TOFP16_EN__SHIFT 16 +#define DPU_OUT_CVT_SCALE_OUT_CVT_SCALE__MASK 0x0000ffff +#define DPU_OUT_CVT_SCALE_OUT_CVT_SCALE__SHIFT 0 +#define REG_DPU_OUT_CVT_SHIFT 0x00004088 +#define DPU_OUT_CVT_SHIFT_CVT_TYPE__MASK 0x80000000 +#define DPU_OUT_CVT_SHIFT_CVT_TYPE__SHIFT 31 +#define DPU_OUT_CVT_SHIFT_CVT_ROUND__MASK 0x40000000 +#define DPU_OUT_CVT_SHIFT_CVT_ROUND__SHIFT 30 +#define DPU_OUT_CVT_SHIFT_RESERVED_0__MASK 0x3ff00000 +#define DPU_OUT_CVT_SHIFT_RESERVED_0__SHIFT 20 +#define DPU_OUT_CVT_SHIFT_MINUS_EXP__MASK 0x000ff000 +#define DPU_OUT_CVT_SHIFT_MINUS_EXP__SHIFT 12 +#define DPU_OUT_CVT_SHIFT_OUT_CVT_SHIFT__MASK 0x00000fff +#define DPU_OUT_CVT_SHIFT_OUT_CVT_SHIFT__SHIFT 0 +#define REG_DPU_EW_OP_VALUE_0 0x00004090 +#define DPU_EW_OP_VALUE_0_EW_OPERAND_0__MASK 0xffffffff +#define DPU_EW_OP_VALUE_0_EW_OPERAND_0__SHIFT 0 +#define REG_DPU_EW_OP_VALUE_1 0x00004094 +#define DPU_EW_OP_VALUE_1_EW_OPERAND_1__MASK 0xffffffff +#define DPU_EW_OP_VALUE_1_EW_OPERAND_1__SHIFT 0 +#define REG_DPU_EW_OP_VALUE_2 0x00004098 +#define DPU_EW_OP_VALUE_2_EW_OPERAND_2__MASK 0xffffffff +#define DPU_EW_OP_VALUE_2_EW_OPERAND_2__SHIFT 0 +#define REG_DPU_EW_OP_VALUE_3 0x0000409c +#define DPU_EW_OP_VALUE_3_EW_OPERAND_3__MASK 0xffffffff +#define DPU_EW_OP_VALUE_3_EW_OPERAND_3__SHIFT 0 +#define REG_DPU_EW_OP_VALUE_4 0x000040a0 +#define DPU_EW_OP_VALUE_4_EW_OPERAND_4__MASK 0xffffffff +#define DPU_EW_OP_VALUE_4_EW_OPERAND_4__SHIFT 0 +#define REG_DPU_EW_OP_VALUE_5 0x000040a4 +#define DPU_EW_OP_VALUE_5_EW_OPERAND_5__MASK 0xffffffff +#define DPU_EW_OP_VALUE_5_EW_OPERAND_5__SHIFT 0 +#define REG_DPU_EW_OP_VALUE_6 0x000040a8 +#define DPU_EW_OP_VALUE_6_EW_OPERAND_6__MASK 0xffffffff +#define DPU_EW_OP_VALUE_6_EW_OPERAND_6__SHIFT 0 +#define REG_DPU_EW_OP_VALUE_7 0x000040ac +#define DPU_EW_OP_VALUE_7_EW_OPERAND_7__MASK 0xffffffff +#define DPU_EW_OP_VALUE_7_EW_OPERAND_7__SHIFT 0 +#define REG_DPU_SURFACE_ADD 0x000040c0 +#define DPU_SURFACE_ADD_SURF_ADD__MASK 0xfffffff0 +#define DPU_SURFACE_ADD_SURF_ADD__SHIFT 4 +#define DPU_SURFACE_ADD_RESERVED_0__MASK 0x0000000f +#define DPU_SURFACE_ADD_RESERVED_0__SHIFT 0 +#define REG_DPU_LUT_ACCESS_CFG 0x00004100 +#define DPU_LUT_ACCESS_CFG_RESERVED_0__MASK 0xfffc0000 +#define DPU_LUT_ACCESS_CFG_RESERVED_0__SHIFT 18 +#define DPU_LUT_ACCESS_CFG_LUT_ACCESS_TYPE__MASK 0x00020000 +#define DPU_LUT_ACCESS_CFG_LUT_ACCESS_TYPE__SHIFT 17 +#define DPU_LUT_ACCESS_CFG_LUT_TABLE_ID__MASK 0x00010000 +#define DPU_LUT_ACCESS_CFG_LUT_TABLE_ID__SHIFT 16 +#define DPU_LUT_ACCESS_CFG_RESERVED_1__MASK 0x0000fc00 +#define DPU_LUT_ACCESS_CFG_RESERVED_1__SHIFT 10 +#define DPU_LUT_ACCESS_CFG_LUT_ADDR__MASK 0x000003ff +#define DPU_LUT_ACCESS_CFG_LUT_ADDR__SHIFT 0 +#define REG_DPU_LUT_ACCESS_DATA 0x00004104 +#define DPU_LUT_ACCESS_DATA_RESERVED_0__MASK 0xffff0000 +#define DPU_LUT_ACCESS_DATA_RESERVED_0__SHIFT 16 +#define DPU_LUT_ACCESS_DATA_LUT_ACCESS_DATA__MASK 0x0000ffff +#define DPU_LUT_ACCESS_DATA_LUT_ACCESS_DATA__SHIFT 0 +#define REG_DPU_LUT_CFG 0x00004108 +#define DPU_LUT_CFG_RESERVED_0__MASK 0xffffff00 +#define DPU_LUT_CFG_RESERVED_0__SHIFT 8 +#define DPU_LUT_CFG_LUT_CAL_SEL__MASK 0x00000080 +#define DPU_LUT_CFG_LUT_CAL_SEL__SHIFT 7 +#define DPU_LUT_CFG_LUT_HYBRID_PRIORITY__MASK 0x00000040 +#define DPU_LUT_CFG_LUT_HYBRID_PRIORITY__SHIFT 6 +#define DPU_LUT_CFG_LUT_OFLOW_PRIORITY__MASK 0x00000020 +#define DPU_LUT_CFG_LUT_OFLOW_PRIORITY__SHIFT 5 +#define DPU_LUT_CFG_LUT_UFLOW_PRIORITY__MASK 0x00000010 +#define DPU_LUT_CFG_LUT_UFLOW_PRIORITY__SHIFT 4 +#define DPU_LUT_CFG_LUT_LO_LE_MUX__MASK 0x0000000c +#define DPU_LUT_CFG_LUT_LO_LE_MUX__SHIFT 2 +#define DPU_LUT_CFG_LUT_EXPAND_EN__MASK 0x00000002 +#define DPU_LUT_CFG_LUT_EXPAND_EN__SHIFT 1 +#define DPU_LUT_CFG_LUT_ROAD_SEL__MASK 0x00000001 +#define DPU_LUT_CFG_LUT_ROAD_SEL__SHIFT 0 +#define REG_DPU_LUT_INFO 0x0000410c +#define DPU_LUT_INFO_RESERVED_0__MASK 0xff000000 +#define DPU_LUT_INFO_RESERVED_0__SHIFT 24 +#define DPU_LUT_INFO_LUT_LO_INDEX_SELECT__MASK 0x00ff0000 +#define DPU_LUT_INFO_LUT_LO_INDEX_SELECT__SHIFT 16 +#define DPU_LUT_INFO_LUT_LE_INDEX_SELECT__MASK 0x0000ff00 +#define DPU_LUT_INFO_LUT_LE_INDEX_SELECT__SHIFT 8 +#define DPU_LUT_INFO_RESERVED_1__MASK 0x000000ff +#define DPU_LUT_INFO_RESERVED_1__SHIFT 0 +#define REG_DPU_LUT_LE_START 0x00004110 +#define DPU_LUT_LE_START_LUT_LE_START__MASK 0xffffffff +#define DPU_LUT_LE_START_LUT_LE_START__SHIFT 0 +#define REG_DPU_LUT_LE_END 0x00004114 +#define DPU_LUT_LE_END_LUT_LE_END__MASK 0xffffffff +#define DPU_LUT_LE_END_LUT_LE_END__SHIFT 0 +#define REG_DPU_LUT_LO_START 0x00004118 +#define DPU_LUT_LO_START_LUT_LO_START__MASK 0xffffffff +#define DPU_LUT_LO_START_LUT_LO_START__SHIFT 0 +#define REG_DPU_LUT_LO_END 0x0000411c +#define DPU_LUT_LO_END_LUT_LO_END__MASK 0xffffffff +#define DPU_LUT_LO_END_LUT_LO_END__SHIFT 0 +#define REG_DPU_LUT_LE_SLOPE_SCALE 0x00004120 +#define DPU_LUT_LE_SLOPE_SCALE_LUT_LE_SLOPE_OFLOW_SCALE__MASK 0xffff0000 +#define DPU_LUT_LE_SLOPE_SCALE_LUT_LE_SLOPE_OFLOW_SCALE__SHIFT 16 +#define DPU_LUT_LE_SLOPE_SCALE_LUT_LE_SLOPE_UFLOW_SCALE__MASK 0x0000ffff +#define DPU_LUT_LE_SLOPE_SCALE_LUT_LE_SLOPE_UFLOW_SCALE__SHIFT 0 +#define REG_DPU_LUT_LE_SLOPE_SHIFT 0x00004124 +#define DPU_LUT_LE_SLOPE_SHIFT_RESERVED_0__MASK 0xfffffc00 +#define DPU_LUT_LE_SLOPE_SHIFT_RESERVED_0__SHIFT 10 +#define DPU_LUT_LE_SLOPE_SHIFT_LUT_LE_SLOPE_OFLOW_SHIFT__MASK 0x000003e0 +#define DPU_LUT_LE_SLOPE_SHIFT_LUT_LE_SLOPE_OFLOW_SHIFT__SHIFT 5 +#define DPU_LUT_LE_SLOPE_SHIFT_LUT_LE_SLOPE_UFLOW_SHIFT__MASK 0x0000001f +#define DPU_LUT_LE_SLOPE_SHIFT_LUT_LE_SLOPE_UFLOW_SHIFT__SHIFT 0 +#define REG_DPU_LUT_LO_SLOPE_SCALE 0x00004128 +#define DPU_LUT_LO_SLOPE_SCALE_LUT_LO_SLOPE_OFLOW_SCALE__MASK 0xffff0000 +#define DPU_LUT_LO_SLOPE_SCALE_LUT_LO_SLOPE_OFLOW_SCALE__SHIFT 16 +#define DPU_LUT_LO_SLOPE_SCALE_LUT_LO_SLOPE_UFLOW_SCALE__MASK 0x0000ffff +#define DPU_LUT_LO_SLOPE_SCALE_LUT_LO_SLOPE_UFLOW_SCALE__SHIFT 0 +#define REG_DPU_LUT_LO_SLOPE_SHIFT 0x0000412c +#define DPU_LUT_LO_SLOPE_SHIFT_RESERVED_0__MASK 0xfffffc00 +#define DPU_LUT_LO_SLOPE_SHIFT_RESERVED_0__SHIFT 10 +#define DPU_LUT_LO_SLOPE_SHIFT_LUT_LO_SLOPE_OFLOW_SHIFT__MASK 0x000003e0 +#define DPU_LUT_LO_SLOPE_SHIFT_LUT_LO_SLOPE_OFLOW_SHIFT__SHIFT 5 +#define DPU_LUT_LO_SLOPE_SHIFT_LUT_LO_SLOPE_UFLOW_SHIFT__MASK 0x0000001f +#define DPU_LUT_LO_SLOPE_SHIFT_LUT_LO_SLOPE_UFLOW_SHIFT__SHIFT 0 +#define REG_DPU_RDMA_RDMA_S_STATUS 0x00005000 +#define DPU_RDMA_RDMA_S_STATUS_RESERVED_0__MASK 0xfffc0000 +#define DPU_RDMA_RDMA_S_STATUS_RESERVED_0__SHIFT 18 +#define DPU_RDMA_RDMA_S_STATUS_STATUS_1__MASK 0x00030000 +#define DPU_RDMA_RDMA_S_STATUS_STATUS_1__SHIFT 16 +#define DPU_RDMA_RDMA_S_STATUS_RESERVED_1__MASK 0x0000fffc +#define DPU_RDMA_RDMA_S_STATUS_RESERVED_1__SHIFT 2 +#define DPU_RDMA_RDMA_S_STATUS_STATUS_0__MASK 0x00000003 +#define DPU_RDMA_RDMA_S_STATUS_STATUS_0__SHIFT 0 +#define REG_DPU_RDMA_RDMA_S_POINTER 0x00005004 +#define DPU_RDMA_RDMA_S_POINTER_RESERVED_0__MASK 0xfffe0000 +#define DPU_RDMA_RDMA_S_POINTER_RESERVED_0__SHIFT 17 +#define DPU_RDMA_RDMA_S_POINTER_EXECUTER__MASK 0x00010000 +#define DPU_RDMA_RDMA_S_POINTER_EXECUTER__SHIFT 16 +#define DPU_RDMA_RDMA_S_POINTER_RESERVED_1__MASK 0x0000ffc0 +#define DPU_RDMA_RDMA_S_POINTER_RESERVED_1__SHIFT 6 +#define DPU_RDMA_RDMA_S_POINTER_EXECUTER_PP_CLEAR__MASK 0x00000020 +#define DPU_RDMA_RDMA_S_POINTER_EXECUTER_PP_CLEAR__SHIFT 5 +#define DPU_RDMA_RDMA_S_POINTER_POINTER_PP_CLEAR__MASK 0x00000010 +#define DPU_RDMA_RDMA_S_POINTER_POINTER_PP_CLEAR__SHIFT 4 +#define DPU_RDMA_RDMA_S_POINTER_POINTER_PP_MODE__MASK 0x00000008 +#define DPU_RDMA_RDMA_S_POINTER_POINTER_PP_MODE__SHIFT 3 +#define DPU_RDMA_RDMA_S_POINTER_EXECUTER_PP_EN__MASK 0x00000004 +#define DPU_RDMA_RDMA_S_POINTER_EXECUTER_PP_EN__SHIFT 2 +#define DPU_RDMA_RDMA_S_POINTER_POINTER_PP_EN__MASK 0x00000002 +#define DPU_RDMA_RDMA_S_POINTER_POINTER_PP_EN__SHIFT 1 +#define DPU_RDMA_RDMA_S_POINTER_POINTER__MASK 0x00000001 +#define DPU_RDMA_RDMA_S_POINTER_POINTER__SHIFT 0 +#define REG_DPU_RDMA_RDMA_OPERATION_ENABLE 0x00005008 +#define DPU_RDMA_RDMA_OPERATION_ENABLE_RESERVED_0__MASK 0xfffffffe +#define DPU_RDMA_RDMA_OPERATION_ENABLE_RESERVED_0__SHIFT 1 +#define DPU_RDMA_RDMA_OPERATION_ENABLE_OP_EN__MASK 0x00000001 +#define DPU_RDMA_RDMA_OPERATION_ENABLE_OP_EN__SHIFT 0 +#define REG_DPU_RDMA_RDMA_DATA_CUBE_WIDTH 0x0000500c +#define DPU_RDMA_RDMA_DATA_CUBE_WIDTH_RESERVED_0__MASK 0xffffe000 +#define DPU_RDMA_RDMA_DATA_CUBE_WIDTH_RESERVED_0__SHIFT 13 +#define DPU_RDMA_RDMA_DATA_CUBE_WIDTH_WIDTH__MASK 0x00001fff +#define DPU_RDMA_RDMA_DATA_CUBE_WIDTH_WIDTH__SHIFT 0 +#define REG_DPU_RDMA_RDMA_DATA_CUBE_HEIGHT 0x00005010 +#define DPU_RDMA_RDMA_DATA_CUBE_HEIGHT_RESERVED_0__MASK 0xe0000000 +#define DPU_RDMA_RDMA_DATA_CUBE_HEIGHT_RESERVED_0__SHIFT 29 +#define DPU_RDMA_RDMA_DATA_CUBE_HEIGHT_EW_LINE_NOTCH_ADDR__MASK 0x1fff0000 +#define DPU_RDMA_RDMA_DATA_CUBE_HEIGHT_EW_LINE_NOTCH_ADDR__SHIFT 16 +#define DPU_RDMA_RDMA_DATA_CUBE_HEIGHT_RESERVED_1__MASK 0x0000e000 +#define DPU_RDMA_RDMA_DATA_CUBE_HEIGHT_RESERVED_1__SHIFT 13 +#define DPU_RDMA_RDMA_DATA_CUBE_HEIGHT_HEIGHT__MASK 0x00001fff +#define DPU_RDMA_RDMA_DATA_CUBE_HEIGHT_HEIGHT__SHIFT 0 +#define REG_DPU_RDMA_RDMA_DATA_CUBE_CHANNEL 0x00005014 +#define DPU_RDMA_RDMA_DATA_CUBE_CHANNEL_RESERVED_0__MASK 0xffffe000 +#define DPU_RDMA_RDMA_DATA_CUBE_CHANNEL_RESERVED_0__SHIFT 13 +#define DPU_RDMA_RDMA_DATA_CUBE_CHANNEL_CHANNEL__MASK 0x00001fff +#define DPU_RDMA_RDMA_DATA_CUBE_CHANNEL_CHANNEL__SHIFT 0 +#define REG_DPU_RDMA_RDMA_SRC_BASE_ADDR 0x00005018 +#define DPU_RDMA_RDMA_SRC_BASE_ADDR_SRC_BASE_ADDR__MASK 0xffffffff +#define DPU_RDMA_RDMA_SRC_BASE_ADDR_SRC_BASE_ADDR__SHIFT 0 +#define REG_DPU_RDMA_RDMA_BRDMA_CFG 0x0000501c +#define DPU_RDMA_RDMA_BRDMA_CFG_RESERVED_0__MASK 0xffffffe0 +#define DPU_RDMA_RDMA_BRDMA_CFG_RESERVED_0__SHIFT 5 +#define DPU_RDMA_RDMA_BRDMA_CFG_BRDMA_DATA_USE__MASK 0x0000001e +#define DPU_RDMA_RDMA_BRDMA_CFG_BRDMA_DATA_USE__SHIFT 1 +#define DPU_RDMA_RDMA_BRDMA_CFG_RESERVED_1__MASK 0x00000001 +#define DPU_RDMA_RDMA_BRDMA_CFG_RESERVED_1__SHIFT 0 +#define REG_DPU_RDMA_RDMA_BS_BASE_ADDR 0x00005020 +#define DPU_RDMA_RDMA_BS_BASE_ADDR_BS_BASE_ADDR__MASK 0xffffffff +#define DPU_RDMA_RDMA_BS_BASE_ADDR_BS_BASE_ADDR__SHIFT 0 +#define REG_DPU_RDMA_RDMA_NRDMA_CFG 0x00005028 +#define DPU_RDMA_RDMA_NRDMA_CFG_RESERVED_0__MASK 0xffffffe0 +#define DPU_RDMA_RDMA_NRDMA_CFG_RESERVED_0__SHIFT 5 +#define DPU_RDMA_RDMA_NRDMA_CFG_NRDMA_DATA_USE__MASK 0x0000001e +#define DPU_RDMA_RDMA_NRDMA_CFG_NRDMA_DATA_USE__SHIFT 1 +#define DPU_RDMA_RDMA_NRDMA_CFG_RESERVED_1__MASK 0x00000001 +#define DPU_RDMA_RDMA_NRDMA_CFG_RESERVED_1__SHIFT 0 +#define REG_DPU_RDMA_RDMA_BN_BASE_ADDR 0x0000502c +#define DPU_RDMA_RDMA_BN_BASE_ADDR_BN_BASE_ADDR__MASK 0xffffffff +#define DPU_RDMA_RDMA_BN_BASE_ADDR_BN_BASE_ADDR__SHIFT 0 +#define REG_DPU_RDMA_RDMA_ERDMA_CFG 0x00005034 +#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_DATA_MODE__MASK 0xc0000000 +#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_DATA_MODE__SHIFT 30 +#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_SURF_MODE__MASK 0x20000000 +#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_SURF_MODE__SHIFT 29 +#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_NONALIGN__MASK 0x10000000 +#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_NONALIGN__SHIFT 28 +#define DPU_RDMA_RDMA_ERDMA_CFG_RESERVED_0__MASK 0x0ffffff0 +#define DPU_RDMA_RDMA_ERDMA_CFG_RESERVED_0__SHIFT 4 +#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_DATA_SIZE__MASK 0x0000000c +#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_DATA_SIZE__SHIFT 2 +#define DPU_RDMA_RDMA_ERDMA_CFG_OV4K_BYPASS__MASK 0x00000002 +#define DPU_RDMA_RDMA_ERDMA_CFG_OV4K_BYPASS__SHIFT 1 +#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_DISABLE__MASK 0x00000001 +#define DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_DISABLE__SHIFT 0 +#define REG_DPU_RDMA_RDMA_EW_BASE_ADDR 0x00005038 +#define DPU_RDMA_RDMA_EW_BASE_ADDR_EW_BASE_ADDR__MASK 0xffffffff +#define DPU_RDMA_RDMA_EW_BASE_ADDR_EW_BASE_ADDR__SHIFT 0 +#define REG_DPU_RDMA_RDMA_EW_SURF_STRIDE 0x00005040 +#define DPU_RDMA_RDMA_EW_SURF_STRIDE_EW_SURF_STRIDE__MASK 0xfffffff0 +#define DPU_RDMA_RDMA_EW_SURF_STRIDE_EW_SURF_STRIDE__SHIFT 4 +#define DPU_RDMA_RDMA_EW_SURF_STRIDE_RESERVED_0__MASK 0x0000000f +#define DPU_RDMA_RDMA_EW_SURF_STRIDE_RESERVED_0__SHIFT 0 +#define REG_DPU_RDMA_RDMA_FEATURE_MODE_CFG 0x00005044 +#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_RESERVED_0__MASK 0xfffc0000 +#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_RESERVED_0__SHIFT 18 +#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_IN_PRECISION__MASK 0x00038000 +#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_IN_PRECISION__SHIFT 15 +#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_BURST_LEN__MASK 0x00007800 +#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_BURST_LEN__SHIFT 11 +#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_COMB_USE__MASK 0x00000700 +#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_COMB_USE__SHIFT 8 +#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_PROC_PRECISION__MASK 0x000000e0 +#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_PROC_PRECISION__SHIFT 5 +#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_MRDMA_DISABLE__MASK 0x00000010 +#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_MRDMA_DISABLE__SHIFT 4 +#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_MRDMA_FP16TOFP32_EN__MASK 0x00000008 +#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_MRDMA_FP16TOFP32_EN__SHIFT 3 +#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_CONV_MODE__MASK 0x00000006 +#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_CONV_MODE__SHIFT 1 +#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_FLYING_MODE__MASK 0x00000001 +#define DPU_RDMA_RDMA_FEATURE_MODE_CFG_FLYING_MODE__SHIFT 0 +#define REG_DPU_RDMA_RDMA_SRC_DMA_CFG 0x00005048 +#define DPU_RDMA_RDMA_SRC_DMA_CFG_LINE_NOTCH_ADDR__MASK 0xfff80000 +#define DPU_RDMA_RDMA_SRC_DMA_CFG_LINE_NOTCH_ADDR__SHIFT 19 +#define DPU_RDMA_RDMA_SRC_DMA_CFG_RESERVED_0__MASK 0x0007c000 +#define DPU_RDMA_RDMA_SRC_DMA_CFG_RESERVED_0__SHIFT 14 +#define DPU_RDMA_RDMA_SRC_DMA_CFG_POOLING_METHOD__MASK 0x00002000 +#define DPU_RDMA_RDMA_SRC_DMA_CFG_POOLING_METHOD__SHIFT 13 +#define DPU_RDMA_RDMA_SRC_DMA_CFG_UNPOOLING_EN__MASK 0x00001000 +#define DPU_RDMA_RDMA_SRC_DMA_CFG_UNPOOLING_EN__SHIFT 12 +#define DPU_RDMA_RDMA_SRC_DMA_CFG_KERNEL_STRIDE_HEIGHT__MASK 0x00000e00 +#define DPU_RDMA_RDMA_SRC_DMA_CFG_KERNEL_STRIDE_HEIGHT__SHIFT 9 +#define DPU_RDMA_RDMA_SRC_DMA_CFG_KERNEL_STRIDE_WIDTH__MASK 0x000001c0 +#define DPU_RDMA_RDMA_SRC_DMA_CFG_KERNEL_STRIDE_WIDTH__SHIFT 6 +#define DPU_RDMA_RDMA_SRC_DMA_CFG_KERNEL_HEIGHT__MASK 0x00000038 +#define DPU_RDMA_RDMA_SRC_DMA_CFG_KERNEL_HEIGHT__SHIFT 3 +#define DPU_RDMA_RDMA_SRC_DMA_CFG_KERNEL_WIDTH__MASK 0x00000007 +#define DPU_RDMA_RDMA_SRC_DMA_CFG_KERNEL_WIDTH__SHIFT 0 +#define REG_DPU_RDMA_RDMA_SURF_NOTCH 0x0000504c +#define DPU_RDMA_RDMA_SURF_NOTCH_SURF_NOTCH_ADDR__MASK 0xfffffff0 +#define DPU_RDMA_RDMA_SURF_NOTCH_SURF_NOTCH_ADDR__SHIFT 4 +#define DPU_RDMA_RDMA_SURF_NOTCH_RESERVED_0__MASK 0x0000000f +#define DPU_RDMA_RDMA_SURF_NOTCH_RESERVED_0__SHIFT 0 +#define REG_DPU_RDMA_RDMA_PAD_CFG 0x00005064 +#define DPU_RDMA_RDMA_PAD_CFG_PAD_VALUE__MASK 0xffff0000 +#define DPU_RDMA_RDMA_PAD_CFG_PAD_VALUE__SHIFT 16 +#define DPU_RDMA_RDMA_PAD_CFG_RESERVED_0__MASK 0x0000ff80 +#define DPU_RDMA_RDMA_PAD_CFG_RESERVED_0__SHIFT 7 +#define DPU_RDMA_RDMA_PAD_CFG_PAD_TOP__MASK 0x00000070 +#define DPU_RDMA_RDMA_PAD_CFG_PAD_TOP__SHIFT 4 +#define DPU_RDMA_RDMA_PAD_CFG_RESERVED_1__MASK 0x00000008 +#define DPU_RDMA_RDMA_PAD_CFG_RESERVED_1__SHIFT 3 +#define DPU_RDMA_RDMA_PAD_CFG_PAD_LEFT__MASK 0x00000007 +#define DPU_RDMA_RDMA_PAD_CFG_PAD_LEFT__SHIFT 0 +#define REG_DPU_RDMA_RDMA_WEIGHT 0x00005068 +#define DPU_RDMA_RDMA_WEIGHT_E_WEIGHT__MASK 0xff000000 +#define DPU_RDMA_RDMA_WEIGHT_E_WEIGHT__SHIFT 24 +#define DPU_RDMA_RDMA_WEIGHT_N_WEIGHT__MASK 0x00ff0000 +#define DPU_RDMA_RDMA_WEIGHT_N_WEIGHT__SHIFT 16 +#define DPU_RDMA_RDMA_WEIGHT_B_WEIGHT__MASK 0x0000ff00 +#define DPU_RDMA_RDMA_WEIGHT_B_WEIGHT__SHIFT 8 +#define DPU_RDMA_RDMA_WEIGHT_M_WEIGHT__MASK 0x000000ff +#define DPU_RDMA_RDMA_WEIGHT_M_WEIGHT__SHIFT 0 +#define REG_DPU_RDMA_RDMA_EW_SURF_NOTCH 0x0000506c +#define DPU_RDMA_RDMA_EW_SURF_NOTCH_EW_SURF_NOTCH__MASK 0xfffffff0 +#define DPU_RDMA_RDMA_EW_SURF_NOTCH_EW_SURF_NOTCH__SHIFT 4 +#define DPU_RDMA_RDMA_EW_SURF_NOTCH_RESERVED_0__MASK 0x0000000f +#define DPU_RDMA_RDMA_EW_SURF_NOTCH_RESERVED_0__SHIFT 0 + +#endif diff --git a/ggml/src/ggml-rocket/ggml-rocket.cpp b/ggml/src/ggml-rocket/ggml-rocket.cpp new file mode 100644 index 000000000..a335c3968 --- /dev/null +++ b/ggml/src/ggml-rocket/ggml-rocket.cpp @@ -0,0 +1,425 @@ +// SPDX-License-Identifier: MIT +// +// ggml-rocket: MUL_MAT backend for the RK3588 NPU via the mainline "rocket" +// DRM-accel driver (/dev/accel/accel0), no vendor RKNPU2 userspace. +// +// Modeled on ggml-blas: an ACCEL device with no weight buffers of its own +// (weights stay in CPU memory); the scheduler offloads MUL_MAT nodes it +// supports and leaves everything else on the CPU backend. +// +// Per MUL_MAT plane: dequantize the weight to F32 (ggml to_float trait), +// requantize per-output-channel to int8, quantize the F32 activation +// per-tensor to int8, run rkt_npu_matmul (int8 GEMM on the NPU with a +// per-channel out_scale array), then dequantize the int8 result to F32. +// +// out_scale is chosen from a Cauchy-Schwarz upper bound on |output| so the +// int8 output never clips: || <= ||w_n||2 * ||x_m||2. This is the +// pragmatic "int8-first" path; a lossless int32-accumulator readout is the +// planned follow-on (needs an OUT_CVT-bypass builder mode). + +#include "ggml-impl.h" +#include "ggml-rocket.h" +#include "ggml-backend-impl.h" +#include "ggml-cpu.h" + +#include +#include +#include +#include +#include + +extern "C" { +#include "librocket.h" +#include "rkt_npu_matmul.h" +} + +// Empirically verified single-op limits on boltzmann's rocket NPU (int8): +// rows per tile up to 32 correct at K=2560 -> use 16 for margin +// N per tile up to 256 correct -> use 128 +// full K correct through 8192, catastrophic at 11008 (vendor int8 K-limit) +#define ROCKET_TILE_M 16 +#define ROCKET_TILE_N 128 +#define ROCKET_K_MAX 8192 + +struct ggml_backend_rocket_context { + int fd = -1; + // reusable scratch (grown as needed, never shrunk) + std::vector wf; // dequantized weight plane [N*K] + std::vector aq; // quantized activations [M*K] + std::vector wq; // quantized weights [N*K] + std::vector yq; // int8 result [M*N] + std::vector ws; // per-channel weight scale [N] + std::vector os; // per-channel out scale [N] +}; + +static inline uint8_t rocket_q8(float v, float inv_s) { + int q = (int)lrintf(v * inv_s); + if (q > 127) q = 127; + if (q < -127) q = -127; + return (uint8_t)(q + 128); // symmetric int8 stored as uint8, zp=128 +} + +static void ggml_backend_rocket_mul_mat(ggml_backend_rocket_context * ctx, struct ggml_tensor * dst) { + const struct ggml_tensor * src0 = dst->src[0]; // weight [K, N] + const struct ggml_tensor * src1 = dst->src[1]; // activ [K, M] + + GGML_TENSOR_BINARY_OP_LOCALS + + const enum ggml_type type = src0->type; + const auto * tt = ggml_get_type_traits(type); + ggml_to_float_t const to_float = tt->to_float; + + const int64_t K = ne00; // == ne10 + const int64_t N = ne01; // output channels + const int64_t M = ne11; // rows (tokens in batch) + + static const bool dbg = getenv("GGML_ROCKET_DEBUG") != NULL; + if (dbg) fprintf(stderr, "[rocket] enter mul_mat M=%lld N=%lld K=%lld type=%s to_float=%p\n", + (long long)M, (long long)N, (long long)K, + ggml_type_name(type), (void*)to_float), fflush(stderr); + if (to_float == NULL || M == 0 || N == 0 || K == 0) { + GGML_ABORT("rocket mul_mat: unexpected op (to_float=%p M=%lld N=%lld K=%lld)", + (void*)to_float, (long long)M, (long long)N, (long long)K); + } + + // broadcast of src0 over src1's higher dims (as in ggml-blas) + const int64_t r2 = ne12 / ne02; + const int64_t r3 = ne13 / ne03; + + ctx->wf.resize((size_t)N * K); + ctx->aq.resize((size_t)M * K); + ctx->wq.resize((size_t)N * K); + ctx->yq.resize((size_t)M * N); + ctx->ws.resize((size_t)N); + ctx->os.resize((size_t)N); + + for (int64_t i13 = 0; i13 < ne13; i13++) { + for (int64_t i12 = 0; i12 < ne12; i12++) { + const int64_t i03 = i13 / r3; + const int64_t i02 = i12 / r2; + + const char * w_plane = (const char *) src0->data + i02*nb02 + i03*nb03; + const char * a_plane = (const char *) src1->data + i12*nb12 + i13*nb13; + char * d_plane = ( char *) dst->data + i12*nb2 + i13*nb3; + + // --- dequantize weight plane to F32, per-channel scale + L2 norm --- + float wn2max = 0.0f; // max_n ||w_n||_2 + for (int64_t n = 0; n < N; n++) { + float * wrow = ctx->wf.data() + (size_t)n * K; + to_float((const char *) w_plane + n*nb01, wrow, K); + float amax = 0.0f, l2 = 0.0f; + for (int64_t k = 0; k < K; k++) { + float v = wrow[k]; + float a = std::fabs(v); + if (a > amax) amax = a; + l2 += v * v; + } + ctx->ws[n] = (amax > 0.0f ? amax : 1e-6f) / 127.0f; + l2 = std::sqrt(l2); + if (l2 > wn2max) wn2max = l2; + } + + // --- quantize activation plane, per-tensor scale + max L2 row norm --- + float amax_a = 0.0f, ym2max = 0.0f; + for (int64_t m = 0; m < M; m++) { + const float * arow = (const float *) (a_plane + m*nb11); + float l2 = 0.0f; + for (int64_t k = 0; k < K; k++) { + float v = arow[k]; + float a = std::fabs(v); + if (a > amax_a) amax_a = a; + l2 += v * v; + } + l2 = std::sqrt(l2); + if (l2 > ym2max) ym2max = l2; + } + const float a_scale = (amax_a > 0.0f ? amax_a : 1e-6f) / 127.0f; + const float inv_a = 1.0f / a_scale; + for (int64_t m = 0; m < M; m++) { + const float * arow = (const float *) (a_plane + m*nb11); + uint8_t * qrow = ctx->aq.data() + (size_t)m * K; + for (int64_t k = 0; k < K; k++) qrow[k] = rocket_q8(arow[k], inv_a); + } + + // Cauchy-Schwarz bound on |output| -> never clips int8 output. + float out_scale = (wn2max * ym2max) / 127.0f; + if (!(out_scale > 0.0f)) out_scale = 1e-6f; + + for (int64_t n = 0; n < N; n++) { + const float inv_w = 1.0f / ctx->ws[n]; + const float * wrow = ctx->wf.data() + (size_t)n * K; + uint8_t * qrow = ctx->wq.data() + (size_t)n * K; + for (int64_t k = 0; k < K; k++) qrow[k] = rocket_q8(wrow[k], inv_w); + ctx->os[n] = out_scale / ctx->ws[n]; + } + + static const bool dbg = getenv("GGML_ROCKET_DEBUG") != NULL; + if (dbg) fprintf(stderr, "[rocket] mul_mat M=%lld N=%lld K=%lld plane(%lld,%lld) type=%s ...", + (long long)M, (long long)N, (long long)K, + (long long)i12, (long long)i13, ggml_type_name(type)), fflush(stderr); + int rc = rkt_npu_matmul(ctx->fd, ctx->aq.data(), ctx->wq.data(), NULL, + (uint32_t)M, (uint32_t)N, (uint32_t)K, + 128, 128, 128, + a_scale, 1.0f, out_scale, ctx->os.data(), + ROCKET_TILE_M, ROCKET_TILE_N, ctx->yq.data()); + if (dbg) fprintf(stderr, " rc=%d\n", rc); + GGML_ASSERT(rc == 0 && "rkt_npu_matmul failed"); + + // --- dequantize int8 result into dst (F32) --- + for (int64_t m = 0; m < M; m++) { + float * drow = (float *) (d_plane + m*nb1); + const uint8_t * yrow = ctx->yq.data() + (size_t)m * N; + for (int64_t n = 0; n < N; n++) + drow[n] = ((int)yrow[n] - 128) * out_scale; + } + } + } +} + +// backend interface + +static const char * ggml_backend_rocket_get_name(ggml_backend_t backend) { + return "Rocket"; + GGML_UNUSED(backend); +} + +static void ggml_backend_rocket_free(ggml_backend_t backend) { + ggml_backend_rocket_context * ctx = (ggml_backend_rocket_context *)backend->context; + delete ctx; + delete backend; +} + +static enum ggml_status ggml_backend_rocket_graph_compute(ggml_backend_t backend, struct ggml_cgraph * cgraph) { + ggml_backend_rocket_context * ctx = (ggml_backend_rocket_context *)backend->context; + + for (int i = 0; i < cgraph->n_nodes; i++) { + struct ggml_tensor * node = cgraph->nodes[i]; + + if ((node->flags & GGML_TENSOR_FLAG_COMPUTE) == 0) { + continue; + } + + switch (node->op) { + case GGML_OP_MUL_MAT: + ggml_backend_rocket_mul_mat(ctx, node); + break; + + case GGML_OP_NONE: + case GGML_OP_RESHAPE: + case GGML_OP_VIEW: + case GGML_OP_PERMUTE: + case GGML_OP_TRANSPOSE: + break; + + default: + GGML_ABORT("%s: unsupported op %s\n", __func__, ggml_op_desc(node)); + } + } + + return GGML_STATUS_SUCCESS; + GGML_UNUSED(backend); +} + +static struct ggml_backend_i rocket_backend_i = { + /* .get_name = */ ggml_backend_rocket_get_name, + /* .free = */ ggml_backend_rocket_free, + /* .set_tensor_async = */ NULL, + /* .get_tensor_async = */ NULL, + /* .set_tensor_2d_async = */ NULL, + /* .get_tensor_2d_async = */ NULL, + /* .cpy_tensor_async = */ NULL, + /* .synchronize = */ NULL, + /* .graph_plan_create = */ NULL, + /* .graph_plan_free = */ NULL, + /* .graph_plan_update = */ NULL, + /* .graph_plan_compute = */ NULL, + /* .graph_compute = */ ggml_backend_rocket_graph_compute, + /* .event_record = */ NULL, + /* .event_wait = */ NULL, + /* .graph_optimize = */ NULL, +}; + +static ggml_guid_t ggml_backend_rocket_guid(void) { + static ggml_guid guid = { 0x52, 0x6f, 0x63, 0x6b, 0x65, 0x74, 0x4e, 0x50, 0x55, 0x33, 0x35, 0x38, 0x38, 0x01, 0x00, 0x01 }; + return &guid; +} + +ggml_backend_t ggml_backend_rocket_init(void) { + int fd = rocket_open("/dev/accel/accel0"); + if (fd < 0) { + GGML_LOG_ERROR("%s: failed to open /dev/accel/accel0 (%d)\n", __func__, fd); + return NULL; + } + + ggml_backend_rocket_context * ctx = new ggml_backend_rocket_context; + ctx->fd = fd; + + ggml_backend_t backend = new ggml_backend { + /* .guid = */ ggml_backend_rocket_guid(), + /* .iface = */ rocket_backend_i, + /* .device = */ ggml_backend_reg_dev_get(ggml_backend_rocket_reg(), 0), + /* .context = */ ctx, + }; + + return backend; +} + +bool ggml_backend_is_rocket(ggml_backend_t backend) { + return backend != NULL && ggml_guid_matches(backend->guid, ggml_backend_rocket_guid()); +} + +// device interface + +static const char * ggml_backend_rocket_device_get_name(ggml_backend_dev_t dev) { + return "Rocket"; + GGML_UNUSED(dev); +} + +static const char * ggml_backend_rocket_device_get_description(ggml_backend_dev_t dev) { + return "RK3588 NPU (mainline rocket accel)"; + GGML_UNUSED(dev); +} + +static void ggml_backend_rocket_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) { + *free = 0; + *total = 0; + GGML_UNUSED(dev); +} + +static enum ggml_backend_dev_type ggml_backend_rocket_device_get_type(ggml_backend_dev_t dev) { + return GGML_BACKEND_DEVICE_TYPE_ACCEL; + GGML_UNUSED(dev); +} + +static void ggml_backend_rocket_device_get_props(ggml_backend_dev_t dev, struct ggml_backend_dev_props * props) { + props->name = ggml_backend_rocket_device_get_name(dev); + props->description = ggml_backend_rocket_device_get_description(dev); + props->type = ggml_backend_rocket_device_get_type(dev); + ggml_backend_rocket_device_get_memory(dev, &props->memory_free, &props->memory_total); + props->caps = { + /* .async = */ false, + /* .host_buffer = */ false, + /* .buffer_from_host_ptr = */ true, + /* .events = */ false, + }; +} + +static ggml_backend_t ggml_backend_rocket_device_init_backend(ggml_backend_dev_t dev, const char * params) { + return ggml_backend_rocket_init(); + GGML_UNUSED(dev); + GGML_UNUSED(params); +} + +static ggml_backend_buffer_type_t ggml_backend_rocket_device_get_buffer_type(ggml_backend_dev_t dev) { + return ggml_backend_cpu_buffer_type(); + GGML_UNUSED(dev); +} + +static ggml_backend_buffer_t ggml_backend_rocket_device_buffer_from_host_ptr(ggml_backend_dev_t dev, void * ptr, size_t size, size_t max_tensor_size) { + return ggml_backend_cpu_buffer_from_ptr(ptr, size); + GGML_UNUSED(dev); + GGML_UNUSED(max_tensor_size); +} + +static bool ggml_backend_rocket_device_supports_op(ggml_backend_dev_t dev, const struct ggml_tensor * op) { + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * src1 = op->src[1]; + + switch (op->op) { + case GGML_OP_NONE: + case GGML_OP_RESHAPE: + case GGML_OP_VIEW: + case GGML_OP_PERMUTE: + case GGML_OP_TRANSPOSE: + return true; + + case GGML_OP_MUL_MAT: + { + const int64_t K = src1->ne[0]; // == src0->ne[0] + + if (src1->type != GGML_TYPE_F32) return false; + if (!ggml_is_contiguous(src0)) return false; + if (!ggml_is_contiguous(src1)) return false; + if (K <= 0 || K > ROCKET_K_MAX) return false; + if (K % 16 != 0) return false; // feature-atomic align + // weight must be dequantizable to F32 (F32 src0 has no to_float + // trait -> leave those on the CPU backend) + if (ggml_get_type_traits(src0->type)->to_float == NULL) return false; + return true; + } + + default: + return false; + } + + GGML_UNUSED(dev); +} + +static bool ggml_backend_rocket_device_supports_buft(ggml_backend_dev_t dev, ggml_backend_buffer_type_t buft) { + return ggml_backend_buft_is_host(buft); + GGML_UNUSED(dev); +} + +static const struct ggml_backend_device_i ggml_backend_rocket_device_i = { + /* .get_name = */ ggml_backend_rocket_device_get_name, + /* .get_description = */ ggml_backend_rocket_device_get_description, + /* .get_memory = */ ggml_backend_rocket_device_get_memory, + /* .get_type = */ ggml_backend_rocket_device_get_type, + /* .get_props = */ ggml_backend_rocket_device_get_props, + /* .init_backend = */ ggml_backend_rocket_device_init_backend, + /* .get_buffer_type = */ ggml_backend_rocket_device_get_buffer_type, + /* .get_host_buffer_type = */ NULL, + /* .buffer_from_host_ptr = */ ggml_backend_rocket_device_buffer_from_host_ptr, + /* .supports_op = */ ggml_backend_rocket_device_supports_op, + /* .supports_buft = */ ggml_backend_rocket_device_supports_buft, + /* .offload_op = */ NULL, + /* .event_new = */ NULL, + /* .event_free = */ NULL, + /* .event_synchronize = */ NULL, +}; + +// backend reg interface + +static const char * ggml_backend_rocket_reg_get_name(ggml_backend_reg_t reg) { + return "Rocket"; + GGML_UNUSED(reg); +} + +static size_t ggml_backend_rocket_reg_get_device_count(ggml_backend_reg_t reg) { + return 1; + GGML_UNUSED(reg); +} + +static ggml_backend_dev_t ggml_backend_rocket_reg_get_device(ggml_backend_reg_t reg, size_t index) { + GGML_ASSERT(index == 0); + + static ggml_backend_device ggml_backend_rocket_device = { + /* .iface = */ ggml_backend_rocket_device_i, + /* .reg = */ reg, + /* .context = */ nullptr, + }; + + return &ggml_backend_rocket_device; + + GGML_UNUSED(reg); + GGML_UNUSED(index); +} + +static const struct ggml_backend_reg_i ggml_backend_rocket_reg_i = { + /* .get_name = */ ggml_backend_rocket_reg_get_name, + /* .get_device_count = */ ggml_backend_rocket_reg_get_device_count, + /* .get_device = */ ggml_backend_rocket_reg_get_device, + /* .get_proc_address = */ NULL, +}; + +ggml_backend_reg_t ggml_backend_rocket_reg(void) { + static struct ggml_backend_reg ggml_backend_rocket_reg = { + /* .api_version = */ GGML_BACKEND_API_VERSION, + /* .iface = */ ggml_backend_rocket_reg_i, + /* .context = */ NULL, + }; + + return &ggml_backend_rocket_reg; +} + +GGML_BACKEND_DL_IMPL(ggml_backend_rocket_reg) diff --git a/ggml/src/ggml-rocket/ggml-rocket.h b/ggml/src/ggml-rocket/ggml-rocket.h new file mode 100644 index 000000000..9a795eb45 --- /dev/null +++ b/ggml/src/ggml-rocket/ggml-rocket.h @@ -0,0 +1,13 @@ +#pragma once + +#include "ggml-backend.h" + +#ifdef __cplusplus +extern "C" { +#endif + +GGML_API ggml_backend_reg_t ggml_backend_rocket_reg(void); + +#ifdef __cplusplus +} +#endif diff --git a/ggml/src/ggml-rocket/librocket.c b/ggml/src/ggml-rocket/librocket.c new file mode 100644 index 000000000..1eb2707fd --- /dev/null +++ b/ggml/src/ggml-rocket/librocket.c @@ -0,0 +1,106 @@ +/* SPDX-License-Identifier: MIT */ +/* + * librocket — thin C wrapper over the Rockchip "rocket" NPU DRM-accel uAPI. + * + * Implementation — pure syscall-surface plumbing. + */ +#include "librocket.h" + +#include +#include +#include +#include +#include +#include + +int rocket_open(const char *path) +{ + int fd = open(path, O_RDWR | O_CLOEXEC); + if (fd < 0) + return -errno; + return fd; +} + +int rocket_create_bo(int fd, uint32_t size, uint32_t *handle_out, + uint64_t *dma_address_out, uint64_t *mmap_offset_out) +{ + struct drm_rocket_create_bo create_bo; + int ret; + + memset(&create_bo, 0, sizeof(create_bo)); + create_bo.size = size; + + ret = ioctl(fd, DRM_IOCTL_ROCKET_CREATE_BO, &create_bo); + if (ret < 0) + return -errno; + + if (handle_out) + *handle_out = create_bo.handle; + if (dma_address_out) + *dma_address_out = create_bo.dma_address; + if (mmap_offset_out) + *mmap_offset_out = create_bo.offset; + + return 0; +} + +void *rocket_mmap_bo(int fd, uint64_t mmap_offset, uint32_t size) +{ + return mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, fd, + mmap_offset); +} + +int rocket_prep_bo(int fd, uint32_t handle, int64_t timeout_ns) +{ + struct drm_rocket_prep_bo prep_bo; + int ret; + + memset(&prep_bo, 0, sizeof(prep_bo)); + prep_bo.handle = handle; + prep_bo.timeout_ns = timeout_ns; + + ret = ioctl(fd, DRM_IOCTL_ROCKET_PREP_BO, &prep_bo); + if (ret < 0) + return -errno; + return 0; +} + +int rocket_fini_bo(int fd, uint32_t handle) +{ + struct drm_rocket_fini_bo fini_bo; + int ret; + + memset(&fini_bo, 0, sizeof(fini_bo)); + fini_bo.handle = handle; + + ret = ioctl(fd, DRM_IOCTL_ROCKET_FINI_BO, &fini_bo); + if (ret < 0) + return -errno; + return 0; +} + +int rocket_submit(int fd, struct drm_rocket_job *jobs, uint32_t job_count) +{ + struct drm_rocket_submit submit; + int ret; + + memset(&submit, 0, sizeof(submit)); + submit.jobs = (uintptr_t)jobs; + submit.job_count = job_count; + submit.job_struct_size = sizeof(struct drm_rocket_job); + + ret = ioctl(fd, DRM_IOCTL_ROCKET_SUBMIT, &submit); + if (ret < 0) + return -errno; + return 0; +} + +int rocket_close_bo(int fd, uint32_t handle) +{ + struct drm_gem_close c; + memset(&c, 0, sizeof c); + c.handle = handle; + if (ioctl(fd, DRM_IOCTL_GEM_CLOSE, &c) < 0) + return -errno; + return 0; +} diff --git a/ggml/src/ggml-rocket/librocket.h b/ggml/src/ggml-rocket/librocket.h new file mode 100644 index 000000000..7427d90f6 --- /dev/null +++ b/ggml/src/ggml-rocket/librocket.h @@ -0,0 +1,87 @@ +/* SPDX-License-Identifier: MIT */ +/* + * librocket — thin C wrapper over the Rockchip "rocket" NPU DRM-accel uAPI. + * + * This is pure syscall-surface plumbing. No NPU/tensor semantics, no register + * commands. Every function returns 0 on success or -errno on failure (except + * rocket_mmap_bo which returns the mmap'd pointer or MAP_FAILED). + */ +#ifndef LIBROCKET_H +#define LIBROCKET_H + +#include "rocket_accel.h" +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** + * rocket_open - open a rocket NPU accel device + * @path: device node path (e.g. "/dev/accel/accel0") + * + * Returns the file descriptor on success, or -errno on failure. + */ +int rocket_open(const char *path); + +/** + * rocket_create_bo - allocate a buffer object + * @fd: rocket device fd + * @size: desired BO size in bytes + * @handle_out: on success, written with the GEM handle + * @dma_address_out: on success, written with the NPU DMA address + * @mmap_offset_out: on success, written with the mmap offset + * + * Returns 0 on success, -errno on failure. + */ +int rocket_create_bo(int fd, uint32_t size, uint32_t *handle_out, + uint64_t *dma_address_out, uint64_t *mmap_offset_out); + +/** + * rocket_mmap_bo - mmap a BO into userspace + * @fd: rocket device fd + * @mmap_offset: offset returned by rocket_create_bo + * @size: number of bytes to map + * + * Returns the mapped pointer, or MAP_FAILED on failure. + */ +void *rocket_mmap_bo(int fd, uint64_t mmap_offset, uint32_t size); + +/** + * rocket_prep_bo - take CPU ownership of a BO (wait for NPU, sync caches) + * @fd: rocket device fd + * @handle: GEM handle of the BO + * @timeout_ns: ABSOLUTE CLOCK_MONOTONIC deadline (ns); INT64_MAX = block until signalled + * + * Returns 0 on success, -errno on failure. + */ +int rocket_prep_bo(int fd, uint32_t handle, int64_t timeout_ns); + +/** + * rocket_fini_bo - hand a BO back to the device (sync caches) + * @fd: rocket device fd + * @handle: GEM handle of the BO + * + * Returns 0 on success, -errno on failure. + */ +int rocket_fini_bo(int fd, uint32_t handle); + +/** + * rocket_submit - submit NPU jobs + * @fd: rocket device fd + * @jobs: pointer to an array of struct drm_rocket_job + * @job_count: number of jobs in the array + * + * Returns 0 on success, -errno on failure. + */ +int rocket_submit(int fd, struct drm_rocket_job *jobs, uint32_t job_count); + +/* Release a BO handle (DRM_IOCTL_GEM_CLOSE). Returns 0 or -errno. */ +int rocket_close_bo(int fd, uint32_t handle); + +#ifdef __cplusplus +} +#endif + +#endif /* LIBROCKET_H */ diff --git a/ggml/src/ggml-rocket/rkt_gemm.c b/ggml/src/ggml-rocket/rkt_gemm.c new file mode 100644 index 000000000..6da9db4f0 --- /dev/null +++ b/ggml/src/ggml-rocket/rkt_gemm.c @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: MIT + * + * rkt_gemm.c — U6 GEMM tiler (see rkt_gemm.h). + */ +#include "rkt_gemm.h" +#include "rkt_matmul.h" + +int rkt_gemm_plan(uint32_t M, uint32_t N, uint32_t tile_m, uint32_t tile_n, + struct rkt_gemm_tile *tiles, int max_tiles) +{ + if (M == 0 || N == 0 || tile_m == 0 || tile_n == 0) + return -1; + + int t = 0; + for (uint32_t r = 0; r < M; r += tile_m) { + for (uint32_t c = 0; c < N; c += tile_n) { + if (t >= max_tiles) + return -1; + tiles[t].row = r; + tiles[t].col = c; + tiles[t].m = (r + tile_m <= M) ? tile_m : (M - r); + tiles[t].n = (c + tile_n <= N) ? tile_n : (N - c); + t++; + } + } + return t; +} + +int rkt_gemm_op_fits(uint32_t m, uint32_t n, uint32_t k) +{ + uint64_t scratch[512]; + /* Addresses/zero-points are placeholders — fit is shape-only. */ + return rkt_build_matmul_regcmd(scratch, 512, m, n, k, + 0x1000, 0x2000, 0x3000, + 128, 128, 128) >= 0; +} diff --git a/ggml/src/ggml-rocket/rkt_gemm.h b/ggml/src/ggml-rocket/rkt_gemm.h new file mode 100644 index 000000000..dd4bccc90 --- /dev/null +++ b/ggml/src/ggml-rocket/rkt_gemm.h @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: MIT + * + * rkt_gemm.h — U6 backend: tile a large INT8 GEMM into rocket-op-sized + * sub-matmuls, each buildable by rkt_build_matmul_regcmd. + * + * A single rocket op has a fixed CBUF budget (12 banks), so a gemma-scale + * GEMM must be blocked. This layer tiles the M (rows/tokens) and N (output + * channels) dimensions. K (contraction / input channels) is NOT tiled here: + * the full K must fit one op — K-splitting needs partial-sum accumulation via + * the DPU add path and is deferred (see rkt_matmul_coredpu add_tensor path). + */ +#ifndef RKT_GEMM_H +#define RKT_GEMM_H + +#include + +struct rkt_gemm_tile { + uint32_t row; /* first row in M this tile covers */ + uint32_t col; /* first col in N this tile covers */ + uint32_t m; /* number of rows */ + uint32_t n; /* number of cols */ +}; + +/* + * Partition an M x N output (full K) into tiles of at most tile_m x tile_n, + * row-major. Writes tiles[] and returns the tile count, or -1 on bad args or + * if more than max_tiles would be produced. Pure geometry — call + * rkt_gemm_op_fits() to confirm a tile actually fits one rocket op. + */ +int rkt_gemm_plan(uint32_t M, uint32_t N, uint32_t tile_m, uint32_t tile_n, + struct rkt_gemm_tile *tiles, int max_tiles); + +/* True (1) if an m x n, full-k matmul fits a single rocket op. */ +int rkt_gemm_op_fits(uint32_t m, uint32_t n, uint32_t k); + +#endif /* RKT_GEMM_H */ diff --git a/ggml/src/ggml-rocket/rkt_matmul.c b/ggml/src/ggml-rocket/rkt_matmul.c new file mode 100644 index 000000000..d570fe5a1 --- /dev/null +++ b/ggml/src/ggml-rocket/rkt_matmul.c @@ -0,0 +1,255 @@ +/* SPDX-License-Identifier: MIT + * + * rkt_matmul.c — unified INT8 matmul regcmd builder. + * + * Implements rkt_build_matmul_regcmd(): given a plain GEMM shape it derives + * every low-level CNA / CORE / DPU field via the NVDLA-style CBUF tiling math + * (ported from Mesa rocket rkt_task.c: fill_task + the single-tile branch of + * rkt_split_tasks) and drives the two emit stages. + * + * The matmul Y[M][N] = X[M][K] * W[K][N] is expressed as a 1x1 convolution: + * input_width = 1, input_height = M, input_channels = K, + * output_width = 1, output_height = M, output_channels = N, + * weights 1x1, stride 1, no padding, no bias, identity requantisation. + * + * SCOPE: single-tile only. If the operation would need CBUF task-splitting + * (input feature map too tall to fit the available input banks) this returns + * -1 rather than emit a wrong single task. The multi-task splitter and real + * scale/bias handling are separate follow-on work. + */ +#include "rkt_matmul.h" +#include "rkt_matmul_cna.h" +#include "rkt_matmul_coredpu.h" + +#include + +/* CBUF geometry — verbatim from Mesa rkt_ml.h (NVDLA v1 convolution buffer). */ +#define CBUF_BANK_SIZE 32768 +#define CBUF_BANKS 12 +#define CBUF_ENTRIES_PER_BANK 256 +#define CBUF_ENTRY_SIZE (CBUF_BANK_SIZE / CBUF_ENTRIES_PER_BANK) /* 128 */ +#define FEATURE_ATOMIC_SIZE 16 +#define ATOMIC_K_SIZE 16 + +#define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d)) +#define MAX2(a, b) ((a) > (b) ? (a) : (b)) + +static unsigned alignu(unsigned v, unsigned a) { return ((v + a - 1) / a) * a; } + +/* Fields Mesa's tiling math reads off rkt_operation, specialised to a matmul. */ +struct op { + unsigned stride; + unsigned input_width, input_height, input_channels; + unsigned output_width, output_height, output_channels; + unsigned weights_width, weights_height; + int depthwise, addition_input, add_tensor; +}; + +static unsigned calc_line_stride(unsigned width) +{ + return width * ATOMIC_K_SIZE * sizeof(uint8_t); +} + +static unsigned calc_entries_per_slice(const struct op *o) +{ + unsigned atomics_per_entry = CBUF_ENTRY_SIZE / FEATURE_ATOMIC_SIZE; + unsigned total_c_atomics = + DIV_ROUND_UP(o->input_channels * sizeof(uint8_t), FEATURE_ATOMIC_SIZE); + unsigned last_c_atomics = total_c_atomics % atomics_per_entry; + unsigned int_c_entries = + (total_c_atomics / atomics_per_entry) * o->input_width; + unsigned frac_c_entries = + (last_c_atomics == 3) + ? o->input_width + : DIV_ROUND_UP(last_c_atomics * o->input_width, atomics_per_entry); + return int_c_entries + frac_c_entries; +} + +static unsigned calc_input_banks(const struct op *o) +{ + return DIV_ROUND_UP(calc_entries_per_slice(o) * o->input_height, + CBUF_ENTRIES_PER_BANK); +} + +static unsigned calc_weights_banks(const struct op *o) +{ + unsigned bytes = o->weights_width * o->weights_height * + o->input_channels * sizeof(uint8_t); + if (!o->depthwise) + bytes *= o->output_channels; + unsigned entries = DIV_ROUND_UP(bytes, CBUF_ENTRY_SIZE); + unsigned banks = DIV_ROUND_UP(entries, CBUF_ENTRIES_PER_BANK); + banks++; /* Mesa: "the calc above might be wrong on this HW" */ + return banks; +} + +int rkt_build_matmul_regcmd_scaled(uint64_t *out, int out_capacity, + uint32_t M, uint32_t N, uint32_t K, + uint64_t input_dma, uint64_t weights_dma, + uint64_t output_dma, + int32_t input_zero_point, + int32_t weight_zero_point, + int32_t output_zero_point, + float input_scale, float weights_scale, + float output_scale, uint64_t bias_dma) +{ + if (M == 0 || N == 0 || K == 0) + return -1; + + struct op o = { + .stride = 1, + .input_width = 1, .input_height = M, .input_channels = K, + .output_width = 1, .output_height = M, .output_channels = N, + .weights_width = 1, .weights_height = 1, + .depthwise = 0, .addition_input = 0, .add_tensor = -1, + }; + + /* ---- fill_task (Mesa rkt_task.c), stride 1 / no pad / no add ---- */ + unsigned t_input_width = o.input_width; /* width != 8, not doubled */ + unsigned t_input_height = o.input_height; + unsigned t_input_channels = + alignu(MAX2(o.input_channels, FEATURE_ATOMIC_SIZE), FEATURE_ATOMIC_SIZE); + unsigned t_input_channels_real = o.input_channels; + + unsigned t_output_width = o.output_width; + unsigned t_output_height = o.output_height; + unsigned t_output_channels_real = o.output_channels; + unsigned t_output_channels = alignu(MAX2(o.output_channels, 32), 32); + /* depthwise output_channels fix-ups skipped (depthwise == 0) */ + + int t_input_line_stride, t_input_surface_stride; + if (t_input_channels_real == 1 && + (t_output_channels_real > 1 || o.addition_input || o.add_tensor != -1)) { + t_input_width = MAX2(t_input_width, FEATURE_ATOMIC_SIZE); + t_input_line_stride = + MAX2((int)(calc_line_stride(o.input_width) / FEATURE_ATOMIC_SIZE), + (int)FEATURE_ATOMIC_SIZE); + t_input_surface_stride = + (int)((float)t_input_line_stride * ((float)t_input_height - 1.0f)); + } else { + t_input_line_stride = (int)(calc_line_stride(o.input_width) / 4); + t_input_surface_stride = (int)((float)t_input_line_stride * + (((float)t_input_height / 4.0f) - 1.0f)); + } + /* input_width == 8 && addition branch skipped */ + + int output_line_stride = (int)calc_line_stride(o.output_width); + int t_output_surface_stride = output_line_stride * (int)t_output_height; + t_output_surface_stride /= FEATURE_ATOMIC_SIZE; + + unsigned t_input_data_entries; + if (t_input_channels_real == 1) + t_input_data_entries = t_input_width * t_input_height; + else if (t_input_width == 40 && t_input_channels_real == 40) + t_input_data_entries = 40; + else + t_input_data_entries = DIV_ROUND_UP( + t_input_width * 2 * + DIV_ROUND_UP(t_input_channels_real, FEATURE_ATOMIC_SIZE), + 8); + + unsigned t_weights_kernels = + o.depthwise ? 1 : alignu(o.output_channels, 2); + + unsigned t_surfaces_per_row = t_output_width * t_output_height * 2; + /* depthwise surfaces_per_row *= 2 skipped */ + + /* ---- rkt_split_tasks, single-tile branch (full weights, full input) ---- */ + unsigned weights_banks_required = calc_weights_banks(&o); + unsigned input_banks_required = calc_input_banks(&o); + int reuse_weights_cbuf; + unsigned available_input_banks; + if (weights_banks_required + 1 < CBUF_BANKS) { + reuse_weights_cbuf = 1; + available_input_banks = CBUF_BANKS - weights_banks_required; + } else { + reuse_weights_cbuf = 0; + available_input_banks = 7; + } + if (input_banks_required > available_input_banks) + return -1; /* needs CBUF task-splitting — unsupported here */ + + unsigned t_input_banks = input_banks_required; + unsigned t_weights_banks = CBUF_BANKS - t_input_banks; + unsigned t_atomic_count = t_output_width * t_output_height; + + /* ---- CNA params ---- */ + struct cna_params c; + memset(&c, 0, sizeof c); + c.input_width = t_input_width; + c.input_height = t_input_height; + c.input_channels = t_input_channels; + c.input_channels_real = t_input_channels_real; + c.output_width = t_output_width; + c.atomic_count = t_atomic_count; + c.weights_width = o.weights_width; + c.weights_height = o.weights_height; + c.weights_kernels = t_weights_kernels; + c.weights_banks = t_weights_banks; + c.input_banks = t_input_banks; + c.input_data_entries = t_input_data_entries; + c.input_line_stride = (uint32_t)t_input_line_stride; + c.input_surface_stride = (uint32_t)t_input_surface_stride; + c.input_dma = input_dma; + c.weights_dma = weights_dma; + c.stride_x = o.stride; + c.stride_y = o.stride; + c.output_zero_point = output_zero_point; + c.weights_zero_point = weight_zero_point; + c.input_zero_point = input_zero_point; + c.depthwise = o.depthwise; + c.reuse_weights_cbuf = reuse_weights_cbuf; + c.addition_input = o.addition_input; + c.add_tensor = o.add_tensor; + c.task_num = 0; + + /* ---- CORE/DPU/PC params ---- */ + struct coredpu_params d; + memset(&d, 0, sizeof d); + d.input_width = t_input_width; + d.output_width = t_output_width; + d.output_height = t_output_height; + d.output_channels = t_output_channels; + d.output_channels_real = t_output_channels_real; + d.output_offset = 0; + d.output_surface_stride = (uint32_t)t_output_surface_stride; + d.surfaces_per_row = t_surfaces_per_row; + d.truncate_bits = 0; + d.output_zero_point = output_zero_point; + d.weights_zero_point = weight_zero_point; + d.depthwise = o.depthwise; + d.addition_offset = 0; + d.add_tensor = o.add_tensor; + d.input_scale = input_scale; + d.weights_scale = weights_scale; + d.output_scale = output_scale; + d.addition_scale = 1.0f; + d.output_addr = output_dma; + d.add_tensor_addr = 0; + d.biases_addr = bias_dma; + + /* ---- emit both stages into one contiguous regcmd buffer ---- */ + int n1 = rkt_emit_cna(out, out_capacity, &c); + if (n1 < 0) + return -1; + int n2 = rkt_emit_coredpu(out + n1, out_capacity - n1, &d); + if (n2 < 0) + return -1; + return n1 + n2; +} + +/* Back-compat: identity requant, no bias. */ +int rkt_build_matmul_regcmd(uint64_t *out, int out_capacity, + uint32_t M, uint32_t N, uint32_t K, + uint64_t input_dma, uint64_t weights_dma, + uint64_t output_dma, + int32_t input_zero_point, + int32_t weight_zero_point, + int32_t output_zero_point) +{ + return rkt_build_matmul_regcmd_scaled(out, out_capacity, M, N, K, + input_dma, weights_dma, output_dma, + input_zero_point, weight_zero_point, + output_zero_point, + 1.0f, 1.0f, 1.0f, 0); +} diff --git a/ggml/src/ggml-rocket/rkt_matmul.h b/ggml/src/ggml-rocket/rkt_matmul.h new file mode 100644 index 000000000..d5c58b72e --- /dev/null +++ b/ggml/src/ggml-rocket/rkt_matmul.h @@ -0,0 +1,64 @@ +/* + * rkt_matmul.h — Rockchip "rocket" NPU matmul regcmd generator + * + * Builds a register-command stream for a single INT8 matmul + * Y[M][N] = X[M][K] * W[K][N] expressed as a 1×1 convolution. + * + * Faithfully adapted from the upstream Mesa "rocket" Gallium driver + * (rkt_regcmd.c, rkt_task.c, rkt_ml.c, rkt_registers.h). + * + * SPDX-License-Identifier: MIT + */ + +#ifndef RKT_MATMUL_H +#define RKT_MATMUL_H + +#include + +/** + * rkt_build_matmul_regcmd - build a single-task NPU regcmd for an INT8 matmul + * + * The matmul is emitted as a 1×1 convolution on the NPU: + * input_channels = K + * kernels = N + * height = M (conv spatial rows) + * width = 1 (conv spatial columns) + * + * @param out caller-provided output buffer + * @param out_capacity max uint64_t entries in @out + * @param M number of rows of X (and rows of Y) + * @param N number of columns of W (and columns of Y) + * @param K number of columns of X / rows of W + * @param input_dma bus address of X[M][K] (must be 256-byte aligned) + * @param weights_dma bus address of W[K][N] (must be 256-byte aligned) + * @param output_dma bus address of Y[M][N] (must be 256-byte aligned) + * @param input_zero_point int8 zero-point for input (0-255, usually 128) + * @param weight_zero_point int8 zero-point for weights (0-255, usually 128) + * @param output_zero_point int8 zero-point for output (0-255, usually 128) + * + * @return number of uint64_t commands written, or -1 if capacity exceeded. + * + * NOTE: Rate/scale parameters are not exposed. This generator assumes + * input_scale = weights_scale = output_scale = 1.0 (identity requantisation). + * Callers requiring proper quantised output must patch REG_DPU_OUT_CVT_SCALE + * and REG_DPU_OUT_CVT_SHIFT after construction, or provide pre-computed + * scale/shift values via an extended API. + */ +int rkt_build_matmul_regcmd_scaled(uint64_t *out, int out_capacity, + uint32_t M, uint32_t N, uint32_t K, + uint64_t input_dma, uint64_t weights_dma, + uint64_t output_dma, + int32_t input_zero_point, int32_t weight_zero_point, + int32_t output_zero_point, + float input_scale, float weights_scale, + float output_scale, uint64_t bias_dma); + +int rkt_build_matmul_regcmd(uint64_t *out, int out_capacity, + uint32_t M, uint32_t N, uint32_t K, + uint64_t input_dma, uint64_t weights_dma, + uint64_t output_dma, + int32_t input_zero_point, + int32_t weight_zero_point, + int32_t output_zero_point); + +#endif /* RKT_MATMUL_H */ diff --git a/ggml/src/ggml-rocket/rkt_matmul_cna.c b/ggml/src/ggml-rocket/rkt_matmul_cna.c new file mode 100644 index 000000000..8130de7a2 --- /dev/null +++ b/ggml/src/ggml-rocket/rkt_matmul_cna.c @@ -0,0 +1,171 @@ +#include "rkt_matmul_cna.h" +#include "cna_defs.h" +#include + +/* + * Packed register command: (target << 48) | (value << 16) | reg + * target = block_enum + 1. CNA=0x200->0x201, DPU=0x1000->0x1001, + * DPU_RDMA=0x2000->0x2001. The target is a property of the register's block + * (Mesa derives it via rkt_get_target(reg)+1), NOT of the emit site — so the + * DPU_RDMA registers that appear inside this CNA sequence still carry 0x2001. + */ +static int rkt_push(uint64_t *out, int *n, int cap, uint32_t target, + uint32_t reg, uint32_t value) +{ + if (*n >= cap) + return -1; + out[*n] = ((uint64_t)target << 48) | ((uint64_t)value << 16) | (uint64_t)reg; + (*n)++; + return 0; +} + +#define EMIT_CNA(reg, val) do { if (rkt_push(out, &n, cap, 0x201, (reg), (val)) < 0) return -1; } while (0) +#define EMIT_DPU(reg, val) do { if (rkt_push(out, &n, cap, 0x1001, (reg), (val)) < 0) return -1; } while (0) +#define EMIT_RDMA(reg, val) do { if (rkt_push(out, &n, cap, 0x2001, (reg), (val)) < 0) return -1; } while (0) + +/* Field-set: (value << FIELD__SHIFT) & FIELD__MASK */ +#define F(v, name) (((uint32_t)(v) << name##__SHIFT) & name##__MASK) + +int rkt_emit_cna(uint64_t *out, int cap, const struct cna_params *p) +{ + int n = 0; + + uint32_t con0 = F(p->weights_banks, CNA_CBUF_CON0_WEIGHT_BANK) | + F(p->input_banks, CNA_CBUF_CON0_DATA_BANK); + if (p->task_num > 0 && p->reuse_weights_cbuf) + con0 |= F(1, CNA_CBUF_CON0_WEIGHT_REUSE); + EMIT_CNA(REG_CNA_CBUF_CON0, con0); + + EMIT_CNA(REG_CNA_DCOMP_REGNUM, 0); + EMIT_CNA(REG_CNA_DCOMP_CTRL, 0); + + uint32_t con1 = 0x0; + if (p->input_channels_real == 1) { + con1 |= F(1, CNA_CONV_CON1_NONALIGN_DMA) | F(1, CNA_CONV_CON1_GROUP_LINE_OFF) | + F(8, CNA_CONV_CON1_ARGB_IN); + } + if (p->depthwise) + con1 |= F(3, CNA_CONV_CON1_CONV_MODE); + EMIT_CNA(REG_CNA_CONV_CON1, con1); + + EMIT_DPU(REG_DPU_S_POINTER, F(1, DPU_S_POINTER_POINTER_PP_MODE) | + F(1, DPU_S_POINTER_EXECUTER_PP_EN) | + F(1, DPU_S_POINTER_POINTER_PP_EN)); + EMIT_RDMA(REG_DPU_RDMA_RDMA_S_POINTER, + F(1, DPU_RDMA_RDMA_S_POINTER_POINTER_PP_MODE) | + F(1, DPU_RDMA_RDMA_S_POINTER_EXECUTER_PP_EN) | + F(1, DPU_RDMA_RDMA_S_POINTER_POINTER_PP_EN)); + EMIT_CNA(REG_CNA_CONV_CON1, con1); + + EMIT_CNA(REG_CNA_CONV_CON2, + F(50 + p->stride_y + 1, CNA_CONV_CON2_FEATURE_GRAINS)); /* Magic: passes the most tests */ + EMIT_CNA(REG_CNA_CONV_CON3, F(p->stride_x, CNA_CONV_CON3_CONV_X_STRIDE) | + F(p->stride_y, CNA_CONV_CON3_CONV_Y_STRIDE)); + EMIT_CNA(REG_CNA_DATA_SIZE0, + F(p->input_width, CNA_DATA_SIZE0_DATAIN_WIDTH) | + F(p->input_height, CNA_DATA_SIZE0_DATAIN_HEIGHT)); + EMIT_CNA(REG_CNA_DATA_SIZE1, + F(p->input_channels_real - 1, CNA_DATA_SIZE1_DATAIN_CHANNEL_REAL) | + F(p->input_channels, CNA_DATA_SIZE1_DATAIN_CHANNEL)); + EMIT_CNA(REG_CNA_DATA_SIZE2, F(p->output_width, CNA_DATA_SIZE2_DATAOUT_WIDTH)); + EMIT_CNA(REG_CNA_DATA_SIZE3, F(p->atomic_count, CNA_DATA_SIZE3_DATAOUT_ATOMICS)); + EMIT_CNA(REG_CNA_WEIGHT_SIZE0, p->weights_width * p->weights_height * + p->input_channels * p->weights_kernels); + EMIT_CNA(REG_CNA_WEIGHT_SIZE1, + p->weights_width * p->weights_height * p->input_channels); + EMIT_CNA(REG_CNA_WEIGHT_SIZE2, + F(p->weights_width, CNA_WEIGHT_SIZE2_WEIGHT_WIDTH) | + F(p->weights_height, CNA_WEIGHT_SIZE2_WEIGHT_HEIGHT) | + F(p->weights_kernels, CNA_WEIGHT_SIZE2_WEIGHT_KERNELS)); + + EMIT_CNA(REG_CNA_CBUF_CON0, con0); + EMIT_CNA(REG_CNA_CBUF_CON1, F(p->input_data_entries, CNA_CBUF_CON1_DATA_ENTRIES)); + + if (p->input_channels_real == 1) { + unsigned truncate = 14; + unsigned scale = 16384; + unsigned offset = 65408; + + if (p->addition_input || p->add_tensor != -1) { + truncate = 15; + scale = 32388; + } + + EMIT_CNA(REG_CNA_CVT_CON0, F(truncate, CNA_CVT_CON0_CVT_TRUNCATE_3) | + F(truncate, CNA_CVT_CON0_CVT_TRUNCATE_2) | + F(truncate, CNA_CVT_CON0_CVT_TRUNCATE_1) | + F(truncate, CNA_CVT_CON0_CVT_TRUNCATE_0)); + EMIT_CNA(REG_CNA_CVT_CON1, + F(scale, CNA_CVT_CON1_CVT_SCALE0) | F(offset, CNA_CVT_CON1_CVT_OFFSET0)); + EMIT_CNA(REG_CNA_CVT_CON2, + F(scale, CNA_CVT_CON2_CVT_SCALE1) | F(offset, CNA_CVT_CON2_CVT_OFFSET1)); + EMIT_CNA(REG_CNA_CVT_CON3, + F(scale, CNA_CVT_CON3_CVT_SCALE2) | F(offset, CNA_CVT_CON3_CVT_OFFSET2)); + EMIT_CNA(REG_CNA_CVT_CON4, + F(scale, CNA_CVT_CON4_CVT_SCALE3) | F(offset, CNA_CVT_CON4_CVT_OFFSET3)); + } else { + EMIT_CNA(REG_CNA_CVT_CON0, F(1, CNA_CVT_CON0_DATA_SIGN) | + F(1, CNA_CVT_CON0_CVT_TYPE) | + F(1, CNA_CVT_CON0_CVT_BYPASS)); + EMIT_CNA(REG_CNA_CVT_CON1, F(1, CNA_CVT_CON1_CVT_SCALE0)); + EMIT_CNA(REG_CNA_CVT_CON2, F(1, CNA_CVT_CON2_CVT_SCALE1)); + EMIT_CNA(REG_CNA_CVT_CON3, F(1, CNA_CVT_CON3_CVT_SCALE2)); + EMIT_CNA(REG_CNA_CVT_CON4, F(1, CNA_CVT_CON4_CVT_SCALE3)); + } + + EMIT_CNA(REG_CNA_FC_CON0, 0); + EMIT_CNA(REG_CNA_FC_CON1, 0); + EMIT_CNA(REG_CNA_PAD_CON0, F(p->pad_left, CNA_PAD_CON0_PAD_LEFT) | + F(p->pad_top, CNA_PAD_CON0_PAD_TOP)); + EMIT_CNA(REG_CNA_FEATURE_DATA_ADDR, (uint32_t)p->input_dma); + EMIT_CNA(REG_CNA_FC_CON2, 0); + EMIT_CNA(REG_CNA_DMA_CON0, + F(15, CNA_DMA_CON0_WEIGHT_BURST_LEN) | F(15, CNA_DMA_CON0_DATA_BURST_LEN)); + EMIT_CNA(REG_CNA_DMA_CON1, F(p->input_line_stride, CNA_DMA_CON1_LINE_STRIDE)); + EMIT_CNA(REG_CNA_DMA_CON2, F(p->input_surface_stride, CNA_DMA_CON2_SURF_STRIDE)); + EMIT_CNA(REG_CNA_FC_DATA_SIZE0, + F(p->input_width, CNA_FC_DATA_SIZE0_DMA_WIDTH) | + F(p->input_height, CNA_FC_DATA_SIZE0_DMA_HEIGHT)); + EMIT_CNA(REG_CNA_FC_DATA_SIZE1, + F(p->input_channels, CNA_FC_DATA_SIZE1_DMA_CHANNEL)); + EMIT_CNA(REG_CNA_DCOMP_CTRL, 0); + EMIT_CNA(REG_CNA_DCOMP_REGNUM, 0); + EMIT_CNA(REG_CNA_DCOMP_ADDR0, (uint32_t)p->weights_dma); + EMIT_CNA(REG_CNA_DCOMP_AMOUNT0, 0); + EMIT_CNA(REG_CNA_DCOMP_AMOUNT1, 0); + EMIT_CNA(REG_CNA_DCOMP_AMOUNT2, 0); + EMIT_CNA(REG_CNA_DCOMP_AMOUNT3, 0); + EMIT_CNA(REG_CNA_DCOMP_AMOUNT4, 0); + EMIT_CNA(REG_CNA_DCOMP_AMOUNT5, 0); + EMIT_CNA(REG_CNA_DCOMP_AMOUNT6, 0); + EMIT_CNA(REG_CNA_DCOMP_AMOUNT7, 0); + EMIT_CNA(REG_CNA_DCOMP_AMOUNT8, 0); + EMIT_CNA(REG_CNA_DCOMP_AMOUNT9, 0); + EMIT_CNA(REG_CNA_DCOMP_AMOUNT10, 0); + EMIT_CNA(REG_CNA_DCOMP_AMOUNT11, 0); + EMIT_CNA(REG_CNA_DCOMP_AMOUNT12, 0); + EMIT_CNA(REG_CNA_DCOMP_AMOUNT13, 0); + EMIT_CNA(REG_CNA_DCOMP_AMOUNT14, 0); + EMIT_CNA(REG_CNA_DCOMP_AMOUNT15, 0); + + if (p->input_channels_real == 1) + EMIT_CNA(REG_CNA_CVT_CON5, 65535); + else + EMIT_CNA(REG_CNA_CVT_CON5, 0); + + int32_t pad_con1; + if (p->weights_width >= 3 && p->input_zero_point == 0x0) + pad_con1 = 0xffff8080; + else + pad_con1 = p->input_zero_point - 0x80; + + if (p->addition_input || p->add_tensor != -1) + pad_con1 = 0xffffff80; + + if (p->depthwise && p->input_zero_point == 0x8b) + pad_con1 = 0x0b0b; + + EMIT_CNA(REG_CNA_PAD_CON1, pad_con1); + + return n; +} diff --git a/ggml/src/ggml-rocket/rkt_matmul_cna.h b/ggml/src/ggml-rocket/rkt_matmul_cna.h new file mode 100644 index 000000000..e8d083ef2 --- /dev/null +++ b/ggml/src/ggml-rocket/rkt_matmul_cna.h @@ -0,0 +1,25 @@ +#ifndef RKT_MATMUL_CNA_H +#define RKT_MATMUL_CNA_H + +#include + +/* All the task-/operation-level fields the Mesa CNA emit reads, flattened. */ +struct cna_params { + uint32_t input_width, input_height, input_channels, input_channels_real; + uint32_t output_width, atomic_count; + uint32_t weights_width, weights_height, weights_kernels; + uint32_t weights_banks, input_banks, input_data_entries; + uint32_t input_line_stride, input_surface_stride; + uint64_t input_dma, weights_dma; /* NPU DMA addresses of the input/weight BOs */ + uint32_t stride_x, stride_y; + uint32_t pad_left, pad_right, pad_top, pad_bottom; + int32_t output_zero_point, weights_zero_point, input_zero_point; + int depthwise, reuse_weights_cbuf, addition_input, add_tensor; + unsigned task_num; /* 0 for the first/only task */ +}; + +/* Append the CNA-stage register commands (packed uint64_t) into out[] (capacity cap). + Returns the number of commands written, or -1 if capacity would be exceeded. */ +int rkt_emit_cna(uint64_t *out, int cap, const struct cna_params *p); + +#endif /* RKT_MATMUL_CNA_H */ diff --git a/ggml/src/ggml-rocket/rkt_matmul_coredpu.c b/ggml/src/ggml-rocket/rkt_matmul_coredpu.c new file mode 100644 index 000000000..b47324b58 --- /dev/null +++ b/ggml/src/ggml-rocket/rkt_matmul_coredpu.c @@ -0,0 +1,298 @@ +#include "rkt_matmul_coredpu.h" +#include "coredpu_defs.h" +#include + +/* Packed register command: (target<<48) | (value<<16) | reg. target = block_enum + 1. + CORE=0x800->0x801, DPU=0x1000->0x1001, PC=0x100->0x101. */ +static int rkt_push(uint64_t *out, int *n, int cap, uint32_t target, + uint32_t reg, uint32_t value) +{ + if (*n >= cap) + return -1; + out[*n] = ((uint64_t)target << 48) | ((uint64_t)value << 16) | (uint64_t)reg; + (*n)++; + return 0; +} + +#define EMIT_CORE(reg, val) do { if (rkt_push(out, &n, cap, 0x801, (reg), (val)) < 0) return -1; } while (0) +#define EMIT_DPU(reg, val) do { if (rkt_push(out, &n, cap, 0x1001, (reg), (val)) < 0) return -1; } while (0) +#define EMIT_RDMA(reg, val) do { if (rkt_push(out, &n, cap, 0x2001, (reg), (val)) < 0) return -1; } while (0) +#define EMIT_PC(reg, val) do { if (rkt_push(out, &n, cap, 0x101, (reg), (val)) < 0) return -1; } while (0) +#define EMIT_RAW(tgt, reg, val) do { if (rkt_push(out, &n, cap, (tgt), (reg), (val)) < 0) return -1; } while (0) + +#define F(v, name) (((uint32_t)(v) << name##__SHIFT) & name##__MASK) + +/* float -> raw IEEE-754 bits (Mesa fui()) */ +static uint32_t fui(float f) { uint32_t u; __builtin_memcpy(&u, &f, sizeof u); return u; } +#define MAX2(a, b) ((a) > (b) ? (a) : (b)) +#define ATOMIC_K_SIZE 16 +/* raw 64-bit command append (Mesa util_dynarray_append_typed(regs, uint64_t, v)) */ +#define RAW64(v) do { if (n >= cap) return -1; out[n++] = (uint64_t)(v); } while (0) + +int rkt_emit_coredpu(uint64_t *out, int cap, const struct coredpu_params *p) +{ + int n = 0; + int num_tasks = 1; /* single-task port */ + int offset = (int)p->output_zero_point - 0x80; /* Mesa CNA local */ + + /* ==== CORE + DPU + PC emit (filled by slice) ==== */ + uint32_t misc_cfg = F(1, CORE_MISC_CFG_QD_EN); + if (p->depthwise) + misc_cfg |= F(1, CORE_MISC_CFG_DW_EN); + EMIT_CORE(REG_CORE_MISC_CFG, misc_cfg); + EMIT_CORE(REG_CORE_DATAOUT_SIZE_0, + F(p->output_height - 1, CORE_DATAOUT_SIZE_0_DATAOUT_HEIGHT) | + F(p->output_width - 1, CORE_DATAOUT_SIZE_0_DATAOUT_WIDTH)); + EMIT_CORE(REG_CORE_DATAOUT_SIZE_1, + F(p->output_channels - 1, CORE_DATAOUT_SIZE_1_DATAOUT_CHANNEL)); + EMIT_CORE(REG_CORE_CLIP_TRUNCATE, + F(p->truncate_bits, CORE_CLIP_TRUNCATE_CLIP_TRUNCATE)); + EMIT_RAW(0x801, 0x3030, 0); + + uint32_t feat_mode_cfg = + F(15, DPU_FEATURE_MODE_CFG_BURST_LEN) | F(2, DPU_FEATURE_MODE_CFG_OUTPUT_MODE); + if (p->depthwise) + feat_mode_cfg |= F(3, DPU_FEATURE_MODE_CFG_CONV_MODE); + + EMIT_DPU(REG_DPU_FEATURE_MODE_CFG, feat_mode_cfg); + EMIT_DPU(REG_DPU_DATA_FORMAT, 0); + EMIT_DPU(REG_DPU_OFFSET_PEND, 0); + EMIT_DPU(REG_DPU_DST_BASE_ADDR, + p->output_addr + + p->output_offset); + EMIT_DPU(REG_DPU_DST_SURF_STRIDE, + F(p->output_surface_stride, DPU_DST_SURF_STRIDE_DST_SURF_STRIDE)); + EMIT_DPU(REG_DPU_DATA_CUBE_WIDTH, + F(p->output_width - 1, DPU_DATA_CUBE_WIDTH_WIDTH)); + EMIT_DPU(REG_DPU_DATA_CUBE_HEIGHT, + F(p->output_height - 1, DPU_DATA_CUBE_HEIGHT_HEIGHT)); + EMIT_DPU(REG_DPU_DATA_CUBE_NOTCH_ADDR, 0); + EMIT_DPU(REG_DPU_DATA_CUBE_CHANNEL, + F(p->output_channels_real - 1, DPU_DATA_CUBE_CHANNEL_ORIG_CHANNEL) | + F(p->output_channels - 1, DPU_DATA_CUBE_CHANNEL_CHANNEL)); + EMIT_DPU(REG_DPU_BS_CFG, F(2, DPU_BS_CFG_BS_ALU_ALGO) | F(1, DPU_BS_CFG_BS_ALU_SRC) | + F(1, DPU_BS_CFG_BS_RELU_BYPASS) | + F(1, DPU_BS_CFG_BS_MUL_BYPASS)); + EMIT_DPU(REG_DPU_BS_ALU_CFG, 0); + EMIT_DPU(REG_DPU_BS_MUL_CFG, 0); + EMIT_DPU(REG_DPU_BS_RELUX_CMP_VALUE, 0); + + if (p->depthwise) { + EMIT_DPU(REG_DPU_BS_OW_CFG, F(3, DPU_BS_OW_CFG_SIZE_E_2) | + F(3, DPU_BS_OW_CFG_SIZE_E_1) | + F(3, DPU_BS_OW_CFG_SIZE_E_0)); + } else { + EMIT_DPU(REG_DPU_BS_OW_CFG, F(1, DPU_BS_OW_CFG_SIZE_E_2) | + F(1, DPU_BS_OW_CFG_SIZE_E_1) | + F(1, DPU_BS_OW_CFG_SIZE_E_0)); + } + + EMIT_DPU(REG_DPU_BS_OW_OP, F(0x80 - p->weights_zero_point, DPU_BS_OW_OP_OW_OP)); + + EMIT_DPU(REG_DPU_WDMA_SIZE_0, + F(p->output_channels - 1, DPU_WDMA_SIZE_0_CHANNEL_WDMA)); + EMIT_DPU(REG_DPU_WDMA_SIZE_1, + F(p->output_height - 1, DPU_WDMA_SIZE_1_HEIGHT_WDMA) | + F(p->output_width - 1, DPU_WDMA_SIZE_1_WIDTH_WDMA)); + EMIT_DPU(REG_DPU_BN_CFG, + F(1, DPU_BN_CFG_BN_RELU_BYPASS) | F(1, DPU_BN_CFG_BN_MUL_BYPASS) | + F(1, DPU_BN_CFG_BN_ALU_BYPASS) | F(1, DPU_BN_CFG_BN_BYPASS)); + EMIT_DPU(REG_DPU_BN_ALU_CFG, 0); + EMIT_DPU(REG_DPU_BN_MUL_CFG, 0); + EMIT_DPU(REG_DPU_BN_RELUX_CMP_VALUE, 0); + + if (p->add_tensor != -1) { + EMIT_DPU(REG_DPU_EW_CFG, + F(1, DPU_EW_CFG_EW_CVT_TYPE) | F(1, DPU_EW_CFG_EW_DATA_MODE) | + F(1, DPU_EW_CFG_EDATA_SIZE) | F(2, DPU_EW_CFG_EW_ALU_ALGO) | + F(1, DPU_EW_CFG_EW_RELU_BYPASS) | F(1, DPU_EW_CFG_EW_LUT_BYPASS) | + F(1, DPU_EW_CFG_EW_OP_SRC)); + + /* See http://nvdla.org/hw/v1/ias/precision.html#element-wise */ + EMIT_DPU(REG_DPU_EW_CVT_OFFSET_VALUE, p->addition_offset); + + float add_scale = + p->addition_scale / (p->input_scale * p->weights_scale); + + uint32_t add_scale_bits = fui(add_scale); + /* Taken from + * https://github.com/pytorch/QNNPACK/blob/master/src/qnnpack/requantization.h#L130 + */ + unsigned add_shift = 127 + 31 - 32 - (add_scale_bits >> 23) + 16; + + unsigned scale = ((add_scale_bits >> 9) & 0x7fff); + if (scale < 1 << 14) + scale |= 1 << 14; + + EMIT_DPU(REG_DPU_EW_CVT_SCALE_VALUE, + F(add_shift - 1, DPU_EW_CVT_SCALE_VALUE_EW_OP_CVT_SHIFT) | + F(scale, DPU_EW_CVT_SCALE_VALUE_EW_OP_CVT_SCALE)); + + EMIT_DPU(REG_DPU_EW_RELUX_CMP_VALUE, 0x0); + + float out_conv_scale = + (p->input_scale * p->weights_scale) / p->output_scale; + uint32_t out_scale_bits = fui(out_conv_scale); + unsigned out_shift = 127 + 31 - 32 - (out_scale_bits >> 23) + 16; + if (p->truncate_bits > 0) + out_shift--; + unsigned out_scale = ((out_scale_bits >> 9) & 0x7fff) + 1; + if (out_scale < 1 << 14) + out_scale |= 1 << 14; + + EMIT_DPU(REG_DPU_OUT_CVT_OFFSET, offset); + EMIT_DPU(REG_DPU_OUT_CVT_SCALE, F(out_scale, DPU_OUT_CVT_SCALE_OUT_CVT_SCALE)); + EMIT_DPU(REG_DPU_OUT_CVT_SHIFT, F(out_shift - 1, DPU_OUT_CVT_SHIFT_OUT_CVT_SHIFT)); + } else { + EMIT_DPU(REG_DPU_EW_CFG, + F(1, DPU_EW_CFG_EW_RELU_BYPASS) | F(1, DPU_EW_CFG_EW_OP_CVT_BYPASS) | + F(1, DPU_EW_CFG_EW_LUT_BYPASS) | F(1, DPU_EW_CFG_EW_OP_BYPASS) | + F(1, DPU_EW_CFG_EW_BYPASS)); + EMIT_DPU(REG_DPU_EW_CVT_OFFSET_VALUE, 0); + EMIT_DPU(REG_DPU_EW_CVT_SCALE_VALUE, F(1, DPU_EW_CVT_SCALE_VALUE_EW_OP_CVT_SCALE)); + EMIT_DPU(REG_DPU_EW_RELUX_CMP_VALUE, 0); + EMIT_DPU(REG_DPU_OUT_CVT_OFFSET, offset); + + float conv_scale = + (p->input_scale * p->weights_scale) / p->output_scale; + // DBG("conv_scale %f\n", conv_scale); + uint32_t scale_bits = fui(conv_scale); + /* Taken from + * https://github.com/pytorch/QNNPACK/blob/master/src/qnnpack/requantization.h#L130 + */ + unsigned shift = 127 + 31 - 32 - (scale_bits >> 23) + 16; + + if (p->truncate_bits > 0) + shift--; + + unsigned scale = ((scale_bits >> 9) & 0x7fff) + 1; + if (scale < 1 << 14) + scale |= 1 << 14; + + EMIT_DPU(REG_DPU_OUT_CVT_SCALE, F(scale, DPU_OUT_CVT_SCALE_OUT_CVT_SCALE)); + EMIT_DPU(REG_DPU_OUT_CVT_SHIFT, F(shift - 1, DPU_OUT_CVT_SHIFT_OUT_CVT_SHIFT)); + } + + EMIT_DPU(REG_DPU_EW_OP_VALUE_0, 0); + EMIT_DPU(REG_DPU_EW_OP_VALUE_1, 0); + EMIT_DPU(REG_DPU_EW_OP_VALUE_2, 0); + EMIT_DPU(REG_DPU_EW_OP_VALUE_3, 0); + EMIT_DPU(REG_DPU_EW_OP_VALUE_4, 0); + EMIT_DPU(REG_DPU_EW_OP_VALUE_5, 0); + EMIT_DPU(REG_DPU_EW_OP_VALUE_6, 0); + EMIT_DPU(REG_DPU_EW_OP_VALUE_7, 0); + EMIT_DPU(REG_DPU_SURFACE_ADD, F(p->surfaces_per_row, DPU_SURFACE_ADD_SURF_ADD)); + EMIT_RAW(0x1001, 0x40c4, 0); + EMIT_DPU(REG_DPU_LUT_ACCESS_CFG, 0); + EMIT_DPU(REG_DPU_LUT_ACCESS_DATA, 0); + EMIT_DPU(REG_DPU_LUT_CFG, 0); + EMIT_DPU(REG_DPU_LUT_INFO, 0); + EMIT_DPU(REG_DPU_LUT_LE_START, 0); + EMIT_DPU(REG_DPU_LUT_LE_END, 0); + EMIT_DPU(REG_DPU_LUT_LO_START, 0); + EMIT_DPU(REG_DPU_LUT_LO_END, 0); + EMIT_DPU(REG_DPU_LUT_LE_SLOPE_SCALE, 0); + EMIT_DPU(REG_DPU_LUT_LE_SLOPE_SHIFT, 0); + EMIT_DPU(REG_DPU_LUT_LO_SLOPE_SCALE, 0); + EMIT_DPU(REG_DPU_LUT_LO_SLOPE_SHIFT, 0); + EMIT_RDMA(REG_DPU_RDMA_RDMA_DATA_CUBE_WIDTH, + F(p->output_width - 1, DPU_RDMA_RDMA_DATA_CUBE_WIDTH_WIDTH)); + EMIT_RDMA(REG_DPU_RDMA_RDMA_DATA_CUBE_HEIGHT, + F(p->output_height - 1, DPU_RDMA_RDMA_DATA_CUBE_HEIGHT_HEIGHT)); + EMIT_RDMA(REG_DPU_RDMA_RDMA_DATA_CUBE_CHANNEL, + F(p->output_channels - 1, DPU_RDMA_RDMA_DATA_CUBE_CHANNEL_CHANNEL)); + + if (p->add_tensor != -1) { + EMIT_RDMA(REG_DPU_RDMA_RDMA_SRC_BASE_ADDR, + p->add_tensor_addr + + p->output_offset); + } else { + EMIT_RDMA(REG_DPU_RDMA_RDMA_SRC_BASE_ADDR, 0); + } + + EMIT_RDMA(REG_DPU_RDMA_RDMA_BRDMA_CFG, F(1, DPU_RDMA_RDMA_BRDMA_CFG_BRDMA_DATA_USE)); + EMIT_RDMA(REG_DPU_RDMA_RDMA_BS_BASE_ADDR, + p->biases_addr); + EMIT_RDMA(REG_DPU_RDMA_RDMA_NRDMA_CFG, 0); + EMIT_RDMA(REG_DPU_RDMA_RDMA_BN_BASE_ADDR, 0); + + unsigned ew_stride = + MAX2(p->output_width * p->output_height, 12); + + if (p->add_tensor != -1) { + EMIT_RDMA(REG_DPU_RDMA_RDMA_ERDMA_CFG, + F(1, DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_DATA_MODE) | + F(1, DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_DATA_SIZE)); + unsigned ew_base_offset = + p->output_width * p->output_height * ATOMIC_K_SIZE; + EMIT_RDMA(REG_DPU_RDMA_RDMA_EW_BASE_ADDR, + p->add_tensor_addr + + p->output_offset + ew_base_offset); + EMIT_RDMA(REG_DPU_RDMA_RDMA_EW_SURF_STRIDE, + F(ew_stride, DPU_RDMA_RDMA_EW_SURF_STRIDE_EW_SURF_STRIDE)); + } else { + EMIT_RDMA(REG_DPU_RDMA_RDMA_ERDMA_CFG, F(1, DPU_RDMA_RDMA_ERDMA_CFG_ERDMA_DISABLE)); + EMIT_RDMA(REG_DPU_RDMA_RDMA_EW_BASE_ADDR, 0); + EMIT_RDMA(REG_DPU_RDMA_RDMA_EW_SURF_STRIDE, 0); + } + + uint32_t rdma_feat_mode_cfg = 0x0; + + if (p->add_tensor != -1) { + rdma_feat_mode_cfg |= F(15, DPU_RDMA_RDMA_FEATURE_MODE_CFG_BURST_LEN) | + F(5, DPU_RDMA_RDMA_FEATURE_MODE_CFG_COMB_USE); + } else { + rdma_feat_mode_cfg |= F(15, DPU_RDMA_RDMA_FEATURE_MODE_CFG_BURST_LEN) | + F(1, DPU_RDMA_RDMA_FEATURE_MODE_CFG_MRDMA_DISABLE); + } + + if (p->depthwise) + rdma_feat_mode_cfg |= F(3, DPU_RDMA_RDMA_FEATURE_MODE_CFG_CONV_MODE); + + EMIT_RDMA(REG_DPU_RDMA_RDMA_FEATURE_MODE_CFG, rdma_feat_mode_cfg); + EMIT_RDMA(REG_DPU_RDMA_RDMA_SRC_DMA_CFG, 0); + + unsigned surf_notch = + ew_stride + + p->output_width * (p->output_height - p->output_height); + + if (p->input_width == 3) { + surf_notch = 15; + } + + if (p->add_tensor != -1) { + EMIT_RDMA(REG_DPU_RDMA_RDMA_SURF_NOTCH, + F(surf_notch, DPU_RDMA_RDMA_SURF_NOTCH_SURF_NOTCH_ADDR)); + } else { + EMIT_RDMA(REG_DPU_RDMA_RDMA_SURF_NOTCH, 0); + } + + EMIT_RDMA(REG_DPU_RDMA_RDMA_PAD_CFG, 0); + EMIT_RDMA(REG_DPU_RDMA_RDMA_WEIGHT, + F(1, DPU_RDMA_RDMA_WEIGHT_E_WEIGHT) | F(1, DPU_RDMA_RDMA_WEIGHT_N_WEIGHT) | + F(1, DPU_RDMA_RDMA_WEIGHT_B_WEIGHT) | F(1, DPU_RDMA_RDMA_WEIGHT_M_WEIGHT)); + + if (p->add_tensor != -1) { + EMIT_RDMA(REG_DPU_RDMA_RDMA_EW_SURF_NOTCH, + F(surf_notch, DPU_RDMA_RDMA_EW_SURF_NOTCH_EW_SURF_NOTCH)); + } else { + EMIT_RDMA(REG_DPU_RDMA_RDMA_EW_SURF_NOTCH, 0x0); + } + + if (num_tasks == 1) + RAW64(0x0); + else + EMIT_PC(REG_PC_BASE_ADDRESS, 0); + + EMIT_PC(REG_PC_REGISTER_AMOUNTS, 0); + + /* TRM: before op_en, 64'h0041_xxxx_xxxx_xxxx must be set. */ + RAW64(0x0041000000000000); + + /* TRM: 64'h0081_0000_007f_0008 will set each block's op_en(CNA, CORE, ..., + * PPU_RDMA). */ + EMIT_RAW(0x81, REG_PC_OPERATION_ENABLE, + F(14, PC_OPERATION_ENABLE_RESERVED_0) | F(1, PC_OPERATION_ENABLE_OP_EN)); + + return n; +} diff --git a/ggml/src/ggml-rocket/rkt_matmul_coredpu.h b/ggml/src/ggml-rocket/rkt_matmul_coredpu.h new file mode 100644 index 000000000..56d3e87ef --- /dev/null +++ b/ggml/src/ggml-rocket/rkt_matmul_coredpu.h @@ -0,0 +1,25 @@ +#ifndef RKT_MATMUL_COREDPU_H +#define RKT_MATMUL_COREDPU_H + +#include + +/* Fields the Mesa CORE/DPU/PC emit reads (task-/operation-level flattened). */ +struct coredpu_params { + uint32_t input_width; + uint32_t output_width, output_height; + uint32_t output_channels, output_channels_real; + uint32_t output_offset, output_surface_stride, surfaces_per_row; + uint32_t truncate_bits; + int32_t output_zero_point, weights_zero_point; + int depthwise; + int32_t addition_offset; + int32_t add_tensor; /* index, or -1 if none */ + float input_scale, weights_scale, output_scale, addition_scale; + uint64_t output_addr, add_tensor_addr, biases_addr; /* pre-computed DMA addresses */ +}; + +/* Append CORE+DPU+PC register commands (packed uint64_t) into out[] (capacity cap). + Returns number written, or -1 on overflow. */ +int rkt_emit_coredpu(uint64_t *out, int cap, const struct coredpu_params *p); + +#endif /* RKT_MATMUL_COREDPU_H */ diff --git a/ggml/src/ggml-rocket/rkt_npu_matmul.c b/ggml/src/ggml-rocket/rkt_npu_matmul.c new file mode 100644 index 000000000..36922446f --- /dev/null +++ b/ggml/src/ggml-rocket/rkt_npu_matmul.c @@ -0,0 +1,165 @@ +/* SPDX-License-Identifier: MIT + * + * rkt_npu_matmul.c — reusable 2D-tiled INT8 matmul on the rocket NPU (see .h). + */ +#define _GNU_SOURCE +#include "rkt_npu_matmul.h" +#include "librocket.h" +#include "rocket_accel.h" +#include "rkt_matmul.h" +#include "rkt_operands.h" +#include "rkt_gemm.h" + +#include +#include +#include +#include +#include + +/* rocket_prep_bo timeout is an ABSOLUTE CLOCK_MONOTONIC deadline. Build one + * rel_ns in the future, for the job-completion wait so a hung NPU job returns + * an error instead of blocking forever (the driver watchdog is ~5s). */ +static int64_t rkt_deadline_ns(int64_t rel_ns) +{ + struct timespec t; + clock_gettime(CLOCK_MONOTONIC, &t); + return (int64_t)t.tv_sec * 1000000000LL + (int64_t)t.tv_nsec + rel_ns; +} + +int rkt_build_matmul_regcmd_scaled(uint64_t *out, int out_capacity, + uint32_t M, uint32_t N, uint32_t K, + uint64_t input_dma, uint64_t weights_dma, + uint64_t output_dma, + int32_t input_zero_point, int32_t weight_zero_point, + int32_t output_zero_point, + float input_scale, float weights_scale, + float output_scale, uint64_t bias_dma); + +struct rkt_bo { uint32_t h; uint64_t dma, off; void *map; uint32_t sz; }; + +static int bo_alloc(int fd, struct rkt_bo *b, uint32_t size) +{ + b->sz = (size + 0xfffu) & ~0xfffu; + if (rocket_create_bo(fd, b->sz, &b->h, &b->dma, &b->off)) + return -1; + b->map = rocket_mmap_bo(fd, b->off, b->sz); + return b->map == MAP_FAILED ? -1 : 0; +} + +/* prep(CPU) -> zero -> copy -> fini(device); prep uses INT64_MAX = block. */ +static void bo_write(int fd, struct rkt_bo *b, const void *src, unsigned n) +{ + rocket_prep_bo(fd, b->h, INT64_MAX); + memset(b->map, 0, b->sz); + if (src) + memcpy(b->map, src, n); + rocket_fini_bo(fd, b->h); +} + +int rkt_npu_matmul(int fd, const uint8_t *X, const uint8_t *Wc, + const int32_t *bias, uint32_t M, uint32_t N, uint32_t K, + uint8_t izp, uint8_t wzp, uint8_t ozp, + float in_scale, float wt_scale, float out_scale, const float *out_scales, + uint32_t tile_m, uint32_t tile_n, uint8_t *Y) +{ + int ret = -1; + int32_t *bias0 = NULL; + if (!bias) { + bias0 = calloc(N, sizeof(int32_t)); + if (!bias0) + return -1; + bias = bias0; + } + + unsigned max_isz = rkt_raw_input_size(1, tile_m, K); + unsigned max_wsz = rkt_packed_weights_size(1, 1, K, tile_n); + uint8_t *Xt = malloc((size_t)tile_m * K); + uint8_t *Wt = malloc((size_t)tile_n * K); + uint8_t *ipk = calloc(1, max_isz); + uint8_t *wpk = calloc(1, max_wsz); + int32_t *bpk = malloc((size_t)tile_n * sizeof(int32_t)); + struct rkt_gemm_tile *tiles = malloc( + ((M + tile_m - 1) / tile_m) * ((N + tile_n - 1) / tile_n) * + sizeof(struct rkt_gemm_tile)); + uint64_t *rc = malloc(4096 * sizeof(uint64_t)); + if (!Xt || !Wt || !ipk || !wpk || !bpk || !tiles || !rc) + goto out; + + int nt = rkt_gemm_plan(M, N, tile_m, tile_n, tiles, + (int)(((M + tile_m - 1) / tile_m) * + ((N + tile_n - 1) / tile_n))); + if (nt <= 0) + goto out; + + for (int t = 0; t < nt; t++) { + unsigned r = tiles[t].row, cc = tiles[t].col; + unsigned m = tiles[t].m, n = tiles[t].n; + + for (unsigned y = 0; y < m; y++) + memcpy(Xt + (size_t)y * K, X + (size_t)(r + y) * K, K); + for (unsigned j = 0; j < n; j++) + memcpy(Wt + (size_t)j * K, Wc + (size_t)(cc + j) * K, K); + rkt_pack_input(Xt, 1, m, K, izp, ipk); + rkt_pack_weights(Wt, 1, 1, K, n, wzp, wpk); + rkt_compute_biases(Wt, bias + cc, 1, 1, K, n, wzp, izp, bpk); + + struct rkt_bo in = {0}, w = {0}, b = {0}, o = {0}, reg = {0}; + if (bo_alloc(fd, &in, rkt_raw_input_size(1, m, K)) || + bo_alloc(fd, &w, rkt_packed_weights_size(1, 1, K, n)) || + bo_alloc(fd, &b, n * sizeof(int32_t)) || + bo_alloc(fd, &o, rkt_raw_output_size(1, m, n)) || + bo_alloc(fd, ®, 0x1000)) + goto out; + bo_write(fd, &in, ipk, rkt_raw_input_size(1, m, K)); + bo_write(fd, &w, wpk, rkt_packed_weights_size(1, 1, K, n)); + bo_write(fd, &b, bpk, n * sizeof(int32_t)); + bo_write(fd, &o, NULL, 0); + + int nw = rkt_build_matmul_regcmd_scaled( + rc, 4096, m, n, K, in.dma, w.dma, o.dma, izp, wzp, ozp, + in_scale, wt_scale, out_scales ? out_scales[cc] : out_scale, b.dma); + if (nw < 0) { + rocket_close_bo(fd, in.h); rocket_close_bo(fd, w.h); + rocket_close_bo(fd, b.h); rocket_close_bo(fd, o.h); + rocket_close_bo(fd, reg.h); + goto out; + } + bo_write(fd, ®, rc, (unsigned)nw * sizeof(uint64_t)); + + struct drm_rocket_task task = { .regcmd = (uint32_t)reg.dma, + .regcmd_count = (uint32_t)nw }; + uint32_t ih[] = { in.h, w.h, b.h, reg.h }, oh[] = { o.h }; + struct drm_rocket_job job; + memset(&job, 0, sizeof job); + job.tasks = (uintptr_t)&task; + job.task_count = 1; + job.task_struct_size = sizeof(struct drm_rocket_task); + job.in_bo_handles = (uintptr_t)ih; + job.in_bo_handle_count = 4; + job.out_bo_handles = (uintptr_t)oh; + job.out_bo_handle_count = 1; + + int sr = rocket_submit(fd, &job, 1); + int wr = sr ? -1 : rocket_prep_bo(fd, o.h, rkt_deadline_ns(6000000000LL)); + if (!sr && !wr) { + uint8_t *got = malloc((size_t)m * n); + if (got) { + rkt_unpack_output(o.map, 1, m, n, got); + for (unsigned y = 0; y < m; y++) + memcpy(Y + (size_t)(r + y) * N + cc, + got + (size_t)y * n, n); + free(got); + } + } + rocket_close_bo(fd, in.h); rocket_close_bo(fd, w.h); + rocket_close_bo(fd, b.h); rocket_close_bo(fd, o.h); + rocket_close_bo(fd, reg.h); + if (sr || wr) + goto out; + } + ret = 0; +out: + free(Xt); free(Wt); free(ipk); free(wpk); free(bpk); + free(tiles); free(rc); free(bias0); + return ret; +} diff --git a/ggml/src/ggml-rocket/rkt_npu_matmul.h b/ggml/src/ggml-rocket/rkt_npu_matmul.h new file mode 100644 index 000000000..705237bc0 --- /dev/null +++ b/ggml/src/ggml-rocket/rkt_npu_matmul.h @@ -0,0 +1,35 @@ +/* SPDX-License-Identifier: MIT + * + * rkt_npu_matmul.h — reusable arbitrary-size INT8 matmul on the rocket NPU. + * + * The U6 core primitive: Y[M][N] = X[M][K] * W[K][N] computed by 2D-tiling + * into single-op rocket matmuls (rkt_gemm_plan + rkt_build_matmul_regcmd + + * librocket). Intended to be called from a ggml MUL_MAT backend. + */ +#ifndef RKT_NPU_MATMUL_H +#define RKT_NPU_MATMUL_H + +#include + +/* + * fd open /dev/accel/accel0 (rocket_open). + * X [M][K] uint8 row-major activations. + * Wc [N][K] uint8 conv-order weights (Wc[out_ch][in_ch]). + * bias [N] int32 per-output-channel bias, or NULL for none. + * izp/wzp/ozp int8 zero-points (0-255). + * in/wt/out_scale per-tensor requant scales. + * tile_m/tile_n desired tile sizes; each (m x n, full-K) tile must satisfy + * rkt_gemm_op_fits (m limited by CBUF input banks, K*n <= ~327k). + * Y [M][N] uint8 output (caller-allocated). + * + * Returns 0 on success, -1 on error (bad args / tile too big / submit fail). + * Allocates weights/bias/input/output/regcmd BOs per tile (correctness-first; + * weight-reuse across M-tiles is a later perf optimization). + */ +int rkt_npu_matmul(int fd, const uint8_t *X, const uint8_t *Wc, + const int32_t *bias, uint32_t M, uint32_t N, uint32_t K, + uint8_t izp, uint8_t wzp, uint8_t ozp, + float in_scale, float wt_scale, float out_scale, const float *out_scales, + uint32_t tile_m, uint32_t tile_n, uint8_t *Y); + +#endif /* RKT_NPU_MATMUL_H */ diff --git a/ggml/src/ggml-rocket/rkt_operands.c b/ggml/src/ggml-rocket/rkt_operands.c new file mode 100644 index 000000000..89afc0d2e --- /dev/null +++ b/ggml/src/ggml-rocket/rkt_operands.c @@ -0,0 +1,127 @@ +/* SPDX-License-Identifier: MIT + * + * rkt_operands.c — NPU operand layout (faithful ports of Mesa rocket). + * + * Non-depthwise conv/matmul. Row-major host tensors: + * weights_in : [oc_real][ww][wh][ic_real] (conv weights) + * input_in : [iw][ih][ic_real] + * output_out : [oh][ow][oc_real] + * + * Ported from Mesa rkt_coefs.c (rkt_fill_weights, rkt_fill_biases) and + * rkt_ml.c (feature tiling in rkt_ml_subgraph_invoke, output de-tile in + * rkt_ml_subgraph_read_outputs). Assumes input_zero_point handling as Mesa. + */ +#include "rkt_operands.h" + +#define FEATURE_ATOMIC_SIZE 16 +#define WEIGHT_ATOMIC_SIZE 32 +#define DIV_ROUND_UP(n, d) (((n) + (d) - 1) / (d)) +#define MAX2(a, b) ((a) > (b) ? (a) : (b)) +#define MIN2(a, b) ((a) < (b) ? (a) : (b)) +static unsigned alignu(unsigned v, unsigned a) { return ((v + a - 1) / a) * a; } + +unsigned rkt_raw_input_size(unsigned iw, unsigned ih, unsigned ic_real) +{ + return iw * ih * (DIV_ROUND_UP(ic_real, FEATURE_ATOMIC_SIZE) * 2) * + FEATURE_ATOMIC_SIZE; +} + +unsigned rkt_raw_output_size(unsigned ow, unsigned oh, unsigned oc_real) +{ + return ow * oh * (DIV_ROUND_UP(oc_real, FEATURE_ATOMIC_SIZE) * 2) * + FEATURE_ATOMIC_SIZE; +} + +unsigned rkt_packed_weights_size(unsigned ww, unsigned wh, unsigned ic_real, + unsigned oc_real) +{ + unsigned ic = alignu(MAX2(ic_real, FEATURE_ATOMIC_SIZE), FEATURE_ATOMIC_SIZE); + unsigned oc = alignu(oc_real, 2); + return ww * wh * oc * alignu(ic, WEIGHT_ATOMIC_SIZE) * 2; +} + +/* Mesa rkt_fill_weights (non-depthwise). */ +unsigned rkt_pack_weights(const uint8_t *win, unsigned ww, unsigned wh, + unsigned ic_real, unsigned oc_real, uint8_t wzp, + uint8_t *out) +{ + unsigned ic = alignu(MAX2(ic_real, FEATURE_ATOMIC_SIZE), FEATURE_ATOMIC_SIZE); + unsigned oc = alignu(oc_real, 2); + unsigned groups = WEIGHT_ATOMIC_SIZE; + unsigned ic1n = DIV_ROUND_UP(ic, groups); + unsigned ic2n = MIN2(ic, groups); + unsigned n = 0; + + for (unsigned oc1 = 0; oc1 < DIV_ROUND_UP(oc, WEIGHT_ATOMIC_SIZE); oc1++) + for (unsigned ic1 = 0; ic1 < ic1n; ic1++) + for (unsigned x = 0; x < ww; x++) + for (unsigned y = 0; y < wh; y++) + for (unsigned oc2 = 0; + oc2 < MIN2(oc, WEIGHT_ATOMIC_SIZE); oc2++) + for (unsigned ic2 = 0; ic2 < ic2n; ic2++) { + unsigned occ = oc1 * WEIGHT_ATOMIC_SIZE + oc2; + unsigned icc = ic1 * groups + ic2; + if (oc_real > 2 && occ >= alignu(oc_real, 2)) + continue; + if (occ >= oc_real) { + out[n++] = 0x0; + } else if (icc >= ic_real) { + if (ic2 < 16 || (ic_real % 32) > 16) + out[n++] = (uint8_t)(wzp - 0x80); + } else { + unsigned idx = ((occ * ww + x) * wh + y) * ic_real + icc; + out[n++] = (uint8_t)(win[idx] - 0x80); + } + } + return n; +} + +/* Mesa rkt_fill_biases (truncate_bits=0 path) + calculate_bias_correction. */ +void rkt_compute_biases(const uint8_t *win, const int32_t *bin, unsigned ww, + unsigned wh, unsigned ic_real, unsigned oc_real, + uint8_t wzp, uint8_t izp, int32_t *bout) +{ + for (unsigned oc = 0; oc < oc_real; oc++) { + int32_t corr = 0; + for (unsigned x = 0; x < ww; x++) + for (unsigned y = 0; y < wh; y++) + for (unsigned ic = 0; ic < ic_real; ic++) { + unsigned idx = ((oc * ww + x) * wh + y) * ic_real + ic; + corr += ((int32_t)win[idx] - wzp) * + ((int32_t)izp - 0x80); + } + bout[oc] = bin[oc] - corr; + } +} + +/* Mesa feature tiling (rkt_ml_subgraph_invoke, input_channels>1 path). */ +unsigned rkt_pack_input(const uint8_t *iin, unsigned iw, unsigned ih, + unsigned ic_real, uint8_t izp, uint8_t *out) +{ + unsigned n = 0; + for (unsigned u = 0; u < DIV_ROUND_UP(ic_real, FEATURE_ATOMIC_SIZE); u++) + for (unsigned x = 0; x < iw; x++) + for (unsigned y = 0; y < ih; y++) + for (unsigned c = 0; c < FEATURE_ATOMIC_SIZE; c++) { + unsigned ich = c + u * FEATURE_ATOMIC_SIZE; + if (ich < ic_real) + out[n++] = (uint8_t)(iin[(x * ih + y) * ic_real + ich] - 0x80); + else + out[n++] = (uint8_t)(izp - 0x80); + } + return n; +} + +/* Mesa output de-tile (rkt_ml_subgraph_read_outputs). */ +void rkt_unpack_output(const uint8_t *raw, unsigned ow, unsigned oh, + unsigned oc_real, uint8_t *out) +{ + for (unsigned oc = 0; oc < oc_real; oc++) + for (unsigned x = 0; x < ow; x++) + for (unsigned y = 0; y < oh; y++) { + unsigned c = oc % FEATURE_ATOMIC_SIZE; + unsigned g = oc / FEATURE_ATOMIC_SIZE; + out[(y * ow + x) * oc_real + oc] = + (uint8_t)(raw[((g * oh + y) * ow + x) * FEATURE_ATOMIC_SIZE + c] + 0x80); + } +} diff --git a/ggml/src/ggml-rocket/rkt_operands.h b/ggml/src/ggml-rocket/rkt_operands.h new file mode 100644 index 000000000..fc76876d8 --- /dev/null +++ b/ggml/src/ggml-rocket/rkt_operands.h @@ -0,0 +1,30 @@ +/* SPDX-License-Identifier: MIT + * + * rkt_operands.h — NPU operand layout ports (see rkt_operands.c). + * Non-depthwise. Row-major host tensors: + * weights_in [oc_real][ww][wh][ic_real], input_in [iw][ih][ic_real], + * output_out [oh][ow][oc_real]. + */ +#ifndef RKT_OPERANDS_H +#define RKT_OPERANDS_H + +#include + +unsigned rkt_raw_input_size(unsigned iw, unsigned ih, unsigned ic_real); +unsigned rkt_raw_output_size(unsigned ow, unsigned oh, unsigned oc_real); +unsigned rkt_packed_weights_size(unsigned ww, unsigned wh, unsigned ic_real, + unsigned oc_real); + +unsigned rkt_pack_weights(const uint8_t *weights_in, unsigned ww, unsigned wh, + unsigned ic_real, unsigned oc_real, uint8_t wzp, + uint8_t *out); +void rkt_compute_biases(const uint8_t *weights_in, const int32_t *biases_in, + unsigned ww, unsigned wh, unsigned ic_real, + unsigned oc_real, uint8_t wzp, uint8_t izp, + int32_t *biases_out); +unsigned rkt_pack_input(const uint8_t *input_in, unsigned iw, unsigned ih, + unsigned ic_real, uint8_t izp, uint8_t *out); +void rkt_unpack_output(const uint8_t *raw, unsigned ow, unsigned oh, + unsigned oc_real, uint8_t *out); + +#endif /* RKT_OPERANDS_H */ diff --git a/ggml/src/ggml-rocket/rocket_accel.h b/ggml/src/ggml-rocket/rocket_accel.h new file mode 100644 index 000000000..d0685e372 --- /dev/null +++ b/ggml/src/ggml-rocket/rocket_accel.h @@ -0,0 +1,192 @@ +/* SPDX-License-Identifier: MIT */ +/* + * Copyright © 2024 Tomeu Vizoso + */ +#ifndef __DRM_UAPI_ROCKET_ACCEL_H__ +#define __DRM_UAPI_ROCKET_ACCEL_H__ + +#include "drm.h" + +#if defined(__cplusplus) +extern "C" { +#endif + +#define DRM_ROCKET_CREATE_BO 0x00 +#define DRM_ROCKET_SUBMIT 0x01 +#define DRM_ROCKET_PREP_BO 0x02 +#define DRM_ROCKET_FINI_BO 0x03 + +#define DRM_IOCTL_ROCKET_CREATE_BO DRM_IOWR(DRM_COMMAND_BASE + DRM_ROCKET_CREATE_BO, struct drm_rocket_create_bo) +#define DRM_IOCTL_ROCKET_SUBMIT DRM_IOW(DRM_COMMAND_BASE + DRM_ROCKET_SUBMIT, struct drm_rocket_submit) +#define DRM_IOCTL_ROCKET_PREP_BO DRM_IOW(DRM_COMMAND_BASE + DRM_ROCKET_PREP_BO, struct drm_rocket_prep_bo) +#define DRM_IOCTL_ROCKET_FINI_BO DRM_IOW(DRM_COMMAND_BASE + DRM_ROCKET_FINI_BO, struct drm_rocket_fini_bo) + +/** + * struct drm_rocket_create_bo - ioctl argument for creating Rocket BOs. + * + */ +struct drm_rocket_create_bo { + /** + * @size: Input: Size of the requested BO. + */ + __u32 size; + + /** + * @handle: Output: GEM handle for the BO. + */ + __u32 handle; + + /** + * @dma_address: Output: DMA address for the BO in the NPU address + * space. This address is private to the DRM fd and is valid for + * the lifetime of the GEM handle. + */ + __u64 dma_address; + + /** + * @offset: Output: Offset into the drm node to use for subsequent + * mmap call. + */ + __u64 offset; +}; + +/** + * struct drm_rocket_prep_bo - ioctl argument for starting CPU ownership of the BO. + * + * Takes care of waiting for any NPU jobs that might still use the NPU and performs cache + * synchronization. + */ +struct drm_rocket_prep_bo { + /** + * @handle: Input: GEM handle of the buffer object. + */ + __u32 handle; + + /** + * @reserved: Reserved, must be zero. + */ + __u32 reserved; + + /** + * @timeout_ns: Input: Amount of time to wait for NPU jobs. + */ + __s64 timeout_ns; +}; + +/** + * struct drm_rocket_fini_bo - ioctl argument for finishing CPU ownership of the BO. + * + * Synchronize caches for NPU access. + */ +struct drm_rocket_fini_bo { + /** + * @handle: Input: GEM handle of the buffer object. + */ + __u32 handle; + + /** + * @reserved: Reserved, must be zero. + */ + __u32 reserved; +}; + +/** + * struct drm_rocket_task - A task to be run on the NPU + * + * A task is the smallest unit of work that can be run on the NPU. + */ +struct drm_rocket_task { + /** + * @regcmd: Input: DMA address to NPU mapping of register command buffer + */ + __u32 regcmd; + + /** + * @regcmd_count: Input: Number of commands in the register command + * buffer + */ + __u32 regcmd_count; +}; + +/** + * struct drm_rocket_job - A job to be run on the NPU + * + * The kernel will schedule the execution of this job taking into account its + * dependencies with other jobs. All tasks in the same job will be executed + * sequentially on the same core, to benefit from memory residency in SRAM. + */ +struct drm_rocket_job { + /** + * @tasks: Input: Pointer to an array of struct drm_rocket_task. + */ + __u64 tasks; + + /** + * @in_bo_handles: Input: Pointer to a u32 array of the BOs that + * are read by the job. + */ + __u64 in_bo_handles; + + /** + * @out_bo_handles: Input: Pointer to a u32 array of the BOs that + * are written to by the job. + */ + __u64 out_bo_handles; + + /** + * @task_count: Input: Number of tasks passed in. + */ + __u32 task_count; + + /** + * @task_struct_size: Input: Size in bytes of the structs in the + * @tasks field. + */ + __u32 task_struct_size; + + /** + * @in_bo_handle_count: Input: Number of input BO handles passed in + * (size is that times 4). + */ + __u32 in_bo_handle_count; + + /** + * @out_bo_handle_count: Input: Number of output BO handles passed in + * (size is that times 4). + */ + __u32 out_bo_handle_count; +}; + +/** + * struct drm_rocket_submit - ioctl argument for submitting commands to the NPU. + * + * The kernel will schedule the execution of these jobs in dependency order. + */ +struct drm_rocket_submit { + /** + * @jobs: Input: Pointer to an array of struct drm_rocket_job. + */ + __u64 jobs; + + /** + * @job_count: Input: Number of jobs passed in. + */ + __u32 job_count; + + /** + * @job_struct_size: Input: Size in bytes of the structs in the + * @jobs field. + */ + __u32 job_struct_size; + + /** + * @reserved: Reserved, must be zero. + */ + __u64 reserved; +}; + +#if defined(__cplusplus) +} +#endif + +#endif /* __DRM_UAPI_ROCKET_ACCEL_H__ */