mfritsche
8e49f43486
New MUL_MAT backend for the RK3588 NPU via the mainline 'rocket' DRM-accel driver (/dev/accel/accel0), with no vendor RKNPU2 userspace. Modeled on ggml-blas: an ACCEL device with CPU-resident weights; the scheduler offloads supported MUL_MAT nodes and leaves the rest on CPU. Per plane: dequantize weight to F32 (ggml to_float), requantize per-output- channel to int8, quantize F32 activation per-tensor to int8, run the tiled int8 GEMM on the NPU (rkt_npu_matmul, ported from the rosenblatt npu-probe: Mesa-rocket regcmd builder + NVDLA CBUF tiling + librocket DRM ioctls), then dequantize the int8 result. out_scale is a Cauchy-Schwarz bound on |output| so the int8 output never clips. Empirically-verified HW limits gate supports_op: TILE_M=16, TILE_N=128, K<=8192 (int8 K-limit), K%16==0, F32 activations, dequantizable weight. Job-completion wait uses a finite 6s absolute deadline so a hung NPU job errors instead of deadlocking. Enable with -DGGML_ROCKET=ON (and -DLLAMA_RKNPU2=OFF so the vendor backend does not claim the weight buffers). Verified end-to-end: gemma-4-E2B-it-Q8_0 generates coherent text on boltzmann's mainline rocket kernel, matmuls offloaded to the NPU, ~3.9 tok/s eval. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01EWpfhDgYNA21tETDP9ueBE
26 lines
1.1 KiB
C
26 lines
1.1 KiB
C
#ifndef RKT_MATMUL_CNA_H
|
|
#define RKT_MATMUL_CNA_H
|
|
|
|
#include <stdint.h>
|
|
|
|
/* All the task-/operation-level fields the Mesa CNA emit reads, flattened. */
|
|
struct cna_params {
|
|
uint32_t input_width, input_height, input_channels, input_channels_real;
|
|
uint32_t output_width, atomic_count;
|
|
uint32_t weights_width, weights_height, weights_kernels;
|
|
uint32_t weights_banks, input_banks, input_data_entries;
|
|
uint32_t input_line_stride, input_surface_stride;
|
|
uint64_t input_dma, weights_dma; /* NPU DMA addresses of the input/weight BOs */
|
|
uint32_t stride_x, stride_y;
|
|
uint32_t pad_left, pad_right, pad_top, pad_bottom;
|
|
int32_t output_zero_point, weights_zero_point, input_zero_point;
|
|
int depthwise, reuse_weights_cbuf, addition_input, add_tensor;
|
|
unsigned task_num; /* 0 for the first/only task */
|
|
};
|
|
|
|
/* Append the CNA-stage register commands (packed uint64_t) into out[] (capacity cap).
|
|
Returns the number of commands written, or -1 if capacity would be exceeded. */
|
|
int rkt_emit_cna(uint64_t *out, int cap, const struct cna_params *p);
|
|
|
|
#endif /* RKT_MATMUL_CNA_H */
|