marfrit/rk-llama.cpp
1
0
Fork 0
Code Issues Pull Requests Actions 31 Packages Projects Releases Wiki Activity

rknpu2: skip dst memset + RMW when single K-segment

Browse Source 
Experiment: with one K-segment (K<=8192, the common case) each dst element is written exactly once (N-segments are disjoint), so the full-MxN memset and the read-modify-write accumulate are dead work. Write each element once; keep the +=/memset path for multi-K. Byte-identical output. Throughput ~neutral (NPU-bound, so CPU-side traffic saving is marginal): E1 pp300/pp512/pp700/tg 37.01/40.25/40.03/2.61 -> E2 35.8/41.1/40.3/2.57 (pp512/pp700 +1-2%, pp300 within run variance). Kept for traffic reduction and as prerequisite for E5 NEON collect.
This commit is contained in:
mfritsche
2026-07-09 00:59:39 +02:00
parent 04d2c6e548
commit c5a47492ff
1 changed files with 20 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/src/ggml-rknpu2/ggml-rknpu2.cpp
+19 -7
View File
@@ -546,9 +546,15 @@ static enum ggml_status ggml_backend_rknpu_graph_compute(ggml_backend_t backend,
b_domain_id = it->second.iommu_domain_id; b_domain_id = it->second.iommu_domain_id;
} }
// Cleaning the C-matrix buffer // Cleaning the C-matrix buffer.
// Accumulation is only real across K-segments; N-segments write disjoint
// N ranges. With a single K-segment each dst element is written exactly
// once, so the zero-fill and the read-modify-write below are dead work.
const bool single_k_segment = (all_k_segments.size() == 1);
float* dst_data = (float*)get_tensor_real_ptr(dst); float* dst_data = (float*)get_tensor_real_ptr(dst);
if (!single_k_segment) {
memset(dst_data, 0, (size_t)M * N * sizeof(float)); memset(dst_data, 0, (size_t)M * N * sizeof(float));
}
// Acquiring the Hadamard vector // Acquiring the Hadamard vector
std::vector<float> s_vec; std::vector<float> s_vec;
@@ -747,8 +753,10 @@ static enum ggml_status ggml_backend_rknpu_graph_compute(ggml_backend_t backend,
float* dst_ptr = dst_data + (size_t)m * N + N_offset; float* dst_ptr = dst_data + (size_t)m * N + N_offset;
float* src_ptr = src_segment_base + (size_t)m * N_segment; float* src_ptr = src_segment_base + (size_t)m * N_segment;
for(int n=0; n<N_segment; ++n) { if (single_k_segment) {
dst_ptr[n] += src_ptr[n] * dequant_scale; for(int n=0; n<N_segment; ++n) dst_ptr[n] = src_ptr[n] * dequant_scale;
} else {
for(int n=0; n<N_segment; ++n) dst_ptr[n] += src_ptr[n] * dequant_scale;
} }
} }
break; break;
@@ -764,8 +772,10 @@ static enum ggml_status ggml_backend_rknpu_graph_compute(ggml_backend_t backend,
float* dst_ptr = dst_data + (size_t)m * N + N_offset; float* dst_ptr = dst_data + (size_t)m * N + N_offset;
int32_t* src_ptr = (int32_t*)mem_C_segments[idx]->virt_addr + (size_t)m * N_segment; int32_t* src_ptr = (int32_t*)mem_C_segments[idx]->virt_addr + (size_t)m * N_segment;
for(int n=0; n<N_segment; ++n) { if (single_k_segment) {
dst_ptr[n] += (float)src_ptr[n] * dequant_scale; for(int n=0; n<N_segment; ++n) dst_ptr[n] = (float)src_ptr[n] * dequant_scale;
} else {
for(int n=0; n<N_segment; ++n) dst_ptr[n] += (float)src_ptr[n] * dequant_scale;
} }
} }
break; break;
@@ -781,8 +791,10 @@ static enum ggml_status ggml_backend_rknpu_graph_compute(ggml_backend_t backend,
float* dst_ptr = dst_data + (size_t)m * N + N_offset; float* dst_ptr = dst_data + (size_t)m * N + N_offset;
int16_t* src_ptr = (int16_t*)mem_C_segments[idx]->virt_addr + (size_t)m * N_segment; int16_t* src_ptr = (int16_t*)mem_C_segments[idx]->virt_addr + (size_t)m * N_segment;
for(int n=0; n<N_segment; ++n) { if (single_k_segment) {
dst_ptr[n] += (float)src_ptr[n] * dequant_scale; for(int n=0; n<N_segment; ++n) dst_ptr[n] = (float)src_ptr[n] * dequant_scale;
} else {
for(int n=0; n<N_segment; ++n) dst_ptr[n] += (float)src_ptr[n] * dequant_scale;
} }
} }
break; break;