marfrit/rk-llama.cpp
1
0
Fork 0
Code Issues Pull Requests Actions 30 Packages Projects Releases Wiki Activity

rknpu2: hoist per-row A-prep scratch to per-thread pools

Browse Source 
Experiment: the A-prep OMP loop heap-allocated ready_row (plus signed_row and full_hadamard_row on the Hadamard path) once per row -- up to M*(1-3) malloc/free per k-segment per matmul. Move to per-thread pools sized omp_get_max_threads(); resize() reuses the buffer after the first row. Byte-identical (consumers read only the fully-written prefix). pp300/512/700 35.8/41.6/40.6 -> 36.5/42.1/42.4 (+2-4%); tg128 2.93. Cumulative vs baseline: pp300 +19%, tg +14%.
This commit is contained in:
mfritsche
2026-07-09 01:37:12 +02:00
parent 8adda8448c
commit 3705e84765
1 changed files with 12 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/src/ggml-rknpu2/ggml-rknpu2.cpp
+12 -3
View File
@@ -647,15 +647,24 @@ static enum ggml_status ggml_backend_rknpu_graph_compute(ggml_backend_t backend,
const int row_stride = (int)(src1->nb[1] / sizeof(float)); const int row_stride = (int)(src1->nb[1] / sizeof(float));
void* dst_base = mem_A_shared->virt_addr; void* dst_base = mem_A_shared->virt_addr;
// Per-thread reusable scratch to avoid malloc/free per row.
const int a_prep_threads = omp_get_max_threads();
std::vector<std::vector<float>> ready_row_pool(a_prep_threads);
std::vector<std::vector<float>> signed_row_pool(a_prep_threads);
std::vector<std::vector<float>> hadamard_row_pool(a_prep_threads);
#pragma omp parallel for if(M >= 8) #pragma omp parallel for if(M >= 8)
for (int m = 0; m < M; ++m) { for (int m = 0; m < M; ++m) {
const float* src_row = x + (size_t)m * row_stride; const float* src_row = x + (size_t)m * row_stride;
std::vector<float> ready_row(K_seg_op); std::vector<float>& ready_row = ready_row_pool[omp_get_thread_num()];
ready_row.resize(K_seg_op);
// Applying Hadamard Transform // Applying Hadamard Transform
if (is_hadamard) { if (is_hadamard) {
std::vector<float> signed_row(K); std::vector<float>& signed_row = signed_row_pool[omp_get_thread_num()];
std::vector<float> full_hadamard_row(K_op); signed_row.resize(K);
std::vector<float>& full_hadamard_row = hadamard_row_pool[omp_get_thread_num()];
full_hadamard_row.resize(K_op);
for(int k=0; k<K; ++k) signed_row[k] = src_row[k] * s_vec[k]; for(int k=0; k<K; ++k) signed_row[k] = src_row[k] * s_vec[k];
rknpu2_calibration::hadamard_transform(full_hadamard_row.data(), signed_row.data(), K, K_op); rknpu2_calibration::hadamard_transform(full_hadamard_row.data(), signed_row.data(), K, K_op);