From 3705e8476591bc10c031adffd1ca2b0dfef5cdac Mon Sep 17 00:00:00 2001 From: mfritsche Date: Thu, 9 Jul 2026 01:37:12 +0200 Subject: [PATCH] rknpu2: hoist per-row A-prep scratch to per-thread pools Experiment: the A-prep OMP loop heap-allocated ready_row (plus signed_row and full_hadamard_row on the Hadamard path) once per row -- up to M*(1-3) malloc/free per k-segment per matmul. Move to per-thread pools sized omp_get_max_threads(); resize() reuses the buffer after the first row. Byte-identical (consumers read only the fully-written prefix). pp300/512/700 35.8/41.6/40.6 -> 36.5/42.1/42.4 (+2-4%); tg128 2.93. Cumulative vs baseline: pp300 +19%, tg +14%. --- ggml/src/ggml-rknpu2/ggml-rknpu2.cpp | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-rknpu2/ggml-rknpu2.cpp b/ggml/src/ggml-rknpu2/ggml-rknpu2.cpp index e6a15fac3..e3397b06c 100644 --- a/ggml/src/ggml-rknpu2/ggml-rknpu2.cpp +++ b/ggml/src/ggml-rknpu2/ggml-rknpu2.cpp @@ -647,15 +647,24 @@ static enum ggml_status ggml_backend_rknpu_graph_compute(ggml_backend_t backend, const int row_stride = (int)(src1->nb[1] / sizeof(float)); void* dst_base = mem_A_shared->virt_addr; + // Per-thread reusable scratch to avoid malloc/free per row. + const int a_prep_threads = omp_get_max_threads(); + std::vector> ready_row_pool(a_prep_threads); + std::vector> signed_row_pool(a_prep_threads); + std::vector> hadamard_row_pool(a_prep_threads); + #pragma omp parallel for if(M >= 8) for (int m = 0; m < M; ++m) { const float* src_row = x + (size_t)m * row_stride; - std::vector ready_row(K_seg_op); + std::vector& ready_row = ready_row_pool[omp_get_thread_num()]; + ready_row.resize(K_seg_op); // Applying Hadamard Transform if (is_hadamard) { - std::vector signed_row(K); - std::vector full_hadamard_row(K_op); + std::vector& signed_row = signed_row_pool[omp_get_thread_num()]; + signed_row.resize(K); + std::vector& full_hadamard_row = hadamard_row_pool[omp_get_thread_num()]; + full_hadamard_row.resize(K_op); for(int k=0; k