From 7eea51881ec97bc697c1767a54dc4f12596bd40a Mon Sep 17 00:00:00 2001 From: kurquhar Date: Sat, 26 Sep 2026 23:02:37 -0700 Subject: [PATCH] hexagon: support tiled Q4_0 and Q8_0 GET_ROWS (llama/29511) * hexagon: support tiled Q4_0 and Q8_0 GET_ROWS * hex-get-rows: fix macros * hex-get-rows: use tiled HVX dequantization Assisted-by: OpenCode * hex-get-rows: fix register spills and clean up checks for unsupported ops * hex-get-rows: improve dma pipeline * hex-get-rows: improve/simplify kernel selection logic * hex-build: reenable vectorizer, didnt notice the regression earlier in the sampler update --------- Co-authored-by: Max Krasnyansky --- ggml/src/ggml-hexagon/ggml-hexagon.cpp | 124 ++++++---- ggml/src/ggml-hexagon/htp-opnode.h | 15 +- .../ggml-hexagon/htp/cmake-toolchain.cmake | 2 +- ggml/src/ggml-hexagon/htp/get-rows-ops.c | 232 ++++++++++++++++-- ggml/src/ggml-hexagon/htp/get-rows-ops.h | 40 ++- 5 files changed, 332 insertions(+), 81 deletions(-) diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index d95687e34..521a97a73 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -5208,49 +5208,51 @@ static void ggml_hexagon_precompute_get_rows_params( const uint32_t ne12 = src1->ne[2]; const uint32_t nr = ne10 * ne11 * ne12; - const size_t nb01 = src0->nb[1]; - const size_t nb1 = dst->nb[1]; + const ggml_tensor * src0_base = src0->view_src ? src0->view_src : src0; + const auto * extra = src0_base->buffer && ggml_backend_buffer_is_hexagon(src0_base->buffer) ? + (const ggml_hexagon_tensor_extra *) src0_base->extra : nullptr; + const bool tiled = src0->type == GGML_TYPE_Q4_0 || (extra && (extra->flags & GGML_HEXAGON_TENSOR_REPACK) != 0) || + sess->needs_repack.count(src0_base) || sess->needs_repack.count(src0); - const bool can_use_dma = (src0->type == dst->type) && (nb01 == nb1); - const bool use_dma = can_use_dma && (ne00 >= 2048); - - kparams->use_dma = use_dma ? 1 : 0; - - uint32_t chunks_per_row = 1; - uint32_t chunk_size = ne00; - uint32_t total_tasks = nr; - - if (use_dma) { - kparams->n_threads = (std::min)((uint32_t)sess->n_threads, nr); - kparams->tasks_per_thread = (nr + kparams->n_threads - 1) / kparams->n_threads; + if (src0->type == dst->type) { + kparams->kernel_type = HTP_GET_ROWS_KERNEL_SAMETYPE; + } else if (tiled) { + kparams->kernel_type = HTP_GET_ROWS_KERNEL_TILED; } else { - if (src0->type == GGML_TYPE_F32 && nr < sess->n_threads) { - const uint32_t min_chunk_size = 1024; - uint32_t max_chunks = ne00 / min_chunk_size; - if (max_chunks == 0) { - max_chunks = 1; - } - chunks_per_row = (std::min)((sess->n_threads + nr - 1) / nr, max_chunks); - chunk_size = (ne00 + chunks_per_row - 1) / chunks_per_row; - total_tasks = nr * chunks_per_row; - } - kparams->n_threads = (std::min)(total_tasks, (uint32_t)sess->n_threads); - kparams->tasks_per_thread = (total_tasks + kparams->n_threads - 1) / kparams->n_threads; + kparams->kernel_type = HTP_GET_ROWS_KERNEL_FLAT; } + const uint32_t chunks_per_row = 1; + const uint32_t chunk_size = ne00; + const uint32_t total_tasks = nr; + + kparams->n_threads = (std::min)((uint32_t)sess->n_threads, total_tasks); + + struct htp_get_rows_vtcm_layout vtcm_layout = {}; + while (kparams->n_threads > 0) { + htp_get_rows_vtcm_layout_build(&vtcm_layout, kparams->kernel_type, src0->type, ne00, kparams->n_threads); + if (vtcm_layout.total_bytes <= sess->vtcm_size) { + break; + } + --kparams->n_threads; + } + + if (kparams->n_threads == 0 && total_tasks > 0) { + htp_get_rows_vtcm_layout_build(&vtcm_layout, kparams->kernel_type, src0->type, ne00, 1); + } + + kparams->vtcm_size = (total_tasks == 0) ? 0 : vtcm_layout.total_bytes; + kparams->tasks_per_thread = kparams->n_threads > 0 ? (total_tasks + kparams->n_threads - 1) / kparams->n_threads : 0; + kparams->chunks_per_row = chunks_per_row; kparams->chunk_size = chunk_size; kparams->total_tasks = total_tasks; - kparams->div_ne10 = init_fastdiv_values(ne10); - kparams->div_ne10_ne11 = init_fastdiv_values(ne10 * ne11); - kparams->div_chunks_per_row = init_fastdiv_values(chunks_per_row); - kparams->div_ne02 = init_fastdiv_values(ne02); - kparams->div_ne03 = init_fastdiv_values(ne03); - - struct htp_get_rows_vtcm_layout vtcm_layout; - htp_get_rows_vtcm_layout_build(&vtcm_layout, src0->type, ne00, kparams->n_threads); - kparams->vtcm_size = vtcm_layout.total_bytes; + kparams->div_ne10 = ne10 > 0 ? init_fastdiv_values(ne10) : fastdiv_values{0, 0}; + kparams->div_ne10_ne11 = (ne10 * ne11) > 0 ? init_fastdiv_values(ne10 * ne11) : fastdiv_values{0, 0}; + kparams->div_chunks_per_row = chunks_per_row > 0 ? init_fastdiv_values(chunks_per_row) : fastdiv_values{0, 0}; + kparams->div_ne02 = ne02 > 0 ? init_fastdiv_values(ne02) : fastdiv_values{0, 0}; + kparams->div_ne03 = ne03 > 0 ? init_fastdiv_values(ne03) : fastdiv_values{0, 0}; } static void ggml_hexagon_precompute_set_rows_params( @@ -6176,19 +6178,36 @@ static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session * const struct ggml_tensor * src1 = op->src[1]; // indices const struct ggml_tensor * dst = op; - if (src0->extra) { - const auto * extra = (const ggml_hexagon_tensor_extra *) src0->extra; - if (extra->flags & GGML_HEXAGON_TENSOR_REPACK) { + if (src0->type == GGML_TYPE_Q4_0 && src0->view_src) { + return false; + } + + const ggml_tensor * src0_base = src0->view_src ? src0->view_src : src0; + bool is_repacked = false; + if (src0_base->buffer && ggml_backend_buffer_is_hexagon(src0_base->buffer) && src0_base->extra) { + const auto * extra = (const ggml_hexagon_tensor_extra *) src0_base->extra; + is_repacked = (extra->flags & GGML_HEXAGON_TENSOR_REPACK) != 0; + if (is_repacked && src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q8_0) { return false; } } + is_repacked = is_repacked || sess->needs_repack.count(src0_base) || sess->needs_repack.count(src0); - if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_I32 && src0->ne[0] < 32) { + // View offsets use the raw quantized layout and cannot address a tiled allocation. + if (src0->view_src && is_repacked) { + return false; + } + + if (src0->type == GGML_TYPE_Q4_0 && src0->buffer && !is_repacked) { + return false; + } + + if (src0->type != dst->type && src0->ne[0] < 32) { return false; } if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 && - src0->type != GGML_TYPE_Q8_0 && src0->type != GGML_TYPE_I32) { + src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q8_0 && src0->type != GGML_TYPE_I32) { return false; } @@ -6196,13 +6215,30 @@ static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session * return false; } - if (src0->type == GGML_TYPE_I32) { - if (dst->type != GGML_TYPE_I32) { + if (src0->type == dst->type) { + if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_I32 && src0->type != GGML_TYPE_F16) { return false; } - } - else if (dst->type != GGML_TYPE_F32) { + } else if (src0->type == GGML_TYPE_I32) { return false; + } else if (dst->type != GGML_TYPE_F32) { + return false; + } + + // Empty recurrent-state gathers are skipped at execution; do not split the graph for them. + if (ggml_is_empty(op)) { + return true; + } + + struct htp_get_rows_kernel_params kparams; + ggml_hexagon_precompute_get_rows_params(sess, src0, src1, dst, &kparams); + if (kparams.n_threads == 0 || (size_t) kparams.vtcm_size > sess->vtcm_size) { + return false; + } + + // Q4_0 has no raw fallback. Mark only accepted tensors for repacking. + if (src0->type == GGML_TYPE_Q4_0 && !src0->buffer) { + sess->needs_repack.insert(src0); } return true; diff --git a/ggml/src/ggml-hexagon/htp-opnode.h b/ggml/src/ggml-hexagon/htp-opnode.h index e3677b267..8ff38545c 100644 --- a/ggml/src/ggml-hexagon/htp-opnode.h +++ b/ggml/src/ggml-hexagon/htp-opnode.h @@ -20,6 +20,7 @@ #include "htp/gated-delta-net-ops.h" #include "htp/softmax-ops.h" #include "htp/argsort-ops.h" +#include "htp/get-rows-ops.h" struct htp_opnode { ggml_tensor * node { nullptr }; @@ -360,9 +361,7 @@ struct htp_opformat { } else if (node.opcode == HTP_OP_GATED_DELTA_NET) { const auto * kparams = (const struct htp_gdn_kernel_params *) node.kernel_params; const char * path = (kparams->kernel_type == HTP_GDN_KERNEL_HMX_CHUNKED) ? "hmx-chunked" : "hvx-recurrent"; - snprintf(str, max_size, "%s-%s vtcm %u", - path, - kparams->kda ? "kda" : "scalar", + snprintf(str, max_size, "%s-%s vtcm %u", path, kparams->kda ? "kda" : "scalar", (unsigned int) (kparams->vtcm_size ? kparams->vtcm_size : kparams->vtcm_per_thread * kparams->n_threads)); } else if (node.opcode == HTP_OP_MUL || node.opcode == HTP_OP_ADD || node.opcode == HTP_OP_ADD_ID || node.opcode == HTP_OP_SUB || node.opcode == HTP_OP_DIV) { @@ -374,9 +373,19 @@ struct htp_opformat { node.opcode == HTP_OP_TOP_K ? "top_k" : "argsort", (int) kparams->n_threads, (int) kparams->n_chunks, (int) kparams->chunk_elems, (int) kparams->vtcm_size); + } else if (node.opcode == HTP_OP_GET_ROWS) { + const auto * kparams = (const struct htp_get_rows_kernel_params *) node.kernel_params; + const char * ktype_str = "unknown"; + switch (kparams->kernel_type) { + case HTP_GET_ROWS_KERNEL_SAMETYPE: ktype_str = "sametype"; break; + case HTP_GET_ROWS_KERNEL_TILED: ktype_str = "tiled"; break; + case HTP_GET_ROWS_KERNEL_FLAT: ktype_str = "flat"; break; + } + snprintf(str, max_size, "%s%s vtcm %u", ktype_str, kparams->n_threads > 1 ? "-multi" : "", (unsigned int) kparams->vtcm_size); } else { snprintf(str, max_size, "----"); } + } void format(const htp_opnode & node) { diff --git a/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake b/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake index 10840185f..3eff2a398 100644 --- a/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake +++ b/ggml/src/ggml-hexagon/htp/cmake-toolchain.cmake @@ -136,7 +136,7 @@ set(CMAKE_SHARED_LIBRARY_SONAME_C_FLAG "-Wl,-soname,") set(CMAKE_SHARED_LIBRARY_SONAME_CXX_FLAG "-Wl,-soname,") # Compiler Options -set(COMMON_FLAGS "${ARCH_FLAGS} -fno-vectorize -fno-slp-vectorize -flto -Wall -Werror -fno-zero-initialized-in-bss -G0 -fdata-sections -fpic ${XQF_ARGS}") +set(COMMON_FLAGS "${ARCH_FLAGS} -fvectorize -flto -Wall -Werror -fno-zero-initialized-in-bss -G0 -fdata-sections -fpic ${XQF_ARGS}") set(CMAKE_CXX_FLAGS_DEBUG "${COMMON_FLAGS} -O0 -D_DEBUG -g") set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${COMMON_FLAGS} -O2 -g") diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.c b/ggml/src/ggml-hexagon/htp/get-rows-ops.c index f51e00c15..f354b813a 100644 --- a/ggml/src/ggml-hexagon/htp/get-rows-ops.c +++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.c @@ -17,6 +17,7 @@ #include "htp-tensor.h" #include "hvx-utils.h" #include "hvx-quant.h" +#include "matmul-ops.h" #include "get-rows-ops.h" #include "work-queue.h" @@ -28,6 +29,9 @@ struct get_rows_context { uint32_t task_start; uint32_t tasks; uint32_t tasks_per_thread; + uint32_t tile_size; + uint32_t tile_stride; + bool index_i32; }; #define get_rows_preamble \ @@ -195,32 +199,191 @@ static void get_rows_thread_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsigned dma_queue_flush(dma_q); \ } -#define F32_BYTES(n) ((n) * sizeof(float)) #define F16_BYTES(n) ((n) * sizeof(__fp16)) #define Q8_0_BYTES(n) (((n) / 32) * sizeof(block_q8_0)) -GET_ROWS_THREAD_DT_FN(f32, F32_BYTES, int32_t, { if (cur_elems > 0) hvx_copy_f32_uu((uint8_t *)dst_spad, (const uint8_t *)src_spad, cur_elems); }) -GET_ROWS_THREAD_DT_FN(f32, F32_BYTES, int64_t, { if (cur_elems > 0) hvx_copy_f32_uu((uint8_t *)dst_spad, (const uint8_t *)src_spad, cur_elems); }) +static __attribute__((noinline)) void compute_get_rows_f16(float * dst_spad, const void * src_spad, uint32_t cur_elems) { + hvx_dequantize_row_f16_f32(dst_spad, src_spad, cur_elems); +} -GET_ROWS_THREAD_DT_FN(f16, F16_BYTES, int32_t, { hvx_dequantize_row_f16_f32((float *)dst_spad, src_spad, ne00); }) -GET_ROWS_THREAD_DT_FN(f16, F16_BYTES, int64_t, { hvx_dequantize_row_f16_f32((float *)dst_spad, src_spad, ne00); }) +static __attribute__((noinline)) void compute_get_rows_q8_0(float * dst_spad, const void * src_spad, uint32_t cur_elems) { + hvx_dequantize_row_q8_0_f32(dst_spad, src_spad, cur_elems); +} -GET_ROWS_THREAD_DT_FN(q8_0, Q8_0_BYTES, int32_t, { hvx_dequantize_row_q8_0_f32((float *)dst_spad, src_spad, ne00); }) -GET_ROWS_THREAD_DT_FN(q8_0, Q8_0_BYTES, int64_t, { hvx_dequantize_row_q8_0_f32((float *)dst_spad, src_spad, ne00); }) +GET_ROWS_THREAD_DT_FN(f16, F16_BYTES, int32_t, { compute_get_rows_f16((float *)dst_spad, src_spad, cur_elems); }) +GET_ROWS_THREAD_DT_FN(f16, F16_BYTES, int64_t, { compute_get_rows_f16((float *)dst_spad, src_spad, cur_elems); }) + +GET_ROWS_THREAD_DT_FN(q8_0, Q8_0_BYTES, int32_t, { compute_get_rows_q8_0((float *)dst_spad, src_spad, cur_elems); }) +GET_ROWS_THREAD_DT_FN(q8_0, Q8_0_BYTES, int64_t, { compute_get_rows_q8_0((float *)dst_spad, src_spad, cur_elems); }) + + +static __attribute__((noinline)) void compute_get_rows_tiled(float * dst, const uint8_t * tile, uint32_t row, bool q4) { + const HVX_VectorPred first2 = Q6_Q_vsetq_R(2); + const HVX_VectorPred first4 = Q6_Q_vsetq_R(4); + HVX_Vector vq = Q6_V_vzero(); + if (q4) { + const HVX_VectorPred first1 = Q6_Q_vsetq_R(1); + const HVX_VectorPred first3 = Q6_Q_vsetq_R(3); + for (int group = 3; group >= 0; --group) { + const HVX_Vector v = Q6_V_vror_VR(hvx_vmem(tile + group * VLEN), row); + // Four planes contribute bytes at 0, 32, 64 and 96 after rotation. + HVX_Vector packed = Q6_V_vmux_QVV(first1, v, Q6_V_vror_VR(v, 31)); + packed = Q6_V_vmux_QVV(first2, packed, Q6_V_vror_VR(v, 62)); + packed = Q6_V_vmux_QVV(first3, packed, Q6_V_vror_VR(v, 93)); + vq = Q6_V_vmux_QVV(first4, packed, Q6_V_vror_VR(vq, VLEN - 4)); + } + const HVX_Vector lo = Q6_V_vand_VV(vq, Q6_Vb_vsplat_R(0x0F)); + const HVX_Vector hi = Q6_Vub_vlsr_VubR(vq, 4); + vq = Q6_V_lo_W(Q6_W_vshuff_VVR(hi, lo, -1)); + vq = Q6_Vb_vsub_VbVb(vq, Q6_Vb_vsplat_R(8)); + } else { + for (int group = 7; group >= 0; --group) { + const HVX_Vector v = Q6_V_vror_VR(hvx_vmem(tile + group * VLEN), 2 * row); + // Two planes contribute halfwords at 0 and 64 after rotation. + const HVX_Vector packed = Q6_V_vmux_QVV(first2, v, Q6_V_vror_VR(v, 62)); + vq = Q6_V_vmux_QVV(first4, packed, Q6_V_vror_VR(vq, VLEN - 4)); + } + } + const HVX_Vector scales = hvx_vmem(tile + (q4 ? 512 : 1024)); + const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row)); + const HVX_Vector scale = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf)); + const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq); + const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16)); + const HVX_Vector values = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale); + *(HVX_Vector *) dst = values; +} + +struct get_rows_tiled_task { + dma_addr_t tile_src_base; + dma_addr_t dst_data; + uint32_t row; +}; + +static inline struct get_rows_tiled_task get_rows_tiled_calc_task( + const struct htp_ops_context * octx, + const struct get_rows_context * grctx, + uint32_t i, + uint32_t n_k_tiles, + uint32_t tile_size +) { + const struct htp_get_rows_kernel_params * kparams = grctx->kparams; + get_rows_preamble; + + const uint32_t i12 = fastdiv(i, &kparams->div_ne10_ne11); + const uint32_t rem = i - i12 * ne11 * ne10; + const uint32_t i11 = fastdiv(rem, &kparams->div_ne10); + const uint32_t i10 = rem - i11 * ne10; + const dma_addr_t src1_data = octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12; + const uint32_t i01 = grctx->index_i32 ? *(const int32_t *)(uintptr_t) src1_data : (uint32_t) *(const int64_t *)(uintptr_t) src1_data; + assert(i01 < ne01); + + const uint32_t q02 = fastdiv(i11, &kparams->div_ne02); + const uint32_t i02 = i11 - q02 * ne02; + const uint32_t q03 = fastdiv(i12, &kparams->div_ne03); + const uint32_t i03 = i12 - q03 * ne03; + const uint32_t column_tile = i01 / HTP_MM_HMX_TILE_N_ROWS; + const uint32_t row = i01 % HTP_MM_HMX_TILE_N_ROWS; + const dma_addr_t matrix = octx->src[0]->data + i02*nb02 + i03*nb03; + + struct get_rows_tiled_task task; + task.tile_src_base = matrix + (column_tile * n_k_tiles) * tile_size; + task.dst_data = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3; + task.row = row; + return task; +} + +static void get_rows_thread_tiled(unsigned int nth, unsigned int ith, void * data) { + struct get_rows_context * grctx = (struct get_rows_context *) data; + struct htp_ops_context * octx = grctx->octx; + const struct htp_get_rows_kernel_params * kparams = grctx->kparams; + get_rows_preamble; + + const uint32_t dr = grctx->tasks_per_thread; + const uint32_t ir0 = grctx->task_start + dr * ith; + if (ir0 >= grctx->task_start + grctx->tasks) { + return; + } + + const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks); + const uint32_t n_k_tiles = ne00 / HTP_MM_HMX_TILE_N_COLS; + const struct htp_get_rows_vtcm_layout * vtcm_layout = &grctx->vtcm_layout; + uint8_t * src_spad_base = grctx->vtcm_base + vtcm_layout->off_src0 + ith * vtcm_layout->src0_bytes_per_thread; + uint8_t * dst_spad_base = grctx->vtcm_base + vtcm_layout->off_dst + ith * vtcm_layout->dst_bytes_per_thread; + dma_queue * dma_q = octx->ctx->dma[ith]; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; + + const uint32_t tile_size = grctx->tile_size; + const uint32_t tile_stride = grctx->tile_stride; + const uint32_t dst_bytes = ne00 * sizeof(float); + const bool is_q4 = (octx->src[0]->type == HTP_TYPE_Q4_0); + + for (uint32_t step = 0, spad_idx = 0; step < ir1 - ir0 && spad_idx < 2; ++step, ++spad_idx) { + const uint32_t i = ir0 + step; + struct get_rows_tiled_task task = get_rows_tiled_calc_task(octx, grctx, i, n_k_tiles, tile_size); + + // Dummy writeback to prime the queue with dst descriptor + dma_queue_push(dma_q, + dma_make_data(task.dst_data, dst_spad_base + spad_idx * vtcm_layout->dst_spad_half_size), + dst_bytes, vtcm_layout->dst_spad_half_size, dst_bytes, 0); + + // Prefetch row tiles + dma_queue_push(dma_q, + dma_make_data(src_spad_base + spad_idx * vtcm_layout->src0_spad_half_size, task.tile_src_base), + tile_stride, tile_size, tile_size, n_k_tiles); + } + + for (uint32_t step = 0; step < ir1 - ir0; ++step) { + const uint32_t i = ir0 + step; + float * dst_spad = (float *) dma_queue_pop(dma_q).src; + uint8_t * src_spad = (uint8_t *) dma_queue_pop(dma_q).dst; + + struct get_rows_tiled_task task = get_rows_tiled_calc_task(octx, grctx, i, n_k_tiles, tile_size); + + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); + for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) { + const uint8_t * tile = src_spad + k_tile * tile_stride; + float * dst_block = dst_spad + k_tile * HTP_MM_HMX_TILE_N_COLS; + compute_get_rows_tiled(dst_block, tile, task.row, is_q4); + } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i); + + // Real writeback of dst_spad + dma_queue_push(dma_q, + dma_make_data(task.dst_data, dst_spad), + dst_bytes, vtcm_layout->dst_spad_half_size, dst_bytes, 1); + + const uint32_t next_step = step + 2; + if (next_step < ir1 - ir0) { + const uint32_t ni = ir0 + next_step; + struct get_rows_tiled_task next_task = get_rows_tiled_calc_task(octx, grctx, ni, n_k_tiles, tile_size); + dma_queue_push(dma_q, + dma_make_data(src_spad, next_task.tile_src_base), + tile_stride, tile_size, tile_size, n_k_tiles); + } + } + + dma_queue_flush(dma_q); +} int op_get_rows(struct htp_ops_context * octx) { const struct htp_get_rows_kernel_params * kparams = (const struct htp_get_rows_kernel_params *) octx->kernel_params; if (octx->src[0]->type != HTP_TYPE_F32 && - octx->src[0]->type != HTP_TYPE_F16 && - octx->src[0]->type != HTP_TYPE_Q8_0 && - octx->src[0]->type != HTP_TYPE_I32) { + octx->src[0]->type != HTP_TYPE_F16 && + octx->src[0]->type != HTP_TYPE_Q4_0 && + octx->src[0]->type != HTP_TYPE_Q8_0 && + octx->src[0]->type != HTP_TYPE_I32) { return HTP_STATUS_NO_SUPPORT; } - if ((octx->src[0]->type == HTP_TYPE_I32 && octx->dst->type != HTP_TYPE_I32) || - (octx->src[0]->type != HTP_TYPE_I32 && octx->dst->type != HTP_TYPE_F32)) { - return HTP_STATUS_NO_SUPPORT; + if (kparams->kernel_type == HTP_GET_ROWS_KERNEL_SAMETYPE) { + if (octx->src[0]->type != octx->dst->type) { + return HTP_STATUS_NO_SUPPORT; + } + } else { + if (octx->dst->type != HTP_TYPE_F32) { + return HTP_STATUS_NO_SUPPORT; + } } if (octx->src[1]->type != HTP_TYPE_I32 && octx->src[1]->type != HTP_TYPE_I64) { @@ -262,33 +425,48 @@ int op_get_rows(struct htp_ops_context * octx) { grctx.vtcm_base = (uint8_t *)octx->ctx->vtcm_base; grctx.task_start = task_start; grctx.tasks = tasks; - grctx.tasks_per_thread = fastdiv(tasks + n_threads - 1, &octx->n_threads_div); + grctx.tasks_per_thread = octx->ctx->mdev.count == 1 ? kparams->tasks_per_thread : fastdiv(tasks + n_threads - 1, &octx->n_threads_div); + grctx.tile_size = octx->src[0]->type == HTP_TYPE_Q4_0 ? HTP_MM_WEIGHT_TILE_SIZE_Q4_0 : HTP_MM_WEIGHT_TILE_SIZE_Q8_0; + grctx.tile_stride = (grctx.tile_size + 127) & ~127; + grctx.index_i32 = octx->src[1]->type == HTP_TYPE_I32; const uint32_t ne00 = octx->src[0]->ne[0]; - htp_get_rows_vtcm_layout_build(&grctx.vtcm_layout, octx->src[0]->type, ne00, n_threads); + htp_get_rows_vtcm_layout_build(&grctx.vtcm_layout, kparams->kernel_type, octx->src[0]->type, ne00, n_threads); + + if (grctx.vtcm_layout.total_bytes > octx->ctx->vtcm_size) { + FARF(ERROR, "get-rows: VTCM reservation %zu is too small, needed %zu\n", + octx->ctx->vtcm_size, grctx.vtcm_layout.total_bytes); + return HTP_STATUS_INVAL_PARAMS; + } const bool is_i32 = (octx->src[1]->type == HTP_TYPE_I32); work_queue_func_t q_func = NULL; - if (kparams->use_dma) { - q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_st_int32_t : get_rows_thread_st_int64_t); - } else { - switch (octx->src[0]->type) { - case HTP_TYPE_F32: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_f32_int32_t : get_rows_thread_f32_int64_t); break; - case HTP_TYPE_F16: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_f16_int32_t : get_rows_thread_f16_int64_t); break; - case HTP_TYPE_Q8_0: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_q8_0_int32_t : get_rows_thread_q8_0_int64_t); break; - case HTP_TYPE_I32: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_st_int32_t : get_rows_thread_st_int64_t); break; - default: return HTP_STATUS_NO_SUPPORT; - } + switch (kparams->kernel_type) { + case HTP_GET_ROWS_KERNEL_SAMETYPE: + q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_st_int32_t : get_rows_thread_st_int64_t); + break; + case HTP_GET_ROWS_KERNEL_TILED: + q_func = get_rows_thread_tiled; + break; + case HTP_GET_ROWS_KERNEL_FLAT: + switch (octx->src[0]->type) { + case HTP_TYPE_F16: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_f16_int32_t : get_rows_thread_f16_int64_t); break; + case HTP_TYPE_Q8_0: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_q8_0_int32_t : get_rows_thread_q8_0_int64_t); break; + default: return HTP_STATUS_NO_SUPPORT; + } + break; + default: + return HTP_STATUS_NO_SUPPORT; } - FARF(HIGH, "get-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu use-dma %d n-threads %d\n", + FARF(HIGH, "get-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu kernel-type %d n-threads %d\n", octx->src[0]->ne[0], octx->src[0]->ne[1], octx->src[0]->ne[2], octx->src[0]->ne[3], octx->src[1]->ne[0], octx->src[1]->ne[1], octx->src[1]->ne[2], octx->src[1]->ne[3], octx->dst->ne[0], octx->dst->ne[1], octx->dst->ne[2], octx->dst->ne[3], grctx.vtcm_layout.src0_bytes_per_thread * n_threads, grctx.vtcm_layout.dst_bytes_per_thread * n_threads, - kparams->use_dma, n_threads); + kparams->kernel_type, n_threads); work_queue_run(octx->ctx->work_queue, q_func, &grctx, n_threads); return HTP_STATUS_OK; diff --git a/ggml/src/ggml-hexagon/htp/get-rows-ops.h b/ggml/src/ggml-hexagon/htp/get-rows-ops.h index 0e7c2ca8c..06ca1ea74 100644 --- a/ggml/src/ggml-hexagon/htp/get-rows-ops.h +++ b/ggml/src/ggml-hexagon/htp/get-rows-ops.h @@ -1,11 +1,21 @@ #ifndef HTP_GET_ROWS_OPS_H #define HTP_GET_ROWS_OPS_H +#include +#include + #include "hex-fastdiv.h" +#include "matmul-ops.h" + +enum htp_get_rows_kernel_type { + HTP_GET_ROWS_KERNEL_SAMETYPE = 0, + HTP_GET_ROWS_KERNEL_TILED, + HTP_GET_ROWS_KERNEL_FLAT, +}; struct htp_get_rows_kernel_params { int32_t n_threads; - int32_t use_dma; + int32_t kernel_type; int32_t chunks_per_row; int32_t chunk_size; int32_t total_tasks; @@ -34,19 +44,37 @@ struct htp_get_rows_vtcm_layout { static inline void htp_get_rows_vtcm_layout_build( struct htp_get_rows_vtcm_layout * vtcm_layout, + int kernel_type, int type, uint32_t ne00, uint32_t n_threads) { + if (kernel_type == HTP_GET_ROWS_KERNEL_SAMETYPE) { + memset(vtcm_layout, 0, sizeof(*vtcm_layout)); + return; + } + + if (kernel_type == HTP_GET_ROWS_KERNEL_TILED) { + const size_t tile_size = type == HTP_TYPE_Q4_0 ? HTP_MM_WEIGHT_TILE_SIZE_Q4_0 : HTP_MM_WEIGHT_TILE_SIZE_Q8_0; + const size_t tile_stride = (tile_size + 127) & ~127; + const uint32_t n_k_tiles = ne00 / HTP_MM_HMX_TILE_N_COLS; + const size_t row_tiles_size = n_k_tiles > 0 ? (n_k_tiles * tile_stride) : tile_stride; + vtcm_layout->src0_spad_half_size = (row_tiles_size + 255) & ~255; + vtcm_layout->dst_spad_half_size = (ne00 * sizeof(float) + 255) & ~255; + vtcm_layout->src0_bytes_per_thread = 2 * vtcm_layout->src0_spad_half_size; + vtcm_layout->dst_bytes_per_thread = 2 * vtcm_layout->dst_spad_half_size; + vtcm_layout->off_src0 = 0; + vtcm_layout->off_dst = vtcm_layout->src0_bytes_per_thread * n_threads; + vtcm_layout->total_bytes = vtcm_layout->off_dst + vtcm_layout->dst_bytes_per_thread * n_threads; + return; + } + uint32_t src0_row_size = 0; switch (type) { - case 0: // HTP_TYPE_F32 - src0_row_size = ne00 * 4; - break; - case 1: // HTP_TYPE_F16 + case HTP_TYPE_F16: src0_row_size = ne00 * 2; break; - case 8: // HTP_TYPE_Q8_0 + case HTP_TYPE_Q8_0: src0_row_size = (ne00 / 32) * 34; break; default: