From b8051db82a66627589b391d12145929cee7ca2a2 Mon Sep 17 00:00:00 2001 From: cqderek Date: Wed, 30 Sep 2026 05:59:13 +0800 Subject: [PATCH] Hexagon f16 activation ops (llama/29209) * hexagon: add F16 support for activation ops (SILU/GELU/GELU_QUICK/GEGLU/SWIGLU) Widens ggml_hexagon_supported_activations() to accept F16 (src0/dst/src1 must agree on type), and adds F16 per-thread worker functions in act-ops.c mirroring the existing F32 workers, backed by new HVX f16 kernels (hvx_sigmoid_f16_aa, hvx_tanh_f16_aa, hvx_mul_mul_f16_aa, hvx_min_scalar_f16 family). SILU, GELU, GELU_QUICK, GEGLU, and SWIGLU are verified correct on-device (QRD8850) via test-backend-ops CPU-diffed correctness tests. SWIGLU_OAI's F16 path is code-complete and builds clean on host + all 4 DSP arch variants (v73/v75/v79/v81), but has no F16 test-case coverage in test-backend-ops and is therefore unverified on-device in this change. * hex-ops: align macros * hex-ops: minor formatting --------- Co-authored-by: Max Krasnyansky --- ggml/src/ggml-hexagon/ggml-hexagon.cpp | 16 +- ggml/src/ggml-hexagon/htp/act-ops.c | 509 +++++++++++++++++++----- ggml/src/ggml-hexagon/htp/hvx-arith.h | 71 ++++ ggml/src/ggml-hexagon/htp/hvx-sigmoid.h | 54 +++ 4 files changed, 543 insertions(+), 107 deletions(-) diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index 161eee248..4b913947d 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -6037,10 +6037,10 @@ static bool ggml_hexagon_supported_activations(const struct ggml_hexagon_session const struct ggml_tensor * src1 = op->src[1]; const struct ggml_tensor * dst = op; - if (src0->type != GGML_TYPE_F32) { + if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) { return false; } - if (dst->type != GGML_TYPE_F32) { + if (dst->type != src0->type) { return false; } @@ -6052,7 +6052,7 @@ static bool ggml_hexagon_supported_activations(const struct ggml_hexagon_session } if (src1) { - if (src1->type != GGML_TYPE_F32) { + if (src1->type != src0->type) { return false; } if (!ggml_are_same_shape(src0, src1)) { @@ -6637,12 +6637,12 @@ static htp_op_code op_remap_to_htp(const ggml_tensor * t) { case GGML_OP_GLU: switch (ggml_get_glu_op(t)) { - case GGML_GLU_OP_SWIGLU: return HTP_OP_GLU_SWIGLU; - case GGML_GLU_OP_SWIGLU_OAI: return HTP_OP_GLU_SWIGLU_OAI; + case GGML_GLU_OP_SWIGLU: return HTP_OP_GLU_SWIGLU; + case GGML_GLU_OP_SWIGLU_OAI: return HTP_OP_GLU_SWIGLU_OAI; case GGML_GLU_OP_SWIGLU_CLAMP: return HTP_OP_GLU_SWIGLU_CLAMP; - case GGML_GLU_OP_GEGLU: return HTP_OP_GLU_GEGLU; - case GGML_GLU_OP_GEGLU_QUICK: return HTP_OP_GLU_GEGLU_QUICK; - case GGML_GLU_OP_GEGLU_ERF: return HTP_OP_GLU_GEGLU_ERF; + case GGML_GLU_OP_GEGLU: return HTP_OP_GLU_GEGLU; + case GGML_GLU_OP_GEGLU_QUICK: return HTP_OP_GLU_GEGLU_QUICK; + case GGML_GLU_OP_GEGLU_ERF: return HTP_OP_GLU_GEGLU_ERF; default: break; } break; diff --git a/ggml/src/ggml-hexagon/htp/act-ops.c b/ggml/src/ggml-hexagon/htp/act-ops.c index 28c581df4..59a0575c4 100644 --- a/ggml/src/ggml-hexagon/htp/act-ops.c +++ b/ggml/src/ggml-hexagon/htp/act-ops.c @@ -55,16 +55,22 @@ struct htp_act_context; -typedef void (*glu_compute_fn_t)(const float * restrict src0, - const float * restrict src1, - float * restrict dst, - const uint32_t num_rows, - const struct htp_act_context * actx); +typedef void (*glu_compute_f32_fn_t)(const float * restrict src0, + const float * restrict src1, + float * restrict dst, + const uint32_t num_rows, + const struct htp_act_context * actx); + +typedef void (*glu_compute_f16_fn_t)(const _Float16 * restrict src0, + const _Float16 * restrict src1, + _Float16 * restrict dst, + const uint32_t num_rows, + const struct htp_act_context * actx); struct htp_act_context { struct htp_ops_context * octx; - glu_compute_fn_t compute; + void * compute; const char * op_str; // Precomputed values @@ -392,6 +398,58 @@ static inline void hvx_geglu_erf_f32_aa(uint8_t * restrict dst, const uint8_t * } } +static inline void hvx_geglu_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src0 % 128 == 0); + assert((unsigned long) src1 % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + const HVX_Vector * restrict vsrc0 = (const HVX_Vector *) src0; + const HVX_Vector * restrict vsrc1 = (const HVX_Vector *) src1; + + const uint32_t epv = 128 / sizeof(_Float16); + const uint32_t nvec = n / epv; + const uint32_t nloe = n % epv; + + const HVX_Vector v_coef_a = hvx_vec_splat_f16((_Float16) GELU_COEF_A); + const HVX_Vector v_sqrt_2_pi = hvx_vec_splat_f16((_Float16) SQRT_2_OVER_PI); + const HVX_Vector v_one = hvx_vec_splat_f16((_Float16) 1.0f); + const HVX_Vector v_half = hvx_vec_splat_f16((_Float16) 0.5f); + + uint32_t i = 0; + + _Pragma("unroll(4)") + for (; i < nvec; i++) { + HVX_Vector x = vsrc0[i]; + HVX_Vector g = vsrc1[i]; + + // gelu(x) = 0.5*x*(1 + tanh(SQRT_2_OVER_PI*x*(1 + GELU_COEF_A*x*x))) + HVX_Vector x2 = hvx_vec_mul_f16_f16(x, x); + HVX_Vector coef = hvx_vec_add_f16_f16(hvx_vec_mul_f16_f16(x2, v_coef_a), v_one); + HVX_Vector inner = hvx_vec_mul_f16_f16(hvx_vec_mul_f16_f16(x, coef), v_sqrt_2_pi); + + HVX_Vector t = hvx_vec_add_f16_f16(hvx_vec_tanh_f16(inner), v_one); + + HVX_Vector gelu_x = hvx_vec_mul_f16_f16(hvx_vec_mul_f16_f16(x, t), v_half); + vdst[i] = hvx_vec_mul_f16_f16(gelu_x, g); + } + + if (nloe) { + HVX_Vector x = vsrc0[i]; + HVX_Vector g = vsrc1[i]; + + HVX_Vector x2 = hvx_vec_mul_f16_f16(x, x); + HVX_Vector coef = hvx_vec_add_f16_f16(hvx_vec_mul_f16_f16(x2, v_coef_a), v_one); + HVX_Vector inner = hvx_vec_mul_f16_f16(hvx_vec_mul_f16_f16(x, coef), v_sqrt_2_pi); + + HVX_Vector t = hvx_vec_add_f16_f16(hvx_vec_tanh_f16(inner), v_one); + + HVX_Vector gelu_x = hvx_vec_mul_f16_f16(hvx_vec_mul_f16_f16(x, t), v_half); + HVX_Vector res = hvx_vec_mul_f16_f16(gelu_x, g); + hvx_vec_store_a((void *) &vdst[i], nloe * sizeof(_Float16), res); + } +} + // geglu(x, g) = gelu(x) * g static void geglu_f32(const float * restrict src0, const float * restrict src1, @@ -443,101 +501,190 @@ static void geglu_erf_f32(const float * restrict src0, } } -static void glu_f32_per_thread(unsigned int nth, unsigned int ith, void * data) { - struct htp_act_context * actx = (struct htp_act_context *) data; - htp_act_preamble; +// swiglu(x) = x1 * sigmoid(x0) +static void swiglu_f16(const _Float16 * restrict src0, + const _Float16 * restrict src1, + _Float16 * restrict dst, + const uint32_t num_rows, + const struct htp_act_context * actx) { + htp_glu_op_preamble; - struct htp_thread_trace * tr = actx->octx->ctx ? &actx->octx->ctx->trace[ith] : NULL; + for (uint32_t ib = 0; ib < num_rows; ib++) { + const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned); + const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned); + uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned); - size_t src0_row_size = actx->src0_row_size; - size_t src1_row_size = actx->src1_row_size; - size_t dst_row_size = actx->dst_row_size; - - size_t src0_row_stride = actx->src0_row_stride; - size_t src1_row_stride = actx->src1_row_stride; - - const uint32_t src0_nrows = actx->src0_nrows; - const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread; - - const uint32_t src0_start_row = actx->row_start + src0_nrows_per_thread * ith; - const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, actx->row_start + src0_nrows); - - /* no work for this thread */ - if (src0_start_row >= src0_end_row) { - return; + hvx_sigmoid_f16_aa(dst_ptr, src0_ptr, nc); + hvx_mul_mul_f16_aa(dst_ptr, src0_ptr, dst_ptr, src1_ptr, nc); } - - const dma_addr_t data_src0 = actx->data_src0; - const dma_addr_t data_src1 = actx->data_src1; - const dma_addr_t data_dst = actx->data_dst; - - const size_t src0_row_size_aligned = actx->src0_row_size_aligned; - const size_t src1_row_size_aligned = actx->src1_row_size_aligned; - const size_t dst_row_size_aligned = actx->dst_row_size_aligned; - - uint8_t * restrict src0_spad_data = actx->vtcm_src0 + (ith * actx->vtcm_src0_size_per_thread); - uint8_t * restrict src1_spad_data = actx->vtcm_src1 + (ith * actx->vtcm_src1_size_per_thread); - uint8_t * restrict dst_spad_data = actx->vtcm_dst + (ith * actx->vtcm_dst_size_per_thread); - - size_t src0_spad_half_size = actx->src0_spad_half_size; - size_t src1_spad_half_size = actx->src1_spad_half_size; - size_t dst_spad_half_size = actx->dst_spad_half_size; - - const int BLOCK = actx->block; - if (BLOCK == 0) { - FARF(ERROR, "%s : VTCM reservation %zu is too small, needed %zu\n", - actx->op_str, actx->vtcm_src0_size_per_thread, src0_row_size_aligned); - return; - } - - dma_queue * dma_q = actx->octx->ctx->dma[ith]; - glu_compute_fn_t compute = actx->compute; - - for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) { - const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); - - /* Dummy DMA transation for sequencing (interleaving dst,src,dst,...) */ - dma_queue_push(dma_q, - dma_make_data(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)), - dst_row_size, dst_row_size_aligned, dst_row_size, 0); - - dma_queue_push(dma_q, - dma_make_data(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_stride)), - src0_row_size_aligned, src0_row_stride, src0_row_size, block_size); - - dma_queue_push(dma_q, - dma_make_data(src1_spad_data + (spad_idx * src1_spad_half_size), data_src1 + (ir * src1_row_stride)), - src1_row_size_aligned, src1_row_stride, src1_row_size, block_size); - } - - for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) { - const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); - - float * dst_spad = (float *) dma_queue_pop(dma_q).src; - float * src0_spad = (float *) dma_queue_pop(dma_q).dst; - float * src1_spad = (float *) dma_queue_pop(dma_q).dst; - - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir); - compute(src0_spad, src1_spad, dst_spad, block_size, actx); - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir); - - dma_queue_push(dma_q, dma_make_data(data_dst + (ir * dst_row_size), dst_spad), - dst_row_size, dst_row_size_aligned, dst_row_size, block_size); - - /* prefetch N+2 loop iteration if any */ - const uint32_t pref_block = (ir + BLOCK * 2); - if (pref_block < src0_end_row) { - const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block); - dma_queue_push(dma_q, dma_make_data(src0_spad, data_src0 + (pref_block * src0_row_stride)), - src0_row_size_aligned, src0_row_stride, src0_row_size, pref_block_size); - dma_queue_push(dma_q, dma_make_data(src1_spad, data_src1 + (pref_block * src1_row_stride)), - src1_row_size_aligned, src1_row_stride, src1_row_size, pref_block_size); - } - } - - dma_queue_flush(dma_q); } +// out = x * sigmoid(alpha * x) * (clamp(y, -limit, limit) + 1.f) +static void swiglu_oai_f16(const _Float16 * restrict src0, + const _Float16 * restrict src1, + _Float16 * restrict dst, + const uint32_t num_rows, + const struct htp_act_context * actx) { + htp_glu_op_preamble; + const _Float16 alpha = (_Float16) ((const float *) (actx->octx->op_params))[2]; + const _Float16 limit = (_Float16) ((const float *) (actx->octx->op_params))[3]; + + for (uint32_t ib = 0; ib < num_rows; ib++) { + const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned); + const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned); + uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned); + + // x (src0_ptr) = std::min(src0_p[k], limit); + hvx_min_scalar_f16((uint8_t *) src0_ptr, src0_ptr, limit, nc); + // y1 (src1_ptr) = std::clamp(src1_p[k], -limit, limit); + hvx_clamp_scalar_f16((uint8_t *) src1_ptr, src1_ptr, -limit, limit, nc); + // y (src1_ptr) = y1 + 1.f + hvx_add_scalar_f16((uint8_t *) src1_ptr, src1_ptr, (_Float16) 1.0f, nc); + // x1 (dst_ptr) = alpha * x + hvx_mul_scalar_f16(dst_ptr, src0_ptr, alpha, nc); + // x2 (dst_ptr) = sigmoid(x1) = 1/(1+exp(-x1)) + hvx_sigmoid_f16_aa(dst_ptr, dst_ptr, nc); + // out = x * sigmoid(alpha * x) * (y + 1.f) + hvx_mul_mul_f16_aa(dst_ptr, src0_ptr, dst_ptr, src1_ptr, nc); + } +} + +static void swiglu_clamp_f16(const _Float16 * restrict src0, + const _Float16 * restrict src1, + _Float16 * restrict dst, + const uint32_t num_rows, + const struct htp_act_context * actx) { + htp_glu_op_preamble; + const _Float16 limit = (_Float16) ((const float *) (actx->octx->op_params))[3]; + + for (uint32_t ib = 0; ib < num_rows; ib++) { + const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned); + const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned); + uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned); + + hvx_min_scalar_f16((uint8_t *) src0_ptr, src0_ptr, limit, nc); + hvx_clamp_scalar_f16((uint8_t *) src1_ptr, src1_ptr, -limit, limit, nc); + hvx_sigmoid_f16_aa(dst_ptr, src0_ptr, nc); + hvx_mul_mul_f16_aa(dst_ptr, src0_ptr, dst_ptr, src1_ptr, nc); + } +} + +// geglu(x, g) = gelu(x) * g, gelu via tanh approximation +static void geglu_f16(const _Float16 * restrict src0, + const _Float16 * restrict src1, + _Float16 * restrict dst, + const uint32_t num_rows, + const struct htp_act_context * actx) { + htp_glu_op_preamble; + + for (uint32_t ib = 0; ib < num_rows; ib++) { + const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned); + const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned); + uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned); + + hvx_geglu_f16_aa(dst_ptr, src0_ptr, src1_ptr, nc); + } +} + +#define DEFINE_GLU_PER_THREAD(TYPE, SUFFIX) \ + static void glu_##SUFFIX##_per_thread(unsigned int nth, unsigned int ith, void * data) { \ + struct htp_act_context * actx = (struct htp_act_context *) data; \ + htp_act_preamble; \ + \ + struct htp_thread_trace * tr = actx->octx->ctx ? &actx->octx->ctx->trace[ith] : NULL; \ + \ + size_t src0_row_size = actx->src0_row_size; \ + size_t src1_row_size = actx->src1_row_size; \ + size_t dst_row_size = actx->dst_row_size; \ + \ + size_t src0_row_stride = actx->src0_row_stride; \ + size_t src1_row_stride = actx->src1_row_stride; \ + \ + const uint32_t src0_nrows = actx->src0_nrows; \ + const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread; \ + \ + const uint32_t src0_start_row = actx->row_start + src0_nrows_per_thread * ith; \ + const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, actx->row_start + src0_nrows); \ + \ + /* no work for this thread */ \ + if (src0_start_row >= src0_end_row) { \ + return; \ + } \ + \ + const dma_addr_t data_src0 = actx->data_src0; \ + const dma_addr_t data_src1 = actx->data_src1; \ + const dma_addr_t data_dst = actx->data_dst; \ + \ + const size_t src0_row_size_aligned = actx->src0_row_size_aligned; \ + const size_t src1_row_size_aligned = actx->src1_row_size_aligned; \ + const size_t dst_row_size_aligned = actx->dst_row_size_aligned; \ + \ + uint8_t * restrict src0_spad_data = actx->vtcm_src0 + (ith * actx->vtcm_src0_size_per_thread); \ + uint8_t * restrict src1_spad_data = actx->vtcm_src1 + (ith * actx->vtcm_src1_size_per_thread); \ + uint8_t * restrict dst_spad_data = actx->vtcm_dst + (ith * actx->vtcm_dst_size_per_thread); \ + \ + size_t src0_spad_half_size = actx->src0_spad_half_size; \ + size_t src1_spad_half_size = actx->src1_spad_half_size; \ + size_t dst_spad_half_size = actx->dst_spad_half_size; \ + \ + const int BLOCK = actx->block; \ + if (BLOCK == 0) { \ + FARF(ERROR, "%s : VTCM reservation %zu is too small, needed %zu\n", \ + actx->op_str, actx->vtcm_src0_size_per_thread, src0_row_size_aligned); \ + return; \ + } \ + \ + dma_queue * dma_q = actx->octx->ctx->dma[ith]; \ + glu_compute_##SUFFIX##_fn_t compute = (glu_compute_##SUFFIX##_fn_t) actx->compute; \ + \ + for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) { \ + const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); \ + \ + /* Dummy DMA transation for sequencing (interleaving dst,src,dst,...) */ \ + dma_queue_push(dma_q, \ + dma_make_data(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)), \ + dst_row_size, dst_row_size_aligned, dst_row_size, 0); \ + \ + dma_queue_push(dma_q, \ + dma_make_data(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_stride)), \ + src0_row_size_aligned, src0_row_stride, src0_row_size, block_size); \ + \ + dma_queue_push(dma_q, \ + dma_make_data(src1_spad_data + (spad_idx * src1_spad_half_size), data_src1 + (ir * src1_row_stride)), \ + src1_row_size_aligned, src1_row_stride, src1_row_size, block_size); \ + } \ + \ + for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) { \ + const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); \ + \ + TYPE * dst_spad = (TYPE *) dma_queue_pop(dma_q).src; \ + TYPE * src0_spad = (TYPE *) dma_queue_pop(dma_q).dst; \ + TYPE * src1_spad = (TYPE *) dma_queue_pop(dma_q).dst; \ + \ + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir); \ + compute(src0_spad, src1_spad, dst_spad, block_size, actx); \ + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir); \ + \ + dma_queue_push(dma_q, dma_make_data(data_dst + (ir * dst_row_size), dst_spad), \ + dst_row_size, dst_row_size_aligned, dst_row_size, block_size); \ + \ + /* prefetch N+2 loop iteration if any */ \ + const uint32_t pref_block = (ir + BLOCK * 2); \ + if (pref_block < src0_end_row) { \ + const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block); \ + dma_queue_push(dma_q, dma_make_data(src0_spad, data_src0 + (pref_block * src0_row_stride)), \ + src0_row_size_aligned, src0_row_stride, src0_row_size, pref_block_size); \ + dma_queue_push(dma_q, dma_make_data(src1_spad, data_src1 + (pref_block * src1_row_stride)), \ + src1_row_size_aligned, src1_row_stride, src1_row_size, pref_block_size); \ + } \ + } \ + \ + dma_queue_flush(dma_q); \ + } + +DEFINE_GLU_PER_THREAD(float, f32) +DEFINE_GLU_PER_THREAD(_Float16, f16) + static int execute_op_activations_f32(struct htp_ops_context * octx) { const struct htp_tensor * src0 = octx->src[0]; const struct htp_tensor * src1 = octx->src[1]; @@ -548,8 +695,8 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - glu_compute_fn_t compute_fn = NULL; - const char * op_type = NULL; + glu_compute_f32_fn_t compute_fn = NULL; + const char * op_type = NULL; switch (octx->op) { case HTP_OP_GLU_SWIGLU: @@ -581,6 +728,7 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) { compute_fn = geglu_erf_f32; op_type = "geglu-erf-f32"; break; + default: FARF(ERROR, "Unsupported activations Op %u\n", octx->op); return HTP_STATUS_NO_SUPPORT; @@ -643,7 +791,7 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) { // Prepare context struct htp_act_context actx; actx.octx = octx; - actx.compute = compute_fn; + actx.compute = (void *) compute_fn; actx.op_str = op_type; actx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); @@ -708,11 +856,174 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) { return HTP_STATUS_OK; } +static int execute_op_activations_f16(struct htp_ops_context * octx) { + const struct htp_tensor * src0 = octx->src[0]; + const struct htp_tensor * src1 = octx->src[1]; + const struct htp_tensor * dst = octx->dst; + + if ((dst->ne[0] * SIZEOF_FP16) != dst->nb[1]) { + FARF(ERROR, "Non-contiguous dst is not supported at this time \n"); + return HTP_STATUS_NO_SUPPORT; + } + + glu_compute_f16_fn_t compute_fn = NULL; + const char * op_type = NULL; + + switch (octx->op) { + case HTP_OP_GLU_SWIGLU: + compute_fn = swiglu_f16; + op_type = "swiglu-f16"; + break; + + case HTP_OP_GLU_SWIGLU_OAI: + compute_fn = swiglu_oai_f16; + op_type = "swiglu-oai-f16"; + break; + + case HTP_OP_GLU_SWIGLU_CLAMP: + compute_fn = swiglu_clamp_f16; + op_type = "swiglu-clamp-f16"; + break; + + case HTP_OP_GLU_GEGLU: + compute_fn = geglu_f16; + op_type = "geglu-f16"; + break; + + default: + FARF(ERROR, "Unsupported activations Op %u\n", octx->op); + return HTP_STATUS_NO_SUPPORT; + } + + const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const size_t dst_row_size = dst->ne[0] * SIZEOF_FP16; + + uint32_t row_start = 0; + uint32_t nrows = src0_nrows; + + if (octx->ctx->mdev.count > 1) { + uint32_t rows_per_chunk = 0; + htp_tensor_mdev_rows_per_chunk(dst, sizeof(_Float16), (uint32_t) dst_row_size, &rows_per_chunk); + const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div); + row_start = range.start; + nrows = range.count; + } + + if (nrows == 0) { + return HTP_STATUS_OK; + } + + const uint32_t n_threads = octx->n_threads; + + // row_size = bytes of useful data per row (what the kernel touches / what DMA copies). + // row_stride = bytes between successive rows in DDR (may exceed row_size for non-contig src). + const size_t nc_bytes = dst_row_size; + const size_t src0_row_size = nc_bytes; + const size_t src1_row_size = nc_bytes; + const size_t src0_row_stride = src0->nb[1]; + const size_t src1_row_stride = src1 ? src1->nb[1] : src0->nb[1]; + + const size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN); + const size_t src1_row_size_aligned = hex_round_up(src1_row_size, VLEN); + const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN); + + struct htp_act_vtcm_layout L; + htp_act_vtcm_layout_build(&L, src0_row_size_aligned, src1_row_size_aligned, dst_row_size_aligned, n_threads, + octx->ctx->vtcm_size); + + // Make sure the reserved vtcm size is sufficient + if (L.vtcm_row_per_thread == 0) { + FARF(ERROR, "act-%s : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", op_type, octx->ctx->vtcm_size, + (src0_row_size_aligned + src1_row_size_aligned + dst_row_size_aligned) * n_threads); + return HTP_STATUS_VTCM_TOO_SMALL; + } + + if (src1) { + FARF(HIGH, "%s: %ux%ux%ux%u x %ux%ux%ux%u -> %ux%ux%ux%u : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n", + op_type, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2], + src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], L.src0_bytes_per_thread * n_threads, + L.src1_bytes_per_thread * n_threads, L.dst_bytes_per_thread * n_threads); + } else { + FARF(HIGH, "%s: %ux%ux%ux%u -> %ux%ux%ux%u : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n", op_type, + src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], + L.src0_bytes_per_thread * n_threads, L.src1_bytes_per_thread * n_threads, L.dst_bytes_per_thread * n_threads); + } + + // Prepare context + struct htp_act_context actx; + actx.octx = octx; + actx.compute = (void *) compute_fn; + actx.op_str = op_type; + + actx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div); + + actx.src0_row_size = src0_row_size; + actx.src1_row_size = src1_row_size; + actx.dst_row_size = dst_row_size; + + actx.src0_row_size_aligned = src0_row_size_aligned; + actx.src1_row_size_aligned = src1_row_size_aligned; + actx.dst_row_size_aligned = dst_row_size_aligned; + + actx.src0_row_stride = src0_row_stride; + actx.src1_row_stride = src1_row_stride; + + uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base; + actx.vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0); + actx.vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1); + actx.vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst); + + actx.vtcm_src0_size_per_thread = L.src0_bytes_per_thread; + actx.vtcm_src1_size_per_thread = L.src1_bytes_per_thread; + actx.vtcm_dst_size_per_thread = L.dst_bytes_per_thread; + + actx.src0_spad_half_size = L.src0_bytes_per_thread / 2; + actx.src1_spad_half_size = L.src1_bytes_per_thread / 2; + actx.dst_spad_half_size = L.dst_bytes_per_thread / 2; + + actx.block = actx.src0_spad_half_size / actx.src0_row_size_aligned; + actx.src0_nrows = nrows; + actx.row_start = row_start; + + actx.nc = dst->ne[0]; + + // Addresses and GLU logic + dma_addr_t data_src0 = src0->data; + dma_addr_t data_src1 = src1 ? src1->data : 0; + + if (!src1 && (octx->op == HTP_OP_GLU_SWIGLU || + octx->op == HTP_OP_GLU_SWIGLU_OAI || + octx->op == HTP_OP_GLU_SWIGLU_CLAMP || + octx->op == HTP_OP_GLU_GEGLU)) { + const int32_t swapped = octx->op_params[1]; + data_src1 = data_src0; + actx.src1_row_size = actx.src0_row_size; + + size_t nc_in_bytes = actx.nc * SIZEOF_FP16; + if (swapped) { + data_src0 += nc_in_bytes; + } else { + data_src1 += nc_in_bytes; + } + } + + actx.data_src0 = data_src0; + actx.data_src1 = data_src1; + actx.data_dst = dst->data; + + work_queue_run(octx->ctx->work_queue, (worker_callback_t)glu_f16_per_thread, &actx, n_threads); + return HTP_STATUS_OK; +} + + int op_activations(struct htp_ops_context * octx) { switch (octx->src[0]->type) { case HTP_TYPE_F32: return execute_op_activations_f32(octx); + case HTP_TYPE_F16: + return execute_op_activations_f16(octx); + default: return HTP_STATUS_NO_SUPPORT; } diff --git a/ggml/src/ggml-hexagon/htp/hvx-arith.h b/ggml/src/ggml-hexagon/htp/hvx-arith.h index c9a776f62..ab2064af3 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-arith.h +++ b/ggml/src/ggml-hexagon/htp/hvx-arith.h @@ -164,6 +164,37 @@ static inline void hvx_mul_mul_f32_aa(uint8_t * restrict dst, const uint8_t * re } } +static inline void hvx_mul_mul_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, const uint8_t * restrict src2, const uint32_t num_elems) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src0 % 128 == 0); + assert((unsigned long) src1 % 128 == 0); + assert((unsigned long) src2 % 128 == 0); + + HVX_Vector * restrict vdst = (HVX_Vector *) dst; + HVX_Vector * restrict vsrc0 = (HVX_Vector *) src0; + HVX_Vector * restrict vsrc1 = (HVX_Vector *) src1; + HVX_Vector * restrict vsrc2 = (HVX_Vector *) src2; + + const uint32_t elem_size = sizeof(_Float16); + const uint32_t epv = 128 / elem_size; + const uint32_t nvec = num_elems / epv; + const uint32_t nloe = num_elems % epv; + + uint32_t i = 0; + + _Pragma("unroll(4)") + for (; i < nvec; i++) { + HVX_Vector v1 = HVX_OP_MUL_F16(vsrc0[i], vsrc1[i]); + vdst[i] = HVX_OP_MUL_F16(v1, vsrc2[i]); + } + + if (nloe) { + HVX_Vector v1 = HVX_OP_MUL_F16(vsrc0[i], vsrc1[i]); + HVX_Vector v2 = HVX_OP_MUL_F16(v1, vsrc2[i]); + hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v2); + } +} + // Scalar Operations #define hvx_scalar_loop_body(dst_type, src_type, elem_size, vec_store, scalar_op_macro) \ @@ -348,6 +379,44 @@ static inline void hvx_max_scalar_f32(uint8_t * restrict dst, const uint8_t * re } } +#define HVX_OP_MIN_SCALAR_F16(v) Q6_Vhf_vmin_VhfVhf(val_vec, v) + +static inline void hvx_min_scalar_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, uint32_t n) { + const HVX_Vector val_vec = hvx_vec_splat_f16(val); + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + hvx_scalar_loop_body(HVX_Vector, HVX_Vector, sizeof(_Float16), hvx_vec_store_a, HVX_OP_MIN_SCALAR_F16); +} + +static inline void hvx_min_scalar_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, uint32_t n) { + const HVX_Vector val_vec = hvx_vec_splat_f16(val); + assert((unsigned long) dst % 128 == 0); + hvx_scalar_loop_body(HVX_Vector, HVX_UVector, sizeof(_Float16), hvx_vec_store_a, HVX_OP_MIN_SCALAR_F16); +} + +static inline void hvx_min_scalar_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, uint32_t n) { + const HVX_Vector val_vec = hvx_vec_splat_f16(val); + assert((unsigned long) src % 128 == 0); + hvx_scalar_loop_body(HVX_UVector, HVX_Vector, sizeof(_Float16), hvx_vec_store_u, HVX_OP_MIN_SCALAR_F16); +} + +static inline void hvx_min_scalar_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, uint32_t n) { + const HVX_Vector val_vec = hvx_vec_splat_f16(val); + hvx_scalar_loop_body(HVX_UVector, HVX_UVector, sizeof(_Float16), hvx_vec_store_u, HVX_OP_MIN_SCALAR_F16); +} + +static inline void hvx_min_scalar_f16(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, const int num_elems) { + if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) { + hvx_min_scalar_f16_aa(dst, src, val, num_elems); + } else if (hex_is_aligned((void *) dst, 128)) { + hvx_min_scalar_f16_au(dst, src, val, num_elems); + } else if (hex_is_aligned((void *) src, 128)) { + hvx_min_scalar_f16_ua(dst, src, val, num_elems); + } else { + hvx_min_scalar_f16_uu(dst, src, val, num_elems); + } +} + // CLAMP Scalar variants #define HVX_OP_CLAMP_SCALAR(v) \ @@ -767,10 +836,12 @@ static inline void hvx_sqr_f16(uint8_t * restrict dst, const uint8_t * restrict #undef hvx_scalar_loop_body #undef HVX_OP_MIN_SCALAR #undef HVX_OP_MAX_SCALAR +#undef HVX_OP_MIN_SCALAR_F16 #undef HVX_OP_CLAMP_SCALAR #undef HVX_OP_CLAMP_SCALAR_F16 #undef HVX_OP_LEAKY_RELU_SCALAR #undef DEFINE_HVX_BINARY_OP_VARIANTS +#undef DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS #undef HVX_BINARY_DISPATCHER #undef UNUSED diff --git a/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h b/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h index 552017309..b7c2d7409 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h +++ b/ggml/src/ggml-hexagon/htp/hvx-sigmoid.h @@ -178,4 +178,58 @@ static inline HVX_Vector hvx_vec_tanh_f16(HVX_Vector x) { return hvx_vec_add_f16_f16(hvx_vec_mul_f16_f16(sig2x, v_two), v_neg_one); } +#define hvx_sigmoid_f16_loop_body(dst_type, src_type, vec_store) \ + do { \ + dst_type * restrict vdst = (dst_type *) dst; \ + src_type * restrict vsrc = (src_type *) src; \ + \ + const uint32_t epv = 128 / sizeof(_Float16); \ + const uint32_t nvec = n / epv; \ + const uint32_t nloe = n % epv; \ + \ + uint32_t i = 0; \ + \ + _Pragma("unroll(4)") \ + for (; i < nvec; i++) { \ + vdst[i] = hvx_vec_fast_sigmoid_f16(vsrc[i]); \ + } \ + if (nloe) { \ + HVX_Vector tmp = hvx_vec_fast_sigmoid_f16(vsrc[i]); \ + vec_store((void *) &vdst[i], nloe * sizeof(_Float16), tmp); \ + } \ + } while(0) + +#define hvx_tanh_f16_loop_body(dst_type, src_type, vec_store) \ + do { \ + dst_type * restrict vdst = (dst_type *) dst; \ + src_type * restrict vsrc = (src_type *) src; \ + \ + const uint32_t epv = 128 / sizeof(_Float16); \ + const uint32_t nvec = n / epv; \ + const uint32_t nloe = n % epv; \ + \ + uint32_t i = 0; \ + \ + _Pragma("unroll(4)") \ + for (; i < nvec; i++) { \ + vdst[i] = hvx_vec_tanh_f16(vsrc[i]); \ + } \ + if (nloe) { \ + HVX_Vector tmp = hvx_vec_tanh_f16(vsrc[i]); \ + vec_store((void *) &vdst[i], nloe * sizeof(_Float16), tmp); \ + } \ + } while(0) + +static inline void hvx_sigmoid_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + hvx_sigmoid_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a); +} + +static inline void hvx_tanh_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) { + assert((unsigned long) dst % 128 == 0); + assert((unsigned long) src % 128 == 0); + hvx_tanh_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a); +} + #endif /* HVX_SIGMOID_H */