public_git/whisper.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/whisper.cpp.git synced 2026-10-12 17:25:39 +02:00
Code Issues Packages Projects Releases Wiki Activity

Hexagon f16 activation ops (llama/29209)

Browse Source 
* hexagon: add F16 support for activation ops (SILU/GELU/GELU_QUICK/GEGLU/SWIGLU)

Widens ggml_hexagon_supported_activations() to accept F16 (src0/dst/src1
must agree on type), and adds F16 per-thread worker functions in
act-ops.c mirroring the existing F32 workers, backed by new HVX f16
kernels (hvx_sigmoid_f16_aa, hvx_tanh_f16_aa, hvx_mul_mul_f16_aa,
hvx_min_scalar_f16 family).

SILU, GELU, GELU_QUICK, GEGLU, and SWIGLU are verified correct on-device
(QRD8850) via test-backend-ops CPU-diffed correctness tests. SWIGLU_OAI's
F16 path is code-complete and builds clean on host + all 4 DSP arch
variants (v73/v75/v79/v81), but has no F16 test-case coverage in
test-backend-ops and is therefore unverified on-device in this change.

* hex-ops: align macros

* hex-ops: minor formatting

---------

Co-authored-by: Max Krasnyansky <maxk@qti.qualcomm.com>
This commit is contained in:
cqderek
2026-10-06 10:38:11 +03:00
committed by Georgi Gerganov
co-authored by Max Krasnyansky
parent 6e48a365f3
commit b8051db82a
4 changed files with 543 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/src/ggml-hexagon/ggml-hexagon.cpp
+8 -8
View File
@@ -6037,10 +6037,10 @@ static bool ggml_hexagon_supported_activations(const struct ggml_hexagon_session
const struct ggml_tensor * src1 = op->src[1];
const struct ggml_tensor * dst = op;
if (src0->type != GGML_TYPE_F32) {
if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16) {
return false;
}
if (dst->type != GGML_TYPE_F32) {
if (dst->type != src0->type) {
return false;
}
@@ -6052,7 +6052,7 @@ static bool ggml_hexagon_supported_activations(const struct ggml_hexagon_session
}
if (src1) {
if (src1->type != GGML_TYPE_F32) {
if (src1->type != src0->type) {
return false;
}
if (!ggml_are_same_shape(src0, src1)) {
@@ -6637,12 +6637,12 @@ static htp_op_code op_remap_to_htp(const ggml_tensor * t) {
case GGML_OP_GLU:
switch (ggml_get_glu_op(t)) {
case GGML_GLU_OP_SWIGLU: return HTP_OP_GLU_SWIGLU;
case GGML_GLU_OP_SWIGLU_OAI: return HTP_OP_GLU_SWIGLU_OAI;
case GGML_GLU_OP_SWIGLU: return HTP_OP_GLU_SWIGLU;
case GGML_GLU_OP_SWIGLU_OAI: return HTP_OP_GLU_SWIGLU_OAI;
case GGML_GLU_OP_SWIGLU_CLAMP: return HTP_OP_GLU_SWIGLU_CLAMP;
case GGML_GLU_OP_GEGLU: return HTP_OP_GLU_GEGLU;
case GGML_GLU_OP_GEGLU_QUICK: return HTP_OP_GLU_GEGLU_QUICK;
case GGML_GLU_OP_GEGLU_ERF: return HTP_OP_GLU_GEGLU_ERF;
case GGML_GLU_OP_GEGLU: return HTP_OP_GLU_GEGLU;
case GGML_GLU_OP_GEGLU_QUICK: return HTP_OP_GLU_GEGLU_QUICK;
case GGML_GLU_OP_GEGLU_ERF: return HTP_OP_GLU_GEGLU_ERF;
default: break;
}
break;
ggml/src/ggml-hexagon/htp/act-ops.c
+410 -99
View File
@@ -55,16 +55,22 @@
struct htp_act_context;
typedef void (*glu_compute_fn_t)(const float * restrict src0,
const float * restrict src1,
float * restrict dst,
const uint32_t num_rows,
const struct htp_act_context * actx);
typedef void (*glu_compute_f32_fn_t)(const float * restrict src0,
const float * restrict src1,
float * restrict dst,
const uint32_t num_rows,
const struct htp_act_context * actx);
typedef void (*glu_compute_f16_fn_t)(const _Float16 * restrict src0,
const _Float16 * restrict src1,
_Float16 * restrict dst,
const uint32_t num_rows,
const struct htp_act_context * actx);
struct htp_act_context {
struct htp_ops_context * octx;
glu_compute_fn_t compute;
void * compute;
const char * op_str;
// Precomputed values
@@ -392,6 +398,58 @@ static inline void hvx_geglu_erf_f32_aa(uint8_t * restrict dst, const uint8_t *
}
}
static inline void hvx_geglu_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src0 % 128 == 0);
assert((unsigned long) src1 % 128 == 0);
HVX_Vector * restrict vdst = (HVX_Vector *) dst;
const HVX_Vector * restrict vsrc0 = (const HVX_Vector *) src0;
const HVX_Vector * restrict vsrc1 = (const HVX_Vector *) src1;
const uint32_t epv = 128 / sizeof(_Float16);
const uint32_t nvec = n / epv;
const uint32_t nloe = n % epv;
const HVX_Vector v_coef_a = hvx_vec_splat_f16((_Float16) GELU_COEF_A);
const HVX_Vector v_sqrt_2_pi = hvx_vec_splat_f16((_Float16) SQRT_2_OVER_PI);
const HVX_Vector v_one = hvx_vec_splat_f16((_Float16) 1.0f);
const HVX_Vector v_half = hvx_vec_splat_f16((_Float16) 0.5f);
uint32_t i = 0;
_Pragma("unroll(4)")
for (; i < nvec; i++) {
HVX_Vector x = vsrc0[i];
HVX_Vector g = vsrc1[i];
// gelu(x) = 0.5*x*(1 + tanh(SQRT_2_OVER_PI*x*(1 + GELU_COEF_A*x*x)))
HVX_Vector x2 = hvx_vec_mul_f16_f16(x, x);
HVX_Vector coef = hvx_vec_add_f16_f16(hvx_vec_mul_f16_f16(x2, v_coef_a), v_one);
HVX_Vector inner = hvx_vec_mul_f16_f16(hvx_vec_mul_f16_f16(x, coef), v_sqrt_2_pi);
HVX_Vector t = hvx_vec_add_f16_f16(hvx_vec_tanh_f16(inner), v_one);
HVX_Vector gelu_x = hvx_vec_mul_f16_f16(hvx_vec_mul_f16_f16(x, t), v_half);
vdst[i] = hvx_vec_mul_f16_f16(gelu_x, g);
}
if (nloe) {
HVX_Vector x = vsrc0[i];
HVX_Vector g = vsrc1[i];
HVX_Vector x2 = hvx_vec_mul_f16_f16(x, x);
HVX_Vector coef = hvx_vec_add_f16_f16(hvx_vec_mul_f16_f16(x2, v_coef_a), v_one);
HVX_Vector inner = hvx_vec_mul_f16_f16(hvx_vec_mul_f16_f16(x, coef), v_sqrt_2_pi);
HVX_Vector t = hvx_vec_add_f16_f16(hvx_vec_tanh_f16(inner), v_one);
HVX_Vector gelu_x = hvx_vec_mul_f16_f16(hvx_vec_mul_f16_f16(x, t), v_half);
HVX_Vector res = hvx_vec_mul_f16_f16(gelu_x, g);
hvx_vec_store_a((void *) &vdst[i], nloe * sizeof(_Float16), res);
}
}
// geglu(x, g) = gelu(x) * g
static void geglu_f32(const float * restrict src0,
const float * restrict src1,
@@ -443,101 +501,190 @@ static void geglu_erf_f32(const float * restrict src0,
}
}
static void glu_f32_per_thread(unsigned int nth, unsigned int ith, void * data) {
struct htp_act_context * actx = (struct htp_act_context *) data;
htp_act_preamble;
// swiglu(x) = x1 * sigmoid(x0)
static void swiglu_f16(const _Float16 * restrict src0,
const _Float16 * restrict src1,
_Float16 * restrict dst,
const uint32_t num_rows,
const struct htp_act_context * actx) {
htp_glu_op_preamble;
struct htp_thread_trace * tr = actx->octx->ctx ? &actx->octx->ctx->trace[ith] : NULL;
for (uint32_t ib = 0; ib < num_rows; ib++) {
const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned);
const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned);
uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned);
size_t src0_row_size = actx->src0_row_size;
size_t src1_row_size = actx->src1_row_size;
size_t dst_row_size = actx->dst_row_size;
size_t src0_row_stride = actx->src0_row_stride;
size_t src1_row_stride = actx->src1_row_stride;
const uint32_t src0_nrows = actx->src0_nrows;
const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread;
const uint32_t src0_start_row = actx->row_start + src0_nrows_per_thread * ith;
const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, actx->row_start + src0_nrows);
/* no work for this thread */
if (src0_start_row >= src0_end_row) {
return;
hvx_sigmoid_f16_aa(dst_ptr, src0_ptr, nc);
hvx_mul_mul_f16_aa(dst_ptr, src0_ptr, dst_ptr, src1_ptr, nc);
}
const dma_addr_t data_src0 = actx->data_src0;
const dma_addr_t data_src1 = actx->data_src1;
const dma_addr_t data_dst = actx->data_dst;
const size_t src0_row_size_aligned = actx->src0_row_size_aligned;
const size_t src1_row_size_aligned = actx->src1_row_size_aligned;
const size_t dst_row_size_aligned = actx->dst_row_size_aligned;
uint8_t * restrict src0_spad_data = actx->vtcm_src0 + (ith * actx->vtcm_src0_size_per_thread);
uint8_t * restrict src1_spad_data = actx->vtcm_src1 + (ith * actx->vtcm_src1_size_per_thread);
uint8_t * restrict dst_spad_data = actx->vtcm_dst + (ith * actx->vtcm_dst_size_per_thread);
size_t src0_spad_half_size = actx->src0_spad_half_size;
size_t src1_spad_half_size = actx->src1_spad_half_size;
size_t dst_spad_half_size = actx->dst_spad_half_size;
const int BLOCK = actx->block;
if (BLOCK == 0) {
FARF(ERROR, "%s : VTCM reservation %zu is too small, needed %zu\n",
actx->op_str, actx->vtcm_src0_size_per_thread, src0_row_size_aligned);
return;
}
dma_queue * dma_q = actx->octx->ctx->dma[ith];
glu_compute_fn_t compute = actx->compute;
for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) {
const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);
/* Dummy DMA transation for sequencing (interleaving dst,src,dst,...) */
dma_queue_push(dma_q,
dma_make_data(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)),
dst_row_size, dst_row_size_aligned, dst_row_size, 0);
dma_queue_push(dma_q,
dma_make_data(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_stride)),
src0_row_size_aligned, src0_row_stride, src0_row_size, block_size);
dma_queue_push(dma_q,
dma_make_data(src1_spad_data + (spad_idx * src1_spad_half_size), data_src1 + (ir * src1_row_stride)),
src1_row_size_aligned, src1_row_stride, src1_row_size, block_size);
}
for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) {
const uint32_t block_size = MIN(BLOCK, src0_end_row - ir);
float * dst_spad = (float *) dma_queue_pop(dma_q).src;
float * src0_spad = (float *) dma_queue_pop(dma_q).dst;
float * src1_spad = (float *) dma_queue_pop(dma_q).dst;
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir);
compute(src0_spad, src1_spad, dst_spad, block_size, actx);
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir);
dma_queue_push(dma_q, dma_make_data(data_dst + (ir * dst_row_size), dst_spad),
dst_row_size, dst_row_size_aligned, dst_row_size, block_size);
/* prefetch N+2 loop iteration if any */
const uint32_t pref_block = (ir + BLOCK * 2);
if (pref_block < src0_end_row) {
const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block);
dma_queue_push(dma_q, dma_make_data(src0_spad, data_src0 + (pref_block * src0_row_stride)),
src0_row_size_aligned, src0_row_stride, src0_row_size, pref_block_size);
dma_queue_push(dma_q, dma_make_data(src1_spad, data_src1 + (pref_block * src1_row_stride)),
src1_row_size_aligned, src1_row_stride, src1_row_size, pref_block_size);
}
}
dma_queue_flush(dma_q);
}
// out = x * sigmoid(alpha * x) * (clamp(y, -limit, limit) + 1.f)
static void swiglu_oai_f16(const _Float16 * restrict src0,
const _Float16 * restrict src1,
_Float16 * restrict dst,
const uint32_t num_rows,
const struct htp_act_context * actx) {
htp_glu_op_preamble;
const _Float16 alpha = (_Float16) ((const float *) (actx->octx->op_params))[2];
const _Float16 limit = (_Float16) ((const float *) (actx->octx->op_params))[3];
for (uint32_t ib = 0; ib < num_rows; ib++) {
const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned);
const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned);
uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned);
// x (src0_ptr) = std::min(src0_p[k], limit);
hvx_min_scalar_f16((uint8_t *) src0_ptr, src0_ptr, limit, nc);
// y1 (src1_ptr) = std::clamp(src1_p[k], -limit, limit);
hvx_clamp_scalar_f16((uint8_t *) src1_ptr, src1_ptr, -limit, limit, nc);
// y (src1_ptr) = y1 + 1.f
hvx_add_scalar_f16((uint8_t *) src1_ptr, src1_ptr, (_Float16) 1.0f, nc);
// x1 (dst_ptr) = alpha * x
hvx_mul_scalar_f16(dst_ptr, src0_ptr, alpha, nc);
// x2 (dst_ptr) = sigmoid(x1) = 1/(1+exp(-x1))
hvx_sigmoid_f16_aa(dst_ptr, dst_ptr, nc);
// out = x * sigmoid(alpha * x) * (y + 1.f)
hvx_mul_mul_f16_aa(dst_ptr, src0_ptr, dst_ptr, src1_ptr, nc);
}
}
static void swiglu_clamp_f16(const _Float16 * restrict src0,
const _Float16 * restrict src1,
_Float16 * restrict dst,
const uint32_t num_rows,
const struct htp_act_context * actx) {
htp_glu_op_preamble;
const _Float16 limit = (_Float16) ((const float *) (actx->octx->op_params))[3];
for (uint32_t ib = 0; ib < num_rows; ib++) {
const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned);
const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned);
uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned);
hvx_min_scalar_f16((uint8_t *) src0_ptr, src0_ptr, limit, nc);
hvx_clamp_scalar_f16((uint8_t *) src1_ptr, src1_ptr, -limit, limit, nc);
hvx_sigmoid_f16_aa(dst_ptr, src0_ptr, nc);
hvx_mul_mul_f16_aa(dst_ptr, src0_ptr, dst_ptr, src1_ptr, nc);
}
}
// geglu(x, g) = gelu(x) * g, gelu via tanh approximation
static void geglu_f16(const _Float16 * restrict src0,
const _Float16 * restrict src1,
_Float16 * restrict dst,
const uint32_t num_rows,
const struct htp_act_context * actx) {
htp_glu_op_preamble;
for (uint32_t ib = 0; ib < num_rows; ib++) {
const uint8_t * restrict src0_ptr = (const uint8_t *) src0 + (ib * src0_row_size_aligned);
const uint8_t * restrict src1_ptr = (const uint8_t *) src1 + (ib * src1_row_size_aligned);
uint8_t * restrict dst_ptr = (uint8_t *) dst + (ib * dst_row_size_aligned);
hvx_geglu_f16_aa(dst_ptr, src0_ptr, src1_ptr, nc);
}
}
#define DEFINE_GLU_PER_THREAD(TYPE, SUFFIX) \
static void glu_##SUFFIX##_per_thread(unsigned int nth, unsigned int ith, void * data) { \
struct htp_act_context * actx = (struct htp_act_context *) data; \
htp_act_preamble; \
\
struct htp_thread_trace * tr = actx->octx->ctx ? &actx->octx->ctx->trace[ith] : NULL; \
\
size_t src0_row_size = actx->src0_row_size; \
size_t src1_row_size = actx->src1_row_size; \
size_t dst_row_size = actx->dst_row_size; \
\
size_t src0_row_stride = actx->src0_row_stride; \
size_t src1_row_stride = actx->src1_row_stride; \
\
const uint32_t src0_nrows = actx->src0_nrows; \
const uint32_t src0_nrows_per_thread = actx->src0_nrows_per_thread; \
\
const uint32_t src0_start_row = actx->row_start + src0_nrows_per_thread * ith; \
const uint32_t src0_end_row = MIN(src0_start_row + src0_nrows_per_thread, actx->row_start + src0_nrows); \
\
/* no work for this thread */ \
if (src0_start_row >= src0_end_row) { \
return; \
} \
\
const dma_addr_t data_src0 = actx->data_src0; \
const dma_addr_t data_src1 = actx->data_src1; \
const dma_addr_t data_dst = actx->data_dst; \
\
const size_t src0_row_size_aligned = actx->src0_row_size_aligned; \
const size_t src1_row_size_aligned = actx->src1_row_size_aligned; \
const size_t dst_row_size_aligned = actx->dst_row_size_aligned; \
\
uint8_t * restrict src0_spad_data = actx->vtcm_src0 + (ith * actx->vtcm_src0_size_per_thread); \
uint8_t * restrict src1_spad_data = actx->vtcm_src1 + (ith * actx->vtcm_src1_size_per_thread); \
uint8_t * restrict dst_spad_data = actx->vtcm_dst + (ith * actx->vtcm_dst_size_per_thread); \
\
size_t src0_spad_half_size = actx->src0_spad_half_size; \
size_t src1_spad_half_size = actx->src1_spad_half_size; \
size_t dst_spad_half_size = actx->dst_spad_half_size; \
\
const int BLOCK = actx->block; \
if (BLOCK == 0) { \
FARF(ERROR, "%s : VTCM reservation %zu is too small, needed %zu\n", \
actx->op_str, actx->vtcm_src0_size_per_thread, src0_row_size_aligned); \
return; \
} \
\
dma_queue * dma_q = actx->octx->ctx->dma[ith]; \
glu_compute_##SUFFIX##_fn_t compute = (glu_compute_##SUFFIX##_fn_t) actx->compute; \
\
for (uint32_t ir = src0_start_row, spad_idx = 0; ir < src0_end_row && spad_idx < 2; ir += BLOCK, spad_idx++) { \
const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); \
\
/* Dummy DMA transation for sequencing (interleaving dst,src,dst,...) */ \
dma_queue_push(dma_q, \
dma_make_data(data_dst, dst_spad_data + (spad_idx * dst_spad_half_size)), \
dst_row_size, dst_row_size_aligned, dst_row_size, 0); \
\
dma_queue_push(dma_q, \
dma_make_data(src0_spad_data + (spad_idx * src0_spad_half_size), data_src0 + (ir * src0_row_stride)), \
src0_row_size_aligned, src0_row_stride, src0_row_size, block_size); \
\
dma_queue_push(dma_q, \
dma_make_data(src1_spad_data + (spad_idx * src1_spad_half_size), data_src1 + (ir * src1_row_stride)), \
src1_row_size_aligned, src1_row_stride, src1_row_size, block_size); \
} \
\
for (uint32_t ir = src0_start_row; ir < src0_end_row; ir += BLOCK) { \
const uint32_t block_size = MIN(BLOCK, src0_end_row - ir); \
\
TYPE * dst_spad = (TYPE *) dma_queue_pop(dma_q).src; \
TYPE * src0_spad = (TYPE *) dma_queue_pop(dma_q).dst; \
TYPE * src1_spad = (TYPE *) dma_queue_pop(dma_q).dst; \
\
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir); \
compute(src0_spad, src1_spad, dst_spad, block_size, actx); \
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir); \
\
dma_queue_push(dma_q, dma_make_data(data_dst + (ir * dst_row_size), dst_spad), \
dst_row_size, dst_row_size_aligned, dst_row_size, block_size); \
\
/* prefetch N+2 loop iteration if any */ \
const uint32_t pref_block = (ir + BLOCK * 2); \
if (pref_block < src0_end_row) { \
const uint32_t pref_block_size = MIN(BLOCK, src0_end_row - pref_block); \
dma_queue_push(dma_q, dma_make_data(src0_spad, data_src0 + (pref_block * src0_row_stride)), \
src0_row_size_aligned, src0_row_stride, src0_row_size, pref_block_size); \
dma_queue_push(dma_q, dma_make_data(src1_spad, data_src1 + (pref_block * src1_row_stride)), \
src1_row_size_aligned, src1_row_stride, src1_row_size, pref_block_size); \
} \
} \
\
dma_queue_flush(dma_q); \
}
DEFINE_GLU_PER_THREAD(float, f32)
DEFINE_GLU_PER_THREAD(_Float16, f16)
static int execute_op_activations_f32(struct htp_ops_context * octx) {
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * src1 = octx->src[1];
@@ -548,8 +695,8 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) {
return HTP_STATUS_NO_SUPPORT;
}
glu_compute_fn_t compute_fn = NULL;
const char * op_type = NULL;
glu_compute_f32_fn_t compute_fn = NULL;
const char * op_type = NULL;
switch (octx->op) {
case HTP_OP_GLU_SWIGLU:
@@ -581,6 +728,7 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) {
compute_fn = geglu_erf_f32;
op_type = "geglu-erf-f32";
break;
default:
FARF(ERROR, "Unsupported activations Op %u\n", octx->op);
return HTP_STATUS_NO_SUPPORT;
@@ -643,7 +791,7 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) {
// Prepare context
struct htp_act_context actx;
actx.octx = octx;
actx.compute = compute_fn;
actx.compute = (void *) compute_fn;
actx.op_str = op_type;
actx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
@@ -708,11 +856,174 @@ static int execute_op_activations_f32(struct htp_ops_context * octx) {
return HTP_STATUS_OK;
}
static int execute_op_activations_f16(struct htp_ops_context * octx) {
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * src1 = octx->src[1];
const struct htp_tensor * dst = octx->dst;
if ((dst->ne[0] * SIZEOF_FP16) != dst->nb[1]) {
FARF(ERROR, "Non-contiguous dst is not supported at this time \n");
return HTP_STATUS_NO_SUPPORT;
}
glu_compute_f16_fn_t compute_fn = NULL;
const char * op_type = NULL;
switch (octx->op) {
case HTP_OP_GLU_SWIGLU:
compute_fn = swiglu_f16;
op_type = "swiglu-f16";
break;
case HTP_OP_GLU_SWIGLU_OAI:
compute_fn = swiglu_oai_f16;
op_type = "swiglu-oai-f16";
break;
case HTP_OP_GLU_SWIGLU_CLAMP:
compute_fn = swiglu_clamp_f16;
op_type = "swiglu-clamp-f16";
break;
case HTP_OP_GLU_GEGLU:
compute_fn = geglu_f16;
op_type = "geglu-f16";
break;
default:
FARF(ERROR, "Unsupported activations Op %u\n", octx->op);
return HTP_STATUS_NO_SUPPORT;
}
const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
const size_t dst_row_size = dst->ne[0] * SIZEOF_FP16;
uint32_t row_start = 0;
uint32_t nrows = src0_nrows;
if (octx->ctx->mdev.count > 1) {
uint32_t rows_per_chunk = 0;
htp_tensor_mdev_rows_per_chunk(dst, sizeof(_Float16), (uint32_t) dst_row_size, &rows_per_chunk);
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(src0_nrows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
row_start = range.start;
nrows = range.count;
}
if (nrows == 0) {
return HTP_STATUS_OK;
}
const uint32_t n_threads = octx->n_threads;
// row_size = bytes of useful data per row (what the kernel touches / what DMA copies).
// row_stride = bytes between successive rows in DDR (may exceed row_size for non-contig src).
const size_t nc_bytes = dst_row_size;
const size_t src0_row_size = nc_bytes;
const size_t src1_row_size = nc_bytes;
const size_t src0_row_stride = src0->nb[1];
const size_t src1_row_stride = src1 ? src1->nb[1] : src0->nb[1];
const size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN);
const size_t src1_row_size_aligned = hex_round_up(src1_row_size, VLEN);
const size_t dst_row_size_aligned = hex_round_up(dst_row_size, VLEN);
struct htp_act_vtcm_layout L;
htp_act_vtcm_layout_build(&L, src0_row_size_aligned, src1_row_size_aligned, dst_row_size_aligned, n_threads,
octx->ctx->vtcm_size);
// Make sure the reserved vtcm size is sufficient
if (L.vtcm_row_per_thread == 0) {
FARF(ERROR, "act-%s : current VTCM reservation %zu is too small for even 1 row per thread, needed at least %zu\n", op_type, octx->ctx->vtcm_size,
(src0_row_size_aligned + src1_row_size_aligned + dst_row_size_aligned) * n_threads);
return HTP_STATUS_VTCM_TOO_SMALL;
}
if (src1) {
FARF(HIGH, "%s: %ux%ux%ux%u x %ux%ux%ux%u -> %ux%ux%ux%u : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n",
op_type, src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], src1->ne[0], src1->ne[1], src1->ne[2],
src1->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3], L.src0_bytes_per_thread * n_threads,
L.src1_bytes_per_thread * n_threads, L.dst_bytes_per_thread * n_threads);
} else {
FARF(HIGH, "%s: %ux%ux%ux%u -> %ux%ux%ux%u : src0-vtcm-size %zu src1-vtcm-size %zu dst-vtcm-size %zu\n", op_type,
src0->ne[0], src0->ne[1], src0->ne[2], src0->ne[3], dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3],
L.src0_bytes_per_thread * n_threads, L.src1_bytes_per_thread * n_threads, L.dst_bytes_per_thread * n_threads);
}
// Prepare context
struct htp_act_context actx;
actx.octx = octx;
actx.compute = (void *) compute_fn;
actx.op_str = op_type;
actx.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
actx.src0_row_size = src0_row_size;
actx.src1_row_size = src1_row_size;
actx.dst_row_size = dst_row_size;
actx.src0_row_size_aligned = src0_row_size_aligned;
actx.src1_row_size_aligned = src1_row_size_aligned;
actx.dst_row_size_aligned = dst_row_size_aligned;
actx.src0_row_stride = src0_row_stride;
actx.src1_row_stride = src1_row_stride;
uint8_t * const base = (uint8_t *) octx->ctx->vtcm_base;
actx.vtcm_src0 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src0);
actx.vtcm_src1 = VTCM_LAYOUT_PTR(uint8_t, base, L.off_src1);
actx.vtcm_dst = VTCM_LAYOUT_PTR(uint8_t, base, L.off_dst);
actx.vtcm_src0_size_per_thread = L.src0_bytes_per_thread;
actx.vtcm_src1_size_per_thread = L.src1_bytes_per_thread;
actx.vtcm_dst_size_per_thread = L.dst_bytes_per_thread;
actx.src0_spad_half_size = L.src0_bytes_per_thread / 2;
actx.src1_spad_half_size = L.src1_bytes_per_thread / 2;
actx.dst_spad_half_size = L.dst_bytes_per_thread / 2;
actx.block = actx.src0_spad_half_size / actx.src0_row_size_aligned;
actx.src0_nrows = nrows;
actx.row_start = row_start;
actx.nc = dst->ne[0];
// Addresses and GLU logic
dma_addr_t data_src0 = src0->data;
dma_addr_t data_src1 = src1 ? src1->data : 0;
if (!src1 && (octx->op == HTP_OP_GLU_SWIGLU ||
octx->op == HTP_OP_GLU_SWIGLU_OAI ||
octx->op == HTP_OP_GLU_SWIGLU_CLAMP ||
octx->op == HTP_OP_GLU_GEGLU)) {
const int32_t swapped = octx->op_params[1];
data_src1 = data_src0;
actx.src1_row_size = actx.src0_row_size;
size_t nc_in_bytes = actx.nc * SIZEOF_FP16;
if (swapped) {
data_src0 += nc_in_bytes;
} else {
data_src1 += nc_in_bytes;
}
}
actx.data_src0 = data_src0;
actx.data_src1 = data_src1;
actx.data_dst = dst->data;
work_queue_run(octx->ctx->work_queue, (worker_callback_t)glu_f16_per_thread, &actx, n_threads);
return HTP_STATUS_OK;
}
int op_activations(struct htp_ops_context * octx) {
switch (octx->src[0]->type) {
case HTP_TYPE_F32:
return execute_op_activations_f32(octx);
case HTP_TYPE_F16:
return execute_op_activations_f16(octx);
default:
return HTP_STATUS_NO_SUPPORT;
}
ggml/src/ggml-hexagon/htp/hvx-arith.h
+71
View File
@@ -164,6 +164,37 @@ static inline void hvx_mul_mul_f32_aa(uint8_t * restrict dst, const uint8_t * re
}
}
static inline void hvx_mul_mul_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src0, const uint8_t * restrict src1, const uint8_t * restrict src2, const uint32_t num_elems) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src0 % 128 == 0);
assert((unsigned long) src1 % 128 == 0);
assert((unsigned long) src2 % 128 == 0);
HVX_Vector * restrict vdst = (HVX_Vector *) dst;
HVX_Vector * restrict vsrc0 = (HVX_Vector *) src0;
HVX_Vector * restrict vsrc1 = (HVX_Vector *) src1;
HVX_Vector * restrict vsrc2 = (HVX_Vector *) src2;
const uint32_t elem_size = sizeof(_Float16);
const uint32_t epv = 128 / elem_size;
const uint32_t nvec = num_elems / epv;
const uint32_t nloe = num_elems % epv;
uint32_t i = 0;
_Pragma("unroll(4)")
for (; i < nvec; i++) {
HVX_Vector v1 = HVX_OP_MUL_F16(vsrc0[i], vsrc1[i]);
vdst[i] = HVX_OP_MUL_F16(v1, vsrc2[i]);
}
if (nloe) {
HVX_Vector v1 = HVX_OP_MUL_F16(vsrc0[i], vsrc1[i]);
HVX_Vector v2 = HVX_OP_MUL_F16(v1, vsrc2[i]);
hvx_vec_store_a((void *) &vdst[i], nloe * elem_size, v2);
}
}
// Scalar Operations
#define hvx_scalar_loop_body(dst_type, src_type, elem_size, vec_store, scalar_op_macro) \
@@ -348,6 +379,44 @@ static inline void hvx_max_scalar_f32(uint8_t * restrict dst, const uint8_t * re
}
}
#define HVX_OP_MIN_SCALAR_F16(v) Q6_Vhf_vmin_VhfVhf(val_vec, v)
static inline void hvx_min_scalar_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, uint32_t n) {
const HVX_Vector val_vec = hvx_vec_splat_f16(val);
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
hvx_scalar_loop_body(HVX_Vector, HVX_Vector, sizeof(_Float16), hvx_vec_store_a, HVX_OP_MIN_SCALAR_F16);
}
static inline void hvx_min_scalar_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, uint32_t n) {
const HVX_Vector val_vec = hvx_vec_splat_f16(val);
assert((unsigned long) dst % 128 == 0);
hvx_scalar_loop_body(HVX_Vector, HVX_UVector, sizeof(_Float16), hvx_vec_store_a, HVX_OP_MIN_SCALAR_F16);
}
static inline void hvx_min_scalar_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, uint32_t n) {
const HVX_Vector val_vec = hvx_vec_splat_f16(val);
assert((unsigned long) src % 128 == 0);
hvx_scalar_loop_body(HVX_UVector, HVX_Vector, sizeof(_Float16), hvx_vec_store_u, HVX_OP_MIN_SCALAR_F16);
}
static inline void hvx_min_scalar_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, uint32_t n) {
const HVX_Vector val_vec = hvx_vec_splat_f16(val);
hvx_scalar_loop_body(HVX_UVector, HVX_UVector, sizeof(_Float16), hvx_vec_store_u, HVX_OP_MIN_SCALAR_F16);
}
static inline void hvx_min_scalar_f16(uint8_t * restrict dst, const uint8_t * restrict src, const _Float16 val, const int num_elems) {
if (hex_is_aligned((void *) dst, 128) && hex_is_aligned((void *) src, 128)) {
hvx_min_scalar_f16_aa(dst, src, val, num_elems);
} else if (hex_is_aligned((void *) dst, 128)) {
hvx_min_scalar_f16_au(dst, src, val, num_elems);
} else if (hex_is_aligned((void *) src, 128)) {
hvx_min_scalar_f16_ua(dst, src, val, num_elems);
} else {
hvx_min_scalar_f16_uu(dst, src, val, num_elems);
}
}
// CLAMP Scalar variants
#define HVX_OP_CLAMP_SCALAR(v) \
@@ -767,10 +836,12 @@ static inline void hvx_sqr_f16(uint8_t * restrict dst, const uint8_t * restrict
#undef hvx_scalar_loop_body
#undef HVX_OP_MIN_SCALAR
#undef HVX_OP_MAX_SCALAR
#undef HVX_OP_MIN_SCALAR_F16
#undef HVX_OP_CLAMP_SCALAR
#undef HVX_OP_CLAMP_SCALAR_F16
#undef HVX_OP_LEAKY_RELU_SCALAR
#undef DEFINE_HVX_BINARY_OP_VARIANTS
#undef DEFINE_HVX_BINARY_SCALAR_OP_VARIANTS
#undef HVX_BINARY_DISPATCHER
#undef UNUSED
ggml/src/ggml-hexagon/htp/hvx-sigmoid.h
+54
View File
@@ -178,4 +178,58 @@ static inline HVX_Vector hvx_vec_tanh_f16(HVX_Vector x) {
return hvx_vec_add_f16_f16(hvx_vec_mul_f16_f16(sig2x, v_two), v_neg_one);
}
#define hvx_sigmoid_f16_loop_body(dst_type, src_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
\
const uint32_t epv = 128 / sizeof(_Float16); \
const uint32_t nvec = n / epv; \
const uint32_t nloe = n % epv; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (; i < nvec; i++) { \
vdst[i] = hvx_vec_fast_sigmoid_f16(vsrc[i]); \
} \
if (nloe) { \
HVX_Vector tmp = hvx_vec_fast_sigmoid_f16(vsrc[i]); \
vec_store((void *) &vdst[i], nloe * sizeof(_Float16), tmp); \
} \
} while(0)
#define hvx_tanh_f16_loop_body(dst_type, src_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
\
const uint32_t epv = 128 / sizeof(_Float16); \
const uint32_t nvec = n / epv; \
const uint32_t nloe = n % epv; \
\
uint32_t i = 0; \
\
_Pragma("unroll(4)") \
for (; i < nvec; i++) { \
vdst[i] = hvx_vec_tanh_f16(vsrc[i]); \
} \
if (nloe) { \
HVX_Vector tmp = hvx_vec_tanh_f16(vsrc[i]); \
vec_store((void *) &vdst[i], nloe * sizeof(_Float16), tmp); \
} \
} while(0)
static inline void hvx_sigmoid_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
hvx_sigmoid_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);
}
static inline void hvx_tanh_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
hvx_tanh_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);
}
#endif /* HVX_SIGMOID_H */