mirror of
https://github.com/ggml-org/whisper.cpp.git
synced 2026-10-12 01:05:43 +02:00
hexagon: support tiled Q4_0 and Q8_0 GET_ROWS (llama/29511)
* hexagon: support tiled Q4_0 and Q8_0 GET_ROWS * hex-get-rows: fix macros * hex-get-rows: use tiled HVX dequantization Assisted-by: OpenCode * hex-get-rows: fix register spills and clean up checks for unsupported ops * hex-get-rows: improve dma pipeline * hex-get-rows: improve/simplify kernel selection logic * hex-build: reenable vectorizer, didnt notice the regression earlier in the sampler update --------- Co-authored-by: Max Krasnyansky <maxk@qti.qualcomm.com>
This commit is contained in:
kurquhar
co-authored by
Max Krasnyansky
Max Krasnyansky
parent
fe06027d27
commit
7eea51881e
@@ -5208,49 +5208,51 @@ static void ggml_hexagon_precompute_get_rows_params(
|
||||
const uint32_t ne12 = src1->ne[2];
|
||||
const uint32_t nr = ne10 * ne11 * ne12;
|
||||
|
||||
const size_t nb01 = src0->nb[1];
|
||||
const size_t nb1 = dst->nb[1];
|
||||
const ggml_tensor * src0_base = src0->view_src ? src0->view_src : src0;
|
||||
const auto * extra = src0_base->buffer && ggml_backend_buffer_is_hexagon(src0_base->buffer) ?
|
||||
(const ggml_hexagon_tensor_extra *) src0_base->extra : nullptr;
|
||||
const bool tiled = src0->type == GGML_TYPE_Q4_0 || (extra && (extra->flags & GGML_HEXAGON_TENSOR_REPACK) != 0) ||
|
||||
sess->needs_repack.count(src0_base) || sess->needs_repack.count(src0);
|
||||
|
||||
const bool can_use_dma = (src0->type == dst->type) && (nb01 == nb1);
|
||||
const bool use_dma = can_use_dma && (ne00 >= 2048);
|
||||
|
||||
kparams->use_dma = use_dma ? 1 : 0;
|
||||
|
||||
uint32_t chunks_per_row = 1;
|
||||
uint32_t chunk_size = ne00;
|
||||
uint32_t total_tasks = nr;
|
||||
|
||||
if (use_dma) {
|
||||
kparams->n_threads = (std::min)((uint32_t)sess->n_threads, nr);
|
||||
kparams->tasks_per_thread = (nr + kparams->n_threads - 1) / kparams->n_threads;
|
||||
if (src0->type == dst->type) {
|
||||
kparams->kernel_type = HTP_GET_ROWS_KERNEL_SAMETYPE;
|
||||
} else if (tiled) {
|
||||
kparams->kernel_type = HTP_GET_ROWS_KERNEL_TILED;
|
||||
} else {
|
||||
if (src0->type == GGML_TYPE_F32 && nr < sess->n_threads) {
|
||||
const uint32_t min_chunk_size = 1024;
|
||||
uint32_t max_chunks = ne00 / min_chunk_size;
|
||||
if (max_chunks == 0) {
|
||||
max_chunks = 1;
|
||||
}
|
||||
chunks_per_row = (std::min)((sess->n_threads + nr - 1) / nr, max_chunks);
|
||||
chunk_size = (ne00 + chunks_per_row - 1) / chunks_per_row;
|
||||
total_tasks = nr * chunks_per_row;
|
||||
}
|
||||
kparams->n_threads = (std::min)(total_tasks, (uint32_t)sess->n_threads);
|
||||
kparams->tasks_per_thread = (total_tasks + kparams->n_threads - 1) / kparams->n_threads;
|
||||
kparams->kernel_type = HTP_GET_ROWS_KERNEL_FLAT;
|
||||
}
|
||||
|
||||
const uint32_t chunks_per_row = 1;
|
||||
const uint32_t chunk_size = ne00;
|
||||
const uint32_t total_tasks = nr;
|
||||
|
||||
kparams->n_threads = (std::min)((uint32_t)sess->n_threads, total_tasks);
|
||||
|
||||
struct htp_get_rows_vtcm_layout vtcm_layout = {};
|
||||
while (kparams->n_threads > 0) {
|
||||
htp_get_rows_vtcm_layout_build(&vtcm_layout, kparams->kernel_type, src0->type, ne00, kparams->n_threads);
|
||||
if (vtcm_layout.total_bytes <= sess->vtcm_size) {
|
||||
break;
|
||||
}
|
||||
--kparams->n_threads;
|
||||
}
|
||||
|
||||
if (kparams->n_threads == 0 && total_tasks > 0) {
|
||||
htp_get_rows_vtcm_layout_build(&vtcm_layout, kparams->kernel_type, src0->type, ne00, 1);
|
||||
}
|
||||
|
||||
kparams->vtcm_size = (total_tasks == 0) ? 0 : vtcm_layout.total_bytes;
|
||||
kparams->tasks_per_thread = kparams->n_threads > 0 ? (total_tasks + kparams->n_threads - 1) / kparams->n_threads : 0;
|
||||
|
||||
kparams->chunks_per_row = chunks_per_row;
|
||||
kparams->chunk_size = chunk_size;
|
||||
kparams->total_tasks = total_tasks;
|
||||
|
||||
kparams->div_ne10 = init_fastdiv_values(ne10);
|
||||
kparams->div_ne10_ne11 = init_fastdiv_values(ne10 * ne11);
|
||||
kparams->div_chunks_per_row = init_fastdiv_values(chunks_per_row);
|
||||
kparams->div_ne02 = init_fastdiv_values(ne02);
|
||||
kparams->div_ne03 = init_fastdiv_values(ne03);
|
||||
|
||||
struct htp_get_rows_vtcm_layout vtcm_layout;
|
||||
htp_get_rows_vtcm_layout_build(&vtcm_layout, src0->type, ne00, kparams->n_threads);
|
||||
kparams->vtcm_size = vtcm_layout.total_bytes;
|
||||
kparams->div_ne10 = ne10 > 0 ? init_fastdiv_values(ne10) : fastdiv_values{0, 0};
|
||||
kparams->div_ne10_ne11 = (ne10 * ne11) > 0 ? init_fastdiv_values(ne10 * ne11) : fastdiv_values{0, 0};
|
||||
kparams->div_chunks_per_row = chunks_per_row > 0 ? init_fastdiv_values(chunks_per_row) : fastdiv_values{0, 0};
|
||||
kparams->div_ne02 = ne02 > 0 ? init_fastdiv_values(ne02) : fastdiv_values{0, 0};
|
||||
kparams->div_ne03 = ne03 > 0 ? init_fastdiv_values(ne03) : fastdiv_values{0, 0};
|
||||
}
|
||||
|
||||
static void ggml_hexagon_precompute_set_rows_params(
|
||||
@@ -6176,19 +6178,36 @@ static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session *
|
||||
const struct ggml_tensor * src1 = op->src[1]; // indices
|
||||
const struct ggml_tensor * dst = op;
|
||||
|
||||
if (src0->extra) {
|
||||
const auto * extra = (const ggml_hexagon_tensor_extra *) src0->extra;
|
||||
if (extra->flags & GGML_HEXAGON_TENSOR_REPACK) {
|
||||
if (src0->type == GGML_TYPE_Q4_0 && src0->view_src) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const ggml_tensor * src0_base = src0->view_src ? src0->view_src : src0;
|
||||
bool is_repacked = false;
|
||||
if (src0_base->buffer && ggml_backend_buffer_is_hexagon(src0_base->buffer) && src0_base->extra) {
|
||||
const auto * extra = (const ggml_hexagon_tensor_extra *) src0_base->extra;
|
||||
is_repacked = (extra->flags & GGML_HEXAGON_TENSOR_REPACK) != 0;
|
||||
if (is_repacked && src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q8_0) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
is_repacked = is_repacked || sess->needs_repack.count(src0_base) || sess->needs_repack.count(src0);
|
||||
|
||||
if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_I32 && src0->ne[0] < 32) {
|
||||
// View offsets use the raw quantized layout and cannot address a tiled allocation.
|
||||
if (src0->view_src && is_repacked) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (src0->type == GGML_TYPE_Q4_0 && src0->buffer && !is_repacked) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (src0->type != dst->type && src0->ne[0] < 32) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 &&
|
||||
src0->type != GGML_TYPE_Q8_0 && src0->type != GGML_TYPE_I32) {
|
||||
src0->type != GGML_TYPE_Q4_0 && src0->type != GGML_TYPE_Q8_0 && src0->type != GGML_TYPE_I32) {
|
||||
return false;
|
||||
}
|
||||
|
||||
@@ -6196,13 +6215,30 @@ static bool ggml_hexagon_supported_get_rows(const struct ggml_hexagon_session *
|
||||
return false;
|
||||
}
|
||||
|
||||
if (src0->type == GGML_TYPE_I32) {
|
||||
if (dst->type != GGML_TYPE_I32) {
|
||||
if (src0->type == dst->type) {
|
||||
if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_I32 && src0->type != GGML_TYPE_F16) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
else if (dst->type != GGML_TYPE_F32) {
|
||||
} else if (src0->type == GGML_TYPE_I32) {
|
||||
return false;
|
||||
} else if (dst->type != GGML_TYPE_F32) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Empty recurrent-state gathers are skipped at execution; do not split the graph for them.
|
||||
if (ggml_is_empty(op)) {
|
||||
return true;
|
||||
}
|
||||
|
||||
struct htp_get_rows_kernel_params kparams;
|
||||
ggml_hexagon_precompute_get_rows_params(sess, src0, src1, dst, &kparams);
|
||||
if (kparams.n_threads == 0 || (size_t) kparams.vtcm_size > sess->vtcm_size) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Q4_0 has no raw fallback. Mark only accepted tensors for repacking.
|
||||
if (src0->type == GGML_TYPE_Q4_0 && !src0->buffer) {
|
||||
sess->needs_repack.insert(src0);
|
||||
}
|
||||
|
||||
return true;
|
||||
|
||||
@@ -20,6 +20,7 @@
|
||||
#include "htp/gated-delta-net-ops.h"
|
||||
#include "htp/softmax-ops.h"
|
||||
#include "htp/argsort-ops.h"
|
||||
#include "htp/get-rows-ops.h"
|
||||
|
||||
struct htp_opnode {
|
||||
ggml_tensor * node { nullptr };
|
||||
@@ -360,9 +361,7 @@ struct htp_opformat {
|
||||
} else if (node.opcode == HTP_OP_GATED_DELTA_NET) {
|
||||
const auto * kparams = (const struct htp_gdn_kernel_params *) node.kernel_params;
|
||||
const char * path = (kparams->kernel_type == HTP_GDN_KERNEL_HMX_CHUNKED) ? "hmx-chunked" : "hvx-recurrent";
|
||||
snprintf(str, max_size, "%s-%s vtcm %u",
|
||||
path,
|
||||
kparams->kda ? "kda" : "scalar",
|
||||
snprintf(str, max_size, "%s-%s vtcm %u", path, kparams->kda ? "kda" : "scalar",
|
||||
(unsigned int) (kparams->vtcm_size ? kparams->vtcm_size : kparams->vtcm_per_thread * kparams->n_threads));
|
||||
} else if (node.opcode == HTP_OP_MUL || node.opcode == HTP_OP_ADD || node.opcode == HTP_OP_ADD_ID ||
|
||||
node.opcode == HTP_OP_SUB || node.opcode == HTP_OP_DIV) {
|
||||
@@ -374,9 +373,19 @@ struct htp_opformat {
|
||||
node.opcode == HTP_OP_TOP_K ? "top_k" : "argsort",
|
||||
(int) kparams->n_threads, (int) kparams->n_chunks,
|
||||
(int) kparams->chunk_elems, (int) kparams->vtcm_size);
|
||||
} else if (node.opcode == HTP_OP_GET_ROWS) {
|
||||
const auto * kparams = (const struct htp_get_rows_kernel_params *) node.kernel_params;
|
||||
const char * ktype_str = "unknown";
|
||||
switch (kparams->kernel_type) {
|
||||
case HTP_GET_ROWS_KERNEL_SAMETYPE: ktype_str = "sametype"; break;
|
||||
case HTP_GET_ROWS_KERNEL_TILED: ktype_str = "tiled"; break;
|
||||
case HTP_GET_ROWS_KERNEL_FLAT: ktype_str = "flat"; break;
|
||||
}
|
||||
snprintf(str, max_size, "%s%s vtcm %u", ktype_str, kparams->n_threads > 1 ? "-multi" : "", (unsigned int) kparams->vtcm_size);
|
||||
} else {
|
||||
snprintf(str, max_size, "----");
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
void format(const htp_opnode & node) {
|
||||
|
||||
@@ -136,7 +136,7 @@ set(CMAKE_SHARED_LIBRARY_SONAME_C_FLAG "-Wl,-soname,")
|
||||
set(CMAKE_SHARED_LIBRARY_SONAME_CXX_FLAG "-Wl,-soname,")
|
||||
|
||||
# Compiler Options
|
||||
set(COMMON_FLAGS "${ARCH_FLAGS} -fno-vectorize -fno-slp-vectorize -flto -Wall -Werror -fno-zero-initialized-in-bss -G0 -fdata-sections -fpic ${XQF_ARGS}")
|
||||
set(COMMON_FLAGS "${ARCH_FLAGS} -fvectorize -flto -Wall -Werror -fno-zero-initialized-in-bss -G0 -fdata-sections -fpic ${XQF_ARGS}")
|
||||
|
||||
set(CMAKE_CXX_FLAGS_DEBUG "${COMMON_FLAGS} -O0 -D_DEBUG -g")
|
||||
set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${COMMON_FLAGS} -O2 -g")
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
#include "htp-tensor.h"
|
||||
#include "hvx-utils.h"
|
||||
#include "hvx-quant.h"
|
||||
#include "matmul-ops.h"
|
||||
#include "get-rows-ops.h"
|
||||
#include "work-queue.h"
|
||||
|
||||
@@ -28,6 +29,9 @@ struct get_rows_context {
|
||||
uint32_t task_start;
|
||||
uint32_t tasks;
|
||||
uint32_t tasks_per_thread;
|
||||
uint32_t tile_size;
|
||||
uint32_t tile_stride;
|
||||
bool index_i32;
|
||||
};
|
||||
|
||||
#define get_rows_preamble \
|
||||
@@ -195,32 +199,191 @@ static void get_rows_thread_##TYPE_NAME##_##IDX_TYPE(unsigned int nth, unsigned
|
||||
dma_queue_flush(dma_q); \
|
||||
}
|
||||
|
||||
#define F32_BYTES(n) ((n) * sizeof(float))
|
||||
#define F16_BYTES(n) ((n) * sizeof(__fp16))
|
||||
#define Q8_0_BYTES(n) (((n) / 32) * sizeof(block_q8_0))
|
||||
|
||||
GET_ROWS_THREAD_DT_FN(f32, F32_BYTES, int32_t, { if (cur_elems > 0) hvx_copy_f32_uu((uint8_t *)dst_spad, (const uint8_t *)src_spad, cur_elems); })
|
||||
GET_ROWS_THREAD_DT_FN(f32, F32_BYTES, int64_t, { if (cur_elems > 0) hvx_copy_f32_uu((uint8_t *)dst_spad, (const uint8_t *)src_spad, cur_elems); })
|
||||
static __attribute__((noinline)) void compute_get_rows_f16(float * dst_spad, const void * src_spad, uint32_t cur_elems) {
|
||||
hvx_dequantize_row_f16_f32(dst_spad, src_spad, cur_elems);
|
||||
}
|
||||
|
||||
GET_ROWS_THREAD_DT_FN(f16, F16_BYTES, int32_t, { hvx_dequantize_row_f16_f32((float *)dst_spad, src_spad, ne00); })
|
||||
GET_ROWS_THREAD_DT_FN(f16, F16_BYTES, int64_t, { hvx_dequantize_row_f16_f32((float *)dst_spad, src_spad, ne00); })
|
||||
static __attribute__((noinline)) void compute_get_rows_q8_0(float * dst_spad, const void * src_spad, uint32_t cur_elems) {
|
||||
hvx_dequantize_row_q8_0_f32(dst_spad, src_spad, cur_elems);
|
||||
}
|
||||
|
||||
GET_ROWS_THREAD_DT_FN(q8_0, Q8_0_BYTES, int32_t, { hvx_dequantize_row_q8_0_f32((float *)dst_spad, src_spad, ne00); })
|
||||
GET_ROWS_THREAD_DT_FN(q8_0, Q8_0_BYTES, int64_t, { hvx_dequantize_row_q8_0_f32((float *)dst_spad, src_spad, ne00); })
|
||||
GET_ROWS_THREAD_DT_FN(f16, F16_BYTES, int32_t, { compute_get_rows_f16((float *)dst_spad, src_spad, cur_elems); })
|
||||
GET_ROWS_THREAD_DT_FN(f16, F16_BYTES, int64_t, { compute_get_rows_f16((float *)dst_spad, src_spad, cur_elems); })
|
||||
|
||||
GET_ROWS_THREAD_DT_FN(q8_0, Q8_0_BYTES, int32_t, { compute_get_rows_q8_0((float *)dst_spad, src_spad, cur_elems); })
|
||||
GET_ROWS_THREAD_DT_FN(q8_0, Q8_0_BYTES, int64_t, { compute_get_rows_q8_0((float *)dst_spad, src_spad, cur_elems); })
|
||||
|
||||
|
||||
static __attribute__((noinline)) void compute_get_rows_tiled(float * dst, const uint8_t * tile, uint32_t row, bool q4) {
|
||||
const HVX_VectorPred first2 = Q6_Q_vsetq_R(2);
|
||||
const HVX_VectorPred first4 = Q6_Q_vsetq_R(4);
|
||||
HVX_Vector vq = Q6_V_vzero();
|
||||
if (q4) {
|
||||
const HVX_VectorPred first1 = Q6_Q_vsetq_R(1);
|
||||
const HVX_VectorPred first3 = Q6_Q_vsetq_R(3);
|
||||
for (int group = 3; group >= 0; --group) {
|
||||
const HVX_Vector v = Q6_V_vror_VR(hvx_vmem(tile + group * VLEN), row);
|
||||
// Four planes contribute bytes at 0, 32, 64 and 96 after rotation.
|
||||
HVX_Vector packed = Q6_V_vmux_QVV(first1, v, Q6_V_vror_VR(v, 31));
|
||||
packed = Q6_V_vmux_QVV(first2, packed, Q6_V_vror_VR(v, 62));
|
||||
packed = Q6_V_vmux_QVV(first3, packed, Q6_V_vror_VR(v, 93));
|
||||
vq = Q6_V_vmux_QVV(first4, packed, Q6_V_vror_VR(vq, VLEN - 4));
|
||||
}
|
||||
const HVX_Vector lo = Q6_V_vand_VV(vq, Q6_Vb_vsplat_R(0x0F));
|
||||
const HVX_Vector hi = Q6_Vub_vlsr_VubR(vq, 4);
|
||||
vq = Q6_V_lo_W(Q6_W_vshuff_VVR(hi, lo, -1));
|
||||
vq = Q6_Vb_vsub_VbVb(vq, Q6_Vb_vsplat_R(8));
|
||||
} else {
|
||||
for (int group = 7; group >= 0; --group) {
|
||||
const HVX_Vector v = Q6_V_vror_VR(hvx_vmem(tile + group * VLEN), 2 * row);
|
||||
// Two planes contribute halfwords at 0 and 64 after rotation.
|
||||
const HVX_Vector packed = Q6_V_vmux_QVV(first2, v, Q6_V_vror_VR(v, 62));
|
||||
vq = Q6_V_vmux_QVV(first4, packed, Q6_V_vror_VR(vq, VLEN - 4));
|
||||
}
|
||||
}
|
||||
const HVX_Vector scales = hvx_vmem(tile + (q4 ? 512 : 1024));
|
||||
const HVX_Vector scale_hf = hvx_vec_repl_f16(Q6_V_vror_VR(scales, 2 * row));
|
||||
const HVX_Vector scale = Q6_V_lo_W(hvx_vec_f16_to_f32(scale_hf));
|
||||
const HVX_VectorPair p16 = Q6_Wh_vunpack_Vb(vq);
|
||||
const HVX_VectorPair p32 = Q6_Ww_vunpack_Vh(Q6_V_lo_W(p16));
|
||||
const HVX_Vector values = hvx_vec_mul_f32_f32(Q6_Vsf_equals_Vw(Q6_V_lo_W(p32)), scale);
|
||||
*(HVX_Vector *) dst = values;
|
||||
}
|
||||
|
||||
struct get_rows_tiled_task {
|
||||
dma_addr_t tile_src_base;
|
||||
dma_addr_t dst_data;
|
||||
uint32_t row;
|
||||
};
|
||||
|
||||
static inline struct get_rows_tiled_task get_rows_tiled_calc_task(
|
||||
const struct htp_ops_context * octx,
|
||||
const struct get_rows_context * grctx,
|
||||
uint32_t i,
|
||||
uint32_t n_k_tiles,
|
||||
uint32_t tile_size
|
||||
) {
|
||||
const struct htp_get_rows_kernel_params * kparams = grctx->kparams;
|
||||
get_rows_preamble;
|
||||
|
||||
const uint32_t i12 = fastdiv(i, &kparams->div_ne10_ne11);
|
||||
const uint32_t rem = i - i12 * ne11 * ne10;
|
||||
const uint32_t i11 = fastdiv(rem, &kparams->div_ne10);
|
||||
const uint32_t i10 = rem - i11 * ne10;
|
||||
const dma_addr_t src1_data = octx->src[1]->data + i10*nb10 + i11*nb11 + i12*nb12;
|
||||
const uint32_t i01 = grctx->index_i32 ? *(const int32_t *)(uintptr_t) src1_data : (uint32_t) *(const int64_t *)(uintptr_t) src1_data;
|
||||
assert(i01 < ne01);
|
||||
|
||||
const uint32_t q02 = fastdiv(i11, &kparams->div_ne02);
|
||||
const uint32_t i02 = i11 - q02 * ne02;
|
||||
const uint32_t q03 = fastdiv(i12, &kparams->div_ne03);
|
||||
const uint32_t i03 = i12 - q03 * ne03;
|
||||
const uint32_t column_tile = i01 / HTP_MM_HMX_TILE_N_ROWS;
|
||||
const uint32_t row = i01 % HTP_MM_HMX_TILE_N_ROWS;
|
||||
const dma_addr_t matrix = octx->src[0]->data + i02*nb02 + i03*nb03;
|
||||
|
||||
struct get_rows_tiled_task task;
|
||||
task.tile_src_base = matrix + (column_tile * n_k_tiles) * tile_size;
|
||||
task.dst_data = octx->dst->data + i10*nb1 + i11*nb2 + i12*nb3;
|
||||
task.row = row;
|
||||
return task;
|
||||
}
|
||||
|
||||
static void get_rows_thread_tiled(unsigned int nth, unsigned int ith, void * data) {
|
||||
struct get_rows_context * grctx = (struct get_rows_context *) data;
|
||||
struct htp_ops_context * octx = grctx->octx;
|
||||
const struct htp_get_rows_kernel_params * kparams = grctx->kparams;
|
||||
get_rows_preamble;
|
||||
|
||||
const uint32_t dr = grctx->tasks_per_thread;
|
||||
const uint32_t ir0 = grctx->task_start + dr * ith;
|
||||
if (ir0 >= grctx->task_start + grctx->tasks) {
|
||||
return;
|
||||
}
|
||||
|
||||
const uint32_t ir1 = MIN(ir0 + dr, grctx->task_start + grctx->tasks);
|
||||
const uint32_t n_k_tiles = ne00 / HTP_MM_HMX_TILE_N_COLS;
|
||||
const struct htp_get_rows_vtcm_layout * vtcm_layout = &grctx->vtcm_layout;
|
||||
uint8_t * src_spad_base = grctx->vtcm_base + vtcm_layout->off_src0 + ith * vtcm_layout->src0_bytes_per_thread;
|
||||
uint8_t * dst_spad_base = grctx->vtcm_base + vtcm_layout->off_dst + ith * vtcm_layout->dst_bytes_per_thread;
|
||||
dma_queue * dma_q = octx->ctx->dma[ith];
|
||||
struct htp_thread_trace * tr = &octx->ctx->trace[ith];
|
||||
|
||||
const uint32_t tile_size = grctx->tile_size;
|
||||
const uint32_t tile_stride = grctx->tile_stride;
|
||||
const uint32_t dst_bytes = ne00 * sizeof(float);
|
||||
const bool is_q4 = (octx->src[0]->type == HTP_TYPE_Q4_0);
|
||||
|
||||
for (uint32_t step = 0, spad_idx = 0; step < ir1 - ir0 && spad_idx < 2; ++step, ++spad_idx) {
|
||||
const uint32_t i = ir0 + step;
|
||||
struct get_rows_tiled_task task = get_rows_tiled_calc_task(octx, grctx, i, n_k_tiles, tile_size);
|
||||
|
||||
// Dummy writeback to prime the queue with dst descriptor
|
||||
dma_queue_push(dma_q,
|
||||
dma_make_data(task.dst_data, dst_spad_base + spad_idx * vtcm_layout->dst_spad_half_size),
|
||||
dst_bytes, vtcm_layout->dst_spad_half_size, dst_bytes, 0);
|
||||
|
||||
// Prefetch row tiles
|
||||
dma_queue_push(dma_q,
|
||||
dma_make_data(src_spad_base + spad_idx * vtcm_layout->src0_spad_half_size, task.tile_src_base),
|
||||
tile_stride, tile_size, tile_size, n_k_tiles);
|
||||
}
|
||||
|
||||
for (uint32_t step = 0; step < ir1 - ir0; ++step) {
|
||||
const uint32_t i = ir0 + step;
|
||||
float * dst_spad = (float *) dma_queue_pop(dma_q).src;
|
||||
uint8_t * src_spad = (uint8_t *) dma_queue_pop(dma_q).dst;
|
||||
|
||||
struct get_rows_tiled_task task = get_rows_tiled_calc_task(octx, grctx, i, n_k_tiles, tile_size);
|
||||
|
||||
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
|
||||
for (uint32_t k_tile = 0; k_tile < n_k_tiles; ++k_tile) {
|
||||
const uint8_t * tile = src_spad + k_tile * tile_stride;
|
||||
float * dst_block = dst_spad + k_tile * HTP_MM_HMX_TILE_N_COLS;
|
||||
compute_get_rows_tiled(dst_block, tile, task.row, is_q4);
|
||||
}
|
||||
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) i);
|
||||
|
||||
// Real writeback of dst_spad
|
||||
dma_queue_push(dma_q,
|
||||
dma_make_data(task.dst_data, dst_spad),
|
||||
dst_bytes, vtcm_layout->dst_spad_half_size, dst_bytes, 1);
|
||||
|
||||
const uint32_t next_step = step + 2;
|
||||
if (next_step < ir1 - ir0) {
|
||||
const uint32_t ni = ir0 + next_step;
|
||||
struct get_rows_tiled_task next_task = get_rows_tiled_calc_task(octx, grctx, ni, n_k_tiles, tile_size);
|
||||
dma_queue_push(dma_q,
|
||||
dma_make_data(src_spad, next_task.tile_src_base),
|
||||
tile_stride, tile_size, tile_size, n_k_tiles);
|
||||
}
|
||||
}
|
||||
|
||||
dma_queue_flush(dma_q);
|
||||
}
|
||||
|
||||
int op_get_rows(struct htp_ops_context * octx) {
|
||||
const struct htp_get_rows_kernel_params * kparams = (const struct htp_get_rows_kernel_params *) octx->kernel_params;
|
||||
|
||||
if (octx->src[0]->type != HTP_TYPE_F32 &&
|
||||
octx->src[0]->type != HTP_TYPE_F16 &&
|
||||
octx->src[0]->type != HTP_TYPE_Q8_0 &&
|
||||
octx->src[0]->type != HTP_TYPE_I32) {
|
||||
octx->src[0]->type != HTP_TYPE_F16 &&
|
||||
octx->src[0]->type != HTP_TYPE_Q4_0 &&
|
||||
octx->src[0]->type != HTP_TYPE_Q8_0 &&
|
||||
octx->src[0]->type != HTP_TYPE_I32) {
|
||||
return HTP_STATUS_NO_SUPPORT;
|
||||
}
|
||||
|
||||
if ((octx->src[0]->type == HTP_TYPE_I32 && octx->dst->type != HTP_TYPE_I32) ||
|
||||
(octx->src[0]->type != HTP_TYPE_I32 && octx->dst->type != HTP_TYPE_F32)) {
|
||||
return HTP_STATUS_NO_SUPPORT;
|
||||
if (kparams->kernel_type == HTP_GET_ROWS_KERNEL_SAMETYPE) {
|
||||
if (octx->src[0]->type != octx->dst->type) {
|
||||
return HTP_STATUS_NO_SUPPORT;
|
||||
}
|
||||
} else {
|
||||
if (octx->dst->type != HTP_TYPE_F32) {
|
||||
return HTP_STATUS_NO_SUPPORT;
|
||||
}
|
||||
}
|
||||
|
||||
if (octx->src[1]->type != HTP_TYPE_I32 && octx->src[1]->type != HTP_TYPE_I64) {
|
||||
@@ -262,33 +425,48 @@ int op_get_rows(struct htp_ops_context * octx) {
|
||||
grctx.vtcm_base = (uint8_t *)octx->ctx->vtcm_base;
|
||||
grctx.task_start = task_start;
|
||||
grctx.tasks = tasks;
|
||||
grctx.tasks_per_thread = fastdiv(tasks + n_threads - 1, &octx->n_threads_div);
|
||||
grctx.tasks_per_thread = octx->ctx->mdev.count == 1 ? kparams->tasks_per_thread : fastdiv(tasks + n_threads - 1, &octx->n_threads_div);
|
||||
grctx.tile_size = octx->src[0]->type == HTP_TYPE_Q4_0 ? HTP_MM_WEIGHT_TILE_SIZE_Q4_0 : HTP_MM_WEIGHT_TILE_SIZE_Q8_0;
|
||||
grctx.tile_stride = (grctx.tile_size + 127) & ~127;
|
||||
grctx.index_i32 = octx->src[1]->type == HTP_TYPE_I32;
|
||||
|
||||
const uint32_t ne00 = octx->src[0]->ne[0];
|
||||
htp_get_rows_vtcm_layout_build(&grctx.vtcm_layout, octx->src[0]->type, ne00, n_threads);
|
||||
htp_get_rows_vtcm_layout_build(&grctx.vtcm_layout, kparams->kernel_type, octx->src[0]->type, ne00, n_threads);
|
||||
|
||||
if (grctx.vtcm_layout.total_bytes > octx->ctx->vtcm_size) {
|
||||
FARF(ERROR, "get-rows: VTCM reservation %zu is too small, needed %zu\n",
|
||||
octx->ctx->vtcm_size, grctx.vtcm_layout.total_bytes);
|
||||
return HTP_STATUS_INVAL_PARAMS;
|
||||
}
|
||||
|
||||
const bool is_i32 = (octx->src[1]->type == HTP_TYPE_I32);
|
||||
|
||||
work_queue_func_t q_func = NULL;
|
||||
if (kparams->use_dma) {
|
||||
q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_st_int32_t : get_rows_thread_st_int64_t);
|
||||
} else {
|
||||
switch (octx->src[0]->type) {
|
||||
case HTP_TYPE_F32: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_f32_int32_t : get_rows_thread_f32_int64_t); break;
|
||||
case HTP_TYPE_F16: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_f16_int32_t : get_rows_thread_f16_int64_t); break;
|
||||
case HTP_TYPE_Q8_0: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_q8_0_int32_t : get_rows_thread_q8_0_int64_t); break;
|
||||
case HTP_TYPE_I32: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_st_int32_t : get_rows_thread_st_int64_t); break;
|
||||
default: return HTP_STATUS_NO_SUPPORT;
|
||||
}
|
||||
switch (kparams->kernel_type) {
|
||||
case HTP_GET_ROWS_KERNEL_SAMETYPE:
|
||||
q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_st_int32_t : get_rows_thread_st_int64_t);
|
||||
break;
|
||||
case HTP_GET_ROWS_KERNEL_TILED:
|
||||
q_func = get_rows_thread_tiled;
|
||||
break;
|
||||
case HTP_GET_ROWS_KERNEL_FLAT:
|
||||
switch (octx->src[0]->type) {
|
||||
case HTP_TYPE_F16: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_f16_int32_t : get_rows_thread_f16_int64_t); break;
|
||||
case HTP_TYPE_Q8_0: q_func = (work_queue_func_t)(is_i32 ? get_rows_thread_q8_0_int32_t : get_rows_thread_q8_0_int64_t); break;
|
||||
default: return HTP_STATUS_NO_SUPPORT;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
return HTP_STATUS_NO_SUPPORT;
|
||||
}
|
||||
|
||||
FARF(HIGH, "get-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu use-dma %d n-threads %d\n",
|
||||
FARF(HIGH, "get-rows: (%ux%ux%ux%u) x (%ux%ux%ux%u) -> (%ux%ux%ux%u) : src0-vtcm-size %zu dst-vtcm-size %zu kernel-type %d n-threads %d\n",
|
||||
octx->src[0]->ne[0], octx->src[0]->ne[1], octx->src[0]->ne[2], octx->src[0]->ne[3],
|
||||
octx->src[1]->ne[0], octx->src[1]->ne[1], octx->src[1]->ne[2], octx->src[1]->ne[3],
|
||||
octx->dst->ne[0], octx->dst->ne[1], octx->dst->ne[2], octx->dst->ne[3],
|
||||
grctx.vtcm_layout.src0_bytes_per_thread * n_threads,
|
||||
grctx.vtcm_layout.dst_bytes_per_thread * n_threads,
|
||||
kparams->use_dma, n_threads);
|
||||
kparams->kernel_type, n_threads);
|
||||
|
||||
work_queue_run(octx->ctx->work_queue, q_func, &grctx, n_threads);
|
||||
return HTP_STATUS_OK;
|
||||
|
||||
@@ -1,11 +1,21 @@
|
||||
#ifndef HTP_GET_ROWS_OPS_H
|
||||
#define HTP_GET_ROWS_OPS_H
|
||||
|
||||
#include <stdbool.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "hex-fastdiv.h"
|
||||
#include "matmul-ops.h"
|
||||
|
||||
enum htp_get_rows_kernel_type {
|
||||
HTP_GET_ROWS_KERNEL_SAMETYPE = 0,
|
||||
HTP_GET_ROWS_KERNEL_TILED,
|
||||
HTP_GET_ROWS_KERNEL_FLAT,
|
||||
};
|
||||
|
||||
struct htp_get_rows_kernel_params {
|
||||
int32_t n_threads;
|
||||
int32_t use_dma;
|
||||
int32_t kernel_type;
|
||||
int32_t chunks_per_row;
|
||||
int32_t chunk_size;
|
||||
int32_t total_tasks;
|
||||
@@ -34,19 +44,37 @@ struct htp_get_rows_vtcm_layout {
|
||||
|
||||
static inline void htp_get_rows_vtcm_layout_build(
|
||||
struct htp_get_rows_vtcm_layout * vtcm_layout,
|
||||
int kernel_type,
|
||||
int type,
|
||||
uint32_t ne00,
|
||||
uint32_t n_threads) {
|
||||
|
||||
if (kernel_type == HTP_GET_ROWS_KERNEL_SAMETYPE) {
|
||||
memset(vtcm_layout, 0, sizeof(*vtcm_layout));
|
||||
return;
|
||||
}
|
||||
|
||||
if (kernel_type == HTP_GET_ROWS_KERNEL_TILED) {
|
||||
const size_t tile_size = type == HTP_TYPE_Q4_0 ? HTP_MM_WEIGHT_TILE_SIZE_Q4_0 : HTP_MM_WEIGHT_TILE_SIZE_Q8_0;
|
||||
const size_t tile_stride = (tile_size + 127) & ~127;
|
||||
const uint32_t n_k_tiles = ne00 / HTP_MM_HMX_TILE_N_COLS;
|
||||
const size_t row_tiles_size = n_k_tiles > 0 ? (n_k_tiles * tile_stride) : tile_stride;
|
||||
vtcm_layout->src0_spad_half_size = (row_tiles_size + 255) & ~255;
|
||||
vtcm_layout->dst_spad_half_size = (ne00 * sizeof(float) + 255) & ~255;
|
||||
vtcm_layout->src0_bytes_per_thread = 2 * vtcm_layout->src0_spad_half_size;
|
||||
vtcm_layout->dst_bytes_per_thread = 2 * vtcm_layout->dst_spad_half_size;
|
||||
vtcm_layout->off_src0 = 0;
|
||||
vtcm_layout->off_dst = vtcm_layout->src0_bytes_per_thread * n_threads;
|
||||
vtcm_layout->total_bytes = vtcm_layout->off_dst + vtcm_layout->dst_bytes_per_thread * n_threads;
|
||||
return;
|
||||
}
|
||||
|
||||
uint32_t src0_row_size = 0;
|
||||
switch (type) {
|
||||
case 0: // HTP_TYPE_F32
|
||||
src0_row_size = ne00 * 4;
|
||||
break;
|
||||
case 1: // HTP_TYPE_F16
|
||||
case HTP_TYPE_F16:
|
||||
src0_row_size = ne00 * 2;
|
||||
break;
|
||||
case 8: // HTP_TYPE_Q8_0
|
||||
case HTP_TYPE_Q8_0:
|
||||
src0_row_size = (ne00 / 32) * 34;
|
||||
break;
|
||||
default:
|
||||
|
||||
Reference in New Issue
Block a user