public_git/whisper.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/whisper.cpp.git synced 2026-10-21 18:19:35 +02:00
Code Issues Packages Projects Releases Wiki Activity

hexagon: CPY/CONCAT/CONT/DUP overhaul to use DMA/HVX for all cases (llama/30067)

Browse Source 
* hex-cpy: replace more paths with dma and simplify l2flush

* hex-cpy: use DMA in all sametype paths

* hex-cpy: rewrite the rest of the copy paths (diff type) to use dma

* hex-concat: use dma for multi-dev path which also removes the need for l2-line alignment

* hex-concat: proper support for mdev splitting

* hex-concat: cleanup ctx and kern params usage

* hex-cpy: cleanup contex and remove left-over non-dma checks

* hex-cpy: clean dma_cpy naming

* hex-cpy: proper kernel params and kernel selection

* hex-build: resolve left-over rebase conflicts

* hex-cpy: update dev guide to clarify 128 byte alignment requirement

* hex-concat: make sure we go through mdev barrier

* hex-concat: make sure to flush dma-queue

* hex-cpy/concat: cleanup kparams and vtcm layout handling

* hex-cpy: remove dead check for contig (routed to diff kernel) and update comments

* hex-dev: update developer guide based on latest changes

* hex-cpy: safe skip of noop copies

* hex-dup: route DUP to CPY
This commit is contained in:
Max Krasnyansky
2026-10-19 09:47:10 +02:00
committed by Daniel Bevenius
parent 2d5d68ea29
commit 8d13197d65
8 changed files with 978 additions and 736 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/src/ggml-hexagon/ggml-hexagon.cpp
+243 -58
View File
@@ -71,6 +71,8 @@
#include "htp/ssm-conv.h"
#include "htp/gated-delta-net-ops.h"
#include "htp/argsort-ops.h"
#include "htp/concat-ops.h"
#include "htp/cpy-ops.h"
#include "htp_iface.h"
#include "htp-drv.h"
@@ -400,6 +402,18 @@ static void ggml_hexagon_precompute_pool_2d_params(
bool is_pool_1d
);
static bool ggml_hexagon_precompute_concat_params(
const struct ggml_hexagon_session * sess,
const struct ggml_tensor * op,
struct htp_concat_kernel_params * kparams
);
static bool ggml_hexagon_precompute_cpy_params(
const struct ggml_hexagon_session * sess,
const struct ggml_tensor * op,
struct htp_copy_kernel_params * kparams
);
static void ggml_hexagon_precompute_fused_mmnx_params(
const struct ggml_hexagon_session * sess,
const struct ggml_tensor * src0,
@@ -4260,6 +4274,11 @@ void ggml_hexagon_session::enqueue_cpy(const ggml_tensor * src, ggml_tensor * ds
if (with_fence) {
cpy_node.name = "CPY+FENCE";
}
const bool ok = ggml_hexagon_precompute_cpy_params(this, node, (struct htp_copy_kernel_params *) cpy_node.kernel_params);
const auto * kparams = (const struct htp_copy_kernel_params *) cpy_node.kernel_params;
if (ok && !with_fence && kparams->total_elems == 0) {
return;
}
this->enqueue_op(cpy_node);
}
@@ -6350,6 +6369,208 @@ static void ggml_hexagon_precompute_pool_2d_params(
kparams->inv_kernel_area = 1.0f / (float) (kparams->kernel_x * kparams->kernel_y);
}
static bool ggml_hexagon_precompute_concat_params(
const struct ggml_hexagon_session * sess,
const struct ggml_tensor * op,
struct htp_concat_kernel_params * kparams
) {
memset(kparams, 0, sizeof(*kparams));
kparams->kernel_type = HTP_CONCAT_KERNEL_UNSUPPORTED;
const struct ggml_tensor * src0 = op->src[0];
const struct ggml_tensor * src1 = op->src[1];
const struct ggml_tensor * dst = op;
if (!src0 || !src1 || !dst) {
return false;
}
int dim = ((const int32_t *) op->op_params)[0];
if (dim < 0 || dim >= GGML_MAX_DIMS) {
return false;
}
kparams->dim = dim;
if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_I32) {
return false;
}
if (src0->type != dst->type || src1->type != dst->type) {
return false;
}
const uint32_t type_size = ggml_type_size(dst->type);
for (int d = 0; d < GGML_MAX_DIMS; d++) {
const int64_t ne_d = (d == dim) ? src0->ne[d] + src1->ne[d] : src0->ne[d];
if (dst->ne[d] != ne_d || (d != dim && src1->ne[d] != dst->ne[d])) {
return false;
}
}
const bool dma_strides_ok = (src0->nb[0] == type_size && src1->nb[0] == type_size && dst->nb[0] == type_size);
if (dma_strides_ok) {
kparams->kernel_type = HTP_CONCAT_KERNEL_REGULAR;
kparams->n_threads = 1;
return true;
}
const bool is_src1_transposed = (src1->nb[0] > src1->nb[1]);
const bool is_src0_transposed = (src0->nb[0] > src0->nb[1]);
const bool transposed_rows_ok = (src0->nb[0] == type_size && src1->nb[1] == type_size && dst->nb[0] == type_size);
if (dim == 0 && is_src1_transposed && !is_src0_transposed && transposed_rows_ok &&
(dst->type == GGML_TYPE_F32 || dst->type == GGML_TYPE_F16)) {
const uint32_t n_threads = sess->n_threads > 0 ? (uint32_t) sess->n_threads : 8;
struct htp_concat_transposed_vtcm_layout layout;
htp_concat_transposed_vtcm_layout_build(&layout, (uint32_t) src0->ne[0], (uint32_t) src1->ne[0], type_size, n_threads);
if (sess->vtcm_size > 0 && layout.total_bytes > sess->vtcm_size) {
return false;
}
kparams->kernel_type = HTP_CONCAT_KERNEL_TRANSPOSED;
kparams->n_threads = n_threads;
kparams->vtcm_size = layout.total_bytes;
kparams->spad0_size_per_thread = layout.src0_spad_size_per_thread;
kparams->spad1_size_per_thread = layout.src1_spad_size_per_thread;
return true;
}
return false;
}
static bool ggml_hexagon_precompute_cpy_params(
const struct ggml_hexagon_session * sess,
const struct ggml_tensor * op,
struct htp_copy_kernel_params * kparams
) {
memset(kparams, 0, sizeof(*kparams));
kparams->kernel_type = HTP_COPY_KERNEL_UNSUPPORTED;
const struct ggml_tensor * src0 = op->src[0];
const struct ggml_tensor * dst = op;
if (!src0 || !dst) {
return false;
}
if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 && src0->type != GGML_TYPE_I32) {
return false;
}
if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16 && dst->type != GGML_TYPE_I32) {
return false;
}
const int64_t nelem_src = ggml_nelements(src0);
const int64_t nelem_dst = ggml_nelements(dst);
if (nelem_src != nelem_dst || nelem_src < 0) {
return false;
}
const uint32_t src_type_size = ggml_type_size(src0->type);
const uint32_t dst_type_size = ggml_type_size(dst->type);
kparams->src0_type_size = (uint8_t) src_type_size;
kparams->dst_type_size = (uint8_t) dst_type_size;
kparams->total_elems = (uint32_t) nelem_src;
if (nelem_src == 0) {
kparams->kernel_type = HTP_COPY_KERNEL_1D_CONTIG;
return true;
}
if (nelem_src == 1) {
if (src0->type == dst->type) {
kparams->kernel_type = HTP_COPY_KERNEL_SCALAR;
return true;
}
if ((src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_I32) ||
(src0->type == GGML_TYPE_I32 && dst->type == GGML_TYPE_F32) ||
(src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16) ||
(src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F32)) {
kparams->kernel_type = HTP_COPY_KERNEL_SCALAR;
return true;
}
return false;
}
const bool sametype = (src0->type == dst->type);
bool same_extents = true;
for (int d = 0; d < GGML_MAX_DIMS; d++) {
if (src0->ne[d] != dst->ne[d]) {
same_extents = false;
break;
}
}
const bool transposed = (src0->nb[0] > src0->nb[1]) || (dst->nb[0] > dst->nb[1]) ||
(src0->nb[0] != src_type_size) || (dst->nb[0] != dst_type_size) ||
(src0->nb[1] < (size_t) src0->ne[0] * src_type_size) || (dst->nb[1] < (size_t) dst->ne[0] * dst_type_size);
const bool sameshape = same_extents && !transposed;
const bool src_is_contiguous = ggml_is_contiguous(src0);
const bool dst_is_contiguous = ggml_is_contiguous(dst);
if (sametype) {
if (src_is_contiguous && dst_is_contiguous) {
kparams->kernel_type = HTP_COPY_KERNEL_1D_CONTIG;
return true;
}
if (sameshape) {
kparams->kernel_type = HTP_COPY_KERNEL_SAMESHAPE_SAMETYPE;
kparams->total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]);
return true;
}
kparams->kernel_type = HTP_COPY_KERNEL_RESHAPE;
kparams->n_threads = sess->n_threads > 0 ? (uint8_t) sess->n_threads : 4;
kparams->u.reshape.div_ne0 = init_fastdiv_values((uint32_t) dst->ne[0]);
kparams->u.reshape.div_ne1_ne0 = init_fastdiv_values((uint32_t) (dst->ne[1] * dst->ne[0]));
kparams->u.reshape.div_ne2_ne1_ne0 = init_fastdiv_values((uint32_t) (dst->ne[2] * dst->ne[1] * dst->ne[0]));
kparams->u.reshape.div_ne00 = init_fastdiv_values((uint32_t) src0->ne[0]);
kparams->u.reshape.div_ne01_ne00 = init_fastdiv_values((uint32_t) (src0->ne[1] * src0->ne[0]));
kparams->u.reshape.div_ne02_ne01_ne00 = init_fastdiv_values((uint32_t) (src0->ne[2] * src0->ne[1] * src0->ne[0]));
return true;
}
if (!sameshape) {
return false;
}
const bool valid_conversion = (src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_F16) ||
(src0->type == GGML_TYPE_F16 && dst->type == GGML_TYPE_F32) ||
(src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_I32) ||
(src0->type == GGML_TYPE_I32 && dst->type == GGML_TYPE_F32);
if (!valid_conversion) {
return false;
}
const uint32_t n_threads = sess->n_threads > 0 ? (uint32_t) sess->n_threads : 4;
struct htp_copy_convert_vtcm_layout layout;
htp_copy_convert_vtcm_layout_build(&layout, (uint32_t) src0->ne[0], src_type_size, dst_type_size, n_threads);
if (sess->vtcm_size > 0 && layout.total_bytes > sess->vtcm_size) {
return false;
}
kparams->kernel_type = HTP_COPY_KERNEL_SAMESHAPE_CONVERT;
kparams->total_rows = (uint32_t) (src0->ne[1] * src0->ne[2] * src0->ne[3]);
kparams->n_threads = (uint8_t) n_threads;
kparams->vtcm_size = layout.total_bytes;
kparams->u.convert.src0_buf_size = layout.src0_buf_size;
kparams->u.convert.dst_buf_size = layout.dst_buf_size;
kparams->u.convert.spad0_size_per_thread = layout.spad0_size_per_thread;
kparams->u.convert.spad1_size_per_thread = layout.spad1_size_per_thread;
kparams->u.convert.div_ne01 = init_fastdiv_values((uint32_t) src0->ne[1]);
kparams->u.convert.div_ne02_ne01 = init_fastdiv_values((uint32_t) (src0->ne[2] * src0->ne[1]));
return true;
}
static void ggml_hexagon_precompute_fused_mmnx_params(
const struct ggml_hexagon_session * sess,
const struct ggml_tensor * src0, // W0
@@ -7416,6 +7637,7 @@ static htp_op_code op_remap_to_htp(const ggml_tensor * t) {
case GGML_OP_ADD_ID: return HTP_OP_ADD_ID;
case GGML_OP_SUB: return HTP_OP_SUB;
case GGML_OP_DIV: return HTP_OP_DIV;
case GGML_OP_DUP:
case GGML_OP_CPY: return HTP_OP_CPY;
case GGML_OP_CONT: return HTP_OP_CPY;
case GGML_OP_GET_ROWS: return HTP_OP_GET_ROWS;
@@ -7716,8 +7938,21 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg
ggml_hexagon_precompute_pool_2d_params(
sess, node.node->src[0], node.dst(),
(struct htp_pool_2d_kernel_params *)node.kernel_params,
node.opcode == HTP_OP_POOL_1D
node.opcode == HTP_OP_POOL_1D);
} else if (node.opcode == HTP_OP_CONCAT) {
ggml_hexagon_precompute_concat_params(sess,
node.node,
(struct htp_concat_kernel_params *) node.kernel_params
);
} else if (node.opcode == HTP_OP_CPY || node.opcode == HTP_OP_CPY_FENCE) {
const bool ok = ggml_hexagon_precompute_cpy_params(sess,
node.node,
(struct htp_copy_kernel_params *) node.kernel_params
);
const auto * kparams = (const struct htp_copy_kernel_params *) node.kernel_params;
if (ok && node.opcode == HTP_OP_CPY && kparams->total_elems == 0) {
continue;
}
}
computed_nodes.push_back(std::move(node));
}
@@ -8346,49 +8581,13 @@ static ggml_backend_buffer_type_t ggml_backend_hexagon_device_get_host_buffer_ty
}
static bool ggml_hexagon_supported_cpy(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
GGML_UNUSED(sess);
const struct ggml_tensor * src0 = op->src[0];
const struct ggml_tensor * dst = op;
if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 &&
src0->type != GGML_TYPE_I32) return false;
if (dst->type != GGML_TYPE_F32 && dst->type != GGML_TYPE_F16 &&
dst->type != GGML_TYPE_I32) return false;
const bool is_scalar = (ggml_nelements(src0) == 1 && ggml_nelements(dst) == 1);
const bool sametype = (src0->type == dst->type);
const bool transposed = !is_scalar && (ggml_is_transposed(src0) || ggml_is_transposed(dst));
const bool sameshape = is_scalar || (!transposed && ggml_are_same_shape(src0, dst));
if (src0->type == GGML_TYPE_I32 || dst->type == GGML_TYPE_I32) {
if (!sameshape) return false;
if (sametype) return true;
if ((src0->type == GGML_TYPE_F32 && dst->type == GGML_TYPE_I32) ||
(src0->type == GGML_TYPE_I32 && dst->type == GGML_TYPE_F32)) {
return true;
}
return false;
}
// can handle any shape and any same-type (pretty slow if reshaping is required)
if (sametype) return true;
// cannot handle re-shaping and type conversion at the same time
if (!sameshape) return false;
return true;
struct htp_copy_kernel_params kparams;
return ggml_hexagon_precompute_cpy_params(sess, op, &kparams);
}
static bool ggml_hexagon_supported_cont(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
GGML_UNUSED(sess);
const struct ggml_tensor * src0 = op->src[0];
// CONT is same-type only and supports F32, F16, and I32.
if (src0->type != GGML_TYPE_F32 && src0->type != GGML_TYPE_F16 &&
src0->type != GGML_TYPE_I32) return false;
return true;
struct htp_copy_kernel_params kparams;
return ggml_hexagon_precompute_cpy_params(sess, op, &kparams);
}
static bool ggml_hexagon_supported_repeat(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
@@ -8415,23 +8614,8 @@ static bool ggml_hexagon_supported_repeat(const struct ggml_hexagon_session * se
}
static bool ggml_hexagon_supported_concat(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
int dim = ((const int32_t *) op->op_params)[0];
if (dim < 0 || dim >= GGML_MAX_DIMS) {
return false;
}
for (int i = 0; i < GGML_MAX_SRC; ++i) {
const struct ggml_tensor * src = op->src[i];
if (!src) {
continue;
}
if (src->type != GGML_TYPE_F32 && src->type != GGML_TYPE_I32 && src->type != GGML_TYPE_F16) {
return false;
}
}
return true;
GGML_UNUSED(sess);
struct htp_concat_kernel_params kparams;
return ggml_hexagon_precompute_concat_params(sess, op, &kparams);
}
static bool ggml_hexagon_supported_fill(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
@@ -8601,6 +8785,7 @@ static bool ggml_backend_hexagon_device_supports_op(ggml_backend_dev_t dev, cons
supp = ggml_hexagon_supported_get_rows(sess, op);
break;
case GGML_OP_DUP:
case GGML_OP_CPY:
supp = ggml_hexagon_supported_cpy(sess, op);
break;
ggml/src/ggml-hexagon/htp/concat-ops.c
+106 -200
View File
@@ -1,11 +1,13 @@
#include "concat-ops.h"
#include "dma-queue.h"
#include "hex-common.h"
#include "hex-cpy-dma.h"
#include "dma-copy.h"
#include "hex-fastdiv.h"
#include "hex-profile.h"
#include "hexagon_protos.h"
#include "hexagon_types.h"
#include "htp-ctx.h"
#include "htp-fence.h"
#include "htp-ops.h"
#include "htp-tensor.h"
#include "htp-vtcm.h"
@@ -16,15 +18,14 @@
struct htp_concat_context {
struct htp_ops_context * octx;
uint32_t dim;
uint32_t nrows_per_thread;
uint32_t row_start;
uint32_t nrows;
uint32_t elem_start;
uint32_t nelems;
uint32_t nplanes;
struct fastdiv_values div_ne0;
struct fastdiv_values div_ne1;
uint8_t * spad0_base;
uint8_t * spad1_base;
uint32_t spad0_size_per_thread;
uint32_t spad1_size_per_thread;
uint32_t row_start;
uint32_t nrows;
uint32_t nrows_per_thread;
uint32_t nplanes;
struct fastdiv_values div_ne2;
};
@@ -52,8 +53,8 @@ static void concat_2d_f32_transposed(unsigned int nth, unsigned int ith, void *
dma_queue * dma_q = octx->ctx->dma[ith];
uint8_t * spad0_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_thread;
uint8_t * spad1_base = octx->src1_spad.data + ith * octx->src1_spad.size_per_thread;
uint8_t * spad0_base = cctx->spad0_base + ith * cctx->spad0_size_per_thread;
uint8_t * spad1_base = cctx->spad1_base + ith * cctx->spad1_size_per_thread;
const uint32_t block_i = 32;
const uint32_t spad1_stride = block_i * sizeof(float);
@@ -127,6 +128,7 @@ static void concat_2d_f32_transposed(unsigned int nth, unsigned int ith, void *
p = np;
i = ni;
}
dma_queue_flush(dma_q);
}
static void concat_2d_f16_transposed(unsigned int nth, unsigned int ith, void * data) {
@@ -147,8 +149,8 @@ static void concat_2d_f16_transposed(unsigned int nth, unsigned int ith, void *
dma_queue * dma_q = octx->ctx->dma[ith];
uint8_t * spad0_base = octx->src0_spad.data + ith * octx->src0_spad.size_per_thread;
uint8_t * spad1_base = octx->src1_spad.data + ith * octx->src1_spad.size_per_thread;
uint8_t * spad0_base = cctx->spad0_base + ith * cctx->spad0_size_per_thread;
uint8_t * spad1_base = cctx->spad1_base + ith * cctx->spad1_size_per_thread;
const uint32_t block_i = 64;
const uint32_t spad1_stride = block_i * sizeof(__fp16);
@@ -222,219 +224,123 @@ static void concat_2d_f16_transposed(unsigned int nth, unsigned int ith, void *
p = np;
i = ni;
}
dma_queue_flush(dma_q);
}
static void concat_generic(unsigned int nth, unsigned int ith, void * data) {
struct htp_concat_context * cctx = (struct htp_concat_context *) data;
struct htp_ops_context * octx = cctx->octx;
static int concat_regular(struct htp_ops_context * octx, int dim, uint32_t type_size) {
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * src1 = octx->src[1];
const struct htp_tensor * dst = octx->dst;
const int dim = cctx->dim;
const uint32_t type_size = (dst->type == HTP_TYPE_F32 || dst->type == HTP_TYPE_I32) ? 4 : 2;
const uint32_t ne[4] = {dst->ne[0], dst->ne[1], dst->ne[2], dst->ne[3]};
// Per-device element range aligned to prevent false sharing
const uint32_t elem_start = cctx->elem_start;
const uint32_t nelems = cctx->nelems;
const uint32_t chunk_size = fastdiv(nelems + nth - 1, &octx->n_threads_div);
const uint32_t start_idx = MIN(elem_start + ith * chunk_size, elem_start + nelems);
const uint32_t end_idx = MIN(start_idx + chunk_size, elem_start + nelems);
// Naive scalar element-wise copy
for (uint32_t idx = start_idx; idx < end_idx; idx++) {
uint32_t idx_div_ne0 = fastdiv(idx, &cctx->div_ne0);
uint32_t i0 = idx - idx_div_ne0 * ne[0];
uint32_t idx_div_ne01 = fastdiv(idx_div_ne0, &cctx->div_ne1);
uint32_t i1 = idx_div_ne0 - idx_div_ne01 * ne[1];
uint32_t idx_div_ne012 = fastdiv(idx_div_ne01, &cctx->div_ne2);
uint32_t i2 = idx_div_ne01 - idx_div_ne012 * ne[2];
uint32_t i3 = idx_div_ne012;
uint8_t * dst_ptr = (uint8_t *)dst->data + i3 * dst->nb[3] + i2 * dst->nb[2] + i1 * dst->nb[1] + i0 * dst->nb[0];
uint32_t idx_dim = 0;
if (dim == 0) idx_dim = i0;
else if (dim == 1) idx_dim = i1;
else if (dim == 2) idx_dim = i2;
else if (dim == 3) idx_dim = i3;
const struct htp_tensor * src = (idx_dim < src0->ne[dim]) ? src0 : src1;
uint32_t s0 = i0;
uint32_t s1 = i1;
uint32_t s2 = i2;
uint32_t s3 = i3;
if (dim == 0 && src == src1) s0 -= src0->ne[0];
if (dim == 1 && src == src1) s1 -= src0->ne[1];
if (dim == 2 && src == src1) s2 -= src0->ne[2];
if (dim == 3 && src == src1) s3 -= src0->ne[3];
uint8_t * src_ptr = (uint8_t *)src->data + s3 * src->nb[3] + s2 * src->nb[2] + s1 * src->nb[1] + s0 * src->nb[0];
if (type_size == 4) {
*(float*)dst_ptr = *(float*)src_ptr;
} else {
*(__fp16*)dst_ptr = *(__fp16*)src_ptr;
}
}
}
static bool concat_dma(struct htp_ops_context * octx, int dim, uint32_t type_size) {
if (dim < 0 || dim >= HTP_OP_MAX_DIMS) {
return false;
}
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * src1 = octx->src[1];
const struct htp_tensor * dst = octx->dst;
// Not partitioned across devices: the row/element-split paths handle that.
if (octx->ctx->mdev.count > 1 ||
(dst->type != HTP_TYPE_F32 && dst->type != HTP_TYPE_F16 && dst->type != HTP_TYPE_I32) ||
src0->type != dst->type || src1->type != dst->type || src0->nb[0] != type_size || src1->nb[0] != type_size ||
dst->nb[0] != type_size || (size_t) dst->ne[0] * type_size > DMA_MAX_SIZE_24B ||
dst->nb[1] > DMA_MAX_STRIDE_24B || src0->nb[1] > DMA_MAX_STRIDE_24B || src1->nb[1] > DMA_MAX_STRIDE_24B) {
return false;
}
for (int d = 0; d < HTP_OP_MAX_DIMS; d++) {
const uint32_t ne_d = (d == dim) ? src0->ne[d] + src1->ne[d] : src0->ne[d];
if (dst->ne[d] != ne_d || (d != dim && src1->ne[d] != dst->ne[d])) {
return false;
}
}
// The two views of dst, shaped like the sources.
struct htp_tensor view0 = *dst;
struct htp_tensor view1 = *dst;
for (int d = 0; d < HTP_OP_MAX_DIMS; d++) {
view0.ne[d] = src0->ne[d];
view1.ne[d] = src1->ne[d];
}
view1.data += (uint64_t) src0->ne[dim] * dst->nb[dim];
view1.data += src0->ne[dim] * dst->nb[dim];
const uint32_t total_rows_0 = src0->ne[1] * src0->ne[2] * src0->ne[3];
const uint32_t total_rows_1 = src1->ne[1] * src1->ne[2] * src1->ne[3];
uint32_t rstart0 = 0, nrows0 = total_rows_0;
uint32_t rstart1 = 0, nrows1 = total_rows_1;
if (octx->ctx->mdev.count > 1) {
const struct htp_tensor_mdev_range range0 = htp_tensor_mdev_partition(
total_rows_0, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
rstart0 = range0.start;
nrows0 = range0.count;
const struct htp_tensor_mdev_range range1 = htp_tensor_mdev_partition(
total_rows_1, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
rstart1 = range1.start;
nrows1 = range1.count;
}
dma_queue * q = octx->ctx->dma[0];
cpy_dma_sametype_sameshape(q, &view0, src0, type_size);
cpy_dma_sametype_sameshape(q, &view1, src1, type_size);
dma_cpy_sametype_sameshape_range(q, &view0, src0, type_size, rstart0, nrows0);
dma_cpy_sametype_sameshape_range(q, &view1, src1, type_size, rstart1, nrows1);
dma_queue_flush(q);
return true;
return HTP_STATUS_OK;
}
int op_concat(struct htp_ops_context * octx) {
int dim = octx->op_params[0];
if (dim < 0 || dim >= HTP_OP_MAX_DIMS) {
return HTP_STATUS_NO_SUPPORT;
static int concat_transposed(struct htp_ops_context * octx, const struct htp_concat_kernel_params * kparams, uint32_t type_size) {
if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
return HTP_STATUS_INVAL_PARAMS;
}
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * src1 = octx->src[1];
const struct htp_tensor * dst = octx->dst;
const struct htp_tensor * dst = octx->dst;
const uint32_t type_size = (dst->type == HTP_TYPE_F32 || dst->type == HTP_TYPE_I32) ? 4 : 2;
bool is_src1_transposed = (src1->nb[0] > src1->nb[1]);
bool is_src0_transposed = (src0->nb[0] > src0->nb[1]);
const uint32_t total_rows = dst->ne[1];
uint32_t row_start = 0;
uint32_t nrows = total_rows;
if (octx->ctx->mdev.count > 1) {
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
row_start = range.start;
nrows = range.count;
}
if (concat_dma(octx, dim, type_size)) {
if (nrows == 0 || dst->ne[2] == 0 || dst->ne[3] == 0) {
return HTP_STATUS_OK;
}
uint32_t n_threads = octx->n_threads;
struct htp_concat_context cctx;
cctx.octx = octx;
cctx.dim = dim;
cctx.div_ne0 = init_fastdiv_values(dst->ne[0]);
cctx.div_ne1 = init_fastdiv_values(dst->ne[1]);
cctx.div_ne2 = init_fastdiv_values(dst->ne[2]);
void (*worker_func)(unsigned int, unsigned int, void *) = concat_generic;
const bool rows_ok = src0->nb[0] == type_size && src1->nb[1] == type_size && dst->nb[0] == type_size;
if (dim == 0 && is_src1_transposed && !is_src0_transposed && rows_ok) {
const uint32_t total_rows = dst->ne[1];
const size_t dst_data_row_size = dst->ne[0] * type_size;
uint32_t row_start = 0;
uint32_t nrows = total_rows;
if (octx->ctx->mdev.count > 1) {
uint32_t rows_per_chunk = 0;
htp_tensor_mdev_rows_per_chunk(dst, type_size, (uint32_t) dst_data_row_size, &rows_per_chunk);
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, rows_per_chunk, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
row_start = range.start;
nrows = range.count;
}
if (nrows == 0) {
return HTP_STATUS_OK;
}
cctx.row_start = row_start;
cctx.nrows = nrows;
cctx.nplanes = dst->ne[2] * dst->ne[3];
uint32_t block_i = (type_size == 4) ? 32 : 64;
cctx.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
// Allocate VTCM
uint32_t spad1_stride = block_i * type_size;
uint32_t src1_ne0_padded = hex_round_up(src1->ne[0], block_i);
// src0 row is right-aligned to VLEN so the gathered src1 part starts aligned
uint32_t spad0_row_bytes = hex_round_up(src0->ne[0] * type_size, VLEN) + src1_ne0_padded * type_size;
octx->src0_spad.size_per_thread = block_i * spad0_row_bytes;
octx->src1_spad.size_per_thread = src1_ne0_padded * spad1_stride;
octx->src0_spad.size = n_threads * octx->src0_spad.size_per_thread;
octx->src1_spad.size = n_threads * octx->src1_spad.size_per_thread;
if (octx->src0_spad.size + octx->src1_spad.size > octx->ctx->vtcm_size) {
return HTP_STATUS_VTCM_TOO_SMALL;
}
octx->src0_spad.data = octx->ctx->vtcm_base;
octx->src1_spad.data = octx->src0_spad.data + octx->src0_spad.size;
octx->src0_spad.src = NULL;
octx->src1_spad.src = NULL;
if (type_size == 4) {
worker_func = concat_2d_f32_transposed;
} else {
worker_func = concat_2d_f16_transposed;
}
} else {
if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(src1) || htp_tensor_is_extended(dst)) {
return HTP_STATUS_NO_SUPPORT;
}
const uint32_t total_elements = dst->ne[0] * dst->ne[1] * dst->ne[2] * dst->ne[3];
uint32_t elem_start = 0;
uint32_t nelems = total_elements;
if (octx->ctx->mdev.count > 1) {
const uint32_t elems_per_chunk = HEX_L2_LINE_SIZE / type_size;
const bool can_split = htp_tensor_mdev_data_aligned(dst) && htp_tensor_is_contiguous(dst, type_size) && !htp_tensor_is_permuted(dst);
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_elements, can_split ? elems_per_chunk : 0, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
elem_start = range.start;
nelems = range.count;
}
if (nelems == 0) {
return HTP_STATUS_OK;
}
cctx.elem_start = elem_start;
cctx.nelems = nelems;
if (kparams->vtcm_size > octx->ctx->vtcm_size) {
return HTP_STATUS_VTCM_TOO_SMALL;
}
const uint32_t n_threads = octx->n_threads;
// layout precomputed on host; kept for reference:
// struct htp_concat_transposed_vtcm_layout layout;
// htp_concat_transposed_vtcm_layout_build(&layout, octx->src[0]->ne[0], octx->src[1]->ne[0], type_size, n_threads);
uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base;
struct htp_concat_context cctx;
cctx.octx = octx;
cctx.spad0_base = vtcm_base;
cctx.spad1_base = vtcm_base + n_threads * kparams->spad0_size_per_thread;
cctx.spad0_size_per_thread = kparams->spad0_size_per_thread;
cctx.spad1_size_per_thread = kparams->spad1_size_per_thread;
cctx.row_start = row_start;
cctx.nrows = nrows;
cctx.nplanes = dst->ne[2] * dst->ne[3];
cctx.div_ne2 = init_fastdiv_values(dst->ne[2]);
cctx.nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
work_queue_func_t worker_func = (type_size == 4) ? concat_2d_f32_transposed : concat_2d_f16_transposed;
work_queue_run(octx->ctx->work_queue, worker_func, &cctx, n_threads);
return HTP_STATUS_OK;
}
int op_concat(struct htp_ops_context * octx) {
const struct htp_concat_kernel_params * kparams = (const struct htp_concat_kernel_params *) octx->kernel_params;
const struct htp_tensor * dst = octx->dst;
const uint32_t type_size = (dst->type == HTP_TYPE_F32 || dst->type == HTP_TYPE_I32) ? 4 : 2;
int status = HTP_STATUS_OK;
switch (kparams->kernel_type) {
case HTP_CONCAT_KERNEL_REGULAR:
status = concat_regular(octx, kparams->dim, type_size);
break;
case HTP_CONCAT_KERNEL_TRANSPOSED:
status = concat_transposed(octx, kparams, type_size);
break;
default:
status = HTP_STATUS_NO_SUPPORT;
break;
}
htp_ops_context_set_status(octx, status);
if (octx->ctx->mdev.count > 1) {
htp_mdev_group_barrier(octx);
}
return octx->status;
}
ggml/src/ggml-hexagon/htp/concat-ops.h
+53
View File
@@ -0,0 +1,53 @@
#ifndef HTP_CONCAT_OPS_H
#define HTP_CONCAT_OPS_H
#include "hex-common.h"
#include <stdint.h>
enum htp_concat_kernel_type {
HTP_CONCAT_KERNEL_UNSUPPORTED = 0,
HTP_CONCAT_KERNEL_REGULAR = 1,
HTP_CONCAT_KERNEL_TRANSPOSED = 2,
};
struct htp_concat_kernel_params {
uint8_t kernel_type;
uint8_t dim;
uint8_t n_threads;
uint8_t pad;
uint32_t vtcm_size;
uint32_t spad0_size_per_thread;
uint32_t spad1_size_per_thread;
};
#if defined(__cplusplus)
static_assert(sizeof(struct htp_concat_kernel_params) <= 128, "htp_concat_kernel_params is too large for kernel_params blob");
#else
_Static_assert(sizeof(struct htp_concat_kernel_params) <= 128, "htp_concat_kernel_params is too large for kernel_params blob");
#endif
struct htp_concat_transposed_vtcm_layout {
uint32_t src0_spad_size_per_thread;
uint32_t src1_spad_size_per_thread;
uint32_t total_bytes;
};
static inline void htp_concat_transposed_vtcm_layout_build(
struct htp_concat_transposed_vtcm_layout * layout,
uint32_t src0_ne0,
uint32_t src1_ne0,
uint32_t type_size,
uint32_t n_threads) {
uint32_t block_i = (type_size == 4) ? 32 : 64;
uint32_t spad1_stride = block_i * type_size;
uint32_t src1_ne0_padded = hex_round_up(src1_ne0, block_i);
uint32_t spad0_row_bytes = hex_round_up(src0_ne0 * type_size, 128) + src1_ne0_padded * type_size;
layout->src0_spad_size_per_thread = block_i * spad0_row_bytes;
layout->src1_spad_size_per_thread = src1_ne0_padded * spad1_stride;
layout->total_bytes = n_threads * (layout->src0_spad_size_per_thread + layout->src1_spad_size_per_thread);
}
#endif // HTP_CONCAT_OPS_H
ggml/src/ggml-hexagon/htp/cpy-ops.c
+339 -427
View File
@@ -11,7 +11,8 @@
#define GGML_COMMON_DECL_C
#include "ggml-common.h"
#include "hex-cpy-dma.h"
#include "cpy-ops.h"
#include "dma-copy.h"
#include "htp-ctx.h"
#include "htp-fence.h"
#include "htp-ops.h"
@@ -19,34 +20,19 @@
#include "hvx-utils.h"
struct htp_copy_context {
struct htp_ops_context * octx;
struct htp_ops_context * octx;
const struct htp_copy_kernel_params * kparams;
uint32_t src0_type_size;
uint32_t src0_block_size;
uint32_t row_start;
uint32_t nrows;
uint32_t src0_nrows_per_thread;
uint32_t dst_type_size;
uint32_t dst_block_size;
uint32_t elem_start;
uint32_t nelem;
uint32_t elem_per_thread;
uint32_t src0_blocks_per_row;
uint32_t dst_blocks_per_row;
uint32_t elem_start;
uint32_t nelem;
uint32_t elem_per_thread;
uint32_t src0_nrows_per_thread;
uint32_t row_start;
uint32_t nrows;
struct fastdiv_values div_ne01;
struct fastdiv_values div_ne02_ne01;
struct fastdiv_values div_ne0;
struct fastdiv_values div_ne1_ne0;
struct fastdiv_values div_ne2_ne1_ne0;
struct fastdiv_values div_ne00;
struct fastdiv_values div_ne01_ne00;
struct fastdiv_values div_ne02_ne01_ne00;
uint8_t * vtcm_src0;
uint8_t * vtcm_dst;
};
#define cpy_preamble \
@@ -73,52 +59,6 @@ struct htp_copy_context {
const uint32_t nb2 = dst->nb[2]; \
const uint32_t nb3 = dst->nb[3];
#define DEFINE_CPY_SAMESHAPE(NAME, ELEM_TYPE, ELEM_SIZE) \
static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, void * data) { \
struct htp_copy_context * ct = (struct htp_copy_context *) data; \
struct htp_ops_context * octx = ct->octx; \
cpy_preamble; \
const uint32_t dr = ct->src0_nrows_per_thread; \
const uint32_t ir0 = ct->row_start + dr * ith; \
const uint32_t ir1 = MIN(ir0 + dr, ct->row_start + ct->nrows); \
if (ir0 >= ir1) return; \
const bool contiguous = htp_tensor_is_contiguous(src0, ELEM_SIZE) && htp_tensor_is_contiguous(dst, ELEM_SIZE); \
if (contiguous) { \
dma_queue * dma_q = octx->ctx->dma[ith]; \
dma_addr_t dst_addr = dst->data + ir0 * ne00 * ELEM_SIZE; \
dma_addr_t src0_addr = src0->data + ir0 * ne00 * ELEM_SIZE; \
cpy_dma_sametype_reshape_contig(dma_q, dst_addr, src0_addr, (ir1 - ir0) * ne00 * ELEM_SIZE); \
dma_queue_flush(dma_q); \
return; \
} \
const uint32_t ne02_ne01 = ne02 * ne01; \
uint32_t i03 = fastdiv(ir0, &ct->div_ne02_ne01); \
uint32_t rem = ir0 - i03 * ne02_ne01; \
uint32_t i02 = fastdiv(rem, &ct->div_ne01); \
uint32_t i01 = rem - i02 * ne01; \
uint8_t * dst_ptr = (uint8_t *) dst->data + i01*nb1 + i02*nb2 + i03*nb3; \
uint8_t * src0_ptr = (uint8_t *) src0->data + i01*nb01 + i02*nb02 + i03*nb03; \
for (uint32_t r = ir0; r < ir1; r++) { \
hex_l2fetch(src0_ptr, ne00 * ELEM_SIZE, nb01, 2); \
hvx_copy_uu(dst_ptr, src0_ptr, ne00, ELEM_SIZE); \
dst_ptr += nb1; \
src0_ptr += nb01; \
if (++i01 == ne01) { \
i01 = 0; \
if (++i02 == ne02) { \
i02 = 0; \
i03++; \
} \
dst_ptr = (uint8_t *) dst->data + i02*nb2 + i03*nb3; \
src0_ptr = (uint8_t *) src0->data + i02*nb02 + i03*nb03; \
} \
} \
}
DEFINE_CPY_SAMESHAPE(f32, float, 4)
DEFINE_CPY_SAMESHAPE(f16, __fp16, 2)
DEFINE_CPY_SAMESHAPE(i32, int32_t, 4)
#define DEFINE_CPY_RESHAPE(NAME, ELEM_TYPE, ELEM_SIZE) \
static void cpy_thread_##NAME##_reshape(unsigned int nth, unsigned int ith, void * data) { \
struct htp_copy_context * ct = (struct htp_copy_context *) data; \
@@ -129,52 +69,45 @@ static void cpy_thread_##NAME##_reshape(unsigned int nth, unsigned int ith, void
const uint32_t th_end = MIN(th_start + th_nelem, ct->elem_start + ct->nelem); \
if (th_start >= th_end) return; \
\
if (htp_tensor_is_contiguous(src0, ELEM_SIZE) && htp_tensor_is_contiguous(dst, ELEM_SIZE)) { \
dma_queue * dma_q = octx->ctx->dma[ith]; \
dma_addr_t dst_addr = dst->data + th_start * ELEM_SIZE; \
dma_addr_t src0_addr = src0->data + th_start * ELEM_SIZE; \
cpy_dma_sametype_reshape_contig(dma_q, dst_addr, src0_addr, (th_end - th_start) * ELEM_SIZE); \
dma_queue_flush(dma_q); \
return; \
} \
dma_queue * dma_q = octx->ctx->dma[ith]; \
\
const uint32_t ne01_ne00 = ne01 * ne00; \
const uint32_t ne02_ne01_ne00 = ne02 * ne01_ne00; \
const uint32_t ne1_ne0 = ne1 * ne0; \
const uint32_t ne2_ne1_ne0 = ne2 * ne1_ne0; \
\
const struct htp_copy_reshape_params * rsh = &ct->kparams->u.reshape; \
uint32_t e = th_start; \
uint32_t i13 = fastdiv(e, &ct->div_ne2_ne1_ne0); \
uint32_t i13 = fastdiv(e, &rsh->div_ne2_ne1_ne0); \
uint32_t rem = e - i13 * ne2_ne1_ne0; \
uint32_t i12 = fastdiv(rem, &ct->div_ne1_ne0); \
uint32_t i12 = fastdiv(rem, &rsh->div_ne1_ne0); \
uint32_t rem2 = rem - i12 * ne1_ne0; \
uint32_t i11 = fastdiv(rem2, &ct->div_ne0); \
uint32_t i11 = fastdiv(rem2, &rsh->div_ne0); \
uint32_t i10 = rem2 - i11 * ne0; \
\
uint32_t i03 = fastdiv(e, &ct->div_ne02_ne01_ne00); \
uint32_t i03 = fastdiv(e, &rsh->div_ne02_ne01_ne00); \
uint32_t rem_s = e - i03 * ne02_ne01_ne00; \
uint32_t i02 = fastdiv(rem_s, &ct->div_ne01_ne00); \
uint32_t i02 = fastdiv(rem_s, &rsh->div_ne01_ne00); \
uint32_t rem2_s = rem_s - i02 * ne01_ne00; \
uint32_t i01 = fastdiv(rem2_s, &ct->div_ne00); \
uint32_t i01 = fastdiv(rem2_s, &rsh->div_ne00); \
uint32_t i00 = rem2_s - i01 * ne00; \
\
char * dst_ptr = (char *) dst->data + i10*nb0 + i11*nb1 + i12*nb2 + i13*nb3; \
const char * src0_ptr = (const char *) src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03; \
dma_addr_t dst_addr = dst->data + i10*nb0 + i11*nb1 + i12*nb2 + i13*nb3; \
dma_addr_t src0_addr = src0->data + i00*nb00 + i01*nb01 + i02*nb02 + i03*nb03; \
\
const bool rows_contig = (nb00 == ELEM_SIZE) && (nb0 == ELEM_SIZE); \
\
while (e < th_end) { \
uint32_t run = 1; \
const uint32_t run = MIN(MIN(ne00 - i00, ne0 - i10), th_end - e); \
if (rows_contig) { \
run = MIN(MIN(ne00 - i00, ne0 - i10), th_end - e); \
hvx_copy_uu((uint8_t *) dst_ptr, (const uint8_t *) src0_ptr, run, ELEM_SIZE); \
dma_cpy_sametype_reshape_contig(dma_q, dst_addr, src0_addr, run * ELEM_SIZE); \
} else { \
*((ELEM_TYPE *) dst_ptr) = *((const ELEM_TYPE *) src0_ptr); \
dma_cpy_push_2d_chunked(dma_q, dst_addr, src0_addr, nb0, nb00, ELEM_SIZE, run); \
} \
e += run; \
\
dst_ptr += run * nb0; \
i10 += run; \
dst_addr += run * nb0; \
i10 += run; \
if (i10 == ne0) { \
i10 = 0; \
if (++i11 == ne1) { \
@@ -184,11 +117,11 @@ static void cpy_thread_##NAME##_reshape(unsigned int nth, unsigned int ith, void
i13++; \
} \
} \
dst_ptr = (char *) dst->data + i11*nb1 + i12*nb2 + i13*nb3; \
dst_addr = dst->data + i11*nb1 + i12*nb2 + i13*nb3; \
} \
\
src0_ptr += run * nb00; \
i00 += run; \
src0_addr += run * nb00; \
i00 += run; \
if (i00 == ne00) { \
i00 = 0; \
if (++i01 == ne01) { \
@@ -198,371 +131,350 @@ static void cpy_thread_##NAME##_reshape(unsigned int nth, unsigned int ith, void
i03++; \
} \
} \
src0_ptr = (const char *) src0->data + i01*nb01 + i02*nb02 + i03*nb03; \
src0_addr = src0->data + i01*nb01 + i02*nb02 + i03*nb03; \
} \
} \
dma_queue_flush(dma_q); \
}
DEFINE_CPY_RESHAPE(f32, float, 4)
DEFINE_CPY_RESHAPE(f16, __fp16, 2)
DEFINE_CPY_RESHAPE(i32, int32_t, 4)
static void cpy_thread_f16_f32_sameshape(unsigned int nth, unsigned int ith, void * data) {
struct htp_copy_context * ct = (struct htp_copy_context *) data;
struct htp_ops_context * octx = ct->octx;
cpy_preamble;
const uint32_t dr = ct->src0_nrows_per_thread;
const uint32_t ir0 = ct->row_start + dr * ith;
const uint32_t ir1 = MIN(ir0 + dr, ct->row_start + ct->nrows);
if (ir0 >= ir1) return;
const uint32_t ne02_ne01 = ne02 * ne01;
uint32_t i03 = fastdiv(ir0, &ct->div_ne02_ne01);
uint32_t rem = ir0 - i03 * ne02_ne01;
uint32_t i02 = fastdiv(rem, &ct->div_ne01);
uint32_t i01 = rem - i02 * ne01;
uint8_t* dst_ptr = (uint8_t*) dst->data + i01*nb1 + i02*nb2 + i03*nb3;
uint8_t* src0_ptr = (uint8_t*) src0->data + i01*nb01 + i02*nb02 + i03*nb03;
for (uint32_t r = ir0; r < ir1; r++) {
hex_l2fetch(src0_ptr, ne00 * sizeof(float), nb01, 2);
hvx_copy_f16_f32_uu(dst_ptr, src0_ptr, ne00);
dst_ptr += nb1;
src0_ptr += nb01;
if (++i01 == ne01) {
i01 = 0;
if (++i02 == ne02) {
i02 = 0;
i03++;
}
dst_ptr = (uint8_t*) dst->data + i02*nb2 + i03*nb3;
src0_ptr = (uint8_t*) src0->data + i02*nb02 + i03*nb03;
}
}
#define DEFINE_CPY_CONVERT_SAMESHAPE(NAME, CONV_FUNC) \
static void cpy_thread_##NAME##_sameshape(unsigned int nth, unsigned int ith, void * data) { \
struct htp_copy_context * ct = (struct htp_copy_context *) data; \
struct htp_ops_context * octx = ct->octx; \
cpy_preamble; \
\
const uint32_t dr = ct->src0_nrows_per_thread; \
const uint32_t ir0 = ct->row_start + dr * ith; \
const uint32_t ir1 = MIN(ir0 + dr, ct->row_start + ct->nrows); \
if (ir0 >= ir1) return; \
const uint32_t nrows_thread = ir1 - ir0; \
\
dma_queue * dma_q = octx->ctx->dma[ith]; \
struct htp_thread_trace * tr = &octx->ctx->trace[ith]; \
\
const struct htp_copy_convert_params * cvt = &ct->kparams->u.convert; \
const uint32_t src0_buf_size = cvt->src0_buf_size; \
const uint32_t dst_buf_size = cvt->dst_buf_size; \
uint8_t * vtcm_src0_base = ct->vtcm_src0 + ith * cvt->spad0_size_per_thread; \
uint8_t * vtcm_dst_base = ct->vtcm_dst + ith * cvt->spad1_size_per_thread; \
const uint32_t src0_row_size = ne00 * ct->kparams->src0_type_size; \
const uint32_t dst_row_size = ne00 * ct->kparams->dst_type_size; \
\
const uint32_t ne02_ne01 = ne02 * ne01; \
uint32_t i03 = fastdiv(ir0, &cvt->div_ne02_ne01); \
uint32_t rem = ir0 - i03 * ne02_ne01; \
uint32_t i02 = fastdiv(rem, &cvt->div_ne01); \
uint32_t i01 = rem - i02 * ne01; \
\
uint32_t f_i01 = i01, f_i02 = i02, f_i03 = i03; \
dma_addr_t f_src0_addr = src0->data + f_i01*nb01 + f_i02*nb02 + f_i03*nb03; \
\
uint32_t c_i01 = i01, c_i02 = i02, c_i03 = i03; \
dma_addr_t c_dst_addr = dst->data + c_i01*nb1 + c_i02*nb2 + c_i03*nb3; \
\
for (uint32_t r = 0; r < nrows_thread && r < 2; ++r) { \
uint8_t * src_spad = vtcm_src0_base + r * src0_buf_size; \
uint8_t * dst_spad = vtcm_dst_base + r * dst_buf_size; \
dma_queue_push(dma_q, dma_make_data(dst->data, dst_spad), \
dst_row_size, dst_buf_size, dst_row_size, 0); \
dma_queue_push(dma_q, dma_make_data(src_spad, f_src0_addr), \
src0_buf_size, src0_row_size, src0_row_size, 1); \
f_src0_addr += nb01; \
if (++f_i01 == ne01) { \
f_i01 = 0; \
if (++f_i02 == ne02) { \
f_i02 = 0; \
f_i03++; \
} \
f_src0_addr = src0->data + f_i02*nb02 + f_i03*nb03; \
} \
} \
\
for (uint32_t r = 0; r < nrows_thread; ++r) { \
uint8_t * dst_spad = (uint8_t *) (uintptr_t) dma_queue_pop(dma_q).src; \
uint8_t * src_spad = (uint8_t *) (uintptr_t) dma_queue_pop(dma_q).dst; \
\
htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); \
CONV_FUNC(dst_spad, src_spad, ne00); \
htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, (uint16_t) r); \
\
dma_queue_push(dma_q, dma_make_data(c_dst_addr, dst_spad), \
dst_row_size, dst_buf_size, dst_row_size, 1); \
c_dst_addr += nb1; \
if (++c_i01 == ne01) { \
c_i01 = 0; \
if (++c_i02 == ne02) { \
c_i02 = 0; \
c_i03++; \
} \
c_dst_addr = dst->data + c_i02*nb2 + c_i03*nb3; \
} \
\
if (r + 2 < nrows_thread) { \
dma_queue_push(dma_q, dma_make_data(src_spad, f_src0_addr), \
src0_buf_size, src0_row_size, src0_row_size, 1); \
f_src0_addr += nb01; \
if (++f_i01 == ne01) { \
f_i01 = 0; \
if (++f_i02 == ne02) { \
f_i02 = 0; \
f_i03++; \
} \
f_src0_addr = src0->data + f_i02*nb02 + f_i03*nb03; \
} \
} \
} \
dma_queue_flush(dma_q); \
}
static void cpy_thread_f32_f16_sameshape(unsigned int nth, unsigned int ith, void * data) {
struct htp_copy_context * ct = (struct htp_copy_context *) data;
struct htp_ops_context * octx = ct->octx;
cpy_preamble;
DEFINE_CPY_CONVERT_SAMESHAPE(f16_f32, hvx_copy_f16_f32_aa)
DEFINE_CPY_CONVERT_SAMESHAPE(f32_f16, hvx_copy_f32_f16_aa)
DEFINE_CPY_CONVERT_SAMESHAPE(i32_f32, hvx_copy_i32_f32_aa)
DEFINE_CPY_CONVERT_SAMESHAPE(f32_i32, hvx_copy_f32_i32_aa)
const uint32_t dr = ct->src0_nrows_per_thread;
const uint32_t ir0 = ct->row_start + dr * ith;
const uint32_t ir1 = MIN(ir0 + dr, ct->row_start + ct->nrows);
if (ir0 >= ir1) return;
const uint32_t ne02_ne01 = ne02 * ne01;
uint32_t i03 = fastdiv(ir0, &ct->div_ne02_ne01);
uint32_t rem = ir0 - i03 * ne02_ne01;
uint32_t i02 = fastdiv(rem, &ct->div_ne01);
uint32_t i01 = rem - i02 * ne01;
uint8_t* dst_ptr = (uint8_t*) dst->data + i01*nb1 + i02*nb2 + i03*nb3;
uint8_t* src0_ptr = (uint8_t*) src0->data + i01*nb01 + i02*nb02 + i03*nb03;
for (uint32_t r = ir0; r < ir1; r++) {
hex_l2fetch(src0_ptr, ne00 * sizeof(__fp16), nb01, 2);
hvx_copy_f32_f16_uu(dst_ptr, src0_ptr, ne00);
dst_ptr += nb1;
src0_ptr += nb01;
if (++i01 == ne01) {
i01 = 0;
if (++i02 == ne02) {
i02 = 0;
i03++;
}
dst_ptr = (uint8_t*) dst->data + i02*nb2 + i03*nb3;
src0_ptr = (uint8_t*) src0->data + i02*nb02 + i03*nb03;
}
}
}
static void cpy_thread_i32_f32_sameshape(unsigned int nth, unsigned int ith, void * data) {
struct htp_copy_context * ct = (struct htp_copy_context *) data;
struct htp_ops_context * octx = ct->octx;
cpy_preamble;
const uint32_t dr = ct->src0_nrows_per_thread;
const uint32_t ir0 = ct->row_start + dr * ith;
const uint32_t ir1 = MIN(ir0 + dr, ct->row_start + ct->nrows);
if (ir0 >= ir1) return;
const uint32_t ne02_ne01 = ne02 * ne01;
uint32_t i03 = fastdiv(ir0, &ct->div_ne02_ne01);
uint32_t rem = ir0 - i03 * ne02_ne01;
uint32_t i02 = fastdiv(rem, &ct->div_ne01);
uint32_t i01 = rem - i02 * ne01;
uint8_t* dst_ptr = (uint8_t*) dst->data + i01*nb1 + i02*nb2 + i03*nb3;
uint8_t* src0_ptr = (uint8_t*) src0->data + i01*nb01 + i02*nb02 + i03*nb03;
for (uint32_t r = ir0; r < ir1; r++) {
hex_l2fetch(src0_ptr, ne00 * sizeof(float), nb01, 2);
const float * restrict src_row = (const float *) src0_ptr;
int32_t * restrict dst_row = (int32_t *) dst_ptr;
for (uint32_t i = 0; i < ne00; i++) {
dst_row[i] = (int32_t) src_row[i];
}
dst_ptr += nb1;
src0_ptr += nb01;
if (++i01 == ne01) {
i01 = 0;
if (++i02 == ne02) {
i02 = 0;
i03++;
}
dst_ptr = (uint8_t*) dst->data + i02*nb2 + i03*nb3;
src0_ptr = (uint8_t*) src0->data + i02*nb02 + i03*nb03;
}
}
}
static void cpy_thread_f32_i32_sameshape(unsigned int nth, unsigned int ith, void * data) {
struct htp_copy_context * ct = (struct htp_copy_context *) data;
struct htp_ops_context * octx = ct->octx;
cpy_preamble;
const uint32_t dr = ct->src0_nrows_per_thread;
const uint32_t ir0 = ct->row_start + dr * ith;
const uint32_t ir1 = MIN(ir0 + dr, ct->row_start + ct->nrows);
if (ir0 >= ir1) return;
const uint32_t ne02_ne01 = ne02 * ne01;
uint32_t i03 = fastdiv(ir0, &ct->div_ne02_ne01);
uint32_t rem = ir0 - i03 * ne02_ne01;
uint32_t i02 = fastdiv(rem, &ct->div_ne01);
uint32_t i01 = rem - i02 * ne01;
uint8_t* dst_ptr = (uint8_t*) dst->data + i01*nb1 + i02*nb2 + i03*nb3;
uint8_t* src0_ptr = (uint8_t*) src0->data + i01*nb01 + i02*nb02 + i03*nb03;
for (uint32_t r = ir0; r < ir1; r++) {
hex_l2fetch(src0_ptr, ne00 * sizeof(int32_t), nb01, 2);
const int32_t * restrict src_row = (const int32_t *) src0_ptr;
float * restrict dst_row = (float *) dst_ptr;
for (uint32_t i = 0; i < ne00; i++) {
dst_row[i] = (float) src_row[i];
}
dst_ptr += nb1;
src0_ptr += nb01;
if (++i01 == ne01) {
i01 = 0;
if (++i02 == ne02) {
i02 = 0;
i03++;
}
dst_ptr = (uint8_t*) dst->data + i02*nb2 + i03*nb3;
src0_ptr = (uint8_t*) src0->data + i02*nb02 + i03*nb03;
}
}
}
static int exec_cpy(struct htp_ops_context * octx, bool * use_dma) {
cpy_preamble;
*use_dma = false;
const uint32_t total_elems_src = ne00 * ne01 * ne02 * ne03;
const uint32_t total_elems_dst = ne0 * ne1 * ne2 * ne3;
if (total_elems_src == 1 && total_elems_dst == 1) {
if (octx->ctx->mdev.count > 1 && octx->ctx->mdev.idx > 0) {
return HTP_STATUS_OK;
}
if (src0->type == HTP_TYPE_F32 && dst->type == HTP_TYPE_I32) {
((int32_t *) dst->data)[0] = (int32_t) (((const float *) src0->data)[0]);
return HTP_STATUS_OK;
}
if (src0->type == HTP_TYPE_I32 && dst->type == HTP_TYPE_F32) {
((float *) dst->data)[0] = (float) (((const int32_t *) src0->data)[0]);
return HTP_STATUS_OK;
}
if (src0->type == HTP_TYPE_I32 && dst->type == HTP_TYPE_I32) {
((int32_t *) dst->data)[0] = ((const int32_t *) src0->data)[0];
return HTP_STATUS_OK;
}
if (src0->type == HTP_TYPE_F32 && dst->type == HTP_TYPE_F32) {
((float *) dst->data)[0] = ((const float *) src0->data)[0];
return HTP_STATUS_OK;
}
if (src0->type == HTP_TYPE_F16 && dst->type == HTP_TYPE_F16) {
((__fp16 *) dst->data)[0] = ((const __fp16 *) src0->data)[0];
return HTP_STATUS_OK;
}
if (src0->type == HTP_TYPE_F32 && dst->type == HTP_TYPE_F16) {
((__fp16 *) dst->data)[0] = (__fp16) (((const float *) src0->data)[0]);
return HTP_STATUS_OK;
}
if (src0->type == HTP_TYPE_F16 && dst->type == HTP_TYPE_F32) {
((float *) dst->data)[0] = (float) (((const __fp16 *) src0->data)[0]);
return HTP_STATUS_OK;
}
static int cpy_scalar(struct htp_ops_context * octx, const struct htp_copy_kernel_params * kparams) {
if (octx->ctx->mdev.count > 1 && octx->ctx->mdev.idx > 0) {
return HTP_STATUS_OK;
}
struct htp_copy_context ct;
ct.octx = octx;
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * dst = octx->dst;
switch (src0->type) {
case HTP_TYPE_F32: ct.src0_type_size = 4; ct.src0_block_size = 1; ct.src0_blocks_per_row = ne00 / 1; break;
case HTP_TYPE_F16: ct.src0_type_size = 2; ct.src0_block_size = 1; ct.src0_blocks_per_row = ne00 / 1; break;
case HTP_TYPE_I32: ct.src0_type_size = 4; ct.src0_block_size = 1; ct.src0_blocks_per_row = ne00 / 1; break;
default:
return HTP_STATUS_NO_SUPPORT;
if (src0->type == dst->type) {
dma_cpy_sametype_reshape_contig(octx->ctx->dma[0], dst->data, src0->data, kparams->src0_type_size);
dma_queue_flush(octx->ctx->dma[0]);
return HTP_STATUS_OK;
}
switch (dst->type) {
case HTP_TYPE_F32: ct.dst_type_size = 4; ct.dst_block_size = 1; ct.dst_blocks_per_row = ne0 / 1; break;
case HTP_TYPE_F16: ct.dst_type_size = 2; ct.dst_block_size = 1; ct.dst_blocks_per_row = ne0 / 1; break;
case HTP_TYPE_I32: ct.dst_type_size = 4; ct.dst_block_size = 1; ct.dst_blocks_per_row = ne0 / 1; break;
default:
return HTP_STATUS_NO_SUPPORT;
}
dma_queue * dma_q = octx->ctx->dma[0];
dma_addr_t s_vtcm = (dma_addr_t)(uintptr_t) octx->ctx->vtcm_base;
dma_addr_t d_vtcm = s_vtcm + VLEN;
const uint32_t s_size = kparams->src0_type_size;
const uint32_t d_size = kparams->dst_type_size;
const bool sametype = (src0->type == dst->type);
const bool transposed = (nb00 > nb01) || (nb0 > nb1) ||
(nb00 != ct.src0_type_size) || (nb0 != ct.dst_type_size) ||
(nb01 < ne00 * ct.src0_type_size) || (nb1 < ne0 * ct.dst_type_size);
const bool sameshape = !transposed && (ne00 == ne0 && ne01 == ne1 && ne02 == ne2 && ne03 == ne3);
dma_queue_push(dma_q, dma_make_data(s_vtcm, src0->data), s_size, s_size, s_size, 1);
dma_queue_pop(dma_q);
const uint32_t n_threads = octx->n_threads;
uint8_t * s_ptr = (uint8_t *) octx->ctx->vtcm_base;
uint8_t * d_ptr = s_ptr + VLEN;
const bool src_is_contiguous = htp_tensor_is_contiguous(src0, ct.src0_type_size);
const bool dst_is_contiguous = htp_tensor_is_contiguous(dst, ct.dst_type_size);
const HVX_Vector v_src = hvx_vmem(s_ptr);
HVX_Vector v_dst;
if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst)) {
if (!sametype) {
return HTP_STATUS_NO_SUPPORT;
}
if (!sameshape && !(src_is_contiguous && dst_is_contiguous && octx->ctx->mdev.count <= 1)) {
return HTP_STATUS_NO_SUPPORT;
}
}
if (sameshape) {
const uint32_t total_rows = ne01 * ne02 * ne03;
const uint32_t row_size = ne00 * ct.dst_type_size;
ct.div_ne01 = init_fastdiv_values(ne01);
ct.div_ne02_ne01 = init_fastdiv_values(ne02 * ne01);
uint32_t row_start = 0;
uint32_t nrows = total_rows;
if (octx->ctx->mdev.count > 1) {
const uint32_t rows_per_chunk = (row_size > 0) ? (HEX_L2_LINE_SIZE / hex_gcd_u32(row_size, HEX_L2_LINE_SIZE)) : 1;
const bool can_split = htp_tensor_mdev_data_aligned(dst) && dst_is_contiguous;
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_rows, can_split ? rows_per_chunk : 0,
octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
row_start = range.start;
nrows = range.count;
}
if (nrows == 0) {
return HTP_STATUS_OK;
}
ct.row_start = row_start;
ct.nrows = nrows;
ct.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
if (sametype && (octx->ctx->mdev.count <= 1 || htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst))) {
if (octx->ctx->mdev.idx == 0) {
*use_dma = true;
cpy_dma_sametype_sameshape(octx->ctx->dma[0], dst, src0, ct.src0_type_size);
dma_queue_flush(octx->ctx->dma[0]);
}
} else {
work_queue_func_t copy_fun = NULL;
if (sametype) {
switch (src0->type) {
case HTP_TYPE_F32: copy_fun = cpy_thread_f32_sameshape; break;
case HTP_TYPE_F16: copy_fun = cpy_thread_f16_sameshape; break;
case HTP_TYPE_I32: copy_fun = cpy_thread_i32_sameshape; break;
default: return HTP_STATUS_NO_SUPPORT;
}
} else if (dst->type == HTP_TYPE_F16 && src0->type == HTP_TYPE_F32) {
copy_fun = cpy_thread_f16_f32_sameshape;
} else if (dst->type == HTP_TYPE_F32 && src0->type == HTP_TYPE_F16) {
copy_fun = cpy_thread_f32_f16_sameshape;
} else if (dst->type == HTP_TYPE_I32 && src0->type == HTP_TYPE_F32) {
copy_fun = cpy_thread_i32_f32_sameshape;
} else if (dst->type == HTP_TYPE_F32 && src0->type == HTP_TYPE_I32) {
copy_fun = cpy_thread_f32_i32_sameshape;
} else {
return HTP_STATUS_NO_SUPPORT;
}
work_queue_run(octx->ctx->work_queue, copy_fun, &ct, n_threads);
}
} else if (sametype) {
const uint32_t total_elems = ne0 * ne1 * ne2 * ne3;
const uint32_t elems_per_line = (ct.dst_type_size == 4) ? 32 : 64;
if (octx->ctx->mdev.count <= 1 && dst_is_contiguous && src_is_contiguous) {
*use_dma = true;
cpy_dma_sametype_reshape_contig(octx->ctx->dma[0], dst->data, src0->data, total_elems * ct.dst_type_size);
dma_queue_flush(octx->ctx->dma[0]);
return HTP_STATUS_OK;
}
ct.div_ne0 = init_fastdiv_values(ne0);
ct.div_ne1_ne0 = init_fastdiv_values(ne1 * ne0);
ct.div_ne2_ne1_ne0 = init_fastdiv_values(ne2 * ne1 * ne0);
ct.div_ne00 = init_fastdiv_values(ne00);
ct.div_ne01_ne00 = init_fastdiv_values(ne01 * ne00);
ct.div_ne02_ne01_ne00 = init_fastdiv_values(ne02 * ne01 * ne00);
uint32_t elem_start = 0;
uint32_t nelem = total_elems;
if (octx->ctx->mdev.count > 1) {
const bool can_split = htp_tensor_mdev_data_aligned(dst) && dst_is_contiguous;
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(total_elems, can_split ? elems_per_line : 0,
octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
elem_start = range.start;
nelem = range.count;
}
if (nelem == 0) {
return HTP_STATUS_OK;
}
ct.elem_start = elem_start;
ct.nelem = nelem;
ct.elem_per_thread = fastdiv(nelem + n_threads - 1, &octx->n_threads_div);
work_queue_func_t copy_fun = NULL;
switch (src0->type) {
case HTP_TYPE_F32: copy_fun = cpy_thread_f32_reshape; break;
case HTP_TYPE_F16: copy_fun = cpy_thread_f16_reshape; break;
case HTP_TYPE_I32: copy_fun = cpy_thread_i32_reshape; break;
default: return HTP_STATUS_NO_SUPPORT;
}
work_queue_run(octx->ctx->work_queue, copy_fun, &ct, n_threads);
if (src0->type == HTP_TYPE_F32 && dst->type == HTP_TYPE_I32) {
v_dst = Q6_Vw_equals_Vsf(v_src);
} else if (src0->type == HTP_TYPE_I32 && dst->type == HTP_TYPE_F32) {
v_dst = Q6_Vsf_equals_Vw(v_src);
} else if (src0->type == HTP_TYPE_F32 && dst->type == HTP_TYPE_F16) {
v_dst = hvx_vec_f32_to_f16(v_src, v_src);
} else if (src0->type == HTP_TYPE_F16 && dst->type == HTP_TYPE_F32) {
v_dst = Q6_V_lo_W(hvx_vec_f16_to_f32(v_src));
} else {
return HTP_STATUS_NO_SUPPORT;
}
hvx_vmem(d_ptr) = v_dst;
dma_queue_push(dma_q, dma_make_data(dst->data, d_vtcm), d_size, d_size, d_size, 1);
dma_queue_flush(dma_q);
return HTP_STATUS_OK;
}
static int cpy_1d_contig(struct htp_ops_context * octx, const struct htp_copy_kernel_params * kparams) {
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * dst = octx->dst;
uint32_t elem_start = 0;
uint32_t nelem = kparams->total_elems;
if (octx->ctx->mdev.count > 1) {
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(
kparams->total_elems, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
elem_start = range.start;
nelem = range.count;
}
if (nelem > 0) {
dma_queue * q = octx->ctx->dma[0];
const uint32_t type_size = kparams->src0_type_size;
dma_addr_t dst_addr = dst->data + elem_start * type_size;
dma_addr_t src0_addr = src0->data + elem_start * type_size;
dma_cpy_sametype_reshape_contig(q, dst_addr, src0_addr, nelem * type_size);
dma_queue_flush(q);
}
return HTP_STATUS_OK;
}
static int cpy_sameshape_sametype(struct htp_ops_context * octx, const struct htp_copy_kernel_params * kparams) {
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * dst = octx->dst;
uint32_t row_start = 0;
uint32_t nrows = kparams->total_rows;
if (octx->ctx->mdev.count > 1) {
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(
kparams->total_rows, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
row_start = range.start;
nrows = range.count;
}
if (nrows > 0) {
dma_queue * q = octx->ctx->dma[0];
dma_cpy_sametype_sameshape_range(q, dst, src0, kparams->src0_type_size, row_start, nrows);
dma_queue_flush(q);
}
return HTP_STATUS_OK;
}
static int cpy_sameshape_convert(struct htp_ops_context * octx, const struct htp_copy_kernel_params * kparams) {
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * dst = octx->dst;
if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst)) {
return HTP_STATUS_NO_SUPPORT;
}
if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
return HTP_STATUS_INVAL_PARAMS;
}
uint32_t row_start = 0;
uint32_t nrows = kparams->total_rows;
if (octx->ctx->mdev.count > 1) {
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(
kparams->total_rows, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
row_start = range.start;
nrows = range.count;
}
if (nrows == 0) {
return HTP_STATUS_OK;
}
if (kparams->vtcm_size > octx->ctx->vtcm_size) {
return HTP_STATUS_VTCM_TOO_SMALL;
}
const uint32_t n_threads = octx->n_threads;
const struct htp_copy_convert_params * cvt = &kparams->u.convert;
struct htp_copy_context ct;
ct.octx = octx;
ct.kparams = kparams;
ct.row_start = row_start;
ct.nrows = nrows;
ct.src0_nrows_per_thread = fastdiv(nrows + n_threads - 1, &octx->n_threads_div);
uint8_t * vtcm_base = (uint8_t *) octx->ctx->vtcm_base;
ct.vtcm_src0 = vtcm_base;
ct.vtcm_dst = vtcm_base + (size_t) n_threads * cvt->spad0_size_per_thread;
work_queue_func_t copy_fun = NULL;
if (dst->type == HTP_TYPE_F16 && src0->type == HTP_TYPE_F32) {
copy_fun = cpy_thread_f16_f32_sameshape;
} else if (dst->type == HTP_TYPE_F32 && src0->type == HTP_TYPE_F16) {
copy_fun = cpy_thread_f32_f16_sameshape;
} else if (dst->type == HTP_TYPE_I32 && src0->type == HTP_TYPE_F32) {
copy_fun = cpy_thread_i32_f32_sameshape;
} else if (dst->type == HTP_TYPE_F32 && src0->type == HTP_TYPE_I32) {
copy_fun = cpy_thread_f32_i32_sameshape;
} else {
return HTP_STATUS_NO_SUPPORT;
}
work_queue_run(octx->ctx->work_queue, copy_fun, &ct, n_threads);
return HTP_STATUS_OK;
}
static int cpy_reshape(struct htp_ops_context * octx, const struct htp_copy_kernel_params * kparams) {
const struct htp_tensor * src0 = octx->src[0];
const struct htp_tensor * dst = octx->dst;
if (htp_tensor_is_extended(src0) || htp_tensor_is_extended(dst)) {
return HTP_STATUS_NO_SUPPORT;
}
if (!htp_ops_context_set_n_threads(octx, kparams->n_threads)) {
return HTP_STATUS_INVAL_PARAMS;
}
uint32_t elem_start = 0;
uint32_t nelem = kparams->total_elems;
if (octx->ctx->mdev.count > 1) {
const struct htp_tensor_mdev_range range = htp_tensor_mdev_partition(
kparams->total_elems, 1, octx->ctx->mdev.idx, octx->ctx->mdev.count, &octx->ctx->mdev.count_div);
elem_start = range.start;
nelem = range.count;
}
if (nelem == 0) {
return HTP_STATUS_OK;
}
const uint32_t n_threads = octx->n_threads;
struct htp_copy_context ct;
ct.octx = octx;
ct.kparams = kparams;
ct.elem_start = elem_start;
ct.nelem = nelem;
ct.elem_per_thread = fastdiv(nelem + n_threads - 1, &octx->n_threads_div);
work_queue_func_t copy_fun = NULL;
switch (src0->type) {
case HTP_TYPE_F32: copy_fun = cpy_thread_f32_reshape; break;
case HTP_TYPE_F16: copy_fun = cpy_thread_f16_reshape; break;
case HTP_TYPE_I32: copy_fun = cpy_thread_i32_reshape; break;
default: return HTP_STATUS_NO_SUPPORT;
}
work_queue_run(octx->ctx->work_queue, copy_fun, &ct, n_threads);
return HTP_STATUS_OK;
}
int op_cpy(struct htp_ops_context * octx) {
bool use_dma = false;
int status = exec_cpy(octx, &use_dma);
const struct htp_copy_kernel_params * kparams = (const struct htp_copy_kernel_params *) octx->kernel_params;
int status = HTP_STATUS_OK;
switch (kparams->kernel_type) {
case HTP_COPY_KERNEL_SCALAR:
status = cpy_scalar(octx, kparams);
break;
case HTP_COPY_KERNEL_1D_CONTIG:
status = cpy_1d_contig(octx, kparams);
break;
case HTP_COPY_KERNEL_SAMESHAPE_SAMETYPE:
status = cpy_sameshape_sametype(octx, kparams);
break;
case HTP_COPY_KERNEL_SAMESHAPE_CONVERT:
status = cpy_sameshape_convert(octx, kparams);
break;
case HTP_COPY_KERNEL_RESHAPE:
status = cpy_reshape(octx, kparams);
break;
default:
status = HTP_STATUS_NO_SUPPORT;
break;
}
htp_ops_context_set_status(octx, status);
if (octx->op == HTP_OP_CPY_FENCE) {
if (!use_dma) {
htp_flush_dirty_ranges(octx->ctx);
}
if (octx->ctx->mdev.count > 1) {
htp_mdev_group_barrier(octx);
}
if (octx->op == HTP_OP_CPY_FENCE) {
if (octx->ctx->mdev.idx == 0) {
const struct htp_tensor * sync = octx->src[1];
if (htp_tensor_is_extended(sync)) {
return HTP_STATUS_NO_SUPPORT;
}
const uint32_t seq = (uint32_t) octx->op_params[0];
atomic_uint * sync_fence = (atomic_uint *) (uintptr_t) sync->data;
htp_fence_write(sync_fence, seq, octx->status);
ggml/src/ggml-hexagon/htp/cpy-ops.h
+79
View File
@@ -0,0 +1,79 @@
#ifndef HTP_CPY_OPS_H
#define HTP_CPY_OPS_H
#include "hex-common.h"
#include "hex-fastdiv.h"
#include <stdint.h>
enum htp_copy_kernel_type {
HTP_COPY_KERNEL_UNSUPPORTED = 0,
HTP_COPY_KERNEL_1D_CONTIG = 1,
HTP_COPY_KERNEL_SAMESHAPE_SAMETYPE = 2,
HTP_COPY_KERNEL_SAMESHAPE_CONVERT = 3,
HTP_COPY_KERNEL_RESHAPE = 4,
HTP_COPY_KERNEL_SCALAR = 5,
};
struct htp_copy_convert_params {
uint32_t src0_buf_size;
uint32_t dst_buf_size;
uint32_t spad0_size_per_thread;
uint32_t spad1_size_per_thread;
struct fastdiv_values div_ne01;
struct fastdiv_values div_ne02_ne01;
};
struct htp_copy_reshape_params {
struct fastdiv_values div_ne0;
struct fastdiv_values div_ne1_ne0;
struct fastdiv_values div_ne2_ne1_ne0;
struct fastdiv_values div_ne00;
struct fastdiv_values div_ne01_ne00;
struct fastdiv_values div_ne02_ne01_ne00;
};
struct htp_copy_kernel_params {
uint8_t kernel_type;
uint8_t src0_type_size;
uint8_t dst_type_size;
uint8_t n_threads;
uint32_t total_elems;
uint32_t total_rows;
uint32_t vtcm_size;
union {
struct htp_copy_convert_params convert;
struct htp_copy_reshape_params reshape;
} u;
};
struct htp_copy_convert_vtcm_layout {
uint32_t src0_buf_size;
uint32_t dst_buf_size;
uint32_t spad0_size_per_thread;
uint32_t spad1_size_per_thread;
uint32_t total_bytes;
};
static inline void htp_copy_convert_vtcm_layout_build(
struct htp_copy_convert_vtcm_layout * layout,
uint32_t ne00,
uint32_t src_type_size,
uint32_t dst_type_size,
uint32_t n_threads) {
layout->src0_buf_size = hex_round_up(ne00 * src_type_size, 256);
layout->dst_buf_size = hex_round_up(ne00 * dst_type_size, 256);
layout->spad0_size_per_thread = 2 * layout->src0_buf_size;
layout->spad1_size_per_thread = 2 * layout->dst_buf_size;
layout->total_bytes = n_threads * (layout->spad0_size_per_thread + layout->spad1_size_per_thread);
}
#if defined(__cplusplus)
static_assert(sizeof(struct htp_copy_kernel_params) <= 128, "htp_copy_kernel_params is too large for kernel_params blob");
#else
_Static_assert(sizeof(struct htp_copy_kernel_params) <= 128, "htp_copy_kernel_params is too large for kernel_params blob");
#endif
#endif // HTP_CPY_OPS_H
ggml/src/ggml-hexagon/htp/hex-cpy-dma.h → ggml/src/ggml-hexagon/htp/dma-copy.h
+64 -21
View File
@@ -1,9 +1,9 @@
#ifndef HEX_CPY_DMA_H
#define HEX_CPY_DMA_H
#ifndef HTP_DMA_COPY_H
#define HTP_DMA_COPY_H
// DDR<->DDR DMA copies of same-type, same-shape tensors with arbitrary strides.
// Used by CPY for the copy itself and by CONCAT, which is two such copies into
// two views of its destination. Every helper only pushes descriptors; the
// two views of its destination. Every helper only pushes descriptors; the
// caller flushes the queue when it needs the data.
#include "dma-queue.h"
@@ -14,7 +14,7 @@
#include <stdint.h>
// Contiguous byte run, as 1d transfers of at most DMA_SAFE_CHUNK_SIZE each.
static inline void cpy_dma_sametype_reshape_contig(dma_queue * dma_q,
static inline void dma_cpy_sametype_reshape_contig(dma_queue * dma_q,
dma_addr_t dst,
dma_addr_t src0,
uint32_t total_bytes) {
@@ -36,7 +36,7 @@ static inline void cpy_dma_sametype_reshape_contig(dma_queue * dma_q,
}
// One 2d transfer, split at the 16-bit nrows field.
static inline void cpy_dma_push_2d_chunked(dma_queue * dma_q,
static inline void dma_cpy_push_2d_chunked(dma_queue * dma_q,
dma_addr_t dst,
dma_addr_t src,
size_t dst_stride,
@@ -59,12 +59,18 @@ static inline void cpy_dma_push_2d_chunked(dma_queue * dma_q,
}
}
// Copy src0 into dst: same type, same ne[], any nb[] above dim 0, dim 0 dense on
// both sides (nb[0] == elem_size).
static inline void cpy_dma_sametype_sameshape(dma_queue * dma_q,
const struct htp_tensor * dst,
const struct htp_tensor * src0,
uint32_t elem_size) {
// Copy a range of rows [row_start, row_start + nrows) from src0 into dst:
// same type, same ne[], any nb[] above dim 0, dim 0 dense on both sides (nb[0] == elem_size).
static inline void dma_cpy_sametype_sameshape_range(dma_queue * dma_q,
const struct htp_tensor * dst,
const struct htp_tensor * src0,
uint32_t elem_size,
uint32_t row_start,
uint32_t nrows) {
if (nrows == 0) {
return;
}
const uint32_t ne00 = src0->ne[0];
const uint32_t ne01 = src0->ne[1];
const uint32_t ne02 = src0->ne[2];
@@ -85,13 +91,16 @@ static inline void cpy_dma_sametype_sameshape(dma_queue * dma_q,
const bool contiguous = htp_tensor_is_contiguous(src0, elem_size) && htp_tensor_is_contiguous(dst, elem_size);
if (contiguous) {
cpy_dma_sametype_reshape_contig(dma_q, dst->data, src0->data, ne00 * elem_size * ne01 * ne02 * ne03);
dma_cpy_sametype_reshape_contig(dma_q,
dst->data + (dma_addr_t) row_start * ne00 * elem_size,
src0->data + (dma_addr_t) row_start * ne00 * elem_size,
nrows * ne00 * elem_size);
return;
}
// The single-descriptor path flattens (i01,i02,i03) into one row index, so every
// row must sit at a constant stride: nb01 on the source, nb1 on the destination.
// Walk the outer dims and require each to continue that progression. A dim of
// Walk the outer dims and require each to continue that progression. A dim of
// extent 1 spans no rows, so it is skipped -- but its own stride must NOT then be
// used to justify the next dim's stride, which is what comparing nb03 against
// ne02*nb02 did: ggml leaves the stride of an extent-1 dim meaningless, so a view
@@ -109,18 +118,52 @@ static inline void cpy_dma_sametype_sameshape(dma_queue * dma_q,
}
if (contiguous_outer) {
uint32_t total_rows = ne01 * ne02 * ne03;
cpy_dma_push_2d_chunked(dma_q, dst->data, src0->data, nb1, nb01, ne00 * elem_size, total_rows);
dma_cpy_push_2d_chunked(dma_q,
dst->data + (dma_addr_t) row_start * nb1,
src0->data + (dma_addr_t) row_start * nb01,
nb1, nb01, ne00 * elem_size, nrows);
return;
}
for (uint32_t i03 = 0; i03 < ne03; i03++) {
for (uint32_t i02 = 0; i02 < ne02; i02++) {
dma_addr_t dst_data = dst->data + i02 * nb2 + i03 * nb3;
dma_addr_t src0_data = src0->data + i02 * nb02 + i03 * nb03;
cpy_dma_push_2d_chunked(dma_q, dst_data, src0_data, nb1, nb01, ne00 * elem_size, ne01);
const uint32_t ne02_ne01 = ne02 * ne01;
uint32_t i03 = row_start / ne02_ne01;
uint32_t rem = row_start - i03 * ne02_ne01;
uint32_t i02 = rem / ne01;
uint32_t i01 = rem - i02 * ne01;
dma_addr_t cur_dst = dst->data + (dma_addr_t) i01 * nb1 + (dma_addr_t) i02 * nb2 + (dma_addr_t) i03 * nb3;
dma_addr_t cur_src0 = src0->data + (dma_addr_t) i01 * nb01 + (dma_addr_t) i02 * nb02 + (dma_addr_t) i03 * nb03;
uint32_t r = row_start;
const uint32_t row_end = row_start + nrows;
while (r < row_end) {
uint32_t cur_rows = MIN(row_end - r, ne01 - i01);
dma_cpy_push_2d_chunked(dma_q, cur_dst, cur_src0, nb1, nb01, ne00 * elem_size, cur_rows);
r += cur_rows;
i01 += cur_rows;
if (i01 == ne01) {
i01 = 0;
if (++i02 == ne02) {
i02 = 0;
i03++;
}
cur_dst = dst->data + (dma_addr_t) i02 * nb2 + (dma_addr_t) i03 * nb3;
cur_src0 = src0->data + (dma_addr_t) i02 * nb02 + (dma_addr_t) i03 * nb03;
} else {
cur_dst += cur_rows * nb1;
cur_src0 += cur_rows * nb01;
}
}
}
#endif /* HEX_CPY_DMA_H */
// Copy src0 into dst: same type, same ne[], any nb[] above dim 0, dim 0 dense on
// both sides (nb[0] == elem_size).
static inline void dma_cpy_sametype_sameshape(dma_queue * dma_q,
const struct htp_tensor * dst,
const struct htp_tensor * src0,
uint32_t elem_size) {
const uint32_t total_rows = src0->ne[1] * src0->ne[2] * src0->ne[3];
dma_cpy_sametype_sameshape_range(dma_q, dst, src0, elem_size, 0, total_rows);
}
#endif /* HTP_DMA_COPY_H */
ggml/src/ggml-hexagon/htp/hex-utils.h
+3 -9
View File
@@ -38,21 +38,15 @@ static inline void hex_l2fetch_block(const void * addr, size_t size) {
}
#define HEX_L2_LINE_SIZE 128
#define HEX_L2_BLOCK_SIZE (HEX_L2_LINE_SIZE * 4) // flush granularity (lines per loop iteration)
#define HEX_L2_BLOCK_SIZE (HEX_L2_LINE_SIZE * 4) // flush granularity (chunks per thread)
#define HEX_L2_FLUSH_WQ_THRESHOLD (4 * 1024)
#define HEX_L2_FLUSH_ALL_THRESHOLD (4 * 1024 * 1024)
static inline void hex_l2flush(void * addr, size_t size) {
if (size == 0) return;
const uint32_t s = ((uint32_t) addr) & ~(HEX_L2_LINE_SIZE - 1);
const uint32_t e = (((uint32_t) addr) + size + HEX_L2_LINE_SIZE - 1) & ~(HEX_L2_LINE_SIZE - 1);
const uint32_t eb = s + ((e - s) & ~(HEX_L2_BLOCK_SIZE - 1));
for (uint32_t i = s; i < eb; i += HEX_L2_BLOCK_SIZE) {
Q6_dccleaninva_A((void *) (i + HEX_L2_LINE_SIZE * 0));
Q6_dccleaninva_A((void *) (i + HEX_L2_LINE_SIZE * 1));
Q6_dccleaninva_A((void *) (i + HEX_L2_LINE_SIZE * 2));
Q6_dccleaninva_A((void *) (i + HEX_L2_LINE_SIZE * 3));
}
for (uint32_t i = eb; i < e; i += HEX_L2_LINE_SIZE) {
for (uint32_t i = s; i < e; i += HEX_L2_LINE_SIZE) {
Q6_dccleaninva_A((void *) i);
}
}
ggml/src/ggml-hexagon/htp/hvx-copy.h
+91 -21
View File
@@ -44,7 +44,7 @@ static inline void hvx_splat_f32_u(void * restrict dst, float v, uint32_t n) {
}
static inline void hvx_splat_f16_a(void * restrict dst, _Float16 v, uint32_t n) {
hvx_splat_u(dst, hvx_vec_splat_f16(v), n, sizeof(__fp16));
hvx_splat_a(dst, hvx_vec_splat_f16(v), n, sizeof(__fp16));
}
static inline void hvx_splat_f16_u(void * restrict dst, _Float16 v, uint32_t n) {
@@ -106,42 +106,42 @@ static inline void hvx_copy_uu(uint8_t * restrict dst, const uint8_t * restrict
hvx_copy_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u);
}
// copy n fp16 elements : source and destination are aligned to HVX Vector (128)
// copy n fp16 elements : destination and source are aligned to HVX Vector (128)
static inline void hvx_copy_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
hvx_copy_aa(dst, src, n, sizeof(__fp16));
}
// copy n fp16 elements : source is aligned, destination is potentially unaligned
// copy n fp16 elements : destination is aligned, source is unaligned
static inline void hvx_copy_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
hvx_copy_au(dst, src, n, sizeof(__fp16));
}
// copy n fp16 elements : source is aligned, destination is potentially unaligned
// copy n fp16 elements : destination is unaligned, source is aligned
static inline void hvx_copy_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
hvx_copy_ua(dst, src, n, sizeof(__fp16));
}
// copy n fp16 elements : source is aligned, destination is potentially unaligned
// copy n fp16 elements : destination and source are unaligned
static inline void hvx_copy_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
hvx_copy_uu(dst, src, n, sizeof(__fp16));
}
// copy n fp32 elements : source and destination are aligned to HVX Vector (128)
// copy n fp32 elements : destination and source are aligned to HVX Vector (128)
static inline void hvx_copy_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
hvx_copy_aa(dst, src, n, sizeof(float));
}
// copy n fp32 elements : source is aligned, destination is unaligned
static inline void hvx_copy_f32_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
hvx_copy_ua(dst, src, n, sizeof(float));
}
// copy n fp32 elements : source is unaligned, destination is aligned
// copy n fp32 elements : destination is aligned, source is unaligned
static inline void hvx_copy_f32_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
hvx_copy_au(dst, src, n, sizeof(float));
}
// copy n fp32 elements : source is unaligned, destination unaligned
// copy n fp32 elements : destination is unaligned, source is aligned
static inline void hvx_copy_f32_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
hvx_copy_ua(dst, src, n, sizeof(float));
}
// copy n fp32 elements : destination and source are unaligned
static inline void hvx_copy_f32_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
hvx_copy_uu(dst, src, n, sizeof(float));
}
@@ -170,26 +170,26 @@ static inline void hvx_copy_f32_uu(uint8_t * restrict dst, const uint8_t * restr
} \
} while(0)
// copy/convert n fp32 elements into n fp16 elements : source is aligned, destination is aligned
// copy/convert n fp32 elements into n fp16 elements : destination and source are aligned
static inline void hvx_copy_f16_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
hvx_copy_f16_f32_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);
}
// copy/convert n fp32 elements into n fp16 elements : source is unaligned, destination is aligned
// copy/convert n fp32 elements into n fp16 elements : destination is aligned, source is unaligned
static inline void hvx_copy_f16_f32_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
hvx_copy_f16_f32_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a);
}
// copy/convert n fp32 elements into n fp16 elements : source is aligned, destination is unaligned
// copy/convert n fp32 elements into n fp16 elements : destination is unaligned, source is aligned
static inline void hvx_copy_f16_f32_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) src % 128 == 0);
hvx_copy_f16_f32_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u);
}
// copy/convert n fp32 elements into n fp16 elements : source is unaligned, destination is unaligned
// copy/convert n fp32 elements into n fp16 elements : destination and source are unaligned
static inline void hvx_copy_f16_f32_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
hvx_copy_f16_f32_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u);
}
@@ -235,28 +235,98 @@ static inline void hvx_copy_f16_f32_uu(uint8_t * restrict dst, const uint8_t * r
} \
} while(0)
// copy/convert n fp16 elements into n fp32 elements : source is aligned, destination is aligned
// copy/convert n fp16 elements into n fp32 elements : destination and source are aligned
static inline void hvx_copy_f32_f16_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
hvx_copy_f32_f16_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);
}
// copy/convert n fp16 elements into n fp32 elements : source is unaligned, destination is aligned
// copy/convert n fp16 elements into n fp32 elements : destination is aligned, source is unaligned
static inline void hvx_copy_f32_f16_au(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
hvx_copy_f32_f16_loop_body(HVX_Vector, HVX_UVector, hvx_vec_store_a);
}
// copy/convert n fp16 elements into n fp32 elements : source is aligned, destination is unaligned
// copy/convert n fp16 elements into n fp32 elements : destination is unaligned, source is aligned
static inline void hvx_copy_f32_f16_ua(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) src % 128 == 0);
hvx_copy_f32_f16_loop_body(HVX_UVector, HVX_Vector, hvx_vec_store_u);
}
// copy/convert n fp16 elements into n fp32 elements : source is unaligned, destination is unaligned
// copy/convert n fp16 elements into n fp32 elements : destination and source are unaligned
static inline void hvx_copy_f32_f16_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
hvx_copy_f32_f16_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u);
}
//// fp32 -> int32
#define hvx_copy_i32_f32_loop_body(dst_type, src_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
\
const uint32_t elem_size = sizeof(int32_t); \
const uint32_t epv = 128 / elem_size; \
const uint32_t nvec = n / epv; \
const uint32_t nloe = n % epv; \
\
uint32_t i = 0; \
_Pragma("unroll(4)") \
for (; i < nvec; i++) { \
vdst[i] = Q6_Vw_equals_Vsf(vsrc[i]); \
} \
if (nloe) { \
HVX_Vector v = Q6_Vw_equals_Vsf(vsrc[i]); \
vec_store((void *) &vdst[i], nloe * elem_size, v); \
} \
} while(0)
// copy/convert n fp32 elements into n int32 elements : destination and source are aligned
static inline void hvx_copy_i32_f32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
hvx_copy_i32_f32_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);
}
// copy/convert n fp32 elements into n int32 elements : destination and source are unaligned
static inline void hvx_copy_i32_f32_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
hvx_copy_i32_f32_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u);
}
//// int32 -> fp32
#define hvx_copy_f32_i32_loop_body(dst_type, src_type, vec_store) \
do { \
dst_type * restrict vdst = (dst_type *) dst; \
src_type * restrict vsrc = (src_type *) src; \
\
const uint32_t elem_size = sizeof(float); \
const uint32_t epv = 128 / elem_size; \
const uint32_t nvec = n / epv; \
const uint32_t nloe = n % epv; \
\
uint32_t i = 0; \
_Pragma("unroll(4)") \
for (; i < nvec; i++) { \
vdst[i] = Q6_Vsf_equals_Vw(vsrc[i]); \
} \
if (nloe) { \
HVX_Vector v = Q6_Vsf_equals_Vw(vsrc[i]); \
vec_store((void *) &vdst[i], nloe * elem_size, v); \
} \
} while(0)
// copy/convert n int32 elements into n fp32 elements : destination and source are aligned
static inline void hvx_copy_f32_i32_aa(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
assert((unsigned long) dst % 128 == 0);
assert((unsigned long) src % 128 == 0);
hvx_copy_f32_i32_loop_body(HVX_Vector, HVX_Vector, hvx_vec_store_a);
}
// copy/convert n int32 elements into n fp32 elements : destination and source are unaligned
static inline void hvx_copy_f32_i32_uu(uint8_t * restrict dst, const uint8_t * restrict src, uint32_t n) {
hvx_copy_f32_i32_loop_body(HVX_UVector, HVX_UVector, hvx_vec_store_u);
}
#endif // HVX_COPY_H