public_git
/
whisper.cpp
mirror of https://github.com/ggml-org/whisper.cpp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

opencl: use flat mv q5_k when weight exceeds image1d_buffer_t limit (llama/26880)

This commit is contained in:
lhez 2026-08-11 22:02:28 -07:00 committed by Georgi Gerganov
parent cdbe455102
commit db3687b297
1 changed files with 18 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
ggml/src/ggml-opencl/ggml-opencl.cpp
View File

@ -7076,6 +7076,19 @@ inline bool enable_adreno_trans_weight(const ggml_backend_opencl_context *backen
return ((elem_num < 128 * 1024 * 1024) && adreno_kernel && shape_ok); // max element num: 2**27 return ((elem_num < 128 * 1024 * 1024) && adreno_kernel && shape_ok); // max element num: 2**27
} }
inline bool enable_adreno_trans_weight_q5_K(const ggml_backend_opencl_context *backend_ctx, const ggml_tensor *tensor) {
if (!use_adreno_kernels(backend_ctx, tensor)) {
return false;
}
const size_t elem_num = ggml_nelements(tensor);
const size_t q_img_width = elem_num / 8;
const size_t qh_img_width = elem_num / 16;
return q_img_width <= backend_ctx->image_max_buffer_size &&
qh_img_width <= backend_ctx->image_max_buffer_size;
}
static inline bool use_flat_gemv_for_large_m_q4_K(const ggml_tensor *tensor) { static inline bool use_flat_gemv_for_large_m_q4_K(const ggml_tensor *tensor) {
// gemv_noshuffle variant perf drops for large M, use flat variant for large M. // gemv_noshuffle variant perf drops for large M, use flat variant for large M.
// threshold is well above typical hidden/FFN dims, but below typical vocab sizes. // threshold is well above typical hidden/FFN dims, but below typical vocab sizes.
@ -9255,7 +9268,7 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS #ifdef GGML_OPENCL_USE_ADRENO_KERNELS
cl_kernel kernel = backend_ctx->kernel_convert_block_q5_K; cl_kernel kernel = backend_ctx->kernel_convert_block_q5_K;
if (use_adreno_kernels(backend_ctx, tensor)) { if (enable_adreno_trans_weight_q5_K(backend_ctx, tensor)) {
kernel = backend_ctx->kernel_convert_block_q5_K_noshuffle; kernel = backend_ctx->kernel_convert_block_q5_K_noshuffle;
} }
#else #else
@ -9290,7 +9303,7 @@ static void ggml_backend_opencl_buffer_set_tensor(ggml_backend_buffer_t buffer,
tensor->extra = extra; tensor->extra = extra;
#ifdef GGML_OPENCL_USE_ADRENO_KERNELS #ifdef GGML_OPENCL_USE_ADRENO_KERNELS
if (use_adreno_kernels(backend_ctx, tensor)) { if (enable_adreno_trans_weight_q5_K(backend_ctx, tensor)) {
int M = tensor->ne[1]; int M = tensor->ne[1];
int K = tensor->ne[0]; int K = tensor->ne[0];
@ -10388,7 +10401,7 @@ static void ggml_backend_opencl_buffer_get_tensor(ggml_backend_buffer_t buffer,
CL_CHECK(clReleaseMemObject(data_device)); CL_CHECK(clReleaseMemObject(data_device));
return; return;
} }
if (use_adreno_kernels(backend_ctx, tensor)) { if (enable_adreno_trans_weight_q5_K(backend_ctx, tensor)) {
int M = tensor->ne[1]; int M = tensor->ne[1];
int K = tensor->ne[0]; int K = tensor->ne[0];
@ -18928,7 +18941,8 @@ static void ggml_cl_mul_mat(ggml_backend_t backend, const ggml_tensor * src0, co
} }
// q5_K x fp32 // q5_K x fp32
if (src0t == GGML_TYPE_Q5_K && src1t == GGML_TYPE_F32) { if (src0t == GGML_TYPE_Q5_K && src1t == GGML_TYPE_F32 &&
enable_adreno_trans_weight_q5_K(backend_ctx, src0)) {
ggml_cl_mul_mat_q5_K_f32_adreno(backend, src0, src1, dst); ggml_cl_mul_mat_q5_K_f32_adreno(backend, src0, src1, dst);
return; return;
} }