From c2dc946f3f548c78f2a32c6e0efbbff477b0ff8a Mon Sep 17 00:00:00 2001 From: Pranesh Gonegandla Date: Thu, 20 Aug 2026 12:36:21 +0000 Subject: [PATCH] CUDA: adding switch points per HW and quant type to tune the mvq->MMQ decode crossover (llama/26079) * CUDA: runtime GGML_CUDA_MMVQ_MAX to tune the mvq->MMQ decode crossover Add a runtime override of the mul_mat_vec_q -> MMQ batch crossover (default MMVQ_MAX_BATCH_SIZE). Lowering it routes batches above the threshold from the CUDA-core vector kernel to the int8 MMQ tensor-core path, which is faster once quantized decode becomes compute-bound at B>1 (measured +23-41% at B=8 on RTX 5090 for Q4_K dense, no low-batch loss). The value is parsed once and clamped to [1, MMVQ_MAX_BATCH_SIZE], since mul_mat_vec_q asserts ncols_dst <= that; invalid input warns and falls back to the default. The override is applied consistently in both the mul_mat_vec_q and MUL_MAT_ID dispatch paths. Default behavior unchanged. * Added Blackwell specific switch point, to reduce dependence on runtime env var. * Add per-HW switch point values for DGX Spark and removing runtime env var * Adding switch points for Ada, tested on RTX 4090 * Modifying DGX Spark numbers based on latest run and adding some comments and small functional changes relating to MoE * Reverting an unnecessary conditional * Update ggml/src/ggml-cuda/mmvq.cu --------- Co-authored-by: praneshgo <227579474+praneshgo@users.noreply.github.com> Co-authored-by: Oliver Simons --- ggml/src/ggml-cuda/mmvq.cu | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/ggml/src/ggml-cuda/mmvq.cu b/ggml/src/ggml-cuda/mmvq.cu index c99923804..970534809 100644 --- a/ggml/src/ggml-cuda/mmvq.cu +++ b/ggml/src/ggml-cuda/mmvq.cu @@ -290,6 +290,42 @@ bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11) { if (!ggml_is_quantized(type)) { return false; } + // k-quants cost more to decode and mvq redoes that per column, so MMQ wins sooner. + // Only list quant-types MMQ supports, others would fall back to cuBLAS. + if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_ADA_LOVELACE) { + switch (type) { // tuned on RTX 4090 + case GGML_TYPE_Q2_K: + return ne11 <= 4; + case GGML_TYPE_Q3_K: + return ne11 <= 6; + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + return ne11 <= 7; + default: + return ne11 <= MMVQ_MAX_BATCH_SIZE; + } + } + if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_BLACKWELL) { + switch (type) { // tuned on RTX 5090 + case GGML_TYPE_Q2_K: + case GGML_TYPE_Q3_K: + case GGML_TYPE_Q4_K: + case GGML_TYPE_Q5_K: + return ne11 <= 5; + case GGML_TYPE_Q6_K: + return ne11 <= 7; + default: + return ne11 <= MMVQ_MAX_BATCH_SIZE; + } + } + if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_DGX_SPARK) { + switch (type) { // tuned on DGX Spark GB10 + case GGML_TYPE_Q2_K: + return ne11 <= 6; + default: + return ne11 <= MMVQ_MAX_BATCH_SIZE; + } + } if (GGML_CUDA_CC_IS_CDNA(cc)) { if (GGML_CUDA_CC_IS_CDNA1(cc)) { switch (type) {