From 60f14a5eccc38a4c71e59b254b366400a84ab167 Mon Sep 17 00:00:00 2001 From: Georgi Gerganov Date: Thu, 20 Aug 2026 17:00:54 +0300 Subject: [PATCH] metal : dequant kv cache only for large batches (llama/27438) --- ggml/src/ggml-metal/ggml-metal-ops.cpp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-metal/ggml-metal-ops.cpp b/ggml/src/ggml-metal/ggml-metal-ops.cpp index 2dde14d8d..8311544b3 100644 --- a/ggml/src/ggml-metal/ggml-metal-ops.cpp +++ b/ggml/src/ggml-metal/ggml-metal-ops.cpp @@ -2806,6 +2806,13 @@ bool ggml_metal_op_flash_attn_ext_use_vec(const ggml_tensor * op) { static bool ggml_metal_op_flash_attn_ext_use_kv_f16(const ggml_tensor * op) { assert(op->op == GGML_OP_FLASH_ATTN_EXT); + // depending on compute/bandwidth ratio, dequant to f16 kv is not always beneficial + // ref: https://github.com/ggml-org/llama.cpp/pull/27390#issuecomment-5355152767 + // TODO: tune per device + if (op->src[0]->ne[1] < 32) { + return false; + } + switch (op->src[1]->type) { case GGML_TYPE_Q4_0: case GGML_TYPE_Q4_1: @@ -2968,9 +2975,10 @@ size_t ggml_metal_op_flash_attn_ext_extra_tmp(const ggml_tensor * op) { size_t ggml_metal_op_flash_attn_ext_extra_kv_f16(const ggml_tensor * op) { assert(op->op == GGML_OP_FLASH_ATTN_EXT); - if (!ggml_metal_op_flash_attn_ext_use_kv_f16(op)) { - return 0; - } + // note: always reserve the temp buffer to avoid graph reallocations + //if (!ggml_metal_op_flash_attn_ext_use_kv_f16(op)) { + // return 0; + //} GGML_TENSOR_LOCALS( int32_t, ne2, op->src[2], ne);