public_git/whisper.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/whisper.cpp.git synced 2026-10-21 09:41:29 +02:00
Code Issues Packages Projects Releases Wiki Activity

CUDA: replace GGML_FA_ALL_QUANTS with GGML_FA_QUANTS, more control over what is compiled (llama/28079)

Browse Source 
* CUDA: add configurable FA quant combinations

Assisted-by: Codex

* remove all flags but , add runtime fallback with warning for uncompiled combination

* Update docs/build.md

Co-authored-by: Johannes Gäßler <johannesg@5d6.de>

* apply code review comments

---------

Co-authored-by: Johannes Gäßler <johannesg@5d6.de>
This commit is contained in:
Piotr Wilkin (ilintar)
2026-09-14 20:45:06 +03:00
committed by Georgi Gerganov
co-authored by Johannes Gäßler
parent dca2df4211
commit 4d506f58e4
7 changed files with 182 additions and 132 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/cmake/common.cmake
+71
View File
@@ -48,3 +48,74 @@ function(ggml_get_system_arch)
set(GGML_SYSTEM_ARCH "UNKNOWN" PARENT_SCOPE)
endif()
endfunction()
# Determines which FlashAttention vector kernel template instances to compile, returns them in OUT_SRCS.
function(ggml_cuda_fattn_vec_instances DIR OUT_SRCS)
set(FA_TYPES q4_0 q4_1 q5_0 q5_1 q8_0 bf16 f16)
string(TOLOWER "${GGML_CUDA_FA_QUANTS}" FA_QUANTS)
string(STRIP "${FA_QUANTS}" FA_QUANTS)
if (GGML_CUDA_FA_ALL_QUANTS)
message(WARNING "GGML_CUDA_FA_ALL_QUANTS is deprecated, use GGML_CUDA_FA_QUANTS=all instead")
set(FA_QUANTS all)
endif()
if (NOT FA_QUANTS)
message(FATAL_ERROR "GGML_CUDA_FA_QUANTS must not be empty")
endif()
if (FA_QUANTS STREQUAL "all")
set(FA_COMBINATIONS "")
foreach (TYPE_V IN LISTS FA_TYPES)
foreach (TYPE_K IN LISTS FA_TYPES)
list(APPEND FA_COMBINATIONS ${TYPE_K}-${TYPE_V})
endforeach()
endforeach()
else()
set(FA_COMBINATIONS f16-f16)
string(REPLACE "," ";" FA_SELECTED "${FA_QUANTS}")
foreach (COMBINATION IN LISTS FA_SELECTED)
string(STRIP "${COMBINATION}" COMBINATION)
if (NOT COMBINATION MATCHES "^([a-z0-9_]+)-([a-z0-9_]+)$")
message(FATAL_ERROR "GGML_CUDA_FA_QUANTS: \"${COMBINATION}\" is not \"all\" or a <type_K>-<type_V> combination")
endif()
set(TYPE_K ${CMAKE_MATCH_1})
set(TYPE_V ${CMAKE_MATCH_2})
foreach (TYPE ${TYPE_K} ${TYPE_V})
if (NOT TYPE IN_LIST FA_TYPES)
message(FATAL_ERROR
"GGML_CUDA_FA_QUANTS: unknown type \"${TYPE}\" in \"${COMBINATION}\", must be one of: ${FA_TYPES}")
endif()
endforeach()
list(APPEND FA_COMBINATIONS ${TYPE_K}-${TYPE_V})
endforeach()
endif()
list(REMOVE_DUPLICATES FA_COMBINATIONS)
string(REPLACE ";" "," FA_QUANTS_DEFINE "${FA_QUANTS}")
add_compile_definitions(GGML_CUDA_FA_QUANTS="${FA_QUANTS_DEFINE}")
foreach (TYPE_V IN LISTS FA_TYPES)
foreach (TYPE_K IN LISTS FA_TYPES)
if ("${TYPE_K}-${TYPE_V}" IN_LIST FA_COMBINATIONS)
set(COMPILED 1)
else()
set(COMPILED 0)
endif()
string(TOUPPER "GGML_CUDA_FA_${TYPE_K}_${TYPE_V}" COMBINATION_DEF)
add_compile_definitions(${COMBINATION_DEF}=${COMPILED})
endforeach()
endforeach()
message(STATUS "FlashAttention K-V type combinations: ${FA_COMBINATIONS}")
set(SRCS "")
foreach (COMBINATION IN LISTS FA_COMBINATIONS)
set(SRC "${DIR}/template-instances/fattn-vec-instance-${COMBINATION}.cu")
if (NOT EXISTS "${SRC}")
message(FATAL_ERROR "FlashAttention template instance \"${SRC}\" does not exist")
endif()
list(APPEND SRCS "${SRC}")
endforeach()
set(${OUT_SRCS} ${SRCS} PARENT_SCOPE)
endfunction()