public_git/whisper.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/whisper.cpp.git synced 2026-09-30 11:36:38 +02:00
Code Issues Packages Projects Releases Wiki Activity

cuda : tune MMVQ to MMQ crossover for SM70 (Volta) (llama/28912)

Browse Source 
* tune MMVQ to MMQ crossover for SM70 (Volta)

Signed-off-by: Yangyu Chen <cyy@cyyself.name>

* Apply suggestion from @JohannesGaessler

* Apply suggestion from @JohannesGaessler

* Apply suggestion from @JohannesGaessler

---------

Signed-off-by: Yangyu Chen <cyy@cyyself.name>
Co-authored-by: Johannes Gäßler <johannesg@5d6.de>
This commit is contained in:
Yangyu Chen
2026-09-23 20:46:47 +03:00
committed by Georgi Gerganov
co-authored by Johannes Gäßler
parent 87b6876bb8
commit 76d02a12b8
1 changed files with 16 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/src/ggml-cuda/mmvq.cu
+16
View File
@@ -365,6 +365,22 @@ bool ggml_cuda_should_use_mmvq(enum ggml_type type, int cc, int64_t ne11) {
return ne11 <= MMVQ_MAX_BATCH_SIZE;
}
}
if (GGML_CUDA_CC_IS_NVIDIA(cc) && cc == GGML_CUDA_CC_VOLTA) {
switch (type) {
case GGML_TYPE_Q2_K:
return ne11 <= 4;
case GGML_TYPE_Q3_K:
return ne11 <= 6;
case GGML_TYPE_Q4_K:
return ne11 <= 5;
case GGML_TYPE_Q5_K:
return ne11 <= 6;
case GGML_TYPE_Q6_K:
return ne11 <= 7;
default:
return ne11 <= MMVQ_MAX_BATCH_SIZE;
}
}
if (GGML_CUDA_CC_IS_CDNA(cc)) {
if (GGML_CUDA_CC_IS_CDNA1(cc)) {
switch (type) {