public_git/whisper.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/whisper.cpp.git synced 2026-10-06 06:21:41 +02:00
Code Issues Packages Projects Releases Wiki Activity

talk-llama : sync llama.cpp

Browse Source
This commit is contained in:
Georgi Gerganov
2026-07-10 13:06:42 +03:00
parent afa6bffd2b
commit 289ecb05c9
32 changed files with 4809 additions and 313 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/talk-llama/llama-model.cpp
+55 -2
View File
@@ -11,6 +11,7 @@
#include "llama-kv-cache.h"
#include "llama-kv-cache-iswa.h"
#include "llama-kv-cache-dsa.h"
#include "llama-kv-cache-dsv4.h"
#include "llama-memory-hybrid.h"
#include "llama-memory-hybrid-iswa.h"
#include "llama-memory-recurrent.h"
@@ -181,6 +182,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
return new llama_model_deepseek2ocr(params);
case LLM_ARCH_DEEPSEEK32:
return new llama_model_deepseek32(params);
case LLM_ARCH_DEEPSEEK4:
return new llama_model_deepseek4(params);
case LLM_ARCH_GLM_DSA:
return new llama_model_glm_dsa(params);
case LLM_ARCH_MISTRAL4:
@@ -291,6 +294,8 @@ static llama_model * llama_model_mapping(llm_arch arch, const llama_model_params
return new llama_model_mistral3(params);
case LLM_ARCH_EAGLE3:
return new llama_model_eagle3(params);
case LLM_ARCH_DFLASH:
return new llama_model_dflash(params);
case LLM_ARCH_MIMO2:
return new llama_model_mimo2(params);
case LLM_ARCH_KIMI_LINEAR:
@@ -815,6 +820,7 @@ static const char * llama_expert_gating_func_name(llama_expert_gating_func_type
switch (type) {
case LLAMA_EXPERT_GATING_FUNC_TYPE_SOFTMAX: return "softmax";
case LLAMA_EXPERT_GATING_FUNC_TYPE_SIGMOID: return "sigmoid";
case LLAMA_EXPERT_GATING_FUNC_TYPE_SQRT_SOFTPLUS: return "sqrtsoftplus";
default: return "unknown";
}
}
@@ -947,6 +953,8 @@ static buft_list_t make_gpu_buft_list(ggml_backend_dev_t dev, llama_split_mode s
if (buft != nullptr) {
buft_list.emplace_back(dev, buft);
}
} else {
throw std::runtime_error(format("device %s does not support split buffers", ggml_backend_dev_name(dev)));
}
}
@@ -981,6 +989,8 @@ struct llama_model::impl {
std::string desc_str;
llama_ftype ftype = LLAMA_FTYPE_ALL_F32;
// model memory mapped files
llama_mmaps mappings;
@@ -1004,9 +1014,17 @@ struct llama_model::impl {
std::vector<layer_dev> dev_layer;
bool has_tensor_overrides;
std::vector<float> tensor_split_owned;
};
llama_model::llama_model(const llama_model_params & params) : params(params), pimpl(std::make_unique<impl>()) {
if (params.tensor_split != nullptr) {
// llama_model_params stores tensor_split as a borrowed pointer, but the model
// may need it later for tensor-parallel KV-cache split metadata.
pimpl->tensor_split_owned.assign(params.tensor_split, params.tensor_split + llama_max_devices());
this->params.tensor_split = pimpl->tensor_split_owned.data();
}
pimpl->has_tensor_overrides = params.tensor_buft_overrides && params.tensor_buft_overrides[0].pattern;
}
@@ -1194,6 +1212,8 @@ void llama_model_base::load_hparams(llama_model_loader & ml) {
pimpl->desc_str = arch_name() + " " + type_name() + " " + ml.ftype_name();
pimpl->ftype = ml.ftype;
if (hparams.f_max_alibi_bias > 0.0f) {
hparams.use_alibi = true;
}
@@ -1640,6 +1660,10 @@ std::string llama_model::desc() const {
return pimpl->desc_str;
}
llama_ftype llama_model::ftype() const {
return pimpl->ftype;
}
size_t llama_model::size() const {
return pimpl->n_bytes;
}
@@ -2154,7 +2178,24 @@ llama_memory_i * llama_model::create_memory(const llama_memory_params & params,
}
}
if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) {
if (arch == LLM_ARCH_DEEPSEEK4) {
GGML_ASSERT(hparams.swa_type != LLAMA_SWA_TYPE_NONE);
res = new llama_kv_cache_dsv4(
*this,
params.type_k,
params.type_v,
!cparams.flash_attn,
cparams.offload_kqv,
params.swa_full,
cparams.kv_unified,
cparams.n_ctx_seq,
cparams.n_seq_max,
cparams.n_ubatch,
1,
filter,
reuse);
} else if (hparams.swa_type != LLAMA_SWA_TYPE_NONE) {
GGML_ASSERT(hparams.is_swa_any());
if (arch == LLM_ARCH_GEMMA4_ASSISTANT) {
@@ -2326,6 +2367,11 @@ int32_t llama_model_n_head_kv(const llama_model * model) {
}
int32_t llama_model_n_swa(const llama_model * model) {
// dsv4 kv-cache has SWA but it cannot be used as a rollback because of
// other compression ratios, so we return 0 here
if (model->arch == LLM_ARCH_DEEPSEEK4) {
return 0;
}
return model->hparams.n_swa;
}
@@ -2407,6 +2453,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
case LLM_ARCH_DEEPSEEK2:
case LLM_ARCH_DEEPSEEK2OCR:
case LLM_ARCH_DEEPSEEK32:
case LLM_ARCH_DEEPSEEK4:
case LLM_ARCH_PLM:
case LLM_ARCH_CHATGLM:
case LLM_ARCH_GRANITE:
@@ -2494,6 +2541,7 @@ llama_rope_type llama_model_rope_type(const llama_model * model) {
case LLM_ARCH_STEP35:
case LLM_ARCH_TALKIE:
case LLM_ARCH_MELLUM:
case LLM_ARCH_DFLASH:
return LLAMA_ROPE_TYPE_NEOX;
case LLM_ARCH_QWEN2VL:
@@ -2586,6 +2634,10 @@ int32_t llama_model_desc(const llama_model * model, char * buf, size_t buf_size)
return snprintf(buf, buf_size, "%s", model->desc().c_str());
}
llama_ftype llama_model_ftype(const llama_model * model) {
return model->ftype();
}
uint64_t llama_model_size(const llama_model * model) {
return model->size();
}
@@ -2617,7 +2669,8 @@ bool llama_model_has_encoder(const llama_model * model) {
switch (model->arch) {
case LLM_ARCH_T5:
case LLM_ARCH_T5ENCODER:
case LLM_ARCH_EAGLE3: return true;
case LLM_ARCH_EAGLE3:
case LLM_ARCH_DFLASH: return true;
default: return false;
}
}