public_git/whisper.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/whisper.cpp.git synced 2026-10-17 03:35:39 +02:00
Code Issues Packages Projects Releases Wiki Activity

ggml : add alloc_buffer_n to buffer type interface (llama/23671)

Browse Source 
* ggml : add `alloc_buffer_n` to buffer type interface

Add alloc_buffer_n method to ggml_backend_buffer_type_i
interface, with a public API ggml_backend_buft_alloc_buffer_n.

- Default implementation in ggml-backend.cpp handles multi-buffer
  splitting and tensor allocation via ggml_tallocr
- Meta buffer type provides custom implementation that creates
  per-device sub-contexts and delegates to simple buffer types
- ggml_backend_alloc_ctx_tensors_from_buft now collects tensors
  into a list and delegates to the new API
- Remove temporary ggml_backend_meta_alloc_ctx_tensors_from_buft
- Add NULL alloc_buffer_n to all existing buffer type
  interfaces (cpu, metal, openvino, hexagon, webgpu, zdnn, virtgpu, repack)

Assisted-by: llama.cpp:local pi

* cont : fix `cur_buf_size` init after flushing a buffer

* ggml : add TODO tag for shared buffer split logic

Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp

* tests : add alloc_buffer_n coverage

Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp

* cont : fix compile warnings

* tests : add descriptions for alloc_buffer_n tests

Assisted-by: pi:llama.cpp/Qwen3.8-27B

* ggml : address review comments on alloc_buffer_n

- restore GGML_LOG_ERROR on buffer alloc / tensor init failure in the
  default impl (name the failing tensor)
- check the malloc result and drop the _impl indirection in
  ggml_backend_alloc_ctx_tensors_from_buft
- remove comments that restate the code
- fix the TAG_ALLOC_SHARED_BUFFER_SPLIT typo

Assisted-by: pi:llama.cpp/Qwen3.8-27B

* ggml : add get_alloc_size_n to buffer type interface

- Add ggml_backend_buft_get_alloc_size_n public API
- Add optional get_alloc_size_n callback to ggml_backend_buffer_type_i
- Share tensor->buffer planning between alloc_buffer_n default and get_alloc_size_n default
- Replace unchecked realloc with std::vector in alloc_buffer_n default
- Make ggml_backend_alloc_ctx_tensors_from_buft_size use the new API
- Add test-alloc coverage for get_alloc_size_n

Assisted-by: pi:llama.cpp/DeepSeek-V4-Flash-Vision-Exp

* cont : report malloc failure
This commit is contained in:
Georgi Gerganov
2026-10-06 10:38:16 +03:00
parent 8291ab849c
commit 4435763c17
25 changed files with 453 additions and 320 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ggml/src/ggml-backend.cpp
+156 -12
View File
@@ -45,6 +45,138 @@ ggml_backend_buffer_t ggml_backend_buft_alloc_buffer(ggml_backend_buffer_type_t
return buft->iface.alloc_buffer(buft, size);
}
// shared planning logic for allocating a list of tensors into one or more buffers of the given type
struct ggml_backend_buft_alloc_buffer_n_plan_item {
size_t size; // total bytes for this buffer
int first; // first tensor index (inclusive)
int last; // last tensor index (exclusive)
};
using ggml_backend_buft_alloc_buffer_n_plan_t = std::vector<ggml_backend_buft_alloc_buffer_n_plan_item>;
static ggml_backend_buft_alloc_buffer_n_plan_t ggml_backend_buft_alloc_buffer_n_plan(
ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors) {
ggml_backend_buft_alloc_buffer_n_plan_t plan;
const size_t alignment = ggml_backend_buft_get_alignment(buft);
const size_t max_size = ggml_backend_buft_get_max_size(buft);
size_t cur_buf_size = 0;
int first = 0;
for (int i = 0; i < n_tensors; i++) {
size_t this_size = 0;
struct ggml_tensor * t = tensors[i];
if (t->data == NULL && t->view_src == NULL) {
this_size = GGML_PAD(ggml_backend_buft_get_alloc_size(buft, t), alignment);
}
// flush the current buffer if adding this tensor would exceed max_size
if (cur_buf_size > 0 && (cur_buf_size + this_size) > max_size) {
plan.push_back({ cur_buf_size, first, i });
cur_buf_size = this_size;
first = i;
} else {
cur_buf_size += this_size;
}
}
if (cur_buf_size > 0) {
plan.push_back({ cur_buf_size, first, n_tensors });
}
return plan;
}
// default implementation of alloc_buffer_n
// allocates tensors from a list into one or more buffers of the given type
static ggml_backend_buffer_t ggml_backend_buft_alloc_buffer_n_default(ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors) {
const ggml_backend_buft_alloc_buffer_n_plan_t plan = ggml_backend_buft_alloc_buffer_n_plan(buft, tensors, n_tensors);
std::vector<ggml_backend_buffer_t> buffers;
buffers.reserve(plan.size());
for (const ggml_backend_buft_alloc_buffer_n_plan_item & item : plan) {
ggml_backend_buffer_t buffer = ggml_backend_buft_alloc_buffer(buft, item.size);
if (buffer == NULL) {
GGML_LOG_ERROR("%s: failed to allocate %s buffer of size %zu\n", __func__, ggml_backend_buft_name(buft), item.size);
for (ggml_backend_buffer_t b : buffers) {
ggml_backend_buffer_free(b);
}
return NULL;
}
struct ggml_tallocr tallocr = ggml_tallocr_new(buffer);
// allocate tensors in the current buffer
struct ggml_tensor * t_failed = NULL;
for (int j = item.first; j < item.last; j++) {
struct ggml_tensor * t = tensors[j];
if (t->data == NULL) {
if (t->view_src == NULL) {
if (ggml_tallocr_alloc(&tallocr, t) != GGML_STATUS_SUCCESS) {
t_failed = t;
break;
}
} else if (t->buffer == NULL) {
if (ggml_backend_view_init(t) != GGML_STATUS_SUCCESS) {
t_failed = t;
break;
}
}
} else {
if (t->view_src != NULL && t->buffer == NULL) {
// view of a pre-allocated tensor
if (ggml_backend_view_init(t) != GGML_STATUS_SUCCESS) {
t_failed = t;
break;
}
}
}
}
if (t_failed != NULL) {
GGML_LOG_ERROR("%s: failed to initialize tensor %s\n", __func__, t_failed->name);
for (ggml_backend_buffer_t b : buffers) {
ggml_backend_buffer_free(b);
}
ggml_backend_buffer_free(buffer);
return NULL;
}
buffers.push_back(buffer);
}
if (buffers.empty()) {
return NULL;
}
if (buffers.size() == 1) {
return buffers[0];
}
return ggml_backend_multi_buffer_alloc_buffer(buffers.data(), buffers.size());
}
// default implementation of get_alloc_size_n
// returns the total size that alloc_buffer_n_default would allocate for the given tensors
static size_t ggml_backend_buft_get_alloc_size_n_default(ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors) {
const ggml_backend_buft_alloc_buffer_n_plan_t plan = ggml_backend_buft_alloc_buffer_n_plan(buft, tensors, n_tensors);
size_t total = 0;
for (const ggml_backend_buft_alloc_buffer_n_plan_item & item : plan) {
total += item.size;
}
return total;
}
ggml_backend_buffer_t ggml_backend_buft_alloc_buffer_n(ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors) {
GGML_ASSERT(buft);
if (buft->iface.alloc_buffer_n) {
return buft->iface.alloc_buffer_n(buft, tensors, n_tensors);
}
return ggml_backend_buft_alloc_buffer_n_default(buft, tensors, n_tensors);
}
size_t ggml_backend_buft_get_alignment(ggml_backend_buffer_type_t buft) {
GGML_ASSERT(buft);
return buft->iface.get_alignment(buft);
@@ -78,6 +210,14 @@ size_t ggml_backend_buft_get_alloc_size(ggml_backend_buffer_type_t buft, const s
return ggml_nbytes(tensor);
}
size_t ggml_backend_buft_get_alloc_size_n(ggml_backend_buffer_type_t buft, struct ggml_tensor ** tensors, int n_tensors) {
GGML_ASSERT(buft);
if (buft->iface.get_alloc_size_n) {
return buft->iface.get_alloc_size_n(buft, tensors, n_tensors);
}
return ggml_backend_buft_get_alloc_size_n_default(buft, tensors, n_tensors);
}
bool ggml_backend_buft_is_host(ggml_backend_buffer_type_t buft) {
GGML_ASSERT(buft);
if (buft->iface.is_host) {
@@ -2486,12 +2626,14 @@ static bool ggml_backend_cpu_buffer_type_is_host(ggml_backend_buffer_type_t buft
ggml_backend_buffer_type_t ggml_backend_cpu_buffer_type(void) {
static struct ggml_backend_buffer_type ggml_backend_cpu_buffer_type = {
/* .iface = */ {
/* .get_name = */ ggml_backend_cpu_buffer_type_get_name,
/* .alloc_buffer = */ ggml_backend_cpu_buffer_type_alloc_buffer,
/* .get_alignment = */ ggml_backend_cpu_buffer_type_get_alignment,
/* .get_max_size = */ NULL, // defaults to SIZE_MAX
/* .get_alloc_size = */ NULL, // defaults to ggml_nbytes
/* .is_host = */ ggml_backend_cpu_buffer_type_is_host,
/* .get_name = */ ggml_backend_cpu_buffer_type_get_name,
/* .alloc_buffer = */ ggml_backend_cpu_buffer_type_alloc_buffer,
/* .alloc_buffer_n = */ NULL,
/* .get_alignment = */ ggml_backend_cpu_buffer_type_get_alignment,
/* .get_max_size = */ NULL, // defaults to SIZE_MAX
/* .get_alloc_size = */ NULL, // defaults to ggml_nbytes
/* .get_alloc_size_n = */ NULL,
/* .is_host = */ ggml_backend_cpu_buffer_type_is_host,
},
/* .device = */ NULL, // FIXME ggml_backend_reg_dev_get(ggml_backend_cpu_reg(), 0),
/* .context = */ NULL,
@@ -2509,12 +2651,14 @@ static const char * ggml_backend_cpu_buffer_from_ptr_type_get_name(ggml_backend_
static ggml_backend_buffer_type_t ggml_backend_cpu_buffer_from_ptr_type(void) {
static struct ggml_backend_buffer_type ggml_backend_cpu_buffer_type = {
/* .iface = */ {
/* .get_name = */ ggml_backend_cpu_buffer_from_ptr_type_get_name,
/* .alloc_buffer = */ ggml_backend_cpu_buffer_type_alloc_buffer,
/* .get_alignment = */ ggml_backend_cpu_buffer_type_get_alignment,
/* .get_max_size = */ NULL, // defaults to SIZE_MAX
/* .get_alloc_size = */ NULL, // defaults to ggml_nbytes
/* .is_host = */ ggml_backend_cpu_buffer_type_is_host,
/* .get_name = */ ggml_backend_cpu_buffer_from_ptr_type_get_name,
/* .alloc_buffer = */ ggml_backend_cpu_buffer_type_alloc_buffer,
/* .alloc_buffer_n = */ NULL,
/* .get_alignment = */ ggml_backend_cpu_buffer_type_get_alignment,
/* .get_max_size = */ NULL, // defaults to SIZE_MAX
/* .get_alloc_size = */ NULL, // defaults to ggml_nbytes
/* .get_alloc_size_n = */ NULL,
/* .is_host = */ ggml_backend_cpu_buffer_type_is_host,
},
/* .device = */ NULL, // FIXME ggml_backend_reg_dev_get(ggml_backend_cpu_reg(), 0),
/* .context = */ NULL,