ggml-vulkan: graceful fallback on pipeline compilation failure

Some Vulkan drivers (observed on Adreno, Qualcomm build 923a446bf8)
fail to compile compute shaders at runtime, reporting
"Failed to link shaders" and returning ErrorUnknown from
createComputePipeline. Previously this threw a C++ exception that
propagated as an uncaught abort, or the resulting null pipeline was
dispatched causing SIGSEGV:

    AdrenoVK-0: Failed to link shaders.
    Fatal signal 11 (SIGSEGV), code 1 (SEGV_MAPERR), fault addr 0xe8
    #01 ggml_vk_dispatch_pipeline<vk_mat_mat_push_constants>+360
    #02 ggml_vk_mul_mat_q_f16+6616
    #03 ggml_backend_vk_graph_compute+41780

Three changes:

1. ggml_vk_create_pipeline_func: catch the exception, increment
   device->pipeline_failures, clean up the shader module, and return
   instead of rethrowing. Also handle null pipeline after creation.

2. ggml_vk_dispatch_pipeline: early-return if the pipeline is null or
   not compiled (safety net against dispatching broken pipelines).

3. ggml_backend_vk_device_supports_op: return false for all ops when
   pipeline_failures > 0, causing the backend scheduler to route
   everything to the CPU backend. The GPU is still used for buffer
   allocation but all compute runs on CPU.
This commit is contained in:
Simao Gomes Viana 2026-03-25 14:01:00 +01:00
parent a80d3d31a3
commit d28a49b837
No known key found for this signature in database
GPG Key ID: 2B861A352E0B987F
1 changed files with 29 additions and 3 deletions

View File

@ -625,6 +625,11 @@ struct vk_device_struct {
bool shader_64b_indexing;
// Number of compute pipelines that failed to compile.
// When > 0, supports_op returns false for all ops so the backend
// scheduler routes everything to the CPU backend.
std::atomic<uint32_t> pipeline_failures {};
bool integer_dot_product;
// 0: default, 1: force mmvq, -1: disable mmvq
int32_t mmvq_mode;
@ -2192,9 +2197,19 @@ static void ggml_vk_create_pipeline_func(vk_device& device, vk_pipeline& pipelin
try {
pipeline->pipeline = device->device.createComputePipeline(VK_NULL_HANDLE, compute_pipeline_create_info).value;
} catch (const vk::SystemError& e) {
std::cerr << "ggml_vulkan: Compute pipeline creation failed for " << pipeline->name << std::endl;
std::cerr << "ggml_vulkan: " << e.what() << std::endl;
throw e;
GGML_LOG_WARN("ggml_vulkan: compute pipeline creation failed for %s: %s\n",
pipeline->name.c_str(), e.what());
device->pipeline_failures.fetch_add(1, std::memory_order_relaxed);
device->device.destroyShaderModule(pipeline->shader_module);
pipeline->shader_module = VK_NULL_HANDLE;
return;
}
if (!pipeline->pipeline) {
GGML_LOG_WARN("ggml_vulkan: compute pipeline is null for %s\n", pipeline->name.c_str());
device->pipeline_failures.fetch_add(1, std::memory_order_relaxed);
device->device.destroyShaderModule(pipeline->shader_module);
pipeline->shader_module = VK_NULL_HANDLE;
return;
}
pipeline->compiled = true;
@ -6495,6 +6510,9 @@ template <typename T, uint32_t N> const T *push_constant_data(const std::array<T
template <typename T>
static void ggml_vk_dispatch_pipeline(ggml_backend_vk_context* ctx, vk_context& subctx, vk_pipeline& pipeline, std::initializer_list<vk::DescriptorBufferInfo> const& descriptor_buffer_infos, const T &push_constants, std::array<uint32_t, 3> elements) {
if (!pipeline || !pipeline->compiled) {
return;
}
const uint32_t wg0 = CEIL_DIV(elements[0], pipeline->wg_denoms[0]);
const uint32_t wg1 = CEIL_DIV(elements[1], pipeline->wg_denoms[1]);
const uint32_t wg2 = CEIL_DIV(elements[2], pipeline->wg_denoms[2]);
@ -15149,6 +15167,14 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
ggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context;
const vk_device& device = ggml_vk_get_device(ctx->device);
// If any compute pipelines failed to compile, the GPU driver is broken
// for these shaders. Return false for all ops so the backend scheduler
// routes everything to the CPU backend instead of dispatching to
// null pipelines.
if (device->pipeline_failures.load(std::memory_order_relaxed) > 0) {
return false;
}
const bool uses_bda = (op->op == GGML_OP_IM2COL || op->op == GGML_OP_IM2COL_3D) &&
device->shader_int64 && device->buffer_device_address;