From d28a49b8371a9669d1bf21eff8d10461e7958a7a Mon Sep 17 00:00:00 2001 From: Simao Gomes Viana Date: Wed, 25 Mar 2026 14:01:00 +0100 Subject: [PATCH] ggml-vulkan: graceful fallback on pipeline compilation failure Some Vulkan drivers (observed on Adreno, Qualcomm build 923a446bf8) fail to compile compute shaders at runtime, reporting "Failed to link shaders" and returning ErrorUnknown from createComputePipeline. Previously this threw a C++ exception that propagated as an uncaught abort, or the resulting null pipeline was dispatched causing SIGSEGV: AdrenoVK-0: Failed to link shaders. Fatal signal 11 (SIGSEGV), code 1 (SEGV_MAPERR), fault addr 0xe8 #01 ggml_vk_dispatch_pipeline+360 #02 ggml_vk_mul_mat_q_f16+6616 #03 ggml_backend_vk_graph_compute+41780 Three changes: 1. ggml_vk_create_pipeline_func: catch the exception, increment device->pipeline_failures, clean up the shader module, and return instead of rethrowing. Also handle null pipeline after creation. 2. ggml_vk_dispatch_pipeline: early-return if the pipeline is null or not compiled (safety net against dispatching broken pipelines). 3. ggml_backend_vk_device_supports_op: return false for all ops when pipeline_failures > 0, causing the backend scheduler to route everything to the CPU backend. The GPU is still used for buffer allocation but all compute runs on CPU. --- ggml/src/ggml-vulkan/ggml-vulkan.cpp | 32 +++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 50efff144..8fa9d1da9 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -625,6 +625,11 @@ struct vk_device_struct { bool shader_64b_indexing; + // Number of compute pipelines that failed to compile. + // When > 0, supports_op returns false for all ops so the backend + // scheduler routes everything to the CPU backend. + std::atomic pipeline_failures {}; + bool integer_dot_product; // 0: default, 1: force mmvq, -1: disable mmvq int32_t mmvq_mode; @@ -2192,9 +2197,19 @@ static void ggml_vk_create_pipeline_func(vk_device& device, vk_pipeline& pipelin try { pipeline->pipeline = device->device.createComputePipeline(VK_NULL_HANDLE, compute_pipeline_create_info).value; } catch (const vk::SystemError& e) { - std::cerr << "ggml_vulkan: Compute pipeline creation failed for " << pipeline->name << std::endl; - std::cerr << "ggml_vulkan: " << e.what() << std::endl; - throw e; + GGML_LOG_WARN("ggml_vulkan: compute pipeline creation failed for %s: %s\n", + pipeline->name.c_str(), e.what()); + device->pipeline_failures.fetch_add(1, std::memory_order_relaxed); + device->device.destroyShaderModule(pipeline->shader_module); + pipeline->shader_module = VK_NULL_HANDLE; + return; + } + if (!pipeline->pipeline) { + GGML_LOG_WARN("ggml_vulkan: compute pipeline is null for %s\n", pipeline->name.c_str()); + device->pipeline_failures.fetch_add(1, std::memory_order_relaxed); + device->device.destroyShaderModule(pipeline->shader_module); + pipeline->shader_module = VK_NULL_HANDLE; + return; } pipeline->compiled = true; @@ -6495,6 +6510,9 @@ template const T *push_constant_data(const std::array static void ggml_vk_dispatch_pipeline(ggml_backend_vk_context* ctx, vk_context& subctx, vk_pipeline& pipeline, std::initializer_list const& descriptor_buffer_infos, const T &push_constants, std::array elements) { + if (!pipeline || !pipeline->compiled) { + return; + } const uint32_t wg0 = CEIL_DIV(elements[0], pipeline->wg_denoms[0]); const uint32_t wg1 = CEIL_DIV(elements[1], pipeline->wg_denoms[1]); const uint32_t wg2 = CEIL_DIV(elements[2], pipeline->wg_denoms[2]); @@ -15149,6 +15167,14 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm ggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context; const vk_device& device = ggml_vk_get_device(ctx->device); + // If any compute pipelines failed to compile, the GPU driver is broken + // for these shaders. Return false for all ops so the backend scheduler + // routes everything to the CPU backend instead of dispatching to + // null pipelines. + if (device->pipeline_failures.load(std::memory_order_relaxed) > 0) { + return false; + } + const bool uses_bda = (op->op == GGML_OP_IM2COL || op->op == GGML_OP_IM2COL_3D) && device->shader_int64 && device->buffer_device_address;