diff --git a/ggml/src/ggml-vulkan/ggml-vulkan.cpp b/ggml/src/ggml-vulkan/ggml-vulkan.cpp index 50efff144..8fa9d1da9 100644 --- a/ggml/src/ggml-vulkan/ggml-vulkan.cpp +++ b/ggml/src/ggml-vulkan/ggml-vulkan.cpp @@ -625,6 +625,11 @@ struct vk_device_struct { bool shader_64b_indexing; + // Number of compute pipelines that failed to compile. + // When > 0, supports_op returns false for all ops so the backend + // scheduler routes everything to the CPU backend. + std::atomic pipeline_failures {}; + bool integer_dot_product; // 0: default, 1: force mmvq, -1: disable mmvq int32_t mmvq_mode; @@ -2192,9 +2197,19 @@ static void ggml_vk_create_pipeline_func(vk_device& device, vk_pipeline& pipelin try { pipeline->pipeline = device->device.createComputePipeline(VK_NULL_HANDLE, compute_pipeline_create_info).value; } catch (const vk::SystemError& e) { - std::cerr << "ggml_vulkan: Compute pipeline creation failed for " << pipeline->name << std::endl; - std::cerr << "ggml_vulkan: " << e.what() << std::endl; - throw e; + GGML_LOG_WARN("ggml_vulkan: compute pipeline creation failed for %s: %s\n", + pipeline->name.c_str(), e.what()); + device->pipeline_failures.fetch_add(1, std::memory_order_relaxed); + device->device.destroyShaderModule(pipeline->shader_module); + pipeline->shader_module = VK_NULL_HANDLE; + return; + } + if (!pipeline->pipeline) { + GGML_LOG_WARN("ggml_vulkan: compute pipeline is null for %s\n", pipeline->name.c_str()); + device->pipeline_failures.fetch_add(1, std::memory_order_relaxed); + device->device.destroyShaderModule(pipeline->shader_module); + pipeline->shader_module = VK_NULL_HANDLE; + return; } pipeline->compiled = true; @@ -6495,6 +6510,9 @@ template const T *push_constant_data(const std::array static void ggml_vk_dispatch_pipeline(ggml_backend_vk_context* ctx, vk_context& subctx, vk_pipeline& pipeline, std::initializer_list const& descriptor_buffer_infos, const T &push_constants, std::array elements) { + if (!pipeline || !pipeline->compiled) { + return; + } const uint32_t wg0 = CEIL_DIV(elements[0], pipeline->wg_denoms[0]); const uint32_t wg1 = CEIL_DIV(elements[1], pipeline->wg_denoms[1]); const uint32_t wg2 = CEIL_DIV(elements[2], pipeline->wg_denoms[2]); @@ -15149,6 +15167,14 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm ggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context; const vk_device& device = ggml_vk_get_device(ctx->device); + // If any compute pipelines failed to compile, the GPU driver is broken + // for these shaders. Return false for all ops so the backend scheduler + // routes everything to the CPU backend instead of dispatching to + // null pipelines. + if (device->pipeline_failures.load(std::memory_order_relaxed) > 0) { + return false; + } + const bool uses_bda = (op->op == GGML_OP_IM2COL || op->op == GGML_OP_IM2COL_3D) && device->shader_int64 && device->buffer_device_address;