ggml-vulkan: graceful fallback on pipeline compilation failure
Some Vulkan drivers (observed on Adreno, Qualcomm build 923a446bf8)
fail to compile compute shaders at runtime, reporting
"Failed to link shaders" and returning ErrorUnknown from
createComputePipeline. Previously this threw a C++ exception that
propagated as an uncaught abort, or the resulting null pipeline was
dispatched causing SIGSEGV:
AdrenoVK-0: Failed to link shaders.
Fatal signal 11 (SIGSEGV), code 1 (SEGV_MAPERR), fault addr 0xe8
#01 ggml_vk_dispatch_pipeline<vk_mat_mat_push_constants>+360
#02 ggml_vk_mul_mat_q_f16+6616
#03 ggml_backend_vk_graph_compute+41780
Three changes:
1. ggml_vk_create_pipeline_func: catch the exception, increment
device->pipeline_failures, clean up the shader module, and return
instead of rethrowing. Also handle null pipeline after creation.
2. ggml_vk_dispatch_pipeline: early-return if the pipeline is null or
not compiled (safety net against dispatching broken pipelines).
3. ggml_backend_vk_device_supports_op: return false for all ops when
pipeline_failures > 0, causing the backend scheduler to route
everything to the CPU backend. The GPU is still used for buffer
allocation but all compute runs on CPU.
This commit is contained in:
parent
a80d3d31a3
commit
d28a49b837
|
|
@ -625,6 +625,11 @@ struct vk_device_struct {
|
|||
|
||||
bool shader_64b_indexing;
|
||||
|
||||
// Number of compute pipelines that failed to compile.
|
||||
// When > 0, supports_op returns false for all ops so the backend
|
||||
// scheduler routes everything to the CPU backend.
|
||||
std::atomic<uint32_t> pipeline_failures {};
|
||||
|
||||
bool integer_dot_product;
|
||||
// 0: default, 1: force mmvq, -1: disable mmvq
|
||||
int32_t mmvq_mode;
|
||||
|
|
@ -2192,9 +2197,19 @@ static void ggml_vk_create_pipeline_func(vk_device& device, vk_pipeline& pipelin
|
|||
try {
|
||||
pipeline->pipeline = device->device.createComputePipeline(VK_NULL_HANDLE, compute_pipeline_create_info).value;
|
||||
} catch (const vk::SystemError& e) {
|
||||
std::cerr << "ggml_vulkan: Compute pipeline creation failed for " << pipeline->name << std::endl;
|
||||
std::cerr << "ggml_vulkan: " << e.what() << std::endl;
|
||||
throw e;
|
||||
GGML_LOG_WARN("ggml_vulkan: compute pipeline creation failed for %s: %s\n",
|
||||
pipeline->name.c_str(), e.what());
|
||||
device->pipeline_failures.fetch_add(1, std::memory_order_relaxed);
|
||||
device->device.destroyShaderModule(pipeline->shader_module);
|
||||
pipeline->shader_module = VK_NULL_HANDLE;
|
||||
return;
|
||||
}
|
||||
if (!pipeline->pipeline) {
|
||||
GGML_LOG_WARN("ggml_vulkan: compute pipeline is null for %s\n", pipeline->name.c_str());
|
||||
device->pipeline_failures.fetch_add(1, std::memory_order_relaxed);
|
||||
device->device.destroyShaderModule(pipeline->shader_module);
|
||||
pipeline->shader_module = VK_NULL_HANDLE;
|
||||
return;
|
||||
}
|
||||
pipeline->compiled = true;
|
||||
|
||||
|
|
@ -6495,6 +6510,9 @@ template <typename T, uint32_t N> const T *push_constant_data(const std::array<T
|
|||
|
||||
template <typename T>
|
||||
static void ggml_vk_dispatch_pipeline(ggml_backend_vk_context* ctx, vk_context& subctx, vk_pipeline& pipeline, std::initializer_list<vk::DescriptorBufferInfo> const& descriptor_buffer_infos, const T &push_constants, std::array<uint32_t, 3> elements) {
|
||||
if (!pipeline || !pipeline->compiled) {
|
||||
return;
|
||||
}
|
||||
const uint32_t wg0 = CEIL_DIV(elements[0], pipeline->wg_denoms[0]);
|
||||
const uint32_t wg1 = CEIL_DIV(elements[1], pipeline->wg_denoms[1]);
|
||||
const uint32_t wg2 = CEIL_DIV(elements[2], pipeline->wg_denoms[2]);
|
||||
|
|
@ -15149,6 +15167,14 @@ static bool ggml_backend_vk_device_supports_op(ggml_backend_dev_t dev, const ggm
|
|||
ggml_backend_vk_device_context * ctx = (ggml_backend_vk_device_context *)dev->context;
|
||||
const vk_device& device = ggml_vk_get_device(ctx->device);
|
||||
|
||||
// If any compute pipelines failed to compile, the GPU driver is broken
|
||||
// for these shaders. Return false for all ops so the backend scheduler
|
||||
// routes everything to the CPU backend instead of dispatching to
|
||||
// null pipelines.
|
||||
if (device->pipeline_failures.load(std::memory_order_relaxed) > 0) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const bool uses_bda = (op->op == GGML_OP_IM2COL || op->op == GGML_OP_IM2COL_3D) &&
|
||||
device->shader_int64 && device->buffer_device_address;
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue