From 51ee92799785b07390b6d376d5d919b0aea334ab Mon Sep 17 00:00:00 2001 From: Neo Zhang Date: Sun, 13 Sep 2026 23:31:34 +0800 Subject: [PATCH] sycl : Fix get mem error (llama/28227) * fix for unsupport zes API * optimize the code * adjust the log level * rm unused head files * Update docs/backend/SYCL.md Co-authored-by: Titaniumtown * fix the error to detect level zero SDK/dev package, stop build after detect the error * update the message * fix the build error when missed to install level zero dev package * rm GGML_SYCL_DEV_DEBUG, mv read env vars in all entry functions --------- Co-authored-by: Neo Zhang Jianyu Co-authored-by: Titaniumtown Co-authored-by: Neo Zhang --- ggml/src/ggml-sycl/CMakeLists.txt | 14 +++-- ggml/src/ggml-sycl/base.hpp | 7 +++ ggml/src/ggml-sycl/ggml-sycl.cpp | 82 +++++++++++++++++++------- ggml/src/ggml-sycl/mem.cpp | 95 ++++++++++++++----------------- 4 files changed, 120 insertions(+), 78 deletions(-) diff --git a/ggml/src/ggml-sycl/CMakeLists.txt b/ggml/src/ggml-sycl/CMakeLists.txt index a8d9c0d80..d2196f74d 100644 --- a/ggml/src/ggml-sycl/CMakeLists.txt +++ b/ggml/src/ggml-sycl/CMakeLists.txt @@ -110,15 +110,21 @@ if (GGML_SYCL_SUPPORT_LEVEL_ZERO_API) # Link against Level Zero loader for direct device memory allocation. # Avoids sycl::malloc_device triggering DMA-buf/TTM system RAM staging # in the xe kernel driver during multi-GPU inference. - find_path(LEVEL_ZERO_INCLUDE_DIR level_zero/ze_api.h HINTS ${ONEAPI_ROOT}/include ${LEVEL_ZERO_V1_SDK_PATH}/include) + find_path(LEVEL_ZERO_DEV_INCLUDE_DIR level_zero/ze_api.h HINTS ${ONEAPI_ROOT}/include ${LEVEL_ZERO_V1_SDK_PATH}/include) find_library(ZE_LOADER_LIB ze_loader HINTS ${ONEAPI_ROOT}/lib ${LEVEL_ZERO_V1_SDK_LIB_PATH} ENV LD_LIBRARY_PATH) - if(ZE_LOADER_LIB AND LEVEL_ZERO_INCLUDE_DIR) + if(ZE_LOADER_LIB AND LEVEL_ZERO_DEV_INCLUDE_DIR) target_link_libraries(ggml-sycl PRIVATE ${ZE_LOADER_LIB}) target_compile_definitions(ggml-sycl PRIVATE GGML_SYCL_SUPPORT_LEVEL_ZERO_API) message(STATUS "Level Zero loader found: ${ZE_LOADER_LIB}") - message(STATUS "Level Zero headers found: ${LEVEL_ZERO_INCLUDE_DIR}") + message(STATUS "Level Zero development headers found: ${LEVEL_ZERO_DEV_INCLUDE_DIR}") else() - message(WARNING "Level Zero loader or headers not found, Level Zero support disabled") + message(WARNING "Level Zero loader or development headers not found, " + "Level Zero API support disabled. " + "Please install the Level Zero SDK/development package " + "to support Level Zero API features. " + "Level Zero API is not mandatory for SYCL backend, " + "but it is required by the special features for better " + "function & performance on Intel GPUs.") endif() endif() diff --git a/ggml/src/ggml-sycl/base.hpp b/ggml/src/ggml-sycl/base.hpp index 3afd57ccb..fe96c4ab8 100644 --- a/ggml/src/ggml-sycl/base.hpp +++ b/ggml/src/ggml-sycl/base.hpp @@ -17,6 +17,7 @@ #include extern int g_ggml_sycl_debug; +extern int g_ggml_sycl_dev_debug; #if defined(__clang__) && __has_builtin(__builtin_expect) // Hint the optimizer to pipeline the more likely following instruction in branches @@ -33,4 +34,10 @@ extern int g_ggml_sycl_debug; fprintf(stderr, __VA_ARGS__); \ } while (0) +#define GGML_SYCL_DEV_DEBUG(...) \ + do { \ + if (UNLIKELY(g_ggml_sycl_dev_debug)) \ + fprintf(stderr, __VA_ARGS__); \ + } while (0) + #endif // GGML_SYCL_BASE_HPP diff --git a/ggml/src/ggml-sycl/ggml-sycl.cpp b/ggml/src/ggml-sycl/ggml-sycl.cpp index f225682f2..1eb82ad5a 100644 --- a/ggml/src/ggml-sycl/ggml-sycl.cpp +++ b/ggml/src/ggml-sycl/ggml-sycl.cpp @@ -91,6 +91,7 @@ static bool g_sycl_loaded = false; int g_ggml_sycl_debug = 0; +int g_ggml_sycl_dev_debug = 0; int g_ggml_sycl_enable_optimize = 1; int g_ggml_sycl_enable_graph = 0; int g_ggml_sycl_enable_dnn = 1; @@ -113,8 +114,8 @@ int g_ggml_sycl_enable_host_pinned_mem = 1; int g_ggml_sycl_host_pinned_mem_2g = 0; int g_ggml_sycl_get_mem_api = MEMORY_API_TYPE_LEVEL_ZERO; - static ggml_sycl_device_info ggml_sycl_init() { + GGML_SYCL_DEBUG("[SYCL] call ggml_sycl_init\n"); ggml_sycl_device_info info = {}; // Do not hard crash when there exists no SYCL devices. @@ -205,12 +206,9 @@ static ggml_sycl_device_info ggml_sycl_init() { } #ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API - // Large buffers can be allocated before ggml_check_sycl() initializes other - // g_ggml_sycl_enable_* globals, so initialize this one as early as we can. + //update g_ggml_sycl_use_level_zero_api according to the device support g_ggml_sycl_use_level_zero_api = - info.ext_oneapi_level_zero && ggml_sycl_get_env("GGML_SYCL_USE_LEVEL_ZERO_API", 1); -#else - g_ggml_sycl_use_level_zero_api = 0; + info.ext_oneapi_level_zero && g_ggml_sycl_use_level_zero_api; #endif return info; @@ -314,23 +312,40 @@ static const char* dev2dev_int2str(int dev2dev) { * It's the first internal function to be called by them in SYCL backend. * This function is used to do initialize work for the SYCL backend and set the global variables. */ +#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API +static ze_result_t init_zes() { + ze_result_t res = zesInit(0); + if (res != ZE_RESULT_SUCCESS) { + GGML_SYCL_DEBUG("Warning: [%s] zesInit failed with code %d. Sysman free-memory query be unavailable.\n", + __func__, (int) res); + } + return res; +} + +ze_result_t get_zes_init_res() { + static ze_result_t zes_init_res = init_zes(); + GGML_SYCL_DEBUG("[SYCL] call %s: zesInit result: %d\n", __func__, (int) zes_init_res); + return zes_init_res; +} +#endif + void initialize_sycl_begining() { #ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API - ze_result_t zes_init = zesInit(0); - if (zes_init != ZE_RESULT_SUCCESS) { - std::cerr << "Warning: zesInit failed [ggml_check_sycl] with code " << static_cast(zes_init) - << ". Sysman free-memory query may be unavailable.\n"; - } + //must be called in initialization stage, before any other Level Zero API calls + GGML_SYCL_DEBUG("[SYCL] call %s\n", __func__); + get_zes_init_res(); #endif } static void ggml_check_sycl() try { + GGML_SYCL_DEBUG("[SYCL] ggml_check_sycl()\n"); static bool initialized = false; if (!initialized) { initialize_sycl_begining(); g_ggml_sycl_debug = ggml_sycl_get_env("GGML_SYCL_DEBUG", 0); + g_ggml_sycl_dev_debug = ggml_sycl_get_env("GGML_SYCL_DEV_DEBUG", 0); g_ggml_sycl_enable_optimize = ggml_sycl_get_env("GGML_SYCL_ENABLE_OPT", 1); g_ggml_sycl_enable_graph = ggml_sycl_get_env("GGML_SYCL_ENABLE_GRAPH", 0); g_ggml_sycl_enable_dnn = ggml_sycl_get_env("GGML_SYCL_ENABLE_DNN", 1); @@ -344,9 +359,13 @@ static void ggml_check_sycl() try { g_ggml_sycl_enable_esimd = ggml_sycl_get_env("GGML_SYCL_ENABLE_ESIMD", 1); g_ggml_sycl_prioritize_dmmv = ggml_sycl_get_env("GGML_SYCL_PRIORITIZE_DMMV", 0); +#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API + g_ggml_sycl_use_level_zero_api = ggml_sycl_get_env("GGML_SYCL_USE_LEVEL_ZERO_API", 1); +#else + g_ggml_sycl_use_level_zero_api = 0; +#endif g_ggml_sycl_dev2dev_memcpy = ggml_sycl_get_env("GGML_SYCL_DEV2DEV_MEMCPY", DEV2DEV_MEMCPY_SYCL); g_ggml_sycl_get_mem_api = ggml_sycl_get_env("GGML_SYCL_GET_MEM_API", MEMORY_API_TYPE_LEVEL_ZERO); - if (g_ggml_sycl_use_level_zero_api == 0) { g_ggml_sycl_dev2dev_memcpy = DEV2DEV_MEMCPY_SYCL; g_ggml_sycl_get_mem_api = MEMORY_API_TYPE_SYCL; @@ -405,6 +424,7 @@ static void ggml_check_sycl() try { GGML_LOG_INFO("Running with Environment Variables:\n"); GGML_LOG_INFO(" GGML_SYCL_DEBUG: %d\n", g_ggml_sycl_debug); + GGML_LOG_INFO(" GGML_SYCL_DEV_DEBUG: %d\n", g_ggml_sycl_dev_debug); #ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API GGML_LOG_INFO(" GGML_SYCL_DEV2DEV_MEMCPY: %d (%s)\n", g_ggml_sycl_dev2dev_memcpy, dev2dev_int2str(g_ggml_sycl_dev2dev_memcpy)); @@ -945,6 +965,7 @@ inline void * aligned_malloc_host(size_t alignment, size_t size) { static ggml_backend_buffer_t ggml_backend_sycl_buffer_type_alloc_buffer(ggml_backend_buffer_type_t buft, size_t size) try { + GGML_SYCL_DEBUG("[SYCL] call %s: size=%zu\n", __func__, size); ggml_check_sycl(); ggml_backend_sycl_buffer_type_context * buft_ctx = (ggml_backend_sycl_buffer_type_context *)buft->context; @@ -1464,10 +1485,11 @@ static ggml_backend_buffer_type_i ggml_backend_sycl_split_buffer_type_interface }; ggml_backend_buffer_type_t ggml_backend_sycl_split_buffer_type(const float * tensor_split) { + GGML_SYCL_DEBUG("[SYCL] call ggml_backend_sycl_split_buffer_type\n"); + static std::mutex mutex; std::lock_guard lock(mutex); - GGML_SYCL_DEBUG("[SYCL] call ggml_backend_sycl_split_buffer_type\n"); ggml_check_sycl(); // FIXME: this is not thread safe static std::map, struct ggml_backend_buffer_type> buft_map; @@ -1520,6 +1542,7 @@ static const char * ggml_backend_sycl_host_buffer_type_name(ggml_backend_buffer_ //host pinned memory static void * ggml_backend_sycl_host_malloc(size_t size) { + GGML_SYCL_DEBUG("[SYCL] call ggml_backend_sycl_host_malloc\n"); void * ptr = nullptr; try { ggml_check_sycl(); @@ -5341,8 +5364,8 @@ catch (sycl::exception const &exc) { } static bool ggml_sycl_compute_forward(ggml_backend_sycl_context & ctx, struct ggml_tensor * dst) try { + GGML_SYCL_DEBUG("[SYCL] ggml_sycl_compute_forward: dst=%s, op=%s\n", dst->name, ggml_op_name(dst->op)); if (!g_sycl_loaded) return false; - initialize_sycl_begining(); if (dst->src[0] != nullptr && ggml_backend_buffer_is_sycl_split(dst->src[0]->buffer)) { ggml_sycl_set_peer_access(dst->src[1]->ne[1], ctx.device); @@ -5725,11 +5748,27 @@ catch (sycl::exception const &exc) { std::exit(1); } +bool sycl_get_mem_info(int device, size_t * free, size_t * total) { + GGML_SYCL_DEBUG("[SYCL] [%s] g_ggml_sycl_get_mem_api=%d\n", + __func__, g_ggml_sycl_get_mem_api); + + MemoryAPIType mem_api_type = MemoryAPIType::MEMORY_API_TYPE_SYCL; + +#ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API + mem_api_type = get_zes_init_res() == ZE_RESULT_SUCCESS ? + (MemoryAPIType) g_ggml_sycl_get_mem_api : MemoryAPIType::MEMORY_API_TYPE_SYCL; +#else + mem_api_type = MemoryAPIType::MEMORY_API_TYPE_SYCL; +#endif + bool res = get_memory_size(dpct::dev_mgr::instance().get_device(device), + *free, *total, mem_api_type); + GGML_SYCL_DEBUG("[SYCL] [%s] total = %zu free = %zu\n", __func__, *total, *free); + return res; +} + void ggml_backend_sycl_get_device_memory(int device, size_t * free, size_t * total) try { GGML_SYCL_DEBUG("[SYCL] call ggml_backend_sycl_get_device_memory\n"); - bool res = get_memory_size(dpct::dev_mgr::instance().get_device(device), *free, *total, - (MemoryAPIType) g_ggml_sycl_get_mem_api); - if (!res) { + if (!sycl_get_mem_info(device, free, total)) { GGML_ABORT("[%s] failed to get device memory size", __func__); } ggml_sycl_memtrace_report_device("device memory query", device, *free, *total); @@ -6177,12 +6216,12 @@ static const char * ggml_backend_sycl_device_get_description(ggml_backend_dev_t } static void ggml_backend_sycl_device_get_memory(ggml_backend_dev_t dev, size_t * free, size_t * total) { + GGML_SYCL_DEBUG("[SYCL] call %s\n", __func__); ggml_backend_sycl_device_context * ctx = (ggml_backend_sycl_device_context *) dev->context; - bool res = get_memory_size(dpct::dev_mgr::instance().get_device(ctx->device), *free, *total, - (MemoryAPIType) g_ggml_sycl_get_mem_api); - if (!res) { + if (!sycl_get_mem_info(ctx->device, free, total)) { GGML_ABORT("[%s] failed to get device memory size", __func__); } + GGML_SYCL_DEBUG("[SYCL] call %s total %zu free %zu\n", __func__, *total, *free); ggml_sycl_memtrace_report_device("device memory query (dev)", ctx->device, *free, *total); } @@ -7061,6 +7100,7 @@ static const ggml_backend_reg_i ggml_backend_sycl_reg_interface = { // backend registry ggml_backend_reg_t ggml_backend_sycl_reg() { + GGML_SYCL_DEBUG("[SYCL] call ggml_backend_sycl_reg\n"); static ggml_backend_reg reg; static bool initialized = false; @@ -7068,7 +7108,7 @@ ggml_backend_reg_t ggml_backend_sycl_reg() { static std::mutex mutex; std::lock_guard lock(mutex); if (!initialized) { - initialize_sycl_begining(); + ggml_check_sycl(); ggml_backend_sycl_reg_context * ctx = new ggml_backend_sycl_reg_context; const int min_batch_size = getenv("GGML_OP_OFFLOAD_MIN_BATCH") ? atoi(getenv("GGML_OP_OFFLOAD_MIN_BATCH")) : 32; diff --git a/ggml/src/ggml-sycl/mem.cpp b/ggml/src/ggml-sycl/mem.cpp index 5ec466420..ad5bfe0ff 100644 --- a/ggml/src/ggml-sycl/mem.cpp +++ b/ggml/src/ggml-sycl/mem.cpp @@ -6,13 +6,13 @@ #include #endif -#include "base.hpp" -#include "mem.hpp" - #include #include #include +#include "base.hpp" +#include "mem.hpp" + const char * mem_api_int2str(int mem_api) { if (mem_api == MEMORY_API_TYPE_SYCL) { return "SYCL API"; @@ -24,7 +24,12 @@ const char * mem_api_int2str(int mem_api) { } #ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API +/* +* Depend on to call zesInit(0) before any other Level Zero API calls, otherwise the Level Zero API calls may fail. +*/ bool query_free_memory_by_ze(sycl::device dev, size_t & free_bytes, size_t & total_bytes) { + GGML_SYCL_DEBUG("[SYCL] call %s: Querying free memory using Level Zero API.\n", __func__); + free_bytes = 0; total_bytes = 0; @@ -37,41 +42,28 @@ bool query_free_memory_by_ze(sycl::device dev, size_t & free_bytes, size_t & tot #endif try { - ze_result_t zes_init = zesInit(0); - if (zes_init != ZE_RESULT_SUCCESS) { - std::cerr << "Warning: zesInit failed with code " << static_cast(zes_init) - << ". Sysman free-memory query may be unavailable.\n"; - } if (dev.get_platform().get_backend() != kL0Backend) { - GGML_SYCL_DEBUG("Device backend is not Level Zero; falling back to SYCL memory query.\n"); - total_bytes = dev.get_info(); - free_bytes = total_bytes; + GGML_SYCL_DEBUG("Device backend is not Level Zero.\n"); return false; } ze_device_handle_t ze_dev = sycl::get_native(dev); if (ze_dev == nullptr) { - GGML_SYCL_DEBUG("Level Zero device handle is null; falling back to SYCL memory query.\n"); - total_bytes = dev.get_info(); - free_bytes = total_bytes; + GGML_SYCL_DEBUG("Level Zero device handle is null.\n"); return false; } ze_result_t r = zesDeviceEnumMemoryModules(ze_dev, &module_count, nullptr); if (r != ZE_RESULT_SUCCESS || module_count == 0) { - GGML_SYCL_DEBUG("Failed to enumerate Level Zero memory modules. Falling back to SYCL memory query.\n"); - total_bytes = dev.get_info(); - free_bytes = total_bytes; + GGML_SYCL_DEBUG("Failed to enumerate Level Zero memory modules.\n"); return false; } std::vector modules(module_count); r = zesDeviceEnumMemoryModules(ze_dev, &module_count, modules.data()); if (r != ZE_RESULT_SUCCESS || module_count == 0) { - GGML_SYCL_DEBUG("Failed to enumerate Level Zero memory modules. Falling back to SYCL memory query.\n"); - total_bytes = dev.get_info(); - free_bytes = total_bytes; + GGML_SYCL_DEBUG("Failed to enumerate Level Zero memory modules.\n"); return false; } @@ -90,73 +82,70 @@ bool query_free_memory_by_ze(sycl::device dev, size_t & free_bytes, size_t & tot } if (total_bytes == 0) { - GGML_SYCL_DEBUG("Level Zero memory query returned zero total bytes. Falling back to SYCL memory query.\n"); - total_bytes = dev.get_info(); - free_bytes = total_bytes; + GGML_SYCL_DEBUG("Level Zero memory query returned zero total bytes.\n"); return false; } - return true; + return total_bytes >= free_bytes; + } catch (const sycl::exception & e) { GGML_SYCL_DEBUG("Level Zero memory query failed: %s\n", e.what()); - total_bytes = dev.get_info(); - free_bytes = total_bytes; return false; } } #endif bool get_memory_size_by_sycl_api(sycl::device dev, size_t & free_bytes, size_t & total_bytes) { - GGML_SYCL_DEBUG("[%s]Querying free memory using SYCL API.\n", __func__); + GGML_SYCL_DEBUG("[SYCL] call %s: Querying free memory using SYCL API.\n", __func__); total_bytes = dev.get_info(); #if (defined(__SYCL_COMPILER_VERSION) && __SYCL_COMPILER_VERSION >= 20221105) if (dev.has(sycl::aspect::ext_intel_free_memory)) { try { - GGML_SYCL_DEBUG("Querying free memory using SYCL aspect::ext_intel_free_memory."); + GGML_SYCL_DEBUG("Querying free memory using SYCL aspect::ext_intel_free_memory.\n"); free_bytes = dev.get_info(); return true; } catch (const sycl::exception &) { GGML_SYCL_DEBUG( - "Failed to query free memory using SYCL aspect::ext_intel_free_memory. Using total memory as free " - "memory."); - free_bytes = total_bytes; + "Failed to query free memory using SYCL aspect::ext_intel_free_memory.\n"); return false; } } else { GGML_SYCL_DEBUG( - "Device does not support SYCL aspect::ext_intel_free_memory. Using total memory as free memory."); - free_bytes = total_bytes; + "Device does not support SYCL aspect::ext_intel_free_memory.\n"); } #else - GGML_SYCL_DEBUG("SYCL Compiler version is older than 20221105. Using total memory as free memory."); - free_bytes = total_bytes; + GGML_SYCL_DEBUG("SYCL Compiler version is older than 20221105.\n"); #endif - return true; + return false; } bool get_memory_size(sycl::device dev, size_t & free_bytes, size_t & total_bytes, MemoryAPIType api_type) { - const auto name = dev.get_info(); - const auto vendor = dev.get_info(); - const auto global_mem = dev.get_info(); - GGML_SYCL_DEBUG("[%s]GPU Name: %s\n", __func__, name.c_str()); - GGML_SYCL_DEBUG("[%s]GPU Vendor: %s\n", __func__, vendor.c_str()); - GGML_SYCL_DEBUG("[%s]GPU Global Memory: %zu bytes\n", __func__, static_cast(global_mem)); + GGML_SYCL_DEBUG("[%s]GPU Name: %s\n", __func__, + dev.get_info().c_str()); + GGML_SYCL_DEBUG("[%s]GPU Vendor: %s\n", __func__, + dev.get_info().c_str()); if (api_type == MEMORY_API_TYPE_LEVEL_ZERO) { #ifdef GGML_SYCL_SUPPORT_LEVEL_ZERO_API - GGML_SYCL_DEBUG("[%s]Querying free memory using Level Zero API.\n", __func__); - if (!query_free_memory_by_ze(dev, free_bytes, total_bytes)) { - //fallback to SYCL API if Level Zero API fails - GGML_SYCL_DEBUG("[%s]Falling back to SYCL API for memory query.\n", __func__); - return get_memory_size_by_sycl_api(dev, free_bytes, total_bytes); + GGML_SYCL_DEBUG("[%s] Querying free memory using Level Zero API.\n", __func__); + if (query_free_memory_by_ze(dev, free_bytes, total_bytes)) { + return true; } - return true; -#else - GGML_SYCL_DEBUG("[%s]Level Zero API support is not enabled. Please enable it to use this feature.\n", __func__); - return false; + //fallback to SYCL API if Level Zero API fails + GGML_SYCL_DEBUG("[%s] Falling back to SYCL API for memory query.\n", __func__); #endif - } else { //MEMORY_API_TYPE_SYCL - return get_memory_size_by_sycl_api(dev, free_bytes, total_bytes); } + + //MEMORY_API_TYPE_SYCL + if(get_memory_size_by_sycl_api(dev, free_bytes, total_bytes)){ + return true; + } + + //Todo, fallback to other methods to get free memory size, such as using OS-specific APIs (e.g., /proc/meminfo on Linux, GlobalMemoryStatusEx on Windows, etc.) + GGML_SYCL_DEBUG( + "[%s] Can't get free mem size by Level Zero and SYCL API. Using total memory as free memory.\n", __func__); + free_bytes = total_bytes; + + return true; }