diff --git a/ggml/cmake/common.cmake b/ggml/cmake/common.cmake index 25eff7a5e..f3610298f 100644 --- a/ggml/cmake/common.cmake +++ b/ggml/cmake/common.cmake @@ -28,6 +28,7 @@ endfunction() function(ggml_get_system_arch) if (CMAKE_OSX_ARCHITECTURES STREQUAL "arm64" OR CMAKE_GENERATOR_PLATFORM_LWR STREQUAL "arm64" OR + (CMAKE_GENERATOR_PLATFORM_LWR STREQUAL "arm64ec" AND MSVC AND NOT CMAKE_C_COMPILER_ID STREQUAL "Clang") OR (NOT CMAKE_OSX_ARCHITECTURES AND NOT CMAKE_GENERATOR_PLATFORM_LWR AND CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm.*|ARM64)$")) set(GGML_SYSTEM_ARCH "ARM" PARENT_SCOPE) diff --git a/ggml/src/ggml-cpu/CMakeLists.txt b/ggml/src/ggml-cpu/CMakeLists.txt index 2765c2727..71f55b490 100644 --- a/ggml/src/ggml-cpu/CMakeLists.txt +++ b/ggml/src/ggml-cpu/CMakeLists.txt @@ -106,54 +106,79 @@ function(ggml_add_cpu_backend_variant_impl tag_name) ggml-cpu/arch/arm/repack.cpp ) + # MSVC proper (not clang-cl) has no -mcpu/-march for ARM: features can only be + # selected by probing the host at configure time and forwarding -D__ARM_FEATURE_* if (MSVC AND NOT CMAKE_C_COMPILER_ID STREQUAL "Clang") - message(FATAL_ERROR "MSVC is not supported for ARM, use clang") + set(GGML_ARM_MSVC ON) else() - check_cxx_compiler_flag(-mfp16-format=ieee GGML_COMPILER_SUPPORTS_FP16_FORMAT_I3E) - if (NOT "${GGML_COMPILER_SUPPORTS_FP16_FORMAT_I3E}" STREQUAL "") - list(APPEND ARCH_FLAGS -mfp16-format=ieee) + set(GGML_ARM_MSVC OFF) + endif() + + if (GGML_ARM_MSVC AND NOT GGML_NATIVE) + message(FATAL_ERROR "MSVC on ARM requires GGML_NATIVE=ON, use clang for GGML_CPU_ARM_ARCH / GGML_CPU_ALL_VARIANTS") + else() + if (NOT GGML_ARM_MSVC) + check_cxx_compiler_flag(-mfp16-format=ieee GGML_COMPILER_SUPPORTS_FP16_FORMAT_I3E) + if (NOT "${GGML_COMPILER_SUPPORTS_FP16_FORMAT_I3E}" STREQUAL "") + list(APPEND ARCH_FLAGS -mfp16-format=ieee) + endif() endif() if (GGML_NATIVE) # -mcpu=native does not always enable all the features in some compilers, # so we check for them manually and enable them if available - execute_process( - COMMAND ${CMAKE_C_COMPILER} -mcpu=native -E -v - - INPUT_FILE "/dev/null" - OUTPUT_QUIET - ERROR_VARIABLE ARM_MCPU - RESULT_VARIABLE ARM_MCPU_RESULT - ) - if (NOT ARM_MCPU_RESULT) - string(REGEX MATCH "-mcpu=[^ ']+" ARM_MCPU_FLAG "${ARM_MCPU}") - string(REGEX MATCH "-march=[^ ']+" ARM_MARCH_FLAG "${ARM_MCPU}") + # cl.exe cannot be queried this way: no /dev/null, and it rejects the flags. + # ARM_NATIVE_FLAG stays empty for MSVC and is never appended below. + if (NOT GGML_ARM_MSVC) + execute_process( + COMMAND ${CMAKE_C_COMPILER} -mcpu=native -E -v - + INPUT_FILE "/dev/null" + OUTPUT_QUIET + ERROR_VARIABLE ARM_MCPU + RESULT_VARIABLE ARM_MCPU_RESULT + ) + if (NOT ARM_MCPU_RESULT) + string(REGEX MATCH "-mcpu=[^ ']+" ARM_MCPU_FLAG "${ARM_MCPU}") + string(REGEX MATCH "-march=[^ ']+" ARM_MARCH_FLAG "${ARM_MCPU}") - # on some old GCC we need to read -march= - if (ARM_MARCH_FLAG AND NOT "${ARM_MARCH_FLAG}" STREQUAL "-march=native") - set(ARM_NATIVE_FLAG "${ARM_MARCH_FLAG}") - elseif(ARM_MCPU_FLAG AND NOT "${ARM_MCPU_FLAG}" STREQUAL "-mcpu=native") - set(ARM_NATIVE_FLAG "${ARM_MCPU_FLAG}") + # on some old GCC we need to read -march= + if (ARM_MARCH_FLAG AND NOT "${ARM_MARCH_FLAG}" STREQUAL "-march=native") + set(ARM_NATIVE_FLAG "${ARM_MARCH_FLAG}") + elseif(ARM_MCPU_FLAG AND NOT "${ARM_MCPU_FLAG}" STREQUAL "-mcpu=native") + set(ARM_NATIVE_FLAG "${ARM_MCPU_FLAG}") + endif() endif() - endif() - if ("${ARM_NATIVE_FLAG}" STREQUAL "") - set(ARM_NATIVE_FLAG -mcpu=native) - message(WARNING "ARM -march/-mcpu not found, -mcpu=native will be used") - else() - message(STATUS "ARM detected flags: ${ARM_NATIVE_FLAG}") + if ("${ARM_NATIVE_FLAG}" STREQUAL "") + set(ARM_NATIVE_FLAG -mcpu=native) + message(WARNING "ARM -march/-mcpu not found, -mcpu=native will be used") + else() + message(STATUS "ARM detected flags: ${ARM_NATIVE_FLAG}") + endif() endif() include(CheckCXXSourceRuns) macro(check_arm_feature tag feature code) set(CMAKE_REQUIRED_FLAGS_SAVE ${CMAKE_REQUIRED_FLAGS}) - set(CMAKE_REQUIRED_FLAGS "${ARM_NATIVE_FLAG}+${tag}") + if (GGML_ARM_MSVC) + set(ARM_PROBE_FLAG "") + set(ARM_PROBE_NOFLAG "") + else() + set(ARM_PROBE_FLAG "${ARM_NATIVE_FLAG}+${tag}") + set(ARM_PROBE_NOFLAG "${ARM_NATIVE_FLAG}+no${tag}") + endif() + set(CMAKE_REQUIRED_FLAGS "${ARM_PROBE_FLAG}") check_cxx_source_runs("${code}" GGML_MACHINE_SUPPORTS_${tag}) if (GGML_MACHINE_SUPPORTS_${tag}) set(ARM_NATIVE_FLAG_FIX "${ARM_NATIVE_FLAG_FIX}+${tag}") + if (GGML_ARM_MSVC) + # forward runtime probing decisions for MSVC + list(APPEND ARCH_FLAGS "-D__ARM_FEATURE_${feature}=1") + endif() else() - set(CMAKE_REQUIRED_FLAGS "${ARM_NATIVE_FLAG}+no${tag}") + set(CMAKE_REQUIRED_FLAGS "${ARM_PROBE_NOFLAG}") check_cxx_source_compiles("int main() { return 0; }" GGML_MACHINE_SUPPORTS_no${tag}) if (GGML_MACHINE_SUPPORTS_no${tag}) set(ARM_NATIVE_FLAG_FIX "${ARM_NATIVE_FLAG_FIX}+no${tag}") @@ -163,12 +188,24 @@ function(ggml_add_cpu_backend_variant_impl tag_name) set(CMAKE_REQUIRED_FLAGS ${CMAKE_REQUIRED_FLAGS_SAVE}) endmacro() - check_arm_feature(dotprod DOTPROD "#include \nint main() { int8x16_t _a, _b; volatile int32x4_t _s = vdotq_s32(_s, _a, _b); return 0; }") - check_arm_feature(i8mm MATMUL_INT8 "#include \nint main() { int8x16_t _a, _b; volatile int32x4_t _s = vmmlaq_s32(_s, _a, _b); return 0; }") - check_arm_feature(sve SVE "#include \nint main() { svfloat32_t _a, _b; volatile svfloat32_t _c = svadd_f32_z(svptrue_b8(), _a, _b); return 0; }") + if (GGML_ARM_MSVC) + # drop volatile + check_arm_feature(dotprod DOTPROD "#include \nint main() { int8x16_t _a = vdupq_n_s8(0), _b = vdupq_n_s8(0); int32x4_t _s = vdupq_n_s32(0); _s = vdotq_s32(_s, _a, _b); return 0; }") + check_arm_feature(i8mm MATMUL_INT8 "#include \nint main() { int8x16_t _a = vdupq_n_s8(0), _b = vdupq_n_s8(0); int32x4_t _s = vdupq_n_s32(0); _s = vmmlaq_s32(_s, _a, _b); return 0; }") + check_arm_feature(sve SVE "#include \nint main() { const svbool_t pg = svptrue_b8(); const svint8_t a = svdup_n_s8(0), b = svdup_n_s8(0); svint32_t s = svdup_n_s32(0); s = svdot_s32(s, a, b); (void) pg; (void) s; return 0; }") + # FMA is mandatory on ARMv8-A but MSVC never defines __ARM_FEATURE_FMA, + # so probe it like the others and forward the define + check_arm_feature(fma FMA "#include \nint main() { float32x4_t _a = vdupq_n_f32(0), _b = vdupq_n_f32(0), _c = vdupq_n_f32(0); _a = vfmaq_f32(_a, _b, _c); return 0; }") + else() + check_arm_feature(dotprod DOTPROD "#include \nint main() { int8x16_t _a, _b; volatile int32x4_t _s = vdotq_s32(_s, _a, _b); return 0; }") + check_arm_feature(i8mm MATMUL_INT8 "#include \nint main() { int8x16_t _a, _b; volatile int32x4_t _s = vmmlaq_s32(_s, _a, _b); return 0; }") + check_arm_feature(sve SVE "#include \nint main() { svfloat32_t _a, _b; volatile svfloat32_t _c = svadd_f32_z(svptrue_b8(), _a, _b); return 0; }") + endif() check_arm_feature(sme SME "#include \n__arm_locally_streaming int main() { __asm__ volatile(\"smstart; smstop;\"); return 0; }") - list(APPEND ARCH_FLAGS "${ARM_NATIVE_FLAG}${ARM_NATIVE_FLAG_FIX}") + if (NOT GGML_ARM_MSVC) + list(APPEND ARCH_FLAGS "${ARM_NATIVE_FLAG}${ARM_NATIVE_FLAG_FIX}") + endif() else() if (GGML_CPU_ARM_ARCH) list(APPEND ARCH_FLAGS -march=${GGML_CPU_ARM_ARCH}) diff --git a/ggml/src/ggml-cpu/arch-fallback.h b/ggml/src/ggml-cpu/arch-fallback.h index 4dbd1982b..e4882984a 100644 --- a/ggml/src/ggml-cpu/arch-fallback.h +++ b/ggml/src/ggml-cpu/arch-fallback.h @@ -75,7 +75,7 @@ #define ggml_gemm_mxfp4_8x8_q8_0_generic ggml_gemm_mxfp4_8x8_q8_0 #define ggml_gemm_q8_0_4x4_q8_0_generic ggml_gemm_q8_0_4x4_q8_0 #define ggml_gemm_q8_0_4x8_q8_0_generic ggml_gemm_q8_0_4x8_q8_0 -#elif defined(__aarch64__) || defined(__arm__) || defined(_M_ARM) || defined(_M_ARM64) +#elif defined(__aarch64__) || defined(__arm__) || defined(_M_ARM) || defined(_M_ARM64) || defined(_M_ARM64EC) // repack.cpp #define ggml_quantize_mat_q8_K_4x4_generic ggml_quantize_mat_q8_K_4x4 #define ggml_quantize_mat_q8_K_4x8_generic ggml_quantize_mat_q8_K_4x8 diff --git a/ggml/src/ggml-cpu/arch/arm/quants.c b/ggml/src/ggml-cpu/arch/arm/quants.c index b988abf99..cfbbf6617 100644 --- a/ggml/src/ggml-cpu/arch/arm/quants.c +++ b/ggml/src/ggml-cpu/arch/arm/quants.c @@ -875,23 +875,23 @@ void ggml_vec_dot_nvfp4_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const vo const int8x8_t q8_3_lo = vld1_s8(y[2*ib+1].qs + 16); const int8x8_t q8_3_hi = vld1_s8(y[2*ib+1].qs + 24); - const int32x4_t sumi = (int32x4_t){ + const int32x4_t sumi = vld1q_s32(((const int32_t[4]) { vaddvq_s32(ggml_nvfp4_dot8(q4_0_lo, q8_0_lo, q4_0_hi, q8_0_hi)), vaddvq_s32(ggml_nvfp4_dot8(q4_1_lo, q8_1_lo, q4_1_hi, q8_1_hi)), vaddvq_s32(ggml_nvfp4_dot8(q4_2_lo, q8_2_lo, q4_2_hi, q8_2_hi)), vaddvq_s32(ggml_nvfp4_dot8(q4_3_lo, q8_3_lo, q4_3_hi, q8_3_hi)), - }; + })); #endif const float dy0 = GGML_CPU_FP16_TO_FP32(y[2*ib].d); const float dy1 = GGML_CPU_FP16_TO_FP32(y[2*ib+1].d); - const float32x4_t nvsc = { + const float32x4_t nvsc = vld1q_f32(((const float[4]) { GGML_CPU_UE4M3_TO_FP32(x[ib].d[0]), GGML_CPU_UE4M3_TO_FP32(x[ib].d[1]), GGML_CPU_UE4M3_TO_FP32(x[ib].d[2]), GGML_CPU_UE4M3_TO_FP32(x[ib].d[3]) - }; - const float32x4_t scales = vmulq_f32(nvsc, (float32x4_t){dy0, dy0, dy1, dy1}); + })); + const float32x4_t scales = vmulq_f32(nvsc, vld1q_f32(((const float[4]) {dy0, dy0, dy1, dy1}))); acc = vfmaq_f32(acc, vcvtq_f32_s32(sumi), scales); } @@ -2591,7 +2591,7 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi memcpy(scales_mins, x0->scales, 12); const uint32_t mins_0_3 = scales_mins[1] & kmask1; const uint32_t mins_4_7 = ((scales_mins[2] >> 4) & kmask2) | (((scales_mins[1] >> 6) & kmask3) << 4); - const uint32x2_t mins = {mins_0_3, mins_4_7}; + const uint32x2_t mins = vcreate_u32((uint64_t) mins_0_3 | ((uint64_t) mins_4_7 << 32)); x0_mins = vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_u32(mins))); uint32_t scales[2]; scales[0] = scales_mins[0] & kmask1; // scales 0~3 @@ -2603,7 +2603,7 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi memcpy(scales_mins, x1->scales, 12); const uint32_t mins_0_3 = scales_mins[1] & kmask1; const uint32_t mins_4_7 = ((scales_mins[2] >> 4) & kmask2) | (((scales_mins[1] >> 6) & kmask3) << 4); - const uint32x2_t mins = {mins_0_3, mins_4_7}; + const uint32x2_t mins = vcreate_u32((uint64_t) mins_0_3 | ((uint64_t) mins_4_7 << 32)); x1_mins = vreinterpretq_s16_u16(vmovl_u8(vreinterpret_u8_u32(mins))); uint32_t scales[2]; scales[0] = scales_mins[0] & kmask1; // scales 0~3 @@ -2611,7 +2611,7 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi memcpy(x1_scales, scales, 8); } - int32x4_t visum = {0}; + int32x4_t visum = vdupq_n_s32(0); // process 64 data points per iteration, totally 256 data points for (int j = 0; j < QK_K / 64; ++j, qx0 += 32, qx1 += 32, qy0 += 64, qy1 += 64) { @@ -2637,14 +2637,14 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi // process 32 data points (share same block scale) per iteration for (int k = 0; k < 2; ++k) { const int blk = j * 2 + k; - const int32x4_t block_scale = { + const int32x4_t block_scale = vld1q_s32(((const int32_t[4]) { x0_scales[blk], x0_scales[blk], x1_scales[blk], x1_scales[blk], - }; + })); - int32x4_t vr = {0}; + int32x4_t vr = vdupq_n_s32(0); for (int l = 0; l < 2; ++l) { const int idx = k * 2 + l; const int64x2_t vx0_s64 = vreinterpretq_s64_s8(vx0[idx]); @@ -2678,20 +2678,22 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi vmull_s16(vget_high_s16(y0_sums), vget_high_s16(x1_mins)))); bias[3] = vaddvq_s32(vaddq_s32(vmull_s16(vget_low_s16(y1_sums), vget_low_s16(x1_mins)), vmull_s16(vget_high_s16(y1_sums), vget_high_s16(x1_mins)))); - const float32x4_t dmins = { + // note: the parentheses around the compound literal are required, vld1q_f32() is a + // function-like macro on MSVC and would otherwise see four separate arguments + const float32x4_t dmins = vld1q_f32(((const float[4]) { GGML_CPU_FP16_TO_FP32(x0->dmin) * y0->d, GGML_CPU_FP16_TO_FP32(x0->dmin) * y1->d, GGML_CPU_FP16_TO_FP32(x1->dmin) * y0->d, GGML_CPU_FP16_TO_FP32(x1->dmin) * y1->d, - }; + })); vfsum = vmlsq_f32(vfsum, vcvtq_f32_s32(vld1q_s32(bias)), dmins); - const float32x4_t superblock_scale = { + const float32x4_t superblock_scale = vld1q_f32(((const float[4]) { GGML_CPU_FP16_TO_FP32(x0->d) * y0->d, GGML_CPU_FP16_TO_FP32(x0->d) * y1->d, GGML_CPU_FP16_TO_FP32(x1->d) * y0->d, GGML_CPU_FP16_TO_FP32(x1->d) * y1->d, - }; + })); vfsum = vmlaq_f32(vfsum, vcvtq_f32_s32(visum), superblock_scale); } } @@ -3261,12 +3263,8 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi qy0 += 16; qy1 += 16; - const int32x4_t block_scale = { - x0->scales[blk], - x0->scales[blk], - x1->scales[blk], - x1->scales[blk], - }; + const int32x4_t block_scale = + vcombine_s32(vdup_n_s32(x0->scales[blk]), vdup_n_s32(x1->scales[blk])); // calculate four results at once with outer product const int8x16_t vx_l = vreinterpretq_s8_s64(vzip1q_s64(vreinterpretq_s64_s8(vx0[k]), vreinterpretq_s64_s8(vx1[k]))); @@ -3319,12 +3317,14 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi const int32x4_t vibias = vmulq_n_s32(vld1q_s32(bias), 32); - const float32x4_t superblock_scale = { + // note: the parentheses around the compound literal are required, vld1q_f32() is a + // function-like macro on MSVC and would otherwise see four separate arguments + const float32x4_t superblock_scale = vld1q_f32(((const float[4]) { GGML_CPU_FP16_TO_FP32(x0->d) * y0->d, GGML_CPU_FP16_TO_FP32(x0->d) * y1->d, GGML_CPU_FP16_TO_FP32(x1->d) * y0->d, GGML_CPU_FP16_TO_FP32(x1->d) * y1->d, - }; + })); visum = vsubq_s32(visum, vibias); vfsum = vmlaq_f32(vfsum, vcvtq_f32_s32(visum), superblock_scale); diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c index 046f17ca0..24c47569c 100644 --- a/ggml/src/ggml-cpu/ggml-cpu.c +++ b/ggml/src/ggml-cpu/ggml-cpu.c @@ -459,7 +459,7 @@ typedef pthread_mutex_t ggml_mutex_t; #define ggml_lock_init(x) UNUSED(x) #define ggml_lock_destroy(x) UNUSED(x) -#if defined(__x86_64__) || (defined(_MSC_VER) && defined(_M_AMD64)) +#if defined(__x86_64__) || (defined(_MSC_VER) && defined(_M_AMD64) && !defined(_M_ARM64EC)) #define ggml_lock_lock(x) _mm_pause() #else #define ggml_lock_lock(x) UNUSED(x) diff --git a/ggml/src/ggml-cpu/llamafile/sgemm.cpp b/ggml/src/ggml-cpu/llamafile/sgemm.cpp index 99b7d5afa..258bc9e2e 100644 --- a/ggml/src/ggml-cpu/llamafile/sgemm.cpp +++ b/ggml/src/ggml-cpu/llamafile/sgemm.cpp @@ -228,7 +228,7 @@ template <> inline vfloat32m8_t madd(vbfloat16m4_t a, vbfloat16m4_t b, vfloat32m //////////////////////////////////////////////////////////////////////////////////////////////////// // VECTORIZED HORIZONTAL SUM -#if defined(__ARM_NEON) +#if defined(__ARM_NEON) || defined(_M_ARM64) || defined(_M_ARM64EC) inline float hsum(float32x4_t x) { return vaddvq_f32(x); } diff --git a/ggml/src/ggml-cuda/CMakeLists.txt b/ggml/src/ggml-cuda/CMakeLists.txt index 2254090cb..dd57ac423 100644 --- a/ggml/src/ggml-cuda/CMakeLists.txt +++ b/ggml/src/ggml-cuda/CMakeLists.txt @@ -1,5 +1,18 @@ cmake_minimum_required(VERSION 3.18) # for CMAKE_CUDA_ARCHITECTURES +# ARM64EC uses the x64 ABI, so it needs the x64 import libraries. The arm64 ones +# hold native ARM64 symbols and cannot satisfy EC references at link time. +# FindCUDAToolkit picks the library dir from the host arch, not the target, so +# anchor its search at lib/x64 here. It derives everything else from CUDA_CUDART. +string(TOLOWER "${CMAKE_GENERATOR_PLATFORM}" GGML_CUDA_PLATFORM_LWR) +if (MSVC AND GGML_CUDA_PLATFORM_LWR STREQUAL "arm64ec") + find_library(CUDA_CUDART + NAMES cudart + HINTS ${CUDAToolkit_ROOT} ENV CUDA_PATH + PATH_SUFFIXES lib/x64 + ) +endif() + find_package(CUDAToolkit) if (CUDAToolkit_FOUND)