ggml: vectorize ggml_vec_dot_q4_1_q8_1 with WASM SIMD128 (llama/22209)
* ggml: vectorize ggml_vec_dot_q4_1_q8_1 with WASM SIMD128 Optimize the inner loop of ggml_vec_dot_q4_1_q8_1_generic using WASM SIMD128 intrinsics, gated behind #ifdef __wasm_simd128__ so non-wasm builds are completely unaffected. Approach: - single wasm_v128_load covers all 32 packed 4-bit weights - nibbles unpacked via AND/SHR into two u8x16 registers - widened to i16 before multiply (WASM SIMD has no i8*i8 instruction) - 4x wasm_i32x4_dot_i16x8 calls accumulate all 32 element pairs - horizontal reduce via 4x wasm_i32x4_extract_lane Benchmark (node v25, emcc -O3 -msimd128, 64 blocks x QK8_1=32, 200k iterations): | impl | ns/call | speedup | |--------|---------|---------| | scalar | 880.7 | 1.00x | | simd | 257.8 | 3.42x | Correctness verified against scalar reference across 10 random seeds with exact output match. * ggml: move q4_1_q8_1 WASM SIMD implementation to wasm backend Relocate the SIMD128 implementation of ggml_vec_dot_q4_1_q8_1 to ggml/src/ggml-cpu/arch/wasm/quants.c to follow architecture-specific layout. Restore the generic implementation in ggml/src/ggml-cpu/quants.c. Move for loop in the else block. * ggml: use generic q4_1_q8_1 fallback in wasm backend
This commit is contained in:
parent
9d6e561f69
commit
991b5a8b4a
|
|
@ -355,6 +355,78 @@ void ggml_vec_dot_q4_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const voi
|
|||
*s = sumf;
|
||||
}
|
||||
|
||||
void ggml_vec_dot_q4_1_q8_1(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
|
||||
const int qk = QK8_1;
|
||||
const int nb = n / qk;
|
||||
|
||||
assert(n % qk == 0);
|
||||
assert(nrc == 1);
|
||||
UNUSED(nrc);
|
||||
UNUSED(bx);
|
||||
UNUSED(by);
|
||||
UNUSED(bs);
|
||||
|
||||
const block_q4_1 * GGML_RESTRICT x = vx;
|
||||
const block_q8_1 * GGML_RESTRICT y = vy;
|
||||
|
||||
float sumf = 0;
|
||||
|
||||
#if defined __wasm_simd128__
|
||||
v128_t sumv = wasm_f32x4_splat(0.0f);
|
||||
float summs = 0.0f;
|
||||
|
||||
for (int ib = 0; ib < nb; ++ib) {
|
||||
const block_q4_1 * GGML_RESTRICT x0 = &x[ib];
|
||||
const block_q8_1 * GGML_RESTRICT y0 = &y[ib];
|
||||
|
||||
summs += GGML_CPU_FP16_TO_FP32(x0->m) * GGML_CPU_FP16_TO_FP32(y0->s);
|
||||
|
||||
const v128_t raw = wasm_v128_load(x0->qs);
|
||||
const v128_t v0s = wasm_v128_and(raw, wasm_i8x16_splat(0x0F));
|
||||
const v128_t v1s = wasm_u8x16_shr(raw, 4);
|
||||
|
||||
const v128_t ys_lo = wasm_v128_load(y0->qs);
|
||||
const v128_t ys_hi = wasm_v128_load(y0->qs + 16);
|
||||
|
||||
const v128_t v0s_l = wasm_u16x8_extend_low_u8x16(v0s);
|
||||
const v128_t v0s_h = wasm_u16x8_extend_high_u8x16(v0s);
|
||||
const v128_t ylo_l = wasm_i16x8_extend_low_i8x16(ys_lo);
|
||||
const v128_t ylo_h = wasm_i16x8_extend_high_i8x16(ys_lo);
|
||||
const v128_t v1s_l = wasm_u16x8_extend_low_u8x16(v1s);
|
||||
const v128_t v1s_h = wasm_u16x8_extend_high_u8x16(v1s);
|
||||
const v128_t yhi_l = wasm_i16x8_extend_low_i8x16(ys_hi);
|
||||
const v128_t yhi_h = wasm_i16x8_extend_high_i8x16(ys_hi);
|
||||
|
||||
const v128_t acc = wasm_i32x4_add(
|
||||
wasm_i32x4_add(
|
||||
wasm_i32x4_dot_i16x8(v0s_l, ylo_l),
|
||||
wasm_i32x4_dot_i16x8(v0s_h, ylo_h)),
|
||||
wasm_i32x4_add(
|
||||
wasm_i32x4_dot_i16x8(v1s_l, yhi_l),
|
||||
wasm_i32x4_dot_i16x8(v1s_h, yhi_h)));
|
||||
|
||||
sumv = wasm_f32x4_add(sumv,
|
||||
wasm_f32x4_mul(
|
||||
wasm_f32x4_convert_i32x4(acc),
|
||||
wasm_f32x4_splat(GGML_CPU_FP16_TO_FP32(x0->d) * GGML_CPU_FP16_TO_FP32(y0->d))));
|
||||
}
|
||||
|
||||
sumf = wasm_f32x4_extract_lane(sumv, 0) + wasm_f32x4_extract_lane(sumv, 1) +
|
||||
wasm_f32x4_extract_lane(sumv, 2) + wasm_f32x4_extract_lane(sumv, 3) + summs;
|
||||
|
||||
*s = sumf;
|
||||
|
||||
#else
|
||||
UNUSED(nb);
|
||||
UNUSED(x);
|
||||
UNUSED(y);
|
||||
UNUSED(sumf);
|
||||
|
||||
ggml_vec_dot_q4_1_q8_1_generic(
|
||||
n, s, bs, vx, bx, vy, by, nrc);
|
||||
#endif
|
||||
}
|
||||
|
||||
void ggml_vec_dot_q5_0_q8_0(int n, float * GGML_RESTRICT s, size_t bs, const void * GGML_RESTRICT vx, size_t bx, const void * GGML_RESTRICT vy, size_t by, int nrc) {
|
||||
const int qk = QK8_0;
|
||||
const int nb = n / qk;
|
||||
|
|
|
|||
Loading…
Reference in New Issue