diff --git a/src/whisper.cpp b/src/whisper.cpp
index 89146e2e4..6a4f5c1b0 100644
--- a/src/whisper.cpp
+++ b/src/whisper.cpp
@@ -2310,7 +2310,9 @@ static struct ggml_cgraph * whisper_build_graph_cross(
                 layer.cross_attn_k_w,
                 cur);
 
+#ifndef GGML_USE_OPENVINO // scaling is handled internally in OpenVINO backend
         Kcross = ggml_scale(ctx0, Kcross, Kscale);
+#endif
 
         struct ggml_tensor * Vcross = ggml_mul_mat(ctx0,
                 layer.cross_attn_v_w,