diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 7ace04e12..aaaa8fe58 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -640,6 +640,8 @@ jobs:
           -DCMAKE_BUILD_TYPE=${{ matrix.build }}
           -DBUILD_SHARED_LIBS=ON
           -DWHISPER_SDL2=${{ matrix.sdl2 }}
+          -DGGML_NATIVE=OFF
+          -DGGML_BMI2=OFF
 
       - name: Build
         run: |