diff --git a/.github/workflows/build-self-hosted.yml b/.github/workflows/build-self-hosted.yml index 2286b63d6..b689df084 100644 --- a/.github/workflows/build-self-hosted.yml +++ b/.github/workflows/build-self-hosted.yml @@ -114,3 +114,146 @@ jobs: run: | vulkaninfo --summary GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/whisper.cpp ~/mnt/whisper.cpp + + npu-amd-windows: + runs-on: [self-hosted, Windows, X64, stx, rai300-400] + timeout-minutes: 60 + continue-on-error: true # advisory while the runner pool is new; revisit later + + env: + FLEXML_URL: https://github.com/lemonade-sdk/whisper.cpp-rocm/releases/download/deps/flexmlrt-1.7.0-win.zip + MODEL: base + + steps: + - name: Clone + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - uses: microsoft/setup-msbuild@v2 + + - name: Install CMake if not available + shell: powershell + run: | + $installed = Get-Command cmake -ErrorAction SilentlyContinue + if (-not $installed) { + $ver = "3.28.1" + $url = "https://github.com/Kitware/CMake/releases/download/v$ver/cmake-$ver-windows-x86_64.msi" + Invoke-WebRequest -Uri $url -OutFile cmake.msi + Start-Process msiexec.exe -ArgumentList "/i cmake.msi /quiet /norestart" -Wait + $p = "C:\Program Files\CMake\bin" + $env:PATH = "$p;$env:PATH" + echo $p >> $env:GITHUB_PATH + cmake --version + if ($LASTEXITCODE -ne 0) { Write-Error "CMake install failed"; exit 1 } + } else { cmake --version } + + - name: Download FlexML runtime + shell: powershell + run: | + Invoke-WebRequest -Uri "${{ env.FLEXML_URL }}" -OutFile flexmlrt.zip + if (-Not (Test-Path "flexmlrt.zip")) { Write-Error "flexmlrt.zip not downloaded"; exit 1 } + if ((Get-Item "flexmlrt.zip").Length -eq 0) { Write-Error "flexmlrt.zip is empty"; exit 1 } + tar xf flexmlrt.zip + if ($LASTEXITCODE -ne 0) { Write-Error "Extraction failed"; exit 1 } + if (-not (Test-Path "flexmlrt")) { Write-Error "No flexmlrt directory after extraction"; exit 1 } + + - name: Setup FlexML, configure and build + shell: cmd + run: | + cd flexmlrt + call setup.bat + if errorlevel 1 ( echo ERROR: FlexML setup.bat failed & exit /b 1 ) + cd .. + cmake -B build -A x64 -DCMAKE_BUILD_TYPE=Release -DWHISPER_VITISAI=ON + if errorlevel 1 ( echo ERROR: CMake configure failed & exit /b 1 ) + cmake --build build --config Release -j + if errorlevel 1 ( echo ERROR: Build failed & exit /b 1 ) + + - name: Copy FlexML DLLs to build output + shell: powershell + run: | + foreach ($d in "flexmlrt/bin", "flexmlrt/lib") { + if (Test-Path "$d/*.dll") { Copy-Item "$d/*.dll" "build/bin/Release/" -Force } + } + if (-not (Test-Path "build/bin/Release/flexmlrt.dll")) { + Write-Error "flexmlrt.dll not staged next to binaries"; exit 1 + } + + - name: Download ggml model + shell: cmd + run: | + call models\download-ggml-model.cmd %MODEL% models + if not exist models\ggml-%MODEL%.bin ( echo ERROR: model download failed & exit /b 1 ) + + - name: Download NPU encoder cache + shell: cmd + run: | + .\models\download-vitisai-model.cmd %MODEL% + if not exist models\ggml-%MODEL%-encoder-vitisai.rai ( echo ERROR: VitisAI encoder cache download failed & exit /b 1 ) + + - name: Run NPU smoke test + shell: cmd + run: | + build\bin\Release\whisper-cli.exe -m models\ggml-%MODEL%.bin -f samples\jfk.wav > vitisai.log 2>&1 + type vitisai.log + findstr /I /C:"vitisai" vitisai.log || ( echo ERROR: no VitisAI activity - encoder likely fell back to CPU & exit /b 1 ) + findstr /I /C:"ask not what your country" vitisai.log || ( echo ERROR: incorrect transcription & exit /b 1 ) + + - name: Upload smoke test log + if: always() + uses: actions/upload-artifact@v4 + with: + name: vitisai-smoke-log-windows + path: vitisai.log + + npu-amd-linux: + runs-on: [self-hosted, Linux, X64, stx, rai300-400] + timeout-minutes: 60 + continue-on-error: true # advisory while the runner pool is new; revisit later + + env: + FLEXML_LINUX_URL: https://github.com/lemonade-sdk/whisper.cpp-rocm/releases/download/deps/flexmlrt-1.8.0-linux.tar.gz + MODEL: base + + steps: + - name: Clone + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - name: Verify NPU device + run: | + lsmod | grep -q amdxdna || { echo "ERROR: amdxdna driver not loaded"; exit 1; } + ls /dev/accel/accel* || { echo "ERROR: no NPU accel device node"; exit 1; } + + - name: Download FlexML runtime (Linux) + run: | + curl -L --fail -o flexmlrt.tar.gz "$FLEXML_LINUX_URL" + tar xf flexmlrt.tar.gz + source flexmlrt/setup.sh + echo "FlexmlRT_DIR=$PWD/flexmlrt/share/cmake/FlexmlRT" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=$PWD/flexmlrt/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + + - name: Configure and build + run: | + cmake -B build -DCMAKE_BUILD_TYPE=Release -DWHISPER_VITISAI=ON + cmake --build build --config Release -j $(nproc) + + - name: Download ggml model + run: | + ./models/download-ggml-model.sh $MODEL + + - name: Download NPU encoder cache + run: | + sh ./models/download-vitisai-model.sh $MODEL + [ -f "models/ggml-$MODEL-encoder-vitisai.rai" ] || { echo "ERROR: VitisAI encoder cache download failed"; exit 1; } + + - name: Run NPU smoke test + run: | + ./build/bin/whisper-cli -m "models/ggml-$MODEL.bin" -f samples/jfk.wav 2>&1 | tee vitisai.log + grep -qi "vitisai" vitisai.log || { echo "ERROR: no VitisAI activity - CPU fallback?"; exit 1; } + grep -qi "ask not what your country" vitisai.log || { echo "ERROR: incorrect transcription"; exit 1; } + + - name: Upload smoke test log + if: always() + uses: actions/upload-artifact@v4 + with: + name: vitisai-smoke-log-linux + path: vitisai.log diff --git a/README.md b/README.md index fd23e1c55..9de9ed5ac 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ High-performance inference of [OpenAI's Whisper](https://github.com/openai/whisp - Support for CPU-only inference - [Efficient GPU support for NVIDIA](#nvidia-gpu-support) - [AMD ROCm GPU support](#amd-rocm-gpu-support) -- [AMD Ryzen AI NPU Support](#amd-ryzen-ai-support-for-npu) +- [AMD Ryzen AI NPU Support](#amd-ryzen-ai-npu-support) - [OpenVINO Support](#openvino-support) - [Ascend NPU Support](#ascend-npu-support) - [Moore Threads GPU Support](#moore-threads-gpu-support) @@ -314,47 +314,80 @@ This can result in significant speedup in encoder performance. Here are the inst For more information about the OpenVINO implementation please refer to PR [#1037](https://github.com/ggml-org/whisper.cpp/pull/1037). -## AMD Ryzen™ AI support for NPU +## AMD Ryzen™ AI NPU support -On AMD's Ryzen™ AI 300 Series with dedicated NPUs for acceleration, you can now run Whisper models with the ability to fully offload the encoder to NPU. This brings significant speedup compared to CPU-only. -> **Note:** -> **Ryzen™ AI NPU acceleration is currently supported on Windows only.** Linux support is planned for upcoming releases. -> For the latest updates on Ryzen AI, check out [the official documentation](https://ryzenai.docs.amd.com/en/latest/). +On AMD Ryzen™ AI 300 and 400 Series processors with a dedicated NPU, whisper.cpp can fully offload the Whisper encoder to the NPU via VitisAI, delivering significant speedup over CPU-only inference. -### Setup environment (Windows only) +### Prerequisites - - Obtain the XRT package and the FlexmlRT package from AMD. Both are distributed as tarballs or wheels. - - Copy the downloaded archives to a local path, extract them, and run the setup script from each extracted package in your shell (for example `source /path/to/xrt/setup.sh` and `source /path/to/flexmlrt/setup.sh`). Run these in every new shell you use to build or run `whisper.cpp`. +Install the XRT runtime and FlexML runtime for your platform: -- Fetch the matching ggml model and prebuilt VitisAI encoder cache: +- **XRT**: provides the NPU kernel driver and `xrt-smi` diagnostic tool — on Windows this is bundled with the NPU driver; on Linux install it separately following the [NPU driver installation guide](https://ryzenai.docs.amd.com/en/latest/linux.html#install-npu-drivers) +- **FlexML runtime** (`flexmlrt`): VitisAI inference engine used by whisper.cpp — download from the [FlexML runtime releases](https://github.com/lemonade-sdk/whisper.cpp-rocm/releases/tag/deps) - ```bash - sh ./models/download-ggml-model.sh base - sh ./models/download-vitisai-model.sh base - ``` +After installing, source the setup scripts in every shell you use to build or run whisper.cpp: - ```cmd - .\models\download-ggml-model.cmd base - .\models\download-vitisai-model.cmd base - ``` +```bash +# Linux +source /opt/xilinx/xrt/setup.sh +source /path/to/flexmlrt/setup.sh +``` - Use the same model name with both scripts. The VitisAI script queries the AMD collection on Hugging Face to list available caches, then downloads the selected `.rai` file as `ggml--encoder-vitisai.rai` alongside the matching `ggml-.bin` file. You can also browse the collection manually at https://huggingface.co/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models. +```cmd +:: Windows +cd /path/to/flexmlrt && call setup.bat +``` - Depending on the downloaded `.rai` cache, VitisAI may offload either the encoder only or the encoder plus cross-projection layers to the AMD NPU. `whisper.cpp` detects the cache contents at runtime and logs the selected offload mode during model initialization. +You can verify the NPU is visible with: -- Build `whisper.cpp` with VitisAI support: +```bash +xrt-smi examine +``` - ```bash - cmake -B build -DWHISPER_VITISAI=1 - cmake --build build -j --config Release - ``` -Your environment is now ready. +### Download models -### Build Whisper.cpp for Ryzen™ AI support +Download the ggml model and the matching prebuilt VitisAI encoder cache: - ```text - $ ./build/bin/whisper-cli -m models/ggml-base.bin -f samples/jfk.wav - ``` +```bash +# Linux / macOS +sh ./models/download-ggml-model.sh base +sh ./models/download-vitisai-model.sh base +``` + +```cmd +:: Windows +.\models\download-ggml-model.cmd base +.\models\download-vitisai-model.cmd base +``` + +Use the same model name with both scripts. To see all available VitisAI encoder caches: + +```bash +sh ./models/download-vitisai-model.sh --list +``` + +```cmd +.\models\download-vitisai-model.cmd --list +``` + +The VitisAI script queries the [AMD Ryzen AI Whisper NPU collection on Hugging Face](https://huggingface.co/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models) and downloads the `.rai` encoder cache as `models/ggml--encoder-vitisai.rai`. + +> Depending on the `.rai` cache, VitisAI may offload the encoder only, or the encoder plus cross-projection layers. whisper.cpp detects this at runtime and logs the selected offload mode during model initialization. + +### Build + +```bash +cmake -B build -DWHISPER_VITISAI=1 +cmake --build build -j --config Release +``` + +### Run + +```bash +./build/bin/whisper-cli -m models/ggml-base.bin -f samples/jfk.wav +``` + +For more information see the [Ryzen AI documentation](https://ryzenai.docs.amd.com/en/latest/). ## NVIDIA GPU support