Merge pull request #3 from iswaryaalex/pr3608-sync
Add NPU runners (linux + windows)
This commit is contained in:
Sachin Kumawat
GitHub
commit
be6e914da6
|
|
@ -114,3 +114,146 @@ jobs:
|
|||
run: |
|
||||
vulkaninfo --summary
|
||||
GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/whisper.cpp ~/mnt/whisper.cpp
|
||||
|
||||
npu-amd-windows:
|
||||
runs-on: [self-hosted, Windows, X64, stx, rai300-400]
|
||||
timeout-minutes: 60
|
||||
continue-on-error: true # advisory while the runner pool is new; revisit later
|
||||
|
||||
env:
|
||||
FLEXML_URL: https://github.com/lemonade-sdk/whisper.cpp-rocm/releases/download/deps/flexmlrt-1.7.0-win.zip
|
||||
MODEL: base
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
|
||||
|
||||
- uses: microsoft/setup-msbuild@v2
|
||||
|
||||
- name: Install CMake if not available
|
||||
shell: powershell
|
||||
run: |
|
||||
$installed = Get-Command cmake -ErrorAction SilentlyContinue
|
||||
if (-not $installed) {
|
||||
$ver = "3.28.1"
|
||||
$url = "https://github.com/Kitware/CMake/releases/download/v$ver/cmake-$ver-windows-x86_64.msi"
|
||||
Invoke-WebRequest -Uri $url -OutFile cmake.msi
|
||||
Start-Process msiexec.exe -ArgumentList "/i cmake.msi /quiet /norestart" -Wait
|
||||
$p = "C:\Program Files\CMake\bin"
|
||||
$env:PATH = "$p;$env:PATH"
|
||||
echo $p >> $env:GITHUB_PATH
|
||||
cmake --version
|
||||
if ($LASTEXITCODE -ne 0) { Write-Error "CMake install failed"; exit 1 }
|
||||
} else { cmake --version }
|
||||
|
||||
- name: Download FlexML runtime
|
||||
shell: powershell
|
||||
run: |
|
||||
Invoke-WebRequest -Uri "${{ env.FLEXML_URL }}" -OutFile flexmlrt.zip
|
||||
if (-Not (Test-Path "flexmlrt.zip")) { Write-Error "flexmlrt.zip not downloaded"; exit 1 }
|
||||
if ((Get-Item "flexmlrt.zip").Length -eq 0) { Write-Error "flexmlrt.zip is empty"; exit 1 }
|
||||
tar xf flexmlrt.zip
|
||||
if ($LASTEXITCODE -ne 0) { Write-Error "Extraction failed"; exit 1 }
|
||||
if (-not (Test-Path "flexmlrt")) { Write-Error "No flexmlrt directory after extraction"; exit 1 }
|
||||
|
||||
- name: Setup FlexML, configure and build
|
||||
shell: cmd
|
||||
run: |
|
||||
cd flexmlrt
|
||||
call setup.bat
|
||||
if errorlevel 1 ( echo ERROR: FlexML setup.bat failed & exit /b 1 )
|
||||
cd ..
|
||||
cmake -B build -A x64 -DCMAKE_BUILD_TYPE=Release -DWHISPER_VITISAI=ON
|
||||
if errorlevel 1 ( echo ERROR: CMake configure failed & exit /b 1 )
|
||||
cmake --build build --config Release -j
|
||||
if errorlevel 1 ( echo ERROR: Build failed & exit /b 1 )
|
||||
|
||||
- name: Copy FlexML DLLs to build output
|
||||
shell: powershell
|
||||
run: |
|
||||
foreach ($d in "flexmlrt/bin", "flexmlrt/lib") {
|
||||
if (Test-Path "$d/*.dll") { Copy-Item "$d/*.dll" "build/bin/Release/" -Force }
|
||||
}
|
||||
if (-not (Test-Path "build/bin/Release/flexmlrt.dll")) {
|
||||
Write-Error "flexmlrt.dll not staged next to binaries"; exit 1
|
||||
}
|
||||
|
||||
- name: Download ggml model
|
||||
shell: cmd
|
||||
run: |
|
||||
call models\download-ggml-model.cmd %MODEL% models
|
||||
if not exist models\ggml-%MODEL%.bin ( echo ERROR: model download failed & exit /b 1 )
|
||||
|
||||
- name: Download NPU encoder cache
|
||||
shell: cmd
|
||||
run: |
|
||||
.\models\download-vitisai-model.cmd %MODEL%
|
||||
if not exist models\ggml-%MODEL%-encoder-vitisai.rai ( echo ERROR: VitisAI encoder cache download failed & exit /b 1 )
|
||||
|
||||
- name: Run NPU smoke test
|
||||
shell: cmd
|
||||
run: |
|
||||
build\bin\Release\whisper-cli.exe -m models\ggml-%MODEL%.bin -f samples\jfk.wav > vitisai.log 2>&1
|
||||
type vitisai.log
|
||||
findstr /I /C:"vitisai" vitisai.log || ( echo ERROR: no VitisAI activity - encoder likely fell back to CPU & exit /b 1 )
|
||||
findstr /I /C:"ask not what your country" vitisai.log || ( echo ERROR: incorrect transcription & exit /b 1 )
|
||||
|
||||
- name: Upload smoke test log
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: vitisai-smoke-log-windows
|
||||
path: vitisai.log
|
||||
|
||||
npu-amd-linux:
|
||||
runs-on: [self-hosted, Linux, X64, stx, rai300-400]
|
||||
timeout-minutes: 60
|
||||
continue-on-error: true # advisory while the runner pool is new; revisit later
|
||||
|
||||
env:
|
||||
FLEXML_LINUX_URL: https://github.com/lemonade-sdk/whisper.cpp-rocm/releases/download/deps/flexmlrt-1.8.0-linux.tar.gz
|
||||
MODEL: base
|
||||
|
||||
steps:
|
||||
- name: Clone
|
||||
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
|
||||
|
||||
- name: Verify NPU device
|
||||
run: |
|
||||
lsmod | grep -q amdxdna || { echo "ERROR: amdxdna driver not loaded"; exit 1; }
|
||||
ls /dev/accel/accel* || { echo "ERROR: no NPU accel device node"; exit 1; }
|
||||
|
||||
- name: Download FlexML runtime (Linux)
|
||||
run: |
|
||||
curl -L --fail -o flexmlrt.tar.gz "$FLEXML_LINUX_URL"
|
||||
tar xf flexmlrt.tar.gz
|
||||
source flexmlrt/setup.sh
|
||||
echo "FlexmlRT_DIR=$PWD/flexmlrt/share/cmake/FlexmlRT" >> $GITHUB_ENV
|
||||
echo "LD_LIBRARY_PATH=$PWD/flexmlrt/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV
|
||||
|
||||
- name: Configure and build
|
||||
run: |
|
||||
cmake -B build -DCMAKE_BUILD_TYPE=Release -DWHISPER_VITISAI=ON
|
||||
cmake --build build --config Release -j $(nproc)
|
||||
|
||||
- name: Download ggml model
|
||||
run: |
|
||||
./models/download-ggml-model.sh $MODEL
|
||||
|
||||
- name: Download NPU encoder cache
|
||||
run: |
|
||||
sh ./models/download-vitisai-model.sh $MODEL
|
||||
[ -f "models/ggml-$MODEL-encoder-vitisai.rai" ] || { echo "ERROR: VitisAI encoder cache download failed"; exit 1; }
|
||||
|
||||
- name: Run NPU smoke test
|
||||
run: |
|
||||
./build/bin/whisper-cli -m "models/ggml-$MODEL.bin" -f samples/jfk.wav 2>&1 | tee vitisai.log
|
||||
grep -qi "vitisai" vitisai.log || { echo "ERROR: no VitisAI activity - CPU fallback?"; exit 1; }
|
||||
grep -qi "ask not what your country" vitisai.log || { echo "ERROR: incorrect transcription"; exit 1; }
|
||||
|
||||
- name: Upload smoke test log
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: vitisai-smoke-log-linux
|
||||
path: vitisai.log
|
||||
|
|
|
|||
93
README.md
93
README.md
|
|
@ -22,7 +22,7 @@ High-performance inference of [OpenAI's Whisper](https://github.com/openai/whisp
|
|||
- Support for CPU-only inference
|
||||
- [Efficient GPU support for NVIDIA](#nvidia-gpu-support)
|
||||
- [AMD ROCm GPU support](#amd-rocm-gpu-support)
|
||||
- [AMD Ryzen AI NPU Support](#amd-ryzen-ai-support-for-npu)
|
||||
- [AMD Ryzen AI NPU Support](#amd-ryzen-ai-npu-support)
|
||||
- [OpenVINO Support](#openvino-support)
|
||||
- [Ascend NPU Support](#ascend-npu-support)
|
||||
- [Moore Threads GPU Support](#moore-threads-gpu-support)
|
||||
|
|
@ -314,47 +314,80 @@ This can result in significant speedup in encoder performance. Here are the inst
|
|||
|
||||
For more information about the OpenVINO implementation please refer to PR [#1037](https://github.com/ggml-org/whisper.cpp/pull/1037).
|
||||
|
||||
## AMD Ryzen™ AI support for NPU
|
||||
## AMD Ryzen™ AI NPU support
|
||||
|
||||
On AMD's Ryzen™ AI 300 Series with dedicated NPUs for acceleration, you can now run Whisper models with the ability to fully offload the encoder to NPU. This brings significant speedup compared to CPU-only.
|
||||
> **Note:**
|
||||
> **Ryzen™ AI NPU acceleration is currently supported on Windows only.** Linux support is planned for upcoming releases.
|
||||
> For the latest updates on Ryzen AI, check out [the official documentation](https://ryzenai.docs.amd.com/en/latest/).
|
||||
On AMD Ryzen™ AI 300 and 400 Series processors with a dedicated NPU, whisper.cpp can fully offload the Whisper encoder to the NPU via VitisAI, delivering significant speedup over CPU-only inference.
|
||||
|
||||
### Setup environment (Windows only)
|
||||
### Prerequisites
|
||||
|
||||
- Obtain the XRT package and the FlexmlRT package from AMD. Both are distributed as tarballs or wheels.
|
||||
- Copy the downloaded archives to a local path, extract them, and run the setup script from each extracted package in your shell (for example `source /path/to/xrt/setup.sh` and `source /path/to/flexmlrt/setup.sh`). Run these in every new shell you use to build or run `whisper.cpp`.
|
||||
Install the XRT runtime and FlexML runtime for your platform:
|
||||
|
||||
- Fetch the matching ggml model and prebuilt VitisAI encoder cache:
|
||||
- **XRT**: provides the NPU kernel driver and `xrt-smi` diagnostic tool — on Windows this is bundled with the NPU driver; on Linux install it separately following the [NPU driver installation guide](https://ryzenai.docs.amd.com/en/latest/linux.html#install-npu-drivers)
|
||||
- **FlexML runtime** (`flexmlrt`): VitisAI inference engine used by whisper.cpp — download from the [FlexML runtime releases](https://github.com/lemonade-sdk/whisper.cpp-rocm/releases/tag/deps)
|
||||
|
||||
```bash
|
||||
sh ./models/download-ggml-model.sh base
|
||||
sh ./models/download-vitisai-model.sh base
|
||||
```
|
||||
After installing, source the setup scripts in every shell you use to build or run whisper.cpp:
|
||||
|
||||
```cmd
|
||||
.\models\download-ggml-model.cmd base
|
||||
.\models\download-vitisai-model.cmd base
|
||||
```
|
||||
```bash
|
||||
# Linux
|
||||
source /opt/xilinx/xrt/setup.sh
|
||||
source /path/to/flexmlrt/setup.sh
|
||||
```
|
||||
|
||||
Use the same model name with both scripts. The VitisAI script queries the AMD collection on Hugging Face to list available caches, then downloads the selected `.rai` file as `ggml-<model>-encoder-vitisai.rai` alongside the matching `ggml-<model>.bin` file. You can also browse the collection manually at https://huggingface.co/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models.
|
||||
```cmd
|
||||
:: Windows
|
||||
cd /path/to/flexmlrt && call setup.bat
|
||||
```
|
||||
|
||||
Depending on the downloaded `.rai` cache, VitisAI may offload either the encoder only or the encoder plus cross-projection layers to the AMD NPU. `whisper.cpp` detects the cache contents at runtime and logs the selected offload mode during model initialization.
|
||||
You can verify the NPU is visible with:
|
||||
|
||||
- Build `whisper.cpp` with VitisAI support:
|
||||
```bash
|
||||
xrt-smi examine
|
||||
```
|
||||
|
||||
```bash
|
||||
cmake -B build -DWHISPER_VITISAI=1
|
||||
cmake --build build -j --config Release
|
||||
```
|
||||
Your environment is now ready.
|
||||
### Download models
|
||||
|
||||
### Build Whisper.cpp for Ryzen™ AI support
|
||||
Download the ggml model and the matching prebuilt VitisAI encoder cache:
|
||||
|
||||
```text
|
||||
$ ./build/bin/whisper-cli -m models/ggml-base.bin -f samples/jfk.wav
|
||||
```
|
||||
```bash
|
||||
# Linux / macOS
|
||||
sh ./models/download-ggml-model.sh base
|
||||
sh ./models/download-vitisai-model.sh base
|
||||
```
|
||||
|
||||
```cmd
|
||||
:: Windows
|
||||
.\models\download-ggml-model.cmd base
|
||||
.\models\download-vitisai-model.cmd base
|
||||
```
|
||||
|
||||
Use the same model name with both scripts. To see all available VitisAI encoder caches:
|
||||
|
||||
```bash
|
||||
sh ./models/download-vitisai-model.sh --list
|
||||
```
|
||||
|
||||
```cmd
|
||||
.\models\download-vitisai-model.cmd --list
|
||||
```
|
||||
|
||||
The VitisAI script queries the [AMD Ryzen AI Whisper NPU collection on Hugging Face](https://huggingface.co/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models) and downloads the `.rai` encoder cache as `models/ggml-<model>-encoder-vitisai.rai`.
|
||||
|
||||
> Depending on the `.rai` cache, VitisAI may offload the encoder only, or the encoder plus cross-projection layers. whisper.cpp detects this at runtime and logs the selected offload mode during model initialization.
|
||||
|
||||
### Build
|
||||
|
||||
```bash
|
||||
cmake -B build -DWHISPER_VITISAI=1
|
||||
cmake --build build -j --config Release
|
||||
```
|
||||
|
||||
### Run
|
||||
|
||||
```bash
|
||||
./build/bin/whisper-cli -m models/ggml-base.bin -f samples/jfk.wav
|
||||
```
|
||||
|
||||
For more information see the [Ryzen AI documentation](https://ryzenai.docs.amd.com/en/latest/).
|
||||
|
||||
|
||||
## NVIDIA GPU support
|
||||
|
|
|
|||
Loading…
Reference in New Issue