mirror of
https://github.com/ggml-org/whisper.cpp.git
synced 2026-10-18 21:55:18 +02:00
vitisai : add VitisAI Plugin for AMD Ryzen AI NPU encoder offload (#3608)
* Add VitisAI Plugin * Added VitisAI encoder module placeholder files * VitisAI build integration * VitisAI encoder offload functional * Clean up vitisai integration * Add c++17 requirement for Windows * Enabled preemption for windows runs * Add model cache override option * Remove vitisai premature log message * Add rai support through file mapping * Fixed flatbuffer loading * Fixed Windows file mapping issue * Update FlexmlRT resolution * Use Flexmlrt wheel pkg to build VitisAI plugin * Clean up * Remove prints * Change flexmlrt target from Shared to Interface * Add c++17 requirement for Windows * Enabled preemption for windows runs * Add rai support through file mapping * Fixed flatbuffer loading * Fixed Windows file mapping issue * Update FlexmlRT resolution * Use Flexmlrt wheel pkg to build VitisAI plugin * Clean up * Remove prints * Change flexmlrt target from Shared to Interface * Cleanup FlexmlRT integration * format fix * Adding AMD Licenses * Update CMakeLists.txt Co-authored-by: Kumawat, Sachin <sachin.kumawat@amd.com> * Update src/CMakeLists.txt Co-authored-by: Kumawat, Sachin <sachin.kumawat@amd.com> * Update whisper.cpp * Added VitisAI encoder readme section * Remove license headers from common files to whisper.cpp --------- Co-authored-by: Sachin Kumawat <sachink@amd.com> Co-authored-by: Jeff Lin <jeffylin@xilinx.com> Co-authored-by: Lin <jefflin@amd.com> Co-authored-by: Lin, Jeff (DCG-ENG) <jeff.lin@amd.com> Co-authored-by: Iswarya Alex <iswaryaalex96@gmail.com> Co-authored-by: Alex, Iswarya <Iswarya.Alex@amd.com> * Update README.md - RAI EULA Links - Updated for RAI Whisper instructions * Cleanup and add runtime print debug guard * turn off profiling * Add VitisAI Plugin * Added VitisAI encoder module placeholder files * VitisAI build integration * VitisAI encoder offload functional * Clean up vitisai integration * Add c++17 requirement for Windows * Enabled preemption for windows runs * Add model cache override option * Remove vitisai premature log message * Add rai support through file mapping * Fixed flatbuffer loading * Fixed Windows file mapping issue * Update FlexmlRT resolution * Use Flexmlrt wheel pkg to build VitisAI plugin * Clean up * Remove prints * Change flexmlrt target from Shared to Interface * Add c++17 requirement for Windows * Enabled preemption for windows runs * Add rai support through file mapping * Fixed flatbuffer loading * Fixed Windows file mapping issue * Update FlexmlRT resolution * Use Flexmlrt wheel pkg to build VitisAI plugin * Clean up * Remove prints * Change flexmlrt target from Shared to Interface * Cleanup FlexmlRT integration * format fix * Adding AMD Licenses * Update CMakeLists.txt Co-authored-by: Kumawat, Sachin <sachin.kumawat@amd.com> * Update src/CMakeLists.txt Co-authored-by: Kumawat, Sachin <sachin.kumawat@amd.com> * Update whisper.cpp * Added VitisAI encoder readme section * Remove license headers from common files to whisper.cpp --------- Co-authored-by: Sachin Kumawat <sachink@amd.com> Co-authored-by: Jeff Lin <jeffylin@xilinx.com> Co-authored-by: Lin <jefflin@amd.com> Co-authored-by: Lin, Jeff (DCG-ENG) <jeff.lin@amd.com> Co-authored-by: Iswarya Alex <iswaryaalex96@gmail.com> Co-authored-by: Alex, Iswarya <Iswarya.Alex@amd.com> * Cleanup and add runtime print debug guard * Update README.md - RAI EULA Links - Updated for RAI Whisper instructions * turn off profiling * Let flexmlrt detect device type * Add VitisAI model download scripts * Add encoder + cross projection layer offload * Add self hosted runner for amd npu * Update runner * Update workflow for linux * Update workflow for linux * Update flexmlrt packages for linux * Update flexmlrt packages for linux * Updated README * readme: clarify xrt * readme: clarify xrt * ci: update test config * Added supported plarform details with python 3.12 requirement for Linux * Use refactored helpers * Deprecate cross_proj .rai naming and cleanup * Remove stale function code * Fix: formatting --------- Co-authored-by: Jeff Lin <jeffylin@xilinx.com> Co-authored-by: Lin <jefflin@amd.com> Co-authored-by: Lin, Jeff (DCG-ENG) <jeff.lin@amd.com> Co-authored-by: Iswarya Alex <iswaryaalex96@gmail.com> Co-authored-by: Alex, Iswarya <Iswarya.Alex@amd.com> Co-authored-by: Iswarya Alex <47045679+iswaryaalex@users.noreply.github.com>
This commit is contained in:
Sachin Kumawat
co-authored by
Jeff Lin
Lin
Lin, Jeff (DCG-ENG)
Iswarya Alex
Alex, Iswarya
Iswarya Alex
Jeff Lin
Lin
Lin, Jeff (DCG-ENG)
Iswarya Alex
Alex, Iswarya
Iswarya Alex
parent
233fe1fc9b
commit
52dec9d889
@@ -114,3 +114,146 @@ jobs:
|
|||||||
run: |
|
run: |
|
||||||
vulkaninfo --summary
|
vulkaninfo --summary
|
||||||
GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/whisper.cpp ~/mnt/whisper.cpp
|
GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/whisper.cpp ~/mnt/whisper.cpp
|
||||||
|
|
||||||
|
npu-amd-windows:
|
||||||
|
runs-on: [self-hosted, Windows, X64, stx, rai300-400]
|
||||||
|
timeout-minutes: 60
|
||||||
|
continue-on-error: true # advisory while the runner pool is new; revisit later
|
||||||
|
|
||||||
|
env:
|
||||||
|
FLEXML_URL: https://github.com/lemonade-sdk/whisper.cpp-rocm/releases/download/deps/flexmlrt-1.7.0-win.zip
|
||||||
|
MODEL: base
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Clone
|
||||||
|
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
|
||||||
|
|
||||||
|
- uses: microsoft/setup-msbuild@v2
|
||||||
|
|
||||||
|
- name: Install CMake if not available
|
||||||
|
shell: powershell
|
||||||
|
run: |
|
||||||
|
$installed = Get-Command cmake -ErrorAction SilentlyContinue
|
||||||
|
if (-not $installed) {
|
||||||
|
$ver = "3.28.1"
|
||||||
|
$url = "https://github.com/Kitware/CMake/releases/download/v$ver/cmake-$ver-windows-x86_64.msi"
|
||||||
|
Invoke-WebRequest -Uri $url -OutFile cmake.msi
|
||||||
|
Start-Process msiexec.exe -ArgumentList "/i cmake.msi /quiet /norestart" -Wait
|
||||||
|
$p = "C:\Program Files\CMake\bin"
|
||||||
|
$env:PATH = "$p;$env:PATH"
|
||||||
|
echo $p >> $env:GITHUB_PATH
|
||||||
|
cmake --version
|
||||||
|
if ($LASTEXITCODE -ne 0) { Write-Error "CMake install failed"; exit 1 }
|
||||||
|
} else { cmake --version }
|
||||||
|
|
||||||
|
- name: Download FlexML runtime
|
||||||
|
shell: powershell
|
||||||
|
run: |
|
||||||
|
Invoke-WebRequest -Uri "${{ env.FLEXML_URL }}" -OutFile flexmlrt.zip
|
||||||
|
if (-Not (Test-Path "flexmlrt.zip")) { Write-Error "flexmlrt.zip not downloaded"; exit 1 }
|
||||||
|
if ((Get-Item "flexmlrt.zip").Length -eq 0) { Write-Error "flexmlrt.zip is empty"; exit 1 }
|
||||||
|
tar xf flexmlrt.zip
|
||||||
|
if ($LASTEXITCODE -ne 0) { Write-Error "Extraction failed"; exit 1 }
|
||||||
|
if (-not (Test-Path "flexmlrt")) { Write-Error "No flexmlrt directory after extraction"; exit 1 }
|
||||||
|
|
||||||
|
- name: Setup FlexML, configure and build
|
||||||
|
shell: cmd
|
||||||
|
run: |
|
||||||
|
cd flexmlrt
|
||||||
|
call setup.bat
|
||||||
|
if errorlevel 1 ( echo ERROR: FlexML setup.bat failed & exit /b 1 )
|
||||||
|
cd ..
|
||||||
|
cmake -B build -A x64 -DCMAKE_BUILD_TYPE=Release -DWHISPER_VITISAI=ON
|
||||||
|
if errorlevel 1 ( echo ERROR: CMake configure failed & exit /b 1 )
|
||||||
|
cmake --build build --config Release -j
|
||||||
|
if errorlevel 1 ( echo ERROR: Build failed & exit /b 1 )
|
||||||
|
|
||||||
|
- name: Copy FlexML DLLs to build output
|
||||||
|
shell: powershell
|
||||||
|
run: |
|
||||||
|
foreach ($d in "flexmlrt/bin", "flexmlrt/lib") {
|
||||||
|
if (Test-Path "$d/*.dll") { Copy-Item "$d/*.dll" "build/bin/Release/" -Force }
|
||||||
|
}
|
||||||
|
if (-not (Test-Path "build/bin/Release/flexmlrt.dll")) {
|
||||||
|
Write-Error "flexmlrt.dll not staged next to binaries"; exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
- name: Download ggml model
|
||||||
|
shell: cmd
|
||||||
|
run: |
|
||||||
|
call models\download-ggml-model.cmd %MODEL% models
|
||||||
|
if not exist models\ggml-%MODEL%.bin ( echo ERROR: model download failed & exit /b 1 )
|
||||||
|
|
||||||
|
- name: Download NPU encoder cache
|
||||||
|
shell: cmd
|
||||||
|
run: |
|
||||||
|
.\models\download-vitisai-model.cmd %MODEL%
|
||||||
|
if not exist models\ggml-%MODEL%-encoder-vitisai.rai ( echo ERROR: VitisAI encoder cache download failed & exit /b 1 )
|
||||||
|
|
||||||
|
- name: Run NPU smoke test
|
||||||
|
shell: cmd
|
||||||
|
run: |
|
||||||
|
build\bin\Release\whisper-cli.exe -m models\ggml-%MODEL%.bin -f samples\jfk.wav > vitisai.log 2>&1
|
||||||
|
type vitisai.log
|
||||||
|
findstr /I /C:"vitisai" vitisai.log || ( echo ERROR: no VitisAI activity - encoder likely fell back to CPU & exit /b 1 )
|
||||||
|
findstr /I /C:"ask not what your country" vitisai.log || ( echo ERROR: incorrect transcription & exit /b 1 )
|
||||||
|
|
||||||
|
- name: Upload smoke test log
|
||||||
|
if: always()
|
||||||
|
uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: vitisai-smoke-log-windows
|
||||||
|
path: vitisai.log
|
||||||
|
|
||||||
|
npu-amd-linux:
|
||||||
|
runs-on: [self-hosted, Linux, X64, stx, rai300-400]
|
||||||
|
timeout-minutes: 60
|
||||||
|
continue-on-error: true # advisory while the runner pool is new; revisit later
|
||||||
|
|
||||||
|
env:
|
||||||
|
FLEXML_LINUX_URL: https://github.com/lemonade-sdk/whisper.cpp-rocm/releases/download/deps/flexmlrt-1.8.0-linux.tar.gz
|
||||||
|
MODEL: base
|
||||||
|
|
||||||
|
steps:
|
||||||
|
- name: Clone
|
||||||
|
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
|
||||||
|
|
||||||
|
- name: Verify NPU device
|
||||||
|
run: |
|
||||||
|
lsmod | grep -q amdxdna || { echo "ERROR: amdxdna driver not loaded"; exit 1; }
|
||||||
|
ls /dev/accel/accel* || { echo "ERROR: no NPU accel device node"; exit 1; }
|
||||||
|
|
||||||
|
- name: Download FlexML runtime (Linux)
|
||||||
|
run: |
|
||||||
|
curl -L --fail -o flexmlrt.tar.gz "$FLEXML_LINUX_URL"
|
||||||
|
tar xf flexmlrt.tar.gz
|
||||||
|
source flexmlrt/setup.sh
|
||||||
|
echo "FlexmlRT_DIR=$PWD/flexmlrt/share/cmake/FlexmlRT" >> $GITHUB_ENV
|
||||||
|
echo "LD_LIBRARY_PATH=$PWD/flexmlrt/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV
|
||||||
|
|
||||||
|
- name: Configure and build
|
||||||
|
run: |
|
||||||
|
cmake -B build -DCMAKE_BUILD_TYPE=Release -DWHISPER_VITISAI=ON
|
||||||
|
cmake --build build --config Release -j $(nproc)
|
||||||
|
|
||||||
|
- name: Download ggml model
|
||||||
|
run: |
|
||||||
|
./models/download-ggml-model.sh $MODEL
|
||||||
|
|
||||||
|
- name: Download NPU encoder cache
|
||||||
|
run: |
|
||||||
|
sh ./models/download-vitisai-model.sh $MODEL
|
||||||
|
[ -f "models/ggml-$MODEL-encoder-vitisai.rai" ] || { echo "ERROR: VitisAI encoder cache download failed"; exit 1; }
|
||||||
|
|
||||||
|
- name: Run NPU smoke test
|
||||||
|
run: |
|
||||||
|
./build/bin/whisper-cli -m "models/ggml-$MODEL.bin" -f samples/jfk.wav 2>&1 | tee vitisai.log
|
||||||
|
grep -qi "vitisai" vitisai.log || { echo "ERROR: no VitisAI activity - CPU fallback?"; exit 1; }
|
||||||
|
grep -qi "ask not what your country" vitisai.log || { echo "ERROR: incorrect transcription"; exit 1; }
|
||||||
|
|
||||||
|
- name: Upload smoke test log
|
||||||
|
if: always()
|
||||||
|
uses: actions/upload-artifact@v4
|
||||||
|
with:
|
||||||
|
name: vitisai-smoke-log-linux
|
||||||
|
path: vitisai.log
|
||||||
|
|||||||
@@ -115,6 +115,7 @@ endif()
|
|||||||
option(WHISPER_COREML "whisper: enable Core ML framework" OFF)
|
option(WHISPER_COREML "whisper: enable Core ML framework" OFF)
|
||||||
option(WHISPER_COREML_ALLOW_FALLBACK "whisper: allow non-CoreML fallback" OFF)
|
option(WHISPER_COREML_ALLOW_FALLBACK "whisper: allow non-CoreML fallback" OFF)
|
||||||
option(WHISPER_OPENVINO "whisper: support for OpenVINO" OFF)
|
option(WHISPER_OPENVINO "whisper: support for OpenVINO" OFF)
|
||||||
|
option(WHISPER_VITISAI "whisper: support for AMD Vitis AI" OFF)
|
||||||
|
|
||||||
# Required for relocatable CMake package
|
# Required for relocatable CMake package
|
||||||
include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/build-info.cmake)
|
include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/build-info.cmake)
|
||||||
|
|||||||
@@ -21,6 +21,7 @@ High-performance inference of [OpenAI's Whisper](https://github.com/openai/whisp
|
|||||||
- Support for CPU-only inference
|
- Support for CPU-only inference
|
||||||
- [Efficient GPU support for NVIDIA](#nvidia-gpu-support)
|
- [Efficient GPU support for NVIDIA](#nvidia-gpu-support)
|
||||||
- [AMD ROCm GPU support](#amd-rocm-gpu-support)
|
- [AMD ROCm GPU support](#amd-rocm-gpu-support)
|
||||||
|
- [AMD Ryzen AI NPU Support](#amd-ryzen-ai-npu-support)
|
||||||
- [OpenVINO Support](#openvino-support)
|
- [OpenVINO Support](#openvino-support)
|
||||||
- [Ascend NPU Support](#ascend-npu-support)
|
- [Ascend NPU Support](#ascend-npu-support)
|
||||||
- [Moore Threads GPU Support](#moore-threads-gpu-support)
|
- [Moore Threads GPU Support](#moore-threads-gpu-support)
|
||||||
@@ -312,6 +313,87 @@ This can result in significant speedup in encoder performance. Here are the inst
|
|||||||
|
|
||||||
For more information about the OpenVINO implementation please refer to PR [#1037](https://github.com/ggml-org/whisper.cpp/pull/1037).
|
For more information about the OpenVINO implementation please refer to PR [#1037](https://github.com/ggml-org/whisper.cpp/pull/1037).
|
||||||
|
|
||||||
|
## AMD Ryzen™ AI NPU support
|
||||||
|
|
||||||
|
On AMD Ryzen™ AI 300 and 400 Series processors with a dedicated NPU, whisper.cpp can fully offload the Whisper encoder to the NPU via VitisAI, delivering significant speedup over CPU-only inference.
|
||||||
|
|
||||||
|
### Prerequisites
|
||||||
|
|
||||||
|
Supported Platforms
|
||||||
|
|
||||||
|
- **Windows 11**
|
||||||
|
- **Linux** (Ubuntu 24.04 LTS, Python 3.12)
|
||||||
|
|
||||||
|
Install the XRT runtime and FlexML runtime for your platform:
|
||||||
|
|
||||||
|
- **XRT**: provides the NPU kernel driver and `xrt-smi` diagnostic tool — on Windows this is bundled with the NPU driver; on Linux install it separately following the [NPU driver installation guide](https://ryzenai.docs.amd.com/en/latest/linux.html#install-npu-drivers)
|
||||||
|
- **FlexML runtime** (`flexmlrt`): VitisAI inference engine used by whisper.cpp — download from the [FlexML runtime releases](https://github.com/lemonade-sdk/whisper.cpp-rocm/releases/tag/deps)
|
||||||
|
|
||||||
|
After installing, source the setup scripts in every shell you use to build or run whisper.cpp:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Linux
|
||||||
|
source /opt/xilinx/xrt/setup.sh
|
||||||
|
source /path/to/flexmlrt/setup.sh
|
||||||
|
```
|
||||||
|
|
||||||
|
```cmd
|
||||||
|
:: Windows
|
||||||
|
cd /path/to/flexmlrt && call setup.bat
|
||||||
|
```
|
||||||
|
|
||||||
|
You can verify the NPU is visible with:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
xrt-smi examine
|
||||||
|
```
|
||||||
|
|
||||||
|
### Download models
|
||||||
|
|
||||||
|
Download the ggml model and the matching prebuilt VitisAI encoder cache:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# Linux / macOS
|
||||||
|
sh ./models/download-ggml-model.sh base
|
||||||
|
sh ./models/download-vitisai-model.sh base
|
||||||
|
```
|
||||||
|
|
||||||
|
```cmd
|
||||||
|
:: Windows
|
||||||
|
.\models\download-ggml-model.cmd base
|
||||||
|
.\models\download-vitisai-model.cmd base
|
||||||
|
```
|
||||||
|
|
||||||
|
Use the same model name with both scripts. To see all available VitisAI encoder caches:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sh ./models/download-vitisai-model.sh --list
|
||||||
|
```
|
||||||
|
|
||||||
|
```cmd
|
||||||
|
.\models\download-vitisai-model.cmd --list
|
||||||
|
```
|
||||||
|
|
||||||
|
The VitisAI script queries the [AMD Ryzen AI Whisper NPU collection on Hugging Face](https://huggingface.co/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models) and downloads the `.rai` encoder cache as `models/ggml-<model>-encoder-vitisai.rai`.
|
||||||
|
|
||||||
|
> Depending on the `.rai` cache, VitisAI may offload the encoder only, or the encoder plus cross-projection layers. whisper.cpp detects this at runtime and logs the selected offload mode during model initialization.
|
||||||
|
|
||||||
|
### Build
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cmake -B build -DWHISPER_VITISAI=1
|
||||||
|
cmake --build build -j --config Release
|
||||||
|
```
|
||||||
|
|
||||||
|
### Run
|
||||||
|
|
||||||
|
```bash
|
||||||
|
./build/bin/whisper-cli -m models/ggml-base.bin -f samples/jfk.wav
|
||||||
|
```
|
||||||
|
|
||||||
|
For more information see the [Ryzen AI documentation](https://ryzenai.docs.amd.com/en/latest/).
|
||||||
|
|
||||||
|
|
||||||
## NVIDIA GPU support
|
## NVIDIA GPU support
|
||||||
|
|
||||||
With NVIDIA cards the processing of the models is done efficiently on the GPU via cuBLAS and custom CUDA kernels.
|
With NVIDIA cards the processing of the models is done efficiently on the GPU via cuBLAS and custom CUDA kernels.
|
||||||
|
|||||||
@@ -0,0 +1,32 @@
|
|||||||
|
@echo off
|
||||||
|
setlocal
|
||||||
|
|
||||||
|
set "script=%~dp0download-vitisai-model.ps1"
|
||||||
|
|
||||||
|
if "%~1"=="" (
|
||||||
|
PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%"
|
||||||
|
exit /b %ERRORLEVEL%
|
||||||
|
)
|
||||||
|
|
||||||
|
if /I "%~1"=="--list" (
|
||||||
|
PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" -List
|
||||||
|
exit /b %ERRORLEVEL%
|
||||||
|
)
|
||||||
|
|
||||||
|
if /I "%~1"=="-l" (
|
||||||
|
PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" -List
|
||||||
|
exit /b %ERRORLEVEL%
|
||||||
|
)
|
||||||
|
|
||||||
|
if /I "%~1"=="list" (
|
||||||
|
PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" -List
|
||||||
|
exit /b %ERRORLEVEL%
|
||||||
|
)
|
||||||
|
|
||||||
|
if "%~2"=="" (
|
||||||
|
PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" -Model "%~1"
|
||||||
|
) else (
|
||||||
|
PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" -Model "%~1" -ModelsPath "%~2"
|
||||||
|
)
|
||||||
|
|
||||||
|
exit /b %ERRORLEVEL%
|
||||||
@@ -0,0 +1,218 @@
|
|||||||
|
param(
|
||||||
|
[Parameter(Position = 0)]
|
||||||
|
[string] $Model,
|
||||||
|
|
||||||
|
[Parameter(Position = 1)]
|
||||||
|
[string] $ModelsPath,
|
||||||
|
|
||||||
|
[switch] $List
|
||||||
|
)
|
||||||
|
|
||||||
|
$ErrorActionPreference = "Stop"
|
||||||
|
|
||||||
|
$Source = "https://huggingface.co"
|
||||||
|
$CollectionApi = "https://huggingface.co/api/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models"
|
||||||
|
$CollectionUrl = "https://huggingface.co/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models"
|
||||||
|
|
||||||
|
$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||||
|
if ($ScriptDir -match "\\bin$") {
|
||||||
|
$DefaultDownloadPath = (Get-Location).Path
|
||||||
|
} else {
|
||||||
|
$DefaultDownloadPath = $ScriptDir
|
||||||
|
}
|
||||||
|
|
||||||
|
if (-not $ModelsPath) {
|
||||||
|
$ModelsPath = $DefaultDownloadPath
|
||||||
|
}
|
||||||
|
|
||||||
|
function Get-HfHeaders {
|
||||||
|
$headers = @{}
|
||||||
|
if ($env:HF_TOKEN) {
|
||||||
|
$headers["Authorization"] = "Bearer $env:HF_TOKEN"
|
||||||
|
}
|
||||||
|
return $headers
|
||||||
|
}
|
||||||
|
|
||||||
|
function Invoke-HfJson {
|
||||||
|
param([string] $Uri)
|
||||||
|
|
||||||
|
$headers = Get-HfHeaders
|
||||||
|
if ($headers.Count -gt 0) {
|
||||||
|
return Invoke-RestMethod -Uri $Uri -Headers $headers
|
||||||
|
}
|
||||||
|
|
||||||
|
return Invoke-RestMethod -Uri $Uri
|
||||||
|
}
|
||||||
|
|
||||||
|
function Normalize-ModelName {
|
||||||
|
param([string] $Name)
|
||||||
|
|
||||||
|
# HF currently publishes ggml-small-en-encoder-vitisai.rai, while the
|
||||||
|
# matching ggml model is ggml-small.en.bin.
|
||||||
|
if ($Name.EndsWith("-en")) {
|
||||||
|
return $Name.Substring(0, $Name.Length - 3) + ".en"
|
||||||
|
}
|
||||||
|
|
||||||
|
return $Name
|
||||||
|
}
|
||||||
|
|
||||||
|
function Get-VitisAiModels {
|
||||||
|
$collection = Invoke-HfJson -Uri $CollectionApi
|
||||||
|
$seen = @{}
|
||||||
|
$rows = New-Object System.Collections.Generic.List[object]
|
||||||
|
|
||||||
|
foreach ($item in $collection.items) {
|
||||||
|
if ($item.type -ne "model") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
$repo = $item.id
|
||||||
|
if (-not $repo) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
$modelInfo = Invoke-HfJson -Uri "$Source/api/models/$repo"
|
||||||
|
foreach ($sibling in $modelInfo.siblings) {
|
||||||
|
$filename = [string] $sibling.rfilename
|
||||||
|
$match = [regex]::Match($filename, "^ggml-(.+)-encoder-vitisai\.rai$")
|
||||||
|
if (-not $match.Success) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
$rawName = $match.Groups[1].Value
|
||||||
|
$modelName = Normalize-ModelName -Name $rawName
|
||||||
|
if ($seen.ContainsKey($modelName)) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
$seen[$modelName] = $true
|
||||||
|
$destination = "ggml-$modelName-encoder-vitisai.rai"
|
||||||
|
$url = "$Source/$repo/resolve/main/$([uri]::EscapeDataString($filename))"
|
||||||
|
|
||||||
|
$rows.Add([pscustomobject]@{
|
||||||
|
Model = $modelName
|
||||||
|
RawName = $rawName
|
||||||
|
Repo = $repo
|
||||||
|
SourceFile = $filename
|
||||||
|
DestinationFile = $destination
|
||||||
|
DownloadUrl = $url
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
$order = @{
|
||||||
|
"tiny" = 10
|
||||||
|
"tiny.en" = 11
|
||||||
|
"base" = 20
|
||||||
|
"base.en" = 21
|
||||||
|
"small" = 30
|
||||||
|
"small.en" = 31
|
||||||
|
"medium" = 40
|
||||||
|
"medium.en" = 41
|
||||||
|
"large-v1" = 50
|
||||||
|
"large-v2" = 60
|
||||||
|
"large-v3" = 70
|
||||||
|
"large-v3-turbo" = 80
|
||||||
|
}
|
||||||
|
|
||||||
|
return $rows | Sort-Object `
|
||||||
|
@{ Expression = { if ($order.ContainsKey($_.Model)) { $order[$_.Model] } else { 1000 } } }, `
|
||||||
|
@{ Expression = { $_.Model } }
|
||||||
|
}
|
||||||
|
|
||||||
|
function Show-Models {
|
||||||
|
$models = Get-VitisAiModels
|
||||||
|
|
||||||
|
Write-Host ""
|
||||||
|
Write-Host "Available VitisAI encoder caches from ${CollectionUrl}:"
|
||||||
|
foreach ($entry in $models) {
|
||||||
|
if ($entry.Model -eq $entry.RawName) {
|
||||||
|
Write-Host (" {0,-18} {1}" -f $entry.Model, $entry.Repo)
|
||||||
|
} else {
|
||||||
|
Write-Host (" {0,-18} {1} (source name: {2})" -f $entry.Model, $entry.Repo, $entry.RawName)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Write-Host ""
|
||||||
|
}
|
||||||
|
|
||||||
|
function Show-Usage {
|
||||||
|
Write-Host "Usage: download-vitisai-model.cmd --list"
|
||||||
|
Write-Host " download-vitisai-model.cmd <model> [models_path]"
|
||||||
|
Write-Host ""
|
||||||
|
Write-Host "Downloads ggml-<model>-encoder-vitisai.rai next to ggml-<model>.bin."
|
||||||
|
Write-Host "Use the same model name as download-ggml-model.cmd."
|
||||||
|
Write-Host ""
|
||||||
|
}
|
||||||
|
|
||||||
|
if ($List -or $Model -eq "--list" -or $Model -eq "-l" -or $Model -eq "list") {
|
||||||
|
Show-Models
|
||||||
|
exit 0
|
||||||
|
}
|
||||||
|
|
||||||
|
if (-not $Model) {
|
||||||
|
Show-Usage
|
||||||
|
Show-Models
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
$models = Get-VitisAiModels
|
||||||
|
$entry = $models | Where-Object { $_.Model -eq $Model -or $_.RawName -eq $Model } | Select-Object -First 1
|
||||||
|
if (-not $entry) {
|
||||||
|
Write-Host "Invalid model: $Model"
|
||||||
|
foreach ($available in $models) {
|
||||||
|
Write-Host " $($available.Model)"
|
||||||
|
}
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
New-Item -ItemType Directory -Force -Path $ModelsPath | Out-Null
|
||||||
|
$destinationPath = Join-Path $ModelsPath $entry.DestinationFile
|
||||||
|
|
||||||
|
Write-Host "Downloading VitisAI encoder cache $($entry.Model) from '$($entry.Repo)' ..."
|
||||||
|
if (Test-Path $destinationPath) {
|
||||||
|
Write-Host "VitisAI encoder cache $($entry.DestinationFile) already exists. Skipping download."
|
||||||
|
exit 0
|
||||||
|
}
|
||||||
|
|
||||||
|
$headers = Get-HfHeaders
|
||||||
|
$downloaded = $false
|
||||||
|
for ($attempt = 1; $attempt -le 5; ++$attempt) {
|
||||||
|
try {
|
||||||
|
if ($headers.Count -gt 0) {
|
||||||
|
Invoke-WebRequest -Uri $entry.DownloadUrl -Headers $headers -OutFile $destinationPath
|
||||||
|
} else {
|
||||||
|
Invoke-WebRequest -Uri $entry.DownloadUrl -OutFile $destinationPath
|
||||||
|
}
|
||||||
|
$downloaded = $true
|
||||||
|
break
|
||||||
|
} catch {
|
||||||
|
if ($attempt -eq 5) {
|
||||||
|
if (Test-Path $destinationPath) {
|
||||||
|
Remove-Item -Force $destinationPath
|
||||||
|
}
|
||||||
|
Write-Host "Failed to download VitisAI encoder cache $($entry.Model) from $($entry.DownloadUrl)"
|
||||||
|
throw
|
||||||
|
}
|
||||||
|
Start-Sleep -Seconds 5
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (-not $downloaded) {
|
||||||
|
exit 1
|
||||||
|
}
|
||||||
|
|
||||||
|
$whisperCmd = "whisper-cli"
|
||||||
|
if (-not (Get-Command whisper-cli -ErrorAction SilentlyContinue)) {
|
||||||
|
$rootPath = Split-Path -Parent $ScriptDir
|
||||||
|
$whisperCmd = Join-Path $rootPath "build\bin\Release\whisper-cli.exe"
|
||||||
|
}
|
||||||
|
|
||||||
|
Write-Host "Done! VitisAI encoder cache '$($entry.Model)' saved in '$destinationPath'"
|
||||||
|
if ($entry.RawName -ne $entry.Model) {
|
||||||
|
Write-Host "Source cache '$($entry.SourceFile)' was renamed to match ggml model name '$($entry.Model)'."
|
||||||
|
}
|
||||||
|
Write-Host "Use it with the matching ggml model:"
|
||||||
|
Write-Host ""
|
||||||
|
Write-Host " $ScriptDir\download-ggml-model.cmd $($entry.Model) $ModelsPath"
|
||||||
|
Write-Host " $whisperCmd -m $ModelsPath\ggml-$($entry.Model).bin -f samples\jfk.wav"
|
||||||
|
Write-Host ""
|
||||||
Executable
+226
@@ -0,0 +1,226 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
|
||||||
|
# This script downloads prebuilt VitisAI encoder cache files for Whisper models.
|
||||||
|
# The cache file is saved next to the ggml model file and follows the loader
|
||||||
|
# convention: ggml-<model>-encoder-vitisai.rai
|
||||||
|
|
||||||
|
src="https://huggingface.co"
|
||||||
|
collection_api="https://huggingface.co/api/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models"
|
||||||
|
collection_url="https://huggingface.co/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models"
|
||||||
|
|
||||||
|
BOLD="\033[1m"
|
||||||
|
RESET='\033[0m'
|
||||||
|
|
||||||
|
# get the path of this script
|
||||||
|
get_script_path() {
|
||||||
|
if [ -x "$(command -v realpath)" ]; then
|
||||||
|
dirname "$(realpath "$0")"
|
||||||
|
else
|
||||||
|
_ret="$(cd -- "$(dirname "$0")" >/dev/null 2>&1 || exit ; pwd -P)"
|
||||||
|
echo "$_ret"
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
find_python() {
|
||||||
|
if command -v python3 >/dev/null 2>&1; then
|
||||||
|
printf "%s\n" "python3"
|
||||||
|
elif command -v python >/dev/null 2>&1; then
|
||||||
|
printf "%s\n" "python"
|
||||||
|
else
|
||||||
|
return 1
|
||||||
|
fi
|
||||||
|
}
|
||||||
|
|
||||||
|
script_path="$(get_script_path)"
|
||||||
|
|
||||||
|
# Check if the script is inside a /bin/ directory
|
||||||
|
case "$script_path" in
|
||||||
|
*/bin) default_download_path="$PWD" ;; # Use current directory as default download path if in /bin/
|
||||||
|
*) default_download_path="$script_path" ;; # Otherwise, use script directory
|
||||||
|
esac
|
||||||
|
|
||||||
|
models_path="${2:-$default_download_path}"
|
||||||
|
|
||||||
|
discover_models() {
|
||||||
|
python_cmd="$(find_python)" || {
|
||||||
|
printf "Python is required to query available VitisAI caches from Hugging Face.\n" >&2
|
||||||
|
return 1
|
||||||
|
}
|
||||||
|
|
||||||
|
"$python_cmd" - "$src" "$collection_api" <<'PY'
|
||||||
|
import json
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
import urllib.parse
|
||||||
|
import urllib.request
|
||||||
|
|
||||||
|
src = sys.argv[1].rstrip("/")
|
||||||
|
collection_api = sys.argv[2]
|
||||||
|
headers = {}
|
||||||
|
token = os.environ.get("HF_TOKEN")
|
||||||
|
if token:
|
||||||
|
headers["Authorization"] = "Bearer " + token
|
||||||
|
|
||||||
|
def load_json(url):
|
||||||
|
req = urllib.request.Request(url, headers=headers)
|
||||||
|
with urllib.request.urlopen(req) as response:
|
||||||
|
return json.load(response)
|
||||||
|
|
||||||
|
def normalize_model_name(name):
|
||||||
|
# HF currently publishes ggml-small-en-encoder-vitisai.rai, while the
|
||||||
|
# matching ggml model is ggml-small.en.bin.
|
||||||
|
if name.endswith("-en"):
|
||||||
|
return name[:-3] + ".en"
|
||||||
|
return name
|
||||||
|
|
||||||
|
collection = load_json(collection_api)
|
||||||
|
rows = []
|
||||||
|
seen = set()
|
||||||
|
|
||||||
|
for item in collection.get("items", []):
|
||||||
|
if item.get("type") != "model":
|
||||||
|
continue
|
||||||
|
|
||||||
|
repo = item.get("id")
|
||||||
|
if not repo:
|
||||||
|
continue
|
||||||
|
|
||||||
|
model_info = load_json(src + "/api/models/" + repo)
|
||||||
|
for sibling in model_info.get("siblings", []):
|
||||||
|
filename = sibling.get("rfilename", "")
|
||||||
|
match = re.match(r"^ggml-(.+)-encoder-vitisai\.rai$", filename)
|
||||||
|
if not match:
|
||||||
|
continue
|
||||||
|
|
||||||
|
raw_name = match.group(1)
|
||||||
|
model_name = normalize_model_name(raw_name)
|
||||||
|
if model_name in seen:
|
||||||
|
continue
|
||||||
|
seen.add(model_name)
|
||||||
|
|
||||||
|
destination = "ggml-%s-encoder-vitisai.rai" % model_name
|
||||||
|
url = "%s/%s/resolve/main/%s" % (src, repo, urllib.parse.quote(filename))
|
||||||
|
rows.append((model_name, raw_name, repo, filename, destination, url))
|
||||||
|
|
||||||
|
order = {
|
||||||
|
"tiny": 10,
|
||||||
|
"tiny.en": 11,
|
||||||
|
"base": 20,
|
||||||
|
"base.en": 21,
|
||||||
|
"small": 30,
|
||||||
|
"small.en": 31,
|
||||||
|
"medium": 40,
|
||||||
|
"medium.en": 41,
|
||||||
|
"large-v1": 50,
|
||||||
|
"large-v2": 60,
|
||||||
|
"large-v3": 70,
|
||||||
|
"large-v3-turbo": 80,
|
||||||
|
}
|
||||||
|
|
||||||
|
for row in sorted(rows, key=lambda item: (order.get(item[0], 1000), item[0])):
|
||||||
|
print("|".join(row))
|
||||||
|
PY
|
||||||
|
}
|
||||||
|
|
||||||
|
list_models() {
|
||||||
|
models="$(discover_models)" || exit 1
|
||||||
|
|
||||||
|
printf "\n"
|
||||||
|
printf "Available VitisAI encoder caches from %s:\n" "$collection_url"
|
||||||
|
printf "%s\n" "$models" | while IFS='|' read -r model raw repo _source _destination _url; do
|
||||||
|
if [ "$model" = "$raw" ]; then
|
||||||
|
printf " %-18s %s\n" "$model" "$repo"
|
||||||
|
else
|
||||||
|
printf " %-18s %s (source name: %s)\n" "$model" "$repo" "$raw"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
printf "\n"
|
||||||
|
}
|
||||||
|
|
||||||
|
usage() {
|
||||||
|
printf "Usage: %s --list\n" "$0"
|
||||||
|
printf " %s <model> [models_path]\n" "$0"
|
||||||
|
printf "\n"
|
||||||
|
printf "Downloads ggml-<model>-encoder-vitisai.rai next to ggml-<model>.bin.\n"
|
||||||
|
printf "Use the same model name as %s/download-ggml-model.sh.\n" "$script_path"
|
||||||
|
printf "\n"
|
||||||
|
}
|
||||||
|
|
||||||
|
if [ "$#" -eq 1 ] && { [ "$1" = "--list" ] || [ "$1" = "-l" ] || [ "$1" = "list" ]; }; then
|
||||||
|
list_models
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ "$#" -lt 1 ] || [ "$#" -gt 2 ]; then
|
||||||
|
usage
|
||||||
|
list_models
|
||||||
|
printf "___________________________________________________________\n"
|
||||||
|
printf "Example: %s ${BOLD}small${RESET} %s\n" "$0" "$default_download_path"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
model=$1
|
||||||
|
models="$(discover_models)" || exit 1
|
||||||
|
|
||||||
|
match="$(printf "%s\n" "$models" | awk -F '|' -v model="$model" '$1 == model || $2 == model { print; exit }')"
|
||||||
|
if [ -z "$match" ]; then
|
||||||
|
printf "Invalid model: %s\n" "$model"
|
||||||
|
printf "%s\n" "$models" | while IFS='|' read -r available _raw _repo _source _destination _url; do
|
||||||
|
printf " %s\n" "$available"
|
||||||
|
done
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
IFS='|' read -r model raw_name repo source_file destination_file download_url <<EOF
|
||||||
|
$match
|
||||||
|
EOF
|
||||||
|
|
||||||
|
printf "Downloading VitisAI encoder cache %s from '%s' ...\n" "$model" "$repo"
|
||||||
|
|
||||||
|
mkdir -p "$models_path" || exit
|
||||||
|
cd "$models_path" || exit
|
||||||
|
|
||||||
|
if [ -f "$destination_file" ]; then
|
||||||
|
printf "VitisAI encoder cache %s already exists. Skipping download.\n" "$destination_file"
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ -x "$(command -v wget2)" ]; then
|
||||||
|
wget2 --no-config --progress bar -O "$destination_file" ${HF_TOKEN:+--header "Authorization: Bearer $HF_TOKEN"} "$download_url"
|
||||||
|
elif [ -x "$(command -v curl)" ]; then
|
||||||
|
curl -L --fail \
|
||||||
|
--retry 5 \
|
||||||
|
--retry-delay 5 \
|
||||||
|
--retry-all-errors \
|
||||||
|
--retry-connrefused \
|
||||||
|
${HF_TOKEN:+--header "Authorization: Bearer $HF_TOKEN"} \
|
||||||
|
--output "$destination_file" "$download_url"
|
||||||
|
elif [ -x "$(command -v wget)" ]; then
|
||||||
|
wget --no-config --quiet --show-progress ${HF_TOKEN:+--header "Authorization: Bearer $HF_TOKEN"} -O "$destination_file" "$download_url"
|
||||||
|
else
|
||||||
|
printf "Either wget2, curl, or wget is required to download VitisAI encoder caches.\n"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
if [ $? -ne 0 ]; then
|
||||||
|
printf "Failed to download VitisAI encoder cache %s from %s\n" "$model" "$download_url"
|
||||||
|
rm -f "$destination_file"
|
||||||
|
exit 1
|
||||||
|
fi
|
||||||
|
|
||||||
|
# Check if 'whisper-cli' is available in the system PATH
|
||||||
|
if command -v whisper-cli >/dev/null 2>&1; then
|
||||||
|
whisper_cmd="whisper-cli"
|
||||||
|
else
|
||||||
|
whisper_cmd="./build/bin/whisper-cli"
|
||||||
|
fi
|
||||||
|
|
||||||
|
printf "Done! VitisAI encoder cache '%s' saved in '%s/%s'\n" "$model" "$models_path" "$destination_file"
|
||||||
|
if [ "$raw_name" != "$model" ]; then
|
||||||
|
printf "Source cache '%s' was renamed to match ggml model name '%s'.\n" "$source_file" "$model"
|
||||||
|
fi
|
||||||
|
printf "Use it with the matching ggml model:\n\n"
|
||||||
|
printf " $ %s/download-ggml-model.sh %s %s\n" "$script_path" "$model" "$models_path"
|
||||||
|
printf " $ %s -m %s/ggml-%s.bin -f samples/jfk.wav\n" "$whisper_cmd" "$models_path" "$model"
|
||||||
|
printf "\n"
|
||||||
@@ -48,6 +48,61 @@ if (WHISPER_OPENVINO)
|
|||||||
find_package(OpenVINO REQUIRED COMPONENTS Runtime)
|
find_package(OpenVINO REQUIRED COMPONENTS Runtime)
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
|
if (WHISPER_VITISAI)
|
||||||
|
find_package(FlexmlRT REQUIRED)
|
||||||
|
|
||||||
|
# Legacy RAI overrides are required by FlexMLRT older than 1.8.0
|
||||||
|
set(WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES_MODE "AUTO" CACHE STRING
|
||||||
|
"Legacy RAI override mode for FlexMLRT (AUTO|ON|OFF)")
|
||||||
|
set_property(CACHE WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES_MODE PROPERTY STRINGS AUTO ON OFF)
|
||||||
|
|
||||||
|
string(TOUPPER "${WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES_MODE}" _flexmlrt_legacy_mode)
|
||||||
|
set(_flexmlrt_legacy_hint "Set -DWHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES_MODE=ON or OFF explicitly.")
|
||||||
|
|
||||||
|
if (NOT _flexmlrt_legacy_mode MATCHES "^(AUTO|ON|OFF)$")
|
||||||
|
message(FATAL_ERROR
|
||||||
|
"Invalid WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES_MODE='${WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES_MODE}'. "
|
||||||
|
"Expected AUTO, ON, or OFF.")
|
||||||
|
endif()
|
||||||
|
|
||||||
|
if (_flexmlrt_legacy_mode STREQUAL "AUTO")
|
||||||
|
if (NOT FlexmlRT_DIR)
|
||||||
|
message(FATAL_ERROR
|
||||||
|
"FlexmlRT_DIR is unset after find_package(FlexmlRT). ${_flexmlrt_legacy_hint}")
|
||||||
|
endif()
|
||||||
|
|
||||||
|
# FlexmlRT_DIR points to <pkg_root>/share/cmake/FlexmlRT.
|
||||||
|
get_filename_component(_flexmlrt_init_py "${FlexmlRT_DIR}/../../../__init__.py" ABSOLUTE)
|
||||||
|
if (NOT EXISTS "${_flexmlrt_init_py}")
|
||||||
|
message(FATAL_ERROR
|
||||||
|
"flexmlrt __init__.py not found at ${_flexmlrt_init_py}. ${_flexmlrt_legacy_hint}")
|
||||||
|
endif()
|
||||||
|
set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${_flexmlrt_init_py}")
|
||||||
|
|
||||||
|
file(STRINGS "${_flexmlrt_init_py}" _flexmlrt_version_lines
|
||||||
|
REGEX "^VERSION[ \t]*=[ \t]*\"[0-9]+\\.[0-9]+\\.[0-9]+")
|
||||||
|
if (NOT _flexmlrt_version_lines MATCHES "\"([0-9]+\\.[0-9]+\\.[0-9]+)")
|
||||||
|
message(FATAL_ERROR
|
||||||
|
"Could not parse flexmlrt VERSION from ${_flexmlrt_init_py}. ${_flexmlrt_legacy_hint}")
|
||||||
|
endif()
|
||||||
|
set(_flexmlrt_version "${CMAKE_MATCH_1}")
|
||||||
|
|
||||||
|
if (_flexmlrt_version VERSION_LESS "1.8.0")
|
||||||
|
set(WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES 1)
|
||||||
|
else()
|
||||||
|
set(WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES 0)
|
||||||
|
endif()
|
||||||
|
message(STATUS "Detected flexmlrt VERSION=${_flexmlrt_version} from ${_flexmlrt_init_py} (legacy overrides=${WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES})")
|
||||||
|
else()
|
||||||
|
if (_flexmlrt_legacy_mode STREQUAL "ON")
|
||||||
|
set(WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES 1)
|
||||||
|
else()
|
||||||
|
set(WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES 0)
|
||||||
|
endif()
|
||||||
|
message(STATUS "FlexMLRT legacy RAI overrides forced ${_flexmlrt_legacy_mode}")
|
||||||
|
endif()
|
||||||
|
endif()
|
||||||
|
|
||||||
#
|
#
|
||||||
# libraries
|
# libraries
|
||||||
#
|
#
|
||||||
@@ -101,6 +156,33 @@ if (WHISPER_OPENVINO)
|
|||||||
set_target_properties(${TARGET} PROPERTIES FOLDER "libs")
|
set_target_properties(${TARGET} PROPERTIES FOLDER "libs")
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
|
if (WHISPER_VITISAI)
|
||||||
|
set(TARGET whisper.vitisai)
|
||||||
|
|
||||||
|
add_library(${TARGET} OBJECT
|
||||||
|
vitisai/whisper-vitisai-helpers.h
|
||||||
|
vitisai/whisper-vitisai-helpers.cpp
|
||||||
|
vitisai/whisper-vitisai-encoder.h
|
||||||
|
vitisai/whisper-vitisai-encoder.cpp
|
||||||
|
)
|
||||||
|
|
||||||
|
target_include_directories(${TARGET} PUBLIC .)
|
||||||
|
|
||||||
|
set_property(TARGET ${TARGET} PROPERTY POSITION_INDEPENDENT_CODE ON)
|
||||||
|
set(WHISPER_EXTRA_FLAGS ${WHISPER_EXTRA_FLAGS} -DWHISPER_USE_VITISAI)
|
||||||
|
|
||||||
|
# FlexMLRT headers and this plugin require C++17. Keep it PRIVATE so the
|
||||||
|
# C++11 requirement of the whisper target is not bumped.
|
||||||
|
target_compile_features(${TARGET} PRIVATE cxx_std_17)
|
||||||
|
|
||||||
|
target_compile_definitions(${TARGET} PRIVATE
|
||||||
|
WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES=${WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES}
|
||||||
|
)
|
||||||
|
|
||||||
|
target_link_libraries(${TARGET} PRIVATE ggml flexmlrt::flexmlrt)
|
||||||
|
set_target_properties(${TARGET} PROPERTIES FOLDER "libs")
|
||||||
|
endif()
|
||||||
|
|
||||||
# whisper
|
# whisper
|
||||||
|
|
||||||
add_library(whisper
|
add_library(whisper
|
||||||
@@ -159,6 +241,10 @@ if (WHISPER_OPENVINO)
|
|||||||
target_link_libraries(whisper PRIVATE whisper.openvino)
|
target_link_libraries(whisper PRIVATE whisper.openvino)
|
||||||
endif()
|
endif()
|
||||||
|
|
||||||
|
if (WHISPER_VITISAI)
|
||||||
|
target_link_libraries(whisper PRIVATE whisper.vitisai)
|
||||||
|
endif()
|
||||||
|
|
||||||
if (WHISPER_MKL)
|
if (WHISPER_MKL)
|
||||||
target_link_libraries(whisper PRIVATE MKL::MKL)
|
target_link_libraries(whisper PRIVATE MKL::MKL)
|
||||||
endif()
|
endif()
|
||||||
|
|||||||
@@ -0,0 +1,571 @@
|
|||||||
|
#ifdef _WIN32
|
||||||
|
#ifndef NOMINMAX
|
||||||
|
#define NOMINMAX
|
||||||
|
#endif
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#include "vitisai/whisper-vitisai-encoder.h"
|
||||||
|
#include "vitisai/whisper-vitisai-helpers.h"
|
||||||
|
#include "FlexMLClient.h"
|
||||||
|
#include "ggml.h"
|
||||||
|
|
||||||
|
#include <cstdio>
|
||||||
|
#include <cstdlib>
|
||||||
|
#include <cmath>
|
||||||
|
#include <memory>
|
||||||
|
#include <string>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
#if defined(WHISPER_DEBUG)
|
||||||
|
#define WHISPER_DBG_TIMER(name) const int64_t name = ggml_time_us()
|
||||||
|
#else
|
||||||
|
#define WHISPER_DBG_TIMER(name) do {} while (0)
|
||||||
|
#endif
|
||||||
|
|
||||||
|
struct whisper_vitisai_context {
|
||||||
|
std::string model_path;
|
||||||
|
std::shared_ptr<flexmlrt::client::Model> runner;
|
||||||
|
uint8_t * fbs_buffer = nullptr;
|
||||||
|
size_t fbs_buffer_size = 0;
|
||||||
|
|
||||||
|
std::vector<float> cross_k_staging;
|
||||||
|
std::vector<float> cross_v_staging;
|
||||||
|
|
||||||
|
int mel_in_idx = -1;
|
||||||
|
int embd_enc_out_idx = -1;
|
||||||
|
int cross_k_out_idx = -1;
|
||||||
|
int cross_v_out_idx = -1;
|
||||||
|
size_t mel_in_expected_bytes = 0;
|
||||||
|
size_t embd_enc_expected_bytes = 0;
|
||||||
|
size_t cross_k_expected_bytes = 0;
|
||||||
|
size_t cross_v_expected_bytes = 0;
|
||||||
|
|
||||||
|
std::vector<flexmlrt::client::ErtTensorType> cached_input_tensors;
|
||||||
|
std::vector<flexmlrt::client::ErtTensorType> cached_output_tensors;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Return cached IO tensor descriptors by reference to avoid per-call deep copies.
|
||||||
|
static bool whisper_vitisai_get_cached_io_tensors(
|
||||||
|
struct whisper_vitisai_context * ctx,
|
||||||
|
std::vector<flexmlrt::client::ErtTensorType> *& input_tensors,
|
||||||
|
std::vector<flexmlrt::client::ErtTensorType> *& output_tensors) {
|
||||||
|
if (!ctx || !ctx->runner) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ctx->cached_input_tensors.empty() || ctx->cached_output_tensors.empty()) {
|
||||||
|
ctx->cached_input_tensors = ctx->runner->getIOTensors("input", false);
|
||||||
|
ctx->cached_output_tensors = ctx->runner->getIOTensors("output", false);
|
||||||
|
}
|
||||||
|
|
||||||
|
input_tensors = &ctx->cached_input_tensors;
|
||||||
|
output_tensors = &ctx->cached_output_tensors;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model) {
|
||||||
|
if (!path_model) {
|
||||||
|
std::fprintf(stderr, "%s: path_model is null\n", __func__);
|
||||||
|
return nullptr;
|
||||||
|
}
|
||||||
|
|
||||||
|
auto * ctx = new whisper_vitisai_context;
|
||||||
|
ctx->model_path = path_model;
|
||||||
|
|
||||||
|
// Override the model path with the environment variable if it is set
|
||||||
|
if (const char * env_model_path = std::getenv("OVERRIDE_VITISAI_MODEL_PATH")) {
|
||||||
|
if (env_model_path[0] != '\0') {
|
||||||
|
ctx->model_path = env_model_path;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 1: Set up the model
|
||||||
|
flexmlrt::client::Options options;
|
||||||
|
options.modelPath = ctx->model_path;
|
||||||
|
options.debug = false;
|
||||||
|
options.executeMode = 2;
|
||||||
|
options.extOptions["enable_preemption"] = true;
|
||||||
|
|
||||||
|
const bool model_is_rai = ctx->model_path.find(".rai") != std::string::npos;
|
||||||
|
|
||||||
|
// Check if model_path is rai file and if so, add fbs_buffer and fbs_buffer_size to the options
|
||||||
|
if (model_is_rai) {
|
||||||
|
if (whisper_vitisai_helpers::map_rai_file(ctx->model_path.c_str(), &ctx->fbs_buffer, &ctx->fbs_buffer_size)) {
|
||||||
|
options.extOptions["fbs_buffer"] = ctx->fbs_buffer;
|
||||||
|
options.extOptions["fbs_buffer_size"] = ctx->fbs_buffer_size;
|
||||||
|
options.extOptions["cache_dir"] = std::string(".");
|
||||||
|
} else {
|
||||||
|
std::fprintf(stderr, "%s: Failed to mmap rai file '%s'\n", __func__, ctx->model_path.c_str());
|
||||||
|
delete ctx;
|
||||||
|
return nullptr;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
options.deviceName = "stx";
|
||||||
|
#if defined(WHISPER_DEBUG)
|
||||||
|
std::fprintf(stderr, "%s: Using default device name 'stx'\n", __func__);
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
if (model_is_rai) {
|
||||||
|
#if WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES
|
||||||
|
options.deviceName = "stx";
|
||||||
|
options.subgraphName = "vaiml_par_0";
|
||||||
|
#if defined(WHISPER_DEBUG)
|
||||||
|
std::fprintf(stderr,
|
||||||
|
"%s: legacy FlexMLRT compile configuration detected; applying RAI overrides (device='stx', subgraph='vaiml_par_0')\n",
|
||||||
|
__func__);
|
||||||
|
#endif // defined(WHISPER_DEBUG)
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
ctx->runner = std::make_shared<flexmlrt::client::Model>(options);
|
||||||
|
if (!ctx->runner || !ctx->runner->good()) {
|
||||||
|
throw std::runtime_error("Runner creation ran into an error");
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx->cached_input_tensors = ctx->runner->getIOTensors("input", false);
|
||||||
|
ctx->cached_output_tensors = ctx->runner->getIOTensors("output", false);
|
||||||
|
|
||||||
|
auto & input_tensors = ctx->cached_input_tensors;
|
||||||
|
auto & output_tensors = ctx->cached_output_tensors;
|
||||||
|
|
||||||
|
whisper_vitisai_helpers::whisper_vitisai_io_binding binding;
|
||||||
|
std::string binding_error;
|
||||||
|
if (!whisper_vitisai_helpers::whisper_vitisai_resolve_io_binding(
|
||||||
|
__func__, input_tensors, output_tensors, &binding, &binding_error)) {
|
||||||
|
throw std::runtime_error(binding_error);
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx->mel_in_idx = binding.mel_in_idx;
|
||||||
|
ctx->embd_enc_out_idx = binding.embd_enc_out_idx;
|
||||||
|
ctx->cross_k_out_idx = binding.cross_k_out_idx;
|
||||||
|
ctx->cross_v_out_idx = binding.cross_v_out_idx;
|
||||||
|
ctx->mel_in_expected_bytes = binding.mel_in_expected_bytes;
|
||||||
|
ctx->embd_enc_expected_bytes = binding.embd_enc_expected_bytes;
|
||||||
|
ctx->cross_k_expected_bytes = binding.cross_k_expected_bytes;
|
||||||
|
ctx->cross_v_expected_bytes = binding.cross_v_expected_bytes;
|
||||||
|
|
||||||
|
#if defined(WHISPER_DEBUG)
|
||||||
|
{
|
||||||
|
std::fprintf(stderr, "%s: model has %zu input tensor(s)\n", __func__, input_tensors.size());
|
||||||
|
for (int i = 0; i < (int) input_tensors.size(); ++i) {
|
||||||
|
const auto & meta = input_tensors[i].getMetadata();
|
||||||
|
std::fprintf(stderr, "%s: input[%d] name='%s' size=%zu shape=",
|
||||||
|
__func__, i, meta.name.c_str(), (size_t) meta.size);
|
||||||
|
whisper_vitisai_helpers::whisper_vitisai_print_shape(meta.shape);
|
||||||
|
std::fprintf(stderr, "\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
std::fprintf(stderr, "%s: model has %zu output tensor(s)\n", __func__, output_tensors.size());
|
||||||
|
for (int i = 0; i < (int) output_tensors.size(); ++i) {
|
||||||
|
const auto & meta = output_tensors[i].getMetadata();
|
||||||
|
std::fprintf(stderr, "%s: output[%d] name='%s' size=%zu shape=",
|
||||||
|
__func__, i, meta.name.c_str(), (size_t) meta.size);
|
||||||
|
whisper_vitisai_helpers::whisper_vitisai_print_shape(meta.shape);
|
||||||
|
std::fprintf(stderr, "\n");
|
||||||
|
}
|
||||||
|
|
||||||
|
std::fprintf(stderr, "%s: input index: mel=%d\n", __func__, ctx->mel_in_idx);
|
||||||
|
std::fprintf(stderr, "%s: output indices: embd_enc=%d cross_k=%d cross_v=%d\n",
|
||||||
|
__func__, ctx->embd_enc_out_idx, ctx->cross_k_out_idx, ctx->cross_v_out_idx);
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
} catch (const std::exception & e) {
|
||||||
|
std::fprintf(stderr, "%s: Exception during Vitis AI runner creation: %s\n", __func__, e.what());
|
||||||
|
whisper_vitisai_free(ctx);
|
||||||
|
return nullptr;
|
||||||
|
}
|
||||||
|
return ctx;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool whisper_vitisai_has_cross_proj(const struct whisper_vitisai_context * ctx) {
|
||||||
|
return ctx && ctx->cross_k_out_idx >= 0 && ctx->cross_v_out_idx >= 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
void whisper_vitisai_free(struct whisper_vitisai_context * ctx) {
|
||||||
|
if (!ctx) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
#if defined(WHISPER_DEBUG)
|
||||||
|
std::fprintf(stderr, "%s: releasing Vitis AI context for model '%s'\n", __func__, ctx->model_path.c_str());
|
||||||
|
#endif
|
||||||
|
if (ctx->fbs_buffer) {
|
||||||
|
whisper_vitisai_helpers::unmap_rai_file(ctx->fbs_buffer, ctx->fbs_buffer_size);
|
||||||
|
}
|
||||||
|
delete ctx;
|
||||||
|
}
|
||||||
|
|
||||||
|
static int whisper_vitisai_forward_impl(
|
||||||
|
struct whisper_vitisai_context * ctx,
|
||||||
|
struct ggml_tensor * mel,
|
||||||
|
struct ggml_tensor * out,
|
||||||
|
std::vector<flexmlrt::client::ErtTensorType> & input_tensors,
|
||||||
|
std::vector<flexmlrt::client::ErtTensorType> & output_tensors,
|
||||||
|
void * cross_k_data,
|
||||||
|
void * cross_v_data) {
|
||||||
|
if (!ctx || !mel || !out) {
|
||||||
|
std::fprintf(stderr, "%s: ctx/mel/out must not be null\n", __func__);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
const bool with_cross = (cross_k_data != nullptr || cross_v_data != nullptr);
|
||||||
|
if (with_cross && (!cross_k_data || !cross_v_data)) {
|
||||||
|
std::fprintf(stderr, "%s: cross_k_data/cross_v_data must both be set\n", __func__);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ggml_n_dims(mel) != 2) {
|
||||||
|
std::fprintf(stderr, "%s: mel tensor expected to have 2 dims, got %d\n", __func__, ggml_n_dims(mel));
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ggml_n_dims(out) != 2) {
|
||||||
|
std::fprintf(stderr, "%s: out tensor expected to have 2 dims, got %d\n", __func__, ggml_n_dims(out));
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ctx->embd_enc_out_idx < 0 || ctx->embd_enc_out_idx >= (int) output_tensors.size()) {
|
||||||
|
std::fprintf(stderr, "%s: invalid embd_enc output index %d for %zu output tensor(s)\n",
|
||||||
|
__func__, ctx->embd_enc_out_idx, output_tensors.size());
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ctx->mel_in_idx < 0 || ctx->mel_in_idx >= (int) input_tensors.size()) {
|
||||||
|
std::fprintf(stderr, "%s: invalid mel input index %d for %zu input tensor(s)\n",
|
||||||
|
__func__, ctx->mel_in_idx, input_tensors.size());
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!whisper_vitisai_helpers::whisper_vitisai_bind_tensor_data(
|
||||||
|
"mel input",
|
||||||
|
mel,
|
||||||
|
{ (size_t) mel->ne[1], (size_t) mel->ne[0] },
|
||||||
|
input_tensors[ctx->mel_in_idx])) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!whisper_vitisai_helpers::whisper_vitisai_bind_tensor_data(
|
||||||
|
"embd_enc output",
|
||||||
|
out,
|
||||||
|
{ (size_t) out->ne[1], (size_t) out->ne[0] },
|
||||||
|
output_tensors[ctx->embd_enc_out_idx])) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<bool> claimed_inputs(input_tensors.size(), false);
|
||||||
|
claimed_inputs[ctx->mel_in_idx] = true;
|
||||||
|
if (!whisper_vitisai_helpers::whisper_vitisai_all_tensors_claimed(
|
||||||
|
__func__, "input", input_tensors, claimed_inputs)) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<bool> claimed_outputs(output_tensors.size(), false);
|
||||||
|
claimed_outputs[ctx->embd_enc_out_idx] = true;
|
||||||
|
if (with_cross) {
|
||||||
|
if (ctx->cross_k_out_idx < 0 || ctx->cross_k_out_idx >= (int) output_tensors.size() ||
|
||||||
|
ctx->cross_v_out_idx < 0 || ctx->cross_v_out_idx >= (int) output_tensors.size()) {
|
||||||
|
std::fprintf(stderr, "%s: invalid cross output indices cross_k=%d cross_v=%d for %zu output tensor(s)\n",
|
||||||
|
__func__, ctx->cross_k_out_idx, ctx->cross_v_out_idx, output_tensors.size());
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
output_tensors[ctx->cross_k_out_idx].data = cross_k_data;
|
||||||
|
output_tensors[ctx->cross_v_out_idx].data = cross_v_data;
|
||||||
|
claimed_outputs[ctx->cross_k_out_idx] = true;
|
||||||
|
claimed_outputs[ctx->cross_v_out_idx] = true;
|
||||||
|
}
|
||||||
|
if (!whisper_vitisai_helpers::whisper_vitisai_all_tensors_claimed(
|
||||||
|
__func__, "output", output_tensors, claimed_outputs)) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
auto clear_bound_data = [&]() {
|
||||||
|
input_tensors[ctx->mel_in_idx].data = nullptr;
|
||||||
|
output_tensors[ctx->embd_enc_out_idx].data = nullptr;
|
||||||
|
if (with_cross) {
|
||||||
|
output_tensors[ctx->cross_k_out_idx].data = nullptr;
|
||||||
|
output_tensors[ctx->cross_v_out_idx].data = nullptr;
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
try {
|
||||||
|
ctx->runner->forward(input_tensors, output_tensors);
|
||||||
|
clear_bound_data();
|
||||||
|
#if defined(WHISPER_DEBUG)
|
||||||
|
std::fprintf(stderr, "%s: Vitis AI model inference %scompleted.\n",
|
||||||
|
__func__, with_cross ? "(encoder + cross proj) " : "");
|
||||||
|
#endif
|
||||||
|
} catch (const std::exception & e) {
|
||||||
|
clear_bound_data();
|
||||||
|
std::fprintf(stderr, "%s: Exception during model inference: %s\n", __func__, e.what());
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
int whisper_vitisai_encode(struct whisper_vitisai_context * ctx, struct ggml_tensor * mel, struct ggml_tensor * out) {
|
||||||
|
std::vector<flexmlrt::client::ErtTensorType> * input_tensors_cached = nullptr;
|
||||||
|
std::vector<flexmlrt::client::ErtTensorType> * output_tensors_cached = nullptr;
|
||||||
|
if (!whisper_vitisai_get_cached_io_tensors(ctx, input_tensors_cached, output_tensors_cached)) {
|
||||||
|
std::fprintf(stderr, "%s: failed to acquire Vitis AI I/O tensors\n", __func__);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
std::vector<flexmlrt::client::ErtTensorType> input_tensors = *input_tensors_cached;
|
||||||
|
std::vector<flexmlrt::client::ErtTensorType> output_tensors = *output_tensors_cached;
|
||||||
|
|
||||||
|
return whisper_vitisai_forward_impl(
|
||||||
|
ctx,
|
||||||
|
mel,
|
||||||
|
out,
|
||||||
|
input_tensors,
|
||||||
|
output_tensors,
|
||||||
|
nullptr,
|
||||||
|
nullptr);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ensure persistent staging buffers are large enough for the given dimensions.
|
||||||
|
static void ensure_staging_buffers(
|
||||||
|
struct whisper_vitisai_context * ctx,
|
||||||
|
size_t count,
|
||||||
|
bool need_k,
|
||||||
|
bool need_v) {
|
||||||
|
if (need_k && ctx->cross_k_staging.size() < count) {
|
||||||
|
ctx->cross_k_staging.resize(count);
|
||||||
|
}
|
||||||
|
if (need_v && ctx->cross_v_staging.size() < count) {
|
||||||
|
ctx->cross_v_staging.resize(count);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
int whisper_vitisai_encode_with_cross(
|
||||||
|
struct whisper_vitisai_context * ctx,
|
||||||
|
struct ggml_tensor * mel,
|
||||||
|
struct ggml_tensor * embd_enc,
|
||||||
|
struct ggml_tensor * kv_cross_k,
|
||||||
|
struct ggml_tensor * kv_cross_v,
|
||||||
|
int n_text_layer,
|
||||||
|
int n_ctx,
|
||||||
|
int n_text_state,
|
||||||
|
int n_text_head,
|
||||||
|
bool flash_attn) {
|
||||||
|
if (!ctx || !mel || !embd_enc || !kv_cross_k || !kv_cross_v) {
|
||||||
|
std::fprintf(stderr, "%s: ctx/mel/embd_enc/kv_cross_k/kv_cross_v must not be null\n", __func__);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (n_text_layer <= 0 || n_ctx <= 0 || n_text_state <= 0 || n_text_head <= 0) {
|
||||||
|
std::fprintf(stderr, "%s: invalid shape parameters layer=%d ctx=%d state=%d head=%d\n",
|
||||||
|
__func__, n_text_layer, n_ctx, n_text_state, n_text_head);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if ((n_text_state % n_text_head) != 0) {
|
||||||
|
std::fprintf(stderr, "%s: invalid head configuration state=%d head=%d\n",
|
||||||
|
__func__, n_text_state, n_text_head);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (kv_cross_k->type != kv_cross_v->type) {
|
||||||
|
std::fprintf(stderr, "%s: kv_cross type mismatch k=%s v=%s\n",
|
||||||
|
__func__,
|
||||||
|
whisper_vitisai_helpers::whisper_kv_type_name(kv_cross_k->type),
|
||||||
|
whisper_vitisai_helpers::whisper_kv_type_name(kv_cross_v->type));
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
const int n_state = n_text_state;
|
||||||
|
const int n_state_head = n_state / n_text_head;
|
||||||
|
const int n_ctx_pad = (n_ctx + 255) & ~255; // GGML_PAD(n_ctx, 256)
|
||||||
|
|
||||||
|
const float Kscale = pow(float(n_state_head), -0.25f);
|
||||||
|
const ggml_type kv_type = kv_cross_k->type;
|
||||||
|
const bool kv_is_f32 = kv_type == GGML_TYPE_F32;
|
||||||
|
const bool kv_is_f16 = kv_type == GGML_TYPE_F16;
|
||||||
|
if (!kv_is_f32 && !kv_is_f16) {
|
||||||
|
std::fprintf(stderr, "%s: unsupported kv_cross tensor type '%s'\n",
|
||||||
|
__func__, whisper_vitisai_helpers::whisper_kv_type_name(kv_type));
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
const size_t elem_size = ggml_type_size(kv_type);
|
||||||
|
const size_t req_layer_elems = (size_t)n_ctx * (size_t)n_state;
|
||||||
|
|
||||||
|
std::vector<flexmlrt::client::ErtTensorType> * input_tensors_cached = nullptr;
|
||||||
|
std::vector<flexmlrt::client::ErtTensorType> * output_tensors_cached = nullptr;
|
||||||
|
if (!whisper_vitisai_get_cached_io_tensors(ctx, input_tensors_cached, output_tensors_cached)) {
|
||||||
|
std::fprintf(stderr, "%s: failed to acquire Vitis AI I/O tensors\n", __func__);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
std::vector<flexmlrt::client::ErtTensorType> input_tensors = *input_tensors_cached;
|
||||||
|
std::vector<flexmlrt::client::ErtTensorType> output_tensors = *output_tensors_cached;
|
||||||
|
|
||||||
|
if (ctx->cross_k_out_idx < 0 || ctx->cross_k_out_idx >= (int) output_tensors.size() ||
|
||||||
|
ctx->cross_v_out_idx < 0 || ctx->cross_v_out_idx >= (int) output_tensors.size()) {
|
||||||
|
std::fprintf(stderr, "%s: invalid cross output indices cross_k=%d cross_v=%d for %zu output tensor(s)\n",
|
||||||
|
__func__, ctx->cross_k_out_idx, ctx->cross_v_out_idx, output_tensors.size());
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
const auto & cross_k_meta = output_tensors[ctx->cross_k_out_idx].getMetadata();
|
||||||
|
const auto & cross_v_meta = output_tensors[ctx->cross_v_out_idx].getMetadata();
|
||||||
|
if (!whisper_vitisai_helpers::whisper_validate_cross_shape("cross_k", cross_k_meta.shape, n_text_layer, n_ctx, n_state) ||
|
||||||
|
!whisper_vitisai_helpers::whisper_validate_cross_shape("cross_v", cross_v_meta.shape, n_text_layer, n_ctx, n_state)) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (ctx->cross_k_expected_bytes == 0 || ctx->cross_v_expected_bytes == 0) {
|
||||||
|
std::fprintf(stderr, "%s: missing cross output metadata sizes\n", __func__);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
if (ctx->cross_k_expected_bytes != ctx->cross_v_expected_bytes) {
|
||||||
|
std::fprintf(stderr, "%s: cross output metadata size mismatch k=%zu v=%zu\n",
|
||||||
|
__func__, ctx->cross_k_expected_bytes, ctx->cross_v_expected_bytes);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
const size_t expected_cross_bytes = (size_t) n_text_layer * req_layer_elems * sizeof(float);
|
||||||
|
if (ctx->cross_k_expected_bytes != expected_cross_bytes) {
|
||||||
|
std::fprintf(stderr,
|
||||||
|
"%s: cross output size mismatch (model=%zu B, expected=%zu B for layer=%d ctx=%d state=%d)\n",
|
||||||
|
__func__, ctx->cross_k_expected_bytes, expected_cross_bytes, n_text_layer, n_ctx, n_state);
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
const size_t model_total_elems = ctx->cross_k_expected_bytes / sizeof(float);
|
||||||
|
const size_t model_layer_elems = req_layer_elems;
|
||||||
|
|
||||||
|
const size_t required_kv_bytes = flash_attn
|
||||||
|
? (size_t)n_text_layer * elem_size * (size_t)n_state * (size_t)n_ctx_pad
|
||||||
|
: (size_t)n_text_layer * elem_size * req_layer_elems;
|
||||||
|
if (ggml_nbytes(kv_cross_k) < required_kv_bytes || ggml_nbytes(kv_cross_v) < required_kv_bytes) {
|
||||||
|
std::fprintf(stderr,
|
||||||
|
"%s: kv_cross buffers are too small (required=%zu B, k=%zu B, v=%zu B)\n",
|
||||||
|
__func__, required_kv_bytes, ggml_nbytes(kv_cross_k), ggml_nbytes(kv_cross_v));
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
const bool direct_k_to_kv = kv_is_f32 && (!flash_attn || n_ctx_pad == n_ctx);
|
||||||
|
const bool direct_v_to_kv = kv_is_f32 && flash_attn && (n_ctx_pad == n_ctx);
|
||||||
|
const bool need_k_staging = !direct_k_to_kv;
|
||||||
|
const bool need_v_staging = !direct_v_to_kv;
|
||||||
|
|
||||||
|
if (need_k_staging || need_v_staging) {
|
||||||
|
ensure_staging_buffers(ctx, model_total_elems, need_k_staging, need_v_staging);
|
||||||
|
}
|
||||||
|
|
||||||
|
void * cross_k_out = direct_k_to_kv
|
||||||
|
? kv_cross_k->data
|
||||||
|
: (void *) ctx->cross_k_staging.data();
|
||||||
|
void * cross_v_out = direct_v_to_kv
|
||||||
|
? kv_cross_v->data
|
||||||
|
: (void *) ctx->cross_v_staging.data();
|
||||||
|
|
||||||
|
whisper_vitisai_helpers::whisper_kv_cross_layout kv_layout;
|
||||||
|
kv_layout.n_layer = n_text_layer;
|
||||||
|
kv_layout.n_ctx = n_ctx;
|
||||||
|
kv_layout.n_state = n_state;
|
||||||
|
kv_layout.src_layer_elems = model_layer_elems;
|
||||||
|
kv_layout.layer_elems = req_layer_elems;
|
||||||
|
kv_layout.kscale = Kscale;
|
||||||
|
|
||||||
|
if (flash_attn) {
|
||||||
|
WHISPER_DBG_TIMER(t_fwd_start);
|
||||||
|
if (!whisper_vitisai_forward_impl(
|
||||||
|
ctx, mel, embd_enc, input_tensors, output_tensors, cross_k_out, cross_v_out)) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
WHISPER_DBG_TIMER(t_fwd_end);
|
||||||
|
WHISPER_DBG_TIMER(t_post_start);
|
||||||
|
|
||||||
|
if (n_ctx_pad == n_ctx) {
|
||||||
|
kv_layout.dst_layer_stride = req_layer_elems * elem_size;
|
||||||
|
if (kv_is_f32) {
|
||||||
|
// V was written straight into the kv cache by the runtime; only K needs scaling.
|
||||||
|
whisper_vitisai_helpers::whisper_kv_cross_scale_k_f32(
|
||||||
|
(float *)kv_cross_k->data,
|
||||||
|
(size_t) n_text_layer * req_layer_elems,
|
||||||
|
Kscale);
|
||||||
|
} else { // kv_is_f16
|
||||||
|
whisper_vitisai_helpers::whisper_kv_cross_store_layers_f16(
|
||||||
|
ctx->cross_k_staging.data(),
|
||||||
|
ctx->cross_v_staging.data(),
|
||||||
|
(uint8_t *)kv_cross_k->data,
|
||||||
|
(uint8_t *)kv_cross_v->data,
|
||||||
|
kv_layout);
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// Runtime decoder uses padded K/V cache. Copy only requested context, leave the pad tail untouched.
|
||||||
|
kv_layout.dst_layer_stride = elem_size * (size_t)n_state * (size_t)n_ctx_pad;
|
||||||
|
if (kv_is_f32) {
|
||||||
|
whisper_vitisai_helpers::whisper_kv_cross_store_layers_f32(
|
||||||
|
ctx->cross_k_staging.data(),
|
||||||
|
ctx->cross_v_staging.data(),
|
||||||
|
(uint8_t *)kv_cross_k->data,
|
||||||
|
(uint8_t *)kv_cross_v->data,
|
||||||
|
kv_layout);
|
||||||
|
} else { // kv_is_f16
|
||||||
|
whisper_vitisai_helpers::whisper_kv_cross_store_layers_f16(
|
||||||
|
ctx->cross_k_staging.data(),
|
||||||
|
ctx->cross_v_staging.data(),
|
||||||
|
(uint8_t *)kv_cross_k->data,
|
||||||
|
(uint8_t *)kv_cross_v->data,
|
||||||
|
kv_layout);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
WHISPER_DBG_TIMER(t_post_end);
|
||||||
|
|
||||||
|
#if defined(WHISPER_DEBUG)
|
||||||
|
const size_t model_ctx = (size_t) n_ctx;
|
||||||
|
std::fprintf(stderr, "%s: vitisai enc+cross forward time = %8.2f ms\n", __func__, (t_fwd_end - t_fwd_start) / 1000.0f);
|
||||||
|
std::fprintf(stderr, "%s: kv_cross post-process time = %8.2f ms (flash, req_ctx=%d, model_ctx=%zu, req_ctx_pad=%d, kv_type=%s)\n",
|
||||||
|
__func__, (t_post_end - t_post_start) / 1000.0f, n_ctx, model_ctx, n_ctx_pad,
|
||||||
|
whisper_vitisai_helpers::whisper_kv_type_name(kv_type));
|
||||||
|
#endif
|
||||||
|
} else {
|
||||||
|
// Non-flash: model outputs contiguous [ctx, state] per layer.
|
||||||
|
WHISPER_DBG_TIMER(t_fwd_start);
|
||||||
|
if (!whisper_vitisai_forward_impl(
|
||||||
|
ctx, mel, embd_enc, input_tensors, output_tensors, cross_k_out, cross_v_out)) {
|
||||||
|
return 0;
|
||||||
|
}
|
||||||
|
WHISPER_DBG_TIMER(t_fwd_end);
|
||||||
|
WHISPER_DBG_TIMER(t_post_start);
|
||||||
|
|
||||||
|
kv_layout.dst_layer_stride = elem_size * (size_t)n_state * (size_t)n_ctx;
|
||||||
|
if (kv_is_f32) {
|
||||||
|
// K was written straight into the kv cache by the runtime and is scaled there.
|
||||||
|
whisper_vitisai_helpers::whisper_kv_cross_scale_k_f32(
|
||||||
|
(float *)kv_cross_k->data,
|
||||||
|
(size_t) n_text_layer * req_layer_elems,
|
||||||
|
Kscale);
|
||||||
|
|
||||||
|
whisper_vitisai_helpers::whisper_kv_cross_transpose_v_layers_f32(
|
||||||
|
ctx->cross_v_staging.data(),
|
||||||
|
(uint8_t *)kv_cross_v->data,
|
||||||
|
kv_layout);
|
||||||
|
} else { // kv_is_f16
|
||||||
|
whisper_vitisai_helpers::whisper_kv_cross_store_k_transpose_v_layers_f16(
|
||||||
|
ctx->cross_k_staging.data(),
|
||||||
|
ctx->cross_v_staging.data(),
|
||||||
|
(uint8_t *)kv_cross_k->data,
|
||||||
|
(uint8_t *)kv_cross_v->data,
|
||||||
|
kv_layout);
|
||||||
|
}
|
||||||
|
|
||||||
|
WHISPER_DBG_TIMER(t_post_end);
|
||||||
|
|
||||||
|
#if defined(WHISPER_DEBUG)
|
||||||
|
const size_t model_ctx = (size_t) n_ctx;
|
||||||
|
std::fprintf(stderr, "%s: vitisai enc+cross forward time = %8.2f ms\n", __func__, (t_fwd_end - t_fwd_start) / 1000.0f);
|
||||||
|
std::fprintf(stderr, "%s: kv_cross post-process time = %8.2f ms (non-flash, req_ctx=%d, model_ctx=%zu, kv_type=%s)\n",
|
||||||
|
__func__, (t_post_end - t_post_start) / 1000.0f, n_ctx, model_ctx,
|
||||||
|
whisper_vitisai_helpers::whisper_kv_type_name(kv_type));
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
return 1;
|
||||||
|
}
|
||||||
@@ -0,0 +1,36 @@
|
|||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include <cstdbool>
|
||||||
|
|
||||||
|
#if __cplusplus
|
||||||
|
extern "C" {
|
||||||
|
#endif
|
||||||
|
|
||||||
|
struct whisper_vitisai_context;
|
||||||
|
|
||||||
|
struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model);
|
||||||
|
void whisper_vitisai_free(struct whisper_vitisai_context * ctx);
|
||||||
|
bool whisper_vitisai_has_cross_proj(const struct whisper_vitisai_context * ctx);
|
||||||
|
|
||||||
|
struct ggml_tensor;
|
||||||
|
|
||||||
|
int whisper_vitisai_encode(
|
||||||
|
struct whisper_vitisai_context * ctx,
|
||||||
|
struct ggml_tensor * mel,
|
||||||
|
struct ggml_tensor * out);
|
||||||
|
|
||||||
|
int whisper_vitisai_encode_with_cross(
|
||||||
|
struct whisper_vitisai_context * ctx,
|
||||||
|
struct ggml_tensor * mel,
|
||||||
|
struct ggml_tensor * embd_enc,
|
||||||
|
struct ggml_tensor * kv_cross_k,
|
||||||
|
struct ggml_tensor * kv_cross_v,
|
||||||
|
int n_text_layer,
|
||||||
|
int n_ctx,
|
||||||
|
int n_text_state,
|
||||||
|
int n_text_head,
|
||||||
|
bool flash_attn);
|
||||||
|
|
||||||
|
#if __cplusplus
|
||||||
|
}
|
||||||
|
#endif
|
||||||
@@ -0,0 +1,472 @@
|
|||||||
|
#ifdef _WIN32
|
||||||
|
#ifndef NOMINMAX
|
||||||
|
#define NOMINMAX
|
||||||
|
#endif
|
||||||
|
#endif
|
||||||
|
|
||||||
|
#include "vitisai/whisper-vitisai-helpers.h"
|
||||||
|
|
||||||
|
#include <algorithm>
|
||||||
|
#include <cstdio>
|
||||||
|
#ifdef _WIN32
|
||||||
|
#include <windows.h>
|
||||||
|
#else
|
||||||
|
#include <sys/mman.h>
|
||||||
|
#include <sys/stat.h>
|
||||||
|
#endif
|
||||||
|
#include <string>
|
||||||
|
#include <utility>
|
||||||
|
|
||||||
|
namespace whisper_vitisai_helpers {
|
||||||
|
|
||||||
|
bool map_rai_file(const char * path, uint8_t ** buffer, size_t * size) {
|
||||||
|
#ifdef _WIN32
|
||||||
|
HANDLE hFile = CreateFileA(path, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
|
||||||
|
if (hFile == INVALID_HANDLE_VALUE) {
|
||||||
|
std::fprintf(stderr, "%s: %d: Failed to open rai file '%s'\n", __func__, __LINE__, path);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
LARGE_INTEGER fileSize;
|
||||||
|
if (!GetFileSizeEx(hFile, &fileSize)) {
|
||||||
|
CloseHandle(hFile);
|
||||||
|
std::fprintf(stderr, "%s: %d: Failed to get file size for rai file '%s'\n", __func__, __LINE__, path);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
HANDLE hMapping = CreateFileMappingA(hFile, NULL, PAGE_READONLY, 0, fileSize.QuadPart, NULL);
|
||||||
|
if (hMapping == NULL) {
|
||||||
|
CloseHandle(hFile);
|
||||||
|
std::fprintf(stderr, "%s: %d: Failed to create file mapping for rai file '%s'\n", __func__, __LINE__, path);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
*buffer = (uint8_t *) MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, fileSize.QuadPart);
|
||||||
|
if (*buffer == NULL) {
|
||||||
|
CloseHandle(hMapping);
|
||||||
|
CloseHandle(hFile);
|
||||||
|
std::fprintf(stderr, "%s: %d: Failed to map rai file '%s'\n", __func__, __LINE__, path);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
CloseHandle(hMapping);
|
||||||
|
CloseHandle(hFile);
|
||||||
|
*size = fileSize.QuadPart;
|
||||||
|
return true;
|
||||||
|
#else
|
||||||
|
FILE * fd = fopen(path, "rb");
|
||||||
|
if (!fd) {
|
||||||
|
std::fprintf(stderr, "%s: %d: Failed to open rai file '%s'\n", __func__, __LINE__, path);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
struct stat st;
|
||||||
|
if (fstat(fileno(fd), &st) == -1) {
|
||||||
|
fclose(fd);
|
||||||
|
std::fprintf(stderr, "%s: %d: Failed to get file size for rai file '%s'\n", __func__, __LINE__, path);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
*buffer = (uint8_t *) mmap(nullptr, st.st_size, PROT_READ, MAP_PRIVATE, fileno(fd), 0);
|
||||||
|
if (*buffer == MAP_FAILED) {
|
||||||
|
fclose(fd);
|
||||||
|
std::fprintf(stderr, "%s: %d: Failed to mmap rai file '%s'\n", __func__, __LINE__, path);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
fclose(fd);
|
||||||
|
*size = st.st_size;
|
||||||
|
return true;
|
||||||
|
#endif // _WIN32
|
||||||
|
}
|
||||||
|
|
||||||
|
void unmap_rai_file(uint8_t * buffer, size_t size) {
|
||||||
|
#ifdef _WIN32
|
||||||
|
UnmapViewOfFile(buffer);
|
||||||
|
#else
|
||||||
|
munmap(buffer, size);
|
||||||
|
#endif // _WIN32
|
||||||
|
}
|
||||||
|
|
||||||
|
const char * whisper_kv_type_name(ggml_type type) {
|
||||||
|
switch (type) {
|
||||||
|
case GGML_TYPE_F32: return "F32";
|
||||||
|
case GGML_TYPE_F16: return "F16";
|
||||||
|
default: return "unsupported";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const char * whisper_flexml_dtype_name(flexmlrt::client::DataType type) {
|
||||||
|
switch (type) {
|
||||||
|
case flexmlrt::client::DataType::Float32: return "Float32";
|
||||||
|
case flexmlrt::client::DataType::Int8: return "Int8";
|
||||||
|
case flexmlrt::client::DataType::UInt8: return "UInt8";
|
||||||
|
case flexmlrt::client::DataType::Int16: return "Int16";
|
||||||
|
case flexmlrt::client::DataType::UInt16: return "UInt16";
|
||||||
|
case flexmlrt::client::DataType::BFloat16: return "BFloat16";
|
||||||
|
case flexmlrt::client::DataType::Bool: return "Bool";
|
||||||
|
case flexmlrt::client::DataType::Float16: return "Float16";
|
||||||
|
case flexmlrt::client::DataType::Int32: return "Int32";
|
||||||
|
case flexmlrt::client::DataType::UInt32: return "UInt32";
|
||||||
|
default: return "Unknown";
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
bool whisper_flexml_dtype_to_ggml_type(
|
||||||
|
flexmlrt::client::DataType type,
|
||||||
|
ggml_type * ggml_dtype) {
|
||||||
|
switch (type) {
|
||||||
|
case flexmlrt::client::DataType::Float32:
|
||||||
|
if (ggml_dtype) {
|
||||||
|
*ggml_dtype = GGML_TYPE_F32;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
case flexmlrt::client::DataType::Float16:
|
||||||
|
if (ggml_dtype) {
|
||||||
|
*ggml_dtype = GGML_TYPE_F16;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
case flexmlrt::client::DataType::BFloat16:
|
||||||
|
if (ggml_dtype) {
|
||||||
|
*ggml_dtype = GGML_TYPE_BF16;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
default:
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool whisper_vitisai_validate_tensor_dtype(
|
||||||
|
const char * tensor_name,
|
||||||
|
flexmlrt::client::DataType model_dtype,
|
||||||
|
ggml_type runtime_dtype) {
|
||||||
|
ggml_type expected_runtime_dtype = GGML_TYPE_COUNT;
|
||||||
|
if (!whisper_flexml_dtype_to_ggml_type(model_dtype, &expected_runtime_dtype)) {
|
||||||
|
std::fprintf(stderr,
|
||||||
|
"%s: unsupported model dtype for %s: %s (supported: Float32/Float16/BFloat16)\n",
|
||||||
|
__func__, tensor_name, whisper_flexml_dtype_name(model_dtype));
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (runtime_dtype != expected_runtime_dtype) {
|
||||||
|
std::fprintf(stderr,
|
||||||
|
"%s: %s dtype mismatch (runtime=%s, model=%s)\n",
|
||||||
|
__func__, tensor_name, ggml_type_name(runtime_dtype), whisper_flexml_dtype_name(model_dtype));
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
static std::string whisper_shape_to_string(const std::vector<size_t> & shape) {
|
||||||
|
std::string out = "[";
|
||||||
|
for (size_t i = 0; i < shape.size(); ++i) {
|
||||||
|
if (i > 0) {
|
||||||
|
out += ", ";
|
||||||
|
}
|
||||||
|
out += std::to_string(shape[i]);
|
||||||
|
}
|
||||||
|
out += "]";
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
static std::vector<size_t> whisper_canonical_shape(const std::vector<std::uint32_t> & shape) {
|
||||||
|
std::vector<size_t> canonical;
|
||||||
|
canonical.reserve(shape.size());
|
||||||
|
for (size_t i = 0; i < shape.size(); ++i) {
|
||||||
|
const size_t dim = (size_t) shape[i];
|
||||||
|
if (dim != 1) {
|
||||||
|
canonical.push_back(dim);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (canonical.empty()) {
|
||||||
|
canonical.push_back(1);
|
||||||
|
}
|
||||||
|
return canonical;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool whisper_validate_shape(
|
||||||
|
const char * tensor_name,
|
||||||
|
const std::vector<std::uint32_t> & model_shape,
|
||||||
|
const std::vector<size_t> & expected_shape) {
|
||||||
|
const std::vector<size_t> shape = whisper_canonical_shape(model_shape);
|
||||||
|
if (shape != expected_shape) {
|
||||||
|
std::fprintf(stderr,
|
||||||
|
"%s: %s shape mismatch (runtime expected=%s, model=%s)\n",
|
||||||
|
__func__,
|
||||||
|
tensor_name,
|
||||||
|
whisper_shape_to_string(expected_shape).c_str(),
|
||||||
|
whisper_shape_to_string(shape).c_str());
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool whisper_validate_cross_shape(
|
||||||
|
const char * tensor_name,
|
||||||
|
const std::vector<std::uint32_t> & model_shape,
|
||||||
|
int n_text_layer,
|
||||||
|
int n_ctx,
|
||||||
|
int n_state) {
|
||||||
|
const std::vector<size_t> expected = {
|
||||||
|
(size_t) n_text_layer,
|
||||||
|
(size_t) n_ctx,
|
||||||
|
(size_t) n_state,
|
||||||
|
};
|
||||||
|
return whisper_validate_shape(tensor_name, model_shape, expected);
|
||||||
|
}
|
||||||
|
|
||||||
|
bool whisper_vitisai_bind_tensor_data(
|
||||||
|
const char * tensor_name,
|
||||||
|
struct ggml_tensor * runtime_tensor,
|
||||||
|
const std::vector<size_t> & expected_shape,
|
||||||
|
flexmlrt::client::ErtTensorType & io_tensor) {
|
||||||
|
const auto & meta = io_tensor.getMetadata();
|
||||||
|
if (!whisper_vitisai_validate_tensor_dtype(tensor_name, meta.type, runtime_tensor->type)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (!whisper_validate_shape(tensor_name, meta.shape, expected_shape)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
const size_t model_bytes = meta.size;
|
||||||
|
const size_t runtime_bytes = ggml_nbytes(runtime_tensor);
|
||||||
|
if (model_bytes == 0 || runtime_bytes == 0) {
|
||||||
|
std::fprintf(stderr, "%s: %s sizes must be non-zero (model=%zu, runtime=%zu)\n",
|
||||||
|
__func__, tensor_name, model_bytes, runtime_bytes);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
if (runtime_bytes != model_bytes) {
|
||||||
|
std::fprintf(stderr,
|
||||||
|
"%s: %s tensor size mismatch (runtime=%zu B, model=%zu B). "
|
||||||
|
"VitisAI .rai requires exact context match; use matching -ac/model artifact.\n",
|
||||||
|
__func__, tensor_name, runtime_bytes, model_bytes);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
io_tensor.data = runtime_tensor->data;
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool whisper_vitisai_resolve_io_binding(
|
||||||
|
[[maybe_unused]] const char * caller,
|
||||||
|
const std::vector<flexmlrt::client::ErtTensorType> & input_tensors,
|
||||||
|
const std::vector<flexmlrt::client::ErtTensorType> & output_tensors,
|
||||||
|
whisper_vitisai_io_binding * binding,
|
||||||
|
std::string * error) {
|
||||||
|
const auto fail = [error](std::string message) {
|
||||||
|
if (error) {
|
||||||
|
*error = std::move(message);
|
||||||
|
}
|
||||||
|
return false;
|
||||||
|
};
|
||||||
|
|
||||||
|
if (input_tensors.empty()) {
|
||||||
|
return fail("Model has no input tensors");
|
||||||
|
}
|
||||||
|
|
||||||
|
binding->mel_in_idx = 0;
|
||||||
|
bool found_named_mel = false;
|
||||||
|
for (int i = 0; i < (int) input_tensors.size(); ++i) {
|
||||||
|
const std::string & name = input_tensors[i].getMetadata().name;
|
||||||
|
if (name == "input" || name == "mel") {
|
||||||
|
binding->mel_in_idx = i;
|
||||||
|
found_named_mel = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!found_named_mel) {
|
||||||
|
#if defined(WHISPER_DEBUG)
|
||||||
|
std::fprintf(stderr, "%s: WARNING: mel input not found by name; falling back to input[0]\n", caller);
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
|
if (output_tensors.empty()) {
|
||||||
|
return fail("Model has no output tensors");
|
||||||
|
}
|
||||||
|
|
||||||
|
for (int i = 0; i < (int) output_tensors.size(); ++i) {
|
||||||
|
const std::string & name = output_tensors[i].getMetadata().name;
|
||||||
|
if (name == "embd_enc") {
|
||||||
|
binding->embd_enc_out_idx = i;
|
||||||
|
} else if (name == "cross_k") {
|
||||||
|
binding->cross_k_out_idx = i;
|
||||||
|
} else if (name == "cross_v") {
|
||||||
|
binding->cross_v_out_idx = i;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if (binding->embd_enc_out_idx < 0) {
|
||||||
|
#if defined(WHISPER_DEBUG)
|
||||||
|
std::fprintf(stderr, "%s: WARNING: embd_enc output not found by name; falling back to output[0]\n", caller);
|
||||||
|
#endif
|
||||||
|
binding->embd_enc_out_idx = 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
const bool has_cross_k = binding->cross_k_out_idx >= 0;
|
||||||
|
const bool has_cross_v = binding->cross_v_out_idx >= 0;
|
||||||
|
if (has_cross_k != has_cross_v) {
|
||||||
|
return fail("Incomplete cross-projection contract: both cross_k and cross_v outputs are required");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (has_cross_k && (binding->cross_k_out_idx == binding->cross_v_out_idx ||
|
||||||
|
binding->cross_k_out_idx == binding->embd_enc_out_idx ||
|
||||||
|
binding->cross_v_out_idx == binding->embd_enc_out_idx)) {
|
||||||
|
return fail("Invalid output mapping: embd_enc/cross_k/cross_v indices overlap");
|
||||||
|
}
|
||||||
|
|
||||||
|
const auto & mel_meta = input_tensors[binding->mel_in_idx].getMetadata();
|
||||||
|
if (!whisper_flexml_dtype_to_ggml_type(mel_meta.type, nullptr)) {
|
||||||
|
return fail(
|
||||||
|
std::string("Unsupported mel input type: ") +
|
||||||
|
whisper_flexml_dtype_name(mel_meta.type) + " (supported: Float32/Float16/BFloat16)");
|
||||||
|
}
|
||||||
|
binding->mel_in_expected_bytes = mel_meta.size;
|
||||||
|
|
||||||
|
const auto & embd_meta = output_tensors[binding->embd_enc_out_idx].getMetadata();
|
||||||
|
if (!whisper_flexml_dtype_to_ggml_type(embd_meta.type, nullptr)) {
|
||||||
|
return fail(
|
||||||
|
std::string("Unsupported embd_enc output type: ") +
|
||||||
|
whisper_flexml_dtype_name(embd_meta.type) + " (supported: Float32/Float16/BFloat16)");
|
||||||
|
}
|
||||||
|
binding->embd_enc_expected_bytes = embd_meta.size;
|
||||||
|
|
||||||
|
if (has_cross_k) {
|
||||||
|
const auto & cross_k_meta = output_tensors[binding->cross_k_out_idx].getMetadata();
|
||||||
|
const auto & cross_v_meta = output_tensors[binding->cross_v_out_idx].getMetadata();
|
||||||
|
if (cross_k_meta.type != flexmlrt::client::DataType::Float32 ||
|
||||||
|
cross_v_meta.type != flexmlrt::client::DataType::Float32) {
|
||||||
|
return fail(
|
||||||
|
std::string("Unsupported cross output type(s): cross_k=") +
|
||||||
|
whisper_flexml_dtype_name(cross_k_meta.type) + ", cross_v=" +
|
||||||
|
whisper_flexml_dtype_name(cross_v_meta.type) + " (cross path currently requires Float32)");
|
||||||
|
}
|
||||||
|
if (cross_k_meta.size != cross_v_meta.size) {
|
||||||
|
return fail("cross_k and cross_v output sizes do not match");
|
||||||
|
}
|
||||||
|
binding->cross_k_expected_bytes = cross_k_meta.size;
|
||||||
|
binding->cross_v_expected_bytes = cross_v_meta.size;
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool whisper_vitisai_all_tensors_claimed(
|
||||||
|
const char * caller,
|
||||||
|
const char * tensor_kind,
|
||||||
|
const std::vector<flexmlrt::client::ErtTensorType> & tensors,
|
||||||
|
const std::vector<bool> & claimed) {
|
||||||
|
for (size_t i = 0; i < tensors.size(); ++i) {
|
||||||
|
if (!claimed[i]) {
|
||||||
|
std::fprintf(stderr,
|
||||||
|
"%s: unsupported extra %s tensor at index %zu (name='%s'); strict contract expects only mapped %ss\n",
|
||||||
|
caller, tensor_kind, i, tensors[i].getMetadata().name.c_str(), tensor_kind);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
void whisper_kv_cross_scale_k_f32(
|
||||||
|
float * k_data,
|
||||||
|
size_t count,
|
||||||
|
float kscale) {
|
||||||
|
for (size_t i = 0; i < count; ++i) {
|
||||||
|
k_data[i] *= kscale;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void whisper_kv_cross_store_layers_f32(
|
||||||
|
const float * src_k,
|
||||||
|
const float * src_v,
|
||||||
|
uint8_t * dst_k,
|
||||||
|
uint8_t * dst_v,
|
||||||
|
const whisper_kv_cross_layout & layout) {
|
||||||
|
for (int il = 0; il < layout.n_layer; ++il) {
|
||||||
|
const float * layer_src_k = src_k + (size_t)il * layout.src_layer_elems;
|
||||||
|
const float * layer_src_v = src_v + (size_t)il * layout.src_layer_elems;
|
||||||
|
float * dk = (float *)(dst_k + layout.dst_layer_stride * (size_t)il);
|
||||||
|
float * dv = (float *)(dst_v + layout.dst_layer_stride * (size_t)il);
|
||||||
|
for (size_t i = 0; i < layout.layer_elems; ++i) {
|
||||||
|
dk[i] = layer_src_k[i] * layout.kscale;
|
||||||
|
dv[i] = layer_src_v[i];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void whisper_kv_cross_store_layers_f16(
|
||||||
|
const float * src_k,
|
||||||
|
const float * src_v,
|
||||||
|
uint8_t * dst_k,
|
||||||
|
uint8_t * dst_v,
|
||||||
|
const whisper_kv_cross_layout & layout) {
|
||||||
|
for (int il = 0; il < layout.n_layer; ++il) {
|
||||||
|
const float * layer_src_k = src_k + (size_t)il * layout.src_layer_elems;
|
||||||
|
const float * layer_src_v = src_v + (size_t)il * layout.src_layer_elems;
|
||||||
|
ggml_fp16_t * dk = (ggml_fp16_t *)(dst_k + layout.dst_layer_stride * (size_t)il);
|
||||||
|
ggml_fp16_t * dv = (ggml_fp16_t *)(dst_v + layout.dst_layer_stride * (size_t)il);
|
||||||
|
for (size_t i = 0; i < layout.layer_elems; ++i) {
|
||||||
|
dk[i] = ggml_fp32_to_fp16(layer_src_k[i] * layout.kscale);
|
||||||
|
dv[i] = ggml_fp32_to_fp16(layer_src_v[i]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void whisper_kv_cross_transpose_v_layers_f32(
|
||||||
|
const float * src_v,
|
||||||
|
uint8_t * dst_v,
|
||||||
|
const whisper_kv_cross_layout & layout) {
|
||||||
|
const int n_ctx = layout.n_ctx;
|
||||||
|
const int n_state = layout.n_state;
|
||||||
|
|
||||||
|
const int BLOCK = 32;
|
||||||
|
for (int il = 0; il < layout.n_layer; ++il) {
|
||||||
|
const float * layer_src_v = src_v + (size_t)il * layout.src_layer_elems;
|
||||||
|
float * dv = (float *)(dst_v + layout.dst_layer_stride * (size_t)il);
|
||||||
|
|
||||||
|
for (int ic = 0; ic < n_ctx; ic += BLOCK) {
|
||||||
|
for (int is = 0; is < n_state; is += BLOCK) {
|
||||||
|
const int ic_end = std::min(ic + BLOCK, n_ctx);
|
||||||
|
const int is_end = std::min(is + BLOCK, n_state);
|
||||||
|
for (int i = ic; i < ic_end; ++i) {
|
||||||
|
for (int j = is; j < is_end; ++j) {
|
||||||
|
dv[j * n_ctx + i] = layer_src_v[i * n_state + j];
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
void whisper_kv_cross_store_k_transpose_v_layers_f16(
|
||||||
|
const float * src_k,
|
||||||
|
const float * src_v,
|
||||||
|
uint8_t * dst_k,
|
||||||
|
uint8_t * dst_v,
|
||||||
|
const whisper_kv_cross_layout & layout) {
|
||||||
|
const int n_ctx = layout.n_ctx;
|
||||||
|
const int n_state = layout.n_state;
|
||||||
|
|
||||||
|
const int BLOCK = 32;
|
||||||
|
for (int il = 0; il < layout.n_layer; ++il) {
|
||||||
|
const float * layer_src_k = src_k + (size_t)il * layout.src_layer_elems;
|
||||||
|
const float * layer_src_v = src_v + (size_t)il * layout.src_layer_elems;
|
||||||
|
ggml_fp16_t * dk = (ggml_fp16_t *)(dst_k + layout.dst_layer_stride * (size_t)il);
|
||||||
|
ggml_fp16_t * dv = (ggml_fp16_t *)(dst_v + layout.dst_layer_stride * (size_t)il);
|
||||||
|
for (size_t i = 0; i < layout.layer_elems; ++i) {
|
||||||
|
dk[i] = ggml_fp32_to_fp16(layer_src_k[i] * layout.kscale);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (int ic = 0; ic < n_ctx; ic += BLOCK) {
|
||||||
|
for (int is = 0; is < n_state; is += BLOCK) {
|
||||||
|
const int ic_end = std::min(ic + BLOCK, n_ctx);
|
||||||
|
const int is_end = std::min(is + BLOCK, n_state);
|
||||||
|
for (int i = ic; i < ic_end; ++i) {
|
||||||
|
for (int j = is; j < is_end; ++j) {
|
||||||
|
dv[j * n_ctx + i] = ggml_fp32_to_fp16(layer_src_v[i * n_state + j]);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
} // namespace whisper_vitisai_helpers
|
||||||
@@ -0,0 +1,117 @@
|
|||||||
|
#pragma once
|
||||||
|
|
||||||
|
#include "FlexMLClient.h"
|
||||||
|
#include "ggml.h"
|
||||||
|
|
||||||
|
#include <cstddef>
|
||||||
|
#include <cstdint>
|
||||||
|
#include <cstdio>
|
||||||
|
#include <string>
|
||||||
|
#include <vector>
|
||||||
|
|
||||||
|
namespace whisper_vitisai_helpers {
|
||||||
|
|
||||||
|
bool map_rai_file(const char * path, uint8_t ** buffer, size_t * size);
|
||||||
|
void unmap_rai_file(uint8_t * buffer, size_t size);
|
||||||
|
|
||||||
|
const char * whisper_kv_type_name(ggml_type type);
|
||||||
|
const char * whisper_flexml_dtype_name(flexmlrt::client::DataType type);
|
||||||
|
bool whisper_flexml_dtype_to_ggml_type(
|
||||||
|
flexmlrt::client::DataType type,
|
||||||
|
ggml_type * ggml_dtype);
|
||||||
|
|
||||||
|
bool whisper_validate_cross_shape(
|
||||||
|
const char * tensor_name,
|
||||||
|
const std::vector<std::uint32_t> & model_shape,
|
||||||
|
int n_text_layer,
|
||||||
|
int n_ctx,
|
||||||
|
int n_state);
|
||||||
|
|
||||||
|
bool whisper_vitisai_bind_tensor_data(
|
||||||
|
const char * tensor_name,
|
||||||
|
struct ggml_tensor * runtime_tensor,
|
||||||
|
const std::vector<size_t> & expected_shape,
|
||||||
|
flexmlrt::client::ErtTensorType & io_tensor);
|
||||||
|
|
||||||
|
#if defined(WHISPER_DEBUG)
|
||||||
|
template <typename T>
|
||||||
|
void whisper_vitisai_print_shape(const std::vector<T> & shape) {
|
||||||
|
std::fprintf(stderr, "[");
|
||||||
|
for (size_t i = 0; i < shape.size(); ++i) {
|
||||||
|
std::fprintf(stderr, "%s%lld", i == 0 ? "" : ", ", (long long) shape[i]);
|
||||||
|
}
|
||||||
|
std::fprintf(stderr, "]");
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
|
// Model IO tensor indices and metadata sizes resolved once at init time.
|
||||||
|
struct whisper_vitisai_io_binding {
|
||||||
|
int mel_in_idx = -1;
|
||||||
|
int embd_enc_out_idx = -1;
|
||||||
|
int cross_k_out_idx = -1;
|
||||||
|
int cross_v_out_idx = -1;
|
||||||
|
size_t mel_in_expected_bytes = 0;
|
||||||
|
size_t embd_enc_expected_bytes = 0;
|
||||||
|
size_t cross_k_expected_bytes = 0;
|
||||||
|
size_t cross_v_expected_bytes = 0;
|
||||||
|
};
|
||||||
|
|
||||||
|
// Warnings are printed with the caller's name; hard failures are returned in *error
|
||||||
|
// so the caller can decide how to report them.
|
||||||
|
bool whisper_vitisai_resolve_io_binding(
|
||||||
|
const char * caller,
|
||||||
|
const std::vector<flexmlrt::client::ErtTensorType> & input_tensors,
|
||||||
|
const std::vector<flexmlrt::client::ErtTensorType> & output_tensors,
|
||||||
|
whisper_vitisai_io_binding * binding,
|
||||||
|
std::string * error);
|
||||||
|
|
||||||
|
bool whisper_vitisai_all_tensors_claimed(
|
||||||
|
const char * caller,
|
||||||
|
const char * tensor_kind,
|
||||||
|
const std::vector<flexmlrt::client::ErtTensorType> & tensors,
|
||||||
|
const std::vector<bool> & claimed);
|
||||||
|
|
||||||
|
// Geometry of one cross K/V transfer from the model output (always f32, contiguous
|
||||||
|
// [ctx, state] per layer) into the runtime kv cache.
|
||||||
|
struct whisper_kv_cross_layout {
|
||||||
|
int n_layer = 0;
|
||||||
|
int n_ctx = 0;
|
||||||
|
int n_state = 0;
|
||||||
|
size_t src_layer_elems = 0; // f32 elements per layer in the model output buffer
|
||||||
|
size_t layer_elems = 0; // elements per layer transferred into the kv cache
|
||||||
|
size_t dst_layer_stride = 0; // bytes per layer in the kv cache
|
||||||
|
float kscale = 1.0f;
|
||||||
|
};
|
||||||
|
|
||||||
|
void whisper_kv_cross_scale_k_f32(
|
||||||
|
float * k_data,
|
||||||
|
size_t count,
|
||||||
|
float kscale);
|
||||||
|
|
||||||
|
void whisper_kv_cross_store_layers_f32(
|
||||||
|
const float * src_k,
|
||||||
|
const float * src_v,
|
||||||
|
uint8_t * dst_k,
|
||||||
|
uint8_t * dst_v,
|
||||||
|
const whisper_kv_cross_layout & layout);
|
||||||
|
|
||||||
|
void whisper_kv_cross_store_layers_f16(
|
||||||
|
const float * src_k,
|
||||||
|
const float * src_v,
|
||||||
|
uint8_t * dst_k,
|
||||||
|
uint8_t * dst_v,
|
||||||
|
const whisper_kv_cross_layout & layout);
|
||||||
|
|
||||||
|
void whisper_kv_cross_transpose_v_layers_f32(
|
||||||
|
const float * src_v,
|
||||||
|
uint8_t * dst_v,
|
||||||
|
const whisper_kv_cross_layout & layout);
|
||||||
|
|
||||||
|
void whisper_kv_cross_store_k_transpose_v_layers_f16(
|
||||||
|
const float * src_k,
|
||||||
|
const float * src_v,
|
||||||
|
uint8_t * dst_k,
|
||||||
|
uint8_t * dst_v,
|
||||||
|
const whisper_kv_cross_layout & layout);
|
||||||
|
|
||||||
|
} // namespace whisper_vitisai_helpers
|
||||||
+88
-3
@@ -14,6 +14,10 @@
|
|||||||
#include "openvino/whisper-openvino-encoder.h"
|
#include "openvino/whisper-openvino-encoder.h"
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
#ifdef WHISPER_USE_VITISAI
|
||||||
|
#include "vitisai/whisper-vitisai-encoder.h"
|
||||||
|
#endif
|
||||||
|
|
||||||
#include <atomic>
|
#include <atomic>
|
||||||
#include <algorithm>
|
#include <algorithm>
|
||||||
#include <cassert>
|
#include <cassert>
|
||||||
@@ -903,6 +907,10 @@ struct whisper_state {
|
|||||||
whisper_openvino_context * ctx_openvino = nullptr;
|
whisper_openvino_context * ctx_openvino = nullptr;
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
#ifdef WHISPER_USE_VITISAI
|
||||||
|
whisper_vitisai_context * ctx_vitisai = nullptr;
|
||||||
|
#endif
|
||||||
|
|
||||||
// [EXPERIMENTAL] token-level timestamps data
|
// [EXPERIMENTAL] token-level timestamps data
|
||||||
int64_t t_beg = 0;
|
int64_t t_beg = 0;
|
||||||
int64_t t_last = 0;
|
int64_t t_last = 0;
|
||||||
@@ -1976,7 +1984,25 @@ static bool whisper_encode_external(const whisper_state & wstate) {
|
|||||||
const bool use_openvino = wstate.ctx_openvino != nullptr;
|
const bool use_openvino = wstate.ctx_openvino != nullptr;
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
return use_coreml || use_openvino;
|
#ifndef WHISPER_USE_VITISAI
|
||||||
|
const bool use_vitisai = false;
|
||||||
|
#else
|
||||||
|
const bool use_vitisai = wstate.ctx_vitisai != nullptr;
|
||||||
|
#endif
|
||||||
|
|
||||||
|
return use_coreml || use_openvino || use_vitisai;
|
||||||
|
}
|
||||||
|
|
||||||
|
static bool whisper_cross_external(const whisper_state & wstate) {
|
||||||
|
GGML_UNUSED(wstate);
|
||||||
|
|
||||||
|
#if defined(WHISPER_USE_VITISAI)
|
||||||
|
const bool use_vitisai_cross = whisper_vitisai_has_cross_proj(wstate.ctx_vitisai);
|
||||||
|
#else
|
||||||
|
const bool use_vitisai_cross = false;
|
||||||
|
#endif
|
||||||
|
|
||||||
|
return use_vitisai_cross;
|
||||||
}
|
}
|
||||||
|
|
||||||
static struct ggml_cgraph * whisper_build_graph_conv(
|
static struct ggml_cgraph * whisper_build_graph_conv(
|
||||||
@@ -2419,6 +2445,21 @@ static bool whisper_encode_internal(
|
|||||||
|
|
||||||
#if defined(WHISPER_USE_COREML)
|
#if defined(WHISPER_USE_COREML)
|
||||||
whisper_coreml_encode(wstate.ctx_coreml, mel->ne[0], mel->ne[1], (float *) mel->data, (float *) wstate.embd_enc->data);
|
whisper_coreml_encode(wstate.ctx_coreml, mel->ne[0], mel->ne[1], (float *) mel->data, (float *) wstate.embd_enc->data);
|
||||||
|
#elif defined(WHISPER_USE_VITISAI)
|
||||||
|
if (whisper_vitisai_has_cross_proj(wstate.ctx_vitisai)) {
|
||||||
|
const auto & hp = wctx.model.hparams;
|
||||||
|
const int n_ctx = wstate.exp_n_audio_ctx > 0
|
||||||
|
? wstate.exp_n_audio_ctx : hp.n_audio_ctx;
|
||||||
|
if (!whisper_vitisai_encode_with_cross(
|
||||||
|
wstate.ctx_vitisai, mel, wstate.embd_enc,
|
||||||
|
wstate.kv_cross.k, wstate.kv_cross.v,
|
||||||
|
hp.n_text_layer, n_ctx, hp.n_text_state,
|
||||||
|
hp.n_text_head, wctx.params.flash_attn)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
} else if (!whisper_vitisai_encode(wstate.ctx_vitisai, mel, wstate.embd_enc)) {
|
||||||
|
return false;
|
||||||
|
}
|
||||||
#elif defined(WHISPER_USE_OPENVINO)
|
#elif defined(WHISPER_USE_OPENVINO)
|
||||||
whisper_openvino_encode(wstate.ctx_openvino, mel, wstate.embd_enc);
|
whisper_openvino_encode(wstate.ctx_openvino, mel, wstate.embd_enc);
|
||||||
#endif
|
#endif
|
||||||
@@ -2442,7 +2483,7 @@ static bool whisper_encode_internal(
|
|||||||
}
|
}
|
||||||
|
|
||||||
// cross
|
// cross
|
||||||
{
|
if (!whisper_cross_external(wstate)) {
|
||||||
auto & sched = wstate.sched_cross.sched;
|
auto & sched = wstate.sched_cross.sched;
|
||||||
|
|
||||||
ggml_cgraph * gf = whisper_build_graph_cross(wctx, wstate);
|
ggml_cgraph * gf = whisper_build_graph_cross(wctx, wstate);
|
||||||
@@ -3358,6 +3399,19 @@ static std::string whisper_get_coreml_path_encoder(std::string path_bin) {
|
|||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
#ifdef WHISPER_USE_VITISAI
|
||||||
|
// replace extension with Vitis AI encoder artifact. Cross projection support is
|
||||||
|
// detected from the model's output tensors, not from the file name.
|
||||||
|
static std::string whisper_get_vitisai_path_encoder_cache(std::string path_bin) {
|
||||||
|
auto pos = path_bin.rfind('.');
|
||||||
|
if (pos != std::string::npos) {
|
||||||
|
path_bin = path_bin.substr(0, pos);
|
||||||
|
}
|
||||||
|
|
||||||
|
return path_bin + "-encoder-vitisai.rai";
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
#ifdef WHISPER_USE_OPENVINO
|
#ifdef WHISPER_USE_OPENVINO
|
||||||
// replace .bin with-encoder-openvino.xml
|
// replace .bin with-encoder-openvino.xml
|
||||||
static std::string whisper_openvino_get_path_encoder(std::string path_bin) {
|
static std::string whisper_openvino_get_path_encoder(std::string path_bin) {
|
||||||
@@ -3467,6 +3521,21 @@ struct whisper_state * whisper_init_state(whisper_context * ctx) {
|
|||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
#ifdef WHISPER_USE_VITISAI
|
||||||
|
const auto path_vitisai = whisper_get_vitisai_path_encoder_cache(ctx->path_model);
|
||||||
|
|
||||||
|
state->ctx_vitisai = whisper_vitisai_init(path_vitisai.c_str());
|
||||||
|
if (!state->ctx_vitisai) {
|
||||||
|
WHISPER_LOG_ERROR("%s: failed to load Vitis AI model from '%s'\n", __func__, path_vitisai.c_str());
|
||||||
|
whisper_free_state(state);
|
||||||
|
return nullptr;
|
||||||
|
} else if (whisper_vitisai_has_cross_proj(state->ctx_vitisai)) {
|
||||||
|
WHISPER_LOG_INFO("%s: Vitis AI encoder + cross projection model loaded\n", __func__);
|
||||||
|
} else {
|
||||||
|
WHISPER_LOG_INFO("%s: Vitis AI encoder model loaded\n", __func__);
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
state->logits.reserve(ctx->vocab.n_vocab * ctx->model.hparams.n_text_ctx);
|
state->logits.reserve(ctx->vocab.n_vocab * ctx->model.hparams.n_text_ctx);
|
||||||
|
|
||||||
state->batch = whisper_batch_init(ctx->model.hparams.n_text_ctx, WHISPER_MAX_DECODERS);
|
state->batch = whisper_batch_init(ctx->model.hparams.n_text_ctx, WHISPER_MAX_DECODERS);
|
||||||
@@ -3514,7 +3583,7 @@ struct whisper_state * whisper_init_state(whisper_context * ctx) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// cross allocator
|
// cross allocator
|
||||||
{
|
if (!whisper_cross_external(*state)) {
|
||||||
bool ok = whisper_sched_graph_init(state->sched_cross, state->backends,
|
bool ok = whisper_sched_graph_init(state->sched_cross, state->backends,
|
||||||
[&]() {
|
[&]() {
|
||||||
return whisper_build_graph_cross(*ctx, *state);
|
return whisper_build_graph_cross(*ctx, *state);
|
||||||
@@ -3847,6 +3916,13 @@ void whisper_free_state(struct whisper_state * state) {
|
|||||||
}
|
}
|
||||||
#endif
|
#endif
|
||||||
|
|
||||||
|
#ifdef WHISPER_USE_VITISAI
|
||||||
|
if (state->ctx_vitisai != nullptr) {
|
||||||
|
whisper_vitisai_free(state->ctx_vitisai);
|
||||||
|
state->ctx_vitisai = nullptr;
|
||||||
|
}
|
||||||
|
#endif
|
||||||
|
|
||||||
whisper_batch_free(state->batch);
|
whisper_batch_free(state->batch);
|
||||||
|
|
||||||
ggml_backend_sched_free(state->sched_conv.sched);
|
ggml_backend_sched_free(state->sched_conv.sched);
|
||||||
@@ -4338,11 +4414,20 @@ static int whisper_has_openvino(void) {
|
|||||||
#endif
|
#endif
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static int whisper_has_vitisai(void) {
|
||||||
|
#ifdef WHISPER_USE_VITISAI
|
||||||
|
return 1;
|
||||||
|
#else
|
||||||
|
return 0;
|
||||||
|
#endif
|
||||||
|
}
|
||||||
|
|
||||||
const char * whisper_print_system_info(void) {
|
const char * whisper_print_system_info(void) {
|
||||||
static std::string s;
|
static std::string s;
|
||||||
|
|
||||||
s = "";
|
s = "";
|
||||||
s += "WHISPER : ";
|
s += "WHISPER : ";
|
||||||
|
s += "VITISAI = " + std::to_string(whisper_has_vitisai()) + " | ";
|
||||||
s += "COREML = " + std::to_string(whisper_has_coreml()) + " | ";
|
s += "COREML = " + std::to_string(whisper_has_coreml()) + " | ";
|
||||||
s += "OPENVINO = " + std::to_string(whisper_has_openvino()) + " | ";
|
s += "OPENVINO = " + std::to_string(whisper_has_openvino()) + " | ";
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user