public_git
/
whisper.cpp
mirror of https://github.com/ggml-org/whisper.cpp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge 62c448e43e into 1fe009caed

This commit is contained in:
Sachin Kumawat 2026-08-14 12:16:10 -07:00 committed by GitHub
parent 1fe009caed 62c448e43e
commit 338c0c2601
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 2112 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

143
.github/workflows/build-self-hosted.yml vendored
View File

@ -114,3 +114,146 @@ jobs:
run: |
vulkaninfo --summary
GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/whisper.cpp ~/mnt/whisper.cpp
npu-amd-windows:
runs-on: [self-hosted, Windows, X64, stx, rai300-400]
timeout-minutes: 60
continue-on-error: true # advisory while the runner pool is new; revisit later
env:
FLEXML_URL: https://github.com/lemonade-sdk/whisper.cpp-rocm/releases/download/deps/flexmlrt-1.7.0-win.zip
MODEL: base
steps:
- name: Clone
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
- uses: microsoft/setup-msbuild@v2
- name: Install CMake if not available
shell: powershell
run: |
$installed = Get-Command cmake -ErrorAction SilentlyContinue
if (-not $installed) {
$ver = "3.28.1"
$url = "https://github.com/Kitware/CMake/releases/download/v$ver/cmake-$ver-windows-x86_64.msi"
Invoke-WebRequest -Uri $url -OutFile cmake.msi
Start-Process msiexec.exe -ArgumentList "/i cmake.msi /quiet /norestart" -Wait
$p = "C:\Program Files\CMake\bin"
$env:PATH = "$p;$env:PATH"
echo $p >> $env:GITHUB_PATH
cmake --version
if ($LASTEXITCODE -ne 0) { Write-Error "CMake install failed"; exit 1 }
} else { cmake --version }
- name: Download FlexML runtime
shell: powershell
run: |
Invoke-WebRequest -Uri "${{ env.FLEXML_URL }}" -OutFile flexmlrt.zip
if (-Not (Test-Path "flexmlrt.zip")) { Write-Error "flexmlrt.zip not downloaded"; exit 1 }
if ((Get-Item "flexmlrt.zip").Length -eq 0) { Write-Error "flexmlrt.zip is empty"; exit 1 }
tar xf flexmlrt.zip
if ($LASTEXITCODE -ne 0) { Write-Error "Extraction failed"; exit 1 }
if (-not (Test-Path "flexmlrt")) { Write-Error "No flexmlrt directory after extraction"; exit 1 }
- name: Setup FlexML, configure and build
shell: cmd
run: |
cd flexmlrt
call setup.bat
if errorlevel 1 ( echo ERROR: FlexML setup.bat failed & exit /b 1 )
cd ..
cmake -B build -A x64 -DCMAKE_BUILD_TYPE=Release -DWHISPER_VITISAI=ON
if errorlevel 1 ( echo ERROR: CMake configure failed & exit /b 1 )
cmake --build build --config Release -j
if errorlevel 1 ( echo ERROR: Build failed & exit /b 1 )
- name: Copy FlexML DLLs to build output
shell: powershell
run: |
foreach ($d in "flexmlrt/bin", "flexmlrt/lib") {
if (Test-Path "$d/*.dll") { Copy-Item "$d/*.dll" "build/bin/Release/" -Force }
}
if (-not (Test-Path "build/bin/Release/flexmlrt.dll")) {
Write-Error "flexmlrt.dll not staged next to binaries"; exit 1
}
- name: Download ggml model
shell: cmd
run: |
call models\download-ggml-model.cmd %MODEL% models
if not exist models\ggml-%MODEL%.bin ( echo ERROR: model download failed & exit /b 1 )
- name: Download NPU encoder cache
shell: cmd
run: |
.\models\download-vitisai-model.cmd %MODEL%
if not exist models\ggml-%MODEL%-encoder-vitisai.rai ( echo ERROR: VitisAI encoder cache download failed & exit /b 1 )
- name: Run NPU smoke test
shell: cmd
run: |
build\bin\Release\whisper-cli.exe -m models\ggml-%MODEL%.bin -f samples\jfk.wav > vitisai.log 2>&1
type vitisai.log
findstr /I /C:"vitisai" vitisai.log || ( echo ERROR: no VitisAI activity - encoder likely fell back to CPU & exit /b 1 )
findstr /I /C:"ask not what your country" vitisai.log || ( echo ERROR: incorrect transcription & exit /b 1 )
- name: Upload smoke test log
if: always()
uses: actions/upload-artifact@v4
with:
name: vitisai-smoke-log-windows
path: vitisai.log
npu-amd-linux:
runs-on: [self-hosted, Linux, X64, stx, rai300-400]
timeout-minutes: 60
continue-on-error: true # advisory while the runner pool is new; revisit later
env:
FLEXML_LINUX_URL: https://github.com/lemonade-sdk/whisper.cpp-rocm/releases/download/deps/flexmlrt-1.8.0-linux.tar.gz
MODEL: base
steps:
- name: Clone
uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6
- name: Verify NPU device
run: |
lsmod | grep -q amdxdna || { echo "ERROR: amdxdna driver not loaded"; exit 1; }
ls /dev/accel/accel* || { echo "ERROR: no NPU accel device node"; exit 1; }
- name: Download FlexML runtime (Linux)
run: |
curl -L --fail -o flexmlrt.tar.gz "$FLEXML_LINUX_URL"
tar xf flexmlrt.tar.gz
source flexmlrt/setup.sh
echo "FlexmlRT_DIR=$PWD/flexmlrt/share/cmake/FlexmlRT" >> $GITHUB_ENV
echo "LD_LIBRARY_PATH=$PWD/flexmlrt/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV
- name: Configure and build
run: |
cmake -B build -DCMAKE_BUILD_TYPE=Release -DWHISPER_VITISAI=ON
cmake --build build --config Release -j $(nproc)
- name: Download ggml model
run: |
./models/download-ggml-model.sh $MODEL
- name: Download NPU encoder cache
run: |
sh ./models/download-vitisai-model.sh $MODEL
[ -f "models/ggml-$MODEL-encoder-vitisai.rai" ] || { echo "ERROR: VitisAI encoder cache download failed"; exit 1; }
- name: Run NPU smoke test
run: |
./build/bin/whisper-cli -m "models/ggml-$MODEL.bin" -f samples/jfk.wav 2>&1 | tee vitisai.log
grep -qi "vitisai" vitisai.log || { echo "ERROR: no VitisAI activity - CPU fallback?"; exit 1; }
grep -qi "ask not what your country" vitisai.log || { echo "ERROR: incorrect transcription"; exit 1; }
- name: Upload smoke test log
if: always()
uses: actions/upload-artifact@v4
with:
name: vitisai-smoke-log-linux
path: vitisai.log

1
CMakeLists.txt
View File

@ -92,6 +92,7 @@ endif()
option(WHISPER_COREML "whisper: enable Core ML framework" OFF)
option(WHISPER_COREML_ALLOW_FALLBACK "whisper: allow non-CoreML fallback" OFF)
option(WHISPER_OPENVINO "whisper: support for OpenVINO" OFF)
option(WHISPER_VITISAI "whisper: support for AMD Vitis AI" OFF)
# Required for relocatable CMake package
include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/build-info.cmake)

82
README.md
View File

@ -22,6 +22,7 @@ High-performance inference of [OpenAI's Whisper](https://github.com/openai/whisp
- Support for CPU-only inference
- [Efficient GPU support for NVIDIA](#nvidia-gpu-support)
- [AMD ROCm GPU support](#amd-rocm-gpu-support)
- [AMD Ryzen AI NPU Support](#amd-ryzen-ai-npu-support)
- [OpenVINO Support](#openvino-support)
- [Ascend NPU Support](#ascend-npu-support)
- [Moore Threads GPU Support](#moore-threads-gpu-support)
@ -313,6 +314,87 @@ This can result in significant speedup in encoder performance. Here are the inst
For more information about the OpenVINO implementation please refer to PR [#1037](https://github.com/ggml-org/whisper.cpp/pull/1037).
## AMD Ryzen™ AI NPU support
On AMD Ryzen™ AI 300 and 400 Series processors with a dedicated NPU, whisper.cpp can fully offload the Whisper encoder to the NPU via VitisAI, delivering significant speedup over CPU-only inference.
### Prerequisites
Supported Platforms
- **Windows 11**
- **Linux** (Ubuntu 24.04 LTS, Python 3.12)
Install the XRT runtime and FlexML runtime for your platform:
- **XRT**: provides the NPU kernel driver and `xrt-smi` diagnostic tool — on Windows this is bundled with the NPU driver; on Linux install it separately following the [NPU driver installation guide](https://ryzenai.docs.amd.com/en/latest/linux.html#install-npu-drivers)
- **FlexML runtime** (`flexmlrt`): VitisAI inference engine used by whisper.cpp — download from the [FlexML runtime releases](https://github.com/lemonade-sdk/whisper.cpp-rocm/releases/tag/deps)
After installing, source the setup scripts in every shell you use to build or run whisper.cpp:
```bash
# Linux
source /opt/xilinx/xrt/setup.sh
source /path/to/flexmlrt/setup.sh
```
```cmd
:: Windows
cd /path/to/flexmlrt && call setup.bat
```
You can verify the NPU is visible with:
```bash
xrt-smi examine
```
### Download models
Download the ggml model and the matching prebuilt VitisAI encoder cache:
```bash
# Linux / macOS
sh ./models/download-ggml-model.sh base
sh ./models/download-vitisai-model.sh base
```
```cmd
:: Windows
.\models\download-ggml-model.cmd base
.\models\download-vitisai-model.cmd base
```
Use the same model name with both scripts. To see all available VitisAI encoder caches:
```bash
sh ./models/download-vitisai-model.sh --list
```
```cmd
.\models\download-vitisai-model.cmd --list
```
The VitisAI script queries the [AMD Ryzen AI Whisper NPU collection on Hugging Face](https://huggingface.co/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models) and downloads the `.rai` encoder cache as `models/ggml-<model>-encoder-vitisai.rai`.
> Depending on the `.rai` cache, VitisAI may offload the encoder only, or the encoder plus cross-projection layers. whisper.cpp detects this at runtime and logs the selected offload mode during model initialization.
### Build
```bash
cmake -B build -DWHISPER_VITISAI=1
cmake --build build -j --config Release
```
### Run
```bash
./build/bin/whisper-cli -m models/ggml-base.bin -f samples/jfk.wav
```
For more information see the [Ryzen AI documentation](https://ryzenai.docs.amd.com/en/latest/).
## NVIDIA GPU support
With NVIDIA cards the processing of the models is done efficiently on the GPU via cuBLAS and custom CUDA kernels.

32
models/download-vitisai-model.cmd Normal file
View File

@ -0,0 +1,32 @@
@echo off
setlocal
set "script=%~dp0download-vitisai-model.ps1"
if "%~1"=="" (
PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%"
exit /b %ERRORLEVEL%
)
if /I "%~1"=="--list" (
PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" -List
exit /b %ERRORLEVEL%
)
if /I "%~1"=="-l" (
PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" -List
exit /b %ERRORLEVEL%
)
if /I "%~1"=="list" (
PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" -List
exit /b %ERRORLEVEL%
)
if "%~2"=="" (
PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" -Model "%~1"
) else (
PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" -Model "%~1" -ModelsPath "%~2"
)
exit /b %ERRORLEVEL%

218
models/download-vitisai-model.ps1 Normal file
View File

@ -0,0 +1,218 @@
param(
[Parameter(Position = 0)]
[string] $Model,
[Parameter(Position = 1)]
[string] $ModelsPath,
[switch] $List
)
$ErrorActionPreference = "Stop"
$Source = "https://huggingface.co"
$CollectionApi = "https://huggingface.co/api/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models"
$CollectionUrl = "https://huggingface.co/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models"
$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
if ($ScriptDir -match "\\bin$") {
$DefaultDownloadPath = (Get-Location).Path
} else {
$DefaultDownloadPath = $ScriptDir
}
if (-not $ModelsPath) {
$ModelsPath = $DefaultDownloadPath
}
function Get-HfHeaders {
$headers = @{}
if ($env:HF_TOKEN) {
$headers["Authorization"] = "Bearer $env:HF_TOKEN"
}
return $headers
}
function Invoke-HfJson {
param([string] $Uri)
$headers = Get-HfHeaders
if ($headers.Count -gt 0) {
return Invoke-RestMethod -Uri $Uri -Headers $headers
}
return Invoke-RestMethod -Uri $Uri
}
function Normalize-ModelName {
param([string] $Name)
# HF currently publishes ggml-small-en-encoder-vitisai.rai, while the
# matching ggml model is ggml-small.en.bin.
if ($Name.EndsWith("-en")) {
return $Name.Substring(0, $Name.Length - 3) + ".en"
}
return $Name
}
function Get-VitisAiModels {
$collection = Invoke-HfJson -Uri $CollectionApi
$seen = @{}
$rows = New-Object System.Collections.Generic.List[object]
foreach ($item in $collection.items) {
if ($item.type -ne "model") {
continue
}
$repo = $item.id
if (-not $repo) {
continue
}
$modelInfo = Invoke-HfJson -Uri "$Source/api/models/$repo"
foreach ($sibling in $modelInfo.siblings) {
$filename = [string] $sibling.rfilename
$match = [regex]::Match($filename, "^ggml-(.+)-encoder-vitisai\.rai$")
if (-not $match.Success) {
continue
}
$rawName = $match.Groups[1].Value
$modelName = Normalize-ModelName -Name $rawName
if ($seen.ContainsKey($modelName)) {
continue
}
$seen[$modelName] = $true
$destination = "ggml-$modelName-encoder-vitisai.rai"
$url = "$Source/$repo/resolve/main/$([uri]::EscapeDataString($filename))"
$rows.Add([pscustomobject]@{
Model = $modelName
RawName = $rawName
Repo = $repo
SourceFile = $filename
DestinationFile = $destination
DownloadUrl = $url
})
}
}
$order = @{
"tiny" = 10
"tiny.en" = 11
"base" = 20
"base.en" = 21
"small" = 30
"small.en" = 31
"medium" = 40
"medium.en" = 41
"large-v1" = 50
"large-v2" = 60
"large-v3" = 70
"large-v3-turbo" = 80
}
return $rows | Sort-Object `
@{ Expression = { if ($order.ContainsKey($_.Model)) { $order[$_.Model] } else { 1000 } } }, `
@{ Expression = { $_.Model } }
}
function Show-Models {
$models = Get-VitisAiModels
Write-Host ""
Write-Host "Available VitisAI encoder caches from ${CollectionUrl}:"
foreach ($entry in $models) {
if ($entry.Model -eq $entry.RawName) {
Write-Host (" {0,-18} {1}" -f $entry.Model, $entry.Repo)
} else {
Write-Host (" {0,-18} {1} (source name: {2})" -f $entry.Model, $entry.Repo, $entry.RawName)
}
}
Write-Host ""
}
function Show-Usage {
Write-Host "Usage: download-vitisai-model.cmd --list"
Write-Host " download-vitisai-model.cmd <model> [models_path]"
Write-Host ""
Write-Host "Downloads ggml-<model>-encoder-vitisai.rai next to ggml-<model>.bin."
Write-Host "Use the same model name as download-ggml-model.cmd."
Write-Host ""
}
if ($List -or $Model -eq "--list" -or $Model -eq "-l" -or $Model -eq "list") {
Show-Models
exit 0
}
if (-not $Model) {
Show-Usage
Show-Models
exit 1
}
$models = Get-VitisAiModels
$entry = $models | Where-Object { $_.Model -eq $Model -or $_.RawName -eq $Model } | Select-Object -First 1
if (-not $entry) {
Write-Host "Invalid model: $Model"
foreach ($available in $models) {
Write-Host " $($available.Model)"
}
exit 1
}
New-Item -ItemType Directory -Force -Path $ModelsPath | Out-Null
$destinationPath = Join-Path $ModelsPath $entry.DestinationFile
Write-Host "Downloading VitisAI encoder cache $($entry.Model) from '$($entry.Repo)' ..."
if (Test-Path $destinationPath) {
Write-Host "VitisAI encoder cache $($entry.DestinationFile) already exists. Skipping download."
exit 0
}
$headers = Get-HfHeaders
$downloaded = $false
for ($attempt = 1; $attempt -le 5; ++$attempt) {
try {
if ($headers.Count -gt 0) {
Invoke-WebRequest -Uri $entry.DownloadUrl -Headers $headers -OutFile $destinationPath
} else {
Invoke-WebRequest -Uri $entry.DownloadUrl -OutFile $destinationPath
}
$downloaded = $true
break
} catch {
if ($attempt -eq 5) {
if (Test-Path $destinationPath) {
Remove-Item -Force $destinationPath
}
Write-Host "Failed to download VitisAI encoder cache $($entry.Model) from $($entry.DownloadUrl)"
throw
}
Start-Sleep -Seconds 5
}
}
if (-not $downloaded) {
exit 1
}
$whisperCmd = "whisper-cli"
if (-not (Get-Command whisper-cli -ErrorAction SilentlyContinue)) {
$rootPath = Split-Path -Parent $ScriptDir
$whisperCmd = Join-Path $rootPath "build\bin\Release\whisper-cli.exe"
}
Write-Host "Done! VitisAI encoder cache '$($entry.Model)' saved in '$destinationPath'"
if ($entry.RawName -ne $entry.Model) {
Write-Host "Source cache '$($entry.SourceFile)' was renamed to match ggml model name '$($entry.Model)'."
}
Write-Host "Use it with the matching ggml model:"
Write-Host ""
Write-Host " $ScriptDir\download-ggml-model.cmd $($entry.Model) $ModelsPath"
Write-Host " $whisperCmd -m $ModelsPath\ggml-$($entry.Model).bin -f samples\jfk.wav"
Write-Host ""

226
models/download-vitisai-model.sh Executable file
View File

@ -0,0 +1,226 @@
#!/bin/sh
# This script downloads prebuilt VitisAI encoder cache files for Whisper models.
# The cache file is saved next to the ggml model file and follows the loader
# convention: ggml-<model>-encoder-vitisai.rai
src="https://huggingface.co"
collection_api="https://huggingface.co/api/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models"
collection_url="https://huggingface.co/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models"
BOLD="\033[1m"
RESET='\033[0m'
# get the path of this script
get_script_path() {
if [ -x "$(command -v realpath)" ]; then
dirname "$(realpath "$0")"
else
_ret="$(cd -- "$(dirname "$0")" >/dev/null 2>&1 || exit ; pwd -P)"
echo "$_ret"
fi
}
find_python() {
if command -v python3 >/dev/null 2>&1; then
printf "%s\n" "python3"
elif command -v python >/dev/null 2>&1; then
printf "%s\n" "python"
else
return 1
fi
}
script_path="$(get_script_path)"
# Check if the script is inside a /bin/ directory
case "$script_path" in
*/bin) default_download_path="$PWD" ;; # Use current directory as default download path if in /bin/
*) default_download_path="$script_path" ;; # Otherwise, use script directory
esac
models_path="${2:-$default_download_path}"
discover_models() {
python_cmd="$(find_python)" || {
printf "Python is required to query available VitisAI caches from Hugging Face.\n" >&2
return 1
}
"$python_cmd" - "$src" "$collection_api" <<'PY'
import json
import os
import re
import sys
import urllib.parse
import urllib.request
src = sys.argv[1].rstrip("/")
collection_api = sys.argv[2]
headers = {}
token = os.environ.get("HF_TOKEN")
if token:
headers["Authorization"] = "Bearer " + token
def load_json(url):
req = urllib.request.Request(url, headers=headers)
with urllib.request.urlopen(req) as response:
return json.load(response)
def normalize_model_name(name):
# HF currently publishes ggml-small-en-encoder-vitisai.rai, while the
# matching ggml model is ggml-small.en.bin.
if name.endswith("-en"):
return name[:-3] + ".en"
return name
collection = load_json(collection_api)
rows = []
seen = set()
for item in collection.get("items", []):
if item.get("type") != "model":
continue
repo = item.get("id")
if not repo:
continue
model_info = load_json(src + "/api/models/" + repo)
for sibling in model_info.get("siblings", []):
filename = sibling.get("rfilename", "")
match = re.match(r"^ggml-(.+)-encoder-vitisai\.rai$", filename)
if not match:
continue
raw_name = match.group(1)
model_name = normalize_model_name(raw_name)
if model_name in seen:
continue
seen.add(model_name)
destination = "ggml-%s-encoder-vitisai.rai" % model_name
url = "%s/%s/resolve/main/%s" % (src, repo, urllib.parse.quote(filename))
rows.append((model_name, raw_name, repo, filename, destination, url))
order = {
"tiny": 10,
"tiny.en": 11,
"base": 20,
"base.en": 21,
"small": 30,
"small.en": 31,
"medium": 40,
"medium.en": 41,
"large-v1": 50,
"large-v2": 60,
"large-v3": 70,
"large-v3-turbo": 80,
}
for row in sorted(rows, key=lambda item: (order.get(item[0], 1000), item[0])):
print("|".join(row))
PY
}
list_models() {
models="$(discover_models)" || exit 1
printf "\n"
printf "Available VitisAI encoder caches from %s:\n" "$collection_url"
printf "%s\n" "$models" | while IFS='|' read -r model raw repo _source _destination _url; do
if [ "$model" = "$raw" ]; then
printf " %-18s %s\n" "$model" "$repo"
else
printf " %-18s %s (source name: %s)\n" "$model" "$repo" "$raw"
fi
done
printf "\n"
}
usage() {
printf "Usage: %s --list\n" "$0"
printf " %s <model> [models_path]\n" "$0"
printf "\n"
printf "Downloads ggml-<model>-encoder-vitisai.rai next to ggml-<model>.bin.\n"
printf "Use the same model name as %s/download-ggml-model.sh.\n" "$script_path"
printf "\n"
}
if [ "$#" -eq 1 ] && { [ "$1" = "--list" ] || [ "$1" = "-l" ] || [ "$1" = "list" ]; }; then
list_models
exit 0
fi
if [ "$#" -lt 1 ] || [ "$#" -gt 2 ]; then
usage
list_models
printf "___________________________________________________________\n"
printf "Example: %s ${BOLD}small${RESET} %s\n" "$0" "$default_download_path"
exit 1
fi
model=$1
models="$(discover_models)" || exit 1
match="$(printf "%s\n" "$models" | awk -F '|' -v model="$model" '$1 == model || $2 == model { print; exit }')"
if [ -z "$match" ]; then
printf "Invalid model: %s\n" "$model"
printf "%s\n" "$models" | while IFS='|' read -r available _raw _repo _source _destination _url; do
printf " %s\n" "$available"
done
exit 1
fi
IFS='|' read -r model raw_name repo source_file destination_file download_url <<EOF
$match
EOF
printf "Downloading VitisAI encoder cache %s from '%s' ...\n" "$model" "$repo"
mkdir -p "$models_path" || exit
cd "$models_path" || exit
if [ -f "$destination_file" ]; then
printf "VitisAI encoder cache %s already exists. Skipping download.\n" "$destination_file"
exit 0
fi
if [ -x "$(command -v wget2)" ]; then
wget2 --no-config --progress bar -O "$destination_file" ${HF_TOKEN:+--header "Authorization: Bearer $HF_TOKEN"} "$download_url"
elif [ -x "$(command -v curl)" ]; then
curl -L --fail \
--retry 5 \
--retry-delay 5 \
--retry-all-errors \
--retry-connrefused \
${HF_TOKEN:+--header "Authorization: Bearer $HF_TOKEN"} \
--output "$destination_file" "$download_url"
elif [ -x "$(command -v wget)" ]; then
wget --no-config --quiet --show-progress ${HF_TOKEN:+--header "Authorization: Bearer $HF_TOKEN"} -O "$destination_file" "$download_url"
else
printf "Either wget2, curl, or wget is required to download VitisAI encoder caches.\n"
exit 1
fi
if [ $? -ne 0 ]; then
printf "Failed to download VitisAI encoder cache %s from %s\n" "$model" "$download_url"
rm -f "$destination_file"
exit 1
fi
# Check if 'whisper-cli' is available in the system PATH
if command -v whisper-cli >/dev/null 2>&1; then
whisper_cmd="whisper-cli"
else
whisper_cmd="./build/bin/whisper-cli"
fi
printf "Done! VitisAI encoder cache '%s' saved in '%s/%s'\n" "$model" "$models_path" "$destination_file"
if [ "$raw_name" != "$model" ]; then
printf "Source cache '%s' was renamed to match ggml model name '%s'.\n" "$source_file" "$model"
fi
printf "Use it with the matching ggml model:\n\n"
printf " $ %s/download-ggml-model.sh %s %s\n" "$script_path" "$model" "$models_path"
printf " $ %s -m %s/ggml-%s.bin -f samples/jfk.wav\n" "$whisper_cmd" "$models_path" "$model"
printf "\n"

88
src/CMakeLists.txt
View File

@ -48,6 +48,61 @@ if (WHISPER_OPENVINO)
find_package(OpenVINO REQUIRED COMPONENTS Runtime)
endif()
if (WHISPER_VITISAI)
find_package(FlexmlRT REQUIRED)
# Legacy RAI overrides are required by FlexMLRT older than 1.8.0
set(WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES_MODE "AUTO" CACHE STRING
"Legacy RAI override mode for FlexMLRT (AUTO|ON|OFF)")
set_property(CACHE WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES_MODE PROPERTY STRINGS AUTO ON OFF)
string(TOUPPER "${WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES_MODE}" _flexmlrt_legacy_mode)
set(_flexmlrt_legacy_hint "Set -DWHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES_MODE=ON or OFF explicitly.")
if (NOT _flexmlrt_legacy_mode MATCHES "^(AUTO|ON|OFF)$")
message(FATAL_ERROR
"Invalid WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES_MODE='${WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES_MODE}'. "
"Expected AUTO, ON, or OFF.")
endif()
if (_flexmlrt_legacy_mode STREQUAL "AUTO")
if (NOT FlexmlRT_DIR)
message(FATAL_ERROR
"FlexmlRT_DIR is unset after find_package(FlexmlRT). ${_flexmlrt_legacy_hint}")
endif()
# FlexmlRT_DIR points to <pkg_root>/share/cmake/FlexmlRT.
get_filename_component(_flexmlrt_init_py "${FlexmlRT_DIR}/../../../__init__.py" ABSOLUTE)
if (NOT EXISTS "${_flexmlrt_init_py}")
message(FATAL_ERROR
"flexmlrt __init__.py not found at ${_flexmlrt_init_py}. ${_flexmlrt_legacy_hint}")
endif()
set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${_flexmlrt_init_py}")
file(STRINGS "${_flexmlrt_init_py}" _flexmlrt_version_lines
REGEX "^VERSION[ \t]*=[ \t]*\"[0-9]+\\.[0-9]+\\.[0-9]+")
if (NOT _flexmlrt_version_lines MATCHES "\"([0-9]+\\.[0-9]+\\.[0-9]+)")
message(FATAL_ERROR
"Could not parse flexmlrt VERSION from ${_flexmlrt_init_py}. ${_flexmlrt_legacy_hint}")
endif()
set(_flexmlrt_version "${CMAKE_MATCH_1}")
if (_flexmlrt_version VERSION_LESS "1.8.0")
set(WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES 1)
else()
set(WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES 0)
endif()
message(STATUS "Detected flexmlrt VERSION=${_flexmlrt_version} from ${_flexmlrt_init_py} (legacy overrides=${WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES})")
else()
if (_flexmlrt_legacy_mode STREQUAL "ON")
set(WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES 1)
else()
set(WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES 0)
endif()
message(STATUS "FlexMLRT legacy RAI overrides forced ${_flexmlrt_legacy_mode}")
endif()
endif()
#
# libraries
#
@ -101,6 +156,35 @@ if (WHISPER_OPENVINO)
set_target_properties(${TARGET} PROPERTIES FOLDER "libs")
endif()
if (WHISPER_VITISAI)
set(TARGET whisper.vitisai)
add_library(${TARGET} OBJECT
vitisai/whisper-vitisai-helpers.h
vitisai/whisper-vitisai-helpers.cpp
vitisai/whisper-vitisai-encoder.h
vitisai/whisper-vitisai-encoder.cpp
)
target_include_directories(${TARGET} PUBLIC
.
)
set_property(TARGET ${TARGET} PROPERTY POSITION_INDEPENDENT_CODE ON)
set(WHISPER_EXTRA_FLAGS ${WHISPER_EXTRA_FLAGS} -DWHISPER_USE_VITISAI)
# FlexMLRT headers and this plugin require C++17. Keep it PRIVATE so the
# C++11 requirement of the whisper target is not bumped.
target_compile_features(${TARGET} PRIVATE cxx_std_17)
target_compile_definitions(${TARGET} PRIVATE
WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES=${WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES}
)
target_link_libraries(${TARGET} PRIVATE ggml flexmlrt::flexmlrt)
set_target_properties(${TARGET} PROPERTIES FOLDER "libs")
endif()
# whisper
add_library(whisper
@ -157,6 +241,10 @@ if (WHISPER_OPENVINO)
target_link_libraries(whisper PRIVATE whisper.openvino)
endif()
if (WHISPER_VITISAI)
target_link_libraries(whisper PRIVATE whisper.vitisai)
endif()
if (WHISPER_MKL)
target_link_libraries(whisper PRIVATE MKL::MKL)
endif()

602
src/vitisai/whisper-vitisai-encoder.cpp Normal file
View File

@ -0,0 +1,602 @@
#ifdef _WIN32
#ifndef NOMINMAX
#define NOMINMAX
#endif
#endif
#include "vitisai/whisper-vitisai-encoder.h"
#include "vitisai/whisper-vitisai-helpers.h"
#include "FlexMLClient.h"
#include "ggml.h"
#include <cstdio>
#include <cstdlib>
#include <cmath>
#include <memory>
#include <string>
#include <vector>
#if defined(WHISPER_DEBUG)
#define WHISPER_DBG_TIMER(name) const int64_t name = ggml_time_us()
#else
#define WHISPER_DBG_TIMER(name) do {} while (0)
#endif
struct whisper_vitisai_context {
std::string model_path;
std::shared_ptr<flexmlrt::client::Model> runner;
uint8_t * fbs_buffer = nullptr;
size_t fbs_buffer_size = 0;
std::vector<float> cross_k_staging;
std::vector<float> cross_v_staging;
int mel_in_idx = -1;
int embd_enc_out_idx = -1;
int cross_k_out_idx = -1;
int cross_v_out_idx = -1;
size_t mel_in_expected_bytes = 0;
size_t embd_enc_expected_bytes = 0;
size_t cross_k_expected_bytes = 0;
size_t cross_v_expected_bytes = 0;
std::vector<flexmlrt::client::ErtTensorType> cached_input_tensors;
std::vector<flexmlrt::client::ErtTensorType> cached_output_tensors;
};
// Return cached IO tensor descriptors by reference to avoid per-call deep copies.
static bool whisper_vitisai_get_cached_io_tensors(
struct whisper_vitisai_context * ctx,
std::vector<flexmlrt::client::ErtTensorType> *& input_tensors,
std::vector<flexmlrt::client::ErtTensorType> *& output_tensors) {
if (!ctx || !ctx->runner) {
return false;
}
if (ctx->cached_input_tensors.empty() || ctx->cached_output_tensors.empty()) {
ctx->cached_input_tensors = ctx->runner->getIOTensors("input", false);
ctx->cached_output_tensors = ctx->runner->getIOTensors("output", false);
}
input_tensors = &ctx->cached_input_tensors;
output_tensors = &ctx->cached_output_tensors;
return true;
}
struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model) {
if (!path_model) {
std::fprintf(stderr, "%s: path_model is null\n", __func__);
return nullptr;
}
auto * ctx = new whisper_vitisai_context;
ctx->model_path = path_model;
// Override the model path with the environment variable if it is set
if (const char * env_model_path = std::getenv("OVERRIDE_VITISAI_MODEL_PATH")) {
if (env_model_path[0] != '\0') {
ctx->model_path = env_model_path;
}
}
// Step 1: Set up the model
flexmlrt::client::Options options;
options.modelPath = ctx->model_path;
options.debug = false;
options.executeMode = 2;
options.extOptions["enable_preemption"] = true;
const bool model_is_rai = ctx->model_path.find(".rai") != std::string::npos;
// Check if model_path is rai file and if so, add fbs_buffer and fbs_buffer_size to the options
if (model_is_rai) {
if (whisper_vitisai_helpers::map_rai_file(ctx->model_path.c_str(), &ctx->fbs_buffer, &ctx->fbs_buffer_size)) {
options.extOptions["fbs_buffer"] = ctx->fbs_buffer;
options.extOptions["fbs_buffer_size"] = ctx->fbs_buffer_size;
options.extOptions["cache_dir"] = std::string(".");
} else {
std::fprintf(stderr, "%s: Failed to mmap rai file '%s'\n", __func__, ctx->model_path.c_str());
delete ctx;
return nullptr;
}
} else {
options.deviceName = "stx";
#if defined(WHISPER_DEBUG)
std::fprintf(stderr, "%s: Using default device name 'stx'\n", __func__);
#endif
}
if (model_is_rai) {
#if WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES
options.deviceName = "stx";
options.subgraphName = "vaiml_par_0";
#if defined(WHISPER_DEBUG)
std::fprintf(stderr,
"%s: legacy FlexMLRT compile configuration detected; applying RAI overrides (device='stx', subgraph='vaiml_par_0')\n",
__func__);
#endif // defined(WHISPER_DEBUG)
#endif
}
try {
ctx->runner = std::make_shared<flexmlrt::client::Model>(options);
if (!ctx->runner || !ctx->runner->good()) {
throw std::runtime_error("Runner creation ran into an error");
}
ctx->cached_input_tensors = ctx->runner->getIOTensors("input", false);
ctx->cached_output_tensors = ctx->runner->getIOTensors("output", false);
auto & input_tensors = ctx->cached_input_tensors;
auto & output_tensors = ctx->cached_output_tensors;
whisper_vitisai_helpers::whisper_vitisai_io_binding binding;
std::string binding_error;
if (!whisper_vitisai_helpers::whisper_vitisai_resolve_io_binding(
__func__, input_tensors, output_tensors, &binding, &binding_error)) {
throw std::runtime_error(binding_error);
}
ctx->mel_in_idx = binding.mel_in_idx;
ctx->embd_enc_out_idx = binding.embd_enc_out_idx;
ctx->cross_k_out_idx = binding.cross_k_out_idx;
ctx->cross_v_out_idx = binding.cross_v_out_idx;
ctx->mel_in_expected_bytes = binding.mel_in_expected_bytes;
ctx->embd_enc_expected_bytes = binding.embd_enc_expected_bytes;
ctx->cross_k_expected_bytes = binding.cross_k_expected_bytes;
ctx->cross_v_expected_bytes = binding.cross_v_expected_bytes;
#if defined(WHISPER_DEBUG)
{
std::fprintf(stderr, "%s: model has %zu input tensor(s)\n", __func__, input_tensors.size());
for (int i = 0; i < (int) input_tensors.size(); ++i) {
const auto & meta = input_tensors[i].getMetadata();
std::fprintf(stderr, "%s: input[%d] name='%s' size=%zu shape=",
__func__, i, meta.name.c_str(), (size_t) meta.size);
whisper_vitisai_helpers::whisper_vitisai_print_shape(meta.shape);
std::fprintf(stderr, "\n");
}
std::fprintf(stderr, "%s: model has %zu output tensor(s)\n", __func__, output_tensors.size());
for (int i = 0; i < (int) output_tensors.size(); ++i) {
const auto & meta = output_tensors[i].getMetadata();
std::fprintf(stderr, "%s: output[%d] name='%s' size=%zu shape=",
__func__, i, meta.name.c_str(), (size_t) meta.size);
whisper_vitisai_helpers::whisper_vitisai_print_shape(meta.shape);
std::fprintf(stderr, "\n");
}
std::fprintf(stderr, "%s: input index: mel=%d\n", __func__, ctx->mel_in_idx);
std::fprintf(stderr, "%s: output indices: embd_enc=%d cross_k=%d cross_v=%d\n",
__func__, ctx->embd_enc_out_idx, ctx->cross_k_out_idx, ctx->cross_v_out_idx);
}
#endif
} catch (const std::exception & e) {
std::fprintf(stderr, "%s: Exception during Vitis AI runner creation: %s\n", __func__, e.what());
whisper_vitisai_free(ctx);
return nullptr;
}
return ctx;
}
bool whisper_vitisai_has_cross_proj(const struct whisper_vitisai_context * ctx) {
return ctx && ctx->cross_k_out_idx >= 0 && ctx->cross_v_out_idx >= 0;
}
void whisper_vitisai_free(struct whisper_vitisai_context * ctx) {
if (!ctx) {
return;
}
#if defined(WHISPER_DEBUG)
std::fprintf(stderr, "%s: releasing Vitis AI context for model '%s'\n", __func__, ctx->model_path.c_str());
#endif
if (ctx->fbs_buffer) {
whisper_vitisai_helpers::unmap_rai_file(ctx->fbs_buffer, ctx->fbs_buffer_size);
}
delete ctx;
}
static int whisper_vitisai_forward_impl(
struct whisper_vitisai_context * ctx,
struct ggml_tensor * mel,
struct ggml_tensor * out,
std::vector<flexmlrt::client::ErtTensorType> & input_tensors,
std::vector<flexmlrt::client::ErtTensorType> & output_tensors,
void * cross_k_data,
void * cross_v_data) {
if (!ctx || !mel || !out) {
std::fprintf(stderr, "%s: ctx/mel/out must not be null\n", __func__);
return 0;
}
const bool with_cross = (cross_k_data != nullptr || cross_v_data != nullptr);
if (with_cross && (!cross_k_data || !cross_v_data)) {
std::fprintf(stderr, "%s: cross_k_data/cross_v_data must both be set\n", __func__);
return 0;
}
if (ggml_n_dims(mel) != 2) {
std::fprintf(stderr, "%s: mel tensor expected to have 2 dims, got %d\n", __func__, ggml_n_dims(mel));
return 0;
}
if (ggml_n_dims(out) != 2) {
std::fprintf(stderr, "%s: out tensor expected to have 2 dims, got %d\n", __func__, ggml_n_dims(out));
return 0;
}
if (ctx->embd_enc_out_idx < 0 || ctx->embd_enc_out_idx >= (int) output_tensors.size()) {
std::fprintf(stderr, "%s: invalid embd_enc output index %d for %zu output tensor(s)\n",
__func__, ctx->embd_enc_out_idx, output_tensors.size());
return 0;
}
if (ctx->mel_in_idx < 0 || ctx->mel_in_idx >= (int) input_tensors.size()) {
std::fprintf(stderr, "%s: invalid mel input index %d for %zu input tensor(s)\n",
__func__, ctx->mel_in_idx, input_tensors.size());
return 0;
}
if (!whisper_vitisai_helpers::whisper_vitisai_bind_tensor_data(
"mel input",
mel,
{ (size_t) mel->ne[1], (size_t) mel->ne[0] },
input_tensors[ctx->mel_in_idx])) {
return 0;
}
if (!whisper_vitisai_helpers::whisper_vitisai_bind_tensor_data(
"embd_enc output",
out,
{ (size_t) out->ne[1], (size_t) out->ne[0] },
output_tensors[ctx->embd_enc_out_idx])) {
return 0;
}
std::vector<bool> claimed_inputs(input_tensors.size(), false);
claimed_inputs[ctx->mel_in_idx] = true;
if (!whisper_vitisai_helpers::whisper_vitisai_all_tensors_claimed(
__func__, "input", input_tensors, claimed_inputs)) {
return 0;
}
std::vector<bool> claimed_outputs(output_tensors.size(), false);
claimed_outputs[ctx->embd_enc_out_idx] = true;
if (with_cross) {
if (ctx->cross_k_out_idx < 0 || ctx->cross_k_out_idx >= (int) output_tensors.size() ||
ctx->cross_v_out_idx < 0 || ctx->cross_v_out_idx >= (int) output_tensors.size()) {
std::fprintf(stderr, "%s: invalid cross output indices cross_k=%d cross_v=%d for %zu output tensor(s)\n",
__func__, ctx->cross_k_out_idx, ctx->cross_v_out_idx, output_tensors.size());
return 0;
}
output_tensors[ctx->cross_k_out_idx].data = cross_k_data;
output_tensors[ctx->cross_v_out_idx].data = cross_v_data;
claimed_outputs[ctx->cross_k_out_idx] = true;
claimed_outputs[ctx->cross_v_out_idx] = true;
}
if (!whisper_vitisai_helpers::whisper_vitisai_all_tensors_claimed(
__func__, "output", output_tensors, claimed_outputs)) {
return 0;
}
auto clear_bound_data = [&]() {
input_tensors[ctx->mel_in_idx].data = nullptr;
output_tensors[ctx->embd_enc_out_idx].data = nullptr;
if (with_cross) {
output_tensors[ctx->cross_k_out_idx].data = nullptr;
output_tensors[ctx->cross_v_out_idx].data = nullptr;
}
};
try {
ctx->runner->forward(input_tensors, output_tensors);
clear_bound_data();
#if defined(WHISPER_DEBUG)
std::fprintf(stderr, "%s: Vitis AI model inference %scompleted.\n",
__func__, with_cross ? "(encoder + cross proj) " : "");
#endif
} catch (const std::exception & e) {
clear_bound_data();
std::fprintf(stderr, "%s: Exception during model inference: %s\n", __func__, e.what());
return 0;
}
return 1;
}
int whisper_vitisai_encode(struct whisper_vitisai_context * ctx, struct ggml_tensor * mel, struct ggml_tensor * out) {
std::vector<flexmlrt::client::ErtTensorType> * input_tensors_cached = nullptr;
std::vector<flexmlrt::client::ErtTensorType> * output_tensors_cached = nullptr;
if (!whisper_vitisai_get_cached_io_tensors(ctx, input_tensors_cached, output_tensors_cached)) {
std::fprintf(stderr, "%s: failed to acquire Vitis AI I/O tensors\n", __func__);
return 0;
}
std::vector<flexmlrt::client::ErtTensorType> input_tensors = *input_tensors_cached;
std::vector<flexmlrt::client::ErtTensorType> output_tensors = *output_tensors_cached;
return whisper_vitisai_forward_impl(
ctx,
mel,
out,
input_tensors,
output_tensors,
nullptr,
nullptr);
}
int whisper_vitisai_run_enc_cross(
struct whisper_vitisai_context * ctx,
struct ggml_tensor * mel,
struct ggml_tensor * out,
void * cross_v_data,
void * cross_k_data) {
if (!cross_v_data || !cross_k_data) {
std::fprintf(stderr, "%s: cross_v_data/cross_k_data must not be null\n", __func__);
return 0;
}
std::vector<flexmlrt::client::ErtTensorType> * input_tensors_cached = nullptr;
std::vector<flexmlrt::client::ErtTensorType> * output_tensors_cached = nullptr;
if (!whisper_vitisai_get_cached_io_tensors(ctx, input_tensors_cached, output_tensors_cached)) {
std::fprintf(stderr, "%s: failed to acquire Vitis AI I/O tensors\n", __func__);
return 0;
}
std::vector<flexmlrt::client::ErtTensorType> input_tensors = *input_tensors_cached;
std::vector<flexmlrt::client::ErtTensorType> output_tensors = *output_tensors_cached;
return whisper_vitisai_forward_impl(
ctx,
mel,
out,
input_tensors,
output_tensors,
cross_k_data,
cross_v_data);
}
// Ensure persistent staging buffers are large enough for the given dimensions.
static void ensure_staging_buffers(
struct whisper_vitisai_context * ctx,
size_t count,
bool need_k,
bool need_v) {
if (need_k && ctx->cross_k_staging.size() < count) {
ctx->cross_k_staging.resize(count);
}
if (need_v && ctx->cross_v_staging.size() < count) {
ctx->cross_v_staging.resize(count);
}
}
int whisper_vitisai_encode_with_cross(
struct whisper_vitisai_context * ctx,
struct ggml_tensor * mel,
struct ggml_tensor * embd_enc,
struct ggml_tensor * kv_cross_k,
struct ggml_tensor * kv_cross_v,
int n_text_layer,
int n_ctx,
int n_text_state,
int n_text_head,
bool flash_attn) {
if (!ctx || !mel || !embd_enc || !kv_cross_k || !kv_cross_v) {
std::fprintf(stderr, "%s: ctx/mel/embd_enc/kv_cross_k/kv_cross_v must not be null\n", __func__);
return 0;
}
if (n_text_layer <= 0 || n_ctx <= 0 || n_text_state <= 0 || n_text_head <= 0) {
std::fprintf(stderr, "%s: invalid shape parameters layer=%d ctx=%d state=%d head=%d\n",
__func__, n_text_layer, n_ctx, n_text_state, n_text_head);
return 0;
}
if ((n_text_state % n_text_head) != 0) {
std::fprintf(stderr, "%s: invalid head configuration state=%d head=%d\n",
__func__, n_text_state, n_text_head);
return 0;
}
if (kv_cross_k->type != kv_cross_v->type) {
std::fprintf(stderr, "%s: kv_cross type mismatch k=%s v=%s\n",
__func__,
whisper_vitisai_helpers::whisper_kv_type_name(kv_cross_k->type),
whisper_vitisai_helpers::whisper_kv_type_name(kv_cross_v->type));
return 0;
}
const int n_state = n_text_state;
const int n_state_head = n_state / n_text_head;
const int n_ctx_pad = (n_ctx + 255) & ~255; // GGML_PAD(n_ctx, 256)
const float Kscale = pow(float(n_state_head), -0.25f);
const ggml_type kv_type = kv_cross_k->type;
const bool kv_is_f32 = kv_type == GGML_TYPE_F32;
const bool kv_is_f16 = kv_type == GGML_TYPE_F16;
if (!kv_is_f32 && !kv_is_f16) {
std::fprintf(stderr, "%s: unsupported kv_cross tensor type '%s'\n",
__func__, whisper_vitisai_helpers::whisper_kv_type_name(kv_type));
return 0;
}
const size_t elem_size = ggml_type_size(kv_type);
const size_t req_layer_elems = (size_t)n_ctx * (size_t)n_state;
std::vector<flexmlrt::client::ErtTensorType> * input_tensors_cached = nullptr;
std::vector<flexmlrt::client::ErtTensorType> * output_tensors_cached = nullptr;
if (!whisper_vitisai_get_cached_io_tensors(ctx, input_tensors_cached, output_tensors_cached)) {
std::fprintf(stderr, "%s: failed to acquire Vitis AI I/O tensors\n", __func__);
return 0;
}
std::vector<flexmlrt::client::ErtTensorType> input_tensors = *input_tensors_cached;
std::vector<flexmlrt::client::ErtTensorType> output_tensors = *output_tensors_cached;
if (ctx->cross_k_out_idx < 0 || ctx->cross_k_out_idx >= (int) output_tensors.size() ||
ctx->cross_v_out_idx < 0 || ctx->cross_v_out_idx >= (int) output_tensors.size()) {
std::fprintf(stderr, "%s: invalid cross output indices cross_k=%d cross_v=%d for %zu output tensor(s)\n",
__func__, ctx->cross_k_out_idx, ctx->cross_v_out_idx, output_tensors.size());
return 0;
}
const auto & cross_k_meta = output_tensors[ctx->cross_k_out_idx].getMetadata();
const auto & cross_v_meta = output_tensors[ctx->cross_v_out_idx].getMetadata();
if (!whisper_vitisai_helpers::whisper_validate_cross_shape("cross_k", cross_k_meta.shape, n_text_layer, n_ctx, n_state) ||
!whisper_vitisai_helpers::whisper_validate_cross_shape("cross_v", cross_v_meta.shape, n_text_layer, n_ctx, n_state)) {
return 0;
}
if (ctx->cross_k_expected_bytes == 0 || ctx->cross_v_expected_bytes == 0) {
std::fprintf(stderr, "%s: missing cross output metadata sizes\n", __func__);
return 0;
}
if (ctx->cross_k_expected_bytes != ctx->cross_v_expected_bytes) {
std::fprintf(stderr, "%s: cross output metadata size mismatch k=%zu v=%zu\n",
__func__, ctx->cross_k_expected_bytes, ctx->cross_v_expected_bytes);
return 0;
}
const size_t expected_cross_bytes = (size_t) n_text_layer * req_layer_elems * sizeof(float);
if (ctx->cross_k_expected_bytes != expected_cross_bytes) {
std::fprintf(stderr,
"%s: cross output size mismatch (model=%zu B, expected=%zu B for layer=%d ctx=%d state=%d)\n",
__func__, ctx->cross_k_expected_bytes, expected_cross_bytes, n_text_layer, n_ctx, n_state);
return 0;
}
const size_t model_total_elems = ctx->cross_k_expected_bytes / sizeof(float);
const size_t model_layer_elems = req_layer_elems;
const size_t required_kv_bytes = flash_attn
? (size_t)n_text_layer * elem_size * (size_t)n_state * (size_t)n_ctx_pad
: (size_t)n_text_layer * elem_size * req_layer_elems;
if (ggml_nbytes(kv_cross_k) < required_kv_bytes || ggml_nbytes(kv_cross_v) < required_kv_bytes) {
std::fprintf(stderr,
"%s: kv_cross buffers are too small (required=%zu B, k=%zu B, v=%zu B)\n",
__func__, required_kv_bytes, ggml_nbytes(kv_cross_k), ggml_nbytes(kv_cross_v));
return 0;
}
const bool direct_k_to_kv = kv_is_f32 && (!flash_attn || n_ctx_pad == n_ctx);
const bool direct_v_to_kv = kv_is_f32 && flash_attn && (n_ctx_pad == n_ctx);
const bool need_k_staging = !direct_k_to_kv;
const bool need_v_staging = !direct_v_to_kv;
if (need_k_staging || need_v_staging) {
ensure_staging_buffers(ctx, model_total_elems, need_k_staging, need_v_staging);
}
void * cross_k_out = direct_k_to_kv
? kv_cross_k->data
: (void *) ctx->cross_k_staging.data();
void * cross_v_out = direct_v_to_kv
? kv_cross_v->data
: (void *) ctx->cross_v_staging.data();
whisper_vitisai_helpers::whisper_kv_cross_layout kv_layout;
kv_layout.n_layer = n_text_layer;
kv_layout.n_ctx = n_ctx;
kv_layout.n_state = n_state;
kv_layout.src_layer_elems = model_layer_elems;
kv_layout.layer_elems = req_layer_elems;
kv_layout.kscale = Kscale;
if (flash_attn) {
WHISPER_DBG_TIMER(t_fwd_start);
if (!whisper_vitisai_forward_impl(
ctx, mel, embd_enc, input_tensors, output_tensors, cross_k_out, cross_v_out)) {
return 0;
}
WHISPER_DBG_TIMER(t_fwd_end);
WHISPER_DBG_TIMER(t_post_start);
if (n_ctx_pad == n_ctx) {
kv_layout.dst_layer_stride = req_layer_elems * elem_size;
if (kv_is_f32) {
// V was written straight into the kv cache by the runtime; only K needs scaling.
whisper_vitisai_helpers::whisper_kv_cross_scale_k_f32(
(float *)kv_cross_k->data,
(size_t) n_text_layer * req_layer_elems,
Kscale);
} else { // kv_is_f16
whisper_vitisai_helpers::whisper_kv_cross_store_layers_f16(
ctx->cross_k_staging.data(),
ctx->cross_v_staging.data(),
(uint8_t *)kv_cross_k->data,
(uint8_t *)kv_cross_v->data,
kv_layout);
}
} else {
// Runtime decoder uses padded K/V cache. Copy only requested context, leave the pad tail untouched.
kv_layout.dst_layer_stride = elem_size * (size_t)n_state * (size_t)n_ctx_pad;
if (kv_is_f32) {
whisper_vitisai_helpers::whisper_kv_cross_store_layers_f32(
ctx->cross_k_staging.data(),
ctx->cross_v_staging.data(),
(uint8_t *)kv_cross_k->data,
(uint8_t *)kv_cross_v->data,
kv_layout);
} else { // kv_is_f16
whisper_vitisai_helpers::whisper_kv_cross_store_layers_f16(
ctx->cross_k_staging.data(),
ctx->cross_v_staging.data(),
(uint8_t *)kv_cross_k->data,
(uint8_t *)kv_cross_v->data,
kv_layout);
}
}
WHISPER_DBG_TIMER(t_post_end);
#if defined(WHISPER_DEBUG)
const size_t model_ctx = (size_t) n_ctx;
std::fprintf(stderr, "%s: vitisai enc+cross forward time = %8.2f ms\n", __func__, (t_fwd_end - t_fwd_start) / 1000.0f);
std::fprintf(stderr, "%s: kv_cross post-process time = %8.2f ms (flash, req_ctx=%d, model_ctx=%zu, req_ctx_pad=%d, kv_type=%s)\n",
__func__, (t_post_end - t_post_start) / 1000.0f, n_ctx, model_ctx, n_ctx_pad,
whisper_vitisai_helpers::whisper_kv_type_name(kv_type));
#endif
} else {
// Non-flash: model outputs contiguous [ctx, state] per layer.
WHISPER_DBG_TIMER(t_fwd_start);
if (!whisper_vitisai_forward_impl(
ctx, mel, embd_enc, input_tensors, output_tensors, cross_k_out, cross_v_out)) {
return 0;
}
WHISPER_DBG_TIMER(t_fwd_end);
WHISPER_DBG_TIMER(t_post_start);
kv_layout.dst_layer_stride = elem_size * (size_t)n_state * (size_t)n_ctx;
if (kv_is_f32) {
// K was written straight into the kv cache by the runtime and is scaled there.
whisper_vitisai_helpers::whisper_kv_cross_scale_k_f32(
(float *)kv_cross_k->data,
(size_t) n_text_layer * req_layer_elems,
Kscale);
whisper_vitisai_helpers::whisper_kv_cross_transpose_v_layers_f32(
ctx->cross_v_staging.data(),
(uint8_t *)kv_cross_v->data,
kv_layout);
} else { // kv_is_f16
whisper_vitisai_helpers::whisper_kv_cross_store_k_transpose_v_layers_f16(
ctx->cross_k_staging.data(),
ctx->cross_v_staging.data(),
(uint8_t *)kv_cross_k->data,
(uint8_t *)kv_cross_v->data,
kv_layout);
}
WHISPER_DBG_TIMER(t_post_end);
#if defined(WHISPER_DEBUG)
const size_t model_ctx = (size_t) n_ctx;
std::fprintf(stderr, "%s: vitisai enc+cross forward time = %8.2f ms\n", __func__, (t_fwd_end - t_fwd_start) / 1000.0f);
std::fprintf(stderr, "%s: kv_cross post-process time = %8.2f ms (non-flash, req_ctx=%d, model_ctx=%zu, kv_type=%s)\n",
__func__, (t_post_end - t_post_start) / 1000.0f, n_ctx, model_ctx,
whisper_vitisai_helpers::whisper_kv_type_name(kv_type));
#endif
}
return 1;
}

43
src/vitisai/whisper-vitisai-encoder.h Normal file
View File

@ -0,0 +1,43 @@
#pragma once
#include <cstdbool>
#if __cplusplus
extern "C" {
#endif
struct whisper_vitisai_context;
struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model);
void whisper_vitisai_free(struct whisper_vitisai_context * ctx);
bool whisper_vitisai_has_cross_proj(const struct whisper_vitisai_context * ctx);
struct ggml_tensor;
int whisper_vitisai_encode(
struct whisper_vitisai_context * ctx,
struct ggml_tensor * mel,
struct ggml_tensor * out);
int whisper_vitisai_run_enc_cross(
struct whisper_vitisai_context * ctx,
struct ggml_tensor * mel,
struct ggml_tensor * out,
void * cross_v_data,
void * cross_k_data);
int whisper_vitisai_encode_with_cross(
struct whisper_vitisai_context * ctx,
struct ggml_tensor * mel,
struct ggml_tensor * embd_enc,
struct ggml_tensor * kv_cross_k,
struct ggml_tensor * kv_cross_v,
int n_text_layer,
int n_ctx,
int n_text_state,
int n_text_head,
bool flash_attn);
#if __cplusplus
}
#endif

472
src/vitisai/whisper-vitisai-helpers.cpp Normal file
View File

@ -0,0 +1,472 @@
#ifdef _WIN32
#ifndef NOMINMAX
#define NOMINMAX
#endif
#endif
#include "vitisai/whisper-vitisai-helpers.h"
#include <algorithm>
#include <cstdio>
#ifdef _WIN32
#include <windows.h>
#else
#include <sys/mman.h>
#include <sys/stat.h>
#endif
#include <string>
#include <utility>
namespace whisper_vitisai_helpers {
bool map_rai_file(const char * path, uint8_t ** buffer, size_t * size) {
#ifdef _WIN32
HANDLE hFile = CreateFileA(path, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
if (hFile == INVALID_HANDLE_VALUE) {
std::fprintf(stderr, "%s: %d: Failed to open rai file '%s'\n", __func__, __LINE__, path);
return false;
}
LARGE_INTEGER fileSize;
if (!GetFileSizeEx(hFile, &fileSize)) {
CloseHandle(hFile);
std::fprintf(stderr, "%s: %d: Failed to get file size for rai file '%s'\n", __func__, __LINE__, path);
return false;
}
HANDLE hMapping = CreateFileMappingA(hFile, NULL, PAGE_READONLY, 0, fileSize.QuadPart, NULL);
if (hMapping == NULL) {
CloseHandle(hFile);
std::fprintf(stderr, "%s: %d: Failed to create file mapping for rai file '%s'\n", __func__, __LINE__, path);
return false;
}
*buffer = (uint8_t *) MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, fileSize.QuadPart);
if (*buffer == NULL) {
CloseHandle(hMapping);
CloseHandle(hFile);
std::fprintf(stderr, "%s: %d: Failed to map rai file '%s'\n", __func__, __LINE__, path);
return false;
}
CloseHandle(hMapping);
CloseHandle(hFile);
*size = fileSize.QuadPart;
return true;
#else
FILE * fd = fopen(path, "rb");
if (!fd) {
std::fprintf(stderr, "%s: %d: Failed to open rai file '%s'\n", __func__, __LINE__, path);
return false;
}
struct stat st;
if (fstat(fileno(fd), &st) == -1) {
fclose(fd);
std::fprintf(stderr, "%s: %d: Failed to get file size for rai file '%s'\n", __func__, __LINE__, path);
return false;
}
*buffer = (uint8_t *) mmap(nullptr, st.st_size, PROT_READ, MAP_PRIVATE, fileno(fd), 0);
if (*buffer == MAP_FAILED) {
fclose(fd);
std::fprintf(stderr, "%s: %d: Failed to mmap rai file '%s'\n", __func__, __LINE__, path);
return false;
}
fclose(fd);
*size = st.st_size;
return true;
#endif // _WIN32
}
void unmap_rai_file(uint8_t * buffer, size_t size) {
#ifdef _WIN32
UnmapViewOfFile(buffer);
#else
munmap(buffer, size);
#endif // _WIN32
}
const char * whisper_kv_type_name(ggml_type type) {
switch (type) {
case GGML_TYPE_F32: return "F32";
case GGML_TYPE_F16: return "F16";
default: return "unsupported";
}
}
const char * whisper_flexml_dtype_name(flexmlrt::client::DataType type) {
switch (type) {
case flexmlrt::client::DataType::Float32: return "Float32";
case flexmlrt::client::DataType::Int8: return "Int8";
case flexmlrt::client::DataType::UInt8: return "UInt8";
case flexmlrt::client::DataType::Int16: return "Int16";
case flexmlrt::client::DataType::UInt16: return "UInt16";
case flexmlrt::client::DataType::BFloat16: return "BFloat16";
case flexmlrt::client::DataType::Bool: return "Bool";
case flexmlrt::client::DataType::Float16: return "Float16";
case flexmlrt::client::DataType::Int32: return "Int32";
case flexmlrt::client::DataType::UInt32: return "UInt32";
default: return "Unknown";
}
}
bool whisper_flexml_dtype_to_ggml_type(
flexmlrt::client::DataType type,
ggml_type * ggml_dtype) {
switch (type) {
case flexmlrt::client::DataType::Float32:
if (ggml_dtype) {
*ggml_dtype = GGML_TYPE_F32;
}
return true;
case flexmlrt::client::DataType::Float16:
if (ggml_dtype) {
*ggml_dtype = GGML_TYPE_F16;
}
return true;
case flexmlrt::client::DataType::BFloat16:
if (ggml_dtype) {
*ggml_dtype = GGML_TYPE_BF16;
}
return true;
default:
return false;
}
}
static bool whisper_vitisai_validate_tensor_dtype(
const char * tensor_name,
flexmlrt::client::DataType model_dtype,
ggml_type runtime_dtype) {
ggml_type expected_runtime_dtype = GGML_TYPE_COUNT;
if (!whisper_flexml_dtype_to_ggml_type(model_dtype, &expected_runtime_dtype)) {
std::fprintf(stderr,
"%s: unsupported model dtype for %s: %s (supported: Float32/Float16/BFloat16)\n",
__func__, tensor_name, whisper_flexml_dtype_name(model_dtype));
return false;
}
if (runtime_dtype != expected_runtime_dtype) {
std::fprintf(stderr,
"%s: %s dtype mismatch (runtime=%s, model=%s)\n",
__func__, tensor_name, ggml_type_name(runtime_dtype), whisper_flexml_dtype_name(model_dtype));
return false;
}
return true;
}
static std::string whisper_shape_to_string(const std::vector<size_t> & shape) {
std::string out = "[";
for (size_t i = 0; i < shape.size(); ++i) {
if (i > 0) {
out += ", ";
}
out += std::to_string(shape[i]);
}
out += "]";
return out;
}
static std::vector<size_t> whisper_canonical_shape(const std::vector<std::uint32_t> & shape) {
std::vector<size_t> canonical;
canonical.reserve(shape.size());
for (size_t i = 0; i < shape.size(); ++i) {
const size_t dim = (size_t) shape[i];
if (dim != 1) {
canonical.push_back(dim);
}
}
if (canonical.empty()) {
canonical.push_back(1);
}
return canonical;
}
static bool whisper_validate_shape(
const char * tensor_name,
const std::vector<std::uint32_t> & model_shape,
const std::vector<size_t> & expected_shape) {
const std::vector<size_t> shape = whisper_canonical_shape(model_shape);
if (shape != expected_shape) {
std::fprintf(stderr,
"%s: %s shape mismatch (runtime expected=%s, model=%s)\n",
__func__,
tensor_name,
whisper_shape_to_string(expected_shape).c_str(),
whisper_shape_to_string(shape).c_str());
return false;
}
return true;
}
bool whisper_validate_cross_shape(
const char * tensor_name,
const std::vector<std::uint32_t> & model_shape,
int n_text_layer,
int n_ctx,
int n_state) {
const std::vector<size_t> expected = {
(size_t) n_text_layer,
(size_t) n_ctx,
(size_t) n_state,
};
return whisper_validate_shape(tensor_name, model_shape, expected);
}
bool whisper_vitisai_bind_tensor_data(
const char * tensor_name,
struct ggml_tensor * runtime_tensor,
const std::vector<size_t> & expected_shape,
flexmlrt::client::ErtTensorType & io_tensor) {
const auto & meta = io_tensor.getMetadata();
if (!whisper_vitisai_validate_tensor_dtype(tensor_name, meta.type, runtime_tensor->type)) {
return false;
}
if (!whisper_validate_shape(tensor_name, meta.shape, expected_shape)) {
return false;
}
const size_t model_bytes = meta.size;
const size_t runtime_bytes = ggml_nbytes(runtime_tensor);
if (model_bytes == 0 || runtime_bytes == 0) {
std::fprintf(stderr, "%s: %s sizes must be non-zero (model=%zu, runtime=%zu)\n",
__func__, tensor_name, model_bytes, runtime_bytes);
return false;
}
if (runtime_bytes != model_bytes) {
std::fprintf(stderr,
"%s: %s tensor size mismatch (runtime=%zu B, model=%zu B). "
"VitisAI .rai requires exact context match; use matching -ac/model artifact.\n",
__func__, tensor_name, runtime_bytes, model_bytes);
return false;
}
io_tensor.data = runtime_tensor->data;
return true;
}
bool whisper_vitisai_resolve_io_binding(
[[maybe_unused]] const char * caller,
const std::vector<flexmlrt::client::ErtTensorType> & input_tensors,
const std::vector<flexmlrt::client::ErtTensorType> & output_tensors,
whisper_vitisai_io_binding * binding,
std::string * error) {
const auto fail = [error](std::string message) {
if (error) {
*error = std::move(message);
}
return false;
};
if (input_tensors.empty()) {
return fail("Model has no input tensors");
}
binding->mel_in_idx = 0;
bool found_named_mel = false;
for (int i = 0; i < (int) input_tensors.size(); ++i) {
const std::string & name = input_tensors[i].getMetadata().name;
if (name == "input" || name == "mel") {
binding->mel_in_idx = i;
found_named_mel = true;
break;
}
}
if (!found_named_mel) {
#if defined(WHISPER_DEBUG)
std::fprintf(stderr, "%s: WARNING: mel input not found by name; falling back to input[0]\n", caller);
#endif
}
if (output_tensors.empty()) {
return fail("Model has no output tensors");
}
for (int i = 0; i < (int) output_tensors.size(); ++i) {
const std::string & name = output_tensors[i].getMetadata().name;
if (name == "embd_enc") {
binding->embd_enc_out_idx = i;
} else if (name == "cross_k") {
binding->cross_k_out_idx = i;
} else if (name == "cross_v") {
binding->cross_v_out_idx = i;
}
}
if (binding->embd_enc_out_idx < 0) {
#if defined(WHISPER_DEBUG)
std::fprintf(stderr, "%s: WARNING: embd_enc output not found by name; falling back to output[0]\n", caller);
#endif
binding->embd_enc_out_idx = 0;
}
const bool has_cross_k = binding->cross_k_out_idx >= 0;
const bool has_cross_v = binding->cross_v_out_idx >= 0;
if (has_cross_k != has_cross_v) {
return fail("Incomplete cross-projection contract: both cross_k and cross_v outputs are required");
}
if (has_cross_k && (binding->cross_k_out_idx == binding->cross_v_out_idx ||
binding->cross_k_out_idx == binding->embd_enc_out_idx ||
binding->cross_v_out_idx == binding->embd_enc_out_idx)) {
return fail("Invalid output mapping: embd_enc/cross_k/cross_v indices overlap");
}
const auto & mel_meta = input_tensors[binding->mel_in_idx].getMetadata();
if (!whisper_flexml_dtype_to_ggml_type(mel_meta.type, nullptr)) {
return fail(
std::string("Unsupported mel input type: ") +
whisper_flexml_dtype_name(mel_meta.type) + " (supported: Float32/Float16/BFloat16)");
}
binding->mel_in_expected_bytes = mel_meta.size;
const auto & embd_meta = output_tensors[binding->embd_enc_out_idx].getMetadata();
if (!whisper_flexml_dtype_to_ggml_type(embd_meta.type, nullptr)) {
return fail(
std::string("Unsupported embd_enc output type: ") +
whisper_flexml_dtype_name(embd_meta.type) + " (supported: Float32/Float16/BFloat16)");
}
binding->embd_enc_expected_bytes = embd_meta.size;
if (has_cross_k) {
const auto & cross_k_meta = output_tensors[binding->cross_k_out_idx].getMetadata();
const auto & cross_v_meta = output_tensors[binding->cross_v_out_idx].getMetadata();
if (cross_k_meta.type != flexmlrt::client::DataType::Float32 ||
cross_v_meta.type != flexmlrt::client::DataType::Float32) {
return fail(
std::string("Unsupported cross output type(s): cross_k=") +
whisper_flexml_dtype_name(cross_k_meta.type) + ", cross_v=" +
whisper_flexml_dtype_name(cross_v_meta.type) + " (cross path currently requires Float32)");
}
if (cross_k_meta.size != cross_v_meta.size) {
return fail("cross_k and cross_v output sizes do not match");
}
binding->cross_k_expected_bytes = cross_k_meta.size;
binding->cross_v_expected_bytes = cross_v_meta.size;
}
return true;
}
bool whisper_vitisai_all_tensors_claimed(
const char * caller,
const char * tensor_kind,
const std::vector<flexmlrt::client::ErtTensorType> & tensors,
const std::vector<bool> & claimed) {
for (size_t i = 0; i < tensors.size(); ++i) {
if (!claimed[i]) {
std::fprintf(stderr,
"%s: unsupported extra %s tensor at index %zu (name='%s'); strict contract expects only mapped %ss\n",
caller, tensor_kind, i, tensors[i].getMetadata().name.c_str(), tensor_kind);
return false;
}
}
return true;
}
void whisper_kv_cross_scale_k_f32(
float * k_data,
size_t count,
float kscale) {
for (size_t i = 0; i < count; ++i) {
k_data[i] *= kscale;
}
}
void whisper_kv_cross_store_layers_f32(
const float * src_k,
const float * src_v,
uint8_t * dst_k,
uint8_t * dst_v,
const whisper_kv_cross_layout & layout) {
for (int il = 0; il < layout.n_layer; ++il) {
const float * layer_src_k = src_k + (size_t)il * layout.src_layer_elems;
const float * layer_src_v = src_v + (size_t)il * layout.src_layer_elems;
float * dk = (float *)(dst_k + layout.dst_layer_stride * (size_t)il);
float * dv = (float *)(dst_v + layout.dst_layer_stride * (size_t)il);
for (size_t i = 0; i < layout.layer_elems; ++i) {
dk[i] = layer_src_k[i] * layout.kscale;
dv[i] = layer_src_v[i];
}
}
}
void whisper_kv_cross_store_layers_f16(
const float * src_k,
const float * src_v,
uint8_t * dst_k,
uint8_t * dst_v,
const whisper_kv_cross_layout & layout) {
for (int il = 0; il < layout.n_layer; ++il) {
const float * layer_src_k = src_k + (size_t)il * layout.src_layer_elems;
const float * layer_src_v = src_v + (size_t)il * layout.src_layer_elems;
ggml_fp16_t * dk = (ggml_fp16_t *)(dst_k + layout.dst_layer_stride * (size_t)il);
ggml_fp16_t * dv = (ggml_fp16_t *)(dst_v + layout.dst_layer_stride * (size_t)il);
for (size_t i = 0; i < layout.layer_elems; ++i) {
dk[i] = ggml_fp32_to_fp16(layer_src_k[i] * layout.kscale);
dv[i] = ggml_fp32_to_fp16(layer_src_v[i]);
}
}
}
void whisper_kv_cross_transpose_v_layers_f32(
const float * src_v,
uint8_t * dst_v,
const whisper_kv_cross_layout & layout) {
const int n_ctx = layout.n_ctx;
const int n_state = layout.n_state;
const int BLOCK = 32;
for (int il = 0; il < layout.n_layer; ++il) {
const float * layer_src_v = src_v + (size_t)il * layout.src_layer_elems;
float * dv = (float *)(dst_v + layout.dst_layer_stride * (size_t)il);
for (int ic = 0; ic < n_ctx; ic += BLOCK) {
for (int is = 0; is < n_state; is += BLOCK) {
const int ic_end = std::min(ic + BLOCK, n_ctx);
const int is_end = std::min(is + BLOCK, n_state);
for (int i = ic; i < ic_end; ++i) {
for (int j = is; j < is_end; ++j) {
dv[j * n_ctx + i] = layer_src_v[i * n_state + j];
}
}
}
}
}
}
void whisper_kv_cross_store_k_transpose_v_layers_f16(
const float * src_k,
const float * src_v,
uint8_t * dst_k,
uint8_t * dst_v,
const whisper_kv_cross_layout & layout) {
const int n_ctx = layout.n_ctx;
const int n_state = layout.n_state;
const int BLOCK = 32;
for (int il = 0; il < layout.n_layer; ++il) {
const float * layer_src_k = src_k + (size_t)il * layout.src_layer_elems;
const float * layer_src_v = src_v + (size_t)il * layout.src_layer_elems;
ggml_fp16_t * dk = (ggml_fp16_t *)(dst_k + layout.dst_layer_stride * (size_t)il);
ggml_fp16_t * dv = (ggml_fp16_t *)(dst_v + layout.dst_layer_stride * (size_t)il);
for (size_t i = 0; i < layout.layer_elems; ++i) {
dk[i] = ggml_fp32_to_fp16(layer_src_k[i] * layout.kscale);
}
for (int ic = 0; ic < n_ctx; ic += BLOCK) {
for (int is = 0; is < n_state; is += BLOCK) {
const int ic_end = std::min(ic + BLOCK, n_ctx);
const int is_end = std::min(is + BLOCK, n_state);
for (int i = ic; i < ic_end; ++i) {
for (int j = is; j < is_end; ++j) {
dv[j * n_ctx + i] = ggml_fp32_to_fp16(layer_src_v[i * n_state + j]);
}
}
}
}
}
}
} // namespace whisper_vitisai_helpers

117
src/vitisai/whisper-vitisai-helpers.h Normal file
View File

@ -0,0 +1,117 @@
#pragma once
#include "FlexMLClient.h"
#include "ggml.h"
#include <cstddef>
#include <cstdint>
#include <cstdio>
#include <string>
#include <vector>
namespace whisper_vitisai_helpers {
bool map_rai_file(const char * path, uint8_t ** buffer, size_t * size);
void unmap_rai_file(uint8_t * buffer, size_t size);
const char * whisper_kv_type_name(ggml_type type);
const char * whisper_flexml_dtype_name(flexmlrt::client::DataType type);
bool whisper_flexml_dtype_to_ggml_type(
flexmlrt::client::DataType type,
ggml_type * ggml_dtype);
bool whisper_validate_cross_shape(
const char * tensor_name,
const std::vector<std::uint32_t> & model_shape,
int n_text_layer,
int n_ctx,
int n_state);
bool whisper_vitisai_bind_tensor_data(
const char * tensor_name,
struct ggml_tensor * runtime_tensor,
const std::vector<size_t> & expected_shape,
flexmlrt::client::ErtTensorType & io_tensor);
#if defined(WHISPER_DEBUG)
template <typename T>
void whisper_vitisai_print_shape(const std::vector<T> & shape) {
std::fprintf(stderr, "[");
for (size_t i = 0; i < shape.size(); ++i) {
std::fprintf(stderr, "%s%lld", i == 0 ? "" : ", ", (long long) shape[i]);
}
std::fprintf(stderr, "]");
}
#endif
// Model IO tensor indices and metadata sizes resolved once at init time.
struct whisper_vitisai_io_binding {
int mel_in_idx = -1;
int embd_enc_out_idx = -1;
int cross_k_out_idx = -1;
int cross_v_out_idx = -1;
size_t mel_in_expected_bytes = 0;
size_t embd_enc_expected_bytes = 0;
size_t cross_k_expected_bytes = 0;
size_t cross_v_expected_bytes = 0;
};
// Warnings are printed with the caller's name; hard failures are returned in *error
// so the caller can decide how to report them.
bool whisper_vitisai_resolve_io_binding(
const char * caller,
const std::vector<flexmlrt::client::ErtTensorType> & input_tensors,
const std::vector<flexmlrt::client::ErtTensorType> & output_tensors,
whisper_vitisai_io_binding * binding,
std::string * error);
bool whisper_vitisai_all_tensors_claimed(
const char * caller,
const char * tensor_kind,
const std::vector<flexmlrt::client::ErtTensorType> & tensors,
const std::vector<bool> & claimed);
// Geometry of one cross K/V transfer from the model output (always f32, contiguous
// [ctx, state] per layer) into the runtime kv cache.
struct whisper_kv_cross_layout {
int n_layer = 0;
int n_ctx = 0;
int n_state = 0;
size_t src_layer_elems = 0; // f32 elements per layer in the model output buffer
size_t layer_elems = 0; // elements per layer transferred into the kv cache
size_t dst_layer_stride = 0; // bytes per layer in the kv cache
float kscale = 1.0f;
};
void whisper_kv_cross_scale_k_f32(
float * k_data,
size_t count,
float kscale);
void whisper_kv_cross_store_layers_f32(
const float * src_k,
const float * src_v,
uint8_t * dst_k,
uint8_t * dst_v,
const whisper_kv_cross_layout & layout);
void whisper_kv_cross_store_layers_f16(
const float * src_k,
const float * src_v,
uint8_t * dst_k,
uint8_t * dst_v,
const whisper_kv_cross_layout & layout);
void whisper_kv_cross_transpose_v_layers_f32(
const float * src_v,
uint8_t * dst_v,
const whisper_kv_cross_layout & layout);
void whisper_kv_cross_store_k_transpose_v_layers_f16(
const float * src_k,
const float * src_v,
uint8_t * dst_k,
uint8_t * dst_v,
const whisper_kv_cross_layout & layout);
} // namespace whisper_vitisai_helpers

91
src/whisper.cpp
View File

@ -14,6 +14,10 @@
#include "openvino/whisper-openvino-encoder.h"
#endif
#ifdef WHISPER_USE_VITISAI
#include "vitisai/whisper-vitisai-encoder.h"
#endif
#include <atomic>
#include <algorithm>
#include <cassert>
@ -903,6 +907,10 @@ struct whisper_state {
whisper_openvino_context * ctx_openvino = nullptr;
#endif
#ifdef WHISPER_USE_VITISAI
whisper_vitisai_context * ctx_vitisai = nullptr;
#endif
// [EXPERIMENTAL] token-level timestamps data
int64_t t_beg = 0;
int64_t t_last = 0;
@ -1976,7 +1984,25 @@ static bool whisper_encode_external(const whisper_state & wstate) {
const bool use_openvino = wstate.ctx_openvino != nullptr;
#endif
return use_coreml || use_openvino;
#ifndef WHISPER_USE_VITISAI
const bool use_vitisai = false;
#else
const bool use_vitisai = wstate.ctx_vitisai != nullptr;
#endif
return use_coreml || use_openvino || use_vitisai;
}
static bool whisper_cross_external(const whisper_state & wstate) {
GGML_UNUSED(wstate);
#if defined(WHISPER_USE_VITISAI)
const bool use_vitisai_cross = whisper_vitisai_has_cross_proj(wstate.ctx_vitisai);
#else
const bool use_vitisai_cross = false;
#endif
return use_vitisai_cross;
}
static struct ggml_cgraph * whisper_build_graph_conv(
@ -2417,6 +2443,21 @@ static bool whisper_encode_internal(
#if defined(WHISPER_USE_COREML)
whisper_coreml_encode(wstate.ctx_coreml, mel->ne[0], mel->ne[1], (float *) mel->data, (float *) wstate.embd_enc->data);
#elif defined(WHISPER_USE_VITISAI)
if (whisper_vitisai_has_cross_proj(wstate.ctx_vitisai)) {
const auto & hp = wctx.model.hparams;
const int n_ctx = wstate.exp_n_audio_ctx > 0
? wstate.exp_n_audio_ctx : hp.n_audio_ctx;
if (!whisper_vitisai_encode_with_cross(
wstate.ctx_vitisai, mel, wstate.embd_enc,
wstate.kv_cross.k, wstate.kv_cross.v,
hp.n_text_layer, n_ctx, hp.n_text_state,
hp.n_text_head, wctx.params.flash_attn)) {
return false;
}
} else if (!whisper_vitisai_encode(wstate.ctx_vitisai, mel, wstate.embd_enc)) {
return false;
}
#elif defined(WHISPER_USE_OPENVINO)
whisper_openvino_encode(wstate.ctx_openvino, mel, wstate.embd_enc);
#endif
@ -2440,7 +2481,7 @@ static bool whisper_encode_internal(
}
// cross
{
if (!whisper_cross_external(wstate)) {
auto & sched = wstate.sched_cross.sched;
ggml_cgraph * gf = whisper_build_graph_cross(wctx, wstate);
@ -3356,6 +3397,19 @@ static std::string whisper_get_coreml_path_encoder(std::string path_bin) {
}
#endif
#ifdef WHISPER_USE_VITISAI
// replace extension with Vitis AI encoder artifact. Cross projection support is
// detected from the model's output tensors, not from the file name.
static std::string whisper_get_vitisai_path_encoder_cache(std::string path_bin) {
auto pos = path_bin.rfind('.');
if (pos != std::string::npos) {
path_bin = path_bin.substr(0, pos);
}
return path_bin + "-encoder-vitisai.rai";
}
#endif
#ifdef WHISPER_USE_OPENVINO
// replace .bin with-encoder-openvino.xml
static std::string whisper_openvino_get_path_encoder(std::string path_bin) {
@ -3465,6 +3519,21 @@ struct whisper_state * whisper_init_state(whisper_context * ctx) {
}
#endif
#ifdef WHISPER_USE_VITISAI
const auto path_vitisai = whisper_get_vitisai_path_encoder_cache(ctx->path_model);
state->ctx_vitisai = whisper_vitisai_init(path_vitisai.c_str());
if (!state->ctx_vitisai) {
WHISPER_LOG_ERROR("%s: failed to load Vitis AI model from '%s'\n", __func__, path_vitisai.c_str());
whisper_free_state(state);
return nullptr;
} else if (whisper_vitisai_has_cross_proj(state->ctx_vitisai)) {
WHISPER_LOG_INFO("%s: Vitis AI encoder + cross projection model loaded\n", __func__);
} else {
WHISPER_LOG_INFO("%s: Vitis AI encoder model loaded\n", __func__);
}
#endif
state->logits.reserve(ctx->vocab.n_vocab * ctx->model.hparams.n_text_ctx);
state->batch = whisper_batch_init(ctx->model.hparams.n_text_ctx, WHISPER_MAX_DECODERS);
@ -3512,7 +3581,7 @@ struct whisper_state * whisper_init_state(whisper_context * ctx) {
}
// cross allocator
{
if (!whisper_cross_external(*state)) {
bool ok = whisper_sched_graph_init(state->sched_cross, state->backends,
[&]() {
return whisper_build_graph_cross(*ctx, *state);
@ -3845,6 +3914,13 @@ void whisper_free_state(struct whisper_state * state) {
}
#endif
#ifdef WHISPER_USE_VITISAI
if (state->ctx_vitisai != nullptr) {
whisper_vitisai_free(state->ctx_vitisai);
state->ctx_vitisai = nullptr;
}
#endif
whisper_batch_free(state->batch);
ggml_backend_sched_free(state->sched_conv.sched);
@ -4336,11 +4412,20 @@ static int whisper_has_openvino(void) {
#endif
}
static int whisper_has_vitisai(void) {
#ifdef WHISPER_USE_VITISAI
return 1;
#else
return 0;
#endif
}
const char * whisper_print_system_info(void) {
static std::string s;
s = "";
s += "WHISPER : ";
s += "VITISAI = " + std::to_string(whisper_has_vitisai()) + " | ";
s += "COREML = " + std::to_string(whisper_has_coreml()) + " | ";
s += "OPENVINO = " + std::to_string(whisper_has_openvino()) + " | ";