diff --git a/.github/workflows/build-self-hosted.yml b/.github/workflows/build-self-hosted.yml index 2286b63d6..b689df084 100644 --- a/.github/workflows/build-self-hosted.yml +++ b/.github/workflows/build-self-hosted.yml @@ -114,3 +114,146 @@ jobs: run: | vulkaninfo --summary GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/whisper.cpp ~/mnt/whisper.cpp + + npu-amd-windows: + runs-on: [self-hosted, Windows, X64, stx, rai300-400] + timeout-minutes: 60 + continue-on-error: true # advisory while the runner pool is new; revisit later + + env: + FLEXML_URL: https://github.com/lemonade-sdk/whisper.cpp-rocm/releases/download/deps/flexmlrt-1.7.0-win.zip + MODEL: base + + steps: + - name: Clone + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - uses: microsoft/setup-msbuild@v2 + + - name: Install CMake if not available + shell: powershell + run: | + $installed = Get-Command cmake -ErrorAction SilentlyContinue + if (-not $installed) { + $ver = "3.28.1" + $url = "https://github.com/Kitware/CMake/releases/download/v$ver/cmake-$ver-windows-x86_64.msi" + Invoke-WebRequest -Uri $url -OutFile cmake.msi + Start-Process msiexec.exe -ArgumentList "/i cmake.msi /quiet /norestart" -Wait + $p = "C:\Program Files\CMake\bin" + $env:PATH = "$p;$env:PATH" + echo $p >> $env:GITHUB_PATH + cmake --version + if ($LASTEXITCODE -ne 0) { Write-Error "CMake install failed"; exit 1 } + } else { cmake --version } + + - name: Download FlexML runtime + shell: powershell + run: | + Invoke-WebRequest -Uri "${{ env.FLEXML_URL }}" -OutFile flexmlrt.zip + if (-Not (Test-Path "flexmlrt.zip")) { Write-Error "flexmlrt.zip not downloaded"; exit 1 } + if ((Get-Item "flexmlrt.zip").Length -eq 0) { Write-Error "flexmlrt.zip is empty"; exit 1 } + tar xf flexmlrt.zip + if ($LASTEXITCODE -ne 0) { Write-Error "Extraction failed"; exit 1 } + if (-not (Test-Path "flexmlrt")) { Write-Error "No flexmlrt directory after extraction"; exit 1 } + + - name: Setup FlexML, configure and build + shell: cmd + run: | + cd flexmlrt + call setup.bat + if errorlevel 1 ( echo ERROR: FlexML setup.bat failed & exit /b 1 ) + cd .. + cmake -B build -A x64 -DCMAKE_BUILD_TYPE=Release -DWHISPER_VITISAI=ON + if errorlevel 1 ( echo ERROR: CMake configure failed & exit /b 1 ) + cmake --build build --config Release -j + if errorlevel 1 ( echo ERROR: Build failed & exit /b 1 ) + + - name: Copy FlexML DLLs to build output + shell: powershell + run: | + foreach ($d in "flexmlrt/bin", "flexmlrt/lib") { + if (Test-Path "$d/*.dll") { Copy-Item "$d/*.dll" "build/bin/Release/" -Force } + } + if (-not (Test-Path "build/bin/Release/flexmlrt.dll")) { + Write-Error "flexmlrt.dll not staged next to binaries"; exit 1 + } + + - name: Download ggml model + shell: cmd + run: | + call models\download-ggml-model.cmd %MODEL% models + if not exist models\ggml-%MODEL%.bin ( echo ERROR: model download failed & exit /b 1 ) + + - name: Download NPU encoder cache + shell: cmd + run: | + .\models\download-vitisai-model.cmd %MODEL% + if not exist models\ggml-%MODEL%-encoder-vitisai.rai ( echo ERROR: VitisAI encoder cache download failed & exit /b 1 ) + + - name: Run NPU smoke test + shell: cmd + run: | + build\bin\Release\whisper-cli.exe -m models\ggml-%MODEL%.bin -f samples\jfk.wav > vitisai.log 2>&1 + type vitisai.log + findstr /I /C:"vitisai" vitisai.log || ( echo ERROR: no VitisAI activity - encoder likely fell back to CPU & exit /b 1 ) + findstr /I /C:"ask not what your country" vitisai.log || ( echo ERROR: incorrect transcription & exit /b 1 ) + + - name: Upload smoke test log + if: always() + uses: actions/upload-artifact@v4 + with: + name: vitisai-smoke-log-windows + path: vitisai.log + + npu-amd-linux: + runs-on: [self-hosted, Linux, X64, stx, rai300-400] + timeout-minutes: 60 + continue-on-error: true # advisory while the runner pool is new; revisit later + + env: + FLEXML_LINUX_URL: https://github.com/lemonade-sdk/whisper.cpp-rocm/releases/download/deps/flexmlrt-1.8.0-linux.tar.gz + MODEL: base + + steps: + - name: Clone + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - name: Verify NPU device + run: | + lsmod | grep -q amdxdna || { echo "ERROR: amdxdna driver not loaded"; exit 1; } + ls /dev/accel/accel* || { echo "ERROR: no NPU accel device node"; exit 1; } + + - name: Download FlexML runtime (Linux) + run: | + curl -L --fail -o flexmlrt.tar.gz "$FLEXML_LINUX_URL" + tar xf flexmlrt.tar.gz + source flexmlrt/setup.sh + echo "FlexmlRT_DIR=$PWD/flexmlrt/share/cmake/FlexmlRT" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=$PWD/flexmlrt/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + + - name: Configure and build + run: | + cmake -B build -DCMAKE_BUILD_TYPE=Release -DWHISPER_VITISAI=ON + cmake --build build --config Release -j $(nproc) + + - name: Download ggml model + run: | + ./models/download-ggml-model.sh $MODEL + + - name: Download NPU encoder cache + run: | + sh ./models/download-vitisai-model.sh $MODEL + [ -f "models/ggml-$MODEL-encoder-vitisai.rai" ] || { echo "ERROR: VitisAI encoder cache download failed"; exit 1; } + + - name: Run NPU smoke test + run: | + ./build/bin/whisper-cli -m "models/ggml-$MODEL.bin" -f samples/jfk.wav 2>&1 | tee vitisai.log + grep -qi "vitisai" vitisai.log || { echo "ERROR: no VitisAI activity - CPU fallback?"; exit 1; } + grep -qi "ask not what your country" vitisai.log || { echo "ERROR: incorrect transcription"; exit 1; } + + - name: Upload smoke test log + if: always() + uses: actions/upload-artifact@v4 + with: + name: vitisai-smoke-log-linux + path: vitisai.log diff --git a/CMakeLists.txt b/CMakeLists.txt index 4c43e65b9..4f81a6f02 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -92,6 +92,7 @@ endif() option(WHISPER_COREML "whisper: enable Core ML framework" OFF) option(WHISPER_COREML_ALLOW_FALLBACK "whisper: allow non-CoreML fallback" OFF) option(WHISPER_OPENVINO "whisper: support for OpenVINO" OFF) +option(WHISPER_VITISAI "whisper: support for AMD Vitis AI" OFF) # Required for relocatable CMake package include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/build-info.cmake) diff --git a/README.md b/README.md index 2b62a6379..6627ed77a 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ High-performance inference of [OpenAI's Whisper](https://github.com/openai/whisp - Support for CPU-only inference - [Efficient GPU support for NVIDIA](#nvidia-gpu-support) - [AMD ROCm GPU support](#amd-rocm-gpu-support) +- [AMD Ryzen AI NPU Support](#amd-ryzen-ai-npu-support) - [OpenVINO Support](#openvino-support) - [Ascend NPU Support](#ascend-npu-support) - [Moore Threads GPU Support](#moore-threads-gpu-support) @@ -313,6 +314,87 @@ This can result in significant speedup in encoder performance. Here are the inst For more information about the OpenVINO implementation please refer to PR [#1037](https://github.com/ggml-org/whisper.cpp/pull/1037). +## AMD Ryzen™ AI NPU support + +On AMD Ryzen™ AI 300 and 400 Series processors with a dedicated NPU, whisper.cpp can fully offload the Whisper encoder to the NPU via VitisAI, delivering significant speedup over CPU-only inference. + +### Prerequisites + +Supported Platforms + +- **Windows 11** +- **Linux** (Ubuntu 24.04 LTS, Python 3.12) + +Install the XRT runtime and FlexML runtime for your platform: + +- **XRT**: provides the NPU kernel driver and `xrt-smi` diagnostic tool — on Windows this is bundled with the NPU driver; on Linux install it separately following the [NPU driver installation guide](https://ryzenai.docs.amd.com/en/latest/linux.html#install-npu-drivers) +- **FlexML runtime** (`flexmlrt`): VitisAI inference engine used by whisper.cpp — download from the [FlexML runtime releases](https://github.com/lemonade-sdk/whisper.cpp-rocm/releases/tag/deps) + +After installing, source the setup scripts in every shell you use to build or run whisper.cpp: + +```bash +# Linux +source /opt/xilinx/xrt/setup.sh +source /path/to/flexmlrt/setup.sh +``` + +```cmd +:: Windows +cd /path/to/flexmlrt && call setup.bat +``` + +You can verify the NPU is visible with: + +```bash +xrt-smi examine +``` + +### Download models + +Download the ggml model and the matching prebuilt VitisAI encoder cache: + +```bash +# Linux / macOS +sh ./models/download-ggml-model.sh base +sh ./models/download-vitisai-model.sh base +``` + +```cmd +:: Windows +.\models\download-ggml-model.cmd base +.\models\download-vitisai-model.cmd base +``` + +Use the same model name with both scripts. To see all available VitisAI encoder caches: + +```bash +sh ./models/download-vitisai-model.sh --list +``` + +```cmd +.\models\download-vitisai-model.cmd --list +``` + +The VitisAI script queries the [AMD Ryzen AI Whisper NPU collection on Hugging Face](https://huggingface.co/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models) and downloads the `.rai` encoder cache as `models/ggml--encoder-vitisai.rai`. + +> Depending on the `.rai` cache, VitisAI may offload the encoder only, or the encoder plus cross-projection layers. whisper.cpp detects this at runtime and logs the selected offload mode during model initialization. + +### Build + +```bash +cmake -B build -DWHISPER_VITISAI=1 +cmake --build build -j --config Release +``` + +### Run + +```bash +./build/bin/whisper-cli -m models/ggml-base.bin -f samples/jfk.wav +``` + +For more information see the [Ryzen AI documentation](https://ryzenai.docs.amd.com/en/latest/). + + ## NVIDIA GPU support With NVIDIA cards the processing of the models is done efficiently on the GPU via cuBLAS and custom CUDA kernels. diff --git a/models/download-vitisai-model.cmd b/models/download-vitisai-model.cmd new file mode 100644 index 000000000..0b768cd1a --- /dev/null +++ b/models/download-vitisai-model.cmd @@ -0,0 +1,32 @@ +@echo off +setlocal + +set "script=%~dp0download-vitisai-model.ps1" + +if "%~1"=="" ( + PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" + exit /b %ERRORLEVEL% +) + +if /I "%~1"=="--list" ( + PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" -List + exit /b %ERRORLEVEL% +) + +if /I "%~1"=="-l" ( + PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" -List + exit /b %ERRORLEVEL% +) + +if /I "%~1"=="list" ( + PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" -List + exit /b %ERRORLEVEL% +) + +if "%~2"=="" ( + PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" -Model "%~1" +) else ( + PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" -Model "%~1" -ModelsPath "%~2" +) + +exit /b %ERRORLEVEL% diff --git a/models/download-vitisai-model.ps1 b/models/download-vitisai-model.ps1 new file mode 100644 index 000000000..5408248cf --- /dev/null +++ b/models/download-vitisai-model.ps1 @@ -0,0 +1,218 @@ +param( + [Parameter(Position = 0)] + [string] $Model, + + [Parameter(Position = 1)] + [string] $ModelsPath, + + [switch] $List +) + +$ErrorActionPreference = "Stop" + +$Source = "https://huggingface.co" +$CollectionApi = "https://huggingface.co/api/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models" +$CollectionUrl = "https://huggingface.co/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models" + +$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +if ($ScriptDir -match "\\bin$") { + $DefaultDownloadPath = (Get-Location).Path +} else { + $DefaultDownloadPath = $ScriptDir +} + +if (-not $ModelsPath) { + $ModelsPath = $DefaultDownloadPath +} + +function Get-HfHeaders { + $headers = @{} + if ($env:HF_TOKEN) { + $headers["Authorization"] = "Bearer $env:HF_TOKEN" + } + return $headers +} + +function Invoke-HfJson { + param([string] $Uri) + + $headers = Get-HfHeaders + if ($headers.Count -gt 0) { + return Invoke-RestMethod -Uri $Uri -Headers $headers + } + + return Invoke-RestMethod -Uri $Uri +} + +function Normalize-ModelName { + param([string] $Name) + + # HF currently publishes ggml-small-en-encoder-vitisai.rai, while the + # matching ggml model is ggml-small.en.bin. + if ($Name.EndsWith("-en")) { + return $Name.Substring(0, $Name.Length - 3) + ".en" + } + + return $Name +} + +function Get-VitisAiModels { + $collection = Invoke-HfJson -Uri $CollectionApi + $seen = @{} + $rows = New-Object System.Collections.Generic.List[object] + + foreach ($item in $collection.items) { + if ($item.type -ne "model") { + continue + } + + $repo = $item.id + if (-not $repo) { + continue + } + + $modelInfo = Invoke-HfJson -Uri "$Source/api/models/$repo" + foreach ($sibling in $modelInfo.siblings) { + $filename = [string] $sibling.rfilename + $match = [regex]::Match($filename, "^ggml-(.+)-encoder-vitisai\.rai$") + if (-not $match.Success) { + continue + } + + $rawName = $match.Groups[1].Value + $modelName = Normalize-ModelName -Name $rawName + if ($seen.ContainsKey($modelName)) { + continue + } + + $seen[$modelName] = $true + $destination = "ggml-$modelName-encoder-vitisai.rai" + $url = "$Source/$repo/resolve/main/$([uri]::EscapeDataString($filename))" + + $rows.Add([pscustomobject]@{ + Model = $modelName + RawName = $rawName + Repo = $repo + SourceFile = $filename + DestinationFile = $destination + DownloadUrl = $url + }) + } + } + + $order = @{ + "tiny" = 10 + "tiny.en" = 11 + "base" = 20 + "base.en" = 21 + "small" = 30 + "small.en" = 31 + "medium" = 40 + "medium.en" = 41 + "large-v1" = 50 + "large-v2" = 60 + "large-v3" = 70 + "large-v3-turbo" = 80 + } + + return $rows | Sort-Object ` + @{ Expression = { if ($order.ContainsKey($_.Model)) { $order[$_.Model] } else { 1000 } } }, ` + @{ Expression = { $_.Model } } +} + +function Show-Models { + $models = Get-VitisAiModels + + Write-Host "" + Write-Host "Available VitisAI encoder caches from ${CollectionUrl}:" + foreach ($entry in $models) { + if ($entry.Model -eq $entry.RawName) { + Write-Host (" {0,-18} {1}" -f $entry.Model, $entry.Repo) + } else { + Write-Host (" {0,-18} {1} (source name: {2})" -f $entry.Model, $entry.Repo, $entry.RawName) + } + } + Write-Host "" +} + +function Show-Usage { + Write-Host "Usage: download-vitisai-model.cmd --list" + Write-Host " download-vitisai-model.cmd [models_path]" + Write-Host "" + Write-Host "Downloads ggml--encoder-vitisai.rai next to ggml-.bin." + Write-Host "Use the same model name as download-ggml-model.cmd." + Write-Host "" +} + +if ($List -or $Model -eq "--list" -or $Model -eq "-l" -or $Model -eq "list") { + Show-Models + exit 0 +} + +if (-not $Model) { + Show-Usage + Show-Models + exit 1 +} + +$models = Get-VitisAiModels +$entry = $models | Where-Object { $_.Model -eq $Model -or $_.RawName -eq $Model } | Select-Object -First 1 +if (-not $entry) { + Write-Host "Invalid model: $Model" + foreach ($available in $models) { + Write-Host " $($available.Model)" + } + exit 1 +} + +New-Item -ItemType Directory -Force -Path $ModelsPath | Out-Null +$destinationPath = Join-Path $ModelsPath $entry.DestinationFile + +Write-Host "Downloading VitisAI encoder cache $($entry.Model) from '$($entry.Repo)' ..." +if (Test-Path $destinationPath) { + Write-Host "VitisAI encoder cache $($entry.DestinationFile) already exists. Skipping download." + exit 0 +} + +$headers = Get-HfHeaders +$downloaded = $false +for ($attempt = 1; $attempt -le 5; ++$attempt) { + try { + if ($headers.Count -gt 0) { + Invoke-WebRequest -Uri $entry.DownloadUrl -Headers $headers -OutFile $destinationPath + } else { + Invoke-WebRequest -Uri $entry.DownloadUrl -OutFile $destinationPath + } + $downloaded = $true + break + } catch { + if ($attempt -eq 5) { + if (Test-Path $destinationPath) { + Remove-Item -Force $destinationPath + } + Write-Host "Failed to download VitisAI encoder cache $($entry.Model) from $($entry.DownloadUrl)" + throw + } + Start-Sleep -Seconds 5 + } +} + +if (-not $downloaded) { + exit 1 +} + +$whisperCmd = "whisper-cli" +if (-not (Get-Command whisper-cli -ErrorAction SilentlyContinue)) { + $rootPath = Split-Path -Parent $ScriptDir + $whisperCmd = Join-Path $rootPath "build\bin\Release\whisper-cli.exe" +} + +Write-Host "Done! VitisAI encoder cache '$($entry.Model)' saved in '$destinationPath'" +if ($entry.RawName -ne $entry.Model) { + Write-Host "Source cache '$($entry.SourceFile)' was renamed to match ggml model name '$($entry.Model)'." +} +Write-Host "Use it with the matching ggml model:" +Write-Host "" +Write-Host " $ScriptDir\download-ggml-model.cmd $($entry.Model) $ModelsPath" +Write-Host " $whisperCmd -m $ModelsPath\ggml-$($entry.Model).bin -f samples\jfk.wav" +Write-Host "" diff --git a/models/download-vitisai-model.sh b/models/download-vitisai-model.sh new file mode 100755 index 000000000..ddae36d22 --- /dev/null +++ b/models/download-vitisai-model.sh @@ -0,0 +1,226 @@ +#!/bin/sh + +# This script downloads prebuilt VitisAI encoder cache files for Whisper models. +# The cache file is saved next to the ggml model file and follows the loader +# convention: ggml--encoder-vitisai.rai + +src="https://huggingface.co" +collection_api="https://huggingface.co/api/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models" +collection_url="https://huggingface.co/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models" + +BOLD="\033[1m" +RESET='\033[0m' + +# get the path of this script +get_script_path() { + if [ -x "$(command -v realpath)" ]; then + dirname "$(realpath "$0")" + else + _ret="$(cd -- "$(dirname "$0")" >/dev/null 2>&1 || exit ; pwd -P)" + echo "$_ret" + fi +} + +find_python() { + if command -v python3 >/dev/null 2>&1; then + printf "%s\n" "python3" + elif command -v python >/dev/null 2>&1; then + printf "%s\n" "python" + else + return 1 + fi +} + +script_path="$(get_script_path)" + +# Check if the script is inside a /bin/ directory +case "$script_path" in + */bin) default_download_path="$PWD" ;; # Use current directory as default download path if in /bin/ + *) default_download_path="$script_path" ;; # Otherwise, use script directory +esac + +models_path="${2:-$default_download_path}" + +discover_models() { + python_cmd="$(find_python)" || { + printf "Python is required to query available VitisAI caches from Hugging Face.\n" >&2 + return 1 + } + + "$python_cmd" - "$src" "$collection_api" <<'PY' +import json +import os +import re +import sys +import urllib.parse +import urllib.request + +src = sys.argv[1].rstrip("/") +collection_api = sys.argv[2] +headers = {} +token = os.environ.get("HF_TOKEN") +if token: + headers["Authorization"] = "Bearer " + token + +def load_json(url): + req = urllib.request.Request(url, headers=headers) + with urllib.request.urlopen(req) as response: + return json.load(response) + +def normalize_model_name(name): + # HF currently publishes ggml-small-en-encoder-vitisai.rai, while the + # matching ggml model is ggml-small.en.bin. + if name.endswith("-en"): + return name[:-3] + ".en" + return name + +collection = load_json(collection_api) +rows = [] +seen = set() + +for item in collection.get("items", []): + if item.get("type") != "model": + continue + + repo = item.get("id") + if not repo: + continue + + model_info = load_json(src + "/api/models/" + repo) + for sibling in model_info.get("siblings", []): + filename = sibling.get("rfilename", "") + match = re.match(r"^ggml-(.+)-encoder-vitisai\.rai$", filename) + if not match: + continue + + raw_name = match.group(1) + model_name = normalize_model_name(raw_name) + if model_name in seen: + continue + seen.add(model_name) + + destination = "ggml-%s-encoder-vitisai.rai" % model_name + url = "%s/%s/resolve/main/%s" % (src, repo, urllib.parse.quote(filename)) + rows.append((model_name, raw_name, repo, filename, destination, url)) + +order = { + "tiny": 10, + "tiny.en": 11, + "base": 20, + "base.en": 21, + "small": 30, + "small.en": 31, + "medium": 40, + "medium.en": 41, + "large-v1": 50, + "large-v2": 60, + "large-v3": 70, + "large-v3-turbo": 80, +} + +for row in sorted(rows, key=lambda item: (order.get(item[0], 1000), item[0])): + print("|".join(row)) +PY +} + +list_models() { + models="$(discover_models)" || exit 1 + + printf "\n" + printf "Available VitisAI encoder caches from %s:\n" "$collection_url" + printf "%s\n" "$models" | while IFS='|' read -r model raw repo _source _destination _url; do + if [ "$model" = "$raw" ]; then + printf " %-18s %s\n" "$model" "$repo" + else + printf " %-18s %s (source name: %s)\n" "$model" "$repo" "$raw" + fi + done + printf "\n" +} + +usage() { + printf "Usage: %s --list\n" "$0" + printf " %s [models_path]\n" "$0" + printf "\n" + printf "Downloads ggml--encoder-vitisai.rai next to ggml-.bin.\n" + printf "Use the same model name as %s/download-ggml-model.sh.\n" "$script_path" + printf "\n" +} + +if [ "$#" -eq 1 ] && { [ "$1" = "--list" ] || [ "$1" = "-l" ] || [ "$1" = "list" ]; }; then + list_models + exit 0 +fi + +if [ "$#" -lt 1 ] || [ "$#" -gt 2 ]; then + usage + list_models + printf "___________________________________________________________\n" + printf "Example: %s ${BOLD}small${RESET} %s\n" "$0" "$default_download_path" + exit 1 +fi + +model=$1 +models="$(discover_models)" || exit 1 + +match="$(printf "%s\n" "$models" | awk -F '|' -v model="$model" '$1 == model || $2 == model { print; exit }')" +if [ -z "$match" ]; then + printf "Invalid model: %s\n" "$model" + printf "%s\n" "$models" | while IFS='|' read -r available _raw _repo _source _destination _url; do + printf " %s\n" "$available" + done + exit 1 +fi + +IFS='|' read -r model raw_name repo source_file destination_file download_url </dev/null 2>&1; then + whisper_cmd="whisper-cli" +else + whisper_cmd="./build/bin/whisper-cli" +fi + +printf "Done! VitisAI encoder cache '%s' saved in '%s/%s'\n" "$model" "$models_path" "$destination_file" +if [ "$raw_name" != "$model" ]; then + printf "Source cache '%s' was renamed to match ggml model name '%s'.\n" "$source_file" "$model" +fi +printf "Use it with the matching ggml model:\n\n" +printf " $ %s/download-ggml-model.sh %s %s\n" "$script_path" "$model" "$models_path" +printf " $ %s -m %s/ggml-%s.bin -f samples/jfk.wav\n" "$whisper_cmd" "$models_path" "$model" +printf "\n" diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4e7c5b24d..2ae7896f5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -48,6 +48,61 @@ if (WHISPER_OPENVINO) find_package(OpenVINO REQUIRED COMPONENTS Runtime) endif() +if (WHISPER_VITISAI) + find_package(FlexmlRT REQUIRED) + + # Legacy RAI overrides are required by FlexMLRT older than 1.8.0 + set(WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES_MODE "AUTO" CACHE STRING + "Legacy RAI override mode for FlexMLRT (AUTO|ON|OFF)") + set_property(CACHE WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES_MODE PROPERTY STRINGS AUTO ON OFF) + + string(TOUPPER "${WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES_MODE}" _flexmlrt_legacy_mode) + set(_flexmlrt_legacy_hint "Set -DWHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES_MODE=ON or OFF explicitly.") + + if (NOT _flexmlrt_legacy_mode MATCHES "^(AUTO|ON|OFF)$") + message(FATAL_ERROR + "Invalid WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES_MODE='${WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES_MODE}'. " + "Expected AUTO, ON, or OFF.") + endif() + + if (_flexmlrt_legacy_mode STREQUAL "AUTO") + if (NOT FlexmlRT_DIR) + message(FATAL_ERROR + "FlexmlRT_DIR is unset after find_package(FlexmlRT). ${_flexmlrt_legacy_hint}") + endif() + + # FlexmlRT_DIR points to /share/cmake/FlexmlRT. + get_filename_component(_flexmlrt_init_py "${FlexmlRT_DIR}/../../../__init__.py" ABSOLUTE) + if (NOT EXISTS "${_flexmlrt_init_py}") + message(FATAL_ERROR + "flexmlrt __init__.py not found at ${_flexmlrt_init_py}. ${_flexmlrt_legacy_hint}") + endif() + set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${_flexmlrt_init_py}") + + file(STRINGS "${_flexmlrt_init_py}" _flexmlrt_version_lines + REGEX "^VERSION[ \t]*=[ \t]*\"[0-9]+\\.[0-9]+\\.[0-9]+") + if (NOT _flexmlrt_version_lines MATCHES "\"([0-9]+\\.[0-9]+\\.[0-9]+)") + message(FATAL_ERROR + "Could not parse flexmlrt VERSION from ${_flexmlrt_init_py}. ${_flexmlrt_legacy_hint}") + endif() + set(_flexmlrt_version "${CMAKE_MATCH_1}") + + if (_flexmlrt_version VERSION_LESS "1.8.0") + set(WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES 1) + else() + set(WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES 0) + endif() + message(STATUS "Detected flexmlrt VERSION=${_flexmlrt_version} from ${_flexmlrt_init_py} (legacy overrides=${WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES})") + else() + if (_flexmlrt_legacy_mode STREQUAL "ON") + set(WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES 1) + else() + set(WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES 0) + endif() + message(STATUS "FlexMLRT legacy RAI overrides forced ${_flexmlrt_legacy_mode}") + endif() +endif() + # # libraries # @@ -101,6 +156,35 @@ if (WHISPER_OPENVINO) set_target_properties(${TARGET} PROPERTIES FOLDER "libs") endif() +if (WHISPER_VITISAI) + set(TARGET whisper.vitisai) + + add_library(${TARGET} OBJECT + vitisai/whisper-vitisai-helpers.h + vitisai/whisper-vitisai-helpers.cpp + vitisai/whisper-vitisai-encoder.h + vitisai/whisper-vitisai-encoder.cpp + ) + + target_include_directories(${TARGET} PUBLIC + . + ) + + set_property(TARGET ${TARGET} PROPERTY POSITION_INDEPENDENT_CODE ON) + set(WHISPER_EXTRA_FLAGS ${WHISPER_EXTRA_FLAGS} -DWHISPER_USE_VITISAI) + + # FlexMLRT headers and this plugin require C++17. Keep it PRIVATE so the + # C++11 requirement of the whisper target is not bumped. + target_compile_features(${TARGET} PRIVATE cxx_std_17) + + target_compile_definitions(${TARGET} PRIVATE + WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES=${WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES} + ) + + target_link_libraries(${TARGET} PRIVATE ggml flexmlrt::flexmlrt) + set_target_properties(${TARGET} PROPERTIES FOLDER "libs") +endif() + # whisper add_library(whisper @@ -157,6 +241,10 @@ if (WHISPER_OPENVINO) target_link_libraries(whisper PRIVATE whisper.openvino) endif() +if (WHISPER_VITISAI) + target_link_libraries(whisper PRIVATE whisper.vitisai) +endif() + if (WHISPER_MKL) target_link_libraries(whisper PRIVATE MKL::MKL) endif() diff --git a/src/vitisai/whisper-vitisai-encoder.cpp b/src/vitisai/whisper-vitisai-encoder.cpp new file mode 100644 index 000000000..24db7dcdf --- /dev/null +++ b/src/vitisai/whisper-vitisai-encoder.cpp @@ -0,0 +1,602 @@ +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#endif + +#include "vitisai/whisper-vitisai-encoder.h" +#include "vitisai/whisper-vitisai-helpers.h" +#include "FlexMLClient.h" +#include "ggml.h" + +#include +#include +#include +#include +#include +#include + +#if defined(WHISPER_DEBUG) +#define WHISPER_DBG_TIMER(name) const int64_t name = ggml_time_us() +#else +#define WHISPER_DBG_TIMER(name) do {} while (0) +#endif + +struct whisper_vitisai_context { + std::string model_path; + std::shared_ptr runner; + uint8_t * fbs_buffer = nullptr; + size_t fbs_buffer_size = 0; + + std::vector cross_k_staging; + std::vector cross_v_staging; + + int mel_in_idx = -1; + int embd_enc_out_idx = -1; + int cross_k_out_idx = -1; + int cross_v_out_idx = -1; + size_t mel_in_expected_bytes = 0; + size_t embd_enc_expected_bytes = 0; + size_t cross_k_expected_bytes = 0; + size_t cross_v_expected_bytes = 0; + + std::vector cached_input_tensors; + std::vector cached_output_tensors; +}; + +// Return cached IO tensor descriptors by reference to avoid per-call deep copies. +static bool whisper_vitisai_get_cached_io_tensors( + struct whisper_vitisai_context * ctx, + std::vector *& input_tensors, + std::vector *& output_tensors) { + if (!ctx || !ctx->runner) { + return false; + } + + if (ctx->cached_input_tensors.empty() || ctx->cached_output_tensors.empty()) { + ctx->cached_input_tensors = ctx->runner->getIOTensors("input", false); + ctx->cached_output_tensors = ctx->runner->getIOTensors("output", false); + } + + input_tensors = &ctx->cached_input_tensors; + output_tensors = &ctx->cached_output_tensors; + return true; +} + +struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model) { + if (!path_model) { + std::fprintf(stderr, "%s: path_model is null\n", __func__); + return nullptr; + } + + auto * ctx = new whisper_vitisai_context; + ctx->model_path = path_model; + + // Override the model path with the environment variable if it is set + if (const char * env_model_path = std::getenv("OVERRIDE_VITISAI_MODEL_PATH")) { + if (env_model_path[0] != '\0') { + ctx->model_path = env_model_path; + } + } + + // Step 1: Set up the model + flexmlrt::client::Options options; + options.modelPath = ctx->model_path; + options.debug = false; + options.executeMode = 2; + options.extOptions["enable_preemption"] = true; + + const bool model_is_rai = ctx->model_path.find(".rai") != std::string::npos; + + // Check if model_path is rai file and if so, add fbs_buffer and fbs_buffer_size to the options + if (model_is_rai) { + if (whisper_vitisai_helpers::map_rai_file(ctx->model_path.c_str(), &ctx->fbs_buffer, &ctx->fbs_buffer_size)) { + options.extOptions["fbs_buffer"] = ctx->fbs_buffer; + options.extOptions["fbs_buffer_size"] = ctx->fbs_buffer_size; + options.extOptions["cache_dir"] = std::string("."); + } else { + std::fprintf(stderr, "%s: Failed to mmap rai file '%s'\n", __func__, ctx->model_path.c_str()); + delete ctx; + return nullptr; + } + } else { + options.deviceName = "stx"; +#if defined(WHISPER_DEBUG) + std::fprintf(stderr, "%s: Using default device name 'stx'\n", __func__); +#endif + } + + if (model_is_rai) { +#if WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES + options.deviceName = "stx"; + options.subgraphName = "vaiml_par_0"; +#if defined(WHISPER_DEBUG) + std::fprintf(stderr, + "%s: legacy FlexMLRT compile configuration detected; applying RAI overrides (device='stx', subgraph='vaiml_par_0')\n", + __func__); +#endif // defined(WHISPER_DEBUG) +#endif + } + + try { + ctx->runner = std::make_shared(options); + if (!ctx->runner || !ctx->runner->good()) { + throw std::runtime_error("Runner creation ran into an error"); + } + + ctx->cached_input_tensors = ctx->runner->getIOTensors("input", false); + ctx->cached_output_tensors = ctx->runner->getIOTensors("output", false); + + auto & input_tensors = ctx->cached_input_tensors; + auto & output_tensors = ctx->cached_output_tensors; + + whisper_vitisai_helpers::whisper_vitisai_io_binding binding; + std::string binding_error; + if (!whisper_vitisai_helpers::whisper_vitisai_resolve_io_binding( + __func__, input_tensors, output_tensors, &binding, &binding_error)) { + throw std::runtime_error(binding_error); + } + + ctx->mel_in_idx = binding.mel_in_idx; + ctx->embd_enc_out_idx = binding.embd_enc_out_idx; + ctx->cross_k_out_idx = binding.cross_k_out_idx; + ctx->cross_v_out_idx = binding.cross_v_out_idx; + ctx->mel_in_expected_bytes = binding.mel_in_expected_bytes; + ctx->embd_enc_expected_bytes = binding.embd_enc_expected_bytes; + ctx->cross_k_expected_bytes = binding.cross_k_expected_bytes; + ctx->cross_v_expected_bytes = binding.cross_v_expected_bytes; + +#if defined(WHISPER_DEBUG) + { + std::fprintf(stderr, "%s: model has %zu input tensor(s)\n", __func__, input_tensors.size()); + for (int i = 0; i < (int) input_tensors.size(); ++i) { + const auto & meta = input_tensors[i].getMetadata(); + std::fprintf(stderr, "%s: input[%d] name='%s' size=%zu shape=", + __func__, i, meta.name.c_str(), (size_t) meta.size); + whisper_vitisai_helpers::whisper_vitisai_print_shape(meta.shape); + std::fprintf(stderr, "\n"); + } + + std::fprintf(stderr, "%s: model has %zu output tensor(s)\n", __func__, output_tensors.size()); + for (int i = 0; i < (int) output_tensors.size(); ++i) { + const auto & meta = output_tensors[i].getMetadata(); + std::fprintf(stderr, "%s: output[%d] name='%s' size=%zu shape=", + __func__, i, meta.name.c_str(), (size_t) meta.size); + whisper_vitisai_helpers::whisper_vitisai_print_shape(meta.shape); + std::fprintf(stderr, "\n"); + } + + std::fprintf(stderr, "%s: input index: mel=%d\n", __func__, ctx->mel_in_idx); + std::fprintf(stderr, "%s: output indices: embd_enc=%d cross_k=%d cross_v=%d\n", + __func__, ctx->embd_enc_out_idx, ctx->cross_k_out_idx, ctx->cross_v_out_idx); + } +#endif + } catch (const std::exception & e) { + std::fprintf(stderr, "%s: Exception during Vitis AI runner creation: %s\n", __func__, e.what()); + whisper_vitisai_free(ctx); + return nullptr; + } + return ctx; +} + +bool whisper_vitisai_has_cross_proj(const struct whisper_vitisai_context * ctx) { + return ctx && ctx->cross_k_out_idx >= 0 && ctx->cross_v_out_idx >= 0; +} + +void whisper_vitisai_free(struct whisper_vitisai_context * ctx) { + if (!ctx) { + return; + } + +#if defined(WHISPER_DEBUG) + std::fprintf(stderr, "%s: releasing Vitis AI context for model '%s'\n", __func__, ctx->model_path.c_str()); +#endif + if (ctx->fbs_buffer) { + whisper_vitisai_helpers::unmap_rai_file(ctx->fbs_buffer, ctx->fbs_buffer_size); + } + delete ctx; +} + +static int whisper_vitisai_forward_impl( + struct whisper_vitisai_context * ctx, + struct ggml_tensor * mel, + struct ggml_tensor * out, + std::vector & input_tensors, + std::vector & output_tensors, + void * cross_k_data, + void * cross_v_data) { + if (!ctx || !mel || !out) { + std::fprintf(stderr, "%s: ctx/mel/out must not be null\n", __func__); + return 0; + } + + const bool with_cross = (cross_k_data != nullptr || cross_v_data != nullptr); + if (with_cross && (!cross_k_data || !cross_v_data)) { + std::fprintf(stderr, "%s: cross_k_data/cross_v_data must both be set\n", __func__); + return 0; + } + + if (ggml_n_dims(mel) != 2) { + std::fprintf(stderr, "%s: mel tensor expected to have 2 dims, got %d\n", __func__, ggml_n_dims(mel)); + return 0; + } + + if (ggml_n_dims(out) != 2) { + std::fprintf(stderr, "%s: out tensor expected to have 2 dims, got %d\n", __func__, ggml_n_dims(out)); + return 0; + } + + if (ctx->embd_enc_out_idx < 0 || ctx->embd_enc_out_idx >= (int) output_tensors.size()) { + std::fprintf(stderr, "%s: invalid embd_enc output index %d for %zu output tensor(s)\n", + __func__, ctx->embd_enc_out_idx, output_tensors.size()); + return 0; + } + + if (ctx->mel_in_idx < 0 || ctx->mel_in_idx >= (int) input_tensors.size()) { + std::fprintf(stderr, "%s: invalid mel input index %d for %zu input tensor(s)\n", + __func__, ctx->mel_in_idx, input_tensors.size()); + return 0; + } + + if (!whisper_vitisai_helpers::whisper_vitisai_bind_tensor_data( + "mel input", + mel, + { (size_t) mel->ne[1], (size_t) mel->ne[0] }, + input_tensors[ctx->mel_in_idx])) { + return 0; + } + + if (!whisper_vitisai_helpers::whisper_vitisai_bind_tensor_data( + "embd_enc output", + out, + { (size_t) out->ne[1], (size_t) out->ne[0] }, + output_tensors[ctx->embd_enc_out_idx])) { + return 0; + } + + std::vector claimed_inputs(input_tensors.size(), false); + claimed_inputs[ctx->mel_in_idx] = true; + if (!whisper_vitisai_helpers::whisper_vitisai_all_tensors_claimed( + __func__, "input", input_tensors, claimed_inputs)) { + return 0; + } + + std::vector claimed_outputs(output_tensors.size(), false); + claimed_outputs[ctx->embd_enc_out_idx] = true; + if (with_cross) { + if (ctx->cross_k_out_idx < 0 || ctx->cross_k_out_idx >= (int) output_tensors.size() || + ctx->cross_v_out_idx < 0 || ctx->cross_v_out_idx >= (int) output_tensors.size()) { + std::fprintf(stderr, "%s: invalid cross output indices cross_k=%d cross_v=%d for %zu output tensor(s)\n", + __func__, ctx->cross_k_out_idx, ctx->cross_v_out_idx, output_tensors.size()); + return 0; + } + output_tensors[ctx->cross_k_out_idx].data = cross_k_data; + output_tensors[ctx->cross_v_out_idx].data = cross_v_data; + claimed_outputs[ctx->cross_k_out_idx] = true; + claimed_outputs[ctx->cross_v_out_idx] = true; + } + if (!whisper_vitisai_helpers::whisper_vitisai_all_tensors_claimed( + __func__, "output", output_tensors, claimed_outputs)) { + return 0; + } + + auto clear_bound_data = [&]() { + input_tensors[ctx->mel_in_idx].data = nullptr; + output_tensors[ctx->embd_enc_out_idx].data = nullptr; + if (with_cross) { + output_tensors[ctx->cross_k_out_idx].data = nullptr; + output_tensors[ctx->cross_v_out_idx].data = nullptr; + } + }; + + try { + ctx->runner->forward(input_tensors, output_tensors); + clear_bound_data(); +#if defined(WHISPER_DEBUG) + std::fprintf(stderr, "%s: Vitis AI model inference %scompleted.\n", + __func__, with_cross ? "(encoder + cross proj) " : ""); +#endif + } catch (const std::exception & e) { + clear_bound_data(); + std::fprintf(stderr, "%s: Exception during model inference: %s\n", __func__, e.what()); + return 0; + } + + return 1; +} + +int whisper_vitisai_encode(struct whisper_vitisai_context * ctx, struct ggml_tensor * mel, struct ggml_tensor * out) { + std::vector * input_tensors_cached = nullptr; + std::vector * output_tensors_cached = nullptr; + if (!whisper_vitisai_get_cached_io_tensors(ctx, input_tensors_cached, output_tensors_cached)) { + std::fprintf(stderr, "%s: failed to acquire Vitis AI I/O tensors\n", __func__); + return 0; + } + + std::vector input_tensors = *input_tensors_cached; + std::vector output_tensors = *output_tensors_cached; + + return whisper_vitisai_forward_impl( + ctx, + mel, + out, + input_tensors, + output_tensors, + nullptr, + nullptr); +} + +int whisper_vitisai_run_enc_cross( + struct whisper_vitisai_context * ctx, + struct ggml_tensor * mel, + struct ggml_tensor * out, + void * cross_v_data, + void * cross_k_data) { + if (!cross_v_data || !cross_k_data) { + std::fprintf(stderr, "%s: cross_v_data/cross_k_data must not be null\n", __func__); + return 0; + } + + std::vector * input_tensors_cached = nullptr; + std::vector * output_tensors_cached = nullptr; + if (!whisper_vitisai_get_cached_io_tensors(ctx, input_tensors_cached, output_tensors_cached)) { + std::fprintf(stderr, "%s: failed to acquire Vitis AI I/O tensors\n", __func__); + return 0; + } + + std::vector input_tensors = *input_tensors_cached; + std::vector output_tensors = *output_tensors_cached; + + return whisper_vitisai_forward_impl( + ctx, + mel, + out, + input_tensors, + output_tensors, + cross_k_data, + cross_v_data); +} + +// Ensure persistent staging buffers are large enough for the given dimensions. +static void ensure_staging_buffers( + struct whisper_vitisai_context * ctx, + size_t count, + bool need_k, + bool need_v) { + if (need_k && ctx->cross_k_staging.size() < count) { + ctx->cross_k_staging.resize(count); + } + if (need_v && ctx->cross_v_staging.size() < count) { + ctx->cross_v_staging.resize(count); + } +} + +int whisper_vitisai_encode_with_cross( + struct whisper_vitisai_context * ctx, + struct ggml_tensor * mel, + struct ggml_tensor * embd_enc, + struct ggml_tensor * kv_cross_k, + struct ggml_tensor * kv_cross_v, + int n_text_layer, + int n_ctx, + int n_text_state, + int n_text_head, + bool flash_attn) { + if (!ctx || !mel || !embd_enc || !kv_cross_k || !kv_cross_v) { + std::fprintf(stderr, "%s: ctx/mel/embd_enc/kv_cross_k/kv_cross_v must not be null\n", __func__); + return 0; + } + + if (n_text_layer <= 0 || n_ctx <= 0 || n_text_state <= 0 || n_text_head <= 0) { + std::fprintf(stderr, "%s: invalid shape parameters layer=%d ctx=%d state=%d head=%d\n", + __func__, n_text_layer, n_ctx, n_text_state, n_text_head); + return 0; + } + + if ((n_text_state % n_text_head) != 0) { + std::fprintf(stderr, "%s: invalid head configuration state=%d head=%d\n", + __func__, n_text_state, n_text_head); + return 0; + } + + if (kv_cross_k->type != kv_cross_v->type) { + std::fprintf(stderr, "%s: kv_cross type mismatch k=%s v=%s\n", + __func__, + whisper_vitisai_helpers::whisper_kv_type_name(kv_cross_k->type), + whisper_vitisai_helpers::whisper_kv_type_name(kv_cross_v->type)); + return 0; + } + + const int n_state = n_text_state; + const int n_state_head = n_state / n_text_head; + const int n_ctx_pad = (n_ctx + 255) & ~255; // GGML_PAD(n_ctx, 256) + + const float Kscale = pow(float(n_state_head), -0.25f); + const ggml_type kv_type = kv_cross_k->type; + const bool kv_is_f32 = kv_type == GGML_TYPE_F32; + const bool kv_is_f16 = kv_type == GGML_TYPE_F16; + if (!kv_is_f32 && !kv_is_f16) { + std::fprintf(stderr, "%s: unsupported kv_cross tensor type '%s'\n", + __func__, whisper_vitisai_helpers::whisper_kv_type_name(kv_type)); + return 0; + } + + const size_t elem_size = ggml_type_size(kv_type); + const size_t req_layer_elems = (size_t)n_ctx * (size_t)n_state; + + std::vector * input_tensors_cached = nullptr; + std::vector * output_tensors_cached = nullptr; + if (!whisper_vitisai_get_cached_io_tensors(ctx, input_tensors_cached, output_tensors_cached)) { + std::fprintf(stderr, "%s: failed to acquire Vitis AI I/O tensors\n", __func__); + return 0; + } + std::vector input_tensors = *input_tensors_cached; + std::vector output_tensors = *output_tensors_cached; + + if (ctx->cross_k_out_idx < 0 || ctx->cross_k_out_idx >= (int) output_tensors.size() || + ctx->cross_v_out_idx < 0 || ctx->cross_v_out_idx >= (int) output_tensors.size()) { + std::fprintf(stderr, "%s: invalid cross output indices cross_k=%d cross_v=%d for %zu output tensor(s)\n", + __func__, ctx->cross_k_out_idx, ctx->cross_v_out_idx, output_tensors.size()); + return 0; + } + + const auto & cross_k_meta = output_tensors[ctx->cross_k_out_idx].getMetadata(); + const auto & cross_v_meta = output_tensors[ctx->cross_v_out_idx].getMetadata(); + if (!whisper_vitisai_helpers::whisper_validate_cross_shape("cross_k", cross_k_meta.shape, n_text_layer, n_ctx, n_state) || + !whisper_vitisai_helpers::whisper_validate_cross_shape("cross_v", cross_v_meta.shape, n_text_layer, n_ctx, n_state)) { + return 0; + } + + if (ctx->cross_k_expected_bytes == 0 || ctx->cross_v_expected_bytes == 0) { + std::fprintf(stderr, "%s: missing cross output metadata sizes\n", __func__); + return 0; + } + if (ctx->cross_k_expected_bytes != ctx->cross_v_expected_bytes) { + std::fprintf(stderr, "%s: cross output metadata size mismatch k=%zu v=%zu\n", + __func__, ctx->cross_k_expected_bytes, ctx->cross_v_expected_bytes); + return 0; + } + const size_t expected_cross_bytes = (size_t) n_text_layer * req_layer_elems * sizeof(float); + if (ctx->cross_k_expected_bytes != expected_cross_bytes) { + std::fprintf(stderr, + "%s: cross output size mismatch (model=%zu B, expected=%zu B for layer=%d ctx=%d state=%d)\n", + __func__, ctx->cross_k_expected_bytes, expected_cross_bytes, n_text_layer, n_ctx, n_state); + return 0; + } + + const size_t model_total_elems = ctx->cross_k_expected_bytes / sizeof(float); + const size_t model_layer_elems = req_layer_elems; + + const size_t required_kv_bytes = flash_attn + ? (size_t)n_text_layer * elem_size * (size_t)n_state * (size_t)n_ctx_pad + : (size_t)n_text_layer * elem_size * req_layer_elems; + if (ggml_nbytes(kv_cross_k) < required_kv_bytes || ggml_nbytes(kv_cross_v) < required_kv_bytes) { + std::fprintf(stderr, + "%s: kv_cross buffers are too small (required=%zu B, k=%zu B, v=%zu B)\n", + __func__, required_kv_bytes, ggml_nbytes(kv_cross_k), ggml_nbytes(kv_cross_v)); + return 0; + } + + const bool direct_k_to_kv = kv_is_f32 && (!flash_attn || n_ctx_pad == n_ctx); + const bool direct_v_to_kv = kv_is_f32 && flash_attn && (n_ctx_pad == n_ctx); + const bool need_k_staging = !direct_k_to_kv; + const bool need_v_staging = !direct_v_to_kv; + + if (need_k_staging || need_v_staging) { + ensure_staging_buffers(ctx, model_total_elems, need_k_staging, need_v_staging); + } + + void * cross_k_out = direct_k_to_kv + ? kv_cross_k->data + : (void *) ctx->cross_k_staging.data(); + void * cross_v_out = direct_v_to_kv + ? kv_cross_v->data + : (void *) ctx->cross_v_staging.data(); + + whisper_vitisai_helpers::whisper_kv_cross_layout kv_layout; + kv_layout.n_layer = n_text_layer; + kv_layout.n_ctx = n_ctx; + kv_layout.n_state = n_state; + kv_layout.src_layer_elems = model_layer_elems; + kv_layout.layer_elems = req_layer_elems; + kv_layout.kscale = Kscale; + + if (flash_attn) { + WHISPER_DBG_TIMER(t_fwd_start); + if (!whisper_vitisai_forward_impl( + ctx, mel, embd_enc, input_tensors, output_tensors, cross_k_out, cross_v_out)) { + return 0; + } + + WHISPER_DBG_TIMER(t_fwd_end); + WHISPER_DBG_TIMER(t_post_start); + + if (n_ctx_pad == n_ctx) { + kv_layout.dst_layer_stride = req_layer_elems * elem_size; + if (kv_is_f32) { + // V was written straight into the kv cache by the runtime; only K needs scaling. + whisper_vitisai_helpers::whisper_kv_cross_scale_k_f32( + (float *)kv_cross_k->data, + (size_t) n_text_layer * req_layer_elems, + Kscale); + } else { // kv_is_f16 + whisper_vitisai_helpers::whisper_kv_cross_store_layers_f16( + ctx->cross_k_staging.data(), + ctx->cross_v_staging.data(), + (uint8_t *)kv_cross_k->data, + (uint8_t *)kv_cross_v->data, + kv_layout); + } + } else { + // Runtime decoder uses padded K/V cache. Copy only requested context, leave the pad tail untouched. + kv_layout.dst_layer_stride = elem_size * (size_t)n_state * (size_t)n_ctx_pad; + if (kv_is_f32) { + whisper_vitisai_helpers::whisper_kv_cross_store_layers_f32( + ctx->cross_k_staging.data(), + ctx->cross_v_staging.data(), + (uint8_t *)kv_cross_k->data, + (uint8_t *)kv_cross_v->data, + kv_layout); + } else { // kv_is_f16 + whisper_vitisai_helpers::whisper_kv_cross_store_layers_f16( + ctx->cross_k_staging.data(), + ctx->cross_v_staging.data(), + (uint8_t *)kv_cross_k->data, + (uint8_t *)kv_cross_v->data, + kv_layout); + } + } + + WHISPER_DBG_TIMER(t_post_end); + +#if defined(WHISPER_DEBUG) + const size_t model_ctx = (size_t) n_ctx; + std::fprintf(stderr, "%s: vitisai enc+cross forward time = %8.2f ms\n", __func__, (t_fwd_end - t_fwd_start) / 1000.0f); + std::fprintf(stderr, "%s: kv_cross post-process time = %8.2f ms (flash, req_ctx=%d, model_ctx=%zu, req_ctx_pad=%d, kv_type=%s)\n", + __func__, (t_post_end - t_post_start) / 1000.0f, n_ctx, model_ctx, n_ctx_pad, + whisper_vitisai_helpers::whisper_kv_type_name(kv_type)); +#endif + } else { + // Non-flash: model outputs contiguous [ctx, state] per layer. + WHISPER_DBG_TIMER(t_fwd_start); + if (!whisper_vitisai_forward_impl( + ctx, mel, embd_enc, input_tensors, output_tensors, cross_k_out, cross_v_out)) { + return 0; + } + WHISPER_DBG_TIMER(t_fwd_end); + WHISPER_DBG_TIMER(t_post_start); + + kv_layout.dst_layer_stride = elem_size * (size_t)n_state * (size_t)n_ctx; + if (kv_is_f32) { + // K was written straight into the kv cache by the runtime and is scaled there. + whisper_vitisai_helpers::whisper_kv_cross_scale_k_f32( + (float *)kv_cross_k->data, + (size_t) n_text_layer * req_layer_elems, + Kscale); + + whisper_vitisai_helpers::whisper_kv_cross_transpose_v_layers_f32( + ctx->cross_v_staging.data(), + (uint8_t *)kv_cross_v->data, + kv_layout); + } else { // kv_is_f16 + whisper_vitisai_helpers::whisper_kv_cross_store_k_transpose_v_layers_f16( + ctx->cross_k_staging.data(), + ctx->cross_v_staging.data(), + (uint8_t *)kv_cross_k->data, + (uint8_t *)kv_cross_v->data, + kv_layout); + } + + WHISPER_DBG_TIMER(t_post_end); + +#if defined(WHISPER_DEBUG) + const size_t model_ctx = (size_t) n_ctx; + std::fprintf(stderr, "%s: vitisai enc+cross forward time = %8.2f ms\n", __func__, (t_fwd_end - t_fwd_start) / 1000.0f); + std::fprintf(stderr, "%s: kv_cross post-process time = %8.2f ms (non-flash, req_ctx=%d, model_ctx=%zu, kv_type=%s)\n", + __func__, (t_post_end - t_post_start) / 1000.0f, n_ctx, model_ctx, + whisper_vitisai_helpers::whisper_kv_type_name(kv_type)); +#endif + } + + return 1; +} diff --git a/src/vitisai/whisper-vitisai-encoder.h b/src/vitisai/whisper-vitisai-encoder.h new file mode 100644 index 000000000..ee96a1557 --- /dev/null +++ b/src/vitisai/whisper-vitisai-encoder.h @@ -0,0 +1,43 @@ +#pragma once + +#include + +#if __cplusplus +extern "C" { +#endif + +struct whisper_vitisai_context; + +struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model); +void whisper_vitisai_free(struct whisper_vitisai_context * ctx); +bool whisper_vitisai_has_cross_proj(const struct whisper_vitisai_context * ctx); + +struct ggml_tensor; + +int whisper_vitisai_encode( + struct whisper_vitisai_context * ctx, + struct ggml_tensor * mel, + struct ggml_tensor * out); + +int whisper_vitisai_run_enc_cross( + struct whisper_vitisai_context * ctx, + struct ggml_tensor * mel, + struct ggml_tensor * out, + void * cross_v_data, + void * cross_k_data); + +int whisper_vitisai_encode_with_cross( + struct whisper_vitisai_context * ctx, + struct ggml_tensor * mel, + struct ggml_tensor * embd_enc, + struct ggml_tensor * kv_cross_k, + struct ggml_tensor * kv_cross_v, + int n_text_layer, + int n_ctx, + int n_text_state, + int n_text_head, + bool flash_attn); + +#if __cplusplus +} +#endif diff --git a/src/vitisai/whisper-vitisai-helpers.cpp b/src/vitisai/whisper-vitisai-helpers.cpp new file mode 100644 index 000000000..417634615 --- /dev/null +++ b/src/vitisai/whisper-vitisai-helpers.cpp @@ -0,0 +1,472 @@ +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#endif + +#include "vitisai/whisper-vitisai-helpers.h" + +#include +#include +#ifdef _WIN32 + #include +#else + #include + #include +#endif +#include +#include + +namespace whisper_vitisai_helpers { + +bool map_rai_file(const char * path, uint8_t ** buffer, size_t * size) { +#ifdef _WIN32 + HANDLE hFile = CreateFileA(path, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + if (hFile == INVALID_HANDLE_VALUE) { + std::fprintf(stderr, "%s: %d: Failed to open rai file '%s'\n", __func__, __LINE__, path); + return false; + } + + LARGE_INTEGER fileSize; + if (!GetFileSizeEx(hFile, &fileSize)) { + CloseHandle(hFile); + std::fprintf(stderr, "%s: %d: Failed to get file size for rai file '%s'\n", __func__, __LINE__, path); + return false; + } + + HANDLE hMapping = CreateFileMappingA(hFile, NULL, PAGE_READONLY, 0, fileSize.QuadPart, NULL); + if (hMapping == NULL) { + CloseHandle(hFile); + std::fprintf(stderr, "%s: %d: Failed to create file mapping for rai file '%s'\n", __func__, __LINE__, path); + return false; + } + + *buffer = (uint8_t *) MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, fileSize.QuadPart); + if (*buffer == NULL) { + CloseHandle(hMapping); + CloseHandle(hFile); + std::fprintf(stderr, "%s: %d: Failed to map rai file '%s'\n", __func__, __LINE__, path); + return false; + } + CloseHandle(hMapping); + CloseHandle(hFile); + *size = fileSize.QuadPart; + return true; +#else + FILE * fd = fopen(path, "rb"); + if (!fd) { + std::fprintf(stderr, "%s: %d: Failed to open rai file '%s'\n", __func__, __LINE__, path); + return false; + } + + struct stat st; + if (fstat(fileno(fd), &st) == -1) { + fclose(fd); + std::fprintf(stderr, "%s: %d: Failed to get file size for rai file '%s'\n", __func__, __LINE__, path); + return false; + } + + *buffer = (uint8_t *) mmap(nullptr, st.st_size, PROT_READ, MAP_PRIVATE, fileno(fd), 0); + if (*buffer == MAP_FAILED) { + fclose(fd); + std::fprintf(stderr, "%s: %d: Failed to mmap rai file '%s'\n", __func__, __LINE__, path); + return false; + } + fclose(fd); + *size = st.st_size; + return true; +#endif // _WIN32 +} + +void unmap_rai_file(uint8_t * buffer, size_t size) { +#ifdef _WIN32 + UnmapViewOfFile(buffer); +#else + munmap(buffer, size); +#endif // _WIN32 +} + +const char * whisper_kv_type_name(ggml_type type) { + switch (type) { + case GGML_TYPE_F32: return "F32"; + case GGML_TYPE_F16: return "F16"; + default: return "unsupported"; + } +} + +const char * whisper_flexml_dtype_name(flexmlrt::client::DataType type) { + switch (type) { + case flexmlrt::client::DataType::Float32: return "Float32"; + case flexmlrt::client::DataType::Int8: return "Int8"; + case flexmlrt::client::DataType::UInt8: return "UInt8"; + case flexmlrt::client::DataType::Int16: return "Int16"; + case flexmlrt::client::DataType::UInt16: return "UInt16"; + case flexmlrt::client::DataType::BFloat16: return "BFloat16"; + case flexmlrt::client::DataType::Bool: return "Bool"; + case flexmlrt::client::DataType::Float16: return "Float16"; + case flexmlrt::client::DataType::Int32: return "Int32"; + case flexmlrt::client::DataType::UInt32: return "UInt32"; + default: return "Unknown"; + } +} + +bool whisper_flexml_dtype_to_ggml_type( + flexmlrt::client::DataType type, + ggml_type * ggml_dtype) { + switch (type) { + case flexmlrt::client::DataType::Float32: + if (ggml_dtype) { + *ggml_dtype = GGML_TYPE_F32; + } + return true; + case flexmlrt::client::DataType::Float16: + if (ggml_dtype) { + *ggml_dtype = GGML_TYPE_F16; + } + return true; + case flexmlrt::client::DataType::BFloat16: + if (ggml_dtype) { + *ggml_dtype = GGML_TYPE_BF16; + } + return true; + default: + return false; + } +} + +static bool whisper_vitisai_validate_tensor_dtype( + const char * tensor_name, + flexmlrt::client::DataType model_dtype, + ggml_type runtime_dtype) { + ggml_type expected_runtime_dtype = GGML_TYPE_COUNT; + if (!whisper_flexml_dtype_to_ggml_type(model_dtype, &expected_runtime_dtype)) { + std::fprintf(stderr, + "%s: unsupported model dtype for %s: %s (supported: Float32/Float16/BFloat16)\n", + __func__, tensor_name, whisper_flexml_dtype_name(model_dtype)); + return false; + } + + if (runtime_dtype != expected_runtime_dtype) { + std::fprintf(stderr, + "%s: %s dtype mismatch (runtime=%s, model=%s)\n", + __func__, tensor_name, ggml_type_name(runtime_dtype), whisper_flexml_dtype_name(model_dtype)); + return false; + } + + return true; +} + +static std::string whisper_shape_to_string(const std::vector & shape) { + std::string out = "["; + for (size_t i = 0; i < shape.size(); ++i) { + if (i > 0) { + out += ", "; + } + out += std::to_string(shape[i]); + } + out += "]"; + return out; +} + +static std::vector whisper_canonical_shape(const std::vector & shape) { + std::vector canonical; + canonical.reserve(shape.size()); + for (size_t i = 0; i < shape.size(); ++i) { + const size_t dim = (size_t) shape[i]; + if (dim != 1) { + canonical.push_back(dim); + } + } + if (canonical.empty()) { + canonical.push_back(1); + } + return canonical; +} + +static bool whisper_validate_shape( + const char * tensor_name, + const std::vector & model_shape, + const std::vector & expected_shape) { + const std::vector shape = whisper_canonical_shape(model_shape); + if (shape != expected_shape) { + std::fprintf(stderr, + "%s: %s shape mismatch (runtime expected=%s, model=%s)\n", + __func__, + tensor_name, + whisper_shape_to_string(expected_shape).c_str(), + whisper_shape_to_string(shape).c_str()); + return false; + } + return true; +} + +bool whisper_validate_cross_shape( + const char * tensor_name, + const std::vector & model_shape, + int n_text_layer, + int n_ctx, + int n_state) { + const std::vector expected = { + (size_t) n_text_layer, + (size_t) n_ctx, + (size_t) n_state, + }; + return whisper_validate_shape(tensor_name, model_shape, expected); +} + +bool whisper_vitisai_bind_tensor_data( + const char * tensor_name, + struct ggml_tensor * runtime_tensor, + const std::vector & expected_shape, + flexmlrt::client::ErtTensorType & io_tensor) { + const auto & meta = io_tensor.getMetadata(); + if (!whisper_vitisai_validate_tensor_dtype(tensor_name, meta.type, runtime_tensor->type)) { + return false; + } + if (!whisper_validate_shape(tensor_name, meta.shape, expected_shape)) { + return false; + } + + const size_t model_bytes = meta.size; + const size_t runtime_bytes = ggml_nbytes(runtime_tensor); + if (model_bytes == 0 || runtime_bytes == 0) { + std::fprintf(stderr, "%s: %s sizes must be non-zero (model=%zu, runtime=%zu)\n", + __func__, tensor_name, model_bytes, runtime_bytes); + return false; + } + if (runtime_bytes != model_bytes) { + std::fprintf(stderr, + "%s: %s tensor size mismatch (runtime=%zu B, model=%zu B). " + "VitisAI .rai requires exact context match; use matching -ac/model artifact.\n", + __func__, tensor_name, runtime_bytes, model_bytes); + return false; + } + + io_tensor.data = runtime_tensor->data; + return true; +} + +bool whisper_vitisai_resolve_io_binding( + [[maybe_unused]] const char * caller, + const std::vector & input_tensors, + const std::vector & output_tensors, + whisper_vitisai_io_binding * binding, + std::string * error) { + const auto fail = [error](std::string message) { + if (error) { + *error = std::move(message); + } + return false; + }; + + if (input_tensors.empty()) { + return fail("Model has no input tensors"); + } + + binding->mel_in_idx = 0; + bool found_named_mel = false; + for (int i = 0; i < (int) input_tensors.size(); ++i) { + const std::string & name = input_tensors[i].getMetadata().name; + if (name == "input" || name == "mel") { + binding->mel_in_idx = i; + found_named_mel = true; + break; + } + } + if (!found_named_mel) { +#if defined(WHISPER_DEBUG) + std::fprintf(stderr, "%s: WARNING: mel input not found by name; falling back to input[0]\n", caller); +#endif + } + + if (output_tensors.empty()) { + return fail("Model has no output tensors"); + } + + for (int i = 0; i < (int) output_tensors.size(); ++i) { + const std::string & name = output_tensors[i].getMetadata().name; + if (name == "embd_enc") { + binding->embd_enc_out_idx = i; + } else if (name == "cross_k") { + binding->cross_k_out_idx = i; + } else if (name == "cross_v") { + binding->cross_v_out_idx = i; + } + } + + if (binding->embd_enc_out_idx < 0) { +#if defined(WHISPER_DEBUG) + std::fprintf(stderr, "%s: WARNING: embd_enc output not found by name; falling back to output[0]\n", caller); +#endif + binding->embd_enc_out_idx = 0; + } + + const bool has_cross_k = binding->cross_k_out_idx >= 0; + const bool has_cross_v = binding->cross_v_out_idx >= 0; + if (has_cross_k != has_cross_v) { + return fail("Incomplete cross-projection contract: both cross_k and cross_v outputs are required"); + } + + if (has_cross_k && (binding->cross_k_out_idx == binding->cross_v_out_idx || + binding->cross_k_out_idx == binding->embd_enc_out_idx || + binding->cross_v_out_idx == binding->embd_enc_out_idx)) { + return fail("Invalid output mapping: embd_enc/cross_k/cross_v indices overlap"); + } + + const auto & mel_meta = input_tensors[binding->mel_in_idx].getMetadata(); + if (!whisper_flexml_dtype_to_ggml_type(mel_meta.type, nullptr)) { + return fail( + std::string("Unsupported mel input type: ") + + whisper_flexml_dtype_name(mel_meta.type) + " (supported: Float32/Float16/BFloat16)"); + } + binding->mel_in_expected_bytes = mel_meta.size; + + const auto & embd_meta = output_tensors[binding->embd_enc_out_idx].getMetadata(); + if (!whisper_flexml_dtype_to_ggml_type(embd_meta.type, nullptr)) { + return fail( + std::string("Unsupported embd_enc output type: ") + + whisper_flexml_dtype_name(embd_meta.type) + " (supported: Float32/Float16/BFloat16)"); + } + binding->embd_enc_expected_bytes = embd_meta.size; + + if (has_cross_k) { + const auto & cross_k_meta = output_tensors[binding->cross_k_out_idx].getMetadata(); + const auto & cross_v_meta = output_tensors[binding->cross_v_out_idx].getMetadata(); + if (cross_k_meta.type != flexmlrt::client::DataType::Float32 || + cross_v_meta.type != flexmlrt::client::DataType::Float32) { + return fail( + std::string("Unsupported cross output type(s): cross_k=") + + whisper_flexml_dtype_name(cross_k_meta.type) + ", cross_v=" + + whisper_flexml_dtype_name(cross_v_meta.type) + " (cross path currently requires Float32)"); + } + if (cross_k_meta.size != cross_v_meta.size) { + return fail("cross_k and cross_v output sizes do not match"); + } + binding->cross_k_expected_bytes = cross_k_meta.size; + binding->cross_v_expected_bytes = cross_v_meta.size; + } + + return true; +} + +bool whisper_vitisai_all_tensors_claimed( + const char * caller, + const char * tensor_kind, + const std::vector & tensors, + const std::vector & claimed) { + for (size_t i = 0; i < tensors.size(); ++i) { + if (!claimed[i]) { + std::fprintf(stderr, + "%s: unsupported extra %s tensor at index %zu (name='%s'); strict contract expects only mapped %ss\n", + caller, tensor_kind, i, tensors[i].getMetadata().name.c_str(), tensor_kind); + return false; + } + } + return true; +} + +void whisper_kv_cross_scale_k_f32( + float * k_data, + size_t count, + float kscale) { + for (size_t i = 0; i < count; ++i) { + k_data[i] *= kscale; + } +} + +void whisper_kv_cross_store_layers_f32( + const float * src_k, + const float * src_v, + uint8_t * dst_k, + uint8_t * dst_v, + const whisper_kv_cross_layout & layout) { + for (int il = 0; il < layout.n_layer; ++il) { + const float * layer_src_k = src_k + (size_t)il * layout.src_layer_elems; + const float * layer_src_v = src_v + (size_t)il * layout.src_layer_elems; + float * dk = (float *)(dst_k + layout.dst_layer_stride * (size_t)il); + float * dv = (float *)(dst_v + layout.dst_layer_stride * (size_t)il); + for (size_t i = 0; i < layout.layer_elems; ++i) { + dk[i] = layer_src_k[i] * layout.kscale; + dv[i] = layer_src_v[i]; + } + } +} + +void whisper_kv_cross_store_layers_f16( + const float * src_k, + const float * src_v, + uint8_t * dst_k, + uint8_t * dst_v, + const whisper_kv_cross_layout & layout) { + for (int il = 0; il < layout.n_layer; ++il) { + const float * layer_src_k = src_k + (size_t)il * layout.src_layer_elems; + const float * layer_src_v = src_v + (size_t)il * layout.src_layer_elems; + ggml_fp16_t * dk = (ggml_fp16_t *)(dst_k + layout.dst_layer_stride * (size_t)il); + ggml_fp16_t * dv = (ggml_fp16_t *)(dst_v + layout.dst_layer_stride * (size_t)il); + for (size_t i = 0; i < layout.layer_elems; ++i) { + dk[i] = ggml_fp32_to_fp16(layer_src_k[i] * layout.kscale); + dv[i] = ggml_fp32_to_fp16(layer_src_v[i]); + } + } +} + +void whisper_kv_cross_transpose_v_layers_f32( + const float * src_v, + uint8_t * dst_v, + const whisper_kv_cross_layout & layout) { + const int n_ctx = layout.n_ctx; + const int n_state = layout.n_state; + + const int BLOCK = 32; + for (int il = 0; il < layout.n_layer; ++il) { + const float * layer_src_v = src_v + (size_t)il * layout.src_layer_elems; + float * dv = (float *)(dst_v + layout.dst_layer_stride * (size_t)il); + + for (int ic = 0; ic < n_ctx; ic += BLOCK) { + for (int is = 0; is < n_state; is += BLOCK) { + const int ic_end = std::min(ic + BLOCK, n_ctx); + const int is_end = std::min(is + BLOCK, n_state); + for (int i = ic; i < ic_end; ++i) { + for (int j = is; j < is_end; ++j) { + dv[j * n_ctx + i] = layer_src_v[i * n_state + j]; + } + } + } + } + } +} + +void whisper_kv_cross_store_k_transpose_v_layers_f16( + const float * src_k, + const float * src_v, + uint8_t * dst_k, + uint8_t * dst_v, + const whisper_kv_cross_layout & layout) { + const int n_ctx = layout.n_ctx; + const int n_state = layout.n_state; + + const int BLOCK = 32; + for (int il = 0; il < layout.n_layer; ++il) { + const float * layer_src_k = src_k + (size_t)il * layout.src_layer_elems; + const float * layer_src_v = src_v + (size_t)il * layout.src_layer_elems; + ggml_fp16_t * dk = (ggml_fp16_t *)(dst_k + layout.dst_layer_stride * (size_t)il); + ggml_fp16_t * dv = (ggml_fp16_t *)(dst_v + layout.dst_layer_stride * (size_t)il); + for (size_t i = 0; i < layout.layer_elems; ++i) { + dk[i] = ggml_fp32_to_fp16(layer_src_k[i] * layout.kscale); + } + + for (int ic = 0; ic < n_ctx; ic += BLOCK) { + for (int is = 0; is < n_state; is += BLOCK) { + const int ic_end = std::min(ic + BLOCK, n_ctx); + const int is_end = std::min(is + BLOCK, n_state); + for (int i = ic; i < ic_end; ++i) { + for (int j = is; j < is_end; ++j) { + dv[j * n_ctx + i] = ggml_fp32_to_fp16(layer_src_v[i * n_state + j]); + } + } + } + } + } +} + +} // namespace whisper_vitisai_helpers diff --git a/src/vitisai/whisper-vitisai-helpers.h b/src/vitisai/whisper-vitisai-helpers.h new file mode 100644 index 000000000..d50a2f4a2 --- /dev/null +++ b/src/vitisai/whisper-vitisai-helpers.h @@ -0,0 +1,117 @@ +#pragma once + +#include "FlexMLClient.h" +#include "ggml.h" + +#include +#include +#include +#include +#include + +namespace whisper_vitisai_helpers { + +bool map_rai_file(const char * path, uint8_t ** buffer, size_t * size); +void unmap_rai_file(uint8_t * buffer, size_t size); + +const char * whisper_kv_type_name(ggml_type type); +const char * whisper_flexml_dtype_name(flexmlrt::client::DataType type); +bool whisper_flexml_dtype_to_ggml_type( + flexmlrt::client::DataType type, + ggml_type * ggml_dtype); + +bool whisper_validate_cross_shape( + const char * tensor_name, + const std::vector & model_shape, + int n_text_layer, + int n_ctx, + int n_state); + +bool whisper_vitisai_bind_tensor_data( + const char * tensor_name, + struct ggml_tensor * runtime_tensor, + const std::vector & expected_shape, + flexmlrt::client::ErtTensorType & io_tensor); + +#if defined(WHISPER_DEBUG) +template +void whisper_vitisai_print_shape(const std::vector & shape) { + std::fprintf(stderr, "["); + for (size_t i = 0; i < shape.size(); ++i) { + std::fprintf(stderr, "%s%lld", i == 0 ? "" : ", ", (long long) shape[i]); + } + std::fprintf(stderr, "]"); +} +#endif + +// Model IO tensor indices and metadata sizes resolved once at init time. +struct whisper_vitisai_io_binding { + int mel_in_idx = -1; + int embd_enc_out_idx = -1; + int cross_k_out_idx = -1; + int cross_v_out_idx = -1; + size_t mel_in_expected_bytes = 0; + size_t embd_enc_expected_bytes = 0; + size_t cross_k_expected_bytes = 0; + size_t cross_v_expected_bytes = 0; +}; + +// Warnings are printed with the caller's name; hard failures are returned in *error +// so the caller can decide how to report them. +bool whisper_vitisai_resolve_io_binding( + const char * caller, + const std::vector & input_tensors, + const std::vector & output_tensors, + whisper_vitisai_io_binding * binding, + std::string * error); + +bool whisper_vitisai_all_tensors_claimed( + const char * caller, + const char * tensor_kind, + const std::vector & tensors, + const std::vector & claimed); + +// Geometry of one cross K/V transfer from the model output (always f32, contiguous +// [ctx, state] per layer) into the runtime kv cache. +struct whisper_kv_cross_layout { + int n_layer = 0; + int n_ctx = 0; + int n_state = 0; + size_t src_layer_elems = 0; // f32 elements per layer in the model output buffer + size_t layer_elems = 0; // elements per layer transferred into the kv cache + size_t dst_layer_stride = 0; // bytes per layer in the kv cache + float kscale = 1.0f; +}; + +void whisper_kv_cross_scale_k_f32( + float * k_data, + size_t count, + float kscale); + +void whisper_kv_cross_store_layers_f32( + const float * src_k, + const float * src_v, + uint8_t * dst_k, + uint8_t * dst_v, + const whisper_kv_cross_layout & layout); + +void whisper_kv_cross_store_layers_f16( + const float * src_k, + const float * src_v, + uint8_t * dst_k, + uint8_t * dst_v, + const whisper_kv_cross_layout & layout); + +void whisper_kv_cross_transpose_v_layers_f32( + const float * src_v, + uint8_t * dst_v, + const whisper_kv_cross_layout & layout); + +void whisper_kv_cross_store_k_transpose_v_layers_f16( + const float * src_k, + const float * src_v, + uint8_t * dst_k, + uint8_t * dst_v, + const whisper_kv_cross_layout & layout); + +} // namespace whisper_vitisai_helpers diff --git a/src/whisper.cpp b/src/whisper.cpp index 89146e2e4..115bdecb4 100644 --- a/src/whisper.cpp +++ b/src/whisper.cpp @@ -14,6 +14,10 @@ #include "openvino/whisper-openvino-encoder.h" #endif +#ifdef WHISPER_USE_VITISAI +#include "vitisai/whisper-vitisai-encoder.h" +#endif + #include #include #include @@ -903,6 +907,10 @@ struct whisper_state { whisper_openvino_context * ctx_openvino = nullptr; #endif +#ifdef WHISPER_USE_VITISAI + whisper_vitisai_context * ctx_vitisai = nullptr; +#endif + // [EXPERIMENTAL] token-level timestamps data int64_t t_beg = 0; int64_t t_last = 0; @@ -1976,7 +1984,25 @@ static bool whisper_encode_external(const whisper_state & wstate) { const bool use_openvino = wstate.ctx_openvino != nullptr; #endif - return use_coreml || use_openvino; +#ifndef WHISPER_USE_VITISAI + const bool use_vitisai = false; +#else + const bool use_vitisai = wstate.ctx_vitisai != nullptr; +#endif + + return use_coreml || use_openvino || use_vitisai; +} + +static bool whisper_cross_external(const whisper_state & wstate) { + GGML_UNUSED(wstate); + +#if defined(WHISPER_USE_VITISAI) + const bool use_vitisai_cross = whisper_vitisai_has_cross_proj(wstate.ctx_vitisai); +#else + const bool use_vitisai_cross = false; +#endif + + return use_vitisai_cross; } static struct ggml_cgraph * whisper_build_graph_conv( @@ -2417,6 +2443,21 @@ static bool whisper_encode_internal( #if defined(WHISPER_USE_COREML) whisper_coreml_encode(wstate.ctx_coreml, mel->ne[0], mel->ne[1], (float *) mel->data, (float *) wstate.embd_enc->data); +#elif defined(WHISPER_USE_VITISAI) + if (whisper_vitisai_has_cross_proj(wstate.ctx_vitisai)) { + const auto & hp = wctx.model.hparams; + const int n_ctx = wstate.exp_n_audio_ctx > 0 + ? wstate.exp_n_audio_ctx : hp.n_audio_ctx; + if (!whisper_vitisai_encode_with_cross( + wstate.ctx_vitisai, mel, wstate.embd_enc, + wstate.kv_cross.k, wstate.kv_cross.v, + hp.n_text_layer, n_ctx, hp.n_text_state, + hp.n_text_head, wctx.params.flash_attn)) { + return false; + } + } else if (!whisper_vitisai_encode(wstate.ctx_vitisai, mel, wstate.embd_enc)) { + return false; + } #elif defined(WHISPER_USE_OPENVINO) whisper_openvino_encode(wstate.ctx_openvino, mel, wstate.embd_enc); #endif @@ -2440,7 +2481,7 @@ static bool whisper_encode_internal( } // cross - { + if (!whisper_cross_external(wstate)) { auto & sched = wstate.sched_cross.sched; ggml_cgraph * gf = whisper_build_graph_cross(wctx, wstate); @@ -3356,6 +3397,19 @@ static std::string whisper_get_coreml_path_encoder(std::string path_bin) { } #endif +#ifdef WHISPER_USE_VITISAI +// replace extension with Vitis AI encoder artifact. Cross projection support is +// detected from the model's output tensors, not from the file name. +static std::string whisper_get_vitisai_path_encoder_cache(std::string path_bin) { + auto pos = path_bin.rfind('.'); + if (pos != std::string::npos) { + path_bin = path_bin.substr(0, pos); + } + + return path_bin + "-encoder-vitisai.rai"; +} +#endif + #ifdef WHISPER_USE_OPENVINO // replace .bin with-encoder-openvino.xml static std::string whisper_openvino_get_path_encoder(std::string path_bin) { @@ -3465,6 +3519,21 @@ struct whisper_state * whisper_init_state(whisper_context * ctx) { } #endif +#ifdef WHISPER_USE_VITISAI + const auto path_vitisai = whisper_get_vitisai_path_encoder_cache(ctx->path_model); + + state->ctx_vitisai = whisper_vitisai_init(path_vitisai.c_str()); + if (!state->ctx_vitisai) { + WHISPER_LOG_ERROR("%s: failed to load Vitis AI model from '%s'\n", __func__, path_vitisai.c_str()); + whisper_free_state(state); + return nullptr; + } else if (whisper_vitisai_has_cross_proj(state->ctx_vitisai)) { + WHISPER_LOG_INFO("%s: Vitis AI encoder + cross projection model loaded\n", __func__); + } else { + WHISPER_LOG_INFO("%s: Vitis AI encoder model loaded\n", __func__); + } +#endif + state->logits.reserve(ctx->vocab.n_vocab * ctx->model.hparams.n_text_ctx); state->batch = whisper_batch_init(ctx->model.hparams.n_text_ctx, WHISPER_MAX_DECODERS); @@ -3512,7 +3581,7 @@ struct whisper_state * whisper_init_state(whisper_context * ctx) { } // cross allocator - { + if (!whisper_cross_external(*state)) { bool ok = whisper_sched_graph_init(state->sched_cross, state->backends, [&]() { return whisper_build_graph_cross(*ctx, *state); @@ -3845,6 +3914,13 @@ void whisper_free_state(struct whisper_state * state) { } #endif +#ifdef WHISPER_USE_VITISAI + if (state->ctx_vitisai != nullptr) { + whisper_vitisai_free(state->ctx_vitisai); + state->ctx_vitisai = nullptr; + } +#endif + whisper_batch_free(state->batch); ggml_backend_sched_free(state->sched_conv.sched); @@ -4336,11 +4412,20 @@ static int whisper_has_openvino(void) { #endif } +static int whisper_has_vitisai(void) { +#ifdef WHISPER_USE_VITISAI + return 1; +#else + return 0; +#endif +} + const char * whisper_print_system_info(void) { static std::string s; s = ""; s += "WHISPER : "; + s += "VITISAI = " + std::to_string(whisper_has_vitisai()) + " | "; s += "COREML = " + std::to_string(whisper_has_coreml()) + " | "; s += "OPENVINO = " + std::to_string(whisper_has_openvino()) + " | ";