From a4660b7cae18276436a7db82721699bf0628787e Mon Sep 17 00:00:00 2001 From: Sachin Kumawat Date: Thu, 23 Jul 2026 17:51:11 -0700 Subject: [PATCH] Add VitisAI model download scripts --- README.md | 51 ++++--- models/download-vitisai-model.cmd | 32 +++++ models/download-vitisai-model.ps1 | 218 ++++++++++++++++++++++++++++ models/download-vitisai-model.sh | 226 ++++++++++++++++++++++++++++++ 4 files changed, 501 insertions(+), 26 deletions(-) create mode 100644 models/download-vitisai-model.cmd create mode 100644 models/download-vitisai-model.ps1 create mode 100755 models/download-vitisai-model.sh diff --git a/README.md b/README.md index 7882f59dd..91641ffb3 100644 --- a/README.md +++ b/README.md @@ -323,37 +323,36 @@ On AMD's Ryzen™ AI 300 Series with dedicated NPUs for acceleration, you can no ### Setup environment (Windows only) -- **Driver:** Make sure you have NPU drivers version **.280 or newer** installed. [Download latest drivers from here](https://account.amd.com/en/forms/downloads/ryzenai-eula-public-xef.html?filename=NPU_RAI1.5_280_WHQL.zip) -- **Runtime libraries:** Download and install the necessary [runtime dependencies from here](https://account.amd.com/en/forms/downloads/ryzenai-eula-public-xef.html?filename=flexmlrt1.7.0-win.zip). -- **Environment:** Extract the runtime package and set up the environment: - ```powershell - tar xvf flexmlrt1.7.0-win.zip - flexmlrt\setup.bat + - Obtain the XRT package and the FlexmlRT package from AMD. Both are distributed as tarballs or wheels. + - Copy the downloaded archives to a local path, extract them, and run the setup script from each extracted package in your shell (for example `source /path/to/xrt/setup.sh` and `source /path/to/flexmlrt/setup.sh`). Run these in every new shell you use to build or run `whisper.cpp`. + +- Fetch the matching ggml model and prebuilt VitisAI encoder cache: + + ```bash + sh ./models/download-ggml-model.sh base + sh ./models/download-vitisai-model.sh base + ``` + + ```cmd + .\models\download-ggml-model.cmd base + .\models\download-vitisai-model.cmd base + ``` + + Use the same model name with both scripts. The VitisAI script queries the AMD collection on Hugging Face to list available caches, then downloads the selected `.rai` file as `ggml--encoder-vitisai.rai` alongside the matching `ggml-.bin` file. You can also browse the collection manually at https://huggingface.co/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models. + +- Build `whisper.cpp` with VitisAI support: + + ```bash + cmake -B build -DWHISPER_VITISAI=1 + cmake --build build -j --config Release ``` Your environment is now ready. ### Build Whisper.cpp for Ryzen™ AI support -```bash -cmake -B build -DWHISPER_VITISAI=1 -cmake --build build -j --config Release -``` - -### Download NPU-optimized models - -- All NPU-supported Whisper models and their compiled `.rai` cache files are available in this collection: - https://huggingface.co/collections/amd/ryzen-ai-16-whisper-npu-optimized-onnx-models -- Download the pre-compiled `.rai` cache file matching your desired model, and place it in your `models/` directory alongside its corresponding `ggml-<...>.bin` file. - The cache file must be named with the `-encoder-vitisai.rai` suffix. For example, if your model file is named `ggml-small.bin`, the cache file should be named `ggml-small-encoder-vitisai.rai`. - - -> **Note:** The ".rai" models from Hugging Face are pre-optimized for Ryzen™ AI NPUs, delivering acceleration benefits from the very first run (aside from any initial CPU-side caching overhead). - -Run the examples as usual: - -```bash -./build/bin/whisper-cli -m models/ggml-small.bin -f samples/jfk.wav -``` + ```text + $ ./build/bin/whisper-cli -m models/ggml-base.bin -f samples/jfk.wav + ``` ## NVIDIA GPU support diff --git a/models/download-vitisai-model.cmd b/models/download-vitisai-model.cmd new file mode 100644 index 000000000..0b768cd1a --- /dev/null +++ b/models/download-vitisai-model.cmd @@ -0,0 +1,32 @@ +@echo off +setlocal + +set "script=%~dp0download-vitisai-model.ps1" + +if "%~1"=="" ( + PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" + exit /b %ERRORLEVEL% +) + +if /I "%~1"=="--list" ( + PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" -List + exit /b %ERRORLEVEL% +) + +if /I "%~1"=="-l" ( + PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" -List + exit /b %ERRORLEVEL% +) + +if /I "%~1"=="list" ( + PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" -List + exit /b %ERRORLEVEL% +) + +if "%~2"=="" ( + PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" -Model "%~1" +) else ( + PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" -Model "%~1" -ModelsPath "%~2" +) + +exit /b %ERRORLEVEL% diff --git a/models/download-vitisai-model.ps1 b/models/download-vitisai-model.ps1 new file mode 100644 index 000000000..5408248cf --- /dev/null +++ b/models/download-vitisai-model.ps1 @@ -0,0 +1,218 @@ +param( + [Parameter(Position = 0)] + [string] $Model, + + [Parameter(Position = 1)] + [string] $ModelsPath, + + [switch] $List +) + +$ErrorActionPreference = "Stop" + +$Source = "https://huggingface.co" +$CollectionApi = "https://huggingface.co/api/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models" +$CollectionUrl = "https://huggingface.co/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models" + +$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +if ($ScriptDir -match "\\bin$") { + $DefaultDownloadPath = (Get-Location).Path +} else { + $DefaultDownloadPath = $ScriptDir +} + +if (-not $ModelsPath) { + $ModelsPath = $DefaultDownloadPath +} + +function Get-HfHeaders { + $headers = @{} + if ($env:HF_TOKEN) { + $headers["Authorization"] = "Bearer $env:HF_TOKEN" + } + return $headers +} + +function Invoke-HfJson { + param([string] $Uri) + + $headers = Get-HfHeaders + if ($headers.Count -gt 0) { + return Invoke-RestMethod -Uri $Uri -Headers $headers + } + + return Invoke-RestMethod -Uri $Uri +} + +function Normalize-ModelName { + param([string] $Name) + + # HF currently publishes ggml-small-en-encoder-vitisai.rai, while the + # matching ggml model is ggml-small.en.bin. + if ($Name.EndsWith("-en")) { + return $Name.Substring(0, $Name.Length - 3) + ".en" + } + + return $Name +} + +function Get-VitisAiModels { + $collection = Invoke-HfJson -Uri $CollectionApi + $seen = @{} + $rows = New-Object System.Collections.Generic.List[object] + + foreach ($item in $collection.items) { + if ($item.type -ne "model") { + continue + } + + $repo = $item.id + if (-not $repo) { + continue + } + + $modelInfo = Invoke-HfJson -Uri "$Source/api/models/$repo" + foreach ($sibling in $modelInfo.siblings) { + $filename = [string] $sibling.rfilename + $match = [regex]::Match($filename, "^ggml-(.+)-encoder-vitisai\.rai$") + if (-not $match.Success) { + continue + } + + $rawName = $match.Groups[1].Value + $modelName = Normalize-ModelName -Name $rawName + if ($seen.ContainsKey($modelName)) { + continue + } + + $seen[$modelName] = $true + $destination = "ggml-$modelName-encoder-vitisai.rai" + $url = "$Source/$repo/resolve/main/$([uri]::EscapeDataString($filename))" + + $rows.Add([pscustomobject]@{ + Model = $modelName + RawName = $rawName + Repo = $repo + SourceFile = $filename + DestinationFile = $destination + DownloadUrl = $url + }) + } + } + + $order = @{ + "tiny" = 10 + "tiny.en" = 11 + "base" = 20 + "base.en" = 21 + "small" = 30 + "small.en" = 31 + "medium" = 40 + "medium.en" = 41 + "large-v1" = 50 + "large-v2" = 60 + "large-v3" = 70 + "large-v3-turbo" = 80 + } + + return $rows | Sort-Object ` + @{ Expression = { if ($order.ContainsKey($_.Model)) { $order[$_.Model] } else { 1000 } } }, ` + @{ Expression = { $_.Model } } +} + +function Show-Models { + $models = Get-VitisAiModels + + Write-Host "" + Write-Host "Available VitisAI encoder caches from ${CollectionUrl}:" + foreach ($entry in $models) { + if ($entry.Model -eq $entry.RawName) { + Write-Host (" {0,-18} {1}" -f $entry.Model, $entry.Repo) + } else { + Write-Host (" {0,-18} {1} (source name: {2})" -f $entry.Model, $entry.Repo, $entry.RawName) + } + } + Write-Host "" +} + +function Show-Usage { + Write-Host "Usage: download-vitisai-model.cmd --list" + Write-Host " download-vitisai-model.cmd [models_path]" + Write-Host "" + Write-Host "Downloads ggml--encoder-vitisai.rai next to ggml-.bin." + Write-Host "Use the same model name as download-ggml-model.cmd." + Write-Host "" +} + +if ($List -or $Model -eq "--list" -or $Model -eq "-l" -or $Model -eq "list") { + Show-Models + exit 0 +} + +if (-not $Model) { + Show-Usage + Show-Models + exit 1 +} + +$models = Get-VitisAiModels +$entry = $models | Where-Object { $_.Model -eq $Model -or $_.RawName -eq $Model } | Select-Object -First 1 +if (-not $entry) { + Write-Host "Invalid model: $Model" + foreach ($available in $models) { + Write-Host " $($available.Model)" + } + exit 1 +} + +New-Item -ItemType Directory -Force -Path $ModelsPath | Out-Null +$destinationPath = Join-Path $ModelsPath $entry.DestinationFile + +Write-Host "Downloading VitisAI encoder cache $($entry.Model) from '$($entry.Repo)' ..." +if (Test-Path $destinationPath) { + Write-Host "VitisAI encoder cache $($entry.DestinationFile) already exists. Skipping download." + exit 0 +} + +$headers = Get-HfHeaders +$downloaded = $false +for ($attempt = 1; $attempt -le 5; ++$attempt) { + try { + if ($headers.Count -gt 0) { + Invoke-WebRequest -Uri $entry.DownloadUrl -Headers $headers -OutFile $destinationPath + } else { + Invoke-WebRequest -Uri $entry.DownloadUrl -OutFile $destinationPath + } + $downloaded = $true + break + } catch { + if ($attempt -eq 5) { + if (Test-Path $destinationPath) { + Remove-Item -Force $destinationPath + } + Write-Host "Failed to download VitisAI encoder cache $($entry.Model) from $($entry.DownloadUrl)" + throw + } + Start-Sleep -Seconds 5 + } +} + +if (-not $downloaded) { + exit 1 +} + +$whisperCmd = "whisper-cli" +if (-not (Get-Command whisper-cli -ErrorAction SilentlyContinue)) { + $rootPath = Split-Path -Parent $ScriptDir + $whisperCmd = Join-Path $rootPath "build\bin\Release\whisper-cli.exe" +} + +Write-Host "Done! VitisAI encoder cache '$($entry.Model)' saved in '$destinationPath'" +if ($entry.RawName -ne $entry.Model) { + Write-Host "Source cache '$($entry.SourceFile)' was renamed to match ggml model name '$($entry.Model)'." +} +Write-Host "Use it with the matching ggml model:" +Write-Host "" +Write-Host " $ScriptDir\download-ggml-model.cmd $($entry.Model) $ModelsPath" +Write-Host " $whisperCmd -m $ModelsPath\ggml-$($entry.Model).bin -f samples\jfk.wav" +Write-Host "" diff --git a/models/download-vitisai-model.sh b/models/download-vitisai-model.sh new file mode 100755 index 000000000..ddae36d22 --- /dev/null +++ b/models/download-vitisai-model.sh @@ -0,0 +1,226 @@ +#!/bin/sh + +# This script downloads prebuilt VitisAI encoder cache files for Whisper models. +# The cache file is saved next to the ggml model file and follows the loader +# convention: ggml--encoder-vitisai.rai + +src="https://huggingface.co" +collection_api="https://huggingface.co/api/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models" +collection_url="https://huggingface.co/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models" + +BOLD="\033[1m" +RESET='\033[0m' + +# get the path of this script +get_script_path() { + if [ -x "$(command -v realpath)" ]; then + dirname "$(realpath "$0")" + else + _ret="$(cd -- "$(dirname "$0")" >/dev/null 2>&1 || exit ; pwd -P)" + echo "$_ret" + fi +} + +find_python() { + if command -v python3 >/dev/null 2>&1; then + printf "%s\n" "python3" + elif command -v python >/dev/null 2>&1; then + printf "%s\n" "python" + else + return 1 + fi +} + +script_path="$(get_script_path)" + +# Check if the script is inside a /bin/ directory +case "$script_path" in + */bin) default_download_path="$PWD" ;; # Use current directory as default download path if in /bin/ + *) default_download_path="$script_path" ;; # Otherwise, use script directory +esac + +models_path="${2:-$default_download_path}" + +discover_models() { + python_cmd="$(find_python)" || { + printf "Python is required to query available VitisAI caches from Hugging Face.\n" >&2 + return 1 + } + + "$python_cmd" - "$src" "$collection_api" <<'PY' +import json +import os +import re +import sys +import urllib.parse +import urllib.request + +src = sys.argv[1].rstrip("/") +collection_api = sys.argv[2] +headers = {} +token = os.environ.get("HF_TOKEN") +if token: + headers["Authorization"] = "Bearer " + token + +def load_json(url): + req = urllib.request.Request(url, headers=headers) + with urllib.request.urlopen(req) as response: + return json.load(response) + +def normalize_model_name(name): + # HF currently publishes ggml-small-en-encoder-vitisai.rai, while the + # matching ggml model is ggml-small.en.bin. + if name.endswith("-en"): + return name[:-3] + ".en" + return name + +collection = load_json(collection_api) +rows = [] +seen = set() + +for item in collection.get("items", []): + if item.get("type") != "model": + continue + + repo = item.get("id") + if not repo: + continue + + model_info = load_json(src + "/api/models/" + repo) + for sibling in model_info.get("siblings", []): + filename = sibling.get("rfilename", "") + match = re.match(r"^ggml-(.+)-encoder-vitisai\.rai$", filename) + if not match: + continue + + raw_name = match.group(1) + model_name = normalize_model_name(raw_name) + if model_name in seen: + continue + seen.add(model_name) + + destination = "ggml-%s-encoder-vitisai.rai" % model_name + url = "%s/%s/resolve/main/%s" % (src, repo, urllib.parse.quote(filename)) + rows.append((model_name, raw_name, repo, filename, destination, url)) + +order = { + "tiny": 10, + "tiny.en": 11, + "base": 20, + "base.en": 21, + "small": 30, + "small.en": 31, + "medium": 40, + "medium.en": 41, + "large-v1": 50, + "large-v2": 60, + "large-v3": 70, + "large-v3-turbo": 80, +} + +for row in sorted(rows, key=lambda item: (order.get(item[0], 1000), item[0])): + print("|".join(row)) +PY +} + +list_models() { + models="$(discover_models)" || exit 1 + + printf "\n" + printf "Available VitisAI encoder caches from %s:\n" "$collection_url" + printf "%s\n" "$models" | while IFS='|' read -r model raw repo _source _destination _url; do + if [ "$model" = "$raw" ]; then + printf " %-18s %s\n" "$model" "$repo" + else + printf " %-18s %s (source name: %s)\n" "$model" "$repo" "$raw" + fi + done + printf "\n" +} + +usage() { + printf "Usage: %s --list\n" "$0" + printf " %s [models_path]\n" "$0" + printf "\n" + printf "Downloads ggml--encoder-vitisai.rai next to ggml-.bin.\n" + printf "Use the same model name as %s/download-ggml-model.sh.\n" "$script_path" + printf "\n" +} + +if [ "$#" -eq 1 ] && { [ "$1" = "--list" ] || [ "$1" = "-l" ] || [ "$1" = "list" ]; }; then + list_models + exit 0 +fi + +if [ "$#" -lt 1 ] || [ "$#" -gt 2 ]; then + usage + list_models + printf "___________________________________________________________\n" + printf "Example: %s ${BOLD}small${RESET} %s\n" "$0" "$default_download_path" + exit 1 +fi + +model=$1 +models="$(discover_models)" || exit 1 + +match="$(printf "%s\n" "$models" | awk -F '|' -v model="$model" '$1 == model || $2 == model { print; exit }')" +if [ -z "$match" ]; then + printf "Invalid model: %s\n" "$model" + printf "%s\n" "$models" | while IFS='|' read -r available _raw _repo _source _destination _url; do + printf " %s\n" "$available" + done + exit 1 +fi + +IFS='|' read -r model raw_name repo source_file destination_file download_url </dev/null 2>&1; then + whisper_cmd="whisper-cli" +else + whisper_cmd="./build/bin/whisper-cli" +fi + +printf "Done! VitisAI encoder cache '%s' saved in '%s/%s'\n" "$model" "$models_path" "$destination_file" +if [ "$raw_name" != "$model" ]; then + printf "Source cache '%s' was renamed to match ggml model name '%s'.\n" "$source_file" "$model" +fi +printf "Use it with the matching ggml model:\n\n" +printf " $ %s/download-ggml-model.sh %s %s\n" "$script_path" "$model" "$models_path" +printf " $ %s -m %s/ggml-%s.bin -f samples/jfk.wav\n" "$whisper_cmd" "$models_path" "$model" +printf "\n"