Add VitisAI model download scripts
This commit is contained in:
Sachin Kumawat
parent
c9f63ad1f3
commit
a4660b7cae
51
README.md
51
README.md
|
|
@ -323,37 +323,36 @@ On AMD's Ryzen™ AI 300 Series with dedicated NPUs for acceleration, you can no
|
|||
|
||||
### Setup environment (Windows only)
|
||||
|
||||
- **Driver:** Make sure you have NPU drivers version **.280 or newer** installed. [Download latest drivers from here](https://account.amd.com/en/forms/downloads/ryzenai-eula-public-xef.html?filename=NPU_RAI1.5_280_WHQL.zip)
|
||||
- **Runtime libraries:** Download and install the necessary [runtime dependencies from here](https://account.amd.com/en/forms/downloads/ryzenai-eula-public-xef.html?filename=flexmlrt1.7.0-win.zip).
|
||||
- **Environment:** Extract the runtime package and set up the environment:
|
||||
```powershell
|
||||
tar xvf flexmlrt1.7.0-win.zip
|
||||
flexmlrt\setup.bat
|
||||
- Obtain the XRT package and the FlexmlRT package from AMD. Both are distributed as tarballs or wheels.
|
||||
- Copy the downloaded archives to a local path, extract them, and run the setup script from each extracted package in your shell (for example `source /path/to/xrt/setup.sh` and `source /path/to/flexmlrt/setup.sh`). Run these in every new shell you use to build or run `whisper.cpp`.
|
||||
|
||||
- Fetch the matching ggml model and prebuilt VitisAI encoder cache:
|
||||
|
||||
```bash
|
||||
sh ./models/download-ggml-model.sh base
|
||||
sh ./models/download-vitisai-model.sh base
|
||||
```
|
||||
|
||||
```cmd
|
||||
.\models\download-ggml-model.cmd base
|
||||
.\models\download-vitisai-model.cmd base
|
||||
```
|
||||
|
||||
Use the same model name with both scripts. The VitisAI script queries the AMD collection on Hugging Face to list available caches, then downloads the selected `.rai` file as `ggml-<model>-encoder-vitisai.rai` alongside the matching `ggml-<model>.bin` file. You can also browse the collection manually at https://huggingface.co/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models.
|
||||
|
||||
- Build `whisper.cpp` with VitisAI support:
|
||||
|
||||
```bash
|
||||
cmake -B build -DWHISPER_VITISAI=1
|
||||
cmake --build build -j --config Release
|
||||
```
|
||||
Your environment is now ready.
|
||||
|
||||
### Build Whisper.cpp for Ryzen™ AI support
|
||||
|
||||
```bash
|
||||
cmake -B build -DWHISPER_VITISAI=1
|
||||
cmake --build build -j --config Release
|
||||
```
|
||||
|
||||
### Download NPU-optimized models
|
||||
|
||||
- All NPU-supported Whisper models and their compiled `.rai` cache files are available in this collection:
|
||||
https://huggingface.co/collections/amd/ryzen-ai-16-whisper-npu-optimized-onnx-models
|
||||
- Download the pre-compiled `.rai` cache file matching your desired model, and place it in your `models/` directory alongside its corresponding `ggml-<...>.bin` file.
|
||||
The cache file must be named with the `-encoder-vitisai.rai` suffix. For example, if your model file is named `ggml-small.bin`, the cache file should be named `ggml-small-encoder-vitisai.rai`.
|
||||
|
||||
|
||||
> **Note:** The ".rai" models from Hugging Face are pre-optimized for Ryzen™ AI NPUs, delivering acceleration benefits from the very first run (aside from any initial CPU-side caching overhead).
|
||||
|
||||
Run the examples as usual:
|
||||
|
||||
```bash
|
||||
./build/bin/whisper-cli -m models/ggml-small.bin -f samples/jfk.wav
|
||||
```
|
||||
```text
|
||||
$ ./build/bin/whisper-cli -m models/ggml-base.bin -f samples/jfk.wav
|
||||
```
|
||||
|
||||
|
||||
## NVIDIA GPU support
|
||||
|
|
|
|||
|
|
@ -0,0 +1,32 @@
|
|||
@echo off
|
||||
setlocal
|
||||
|
||||
set "script=%~dp0download-vitisai-model.ps1"
|
||||
|
||||
if "%~1"=="" (
|
||||
PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%"
|
||||
exit /b %ERRORLEVEL%
|
||||
)
|
||||
|
||||
if /I "%~1"=="--list" (
|
||||
PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" -List
|
||||
exit /b %ERRORLEVEL%
|
||||
)
|
||||
|
||||
if /I "%~1"=="-l" (
|
||||
PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" -List
|
||||
exit /b %ERRORLEVEL%
|
||||
)
|
||||
|
||||
if /I "%~1"=="list" (
|
||||
PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" -List
|
||||
exit /b %ERRORLEVEL%
|
||||
)
|
||||
|
||||
if "%~2"=="" (
|
||||
PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" -Model "%~1"
|
||||
) else (
|
||||
PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" -Model "%~1" -ModelsPath "%~2"
|
||||
)
|
||||
|
||||
exit /b %ERRORLEVEL%
|
||||
|
|
@ -0,0 +1,218 @@
|
|||
param(
|
||||
[Parameter(Position = 0)]
|
||||
[string] $Model,
|
||||
|
||||
[Parameter(Position = 1)]
|
||||
[string] $ModelsPath,
|
||||
|
||||
[switch] $List
|
||||
)
|
||||
|
||||
$ErrorActionPreference = "Stop"
|
||||
|
||||
$Source = "https://huggingface.co"
|
||||
$CollectionApi = "https://huggingface.co/api/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models"
|
||||
$CollectionUrl = "https://huggingface.co/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models"
|
||||
|
||||
$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path
|
||||
if ($ScriptDir -match "\\bin$") {
|
||||
$DefaultDownloadPath = (Get-Location).Path
|
||||
} else {
|
||||
$DefaultDownloadPath = $ScriptDir
|
||||
}
|
||||
|
||||
if (-not $ModelsPath) {
|
||||
$ModelsPath = $DefaultDownloadPath
|
||||
}
|
||||
|
||||
function Get-HfHeaders {
|
||||
$headers = @{}
|
||||
if ($env:HF_TOKEN) {
|
||||
$headers["Authorization"] = "Bearer $env:HF_TOKEN"
|
||||
}
|
||||
return $headers
|
||||
}
|
||||
|
||||
function Invoke-HfJson {
|
||||
param([string] $Uri)
|
||||
|
||||
$headers = Get-HfHeaders
|
||||
if ($headers.Count -gt 0) {
|
||||
return Invoke-RestMethod -Uri $Uri -Headers $headers
|
||||
}
|
||||
|
||||
return Invoke-RestMethod -Uri $Uri
|
||||
}
|
||||
|
||||
function Normalize-ModelName {
|
||||
param([string] $Name)
|
||||
|
||||
# HF currently publishes ggml-small-en-encoder-vitisai.rai, while the
|
||||
# matching ggml model is ggml-small.en.bin.
|
||||
if ($Name.EndsWith("-en")) {
|
||||
return $Name.Substring(0, $Name.Length - 3) + ".en"
|
||||
}
|
||||
|
||||
return $Name
|
||||
}
|
||||
|
||||
function Get-VitisAiModels {
|
||||
$collection = Invoke-HfJson -Uri $CollectionApi
|
||||
$seen = @{}
|
||||
$rows = New-Object System.Collections.Generic.List[object]
|
||||
|
||||
foreach ($item in $collection.items) {
|
||||
if ($item.type -ne "model") {
|
||||
continue
|
||||
}
|
||||
|
||||
$repo = $item.id
|
||||
if (-not $repo) {
|
||||
continue
|
||||
}
|
||||
|
||||
$modelInfo = Invoke-HfJson -Uri "$Source/api/models/$repo"
|
||||
foreach ($sibling in $modelInfo.siblings) {
|
||||
$filename = [string] $sibling.rfilename
|
||||
$match = [regex]::Match($filename, "^ggml-(.+)-encoder-vitisai\.rai$")
|
||||
if (-not $match.Success) {
|
||||
continue
|
||||
}
|
||||
|
||||
$rawName = $match.Groups[1].Value
|
||||
$modelName = Normalize-ModelName -Name $rawName
|
||||
if ($seen.ContainsKey($modelName)) {
|
||||
continue
|
||||
}
|
||||
|
||||
$seen[$modelName] = $true
|
||||
$destination = "ggml-$modelName-encoder-vitisai.rai"
|
||||
$url = "$Source/$repo/resolve/main/$([uri]::EscapeDataString($filename))"
|
||||
|
||||
$rows.Add([pscustomobject]@{
|
||||
Model = $modelName
|
||||
RawName = $rawName
|
||||
Repo = $repo
|
||||
SourceFile = $filename
|
||||
DestinationFile = $destination
|
||||
DownloadUrl = $url
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
$order = @{
|
||||
"tiny" = 10
|
||||
"tiny.en" = 11
|
||||
"base" = 20
|
||||
"base.en" = 21
|
||||
"small" = 30
|
||||
"small.en" = 31
|
||||
"medium" = 40
|
||||
"medium.en" = 41
|
||||
"large-v1" = 50
|
||||
"large-v2" = 60
|
||||
"large-v3" = 70
|
||||
"large-v3-turbo" = 80
|
||||
}
|
||||
|
||||
return $rows | Sort-Object `
|
||||
@{ Expression = { if ($order.ContainsKey($_.Model)) { $order[$_.Model] } else { 1000 } } }, `
|
||||
@{ Expression = { $_.Model } }
|
||||
}
|
||||
|
||||
function Show-Models {
|
||||
$models = Get-VitisAiModels
|
||||
|
||||
Write-Host ""
|
||||
Write-Host "Available VitisAI encoder caches from ${CollectionUrl}:"
|
||||
foreach ($entry in $models) {
|
||||
if ($entry.Model -eq $entry.RawName) {
|
||||
Write-Host (" {0,-18} {1}" -f $entry.Model, $entry.Repo)
|
||||
} else {
|
||||
Write-Host (" {0,-18} {1} (source name: {2})" -f $entry.Model, $entry.Repo, $entry.RawName)
|
||||
}
|
||||
}
|
||||
Write-Host ""
|
||||
}
|
||||
|
||||
function Show-Usage {
|
||||
Write-Host "Usage: download-vitisai-model.cmd --list"
|
||||
Write-Host " download-vitisai-model.cmd <model> [models_path]"
|
||||
Write-Host ""
|
||||
Write-Host "Downloads ggml-<model>-encoder-vitisai.rai next to ggml-<model>.bin."
|
||||
Write-Host "Use the same model name as download-ggml-model.cmd."
|
||||
Write-Host ""
|
||||
}
|
||||
|
||||
if ($List -or $Model -eq "--list" -or $Model -eq "-l" -or $Model -eq "list") {
|
||||
Show-Models
|
||||
exit 0
|
||||
}
|
||||
|
||||
if (-not $Model) {
|
||||
Show-Usage
|
||||
Show-Models
|
||||
exit 1
|
||||
}
|
||||
|
||||
$models = Get-VitisAiModels
|
||||
$entry = $models | Where-Object { $_.Model -eq $Model -or $_.RawName -eq $Model } | Select-Object -First 1
|
||||
if (-not $entry) {
|
||||
Write-Host "Invalid model: $Model"
|
||||
foreach ($available in $models) {
|
||||
Write-Host " $($available.Model)"
|
||||
}
|
||||
exit 1
|
||||
}
|
||||
|
||||
New-Item -ItemType Directory -Force -Path $ModelsPath | Out-Null
|
||||
$destinationPath = Join-Path $ModelsPath $entry.DestinationFile
|
||||
|
||||
Write-Host "Downloading VitisAI encoder cache $($entry.Model) from '$($entry.Repo)' ..."
|
||||
if (Test-Path $destinationPath) {
|
||||
Write-Host "VitisAI encoder cache $($entry.DestinationFile) already exists. Skipping download."
|
||||
exit 0
|
||||
}
|
||||
|
||||
$headers = Get-HfHeaders
|
||||
$downloaded = $false
|
||||
for ($attempt = 1; $attempt -le 5; ++$attempt) {
|
||||
try {
|
||||
if ($headers.Count -gt 0) {
|
||||
Invoke-WebRequest -Uri $entry.DownloadUrl -Headers $headers -OutFile $destinationPath
|
||||
} else {
|
||||
Invoke-WebRequest -Uri $entry.DownloadUrl -OutFile $destinationPath
|
||||
}
|
||||
$downloaded = $true
|
||||
break
|
||||
} catch {
|
||||
if ($attempt -eq 5) {
|
||||
if (Test-Path $destinationPath) {
|
||||
Remove-Item -Force $destinationPath
|
||||
}
|
||||
Write-Host "Failed to download VitisAI encoder cache $($entry.Model) from $($entry.DownloadUrl)"
|
||||
throw
|
||||
}
|
||||
Start-Sleep -Seconds 5
|
||||
}
|
||||
}
|
||||
|
||||
if (-not $downloaded) {
|
||||
exit 1
|
||||
}
|
||||
|
||||
$whisperCmd = "whisper-cli"
|
||||
if (-not (Get-Command whisper-cli -ErrorAction SilentlyContinue)) {
|
||||
$rootPath = Split-Path -Parent $ScriptDir
|
||||
$whisperCmd = Join-Path $rootPath "build\bin\Release\whisper-cli.exe"
|
||||
}
|
||||
|
||||
Write-Host "Done! VitisAI encoder cache '$($entry.Model)' saved in '$destinationPath'"
|
||||
if ($entry.RawName -ne $entry.Model) {
|
||||
Write-Host "Source cache '$($entry.SourceFile)' was renamed to match ggml model name '$($entry.Model)'."
|
||||
}
|
||||
Write-Host "Use it with the matching ggml model:"
|
||||
Write-Host ""
|
||||
Write-Host " $ScriptDir\download-ggml-model.cmd $($entry.Model) $ModelsPath"
|
||||
Write-Host " $whisperCmd -m $ModelsPath\ggml-$($entry.Model).bin -f samples\jfk.wav"
|
||||
Write-Host ""
|
||||
|
|
@ -0,0 +1,226 @@
|
|||
#!/bin/sh
|
||||
|
||||
# This script downloads prebuilt VitisAI encoder cache files for Whisper models.
|
||||
# The cache file is saved next to the ggml model file and follows the loader
|
||||
# convention: ggml-<model>-encoder-vitisai.rai
|
||||
|
||||
src="https://huggingface.co"
|
||||
collection_api="https://huggingface.co/api/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models"
|
||||
collection_url="https://huggingface.co/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models"
|
||||
|
||||
BOLD="\033[1m"
|
||||
RESET='\033[0m'
|
||||
|
||||
# get the path of this script
|
||||
get_script_path() {
|
||||
if [ -x "$(command -v realpath)" ]; then
|
||||
dirname "$(realpath "$0")"
|
||||
else
|
||||
_ret="$(cd -- "$(dirname "$0")" >/dev/null 2>&1 || exit ; pwd -P)"
|
||||
echo "$_ret"
|
||||
fi
|
||||
}
|
||||
|
||||
find_python() {
|
||||
if command -v python3 >/dev/null 2>&1; then
|
||||
printf "%s\n" "python3"
|
||||
elif command -v python >/dev/null 2>&1; then
|
||||
printf "%s\n" "python"
|
||||
else
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
script_path="$(get_script_path)"
|
||||
|
||||
# Check if the script is inside a /bin/ directory
|
||||
case "$script_path" in
|
||||
*/bin) default_download_path="$PWD" ;; # Use current directory as default download path if in /bin/
|
||||
*) default_download_path="$script_path" ;; # Otherwise, use script directory
|
||||
esac
|
||||
|
||||
models_path="${2:-$default_download_path}"
|
||||
|
||||
discover_models() {
|
||||
python_cmd="$(find_python)" || {
|
||||
printf "Python is required to query available VitisAI caches from Hugging Face.\n" >&2
|
||||
return 1
|
||||
}
|
||||
|
||||
"$python_cmd" - "$src" "$collection_api" <<'PY'
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import urllib.parse
|
||||
import urllib.request
|
||||
|
||||
src = sys.argv[1].rstrip("/")
|
||||
collection_api = sys.argv[2]
|
||||
headers = {}
|
||||
token = os.environ.get("HF_TOKEN")
|
||||
if token:
|
||||
headers["Authorization"] = "Bearer " + token
|
||||
|
||||
def load_json(url):
|
||||
req = urllib.request.Request(url, headers=headers)
|
||||
with urllib.request.urlopen(req) as response:
|
||||
return json.load(response)
|
||||
|
||||
def normalize_model_name(name):
|
||||
# HF currently publishes ggml-small-en-encoder-vitisai.rai, while the
|
||||
# matching ggml model is ggml-small.en.bin.
|
||||
if name.endswith("-en"):
|
||||
return name[:-3] + ".en"
|
||||
return name
|
||||
|
||||
collection = load_json(collection_api)
|
||||
rows = []
|
||||
seen = set()
|
||||
|
||||
for item in collection.get("items", []):
|
||||
if item.get("type") != "model":
|
||||
continue
|
||||
|
||||
repo = item.get("id")
|
||||
if not repo:
|
||||
continue
|
||||
|
||||
model_info = load_json(src + "/api/models/" + repo)
|
||||
for sibling in model_info.get("siblings", []):
|
||||
filename = sibling.get("rfilename", "")
|
||||
match = re.match(r"^ggml-(.+)-encoder-vitisai\.rai$", filename)
|
||||
if not match:
|
||||
continue
|
||||
|
||||
raw_name = match.group(1)
|
||||
model_name = normalize_model_name(raw_name)
|
||||
if model_name in seen:
|
||||
continue
|
||||
seen.add(model_name)
|
||||
|
||||
destination = "ggml-%s-encoder-vitisai.rai" % model_name
|
||||
url = "%s/%s/resolve/main/%s" % (src, repo, urllib.parse.quote(filename))
|
||||
rows.append((model_name, raw_name, repo, filename, destination, url))
|
||||
|
||||
order = {
|
||||
"tiny": 10,
|
||||
"tiny.en": 11,
|
||||
"base": 20,
|
||||
"base.en": 21,
|
||||
"small": 30,
|
||||
"small.en": 31,
|
||||
"medium": 40,
|
||||
"medium.en": 41,
|
||||
"large-v1": 50,
|
||||
"large-v2": 60,
|
||||
"large-v3": 70,
|
||||
"large-v3-turbo": 80,
|
||||
}
|
||||
|
||||
for row in sorted(rows, key=lambda item: (order.get(item[0], 1000), item[0])):
|
||||
print("|".join(row))
|
||||
PY
|
||||
}
|
||||
|
||||
list_models() {
|
||||
models="$(discover_models)" || exit 1
|
||||
|
||||
printf "\n"
|
||||
printf "Available VitisAI encoder caches from %s:\n" "$collection_url"
|
||||
printf "%s\n" "$models" | while IFS='|' read -r model raw repo _source _destination _url; do
|
||||
if [ "$model" = "$raw" ]; then
|
||||
printf " %-18s %s\n" "$model" "$repo"
|
||||
else
|
||||
printf " %-18s %s (source name: %s)\n" "$model" "$repo" "$raw"
|
||||
fi
|
||||
done
|
||||
printf "\n"
|
||||
}
|
||||
|
||||
usage() {
|
||||
printf "Usage: %s --list\n" "$0"
|
||||
printf " %s <model> [models_path]\n" "$0"
|
||||
printf "\n"
|
||||
printf "Downloads ggml-<model>-encoder-vitisai.rai next to ggml-<model>.bin.\n"
|
||||
printf "Use the same model name as %s/download-ggml-model.sh.\n" "$script_path"
|
||||
printf "\n"
|
||||
}
|
||||
|
||||
if [ "$#" -eq 1 ] && { [ "$1" = "--list" ] || [ "$1" = "-l" ] || [ "$1" = "list" ]; }; then
|
||||
list_models
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$#" -lt 1 ] || [ "$#" -gt 2 ]; then
|
||||
usage
|
||||
list_models
|
||||
printf "___________________________________________________________\n"
|
||||
printf "Example: %s ${BOLD}small${RESET} %s\n" "$0" "$default_download_path"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
model=$1
|
||||
models="$(discover_models)" || exit 1
|
||||
|
||||
match="$(printf "%s\n" "$models" | awk -F '|' -v model="$model" '$1 == model || $2 == model { print; exit }')"
|
||||
if [ -z "$match" ]; then
|
||||
printf "Invalid model: %s\n" "$model"
|
||||
printf "%s\n" "$models" | while IFS='|' read -r available _raw _repo _source _destination _url; do
|
||||
printf " %s\n" "$available"
|
||||
done
|
||||
exit 1
|
||||
fi
|
||||
|
||||
IFS='|' read -r model raw_name repo source_file destination_file download_url <<EOF
|
||||
$match
|
||||
EOF
|
||||
|
||||
printf "Downloading VitisAI encoder cache %s from '%s' ...\n" "$model" "$repo"
|
||||
|
||||
mkdir -p "$models_path" || exit
|
||||
cd "$models_path" || exit
|
||||
|
||||
if [ -f "$destination_file" ]; then
|
||||
printf "VitisAI encoder cache %s already exists. Skipping download.\n" "$destination_file"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ -x "$(command -v wget2)" ]; then
|
||||
wget2 --no-config --progress bar -O "$destination_file" ${HF_TOKEN:+--header "Authorization: Bearer $HF_TOKEN"} "$download_url"
|
||||
elif [ -x "$(command -v curl)" ]; then
|
||||
curl -L --fail \
|
||||
--retry 5 \
|
||||
--retry-delay 5 \
|
||||
--retry-all-errors \
|
||||
--retry-connrefused \
|
||||
${HF_TOKEN:+--header "Authorization: Bearer $HF_TOKEN"} \
|
||||
--output "$destination_file" "$download_url"
|
||||
elif [ -x "$(command -v wget)" ]; then
|
||||
wget --no-config --quiet --show-progress ${HF_TOKEN:+--header "Authorization: Bearer $HF_TOKEN"} -O "$destination_file" "$download_url"
|
||||
else
|
||||
printf "Either wget2, curl, or wget is required to download VitisAI encoder caches.\n"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
printf "Failed to download VitisAI encoder cache %s from %s\n" "$model" "$download_url"
|
||||
rm -f "$destination_file"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if 'whisper-cli' is available in the system PATH
|
||||
if command -v whisper-cli >/dev/null 2>&1; then
|
||||
whisper_cmd="whisper-cli"
|
||||
else
|
||||
whisper_cmd="./build/bin/whisper-cli"
|
||||
fi
|
||||
|
||||
printf "Done! VitisAI encoder cache '%s' saved in '%s/%s'\n" "$model" "$models_path" "$destination_file"
|
||||
if [ "$raw_name" != "$model" ]; then
|
||||
printf "Source cache '%s' was renamed to match ggml model name '%s'.\n" "$source_file" "$model"
|
||||
fi
|
||||
printf "Use it with the matching ggml model:\n\n"
|
||||
printf " $ %s/download-ggml-model.sh %s %s\n" "$script_path" "$model" "$models_path"
|
||||
printf " $ %s -m %s/ggml-%s.bin -f samples/jfk.wav\n" "$whisper_cmd" "$models_path" "$model"
|
||||
printf "\n"
|
||||
Loading…
Reference in New Issue