From 66e882aeedf387614316e0c94f7d59a815766c9b Mon Sep 17 00:00:00 2001 From: "Kumawat, Sachin" Date: Tue, 13 Jan 2026 14:14:27 -0800 Subject: [PATCH 01/24] Add VitisAI Plugin * Added VitisAI encoder module placeholder files * VitisAI build integration * VitisAI encoder offload functional * Clean up vitisai integration * Add c++17 requirement for Windows * Enabled preemption for windows runs * Add model cache override option * Remove vitisai premature log message * Add rai support through file mapping * Fixed flatbuffer loading * Fixed Windows file mapping issue * Update FlexmlRT resolution * Use Flexmlrt wheel pkg to build VitisAI plugin * Clean up * Remove prints * Change flexmlrt target from Shared to Interface * Add c++17 requirement for Windows * Enabled preemption for windows runs * Add rai support through file mapping * Fixed flatbuffer loading * Fixed Windows file mapping issue * Update FlexmlRT resolution * Use Flexmlrt wheel pkg to build VitisAI plugin * Clean up * Remove prints * Change flexmlrt target from Shared to Interface * Cleanup FlexmlRT integration * format fix * Adding AMD Licenses * Update CMakeLists.txt Co-authored-by: Kumawat, Sachin * Update src/CMakeLists.txt Co-authored-by: Kumawat, Sachin * Update whisper.cpp * Added VitisAI encoder readme section * Remove license headers from common files to whisper.cpp --------- Co-authored-by: Sachin Kumawat Co-authored-by: Jeff Lin Co-authored-by: Lin Co-authored-by: Lin, Jeff (DCG-ENG) Co-authored-by: Iswarya Alex Co-authored-by: Alex, Iswarya --- CMakeLists.txt | 1 + README.md | 29 ++++ src/CMakeLists.txt | 32 ++++ src/vitisai/whisper-vitisai-encoder.cpp | 204 ++++++++++++++++++++++++ src/vitisai/whisper-vitisai-encoder.h | 32 ++++ src/whisper.cpp | 61 ++++++- 6 files changed, 358 insertions(+), 1 deletion(-) create mode 100644 src/vitisai/whisper-vitisai-encoder.cpp create mode 100644 src/vitisai/whisper-vitisai-encoder.h diff --git a/CMakeLists.txt b/CMakeLists.txt index b60bb0452..a8c7347a6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -91,6 +91,7 @@ endif() option(WHISPER_COREML "whisper: enable Core ML framework" OFF) option(WHISPER_COREML_ALLOW_FALLBACK "whisper: allow non-CoreML fallback" OFF) option(WHISPER_OPENVINO "whisper: support for OpenVINO" OFF) +option(WHISPER_VITISAI "whisper: support for AMD Vitis AI" OFF) # Required for relocatable CMake package include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/build-info.cmake) diff --git a/README.md b/README.md index 6d4988e6f..0369f142e 100644 --- a/README.md +++ b/README.md @@ -312,6 +312,35 @@ This can result in significant speedup in encoder performance. Here are the inst For more information about the OpenVINO implementation please refer to PR [#1037](https://github.com/ggml-org/whisper.cpp/pull/1037). +## VitisAI encoder support + +On AMD Ryzen AI NPU devices, you can run the Encoder via the VitisAI plugin to significantly accelerate the whisper models. + +- Prepare the AMD runtime packages (required before building): + + - Obtain the XRT package and the FlexmlRT package from AMD. Both are distributed as tarballs or wheels. + - Copy the downloaded archives to a local path, extract them, and run the setup script from each extracted package in your shell (for example `source /path/to/xrt/setup.sh` and `source /path/to/flexmlrt/setup.sh`). Run these in every new shell you use to build or run `whisper.cpp`. + +- Fetch the prebuilt VitisAI encoder cache: + + - Download the appropriate Whisper encoder `.rai` cache for your model size from the AMD collection on Hugging Face: https://huggingface.co/collections/amd/ryzen-ai-16-whisper-npu-optimized-onnx-models + - Place and rename the downloaded `.rai` file as `-encoder-vitisai.rai` alongside your ggml model files `.bin`. + +- Build `whisper.cpp` with VitisAI support: + + ```bash + cmake -B build -DWHISPER_VITISAI=1 + cmake --build build -j --config Release + ``` + +- Run the examples as usual. For example: + + ```text + $ ./build/bin/whisper-cli -m models/ggml-base.en.bin -f samples/jfk.wav + ``` + +The VitisAI artifact from Huggingface is already optimized for Ryzen AI NPUs, there is no slow compilation needed. The acceleration advantage should be seen from first run itself apart from CPU caching overheads. + ## NVIDIA GPU support With NVIDIA cards the processing of the models is done efficiently on the GPU via cuBLAS and custom CUDA kernels. diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 095a2791d..6cba1c6e3 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -48,6 +48,10 @@ if (WHISPER_OPENVINO) find_package(OpenVINO REQUIRED COMPONENTS Runtime) endif() +if (WHISPER_VITISAI) + find_package(FlexmlRT REQUIRED) +endif() + # # libraries # @@ -101,6 +105,30 @@ if (WHISPER_OPENVINO) set_target_properties(${TARGET} PROPERTIES FOLDER "libs") endif() +if (WHISPER_VITISAI) + set(TARGET whisper.vitisai) + + add_library(${TARGET} OBJECT + vitisai/whisper-vitisai-encoder.h + vitisai/whisper-vitisai-encoder.cpp + ) + + target_include_directories(${TARGET} PUBLIC + . + ) + + set_property(TARGET ${TARGET} PROPERTY POSITION_INDEPENDENT_CODE ON) + set(WHISPER_EXTRA_FLAGS ${WHISPER_EXTRA_FLAGS} -DWHISPER_USE_VITISAI) + + # Add C++17 standard for MSVC + if (MSVC) + target_compile_options(${TARGET} PRIVATE /std:c++17) + endif() + + target_link_libraries(${TARGET} PRIVATE ggml flexmlrt::flexmlrt) + set_target_properties(${TARGET} PROPERTIES FOLDER "libs") +endif() + # whisper add_library(whisper @@ -137,6 +165,10 @@ if (WHISPER_OPENVINO) target_link_libraries(whisper PRIVATE whisper.openvino) endif() +if (WHISPER_VITISAI) + target_link_libraries(whisper PRIVATE whisper.vitisai) +endif() + if (WHISPER_MKL) target_link_libraries(whisper PRIVATE MKL::MKL) endif() diff --git a/src/vitisai/whisper-vitisai-encoder.cpp b/src/vitisai/whisper-vitisai-encoder.cpp new file mode 100644 index 000000000..a6d20a88c --- /dev/null +++ b/src/vitisai/whisper-vitisai-encoder.cpp @@ -0,0 +1,204 @@ +// Copyright(C) 2025 Advanced Micro Devices, Inc. All rights reserved. +#include "vitisai/whisper-vitisai-encoder.h" +#include "FlexMLClient.h" +#include "ggml.h" +#include "ggml-backend.h" + +#include +#include +#ifdef _WIN32 + #include +#else + #include + #include + #include +#endif +#include +#include + +struct whisper_vitisai_context { + std::string model_path; + std::shared_ptr runner; + uint8_t * fbs_buffer; + size_t fbs_buffer_size; +}; + +// Function to mmap rai file for Linux and MapViewOfFile for Windows +bool map_rai_file(const char * path, uint8_t ** buffer, size_t * size) { +#ifdef _WIN32 + // Open the file + HANDLE hFile = CreateFileA(path, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + if (hFile == INVALID_HANDLE_VALUE) { + std::fprintf(stderr, "%s: %d: Failed to open rai file '%s'\n", __func__, __LINE__, path); + return false; + } + + // Get the file size + LARGE_INTEGER fileSize; + if (!GetFileSizeEx(hFile, &fileSize)) { + CloseHandle(hFile); + std::fprintf(stderr, "%s: %d: Failed to get file size for rai file '%s'\n", __func__, __LINE__, path); + return false; + } + + // Create a file mapping object + HANDLE hMapping = CreateFileMappingA(hFile, NULL, PAGE_READONLY, 0, fileSize.QuadPart, NULL); + if (hMapping == NULL) { + CloseHandle(hFile); + std::fprintf(stderr, "%s: %d: Failed to create file mapping for rai file '%s'\n", __func__, __LINE__, path); + return false; + } + + // Map the file + *buffer = (uint8_t *)MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, fileSize.QuadPart); + if (*buffer == NULL) { + CloseHandle(hMapping); + CloseHandle(hFile); + std::fprintf(stderr, "%s: %d: Failed to map rai file '%s'\n", __func__, __LINE__, path); + return false; + } + *size = fileSize.QuadPart; + return true; +#else + // Open the file + FILE * fd = fopen(path, "rb"); + if (!fd) { + std::fprintf(stderr, "%s: %d: Failed to open rai file '%s'\n", __func__, __LINE__, path); + return false; + } + + // Get the file size + struct stat st; + if (fstat(fileno(fd), &st) == -1) { + fclose(fd); + std::fprintf(stderr, "%s: %d: Failed to get file size for rai file '%s'\n", __func__, __LINE__, path); + return false; + } + + // Mmap the file + *buffer = (uint8_t *)mmap(nullptr, st.st_size, PROT_READ, MAP_SHARED, fileno(fd), 0); + if (*buffer == MAP_FAILED) { + fclose(fd); + std::fprintf(stderr, "%s: %d: Failed to mmap rai file '%s'\n", __func__, __LINE__, path); + return false; + } + *size = st.st_size; + return true; +#endif // _WIN32 +} + +void unmap_rai_file(uint8_t * buffer, size_t size) { +#ifdef _WIN32 + UnmapViewOfFile(buffer); +#else + munmap(buffer, size); +#endif // _WIN32 +} + +struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model) { + if (!path_model) { + std::fprintf(stderr, "%s: path_model is null\n", __func__); + return nullptr; + } + + auto * ctx = new whisper_vitisai_context; + ctx->model_path = path_model; + + // Override the model path with the environment variable if it is set + if (const char * env_model_path = std::getenv("OVERRIDE_VITISAI_MODEL_PATH")) { + if (env_model_path[0] != '\0') { + ctx->model_path = env_model_path; + } + } + + // Step 1: Set up the model + flexmlrt::client::Options options; + options.modelPath = ctx->model_path; + options.deviceName = "stx"; + options.debug = false; + options.executeMode = 2; + options.extOptions["ai_analyzer_profiling"] = true; // Enable AIA profiling + options.extOptions["enable_preemption"] = true; + + // Check if model_path is rai file and if so, add fbs_buffer and fbs_buffer_size to the options + if (ctx->model_path.find(".rai") != std::string::npos) { + // mmap rai file for both Linux and Windows and pass the buffer to the options + ctx->fbs_buffer = nullptr; + ctx->fbs_buffer_size = 0; + if (map_rai_file(ctx->model_path.c_str(), &ctx->fbs_buffer, &ctx->fbs_buffer_size)) { + options.extOptions["fbs_buffer"] = ctx->fbs_buffer; + options.extOptions["fbs_buffer_size"] = ctx->fbs_buffer_size; + options.subgraphName = "vaiml_par_0"; + options.extOptions["cache_dir"] = std::string("."); + } else { + std::fprintf(stderr, "%s: Failed to mmap rai file '%s'\n", __func__, ctx->model_path.c_str()); + delete ctx; + return nullptr; + } + } + + try { + ctx->runner = std::make_shared(options); + + if (!ctx->runner->good()) { + throw std::runtime_error("Runner creation ran into an error"); + } + } catch (const std::exception & e) { + std::fprintf(stderr, "%s: Exception during Vitis AI runner creation: %s\n", __func__, e.what()); + delete ctx; + return nullptr; + } + return ctx; +} + +void whisper_vitisai_free(struct whisper_vitisai_context * ctx) { + if (!ctx) { + return; + } + + std::fprintf(stderr, "%s: releasing Vitis AI encoder context for model '%s'\n", __func__, ctx->model_path.c_str()); + if (ctx->fbs_buffer) { + unmap_rai_file(ctx->fbs_buffer, ctx->fbs_buffer_size); + } + delete ctx; +} + +int whisper_vitisai_encode(struct whisper_vitisai_context * ctx, struct ggml_tensor * mel, struct ggml_tensor * out) { + if (!ctx || !mel || !out) { + std::fprintf(stderr, "%s: ctx/mel/out must not be null\n", __func__); + return 0; + } + + if (ggml_n_dims(mel) != 2) { + std::fprintf(stderr, "%s: mel tensor expected to have 2 dims, got %d\n", __func__, ggml_n_dims(mel)); + return 0; + } + + if (ggml_n_dims(out) != 2) { + std::fprintf(stderr, "%s: out tensor expected to have 2 dims, got %d\n", __func__, ggml_n_dims(out)); + return 0; + } + + // setup input and output tensors for Vitis AI model + std::vector input_tensors, output_tensors; + auto model = ctx->runner; + + // Get tensors as CPU tensors (hwTensor = false) + input_tensors = model->getIOTensors("input", false); + output_tensors = model->getIOTensors("output", false); + + // TODO: add assert checks for tensor numbers and shapes + + input_tensors[0].data = mel->data; + output_tensors[0].data = out->data; + + try { + model->forward(input_tensors, output_tensors); + std::fprintf(stdout, "%s: Vitis AI model inference completed.\n", __func__); + } catch (const std::exception & e) { + std::fprintf(stderr, "%s: Exception during model inference: %s\n", __func__, e.what()); + return 0; + } + + return 1; +} diff --git a/src/vitisai/whisper-vitisai-encoder.h b/src/vitisai/whisper-vitisai-encoder.h new file mode 100644 index 000000000..05dc812be --- /dev/null +++ b/src/vitisai/whisper-vitisai-encoder.h @@ -0,0 +1,32 @@ +// Copyright(C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +#pragma once + +#include +#include +#include + +#if __cplusplus +extern "C" { +#endif + +struct whisper_vitisai_context; + +struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model); +void whisper_vitisai_free(struct whisper_vitisai_context * ctx); + +// Function to mmap rai file for Linux and MapViewOfFile for Windows +bool map_rai_file(const char * path, uint8_t ** buffer, size_t * size); +// Function to unmap rai file for Linux and UnmapViewOfFile for Windows +void unmap_rai_file(uint8_t * buffer, size_t size); + +struct ggml_tensor; + +int whisper_vitisai_encode( + struct whisper_vitisai_context * ctx, + struct ggml_tensor * mel, + struct ggml_tensor * out); + +#if __cplusplus +} +#endif diff --git a/src/whisper.cpp b/src/whisper.cpp index 5b6e4b4be..59dd59c50 100644 --- a/src/whisper.cpp +++ b/src/whisper.cpp @@ -14,6 +14,10 @@ #include "openvino/whisper-openvino-encoder.h" #endif +#ifdef WHISPER_USE_VITISAI +#include "vitisai/whisper-vitisai-encoder.h" +#endif + #include #include #include @@ -903,6 +907,10 @@ struct whisper_state { whisper_openvino_context * ctx_openvino = nullptr; #endif +#ifdef WHISPER_USE_VITISAI + whisper_vitisai_context * ctx_vitisai = nullptr; +#endif + // [EXPERIMENTAL] token-level timestamps data int64_t t_beg = 0; int64_t t_last = 0; @@ -1970,7 +1978,13 @@ static bool whisper_encode_external(const whisper_state & wstate) { const bool use_openvino = wstate.ctx_openvino != nullptr; #endif - return use_coreml || use_openvino; +#ifndef WHISPER_USE_VITISAI + const bool use_vitisai = false; +#else + const bool use_vitisai = wstate.ctx_vitisai != nullptr; +#endif + + return use_coreml || use_openvino || use_vitisai; } static struct ggml_cgraph * whisper_build_graph_conv( @@ -2411,6 +2425,8 @@ static bool whisper_encode_internal( #if defined(WHISPER_USE_COREML) whisper_coreml_encode(wstate.ctx_coreml, mel->ne[0], mel->ne[1], (float *) mel->data, (float *) wstate.embd_enc->data); +#elif defined(WHISPER_USE_VITISAI) + whisper_vitisai_encode(wstate.ctx_vitisai, mel, wstate.embd_enc); #elif defined(WHISPER_USE_OPENVINO) whisper_openvino_encode(wstate.ctx_openvino, mel, wstate.embd_enc); #endif @@ -3346,6 +3362,20 @@ static std::string whisper_get_coreml_path_encoder(std::string path_bin) { } #endif +#ifdef WHISPER_USE_VITISAI +// replace extension with Vitis AI encoder artifact +static std::string whisper_get_vitisai_path_encoder_cache(std::string path_bin) { + auto pos = path_bin.rfind('.'); + if (pos != std::string::npos) { + path_bin = path_bin.substr(0, pos); + } + + path_bin += "-encoder-vitisai.rai"; + + return path_bin; +} +#endif + #ifdef WHISPER_USE_OPENVINO // replace .bin with-encoder-openvino.xml static std::string whisper_openvino_get_path_encoder(std::string path_bin) { @@ -3455,6 +3485,19 @@ struct whisper_state * whisper_init_state(whisper_context * ctx) { } #endif +#ifdef WHISPER_USE_VITISAI + const auto path_vitisai = whisper_get_vitisai_path_encoder_cache(ctx->path_model); + + state->ctx_vitisai = whisper_vitisai_init(path_vitisai.c_str()); + if (!state->ctx_vitisai) { + WHISPER_LOG_ERROR("%s: failed to load Vitis AI model from '%s'\n", __func__, path_vitisai.c_str()); + whisper_free_state(state); + return nullptr; + } else { + WHISPER_LOG_INFO("%s: Vitis AI model loaded\n", __func__); + } +#endif + state->logits.reserve(ctx->vocab.n_vocab * ctx->model.hparams.n_text_ctx); state->batch = whisper_batch_init(ctx->model.hparams.n_text_ctx, WHISPER_MAX_DECODERS); @@ -3821,6 +3864,13 @@ void whisper_free_state(struct whisper_state * state) { } #endif +#ifdef WHISPER_USE_VITISAI + if (state->ctx_vitisai != nullptr) { + whisper_vitisai_free(state->ctx_vitisai); + state->ctx_vitisai = nullptr; + } +#endif + whisper_batch_free(state->batch); ggml_backend_sched_free(state->sched_conv.sched); @@ -4312,11 +4362,20 @@ static int whisper_has_openvino(void) { #endif } +static int whisper_has_vitisai(void) { +#ifdef WHISPER_USE_VITISAI + return 1; +#else + return 0; +#endif +} + const char * whisper_print_system_info(void) { static std::string s; s = ""; s += "WHISPER : "; + s += "VITISAI = " + std::to_string(whisper_has_vitisai()) + " | "; s += "COREML = " + std::to_string(whisper_has_coreml()) + " | "; s += "OPENVINO = " + std::to_string(whisper_has_openvino()) + " | "; From 1a98960e5c21e8d2aadf5316d65b8615cfdb2eee Mon Sep 17 00:00:00 2001 From: Iswarya Alex <47045679+iswaryaalex@users.noreply.github.com> Date: Tue, 27 Jan 2026 11:51:47 -0800 Subject: [PATCH 02/24] Update README.md - RAI EULA Links - Updated for RAI Whisper instructions --- README.md | 56 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index 0369f142e..91a4b114c 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ High-performance inference of [OpenAI's Whisper](https://github.com/openai/whisp - [Vulkan support](#vulkan-gpu-support) - Support for CPU-only inference - [Efficient GPU support for NVIDIA](#nvidia-gpu-support) +- [AMD Ryzen AI NPU Support](#amd-ryzen-ai-support-for-npu) - [OpenVINO Support](#openvino-support) - [Ascend NPU Support](#ascend-npu-support) - [Moore Threads GPU Support](#moore-threads-gpu-support) @@ -312,34 +313,47 @@ This can result in significant speedup in encoder performance. Here are the inst For more information about the OpenVINO implementation please refer to PR [#1037](https://github.com/ggml-org/whisper.cpp/pull/1037). -## VitisAI encoder support +## AMD Ryzen™ AI support for NPU -On AMD Ryzen AI NPU devices, you can run the Encoder via the VitisAI plugin to significantly accelerate the whisper models. +On AMD's Ryzen™ AI 300 Series with dedicated NPUs for acceleration, you can now run Whisper models with the ability to fully offload the encoder to NPU. This brings significant speedup compared to CPU-only. +> **Note:** +> **Ryzen™ AI NPU acceleration is currently supported on Windows only.** Linux support is planned for upcoming releases. +> For the latest updates on Ryzen AI, check out [the official documentation](https://ryzenai.docs.amd.com/en/latest/). -- Prepare the AMD runtime packages (required before building): +### Setup environment (Windows only) - - Obtain the XRT package and the FlexmlRT package from AMD. Both are distributed as tarballs or wheels. - - Copy the downloaded archives to a local path, extract them, and run the setup script from each extracted package in your shell (for example `source /path/to/xrt/setup.sh` and `source /path/to/flexmlrt/setup.sh`). Run these in every new shell you use to build or run `whisper.cpp`. - -- Fetch the prebuilt VitisAI encoder cache: - - - Download the appropriate Whisper encoder `.rai` cache for your model size from the AMD collection on Hugging Face: https://huggingface.co/collections/amd/ryzen-ai-16-whisper-npu-optimized-onnx-models - - Place and rename the downloaded `.rai` file as `-encoder-vitisai.rai` alongside your ggml model files `.bin`. - -- Build `whisper.cpp` with VitisAI support: - - ```bash - cmake -B build -DWHISPER_VITISAI=1 - cmake --build build -j --config Release +- **Driver:** Make sure you have NPU drivers version **.280 or newer** installed. [Download latest drivers from here](https://account.amd.com/en/forms/downloads/ryzenai-eula-public-xef.html?filename=NPU_RAI1.5_280_WHQL.zip) +- **Runtime libraries:** Download and install the necessary [runtime dependencies from here](https://account.amd.com/en/forms/downloads/ryzenai-eula-public-xef.html?filename=flexmlrt1.7.0-win.zip). +- **Environment:** Extract the runtime package and set up the environment: + ```powershell + tar xvf flexmlrt1.7.0-win.zip + flexmlrt\setup.bat ``` +Your environment is now ready. -- Run the examples as usual. For example: +### Build Whisper.cpp for Ryzen™ AI support - ```text - $ ./build/bin/whisper-cli -m models/ggml-base.en.bin -f samples/jfk.wav - ``` +```bash +cmake -B build -DWHISPER_VITISAI=1 +cmake --build build -j --config Release +``` + +### Download NPU-optimized models + +- All NPU-supported Whisper models and their compiled `.rai` cache files are available in this collection: + https://huggingface.co/collections/amd/ryzen-ai-16-whisper-npu-optimized-onnx-models +- Download the pre-compiled `.rai` cache file matching your desired model, and place it in your `models/` directory alongside its corresponding `ggml-<...>.bin` file. + The cache file must be named with the `-encoder-vitisai.rai` suffix. For example, if your model file is named `ggml-small.bin`, the cache file should be named `ggml-small-encoder-vitisai.rai`. + + +> **Note:** The ".rai" models from Hugging Face are pre-optimized for Ryzen™ AI NPUs, delivering acceleration benefits from the very first run (aside from any initial CPU-side caching overhead). + +Run the examples as usual: + +```bash +./build/bin/whisper-cli -m models/ggml-small.bin -f samples/jfk.wav +``` -The VitisAI artifact from Huggingface is already optimized for Ryzen AI NPUs, there is no slow compilation needed. The acceleration advantage should be seen from first run itself apart from CPU caching overheads. ## NVIDIA GPU support From 175b9a53451b13e103b37194c8cb9f66de5c91fa Mon Sep 17 00:00:00 2001 From: Sachin Kumawat Date: Thu, 26 Feb 2026 12:42:08 -0800 Subject: [PATCH 03/24] Cleanup and add runtime print debug guard --- src/vitisai/whisper-vitisai-encoder.cpp | 9 +++++---- src/vitisai/whisper-vitisai-encoder.h | 7 ------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/src/vitisai/whisper-vitisai-encoder.cpp b/src/vitisai/whisper-vitisai-encoder.cpp index a6d20a88c..c10e1c37a 100644 --- a/src/vitisai/whisper-vitisai-encoder.cpp +++ b/src/vitisai/whisper-vitisai-encoder.cpp @@ -1,4 +1,3 @@ -// Copyright(C) 2025 Advanced Micro Devices, Inc. All rights reserved. #include "vitisai/whisper-vitisai-encoder.h" #include "FlexMLClient.h" #include "ggml.h" @@ -24,7 +23,7 @@ struct whisper_vitisai_context { }; // Function to mmap rai file for Linux and MapViewOfFile for Windows -bool map_rai_file(const char * path, uint8_t ** buffer, size_t * size) { +static bool map_rai_file(const char * path, uint8_t ** buffer, size_t * size) { #ifdef _WIN32 // Open the file HANDLE hFile = CreateFileA(path, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); @@ -87,7 +86,7 @@ bool map_rai_file(const char * path, uint8_t ** buffer, size_t * size) { #endif // _WIN32 } -void unmap_rai_file(uint8_t * buffer, size_t size) { +static void unmap_rai_file(uint8_t * buffer, size_t size) { #ifdef _WIN32 UnmapViewOfFile(buffer); #else @@ -194,7 +193,9 @@ int whisper_vitisai_encode(struct whisper_vitisai_context * ctx, struct ggml_ten try { model->forward(input_tensors, output_tensors); - std::fprintf(stdout, "%s: Vitis AI model inference completed.\n", __func__); +#if defined(WHISPER_DEBUG) + std::fprintf(stderr, "%s: Vitis AI model inference completed.\n", __func__); +#endif } catch (const std::exception & e) { std::fprintf(stderr, "%s: Exception during model inference: %s\n", __func__, e.what()); return 0; diff --git a/src/vitisai/whisper-vitisai-encoder.h b/src/vitisai/whisper-vitisai-encoder.h index 05dc812be..840ce6941 100644 --- a/src/vitisai/whisper-vitisai-encoder.h +++ b/src/vitisai/whisper-vitisai-encoder.h @@ -1,5 +1,3 @@ -// Copyright(C) 2025 Advanced Micro Devices, Inc. All rights reserved. - #pragma once #include @@ -15,11 +13,6 @@ struct whisper_vitisai_context; struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model); void whisper_vitisai_free(struct whisper_vitisai_context * ctx); -// Function to mmap rai file for Linux and MapViewOfFile for Windows -bool map_rai_file(const char * path, uint8_t ** buffer, size_t * size); -// Function to unmap rai file for Linux and UnmapViewOfFile for Windows -void unmap_rai_file(uint8_t * buffer, size_t size); - struct ggml_tensor; int whisper_vitisai_encode( From 988a4af6f1b7a1153b5d37e483d4e7f5caa605b4 Mon Sep 17 00:00:00 2001 From: Sachin Kumawat Date: Tue, 3 Mar 2026 16:37:29 -0800 Subject: [PATCH 04/24] turn off profiling --- src/vitisai/whisper-vitisai-encoder.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/vitisai/whisper-vitisai-encoder.cpp b/src/vitisai/whisper-vitisai-encoder.cpp index c10e1c37a..580bcfe36 100644 --- a/src/vitisai/whisper-vitisai-encoder.cpp +++ b/src/vitisai/whisper-vitisai-encoder.cpp @@ -116,7 +116,6 @@ struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model) { options.deviceName = "stx"; options.debug = false; options.executeMode = 2; - options.extOptions["ai_analyzer_profiling"] = true; // Enable AIA profiling options.extOptions["enable_preemption"] = true; // Check if model_path is rai file and if so, add fbs_buffer and fbs_buffer_size to the options From c3e8af4b7bf5eeaded9798b9281d1facaa56eaac Mon Sep 17 00:00:00 2001 From: "Kumawat, Sachin" Date: Tue, 13 Jan 2026 14:14:27 -0800 Subject: [PATCH 05/24] Add VitisAI Plugin * Added VitisAI encoder module placeholder files * VitisAI build integration * VitisAI encoder offload functional * Clean up vitisai integration * Add c++17 requirement for Windows * Enabled preemption for windows runs * Add model cache override option * Remove vitisai premature log message * Add rai support through file mapping * Fixed flatbuffer loading * Fixed Windows file mapping issue * Update FlexmlRT resolution * Use Flexmlrt wheel pkg to build VitisAI plugin * Clean up * Remove prints * Change flexmlrt target from Shared to Interface * Add c++17 requirement for Windows * Enabled preemption for windows runs * Add rai support through file mapping * Fixed flatbuffer loading * Fixed Windows file mapping issue * Update FlexmlRT resolution * Use Flexmlrt wheel pkg to build VitisAI plugin * Clean up * Remove prints * Change flexmlrt target from Shared to Interface * Cleanup FlexmlRT integration * format fix * Adding AMD Licenses * Update CMakeLists.txt Co-authored-by: Kumawat, Sachin * Update src/CMakeLists.txt Co-authored-by: Kumawat, Sachin * Update whisper.cpp * Added VitisAI encoder readme section * Remove license headers from common files to whisper.cpp --------- Co-authored-by: Sachin Kumawat Co-authored-by: Jeff Lin Co-authored-by: Lin Co-authored-by: Lin, Jeff (DCG-ENG) Co-authored-by: Iswarya Alex Co-authored-by: Alex, Iswarya --- CMakeLists.txt | 1 + README.md | 29 ++++ src/CMakeLists.txt | 32 ++++ src/vitisai/whisper-vitisai-encoder.cpp | 204 ++++++++++++++++++++++++ src/vitisai/whisper-vitisai-encoder.h | 32 ++++ src/whisper.cpp | 61 ++++++- 6 files changed, 358 insertions(+), 1 deletion(-) create mode 100644 src/vitisai/whisper-vitisai-encoder.cpp create mode 100644 src/vitisai/whisper-vitisai-encoder.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 26037c265..e25d9fc33 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -92,6 +92,7 @@ endif() option(WHISPER_COREML "whisper: enable Core ML framework" OFF) option(WHISPER_COREML_ALLOW_FALLBACK "whisper: allow non-CoreML fallback" OFF) option(WHISPER_OPENVINO "whisper: support for OpenVINO" OFF) +option(WHISPER_VITISAI "whisper: support for AMD Vitis AI" OFF) # Required for relocatable CMake package include(${CMAKE_CURRENT_SOURCE_DIR}/cmake/build-info.cmake) diff --git a/README.md b/README.md index 0e2d5f100..e96205845 100644 --- a/README.md +++ b/README.md @@ -313,6 +313,35 @@ This can result in significant speedup in encoder performance. Here are the inst For more information about the OpenVINO implementation please refer to PR [#1037](https://github.com/ggml-org/whisper.cpp/pull/1037). +## VitisAI encoder support + +On AMD Ryzen AI NPU devices, you can run the Encoder via the VitisAI plugin to significantly accelerate the whisper models. + +- Prepare the AMD runtime packages (required before building): + + - Obtain the XRT package and the FlexmlRT package from AMD. Both are distributed as tarballs or wheels. + - Copy the downloaded archives to a local path, extract them, and run the setup script from each extracted package in your shell (for example `source /path/to/xrt/setup.sh` and `source /path/to/flexmlrt/setup.sh`). Run these in every new shell you use to build or run `whisper.cpp`. + +- Fetch the prebuilt VitisAI encoder cache: + + - Download the appropriate Whisper encoder `.rai` cache for your model size from the AMD collection on Hugging Face: https://huggingface.co/collections/amd/ryzen-ai-16-whisper-npu-optimized-onnx-models + - Place and rename the downloaded `.rai` file as `-encoder-vitisai.rai` alongside your ggml model files `.bin`. + +- Build `whisper.cpp` with VitisAI support: + + ```bash + cmake -B build -DWHISPER_VITISAI=1 + cmake --build build -j --config Release + ``` + +- Run the examples as usual. For example: + + ```text + $ ./build/bin/whisper-cli -m models/ggml-base.en.bin -f samples/jfk.wav + ``` + +The VitisAI artifact from Huggingface is already optimized for Ryzen AI NPUs, there is no slow compilation needed. The acceleration advantage should be seen from first run itself apart from CPU caching overheads. + ## NVIDIA GPU support With NVIDIA cards the processing of the models is done efficiently on the GPU via cuBLAS and custom CUDA kernels. diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4e7c5b24d..de4431d9b 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -48,6 +48,10 @@ if (WHISPER_OPENVINO) find_package(OpenVINO REQUIRED COMPONENTS Runtime) endif() +if (WHISPER_VITISAI) + find_package(FlexmlRT REQUIRED) +endif() + # # libraries # @@ -101,6 +105,30 @@ if (WHISPER_OPENVINO) set_target_properties(${TARGET} PROPERTIES FOLDER "libs") endif() +if (WHISPER_VITISAI) + set(TARGET whisper.vitisai) + + add_library(${TARGET} OBJECT + vitisai/whisper-vitisai-encoder.h + vitisai/whisper-vitisai-encoder.cpp + ) + + target_include_directories(${TARGET} PUBLIC + . + ) + + set_property(TARGET ${TARGET} PROPERTY POSITION_INDEPENDENT_CODE ON) + set(WHISPER_EXTRA_FLAGS ${WHISPER_EXTRA_FLAGS} -DWHISPER_USE_VITISAI) + + # Add C++17 standard for MSVC + if (MSVC) + target_compile_options(${TARGET} PRIVATE /std:c++17) + endif() + + target_link_libraries(${TARGET} PRIVATE ggml flexmlrt::flexmlrt) + set_target_properties(${TARGET} PROPERTIES FOLDER "libs") +endif() + # whisper add_library(whisper @@ -157,6 +185,10 @@ if (WHISPER_OPENVINO) target_link_libraries(whisper PRIVATE whisper.openvino) endif() +if (WHISPER_VITISAI) + target_link_libraries(whisper PRIVATE whisper.vitisai) +endif() + if (WHISPER_MKL) target_link_libraries(whisper PRIVATE MKL::MKL) endif() diff --git a/src/vitisai/whisper-vitisai-encoder.cpp b/src/vitisai/whisper-vitisai-encoder.cpp new file mode 100644 index 000000000..a6d20a88c --- /dev/null +++ b/src/vitisai/whisper-vitisai-encoder.cpp @@ -0,0 +1,204 @@ +// Copyright(C) 2025 Advanced Micro Devices, Inc. All rights reserved. +#include "vitisai/whisper-vitisai-encoder.h" +#include "FlexMLClient.h" +#include "ggml.h" +#include "ggml-backend.h" + +#include +#include +#ifdef _WIN32 + #include +#else + #include + #include + #include +#endif +#include +#include + +struct whisper_vitisai_context { + std::string model_path; + std::shared_ptr runner; + uint8_t * fbs_buffer; + size_t fbs_buffer_size; +}; + +// Function to mmap rai file for Linux and MapViewOfFile for Windows +bool map_rai_file(const char * path, uint8_t ** buffer, size_t * size) { +#ifdef _WIN32 + // Open the file + HANDLE hFile = CreateFileA(path, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + if (hFile == INVALID_HANDLE_VALUE) { + std::fprintf(stderr, "%s: %d: Failed to open rai file '%s'\n", __func__, __LINE__, path); + return false; + } + + // Get the file size + LARGE_INTEGER fileSize; + if (!GetFileSizeEx(hFile, &fileSize)) { + CloseHandle(hFile); + std::fprintf(stderr, "%s: %d: Failed to get file size for rai file '%s'\n", __func__, __LINE__, path); + return false; + } + + // Create a file mapping object + HANDLE hMapping = CreateFileMappingA(hFile, NULL, PAGE_READONLY, 0, fileSize.QuadPart, NULL); + if (hMapping == NULL) { + CloseHandle(hFile); + std::fprintf(stderr, "%s: %d: Failed to create file mapping for rai file '%s'\n", __func__, __LINE__, path); + return false; + } + + // Map the file + *buffer = (uint8_t *)MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, fileSize.QuadPart); + if (*buffer == NULL) { + CloseHandle(hMapping); + CloseHandle(hFile); + std::fprintf(stderr, "%s: %d: Failed to map rai file '%s'\n", __func__, __LINE__, path); + return false; + } + *size = fileSize.QuadPart; + return true; +#else + // Open the file + FILE * fd = fopen(path, "rb"); + if (!fd) { + std::fprintf(stderr, "%s: %d: Failed to open rai file '%s'\n", __func__, __LINE__, path); + return false; + } + + // Get the file size + struct stat st; + if (fstat(fileno(fd), &st) == -1) { + fclose(fd); + std::fprintf(stderr, "%s: %d: Failed to get file size for rai file '%s'\n", __func__, __LINE__, path); + return false; + } + + // Mmap the file + *buffer = (uint8_t *)mmap(nullptr, st.st_size, PROT_READ, MAP_SHARED, fileno(fd), 0); + if (*buffer == MAP_FAILED) { + fclose(fd); + std::fprintf(stderr, "%s: %d: Failed to mmap rai file '%s'\n", __func__, __LINE__, path); + return false; + } + *size = st.st_size; + return true; +#endif // _WIN32 +} + +void unmap_rai_file(uint8_t * buffer, size_t size) { +#ifdef _WIN32 + UnmapViewOfFile(buffer); +#else + munmap(buffer, size); +#endif // _WIN32 +} + +struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model) { + if (!path_model) { + std::fprintf(stderr, "%s: path_model is null\n", __func__); + return nullptr; + } + + auto * ctx = new whisper_vitisai_context; + ctx->model_path = path_model; + + // Override the model path with the environment variable if it is set + if (const char * env_model_path = std::getenv("OVERRIDE_VITISAI_MODEL_PATH")) { + if (env_model_path[0] != '\0') { + ctx->model_path = env_model_path; + } + } + + // Step 1: Set up the model + flexmlrt::client::Options options; + options.modelPath = ctx->model_path; + options.deviceName = "stx"; + options.debug = false; + options.executeMode = 2; + options.extOptions["ai_analyzer_profiling"] = true; // Enable AIA profiling + options.extOptions["enable_preemption"] = true; + + // Check if model_path is rai file and if so, add fbs_buffer and fbs_buffer_size to the options + if (ctx->model_path.find(".rai") != std::string::npos) { + // mmap rai file for both Linux and Windows and pass the buffer to the options + ctx->fbs_buffer = nullptr; + ctx->fbs_buffer_size = 0; + if (map_rai_file(ctx->model_path.c_str(), &ctx->fbs_buffer, &ctx->fbs_buffer_size)) { + options.extOptions["fbs_buffer"] = ctx->fbs_buffer; + options.extOptions["fbs_buffer_size"] = ctx->fbs_buffer_size; + options.subgraphName = "vaiml_par_0"; + options.extOptions["cache_dir"] = std::string("."); + } else { + std::fprintf(stderr, "%s: Failed to mmap rai file '%s'\n", __func__, ctx->model_path.c_str()); + delete ctx; + return nullptr; + } + } + + try { + ctx->runner = std::make_shared(options); + + if (!ctx->runner->good()) { + throw std::runtime_error("Runner creation ran into an error"); + } + } catch (const std::exception & e) { + std::fprintf(stderr, "%s: Exception during Vitis AI runner creation: %s\n", __func__, e.what()); + delete ctx; + return nullptr; + } + return ctx; +} + +void whisper_vitisai_free(struct whisper_vitisai_context * ctx) { + if (!ctx) { + return; + } + + std::fprintf(stderr, "%s: releasing Vitis AI encoder context for model '%s'\n", __func__, ctx->model_path.c_str()); + if (ctx->fbs_buffer) { + unmap_rai_file(ctx->fbs_buffer, ctx->fbs_buffer_size); + } + delete ctx; +} + +int whisper_vitisai_encode(struct whisper_vitisai_context * ctx, struct ggml_tensor * mel, struct ggml_tensor * out) { + if (!ctx || !mel || !out) { + std::fprintf(stderr, "%s: ctx/mel/out must not be null\n", __func__); + return 0; + } + + if (ggml_n_dims(mel) != 2) { + std::fprintf(stderr, "%s: mel tensor expected to have 2 dims, got %d\n", __func__, ggml_n_dims(mel)); + return 0; + } + + if (ggml_n_dims(out) != 2) { + std::fprintf(stderr, "%s: out tensor expected to have 2 dims, got %d\n", __func__, ggml_n_dims(out)); + return 0; + } + + // setup input and output tensors for Vitis AI model + std::vector input_tensors, output_tensors; + auto model = ctx->runner; + + // Get tensors as CPU tensors (hwTensor = false) + input_tensors = model->getIOTensors("input", false); + output_tensors = model->getIOTensors("output", false); + + // TODO: add assert checks for tensor numbers and shapes + + input_tensors[0].data = mel->data; + output_tensors[0].data = out->data; + + try { + model->forward(input_tensors, output_tensors); + std::fprintf(stdout, "%s: Vitis AI model inference completed.\n", __func__); + } catch (const std::exception & e) { + std::fprintf(stderr, "%s: Exception during model inference: %s\n", __func__, e.what()); + return 0; + } + + return 1; +} diff --git a/src/vitisai/whisper-vitisai-encoder.h b/src/vitisai/whisper-vitisai-encoder.h new file mode 100644 index 000000000..05dc812be --- /dev/null +++ b/src/vitisai/whisper-vitisai-encoder.h @@ -0,0 +1,32 @@ +// Copyright(C) 2025 Advanced Micro Devices, Inc. All rights reserved. + +#pragma once + +#include +#include +#include + +#if __cplusplus +extern "C" { +#endif + +struct whisper_vitisai_context; + +struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model); +void whisper_vitisai_free(struct whisper_vitisai_context * ctx); + +// Function to mmap rai file for Linux and MapViewOfFile for Windows +bool map_rai_file(const char * path, uint8_t ** buffer, size_t * size); +// Function to unmap rai file for Linux and UnmapViewOfFile for Windows +void unmap_rai_file(uint8_t * buffer, size_t size); + +struct ggml_tensor; + +int whisper_vitisai_encode( + struct whisper_vitisai_context * ctx, + struct ggml_tensor * mel, + struct ggml_tensor * out); + +#if __cplusplus +} +#endif diff --git a/src/whisper.cpp b/src/whisper.cpp index 2a95bdb1e..896dd4c21 100644 --- a/src/whisper.cpp +++ b/src/whisper.cpp @@ -14,6 +14,10 @@ #include "openvino/whisper-openvino-encoder.h" #endif +#ifdef WHISPER_USE_VITISAI +#include "vitisai/whisper-vitisai-encoder.h" +#endif + #include #include #include @@ -903,6 +907,10 @@ struct whisper_state { whisper_openvino_context * ctx_openvino = nullptr; #endif +#ifdef WHISPER_USE_VITISAI + whisper_vitisai_context * ctx_vitisai = nullptr; +#endif + // [EXPERIMENTAL] token-level timestamps data int64_t t_beg = 0; int64_t t_last = 0; @@ -1970,7 +1978,13 @@ static bool whisper_encode_external(const whisper_state & wstate) { const bool use_openvino = wstate.ctx_openvino != nullptr; #endif - return use_coreml || use_openvino; +#ifndef WHISPER_USE_VITISAI + const bool use_vitisai = false; +#else + const bool use_vitisai = wstate.ctx_vitisai != nullptr; +#endif + + return use_coreml || use_openvino || use_vitisai; } static struct ggml_cgraph * whisper_build_graph_conv( @@ -2411,6 +2425,8 @@ static bool whisper_encode_internal( #if defined(WHISPER_USE_COREML) whisper_coreml_encode(wstate.ctx_coreml, mel->ne[0], mel->ne[1], (float *) mel->data, (float *) wstate.embd_enc->data); +#elif defined(WHISPER_USE_VITISAI) + whisper_vitisai_encode(wstate.ctx_vitisai, mel, wstate.embd_enc); #elif defined(WHISPER_USE_OPENVINO) whisper_openvino_encode(wstate.ctx_openvino, mel, wstate.embd_enc); #endif @@ -3346,6 +3362,20 @@ static std::string whisper_get_coreml_path_encoder(std::string path_bin) { } #endif +#ifdef WHISPER_USE_VITISAI +// replace extension with Vitis AI encoder artifact +static std::string whisper_get_vitisai_path_encoder_cache(std::string path_bin) { + auto pos = path_bin.rfind('.'); + if (pos != std::string::npos) { + path_bin = path_bin.substr(0, pos); + } + + path_bin += "-encoder-vitisai.rai"; + + return path_bin; +} +#endif + #ifdef WHISPER_USE_OPENVINO // replace .bin with-encoder-openvino.xml static std::string whisper_openvino_get_path_encoder(std::string path_bin) { @@ -3455,6 +3485,19 @@ struct whisper_state * whisper_init_state(whisper_context * ctx) { } #endif +#ifdef WHISPER_USE_VITISAI + const auto path_vitisai = whisper_get_vitisai_path_encoder_cache(ctx->path_model); + + state->ctx_vitisai = whisper_vitisai_init(path_vitisai.c_str()); + if (!state->ctx_vitisai) { + WHISPER_LOG_ERROR("%s: failed to load Vitis AI model from '%s'\n", __func__, path_vitisai.c_str()); + whisper_free_state(state); + return nullptr; + } else { + WHISPER_LOG_INFO("%s: Vitis AI model loaded\n", __func__); + } +#endif + state->logits.reserve(ctx->vocab.n_vocab * ctx->model.hparams.n_text_ctx); state->batch = whisper_batch_init(ctx->model.hparams.n_text_ctx, WHISPER_MAX_DECODERS); @@ -3835,6 +3878,13 @@ void whisper_free_state(struct whisper_state * state) { } #endif +#ifdef WHISPER_USE_VITISAI + if (state->ctx_vitisai != nullptr) { + whisper_vitisai_free(state->ctx_vitisai); + state->ctx_vitisai = nullptr; + } +#endif + whisper_batch_free(state->batch); ggml_backend_sched_free(state->sched_conv.sched); @@ -4326,11 +4376,20 @@ static int whisper_has_openvino(void) { #endif } +static int whisper_has_vitisai(void) { +#ifdef WHISPER_USE_VITISAI + return 1; +#else + return 0; +#endif +} + const char * whisper_print_system_info(void) { static std::string s; s = ""; s += "WHISPER : "; + s += "VITISAI = " + std::to_string(whisper_has_vitisai()) + " | "; s += "COREML = " + std::to_string(whisper_has_coreml()) + " | "; s += "OPENVINO = " + std::to_string(whisper_has_openvino()) + " | "; From def9eea2bfb117d065feaca2f8285f076bb1eb85 Mon Sep 17 00:00:00 2001 From: Sachin Kumawat Date: Thu, 26 Feb 2026 12:42:08 -0800 Subject: [PATCH 06/24] Cleanup and add runtime print debug guard --- src/vitisai/whisper-vitisai-encoder.cpp | 9 +++++---- src/vitisai/whisper-vitisai-encoder.h | 7 ------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/src/vitisai/whisper-vitisai-encoder.cpp b/src/vitisai/whisper-vitisai-encoder.cpp index a6d20a88c..c10e1c37a 100644 --- a/src/vitisai/whisper-vitisai-encoder.cpp +++ b/src/vitisai/whisper-vitisai-encoder.cpp @@ -1,4 +1,3 @@ -// Copyright(C) 2025 Advanced Micro Devices, Inc. All rights reserved. #include "vitisai/whisper-vitisai-encoder.h" #include "FlexMLClient.h" #include "ggml.h" @@ -24,7 +23,7 @@ struct whisper_vitisai_context { }; // Function to mmap rai file for Linux and MapViewOfFile for Windows -bool map_rai_file(const char * path, uint8_t ** buffer, size_t * size) { +static bool map_rai_file(const char * path, uint8_t ** buffer, size_t * size) { #ifdef _WIN32 // Open the file HANDLE hFile = CreateFileA(path, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); @@ -87,7 +86,7 @@ bool map_rai_file(const char * path, uint8_t ** buffer, size_t * size) { #endif // _WIN32 } -void unmap_rai_file(uint8_t * buffer, size_t size) { +static void unmap_rai_file(uint8_t * buffer, size_t size) { #ifdef _WIN32 UnmapViewOfFile(buffer); #else @@ -194,7 +193,9 @@ int whisper_vitisai_encode(struct whisper_vitisai_context * ctx, struct ggml_ten try { model->forward(input_tensors, output_tensors); - std::fprintf(stdout, "%s: Vitis AI model inference completed.\n", __func__); +#if defined(WHISPER_DEBUG) + std::fprintf(stderr, "%s: Vitis AI model inference completed.\n", __func__); +#endif } catch (const std::exception & e) { std::fprintf(stderr, "%s: Exception during model inference: %s\n", __func__, e.what()); return 0; diff --git a/src/vitisai/whisper-vitisai-encoder.h b/src/vitisai/whisper-vitisai-encoder.h index 05dc812be..840ce6941 100644 --- a/src/vitisai/whisper-vitisai-encoder.h +++ b/src/vitisai/whisper-vitisai-encoder.h @@ -1,5 +1,3 @@ -// Copyright(C) 2025 Advanced Micro Devices, Inc. All rights reserved. - #pragma once #include @@ -15,11 +13,6 @@ struct whisper_vitisai_context; struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model); void whisper_vitisai_free(struct whisper_vitisai_context * ctx); -// Function to mmap rai file for Linux and MapViewOfFile for Windows -bool map_rai_file(const char * path, uint8_t ** buffer, size_t * size); -// Function to unmap rai file for Linux and UnmapViewOfFile for Windows -void unmap_rai_file(uint8_t * buffer, size_t size); - struct ggml_tensor; int whisper_vitisai_encode( From 83bd0c012c9f806b5f2a7287e80fec1e66cc4bd5 Mon Sep 17 00:00:00 2001 From: Iswarya Alex <47045679+iswaryaalex@users.noreply.github.com> Date: Tue, 27 Jan 2026 11:51:47 -0800 Subject: [PATCH 07/24] Update README.md - RAI EULA Links - Updated for RAI Whisper instructions --- README.md | 56 ++++++++++++++++++++++++++++++++++--------------------- 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index e96205845..7882f59dd 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ High-performance inference of [OpenAI's Whisper](https://github.com/openai/whisp - Support for CPU-only inference - [Efficient GPU support for NVIDIA](#nvidia-gpu-support) - [AMD ROCm GPU support](#amd-rocm-gpu-support) +- [AMD Ryzen AI NPU Support](#amd-ryzen-ai-support-for-npu) - [OpenVINO Support](#openvino-support) - [Ascend NPU Support](#ascend-npu-support) - [Moore Threads GPU Support](#moore-threads-gpu-support) @@ -313,34 +314,47 @@ This can result in significant speedup in encoder performance. Here are the inst For more information about the OpenVINO implementation please refer to PR [#1037](https://github.com/ggml-org/whisper.cpp/pull/1037). -## VitisAI encoder support +## AMD Ryzen™ AI support for NPU -On AMD Ryzen AI NPU devices, you can run the Encoder via the VitisAI plugin to significantly accelerate the whisper models. +On AMD's Ryzen™ AI 300 Series with dedicated NPUs for acceleration, you can now run Whisper models with the ability to fully offload the encoder to NPU. This brings significant speedup compared to CPU-only. +> **Note:** +> **Ryzen™ AI NPU acceleration is currently supported on Windows only.** Linux support is planned for upcoming releases. +> For the latest updates on Ryzen AI, check out [the official documentation](https://ryzenai.docs.amd.com/en/latest/). -- Prepare the AMD runtime packages (required before building): +### Setup environment (Windows only) - - Obtain the XRT package and the FlexmlRT package from AMD. Both are distributed as tarballs or wheels. - - Copy the downloaded archives to a local path, extract them, and run the setup script from each extracted package in your shell (for example `source /path/to/xrt/setup.sh` and `source /path/to/flexmlrt/setup.sh`). Run these in every new shell you use to build or run `whisper.cpp`. - -- Fetch the prebuilt VitisAI encoder cache: - - - Download the appropriate Whisper encoder `.rai` cache for your model size from the AMD collection on Hugging Face: https://huggingface.co/collections/amd/ryzen-ai-16-whisper-npu-optimized-onnx-models - - Place and rename the downloaded `.rai` file as `-encoder-vitisai.rai` alongside your ggml model files `.bin`. - -- Build `whisper.cpp` with VitisAI support: - - ```bash - cmake -B build -DWHISPER_VITISAI=1 - cmake --build build -j --config Release +- **Driver:** Make sure you have NPU drivers version **.280 or newer** installed. [Download latest drivers from here](https://account.amd.com/en/forms/downloads/ryzenai-eula-public-xef.html?filename=NPU_RAI1.5_280_WHQL.zip) +- **Runtime libraries:** Download and install the necessary [runtime dependencies from here](https://account.amd.com/en/forms/downloads/ryzenai-eula-public-xef.html?filename=flexmlrt1.7.0-win.zip). +- **Environment:** Extract the runtime package and set up the environment: + ```powershell + tar xvf flexmlrt1.7.0-win.zip + flexmlrt\setup.bat ``` +Your environment is now ready. -- Run the examples as usual. For example: +### Build Whisper.cpp for Ryzen™ AI support - ```text - $ ./build/bin/whisper-cli -m models/ggml-base.en.bin -f samples/jfk.wav - ``` +```bash +cmake -B build -DWHISPER_VITISAI=1 +cmake --build build -j --config Release +``` + +### Download NPU-optimized models + +- All NPU-supported Whisper models and their compiled `.rai` cache files are available in this collection: + https://huggingface.co/collections/amd/ryzen-ai-16-whisper-npu-optimized-onnx-models +- Download the pre-compiled `.rai` cache file matching your desired model, and place it in your `models/` directory alongside its corresponding `ggml-<...>.bin` file. + The cache file must be named with the `-encoder-vitisai.rai` suffix. For example, if your model file is named `ggml-small.bin`, the cache file should be named `ggml-small-encoder-vitisai.rai`. + + +> **Note:** The ".rai" models from Hugging Face are pre-optimized for Ryzen™ AI NPUs, delivering acceleration benefits from the very first run (aside from any initial CPU-side caching overhead). + +Run the examples as usual: + +```bash +./build/bin/whisper-cli -m models/ggml-small.bin -f samples/jfk.wav +``` -The VitisAI artifact from Huggingface is already optimized for Ryzen AI NPUs, there is no slow compilation needed. The acceleration advantage should be seen from first run itself apart from CPU caching overheads. ## NVIDIA GPU support From 13735c1bc5b6bcca1e6f005e5a31846423810829 Mon Sep 17 00:00:00 2001 From: Sachin Kumawat Date: Tue, 3 Mar 2026 16:37:29 -0800 Subject: [PATCH 08/24] turn off profiling --- src/vitisai/whisper-vitisai-encoder.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/vitisai/whisper-vitisai-encoder.cpp b/src/vitisai/whisper-vitisai-encoder.cpp index c10e1c37a..580bcfe36 100644 --- a/src/vitisai/whisper-vitisai-encoder.cpp +++ b/src/vitisai/whisper-vitisai-encoder.cpp @@ -116,7 +116,6 @@ struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model) { options.deviceName = "stx"; options.debug = false; options.executeMode = 2; - options.extOptions["ai_analyzer_profiling"] = true; // Enable AIA profiling options.extOptions["enable_preemption"] = true; // Check if model_path is rai file and if so, add fbs_buffer and fbs_buffer_size to the options From c9f63ad1f3423dea735fbe52268b5493325504f8 Mon Sep 17 00:00:00 2001 From: Sachin Kumawat Date: Tue, 23 Jun 2026 15:46:37 -0700 Subject: [PATCH 09/24] Let flexmlrt detect device type --- src/vitisai/whisper-vitisai-encoder.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/vitisai/whisper-vitisai-encoder.cpp b/src/vitisai/whisper-vitisai-encoder.cpp index 580bcfe36..ef151dff5 100644 --- a/src/vitisai/whisper-vitisai-encoder.cpp +++ b/src/vitisai/whisper-vitisai-encoder.cpp @@ -102,6 +102,8 @@ struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model) { auto * ctx = new whisper_vitisai_context; ctx->model_path = path_model; + ctx->fbs_buffer = nullptr; + ctx->fbs_buffer_size = 0; // Override the model path with the environment variable if it is set if (const char * env_model_path = std::getenv("OVERRIDE_VITISAI_MODEL_PATH")) { @@ -113,7 +115,6 @@ struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model) { // Step 1: Set up the model flexmlrt::client::Options options; options.modelPath = ctx->model_path; - options.deviceName = "stx"; options.debug = false; options.executeMode = 2; options.extOptions["enable_preemption"] = true; @@ -121,18 +122,20 @@ struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model) { // Check if model_path is rai file and if so, add fbs_buffer and fbs_buffer_size to the options if (ctx->model_path.find(".rai") != std::string::npos) { // mmap rai file for both Linux and Windows and pass the buffer to the options - ctx->fbs_buffer = nullptr; - ctx->fbs_buffer_size = 0; if (map_rai_file(ctx->model_path.c_str(), &ctx->fbs_buffer, &ctx->fbs_buffer_size)) { options.extOptions["fbs_buffer"] = ctx->fbs_buffer; options.extOptions["fbs_buffer_size"] = ctx->fbs_buffer_size; - options.subgraphName = "vaiml_par_0"; options.extOptions["cache_dir"] = std::string("."); } else { std::fprintf(stderr, "%s: Failed to mmap rai file '%s'\n", __func__, ctx->model_path.c_str()); delete ctx; return nullptr; } + } else { + options.deviceName = "stx"; +#if defined(WHISPER_DEBUG) + std::fprintf(stderr, "%s: Using default device name 'stx'\n", __func__); +#endif } try { From a4660b7cae18276436a7db82721699bf0628787e Mon Sep 17 00:00:00 2001 From: Sachin Kumawat Date: Thu, 23 Jul 2026 17:51:11 -0700 Subject: [PATCH 10/24] Add VitisAI model download scripts --- README.md | 51 ++++--- models/download-vitisai-model.cmd | 32 +++++ models/download-vitisai-model.ps1 | 218 ++++++++++++++++++++++++++++ models/download-vitisai-model.sh | 226 ++++++++++++++++++++++++++++++ 4 files changed, 501 insertions(+), 26 deletions(-) create mode 100644 models/download-vitisai-model.cmd create mode 100644 models/download-vitisai-model.ps1 create mode 100755 models/download-vitisai-model.sh diff --git a/README.md b/README.md index 7882f59dd..91641ffb3 100644 --- a/README.md +++ b/README.md @@ -323,37 +323,36 @@ On AMD's Ryzen™ AI 300 Series with dedicated NPUs for acceleration, you can no ### Setup environment (Windows only) -- **Driver:** Make sure you have NPU drivers version **.280 or newer** installed. [Download latest drivers from here](https://account.amd.com/en/forms/downloads/ryzenai-eula-public-xef.html?filename=NPU_RAI1.5_280_WHQL.zip) -- **Runtime libraries:** Download and install the necessary [runtime dependencies from here](https://account.amd.com/en/forms/downloads/ryzenai-eula-public-xef.html?filename=flexmlrt1.7.0-win.zip). -- **Environment:** Extract the runtime package and set up the environment: - ```powershell - tar xvf flexmlrt1.7.0-win.zip - flexmlrt\setup.bat + - Obtain the XRT package and the FlexmlRT package from AMD. Both are distributed as tarballs or wheels. + - Copy the downloaded archives to a local path, extract them, and run the setup script from each extracted package in your shell (for example `source /path/to/xrt/setup.sh` and `source /path/to/flexmlrt/setup.sh`). Run these in every new shell you use to build or run `whisper.cpp`. + +- Fetch the matching ggml model and prebuilt VitisAI encoder cache: + + ```bash + sh ./models/download-ggml-model.sh base + sh ./models/download-vitisai-model.sh base + ``` + + ```cmd + .\models\download-ggml-model.cmd base + .\models\download-vitisai-model.cmd base + ``` + + Use the same model name with both scripts. The VitisAI script queries the AMD collection on Hugging Face to list available caches, then downloads the selected `.rai` file as `ggml--encoder-vitisai.rai` alongside the matching `ggml-.bin` file. You can also browse the collection manually at https://huggingface.co/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models. + +- Build `whisper.cpp` with VitisAI support: + + ```bash + cmake -B build -DWHISPER_VITISAI=1 + cmake --build build -j --config Release ``` Your environment is now ready. ### Build Whisper.cpp for Ryzen™ AI support -```bash -cmake -B build -DWHISPER_VITISAI=1 -cmake --build build -j --config Release -``` - -### Download NPU-optimized models - -- All NPU-supported Whisper models and their compiled `.rai` cache files are available in this collection: - https://huggingface.co/collections/amd/ryzen-ai-16-whisper-npu-optimized-onnx-models -- Download the pre-compiled `.rai` cache file matching your desired model, and place it in your `models/` directory alongside its corresponding `ggml-<...>.bin` file. - The cache file must be named with the `-encoder-vitisai.rai` suffix. For example, if your model file is named `ggml-small.bin`, the cache file should be named `ggml-small-encoder-vitisai.rai`. - - -> **Note:** The ".rai" models from Hugging Face are pre-optimized for Ryzen™ AI NPUs, delivering acceleration benefits from the very first run (aside from any initial CPU-side caching overhead). - -Run the examples as usual: - -```bash -./build/bin/whisper-cli -m models/ggml-small.bin -f samples/jfk.wav -``` + ```text + $ ./build/bin/whisper-cli -m models/ggml-base.bin -f samples/jfk.wav + ``` ## NVIDIA GPU support diff --git a/models/download-vitisai-model.cmd b/models/download-vitisai-model.cmd new file mode 100644 index 000000000..0b768cd1a --- /dev/null +++ b/models/download-vitisai-model.cmd @@ -0,0 +1,32 @@ +@echo off +setlocal + +set "script=%~dp0download-vitisai-model.ps1" + +if "%~1"=="" ( + PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" + exit /b %ERRORLEVEL% +) + +if /I "%~1"=="--list" ( + PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" -List + exit /b %ERRORLEVEL% +) + +if /I "%~1"=="-l" ( + PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" -List + exit /b %ERRORLEVEL% +) + +if /I "%~1"=="list" ( + PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" -List + exit /b %ERRORLEVEL% +) + +if "%~2"=="" ( + PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" -Model "%~1" +) else ( + PowerShell -NoProfile -ExecutionPolicy Bypass -File "%script%" -Model "%~1" -ModelsPath "%~2" +) + +exit /b %ERRORLEVEL% diff --git a/models/download-vitisai-model.ps1 b/models/download-vitisai-model.ps1 new file mode 100644 index 000000000..5408248cf --- /dev/null +++ b/models/download-vitisai-model.ps1 @@ -0,0 +1,218 @@ +param( + [Parameter(Position = 0)] + [string] $Model, + + [Parameter(Position = 1)] + [string] $ModelsPath, + + [switch] $List +) + +$ErrorActionPreference = "Stop" + +$Source = "https://huggingface.co" +$CollectionApi = "https://huggingface.co/api/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models" +$CollectionUrl = "https://huggingface.co/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models" + +$ScriptDir = Split-Path -Parent $MyInvocation.MyCommand.Path +if ($ScriptDir -match "\\bin$") { + $DefaultDownloadPath = (Get-Location).Path +} else { + $DefaultDownloadPath = $ScriptDir +} + +if (-not $ModelsPath) { + $ModelsPath = $DefaultDownloadPath +} + +function Get-HfHeaders { + $headers = @{} + if ($env:HF_TOKEN) { + $headers["Authorization"] = "Bearer $env:HF_TOKEN" + } + return $headers +} + +function Invoke-HfJson { + param([string] $Uri) + + $headers = Get-HfHeaders + if ($headers.Count -gt 0) { + return Invoke-RestMethod -Uri $Uri -Headers $headers + } + + return Invoke-RestMethod -Uri $Uri +} + +function Normalize-ModelName { + param([string] $Name) + + # HF currently publishes ggml-small-en-encoder-vitisai.rai, while the + # matching ggml model is ggml-small.en.bin. + if ($Name.EndsWith("-en")) { + return $Name.Substring(0, $Name.Length - 3) + ".en" + } + + return $Name +} + +function Get-VitisAiModels { + $collection = Invoke-HfJson -Uri $CollectionApi + $seen = @{} + $rows = New-Object System.Collections.Generic.List[object] + + foreach ($item in $collection.items) { + if ($item.type -ne "model") { + continue + } + + $repo = $item.id + if (-not $repo) { + continue + } + + $modelInfo = Invoke-HfJson -Uri "$Source/api/models/$repo" + foreach ($sibling in $modelInfo.siblings) { + $filename = [string] $sibling.rfilename + $match = [regex]::Match($filename, "^ggml-(.+)-encoder-vitisai\.rai$") + if (-not $match.Success) { + continue + } + + $rawName = $match.Groups[1].Value + $modelName = Normalize-ModelName -Name $rawName + if ($seen.ContainsKey($modelName)) { + continue + } + + $seen[$modelName] = $true + $destination = "ggml-$modelName-encoder-vitisai.rai" + $url = "$Source/$repo/resolve/main/$([uri]::EscapeDataString($filename))" + + $rows.Add([pscustomobject]@{ + Model = $modelName + RawName = $rawName + Repo = $repo + SourceFile = $filename + DestinationFile = $destination + DownloadUrl = $url + }) + } + } + + $order = @{ + "tiny" = 10 + "tiny.en" = 11 + "base" = 20 + "base.en" = 21 + "small" = 30 + "small.en" = 31 + "medium" = 40 + "medium.en" = 41 + "large-v1" = 50 + "large-v2" = 60 + "large-v3" = 70 + "large-v3-turbo" = 80 + } + + return $rows | Sort-Object ` + @{ Expression = { if ($order.ContainsKey($_.Model)) { $order[$_.Model] } else { 1000 } } }, ` + @{ Expression = { $_.Model } } +} + +function Show-Models { + $models = Get-VitisAiModels + + Write-Host "" + Write-Host "Available VitisAI encoder caches from ${CollectionUrl}:" + foreach ($entry in $models) { + if ($entry.Model -eq $entry.RawName) { + Write-Host (" {0,-18} {1}" -f $entry.Model, $entry.Repo) + } else { + Write-Host (" {0,-18} {1} (source name: {2})" -f $entry.Model, $entry.Repo, $entry.RawName) + } + } + Write-Host "" +} + +function Show-Usage { + Write-Host "Usage: download-vitisai-model.cmd --list" + Write-Host " download-vitisai-model.cmd [models_path]" + Write-Host "" + Write-Host "Downloads ggml--encoder-vitisai.rai next to ggml-.bin." + Write-Host "Use the same model name as download-ggml-model.cmd." + Write-Host "" +} + +if ($List -or $Model -eq "--list" -or $Model -eq "-l" -or $Model -eq "list") { + Show-Models + exit 0 +} + +if (-not $Model) { + Show-Usage + Show-Models + exit 1 +} + +$models = Get-VitisAiModels +$entry = $models | Where-Object { $_.Model -eq $Model -or $_.RawName -eq $Model } | Select-Object -First 1 +if (-not $entry) { + Write-Host "Invalid model: $Model" + foreach ($available in $models) { + Write-Host " $($available.Model)" + } + exit 1 +} + +New-Item -ItemType Directory -Force -Path $ModelsPath | Out-Null +$destinationPath = Join-Path $ModelsPath $entry.DestinationFile + +Write-Host "Downloading VitisAI encoder cache $($entry.Model) from '$($entry.Repo)' ..." +if (Test-Path $destinationPath) { + Write-Host "VitisAI encoder cache $($entry.DestinationFile) already exists. Skipping download." + exit 0 +} + +$headers = Get-HfHeaders +$downloaded = $false +for ($attempt = 1; $attempt -le 5; ++$attempt) { + try { + if ($headers.Count -gt 0) { + Invoke-WebRequest -Uri $entry.DownloadUrl -Headers $headers -OutFile $destinationPath + } else { + Invoke-WebRequest -Uri $entry.DownloadUrl -OutFile $destinationPath + } + $downloaded = $true + break + } catch { + if ($attempt -eq 5) { + if (Test-Path $destinationPath) { + Remove-Item -Force $destinationPath + } + Write-Host "Failed to download VitisAI encoder cache $($entry.Model) from $($entry.DownloadUrl)" + throw + } + Start-Sleep -Seconds 5 + } +} + +if (-not $downloaded) { + exit 1 +} + +$whisperCmd = "whisper-cli" +if (-not (Get-Command whisper-cli -ErrorAction SilentlyContinue)) { + $rootPath = Split-Path -Parent $ScriptDir + $whisperCmd = Join-Path $rootPath "build\bin\Release\whisper-cli.exe" +} + +Write-Host "Done! VitisAI encoder cache '$($entry.Model)' saved in '$destinationPath'" +if ($entry.RawName -ne $entry.Model) { + Write-Host "Source cache '$($entry.SourceFile)' was renamed to match ggml model name '$($entry.Model)'." +} +Write-Host "Use it with the matching ggml model:" +Write-Host "" +Write-Host " $ScriptDir\download-ggml-model.cmd $($entry.Model) $ModelsPath" +Write-Host " $whisperCmd -m $ModelsPath\ggml-$($entry.Model).bin -f samples\jfk.wav" +Write-Host "" diff --git a/models/download-vitisai-model.sh b/models/download-vitisai-model.sh new file mode 100755 index 000000000..ddae36d22 --- /dev/null +++ b/models/download-vitisai-model.sh @@ -0,0 +1,226 @@ +#!/bin/sh + +# This script downloads prebuilt VitisAI encoder cache files for Whisper models. +# The cache file is saved next to the ggml model file and follows the loader +# convention: ggml--encoder-vitisai.rai + +src="https://huggingface.co" +collection_api="https://huggingface.co/api/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models" +collection_url="https://huggingface.co/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models" + +BOLD="\033[1m" +RESET='\033[0m' + +# get the path of this script +get_script_path() { + if [ -x "$(command -v realpath)" ]; then + dirname "$(realpath "$0")" + else + _ret="$(cd -- "$(dirname "$0")" >/dev/null 2>&1 || exit ; pwd -P)" + echo "$_ret" + fi +} + +find_python() { + if command -v python3 >/dev/null 2>&1; then + printf "%s\n" "python3" + elif command -v python >/dev/null 2>&1; then + printf "%s\n" "python" + else + return 1 + fi +} + +script_path="$(get_script_path)" + +# Check if the script is inside a /bin/ directory +case "$script_path" in + */bin) default_download_path="$PWD" ;; # Use current directory as default download path if in /bin/ + *) default_download_path="$script_path" ;; # Otherwise, use script directory +esac + +models_path="${2:-$default_download_path}" + +discover_models() { + python_cmd="$(find_python)" || { + printf "Python is required to query available VitisAI caches from Hugging Face.\n" >&2 + return 1 + } + + "$python_cmd" - "$src" "$collection_api" <<'PY' +import json +import os +import re +import sys +import urllib.parse +import urllib.request + +src = sys.argv[1].rstrip("/") +collection_api = sys.argv[2] +headers = {} +token = os.environ.get("HF_TOKEN") +if token: + headers["Authorization"] = "Bearer " + token + +def load_json(url): + req = urllib.request.Request(url, headers=headers) + with urllib.request.urlopen(req) as response: + return json.load(response) + +def normalize_model_name(name): + # HF currently publishes ggml-small-en-encoder-vitisai.rai, while the + # matching ggml model is ggml-small.en.bin. + if name.endswith("-en"): + return name[:-3] + ".en" + return name + +collection = load_json(collection_api) +rows = [] +seen = set() + +for item in collection.get("items", []): + if item.get("type") != "model": + continue + + repo = item.get("id") + if not repo: + continue + + model_info = load_json(src + "/api/models/" + repo) + for sibling in model_info.get("siblings", []): + filename = sibling.get("rfilename", "") + match = re.match(r"^ggml-(.+)-encoder-vitisai\.rai$", filename) + if not match: + continue + + raw_name = match.group(1) + model_name = normalize_model_name(raw_name) + if model_name in seen: + continue + seen.add(model_name) + + destination = "ggml-%s-encoder-vitisai.rai" % model_name + url = "%s/%s/resolve/main/%s" % (src, repo, urllib.parse.quote(filename)) + rows.append((model_name, raw_name, repo, filename, destination, url)) + +order = { + "tiny": 10, + "tiny.en": 11, + "base": 20, + "base.en": 21, + "small": 30, + "small.en": 31, + "medium": 40, + "medium.en": 41, + "large-v1": 50, + "large-v2": 60, + "large-v3": 70, + "large-v3-turbo": 80, +} + +for row in sorted(rows, key=lambda item: (order.get(item[0], 1000), item[0])): + print("|".join(row)) +PY +} + +list_models() { + models="$(discover_models)" || exit 1 + + printf "\n" + printf "Available VitisAI encoder caches from %s:\n" "$collection_url" + printf "%s\n" "$models" | while IFS='|' read -r model raw repo _source _destination _url; do + if [ "$model" = "$raw" ]; then + printf " %-18s %s\n" "$model" "$repo" + else + printf " %-18s %s (source name: %s)\n" "$model" "$repo" "$raw" + fi + done + printf "\n" +} + +usage() { + printf "Usage: %s --list\n" "$0" + printf " %s [models_path]\n" "$0" + printf "\n" + printf "Downloads ggml--encoder-vitisai.rai next to ggml-.bin.\n" + printf "Use the same model name as %s/download-ggml-model.sh.\n" "$script_path" + printf "\n" +} + +if [ "$#" -eq 1 ] && { [ "$1" = "--list" ] || [ "$1" = "-l" ] || [ "$1" = "list" ]; }; then + list_models + exit 0 +fi + +if [ "$#" -lt 1 ] || [ "$#" -gt 2 ]; then + usage + list_models + printf "___________________________________________________________\n" + printf "Example: %s ${BOLD}small${RESET} %s\n" "$0" "$default_download_path" + exit 1 +fi + +model=$1 +models="$(discover_models)" || exit 1 + +match="$(printf "%s\n" "$models" | awk -F '|' -v model="$model" '$1 == model || $2 == model { print; exit }')" +if [ -z "$match" ]; then + printf "Invalid model: %s\n" "$model" + printf "%s\n" "$models" | while IFS='|' read -r available _raw _repo _source _destination _url; do + printf " %s\n" "$available" + done + exit 1 +fi + +IFS='|' read -r model raw_name repo source_file destination_file download_url </dev/null 2>&1; then + whisper_cmd="whisper-cli" +else + whisper_cmd="./build/bin/whisper-cli" +fi + +printf "Done! VitisAI encoder cache '%s' saved in '%s/%s'\n" "$model" "$models_path" "$destination_file" +if [ "$raw_name" != "$model" ]; then + printf "Source cache '%s' was renamed to match ggml model name '%s'.\n" "$source_file" "$model" +fi +printf "Use it with the matching ggml model:\n\n" +printf " $ %s/download-ggml-model.sh %s %s\n" "$script_path" "$model" "$models_path" +printf " $ %s -m %s/ggml-%s.bin -f samples/jfk.wav\n" "$whisper_cmd" "$models_path" "$model" +printf "\n" From 951b1f0eadde0fe05537dfbfb216c280647542c6 Mon Sep 17 00:00:00 2001 From: Sachin Kumawat Date: Tue, 28 Jul 2026 23:43:12 -0700 Subject: [PATCH 11/24] Add encoder + cross projection layer offload --- README.md | 2 + src/CMakeLists.txt | 55 +++ src/vitisai/whisper-vitisai-encoder.cpp | 416 +++++++++++++++++++- src/vitisai/whisper-vitisai-encoder.h | 23 +- src/vitisai/whisper-vitisai-helpers.cpp | 481 ++++++++++++++++++++++++ src/vitisai/whisper-vitisai-helpers.h | 118 ++++++ src/whisper.cpp | 43 ++- 7 files changed, 1120 insertions(+), 18 deletions(-) create mode 100644 src/vitisai/whisper-vitisai-helpers.cpp create mode 100644 src/vitisai/whisper-vitisai-helpers.h diff --git a/README.md b/README.md index 91641ffb3..fd23e1c55 100644 --- a/README.md +++ b/README.md @@ -340,6 +340,8 @@ On AMD's Ryzen™ AI 300 Series with dedicated NPUs for acceleration, you can no Use the same model name with both scripts. The VitisAI script queries the AMD collection on Hugging Face to list available caches, then downloads the selected `.rai` file as `ggml--encoder-vitisai.rai` alongside the matching `ggml-.bin` file. You can also browse the collection manually at https://huggingface.co/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models. + Depending on the downloaded `.rai` cache, VitisAI may offload either the encoder only or the encoder plus cross-projection layers to the AMD NPU. `whisper.cpp` detects the cache contents at runtime and logs the selected offload mode during model initialization. + - Build `whisper.cpp` with VitisAI support: ```bash diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index de4431d9b..4dc39dbba 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -50,6 +50,57 @@ endif() if (WHISPER_VITISAI) find_package(FlexmlRT REQUIRED) + + # Legacy RAI overrides are required by FlexMLRT older than 1.8.0 + set(WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES_MODE "AUTO" CACHE STRING + "Legacy RAI override mode for FlexMLRT (AUTO|ON|OFF)") + set_property(CACHE WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES_MODE PROPERTY STRINGS AUTO ON OFF) + + string(TOUPPER "${WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES_MODE}" _flexmlrt_legacy_mode) + set(_flexmlrt_legacy_hint "Set -DWHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES_MODE=ON or OFF explicitly.") + + if (NOT _flexmlrt_legacy_mode MATCHES "^(AUTO|ON|OFF)$") + message(FATAL_ERROR + "Invalid WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES_MODE='${WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES_MODE}'. " + "Expected AUTO, ON, or OFF.") + endif() + + if (_flexmlrt_legacy_mode STREQUAL "AUTO") + if (NOT FlexmlRT_DIR) + message(FATAL_ERROR + "FlexmlRT_DIR is unset after find_package(FlexmlRT). ${_flexmlrt_legacy_hint}") + endif() + + # FlexmlRT_DIR points to /share/cmake/FlexmlRT. + get_filename_component(_flexmlrt_init_py "${FlexmlRT_DIR}/../../../__init__.py" ABSOLUTE) + if (NOT EXISTS "${_flexmlrt_init_py}") + message(FATAL_ERROR + "flexmlrt __init__.py not found at ${_flexmlrt_init_py}. ${_flexmlrt_legacy_hint}") + endif() + set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${_flexmlrt_init_py}") + + file(STRINGS "${_flexmlrt_init_py}" _flexmlrt_version_lines + REGEX "^VERSION[ \t]*=[ \t]*\"[0-9]+\\.[0-9]+\\.[0-9]+") + if (NOT _flexmlrt_version_lines MATCHES "\"([0-9]+\\.[0-9]+\\.[0-9]+)") + message(FATAL_ERROR + "Could not parse flexmlrt VERSION from ${_flexmlrt_init_py}. ${_flexmlrt_legacy_hint}") + endif() + set(_flexmlrt_version "${CMAKE_MATCH_1}") + + if (_flexmlrt_version VERSION_LESS "1.8.0") + set(WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES 1) + else() + set(WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES 0) + endif() + message(STATUS "Detected flexmlrt VERSION=${_flexmlrt_version} from ${_flexmlrt_init_py} (legacy overrides=${WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES})") + else() + if (_flexmlrt_legacy_mode STREQUAL "ON") + set(WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES 1) + else() + set(WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES 0) + endif() + message(STATUS "FlexMLRT legacy RAI overrides forced ${_flexmlrt_legacy_mode}") + endif() endif() # @@ -125,6 +176,10 @@ if (WHISPER_VITISAI) target_compile_options(${TARGET} PRIVATE /std:c++17) endif() + target_compile_definitions(${TARGET} PRIVATE + WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES=${WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES} + ) + target_link_libraries(${TARGET} PRIVATE ggml flexmlrt::flexmlrt) set_target_properties(${TARGET} PROPERTIES FOLDER "libs") endif() diff --git a/src/vitisai/whisper-vitisai-encoder.cpp b/src/vitisai/whisper-vitisai-encoder.cpp index ef151dff5..9586c24a9 100644 --- a/src/vitisai/whisper-vitisai-encoder.cpp +++ b/src/vitisai/whisper-vitisai-encoder.cpp @@ -1,3 +1,9 @@ +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#endif + #include "vitisai/whisper-vitisai-encoder.h" #include "FlexMLClient.h" #include "ggml.h" @@ -12,14 +18,45 @@ #include #include #endif +#include +#include #include +#include #include +#include + +#if defined(WHISPER_DEBUG) +#define WHISPER_DBG_TIMER(name) const int64_t name = ggml_time_us() +#else +#define WHISPER_DBG_TIMER(name) do {} while (0) +#endif + +#if defined(WHISPER_DEBUG) +template +static void whisper_vitisai_print_shape(const std::vector & shape) { + std::fprintf(stderr, "["); + for (size_t i = 0; i < shape.size(); ++i) { + std::fprintf(stderr, "%s%lld", i == 0 ? "" : ", ", (long long) shape[i]); + } + std::fprintf(stderr, "]"); +} +#endif struct whisper_vitisai_context { std::string model_path; std::shared_ptr runner; - uint8_t * fbs_buffer; - size_t fbs_buffer_size; + uint8_t * fbs_buffer = nullptr; + size_t fbs_buffer_size = 0; + + std::vector cross_k_staging; + std::vector cross_v_staging; + + int embd_enc_out_idx = -1; + int cross_k_out_idx = -1; + int cross_v_out_idx = -1; + + std::vector cached_input_tensors; + std::vector cached_output_tensors; }; // Function to mmap rai file for Linux and MapViewOfFile for Windows @@ -94,6 +131,38 @@ static void unmap_rai_file(uint8_t * buffer, size_t size) { #endif // _WIN32 } +bool whisper_vitisai_file_exists(const char * path) { + if (!path) { + return false; + } + + FILE * file = fopen(path, "rb"); + if (!file) { + return false; + } + fclose(file); + return true; +} + +// Reuse cached tensor descriptors to avoid repeated getIOTensors() lookups. +static bool whisper_vitisai_get_io_tensors( + struct whisper_vitisai_context * ctx, + std::vector & input_tensors, + std::vector & output_tensors) { + if (!ctx || !ctx->runner) { + return false; + } + + if (ctx->cached_input_tensors.empty() || ctx->cached_output_tensors.empty()) { + ctx->cached_input_tensors = ctx->runner->getIOTensors("input", false); + ctx->cached_output_tensors = ctx->runner->getIOTensors("output", false); + } + + input_tensors = ctx->cached_input_tensors; + output_tensors = ctx->cached_output_tensors; + return true; +} + struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model) { if (!path_model) { std::fprintf(stderr, "%s: path_model is null\n", __func__); @@ -102,8 +171,6 @@ struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model) { auto * ctx = new whisper_vitisai_context; ctx->model_path = path_model; - ctx->fbs_buffer = nullptr; - ctx->fbs_buffer_size = 0; // Override the model path with the environment variable if it is set if (const char * env_model_path = std::getenv("OVERRIDE_VITISAI_MODEL_PATH")) { @@ -121,7 +188,6 @@ struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model) { // Check if model_path is rai file and if so, add fbs_buffer and fbs_buffer_size to the options if (ctx->model_path.find(".rai") != std::string::npos) { - // mmap rai file for both Linux and Windows and pass the buffer to the options if (map_rai_file(ctx->model_path.c_str(), &ctx->fbs_buffer, &ctx->fbs_buffer_size)) { options.extOptions["fbs_buffer"] = ctx->fbs_buffer; options.extOptions["fbs_buffer_size"] = ctx->fbs_buffer_size; @@ -138,12 +204,72 @@ struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model) { #endif } + const bool model_is_rai = ctx->model_path.find(".rai") != std::string::npos; + if (model_is_rai) { +#if WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES + options.deviceName = "stx"; + options.subgraphName = "vaiml_par_0"; +#if defined(WHISPER_DEBUG) + std::fprintf(stderr, + "%s: legacy FlexMLRT compile configuration detected; applying RAI overrides (device='stx', subgraph='vaiml_par_0')\n", + __func__); +#endif // defined(WHISPER_DEBUG) +#endif + } + try { ctx->runner = std::make_shared(options); if (!ctx->runner->good()) { throw std::runtime_error("Runner creation ran into an error"); } + + ctx->cached_input_tensors = ctx->runner->getIOTensors("input", false); + ctx->cached_output_tensors = ctx->runner->getIOTensors("output", false); + + auto & output_tensors = ctx->cached_output_tensors; + for (int i = 0; i < (int) output_tensors.size(); ++i) { + const std::string & name = output_tensors[i].getMetadata().name; + if (name == "embd_enc") { + ctx->embd_enc_out_idx = i; + } else if (name == "cross_k") { + ctx->cross_k_out_idx = i; + } else if (name == "cross_v") { + ctx->cross_v_out_idx = i; + } + } + + if (ctx->embd_enc_out_idx < 0) { + std::fprintf(stderr, "%s: WARNING: embd_enc output not found by name; falling back to output[0]\n", __func__); + ctx->embd_enc_out_idx = 0; + } + +#if defined(WHISPER_DEBUG) + { + auto & input_tensors = ctx->cached_input_tensors; + + std::fprintf(stderr, "%s: model has %zu input tensor(s)\n", __func__, input_tensors.size()); + for (int i = 0; i < (int) input_tensors.size(); ++i) { + const auto & meta = input_tensors[i].getMetadata(); + std::fprintf(stderr, "%s: input[%d] name='%s' size=%zu shape=", + __func__, i, meta.name.c_str(), (size_t) meta.size); + whisper_vitisai_print_shape(meta.shape); + std::fprintf(stderr, "\n"); + } + + std::fprintf(stderr, "%s: model has %zu output tensor(s)\n", __func__, output_tensors.size()); + for (int i = 0; i < (int) output_tensors.size(); ++i) { + const auto & meta = output_tensors[i].getMetadata(); + std::fprintf(stderr, "%s: output[%d] name='%s' size=%zu shape=", + __func__, i, meta.name.c_str(), (size_t) meta.size); + whisper_vitisai_print_shape(meta.shape); + std::fprintf(stderr, "\n"); + } + + std::fprintf(stderr, "%s: output indices: embd_enc=%d cross_k=%d cross_v=%d\n", + __func__, ctx->embd_enc_out_idx, ctx->cross_k_out_idx, ctx->cross_v_out_idx); + } +#endif } catch (const std::exception & e) { std::fprintf(stderr, "%s: Exception during Vitis AI runner creation: %s\n", __func__, e.what()); delete ctx; @@ -152,12 +278,18 @@ struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model) { return ctx; } +bool whisper_vitisai_has_cross_proj(const struct whisper_vitisai_context * ctx) { + return ctx && ctx->cross_k_out_idx >= 0 && ctx->cross_v_out_idx >= 0; +} + void whisper_vitisai_free(struct whisper_vitisai_context * ctx) { if (!ctx) { return; } - std::fprintf(stderr, "%s: releasing Vitis AI encoder context for model '%s'\n", __func__, ctx->model_path.c_str()); +#if defined(WHISPER_DEBUG) + std::fprintf(stderr, "%s: releasing Vitis AI context for model '%s'\n", __func__, ctx->model_path.c_str()); +#endif if (ctx->fbs_buffer) { unmap_rai_file(ctx->fbs_buffer, ctx->fbs_buffer_size); } @@ -184,14 +316,21 @@ int whisper_vitisai_encode(struct whisper_vitisai_context * ctx, struct ggml_ten std::vector input_tensors, output_tensors; auto model = ctx->runner; - // Get tensors as CPU tensors (hwTensor = false) - input_tensors = model->getIOTensors("input", false); - output_tensors = model->getIOTensors("output", false); + if (!whisper_vitisai_get_io_tensors(ctx, input_tensors, output_tensors)) { + std::fprintf(stderr, "%s: failed to acquire Vitis AI I/O tensors\n", __func__); + return 0; + } // TODO: add assert checks for tensor numbers and shapes + if (ctx->embd_enc_out_idx < 0 || ctx->embd_enc_out_idx >= (int) output_tensors.size()) { + std::fprintf(stderr, "%s: invalid embd_enc output index %d for %zu output tensor(s)\n", + __func__, ctx->embd_enc_out_idx, output_tensors.size()); + return 0; + } + input_tensors[0].data = mel->data; - output_tensors[0].data = out->data; + output_tensors[ctx->embd_enc_out_idx].data = out->data; try { model->forward(input_tensors, output_tensors); @@ -205,3 +344,260 @@ int whisper_vitisai_encode(struct whisper_vitisai_context * ctx, struct ggml_ten return 1; } + +int whisper_vitisai_run_enc_cross( + struct whisper_vitisai_context * ctx, + struct ggml_tensor * mel, + struct ggml_tensor * out, + void * cross_v_data, + void * cross_k_data) { + if (!ctx || !mel || !out || !cross_v_data || !cross_k_data) { + std::fprintf(stderr, "%s: ctx/mel/out/cross_v_data/cross_k_data must not be null\n", __func__); + return 0; + } + + if (ggml_n_dims(mel) != 2) { + std::fprintf(stderr, "%s: mel tensor expected to have 2 dims, got %d\n", __func__, ggml_n_dims(mel)); + return 0; + } + + if (ggml_n_dims(out) != 2) { + std::fprintf(stderr, "%s: out tensor expected to have 2 dims, got %d\n", __func__, ggml_n_dims(out)); + return 0; + } + + std::vector input_tensors, output_tensors; + auto model = ctx->runner; + + if (!whisper_vitisai_get_io_tensors(ctx, input_tensors, output_tensors)) { + std::fprintf(stderr, "%s: failed to acquire Vitis AI I/O tensors\n", __func__); + return 0; + } + + if (output_tensors.size() != 3) { + std::fprintf(stderr, "%s: expected 3 output tensors, got %zu\n", __func__, output_tensors.size()); + return 0; + } + + if (ctx->embd_enc_out_idx < 0 || ctx->embd_enc_out_idx >= (int) output_tensors.size() || + ctx->cross_k_out_idx < 0 || ctx->cross_k_out_idx >= (int) output_tensors.size() || + ctx->cross_v_out_idx < 0 || ctx->cross_v_out_idx >= (int) output_tensors.size()) { + std::fprintf(stderr, "%s: invalid output indices embd_enc=%d cross_k=%d cross_v=%d for %zu output tensor(s)\n", + __func__, ctx->embd_enc_out_idx, ctx->cross_k_out_idx, ctx->cross_v_out_idx, output_tensors.size()); + return 0; + } + + input_tensors[0].data = mel->data; + output_tensors[ctx->embd_enc_out_idx].data = out->data; + output_tensors[ctx->cross_v_out_idx].data = cross_v_data; + output_tensors[ctx->cross_k_out_idx].data = cross_k_data; + + try { + model->forward(input_tensors, output_tensors); +#if defined(WHISPER_DEBUG) + std::fprintf(stderr, "%s: Vitis AI model inference (encoder + cross proj) completed.\n", __func__); +#endif + } catch (const std::exception & e) { + std::fprintf(stderr, "%s: Exception during model inference: %s\n", __func__, e.what()); + return 0; + } + + return 1; +} + +// Ensure persistent staging buffers are large enough for the given dimensions. +static void ensure_staging_buffers( + struct whisper_vitisai_context * ctx, + size_t count, bool need_k) { + if (need_k && ctx->cross_k_staging.size() < count) { + ctx->cross_k_staging.resize(count); + } + if (ctx->cross_v_staging.size() < count) { + ctx->cross_v_staging.resize(count); + } +} + +int whisper_vitisai_encode_with_cross( + struct whisper_vitisai_context * ctx, + struct ggml_tensor * mel, + struct ggml_tensor * embd_enc, + struct ggml_tensor * kv_cross_k, + struct ggml_tensor * kv_cross_v, + int n_text_layer, + int n_ctx, + int n_text_state, + int n_text_head, + bool flash_attn) { + if (!ctx || !mel || !embd_enc || !kv_cross_k || !kv_cross_v) { + std::fprintf(stderr, "%s: null argument\n", __func__); + return 0; + } + + const int n_state = n_text_state; + const int n_state_head = n_state / n_text_head; + const int n_ctx_pad = (n_ctx + 255) & ~255; // GGML_PAD(n_ctx, 256) + + const float Kscale = pow(float(n_state_head), -0.25f); + const ggml_type kv_type = kv_cross_k->type; + const size_t elem_size = ggml_type_size(kv_type); + const size_t layer_elems = (size_t)n_ctx * n_state; + const size_t buf_count = (size_t)n_text_layer * layer_elems; + + if (flash_attn) { + WHISPER_DBG_TIMER(t_fwd_start); + + if (n_ctx_pad == n_ctx) { + // No padding gap -- plugin writes directly into kv_cross. + if (!whisper_vitisai_run_enc_cross( + ctx, mel, embd_enc, + kv_cross_v->data, kv_cross_k->data)) { + return 0; + } + + WHISPER_DBG_TIMER(t_fwd_end); + WHISPER_DBG_TIMER(t_post_start); + + if (kv_type == GGML_TYPE_F32) { + float * kdata = (float *)kv_cross_k->data; + for (size_t i = 0; i < buf_count; ++i) { + kdata[i] *= Kscale; + } + } else if (kv_type == GGML_TYPE_F16) { + ggml_fp16_t * kdata = (ggml_fp16_t *)kv_cross_k->data; + for (size_t i = 0; i < buf_count; ++i) { + kdata[i] = ggml_fp32_to_fp16(ggml_fp16_to_fp32(kdata[i]) * Kscale); + } + } + + WHISPER_DBG_TIMER(t_post_end); + +#if defined(WHISPER_DEBUG) + std::fprintf(stderr, "%s: vitisai enc+cross forward time = %8.2f ms\n", __func__, (t_fwd_end - t_fwd_start) / 1000.0f); + std::fprintf(stderr, "%s: kv_cross post-process time = %8.2f ms (flash, no-pad direct)\n", __func__, (t_post_end - t_post_start) / 1000.0f); +#endif + } else { + // Padding gap -- use persistent staging buffers. + ensure_staging_buffers(ctx, buf_count, true); + float * cross_k_buf = ctx->cross_k_staging.data(); + float * cross_v_buf = ctx->cross_v_staging.data(); + + if (!whisper_vitisai_run_enc_cross( + ctx, mel, embd_enc, + cross_v_buf, cross_k_buf)) { + return 0; + } + + WHISPER_DBG_TIMER(t_fwd_end); + WHISPER_DBG_TIMER(t_post_start); + + // Combined per-layer K+V scatter for better cache locality. + const size_t padded_layer_stride = elem_size * n_state * n_ctx_pad; + + for (int il = 0; il < n_text_layer; ++il) { + const float * src_k = cross_k_buf + (size_t)il * layer_elems; + const float * src_v = cross_v_buf + (size_t)il * layer_elems; + uint8_t * dst_k = (uint8_t *)kv_cross_k->data + padded_layer_stride * il; + uint8_t * dst_v = (uint8_t *)kv_cross_v->data + padded_layer_stride * il; + + if (kv_type == GGML_TYPE_F32) { + float * dk = (float *)dst_k; + for (size_t i = 0; i < layer_elems; ++i) { + dk[i] = src_k[i] * Kscale; + } + memcpy(dst_v, src_v, layer_elems * sizeof(float)); + } else if (kv_type == GGML_TYPE_F16) { + ggml_fp16_t * dk = (ggml_fp16_t *)dst_k; + ggml_fp16_t * dv = (ggml_fp16_t *)dst_v; + for (size_t i = 0; i < layer_elems; ++i) { + dk[i] = ggml_fp32_to_fp16(src_k[i] * Kscale); + dv[i] = ggml_fp32_to_fp16(src_v[i]); + } + } + } + + WHISPER_DBG_TIMER(t_post_end); + +#if defined(WHISPER_DEBUG) + std::fprintf(stderr, "%s: vitisai enc+cross forward time = %8.2f ms\n", __func__, (t_fwd_end - t_fwd_start) / 1000.0f); + std::fprintf(stderr, "%s: kv_cross post-process time = %8.2f ms (flash, padded, n_ctx=%d, n_ctx_pad=%d, kv_type=%s)\n", + __func__, (t_post_end - t_post_start) / 1000.0f, + n_ctx, n_ctx_pad, + kv_type == GGML_TYPE_F32 ? "F32" : kv_type == GGML_TYPE_F16 ? "F16" : "other"); +#endif + } + } else { + // Non-flash: layers are contiguous (stride = n_state * n_ctx). + // K: plugin writes directly into kv_cross_k, then in-place Kscale. + // V: persistent staging buffer + cache-friendly blocked transpose. + ensure_staging_buffers(ctx, buf_count, false); + float * cross_v_buf = ctx->cross_v_staging.data(); + + WHISPER_DBG_TIMER(t_fwd_start); + + if (!whisper_vitisai_run_enc_cross( + ctx, mel, embd_enc, + cross_v_buf, kv_cross_k->data)) { + return 0; + } + + WHISPER_DBG_TIMER(t_fwd_end); + WHISPER_DBG_TIMER(t_post_start); + + if (kv_type == GGML_TYPE_F32) { + float * kdata = (float *)kv_cross_k->data; + for (size_t i = 0; i < buf_count; ++i) { + kdata[i] *= Kscale; + } + + const int BLOCK = 32; + for (int il = 0; il < n_text_layer; ++il) { + const float * src_v = cross_v_buf + (size_t)il * layer_elems; + float * dst_v = (float *)kv_cross_v->data + (size_t)il * layer_elems; + + for (int ic = 0; ic < n_ctx; ic += BLOCK) { + for (int is = 0; is < n_state; is += BLOCK) { + const int ic_end = std::min(ic + BLOCK, n_ctx); + const int is_end = std::min(is + BLOCK, n_state); + for (int i = ic; i < ic_end; ++i) { + for (int j = is; j < is_end; ++j) { + dst_v[j * n_ctx + i] = src_v[i * n_state + j]; + } + } + } + } + } + } else if (kv_type == GGML_TYPE_F16) { + ggml_fp16_t * kdata = (ggml_fp16_t *)kv_cross_k->data; + for (size_t i = 0; i < buf_count; ++i) { + kdata[i] = ggml_fp32_to_fp16(ggml_fp16_to_fp32(kdata[i]) * Kscale); + } + + const int BLOCK = 32; + for (int il = 0; il < n_text_layer; ++il) { + const float * src_v = cross_v_buf + (size_t)il * layer_elems; + ggml_fp16_t * dst_v = (ggml_fp16_t *)((uint8_t *)kv_cross_v->data + elem_size * n_state * n_ctx * il); + + for (int ic = 0; ic < n_ctx; ic += BLOCK) { + for (int is = 0; is < n_state; is += BLOCK) { + const int ic_end = std::min(ic + BLOCK, n_ctx); + const int is_end = std::min(is + BLOCK, n_state); + for (int i = ic; i < ic_end; ++i) { + for (int j = is; j < is_end; ++j) { + dst_v[j * n_ctx + i] = ggml_fp32_to_fp16(src_v[i * n_state + j]); + } + } + } + } + } + } + + WHISPER_DBG_TIMER(t_post_end); + +#if defined(WHISPER_DEBUG) + std::fprintf(stderr, "%s: vitisai enc+cross forward time = %8.2f ms\n", __func__, (t_fwd_end - t_fwd_start) / 1000.0f); + std::fprintf(stderr, "%s: kv_cross post-process time = %8.2f ms (non-flash)\n", __func__, (t_post_end - t_post_start) / 1000.0f); +#endif + } + + return 1; +} diff --git a/src/vitisai/whisper-vitisai-encoder.h b/src/vitisai/whisper-vitisai-encoder.h index 840ce6941..f09003a64 100644 --- a/src/vitisai/whisper-vitisai-encoder.h +++ b/src/vitisai/whisper-vitisai-encoder.h @@ -1,8 +1,6 @@ #pragma once -#include #include -#include #if __cplusplus extern "C" { @@ -12,6 +10,8 @@ struct whisper_vitisai_context; struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model); void whisper_vitisai_free(struct whisper_vitisai_context * ctx); +bool whisper_vitisai_has_cross_proj(const struct whisper_vitisai_context * ctx); +bool whisper_vitisai_file_exists(const char * path); struct ggml_tensor; @@ -20,6 +20,25 @@ int whisper_vitisai_encode( struct ggml_tensor * mel, struct ggml_tensor * out); +int whisper_vitisai_run_enc_cross( + struct whisper_vitisai_context * ctx, + struct ggml_tensor * mel, + struct ggml_tensor * out, + void * cross_v_data, + void * cross_k_data); + +int whisper_vitisai_encode_with_cross( + struct whisper_vitisai_context * ctx, + struct ggml_tensor * mel, + struct ggml_tensor * embd_enc, + struct ggml_tensor * kv_cross_k, + struct ggml_tensor * kv_cross_v, + int n_text_layer, + int n_ctx, + int n_text_state, + int n_text_head, + bool flash_attn); + #if __cplusplus } #endif diff --git a/src/vitisai/whisper-vitisai-helpers.cpp b/src/vitisai/whisper-vitisai-helpers.cpp new file mode 100644 index 000000000..2ff509447 --- /dev/null +++ b/src/vitisai/whisper-vitisai-helpers.cpp @@ -0,0 +1,481 @@ +#ifdef _WIN32 +#ifndef NOMINMAX +#define NOMINMAX +#endif +#endif + +#include "vitisai/whisper-vitisai-helpers.h" + +#include +#include +#ifdef _WIN32 + #include +#else + #include + #include +#endif +#include +#include + +namespace whisper_vitisai_helpers { + +bool map_rai_file(const char * path, uint8_t ** buffer, size_t * size) { +#ifdef _WIN32 + HANDLE hFile = CreateFileA(path, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); + if (hFile == INVALID_HANDLE_VALUE) { + std::fprintf(stderr, "%s: %d: Failed to open rai file '%s'\n", __func__, __LINE__, path); + return false; + } + + LARGE_INTEGER fileSize; + if (!GetFileSizeEx(hFile, &fileSize)) { + CloseHandle(hFile); + std::fprintf(stderr, "%s: %d: Failed to get file size for rai file '%s'\n", __func__, __LINE__, path); + return false; + } + + HANDLE hMapping = CreateFileMappingA(hFile, NULL, PAGE_READONLY, 0, fileSize.QuadPart, NULL); + if (hMapping == NULL) { + CloseHandle(hFile); + std::fprintf(stderr, "%s: %d: Failed to create file mapping for rai file '%s'\n", __func__, __LINE__, path); + return false; + } + + *buffer = (uint8_t *) MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, fileSize.QuadPart); + if (*buffer == NULL) { + CloseHandle(hMapping); + CloseHandle(hFile); + std::fprintf(stderr, "%s: %d: Failed to map rai file '%s'\n", __func__, __LINE__, path); + return false; + } + CloseHandle(hMapping); + CloseHandle(hFile); + *size = fileSize.QuadPart; + return true; +#else + FILE * fd = fopen(path, "rb"); + if (!fd) { + std::fprintf(stderr, "%s: %d: Failed to open rai file '%s'\n", __func__, __LINE__, path); + return false; + } + + struct stat st; + if (fstat(fileno(fd), &st) == -1) { + fclose(fd); + std::fprintf(stderr, "%s: %d: Failed to get file size for rai file '%s'\n", __func__, __LINE__, path); + return false; + } + + *buffer = (uint8_t *) mmap(nullptr, st.st_size, PROT_READ, MAP_SHARED, fileno(fd), 0); + if (*buffer == MAP_FAILED) { + fclose(fd); + std::fprintf(stderr, "%s: %d: Failed to mmap rai file '%s'\n", __func__, __LINE__, path); + return false; + } + fclose(fd); + *size = st.st_size; + return true; +#endif // _WIN32 +} + +void unmap_rai_file(uint8_t * buffer, size_t size) { +#ifdef _WIN32 + UnmapViewOfFile(buffer); +#else + munmap(buffer, size); +#endif // _WIN32 +} + +bool file_exists(const char * path) { + if (!path) { + return false; + } + + FILE * file = fopen(path, "rb"); + if (!file) { + return false; + } + fclose(file); + return true; +} + +const char * whisper_kv_type_name(ggml_type type) { + switch (type) { + case GGML_TYPE_F32: return "F32"; + case GGML_TYPE_F16: return "F16"; + default: return "unsupported"; + } +} + +const char * whisper_flexml_dtype_name(flexmlrt::client::DataType type) { + switch (type) { + case flexmlrt::client::DataType::Float32: return "Float32"; + case flexmlrt::client::DataType::Int8: return "Int8"; + case flexmlrt::client::DataType::UInt8: return "UInt8"; + case flexmlrt::client::DataType::Int16: return "Int16"; + case flexmlrt::client::DataType::UInt16: return "UInt16"; + case flexmlrt::client::DataType::BFloat16: return "BFloat16"; + case flexmlrt::client::DataType::Bool: return "Bool"; + case flexmlrt::client::DataType::Float16: return "Float16"; + case flexmlrt::client::DataType::Int32: return "Int32"; + case flexmlrt::client::DataType::UInt32: return "UInt32"; + default: return "Unknown"; + } +} + +bool whisper_flexml_dtype_to_ggml_type( + flexmlrt::client::DataType type, + ggml_type * ggml_dtype) { + switch (type) { + case flexmlrt::client::DataType::Float32: + if (ggml_dtype) { + *ggml_dtype = GGML_TYPE_F32; + } + return true; + case flexmlrt::client::DataType::Float16: + if (ggml_dtype) { + *ggml_dtype = GGML_TYPE_F16; + } + return true; + case flexmlrt::client::DataType::BFloat16: + if (ggml_dtype) { + *ggml_dtype = GGML_TYPE_BF16; + } + return true; + default: + return false; + } +} + +static bool whisper_vitisai_validate_tensor_dtype( + const char * tensor_name, + flexmlrt::client::DataType model_dtype, + ggml_type runtime_dtype) { + ggml_type expected_runtime_dtype = GGML_TYPE_COUNT; + if (!whisper_flexml_dtype_to_ggml_type(model_dtype, &expected_runtime_dtype)) { + std::fprintf(stderr, + "%s: unsupported model dtype for %s: %s (supported: Float32/Float16/BFloat16)\n", + __func__, tensor_name, whisper_flexml_dtype_name(model_dtype)); + return false; + } + + if (runtime_dtype != expected_runtime_dtype) { + std::fprintf(stderr, + "%s: %s dtype mismatch (runtime=%s, model=%s)\n", + __func__, tensor_name, ggml_type_name(runtime_dtype), whisper_flexml_dtype_name(model_dtype)); + return false; + } + + return true; +} + +static std::string whisper_shape_to_string(const std::vector & shape) { + std::string out = "["; + for (size_t i = 0; i < shape.size(); ++i) { + if (i > 0) { + out += ", "; + } + out += std::to_string(shape[i]); + } + out += "]"; + return out; +} + +static std::vector whisper_canonical_shape(const std::vector & shape) { + std::vector canonical; + canonical.reserve(shape.size()); + for (size_t i = 0; i < shape.size(); ++i) { + const size_t dim = (size_t) shape[i]; + if (dim != 1) { + canonical.push_back(dim); + } + } + if (canonical.empty()) { + canonical.push_back(1); + } + return canonical; +} + +static bool whisper_validate_shape( + const char * tensor_name, + const std::vector & model_shape, + const std::vector & expected_shape) { + const std::vector shape = whisper_canonical_shape(model_shape); + if (shape != expected_shape) { + std::fprintf(stderr, + "%s: %s shape mismatch (runtime expected=%s, model=%s)\n", + __func__, + tensor_name, + whisper_shape_to_string(expected_shape).c_str(), + whisper_shape_to_string(shape).c_str()); + return false; + } + return true; +} + +bool whisper_validate_cross_shape( + const char * tensor_name, + const std::vector & model_shape, + int n_text_layer, + int n_ctx, + int n_state) { + const std::vector expected = { + (size_t) n_text_layer, + (size_t) n_ctx, + (size_t) n_state, + }; + return whisper_validate_shape(tensor_name, model_shape, expected); +} + +bool whisper_vitisai_bind_tensor_data( + const char * tensor_name, + struct ggml_tensor * runtime_tensor, + const std::vector & expected_shape, + flexmlrt::client::ErtTensorType & io_tensor) { + const auto & meta = io_tensor.getMetadata(); + if (!whisper_vitisai_validate_tensor_dtype(tensor_name, meta.type, runtime_tensor->type)) { + return false; + } + if (!whisper_validate_shape(tensor_name, meta.shape, expected_shape)) { + return false; + } + + const size_t model_bytes = meta.size; + const size_t runtime_bytes = ggml_nbytes(runtime_tensor); + if (model_bytes == 0 || runtime_bytes == 0) { + std::fprintf(stderr, "%s: %s sizes must be non-zero (model=%zu, runtime=%zu)\n", + __func__, tensor_name, model_bytes, runtime_bytes); + return false; + } + if (runtime_bytes != model_bytes) { + std::fprintf(stderr, + "%s: %s tensor size mismatch (runtime=%zu B, model=%zu B). " + "VitisAI .rai requires exact context match; use matching -ac/model artifact.\n", + __func__, tensor_name, runtime_bytes, model_bytes); + return false; + } + + io_tensor.data = runtime_tensor->data; + return true; +} + +bool whisper_vitisai_resolve_io_binding( + const char * caller, + const std::vector & input_tensors, + const std::vector & output_tensors, + whisper_vitisai_io_binding * binding, + std::string * error) { + const auto fail = [error](std::string message) { + if (error) { + *error = std::move(message); + } + return false; + }; + + if (input_tensors.empty()) { + return fail("Model has no input tensors"); + } + + binding->mel_in_idx = 0; + bool found_named_mel = false; + for (int i = 0; i < (int) input_tensors.size(); ++i) { + const std::string & name = input_tensors[i].getMetadata().name; + if (name == "input" || name == "mel") { + binding->mel_in_idx = i; + found_named_mel = true; + break; + } + } + if (!found_named_mel) { + std::fprintf(stderr, "%s: WARNING: mel input not found by name; falling back to input[0]\n", caller); + } + + if (output_tensors.empty()) { + return fail("Model has no output tensors"); + } + + for (int i = 0; i < (int) output_tensors.size(); ++i) { + const std::string & name = output_tensors[i].getMetadata().name; + if (name == "embd_enc") { + binding->embd_enc_out_idx = i; + } else if (name == "cross_k") { + binding->cross_k_out_idx = i; + } else if (name == "cross_v") { + binding->cross_v_out_idx = i; + } + } + + if (binding->embd_enc_out_idx < 0) { + std::fprintf(stderr, "%s: WARNING: embd_enc output not found by name; falling back to output[0]\n", caller); + binding->embd_enc_out_idx = 0; + } + + const bool has_cross_k = binding->cross_k_out_idx >= 0; + const bool has_cross_v = binding->cross_v_out_idx >= 0; + if (has_cross_k != has_cross_v) { + return fail("Incomplete cross-projection contract: both cross_k and cross_v outputs are required"); + } + + if (has_cross_k && (binding->cross_k_out_idx == binding->cross_v_out_idx || + binding->cross_k_out_idx == binding->embd_enc_out_idx || + binding->cross_v_out_idx == binding->embd_enc_out_idx)) { + return fail("Invalid output mapping: embd_enc/cross_k/cross_v indices overlap"); + } + + const auto & mel_meta = input_tensors[binding->mel_in_idx].getMetadata(); + if (!whisper_flexml_dtype_to_ggml_type(mel_meta.type, nullptr)) { + return fail( + std::string("Unsupported mel input type: ") + + whisper_flexml_dtype_name(mel_meta.type) + " (supported: Float32/Float16/BFloat16)"); + } + binding->mel_in_expected_bytes = mel_meta.size; + + const auto & embd_meta = output_tensors[binding->embd_enc_out_idx].getMetadata(); + if (!whisper_flexml_dtype_to_ggml_type(embd_meta.type, nullptr)) { + return fail( + std::string("Unsupported embd_enc output type: ") + + whisper_flexml_dtype_name(embd_meta.type) + " (supported: Float32/Float16/BFloat16)"); + } + binding->embd_enc_expected_bytes = embd_meta.size; + + if (has_cross_k) { + const auto & cross_k_meta = output_tensors[binding->cross_k_out_idx].getMetadata(); + const auto & cross_v_meta = output_tensors[binding->cross_v_out_idx].getMetadata(); + if (cross_k_meta.type != flexmlrt::client::DataType::Float32 || + cross_v_meta.type != flexmlrt::client::DataType::Float32) { + return fail( + std::string("Unsupported cross output type(s): cross_k=") + + whisper_flexml_dtype_name(cross_k_meta.type) + ", cross_v=" + + whisper_flexml_dtype_name(cross_v_meta.type) + " (cross path currently requires Float32)"); + } + if (cross_k_meta.size != cross_v_meta.size) { + return fail("cross_k and cross_v output sizes do not match"); + } + binding->cross_k_expected_bytes = cross_k_meta.size; + binding->cross_v_expected_bytes = cross_v_meta.size; + } + + return true; +} + +bool whisper_vitisai_all_tensors_claimed( + const char * caller, + const char * tensor_kind, + const std::vector & tensors, + const std::vector & claimed) { + for (size_t i = 0; i < tensors.size(); ++i) { + if (!claimed[i]) { + std::fprintf(stderr, + "%s: unsupported extra %s tensor at index %zu (name='%s'); strict contract expects only mapped %ss\n", + caller, tensor_kind, i, tensors[i].getMetadata().name.c_str(), tensor_kind); + return false; + } + } + return true; +} + +void whisper_kv_cross_scale_k_f32( + float * k_data, + size_t count, + float kscale) { + for (size_t i = 0; i < count; ++i) { + k_data[i] *= kscale; + } +} + +void whisper_kv_cross_store_layers_f32( + const float * src_k, + const float * src_v, + uint8_t * dst_k, + uint8_t * dst_v, + const whisper_kv_cross_layout & layout) { + for (int il = 0; il < layout.n_layer; ++il) { + const float * layer_src_k = src_k + (size_t)il * layout.src_layer_elems; + const float * layer_src_v = src_v + (size_t)il * layout.src_layer_elems; + float * dk = (float *)(dst_k + layout.dst_layer_stride * (size_t)il); + float * dv = (float *)(dst_v + layout.dst_layer_stride * (size_t)il); + for (size_t i = 0; i < layout.layer_elems; ++i) { + dk[i] = layer_src_k[i] * layout.kscale; + dv[i] = layer_src_v[i]; + } + } +} + +void whisper_kv_cross_store_layers_f16( + const float * src_k, + const float * src_v, + uint8_t * dst_k, + uint8_t * dst_v, + const whisper_kv_cross_layout & layout) { + for (int il = 0; il < layout.n_layer; ++il) { + const float * layer_src_k = src_k + (size_t)il * layout.src_layer_elems; + const float * layer_src_v = src_v + (size_t)il * layout.src_layer_elems; + ggml_fp16_t * dk = (ggml_fp16_t *)(dst_k + layout.dst_layer_stride * (size_t)il); + ggml_fp16_t * dv = (ggml_fp16_t *)(dst_v + layout.dst_layer_stride * (size_t)il); + for (size_t i = 0; i < layout.layer_elems; ++i) { + dk[i] = ggml_fp32_to_fp16(layer_src_k[i] * layout.kscale); + dv[i] = ggml_fp32_to_fp16(layer_src_v[i]); + } + } +} + +void whisper_kv_cross_transpose_v_layers_f32( + const float * src_v, + uint8_t * dst_v, + const whisper_kv_cross_layout & layout) { + const int n_ctx = layout.n_ctx; + const int n_state = layout.n_state; + + const int BLOCK = 32; + for (int il = 0; il < layout.n_layer; ++il) { + const float * layer_src_v = src_v + (size_t)il * layout.src_layer_elems; + float * dv = (float *)(dst_v + layout.dst_layer_stride * (size_t)il); + + for (int ic = 0; ic < n_ctx; ic += BLOCK) { + for (int is = 0; is < n_state; is += BLOCK) { + const int ic_end = std::min(ic + BLOCK, n_ctx); + const int is_end = std::min(is + BLOCK, n_state); + for (int i = ic; i < ic_end; ++i) { + for (int j = is; j < is_end; ++j) { + dv[j * n_ctx + i] = layer_src_v[i * n_state + j]; + } + } + } + } + } +} + +void whisper_kv_cross_store_k_transpose_v_layers_f16( + const float * src_k, + const float * src_v, + uint8_t * dst_k, + uint8_t * dst_v, + const whisper_kv_cross_layout & layout) { + const int n_ctx = layout.n_ctx; + const int n_state = layout.n_state; + + const int BLOCK = 32; + for (int il = 0; il < layout.n_layer; ++il) { + const float * layer_src_k = src_k + (size_t)il * layout.src_layer_elems; + const float * layer_src_v = src_v + (size_t)il * layout.src_layer_elems; + ggml_fp16_t * dk = (ggml_fp16_t *)(dst_k + layout.dst_layer_stride * (size_t)il); + ggml_fp16_t * dv = (ggml_fp16_t *)(dst_v + layout.dst_layer_stride * (size_t)il); + for (size_t i = 0; i < layout.layer_elems; ++i) { + dk[i] = ggml_fp32_to_fp16(layer_src_k[i] * layout.kscale); + } + + for (int ic = 0; ic < n_ctx; ic += BLOCK) { + for (int is = 0; is < n_state; is += BLOCK) { + const int ic_end = std::min(ic + BLOCK, n_ctx); + const int is_end = std::min(is + BLOCK, n_state); + for (int i = ic; i < ic_end; ++i) { + for (int j = is; j < is_end; ++j) { + dv[j * n_ctx + i] = ggml_fp32_to_fp16(layer_src_v[i * n_state + j]); + } + } + } + } + } +} + +} // namespace whisper_vitisai_helpers diff --git a/src/vitisai/whisper-vitisai-helpers.h b/src/vitisai/whisper-vitisai-helpers.h new file mode 100644 index 000000000..f6ab53902 --- /dev/null +++ b/src/vitisai/whisper-vitisai-helpers.h @@ -0,0 +1,118 @@ +#pragma once + +#include "FlexMLClient.h" +#include "ggml.h" + +#include +#include +#include +#include +#include + +namespace whisper_vitisai_helpers { + +bool map_rai_file(const char * path, uint8_t ** buffer, size_t * size); +void unmap_rai_file(uint8_t * buffer, size_t size); +bool file_exists(const char * path); + +const char * whisper_kv_type_name(ggml_type type); +const char * whisper_flexml_dtype_name(flexmlrt::client::DataType type); +bool whisper_flexml_dtype_to_ggml_type( + flexmlrt::client::DataType type, + ggml_type * ggml_dtype); + +bool whisper_validate_cross_shape( + const char * tensor_name, + const std::vector & model_shape, + int n_text_layer, + int n_ctx, + int n_state); + +bool whisper_vitisai_bind_tensor_data( + const char * tensor_name, + struct ggml_tensor * runtime_tensor, + const std::vector & expected_shape, + flexmlrt::client::ErtTensorType & io_tensor); + +#if defined(WHISPER_DEBUG) +template +void whisper_vitisai_print_shape(const std::vector & shape) { + std::fprintf(stderr, "["); + for (size_t i = 0; i < shape.size(); ++i) { + std::fprintf(stderr, "%s%lld", i == 0 ? "" : ", ", (long long) shape[i]); + } + std::fprintf(stderr, "]"); +} +#endif + +// Model IO tensor indices and metadata sizes resolved once at init time. +struct whisper_vitisai_io_binding { + int mel_in_idx = -1; + int embd_enc_out_idx = -1; + int cross_k_out_idx = -1; + int cross_v_out_idx = -1; + size_t mel_in_expected_bytes = 0; + size_t embd_enc_expected_bytes = 0; + size_t cross_k_expected_bytes = 0; + size_t cross_v_expected_bytes = 0; +}; + +// Warnings are printed with the caller's name; hard failures are returned in *error +// so the caller can decide how to report them. +bool whisper_vitisai_resolve_io_binding( + const char * caller, + const std::vector & input_tensors, + const std::vector & output_tensors, + whisper_vitisai_io_binding * binding, + std::string * error); + +bool whisper_vitisai_all_tensors_claimed( + const char * caller, + const char * tensor_kind, + const std::vector & tensors, + const std::vector & claimed); + +// Geometry of one cross K/V transfer from the model output (always f32, contiguous +// [ctx, state] per layer) into the runtime kv cache. +struct whisper_kv_cross_layout { + int n_layer = 0; + int n_ctx = 0; + int n_state = 0; + size_t src_layer_elems = 0; // f32 elements per layer in the model output buffer + size_t layer_elems = 0; // elements per layer transferred into the kv cache + size_t dst_layer_stride = 0; // bytes per layer in the kv cache + float kscale = 1.0f; +}; + +void whisper_kv_cross_scale_k_f32( + float * k_data, + size_t count, + float kscale); + +void whisper_kv_cross_store_layers_f32( + const float * src_k, + const float * src_v, + uint8_t * dst_k, + uint8_t * dst_v, + const whisper_kv_cross_layout & layout); + +void whisper_kv_cross_store_layers_f16( + const float * src_k, + const float * src_v, + uint8_t * dst_k, + uint8_t * dst_v, + const whisper_kv_cross_layout & layout); + +void whisper_kv_cross_transpose_v_layers_f32( + const float * src_v, + uint8_t * dst_v, + const whisper_kv_cross_layout & layout); + +void whisper_kv_cross_store_k_transpose_v_layers_f16( + const float * src_k, + const float * src_v, + uint8_t * dst_k, + uint8_t * dst_v, + const whisper_kv_cross_layout & layout); + +} // namespace whisper_vitisai_helpers diff --git a/src/whisper.cpp b/src/whisper.cpp index 896dd4c21..c64b210b0 100644 --- a/src/whisper.cpp +++ b/src/whisper.cpp @@ -1987,6 +1987,18 @@ static bool whisper_encode_external(const whisper_state & wstate) { return use_coreml || use_openvino || use_vitisai; } +static bool whisper_cross_external(const whisper_state & wstate) { + GGML_UNUSED(wstate); + +#if defined(WHISPER_USE_VITISAI) + const bool use_vitisai_cross = whisper_vitisai_has_cross_proj(wstate.ctx_vitisai); +#else + const bool use_vitisai_cross = false; +#endif + + return use_vitisai_cross; +} + static struct ggml_cgraph * whisper_build_graph_conv( whisper_context & wctx, whisper_state & wstate) { @@ -2426,7 +2438,20 @@ static bool whisper_encode_internal( #if defined(WHISPER_USE_COREML) whisper_coreml_encode(wstate.ctx_coreml, mel->ne[0], mel->ne[1], (float *) mel->data, (float *) wstate.embd_enc->data); #elif defined(WHISPER_USE_VITISAI) - whisper_vitisai_encode(wstate.ctx_vitisai, mel, wstate.embd_enc); + if (whisper_vitisai_has_cross_proj(wstate.ctx_vitisai)) { + const auto & hp = wctx.model.hparams; + const int n_ctx = wstate.exp_n_audio_ctx > 0 + ? wstate.exp_n_audio_ctx : hp.n_audio_ctx; + if (!whisper_vitisai_encode_with_cross( + wstate.ctx_vitisai, mel, wstate.embd_enc, + wstate.kv_cross.k, wstate.kv_cross.v, + hp.n_text_layer, n_ctx, hp.n_text_state, + hp.n_text_head, wctx.params.flash_attn)) { + return false; + } + } else if (!whisper_vitisai_encode(wstate.ctx_vitisai, mel, wstate.embd_enc)) { + return false; + } #elif defined(WHISPER_USE_OPENVINO) whisper_openvino_encode(wstate.ctx_openvino, mel, wstate.embd_enc); #endif @@ -2450,7 +2475,7 @@ static bool whisper_encode_internal( } // cross - { + if (!whisper_cross_external(wstate)) { auto & sched = wstate.sched_cross.sched; ggml_cgraph * gf = whisper_build_graph_cross(wctx, wstate); @@ -3370,9 +3395,13 @@ static std::string whisper_get_vitisai_path_encoder_cache(std::string path_bin) path_bin = path_bin.substr(0, pos); } - path_bin += "-encoder-vitisai.rai"; + const std::string path_vitisai_cross = path_bin + "-encoder-cross-vitisai.rai"; + if (FILE * file = fopen(path_vitisai_cross.c_str(), "rb")) { + fclose(file); + return path_vitisai_cross; + } - return path_bin; + return path_bin + "-encoder-vitisai.rai"; } #endif @@ -3493,8 +3522,10 @@ struct whisper_state * whisper_init_state(whisper_context * ctx) { WHISPER_LOG_ERROR("%s: failed to load Vitis AI model from '%s'\n", __func__, path_vitisai.c_str()); whisper_free_state(state); return nullptr; + } else if (whisper_vitisai_has_cross_proj(state->ctx_vitisai)) { + WHISPER_LOG_INFO("%s: Vitis AI encoder + cross projection model loaded\n", __func__); } else { - WHISPER_LOG_INFO("%s: Vitis AI model loaded\n", __func__); + WHISPER_LOG_INFO("%s: Vitis AI encoder model loaded\n", __func__); } #endif @@ -3545,7 +3576,7 @@ struct whisper_state * whisper_init_state(whisper_context * ctx) { } // cross allocator - { + if (!whisper_cross_external(*state)) { bool ok = whisper_sched_graph_init(state->sched_cross, state->backends, [&]() { return whisper_build_graph_cross(*ctx, *state); From 41b665011f41bd85d3a5d73c3d0f178583ac0847 Mon Sep 17 00:00:00 2001 From: Iswarya Alex Date: Wed, 29 Jul 2026 21:55:27 -0700 Subject: [PATCH 12/24] Add self hosted runner for amd npu --- .github/workflows/build-self-hosted.yml | 153 ++++++++++++++++++++++++ 1 file changed, 153 insertions(+) diff --git a/.github/workflows/build-self-hosted.yml b/.github/workflows/build-self-hosted.yml index 2286b63d6..f9e8715f1 100644 --- a/.github/workflows/build-self-hosted.yml +++ b/.github/workflows/build-self-hosted.yml @@ -114,3 +114,156 @@ jobs: run: | vulkaninfo --summary GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/whisper.cpp ~/mnt/whisper.cpp + + amd-npu-windows: + runs-on: [self-hosted, Windows, stx, rai300_400] + timeout-minutes: 60 + continue-on-error: true # advisory while the runner pool is new; revisit later + + env: + FLEXML_URL: https://github.com/lemonade-sdk/whisper.cpp/releases/download/deps/flexmlrt1.7.0-win.zip + MODEL: base + + steps: + - name: Clone + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - uses: microsoft/setup-msbuild@v2 + + - name: Install CMake if not available + shell: powershell + run: | + $installed = Get-Command cmake -ErrorAction SilentlyContinue + if (-not $installed) { + $ver = "3.28.1" + $url = "https://github.com/Kitware/CMake/releases/download/v$ver/cmake-$ver-windows-x86_64.msi" + Invoke-WebRequest -Uri $url -OutFile cmake.msi + Start-Process msiexec.exe -ArgumentList "/i cmake.msi /quiet /norestart" -Wait + $p = "C:\Program Files\CMake\bin" + $env:PATH = "$p;$env:PATH" + echo $p >> $env:GITHUB_PATH + cmake --version + if ($LASTEXITCODE -ne 0) { Write-Error "CMake install failed"; exit 1 } + } else { cmake --version } + + - name: Download FlexML runtime + shell: powershell + run: | + Invoke-WebRequest -Uri "${{ env.FLEXML_URL }}" -OutFile flexmlrt.zip + if (-Not (Test-Path "flexmlrt.zip")) { Write-Error "flexmlrt.zip not downloaded"; exit 1 } + if ((Get-Item "flexmlrt.zip").Length -eq 0) { Write-Error "flexmlrt.zip is empty"; exit 1 } + tar xf flexmlrt.zip + if ($LASTEXITCODE -ne 0) { Write-Error "Extraction failed"; exit 1 } + if (-not (Test-Path "flexmlrt")) { Write-Error "No flexmlrt directory after extraction"; exit 1 } + + - name: Setup FlexML, configure and build + shell: cmd + run: | + cd flexmlrt + call setup.bat + if errorlevel 1 ( echo ERROR: FlexML setup.bat failed & exit /b 1 ) + cd .. + cmake -B build -A x64 -DCMAKE_BUILD_TYPE=Release -DWHISPER_VITISAI=ON + if errorlevel 1 ( echo ERROR: CMake configure failed & exit /b 1 ) + cmake --build build --config Release -j + if errorlevel 1 ( echo ERROR: Build failed & exit /b 1 ) + + - name: Copy FlexML DLLs to build output + shell: powershell + run: | + foreach ($d in "flexmlrt/bin", "flexmlrt/lib") { + if (Test-Path "$d/*.dll") { Copy-Item "$d/*.dll" "build/bin/Release/" -Force } + } + if (-not (Test-Path "build/bin/Release/flexmlrt.dll")) { + Write-Error "flexmlrt.dll not staged next to binaries"; exit 1 + } + + - name: Download ggml model + shell: cmd + run: | + call models\download-ggml-model.cmd %MODEL% + if not exist models\ggml-%MODEL%.bin ( echo ERROR: model download failed & exit /b 1 ) + + - name: Download NPU encoder cache + shell: powershell + run: | + curl.exe -L --fail -o "models/ggml-$env:MODEL-encoder-vitisai.rai" ` + "https://huggingface.co/amd/whisper-$env:MODEL-onnx-npu/resolve/main/ggml-$env:MODEL-encoder-vitisai.rai" + $f = Get-Item "models/ggml-$env:MODEL-encoder-vitisai.rai" + Write-Host ".rai size: $([math]::Round($f.Length/1MB,2)) MB" + if ($f.Length -lt 1MB) { Write-Error ".rai suspiciously small - wrong URL or LFS pointer?"; exit 1 } + + - name: Run NPU smoke test + shell: cmd + run: | + build\bin\Release\whisper-cli.exe -m models\ggml-%MODEL%.bin -f samples\jfk.wav > vitisai.log 2>&1 + type vitisai.log + findstr /I /C:"vitisai" vitisai.log || ( echo ERROR: no VitisAI activity - encoder likely fell back to CPU & exit /b 1 ) + findstr /I /C:"ask not what your country" vitisai.log || ( echo ERROR: incorrect transcription & exit /b 1 ) + + - name: Upload smoke test log + if: always() + uses: actions/upload-artifact@v4 + with: + name: vitisai-smoke-log-windows + path: vitisai.log + + amd-npu-linux: + runs-on: [self-hosted, Linux, X64, stx, rai300-400] + timeout-minutes: 60 + continue-on-error: true # advisory while the runner pool is new; revisit later + + env: + FLEXML_LINUX_URL: # TODO: fill in Linux FlexML runtime URL + MODEL: base + + steps: + - name: Clone + uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + + - name: Install system deps + run: | + sudo apt-get update + sudo apt-get install -y build-essential cmake git \ + python3.12 python3.12-venv libboost-filesystem1.74.0 + + - name: Verify NPU device + run: | + lsmod | grep -q amdxdna || { echo "ERROR: amdxdna driver not loaded"; exit 1; } + ls /dev/accel/accel* || { echo "ERROR: no NPU accel device node"; exit 1; } + + - name: Download FlexML runtime (Linux) + run: | + curl -L --fail -o flexmlrt.tar.gz "$FLEXML_LINUX_URL" + tar xf flexmlrt.tar.gz + # TODO: add Linux FlexML environment setup (equivalent of Windows setup.bat) + echo "FlexmlRT_DIR=$PWD/flexmlrt/share/cmake/FlexmlRT" >> $GITHUB_ENV + echo "LD_LIBRARY_PATH=$PWD/flexmlrt/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV + + - name: Configure and build + run: | + cmake -B build -DCMAKE_BUILD_TYPE=Release -DWHISPER_VITISAI=ON + cmake --build build --config Release -j $(nproc) + + - name: Download ggml model + run: | + ./models/download-ggml-model.sh $MODEL + + - name: Download NPU encoder cache + run: | + curl -L --fail -o "models/ggml-$MODEL-encoder-vitisai.rai" \ + "https://huggingface.co/amd/whisper-$MODEL-onnx-npu/resolve/main/ggml-$MODEL-encoder-vitisai.rai" + [ $(stat -c%s "models/ggml-$MODEL-encoder-vitisai.rai") -gt 1000000 ] || { echo "ERROR: .rai too small"; exit 1; } + + - name: Run NPU smoke test + run: | + ./build/bin/whisper-cli -m "models/ggml-$MODEL.bin" -f samples/jfk.wav 2>&1 | tee vitisai.log + grep -qi "vitisai" vitisai.log || { echo "ERROR: no VitisAI activity - CPU fallback?"; exit 1; } + grep -qi "ask not what your country" vitisai.log || { echo "ERROR: incorrect transcription"; exit 1; } + + - name: Upload smoke test log + if: always() + uses: actions/upload-artifact@v4 + with: + name: vitisai-smoke-log-linux + path: vitisai.log From 240ed091d57d34d19247ddca7eba4b6e9621eb4f Mon Sep 17 00:00:00 2001 From: Iswarya Alex Date: Wed, 29 Jul 2026 22:02:11 -0700 Subject: [PATCH 13/24] Update runner --- .github/workflows/build-self-hosted.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-self-hosted.yml b/.github/workflows/build-self-hosted.yml index f9e8715f1..b9be60583 100644 --- a/.github/workflows/build-self-hosted.yml +++ b/.github/workflows/build-self-hosted.yml @@ -116,7 +116,7 @@ jobs: GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/whisper.cpp ~/mnt/whisper.cpp amd-npu-windows: - runs-on: [self-hosted, Windows, stx, rai300_400] + runs-on: [self-hosted, Windows, X64, stx, rai300-400] timeout-minutes: 60 continue-on-error: true # advisory while the runner pool is new; revisit later From 29bc8871fb40f31a45496dd51bd62df6cb13b2ad Mon Sep 17 00:00:00 2001 From: Iswarya Alex Date: Wed, 29 Jul 2026 22:07:11 -0700 Subject: [PATCH 14/24] Update workflow for linux --- .github/workflows/build-self-hosted.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build-self-hosted.yml b/.github/workflows/build-self-hosted.yml index b9be60583..dfafd4bd7 100644 --- a/.github/workflows/build-self-hosted.yml +++ b/.github/workflows/build-self-hosted.yml @@ -181,7 +181,9 @@ jobs: - name: Download ggml model shell: cmd run: | - call models\download-ggml-model.cmd %MODEL% + cd models + call download-ggml-model.cmd %MODEL% + cd .. if not exist models\ggml-%MODEL%.bin ( echo ERROR: model download failed & exit /b 1 ) - name: Download NPU encoder cache @@ -221,12 +223,6 @@ jobs: - name: Clone uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 - - name: Install system deps - run: | - sudo apt-get update - sudo apt-get install -y build-essential cmake git \ - python3.12 python3.12-venv libboost-filesystem1.74.0 - - name: Verify NPU device run: | lsmod | grep -q amdxdna || { echo "ERROR: amdxdna driver not loaded"; exit 1; } From aa1a5cc02a4c46a0dcfefc0541e9ad203f76a724 Mon Sep 17 00:00:00 2001 From: Iswarya Alex Date: Wed, 29 Jul 2026 22:08:14 -0700 Subject: [PATCH 15/24] Update workflow for linux --- .github/workflows/build-self-hosted.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/build-self-hosted.yml b/.github/workflows/build-self-hosted.yml index dfafd4bd7..43f52fe72 100644 --- a/.github/workflows/build-self-hosted.yml +++ b/.github/workflows/build-self-hosted.yml @@ -181,9 +181,7 @@ jobs: - name: Download ggml model shell: cmd run: | - cd models - call download-ggml-model.cmd %MODEL% - cd .. + call models\download-ggml-model.cmd %MODEL% models if not exist models\ggml-%MODEL%.bin ( echo ERROR: model download failed & exit /b 1 ) - name: Download NPU encoder cache From fae02367f1838e345be847383a9a32df73edf08f Mon Sep 17 00:00:00 2001 From: Iswarya Alex Date: Wed, 29 Jul 2026 22:51:34 -0700 Subject: [PATCH 16/24] Update flexmlrt packages for linux --- .github/workflows/build-self-hosted.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build-self-hosted.yml b/.github/workflows/build-self-hosted.yml index 43f52fe72..2b9311d46 100644 --- a/.github/workflows/build-self-hosted.yml +++ b/.github/workflows/build-self-hosted.yml @@ -121,7 +121,7 @@ jobs: continue-on-error: true # advisory while the runner pool is new; revisit later env: - FLEXML_URL: https://github.com/lemonade-sdk/whisper.cpp/releases/download/deps/flexmlrt1.7.0-win.zip + FLEXML_URL: https://github.com/lemonade-sdk/whisper.cpp-rocm/releases/download/deps/flexmlrt-1.7.0-win.zip MODEL: base steps: @@ -214,7 +214,7 @@ jobs: continue-on-error: true # advisory while the runner pool is new; revisit later env: - FLEXML_LINUX_URL: # TODO: fill in Linux FlexML runtime URL + FLEXML_LINUX_URL: https://github.com/lemonade-sdk/whisper.cpp-rocm/releases/download/deps/flexmlrt-1.8.0-linux.tar.gz MODEL: base steps: @@ -229,8 +229,9 @@ jobs: - name: Download FlexML runtime (Linux) run: | curl -L --fail -o flexmlrt.tar.gz "$FLEXML_LINUX_URL" - tar xf flexmlrt.tar.gz - # TODO: add Linux FlexML environment setup (equivalent of Windows setup.bat) + mkdir -p flexmlrt + tar xf flexmlrt.tar.gz -C flexmlrt + source flexmlrt/setup.sh echo "FlexmlRT_DIR=$PWD/flexmlrt/share/cmake/FlexmlRT" >> $GITHUB_ENV echo "LD_LIBRARY_PATH=$PWD/flexmlrt/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV From 622fe01bf2d7f3d191b2620e6db40a6745018758 Mon Sep 17 00:00:00 2001 From: Iswarya Alex Date: Wed, 29 Jul 2026 22:53:47 -0700 Subject: [PATCH 17/24] Update flexmlrt packages for linux --- .github/workflows/build-self-hosted.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/build-self-hosted.yml b/.github/workflows/build-self-hosted.yml index 2b9311d46..dd31b1d15 100644 --- a/.github/workflows/build-self-hosted.yml +++ b/.github/workflows/build-self-hosted.yml @@ -229,8 +229,7 @@ jobs: - name: Download FlexML runtime (Linux) run: | curl -L --fail -o flexmlrt.tar.gz "$FLEXML_LINUX_URL" - mkdir -p flexmlrt - tar xf flexmlrt.tar.gz -C flexmlrt + tar xf flexmlrt.tar.gz source flexmlrt/setup.sh echo "FlexmlRT_DIR=$PWD/flexmlrt/share/cmake/FlexmlRT" >> $GITHUB_ENV echo "LD_LIBRARY_PATH=$PWD/flexmlrt/lib:$LD_LIBRARY_PATH" >> $GITHUB_ENV From e3084ea0d4faa7f74698f2ab7b63510569a72f5f Mon Sep 17 00:00:00 2001 From: Iswarya Alex Date: Fri, 31 Jul 2026 14:27:50 -0700 Subject: [PATCH 18/24] Updated README --- .github/workflows/build-self-hosted.yml | 4 +- README.md | 91 +++++++++++++++++-------- 2 files changed, 64 insertions(+), 31 deletions(-) diff --git a/.github/workflows/build-self-hosted.yml b/.github/workflows/build-self-hosted.yml index dd31b1d15..9b4f5d294 100644 --- a/.github/workflows/build-self-hosted.yml +++ b/.github/workflows/build-self-hosted.yml @@ -115,7 +115,7 @@ jobs: vulkaninfo --summary GG_BUILD_VULKAN=1 bash ./ci/run.sh ~/results/whisper.cpp ~/mnt/whisper.cpp - amd-npu-windows: + npu-amd-windows: runs-on: [self-hosted, Windows, X64, stx, rai300-400] timeout-minutes: 60 continue-on-error: true # advisory while the runner pool is new; revisit later @@ -208,7 +208,7 @@ jobs: name: vitisai-smoke-log-windows path: vitisai.log - amd-npu-linux: + npu-amd-linux: runs-on: [self-hosted, Linux, X64, stx, rai300-400] timeout-minutes: 60 continue-on-error: true # advisory while the runner pool is new; revisit later diff --git a/README.md b/README.md index fd23e1c55..18f37d563 100644 --- a/README.md +++ b/README.md @@ -314,47 +314,80 @@ This can result in significant speedup in encoder performance. Here are the inst For more information about the OpenVINO implementation please refer to PR [#1037](https://github.com/ggml-org/whisper.cpp/pull/1037). -## AMD Ryzen™ AI support for NPU +## AMD Ryzen™ AI NPU support -On AMD's Ryzen™ AI 300 Series with dedicated NPUs for acceleration, you can now run Whisper models with the ability to fully offload the encoder to NPU. This brings significant speedup compared to CPU-only. -> **Note:** -> **Ryzen™ AI NPU acceleration is currently supported on Windows only.** Linux support is planned for upcoming releases. -> For the latest updates on Ryzen AI, check out [the official documentation](https://ryzenai.docs.amd.com/en/latest/). +On AMD Ryzen™ AI 300 and 400 Series processors with a dedicated NPU, whisper.cpp can fully offload the Whisper encoder to the NPU via VitisAI, delivering significant speedup over CPU-only inference. -### Setup environment (Windows only) +### Prerequisites - - Obtain the XRT package and the FlexmlRT package from AMD. Both are distributed as tarballs or wheels. - - Copy the downloaded archives to a local path, extract them, and run the setup script from each extracted package in your shell (for example `source /path/to/xrt/setup.sh` and `source /path/to/flexmlrt/setup.sh`). Run these in every new shell you use to build or run `whisper.cpp`. +Install the XRT runtime and FlexML runtime for your platform: -- Fetch the matching ggml model and prebuilt VitisAI encoder cache: +- **XRT**: provides the NPU kernel driver and `xrt-smi` diagnostic tool +- **FlexML runtime** (`flexmlrt`): VitisAI inference engine used by whisper.cpp — download from the [FlexML runtime releases](https://github.com/lemonade-sdk/whisper.cpp-rocm/releases/tag/deps) - ```bash - sh ./models/download-ggml-model.sh base - sh ./models/download-vitisai-model.sh base - ``` +After installing, source the setup scripts in every shell you use to build or run whisper.cpp: - ```cmd - .\models\download-ggml-model.cmd base - .\models\download-vitisai-model.cmd base - ``` +```bash +# Linux +source /opt/xilinx/xrt/setup.sh +source /path/to/flexmlrt/setup.sh +``` - Use the same model name with both scripts. The VitisAI script queries the AMD collection on Hugging Face to list available caches, then downloads the selected `.rai` file as `ggml--encoder-vitisai.rai` alongside the matching `ggml-.bin` file. You can also browse the collection manually at https://huggingface.co/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models. +```cmd +:: Windows +cd /path/to/flexmlrt && call setup.bat +``` - Depending on the downloaded `.rai` cache, VitisAI may offload either the encoder only or the encoder plus cross-projection layers to the AMD NPU. `whisper.cpp` detects the cache contents at runtime and logs the selected offload mode during model initialization. +You can verify the NPU is visible with: -- Build `whisper.cpp` with VitisAI support: +```bash +xrt-smi examine +``` - ```bash - cmake -B build -DWHISPER_VITISAI=1 - cmake --build build -j --config Release - ``` -Your environment is now ready. +### Download models -### Build Whisper.cpp for Ryzen™ AI support +Download the ggml model and the matching prebuilt VitisAI encoder cache: - ```text - $ ./build/bin/whisper-cli -m models/ggml-base.bin -f samples/jfk.wav - ``` +```bash +# Linux / macOS +sh ./models/download-ggml-model.sh base +sh ./models/download-vitisai-model.sh base +``` + +```cmd +:: Windows +.\models\download-ggml-model.cmd base +.\models\download-vitisai-model.cmd base +``` + +Use the same model name with both scripts. To see all available VitisAI encoder caches: + +```bash +sh ./models/download-vitisai-model.sh --list +``` + +```cmd +.\models\download-vitisai-model.cmd --list +``` + +The VitisAI script queries the [AMD Ryzen AI Whisper NPU collection on Hugging Face](https://huggingface.co/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models) and downloads the `.rai` encoder cache as `models/ggml--encoder-vitisai.rai`. + +> Depending on the `.rai` cache, VitisAI may offload the encoder only, or the encoder plus cross-projection layers. whisper.cpp detects this at runtime and logs the selected offload mode during model initialization. + +### Build + +```bash +cmake -B build -DWHISPER_VITISAI=1 +cmake --build build -j --config Release +``` + +### Run + +```bash +./build/bin/whisper-cli -m models/ggml-base.bin -f samples/jfk.wav +``` + +For more information see the [Ryzen AI documentation](https://ryzenai.docs.amd.com/en/latest/). ## NVIDIA GPU support From 2d5830d7911981b6f61b789e1b897900fa497f22 Mon Sep 17 00:00:00 2001 From: Iswarya Alex Date: Fri, 31 Jul 2026 15:13:26 -0700 Subject: [PATCH 19/24] readme: clarify xrt --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 18f37d563..3f80acbec 100644 --- a/README.md +++ b/README.md @@ -322,7 +322,7 @@ On AMD Ryzen™ AI 300 and 400 Series processors with a dedicated NPU, whisper.c Install the XRT runtime and FlexML runtime for your platform: -- **XRT**: provides the NPU kernel driver and `xrt-smi` diagnostic tool +- **XRT**: provides the NPU kernel driver and `xrt-smi` diagnostic tool — on Windows this is bundled with the NPU driver; on Linux install it separately following the [NPU driver installation guide](https://ryzenai.docs.amd.com/en/latest/linux.html#install-npu-drivers) - **FlexML runtime** (`flexmlrt`): VitisAI inference engine used by whisper.cpp — download from the [FlexML runtime releases](https://github.com/lemonade-sdk/whisper.cpp-rocm/releases/tag/deps) After installing, source the setup scripts in every shell you use to build or run whisper.cpp: From 7233ef5d4224766b1f9f35679a0f2d22711a9fc5 Mon Sep 17 00:00:00 2001 From: Iswarya Alex Date: Fri, 31 Jul 2026 15:16:17 -0700 Subject: [PATCH 20/24] readme: clarify xrt --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3f80acbec..9de9ed5ac 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ High-performance inference of [OpenAI's Whisper](https://github.com/openai/whisp - Support for CPU-only inference - [Efficient GPU support for NVIDIA](#nvidia-gpu-support) - [AMD ROCm GPU support](#amd-rocm-gpu-support) -- [AMD Ryzen AI NPU Support](#amd-ryzen-ai-support-for-npu) +- [AMD Ryzen AI NPU Support](#amd-ryzen-ai-npu-support) - [OpenVINO Support](#openvino-support) - [Ascend NPU Support](#ascend-npu-support) - [Moore Threads GPU Support](#moore-threads-gpu-support) From 7469ea099a009006ac30703977c4afe95802f451 Mon Sep 17 00:00:00 2001 From: Iswarya Alex Date: Fri, 31 Jul 2026 15:27:03 -0700 Subject: [PATCH 21/24] ci: update test config --- .github/workflows/build-self-hosted.yml | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build-self-hosted.yml b/.github/workflows/build-self-hosted.yml index 9b4f5d294..b689df084 100644 --- a/.github/workflows/build-self-hosted.yml +++ b/.github/workflows/build-self-hosted.yml @@ -185,13 +185,10 @@ jobs: if not exist models\ggml-%MODEL%.bin ( echo ERROR: model download failed & exit /b 1 ) - name: Download NPU encoder cache - shell: powershell + shell: cmd run: | - curl.exe -L --fail -o "models/ggml-$env:MODEL-encoder-vitisai.rai" ` - "https://huggingface.co/amd/whisper-$env:MODEL-onnx-npu/resolve/main/ggml-$env:MODEL-encoder-vitisai.rai" - $f = Get-Item "models/ggml-$env:MODEL-encoder-vitisai.rai" - Write-Host ".rai size: $([math]::Round($f.Length/1MB,2)) MB" - if ($f.Length -lt 1MB) { Write-Error ".rai suspiciously small - wrong URL or LFS pointer?"; exit 1 } + .\models\download-vitisai-model.cmd %MODEL% + if not exist models\ggml-%MODEL%-encoder-vitisai.rai ( echo ERROR: VitisAI encoder cache download failed & exit /b 1 ) - name: Run NPU smoke test shell: cmd @@ -245,9 +242,8 @@ jobs: - name: Download NPU encoder cache run: | - curl -L --fail -o "models/ggml-$MODEL-encoder-vitisai.rai" \ - "https://huggingface.co/amd/whisper-$MODEL-onnx-npu/resolve/main/ggml-$MODEL-encoder-vitisai.rai" - [ $(stat -c%s "models/ggml-$MODEL-encoder-vitisai.rai") -gt 1000000 ] || { echo "ERROR: .rai too small"; exit 1; } + sh ./models/download-vitisai-model.sh $MODEL + [ -f "models/ggml-$MODEL-encoder-vitisai.rai" ] || { echo "ERROR: VitisAI encoder cache download failed"; exit 1; } - name: Run NPU smoke test run: | From ee41f481ac03090c8485916d0cc79c23e8edce9e Mon Sep 17 00:00:00 2001 From: Sachin Kumawat Date: Wed, 5 Aug 2026 14:49:47 -0700 Subject: [PATCH 22/24] Added supported plarform details with python 3.12 requirement for Linux --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 9de9ed5ac..508fce785 100644 --- a/README.md +++ b/README.md @@ -320,6 +320,11 @@ On AMD Ryzen™ AI 300 and 400 Series processors with a dedicated NPU, whisper.c ### Prerequisites +Supported Platforms + +- **Windows 11** +- **Linux** (Ubuntu 24.04 LTS, Python 3.12) + Install the XRT runtime and FlexML runtime for your platform: - **XRT**: provides the NPU kernel driver and `xrt-smi` diagnostic tool — on Windows this is bundled with the NPU driver; on Linux install it separately following the [NPU driver installation guide](https://ryzenai.docs.amd.com/en/latest/linux.html#install-npu-drivers) From c801520de39d0cce293af6bc1a14c12713526e35 Mon Sep 17 00:00:00 2001 From: Sachin Kumawat Date: Fri, 7 Aug 2026 11:54:12 -0700 Subject: [PATCH 23/24] Use refactored helpers --- src/CMakeLists.txt | 2 + src/vitisai/whisper-vitisai-encoder.cpp | 674 ++++++++++++------------ 2 files changed, 340 insertions(+), 336 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 4dc39dbba..d4f9bfb51 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -160,6 +160,8 @@ if (WHISPER_VITISAI) set(TARGET whisper.vitisai) add_library(${TARGET} OBJECT + vitisai/whisper-vitisai-helpers.h + vitisai/whisper-vitisai-helpers.cpp vitisai/whisper-vitisai-encoder.h vitisai/whisper-vitisai-encoder.cpp ) diff --git a/src/vitisai/whisper-vitisai-encoder.cpp b/src/vitisai/whisper-vitisai-encoder.cpp index 9586c24a9..2a1b6d280 100644 --- a/src/vitisai/whisper-vitisai-encoder.cpp +++ b/src/vitisai/whisper-vitisai-encoder.cpp @@ -5,22 +5,13 @@ #endif #include "vitisai/whisper-vitisai-encoder.h" +#include "vitisai/whisper-vitisai-helpers.h" #include "FlexMLClient.h" #include "ggml.h" -#include "ggml-backend.h" #include #include -#ifdef _WIN32 - #include -#else - #include - #include - #include -#endif -#include #include -#include #include #include #include @@ -31,17 +22,6 @@ #define WHISPER_DBG_TIMER(name) do {} while (0) #endif -#if defined(WHISPER_DEBUG) -template -static void whisper_vitisai_print_shape(const std::vector & shape) { - std::fprintf(stderr, "["); - for (size_t i = 0; i < shape.size(); ++i) { - std::fprintf(stderr, "%s%lld", i == 0 ? "" : ", ", (long long) shape[i]); - } - std::fprintf(stderr, "]"); -} -#endif - struct whisper_vitisai_context { std::string model_path; std::shared_ptr runner; @@ -51,104 +31,24 @@ struct whisper_vitisai_context { std::vector cross_k_staging; std::vector cross_v_staging; + int mel_in_idx = -1; int embd_enc_out_idx = -1; int cross_k_out_idx = -1; int cross_v_out_idx = -1; + size_t mel_in_expected_bytes = 0; + size_t embd_enc_expected_bytes = 0; + size_t cross_k_expected_bytes = 0; + size_t cross_v_expected_bytes = 0; std::vector cached_input_tensors; std::vector cached_output_tensors; }; -// Function to mmap rai file for Linux and MapViewOfFile for Windows -static bool map_rai_file(const char * path, uint8_t ** buffer, size_t * size) { -#ifdef _WIN32 - // Open the file - HANDLE hFile = CreateFileA(path, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL); - if (hFile == INVALID_HANDLE_VALUE) { - std::fprintf(stderr, "%s: %d: Failed to open rai file '%s'\n", __func__, __LINE__, path); - return false; - } - - // Get the file size - LARGE_INTEGER fileSize; - if (!GetFileSizeEx(hFile, &fileSize)) { - CloseHandle(hFile); - std::fprintf(stderr, "%s: %d: Failed to get file size for rai file '%s'\n", __func__, __LINE__, path); - return false; - } - - // Create a file mapping object - HANDLE hMapping = CreateFileMappingA(hFile, NULL, PAGE_READONLY, 0, fileSize.QuadPart, NULL); - if (hMapping == NULL) { - CloseHandle(hFile); - std::fprintf(stderr, "%s: %d: Failed to create file mapping for rai file '%s'\n", __func__, __LINE__, path); - return false; - } - - // Map the file - *buffer = (uint8_t *)MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, fileSize.QuadPart); - if (*buffer == NULL) { - CloseHandle(hMapping); - CloseHandle(hFile); - std::fprintf(stderr, "%s: %d: Failed to map rai file '%s'\n", __func__, __LINE__, path); - return false; - } - *size = fileSize.QuadPart; - return true; -#else - // Open the file - FILE * fd = fopen(path, "rb"); - if (!fd) { - std::fprintf(stderr, "%s: %d: Failed to open rai file '%s'\n", __func__, __LINE__, path); - return false; - } - - // Get the file size - struct stat st; - if (fstat(fileno(fd), &st) == -1) { - fclose(fd); - std::fprintf(stderr, "%s: %d: Failed to get file size for rai file '%s'\n", __func__, __LINE__, path); - return false; - } - - // Mmap the file - *buffer = (uint8_t *)mmap(nullptr, st.st_size, PROT_READ, MAP_SHARED, fileno(fd), 0); - if (*buffer == MAP_FAILED) { - fclose(fd); - std::fprintf(stderr, "%s: %d: Failed to mmap rai file '%s'\n", __func__, __LINE__, path); - return false; - } - *size = st.st_size; - return true; -#endif // _WIN32 -} - -static void unmap_rai_file(uint8_t * buffer, size_t size) { -#ifdef _WIN32 - UnmapViewOfFile(buffer); -#else - munmap(buffer, size); -#endif // _WIN32 -} - -bool whisper_vitisai_file_exists(const char * path) { - if (!path) { - return false; - } - - FILE * file = fopen(path, "rb"); - if (!file) { - return false; - } - fclose(file); - return true; -} - -// Reuse cached tensor descriptors to avoid repeated getIOTensors() lookups. -static bool whisper_vitisai_get_io_tensors( +// Return cached IO tensor descriptors by reference to avoid per-call deep copies. +static bool whisper_vitisai_get_cached_io_tensors( struct whisper_vitisai_context * ctx, - std::vector & input_tensors, - std::vector & output_tensors) { + std::vector *& input_tensors, + std::vector *& output_tensors) { if (!ctx || !ctx->runner) { return false; } @@ -158,8 +58,8 @@ static bool whisper_vitisai_get_io_tensors( ctx->cached_output_tensors = ctx->runner->getIOTensors("output", false); } - input_tensors = ctx->cached_input_tensors; - output_tensors = ctx->cached_output_tensors; + input_tensors = &ctx->cached_input_tensors; + output_tensors = &ctx->cached_output_tensors; return true; } @@ -188,7 +88,7 @@ struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model) { // Check if model_path is rai file and if so, add fbs_buffer and fbs_buffer_size to the options if (ctx->model_path.find(".rai") != std::string::npos) { - if (map_rai_file(ctx->model_path.c_str(), &ctx->fbs_buffer, &ctx->fbs_buffer_size)) { + if (whisper_vitisai_helpers::map_rai_file(ctx->model_path.c_str(), &ctx->fbs_buffer, &ctx->fbs_buffer_size)) { options.extOptions["fbs_buffer"] = ctx->fbs_buffer; options.extOptions["fbs_buffer_size"] = ctx->fbs_buffer_size; options.extOptions["cache_dir"] = std::string("."); @@ -219,41 +119,40 @@ struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model) { try { ctx->runner = std::make_shared(options); - - if (!ctx->runner->good()) { + if (!ctx->runner || !ctx->runner->good()) { throw std::runtime_error("Runner creation ran into an error"); } ctx->cached_input_tensors = ctx->runner->getIOTensors("input", false); ctx->cached_output_tensors = ctx->runner->getIOTensors("output", false); + auto & input_tensors = ctx->cached_input_tensors; auto & output_tensors = ctx->cached_output_tensors; - for (int i = 0; i < (int) output_tensors.size(); ++i) { - const std::string & name = output_tensors[i].getMetadata().name; - if (name == "embd_enc") { - ctx->embd_enc_out_idx = i; - } else if (name == "cross_k") { - ctx->cross_k_out_idx = i; - } else if (name == "cross_v") { - ctx->cross_v_out_idx = i; - } + + whisper_vitisai_helpers::whisper_vitisai_io_binding binding; + std::string binding_error; + if (!whisper_vitisai_helpers::whisper_vitisai_resolve_io_binding( + __func__, input_tensors, output_tensors, &binding, &binding_error)) { + throw std::runtime_error(binding_error); } - if (ctx->embd_enc_out_idx < 0) { - std::fprintf(stderr, "%s: WARNING: embd_enc output not found by name; falling back to output[0]\n", __func__); - ctx->embd_enc_out_idx = 0; - } + ctx->mel_in_idx = binding.mel_in_idx; + ctx->embd_enc_out_idx = binding.embd_enc_out_idx; + ctx->cross_k_out_idx = binding.cross_k_out_idx; + ctx->cross_v_out_idx = binding.cross_v_out_idx; + ctx->mel_in_expected_bytes = binding.mel_in_expected_bytes; + ctx->embd_enc_expected_bytes = binding.embd_enc_expected_bytes; + ctx->cross_k_expected_bytes = binding.cross_k_expected_bytes; + ctx->cross_v_expected_bytes = binding.cross_v_expected_bytes; #if defined(WHISPER_DEBUG) { - auto & input_tensors = ctx->cached_input_tensors; - std::fprintf(stderr, "%s: model has %zu input tensor(s)\n", __func__, input_tensors.size()); for (int i = 0; i < (int) input_tensors.size(); ++i) { const auto & meta = input_tensors[i].getMetadata(); std::fprintf(stderr, "%s: input[%d] name='%s' size=%zu shape=", __func__, i, meta.name.c_str(), (size_t) meta.size); - whisper_vitisai_print_shape(meta.shape); + whisper_vitisai_helpers::whisper_vitisai_print_shape(meta.shape); std::fprintf(stderr, "\n"); } @@ -262,17 +161,18 @@ struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model) { const auto & meta = output_tensors[i].getMetadata(); std::fprintf(stderr, "%s: output[%d] name='%s' size=%zu shape=", __func__, i, meta.name.c_str(), (size_t) meta.size); - whisper_vitisai_print_shape(meta.shape); + whisper_vitisai_helpers::whisper_vitisai_print_shape(meta.shape); std::fprintf(stderr, "\n"); } + std::fprintf(stderr, "%s: input index: mel=%d\n", __func__, ctx->mel_in_idx); std::fprintf(stderr, "%s: output indices: embd_enc=%d cross_k=%d cross_v=%d\n", __func__, ctx->embd_enc_out_idx, ctx->cross_k_out_idx, ctx->cross_v_out_idx); } #endif } catch (const std::exception & e) { std::fprintf(stderr, "%s: Exception during Vitis AI runner creation: %s\n", __func__, e.what()); - delete ctx; + whisper_vitisai_free(ctx); return nullptr; } return ctx; @@ -282,6 +182,10 @@ bool whisper_vitisai_has_cross_proj(const struct whisper_vitisai_context * ctx) return ctx && ctx->cross_k_out_idx >= 0 && ctx->cross_v_out_idx >= 0; } +bool whisper_vitisai_file_exists(const char * path) { + return whisper_vitisai_helpers::file_exists(path); +} + void whisper_vitisai_free(struct whisper_vitisai_context * ctx) { if (!ctx) { return; @@ -291,17 +195,30 @@ void whisper_vitisai_free(struct whisper_vitisai_context * ctx) { std::fprintf(stderr, "%s: releasing Vitis AI context for model '%s'\n", __func__, ctx->model_path.c_str()); #endif if (ctx->fbs_buffer) { - unmap_rai_file(ctx->fbs_buffer, ctx->fbs_buffer_size); + whisper_vitisai_helpers::unmap_rai_file(ctx->fbs_buffer, ctx->fbs_buffer_size); } delete ctx; } -int whisper_vitisai_encode(struct whisper_vitisai_context * ctx, struct ggml_tensor * mel, struct ggml_tensor * out) { +static int whisper_vitisai_forward_impl( + struct whisper_vitisai_context * ctx, + struct ggml_tensor * mel, + struct ggml_tensor * out, + std::vector & input_tensors, + std::vector & output_tensors, + void * cross_k_data, + void * cross_v_data) { if (!ctx || !mel || !out) { std::fprintf(stderr, "%s: ctx/mel/out must not be null\n", __func__); return 0; } + const bool with_cross = (cross_k_data != nullptr || cross_v_data != nullptr); + if (with_cross && (!cross_k_data || !cross_v_data)) { + std::fprintf(stderr, "%s: cross_k_data/cross_v_data must both be set\n", __func__); + return 0; + } + if (ggml_n_dims(mel) != 2) { std::fprintf(stderr, "%s: mel tensor expected to have 2 dims, got %d\n", __func__, ggml_n_dims(mel)); return 0; @@ -312,32 +229,78 @@ int whisper_vitisai_encode(struct whisper_vitisai_context * ctx, struct ggml_ten return 0; } - // setup input and output tensors for Vitis AI model - std::vector input_tensors, output_tensors; - auto model = ctx->runner; - - if (!whisper_vitisai_get_io_tensors(ctx, input_tensors, output_tensors)) { - std::fprintf(stderr, "%s: failed to acquire Vitis AI I/O tensors\n", __func__); - return 0; - } - - // TODO: add assert checks for tensor numbers and shapes - if (ctx->embd_enc_out_idx < 0 || ctx->embd_enc_out_idx >= (int) output_tensors.size()) { std::fprintf(stderr, "%s: invalid embd_enc output index %d for %zu output tensor(s)\n", __func__, ctx->embd_enc_out_idx, output_tensors.size()); return 0; } - input_tensors[0].data = mel->data; - output_tensors[ctx->embd_enc_out_idx].data = out->data; + if (ctx->mel_in_idx < 0 || ctx->mel_in_idx >= (int) input_tensors.size()) { + std::fprintf(stderr, "%s: invalid mel input index %d for %zu input tensor(s)\n", + __func__, ctx->mel_in_idx, input_tensors.size()); + return 0; + } + + if (!whisper_vitisai_helpers::whisper_vitisai_bind_tensor_data( + "mel input", + mel, + { (size_t) mel->ne[1], (size_t) mel->ne[0] }, + input_tensors[ctx->mel_in_idx])) { + return 0; + } + + if (!whisper_vitisai_helpers::whisper_vitisai_bind_tensor_data( + "embd_enc output", + out, + { (size_t) out->ne[1], (size_t) out->ne[0] }, + output_tensors[ctx->embd_enc_out_idx])) { + return 0; + } + + std::vector claimed_inputs(input_tensors.size(), false); + claimed_inputs[ctx->mel_in_idx] = true; + if (!whisper_vitisai_helpers::whisper_vitisai_all_tensors_claimed( + __func__, "input", input_tensors, claimed_inputs)) { + return 0; + } + + std::vector claimed_outputs(output_tensors.size(), false); + claimed_outputs[ctx->embd_enc_out_idx] = true; + if (with_cross) { + if (ctx->cross_k_out_idx < 0 || ctx->cross_k_out_idx >= (int) output_tensors.size() || + ctx->cross_v_out_idx < 0 || ctx->cross_v_out_idx >= (int) output_tensors.size()) { + std::fprintf(stderr, "%s: invalid cross output indices cross_k=%d cross_v=%d for %zu output tensor(s)\n", + __func__, ctx->cross_k_out_idx, ctx->cross_v_out_idx, output_tensors.size()); + return 0; + } + output_tensors[ctx->cross_k_out_idx].data = cross_k_data; + output_tensors[ctx->cross_v_out_idx].data = cross_v_data; + claimed_outputs[ctx->cross_k_out_idx] = true; + claimed_outputs[ctx->cross_v_out_idx] = true; + } + if (!whisper_vitisai_helpers::whisper_vitisai_all_tensors_claimed( + __func__, "output", output_tensors, claimed_outputs)) { + return 0; + } + + auto clear_bound_data = [&]() { + input_tensors[ctx->mel_in_idx].data = nullptr; + output_tensors[ctx->embd_enc_out_idx].data = nullptr; + if (with_cross) { + output_tensors[ctx->cross_k_out_idx].data = nullptr; + output_tensors[ctx->cross_v_out_idx].data = nullptr; + } + }; try { - model->forward(input_tensors, output_tensors); + ctx->runner->forward(input_tensors, output_tensors); + clear_bound_data(); #if defined(WHISPER_DEBUG) - std::fprintf(stderr, "%s: Vitis AI model inference completed.\n", __func__); + std::fprintf(stderr, "%s: Vitis AI model inference %scompleted.\n", + __func__, with_cross ? "(encoder + cross proj) " : ""); #endif } catch (const std::exception & e) { + clear_bound_data(); std::fprintf(stderr, "%s: Exception during model inference: %s\n", __func__, e.what()); return 0; } @@ -345,74 +308,68 @@ int whisper_vitisai_encode(struct whisper_vitisai_context * ctx, struct ggml_ten return 1; } +int whisper_vitisai_encode(struct whisper_vitisai_context * ctx, struct ggml_tensor * mel, struct ggml_tensor * out) { + std::vector * input_tensors_cached = nullptr; + std::vector * output_tensors_cached = nullptr; + if (!whisper_vitisai_get_cached_io_tensors(ctx, input_tensors_cached, output_tensors_cached)) { + std::fprintf(stderr, "%s: failed to acquire Vitis AI I/O tensors\n", __func__); + return 0; + } + + std::vector input_tensors = *input_tensors_cached; + std::vector output_tensors = *output_tensors_cached; + + return whisper_vitisai_forward_impl( + ctx, + mel, + out, + input_tensors, + output_tensors, + nullptr, + nullptr); +} + int whisper_vitisai_run_enc_cross( struct whisper_vitisai_context * ctx, struct ggml_tensor * mel, struct ggml_tensor * out, void * cross_v_data, void * cross_k_data) { - if (!ctx || !mel || !out || !cross_v_data || !cross_k_data) { - std::fprintf(stderr, "%s: ctx/mel/out/cross_v_data/cross_k_data must not be null\n", __func__); + if (!cross_v_data || !cross_k_data) { + std::fprintf(stderr, "%s: cross_v_data/cross_k_data must not be null\n", __func__); return 0; } - if (ggml_n_dims(mel) != 2) { - std::fprintf(stderr, "%s: mel tensor expected to have 2 dims, got %d\n", __func__, ggml_n_dims(mel)); - return 0; - } - - if (ggml_n_dims(out) != 2) { - std::fprintf(stderr, "%s: out tensor expected to have 2 dims, got %d\n", __func__, ggml_n_dims(out)); - return 0; - } - - std::vector input_tensors, output_tensors; - auto model = ctx->runner; - - if (!whisper_vitisai_get_io_tensors(ctx, input_tensors, output_tensors)) { + std::vector * input_tensors_cached = nullptr; + std::vector * output_tensors_cached = nullptr; + if (!whisper_vitisai_get_cached_io_tensors(ctx, input_tensors_cached, output_tensors_cached)) { std::fprintf(stderr, "%s: failed to acquire Vitis AI I/O tensors\n", __func__); return 0; } - if (output_tensors.size() != 3) { - std::fprintf(stderr, "%s: expected 3 output tensors, got %zu\n", __func__, output_tensors.size()); - return 0; - } + std::vector input_tensors = *input_tensors_cached; + std::vector output_tensors = *output_tensors_cached; - if (ctx->embd_enc_out_idx < 0 || ctx->embd_enc_out_idx >= (int) output_tensors.size() || - ctx->cross_k_out_idx < 0 || ctx->cross_k_out_idx >= (int) output_tensors.size() || - ctx->cross_v_out_idx < 0 || ctx->cross_v_out_idx >= (int) output_tensors.size()) { - std::fprintf(stderr, "%s: invalid output indices embd_enc=%d cross_k=%d cross_v=%d for %zu output tensor(s)\n", - __func__, ctx->embd_enc_out_idx, ctx->cross_k_out_idx, ctx->cross_v_out_idx, output_tensors.size()); - return 0; - } - - input_tensors[0].data = mel->data; - output_tensors[ctx->embd_enc_out_idx].data = out->data; - output_tensors[ctx->cross_v_out_idx].data = cross_v_data; - output_tensors[ctx->cross_k_out_idx].data = cross_k_data; - - try { - model->forward(input_tensors, output_tensors); -#if defined(WHISPER_DEBUG) - std::fprintf(stderr, "%s: Vitis AI model inference (encoder + cross proj) completed.\n", __func__); -#endif - } catch (const std::exception & e) { - std::fprintf(stderr, "%s: Exception during model inference: %s\n", __func__, e.what()); - return 0; - } - - return 1; + return whisper_vitisai_forward_impl( + ctx, + mel, + out, + input_tensors, + output_tensors, + cross_k_data, + cross_v_data); } // Ensure persistent staging buffers are large enough for the given dimensions. static void ensure_staging_buffers( struct whisper_vitisai_context * ctx, - size_t count, bool need_k) { + size_t count, + bool need_k, + bool need_v) { if (need_k && ctx->cross_k_staging.size() < count) { ctx->cross_k_staging.resize(count); } - if (ctx->cross_v_staging.size() < count) { + if (need_v && ctx->cross_v_staging.size() < count) { ctx->cross_v_staging.resize(count); } } @@ -429,7 +386,27 @@ int whisper_vitisai_encode_with_cross( int n_text_head, bool flash_attn) { if (!ctx || !mel || !embd_enc || !kv_cross_k || !kv_cross_v) { - std::fprintf(stderr, "%s: null argument\n", __func__); + std::fprintf(stderr, "%s: ctx/mel/embd_enc/kv_cross_k/kv_cross_v must not be null\n", __func__); + return 0; + } + + if (n_text_layer <= 0 || n_ctx <= 0 || n_text_state <= 0 || n_text_head <= 0) { + std::fprintf(stderr, "%s: invalid shape parameters layer=%d ctx=%d state=%d head=%d\n", + __func__, n_text_layer, n_ctx, n_text_state, n_text_head); + return 0; + } + + if ((n_text_state % n_text_head) != 0) { + std::fprintf(stderr, "%s: invalid head configuration state=%d head=%d\n", + __func__, n_text_state, n_text_head); + return 0; + } + + if (kv_cross_k->type != kv_cross_v->type) { + std::fprintf(stderr, "%s: kv_cross type mismatch k=%s v=%s\n", + __func__, + whisper_vitisai_helpers::whisper_kv_type_name(kv_cross_k->type), + whisper_vitisai_helpers::whisper_kv_type_name(kv_cross_v->type)); return 0; } @@ -439,163 +416,188 @@ int whisper_vitisai_encode_with_cross( const float Kscale = pow(float(n_state_head), -0.25f); const ggml_type kv_type = kv_cross_k->type; + const bool kv_is_f32 = kv_type == GGML_TYPE_F32; + const bool kv_is_f16 = kv_type == GGML_TYPE_F16; + if (!kv_is_f32 && !kv_is_f16) { + std::fprintf(stderr, "%s: unsupported kv_cross tensor type '%s'\n", + __func__, whisper_vitisai_helpers::whisper_kv_type_name(kv_type)); + return 0; + } + const size_t elem_size = ggml_type_size(kv_type); - const size_t layer_elems = (size_t)n_ctx * n_state; - const size_t buf_count = (size_t)n_text_layer * layer_elems; + const size_t req_layer_elems = (size_t)n_ctx * (size_t)n_state; + + std::vector * input_tensors_cached = nullptr; + std::vector * output_tensors_cached = nullptr; + if (!whisper_vitisai_get_cached_io_tensors(ctx, input_tensors_cached, output_tensors_cached)) { + std::fprintf(stderr, "%s: failed to acquire Vitis AI I/O tensors\n", __func__); + return 0; + } + std::vector input_tensors = *input_tensors_cached; + std::vector output_tensors = *output_tensors_cached; + + if (ctx->cross_k_out_idx < 0 || ctx->cross_k_out_idx >= (int) output_tensors.size() || + ctx->cross_v_out_idx < 0 || ctx->cross_v_out_idx >= (int) output_tensors.size()) { + std::fprintf(stderr, "%s: invalid cross output indices cross_k=%d cross_v=%d for %zu output tensor(s)\n", + __func__, ctx->cross_k_out_idx, ctx->cross_v_out_idx, output_tensors.size()); + return 0; + } + + const auto & cross_k_meta = output_tensors[ctx->cross_k_out_idx].getMetadata(); + const auto & cross_v_meta = output_tensors[ctx->cross_v_out_idx].getMetadata(); + if (!whisper_vitisai_helpers::whisper_validate_cross_shape("cross_k", cross_k_meta.shape, n_text_layer, n_ctx, n_state) || + !whisper_vitisai_helpers::whisper_validate_cross_shape("cross_v", cross_v_meta.shape, n_text_layer, n_ctx, n_state)) { + return 0; + } + + if (ctx->cross_k_expected_bytes == 0 || ctx->cross_v_expected_bytes == 0) { + std::fprintf(stderr, "%s: missing cross output metadata sizes\n", __func__); + return 0; + } + if (ctx->cross_k_expected_bytes != ctx->cross_v_expected_bytes) { + std::fprintf(stderr, "%s: cross output metadata size mismatch k=%zu v=%zu\n", + __func__, ctx->cross_k_expected_bytes, ctx->cross_v_expected_bytes); + return 0; + } + const size_t expected_cross_bytes = (size_t) n_text_layer * req_layer_elems * sizeof(float); + if (ctx->cross_k_expected_bytes != expected_cross_bytes) { + std::fprintf(stderr, + "%s: cross output size mismatch (model=%zu B, expected=%zu B for layer=%d ctx=%d state=%d)\n", + __func__, ctx->cross_k_expected_bytes, expected_cross_bytes, n_text_layer, n_ctx, n_state); + return 0; + } + + const size_t model_total_elems = ctx->cross_k_expected_bytes / sizeof(float); + const size_t model_layer_elems = req_layer_elems; + + const size_t required_kv_bytes = flash_attn + ? (size_t)n_text_layer * elem_size * (size_t)n_state * (size_t)n_ctx_pad + : (size_t)n_text_layer * elem_size * req_layer_elems; + if (ggml_nbytes(kv_cross_k) < required_kv_bytes || ggml_nbytes(kv_cross_v) < required_kv_bytes) { + std::fprintf(stderr, + "%s: kv_cross buffers are too small (required=%zu B, k=%zu B, v=%zu B)\n", + __func__, required_kv_bytes, ggml_nbytes(kv_cross_k), ggml_nbytes(kv_cross_v)); + return 0; + } + + const bool direct_k_to_kv = kv_is_f32 && (!flash_attn || n_ctx_pad == n_ctx); + const bool direct_v_to_kv = kv_is_f32 && flash_attn && (n_ctx_pad == n_ctx); + const bool need_k_staging = !direct_k_to_kv; + const bool need_v_staging = !direct_v_to_kv; + + if (need_k_staging || need_v_staging) { + ensure_staging_buffers(ctx, model_total_elems, need_k_staging, need_v_staging); + } + + void * cross_k_out = direct_k_to_kv + ? kv_cross_k->data + : (void *) ctx->cross_k_staging.data(); + void * cross_v_out = direct_v_to_kv + ? kv_cross_v->data + : (void *) ctx->cross_v_staging.data(); + + whisper_vitisai_helpers::whisper_kv_cross_layout kv_layout; + kv_layout.n_layer = n_text_layer; + kv_layout.n_ctx = n_ctx; + kv_layout.n_state = n_state; + kv_layout.src_layer_elems = model_layer_elems; + kv_layout.layer_elems = req_layer_elems; + kv_layout.kscale = Kscale; if (flash_attn) { WHISPER_DBG_TIMER(t_fwd_start); - - if (n_ctx_pad == n_ctx) { - // No padding gap -- plugin writes directly into kv_cross. - if (!whisper_vitisai_run_enc_cross( - ctx, mel, embd_enc, - kv_cross_v->data, kv_cross_k->data)) { - return 0; - } - - WHISPER_DBG_TIMER(t_fwd_end); - WHISPER_DBG_TIMER(t_post_start); - - if (kv_type == GGML_TYPE_F32) { - float * kdata = (float *)kv_cross_k->data; - for (size_t i = 0; i < buf_count; ++i) { - kdata[i] *= Kscale; - } - } else if (kv_type == GGML_TYPE_F16) { - ggml_fp16_t * kdata = (ggml_fp16_t *)kv_cross_k->data; - for (size_t i = 0; i < buf_count; ++i) { - kdata[i] = ggml_fp32_to_fp16(ggml_fp16_to_fp32(kdata[i]) * Kscale); - } - } - - WHISPER_DBG_TIMER(t_post_end); - -#if defined(WHISPER_DEBUG) - std::fprintf(stderr, "%s: vitisai enc+cross forward time = %8.2f ms\n", __func__, (t_fwd_end - t_fwd_start) / 1000.0f); - std::fprintf(stderr, "%s: kv_cross post-process time = %8.2f ms (flash, no-pad direct)\n", __func__, (t_post_end - t_post_start) / 1000.0f); -#endif - } else { - // Padding gap -- use persistent staging buffers. - ensure_staging_buffers(ctx, buf_count, true); - float * cross_k_buf = ctx->cross_k_staging.data(); - float * cross_v_buf = ctx->cross_v_staging.data(); - - if (!whisper_vitisai_run_enc_cross( - ctx, mel, embd_enc, - cross_v_buf, cross_k_buf)) { - return 0; - } - - WHISPER_DBG_TIMER(t_fwd_end); - WHISPER_DBG_TIMER(t_post_start); - - // Combined per-layer K+V scatter for better cache locality. - const size_t padded_layer_stride = elem_size * n_state * n_ctx_pad; - - for (int il = 0; il < n_text_layer; ++il) { - const float * src_k = cross_k_buf + (size_t)il * layer_elems; - const float * src_v = cross_v_buf + (size_t)il * layer_elems; - uint8_t * dst_k = (uint8_t *)kv_cross_k->data + padded_layer_stride * il; - uint8_t * dst_v = (uint8_t *)kv_cross_v->data + padded_layer_stride * il; - - if (kv_type == GGML_TYPE_F32) { - float * dk = (float *)dst_k; - for (size_t i = 0; i < layer_elems; ++i) { - dk[i] = src_k[i] * Kscale; - } - memcpy(dst_v, src_v, layer_elems * sizeof(float)); - } else if (kv_type == GGML_TYPE_F16) { - ggml_fp16_t * dk = (ggml_fp16_t *)dst_k; - ggml_fp16_t * dv = (ggml_fp16_t *)dst_v; - for (size_t i = 0; i < layer_elems; ++i) { - dk[i] = ggml_fp32_to_fp16(src_k[i] * Kscale); - dv[i] = ggml_fp32_to_fp16(src_v[i]); - } - } - } - - WHISPER_DBG_TIMER(t_post_end); - -#if defined(WHISPER_DEBUG) - std::fprintf(stderr, "%s: vitisai enc+cross forward time = %8.2f ms\n", __func__, (t_fwd_end - t_fwd_start) / 1000.0f); - std::fprintf(stderr, "%s: kv_cross post-process time = %8.2f ms (flash, padded, n_ctx=%d, n_ctx_pad=%d, kv_type=%s)\n", - __func__, (t_post_end - t_post_start) / 1000.0f, - n_ctx, n_ctx_pad, - kv_type == GGML_TYPE_F32 ? "F32" : kv_type == GGML_TYPE_F16 ? "F16" : "other"); -#endif - } - } else { - // Non-flash: layers are contiguous (stride = n_state * n_ctx). - // K: plugin writes directly into kv_cross_k, then in-place Kscale. - // V: persistent staging buffer + cache-friendly blocked transpose. - ensure_staging_buffers(ctx, buf_count, false); - float * cross_v_buf = ctx->cross_v_staging.data(); - - WHISPER_DBG_TIMER(t_fwd_start); - - if (!whisper_vitisai_run_enc_cross( - ctx, mel, embd_enc, - cross_v_buf, kv_cross_k->data)) { + if (!whisper_vitisai_forward_impl( + ctx, mel, embd_enc, input_tensors, output_tensors, cross_k_out, cross_v_out)) { return 0; } WHISPER_DBG_TIMER(t_fwd_end); WHISPER_DBG_TIMER(t_post_start); - if (kv_type == GGML_TYPE_F32) { - float * kdata = (float *)kv_cross_k->data; - for (size_t i = 0; i < buf_count; ++i) { - kdata[i] *= Kscale; + if (n_ctx_pad == n_ctx) { + kv_layout.dst_layer_stride = req_layer_elems * elem_size; + if (kv_is_f32) { + // V was written straight into the kv cache by the runtime; only K needs scaling. + whisper_vitisai_helpers::whisper_kv_cross_scale_k_f32( + (float *)kv_cross_k->data, + (size_t) n_text_layer * req_layer_elems, + Kscale); + } else { // kv_is_f16 + whisper_vitisai_helpers::whisper_kv_cross_store_layers_f16( + ctx->cross_k_staging.data(), + ctx->cross_v_staging.data(), + (uint8_t *)kv_cross_k->data, + (uint8_t *)kv_cross_v->data, + kv_layout); } - - const int BLOCK = 32; - for (int il = 0; il < n_text_layer; ++il) { - const float * src_v = cross_v_buf + (size_t)il * layer_elems; - float * dst_v = (float *)kv_cross_v->data + (size_t)il * layer_elems; - - for (int ic = 0; ic < n_ctx; ic += BLOCK) { - for (int is = 0; is < n_state; is += BLOCK) { - const int ic_end = std::min(ic + BLOCK, n_ctx); - const int is_end = std::min(is + BLOCK, n_state); - for (int i = ic; i < ic_end; ++i) { - for (int j = is; j < is_end; ++j) { - dst_v[j * n_ctx + i] = src_v[i * n_state + j]; - } - } - } - } - } - } else if (kv_type == GGML_TYPE_F16) { - ggml_fp16_t * kdata = (ggml_fp16_t *)kv_cross_k->data; - for (size_t i = 0; i < buf_count; ++i) { - kdata[i] = ggml_fp32_to_fp16(ggml_fp16_to_fp32(kdata[i]) * Kscale); - } - - const int BLOCK = 32; - for (int il = 0; il < n_text_layer; ++il) { - const float * src_v = cross_v_buf + (size_t)il * layer_elems; - ggml_fp16_t * dst_v = (ggml_fp16_t *)((uint8_t *)kv_cross_v->data + elem_size * n_state * n_ctx * il); - - for (int ic = 0; ic < n_ctx; ic += BLOCK) { - for (int is = 0; is < n_state; is += BLOCK) { - const int ic_end = std::min(ic + BLOCK, n_ctx); - const int is_end = std::min(is + BLOCK, n_state); - for (int i = ic; i < ic_end; ++i) { - for (int j = is; j < is_end; ++j) { - dst_v[j * n_ctx + i] = ggml_fp32_to_fp16(src_v[i * n_state + j]); - } - } - } - } + } else { + // Runtime decoder uses padded K/V cache. Copy only requested context, leave the pad tail untouched. + kv_layout.dst_layer_stride = elem_size * (size_t)n_state * (size_t)n_ctx_pad; + if (kv_is_f32) { + whisper_vitisai_helpers::whisper_kv_cross_store_layers_f32( + ctx->cross_k_staging.data(), + ctx->cross_v_staging.data(), + (uint8_t *)kv_cross_k->data, + (uint8_t *)kv_cross_v->data, + kv_layout); + } else { // kv_is_f16 + whisper_vitisai_helpers::whisper_kv_cross_store_layers_f16( + ctx->cross_k_staging.data(), + ctx->cross_v_staging.data(), + (uint8_t *)kv_cross_k->data, + (uint8_t *)kv_cross_v->data, + kv_layout); } } WHISPER_DBG_TIMER(t_post_end); #if defined(WHISPER_DEBUG) + const size_t model_ctx = (size_t) n_ctx; std::fprintf(stderr, "%s: vitisai enc+cross forward time = %8.2f ms\n", __func__, (t_fwd_end - t_fwd_start) / 1000.0f); - std::fprintf(stderr, "%s: kv_cross post-process time = %8.2f ms (non-flash)\n", __func__, (t_post_end - t_post_start) / 1000.0f); + std::fprintf(stderr, "%s: kv_cross post-process time = %8.2f ms (flash, req_ctx=%d, model_ctx=%zu, req_ctx_pad=%d, kv_type=%s)\n", + __func__, (t_post_end - t_post_start) / 1000.0f, n_ctx, model_ctx, n_ctx_pad, + whisper_vitisai_helpers::whisper_kv_type_name(kv_type)); +#endif + } else { + // Non-flash: model outputs contiguous [ctx, state] per layer. + WHISPER_DBG_TIMER(t_fwd_start); + if (!whisper_vitisai_forward_impl( + ctx, mel, embd_enc, input_tensors, output_tensors, cross_k_out, cross_v_out)) { + return 0; + } + WHISPER_DBG_TIMER(t_fwd_end); + WHISPER_DBG_TIMER(t_post_start); + + kv_layout.dst_layer_stride = elem_size * (size_t)n_state * (size_t)n_ctx; + if (kv_is_f32) { + // K was written straight into the kv cache by the runtime and is scaled there. + whisper_vitisai_helpers::whisper_kv_cross_scale_k_f32( + (float *)kv_cross_k->data, + (size_t) n_text_layer * req_layer_elems, + Kscale); + + whisper_vitisai_helpers::whisper_kv_cross_transpose_v_layers_f32( + ctx->cross_v_staging.data(), + (uint8_t *)kv_cross_v->data, + kv_layout); + } else { // kv_is_f16 + whisper_vitisai_helpers::whisper_kv_cross_store_k_transpose_v_layers_f16( + ctx->cross_k_staging.data(), + ctx->cross_v_staging.data(), + (uint8_t *)kv_cross_k->data, + (uint8_t *)kv_cross_v->data, + kv_layout); + } + + WHISPER_DBG_TIMER(t_post_end); + +#if defined(WHISPER_DEBUG) + const size_t model_ctx = (size_t) n_ctx; + std::fprintf(stderr, "%s: vitisai enc+cross forward time = %8.2f ms\n", __func__, (t_fwd_end - t_fwd_start) / 1000.0f); + std::fprintf(stderr, "%s: kv_cross post-process time = %8.2f ms (non-flash, req_ctx=%d, model_ctx=%zu, kv_type=%s)\n", + __func__, (t_post_end - t_post_start) / 1000.0f, n_ctx, model_ctx, + whisper_vitisai_helpers::whisper_kv_type_name(kv_type)); #endif } From 62c448e43e18a25050f56db61add805076b811e3 Mon Sep 17 00:00:00 2001 From: Sachin Kumawat Date: Fri, 7 Aug 2026 14:07:10 -0700 Subject: [PATCH 24/24] Deprecate cross_proj .rai naming and cleanup --- src/CMakeLists.txt | 7 +++---- src/vitisai/whisper-vitisai-encoder.cpp | 9 +++------ src/vitisai/whisper-vitisai-encoder.h | 1 - src/vitisai/whisper-vitisai-helpers.cpp | 21 ++++++--------------- src/vitisai/whisper-vitisai-helpers.h | 1 - src/whisper.cpp | 9 ++------- 6 files changed, 14 insertions(+), 34 deletions(-) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d4f9bfb51..2ae7896f5 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -173,10 +173,9 @@ if (WHISPER_VITISAI) set_property(TARGET ${TARGET} PROPERTY POSITION_INDEPENDENT_CODE ON) set(WHISPER_EXTRA_FLAGS ${WHISPER_EXTRA_FLAGS} -DWHISPER_USE_VITISAI) - # Add C++17 standard for MSVC - if (MSVC) - target_compile_options(${TARGET} PRIVATE /std:c++17) - endif() + # FlexMLRT headers and this plugin require C++17. Keep it PRIVATE so the + # C++11 requirement of the whisper target is not bumped. + target_compile_features(${TARGET} PRIVATE cxx_std_17) target_compile_definitions(${TARGET} PRIVATE WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES=${WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES} diff --git a/src/vitisai/whisper-vitisai-encoder.cpp b/src/vitisai/whisper-vitisai-encoder.cpp index 2a1b6d280..24db7dcdf 100644 --- a/src/vitisai/whisper-vitisai-encoder.cpp +++ b/src/vitisai/whisper-vitisai-encoder.cpp @@ -86,8 +86,10 @@ struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model) { options.executeMode = 2; options.extOptions["enable_preemption"] = true; + const bool model_is_rai = ctx->model_path.find(".rai") != std::string::npos; + // Check if model_path is rai file and if so, add fbs_buffer and fbs_buffer_size to the options - if (ctx->model_path.find(".rai") != std::string::npos) { + if (model_is_rai) { if (whisper_vitisai_helpers::map_rai_file(ctx->model_path.c_str(), &ctx->fbs_buffer, &ctx->fbs_buffer_size)) { options.extOptions["fbs_buffer"] = ctx->fbs_buffer; options.extOptions["fbs_buffer_size"] = ctx->fbs_buffer_size; @@ -104,7 +106,6 @@ struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model) { #endif } - const bool model_is_rai = ctx->model_path.find(".rai") != std::string::npos; if (model_is_rai) { #if WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES options.deviceName = "stx"; @@ -182,10 +183,6 @@ bool whisper_vitisai_has_cross_proj(const struct whisper_vitisai_context * ctx) return ctx && ctx->cross_k_out_idx >= 0 && ctx->cross_v_out_idx >= 0; } -bool whisper_vitisai_file_exists(const char * path) { - return whisper_vitisai_helpers::file_exists(path); -} - void whisper_vitisai_free(struct whisper_vitisai_context * ctx) { if (!ctx) { return; diff --git a/src/vitisai/whisper-vitisai-encoder.h b/src/vitisai/whisper-vitisai-encoder.h index f09003a64..ee96a1557 100644 --- a/src/vitisai/whisper-vitisai-encoder.h +++ b/src/vitisai/whisper-vitisai-encoder.h @@ -11,7 +11,6 @@ struct whisper_vitisai_context; struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model); void whisper_vitisai_free(struct whisper_vitisai_context * ctx); bool whisper_vitisai_has_cross_proj(const struct whisper_vitisai_context * ctx); -bool whisper_vitisai_file_exists(const char * path); struct ggml_tensor; diff --git a/src/vitisai/whisper-vitisai-helpers.cpp b/src/vitisai/whisper-vitisai-helpers.cpp index 2ff509447..417634615 100644 --- a/src/vitisai/whisper-vitisai-helpers.cpp +++ b/src/vitisai/whisper-vitisai-helpers.cpp @@ -66,7 +66,7 @@ bool map_rai_file(const char * path, uint8_t ** buffer, size_t * size) { return false; } - *buffer = (uint8_t *) mmap(nullptr, st.st_size, PROT_READ, MAP_SHARED, fileno(fd), 0); + *buffer = (uint8_t *) mmap(nullptr, st.st_size, PROT_READ, MAP_PRIVATE, fileno(fd), 0); if (*buffer == MAP_FAILED) { fclose(fd); std::fprintf(stderr, "%s: %d: Failed to mmap rai file '%s'\n", __func__, __LINE__, path); @@ -86,19 +86,6 @@ void unmap_rai_file(uint8_t * buffer, size_t size) { #endif // _WIN32 } -bool file_exists(const char * path) { - if (!path) { - return false; - } - - FILE * file = fopen(path, "rb"); - if (!file) { - return false; - } - fclose(file); - return true; -} - const char * whisper_kv_type_name(ggml_type type) { switch (type) { case GGML_TYPE_F32: return "F32"; @@ -260,7 +247,7 @@ bool whisper_vitisai_bind_tensor_data( } bool whisper_vitisai_resolve_io_binding( - const char * caller, + [[maybe_unused]] const char * caller, const std::vector & input_tensors, const std::vector & output_tensors, whisper_vitisai_io_binding * binding, @@ -287,7 +274,9 @@ bool whisper_vitisai_resolve_io_binding( } } if (!found_named_mel) { +#if defined(WHISPER_DEBUG) std::fprintf(stderr, "%s: WARNING: mel input not found by name; falling back to input[0]\n", caller); +#endif } if (output_tensors.empty()) { @@ -306,7 +295,9 @@ bool whisper_vitisai_resolve_io_binding( } if (binding->embd_enc_out_idx < 0) { +#if defined(WHISPER_DEBUG) std::fprintf(stderr, "%s: WARNING: embd_enc output not found by name; falling back to output[0]\n", caller); +#endif binding->embd_enc_out_idx = 0; } diff --git a/src/vitisai/whisper-vitisai-helpers.h b/src/vitisai/whisper-vitisai-helpers.h index f6ab53902..d50a2f4a2 100644 --- a/src/vitisai/whisper-vitisai-helpers.h +++ b/src/vitisai/whisper-vitisai-helpers.h @@ -13,7 +13,6 @@ namespace whisper_vitisai_helpers { bool map_rai_file(const char * path, uint8_t ** buffer, size_t * size); void unmap_rai_file(uint8_t * buffer, size_t size); -bool file_exists(const char * path); const char * whisper_kv_type_name(ggml_type type); const char * whisper_flexml_dtype_name(flexmlrt::client::DataType type); diff --git a/src/whisper.cpp b/src/whisper.cpp index c64b210b0..317827142 100644 --- a/src/whisper.cpp +++ b/src/whisper.cpp @@ -3388,19 +3388,14 @@ static std::string whisper_get_coreml_path_encoder(std::string path_bin) { #endif #ifdef WHISPER_USE_VITISAI -// replace extension with Vitis AI encoder artifact +// replace extension with Vitis AI encoder artifact. Cross projection support is +// detected from the model's output tensors, not from the file name. static std::string whisper_get_vitisai_path_encoder_cache(std::string path_bin) { auto pos = path_bin.rfind('.'); if (pos != std::string::npos) { path_bin = path_bin.substr(0, pos); } - const std::string path_vitisai_cross = path_bin + "-encoder-cross-vitisai.rai"; - if (FILE * file = fopen(path_vitisai_cross.c_str(), "rb")) { - fclose(file); - return path_vitisai_cross; - } - return path_bin + "-encoder-vitisai.rai"; } #endif