public_git
/
whisper.cpp
mirror of https://github.com/ggml-org/whisper.cpp.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add encoder + cross projection layer offload

This commit is contained in:
Sachin Kumawat 2026-07-28 23:43:12 -07:00
parent a4660b7cae
commit 951b1f0ead
7 changed files with 1120 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -340,6 +340,8 @@ On AMD's Ryzen™ AI 300 Series with dedicated NPUs for acceleration, you can no
Use the same model name with both scripts. The VitisAI script queries the AMD collection on Hugging Face to list available caches, then downloads the selected `.rai` file as `ggml-<model>-encoder-vitisai.rai` alongside the matching `ggml-<model>.bin` file. You can also browse the collection manually at https://huggingface.co/collections/amd/ryzen-ai-whisper-npu-optimized-onnx-models.
Depending on the downloaded `.rai` cache, VitisAI may offload either the encoder only or the encoder plus cross-projection layers to the AMD NPU. `whisper.cpp` detects the cache contents at runtime and logs the selected offload mode during model initialization.
- Build `whisper.cpp` with VitisAI support:
```bash

55
src/CMakeLists.txt
View File

@ -50,6 +50,57 @@ endif()
if (WHISPER_VITISAI)
find_package(FlexmlRT REQUIRED)
# Legacy RAI overrides are required by FlexMLRT older than 1.8.0
set(WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES_MODE "AUTO" CACHE STRING
"Legacy RAI override mode for FlexMLRT (AUTO|ON|OFF)")
set_property(CACHE WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES_MODE PROPERTY STRINGS AUTO ON OFF)
string(TOUPPER "${WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES_MODE}" _flexmlrt_legacy_mode)
set(_flexmlrt_legacy_hint "Set -DWHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES_MODE=ON or OFF explicitly.")
if (NOT _flexmlrt_legacy_mode MATCHES "^(AUTO|ON|OFF)$")
message(FATAL_ERROR
"Invalid WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES_MODE='${WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES_MODE}'. "
"Expected AUTO, ON, or OFF.")
endif()
if (_flexmlrt_legacy_mode STREQUAL "AUTO")
if (NOT FlexmlRT_DIR)
message(FATAL_ERROR
"FlexmlRT_DIR is unset after find_package(FlexmlRT). ${_flexmlrt_legacy_hint}")
endif()
# FlexmlRT_DIR points to <pkg_root>/share/cmake/FlexmlRT.
get_filename_component(_flexmlrt_init_py "${FlexmlRT_DIR}/../../../__init__.py" ABSOLUTE)
if (NOT EXISTS "${_flexmlrt_init_py}")
message(FATAL_ERROR
"flexmlrt __init__.py not found at ${_flexmlrt_init_py}. ${_flexmlrt_legacy_hint}")
endif()
set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS "${_flexmlrt_init_py}")
file(STRINGS "${_flexmlrt_init_py}" _flexmlrt_version_lines
REGEX "^VERSION[ \t]*=[ \t]*\"[0-9]+\\.[0-9]+\\.[0-9]+")
if (NOT _flexmlrt_version_lines MATCHES "\"([0-9]+\\.[0-9]+\\.[0-9]+)")
message(FATAL_ERROR
"Could not parse flexmlrt VERSION from ${_flexmlrt_init_py}. ${_flexmlrt_legacy_hint}")
endif()
set(_flexmlrt_version "${CMAKE_MATCH_1}")
if (_flexmlrt_version VERSION_LESS "1.8.0")
set(WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES 1)
else()
set(WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES 0)
endif()
message(STATUS "Detected flexmlrt VERSION=${_flexmlrt_version} from ${_flexmlrt_init_py} (legacy overrides=${WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES})")
else()
if (_flexmlrt_legacy_mode STREQUAL "ON")
set(WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES 1)
else()
set(WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES 0)
endif()
message(STATUS "FlexMLRT legacy RAI overrides forced ${_flexmlrt_legacy_mode}")
endif()
endif()
#
@ -125,6 +176,10 @@ if (WHISPER_VITISAI)
target_compile_options(${TARGET} PRIVATE /std:c++17)
endif()
target_compile_definitions(${TARGET} PRIVATE
WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES=${WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES}
)
target_link_libraries(${TARGET} PRIVATE ggml flexmlrt::flexmlrt)
set_target_properties(${TARGET} PROPERTIES FOLDER "libs")
endif()

416
src/vitisai/whisper-vitisai-encoder.cpp
View File

@ -1,3 +1,9 @@
#ifdef _WIN32
#ifndef NOMINMAX
#define NOMINMAX
#endif
#endif
#include "vitisai/whisper-vitisai-encoder.h"
#include "FlexMLClient.h"
#include "ggml.h"
@ -12,14 +18,45 @@
#include <sys/stat.h>
#include <fcntl.h>
#endif
#include <algorithm>
#include <cmath>
#include <cstring>
#include <memory>
#include <string>
#include <vector>
#if defined(WHISPER_DEBUG)
#define WHISPER_DBG_TIMER(name) const int64_t name = ggml_time_us()
#else
#define WHISPER_DBG_TIMER(name) do {} while (0)
#endif
#if defined(WHISPER_DEBUG)
template <typename T>
static void whisper_vitisai_print_shape(const std::vector<T> & shape) {
std::fprintf(stderr, "[");
for (size_t i = 0; i < shape.size(); ++i) {
std::fprintf(stderr, "%s%lld", i == 0 ? "" : ", ", (long long) shape[i]);
}
std::fprintf(stderr, "]");
}
#endif
struct whisper_vitisai_context {
std::string model_path;
std::shared_ptr<flexmlrt::client::Model> runner;
uint8_t * fbs_buffer;
size_t fbs_buffer_size;
uint8_t * fbs_buffer = nullptr;
size_t fbs_buffer_size = 0;
std::vector<float> cross_k_staging;
std::vector<float> cross_v_staging;
int embd_enc_out_idx = -1;
int cross_k_out_idx = -1;
int cross_v_out_idx = -1;
std::vector<flexmlrt::client::ErtTensorType> cached_input_tensors;
std::vector<flexmlrt::client::ErtTensorType> cached_output_tensors;
};
// Function to mmap rai file for Linux and MapViewOfFile for Windows
@ -94,6 +131,38 @@ static void unmap_rai_file(uint8_t * buffer, size_t size) {
#endif // _WIN32
}
bool whisper_vitisai_file_exists(const char * path) {
if (!path) {
return false;
}
FILE * file = fopen(path, "rb");
if (!file) {
return false;
}
fclose(file);
return true;
}
// Reuse cached tensor descriptors to avoid repeated getIOTensors() lookups.
static bool whisper_vitisai_get_io_tensors(
struct whisper_vitisai_context * ctx,
std::vector<flexmlrt::client::ErtTensorType> & input_tensors,
std::vector<flexmlrt::client::ErtTensorType> & output_tensors) {
if (!ctx || !ctx->runner) {
return false;
}
if (ctx->cached_input_tensors.empty() || ctx->cached_output_tensors.empty()) {
ctx->cached_input_tensors = ctx->runner->getIOTensors("input", false);
ctx->cached_output_tensors = ctx->runner->getIOTensors("output", false);
}
input_tensors = ctx->cached_input_tensors;
output_tensors = ctx->cached_output_tensors;
return true;
}
struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model) {
if (!path_model) {
std::fprintf(stderr, "%s: path_model is null\n", __func__);
@ -102,8 +171,6 @@ struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model) {
auto * ctx = new whisper_vitisai_context;
ctx->model_path = path_model;
ctx->fbs_buffer = nullptr;
ctx->fbs_buffer_size = 0;
// Override the model path with the environment variable if it is set
if (const char * env_model_path = std::getenv("OVERRIDE_VITISAI_MODEL_PATH")) {
@ -121,7 +188,6 @@ struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model) {
// Check if model_path is rai file and if so, add fbs_buffer and fbs_buffer_size to the options
if (ctx->model_path.find(".rai") != std::string::npos) {
// mmap rai file for both Linux and Windows and pass the buffer to the options
if (map_rai_file(ctx->model_path.c_str(), &ctx->fbs_buffer, &ctx->fbs_buffer_size)) {
options.extOptions["fbs_buffer"] = ctx->fbs_buffer;
options.extOptions["fbs_buffer_size"] = ctx->fbs_buffer_size;
@ -138,12 +204,72 @@ struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model) {
#endif
}
const bool model_is_rai = ctx->model_path.find(".rai") != std::string::npos;
if (model_is_rai) {
#if WHISPER_FLEXMLRT_LEGACY_RAI_OVERRIDES
options.deviceName = "stx";
options.subgraphName = "vaiml_par_0";
#if defined(WHISPER_DEBUG)
std::fprintf(stderr,
"%s: legacy FlexMLRT compile configuration detected; applying RAI overrides (device='stx', subgraph='vaiml_par_0')\n",
__func__);
#endif // defined(WHISPER_DEBUG)
#endif
}
try {
ctx->runner = std::make_shared<flexmlrt::client::Model>(options);
if (!ctx->runner->good()) {
throw std::runtime_error("Runner creation ran into an error");
}
ctx->cached_input_tensors = ctx->runner->getIOTensors("input", false);
ctx->cached_output_tensors = ctx->runner->getIOTensors("output", false);
auto & output_tensors = ctx->cached_output_tensors;
for (int i = 0; i < (int) output_tensors.size(); ++i) {
const std::string & name = output_tensors[i].getMetadata().name;
if (name == "embd_enc") {
ctx->embd_enc_out_idx = i;
} else if (name == "cross_k") {
ctx->cross_k_out_idx = i;
} else if (name == "cross_v") {
ctx->cross_v_out_idx = i;
}
}
if (ctx->embd_enc_out_idx < 0) {
std::fprintf(stderr, "%s: WARNING: embd_enc output not found by name; falling back to output[0]\n", __func__);
ctx->embd_enc_out_idx = 0;
}
#if defined(WHISPER_DEBUG)
{
auto & input_tensors = ctx->cached_input_tensors;
std::fprintf(stderr, "%s: model has %zu input tensor(s)\n", __func__, input_tensors.size());
for (int i = 0; i < (int) input_tensors.size(); ++i) {
const auto & meta = input_tensors[i].getMetadata();
std::fprintf(stderr, "%s: input[%d] name='%s' size=%zu shape=",
__func__, i, meta.name.c_str(), (size_t) meta.size);
whisper_vitisai_print_shape(meta.shape);
std::fprintf(stderr, "\n");
}
std::fprintf(stderr, "%s: model has %zu output tensor(s)\n", __func__, output_tensors.size());
for (int i = 0; i < (int) output_tensors.size(); ++i) {
const auto & meta = output_tensors[i].getMetadata();
std::fprintf(stderr, "%s: output[%d] name='%s' size=%zu shape=",
__func__, i, meta.name.c_str(), (size_t) meta.size);
whisper_vitisai_print_shape(meta.shape);
std::fprintf(stderr, "\n");
}
std::fprintf(stderr, "%s: output indices: embd_enc=%d cross_k=%d cross_v=%d\n",
__func__, ctx->embd_enc_out_idx, ctx->cross_k_out_idx, ctx->cross_v_out_idx);
}
#endif
} catch (const std::exception & e) {
std::fprintf(stderr, "%s: Exception during Vitis AI runner creation: %s\n", __func__, e.what());
delete ctx;
@ -152,12 +278,18 @@ struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model) {
return ctx;
}
bool whisper_vitisai_has_cross_proj(const struct whisper_vitisai_context * ctx) {
return ctx && ctx->cross_k_out_idx >= 0 && ctx->cross_v_out_idx >= 0;
}
void whisper_vitisai_free(struct whisper_vitisai_context * ctx) {
if (!ctx) {
return;
}
std::fprintf(stderr, "%s: releasing Vitis AI encoder context for model '%s'\n", __func__, ctx->model_path.c_str());
#if defined(WHISPER_DEBUG)
std::fprintf(stderr, "%s: releasing Vitis AI context for model '%s'\n", __func__, ctx->model_path.c_str());
#endif
if (ctx->fbs_buffer) {
unmap_rai_file(ctx->fbs_buffer, ctx->fbs_buffer_size);
}
@ -184,14 +316,21 @@ int whisper_vitisai_encode(struct whisper_vitisai_context * ctx, struct ggml_ten
std::vector<flexmlrt::client::ErtTensorType> input_tensors, output_tensors;
auto model = ctx->runner;
// Get tensors as CPU tensors (hwTensor = false)
input_tensors = model->getIOTensors("input", false);
output_tensors = model->getIOTensors("output", false);
if (!whisper_vitisai_get_io_tensors(ctx, input_tensors, output_tensors)) {
std::fprintf(stderr, "%s: failed to acquire Vitis AI I/O tensors\n", __func__);
return 0;
}
// TODO: add assert checks for tensor numbers and shapes
if (ctx->embd_enc_out_idx < 0 || ctx->embd_enc_out_idx >= (int) output_tensors.size()) {
std::fprintf(stderr, "%s: invalid embd_enc output index %d for %zu output tensor(s)\n",
__func__, ctx->embd_enc_out_idx, output_tensors.size());
return 0;
}
input_tensors[0].data = mel->data;
output_tensors[0].data = out->data;
output_tensors[ctx->embd_enc_out_idx].data = out->data;
try {
model->forward(input_tensors, output_tensors);
@ -205,3 +344,260 @@ int whisper_vitisai_encode(struct whisper_vitisai_context * ctx, struct ggml_ten
return 1;
}
int whisper_vitisai_run_enc_cross(
struct whisper_vitisai_context * ctx,
struct ggml_tensor * mel,
struct ggml_tensor * out,
void * cross_v_data,
void * cross_k_data) {
if (!ctx || !mel || !out || !cross_v_data || !cross_k_data) {
std::fprintf(stderr, "%s: ctx/mel/out/cross_v_data/cross_k_data must not be null\n", __func__);
return 0;
}
if (ggml_n_dims(mel) != 2) {
std::fprintf(stderr, "%s: mel tensor expected to have 2 dims, got %d\n", __func__, ggml_n_dims(mel));
return 0;
}
if (ggml_n_dims(out) != 2) {
std::fprintf(stderr, "%s: out tensor expected to have 2 dims, got %d\n", __func__, ggml_n_dims(out));
return 0;
}
std::vector<flexmlrt::client::ErtTensorType> input_tensors, output_tensors;
auto model = ctx->runner;
if (!whisper_vitisai_get_io_tensors(ctx, input_tensors, output_tensors)) {
std::fprintf(stderr, "%s: failed to acquire Vitis AI I/O tensors\n", __func__);
return 0;
}
if (output_tensors.size() != 3) {
std::fprintf(stderr, "%s: expected 3 output tensors, got %zu\n", __func__, output_tensors.size());
return 0;
}
if (ctx->embd_enc_out_idx < 0 || ctx->embd_enc_out_idx >= (int) output_tensors.size() ||
ctx->cross_k_out_idx < 0 || ctx->cross_k_out_idx >= (int) output_tensors.size() ||
ctx->cross_v_out_idx < 0 || ctx->cross_v_out_idx >= (int) output_tensors.size()) {
std::fprintf(stderr, "%s: invalid output indices embd_enc=%d cross_k=%d cross_v=%d for %zu output tensor(s)\n",
__func__, ctx->embd_enc_out_idx, ctx->cross_k_out_idx, ctx->cross_v_out_idx, output_tensors.size());
return 0;
}
input_tensors[0].data = mel->data;
output_tensors[ctx->embd_enc_out_idx].data = out->data;
output_tensors[ctx->cross_v_out_idx].data = cross_v_data;
output_tensors[ctx->cross_k_out_idx].data = cross_k_data;
try {
model->forward(input_tensors, output_tensors);
#if defined(WHISPER_DEBUG)
std::fprintf(stderr, "%s: Vitis AI model inference (encoder + cross proj) completed.\n", __func__);
#endif
} catch (const std::exception & e) {
std::fprintf(stderr, "%s: Exception during model inference: %s\n", __func__, e.what());
return 0;
}
return 1;
}
// Ensure persistent staging buffers are large enough for the given dimensions.
static void ensure_staging_buffers(
struct whisper_vitisai_context * ctx,
size_t count, bool need_k) {
if (need_k && ctx->cross_k_staging.size() < count) {
ctx->cross_k_staging.resize(count);
}
if (ctx->cross_v_staging.size() < count) {
ctx->cross_v_staging.resize(count);
}
}
int whisper_vitisai_encode_with_cross(
struct whisper_vitisai_context * ctx,
struct ggml_tensor * mel,
struct ggml_tensor * embd_enc,
struct ggml_tensor * kv_cross_k,
struct ggml_tensor * kv_cross_v,
int n_text_layer,
int n_ctx,
int n_text_state,
int n_text_head,
bool flash_attn) {
if (!ctx || !mel || !embd_enc || !kv_cross_k || !kv_cross_v) {
std::fprintf(stderr, "%s: null argument\n", __func__);
return 0;
}
const int n_state = n_text_state;
const int n_state_head = n_state / n_text_head;
const int n_ctx_pad = (n_ctx + 255) & ~255; // GGML_PAD(n_ctx, 256)
const float Kscale = pow(float(n_state_head), -0.25f);
const ggml_type kv_type = kv_cross_k->type;
const size_t elem_size = ggml_type_size(kv_type);
const size_t layer_elems = (size_t)n_ctx * n_state;
const size_t buf_count = (size_t)n_text_layer * layer_elems;
if (flash_attn) {
WHISPER_DBG_TIMER(t_fwd_start);
if (n_ctx_pad == n_ctx) {
// No padding gap -- plugin writes directly into kv_cross.
if (!whisper_vitisai_run_enc_cross(
ctx, mel, embd_enc,
kv_cross_v->data, kv_cross_k->data)) {
return 0;
}
WHISPER_DBG_TIMER(t_fwd_end);
WHISPER_DBG_TIMER(t_post_start);
if (kv_type == GGML_TYPE_F32) {
float * kdata = (float *)kv_cross_k->data;
for (size_t i = 0; i < buf_count; ++i) {
kdata[i] *= Kscale;
}
} else if (kv_type == GGML_TYPE_F16) {
ggml_fp16_t * kdata = (ggml_fp16_t *)kv_cross_k->data;
for (size_t i = 0; i < buf_count; ++i) {
kdata[i] = ggml_fp32_to_fp16(ggml_fp16_to_fp32(kdata[i]) * Kscale);
}
}
WHISPER_DBG_TIMER(t_post_end);
#if defined(WHISPER_DEBUG)
std::fprintf(stderr, "%s: vitisai enc+cross forward time = %8.2f ms\n", __func__, (t_fwd_end - t_fwd_start) / 1000.0f);
std::fprintf(stderr, "%s: kv_cross post-process time = %8.2f ms (flash, no-pad direct)\n", __func__, (t_post_end - t_post_start) / 1000.0f);
#endif
} else {
// Padding gap -- use persistent staging buffers.
ensure_staging_buffers(ctx, buf_count, true);
float * cross_k_buf = ctx->cross_k_staging.data();
float * cross_v_buf = ctx->cross_v_staging.data();
if (!whisper_vitisai_run_enc_cross(
ctx, mel, embd_enc,
cross_v_buf, cross_k_buf)) {
return 0;
}
WHISPER_DBG_TIMER(t_fwd_end);
WHISPER_DBG_TIMER(t_post_start);
// Combined per-layer K+V scatter for better cache locality.
const size_t padded_layer_stride = elem_size * n_state * n_ctx_pad;
for (int il = 0; il < n_text_layer; ++il) {
const float * src_k = cross_k_buf + (size_t)il * layer_elems;
const float * src_v = cross_v_buf + (size_t)il * layer_elems;
uint8_t * dst_k = (uint8_t *)kv_cross_k->data + padded_layer_stride * il;
uint8_t * dst_v = (uint8_t *)kv_cross_v->data + padded_layer_stride * il;
if (kv_type == GGML_TYPE_F32) {
float * dk = (float *)dst_k;
for (size_t i = 0; i < layer_elems; ++i) {
dk[i] = src_k[i] * Kscale;
}
memcpy(dst_v, src_v, layer_elems * sizeof(float));
} else if (kv_type == GGML_TYPE_F16) {
ggml_fp16_t * dk = (ggml_fp16_t *)dst_k;
ggml_fp16_t * dv = (ggml_fp16_t *)dst_v;
for (size_t i = 0; i < layer_elems; ++i) {
dk[i] = ggml_fp32_to_fp16(src_k[i] * Kscale);
dv[i] = ggml_fp32_to_fp16(src_v[i]);
}
}
}
WHISPER_DBG_TIMER(t_post_end);
#if defined(WHISPER_DEBUG)
std::fprintf(stderr, "%s: vitisai enc+cross forward time = %8.2f ms\n", __func__, (t_fwd_end - t_fwd_start) / 1000.0f);
std::fprintf(stderr, "%s: kv_cross post-process time = %8.2f ms (flash, padded, n_ctx=%d, n_ctx_pad=%d, kv_type=%s)\n",
__func__, (t_post_end - t_post_start) / 1000.0f,
n_ctx, n_ctx_pad,
kv_type == GGML_TYPE_F32 ? "F32" : kv_type == GGML_TYPE_F16 ? "F16" : "other");
#endif
}
} else {
// Non-flash: layers are contiguous (stride = n_state * n_ctx).
// K: plugin writes directly into kv_cross_k, then in-place Kscale.
// V: persistent staging buffer + cache-friendly blocked transpose.
ensure_staging_buffers(ctx, buf_count, false);
float * cross_v_buf = ctx->cross_v_staging.data();
WHISPER_DBG_TIMER(t_fwd_start);
if (!whisper_vitisai_run_enc_cross(
ctx, mel, embd_enc,
cross_v_buf, kv_cross_k->data)) {
return 0;
}
WHISPER_DBG_TIMER(t_fwd_end);
WHISPER_DBG_TIMER(t_post_start);
if (kv_type == GGML_TYPE_F32) {
float * kdata = (float *)kv_cross_k->data;
for (size_t i = 0; i < buf_count; ++i) {
kdata[i] *= Kscale;
}
const int BLOCK = 32;
for (int il = 0; il < n_text_layer; ++il) {
const float * src_v = cross_v_buf + (size_t)il * layer_elems;
float * dst_v = (float *)kv_cross_v->data + (size_t)il * layer_elems;
for (int ic = 0; ic < n_ctx; ic += BLOCK) {
for (int is = 0; is < n_state; is += BLOCK) {
const int ic_end = std::min(ic + BLOCK, n_ctx);
const int is_end = std::min(is + BLOCK, n_state);
for (int i = ic; i < ic_end; ++i) {
for (int j = is; j < is_end; ++j) {
dst_v[j * n_ctx + i] = src_v[i * n_state + j];
}
}
}
}
}
} else if (kv_type == GGML_TYPE_F16) {
ggml_fp16_t * kdata = (ggml_fp16_t *)kv_cross_k->data;
for (size_t i = 0; i < buf_count; ++i) {
kdata[i] = ggml_fp32_to_fp16(ggml_fp16_to_fp32(kdata[i]) * Kscale);
}
const int BLOCK = 32;
for (int il = 0; il < n_text_layer; ++il) {
const float * src_v = cross_v_buf + (size_t)il * layer_elems;
ggml_fp16_t * dst_v = (ggml_fp16_t *)((uint8_t *)kv_cross_v->data + elem_size * n_state * n_ctx * il);
for (int ic = 0; ic < n_ctx; ic += BLOCK) {
for (int is = 0; is < n_state; is += BLOCK) {
const int ic_end = std::min(ic + BLOCK, n_ctx);
const int is_end = std::min(is + BLOCK, n_state);
for (int i = ic; i < ic_end; ++i) {
for (int j = is; j < is_end; ++j) {
dst_v[j * n_ctx + i] = ggml_fp32_to_fp16(src_v[i * n_state + j]);
}
}
}
}
}
}
WHISPER_DBG_TIMER(t_post_end);
#if defined(WHISPER_DEBUG)
std::fprintf(stderr, "%s: vitisai enc+cross forward time = %8.2f ms\n", __func__, (t_fwd_end - t_fwd_start) / 1000.0f);
std::fprintf(stderr, "%s: kv_cross post-process time = %8.2f ms (non-flash)\n", __func__, (t_post_end - t_post_start) / 1000.0f);
#endif
}
return 1;
}

23
src/vitisai/whisper-vitisai-encoder.h
View File

@ -1,8 +1,6 @@
#pragma once
#include <cstddef>
#include <cstdbool>
#include <cstdint>
#if __cplusplus
extern "C" {
@ -12,6 +10,8 @@ struct whisper_vitisai_context;
struct whisper_vitisai_context * whisper_vitisai_init(const char * path_model);
void whisper_vitisai_free(struct whisper_vitisai_context * ctx);
bool whisper_vitisai_has_cross_proj(const struct whisper_vitisai_context * ctx);
bool whisper_vitisai_file_exists(const char * path);
struct ggml_tensor;
@ -20,6 +20,25 @@ int whisper_vitisai_encode(
struct ggml_tensor * mel,
struct ggml_tensor * out);
int whisper_vitisai_run_enc_cross(
struct whisper_vitisai_context * ctx,
struct ggml_tensor * mel,
struct ggml_tensor * out,
void * cross_v_data,
void * cross_k_data);
int whisper_vitisai_encode_with_cross(
struct whisper_vitisai_context * ctx,
struct ggml_tensor * mel,
struct ggml_tensor * embd_enc,
struct ggml_tensor * kv_cross_k,
struct ggml_tensor * kv_cross_v,
int n_text_layer,
int n_ctx,
int n_text_state,
int n_text_head,
bool flash_attn);
#if __cplusplus
}
#endif

481
src/vitisai/whisper-vitisai-helpers.cpp Normal file
View File

@ -0,0 +1,481 @@
#ifdef _WIN32
#ifndef NOMINMAX
#define NOMINMAX
#endif
#endif
#include "vitisai/whisper-vitisai-helpers.h"
#include <algorithm>
#include <cstdio>
#ifdef _WIN32
#include <windows.h>
#else
#include <sys/mman.h>
#include <sys/stat.h>
#endif
#include <string>
#include <utility>
namespace whisper_vitisai_helpers {
bool map_rai_file(const char * path, uint8_t ** buffer, size_t * size) {
#ifdef _WIN32
HANDLE hFile = CreateFileA(path, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
if (hFile == INVALID_HANDLE_VALUE) {
std::fprintf(stderr, "%s: %d: Failed to open rai file '%s'\n", __func__, __LINE__, path);
return false;
}
LARGE_INTEGER fileSize;
if (!GetFileSizeEx(hFile, &fileSize)) {
CloseHandle(hFile);
std::fprintf(stderr, "%s: %d: Failed to get file size for rai file '%s'\n", __func__, __LINE__, path);
return false;
}
HANDLE hMapping = CreateFileMappingA(hFile, NULL, PAGE_READONLY, 0, fileSize.QuadPart, NULL);
if (hMapping == NULL) {
CloseHandle(hFile);
std::fprintf(stderr, "%s: %d: Failed to create file mapping for rai file '%s'\n", __func__, __LINE__, path);
return false;
}
*buffer = (uint8_t *) MapViewOfFile(hMapping, FILE_MAP_READ, 0, 0, fileSize.QuadPart);
if (*buffer == NULL) {
CloseHandle(hMapping);
CloseHandle(hFile);
std::fprintf(stderr, "%s: %d: Failed to map rai file '%s'\n", __func__, __LINE__, path);
return false;
}
CloseHandle(hMapping);
CloseHandle(hFile);
*size = fileSize.QuadPart;
return true;
#else
FILE * fd = fopen(path, "rb");
if (!fd) {
std::fprintf(stderr, "%s: %d: Failed to open rai file '%s'\n", __func__, __LINE__, path);
return false;
}
struct stat st;
if (fstat(fileno(fd), &st) == -1) {
fclose(fd);
std::fprintf(stderr, "%s: %d: Failed to get file size for rai file '%s'\n", __func__, __LINE__, path);
return false;
}
*buffer = (uint8_t *) mmap(nullptr, st.st_size, PROT_READ, MAP_SHARED, fileno(fd), 0);
if (*buffer == MAP_FAILED) {
fclose(fd);
std::fprintf(stderr, "%s: %d: Failed to mmap rai file '%s'\n", __func__, __LINE__, path);
return false;
}
fclose(fd);
*size = st.st_size;
return true;
#endif // _WIN32
}
void unmap_rai_file(uint8_t * buffer, size_t size) {
#ifdef _WIN32
UnmapViewOfFile(buffer);
#else
munmap(buffer, size);
#endif // _WIN32
}
bool file_exists(const char * path) {
if (!path) {
return false;
}
FILE * file = fopen(path, "rb");
if (!file) {
return false;
}
fclose(file);
return true;
}
const char * whisper_kv_type_name(ggml_type type) {
switch (type) {
case GGML_TYPE_F32: return "F32";
case GGML_TYPE_F16: return "F16";
default: return "unsupported";
}
}
const char * whisper_flexml_dtype_name(flexmlrt::client::DataType type) {
switch (type) {
case flexmlrt::client::DataType::Float32: return "Float32";
case flexmlrt::client::DataType::Int8: return "Int8";
case flexmlrt::client::DataType::UInt8: return "UInt8";
case flexmlrt::client::DataType::Int16: return "Int16";
case flexmlrt::client::DataType::UInt16: return "UInt16";
case flexmlrt::client::DataType::BFloat16: return "BFloat16";
case flexmlrt::client::DataType::Bool: return "Bool";
case flexmlrt::client::DataType::Float16: return "Float16";
case flexmlrt::client::DataType::Int32: return "Int32";
case flexmlrt::client::DataType::UInt32: return "UInt32";
default: return "Unknown";
}
}
bool whisper_flexml_dtype_to_ggml_type(
flexmlrt::client::DataType type,
ggml_type * ggml_dtype) {
switch (type) {
case flexmlrt::client::DataType::Float32:
if (ggml_dtype) {
*ggml_dtype = GGML_TYPE_F32;
}
return true;
case flexmlrt::client::DataType::Float16:
if (ggml_dtype) {
*ggml_dtype = GGML_TYPE_F16;
}
return true;
case flexmlrt::client::DataType::BFloat16:
if (ggml_dtype) {
*ggml_dtype = GGML_TYPE_BF16;
}
return true;
default:
return false;
}
}
static bool whisper_vitisai_validate_tensor_dtype(
const char * tensor_name,
flexmlrt::client::DataType model_dtype,
ggml_type runtime_dtype) {
ggml_type expected_runtime_dtype = GGML_TYPE_COUNT;
if (!whisper_flexml_dtype_to_ggml_type(model_dtype, &expected_runtime_dtype)) {
std::fprintf(stderr,
"%s: unsupported model dtype for %s: %s (supported: Float32/Float16/BFloat16)\n",
__func__, tensor_name, whisper_flexml_dtype_name(model_dtype));
return false;
}
if (runtime_dtype != expected_runtime_dtype) {
std::fprintf(stderr,
"%s: %s dtype mismatch (runtime=%s, model=%s)\n",
__func__, tensor_name, ggml_type_name(runtime_dtype), whisper_flexml_dtype_name(model_dtype));
return false;
}
return true;
}
static std::string whisper_shape_to_string(const std::vector<size_t> & shape) {
std::string out = "[";
for (size_t i = 0; i < shape.size(); ++i) {
if (i > 0) {
out += ", ";
}
out += std::to_string(shape[i]);
}
out += "]";
return out;
}
static std::vector<size_t> whisper_canonical_shape(const std::vector<std::uint32_t> & shape) {
std::vector<size_t> canonical;
canonical.reserve(shape.size());
for (size_t i = 0; i < shape.size(); ++i) {
const size_t dim = (size_t) shape[i];
if (dim != 1) {
canonical.push_back(dim);
}
}
if (canonical.empty()) {
canonical.push_back(1);
}
return canonical;
}
static bool whisper_validate_shape(
const char * tensor_name,
const std::vector<std::uint32_t> & model_shape,
const std::vector<size_t> & expected_shape) {
const std::vector<size_t> shape = whisper_canonical_shape(model_shape);
if (shape != expected_shape) {
std::fprintf(stderr,
"%s: %s shape mismatch (runtime expected=%s, model=%s)\n",
__func__,
tensor_name,
whisper_shape_to_string(expected_shape).c_str(),
whisper_shape_to_string(shape).c_str());
return false;
}
return true;
}
bool whisper_validate_cross_shape(
const char * tensor_name,
const std::vector<std::uint32_t> & model_shape,
int n_text_layer,
int n_ctx,
int n_state) {
const std::vector<size_t> expected = {
(size_t) n_text_layer,
(size_t) n_ctx,
(size_t) n_state,
};
return whisper_validate_shape(tensor_name, model_shape, expected);
}
bool whisper_vitisai_bind_tensor_data(
const char * tensor_name,
struct ggml_tensor * runtime_tensor,
const std::vector<size_t> & expected_shape,
flexmlrt::client::ErtTensorType & io_tensor) {
const auto & meta = io_tensor.getMetadata();
if (!whisper_vitisai_validate_tensor_dtype(tensor_name, meta.type, runtime_tensor->type)) {
return false;
}
if (!whisper_validate_shape(tensor_name, meta.shape, expected_shape)) {
return false;
}
const size_t model_bytes = meta.size;
const size_t runtime_bytes = ggml_nbytes(runtime_tensor);
if (model_bytes == 0 || runtime_bytes == 0) {
std::fprintf(stderr, "%s: %s sizes must be non-zero (model=%zu, runtime=%zu)\n",
__func__, tensor_name, model_bytes, runtime_bytes);
return false;
}
if (runtime_bytes != model_bytes) {
std::fprintf(stderr,
"%s: %s tensor size mismatch (runtime=%zu B, model=%zu B). "
"VitisAI .rai requires exact context match; use matching -ac/model artifact.\n",
__func__, tensor_name, runtime_bytes, model_bytes);
return false;
}
io_tensor.data = runtime_tensor->data;
return true;
}
bool whisper_vitisai_resolve_io_binding(
const char * caller,
const std::vector<flexmlrt::client::ErtTensorType> & input_tensors,
const std::vector<flexmlrt::client::ErtTensorType> & output_tensors,
whisper_vitisai_io_binding * binding,
std::string * error) {
const auto fail = [error](std::string message) {
if (error) {
*error = std::move(message);
}
return false;
};
if (input_tensors.empty()) {
return fail("Model has no input tensors");
}
binding->mel_in_idx = 0;
bool found_named_mel = false;
for (int i = 0; i < (int) input_tensors.size(); ++i) {
const std::string & name = input_tensors[i].getMetadata().name;
if (name == "input" || name == "mel") {
binding->mel_in_idx = i;
found_named_mel = true;
break;
}
}
if (!found_named_mel) {
std::fprintf(stderr, "%s: WARNING: mel input not found by name; falling back to input[0]\n", caller);
}
if (output_tensors.empty()) {
return fail("Model has no output tensors");
}
for (int i = 0; i < (int) output_tensors.size(); ++i) {
const std::string & name = output_tensors[i].getMetadata().name;
if (name == "embd_enc") {
binding->embd_enc_out_idx = i;
} else if (name == "cross_k") {
binding->cross_k_out_idx = i;
} else if (name == "cross_v") {
binding->cross_v_out_idx = i;
}
}
if (binding->embd_enc_out_idx < 0) {
std::fprintf(stderr, "%s: WARNING: embd_enc output not found by name; falling back to output[0]\n", caller);
binding->embd_enc_out_idx = 0;
}
const bool has_cross_k = binding->cross_k_out_idx >= 0;
const bool has_cross_v = binding->cross_v_out_idx >= 0;
if (has_cross_k != has_cross_v) {
return fail("Incomplete cross-projection contract: both cross_k and cross_v outputs are required");
}
if (has_cross_k && (binding->cross_k_out_idx == binding->cross_v_out_idx ||
binding->cross_k_out_idx == binding->embd_enc_out_idx ||
binding->cross_v_out_idx == binding->embd_enc_out_idx)) {
return fail("Invalid output mapping: embd_enc/cross_k/cross_v indices overlap");
}
const auto & mel_meta = input_tensors[binding->mel_in_idx].getMetadata();
if (!whisper_flexml_dtype_to_ggml_type(mel_meta.type, nullptr)) {
return fail(
std::string("Unsupported mel input type: ") +
whisper_flexml_dtype_name(mel_meta.type) + " (supported: Float32/Float16/BFloat16)");
}
binding->mel_in_expected_bytes = mel_meta.size;
const auto & embd_meta = output_tensors[binding->embd_enc_out_idx].getMetadata();
if (!whisper_flexml_dtype_to_ggml_type(embd_meta.type, nullptr)) {
return fail(
std::string("Unsupported embd_enc output type: ") +
whisper_flexml_dtype_name(embd_meta.type) + " (supported: Float32/Float16/BFloat16)");
}
binding->embd_enc_expected_bytes = embd_meta.size;
if (has_cross_k) {
const auto & cross_k_meta = output_tensors[binding->cross_k_out_idx].getMetadata();
const auto & cross_v_meta = output_tensors[binding->cross_v_out_idx].getMetadata();
if (cross_k_meta.type != flexmlrt::client::DataType::Float32 ||
cross_v_meta.type != flexmlrt::client::DataType::Float32) {
return fail(
std::string("Unsupported cross output type(s): cross_k=") +
whisper_flexml_dtype_name(cross_k_meta.type) + ", cross_v=" +
whisper_flexml_dtype_name(cross_v_meta.type) + " (cross path currently requires Float32)");
}
if (cross_k_meta.size != cross_v_meta.size) {
return fail("cross_k and cross_v output sizes do not match");
}
binding->cross_k_expected_bytes = cross_k_meta.size;
binding->cross_v_expected_bytes = cross_v_meta.size;
}
return true;
}
bool whisper_vitisai_all_tensors_claimed(
const char * caller,
const char * tensor_kind,
const std::vector<flexmlrt::client::ErtTensorType> & tensors,
const std::vector<bool> & claimed) {
for (size_t i = 0; i < tensors.size(); ++i) {
if (!claimed[i]) {
std::fprintf(stderr,
"%s: unsupported extra %s tensor at index %zu (name='%s'); strict contract expects only mapped %ss\n",
caller, tensor_kind, i, tensors[i].getMetadata().name.c_str(), tensor_kind);
return false;
}
}
return true;
}
void whisper_kv_cross_scale_k_f32(
float * k_data,
size_t count,
float kscale) {
for (size_t i = 0; i < count; ++i) {
k_data[i] *= kscale;
}
}
void whisper_kv_cross_store_layers_f32(
const float * src_k,
const float * src_v,
uint8_t * dst_k,
uint8_t * dst_v,
const whisper_kv_cross_layout & layout) {
for (int il = 0; il < layout.n_layer; ++il) {
const float * layer_src_k = src_k + (size_t)il * layout.src_layer_elems;
const float * layer_src_v = src_v + (size_t)il * layout.src_layer_elems;
float * dk = (float *)(dst_k + layout.dst_layer_stride * (size_t)il);
float * dv = (float *)(dst_v + layout.dst_layer_stride * (size_t)il);
for (size_t i = 0; i < layout.layer_elems; ++i) {
dk[i] = layer_src_k[i] * layout.kscale;
dv[i] = layer_src_v[i];
}
}
}
void whisper_kv_cross_store_layers_f16(
const float * src_k,
const float * src_v,
uint8_t * dst_k,
uint8_t * dst_v,
const whisper_kv_cross_layout & layout) {
for (int il = 0; il < layout.n_layer; ++il) {
const float * layer_src_k = src_k + (size_t)il * layout.src_layer_elems;
const float * layer_src_v = src_v + (size_t)il * layout.src_layer_elems;
ggml_fp16_t * dk = (ggml_fp16_t *)(dst_k + layout.dst_layer_stride * (size_t)il);
ggml_fp16_t * dv = (ggml_fp16_t *)(dst_v + layout.dst_layer_stride * (size_t)il);
for (size_t i = 0; i < layout.layer_elems; ++i) {
dk[i] = ggml_fp32_to_fp16(layer_src_k[i] * layout.kscale);
dv[i] = ggml_fp32_to_fp16(layer_src_v[i]);
}
}
}
void whisper_kv_cross_transpose_v_layers_f32(
const float * src_v,
uint8_t * dst_v,
const whisper_kv_cross_layout & layout) {
const int n_ctx = layout.n_ctx;
const int n_state = layout.n_state;
const int BLOCK = 32;
for (int il = 0; il < layout.n_layer; ++il) {
const float * layer_src_v = src_v + (size_t)il * layout.src_layer_elems;
float * dv = (float *)(dst_v + layout.dst_layer_stride * (size_t)il);
for (int ic = 0; ic < n_ctx; ic += BLOCK) {
for (int is = 0; is < n_state; is += BLOCK) {
const int ic_end = std::min(ic + BLOCK, n_ctx);
const int is_end = std::min(is + BLOCK, n_state);
for (int i = ic; i < ic_end; ++i) {
for (int j = is; j < is_end; ++j) {
dv[j * n_ctx + i] = layer_src_v[i * n_state + j];
}
}
}
}
}
}
void whisper_kv_cross_store_k_transpose_v_layers_f16(
const float * src_k,
const float * src_v,
uint8_t * dst_k,
uint8_t * dst_v,
const whisper_kv_cross_layout & layout) {
const int n_ctx = layout.n_ctx;
const int n_state = layout.n_state;
const int BLOCK = 32;
for (int il = 0; il < layout.n_layer; ++il) {
const float * layer_src_k = src_k + (size_t)il * layout.src_layer_elems;
const float * layer_src_v = src_v + (size_t)il * layout.src_layer_elems;
ggml_fp16_t * dk = (ggml_fp16_t *)(dst_k + layout.dst_layer_stride * (size_t)il);
ggml_fp16_t * dv = (ggml_fp16_t *)(dst_v + layout.dst_layer_stride * (size_t)il);
for (size_t i = 0; i < layout.layer_elems; ++i) {
dk[i] = ggml_fp32_to_fp16(layer_src_k[i] * layout.kscale);
}
for (int ic = 0; ic < n_ctx; ic += BLOCK) {
for (int is = 0; is < n_state; is += BLOCK) {
const int ic_end = std::min(ic + BLOCK, n_ctx);
const int is_end = std::min(is + BLOCK, n_state);
for (int i = ic; i < ic_end; ++i) {
for (int j = is; j < is_end; ++j) {
dv[j * n_ctx + i] = ggml_fp32_to_fp16(layer_src_v[i * n_state + j]);
}
}
}
}
}
}
} // namespace whisper_vitisai_helpers

118
src/vitisai/whisper-vitisai-helpers.h Normal file
View File

@ -0,0 +1,118 @@
#pragma once
#include "FlexMLClient.h"
#include "ggml.h"
#include <cstddef>
#include <cstdint>
#include <cstdio>
#include <string>
#include <vector>
namespace whisper_vitisai_helpers {
bool map_rai_file(const char * path, uint8_t ** buffer, size_t * size);
void unmap_rai_file(uint8_t * buffer, size_t size);
bool file_exists(const char * path);
const char * whisper_kv_type_name(ggml_type type);
const char * whisper_flexml_dtype_name(flexmlrt::client::DataType type);
bool whisper_flexml_dtype_to_ggml_type(
flexmlrt::client::DataType type,
ggml_type * ggml_dtype);
bool whisper_validate_cross_shape(
const char * tensor_name,
const std::vector<std::uint32_t> & model_shape,
int n_text_layer,
int n_ctx,
int n_state);
bool whisper_vitisai_bind_tensor_data(
const char * tensor_name,
struct ggml_tensor * runtime_tensor,
const std::vector<size_t> & expected_shape,
flexmlrt::client::ErtTensorType & io_tensor);
#if defined(WHISPER_DEBUG)
template <typename T>
void whisper_vitisai_print_shape(const std::vector<T> & shape) {
std::fprintf(stderr, "[");
for (size_t i = 0; i < shape.size(); ++i) {
std::fprintf(stderr, "%s%lld", i == 0 ? "" : ", ", (long long) shape[i]);
}
std::fprintf(stderr, "]");
}
#endif
// Model IO tensor indices and metadata sizes resolved once at init time.
struct whisper_vitisai_io_binding {
int mel_in_idx = -1;
int embd_enc_out_idx = -1;
int cross_k_out_idx = -1;
int cross_v_out_idx = -1;
size_t mel_in_expected_bytes = 0;
size_t embd_enc_expected_bytes = 0;
size_t cross_k_expected_bytes = 0;
size_t cross_v_expected_bytes = 0;
};
// Warnings are printed with the caller's name; hard failures are returned in *error
// so the caller can decide how to report them.
bool whisper_vitisai_resolve_io_binding(
const char * caller,
const std::vector<flexmlrt::client::ErtTensorType> & input_tensors,
const std::vector<flexmlrt::client::ErtTensorType> & output_tensors,
whisper_vitisai_io_binding * binding,
std::string * error);
bool whisper_vitisai_all_tensors_claimed(
const char * caller,
const char * tensor_kind,
const std::vector<flexmlrt::client::ErtTensorType> & tensors,
const std::vector<bool> & claimed);
// Geometry of one cross K/V transfer from the model output (always f32, contiguous
// [ctx, state] per layer) into the runtime kv cache.
struct whisper_kv_cross_layout {
int n_layer = 0;
int n_ctx = 0;
int n_state = 0;
size_t src_layer_elems = 0; // f32 elements per layer in the model output buffer
size_t layer_elems = 0; // elements per layer transferred into the kv cache
size_t dst_layer_stride = 0; // bytes per layer in the kv cache
float kscale = 1.0f;
};
void whisper_kv_cross_scale_k_f32(
float * k_data,
size_t count,
float kscale);
void whisper_kv_cross_store_layers_f32(
const float * src_k,
const float * src_v,
uint8_t * dst_k,
uint8_t * dst_v,
const whisper_kv_cross_layout & layout);
void whisper_kv_cross_store_layers_f16(
const float * src_k,
const float * src_v,
uint8_t * dst_k,
uint8_t * dst_v,
const whisper_kv_cross_layout & layout);
void whisper_kv_cross_transpose_v_layers_f32(
const float * src_v,
uint8_t * dst_v,
const whisper_kv_cross_layout & layout);
void whisper_kv_cross_store_k_transpose_v_layers_f16(
const float * src_k,
const float * src_v,
uint8_t * dst_k,
uint8_t * dst_v,
const whisper_kv_cross_layout & layout);
} // namespace whisper_vitisai_helpers

43
src/whisper.cpp
View File

@ -1987,6 +1987,18 @@ static bool whisper_encode_external(const whisper_state & wstate) {
return use_coreml || use_openvino || use_vitisai;
}
static bool whisper_cross_external(const whisper_state & wstate) {
GGML_UNUSED(wstate);
#if defined(WHISPER_USE_VITISAI)
const bool use_vitisai_cross = whisper_vitisai_has_cross_proj(wstate.ctx_vitisai);
#else
const bool use_vitisai_cross = false;
#endif
return use_vitisai_cross;
}
static struct ggml_cgraph * whisper_build_graph_conv(
whisper_context & wctx,
whisper_state & wstate) {
@ -2426,7 +2438,20 @@ static bool whisper_encode_internal(
#if defined(WHISPER_USE_COREML)
whisper_coreml_encode(wstate.ctx_coreml, mel->ne[0], mel->ne[1], (float *) mel->data, (float *) wstate.embd_enc->data);
#elif defined(WHISPER_USE_VITISAI)
whisper_vitisai_encode(wstate.ctx_vitisai, mel, wstate.embd_enc);
if (whisper_vitisai_has_cross_proj(wstate.ctx_vitisai)) {
const auto & hp = wctx.model.hparams;
const int n_ctx = wstate.exp_n_audio_ctx > 0
? wstate.exp_n_audio_ctx : hp.n_audio_ctx;
if (!whisper_vitisai_encode_with_cross(
wstate.ctx_vitisai, mel, wstate.embd_enc,
wstate.kv_cross.k, wstate.kv_cross.v,
hp.n_text_layer, n_ctx, hp.n_text_state,
hp.n_text_head, wctx.params.flash_attn)) {
return false;
}
} else if (!whisper_vitisai_encode(wstate.ctx_vitisai, mel, wstate.embd_enc)) {
return false;
}
#elif defined(WHISPER_USE_OPENVINO)
whisper_openvino_encode(wstate.ctx_openvino, mel, wstate.embd_enc);
#endif
@ -2450,7 +2475,7 @@ static bool whisper_encode_internal(
}
// cross
{
if (!whisper_cross_external(wstate)) {
auto & sched = wstate.sched_cross.sched;
ggml_cgraph * gf = whisper_build_graph_cross(wctx, wstate);
@ -3370,9 +3395,13 @@ static std::string whisper_get_vitisai_path_encoder_cache(std::string path_bin)
path_bin = path_bin.substr(0, pos);
}
path_bin += "-encoder-vitisai.rai";
const std::string path_vitisai_cross = path_bin + "-encoder-cross-vitisai.rai";
if (FILE * file = fopen(path_vitisai_cross.c_str(), "rb")) {
fclose(file);
return path_vitisai_cross;
}
return path_bin;
return path_bin + "-encoder-vitisai.rai";
}
#endif
@ -3493,8 +3522,10 @@ struct whisper_state * whisper_init_state(whisper_context * ctx) {
WHISPER_LOG_ERROR("%s: failed to load Vitis AI model from '%s'\n", __func__, path_vitisai.c_str());
whisper_free_state(state);
return nullptr;
} else if (whisper_vitisai_has_cross_proj(state->ctx_vitisai)) {
WHISPER_LOG_INFO("%s: Vitis AI encoder + cross projection model loaded\n", __func__);
} else {
WHISPER_LOG_INFO("%s: Vitis AI model loaded\n", __func__);
WHISPER_LOG_INFO("%s: Vitis AI encoder model loaded\n", __func__);
}
#endif
@ -3545,7 +3576,7 @@ struct whisper_state * whisper_init_state(whisper_context * ctx) {
}
// cross allocator
{
if (!whisper_cross_external(*state)) {
bool ok = whisper_sched_graph_init(state->sched_cross, state->backends,
[&]() {
return whisper_build_graph_cross(*ctx, *state);