public_git/whisper.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/whisper.cpp.git synced 2026-09-30 19:41:14 +02:00
Code Issues Packages Projects Releases Wiki Activity

vad : add initial Voice Activity Detection (VAD) support (#3065)

Browse Source 
* vad : add initial Voice Activity Detection (VAD) support

This commit add support for Voice Activity Detection (VAD). When enabled
this feature will process the audio input and detect speech segments.
This information is then used to reduce the number of samples that need
to be processed by whisper_full.

Resolves: https://github.com/ggml-org/whisper.cpp/issues/3003

---------

Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
This commit is contained in:
Daniel Bevenius
2025-05-12 16:10:11 +02:00
committed by GitHub
co-authored by Georgi Gerganov
parent e39ba750cd
commit e41bc5c61a
11 changed files with 1972 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/whisper-arch.h
+56
View File
@@ -139,3 +139,59 @@ static const std::map<asr_tensor, ggml_op> ASR_TENSOR_INFO = {
{ASR_TENSOR_ATTN_OUT_WEIGHT, GGML_OP_MUL_MAT},
{ASR_TENSOR_ATTN_OUT_BIAS, GGML_OP_ADD},
};
enum vad_tensor {
VAD_TENSOR_STFT_BASIS,
VAD_TENSOR_ENC_0_WEIGHT,
VAD_TENSOR_ENC_0_BIAS,
VAD_TENSOR_ENC_1_WEIGHT,
VAD_TENSOR_ENC_1_BIAS,
VAD_TENSOR_ENC_2_WEIGHT,
VAD_TENSOR_ENC_2_BIAS,
VAD_TENSOR_ENC_3_WEIGHT,
VAD_TENSOR_ENC_3_BIAS,
VAD_TENSOR_LSTM_WEIGHT_IH,
VAD_TENSOR_LSTM_WEIGHT_HH,
VAD_TENSOR_LSTM_BIAS_IH,
VAD_TENSOR_LSTM_BIAS_HH,
VAD_TENSOR_FINAL_CONV_WEIGHT,
VAD_TENSOR_FINAL_CONV_BIAS,
};
static const std::map<vad_tensor, ggml_op> VAD_TENSOR_OPS = {
{VAD_TENSOR_STFT_BASIS, GGML_OP_IM2COL},
{VAD_TENSOR_ENC_0_WEIGHT, GGML_OP_IM2COL},
{VAD_TENSOR_ENC_0_BIAS, GGML_OP_ADD},
{VAD_TENSOR_ENC_1_WEIGHT, GGML_OP_IM2COL},
{VAD_TENSOR_ENC_1_BIAS, GGML_OP_ADD},
{VAD_TENSOR_ENC_2_WEIGHT, GGML_OP_IM2COL},
{VAD_TENSOR_ENC_2_BIAS, GGML_OP_ADD},
{VAD_TENSOR_ENC_3_WEIGHT, GGML_OP_IM2COL},
{VAD_TENSOR_ENC_3_BIAS, GGML_OP_ADD},
{VAD_TENSOR_LSTM_WEIGHT_IH, GGML_OP_MUL_MAT},
{VAD_TENSOR_LSTM_WEIGHT_HH, GGML_OP_MUL_MAT},
{VAD_TENSOR_LSTM_BIAS_IH, GGML_OP_ADD},
{VAD_TENSOR_LSTM_BIAS_HH, GGML_OP_ADD},
{VAD_TENSOR_FINAL_CONV_WEIGHT, GGML_OP_IM2COL},
{VAD_TENSOR_FINAL_CONV_BIAS, GGML_OP_ADD}
};
static const std::map<vad_tensor, const char *> VAD_TENSOR_NAMES = {
{VAD_TENSOR_STFT_BASIS, "_model.stft.forward_basis_buffer"},
{VAD_TENSOR_ENC_0_WEIGHT, "_model.encoder.0.reparam_conv.weight"},
{VAD_TENSOR_ENC_0_BIAS, "_model.encoder.0.reparam_conv.bias"},
{VAD_TENSOR_ENC_1_WEIGHT, "_model.encoder.1.reparam_conv.weight"},
{VAD_TENSOR_ENC_1_BIAS, "_model.encoder.1.reparam_conv.bias"},
{VAD_TENSOR_ENC_2_WEIGHT, "_model.encoder.2.reparam_conv.weight"},
{VAD_TENSOR_ENC_2_BIAS, "_model.encoder.2.reparam_conv.bias"},
{VAD_TENSOR_ENC_3_WEIGHT, "_model.encoder.3.reparam_conv.weight"},
{VAD_TENSOR_ENC_3_BIAS, "_model.encoder.3.reparam_conv.bias"},
{VAD_TENSOR_LSTM_WEIGHT_IH, "_model.decoder.rnn.weight_ih"},
{VAD_TENSOR_LSTM_WEIGHT_HH, "_model.decoder.rnn.weight_hh"},
{VAD_TENSOR_LSTM_BIAS_IH, "_model.decoder.rnn.bias_ih"},
{VAD_TENSOR_LSTM_BIAS_HH, "_model.decoder.rnn.bias_hh"},
{VAD_TENSOR_FINAL_CONV_WEIGHT, "_model.decoder.decoder.2.weight"},
{VAD_TENSOR_FINAL_CONV_BIAS, "_model.decoder.decoder.2.bias"}
};
src/whisper.cpp
+1381 -11
View File
File diff suppressed because it is too large Load Diff