mirror of
https://github.com/ggml-org/whisper.cpp.git
synced 2026-09-30 11:36:38 +02:00
parakeet : add support for NVIDIA Parakeet (#3735)
* parakeet : add support for NVIDIA Parakeet Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
This commit is contained in:
Daniel Bevenius
co-authored by
Georgi Gerganov
Georgi Gerganov
parent
3805e602d3
commit
9efddafb91
@@ -107,6 +107,8 @@ else()
|
||||
add_subdirectory(server)
|
||||
add_subdirectory(quantize)
|
||||
add_subdirectory(vad-speech-segments)
|
||||
add_subdirectory(parakeet-cli)
|
||||
add_subdirectory(parakeet-quantize)
|
||||
if (WHISPER_SDL2)
|
||||
add_subdirectory(stream)
|
||||
add_subdirectory(command)
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
set(TARGET parakeet-cli)
|
||||
add_executable(${TARGET} parakeet-cli.cpp)
|
||||
|
||||
include(DefaultTargetOptions)
|
||||
|
||||
target_link_libraries(${TARGET} PRIVATE common parakeet ${FFMPEG_LIBRARIES} ${CMAKE_THREAD_LIBS_INIT})
|
||||
|
||||
install(TARGETS ${TARGET} RUNTIME)
|
||||
@@ -0,0 +1,106 @@
|
||||
# whisper.cpp/examples/parakeet-cli
|
||||
|
||||
This is an example of using the [Parakeet] model in whisper.cpp.
|
||||
|
||||
### Download converted model
|
||||
```console
|
||||
$ hf download ggml-org/parakeet-GGUF parakeet-tdt-0.6b-v3-f16.bin --local-dir models
|
||||
```
|
||||
|
||||
### Building
|
||||
```console
|
||||
$ cmake -B build -S .
|
||||
$ cmake --build build --target parakeet-cli -j 12
|
||||
```
|
||||
|
||||
### Usage
|
||||
```console
|
||||
$ ./build/bin/parakeet-cli --help
|
||||
|
||||
usage: ./build/bin/parakeet-cli [options] file0 file1 ...
|
||||
supported audio formats: flac, mp3, ogg, wav
|
||||
|
||||
options:
|
||||
-h, --help [default] show this help message and exit
|
||||
-t N, --threads N [4 ] number of threads to use during computation
|
||||
-m, --model FILE [models/ggml-parakeet-tdt-0.6b-v3.bin] model path
|
||||
-f, --file FILE [ ] input audio file
|
||||
-ng, --no-gpu [false ] disable GPU
|
||||
-dev N, --device N [0 ] GPU device to use
|
||||
-ps, --print-segments [false ] print segment information
|
||||
```
|
||||
|
||||
### Example
|
||||
```console
|
||||
$ ./build/bin/parakeet-cli -m models/parakeet-tdt-0.6b-v3-f16.bin -f samples/jfk.wav
|
||||
Processing audio (176000 samples, 11.00 seconds)
|
||||
Processing audio: total_frames=1101, chunk_size=1101
|
||||
parakeet_decode: starting decode with n_frames=138
|
||||
And so, my fellow Americans, ask not what your country can do for you, ask what you can do for your country.
|
||||
```
|
||||
|
||||
To print segment information:
|
||||
```console
|
||||
$ ./build/bin/parakeet-cli -m models/parakeet-tdt-0.6b-v3-f16.bin -f samples/jfk.wav --print-segments
|
||||
Processing audio (176000 samples, 11.00 seconds)
|
||||
Processing audio: total_frames=1101, chunk_size=1101
|
||||
parakeet_decode: starting decode with n_frames=138
|
||||
And so, my fellow Americans, ask not what your country can do for you, ask what you can do for your country.
|
||||
|
||||
Segments (1):
|
||||
Segment 0: [0 -> 1101] "And so, my fellow Americans, ask not what your country can do for you, ask what you can do for your country."
|
||||
Tokens [38]:
|
||||
[ 0] id= 1976 frame= 3 dur_idx= 4 dur_val= 4 p=0.9996 plog=-15.6206 t0= 24 t1= 56 word_start=true "▁And"
|
||||
[ 1] id= 547 frame= 7 dur_idx= 4 dur_val= 4 p=0.9999 plog=-18.7922 t0= 56 t1= 88 word_start=true "▁so"
|
||||
[ 2] id= 7877 frame= 11 dur_idx= 2 dur_val= 2 p=0.8451 plog=-14.5929 t0= 88 t1= 88 word_start=false ","
|
||||
[ 3] id= 1103 frame= 13 dur_idx= 3 dur_val= 3 p=0.9996 plog=-15.6127 t0= 104 t1= 128 word_start=true "▁my"
|
||||
[ 4] id= 309 frame= 16 dur_idx= 1 dur_val= 1 p=0.9912 plog=-11.9635 t0= 128 t1= 136 word_start=true "▁f"
|
||||
[ 5] id= 530 frame= 17 dur_idx= 2 dur_val= 2 p=1.0000 plog=-13.5239 t0= 136 t1= 152 word_start=false "ell"
|
||||
[ 6] id= 596 frame= 19 dur_idx= 3 dur_val= 3 p=1.0000 plog=-16.3120 t0= 152 t1= 176 word_start=false "ow"
|
||||
[ 7] id= 3213 frame= 22 dur_idx= 4 dur_val= 4 p=0.9999 plog=-10.1462 t0= 176 t1= 208 word_start=true "▁Amer"
|
||||
[ 8] id= 404 frame= 26 dur_idx= 4 dur_val= 4 p=1.0000 plog=-25.0910 t0= 208 t1= 240 word_start=false "ic"
|
||||
[ 9] id= 667 frame= 30 dur_idx= 4 dur_val= 4 p=1.0000 plog=-27.1707 t0= 240 t1= 272 word_start=false "ans"
|
||||
[10] id= 7877 frame= 37 dur_idx= 4 dur_val= 4 p=0.9094 plog=-16.3405 t0= 272 t1= 272 word_start=false ","
|
||||
[11] id= 279 frame= 41 dur_idx= 4 dur_val= 4 p=0.9980 plog=-19.7244 t0= 328 t1= 360 word_start=true "▁a"
|
||||
[12] id= 583 frame= 45 dur_idx= 4 dur_val= 4 p=1.0000 plog=-24.5312 t0= 360 t1= 392 word_start=false "sk"
|
||||
[13] id= 1491 frame= 53 dur_idx= 4 dur_val= 4 p=1.0000 plog=-23.2991 t0= 424 t1= 456 word_start=true "▁not"
|
||||
[14] id= 3470 frame= 65 dur_idx= 4 dur_val= 4 p=0.9995 plog=-16.7306 t0= 520 t1= 552 word_start=true "▁what"
|
||||
[15] id= 3629 frame= 69 dur_idx= 2 dur_val= 2 p=0.8139 plog=-11.6486 t0= 552 t1= 568 word_start=true "▁your"
|
||||
[16] id= 867 frame= 75 dur_idx= 1 dur_val= 1 p=0.9980 plog=-12.5265 t0= 600 t1= 608 word_start=true "▁co"
|
||||
[17] id= 331 frame= 76 dur_idx= 2 dur_val= 2 p=1.0000 plog=-11.6697 t0= 608 t1= 624 word_start=false "un"
|
||||
[18] id= 958 frame= 78 dur_idx= 2 dur_val= 2 p=1.0000 plog=-11.3621 t0= 624 t1= 640 word_start=false "tr"
|
||||
[19] id= 7893 frame= 80 dur_idx= 2 dur_val= 2 p=1.0000 plog=-14.3245 t0= 640 t1= 656 word_start=false "y"
|
||||
[20] id= 2059 frame= 82 dur_idx= 3 dur_val= 3 p=1.0000 plog=-17.7694 t0= 656 t1= 680 word_start=true "▁can"
|
||||
[21] id= 458 frame= 85 dur_idx= 4 dur_val= 4 p=1.0000 plog=-23.2510 t0= 680 t1= 712 word_start=true "▁do"
|
||||
[22] id= 509 frame= 89 dur_idx= 4 dur_val= 4 p=1.0000 plog=-23.0688 t0= 712 t1= 744 word_start=true "▁for"
|
||||
[23] id= 1180 frame= 93 dur_idx= 4 dur_val= 4 p=0.9999 plog=-25.0567 t0= 744 t1= 776 word_start=true "▁you"
|
||||
[24] id= 7877 frame= 98 dur_idx= 4 dur_val= 4 p=0.8820 plog=-14.2549 t0= 776 t1= 776 word_start=false ","
|
||||
[25] id= 279 frame=102 dur_idx= 3 dur_val= 3 p=0.9992 plog=-16.8176 t0= 816 t1= 840 word_start=true "▁a"
|
||||
[26] id= 583 frame=105 dur_idx= 4 dur_val= 4 p=1.0000 plog=-21.0352 t0= 840 t1= 872 word_start=false "sk"
|
||||
[27] id= 3470 frame=109 dur_idx= 3 dur_val= 3 p=0.9999 plog=-15.4659 t0= 872 t1= 896 word_start=true "▁what"
|
||||
[28] id= 1180 frame=112 dur_idx= 4 dur_val= 4 p=0.9997 plog=-17.6392 t0= 896 t1= 928 word_start=true "▁you"
|
||||
[29] id= 2059 frame=116 dur_idx= 3 dur_val= 3 p=0.9999 plog=-15.5484 t0= 928 t1= 952 word_start=true "▁can"
|
||||
[30] id= 458 frame=119 dur_idx= 2 dur_val= 2 p=1.0000 plog=-15.9953 t0= 952 t1= 968 word_start=true "▁do"
|
||||
[31] id= 509 frame=121 dur_idx= 3 dur_val= 3 p=1.0000 plog=-15.9605 t0= 968 t1= 992 word_start=true "▁for"
|
||||
[32] id= 3629 frame=124 dur_idx= 2 dur_val= 2 p=0.9994 plog=-12.2083 t0= 992 t1=1008 word_start=true "▁your"
|
||||
[33] id= 867 frame=126 dur_idx= 2 dur_val= 2 p=0.9969 plog=-9.1252 t0=1008 t1=1024 word_start=true "▁co"
|
||||
[34] id= 331 frame=128 dur_idx= 1 dur_val= 1 p=0.9999 plog=-12.6911 t0=1024 t1=1032 word_start=false "un"
|
||||
[35] id= 958 frame=129 dur_idx= 1 dur_val= 1 p=1.0000 plog=-8.8885 t0=1032 t1=1040 word_start=false "tr"
|
||||
[36] id= 7893 frame=130 dur_idx= 2 dur_val= 2 p=1.0000 plog=-14.1441 t0=1040 t1=1056 word_start=false "y"
|
||||
[37] id= 7883 frame=132 dur_idx= 4 dur_val= 4 p=0.9567 plog=-11.5227 t0=1056 t1=1056 word_start=false "."
|
||||
```
|
||||
|
||||
### Model conversion
|
||||
Clone the original model from Hugging Face:
|
||||
```console
|
||||
$ git clone https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3
|
||||
```
|
||||
Convert the model:
|
||||
```console
|
||||
(venv) $ python models/convert-parakeet-to-ggml.py \
|
||||
--model <path to cloned model> \
|
||||
--out-dir models \
|
||||
--out-name ggml-parakeet-tdt-0.6b-v3-f16.bin
|
||||
```
|
||||
|
||||
[Parakeet]: https://huggingface.co/nvidia/parakeet-tdt-0.6b-v3
|
||||
@@ -0,0 +1,243 @@
|
||||
#include "parakeet.h"
|
||||
#include "common-whisper.h"
|
||||
|
||||
#include <cstdio>
|
||||
#include <string>
|
||||
#include <thread>
|
||||
#include <vector>
|
||||
#include <cstring>
|
||||
#include <fstream>
|
||||
|
||||
// command-line parameters
|
||||
struct parakeet_params {
|
||||
int32_t n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
|
||||
|
||||
bool use_gpu = true;
|
||||
int32_t gpu_device = 0;
|
||||
|
||||
bool print_segments = false;
|
||||
bool output_txt = false;
|
||||
bool no_prints = false;
|
||||
|
||||
std::string model = "models/ggml-parakeet-tdt-0.6b-v3.bin";
|
||||
std::string output_file = "";
|
||||
std::vector<std::string> fname_inp = {};
|
||||
};
|
||||
|
||||
static void parakeet_print_usage(int argc, char ** argv, const parakeet_params & params);
|
||||
|
||||
static char * requires_value_error(const std::string & arg) {
|
||||
fprintf(stderr, "error: argument %s requires value\n", arg.c_str());
|
||||
exit(1);
|
||||
}
|
||||
|
||||
static bool parakeet_params_parse(int argc, char ** argv, parakeet_params & params) {
|
||||
if (const char * env_device = std::getenv("PARAKEET_ARG_DEVICE")) {
|
||||
params.gpu_device = std::stoi(env_device);
|
||||
}
|
||||
|
||||
for (int i = 1; i < argc; i++) {
|
||||
std::string arg = argv[i];
|
||||
|
||||
if (arg == "-"){
|
||||
params.fname_inp.push_back(arg);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg[0] != '-') {
|
||||
params.fname_inp.push_back(arg);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (arg == "-h" || arg == "--help") {
|
||||
parakeet_print_usage(argc, argv, params);
|
||||
exit(0);
|
||||
}
|
||||
#define ARGV_NEXT (((i + 1) < argc) ? argv[++i] : requires_value_error(arg))
|
||||
else if (arg == "-t" || arg == "--threads") { params.n_threads = std::stoi(ARGV_NEXT); }
|
||||
else if (arg == "-m" || arg == "--model") { params.model = ARGV_NEXT; }
|
||||
else if (arg == "-f" || arg == "--file") { params.fname_inp.emplace_back(ARGV_NEXT); }
|
||||
else if (arg == "-ng" || arg == "--no-gpu") { params.use_gpu = false; }
|
||||
else if (arg == "-dev" || arg == "--device") { params.gpu_device = std::stoi(ARGV_NEXT); }
|
||||
else if (arg == "-ps" || arg == "--print-segments") { params.print_segments = true; }
|
||||
else if (arg == "-otxt" || arg == "--output-txt") { params.output_txt = true; }
|
||||
else if (arg == "-of" || arg == "--output-file") { params.output_file = ARGV_NEXT; }
|
||||
else if (arg == "-np" || arg == "--no-prints") { params.no_prints = true; }
|
||||
else {
|
||||
fprintf(stderr, "error: unknown argument: %s\n", arg.c_str());
|
||||
parakeet_print_usage(argc, argv, params);
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
static void parakeet_print_usage(int /*argc*/, char ** argv, const parakeet_params & params) {
|
||||
fprintf(stderr, "\n");
|
||||
fprintf(stderr, "usage: %s [options] file0 file1 ...\n", argv[0]);
|
||||
fprintf(stderr, "supported audio formats: flac, mp3, ogg, wav\n");
|
||||
fprintf(stderr, "\n");
|
||||
fprintf(stderr, "options:\n");
|
||||
fprintf(stderr, " -h, --help [default] show this help message and exit\n");
|
||||
fprintf(stderr, " -t N, --threads N [%-7d] number of threads to use during computation\n", params.n_threads);
|
||||
fprintf(stderr, " -m, --model FILE [%-7s] model path\n", params.model.c_str());
|
||||
fprintf(stderr, " -f, --file FILE [%-7s] input audio file\n", "");
|
||||
fprintf(stderr, " -ng, --no-gpu [%-7s] disable GPU\n", params.use_gpu ? "false" : "true");
|
||||
fprintf(stderr, " -dev N, --device N [%-7d] GPU device to use\n", params.gpu_device);
|
||||
fprintf(stderr, " -ps, --print-segments [%-7s] print segment information\n", params.print_segments ? "true" : "false");
|
||||
fprintf(stderr, " -otxt, --output-txt [%-7s] output result in a text file\n", params.output_txt ? "true" : "false");
|
||||
fprintf(stderr, " -of, --output-file FILE [%-7s] output file path (without file extension)\n", "");
|
||||
fprintf(stderr, " -np, --no-prints [%-7s] do not print anything other than the results\n", params.no_prints ? "true" : "false");
|
||||
fprintf(stderr, "\n");
|
||||
}
|
||||
|
||||
void token_callback(parakeet_context * ctx, parakeet_state * state, const parakeet_token_data * token_data, void * user_data) {
|
||||
bool * is_first = (bool *) user_data;
|
||||
|
||||
const char * token_str = parakeet_token_to_str(ctx, token_data->id);
|
||||
char text_buf[256];
|
||||
parakeet_token_to_text(token_str, *is_first, text_buf, sizeof(text_buf));
|
||||
printf("%s", text_buf);
|
||||
fflush(stdout);
|
||||
|
||||
*is_first = false;
|
||||
}
|
||||
|
||||
static void cb_log_disable(enum ggml_log_level , const char * , void * ) { }
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
ggml_backend_load_all();
|
||||
|
||||
parakeet_params params;
|
||||
|
||||
if (parakeet_params_parse(argc, argv, params) == false) {
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (params.no_prints) {
|
||||
parakeet_log_set(cb_log_disable, NULL);
|
||||
}
|
||||
|
||||
if (params.fname_inp.empty()) {
|
||||
fprintf(stderr, "error: no input files specified\n");
|
||||
parakeet_print_usage(argc, argv, params);
|
||||
return 1;
|
||||
}
|
||||
|
||||
struct parakeet_context_params ctx_params = parakeet_context_default_params();
|
||||
ctx_params.use_gpu = params.use_gpu;
|
||||
ctx_params.gpu_device = params.gpu_device;
|
||||
|
||||
if (!params.no_prints) {
|
||||
fprintf(stderr, "Loading Parakeet model from: %s\n", params.model.c_str());
|
||||
}
|
||||
|
||||
|
||||
struct parakeet_context * pctx = parakeet_init_from_file_with_params(params.model.c_str(), ctx_params);
|
||||
if (pctx == nullptr) {
|
||||
fprintf(stderr, "error: failed to load Parakeet model from '%s'\n", params.model.c_str());
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (!params.no_prints) {
|
||||
fprintf(stderr, "Successfully loaded Parakeet model\n");
|
||||
fprintf(stderr, "system_info: n_threads = %d / %d | %s\n",
|
||||
params.n_threads, (int32_t) std::thread::hardware_concurrency(), parakeet_print_system_info());
|
||||
}
|
||||
|
||||
// Process each input file
|
||||
for (const auto & fname : params.fname_inp) {
|
||||
if (!params.no_prints) {
|
||||
fprintf(stderr, "\nProcessing file: %s\n", fname.c_str());
|
||||
}
|
||||
|
||||
std::vector<float> pcmf32;
|
||||
std::vector<std::vector<float>> pcmf32s;
|
||||
if (!read_audio_data(fname.c_str(), pcmf32, pcmf32s, false)) {
|
||||
fprintf(stderr, "error: failed to read audio file '%s'\n", fname.c_str());
|
||||
continue;
|
||||
}
|
||||
|
||||
if (pcmf32.empty()) {
|
||||
fprintf(stderr, "error: no audio data in file '%s'\n", fname.c_str());
|
||||
continue;
|
||||
}
|
||||
|
||||
bool is_first = true;
|
||||
struct parakeet_full_params full_params = parakeet_full_default_params(PARAKEET_SAMPLING_GREEDY);
|
||||
full_params.n_threads = params.n_threads;
|
||||
full_params.new_token_callback = token_callback;
|
||||
full_params.new_token_callback_user_data = &is_first;
|
||||
|
||||
const int mel_frames = (int)(pcmf32.size() / PARAKEET_HOP_LENGTH);
|
||||
int ret = parakeet_full(pctx, full_params, pcmf32.data(), pcmf32.size());
|
||||
|
||||
if (ret != 0) {
|
||||
fprintf(stderr, "error: failed to process audio file '%s'\n", fname.c_str());
|
||||
continue;
|
||||
}
|
||||
|
||||
printf("\n");
|
||||
|
||||
if (params.output_txt) {
|
||||
const std::string fname_out = (!params.output_file.empty() ? params.output_file : fname) + ".txt";
|
||||
|
||||
std::ofstream fout(fname_out);
|
||||
if (fout.is_open()) {
|
||||
const int n_segments = parakeet_full_n_segments(pctx);
|
||||
for (int i = 0; i < n_segments; ++i) {
|
||||
const char * text = parakeet_full_get_segment_text(pctx, i);
|
||||
fout << text << "\n";
|
||||
}
|
||||
fout.close();
|
||||
if (!params.no_prints) {
|
||||
fprintf(stderr, "Output written to: %s\n", fname_out.c_str());
|
||||
}
|
||||
} else {
|
||||
fprintf(stderr, "error: failed to open '%s' for writing\n", fname_out.c_str());
|
||||
}
|
||||
}
|
||||
|
||||
if (!params.no_prints) {
|
||||
parakeet_print_timings(pctx);
|
||||
}
|
||||
|
||||
if (params.print_segments) {
|
||||
const int n_segments = parakeet_full_n_segments(pctx);
|
||||
fprintf(stderr, "\nSegments (%d):\n", n_segments);
|
||||
|
||||
for (int i = 0; i < n_segments; i++) {
|
||||
const char * text = parakeet_full_get_segment_text(pctx, i);
|
||||
const int64_t t0 = parakeet_full_get_segment_t0(pctx, i);
|
||||
const int64_t t1 = parakeet_full_get_segment_t1(pctx, i);
|
||||
const int n_tokens = parakeet_full_n_tokens(pctx, i);
|
||||
|
||||
fprintf(stderr, "Segment %d: [%lld -> %lld] \"%s\"\n", i, (long long)t0, (long long)t1, text);
|
||||
fprintf(stderr, "Tokens [%d]:\n", n_tokens);
|
||||
|
||||
for (int j = 0; j < n_tokens; j++) {
|
||||
parakeet_token_data token_data = parakeet_full_get_token_data(pctx, i, j);
|
||||
const char * token_str = parakeet_token_to_str(pctx, token_data.id);
|
||||
|
||||
fprintf(stderr, " [%2d] id=%5d frame=%3d dur_idx=%2d dur_val=%2d p=%.4f plog=%.4f t0=%4lld t1=%4lld word_start=%s \"%s\"\n",
|
||||
j,
|
||||
token_data.id,
|
||||
token_data.frame_index,
|
||||
token_data.duration_idx,
|
||||
token_data.duration_value,
|
||||
token_data.p,
|
||||
token_data.plog,
|
||||
(long long)token_data.t0,
|
||||
(long long)token_data.t1,
|
||||
token_data.is_word_start ? "true": "false",
|
||||
token_str);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
parakeet_free(pctx);
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
set(TARGET parakeet-quantize)
|
||||
add_executable(${TARGET} parakeet-quantize.cpp)
|
||||
|
||||
include(DefaultTargetOptions)
|
||||
|
||||
target_link_libraries(${TARGET} PRIVATE common parakeet ${CMAKE_THREAD_LIBS_INIT})
|
||||
install(TARGETS ${TARGET} RUNTIME)
|
||||
@@ -0,0 +1,230 @@
|
||||
#include "ggml.h"
|
||||
#include "ggml-backend.h"
|
||||
|
||||
#include "common-ggml.h"
|
||||
|
||||
#include <cassert>
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
#include <fstream>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
struct parakeet_hparams {
|
||||
int32_t n_vocab = 0;
|
||||
int32_t n_audio_ctx = 0;
|
||||
int32_t n_audio_state = 0;
|
||||
int32_t n_audio_head = 0;
|
||||
int32_t n_audio_layer = 0;
|
||||
int32_t n_mels = 0;
|
||||
int32_t ftype = 0;
|
||||
int32_t n_fft = 0;
|
||||
int32_t subsampling_factor = 0;
|
||||
int32_t n_subsampling_channels = 0;
|
||||
int32_t n_conv_kernel = 0;
|
||||
int32_t n_pred_dim = 0;
|
||||
int32_t n_pred_layers = 0;
|
||||
int32_t n_tdt_durations = 0;
|
||||
int32_t n_max_tokens = 0;
|
||||
};
|
||||
|
||||
static bool parakeet_model_quantize(const std::string & fname_inp, const std::string & fname_out, ggml_ftype ftype) {
|
||||
printf("%s: loading model from '%s'\n", __func__, fname_inp.c_str());
|
||||
|
||||
auto finp = std::ifstream(fname_inp, std::ios::binary);
|
||||
if (!finp) {
|
||||
fprintf(stderr, "%s: failed to open '%s' for reading\n", __func__, fname_inp.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
auto fout = std::ofstream(fname_out, std::ios::binary);
|
||||
if (!fout) {
|
||||
fprintf(stderr, "%s: failed to open '%s' for writing\n", __func__, fname_out.c_str());
|
||||
return false;
|
||||
}
|
||||
|
||||
// magic
|
||||
{
|
||||
uint32_t magic;
|
||||
finp.read((char *) &magic, sizeof(magic));
|
||||
if (magic != GGML_FILE_MAGIC) {
|
||||
fprintf(stderr, "%s: invalid model file (bad magic)\n", __func__);
|
||||
return false;
|
||||
}
|
||||
fout.write((char *) &magic, sizeof(magic));
|
||||
}
|
||||
|
||||
// hparams
|
||||
parakeet_hparams hparams;
|
||||
{
|
||||
finp.read((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
|
||||
finp.read((char *) &hparams.n_audio_ctx, sizeof(hparams.n_audio_ctx));
|
||||
finp.read((char *) &hparams.n_audio_state, sizeof(hparams.n_audio_state));
|
||||
finp.read((char *) &hparams.n_audio_head, sizeof(hparams.n_audio_head));
|
||||
finp.read((char *) &hparams.n_audio_layer, sizeof(hparams.n_audio_layer));
|
||||
finp.read((char *) &hparams.n_mels, sizeof(hparams.n_mels));
|
||||
finp.read((char *) &hparams.ftype, sizeof(hparams.ftype));
|
||||
finp.read((char *) &hparams.n_fft, sizeof(hparams.n_fft));
|
||||
finp.read((char *) &hparams.subsampling_factor, sizeof(hparams.subsampling_factor));
|
||||
finp.read((char *) &hparams.n_subsampling_channels, sizeof(hparams.n_subsampling_channels));
|
||||
finp.read((char *) &hparams.n_conv_kernel, sizeof(hparams.n_conv_kernel));
|
||||
finp.read((char *) &hparams.n_pred_dim, sizeof(hparams.n_pred_dim));
|
||||
finp.read((char *) &hparams.n_pred_layers, sizeof(hparams.n_pred_layers));
|
||||
finp.read((char *) &hparams.n_tdt_durations, sizeof(hparams.n_tdt_durations));
|
||||
finp.read((char *) &hparams.n_max_tokens, sizeof(hparams.n_max_tokens));
|
||||
|
||||
const int32_t qntvr_src = hparams.ftype / GGML_QNT_VERSION_FACTOR;
|
||||
const int32_t ftype_dst = GGML_QNT_VERSION * GGML_QNT_VERSION_FACTOR + ftype;
|
||||
|
||||
fprintf(stderr, "%s: n_vocab = %d\n", __func__, hparams.n_vocab);
|
||||
fprintf(stderr, "%s: n_audio_state = %d\n", __func__, hparams.n_audio_state);
|
||||
fprintf(stderr, "%s: n_audio_layer = %d\n", __func__, hparams.n_audio_layer);
|
||||
fprintf(stderr, "%s: n_mels = %d\n", __func__, hparams.n_mels);
|
||||
fprintf(stderr, "%s: ftype (src) = %d\n", __func__, hparams.ftype);
|
||||
fprintf(stderr, "%s: qntvr (src) = %d\n", __func__, qntvr_src);
|
||||
fprintf(stderr, "%s: ftype (dst) = %d\n", __func__, ftype_dst);
|
||||
fprintf(stderr, "%s: qntvr (dst) = %d\n", __func__, GGML_QNT_VERSION);
|
||||
|
||||
fout.write((char *) &hparams.n_vocab, sizeof(hparams.n_vocab));
|
||||
fout.write((char *) &hparams.n_audio_ctx, sizeof(hparams.n_audio_ctx));
|
||||
fout.write((char *) &hparams.n_audio_state, sizeof(hparams.n_audio_state));
|
||||
fout.write((char *) &hparams.n_audio_head, sizeof(hparams.n_audio_head));
|
||||
fout.write((char *) &hparams.n_audio_layer, sizeof(hparams.n_audio_layer));
|
||||
fout.write((char *) &hparams.n_mels, sizeof(hparams.n_mels));
|
||||
fout.write((char *) &ftype_dst, sizeof(ftype_dst));
|
||||
fout.write((char *) &hparams.n_fft, sizeof(hparams.n_fft));
|
||||
fout.write((char *) &hparams.subsampling_factor, sizeof(hparams.subsampling_factor));
|
||||
fout.write((char *) &hparams.n_subsampling_channels, sizeof(hparams.n_subsampling_channels));
|
||||
fout.write((char *) &hparams.n_conv_kernel, sizeof(hparams.n_conv_kernel));
|
||||
fout.write((char *) &hparams.n_pred_dim, sizeof(hparams.n_pred_dim));
|
||||
fout.write((char *) &hparams.n_pred_layers, sizeof(hparams.n_pred_layers));
|
||||
fout.write((char *) &hparams.n_tdt_durations, sizeof(hparams.n_tdt_durations));
|
||||
fout.write((char *) &hparams.n_max_tokens, sizeof(hparams.n_max_tokens));
|
||||
}
|
||||
|
||||
// mel filterbank
|
||||
{
|
||||
int32_t n_mel, n_fb;
|
||||
finp.read((char *) &n_mel, sizeof(n_mel));
|
||||
fout.write((char *) &n_mel, sizeof(n_mel));
|
||||
finp.read((char *) &n_fb, sizeof(n_fb));
|
||||
fout.write((char *) &n_fb, sizeof(n_fb));
|
||||
|
||||
const size_t n = (size_t) n_mel * n_fb;
|
||||
std::vector<float> buf(n);
|
||||
finp.read((char *) buf.data(), n * sizeof(float));
|
||||
fout.write((char *) buf.data(), n * sizeof(float));
|
||||
}
|
||||
|
||||
// window function
|
||||
{
|
||||
int32_t n_window;
|
||||
finp.read((char *) &n_window, sizeof(n_window));
|
||||
fout.write((char *) &n_window, sizeof(n_window));
|
||||
|
||||
std::vector<float> buf(n_window);
|
||||
finp.read((char *) buf.data(), n_window * sizeof(float));
|
||||
fout.write((char *) buf.data(), n_window * sizeof(float));
|
||||
}
|
||||
|
||||
// TDT durations
|
||||
{
|
||||
std::vector<uint32_t> buf(hparams.n_tdt_durations);
|
||||
finp.read((char *) buf.data(), hparams.n_tdt_durations * sizeof(uint32_t));
|
||||
fout.write((char *) buf.data(), hparams.n_tdt_durations * sizeof(uint32_t));
|
||||
}
|
||||
|
||||
// vocab
|
||||
{
|
||||
int32_t n_tokens;
|
||||
finp.read((char *) &n_tokens, sizeof(n_tokens));
|
||||
fout.write((char *) &n_tokens, sizeof(n_tokens));
|
||||
|
||||
for (int i = 0; i < n_tokens; ++i) {
|
||||
int32_t len;
|
||||
finp.read((char *) &len, sizeof(len));
|
||||
fout.write((char *) &len, sizeof(len));
|
||||
|
||||
std::string token(len, '\0');
|
||||
finp.read(&token[0], len);
|
||||
fout.write(&token[0], len);
|
||||
}
|
||||
}
|
||||
|
||||
// tensors — quantize 2D weights skipping tensors that must stay F32:
|
||||
// ggml_ssm_conv / ggml_conv2d_dw CUDA kernels require F32 weights.
|
||||
// pos_bias_u / pos_bias_v are declared F32 in the loader.
|
||||
const std::vector<std::string> to_quant = { ".*" };
|
||||
std::vector<std::string> to_skip = {
|
||||
// CUDA kernel constraints (ggml_ssm_conv / ggml_conv2d_dw require F32 weights)
|
||||
"encoder\\.layers\\..+\\.conv\\.depthwise_conv\\.weight",
|
||||
// Declared F32 in loader (pos_bias tensors)
|
||||
"encoder\\.layers\\..+\\.self_attn\\.pos_bias_u",
|
||||
"encoder\\.layers\\..+\\.self_attn\\.pos_bias_v",
|
||||
};
|
||||
|
||||
// Prediction/joint tensors use n_pred_dim as their inner dimension. K-quant
|
||||
// types (block size 256) cannot quantize 640 evenly, so keep them F32. For
|
||||
// other types (Q8_0, Q4_0, block size 32) 640 is divisible and they can be
|
||||
// quantized normally. The loader mirrors this logic at load time.
|
||||
{
|
||||
const ggml_type qtype = ggml_ftype_to_ggml_type(ftype);
|
||||
const int32_t blck = ggml_blck_size(qtype);
|
||||
if (blck > 1 && hparams.n_pred_dim % blck != 0) {
|
||||
to_skip.push_back("decoder\\.prediction\\.embed\\.weight");
|
||||
to_skip.push_back("decoder\\.prediction\\.dec_rnn\\.lstm\\.weight_ih_l.*");
|
||||
to_skip.push_back("decoder\\.prediction\\.dec_rnn\\.lstm\\.weight_hh_l.*");
|
||||
to_skip.push_back("joint\\.pred\\.weight");
|
||||
to_skip.push_back("joint\\.joint_net\\.2\\.weight");
|
||||
}
|
||||
}
|
||||
|
||||
if (!ggml_common_quantize_0(finp, fout, ftype, to_quant, to_skip)) {
|
||||
fprintf(stderr, "%s: failed to quantize tensors\n", __func__);
|
||||
return false;
|
||||
}
|
||||
|
||||
finp.close();
|
||||
fout.close();
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
int main(int argc, char ** argv) {
|
||||
ggml_backend_load_all();
|
||||
|
||||
if (argc != 4) {
|
||||
fprintf(stderr, "usage: %s model-f32.bin model-quant.bin type\n", argv[0]);
|
||||
ggml_print_ftypes(stderr);
|
||||
return 1;
|
||||
}
|
||||
|
||||
// initialise F16 lookup tables
|
||||
{
|
||||
struct ggml_init_params params = { 0, NULL, false };
|
||||
struct ggml_context * ctx = ggml_init(params);
|
||||
ggml_free(ctx);
|
||||
}
|
||||
|
||||
const std::string fname_inp = argv[1];
|
||||
const std::string fname_out = argv[2];
|
||||
const ggml_ftype ftype = ggml_parse_ftype(argv[3]);
|
||||
|
||||
if (ftype == GGML_FTYPE_UNKNOWN) {
|
||||
fprintf(stderr, "%s: invalid quantization type\n", argv[0]);
|
||||
ggml_print_ftypes(stderr);
|
||||
return 1;
|
||||
}
|
||||
|
||||
const int64_t t_start_us = ggml_time_us();
|
||||
|
||||
if (!parakeet_model_quantize(fname_inp, fname_out, ftype)) {
|
||||
fprintf(stderr, "%s: failed to quantize model from '%s'\n", argv[0], fname_inp.c_str());
|
||||
return 1;
|
||||
}
|
||||
|
||||
printf("\n%s: quantize time = %8.2f ms\n", argv[0], (ggml_time_us() - t_start_us) / 1000.0f);
|
||||
printf("%s: output model = %s\n", argv[0], fname_out.c_str());
|
||||
|
||||
return 0;
|
||||
}
|
||||
Reference in New Issue
Block a user