public_git/whisper.cpp
1
0
Fork 0
mirror of https://github.com/ggml-org/whisper.cpp.git synced 2026-09-30 03:21:18 +02:00
Code Issues Packages Projects Releases Wiki Activity

server : merge split utf-8 token text in verbose json (#3850)

Browse Source
This commit is contained in:
Noah Lyons
2026-06-02 13:10:27 +02:00
committed by GitHub
parent ef24de1e58
commit e5d4412578
6 changed files with 94 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files
examples/cli/cli.cpp
-33
View File
@@ -31,39 +31,6 @@ static void replace_all(std::string & s, const std::string & search, const std::
}
}
// Returns the number of trailing continuation bytes still needed for `s` to end
// on a complete UTF-8 codepoint. Returns 0 if the tail of `s` is already a
// complete codepoint (or if the tail looks malformed and we should stop merging).
// Used to merge whisper tokens whose bytes split a multi-byte UTF-8 character
// (e.g. CJK), so the JSON output stays valid UTF-8. See https://github.com/ggml-org/whisper.cpp/issues/1798.
static int utf8_trailing_bytes_needed(const std::string & s) {
const int n = (int) s.size();
int i = n - 1;
// walk back past continuation bytes (10xxxxxx)
while (i >= 0 && ((unsigned char) s[i] & 0xC0) == 0x80) {
--i;
}
if (i < 0) {
// all continuation bytes, or empty — nothing we can do
return 0;
}
const unsigned char c = (unsigned char) s[i];
int expected;
if ((c & 0x80) == 0x00) {
expected = 1; // ASCII
} else if ((c & 0xE0) == 0xC0) {
expected = 2;
} else if ((c & 0xF0) == 0xE0) {
expected = 3;
} else if ((c & 0xF8) == 0xF0) {
expected = 4;
} else {
return 0; // malformed lead, give up
}
const int have = n - i;
return have >= expected ? 0 : (expected - have);
}
// command-line parameters
struct whisper_params {
int32_t n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());