diff --git a/examples/cli/cli.cpp b/examples/cli/cli.cpp index c64976a56..5254326d9 100644 --- a/examples/cli/cli.cpp +++ b/examples/cli/cli.cpp @@ -523,10 +523,16 @@ static char * escape_double_quotes_and_backslashes(const char * str) { return NULL; } - size_t escaped_length = strlen(str) + 1; + // Drop bytes that are not valid UTF-8 so the emitted JSON is always well-formed: + // byte-level decoder tokens can leave a lone UTF-8 lead byte at a segment + // boundary, which would otherwise produce invalid UTF-8 output (issue #3760). + const std::string sanitized = utf8_sanitize(str); + const char * s = sanitized.c_str(); - for (size_t i = 0; str[i] != '\0'; i++) { - if (str[i] == '"' || str[i] == '\\') { + size_t escaped_length = sanitized.size() + 1; + + for (size_t i = 0; s[i] != '\0'; i++) { + if (s[i] == '"' || s[i] == '\\') { escaped_length++; } } @@ -537,11 +543,11 @@ static char * escape_double_quotes_and_backslashes(const char * str) { } size_t pos = 0; - for (size_t i = 0; str[i] != '\0'; i++) { - if (str[i] == '"' || str[i] == '\\') { + for (size_t i = 0; s[i] != '\0'; i++) { + if (s[i] == '"' || s[i] == '\\') { escaped[pos++] = '\\'; } - escaped[pos++] = str[i]; + escaped[pos++] = s[i]; } // no need to set zero due to calloc() being used prior diff --git a/examples/common-whisper.cpp b/examples/common-whisper.cpp index b12481c01..1cdd2a65e 100644 --- a/examples/common-whisper.cpp +++ b/examples/common-whisper.cpp @@ -226,6 +226,62 @@ int utf8_trailing_bytes_needed(const std::string & s) { return have >= expected ? 0 : (expected - have); } +std::string utf8_sanitize(const std::string & s) { + // Keep only well-formed UTF-8 sequences (Unicode Table 3-7). Any byte that + // starts an invalid/overlong/surrogate sequence, is an orphan continuation + // byte, or belongs to a truncated trailing sequence is dropped. + std::string out; + out.reserve(s.size()); + + const size_t n = s.size(); + size_t i = 0; + while (i < n) { + const unsigned char c = (unsigned char) s[i]; + + size_t len; + unsigned char lo = 0x80; // valid range for the first continuation byte + unsigned char hi = 0xBF; + if (c <= 0x7F) { + out.push_back((char) c); + ++i; + continue; + } else if (c >= 0xC2 && c <= 0xDF) { + len = 2; + } else if (c >= 0xE0 && c <= 0xEF) { + len = 3; + if (c == 0xE0) { lo = 0xA0; } // exclude overlong encodings + else if (c == 0xED) { hi = 0x9F; } // exclude UTF-16 surrogates + } else if (c >= 0xF0 && c <= 0xF4) { + len = 4; + if (c == 0xF0) { lo = 0x90; } // exclude overlong encodings + else if (c == 0xF4) { hi = 0x8F; } // exclude code points > U+10FFFF + } else { + ++i; // invalid lead byte (incl. orphan continuation 0x80..0xBF) + continue; + } + + if (i + len > n) { + ++i; // truncated sequence: drop the lead and rescan the rest + continue; + } + + bool ok = ((unsigned char) s[i + 1] >= lo && (unsigned char) s[i + 1] <= hi); + for (size_t k = 2; ok && k < len; ++k) { + const unsigned char cc = (unsigned char) s[i + k]; + ok = (cc >= 0x80 && cc <= 0xBF); + } + if (!ok) { + ++i; // malformed continuation byte: drop the lead and rescan + continue; + } + + out.append(s, i, len); + i += len; + } + + return out; +} + bool speak_with_file(const std::string & command, const std::string & text, const std::string & path, int voice_id) { std::ofstream speak_file(path.c_str()); if (speak_file.fail()) { diff --git a/examples/common-whisper.h b/examples/common-whisper.h index aec430d36..70b8a6c91 100644 --- a/examples/common-whisper.h +++ b/examples/common-whisper.h @@ -31,5 +31,10 @@ int timestamp_to_sample(int64_t t, int n_samples, int whisper_sample_rate); // Returns the number of trailing bytes still needed for s to end on a complete UTF-8 codepoint. int utf8_trailing_bytes_needed(const std::string & s); +// Returns s with every byte that is not part of a valid, complete UTF-8 sequence removed. +// Byte-level decoder tokens can leave a lone UTF-8 lead byte or an orphan continuation byte +// at a segment boundary; dropping them keeps text output well-formed UTF-8 (issue #3760). +std::string utf8_sanitize(const std::string & s); + // write text to file, and call system("command voice_id file") bool speak_with_file(const std::string & command, const std::string & text, const std::string & path, int voice_id); diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index aecc6f3b2..765d634c7 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -94,7 +94,7 @@ add_executable(${UTF8_TEST} ${UTF8_TEST}.cpp) target_include_directories(${UTF8_TEST} PRIVATE ../examples) target_link_libraries(${UTF8_TEST} PRIVATE common) add_test(NAME ${UTF8_TEST} COMMAND ${UTF8_TEST}) -set_tests_properties(${UTF8_TEST} PROPERTIES LABELS "unit") +set_tests_properties(${UTF8_TEST} PROPERTIES LABELS "unit;gh") # VAD test tests VAD in isolation set(VAD_TEST test-vad) diff --git a/tests/test-common-utf8.cpp b/tests/test-common-utf8.cpp index 91c73a742..81ebfe11b 100644 --- a/tests/test-common-utf8.cpp +++ b/tests/test-common-utf8.cpp @@ -12,6 +12,14 @@ static void expect_needed(const std::string & input, int expected) { } } +static void expect_sanitized(const std::string & input, const std::string & expected) { + const std::string actual = utf8_sanitize(input); + if (actual != expected) { + fprintf(stderr, "utf8_sanitize: expected %zu bytes, got %zu\n", expected.size(), actual.size()); + std::abort(); + } +} + int main() { expect_needed("", 0); expect_needed("plain ascii", 0); @@ -30,5 +38,31 @@ int main() { expect_needed("\x80\x80", 0); expect_needed("\xFF", 0); + // utf8_sanitize: valid input is preserved byte-for-byte + expect_sanitized("", ""); + expect_sanitized("plain ascii", "plain ascii"); + expect_sanitized(cjk, cjk); + expect_sanitized(emoji, emoji); + expect_sanitized("hi " + cjk + "!", "hi " + cjk + "!"); + + // issue #3760: a lone UTF-8 lead byte emitted as a whole segment's text + expect_sanitized("\xC3", ""); + expect_sanitized("\xC5", ""); + // a valid prefix followed by a trailing incomplete lead byte + expect_sanitized("A\xC3", "A"); + expect_sanitized(cjk + "\xC3", cjk); + // orphan continuation byte (the split character's tail in the next segment) + expect_sanitized("\x80", ""); + expect_sanitized("\xBF next", " next"); + // truncated multi-byte sequences + expect_sanitized(emoji.substr(0, 3), ""); + expect_sanitized(cjk.substr(0, 2), ""); + // invalid lead bytes and overlong / surrogate encodings + expect_sanitized("\xFF", ""); + expect_sanitized("\xC0\x80", ""); // overlong NUL + expect_sanitized("\xC1\xBF", ""); // overlong + expect_sanitized("\xED\xA0\x80", ""); // UTF-16 surrogate U+D800 + expect_sanitized("\xF4\x90\x80\x80", ""); // > U+10FFFF + return 0; }