Merge 6a27b12f27 into 1fe009caed

examples/cli/cli.cpp

@@ -523,10 +523,16 @@ static char * escape_double_quotes_and_backslashes(const char * str) {
         return NULL;
     }
 
-    size_t escaped_length = strlen(str) + 1;
+    // Drop bytes that are not valid UTF-8 so the emitted JSON is always well-formed:
+    // byte-level decoder tokens can leave a lone UTF-8 lead byte at a segment
+    // boundary, which would otherwise produce invalid UTF-8 output (issue #3760).
+    const std::string sanitized = utf8_sanitize(str);
+    const char * s = sanitized.c_str();
 
-    for (size_t i = 0; str[i] != '\0'; i++) {
-        if (str[i] == '"' || str[i] == '\\') {
+    size_t escaped_length = sanitized.size() + 1;
+
+    for (size_t i = 0; s[i] != '\0'; i++) {
+        if (s[i] == '"' || s[i] == '\\') {
             escaped_length++;
         }
     }
@@ -537,11 +543,11 @@ static char * escape_double_quotes_and_backslashes(const char * str) {
     }
 
     size_t pos = 0;
-    for (size_t i = 0; str[i] != '\0'; i++) {
-        if (str[i] == '"' || str[i] == '\\') {
+    for (size_t i = 0; s[i] != '\0'; i++) {
+        if (s[i] == '"' || s[i] == '\\') {
             escaped[pos++] = '\\';
         }
-        escaped[pos++] = str[i];
+        escaped[pos++] = s[i];
     }
 
     // no need to set zero due to calloc() being used prior

examples/common-whisper.cpp

@@ -226,6 +226,62 @@ int utf8_trailing_bytes_needed(const std::string & s) {
     return have >= expected ? 0 : (expected - have);
 }
 
+std::string utf8_sanitize(const std::string & s) {
+    // Keep only well-formed UTF-8 sequences (Unicode Table 3-7). Any byte that
+    // starts an invalid/overlong/surrogate sequence, is an orphan continuation
+    // byte, or belongs to a truncated trailing sequence is dropped.
+    std::string out;
+    out.reserve(s.size());
+
+    const size_t n = s.size();
+    size_t i = 0;
+    while (i < n) {
+        const unsigned char c = (unsigned char) s[i];
+
+        size_t len;
+        unsigned char lo = 0x80; // valid range for the first continuation byte
+        unsigned char hi = 0xBF;
+        if (c <= 0x7F) {
+            out.push_back((char) c);
+            ++i;
+            continue;
+        } else if (c >= 0xC2 && c <= 0xDF) {
+            len = 2;
+        } else if (c >= 0xE0 && c <= 0xEF) {
+            len = 3;
+            if      (c == 0xE0) { lo = 0xA0; } // exclude overlong encodings
+            else if (c == 0xED) { hi = 0x9F; } // exclude UTF-16 surrogates
+        } else if (c >= 0xF0 && c <= 0xF4) {
+            len = 4;
+            if      (c == 0xF0) { lo = 0x90; } // exclude overlong encodings
+            else if (c == 0xF4) { hi = 0x8F; } // exclude code points > U+10FFFF
+        } else {
+            ++i; // invalid lead byte (incl. orphan continuation 0x80..0xBF)
+            continue;
+        }
+
+        if (i + len > n) {
+            ++i; // truncated sequence: drop the lead and rescan the rest
+            continue;
+        }
+
+        bool ok = ((unsigned char) s[i + 1] >= lo && (unsigned char) s[i + 1] <= hi);
+        for (size_t k = 2; ok && k < len; ++k) {
+            const unsigned char cc = (unsigned char) s[i + k];
+            ok = (cc >= 0x80 && cc <= 0xBF);
+        }
+        if (!ok) {
+            ++i; // malformed continuation byte: drop the lead and rescan
+            continue;
+        }
+
+        out.append(s, i, len);
+        i += len;
+    }
+
+    return out;
+}
+
 bool speak_with_file(const std::string & command, const std::string & text, const std::string & path, int voice_id) {
     std::ofstream speak_file(path.c_str());
     if (speak_file.fail()) {

examples/common-whisper.h

@@ -31,5 +31,10 @@ int timestamp_to_sample(int64_t t, int n_samples, int whisper_sample_rate);
 // Returns the number of trailing bytes still needed for s to end on a complete UTF-8 codepoint.
 int utf8_trailing_bytes_needed(const std::string & s);
 
+// Returns s with every byte that is not part of a valid, complete UTF-8 sequence removed.
+// Byte-level decoder tokens can leave a lone UTF-8 lead byte or an orphan continuation byte
+// at a segment boundary; dropping them keeps text output well-formed UTF-8 (issue #3760).
+std::string utf8_sanitize(const std::string & s);
+
 // write text to file, and call system("command voice_id file")
 bool speak_with_file(const std::string & command, const std::string & text, const std::string & path, int voice_id);

tests/CMakeLists.txt

@@ -94,7 +94,7 @@ add_executable(${UTF8_TEST} ${UTF8_TEST}.cpp)
 target_include_directories(${UTF8_TEST} PRIVATE ../examples)
 target_link_libraries(${UTF8_TEST} PRIVATE common)
 add_test(NAME ${UTF8_TEST} COMMAND ${UTF8_TEST})
-set_tests_properties(${UTF8_TEST} PROPERTIES LABELS "unit")
+set_tests_properties(${UTF8_TEST} PROPERTIES LABELS "unit;gh")
 
 # VAD test tests VAD in isolation
 set(VAD_TEST test-vad)

tests/test-common-utf8.cpp

@@ -12,6 +12,14 @@ static void expect_needed(const std::string & input, int expected) {
     }
 }
 
+static void expect_sanitized(const std::string & input, const std::string & expected) {
+    const std::string actual = utf8_sanitize(input);
+    if (actual != expected) {
+        fprintf(stderr, "utf8_sanitize: expected %zu bytes, got %zu\n", expected.size(), actual.size());
+        std::abort();
+    }
+}
+
 int main() {
     expect_needed("", 0);
     expect_needed("plain ascii", 0);
@@ -30,5 +38,31 @@ int main() {
     expect_needed("\x80\x80", 0);
     expect_needed("\xFF", 0);
 
+    // utf8_sanitize: valid input is preserved byte-for-byte
+    expect_sanitized("", "");
+    expect_sanitized("plain ascii", "plain ascii");
+    expect_sanitized(cjk, cjk);
+    expect_sanitized(emoji, emoji);
+    expect_sanitized("hi " + cjk + "!", "hi " + cjk + "!");
+
+    // issue #3760: a lone UTF-8 lead byte emitted as a whole segment's text
+    expect_sanitized("\xC3", "");
+    expect_sanitized("\xC5", "");
+    // a valid prefix followed by a trailing incomplete lead byte
+    expect_sanitized("A\xC3", "A");
+    expect_sanitized(cjk + "\xC3", cjk);
+    // orphan continuation byte (the split character's tail in the next segment)
+    expect_sanitized("\x80", "");
+    expect_sanitized("\xBF next", " next");
+    // truncated multi-byte sequences
+    expect_sanitized(emoji.substr(0, 3), "");
+    expect_sanitized(cjk.substr(0, 2), "");
+    // invalid lead bytes and overlong / surrogate encodings
+    expect_sanitized("\xFF", "");
+    expect_sanitized("\xC0\x80", "");         // overlong NUL
+    expect_sanitized("\xC1\xBF", "");         // overlong
+    expect_sanitized("\xED\xA0\x80", "");     // UTF-16 surrogate U+D800
+    expect_sanitized("\xF4\x90\x80\x80", "");  // > U+10FFFF
+
     return 0;
 }