diff --git a/android/app/src/test/cpp/CMakeLists.txt b/android/app/src/test/cpp/CMakeLists.txt index 6d47e4755..adf181e40 100644 --- a/android/app/src/test/cpp/CMakeLists.txt +++ b/android/app/src/test/cpp/CMakeLists.txt @@ -13,3 +13,8 @@ add_executable(ffmpeg_audio_buffer_test ffmpeg_audio_buffer_test.cpp) target_compile_features(ffmpeg_audio_buffer_test PRIVATE cxx_std_17) add_test(NAME ffmpeg_audio_buffer_test COMMAND ffmpeg_audio_buffer_test) + +add_executable(mpv_utf8_convert_test mpv_utf8_convert_test.cpp) +target_compile_features(mpv_utf8_convert_test PRIVATE cxx_std_17) + +add_test(NAME mpv_utf8_convert_test COMMAND mpv_utf8_convert_test) diff --git a/android/app/src/test/cpp/mpv_utf8_convert_test.cpp b/android/app/src/test/cpp/mpv_utf8_convert_test.cpp new file mode 100644 index 000000000..e99b9b81e --- /dev/null +++ b/android/app/src/test/cpp/mpv_utf8_convert_test.cpp @@ -0,0 +1,59 @@ +#include +#include + +#include "../../../../libmpv/src/main/cpp/utf8_convert.h" + +namespace { + +using plezy::utf8::FromUtf16; +using plezy::utf8::ToUtf16; + +bool check(bool condition, const char* message) { + if (!condition) std::fprintf(stderr, "%s\n", message); + return condition; +} + +bool roundTripsAsciiBmpAndSupplementary() { + // "a" U+00E9 U+4E2D U+1F3AC (clapper board) — 1/2/3/4-byte sequences. + const std::string utf8 = "a\xC3\xA9\xE4\xB8\xAD\xF0\x9F\x8E\xAC"; + const std::u16string utf16 = ToUtf16(utf8.c_str()); + return check(utf16 == u"a\u00E9\u4E2D\U0001F3AC", "UTF-8 -> UTF-16 mismatch") && + check(FromUtf16(utf16.data(), utf16.size()) == utf8, "UTF-16 -> UTF-8 mismatch"); +} + +bool doesNotEmitModifiedUtf8() { + // JNI's modified UTF-8 would encode U+1F3AC as a 6-byte CESU-8 surrogate + // pair; mpv/open() need the real 4-byte form. + const std::u16string clapper = u"\U0001F3AC"; + return check(FromUtf16(clapper.data(), clapper.size()) == "\xF0\x9F\x8E\xAC", "supplementary char not 4 bytes") && + check( + ToUtf16("\xED\xA0\xBC\xED\xBE\xAC") == u"\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD\uFFFD", + "CESU-8 surrogate bytes accepted as UTF-8"); +} + +bool replacesMalformedBytesOneAtATime() { + return check(ToUtf16("ok\xFF\xFEz") == u"ok\uFFFD\uFFFDz", "stray bytes not replaced individually") && + check(ToUtf16("\xC0\x80") == u"\uFFFD\uFFFD", "overlong NUL accepted") && + check(ToUtf16("\xE2\x82") == u"\uFFFD\uFFFD", "truncated sequence not replaced") && + check(ToUtf16("\xF4\x90\x80\x80") == u"\uFFFD\uFFFD\uFFFD\uFFFD", "code point above U+10FFFF accepted") && + check(ToUtf16("\xE0\x80\xAF") == u"\uFFFD\uFFFD\uFFFD", "overlong 3-byte form accepted"); +} + +bool replacesLoneSurrogates() { + const std::u16string lone = u"x\xD83Cy\xDFACz"; + return check(FromUtf16(lone.data(), lone.size()) == "x\xEF\xBF\xBDy\xEF\xBF\xBDz", "lone surrogates not replaced"); +} + +bool handlesNullAndEmpty() { + return check(ToUtf16(nullptr).empty(), "NULL input not empty") && + check(ToUtf16("").empty(), "empty input not empty") && + check(FromUtf16(nullptr, 0).empty(), "NULL UTF-16 not empty"); +} + +} // namespace + +int main() { + const bool ok = roundTripsAsciiBmpAndSupplementary() && doesNotEmitModifiedUtf8() && + replacesMalformedBytesOneAtATime() && replacesLoneSurrogates() && handlesNullAndEmpty(); + return ok ? 0 : 1; +} diff --git a/android/libmpv/src/main/cpp/event.cpp b/android/libmpv/src/main/cpp/event.cpp index 479f2e413..6f0e27e94 100644 --- a/android/libmpv/src/main/cpp/event.cpp +++ b/android/libmpv/src/main/cpp/event.cpp @@ -6,7 +6,7 @@ #include "log.h" static void sendPropertyUpdateToJava(JNIEnv* env, mpv_event_property* prop) { - jstring jprop = env->NewStringUTF(prop->name); + jstring jprop = new_java_string(env, prop->name); jstring jvalue = NULL; switch (prop->format) { case MPV_FORMAT_NONE: @@ -22,7 +22,7 @@ static void sendPropertyUpdateToJava(JNIEnv* env, mpv_event_property* prop) { env->CallStaticVoidMethod(mpv_MpvPlayer, mpv_MpvPlayer_onPropertyChanged_Sd, jprop, *(double*)prop->data); break; case MPV_FORMAT_STRING: - jvalue = env->NewStringUTF(*(const char**)prop->data); + jvalue = new_java_string(env, *(const char**)prop->data); env->CallStaticVoidMethod(mpv_MpvPlayer, mpv_MpvPlayer_onPropertyChanged_SS, jprop, jvalue); break; default: @@ -42,16 +42,9 @@ static void sendEndFileToJava(JNIEnv* env, mpv_event* event) { env->CallStaticVoidMethod(mpv_MpvPlayer, mpv_MpvPlayer_onEndFile, (jint)reason); } -static inline bool invalid_utf8(unsigned char c) { return c == 0xc0 || c == 0xc1 || c >= 0xf5; } - static void sendLogMessageToJava(JNIEnv* env, mpv_event_log_message* msg) { - const auto invalid_utf8 = [](unsigned char c) { return c == 0xc0 || c == 0xc1 || c >= 0xf5; }; - for (int i = 0; msg->text[i]; i++) { - if (invalid_utf8(static_cast(msg->text[i]))) return; - } - - jstring jprefix = env->NewStringUTF(msg->prefix); - jstring jtext = env->NewStringUTF(msg->text); + jstring jprefix = new_java_string(env, msg->prefix); + jstring jtext = new_java_string(env, msg->text); env->CallStaticVoidMethod(mpv_MpvPlayer, mpv_MpvPlayer_onLogMessage, jprefix, (jint)msg->log_level, jtext); diff --git a/android/libmpv/src/main/cpp/jni_utils.cpp b/android/libmpv/src/main/cpp/jni_utils.cpp index 5cc93033e..af537d4eb 100644 --- a/android/libmpv/src/main/cpp/jni_utils.cpp +++ b/android/libmpv/src/main/cpp/jni_utils.cpp @@ -5,6 +5,24 @@ #include +#include "utf8_convert.h" + +jstring new_java_string(JNIEnv* env, const char* utf8) { + if (!utf8) return NULL; + const std::u16string u16 = plezy::utf8::ToUtf16(utf8); + return env->NewString(reinterpret_cast(u16.data()), static_cast(u16.size())); +} + +std::string java_string_to_utf8(JNIEnv* env, jstring jstr) { + if (!jstr) return std::string(); + const jsize len = env->GetStringLength(jstr); + const jchar* chars = env->GetStringChars(jstr, NULL); + if (!chars) return std::string(); + std::string out = plezy::utf8::FromUtf16(reinterpret_cast(chars), static_cast(len)); + env->ReleaseStringChars(jstr, chars); + return out; +} + bool acquire_jni_env(JavaVM* vm, JNIEnv** env) { int ret = vm->GetEnv((void**)env, JNI_VERSION_1_6); if (ret == JNI_EDETACHED) diff --git a/android/libmpv/src/main/cpp/jni_utils.h b/android/libmpv/src/main/cpp/jni_utils.h index 69945fe8f..8d0c6b051 100644 --- a/android/libmpv/src/main/cpp/jni_utils.h +++ b/android/libmpv/src/main/cpp/jni_utils.h @@ -2,6 +2,8 @@ #include +#include + #define jni_func_name(name) Java_com_edde746_plezy_libmpv_MpvPlayer_##name #define jni_func(return_type, name, ...) \ JNIEXPORT return_type JNICALL jni_func_name(name)(JNIEnv * env, jobject obj, ##__VA_ARGS__) @@ -9,6 +11,12 @@ bool acquire_jni_env(JavaVM* vm, JNIEnv** env); void init_methods_cache(JNIEnv* env); +// Standard-UTF-8 string crossings; see utf8_convert.h for why NewStringUTF / +// GetStringUTFChars are wrong for mpv data. `utf8` may be NULL (-> NULL). +jstring new_java_string(JNIEnv* env, const char* utf8); +// `jstr` may be NULL (-> empty). +std::string java_string_to_utf8(JNIEnv* env, jstring jstr); + #ifndef UTIL_EXTERN #define UTIL_EXTERN extern #endif diff --git a/android/libmpv/src/main/cpp/main.cpp b/android/libmpv/src/main/cpp/main.cpp index 309923b42..8ae03f69c 100644 --- a/android/libmpv/src/main/cpp/main.cpp +++ b/android/libmpv/src/main/cpp/main.cpp @@ -8,6 +8,8 @@ #include #include #include +#include +#include extern "C" { #include @@ -137,16 +139,19 @@ jni_func(void, nativeCommand, jobjectArray jarray) { const char* arguments[128] = {0}; int len = env->GetArrayLength(jarray); - if (len >= ARRAYLEN(arguments)) { + if (len >= (int)ARRAYLEN(arguments)) { die("too many command arguments"); return; } - for (int i = 0; i < len; ++i) - arguments[i] = env->GetStringUTFChars((jstring)env->GetObjectArrayElement(jarray, i), NULL); + std::vector storage; + storage.reserve(len); + for (int i = 0; i < len; ++i) { + jstring jarg = (jstring)env->GetObjectArrayElement(jarray, i); + storage.push_back(java_string_to_utf8(env, jarg)); + arguments[i] = storage.back().c_str(); + env->DeleteLocalRef(jarg); + } mpv_command(g_mpv, arguments); - - for (int i = 0; i < len; ++i) - env->ReleaseStringUTFChars((jstring)env->GetObjectArrayElement(jarray, i), arguments[i]); } diff --git a/android/libmpv/src/main/cpp/property.cpp b/android/libmpv/src/main/cpp/property.cpp index 983838e4e..2336514cf 100644 --- a/android/libmpv/src/main/cpp/property.cpp +++ b/android/libmpv/src/main/cpp/property.cpp @@ -2,6 +2,7 @@ #include #include +#include #include "globals.h" #include "jni_utils.h" @@ -26,12 +27,11 @@ jni_func(jint, nativeSetOptionString, jstring joption, jstring jvalue) { CHECK_MPV_INIT_RET(0); const char* option = env->GetStringUTFChars(joption, NULL); - const char* value = env->GetStringUTFChars(jvalue, NULL); + const std::string value = java_string_to_utf8(env, jvalue); - int result = mpv_set_option_string(g_mpv, option, value); + int result = mpv_set_option_string(g_mpv, option, value.c_str()); env->ReleaseStringUTFChars(joption, option); - env->ReleaseStringUTFChars(jvalue, value); return result; } @@ -80,7 +80,7 @@ jni_func(jobject, nativeGetPropertyBoolean, jstring jproperty) { jni_func(jstring, nativeGetPropertyString, jstring jproperty) { char* value; if (common_get_property(env, jproperty, MPV_FORMAT_STRING, &value) < 0) return NULL; - jstring jvalue = env->NewStringUTF(value); + jstring jvalue = new_java_string(env, value); mpv_free(value); return jvalue; } @@ -101,9 +101,9 @@ jni_func(void, nativeSetPropertyBoolean, jstring jproperty, jboolean jvalue) { } jni_func(void, nativeSetPropertyString, jstring jproperty, jstring jvalue) { - const char* value = env->GetStringUTFChars(jvalue, NULL); - common_set_property(env, jproperty, MPV_FORMAT_STRING, &value); - env->ReleaseStringUTFChars(jvalue, value); + const std::string value = java_string_to_utf8(env, jvalue); + const char* value_ptr = value.c_str(); + common_set_property(env, jproperty, MPV_FORMAT_STRING, &value_ptr); } jni_func(void, nativeObserveProperty, jstring property, jint format) { diff --git a/android/libmpv/src/main/cpp/utf8_convert.h b/android/libmpv/src/main/cpp/utf8_convert.h new file mode 100644 index 000000000..dc2433f1a --- /dev/null +++ b/android/libmpv/src/main/cpp/utf8_convert.h @@ -0,0 +1,126 @@ +#pragma once + +#include +#include +#include + +// UTF-8 <-> UTF-16 transcoding for the JNI boundary. +// +// mpv speaks standard UTF-8. JNI's NewStringUTF/GetStringUTFChars speak +// *modified* UTF-8: supplementary-plane characters are CESU-8 surrogate pairs +// and NUL is 0xC0 0x80. Feeding one encoding to the other corrupts emoji in +// file names and titles, and malformed bytes from mpv (log lines, ID3 tags, +// system-encoded paths) abort under CheckJNI. Both directions therefore go +// through UTF-16 with NewString/GetStringChars; malformed input is replaced +// with U+FFFD one unit at a time, matching shared/cpp/sanitize_utf8.h on +// desktop. Header-only and JNI-free so the host test harness can exercise it. + +namespace plezy { +namespace utf8 { + +// Decodes one scalar value at `s`. Returns the number of bytes consumed, or 0 +// when `s` does not start a well-formed sequence (Unicode Table 3-7). +inline size_t DecodeOne(const unsigned char* s, size_t len, uint32_t* cp) { + const unsigned char c = s[0]; + if (c < 0x80) { + *cp = c; + return 1; + } + size_t need; + unsigned char lo = 0x80, hi = 0xBF; + if (c >= 0xC2 && c <= 0xDF) { + need = 2; + *cp = c & 0x1F; + } else if (c >= 0xE0 && c <= 0xEF) { + need = 3; + *cp = c & 0x0F; + if (c == 0xE0) lo = 0xA0; + if (c == 0xED) hi = 0x9F; // no surrogates + } else if (c >= 0xF0 && c <= 0xF4) { + need = 4; + *cp = c & 0x07; + if (c == 0xF0) lo = 0x90; + if (c == 0xF4) hi = 0x8F; // <= U+10FFFF + } else { + return 0; + } + if (len < need) return 0; + for (size_t i = 1; i < need; ++i) { + const unsigned char b = s[i]; + if (b < lo || b > hi) return 0; + lo = 0x80; + hi = 0xBF; + *cp = (*cp << 6) | (b & 0x3F); + } + return need; +} + +// Standard UTF-8 -> UTF-16. Malformed bytes become U+FFFD. +inline std::u16string ToUtf16(const char* input, size_t len) { + std::u16string out; + if (!input) return out; + out.reserve(len); + const unsigned char* s = reinterpret_cast(input); + size_t pos = 0; + while (pos < len) { + uint32_t cp; + const size_t n = DecodeOne(s + pos, len - pos, &cp); + if (n == 0) { + out.push_back(u'\uFFFD'); + pos += 1; + continue; + } + pos += n; + if (cp < 0x10000) { + out.push_back(static_cast(cp)); + } else { + cp -= 0x10000; + out.push_back(static_cast(0xD800 | (cp >> 10))); + out.push_back(static_cast(0xDC00 | (cp & 0x3FF))); + } + } + return out; +} + +inline std::u16string ToUtf16(const char* input) { + return input ? ToUtf16(input, std::char_traits::length(input)) : std::u16string(); +} + +// UTF-16 -> standard UTF-8. Lone surrogates become U+FFFD. +inline std::string FromUtf16(const char16_t* input, size_t len) { + std::string out; + if (!input) return out; + out.reserve(len * 3); + for (size_t i = 0; i < len; ++i) { + uint32_t cp = input[i]; + if (cp >= 0xD800 && cp <= 0xDBFF) { + if (i + 1 < len && input[i + 1] >= 0xDC00 && input[i + 1] <= 0xDFFF) { + cp = 0x10000 + ((cp - 0xD800) << 10) + (input[i + 1] - 0xDC00); + ++i; + } else { + cp = 0xFFFD; + } + } else if (cp >= 0xDC00 && cp <= 0xDFFF) { + cp = 0xFFFD; + } + if (cp < 0x80) { + out.push_back(static_cast(cp)); + } else if (cp < 0x800) { + out.push_back(static_cast(0xC0 | (cp >> 6))); + out.push_back(static_cast(0x80 | (cp & 0x3F))); + } else if (cp < 0x10000) { + out.push_back(static_cast(0xE0 | (cp >> 12))); + out.push_back(static_cast(0x80 | ((cp >> 6) & 0x3F))); + out.push_back(static_cast(0x80 | (cp & 0x3F))); + } else { + out.push_back(static_cast(0xF0 | (cp >> 18))); + out.push_back(static_cast(0x80 | ((cp >> 12) & 0x3F))); + out.push_back(static_cast(0x80 | ((cp >> 6) & 0x3F))); + out.push_back(static_cast(0x80 | (cp & 0x3F))); + } + } + return out; +} + +} // namespace utf8 +} // namespace plezy diff --git a/android/libmpv/src/main/java/com/edde746/plezy/libmpv/MpvPlayer.kt b/android/libmpv/src/main/java/com/edde746/plezy/libmpv/MpvPlayer.kt index f568bd7f7..e6e1ffc44 100644 --- a/android/libmpv/src/main/java/com/edde746/plezy/libmpv/MpvPlayer.kt +++ b/android/libmpv/src/main/java/com/edde746/plezy/libmpv/MpvPlayer.kt @@ -75,9 +75,7 @@ class MpvPlayer private constructor() : AutoCloseable { @JvmStatic fun onPropertyChanged(name: String, value: String) { - instance.get()?.rawPropertyChanges?.trySend( - PropertyChange.Str(name, sanitizeString(value)) - ) + instance.get()?.rawPropertyChanges?.trySend(PropertyChange.Str(name, value)) } @JvmStatic @@ -96,34 +94,7 @@ class MpvPlayer private constructor() : AutoCloseable { @JvmStatic fun onLogMessage(prefix: String, level: Int, text: String) { val logLevel = LogLevel.fromNative(level) ?: return - instance.get()?.rawLogMessages?.trySend( - LogMessage(prefix, logLevel, sanitizeString(text).trimEnd()) - ) - } - - private fun sanitizeString(s: String): String { - val sb = StringBuilder(s.length) - var i = 0 - while (i < s.length) { - val c = s[i] - if (c.isHighSurrogate()) { - if (i + 1 < s.length && s[i + 1].isLowSurrogate()) { - sb.append(c) - sb.append(s[i + 1]) - i += 2 - } else { - sb.append('\uFFFD') - i++ - } - } else if (c.isLowSurrogate()) { - sb.append('\uFFFD') - i++ - } else { - sb.append(c) - i++ - } - } - return sb.toString() + instance.get()?.rawLogMessages?.trySend(LogMessage(prefix, logLevel, text.trimEnd())) } // JNI native declarations — private to avoid internal name mangling