refactor: shared Unicode display-width utility with grapheme cluster support
Consolidate two inconsistent codepoint_width implementations from Screen.cpp and EventEditor.cpp into nocal/tui/unicode.hpp. The unified table covers all CJK, Hangul, emoji, symbol, and combining mark ranges. display_width and fit_text handle grapheme clusters (ZWJ sequences, skin-tone/hair modifiers, regional indicator pairs, variation selectors) so calendar text with emoji or East Asian characters renders correctly.
This commit is contained in:
248
tests/unicode_tests.cpp
Normal file
248
tests/unicode_tests.cpp
Normal file
@@ -0,0 +1,248 @@
|
||||
#include "nocal/tui/unicode.hpp"
|
||||
|
||||
#include <iostream>
|
||||
#include <string>
|
||||
#include <string_view>
|
||||
|
||||
namespace {
|
||||
|
||||
int failures = 0;
|
||||
|
||||
void check(const bool condition, const std::string_view message)
|
||||
{
|
||||
if (!condition) {
|
||||
++failures;
|
||||
std::cerr << "FAIL: " << message << '\n';
|
||||
}
|
||||
}
|
||||
|
||||
void test_decode_codepoint()
|
||||
{
|
||||
// ASCII
|
||||
{
|
||||
const auto d = nocal::tui::decode_codepoint("A", 0);
|
||||
check(d.value == 0x41U && d.length == 1, "ASCII decode");
|
||||
}
|
||||
|
||||
// Two-byte UTF-8 (U+00A9 ©)
|
||||
{
|
||||
const auto d = nocal::tui::decode_codepoint("\u00a9", 0);
|
||||
check(d.value == 0xa9U && d.length == 2, "2-byte decode");
|
||||
}
|
||||
|
||||
// Three-byte UTF-8 (U+4E00 一)
|
||||
{
|
||||
const auto d = nocal::tui::decode_codepoint("\u4e00", 0);
|
||||
check(d.value == 0x4e00U && d.length == 3, "3-byte decode");
|
||||
}
|
||||
|
||||
// Four-byte UTF-8 (U+1F600 😀)
|
||||
{
|
||||
const auto d = nocal::tui::decode_codepoint("\U0001f600", 0);
|
||||
check(d.value == 0x1f600U && d.length == 4, "4-byte decode");
|
||||
}
|
||||
|
||||
// Invalid continuation byte → replacement character
|
||||
{
|
||||
const auto d = nocal::tui::decode_codepoint("\x80", 0);
|
||||
check(d.value == 0xfffdU && d.length == 1, "invalid byte yields replacement");
|
||||
}
|
||||
|
||||
// Truncated sequence
|
||||
{
|
||||
const auto d = nocal::tui::decode_codepoint("\xc0", 0);
|
||||
check(d.value == 0xfffdU && d.length == 1, "truncated sequence yields replacement");
|
||||
}
|
||||
}
|
||||
|
||||
void test_codepoint_width()
|
||||
{
|
||||
// ASCII
|
||||
check(nocal::tui::codepoint_width(0x41U) == 1, "ASCII letter width 1");
|
||||
|
||||
// Combining marks
|
||||
check(nocal::tui::codepoint_width(0x0300U) == 0, "combining grave accent width 0");
|
||||
check(nocal::tui::codepoint_width(0x036fU) == 0, "combining mark end width 0");
|
||||
check(nocal::tui::codepoint_width(0x1ab0U) == 0, "combining half mark start width 0");
|
||||
check(nocal::tui::codepoint_width(0x1affU) == 0, "combining half mark end width 0");
|
||||
check(nocal::tui::codepoint_width(0x1dc0U) == 0, "combining diacritical extended start width 0");
|
||||
check(nocal::tui::codepoint_width(0x1dffU) == 0, "combining diacritical extended end width 0");
|
||||
check(nocal::tui::codepoint_width(0x20d0U) == 0, "combining enclosing key symbols start width 0");
|
||||
check(nocal::tui::codepoint_width(0x20ffU) == 0, "combining enclosing key symbols end width 0");
|
||||
check(nocal::tui::codepoint_width(0xfe00U) == 0, "variation selector-1 width 0");
|
||||
check(nocal::tui::codepoint_width(0xfe0fU) == 0, "variation selector-16 width 0");
|
||||
check(nocal::tui::codepoint_width(0xfe20U) == 0, "combining ligature start width 0");
|
||||
check(nocal::tui::codepoint_width(0xfe2fU) == 0, "combining ligature end width 0");
|
||||
check(nocal::tui::codepoint_width(0x200dU) == 0, "ZWJ width 0");
|
||||
check(nocal::tui::codepoint_width(0x1f3fbU) == 2, "skin tone modifier standalone width 2");
|
||||
|
||||
// Double-width CJK
|
||||
check(nocal::tui::codepoint_width(0x1100U) == 2, "Hangul Jamo start width 2");
|
||||
check(nocal::tui::codepoint_width(0x115fU) == 2, "Hangul Jamo end width 2");
|
||||
check(nocal::tui::codepoint_width(0x2329U) == 2, "left-pointing angle bracket width 2");
|
||||
check(nocal::tui::codepoint_width(0x232aU) == 2, "right-pointing angle bracket width 2");
|
||||
check(nocal::tui::codepoint_width(0x2e80U) == 2, "CJK radical start width 2");
|
||||
check(nocal::tui::codepoint_width(0xa4cfU) == 2, "Yi syllable end width 2");
|
||||
check(nocal::tui::codepoint_width(0xac00U) == 2, "Hangul syllable start width 2");
|
||||
check(nocal::tui::codepoint_width(0xd7a3U) == 2, "Hangul syllable end width 2");
|
||||
check(nocal::tui::codepoint_width(0xf900U) == 2, "CJK compat ideograph start width 2");
|
||||
check(nocal::tui::codepoint_width(0xfaffU) == 2, "CJK compat ideograph end width 2");
|
||||
check(nocal::tui::codepoint_width(0xfe10U) == 2, "vertical form start width 2");
|
||||
check(nocal::tui::codepoint_width(0xfe19U) == 2, "vertical form end width 2");
|
||||
check(nocal::tui::codepoint_width(0xfe30U) == 2, "CJK compat form start width 2");
|
||||
check(nocal::tui::codepoint_width(0xfe6fU) == 2, "CJK compat form end width 2");
|
||||
check(nocal::tui::codepoint_width(0xff00U) == 2, "fullwidth ASCII start width 2");
|
||||
check(nocal::tui::codepoint_width(0xff60U) == 2, "fullwidth ASCII end width 2");
|
||||
check(nocal::tui::codepoint_width(0xffe0U) == 2, "fullwidth symbol start width 2");
|
||||
check(nocal::tui::codepoint_width(0xffe6U) == 2, "fullwidth symbol end width 2");
|
||||
check(nocal::tui::codepoint_width(0x2600U) == 2, "misc symbol start width 2");
|
||||
check(nocal::tui::codepoint_width(0x27bfU) == 2, "misc symbol end width 2");
|
||||
check(nocal::tui::codepoint_width(0x1f300U) == 2, "misc symbol pictograph start width 2");
|
||||
check(nocal::tui::codepoint_width(0x1faffU) == 2, "misc symbol pictograph end width 2");
|
||||
check(nocal::tui::codepoint_width(0x20000U) == 2, "CJK ext B start width 2");
|
||||
check(nocal::tui::codepoint_width(0x3fffdU) == 2, "CJK ext B end width 2");
|
||||
|
||||
// Regular CJK character (U+4E00)
|
||||
check(nocal::tui::codepoint_width(0x4e00U) == 2, "CJK ideograph width 2");
|
||||
}
|
||||
|
||||
void test_display_width()
|
||||
{
|
||||
using nocal::tui::display_width;
|
||||
|
||||
// ASCII text: width equals byte count
|
||||
check(display_width("hello") == 5, "ASCII text width");
|
||||
check(display_width("") == 0, "empty string width");
|
||||
|
||||
// CJK characters: each is width 2
|
||||
check(display_width("\u4e00") == 2, "single CJK width 2");
|
||||
check(display_width("\u4e00\u4e00") == 4, "two CJK width 4");
|
||||
check(display_width("a\u4e00b") == 4, "mixed ASCII+CJK width");
|
||||
|
||||
// Combining marks: absorbed by base
|
||||
check(display_width("a\u0300") == 1, "letter+combining mark width 1");
|
||||
check(display_width("\u4e00\u0300") == 2, "CJK+combining mark width 2");
|
||||
|
||||
// Emoji: width 2
|
||||
check(display_width("\U0001f600") == 2, "emoji width 2");
|
||||
check(display_width("\U0001f600\U0001f601") == 4, "two emoji width 4");
|
||||
|
||||
// Skin tone modifiers: absorbed (e.g. 👋🏽)
|
||||
check(display_width("\U0001f44b\U0001f3fd") == 2, "emoji+skin tone width 2");
|
||||
|
||||
// Regional indicator pairs: width 2 (e.g. 🇺🇸)
|
||||
check(display_width("\U0001f1fa\U0001f1f8") == 2, "US flag width 2");
|
||||
}
|
||||
|
||||
void test_fit_text()
|
||||
{
|
||||
using nocal::tui::fit_text;
|
||||
|
||||
// Empty string
|
||||
check(fit_text("", 5) == " ", "empty fit to 5");
|
||||
check(fit_text("", 0) == "", "empty fit to 0");
|
||||
check(fit_text("", -1) == "", "empty fit to negative");
|
||||
|
||||
// Truncated to 0 width
|
||||
check(fit_text("hello", 0) == "", "fit to 0 width");
|
||||
check(fit_text("hello", -5) == "", "fit to negative width");
|
||||
|
||||
// Text fits exactly
|
||||
check(fit_text("hi", 2) == "hi", "exact fit");
|
||||
check(fit_text("hi", 3) == "hi ", "pad to 3");
|
||||
check(fit_text("hi", 5) == "hi ", "pad to 5");
|
||||
|
||||
// Truncation with ellipsis
|
||||
check(fit_text("hello", 4) == "hel\u2026", "truncate with ellipsis width 4");
|
||||
check(fit_text("hello", 3) == "he\u2026", "truncate with ellipsis width 3");
|
||||
check(fit_text("hello", 2) == "h\u2026", "truncate with ellipsis width 2");
|
||||
check(fit_text("hello", 1) == "h", "truncate width 1 (no ellipsis)");
|
||||
|
||||
// Truncation without ellipsis
|
||||
check(fit_text("hello", 3, false) == "hel", "truncate without ellipsis");
|
||||
|
||||
// CJK truncation (no ellipsis)
|
||||
check(fit_text("\u4e00\u4e00\u4e00", 4, false) == "\u4e00\u4e00", "CJK truncate 2 chars no ellipsis");
|
||||
check(fit_text("\u4e00\u4e00\u4e00", 5, false) == "\u4e00\u4e00 ", "CJK fit 2 chars pad no ellipsis");
|
||||
|
||||
// CJK with ellipsis
|
||||
check(fit_text("\u4e00\u4e00\u4e00", 5, true) == "\u4e00\u4e00\u2026", "CJK truncate with ellipsis");
|
||||
}
|
||||
|
||||
void test_grapheme_clusters()
|
||||
{
|
||||
using nocal::tui::display_width;
|
||||
using nocal::tui::fit_text;
|
||||
|
||||
// ZWJ sequences (e.g. family emoji)
|
||||
// U+1F468 (man) + ZWJ + U+1F469 (woman) + ZWJ + U+1F467 (girl) + ZWJ + U+1F466 (boy)
|
||||
// This is a single grapheme cluster of width 2
|
||||
const std::string family = "\U0001f468\u200d\U0001f469\u200d\U0001f467\u200d\U0001f466";
|
||||
check(display_width(family) == 2, "ZWJ family emoji width 2");
|
||||
|
||||
// Skin tone modifiers
|
||||
// Hand wave + skin tone
|
||||
const std::string wave = "\U0001f44b\U0001f3fd";
|
||||
check(display_width(wave) == 2, "wave+skin tone width 2");
|
||||
|
||||
// Hair style modifiers
|
||||
// Person + red hair
|
||||
const std::string red_hair = "\U0001f9d1\U0001f9b0";
|
||||
check(display_width(red_hair) == 2, "person+red hair width 2");
|
||||
|
||||
// fit_text with ZWJ: should not break inside cluster
|
||||
check(fit_text(family, 2) == family, "fit ZWJ at exact width");
|
||||
// When truncating to 1, the entire cluster should be dropped (no partial cluster)
|
||||
check(fit_text(family, 1) == " ", "fit ZWJ at width 1 drops cluster");
|
||||
}
|
||||
|
||||
void test_regional_indicator_pairs()
|
||||
{
|
||||
using nocal::tui::display_width;
|
||||
using nocal::tui::fit_text;
|
||||
|
||||
// US flag
|
||||
const std::string us = "\U0001f1fa\U0001f1f8";
|
||||
check(display_width(us) == 2, "US flag width 2");
|
||||
|
||||
// Japan flag
|
||||
const std::string jp = "\U0001f1ef\U0001f1f5";
|
||||
check(display_width(jp) == 2, "Japan flag width 2");
|
||||
|
||||
// fit_text should not split a flag pair
|
||||
check(fit_text(us, 2) == us, "fit US flag at exact width");
|
||||
check(fit_text(us, 1) == " ", "fit US flag at width 1 drops pair");
|
||||
}
|
||||
|
||||
void test_variation_selectors()
|
||||
{
|
||||
using nocal::tui::display_width;
|
||||
|
||||
// Emoji + variation selector
|
||||
check(display_width("\u2600") == 2, "sun without VS width 2");
|
||||
check(display_width("\u2600\ufe0f") == 2, "sun with VS16 width 2");
|
||||
|
||||
// CJK + variation selector
|
||||
check(display_width("\u4e00") == 2, "CJK without VS width 2");
|
||||
check(display_width("\u4e00\ufe00") == 2, "CJK with VS1 width 2");
|
||||
}
|
||||
|
||||
} // anonymous namespace
|
||||
|
||||
int main()
|
||||
{
|
||||
test_decode_codepoint();
|
||||
test_codepoint_width();
|
||||
test_display_width();
|
||||
test_fit_text();
|
||||
test_grapheme_clusters();
|
||||
test_regional_indicator_pairs();
|
||||
test_variation_selectors();
|
||||
|
||||
if (failures > 0) {
|
||||
std::cerr << "FAIL: " << failures << " test(s) failed\n";
|
||||
return 1;
|
||||
}
|
||||
std::cout << "OK: all unicode tests passed\n";
|
||||
return 0;
|
||||
}
|
||||
Reference in New Issue
Block a user