Skip to content

Commit f21778c

Browse files
author
i2cjak
authored
Merge pull request #144 from i2cjak/backplane/i18n-cjk
Japanese text in the window: JP forms, two-cell width, kinsoku
2 parents 62892d1 + 0ffb40a commit f21778c

9 files changed

Lines changed: 776 additions & 9 deletions

File tree

‎LAWS.bend‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@ import ./test/fixtures/i18n_fallback.bend as I18F
44
import ./src/core/usage.bend as UZ
55
import ./src/core/mail.bend as ML # mail 1
66
import ./src/core/ref.bend as REF
7+
# ---- i18n cjk ----
8+
import ./src/core/cjk.bend as CJK
9+
# ---- end i18n cjk ----
710
import ./src/core/tz.bend as TZ
811
import ./src/core/when.bend as WH
912
import ./src/core/sched.bend as SCH
@@ -8712,3 +8715,35 @@ law usage_boat_due_never_read:
87128715
# credit-seconds are shown as exact dollars, never rounded
87138716
law usage_boat_usd_exact:
87148717
{UZ.Bj.usd(158n) == "$0.00158" : String}
8718+
# ---- i18n cjk ----
8719+
law cjk_segs_lossless:
8720+
{CJK.Cjk.join(CJK.Cjk.segs("日本語。")) == "日本語。" : String}
8721+
8722+
law cjk_segs_lossless_mixed:
8723+
{CJK.Cjk.join(CJK.Cjk.segs("PRを開いた。")) == "PRを開いた。" : String}
8724+
8725+
law cjk_segs_lossless_spaces:
8726+
{CJK.Cjk.join(CJK.Cjk.segs(" あ い ")) == " あ い " : String}
8727+
8728+
law cjk_segs_lossless_empty:
8729+
{CJK.Cjk.join(CJK.Cjk.segs("")) == "" : String}
8730+
8731+
law cjk_no_start_stop:
8732+
{CJK.Cjk.can_break(12354, 12290) == False{} : Bool}
8733+
8734+
law cjk_no_end_quote:
8735+
{CJK.Cjk.can_break(12300, 12354) == False{} : Bool}
8736+
8737+
law cjk_kana_break:
8738+
{CJK.Cjk.can_break(12354, 12356) == True{} : Bool}
8739+
8740+
law cjk_ascii_no_break:
8741+
{CJK.Cjk.can_break(65, 66) == False{} : Bool}
8742+
8743+
law cjk_terminal_cells:
8744+
{L.Term.cells("A日本B", 0) == 6 : U32}
8745+
8746+
law cjk_terminal_combining_cells:
8747+
{L.Term.cells("é日", 0) == 3 : U32}
8748+
8749+
# ---- end i18n cjk ----

‎PROOF.bend‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10610,3 +10610,35 @@ def Laws.usage_boat_due_never_read(now):
1061010610

1061110611
def Laws.usage_boat_usd_exact():
1061210612
{==}
10613+
# ---- i18n cjk ----
10614+
def Laws.cjk_segs_lossless():
10615+
{==}
10616+
10617+
def Laws.cjk_segs_lossless_mixed():
10618+
{==}
10619+
10620+
def Laws.cjk_segs_lossless_spaces():
10621+
{==}
10622+
10623+
def Laws.cjk_segs_lossless_empty():
10624+
{==}
10625+
10626+
def Laws.cjk_no_start_stop():
10627+
{==}
10628+
10629+
def Laws.cjk_no_end_quote():
10630+
{==}
10631+
10632+
def Laws.cjk_kana_break():
10633+
{==}
10634+
10635+
def Laws.cjk_ascii_no_break():
10636+
{==}
10637+
10638+
def Laws.cjk_terminal_cells():
10639+
{==}
10640+
10641+
def Laws.cjk_terminal_combining_cells():
10642+
{==}
10643+
10644+
# ---- end i18n cjk ----

‎src/app/layout.bend‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3616,12 +3616,20 @@ def Lay.settings(+ty: Ty, +st: M.State, +n: S.Nui, +info: Map<&2, String>, +v: S
36163616
# Terminal
36173617
# --------
36183618

3619+
# Terminal cells, including combining marks (0) and East Asian wide (2).
3620+
def Term.cells(s: String, acc: U32) -> U32:
3621+
match s:
3622+
case SNil{}:
3623+
acc
3624+
case SCon{Chr{cp}, t}:
3625+
Term.cells(t, (acc + VT.Vt.width(cp) : U32))
3626+
36193627
def Term.runs(+ty: Ty, rs: List<&2, VT.VRun>, +cw: U32, +c: D.Clip, +x: U32, +y: U32) -> List<&2, R.Op>:
36203628
match rs:
36213629
case Nil{}:
36223630
Nil{}
36233631
case VT.VRun{+t, +fg, +bg, +bold, _} <> rest:
3624-
+w = {(U32.from_nat(String.length(t)) * cw : U32) : U32}
3632+
+w = {(Term.cells(t, 0) * cw : U32) : U32}
36253633
List.concat(&2, R.Op, [
36263634
Bool.pick(List<&2, R.Op>, U32.is_eq(bg, VT.Vt.bg()), Nil{}, [D.Draw.rect(c, x, y, w, 18, bg, 255)]),
36273635
Ty.text(ty, Bool.pick(Nat, bold, 1n, 4n), t, c, x, y, 18, fg),

‎src/core/cjk.bend‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
import Base
2+
3+
# East Asian cell widths match Vt.wide. Keep these ranges in step with
4+
# the terminal, including supplementary ideographs and wide emoji.
5+
def Cjk.in(+cp: U32, +a: U32, +b: U32) -> Bool:
6+
U32.is_ge(cp, a) && U32.is_le(cp, b)
7+
8+
def Cjk.wide(+cp: U32) -> Bool:
9+
Cjk.in(cp, 4352, 4447) || Cjk.in(cp, 11904, 12350) || Cjk.in(cp, 12353, 13311) || Cjk.in(cp, 13312, 19903) || Cjk.in(cp, 19968, 40959) || Cjk.in(cp, 40960, 42191) || Cjk.in(cp, 44032, 55203) || Cjk.in(cp, 63744, 64255) || Cjk.in(cp, 65072, 65103) || Cjk.in(cp, 65280, 65376) || Cjk.in(cp, 65504, 65510) || Cjk.in(cp, 127744, 128591) || Cjk.in(cp, 128640, 128767) || Cjk.in(cp, 129280, 129535) || Cjk.in(cp, 131072, 262141)
10+
11+
def Cjk.no_start(+cp: U32) -> Bool:
12+
String.contains("、。,.・:;?!ー~)」』】〕〉》}]⦆〙〗’”ぁぃぅぇぉっゃゅょゎァィゥェォッャュョヮヵヶ々〻‐゠–〜)]},.:;!?", SCon{Chr{cp}, ""})
13+
14+
def Cjk.no_end(+cp: U32) -> Bool:
15+
String.contains("(「『【〔〈《{[⦅〘〖‘“([{", SCon{Chr{cp}, ""})
16+
17+
def Cjk.can_break.k(wide: Bool, +a: U32, +b: U32) -> Bool:
18+
match wide:
19+
case False{}:
20+
False{}
21+
case True{}:
22+
Bool.not(Cjk.no_end(a)) && Bool.not(Cjk.no_start(b))
23+
24+
def Cjk.can_break(+a: U32, +b: U32) -> Bool:
25+
Cjk.can_break.k(Cjk.wide(a) || Cjk.wide(b), a, b)
26+
27+
# Test once per paragraph; ordinary Latin text keeps the old word path.
28+
def Cjk.has.go(s: String, hit: Bool) -> Bool:
29+
match s hit:
30+
case _ True{}:
31+
True{}
32+
case SNil{} False{}:
33+
False{}
34+
case SCon{Chr{cp}, t} False{}:
35+
Cjk.has.go(t, U32.is_ge(cp, 11904))
36+
37+
def Cjk.has(s: String) -> Bool:
38+
Cjk.has.go(s, False{})
39+
40+
# Each gap is exactly one original space. Empty segments preserve a
41+
# leading, repeated or trailing space. Characters in rev are newest first.
42+
type Seg is Data:
43+
Seg{text: String, gap: Bool}
44+
45+
type SegAcc is Data:
46+
SegAcc{done: List<&2, Seg>, rev: String, gap: Bool, prev: U32, have: Bool}
47+
48+
def Cjk.rev(s: String, acc: String) -> String:
49+
match s:
50+
case SNil{}:
51+
acc
52+
case SCon{c, t}:
53+
Cjk.rev(t, SCon{c, acc})
54+
55+
def Cjk.step.k(space: Bool, cut: Bool, +cp: U32, a: SegAcc) -> SegAcc:
56+
match space cut a:
57+
case True{} _ SegAcc{ls, rev, gap, _, _}:
58+
SegAcc{Seg{Cjk.rev(rev, ""), gap} <> ls, "", True{}, 0, False{}}
59+
case False{} True{} SegAcc{ls, rev, gap, _, _}:
60+
SegAcc{Seg{Cjk.rev(rev, ""), gap} <> ls, SCon{Chr{cp}, ""}, False{}, cp, True{}}
61+
case False{} False{} SegAcc{ls, rev, gap, _, _}:
62+
SegAcc{ls, SCon{Chr{cp}, rev}, gap, cp, True{}}
63+
64+
def Cjk.step(+cp: U32, a: SegAcc) -> SegAcc:
65+
match a:
66+
case SegAcc{ls, rev, gap, +prev, +have}:
67+
Cjk.step.k(U32.is_eq(cp, 32), Bool.and(have, Cjk.can_break(prev, cp)), cp, SegAcc{ls, rev, gap, prev, have})
68+
69+
def Cjk.segs.go(s: String, a: SegAcc) -> List<&2, Seg>:
70+
match s a:
71+
case SNil{} SegAcc{ls, rev, gap, _, _}:
72+
List.reverse(&2, Seg, Seg{Cjk.rev(rev, ""), gap} <> ls)
73+
case SCon{Chr{cp}, t} a:
74+
Cjk.segs.go(t, Cjk.step(cp, a))
75+
76+
def Cjk.segs(s: String) -> List<&2, Seg>:
77+
Cjk.segs.go(s, SegAcc{Nil{}, "", False{}, 0, False{}})
78+
79+
# Reconstruction also uses reversed accumulators, with no recursive
80+
# result appended outside the tail call.
81+
def Cjk.join.go(ss: List<&2, Seg>, +acc: String) -> String:
82+
match ss:
83+
case Nil{}:
84+
Cjk.rev(acc, "")
85+
case Seg{text, gap} <> t:
86+
Cjk.join.go(t, Cjk.rev(text, Bool.pick(String, gap, SCon{' ', acc}, acc)))
87+
88+
def Cjk.join(ss: List<&2, Seg>) -> String:
89+
Cjk.join.go(ss, "")

‎src/gfx/effects/font.c‎

Lines changed: 75 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -326,6 +326,7 @@ typedef struct {
326326
FxFace* face;
327327
u32 px;
328328
long style;
329+
FxFace* primary; // fixed/proportional policy, cell and line-box metrics
329330
} FxFallback;
330331

331332
static FxFallback fx_fb[FX_FB_MAX];
@@ -340,7 +341,52 @@ static unsigned fx_has(FxFace* f, u32 cp) {
340341
return gi;
341342
}
342343

343-
static FxFace* fx_fb_open(void* pat, u32 cp, u32 px, long style) {
344+
// Read preference once. Locale precedence follows LC_ALL, LC_MESSAGES,
345+
// LANG; a non-CJK locale deliberately defaults to Japanese glyph forms.
346+
static const char* fx_cjk_lang(void) {
347+
static const char* lang = NULL;
348+
if (lang != NULL) return lang;
349+
const char* override = getenv("BACKPLANE_CJK_LANG");
350+
const char* loc = override;
351+
char tag[64];
352+
for (int attempt = 0; attempt < 2; attempt += 1) {
353+
if (attempt || loc == NULL || !*loc) {
354+
loc = getenv("LC_ALL");
355+
if (loc == NULL || !*loc) loc = getenv("LC_MESSAGES");
356+
if (loc == NULL || !*loc) loc = getenv("LANG");
357+
}
358+
unsigned i = 0;
359+
while (loc && loc[i] && loc[i] != '.' && loc[i] != '@' && i < sizeof(tag) - 1) {
360+
char ch = loc[i];
361+
tag[i++] = ch == '_' ? '-' : (ch >= 'A' && ch <= 'Z' ? ch + 'a' - 'A' : ch);
362+
}
363+
tag[i] = 0;
364+
if (override && *override && !attempt) {
365+
if (!strcmp(tag, "ja") || !strcmp(tag, "zh-cn") || !strcmp(tag, "zh-tw") || !strcmp(tag, "ko")) break;
366+
} else break;
367+
}
368+
lang = "ja";
369+
if (!strncmp(tag, "ko", 2) && (!tag[2] || tag[2] == '-')) lang = "ko";
370+
if (!strncmp(tag, "zh", 2) && (!tag[2] || tag[2] == '-')) {
371+
lang = strstr(tag, "-tw") || strstr(tag, "-hk") || strstr(tag, "-mo") || strstr(tag, "-hant") ? "zh-tw" : "zh-cn";
372+
}
373+
return lang;
374+
}
375+
376+
// Same ranges as core/cjk.bend and Vt.wide (no Unicode library needed).
377+
static int fx_cjk_wide(u32 cp) {
378+
static const u32 ranges[][2] = {
379+
{4352,4447}, {11904,12350}, {12353,13311}, {13312,19903},
380+
{19968,40959}, {40960,42191}, {44032,55203}, {63744,64255},
381+
{65072,65103}, {65280,65376}, {65504,65510}, {127744,128591},
382+
{128640,128767}, {129280,129535}, {131072,262141}
383+
};
384+
for (unsigned i = 0; i < sizeof(ranges) / sizeof(ranges[0]); i += 1)
385+
if (cp >= ranges[i][0] && cp <= ranges[i][1]) return 1;
386+
return 0;
387+
}
388+
389+
static FxFace* fx_fb_open(void* pat, u32 cp, u32 px, long style, FxFace* primary) {
344390
unsigned char* file = NULL;
345391
int index = 0;
346392
int color = 0;
@@ -365,6 +411,7 @@ static FxFace* fx_fb_open(void* pat, u32 cp, u32 px, long style) {
365411
fx_fb[fx_nfb].face = f;
366412
fx_fb[fx_nfb].px = px;
367413
fx_fb[fx_nfb].style = style;
414+
fx_fb[fx_nfb].primary = primary;
368415
fx_nfb += 1;
369416
return f;
370417
}
@@ -374,7 +421,7 @@ static FxFace* fx_fallback(FxFace* face, u32 cp) {
374421
u32 px = face->size->metrics.y_ppem;
375422
long style = face->style_flags & (FX_STYLE_ITALIC | FX_STYLE_BOLD);
376423
for (u32 i = 0; i < fx_nfb; i += 1) {
377-
if (fx_fb[i].px == px && fx_fb[i].style == style && fx_has(fx_fb[i].face, cp) != 0) {
424+
if (fx_fb[i].primary == face && fx_fb[i].px == px && fx_fb[i].style == style && fx_has(fx_fb[i].face, cp) != 0) {
378425
return fx_fb[i].face;
379426
}
380427
}
@@ -394,6 +441,7 @@ static FxFace* fx_fallback(FxFace* face, u32 cp) {
394441
return NULL;
395442
}
396443
fx_FcCharSetAddChar(cs, cp);
444+
fx_FcPatternAddString(p, "lang", (const unsigned char*)fx_cjk_lang());
397445
fx_FcPatternAddCharSet(p, "charset", cs);
398446
fx_FcPatternAddString(p, "family", (const unsigned char*)((face->face_flags & FX_FACE_FIXED_WIDTH) ? "monospace" : "sans-serif"));
399447
if (style & FX_STYLE_BOLD) {
@@ -409,7 +457,7 @@ static FxFace* fx_fallback(FxFace* face, u32 cp) {
409457
FxFace* found = NULL;
410458
if (fs != NULL) {
411459
for (int i = 0; i < fs->nfont && i < 24 && found == NULL; i += 1) {
412-
found = fx_fb_open(fs->fonts[i], cp, px, style);
460+
found = fx_fb_open(fs->fonts[i], cp, px, style, face);
413461
}
414462
fx_FcFontSetDestroy(fs);
415463
}
@@ -434,14 +482,29 @@ Term font_glyph_run(Env e, Term* f, IoWork* w) {
434482
if (face == NULL) {
435483
return fx_list(e, NULL, 0);
436484
}
485+
FxFace* primary = face;
486+
long cell = 0;
437487
unsigned gi = fx_FT_Get_Char_Index(face, cp);
488+
if (gi == 0 && (primary->face_flags & FX_FACE_FIXED_WIDTH)) {
489+
unsigned zero = fx_FT_Get_Char_Index(primary, '0');
490+
if (zero && fx_FT_Load_Glyph(primary, zero, FX_LOAD_TARGET_LIGHT | FX_LOAD_NO_BITMAP) == 0)
491+
cell = primary->glyph->advance.x;
492+
}
438493
if (gi == 0 && cp >= 128) {
439494
FxFace* fb = fx_fallback(face, cp);
440495
if (fb != NULL) {
441496
face = fb;
442497
gi = fx_FT_Get_Char_Index(face, cp);
443498
}
444499
}
500+
if (gi == 0 && fx_cjk_wide(cp)) {
501+
static int warned = 0;
502+
if (!warned) {
503+
fprintf(stderr, "no CJK font: install fonts-noto-cjk\n");
504+
warned = 1;
505+
}
506+
return fx_list(e, NULL, 0);
507+
}
445508
if (fx_FT_Load_Glyph(face, gi, FX_LOAD_TARGET_LIGHT | FX_LOAD_NO_BITMAP) != 0
446509
|| fx_FT_Render_Glyph(face->glyph, FX_RENDER_MODE_NORMAL) != 0) {
447510
return fx_list(e, NULL, 0);
@@ -456,6 +519,15 @@ Term font_glyph_run(Env e, Term* f, IoWork* w) {
456519
ws[0] = gi != 0;
457520
ws[1] = (u32)s->advance.x;
458521
ws[2] = (u32)(s->bitmap_left + FX_BIAS);
522+
if (face != primary && cell > 0) {
523+
long advance = cell * (fx_cjk_wide(cp) ? 2 : 1);
524+
ws[1] = (u32)advance;
525+
// Centre the advance box, preserving the font's own bearing. CJK
526+
// punctuation intentionally sits at an edge of its em box.
527+
long delta = advance - s->advance.x;
528+
long shift = delta >= 0 ? (delta + 64) / 128 : -((-delta + 64) / 128);
529+
ws[2] = (u32)(s->bitmap_left + shift + FX_BIAS);
530+
}
459531
ws[3] = (u32)(s->bitmap_top + FX_BIAS);
460532
ws[4] = bw;
461533
ws[5] = bh;

0 commit comments

Comments
 (0)