Skip to content

Commit 5ca5c59

Browse files
authored
feat(workbook): read every Japanese form under a pinned type (#2998)
1 parent 51bf8c7 commit 5ca5c59

17 files changed

Lines changed: 1038 additions & 172 deletions

File tree

‎conformance/spec077_xlsx/extract_test.go‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -143,6 +143,10 @@ type extracted struct {
143143
Contact string `json:"contact"`
144144
Title string `json:"title"`
145145
Terms string `json:"terms"`
146+
// Delivered and Accepted are the datetime and boolean fields of the
147+
// Japanese quote.
148+
Delivered string `json:"delivered"`
149+
Accepted bool `json:"accepted"`
146150
// TotalText is the total rendered inside a quoted string, so a null
147151
// total can be told from a number.
148152
TotalText string `json:"total_text"`
@@ -283,6 +287,9 @@ func TestXlsxExtractJapanese(t *testing.T) {
283287
require.Equal(t, "Q-2026-001", out.QuoteNo, "a string field keeps the text as it is")
284288
require.Equal(t, "2026-10-15", out.Delivery, "an era date pinned to date")
285289
require.Equal(t, float64(123000), out.Total, "a yen amount pinned to number")
290+
require.Equal(t, "2026-10-15T14:30:00", out.Delivered, "an era date with 午後 pinned to date-time")
291+
require.True(t, out.Accepted, "済 pinned to boolean")
292+
require.Equal(t, "Sheet1!B6", out.Cells["accepted"])
286293
}
287294

288295
// dagu xlsx cache clear and dagu rm --history drop the cached cells.

‎conformance/spec077_xlsx/gaps_merge_test.go‎

Lines changed: 44 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -62,31 +62,57 @@ func TestXlsxMergedCellErrors(t *testing.T) {
6262
}
6363
}
6464

65-
// Spec 077 "Typing": Japanese text under a pinned type reads as its ASCII
66-
// form, a yen amount, a kanji date, or an era date; text that is none of
67-
// these still fails the type, quoted as the cell holds it.
65+
// Spec 077 "Japanese text under a pinned type": every family of amount,
66+
// date, time, and yes/no form reads under its pinned type; text outside
67+
// the forms fails the type, quoted as the cell holds it.
6868
func TestXlsxTypesJapanese(t *testing.T) {
6969
t.Parallel()
7070
dagu := harness.NewRunner(t)
7171
dagu.Run("start", "types_japanese.yaml").ExpectExitCode(0)
7272

73-
var rows []map[string]any
74-
readJSON(t, dagu, "out.json", &rows)
75-
require.Len(t, rows, 4)
76-
require.Equal(t, float64(123456), rows[0]["Amount"], "full-width digits and comma")
77-
require.Equal(t, "2026-10-03", rows[0]["Due"], "full-width slashes")
78-
require.Equal(t, "007", rows[0]["Code"], "an unpinned column keeps its text")
79-
require.Equal(t, float64(123000), rows[1]["Amount"], "a yen sign before the amount")
80-
require.Equal(t, "2026-10-03", rows[1]["Due"], "an era date, long")
81-
require.Equal(t, float64(123000), rows[2]["Amount"], "円 after the amount")
82-
require.Equal(t, "2026-10-03", rows[2]["Due"], "an era date, short")
83-
require.Equal(t, float64(-5), rows[3]["Amount"], "the minus sign U+2212")
84-
require.Equal(t, "2019-05-01", rows[3]["Due"], "the first year of an era")
73+
var out struct {
74+
Rows []map[string]any `json:"rows"`
75+
Warnings []string `json:"warnings"`
76+
}
77+
readJSON(t, dagu, "out.json", &out)
78+
require.Len(t, out.Rows, 14)
79+
for i, want := range []struct {
80+
amount float64
81+
due string
82+
when string
83+
done any
84+
note string
85+
}{
86+
{123456, "2026-10-03", "2026-10-03T14:30:00", true, "full-width digits; a slash date with a time; a circle"},
87+
{123000, "2026-10-03", "2026-10-03T14:30:00", false, "a yen sign; an era date; 時分; a cross"},
88+
{123000, "2026-10-03", "2026-10-03T14:30:15", true, "円 after; a short era date; 時分秒; はい"},
89+
{-5, "2019-05-01", "2026-10-03T14:30:15", false, "U+2212; the first year of an era; a dot date with seconds; いいえ"},
90+
{123500, "2026-10-03", "2026-10-03T14:30:00", true, "Arabic digits with 万; a weekday mark; 午後; 済"},
91+
{-1000, "2026-10-01", "2026-10-03T14:00:00", false, "▲; an era month; 時 alone; 未"},
92+
{-1000, "2026-10-01", "2026-10-03T09:00:00", false, "accounting parentheses; a kanji month; 午前; 無"},
93+
{123000, "2026-10-03", "2026-10-03T14:30:00", nil, "the trailing dash; kanji digits in place; (Fri); △ is not a boolean"},
94+
{10, "2026-10-03", "2026-10-03T14:30:00", false, "percent; an era ligature; a weekday before the time; a dash for none"},
95+
{123000, "2019-04-30", "2026-10-03T12:00:00", false, "a receipt in daiji; kanji numerals with units; 午後0時; an empty box"},
96+
{1000, "2026-10-03", "2026-10-03T00:00:00", true, "a price tag; a dot date; 午前12時; 有"},
97+
{150000000, "2019-04-30", "2026-10-03T00:00:00", true, "億; a dashed era date; a date alone; a check mark"},
98+
{1000, "2026-10-03", "2026-10-03T14:30:00", false, "a tag in parentheses; a kanji era date; a colon time; 不要"},
99+
{1000000, "2026-10-01", "2026-10-03T14:00:00", true, "百万; a slash month; 金曜日 before the time; 完了"},
100+
} {
101+
row := out.Rows[i]
102+
require.Equal(t, want.amount, row["Amount"], "row %d amount: %s", i+2, want.note)
103+
require.Equal(t, want.due, row["Due"], "row %d due: %s", i+2, want.note)
104+
require.Equal(t, want.when, row["When"], "row %d when: %s", i+2, want.note)
105+
require.Equal(t, want.done, row["Done"], "row %d done: %s", i+2, want.note)
106+
}
107+
require.Equal(t, "007", out.Rows[0]["Code"], "an unpinned column keeps its text")
108+
require.Equal(t, []string{`Sheet1!D9: expected boolean, found "△"`}, out.Warnings)
85109

86110
var warnings []string
87111
readJSON(t, dagu, "warnings.json", &warnings)
88112
require.Equal(t, []string{
89-
`Sheet1!C4: expected number, found "abc"`,
90-
`Sheet1!C5: expected number, found "x"`,
91-
}, warnings, "full-width digits pinned to number convert; other text is quoted as it is")
113+
`Sheet1!E4: expected number, found "abc"`,
114+
`Sheet1!E5: expected number, found "1,000(千円)"`,
115+
`Sheet1!E6: expected number, found "12万3万"`,
116+
`Sheet1!D9: expected boolean, found "△"`,
117+
}, warnings, "text outside the forms is quoted as the cell holds it")
92118
}

‎conformance/spec077_xlsx/testdata/extract_japanese.yaml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ steps:
1818
- ["見積番号", "Q-2026-001"]
1919
- ["納期", "令和8年10月15日"]
2020
- ["合計金額", "¥123,000"]
21+
- ["納品日時", "令和8年10月15日 午後2時30分"]
22+
- ["検収", "済"]
2123
- id: fields
2224
depends: form
2325
action: xlsx.extract
@@ -30,9 +32,11 @@ steps:
3032
quote_no: {type: string, description: 見積番号}
3133
delivery: {type: string, format: date, description: 納期}
3234
total: {type: number, description: 合計金額}
35+
delivered: {type: string, format: date-time, description: 納品日時}
36+
accepted: {type: boolean, description: 検収}
3337
- id: out
3438
depends: fields
3539
action: file.write
3640
with:
3741
path: out.json
38-
content: '{"quote_no": "${steps.fields.outputs.quote_no}", "delivery": "${steps.fields.outputs.delivery}", "total": ${steps.fields.outputs.total}, "cells": ${steps.fields.outputs.cells}, "source": "${steps.fields.outputs.source}"}'
42+
content: '{"quote_no": "${steps.fields.outputs.quote_no}", "delivery": "${steps.fields.outputs.delivery}", "total": ${steps.fields.outputs.total}, "delivered": "${steps.fields.outputs.delivered}", "accepted": ${steps.fields.outputs.accepted}, "cells": ${steps.fields.outputs.cells}, "source": "${steps.fields.outputs.source}"}'

‎conformance/spec077_xlsx/testdata/types_japanese.yaml‎

Lines changed: 19 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -4,30 +4,41 @@ steps:
44
action: xlsx.write
55
with:
66
path: amounts.xlsx
7-
columns: [Amount, Due, Code]
7+
columns: [Amount, Due, When, Done, Code]
88
rows:
9-
- {"Amount": "123,456", "Due": "2026/10/03", "Code": "007"}
10-
- {"Amount": "¥123,000", "Due": "令和8年10月3日", "Code": "123"}
11-
- {"Amount": "123,000円", "Due": "R8.10.3", "Code": "abc"}
12-
- {"Amount": "−5", "Due": "令和元年5月1日", "Code": "x"}
9+
- {"Amount": "123,456", "Due": "2026/10/03", "When": "2026/10/3 14:30", "Done": "○", "Code": "007"}
10+
- {"Amount": "¥123,000", "Due": "令和8年10月3日", "When": "2026年10月3日 14時30分", "Done": "×", "Code": "123"}
11+
- {"Amount": "123,000円", "Due": "R8.10.3", "When": "令和8年10月3日 14時30分15秒", "Done": "はい", "Code": "abc"}
12+
- {"Amount": "−5", "Due": "令和元年5月1日", "When": "2026.10.3 14:30:15", "Done": "いいえ", "Code": "1,000(千円)"}
13+
- {"Amount": "12万3,500円", "Due": "令和8年10月3日(金)", "When": "2026年10月3日 午後2時30分", "Done": "済", "Code": "12万3万"}
14+
- {"Amount": "▲1,000", "Due": "R8.10", "When": "2026年10月3日14時", "Done": "未", "Code": "100"}
15+
- {"Amount": "(1,000)", "Due": "2026年10月", "When": "令和8年10月3日 午前9時", "Done": "無", "Code": "100"}
16+
- {"Amount": "¥123,000-", "Due": "二〇二六年十月三日", "When": "2026/10/3 (Fri) 14:30", "Done": "△", "Code": "100"}
17+
- {"Amount": "10%", "Due": "㋿8.10.3", "When": "2026年10月3日(金)14:30", "Done": "ー", "Code": "100"}
18+
- {"Amount": "金壱拾弐万参千円也", "Due": "平成三十一年四月三十日", "When": "2026年10月3日 午後0時", "Done": "☐", "Code": "100"}
19+
- {"Amount": "税込1,000", "Due": "2026.10.3", "When": "2026年10月3日 午前12時", "Done": "有", "Code": "100"}
20+
- {"Amount": "1.5億", "Due": "H31/4/30", "When": "2026年10月3日", "Done": "✓", "Code": "100"}
21+
- {"Amount": "1,000(税込)", "Due": "令和八年十月三日", "When": "令和8年10月3日 14:30", "Done": "不要", "Code": "100"}
22+
- {"Amount": "百万", "Due": "2026/10", "When": "2026年10月3日 金曜日 14時", "Done": "完了", "Code": "100"}
1323
- id: read
1424
depends: write
1525
action: xlsx.read
1626
with:
1727
path: amounts.xlsx
18-
types: {Amount: number, Due: date}
28+
types: {Amount: number, Due: date, When: datetime, Done: boolean}
29+
on_type_error: warn
1930
- id: out
2031
depends: read
2132
action: file.write
2233
with:
2334
path: out.json
24-
content: ${steps.read.outputs.rows}
35+
content: '{"rows": ${steps.read.outputs.rows}, "warnings": ${steps.read.outputs.warnings}}'
2536
- id: bad
2637
depends: write
2738
action: xlsx.read
2839
with:
2940
path: amounts.xlsx
30-
types: {Code: number}
41+
types: {Code: number, Done: boolean}
3142
on_type_error: warn
3243
- id: warnings
3344
depends: bad

‎internal/cmn/workbook/boolean.go‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
// Copyright (C) 2026 Yota Hamada
2+
// SPDX-License-Identifier: GPL-3.0-or-later
3+
4+
package workbook
5+
6+
import "strings"
7+
8+
// booleanWords maps the words and marks a form writes in a yes/no column
9+
// to their values, after folding and lowercasing. Nothing outside the map
10+
// is a boolean: a prefix such as 未 or a suffix such as 済 does not decide
11+
// a word, so 未定 and 対応済み fail rather than guess.
12+
var booleanWords = map[string]bool{
13+
// English.
14+
"true": true, "yes": true, "y": true, "on": true, "ok": true, "1": true,
15+
"false": false, "no": false, "n": false, "off": false, "ng": false, "0": false,
16+
// Words.
17+
"はい": true, "有": true, "有り": true, "あり": true, "済": true, "済み": true, "完了": true, "完": true, "了": true,
18+
"可": true, "要": true, "必要": true, "対象": true, "該当": true, "真": true,
19+
"いいえ": false, "無": false, "無し": false, "なし": false, "未": false, "未済": false, "未了": false, "未完": false,
20+
"未完了": false, "不可": false, "否": false, "不要": false, "対象外": false, "非該当": false, "偽": false,
21+
// Marks.
22+
"○": true, "〇": true, "◯": true, "◎": true, "●": true, "✓": true, "✔": true, "✅": true, "☑": true, "レ": true,
23+
"×": false, "✕": false, "✗": false, "✘": false, "❌": false, "☐": false, "-": false, "ー": false, "―": false, "—": false,
24+
}
25+
26+
// parseBooleanText reads a yes/no word or mark the way a Japanese form
27+
// writes it. See spec 077, Japanese text under a pinned type.
28+
func parseBooleanText(text string) (bool, bool) {
29+
value, ok := booleanWords[strings.ToLower(foldWidth(text))]
30+
return value, ok
31+
}
Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,54 @@
1+
// Copyright (C) 2026 Yota Hamada
2+
// SPDX-License-Identifier: GPL-3.0-or-later
3+
4+
package workbook
5+
6+
import (
7+
"testing"
8+
9+
"github.com/stretchr/testify/assert"
10+
"github.com/stretchr/testify/require"
11+
)
12+
13+
// Every yes/no word and mark spec 077 lists under Japanese text under a
14+
// pinned type, and what it says is not read.
15+
func TestParseBooleanText(t *testing.T) {
16+
t.Parallel()
17+
for _, in := range []string{
18+
"true", "TRUE", "yes", "y", "on", "ok", "OK", "1", "1",
19+
"はい", "有", "有り", "あり", "済", "済み", " 済 ", "完了", "完", "了", "可", "要", "必要", "対象", "該当", "真",
20+
"○", "〇", "◯", "◎", "●", "✓", "✔", "✅", "☑", "レ", "レ",
21+
} {
22+
v, ok := parseBooleanText(in)
23+
require.True(t, ok, in)
24+
assert.True(t, v, in)
25+
}
26+
for _, in := range []string{
27+
"false", "no", "n", "off", "ng", "0", "0",
28+
"いいえ", "無", "無し", "なし", "未", "未済", "未了", "未完", "未完了", "不可", "否", "不要", "対象外", "非該当", "偽",
29+
"×", "✕", "✗", "✘", "❌", "☐", "-", "-", "−", "ー", "―", "—",
30+
} {
31+
v, ok := parseBooleanText(in)
32+
require.True(t, ok, in)
33+
assert.False(t, v, in)
34+
}
35+
for _, in := range []string{"△", "☒", "?", "?", "未定", "保留", "TBD", "-1", "2", "対応済み", "", "yes no"} {
36+
_, ok := parseBooleanText(in)
37+
assert.False(t, ok, in)
38+
}
39+
}
40+
41+
// Through coerce: text, real booleans, and 0 and 1 all read; a failure
42+
// quotes the cell's text.
43+
func TestCoerceJapaneseBooleans(t *testing.T) {
44+
t.Parallel()
45+
for in, want := range map[any]bool{"済": true, "×": false, "TRUE": true, "1": true, true: true, int64(0): false, float64(1): true} {
46+
v, err := coerce(in, TypeBoolean, false)
47+
require.NoError(t, err, in)
48+
assert.Equal(t, want, v, in)
49+
}
50+
_, err := coerce("未定", TypeBoolean, false)
51+
require.EqualError(t, err, `expected boolean, found "未定"`)
52+
_, err = coerce(int64(2), TypeBoolean, false)
53+
require.EqualError(t, err, `expected boolean, found 2`)
54+
}

0 commit comments

Comments
 (0)