Skip to content

Commit 4139938

Browse files
steipetesteipete
committed
fix(docs): preserve translated list structure
1 parent fbfa137 commit 4139938

7 files changed

Lines changed: 179 additions & 4 deletions

File tree

‎scripts/docs-i18n/doc_chunked_raw.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ type docChunkStructure struct {
2929
fenceCount int
3030
tagCounts map[string]int
3131
headingLevels []int
32+
listShapes []markdownListShape
3233
inlineCodeSpans []string
3334
fencedPlaceholders []string
3435
fencedProtocolTokens []string
@@ -69,6 +70,9 @@ func validateDocBodyFencedLiterals(source, translated string) error {
6970
}
7071
sourceStructure := summarizeDocChunkStructure(source)
7172
translatedStructure := summarizeDocChunkStructure(translated)
73+
if !slices.Equal(sourceStructure.listShapes, translatedStructure.listShapes) {
74+
return fmt.Errorf("list structure mismatch: source=%v translated=%v", sourceStructure.listShapes, translatedStructure.listShapes)
75+
}
7276
if !slices.Equal(sourceStructure.fencedPlaceholders, translatedStructure.fencedPlaceholders) {
7377
return fmt.Errorf("fenced placeholder mismatch: source=%d translated=%d", len(sourceStructure.fencedPlaceholders), len(translatedStructure.fencedPlaceholders))
7478
}
@@ -234,6 +238,9 @@ func validateDocChunkTranslation(source, translated string) error {
234238
if !slices.Equal(sourceStructure.headingLevels, translatedStructure.headingLevels) {
235239
return fmt.Errorf("heading structure mismatch: source=%v translated=%v", sourceStructure.headingLevels, translatedStructure.headingLevels)
236240
}
241+
if !slices.Equal(sourceStructure.listShapes, translatedStructure.listShapes) {
242+
return fmt.Errorf("list structure mismatch: source=%v translated=%v", sourceStructure.listShapes, translatedStructure.listShapes)
243+
}
237244
if !sameStringMultiset(sourceStructure.inlineCodeSpans, translatedStructure.inlineCodeSpans) {
238245
return fmt.Errorf("inline code mismatch: source=%d translated=%d", len(sourceStructure.inlineCodeSpans), len(translatedStructure.inlineCodeSpans))
239246
}
@@ -392,6 +399,7 @@ func summarizeDocChunkStructure(text string) docChunkStructure {
392399
fenceCount: counts["__fence_toggle__"],
393400
tagCounts: countsWithoutFence(counts),
394401
headingLevels: extractMarkdownHeadingLevels(text),
402+
listShapes: extractMarkdownListShapes(text),
395403
inlineCodeSpans: extractMarkdownInlineCodeValues(text),
396404
fencedPlaceholders: fencedPlaceholders,
397405
fencedProtocolTokens: fencedProtocolTokens,

‎scripts/docs-i18n/doc_mode_test.go‎

Lines changed: 97 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -197,6 +197,21 @@ func (uppercaseWrapperTranslator) TranslateRaw(_ context.Context, text, _, _ str
197197

198198
func (uppercaseWrapperTranslator) Close() {}
199199

200+
type boundaryWrapperTranslator struct{}
201+
202+
func (boundaryWrapperTranslator) Translate(_ context.Context, text, _, _ string) (string, error) {
203+
return text, nil
204+
}
205+
206+
func (boundaryWrapperTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
207+
if strings.Contains(text, "Intro paragraph") {
208+
return "<body>\nEinleitung\n</body>", nil
209+
}
210+
return strings.NewReplacer("First item", "Erster Eintrag", "Second item", "Zweiter Eintrag").Replace(text), nil
211+
}
212+
213+
func (boundaryWrapperTranslator) Close() {}
214+
200215
type oversizedBlockTranslator struct {
201216
rawInputs []string
202217
}
@@ -513,6 +528,73 @@ func TestValidateDocChunkTranslationAcceptsTranslatedHeadingText(t *testing.T) {
513528
}
514529
}
515530

531+
func TestValidateDocChunkTranslationRejectsAccidentalOrderedListFromTranslatedDate(t *testing.T) {
532+
t.Parallel()
533+
534+
source := "Catalog pricing uses the introductory rate through August 31, 2026, then the standard rate from\nSeptember 1. The regional endpoints use a premium.\n"
535+
translated := "Die Katalogpreise verwenden den Einführungstarif bis zum 31. August 2026 und danach den Standardtarif ab\n1. September. Für die regionalen Endpunkte gilt ein Aufschlag.\n"
536+
537+
err := validateDocChunkTranslation(source, translated)
538+
if err == nil {
539+
t.Fatal("expected an accidental ordered list from a translated date to be rejected")
540+
}
541+
if !strings.Contains(err.Error(), "list structure mismatch") {
542+
t.Fatalf("expected list structure mismatch, got %v", err)
543+
}
544+
}
545+
546+
func TestValidateDocChunkTranslationAcceptsTranslatedDateWithoutListChange(t *testing.T) {
547+
t.Parallel()
548+
549+
source := "Catalog pricing uses the introductory rate through August 31, 2026, then the standard rate from\nSeptember 1. The regional endpoints use a premium.\n"
550+
translated := "Die Katalogpreise verwenden den Einführungstarif bis zum 31. August 2026. Ab dem 1. September gilt der Standardtarif.\nFür die regionalen Endpunkte gilt ein Aufschlag.\n"
551+
552+
if err := validateDocChunkTranslation(source, translated); err != nil {
553+
t.Fatalf("expected translated prose without a list-shape change to pass, got %v", err)
554+
}
555+
}
556+
557+
func TestValidateDocChunkTranslationPreservesNestedListShape(t *testing.T) {
558+
t.Parallel()
559+
560+
source := "- First\n 3. Nested first\n 4. Nested second\n- Second\n"
561+
translated := "- Erstens\n 3. Verschachtelt eins\n 4. Verschachtelt zwei\n- Zweitens\n"
562+
563+
if err := validateDocChunkTranslation(source, translated); err != nil {
564+
t.Fatalf("expected equivalent translated list structure to pass, got %v", err)
565+
}
566+
}
567+
568+
func TestValidateDocChunkTranslationRejectsChangedListNesting(t *testing.T) {
569+
t.Parallel()
570+
571+
source := "- First\n - Nested\n- Second\n"
572+
translated := "- Erstens\n- Nicht mehr verschachtelt\n- Zweitens\n"
573+
574+
err := validateDocChunkTranslation(source, translated)
575+
if err == nil {
576+
t.Fatal("expected changed list nesting to be rejected")
577+
}
578+
if !strings.Contains(err.Error(), "list structure mismatch") {
579+
t.Fatalf("expected list structure mismatch, got %v", err)
580+
}
581+
}
582+
583+
func TestValidateDocChunkTranslationRejectsNestedListMovedToDifferentParentItem(t *testing.T) {
584+
t.Parallel()
585+
586+
source := "- First\n - Nested under first\n- Second\n"
587+
translated := "- Erstens\n- Zweitens\n - Unter dem zweiten verschachtelt\n"
588+
589+
err := validateDocChunkTranslation(source, translated)
590+
if err == nil {
591+
t.Fatal("expected a nested list moved to another parent item to be rejected")
592+
}
593+
if !strings.Contains(err.Error(), "list structure mismatch") {
594+
t.Fatalf("expected list structure mismatch, got %v", err)
595+
}
596+
}
597+
516598
func TestValidateDocChunkTranslationRejectsTranslatedInlineCode(t *testing.T) {
517599
t.Parallel()
518600

@@ -1589,6 +1671,19 @@ func TestTranslateDocBodyChunkedStripsUppercaseBodyWrapper(t *testing.T) {
15891671
}
15901672
}
15911673

1674+
func TestTranslateDocBodyChunkedRejectsListCorruptionAcrossSanitizedChunkBoundary(t *testing.T) {
1675+
body := "Intro paragraph.\n\n1. First item\n2. Second item\n\n"
1676+
1677+
t.Setenv("OPENCLAW_DOCS_I18N_DOC_CHUNK_MAX_BYTES", "20")
1678+
_, err := translateDocBodyChunked(context.Background(), boundaryWrapperTranslator{}, "example.md", body, "en", "de")
1679+
if err == nil {
1680+
t.Fatal("expected final-document list corruption across chunk boundaries to be rejected")
1681+
}
1682+
if !strings.Contains(err.Error(), "final document validation: list structure mismatch") {
1683+
t.Fatalf("expected final list structure mismatch, got %v", err)
1684+
}
1685+
}
1686+
15921687
func TestSanitizeDocChunkProtocolWrappersKeepsBodyOnlyWrapperWhenSourceMentionsBodyTag(t *testing.T) {
15931688
t.Parallel()
15941689

@@ -1932,8 +2027,8 @@ func TestProcessFileDocUsesFieldLevelFrontmatterTranslation(t *testing.T) {
19322027
if !strings.Contains(text, "在 Fly.io 上部署 OpenClaw") {
19332028
t.Fatalf("expected translated read_when entry in output:\n%s", text)
19342029
}
1935-
if !strings.Contains(text, "prompt_version: 10") {
1936-
t.Fatalf("expected prompt version 10 in output metadata:\n%s", text)
2030+
if !strings.Contains(text, "prompt_version: 11") {
2031+
t.Fatalf("expected prompt version 11 in output metadata:\n%s", text)
19372032
}
19382033
}
19392034

‎scripts/docs-i18n/markdown_segments.go‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package main
22

33
import (
44
"sort"
5+
"strconv"
56
"strings"
67

78
"github.com/yuin/goldmark"
@@ -111,6 +112,75 @@ func extractMarkdownHeadingLevels(body string) []int {
111112
return levels
112113
}
113114

115+
type markdownListShape struct {
116+
ordered bool
117+
start int
118+
depth int
119+
items int
120+
parentItemPath string
121+
}
122+
123+
func extractMarkdownListShapes(body string) []markdownListShape {
124+
parseSource := []byte(normalizeDocComponentsForMarkdownParse(body))
125+
doc := goldmark.New(goldmark.WithExtensions(extension.GFM)).Parser().Parse(text.NewReader(parseSource))
126+
shapes := []markdownListShape{}
127+
_ = ast.Walk(doc, func(node ast.Node, entering bool) (ast.WalkStatus, error) {
128+
if !entering {
129+
return ast.WalkContinue, nil
130+
}
131+
list, ok := node.(*ast.List)
132+
if !ok {
133+
return ast.WalkContinue, nil
134+
}
135+
depth := 0
136+
for parent := list.Parent(); parent != nil; parent = parent.Parent() {
137+
if _, ok := parent.(*ast.List); ok {
138+
depth++
139+
}
140+
}
141+
items := 0
142+
for child := list.FirstChild(); child != nil; child = child.NextSibling() {
143+
if _, ok := child.(*ast.ListItem); ok {
144+
items++
145+
}
146+
}
147+
shapes = append(shapes, markdownListShape{
148+
ordered: list.IsOrdered(),
149+
start: list.Start,
150+
depth: depth,
151+
items: items,
152+
parentItemPath: markdownListParentItemPath(list),
153+
})
154+
return ast.WalkContinue, nil
155+
})
156+
return shapes
157+
}
158+
159+
func markdownListParentItemPath(list *ast.List) string {
160+
indices := []int{}
161+
for parent := list.Parent(); parent != nil; parent = parent.Parent() {
162+
item, ok := parent.(*ast.ListItem)
163+
if !ok {
164+
continue
165+
}
166+
index := 0
167+
for sibling := item.Parent().FirstChild(); sibling != item; sibling = sibling.NextSibling() {
168+
if _, ok := sibling.(*ast.ListItem); ok {
169+
index++
170+
}
171+
}
172+
indices = append(indices, index)
173+
}
174+
for left, right := 0, len(indices)-1; left < right; left, right = left+1, right-1 {
175+
indices[left], indices[right] = indices[right], indices[left]
176+
}
177+
parts := make([]string, len(indices))
178+
for index, itemIndex := range indices {
179+
parts[index] = strconv.Itoa(itemIndex)
180+
}
181+
return strings.Join(parts, ".")
182+
}
183+
114184
func extractMarkdownInlineCodeValues(body string) []string {
115185
parseSource := []byte(normalizeDocComponentsForMarkdownParse(body))
116186
doc := goldmark.New(goldmark.WithExtensions(extension.GFM)).Parser().Parse(text.NewReader(parseSource))

‎scripts/docs-i18n/prompt.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,7 @@ Rules:
129129
- Preserve YAML structure inside <frontmatter>; translate only values.
130130
- Preserve every [[[FM_*]]] marker exactly and translate only text between its START/END pair.
131131
- Preserve Markdown structure exactly: headings, list nesting, tables, links, emphasis, and line-level content order.
132+
- Preserve Markdown list nodes exactly: ordered versus unordered kind, nesting, item count, and ordered-list starting number. Do not let translated prose accidentally become a list item; for example, if a wrapped date would begin a line with “1.”, rephrase it or keep it on the preceding line when the source is not a list.
132133
- Preserve HTML/MDX tag names, attribute names, nesting, and structural attribute values exactly. Never change resource or behavior attributes such as “href”, “src”, “id”, “icon”, “path”, “type”, or “default”.
133134
- Translate user-visible prose inside string-valued component attributes such as “title”, “label”, “description”, and “placeholder”, unless a higher-precedence literal UI-label rule protects that value. Do not translate code-like attribute values.
134135
- Do not translate or modify code spans, executable code or config blocks, config keys, CLI flags, environment variables, commands, or placeholders such as __OC_I18N_####__.

‎scripts/docs-i18n/prompt_test.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ func TestTranslationPromptUsesSharedContractAndLocaleOverlayForEverySupportedLoc
4242
"Keep authentication, authorization, credentials, tokens, passwords, secrets, identities, and accounts distinct",
4343
"Preserve actors, objects, temporal order, negation, conditions, scope, singular/plural meaning, and requirement strength",
4444
"Preserve every factual value exactly, including numbers, units, versions, ports, limits, durations, paths, and comparison operators",
45+
"Preserve Markdown list nodes exactly: ordered versus unordered kind, nesting, item count, and ordered-list starting number",
4546
"Preserve HTML/MDX tag names, attribute names, nesting, and structural attribute values exactly",
4647
"Fenced text, transcript, output, and documentation examples are an exception to the preceding block rule",
4748
"Translate user-visible prose inside string-valued component attributes such as “title”, “label”, “description”, and “placeholder”",

‎scripts/docs-i18n/util.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ import (
1212

1313
const (
1414
workflowVersion = 16
15-
promptVersion = 10
15+
promptVersion = 11
1616
docsI18nEngineName = "codex"
1717
envDocsI18nProvider = "OPENCLAW_DOCS_I18N_PROVIDER"
1818
envDocsI18nModel = "OPENCLAW_DOCS_I18N_MODEL"

‎scripts/docs-i18n/util_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import (
88
func TestCacheNamespaceIncludesPromptVersion(t *testing.T) {
99
t.Parallel()
1010

11-
if want := "prompt=10"; !strings.Contains(cacheNamespace(), want) {
11+
if want := "prompt=11"; !strings.Contains(cacheNamespace(), want) {
1212
t.Fatalf("expected cache namespace to contain %q, got %q", want, cacheNamespace())
1313
}
1414
}

0 commit comments

Comments
 (0)