Skip to content

Commit cab86dc

Browse files
vincentkocvincentkoc
committed
fix(docs): allow partial i18n doc batches
1 parent 5e8c396 commit cab86dc

2 files changed

Lines changed: 97 additions & 36 deletions

File tree

‎scripts/docs-i18n/main.go‎

Lines changed: 43 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -27,28 +27,30 @@ type docResult struct {
2727
}
2828

2929
type runConfig struct {
30-
targetLang string
31-
sourceLang string
32-
docsRoot string
33-
tmPath string
34-
mode string
35-
thinking string
36-
overwrite bool
37-
maxFiles int
38-
parallel int
30+
targetLang string
31+
sourceLang string
32+
docsRoot string
33+
tmPath string
34+
mode string
35+
thinking string
36+
overwrite bool
37+
allowPartial bool
38+
maxFiles int
39+
parallel int
3940
}
4041

4142
func main() {
4243
var (
43-
targetLang = flag.String("lang", "zh-CN", "target language (e.g., zh-CN)")
44-
sourceLang = flag.String("src", "en", "source language")
45-
docsRoot = flag.String("docs", "docs", "docs root")
46-
tmPath = flag.String("tm", "", "translation memory path")
47-
mode = flag.String("mode", "segment", "translation mode (segment|doc)")
48-
thinking = flag.String("thinking", "high", "thinking level (low|medium|high|xhigh)")
49-
overwrite = flag.Bool("overwrite", false, "overwrite existing translations")
50-
maxFiles = flag.Int("max", 0, "max files to process (0 = all)")
51-
parallel = flag.Int("parallel", 1, "parallel workers for doc mode")
44+
targetLang = flag.String("lang", "zh-CN", "target language (e.g., zh-CN)")
45+
sourceLang = flag.String("src", "en", "source language")
46+
docsRoot = flag.String("docs", "docs", "docs root")
47+
tmPath = flag.String("tm", "", "translation memory path")
48+
mode = flag.String("mode", "segment", "translation mode (segment|doc)")
49+
thinking = flag.String("thinking", "high", "thinking level (low|medium|high|xhigh)")
50+
overwrite = flag.Bool("overwrite", false, "overwrite existing translations")
51+
allowPartial = flag.Bool("allow-partial", false, "write successful doc-mode outputs even when another file fails")
52+
maxFiles = flag.Int("max", 0, "max files to process (0 = all)")
53+
parallel = flag.Int("parallel", 1, "parallel workers for doc mode")
5254
)
5355
flag.Parse()
5456
files := flag.Args()
@@ -57,15 +59,16 @@ func main() {
5759
}
5860

5961
if err := runDocsI18N(context.Background(), runConfig{
60-
targetLang: *targetLang,
61-
sourceLang: *sourceLang,
62-
docsRoot: *docsRoot,
63-
tmPath: *tmPath,
64-
mode: *mode,
65-
thinking: *thinking,
66-
overwrite: *overwrite,
67-
maxFiles: *maxFiles,
68-
parallel: *parallel,
62+
targetLang: *targetLang,
63+
sourceLang: *sourceLang,
64+
docsRoot: *docsRoot,
65+
tmPath: *tmPath,
66+
mode: *mode,
67+
thinking: *thinking,
68+
overwrite: *overwrite,
69+
allowPartial: *allowPartial,
70+
maxFiles: *maxFiles,
71+
parallel: *parallel,
6972
}, files, func(srcLang, tgtLang string, glossary []GlossaryEntry, thinking string) (docsTranslator, error) {
7073
return NewCodexTranslator(srcLang, tgtLang, glossary, thinking)
7174
}); err != nil {
@@ -127,7 +130,7 @@ func runDocsI18N(ctx context.Context, cfg runConfig, files []string, newTranslat
127130
processed := 0
128131
skipped := 0
129132
localizedFiles := []string{}
130-
var runErr error
133+
var translationErr error
131134

132135
log.Printf("docs-i18n: mode=%s total=%d pending=%d pre_skipped=%d overwrite=%t thinking=%s parallel=%d", cfg.mode, totalFiles, len(ordered), preSkipped, cfg.overwrite, cfg.thinking, parallel)
133136
switch cfg.mode {
@@ -138,7 +141,7 @@ func runDocsI18N(ctx context.Context, cfg runConfig, files []string, newTranslat
138141
skipped += skip
139142
localizedFiles = append(localizedFiles, outputs...)
140143
if err != nil {
141-
runErr = err
144+
translationErr = err
142145
}
143146
} else {
144147
translator, err := newTranslator(cfg.sourceLang, cfg.targetLang, glossary, cfg.thinking)
@@ -151,7 +154,7 @@ func runDocsI18N(ctx context.Context, cfg runConfig, files []string, newTranslat
151154
skipped += skip
152155
localizedFiles = append(localizedFiles, outputs...)
153156
if err != nil {
154-
runErr = err
157+
translationErr = err
155158
}
156159
}
157160
case "segment":
@@ -167,21 +170,25 @@ func runDocsI18N(ctx context.Context, cfg runConfig, files []string, newTranslat
167170
processed += proc
168171
localizedFiles = append(localizedFiles, outputs...)
169172
if err != nil {
170-
runErr = err
173+
translationErr = err
171174
}
172175
default:
173176
return fmt.Errorf("unknown mode: %s", cfg.mode)
174177
}
175178

176-
if err := tm.Save(); err != nil && runErr == nil {
177-
runErr = err
179+
if err := tm.Save(); err != nil {
180+
return err
178181
}
179-
if err := postprocessLocalizedDocs(resolvedDocsRoot, cfg.targetLang, localizedFiles); err != nil && runErr == nil {
180-
runErr = err
182+
if err := postprocessLocalizedDocs(resolvedDocsRoot, cfg.targetLang, localizedFiles); err != nil {
183+
return err
181184
}
182185
elapsed := time.Since(start).Round(time.Millisecond)
183186
log.Printf("docs-i18n: completed processed=%d skipped=%d elapsed=%s", processed, skipped, elapsed)
184-
return runErr
187+
if translationErr != nil && cfg.allowPartial && cfg.mode == "doc" && processed > 0 {
188+
log.Printf("docs-i18n: allowing partial doc output after translation error: %v", translationErr)
189+
return nil
190+
}
191+
return translationErr
185192
}
186193

187194
func runDocSequential(ctx context.Context, ordered []string, translator docsTranslator, docsRoot, srcLang, tgtLang string, overwrite bool) (int, int, []string, error) {

‎scripts/docs-i18n/main_test.go‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@ package main
22

33
import (
44
"context"
5+
"errors"
6+
"os"
57
"path/filepath"
68
"strings"
79
"testing"
@@ -49,6 +51,24 @@ func (transcriptFrontmatterTranslator) TranslateRaw(_ context.Context, text, _,
4951

5052
func (transcriptFrontmatterTranslator) Close() {}
5153

54+
type partialFailTranslator struct{}
55+
56+
func (partialFailTranslator) Translate(_ context.Context, text, _, _ string) (string, error) {
57+
if strings.Contains(text, "FAIL") {
58+
return "", errors.New("translation failed")
59+
}
60+
return text, nil
61+
}
62+
63+
func (partialFailTranslator) TranslateRaw(_ context.Context, text, _, _ string) (string, error) {
64+
if strings.Contains(text, "FAIL") {
65+
return "", errors.New("translation failed")
66+
}
67+
return text, nil
68+
}
69+
70+
func (partialFailTranslator) Close() {}
71+
5272
func TestRunDocsI18NRewritesFinalLocalizedPageLinks(t *testing.T) {
5373
t.Parallel()
5474

@@ -99,6 +119,40 @@ func TestRunDocsI18NRewritesFinalLocalizedPageLinks(t *testing.T) {
99119
}
100120
}
101121

122+
func TestRunDocsI18NAllowPartialKeepsSuccessfulDocOutputs(t *testing.T) {
123+
t.Parallel()
124+
125+
docsRoot := t.TempDir()
126+
writeFile(t, filepath.Join(docsRoot, ".i18n", "glossary.zh-CN.json"), "[]")
127+
writeFile(t, filepath.Join(docsRoot, "docs.json"), `{"redirects":[]}`)
128+
okPath := filepath.Join(docsRoot, "aaa-ok.md")
129+
failPath := filepath.Join(docsRoot, "zzz-fail.md")
130+
writeFile(t, okPath, "# Gateway\n")
131+
writeFile(t, failPath, "# FAIL\n")
132+
133+
err := runDocsI18N(context.Background(), runConfig{
134+
targetLang: "zh-CN",
135+
sourceLang: "en",
136+
docsRoot: docsRoot,
137+
mode: "doc",
138+
thinking: "high",
139+
overwrite: true,
140+
allowPartial: true,
141+
parallel: 1,
142+
}, []string{okPath, failPath}, func(_, _ string, _ []GlossaryEntry, _ string) (docsTranslator, error) {
143+
return partialFailTranslator{}, nil
144+
})
145+
if err != nil {
146+
t.Fatalf("runDocsI18N failed despite partial output: %v", err)
147+
}
148+
if got := mustReadFile(t, filepath.Join(docsRoot, "zh-CN", "aaa-ok.md")); !strings.Contains(got, "# Gateway") {
149+
t.Fatalf("expected successful output to be written, got:\n%s", got)
150+
}
151+
if _, err := os.Stat(filepath.Join(docsRoot, "zh-CN", "zzz-fail.md")); err == nil {
152+
t.Fatal("did not expect failed output to be written")
153+
}
154+
}
155+
102156
func TestTranslateSnippetDoesNotCacheFallbackToSource(t *testing.T) {
103157
t.Parallel()
104158

0 commit comments

Comments
 (0)