Skip to content

Commit eaeedbf

Browse files
hxy91819clawsweeper[bot]
hxy91819
and
clawsweeper[bot]
authored
fix(docs): finalize i18n postprocess before skip (#92668)
Summary: - Merged fix(docs): finalize i18n postprocess before skip after ClawSweeper review. Automerge notes: - No ClawSweeper repair was needed after automerge opt-in. Validation: - ClawSweeper review passed for head ad79445. - Required merge gates passed before the squash merge. Prepared head SHA: ad79445 Review: #92668 (comment) Co-authored-by: Mason Huang <[email protected]> Co-authored-by: clawsweeper[bot] <274271284+clawsweeper[bot]@users.noreply.github.com> Approved-by: hxy91819 Co-authored-by: hxy91819 <[email protected]>
1 parent dc493bc commit eaeedbf

6 files changed

Lines changed: 378 additions & 32 deletions

File tree

‎scripts/docs-i18n/doc_mode.go‎

Lines changed: 52 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,14 @@ const (
1818
bodyTagEnd = "</body>"
1919
)
2020

21+
type docOutputStatus int
22+
23+
const (
24+
docOutputNeedsTranslation docOutputStatus = iota
25+
docOutputReady
26+
docOutputNeedsPostprocess
27+
)
28+
2129
func processFileDoc(ctx context.Context, translator docsTranslator, docsRoot, filePath, srcLang, tgtLang string, overwrite bool) (bool, string, error) {
2230
absPath, relPath, err := resolveDocsPath(docsRoot, filePath)
2331
if err != nil {
@@ -32,12 +40,15 @@ func processFileDoc(ctx context.Context, translator docsTranslator, docsRoot, fi
3240

3341
outputPath := filepath.Join(docsRoot, tgtLang, relPath)
3442
if !overwrite {
35-
skip, err := shouldSkipDoc(outputPath, currentHash)
43+
status, err := classifyDocOutput(outputPath, currentHash, tgtLang)
3644
if err != nil {
3745
return false, "", err
3846
}
39-
if skip {
47+
switch status {
48+
case docOutputReady:
4049
return true, "", nil
50+
case docOutputNeedsPostprocess:
51+
return true, outputPath, nil
4152
}
4253
}
4354

@@ -138,31 +149,42 @@ func trimTagNewlines(value string) string {
138149
return value
139150
}
140151

141-
func shouldSkipDoc(outputPath string, sourceHash string) (bool, error) {
152+
func classifyDocOutput(outputPath string, sourceHash string, targetLang string) (docOutputStatus, error) {
142153
data, err := os.ReadFile(outputPath)
143154
if err != nil {
144155
if os.IsNotExist(err) {
145-
return false, nil
156+
return docOutputNeedsTranslation, nil
146157
}
147-
return false, err
158+
return docOutputNeedsTranslation, err
148159
}
149160
frontMatter, _ := splitFrontMatter(string(data))
150161
if frontMatter == "" {
151-
return false, nil
162+
return docOutputNeedsTranslation, nil
152163
}
153164
frontData := map[string]any{}
154165
if err := yaml.Unmarshal([]byte(frontMatter), &frontData); err != nil {
155-
return false, nil
166+
return docOutputNeedsTranslation, nil
156167
}
157168
storedHash := extractSourceHash(frontData)
158169
if storedHash == "" {
159-
return false, nil
170+
return docOutputNeedsTranslation, nil
171+
}
172+
if !strings.EqualFold(storedHash, sourceHash) {
173+
return docOutputNeedsTranslation, nil
174+
}
175+
if strings.EqualFold(strings.TrimSpace(targetLang), "en") {
176+
return docOutputReady, nil
160177
}
161-
return strings.EqualFold(storedHash, sourceHash), nil
178+
179+
postprocessVersion := extractPostprocessVersion(frontData)
180+
if strings.EqualFold(postprocessVersion, localizedLinkPostprocessVersion) {
181+
return docOutputReady, nil
182+
}
183+
return docOutputNeedsPostprocess, nil
162184
}
163185

164186
func extractSourceHash(frontData map[string]any) string {
165-
xi, ok := frontData["x-i18n"].(map[string]any)
187+
xi, ok := extractXI18N(frontData)
166188
if !ok {
167189
return ""
168190
}
@@ -173,6 +195,26 @@ func extractSourceHash(frontData map[string]any) string {
173195
return strings.TrimSpace(value)
174196
}
175197

198+
func extractPostprocessVersion(frontData map[string]any) string {
199+
xi, ok := extractXI18N(frontData)
200+
if !ok {
201+
return ""
202+
}
203+
value, ok := xi["postprocess_version"].(string)
204+
if !ok {
205+
return ""
206+
}
207+
return strings.TrimSpace(value)
208+
}
209+
210+
func extractXI18N(frontData map[string]any) (map[string]any, bool) {
211+
xi, ok := frontData["x-i18n"].(map[string]any)
212+
if ok {
213+
return xi, true
214+
}
215+
return nil, false
216+
}
217+
176218
func logDocChunkPlan(relPath string, blocks []string, groups [][]string) {
177219
totalBytes := 0
178220
for _, block := range blocks {

‎scripts/docs-i18n/main.go‎

Lines changed: 31 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -109,15 +109,17 @@ func runDocsI18N(ctx context.Context, cfg runConfig, files []string, newTranslat
109109
}
110110
totalFiles := len(ordered)
111111
preSkipped := 0
112+
prePostprocessFiles := []string{}
112113
if cfg.mode == "doc" && !cfg.overwrite {
113-
filtered, skipped, err := filterDocQueue(resolvedDocsRoot, cfg.targetLang, ordered)
114+
filtered, skipped, existingOutputs, err := filterDocQueue(resolvedDocsRoot, cfg.targetLang, ordered, cfg.maxFiles)
114115
if err != nil {
115116
return err
116117
}
117118
ordered = filtered
118119
preSkipped = skipped
120+
prePostprocessFiles = append(prePostprocessFiles, existingOutputs...)
119121
}
120-
if cfg.maxFiles > 0 && cfg.maxFiles < len(ordered) {
122+
if (cfg.mode != "doc" || cfg.overwrite) && cfg.maxFiles > 0 && cfg.maxFiles < len(ordered) {
121123
ordered = ordered[:cfg.maxFiles]
122124
}
123125

@@ -130,7 +132,7 @@ func runDocsI18N(ctx context.Context, cfg runConfig, files []string, newTranslat
130132
start := time.Now()
131133
processed := 0
132134
skipped := 0
133-
localizedFiles := []string{}
135+
localizedFiles := append([]string{}, prePostprocessFiles...)
134136
var translationErr error
135137

136138
log.Printf("docs-i18n: mode=%s total=%d pending=%d pre_skipped=%d overwrite=%t thinking=%s parallel=%d", cfg.mode, totalFiles, len(ordered), preSkipped, cfg.overwrite, cfg.thinking, parallel)
@@ -217,6 +219,9 @@ func runDocSequential(ctx context.Context, ordered []string, translator docsTran
217219
}
218220
if skip {
219221
skipped++
222+
if outputPath != "" {
223+
outputs = append(outputs, outputPath)
224+
}
220225
log.Printf("docs-i18n: [%d/%d] skipped %s (%s)", index+1, len(ordered), relPath, time.Since(start).Round(time.Millisecond))
221226
} else {
222227
processed++
@@ -294,6 +299,9 @@ func runDocParallel(ctx context.Context, ordered []string, docsRoot, srcLang, tg
294299
}
295300
if result.skipped {
296301
skipped++
302+
if result.output != "" {
303+
outputs = append(outputs, result.output)
304+
}
297305
log.Printf("docs-i18n: [w* %d/%d] skipped %s (%s)", result.index, len(ordered), result.rel, result.duration.Round(time.Millisecond))
298306
} else if result.err != nil {
299307
log.Printf("docs-i18n: [w* %d/%d] failed %s (%s): %v", result.index, len(ordered), result.rel, result.duration.Round(time.Millisecond), result.err)
@@ -339,29 +347,40 @@ func resolveRelPath(docsRoot, file string) string {
339347
return relPath
340348
}
341349

342-
func filterDocQueue(docsRoot, targetLang string, ordered []string) ([]string, int, error) {
350+
func filterDocQueue(docsRoot, targetLang string, ordered []string, maxFiles int) ([]string, int, []string, error) {
343351
pending := make([]string, 0, len(ordered))
352+
existingOutputs := []string{}
344353
skipped := 0
345354
for _, file := range ordered {
346355
absPath, relPath, err := resolveDocsPath(docsRoot, file)
347356
if err != nil {
348-
return nil, skipped, err
357+
return nil, skipped, nil, err
349358
}
350359
content, err := os.ReadFile(absPath)
351360
if err != nil {
352-
return nil, skipped, err
361+
return nil, skipped, nil, err
353362
}
354363
sourceHash := hashBytes(content)
355364
outputPath := filepath.Join(docsRoot, targetLang, relPath)
356-
skip, err := shouldSkipDoc(outputPath, sourceHash)
365+
status, err := classifyDocOutput(outputPath, sourceHash, targetLang)
357366
if err != nil {
358-
return nil, skipped, err
367+
return nil, skipped, nil, err
359368
}
360-
if skip {
369+
switch status {
370+
case docOutputReady:
361371
skipped++
362-
continue
372+
case docOutputNeedsPostprocess:
373+
if maxFiles > 0 && len(pending)+len(existingOutputs) >= maxFiles {
374+
continue
375+
}
376+
skipped++
377+
existingOutputs = append(existingOutputs, outputPath)
378+
case docOutputNeedsTranslation:
379+
if maxFiles > 0 && len(pending)+len(existingOutputs) >= maxFiles {
380+
continue
381+
}
382+
pending = append(pending, file)
363383
}
364-
pending = append(pending, file)
365384
}
366-
return pending, skipped, nil
385+
return pending, skipped, existingOutputs, nil
367386
}

0 commit comments

Comments
 (0)