Skip to content

Commit e709847

Browse files
joshp123joshp123
committed
Docs i18n: harden doc-mode pipeline
1 parent da9f28d commit e709847

7 files changed

Lines changed: 760 additions & 37 deletions

File tree

‎scripts/docs-i18n/doc_mode.go‎

Lines changed: 272 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,272 @@
1+
package main
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"os"
7+
"path/filepath"
8+
"strings"
9+
10+
"gopkg.in/yaml.v3"
11+
)
12+
13+
const (
14+
frontmatterTagStart = "<frontmatter>"
15+
frontmatterTagEnd = "</frontmatter>"
16+
bodyTagStart = "<body>"
17+
bodyTagEnd = "</body>"
18+
)
19+
20+
func processFileDoc(ctx context.Context, translator *PiTranslator, docsRoot, filePath, srcLang, tgtLang string, overwrite bool) (bool, error) {
21+
absPath, relPath, err := resolveDocsPath(docsRoot, filePath)
22+
if err != nil {
23+
return false, err
24+
}
25+
26+
content, err := os.ReadFile(absPath)
27+
if err != nil {
28+
return false, err
29+
}
30+
currentHash := hashBytes(content)
31+
32+
outputPath := filepath.Join(docsRoot, tgtLang, relPath)
33+
if !overwrite {
34+
skip, err := shouldSkipDoc(outputPath, currentHash)
35+
if err != nil {
36+
return false, err
37+
}
38+
if skip {
39+
return true, nil
40+
}
41+
}
42+
43+
sourceFront, sourceBody := splitFrontMatter(string(content))
44+
frontData := map[string]any{}
45+
if strings.TrimSpace(sourceFront) != "" {
46+
if err := yaml.Unmarshal([]byte(sourceFront), &frontData); err != nil {
47+
return false, fmt.Errorf("frontmatter parse failed for %s: %w", relPath, err)
48+
}
49+
}
50+
frontTemplate, markers := buildFrontmatterTemplate(frontData)
51+
taggedInput := formatTaggedDocument(frontTemplate, sourceBody)
52+
53+
translatedDoc, err := translator.TranslateRaw(ctx, taggedInput, srcLang, tgtLang)
54+
if err != nil {
55+
return false, fmt.Errorf("translate failed (%s): %w", relPath, err)
56+
}
57+
58+
translatedFront, translatedBody, err := parseTaggedDocument(translatedDoc)
59+
if err != nil {
60+
return false, fmt.Errorf("tagged output invalid for %s: %w", relPath, err)
61+
}
62+
if sourceFront != "" && strings.TrimSpace(translatedFront) == "" {
63+
return false, fmt.Errorf("translation removed frontmatter for %s", relPath)
64+
}
65+
if err := applyFrontmatterTranslations(frontData, markers, translatedFront); err != nil {
66+
return false, fmt.Errorf("frontmatter translation failed for %s: %w", relPath, err)
67+
}
68+
69+
updatedFront, err := encodeFrontMatter(frontData, relPath, content)
70+
if err != nil {
71+
return false, err
72+
}
73+
74+
if err := os.MkdirAll(filepath.Dir(outputPath), 0o755); err != nil {
75+
return false, err
76+
}
77+
78+
output := updatedFront + translatedBody
79+
return false, os.WriteFile(outputPath, []byte(output), 0o644)
80+
}
81+
82+
func formatTaggedDocument(frontMatter, body string) string {
83+
return fmt.Sprintf("%s\n%s\n%s\n%s\n%s\n%s", frontmatterTagStart, frontMatter, frontmatterTagEnd, bodyTagStart, body, bodyTagEnd)
84+
}
85+
86+
func parseTaggedDocument(text string) (string, string, error) {
87+
frontStart := strings.Index(text, frontmatterTagStart)
88+
if frontStart == -1 {
89+
return "", "", fmt.Errorf("missing %s", frontmatterTagStart)
90+
}
91+
frontStart += len(frontmatterTagStart)
92+
frontEnd := strings.Index(text[frontStart:], frontmatterTagEnd)
93+
if frontEnd == -1 {
94+
return "", "", fmt.Errorf("missing %s", frontmatterTagEnd)
95+
}
96+
frontEnd += frontStart
97+
98+
bodyStart := strings.Index(text[frontEnd:], bodyTagStart)
99+
if bodyStart == -1 {
100+
return "", "", fmt.Errorf("missing %s", bodyTagStart)
101+
}
102+
bodyStart += frontEnd + len(bodyTagStart)
103+
bodyEnd := strings.Index(text[bodyStart:], bodyTagEnd)
104+
if bodyEnd == -1 {
105+
return "", "", fmt.Errorf("missing %s", bodyTagEnd)
106+
}
107+
bodyEnd += bodyStart
108+
109+
prefix := strings.TrimSpace(text[:frontStart-len(frontmatterTagStart)])
110+
suffix := strings.TrimSpace(text[bodyEnd+len(bodyTagEnd):])
111+
if prefix != "" || suffix != "" {
112+
return "", "", fmt.Errorf("unexpected text outside tagged sections")
113+
}
114+
115+
frontMatter := trimTagNewlines(text[frontStart:frontEnd])
116+
body := trimTagNewlines(text[bodyStart:bodyEnd])
117+
return frontMatter, body, nil
118+
}
119+
120+
func trimTagNewlines(value string) string {
121+
value = strings.TrimPrefix(value, "\n")
122+
value = strings.TrimSuffix(value, "\n")
123+
return value
124+
}
125+
126+
type frontmatterMarker struct {
127+
Field string
128+
Index int
129+
Start string
130+
End string
131+
}
132+
133+
func buildFrontmatterTemplate(data map[string]any) (string, []frontmatterMarker) {
134+
if len(data) == 0 {
135+
return "", nil
136+
}
137+
markers := []frontmatterMarker{}
138+
lines := []string{}
139+
140+
if summary, ok := data["summary"].(string); ok {
141+
start, end := markerPair("SUMMARY", 0)
142+
markers = append(markers, frontmatterMarker{Field: "summary", Index: 0, Start: start, End: end})
143+
lines = append(lines, fmt.Sprintf("summary: %s%s%s", start, summary, end))
144+
}
145+
146+
if title, ok := data["title"].(string); ok {
147+
start, end := markerPair("TITLE", 0)
148+
markers = append(markers, frontmatterMarker{Field: "title", Index: 0, Start: start, End: end})
149+
lines = append(lines, fmt.Sprintf("title: %s%s%s", start, title, end))
150+
}
151+
152+
if readWhen, ok := data["read_when"].([]any); ok {
153+
lines = append(lines, "read_when:")
154+
for idx, item := range readWhen {
155+
textValue, ok := item.(string)
156+
if !ok {
157+
lines = append(lines, fmt.Sprintf(" - %v", item))
158+
continue
159+
}
160+
start, end := markerPair("READ_WHEN", idx)
161+
markers = append(markers, frontmatterMarker{Field: "read_when", Index: idx, Start: start, End: end})
162+
lines = append(lines, fmt.Sprintf(" - %s%s%s", start, textValue, end))
163+
}
164+
}
165+
166+
return strings.Join(lines, "\n"), markers
167+
}
168+
169+
func markerPair(field string, index int) (string, string) {
170+
return fmt.Sprintf("[[[FM_%s_%d_START]]]", field, index), fmt.Sprintf("[[[FM_%s_%d_END]]]", field, index)
171+
}
172+
173+
func applyFrontmatterTranslations(data map[string]any, markers []frontmatterMarker, translatedFront string) error {
174+
if len(markers) == 0 {
175+
return nil
176+
}
177+
for _, marker := range markers {
178+
value, err := extractMarkerValue(translatedFront, marker.Start, marker.End)
179+
if err != nil {
180+
return err
181+
}
182+
value = strings.TrimSpace(value)
183+
switch marker.Field {
184+
case "summary":
185+
data["summary"] = value
186+
case "title":
187+
data["title"] = value
188+
case "read_when":
189+
data["read_when"] = setReadWhenValue(data["read_when"], marker.Index, value)
190+
}
191+
}
192+
return nil
193+
}
194+
195+
func extractMarkerValue(text, start, end string) (string, error) {
196+
startIndex := strings.Index(text, start)
197+
if startIndex == -1 {
198+
return "", fmt.Errorf("missing marker %s", start)
199+
}
200+
startIndex += len(start)
201+
endIndex := strings.Index(text[startIndex:], end)
202+
if endIndex == -1 {
203+
return "", fmt.Errorf("missing marker %s", end)
204+
}
205+
endIndex += startIndex
206+
return text[startIndex:endIndex], nil
207+
}
208+
209+
func setReadWhenValue(existing any, index int, value string) []any {
210+
readWhen, ok := existing.([]any)
211+
if !ok {
212+
readWhen = []any{}
213+
}
214+
for len(readWhen) <= index {
215+
readWhen = append(readWhen, "")
216+
}
217+
readWhen[index] = value
218+
return readWhen
219+
}
220+
221+
func shouldSkipDoc(outputPath string, sourceHash string) (bool, error) {
222+
data, err := os.ReadFile(outputPath)
223+
if err != nil {
224+
if os.IsNotExist(err) {
225+
return false, nil
226+
}
227+
return false, err
228+
}
229+
frontMatter, _ := splitFrontMatter(string(data))
230+
if frontMatter == "" {
231+
return false, nil
232+
}
233+
frontData := map[string]any{}
234+
if err := yaml.Unmarshal([]byte(frontMatter), &frontData); err != nil {
235+
return false, nil
236+
}
237+
storedHash := extractSourceHash(frontData)
238+
if storedHash == "" {
239+
return false, nil
240+
}
241+
return strings.EqualFold(storedHash, sourceHash), nil
242+
}
243+
244+
func extractSourceHash(frontData map[string]any) string {
245+
xi, ok := frontData["x-i18n"].(map[string]any)
246+
if !ok {
247+
return ""
248+
}
249+
value, ok := xi["source_hash"].(string)
250+
if !ok {
251+
return ""
252+
}
253+
return strings.TrimSpace(value)
254+
}
255+
256+
func resolveDocsPath(docsRoot, filePath string) (string, string, error) {
257+
absPath, err := filepath.Abs(filePath)
258+
if err != nil {
259+
return "", "", err
260+
}
261+
relPath, err := filepath.Rel(docsRoot, absPath)
262+
if err != nil {
263+
return "", "", err
264+
}
265+
if relPath == "." || relPath == "" {
266+
return "", "", fmt.Errorf("file %s resolves to docs root %s", absPath, docsRoot)
267+
}
268+
if filepath.IsAbs(relPath) || relPath == ".." || strings.HasPrefix(relPath, ".."+string(filepath.Separator)) {
269+
return "", "", fmt.Errorf("file %s not under docs root %s", absPath, docsRoot)
270+
}
271+
return absPath, relPath, nil
272+
}

0 commit comments

Comments
 (0)