Skip to content

Commit 0cc0de3

Browse files
tbpgtbpg
authored
feat(docfx): add support for examples (#2884)
Examples are located in _test.go files, which go/packages parses into a separate foo_test package. So, this requires an extra step of collecting all files from the package, including test packages, then parsing them. That way, the go/doc package can associate types/functions/packages with the examples for them (rather than an unassociated list).
1 parent 80f55c7 commit 0cc0de3

2 files changed

Lines changed: 246 additions & 79 deletions

File tree

‎internal/godocfx/parse.go‎

Lines changed: 147 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,14 @@
1717
package main
1818

1919
import (
20+
"bytes"
2021
"fmt"
2122
"go/ast"
2223
"go/doc"
24+
"go/format"
25+
"go/parser"
26+
"go/printer"
27+
"go/token"
2328
"log"
2429
"sort"
2530
"strings"
@@ -58,17 +63,23 @@ type syntax struct {
5863
Content string `yaml:"content,omitempty"`
5964
}
6065

66+
type example struct {
67+
Content string `yaml:"content,omitempty"`
68+
Name string `yaml:"name,omitempty"`
69+
}
70+
6171
// item represents a DocFX item.
6272
type item struct {
63-
UID string `yaml:"uid"`
64-
Name string `yaml:"name,omitempty"`
65-
ID string `yaml:"id,omitempty"`
66-
Summary string `yaml:"summary,omitempty"`
67-
Parent string `yaml:"parent,omitempty"`
68-
Type string `yaml:"type,omitempty"`
69-
Langs []string `yaml:"langs,omitempty"`
70-
Syntax syntax `yaml:"syntax,omitempty"`
71-
Children []child `yaml:"children,omitempty"`
73+
UID string `yaml:"uid"`
74+
Name string `yaml:"name,omitempty"`
75+
ID string `yaml:"id,omitempty"`
76+
Summary string `yaml:"summary,omitempty"`
77+
Parent string `yaml:"parent,omitempty"`
78+
Type string `yaml:"type,omitempty"`
79+
Langs []string `yaml:"langs,omitempty"`
80+
Syntax syntax `yaml:"syntax,omitempty"`
81+
Examples []example `yaml:"codeexamples,omitempty"`
82+
Children []child `yaml:"children,omitempty"`
7283
}
7384

7485
func (p *page) addItem(i *item) {
@@ -109,97 +120,116 @@ func parse(glob string) (map[string]*page, tableOfContents, *packages.Module, er
109120

110121
log.Printf("Processing %s@%s", module.Path, module.Version)
111122

123+
// First, collect all of the files grouped by package, including test
124+
// packages.
125+
pkgFiles := map[string][]string{}
112126
for _, pkg := range pkgs {
113-
if pkg == nil || pkg.Module == nil {
127+
id := pkg.ID
128+
// See https://pkg.go.dev/golang.org/x/tools/go/packages#Config.
129+
// The uncompiled test package shows up as "foo_test [foo.test]".
130+
if strings.HasSuffix(id, ".test") ||
131+
strings.Contains(id, "internal") ||
132+
(strings.Contains(id, " [") && !strings.Contains(id, "_test [")) {
114133
continue
115134
}
116-
if pkg.Module.Path != module.Path {
117-
skippedModules[pkg.Module.Path] = struct{}{}
118-
continue
135+
if strings.Contains(id, "_test") {
136+
id = id[0:strings.Index(id, "_test [")]
137+
} else {
138+
// The test package doesn't have Module set.
139+
if pkg.Module.Path != module.Path {
140+
skippedModules[pkg.Module.Path] = struct{}{}
141+
continue
142+
}
119143
}
120-
// Don't generate docs for tests or internal.
121-
switch {
122-
case strings.HasSuffix(pkg.ID, ".test"),
123-
strings.HasSuffix(pkg.ID, ".test]"),
124-
strings.Contains(pkg.ID, "internal"):
125-
continue
144+
for _, f := range pkg.Syntax {
145+
name := pkg.Fset.File(f.Pos()).Name()
146+
if strings.HasSuffix(name, ".go") {
147+
pkgFiles[id] = append(pkgFiles[id], name)
148+
}
126149
}
150+
}
127151

128-
// Collect all .go files.
129-
files := []*ast.File{}
130-
for _, f := range pkg.Syntax {
131-
tf := pkg.Fset.File(f.Pos())
132-
if strings.HasSuffix(tf.Name(), ".go") {
133-
files = append(files, f)
152+
// Once the files are grouped by package, process each package
153+
// independently.
154+
for pkgPath, files := range pkgFiles {
155+
parsedFiles := []*ast.File{}
156+
fset := token.NewFileSet()
157+
for _, f := range files {
158+
pf, err := parser.ParseFile(fset, f, nil, parser.ParseComments)
159+
if err != nil {
160+
return nil, nil, nil, fmt.Errorf("ParseFile: %v", err)
134161
}
162+
parsedFiles = append(parsedFiles, pf)
135163
}
136164

137165
// Parse out GoDoc.
138-
docPkg, err := doc.NewFromFiles(pkg.Fset, files, pkg.PkgPath)
166+
docPkg, err := doc.NewFromFiles(fset, parsedFiles, pkgPath)
139167
if err != nil {
140168
return nil, nil, nil, fmt.Errorf("doc.NewFromFiles: %v", err)
141169
}
142170

143171
toc = append(toc, &tocItem{
144-
UID: pkg.ID,
145-
Name: pkg.PkgPath,
172+
UID: docPkg.ImportPath,
173+
Name: docPkg.ImportPath,
146174
})
147175

148176
pkgItem := &item{
149-
UID: pkg.ID,
150-
Name: pkg.PkgPath,
151-
ID: pkg.Name,
152-
Summary: docPkg.Doc,
153-
Langs: onlyGo,
154-
Type: "package",
177+
UID: docPkg.ImportPath,
178+
Name: docPkg.ImportPath,
179+
ID: docPkg.Name,
180+
Summary: docPkg.Doc,
181+
Langs: onlyGo,
182+
Type: "package",
183+
Examples: processExamples(docPkg.Examples, fset),
155184
}
156185
pkgPage := &page{Items: []*item{pkgItem}}
157-
pages[pkg.PkgPath] = pkgPage
186+
pages[pkgPath] = pkgPage
158187

159188
for _, c := range docPkg.Consts {
160189
name := strings.Join(c.Names, ", ")
161190
id := strings.Join(c.Names, ",")
162-
uid := pkg.PkgPath + "." + id
191+
uid := docPkg.ImportPath + "." + id
163192
pkgItem.addChild(child(uid))
164193
pkgPage.addItem(&item{
165194
UID: uid,
166195
Name: name,
167196
ID: id,
168-
Parent: pkg.PkgPath,
197+
Parent: docPkg.ImportPath,
169198
Type: "const",
170199
Summary: c.Doc,
171200
Langs: onlyGo,
172-
Syntax: syntax{Content: pkgsite.PrintType(pkg.Fset, c.Decl)},
201+
Syntax: syntax{Content: pkgsite.PrintType(fset, c.Decl)},
173202
})
174203
}
175204
for _, v := range docPkg.Vars {
176205
name := strings.Join(v.Names, ", ")
177206
id := strings.Join(v.Names, ",")
178-
uid := pkg.PkgPath + "." + id
207+
uid := docPkg.ImportPath + "." + id
179208
pkgItem.addChild(child(uid))
180209
pkgPage.addItem(&item{
181210
UID: uid,
182211
Name: name,
183212
ID: id,
184-
Parent: pkg.PkgPath,
213+
Parent: docPkg.ImportPath,
185214
Type: "variable",
186215
Summary: v.Doc,
187216
Langs: onlyGo,
188-
Syntax: syntax{Content: pkgsite.PrintType(pkg.Fset, v.Decl)},
217+
Syntax: syntax{Content: pkgsite.PrintType(fset, v.Decl)},
189218
})
190219
}
191220
for _, t := range docPkg.Types {
192-
uid := pkg.PkgPath + "." + t.Name
221+
uid := docPkg.ImportPath + "." + t.Name
193222
pkgItem.addChild(child(uid))
194223
typeItem := &item{
195-
UID: uid,
196-
Name: t.Name,
197-
ID: t.Name,
198-
Parent: pkg.PkgPath,
199-
Type: "type",
200-
Summary: t.Doc,
201-
Langs: onlyGo,
202-
Syntax: syntax{Content: pkgsite.PrintType(pkg.Fset, t.Decl)},
224+
UID: uid,
225+
Name: t.Name,
226+
ID: t.Name,
227+
Parent: docPkg.ImportPath,
228+
Type: "type",
229+
Summary: t.Doc,
230+
Langs: onlyGo,
231+
Syntax: syntax{Content: pkgsite.PrintType(fset, t.Decl)},
232+
Examples: processExamples(t.Examples, fset),
203233
}
204234
// TODO: items are added as page.Children, rather than
205235
// typeItem.Children, as a workaround for the DocFX template.
@@ -208,7 +238,7 @@ func parse(glob string) (map[string]*page, tableOfContents, *packages.Module, er
208238
for _, c := range t.Consts {
209239
name := strings.Join(c.Names, ", ")
210240
id := strings.Join(c.Names, ",")
211-
cUID := pkg.PkgPath + "." + id
241+
cUID := docPkg.ImportPath + "." + id
212242
pkgItem.addChild(child(cUID))
213243
pkgPage.addItem(&item{
214244
UID: cUID,
@@ -218,13 +248,13 @@ func parse(glob string) (map[string]*page, tableOfContents, *packages.Module, er
218248
Type: "const",
219249
Summary: c.Doc,
220250
Langs: onlyGo,
221-
Syntax: syntax{Content: pkgsite.PrintType(pkg.Fset, c.Decl)},
251+
Syntax: syntax{Content: pkgsite.PrintType(fset, c.Decl)},
222252
})
223253
}
224254
for _, v := range t.Vars {
225255
name := strings.Join(v.Names, ", ")
226256
id := strings.Join(v.Names, ",")
227-
cUID := pkg.PkgPath + "." + id
257+
cUID := docPkg.ImportPath + "." + id
228258
pkgItem.addChild(child(cUID))
229259
pkgPage.addItem(&item{
230260
UID: cUID,
@@ -234,51 +264,54 @@ func parse(glob string) (map[string]*page, tableOfContents, *packages.Module, er
234264
Type: "variable",
235265
Summary: v.Doc,
236266
Langs: onlyGo,
237-
Syntax: syntax{Content: pkgsite.PrintType(pkg.Fset, v.Decl)},
267+
Syntax: syntax{Content: pkgsite.PrintType(fset, v.Decl)},
238268
})
239269
}
240270

241271
for _, fn := range t.Funcs {
242272
fnUID := uid + "." + fn.Name
243273
pkgItem.addChild(child(fnUID))
244274
pkgPage.addItem(&item{
245-
UID: fnUID,
246-
Name: fmt.Sprintf("func %s\n", fn.Name),
247-
ID: fn.Name,
248-
Parent: uid,
249-
Type: "function",
250-
Summary: fn.Doc,
251-
Langs: onlyGo,
252-
Syntax: syntax{Content: pkgsite.Synopsis(pkg.Fset, fn.Decl)},
275+
UID: fnUID,
276+
Name: fmt.Sprintf("func %s\n", fn.Name),
277+
ID: fn.Name,
278+
Parent: uid,
279+
Type: "function",
280+
Summary: fn.Doc,
281+
Langs: onlyGo,
282+
Syntax: syntax{Content: pkgsite.Synopsis(fset, fn.Decl)},
283+
Examples: processExamples(fn.Examples, fset),
253284
})
254285
}
255286
for _, fn := range t.Methods {
256287
fnUID := uid + "." + fn.Name
257288
pkgItem.addChild(child(fnUID))
258289
pkgPage.addItem(&item{
259-
UID: fnUID,
260-
Name: fmt.Sprintf("func (%s) %s\n", fn.Recv, fn.Name),
261-
ID: fn.Name,
262-
Parent: uid,
263-
Type: "function", // Note: this is actually a method.
264-
Summary: fn.Doc,
265-
Langs: onlyGo,
266-
Syntax: syntax{Content: pkgsite.Synopsis(pkg.Fset, fn.Decl)},
290+
UID: fnUID,
291+
Name: fmt.Sprintf("func (%s) %s\n", fn.Recv, fn.Name),
292+
ID: fn.Name,
293+
Parent: uid,
294+
Type: "function", // Note: this is actually a method.
295+
Summary: fn.Doc,
296+
Langs: onlyGo,
297+
Syntax: syntax{Content: pkgsite.Synopsis(fset, fn.Decl)},
298+
Examples: processExamples(fn.Examples, fset),
267299
})
268300
}
269301
}
270302
for _, fn := range docPkg.Funcs {
271-
uid := pkg.PkgPath + "." + fn.Name
303+
uid := docPkg.ImportPath + "." + fn.Name
272304
pkgItem.addChild(child(uid))
273305
pkgPage.addItem(&item{
274-
UID: uid,
275-
Name: fmt.Sprintf("func %s\n", fn.Name),
276-
ID: fn.Name,
277-
Parent: pkg.PkgPath,
278-
Type: "function",
279-
Summary: fn.Doc,
280-
Langs: onlyGo,
281-
Syntax: syntax{Content: pkgsite.Synopsis(pkg.Fset, fn.Decl)},
306+
UID: uid,
307+
Name: fmt.Sprintf("func %s\n", fn.Name),
308+
ID: fn.Name,
309+
Parent: docPkg.ImportPath,
310+
Type: "function",
311+
Summary: fn.Doc,
312+
Langs: onlyGo,
313+
Syntax: syntax{Content: pkgsite.Synopsis(fset, fn.Decl)},
314+
Examples: processExamples(fn.Examples, fset),
282315
})
283316
}
284317
}
@@ -292,3 +325,38 @@ func parse(glob string) (map[string]*page, tableOfContents, *packages.Module, er
292325
}
293326
return pages, toc, module, nil
294327
}
328+
329+
// processExamples converts the examples to []example.
330+
//
331+
// Surrounding braces and indentation is removed.
332+
func processExamples(exs []*doc.Example, fset *token.FileSet) []example {
333+
result := []example{}
334+
for _, ex := range exs {
335+
buf := &bytes.Buffer{}
336+
var node interface{} = &printer.CommentedNode{
337+
Node: ex.Code,
338+
Comments: ex.Comments,
339+
}
340+
if ex.Play != nil {
341+
node = ex.Play
342+
}
343+
if err := format.Node(buf, fset, node); err != nil {
344+
log.Fatal(err)
345+
}
346+
s := buf.String()
347+
if strings.HasPrefix(s, "{\n") && strings.HasSuffix(s, "\n}") {
348+
lines := strings.Split(s, "\n")
349+
builder := strings.Builder{}
350+
for _, line := range lines[1 : len(lines)-1] {
351+
builder.WriteString(strings.TrimPrefix(line, "\t"))
352+
builder.WriteString("\n")
353+
}
354+
s = builder.String()
355+
}
356+
result = append(result, example{
357+
Content: s,
358+
Name: ex.Suffix,
359+
})
360+
}
361+
return result
362+
}

0 commit comments

Comments
 (0)