Skip to content

Commit f084690

Browse files
tbpgtbpg
authored
feat(godocfx): include README in output (#2927)
I also changed the return type of parse to be a result type to simplify passing the... results around. Related to googleapis/doc-pipeline#39.
1 parent cd2180a commit f084690

5 files changed

Lines changed: 114 additions & 32 deletions

File tree

‎internal/godocfx/godocfx_test.go‎

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -36,21 +36,21 @@ func TestMain(m *testing.M) {
3636

3737
func TestParse(t *testing.T) {
3838
testPath := "cloud.google.com/go/storage"
39-
pages, toc, module, err := parse(testPath)
39+
r, err := parse(testPath, nil)
4040
if err != nil {
4141
t.Fatalf("Parse: %v", err)
4242
}
43-
if got, want := len(toc), 1; got != want {
43+
if got, want := len(r.toc), 1; got != want {
4444
t.Fatalf("Parse got len(toc) = %d, want %d", got, want)
4545
}
46-
if got, want := len(pages), 1; got != want {
46+
if got, want := len(r.pages), 1; got != want {
4747
t.Errorf("Parse got len(pages) = %d, want %d", got, want)
4848
}
49-
if got := module.Path; got != testPath {
49+
if got := r.module.Path; got != testPath {
5050
t.Fatalf("Parse got module = %q, want %q", got, testPath)
5151
}
5252

53-
page := pages[testPath]
53+
page := r.pages[testPath]
5454

5555
// Check invariants for every item.
5656
for _, item := range page.Items {
@@ -64,8 +64,7 @@ func TestParse(t *testing.T) {
6464
}
6565

6666
// Check there is at least one type, const, variable, and function.
67-
// Note: no method because they aren't printed for Namespaces yet.
68-
wants := []string{"type", "const", "variable", "function"}
67+
wants := []string{"type", "const", "variable", "function", "method"}
6968
for _, want := range wants {
7069
found := false
7170
for _, c := range page.Items {
@@ -83,9 +82,10 @@ func TestParse(t *testing.T) {
8382
func TestGoldens(t *testing.T) {
8483
gotDir := "testdata/out"
8584
goldenDir := "testdata/golden"
85+
extraFiles := []string{"README.md"}
8686

8787
testPath := "cloud.google.com/go/storage"
88-
pages, toc, module, err := parse(testPath)
88+
r, err := parse(testPath, extraFiles)
8989
if err != nil {
9090
t.Fatalf("parse: %v", err)
9191
}
@@ -95,7 +95,7 @@ func TestGoldens(t *testing.T) {
9595
if updateGoldens {
9696
os.RemoveAll(goldenDir)
9797

98-
if err := write(goldenDir, pages, toc, module); err != nil {
98+
if err := write(goldenDir, r, extraFiles); err != nil {
9999
t.Fatalf("write: %v", err)
100100
}
101101

@@ -110,7 +110,7 @@ func TestGoldens(t *testing.T) {
110110
return
111111
}
112112

113-
if err := write(gotDir, pages, toc, module); err != nil {
113+
if err := write(gotDir, r, extraFiles); err != nil {
114114
t.Fatalf("write: %v", err)
115115
}
116116

‎internal/godocfx/main.go‎

Lines changed: 28 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -39,13 +39,13 @@ package main
3939
import (
4040
"flag"
4141
"fmt"
42+
"io"
4243
"log"
4344
"os"
4445
"path/filepath"
4546
"strings"
4647
"time"
4748

48-
"golang.org/x/tools/go/packages"
4949
"gopkg.in/yaml.v2"
5050
)
5151

@@ -58,17 +58,20 @@ func main() {
5858
log.Fatalf("%s missing required argument: module path", os.Args[0])
5959
}
6060

61-
pages, toc, module, err := parse(flag.Arg(0))
61+
extraFiles := []string{
62+
"README.md",
63+
}
64+
r, err := parse(flag.Arg(0), extraFiles)
6265
if err != nil {
6366
log.Fatal(err)
6467
}
6568

6669
if *print {
67-
if err := yaml.NewEncoder(os.Stdout).Encode(pages); err != nil {
70+
if err := yaml.NewEncoder(os.Stdout).Encode(r.pages); err != nil {
6871
log.Fatal(err)
6972
}
7073
fmt.Println("----- toc.yaml")
71-
if err := yaml.NewEncoder(os.Stdout).Encode(toc); err != nil {
74+
if err := yaml.NewEncoder(os.Stdout).Encode(r.toc); err != nil {
7275
log.Fatal(err)
7376
}
7477
return
@@ -78,22 +81,22 @@ func main() {
7881
os.RemoveAll(*outDir)
7982
}
8083

81-
if err := write(*outDir, pages, toc, module); err != nil {
84+
if err := write(*outDir, r, extraFiles); err != nil {
8285
log.Fatalf("write: %v", err)
8386
}
8487
}
8588

86-
func write(outDir string, pages map[string]*page, toc tableOfContents, module *packages.Module) error {
89+
func write(outDir string, r *result, extraFiles []string) error {
8790
if err := os.MkdirAll(outDir, os.ModePerm); err != nil {
8891
return fmt.Errorf("os.MkdirAll: %v", err)
8992
}
90-
for path, p := range pages {
93+
for path, p := range r.pages {
9194
// Make the module root page the index.
92-
if path == module.Path {
95+
if path == r.module.Path {
9396
path = "index"
9497
}
9598
// Trim the module path from all other paths.
96-
path = strings.TrimPrefix(path, module.Path+"/")
99+
path = strings.TrimPrefix(path, r.module.Path+"/")
97100
path = filepath.Join(outDir, path+".yml")
98101
if err := os.MkdirAll(filepath.Dir(path), os.ModePerm); err != nil {
99102
return fmt.Errorf("os.MkdirAll: %v", err)
@@ -115,9 +118,23 @@ func write(outDir string, pages map[string]*page, toc tableOfContents, module *p
115118
}
116119
defer f.Close()
117120
fmt.Fprintln(f, "### YamlMime:TableOfContent")
118-
if err := yaml.NewEncoder(f).Encode(toc); err != nil {
121+
if err := yaml.NewEncoder(f).Encode(r.toc); err != nil {
122+
return err
123+
}
124+
}
125+
126+
for _, path := range extraFiles {
127+
src, err := os.Open(filepath.Join(r.module.Dir, path))
128+
if err != nil {
119129
return err
120130
}
131+
dst, err := os.Create(filepath.Join(outDir, path))
132+
if err != nil {
133+
return err
134+
}
135+
if _, err := io.Copy(dst, src); err != nil {
136+
return nil
137+
}
121138
}
122139

123140
// Write the docuploader docs.metadata file. Not for DocFX.
@@ -145,6 +162,6 @@ func write(outDir string, pages map[string]*page, toc tableOfContents, module *p
145162
}
146163
name: %q
147164
version: %q
148-
language: "go"`, now.Unix(), now.Nanosecond(), module.Path, module.Version)
165+
language: "go"`, now.Unix(), now.Nanosecond(), r.module.Path, r.module.Version)
149166
return nil
150167
}

‎internal/godocfx/parse.go‎

Lines changed: 41 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626
"go/printer"
2727
"go/token"
2828
"log"
29+
"path/filepath"
2930
"sort"
3031
"strings"
3132

@@ -41,6 +42,7 @@ type tocItem struct {
4142
UID string `yaml:"uid,omitempty"`
4243
Name string `yaml:"name,omitempty"`
4344
Items []*tocItem `yaml:"items,omitempty"`
45+
Href string `yaml:"href,omitempty"`
4446
}
4547

4648
func (t *tocItem) addItem(i *tocItem) {
@@ -92,24 +94,32 @@ func (i *item) addChild(c child) {
9294

9395
var onlyGo = []string{"go"}
9496

97+
type result struct {
98+
pages map[string]*page
99+
toc tableOfContents
100+
module *packages.Module
101+
}
102+
95103
// parse parses the directory into a map of import path -> page and a TOC.
96104
//
97105
// glob is the path to parse, usually ending in `...`. glob is passed directly
98106
// to packages.Load as-is.
99-
func parse(glob string) (map[string]*page, tableOfContents, *packages.Module, error) {
107+
//
108+
// extraFiles is a list of paths relative to the module root to include.
109+
func parse(glob string, extraFiles []string) (*result, error) {
100110
config := &packages.Config{
101111
Mode: packages.NeedName | packages.NeedSyntax | packages.NeedTypes | packages.NeedTypesInfo | packages.NeedModule,
102112
Tests: true,
103113
}
104114

105115
pkgs, err := packages.Load(config, glob)
106116
if err != nil {
107-
return nil, nil, nil, fmt.Errorf("packages.Load: %v", err)
117+
return nil, fmt.Errorf("packages.Load: %v", err)
108118
}
109119
packages.PrintErrors(pkgs) // Don't fail everything because of one package.
110120

111121
if len(pkgs) == 0 {
112-
return nil, nil, nil, fmt.Errorf("pattern %q matched 0 packages", glob)
122+
return nil, fmt.Errorf("pattern %q matched 0 packages", glob)
113123
}
114124

115125
pages := map[string]*page{}
@@ -122,7 +132,7 @@ func parse(glob string) (map[string]*page, tableOfContents, *packages.Module, er
122132

123133
// First, collect all of the files grouped by package, including test
124134
// packages.
125-
allPkgFiles := map[string][]string{}
135+
goPkgFiles := map[string][]string{}
126136
for _, pkg := range pkgs {
127137
id := pkg.ID
128138
// See https://pkg.go.dev/golang.org/x/tools/go/packages#Config.
@@ -144,15 +154,15 @@ func parse(glob string) (map[string]*page, tableOfContents, *packages.Module, er
144154
for _, f := range pkg.Syntax {
145155
name := pkg.Fset.File(f.Pos()).Name()
146156
if strings.HasSuffix(name, ".go") {
147-
allPkgFiles[id] = append(allPkgFiles[id], name)
157+
goPkgFiles[id] = append(goPkgFiles[id], name)
148158
}
149159
}
150160
}
151161

152162
// Test files don't have Module set. Filter out packages in skipped modules.
153163
pkgFiles := map[string][]string{}
154164
pkgNames := []string{}
155-
for pkgPath, files := range allPkgFiles {
165+
for pkgPath, files := range goPkgFiles {
156166
skip := false
157167
for skipped := range skippedModules {
158168
if strings.HasPrefix(pkgPath, skipped) {
@@ -175,26 +185,41 @@ func parse(glob string) (map[string]*page, tableOfContents, *packages.Module, er
175185
for _, f := range pkgFiles[pkgPath] {
176186
pf, err := parser.ParseFile(fset, f, nil, parser.ParseComments)
177187
if err != nil {
178-
return nil, nil, nil, fmt.Errorf("ParseFile: %v", err)
188+
return nil, fmt.Errorf("ParseFile: %v", err)
179189
}
180190
parsedFiles = append(parsedFiles, pf)
181191
}
182192

183193
// Parse out GoDoc.
184194
docPkg, err := doc.NewFromFiles(fset, parsedFiles, pkgPath)
185195
if err != nil {
186-
return nil, nil, nil, fmt.Errorf("doc.NewFromFiles: %v", err)
196+
return nil, fmt.Errorf("doc.NewFromFiles: %v", err)
187197
}
188198

189199
// Extra filter in case the file filtering didn't catch everything.
190200
if !strings.HasPrefix(docPkg.ImportPath, module.Path) {
191201
continue
192202
}
193203

194-
toc = append(toc, &tocItem{
204+
pkgTOCITem := &tocItem{
195205
UID: docPkg.ImportPath,
196206
Name: docPkg.ImportPath,
197-
})
207+
}
208+
toc = append(toc, pkgTOCITem)
209+
210+
// If the package path == module path, add the extra files to the TOC
211+
// as a child of the module==pkg item.
212+
if pkgPath == module.Path {
213+
for _, path := range extraFiles {
214+
base := filepath.Base(path)
215+
name := strings.TrimSuffix(base, filepath.Ext(base))
216+
name = strings.Title(name)
217+
pkgTOCITem.addItem(&tocItem{
218+
Href: path,
219+
Name: name,
220+
})
221+
}
222+
}
198223

199224
pkgItem := &item{
200225
UID: docPkg.ImportPath,
@@ -345,7 +370,12 @@ func parse(glob string) (map[string]*page, tableOfContents, *packages.Module, er
345370
sort.Strings(skipped)
346371
log.Printf("Warning: Only processed %s@%s, skipped:\n%s\n", module.Path, module.Version, strings.Join(skipped, "\n"))
347372
}
348-
return pages, toc, module, nil
373+
374+
return &result{
375+
pages: pages,
376+
toc: toc,
377+
module: module,
378+
}, nil
349379
}
350380

351381
// processExamples converts the examples to []example.

‎internal/godocfx/testdata/golden/README.md‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
## Cloud Storage [![GoDoc](https://godoc.org/cloud.google.com/go/storage?status.svg)](https://godoc.org/cloud.google.com/go/storage)
2+
3+
- [About Cloud Storage](https://cloud.google.com/storage/)
4+
- [API documentation](https://cloud.google.com/storage/docs)
5+
- [Go client documentation](https://godoc.org/cloud.google.com/go/storage)
6+
- [Complete sample programs](https://github.com/GoogleCloudPlatform/golang-samples/tree/master/storage)
7+
8+
### Example Usage
9+
10+
First create a `storage.Client` to use throughout your application:
11+
12+
[snip]:# (storage-1)
13+
```go
14+
client, err := storage.NewClient(ctx)
15+
if err != nil {
16+
log.Fatal(err)
17+
}
18+
```
19+
20+
[snip]:# (storage-2)
21+
```go
22+
// Read the object1 from bucket.
23+
rc, err := client.Bucket("bucket").Object("object1").NewReader(ctx)
24+
if err != nil {
25+
log.Fatal(err)
26+
}
27+
defer rc.Close()
28+
body, err := ioutil.ReadAll(rc)
29+
if err != nil {
30+
log.Fatal(err)
31+
}
32+
```

‎internal/godocfx/testdata/golden/toc.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,6 @@
11
### YamlMime:TableOfContent
22
- uid: cloud.google.com/go/storage
33
name: cloud.google.com/go/storage
4+
items:
5+
- name: README
6+
href: README.md

0 commit comments

Comments
 (0)