Skip to content

Commit dad0ed7

Browse files
committed
create report entries for the manifest files to unify logic
1 parent f9d1aaf commit dad0ed7

5 files changed

Lines changed: 172 additions & 84 deletions

File tree

‎internal/fsutil/create.go‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,35 @@ func Create(options *CreateOptions) (*Entry, error) {
7575
return entry, nil
7676
}
7777

78+
// CreateWriter handles the creation of a regular file and collects the
79+
// information recorded in Entry. The Hash and Size attributes are set on
80+
// calling Close() on the Writer.
81+
func CreateWriter(options *CreateOptions) (io.WriteCloser, *Entry, error) {
82+
if !options.Mode.IsRegular() {
83+
return nil, nil, fmt.Errorf("unsupported file type: %s", options.Path)
84+
}
85+
if options.MakeParents {
86+
if err := os.MkdirAll(filepath.Dir(options.Path), 0755); err != nil {
87+
return nil, nil, err
88+
}
89+
}
90+
file, err := os.OpenFile(options.Path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, options.Mode)
91+
if err != nil {
92+
return nil, nil, err
93+
}
94+
entry := &Entry{
95+
Path: options.Path,
96+
Mode: options.Mode,
97+
}
98+
wp := &writerProxy{
99+
entry: entry,
100+
inner: file,
101+
h: sha256.New(),
102+
size: 0,
103+
}
104+
return wp, entry, nil
105+
}
106+
78107
func createDir(o *CreateOptions) error {
79108
debugf("Creating directory: %s (mode %#o)", o.Path, o.Mode)
80109
err := os.Mkdir(o.Path, o.Mode)
@@ -137,3 +166,28 @@ func (rp *readerProxy) Read(p []byte) (n int, err error) {
137166
rp.size += n
138167
return n, err
139168
}
169+
170+
// writerProxy implements the io.WriteCloser interface proxying the calls to its
171+
// inner io.WriteCloser. On each write, the proxy keeps track of the file size
172+
// and hash.
173+
type writerProxy struct {
174+
inner io.WriteCloser
175+
h hash.Hash
176+
size int
177+
entry *Entry
178+
}
179+
180+
var _ io.WriteCloser = (*writerProxy)(nil)
181+
182+
func (rp *writerProxy) Write(p []byte) (n int, err error) {
183+
n, err = rp.inner.Write(p)
184+
rp.h.Write(p[:n])
185+
rp.size += n
186+
return n, err
187+
}
188+
189+
func (rp *writerProxy) Close() error {
190+
rp.entry.Hash = hex.EncodeToString(rp.h.Sum(nil))
191+
rp.entry.Size = rp.size
192+
return rp.inner.Close()
193+
}

‎internal/fsutil/create_test.go‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,3 +133,100 @@ func (s *S) TestCreate(c *C) {
133133
c.Assert(testutil.TreeDumpEntry(entry), DeepEquals, test.result[slashPath])
134134
}
135135
}
136+
137+
type createWriterTest struct {
138+
options fsutil.CreateOptions
139+
data []byte
140+
hackdir func(c *C, dir string)
141+
result map[string]string
142+
error string
143+
}
144+
145+
var createWriterTests = []createWriterTest{{
146+
options: fsutil.CreateOptions{
147+
Path: "foo",
148+
Mode: 0644,
149+
},
150+
data: []byte("foo"),
151+
result: map[string]string{
152+
"/foo": "file 0644 2c26b46b",
153+
},
154+
}, {
155+
options: fsutil.CreateOptions{
156+
Path: "foo",
157+
Mode: 0644 | fs.ModeDir,
158+
},
159+
error: `unsupported file type: \/[a-z0-9\-\/]*foo`,
160+
}, {
161+
options: fsutil.CreateOptions{
162+
Path: "foo",
163+
Mode: 0644 | fs.ModeSymlink,
164+
},
165+
error: `unsupported file type: /[a-z0-9\-\/]*/foo`,
166+
}, {
167+
options: fsutil.CreateOptions{
168+
Path: "foo/bar",
169+
Mode: 0644,
170+
MakeParents: true,
171+
},
172+
data: []byte("foo"),
173+
result: map[string]string{
174+
"/foo/": "dir 0755",
175+
"/foo/bar": "file 0644 2c26b46b",
176+
},
177+
}, {
178+
options: fsutil.CreateOptions{
179+
Path: "foo/bar",
180+
Mode: 0644,
181+
MakeParents: false,
182+
},
183+
error: `open /[a-z0-9\-\/]*/foo/bar: no such file or directory`,
184+
}}
185+
186+
func (s *S) TestCreateWriter(c *C) {
187+
oldUmask := syscall.Umask(0)
188+
defer func() {
189+
syscall.Umask(oldUmask)
190+
}()
191+
192+
for _, test := range createWriterTests {
193+
if test.result == nil {
194+
// Empty map for no files created.
195+
test.result = make(map[string]string)
196+
}
197+
c.Logf("Options: %v", test.options)
198+
dir := c.MkDir()
199+
if test.hackdir != nil {
200+
test.hackdir(c, dir)
201+
}
202+
options := test.options
203+
options.Path = filepath.Join(dir, options.Path)
204+
writer, entry, err := fsutil.CreateWriter(&options)
205+
if test.error != "" {
206+
c.Assert(err, ErrorMatches, test.error)
207+
continue
208+
}
209+
c.Assert(err, IsNil)
210+
211+
// Hash and Size are only set when the writer is closed.
212+
_, err = writer.Write(test.data)
213+
c.Assert(err, IsNil)
214+
c.Assert(entry.Path, Equals, options.Path)
215+
c.Assert(entry.Mode, Equals, options.Mode)
216+
c.Assert(entry.Hash, Equals, "")
217+
c.Assert(entry.Size, Equals, 0)
218+
err = writer.Close()
219+
c.Assert(err, IsNil)
220+
221+
c.Assert(testutil.TreeDump(dir), DeepEquals, test.result)
222+
// [fsutil.CreateWriter] does not return information about parent
223+
// directories created implicitly. We only check for the requested path.
224+
entry.Path = strings.TrimPrefix(entry.Path, dir)
225+
// Add the slashes that TreeDump adds to the path.
226+
slashPath := "/" + test.options.Path
227+
if test.options.Mode.IsDir() {
228+
slashPath = slashPath + "/"
229+
}
230+
c.Assert(testutil.TreeDumpEntry(entry), DeepEquals, test.result[slashPath])
231+
}
232+
}

‎internal/manifest/manifest.go‎

Lines changed: 0 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@ import (
44
"fmt"
55
"io"
66
"io/fs"
7-
"os"
87
"path/filepath"
98
"slices"
109
"sort"
@@ -187,7 +186,6 @@ type WriteOptions struct {
187186
// LocateManifestSlices.
188187
ManifestSlices map[string][]*setup.Slice
189188
Report *Report
190-
ManifestMode os.FileMode
191189
}
192190

193191
func Write(options *WriteOptions, writer io.Writer) error {
@@ -214,11 +212,6 @@ func Write(options *WriteOptions, writer io.Writer) error {
214212
return err
215213
}
216214

217-
err = manifestAddManifestPaths(dbw, options.ManifestMode, options.ManifestSlices)
218-
if err != nil {
219-
return err
220-
}
221-
222215
_, err = dbw.WriteTo(writer)
223216
return err
224217
}
@@ -307,34 +300,6 @@ func manifestAddReport(dbw *jsonwall.DBWriter, entries map[string]ReportEntry) e
307300
return nil
308301
}
309302

310-
func manifestAddManifestPaths(dbw *jsonwall.DBWriter, manifestMode os.FileMode, manifestSlices map[string][]*setup.Slice) error {
311-
for path, slices := range manifestSlices {
312-
sliceNames := []string{}
313-
for _, slice := range slices {
314-
err := dbw.Add(&Content{
315-
Kind: "content",
316-
Slice: slice.String(),
317-
Path: path,
318-
})
319-
if err != nil {
320-
return err
321-
}
322-
sliceNames = append(sliceNames, slice.String())
323-
}
324-
sort.Strings(sliceNames)
325-
err := dbw.Add(&Path{
326-
Kind: "path",
327-
Path: path,
328-
Mode: fmt.Sprintf("0%o", unixPerm(manifestMode)),
329-
Slices: sliceNames,
330-
})
331-
if err != nil {
332-
return err
333-
}
334-
}
335-
return nil
336-
}
337-
338303
func unixPerm(mode fs.FileMode) (perm uint32) {
339304
perm = uint32(mode.Perm())
340305
if mode&fs.ModeSticky != 0 {

‎internal/manifest/manifest_test.go‎

Lines changed: 4 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -273,16 +273,6 @@ func (s *S) TestGenerateManifests(c *C) {
273273
slice1 := &setup.Slice{
274274
Package: "package1",
275275
Name: "slice1",
276-
Contents: map[string]setup.PathInfo{
277-
"/dir/**": {
278-
Kind: "generate",
279-
Generate: "manifest",
280-
},
281-
"/other-dir/**": {
282-
Kind: "generate",
283-
Generate: "manifest",
284-
},
285-
},
286276
}
287277
slice2 := &setup.Slice{
288278
Package: "package2",
@@ -321,11 +311,6 @@ func (s *S) TestGenerateManifests(c *C) {
321311

322312
expected := &manifestContents{
323313
Paths: []*manifest.Path{{
324-
Kind: "path",
325-
Path: "/dir/manifest.wall",
326-
Mode: "0645",
327-
Slices: []string{"package1_slice1"},
328-
}, {
329314
Kind: "path",
330315
Path: "/file",
331316
Mode: "0456",
@@ -339,11 +324,6 @@ func (s *S) TestGenerateManifests(c *C) {
339324
Link: "/target",
340325
Mode: "0567",
341326
Slices: []string{"package1_slice1", "package2_slice2"},
342-
}, {
343-
Kind: "path",
344-
Path: "/other-dir/manifest.wall",
345-
Mode: "0645",
346-
Slices: []string{"package1_slice1"},
347327
}},
348328
Packages: []*manifest.Package{{
349329
Kind: "package",
@@ -366,21 +346,13 @@ func (s *S) TestGenerateManifests(c *C) {
366346
Name: "package2_slice2",
367347
}},
368348
Contents: []*manifest.Content{{
369-
Kind: "content",
370-
Slice: "package1_slice1",
371-
Path: "/dir/manifest.wall",
372-
}, {
373349
Kind: "content",
374350
Slice: "package1_slice1",
375351
Path: "/file",
376352
}, {
377353
Kind: "content",
378354
Slice: "package1_slice1",
379355
Path: "/link",
380-
}, {
381-
Kind: "content",
382-
Slice: "package1_slice1",
383-
Path: "/other-dir/manifest.wall",
384356
}, {
385357
Kind: "content",
386358
Slice: "package2_slice2",
@@ -393,10 +365,9 @@ func (s *S) TestGenerateManifests(c *C) {
393365
"/dir/manifest.wall": {slice1},
394366
"/other-dir/manifest.wall": {slice1},
395367
},
396-
PackageInfo: packageInfo,
397-
Selection: []*setup.Slice{slice1, slice2},
398-
Report: report,
399-
ManifestMode: 0645,
368+
PackageInfo: packageInfo,
369+
Selection: []*setup.Slice{slice1, slice2},
370+
Report: report,
400371
}
401372
var buffer bytes.Buffer
402373
err := manifest.Write(options, &buffer)
@@ -410,14 +381,8 @@ func (s *S) TestGenerateManifests(c *C) {
410381
}
411382

412383
func (s *S) TestGenerateNoManifests(c *C) {
413-
options := &manifest.WriteOptions{
414-
PackageInfo: nil,
415-
Selection: nil,
416-
Report: nil,
417-
ManifestMode: 0645,
418-
}
419384
var buffer bytes.Buffer
420-
err := manifest.Write(options, &buffer)
385+
err := manifest.Write(&manifest.WriteOptions{}, &buffer)
421386
c.Assert(err, IsNil)
422387
var reader io.Reader = &buffer
423388
var bs []byte

‎internal/slicer/slicer.go‎

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -332,21 +332,29 @@ func Run(options *RunOptions) error {
332332

333333
manifestSlices := manifest.LocateManifestSlices(options.Selection.Slices, manifestFilename)
334334
if len(manifestSlices) > 0 {
335-
var files []io.Writer
336-
for relPath := range manifestSlices {
335+
var writers []io.Writer
336+
for relPath, slices := range manifestSlices {
337337
logf("Generating manifest at %s...", relPath)
338-
absPath := filepath.Join(options.TargetDir, relPath)
339-
if err := os.MkdirAll(filepath.Dir(absPath), 0755); err != nil {
340-
return err
338+
absPath := filepath.Join(targetDir, relPath)
339+
createOptions := &fsutil.CreateOptions{
340+
Path: absPath,
341+
Mode: manifestMode,
342+
MakeParents: true,
341343
}
342-
file, err := os.OpenFile(absPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, manifestMode)
344+
writer, info, err := fsutil.CreateWriter(createOptions)
343345
if err != nil {
344346
return err
345347
}
346-
files = append(files, file)
347-
defer file.Close()
348+
defer writer.Close()
349+
writers = append(writers, writer)
350+
for _, slice := range slices {
351+
err := report.Add(slice, info)
352+
if err != nil {
353+
return err
354+
}
355+
}
348356
}
349-
w, err := zstd.NewWriter(io.MultiWriter(files...))
357+
w, err := zstd.NewWriter(io.MultiWriter(writers...))
350358
if err != nil {
351359
return err
352360
}
@@ -356,7 +364,6 @@ func Run(options *RunOptions) error {
356364
PackageInfo: pkgInfos,
357365
Selection: options.Selection.Slices,
358366
Report: report,
359-
ManifestMode: manifestMode,
360367
}
361368
err = manifest.Write(writeOptions, w)
362369
return err

0 commit comments

Comments
 (0)