Skip to content

Commit a97d660

Browse files
authored
Merge pull request #2125 from hiddeco/v5/format-input-bounds
v5: plumbing: format decoder input bounds and contracts
2 parents 170b881 + aeaa125 commit a97d660

8 files changed

Lines changed: 717 additions & 18 deletions

File tree

‎plumbing/format/idxfile/decoder.go‎

Lines changed: 159 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"errors"
88
"fmt"
99
"io"
10+
"io/fs"
1011

1112
"github.com/go-git/go-git/v5/plumbing/hash"
1213
"github.com/go-git/go-git/v5/utils/binary"
@@ -25,35 +26,88 @@ const (
2526
objectIDLength = hash.Size
2627
)
2728

29+
// Byte sizes of the idx v2 layout elements, used by the size formula
30+
// in [validateIdxV2Size]. See [gitformat-pack] for the canonical
31+
// layout.
32+
//
33+
// [gitformat-pack]: https://git-scm.com/docs/gitformat-pack
34+
const (
35+
headerLen = 8 // magic + version
36+
fanoutLen = fanout * 4 // uint32 per bucket
37+
crc32Len = 4 // CRC32 per object
38+
offset32Len = 4 // 32-bit offset per object
39+
offset64Len = 8 // 64-bit overflow offset
40+
trailerHashes = 2 // pack checksum + idx checksum, each hashsz
41+
)
42+
43+
// statInput is the optional shape the [Decoder] probes for at the
44+
// start of [Decoder.Decode] to learn the on-disk length of the idx
45+
// blob, which it uses to validate the canonical-Git size formula
46+
// before any allocations driven by the fanout table. Callers that
47+
// pass an [*os.File] or a `billy.File` backed by an `*os.File`
48+
// (the production call sites in `storage/filesystem`) satisfy it
49+
// directly; arbitrary [io.Reader]s do not, and decode for them
50+
// retains the pre-existing behaviour of erroring out at the
51+
// truncated-payload boundary instead.
52+
//
53+
// The interface is intentionally unexported so the public
54+
// [NewDecoder] signature stays compatible with v5.
55+
type statInput interface {
56+
Stat() (fs.FileInfo, error)
57+
}
58+
2859
// Decoder reads and decodes idx files from an input stream.
2960
type Decoder struct {
3061
io.Reader
31-
h hash.Hash
62+
src io.Reader
63+
h hash.Hash
3264
}
3365

3466
// NewDecoder builds a new idx stream decoder, that reads from r.
3567
func NewDecoder(r io.Reader) *Decoder {
3668
h := hash.New(crypto.SHA1)
3769
tr := io.TeeReader(r, h)
38-
return &Decoder{tr, h}
70+
return &Decoder{tr, r, h}
3971
}
4072

4173
// Decode reads from the stream and decode the content into the MemoryIndex struct.
4274
func (d *Decoder) Decode(idx *MemoryIndex) error {
75+
idxSize := int64(-1)
76+
if in, ok := d.src.(statInput); ok {
77+
fi, err := in.Stat()
78+
if err != nil {
79+
return fmt.Errorf("%w: stat input: %w", ErrMalformedIdxFile, err)
80+
}
81+
idxSize = fi.Size()
82+
}
83+
4384
if err := validateHeader(d); err != nil {
4485
return err
4586
}
4687

47-
flow := []func(*MemoryIndex, io.Reader) error{
88+
headerFlow := []func(*MemoryIndex, io.Reader) error{
4889
readVersion,
4990
readFanout,
91+
}
92+
for _, f := range headerFlow {
93+
if err := f(idx, d); err != nil {
94+
return err
95+
}
96+
}
97+
98+
if idxSize >= 0 {
99+
if err := validateIdxV2Size(idx, idxSize); err != nil {
100+
return err
101+
}
102+
}
103+
104+
bodyFlow := []func(*MemoryIndex, io.Reader) error{
50105
readObjectNames,
51106
readCRC32,
52107
readOffsets,
53108
readPackChecksum,
54109
}
55-
56-
for _, f := range flow {
110+
for _, f := range bodyFlow {
57111
if err := f(idx, d); err != nil {
58112
return err
59113
}
@@ -199,3 +253,103 @@ func readIdxChecksum(idx *MemoryIndex, r io.Reader) error {
199253

200254
return nil
201255
}
256+
257+
// validateIdxV2Size enforces the size formula used by canonical Git
258+
// load_idx for idx v2 files: the on-disk length must lie within
259+
// [minSize, maxSize] where
260+
//
261+
// perObject = hashsz + crc32Len + offset32Len
262+
// minSize = headerLen + fanoutLen + trailerHashes*hashsz + nr*perObject
263+
// maxSize = minSize + (nr-1)*offset64Len when nr > 0
264+
//
265+
// with nr taken from the last fanout entry and hashsz fixed at
266+
// [objectIDLength] (SHA-1 in v5). Multiplications use a self-checking
267+
// overflow guard so inputs whose claimed object count overflows the
268+
// formula are rejected rather than wrapping into a smaller value.
269+
func validateIdxV2Size(idx *MemoryIndex, idxSize int64) error {
270+
nr := int64(idx.Fanout[fanout-1])
271+
hashsz := int64(objectIDLength)
272+
273+
minSize := minIdxV2Size(nr, hashsz)
274+
maxSize := maxIdxV2Size(nr, hashsz)
275+
if minSize < 0 || maxSize < 0 {
276+
return fmt.Errorf("%w: object count %d is inconsistent with file size", ErrMalformedIdxFile, nr)
277+
}
278+
279+
if idxSize < minSize || idxSize > maxSize {
280+
return fmt.Errorf("%w: file size %d is inconsistent with object count %d", ErrMalformedIdxFile, idxSize, nr)
281+
}
282+
return nil
283+
}
284+
285+
// minIdxV2Size returns the minimum on-disk size of an idx v2 file
286+
// holding nr objects with the given hash size, mirroring the
287+
// computation in canonical Git load_idx. Returns -1 when any
288+
// intermediate multiplication or addition would overflow int64.
289+
func minIdxV2Size(nr, hashsz int64) int64 {
290+
perObject := hashsz + crc32Len + offset32Len
291+
fixed := int64(headerLen+fanoutLen) + trailerHashes*hashsz
292+
293+
objects, ok := mulInt64(nr, perObject)
294+
if !ok {
295+
return -1
296+
}
297+
sum, ok := addInt64(fixed, objects)
298+
if !ok {
299+
return -1
300+
}
301+
return sum
302+
}
303+
304+
// maxIdxV2Size returns the maximum on-disk size of an idx v2 file
305+
// holding nr objects with the given hash size, mirroring the
306+
// computation in canonical Git load_idx. Returns -1 on overflow.
307+
func maxIdxV2Size(nr, hashsz int64) int64 {
308+
minSize := minIdxV2Size(nr, hashsz)
309+
if minSize < 0 {
310+
return -1
311+
}
312+
if nr == 0 {
313+
return minSize
314+
}
315+
overflow, ok := mulInt64(nr-1, offset64Len)
316+
if !ok {
317+
return -1
318+
}
319+
sum, ok := addInt64(minSize, overflow)
320+
if !ok {
321+
return -1
322+
}
323+
return sum
324+
}
325+
326+
// mulInt64 returns a*b and whether the result fits in an int64 without
327+
// overflow. Negative operands or overflow yield ok=false. The overflow
328+
// check uses the standard self-inverse identity: a*b/b == a only when
329+
// the multiplication did not wrap.
330+
func mulInt64(a, b int64) (int64, bool) {
331+
if a < 0 || b < 0 {
332+
return 0, false
333+
}
334+
if a == 0 || b == 0 {
335+
return 0, true
336+
}
337+
c := a * b
338+
if c/b != a {
339+
return 0, false
340+
}
341+
return c, true
342+
}
343+
344+
// addInt64 returns a+b and whether the result fits in an int64 without
345+
// overflow. Negative operands or overflow yield ok=false.
346+
func addInt64(a, b int64) (int64, bool) {
347+
if a < 0 || b < 0 {
348+
return 0, false
349+
}
350+
c := a + b
351+
if c < a {
352+
return 0, false
353+
}
354+
return c, true
355+
}

‎plumbing/format/idxfile/decoder_test.go‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"bytes"
55
"encoding/base64"
66
"encoding/binary"
7+
"errors"
78
"fmt"
89
"io"
910
"testing"
@@ -289,3 +290,110 @@ func idxV2Header() []byte {
289290
binary.Write(&buf, binary.BigEndian, uint32(2))
290291
return buf.Bytes()
291292
}
293+
294+
// TestDecoderSizeFormulaBoundary exercises the [minSize, maxSize] range
295+
// enforced by the size formula when the input exposes Stat. The
296+
// boundary is asymmetric for nr > 1: each extra 8-byte offset64 slot
297+
// extends maxSize by 8. Inputs at the edge of the legal range must
298+
// pass the size check (failing later with the trailing checksum
299+
// mismatch, since the payload is zero-filled); inputs one byte
300+
// outside must be rejected with a size-related error.
301+
func TestDecoderSizeFormulaBoundary(t *testing.T) {
302+
t.Parallel()
303+
304+
const hashsz = 20 // SHA-1
305+
const headerLen = 8 + 4*256
306+
minSize := func(nr int64) int64 {
307+
return headerLen + nr*(hashsz+8) + 2*hashsz
308+
}
309+
maxSize := func(nr int64) int64 {
310+
m := minSize(nr)
311+
if nr > 0 {
312+
m += (nr - 1) * 8
313+
}
314+
return m
315+
}
316+
build := func(nr uint32, total int64) []byte {
317+
buf := append(idxV2Header(), writeFanout(nr, nil)...)
318+
if int64(len(buf)) > total {
319+
t.Fatalf("header+fanout exceeds requested total: %d > %d", len(buf), total)
320+
}
321+
return append(buf, make([]byte, total-int64(len(buf)))...)
322+
}
323+
324+
tests := []struct {
325+
name string
326+
nr uint32
327+
size int64
328+
passesLen bool // true: size check passes (later parsing may fail)
329+
}{
330+
{"nr=1, at minSize", 1, minSize(1), true},
331+
{"nr=1, one byte below minSize", 1, minSize(1) - 1, false},
332+
{"nr=1, one byte above maxSize", 1, maxSize(1) + 1, false},
333+
{"nr=2, at minSize", 2, minSize(2), true},
334+
{"nr=2, at maxSize", 2, maxSize(2), true},
335+
{"nr=2, one byte below minSize", 2, minSize(2) - 1, false},
336+
{"nr=2, one byte above maxSize", 2, maxSize(2) + 1, false},
337+
}
338+
339+
for _, tt := range tests {
340+
t.Run(tt.name, func(t *testing.T) {
341+
t.Parallel()
342+
343+
idx := new(MemoryIndex)
344+
d := NewDecoder(fromBytes(build(tt.nr, tt.size)))
345+
err := d.Decode(idx)
346+
require.Error(t, err)
347+
require.ErrorIs(t, err, ErrMalformedIdxFile)
348+
if tt.passesLen {
349+
// Payload is zero-filled, so we reach the trailing
350+
// checksum comparison and fail there. The size check
351+
// itself must not fire.
352+
require.ErrorContains(t, err, "checksum mismatch")
353+
} else {
354+
require.ErrorContains(t, err, "inconsistent with object count")
355+
}
356+
})
357+
}
358+
}
359+
360+
// TestDecoderRejectsInconsistentObjectCount asserts that a fanout
361+
// table whose object count would overflow the size formula is
362+
// rejected when the input exposes Stat.
363+
func TestDecoderRejectsInconsistentObjectCount(t *testing.T) {
364+
t.Parallel()
365+
366+
// Header (\xff t O c) + version 2 + fanout where fanout[0..255] all
367+
// claim 0x4C4C4C4C objects. The byte count cannot possibly accommodate
368+
// that many object names.
369+
var buf bytes.Buffer
370+
buf.Write([]byte{0xff, 't', 'O', 'c', 0, 0, 0, 2})
371+
for range 256 {
372+
_ = binary.Write(&buf, binary.BigEndian, uint32(0x4C4C4C4C))
373+
}
374+
375+
idx := new(MemoryIndex)
376+
d := NewDecoder(fromBytes(buf.Bytes()))
377+
err := d.Decode(idx)
378+
if !errors.Is(err, ErrMalformedIdxFile) {
379+
t.Fatalf("expected ErrMalformedIdxFile, got %v", err)
380+
}
381+
}
382+
383+
// TestDecoderSizeCheckSkippedForBareReader documents the
384+
// backward-compat path: when the input does not expose Stat (here a
385+
// raw [bytes.Reader]), the size formula is not applied, and a
386+
// truncated payload surfaces as the underlying read error. The
387+
// production call sites in `storage/filesystem` always pass a
388+
// `billy.File` whose concrete type has Stat, so the formula does run
389+
// there.
390+
func TestDecoderSizeCheckSkippedForBareReader(t *testing.T) {
391+
t.Parallel()
392+
393+
buf := append(idxV2Header(), writeFanout(1, nil)...)
394+
395+
idx := new(MemoryIndex)
396+
d := NewDecoder(bytes.NewReader(buf))
397+
err := d.Decode(idx)
398+
require.ErrorIs(t, err, io.EOF)
399+
}
Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
package idxfile_test
2+
3+
import (
4+
"bytes"
5+
"io/fs"
6+
"time"
7+
)
8+
9+
// bytesInput wraps an in-memory idx blob with a Stat method so that
10+
// the [Decoder] can probe its on-disk length and apply the size
11+
// formula. The interface is unexported in [Decoder]; the type
12+
// assertion succeeds on any value that exposes both
13+
// [io.Reader] and `Stat() (fs.FileInfo, error)`.
14+
type bytesInput struct {
15+
*bytes.Reader
16+
size int64
17+
}
18+
19+
func fromBytes(b []byte) *bytesInput {
20+
return &bytesInput{Reader: bytes.NewReader(b), size: int64(len(b))}
21+
}
22+
23+
func (b *bytesInput) Stat() (fs.FileInfo, error) {
24+
return bytesInputInfo(b.size), nil
25+
}
26+
27+
type bytesInputInfo int64
28+
29+
func (i bytesInputInfo) Name() string { return "" }
30+
func (i bytesInputInfo) Size() int64 { return int64(i) }
31+
func (i bytesInputInfo) Mode() fs.FileMode { return 0 }
32+
func (i bytesInputInfo) ModTime() time.Time { return time.Time{} }
33+
func (i bytesInputInfo) IsDir() bool { return false }
34+
func (i bytesInputInfo) Sys() any { return nil }

‎plumbing/format/objfile/reader.go‎

Lines changed: 15 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -11,9 +11,10 @@ import (
1111
)
1212

1313
var (
14-
ErrClosed = errors.New("objfile: already closed")
15-
ErrHeader = errors.New("objfile: invalid header")
16-
ErrNegativeSize = errors.New("objfile: negative object size")
14+
ErrClosed = errors.New("objfile: already closed")
15+
ErrHeader = errors.New("objfile: invalid header")
16+
ErrHeaderNotRead = errors.New("objfile: Header must be called before Read")
17+
ErrNegativeSize = errors.New("objfile: negative object size")
1718
)
1819

1920
// Reader reads and decodes compressed objfile data from a provided io.Reader.
@@ -100,12 +101,23 @@ func (r *Reader) prepareForRead(t plumbing.ObjectType, size int64) {
100101
//
101102
// If Read encounters the end of the data stream it will return err == io.EOF,
102103
// either in the current call if n > 0 or in a subsequent call.
104+
//
105+
// Read returns ErrHeaderNotRead if Header has not been called successfully.
103106
func (r *Reader) Read(p []byte) (n int, err error) {
107+
if r.multi == nil {
108+
return 0, ErrHeaderNotRead
109+
}
104110
return r.multi.Read(p)
105111
}
106112

107113
// Hash returns the hash of the object data stream that has been read so far.
114+
// It returns the zero plumbing.Hash if Header has not been called
115+
// successfully — guarding against the nil hasher that prepareForRead has
116+
// not yet allocated.
108117
func (r *Reader) Hash() plumbing.Hash {
118+
if r.multi == nil {
119+
return plumbing.ZeroHash
120+
}
109121
return r.hasher.Sum()
110122
}
111123

0 commit comments

Comments
 (0)