Move the tar-split digest value into the TOC

mtrmac · mtrmac · commit b5413c2bd647 · 2024-05-14T10:53:03.000+02:00
... so that we can uniquely identify partially-pulled layers
by the TOC digest.

Signed-off-by: Miloslav Trmač &lt;mitr@redhat.com&gt;
diff --git a/pkg/chunked/cache_linux.go b/pkg/chunked/cache_linux.go
@@ -897,6 +897,14 @@ func unmarshalToc(manifest []byte) (*internal.TOC, error) {
 				toc.Entries = append(toc.Entries, m)
 			}
 
+		case "tarsplitdigest": // strings.ToLower("tarSplitDigest")
+			s := iter.ReadString()
+			d, err := digest.Parse(s)
+			if err != nil {
+				return nil, fmt.Errorf("Invalid tarSplitDigest %q: %w", s, err)
+			}
+			toc.TarSplitDigest = d
+
 		default:
 			iter.Skip()
 		}
diff --git a/pkg/chunked/compression_linux.go b/pkg/chunked/compression_linux.go
@@ -147,12 +147,10 @@ func readZstdChunkedManifest(blobStream ImageSourceSeekable, tocDigest digest.Di
 	// The tarSplit… values are valid if tarSplitChunk.Offset > 0
 	var tarSplitChunk ImageSourceChunk
 	var tarSplitLengthUncompressed uint64
-	var tarSplitChecksum string
 	if tarSplitInfoKeyAnnotation, found := annotations[internal.TarSplitInfoKey]; found {
 		if _, err := fmt.Sscanf(tarSplitInfoKeyAnnotation, "%d:%d:%d", &tarSplitChunk.Offset, &tarSplitChunk.Length, &tarSplitLengthUncompressed); err != nil {
 			return nil, nil, nil, 0, err
 		}
-		tarSplitChecksum = annotations[internal.TarSplitChecksumKey]
 	}
 
 	if manifestType != internal.ManifestTypeCRFS {
@@ -217,7 +215,7 @@ func readZstdChunkedManifest(blobStream ImageSourceSeekable, tocDigest digest.Di
 			return nil, nil, nil, 0, err
 		}
 
-		decodedTarSplit, err = decodeAndValidateBlob(tarSplit, tarSplitLengthUncompressed, tarSplitChecksum)
+		decodedTarSplit, err = decodeAndValidateBlob(tarSplit, tarSplitLengthUncompressed, toc.TarSplitDigest.String())
 		if err != nil {
 			return nil, nil, nil, 0, err
 		}
diff --git a/pkg/chunked/internal/compression.go b/pkg/chunked/internal/compression.go
@@ -18,8 +18,9 @@ import (
 )
 
 type TOC struct {
-	Version int            `json:"version"`
-	Entries []FileMetadata `json:"entries"`
+	Version        int            `json:"version"`
+	Entries        []FileMetadata `json:"entries"`
+	TarSplitDigest digest.Digest  `json:"tarSplitDigest,omitempty"`
 }
 
 type FileMetadata struct {
@@ -84,7 +85,7 @@ func GetType(t byte) (string, error) {
 const (
 	ManifestChecksumKey = "io.github.containers.zstd-chunked.manifest-checksum"
 	ManifestInfoKey     = "io.github.containers.zstd-chunked.manifest-position"
-	TarSplitChecksumKey = "io.github.containers.zstd-chunked.tarsplit-checksum"
+	TarSplitChecksumKey = "io.github.containers.zstd-chunked.tarsplit-checksum" // Deprecated: Use the TOC.TarSplitDigest field instead, this one is not authenticated by ManifestChecksumKey.
 	TarSplitInfoKey     = "io.github.containers.zstd-chunked.tarsplit-position"
 
 	// ManifestTypeCRFS is a manifest file compatible with the CRFS TOC file.
@@ -133,8 +134,9 @@ func WriteZstdChunkedManifest(dest io.Writer, outMetadata map[string]string, off
 	manifestOffset := offset + zstdSkippableFrameHeader
 
 	toc := TOC{
-		Version: 1,
-		Entries: metadata,
+		Version:        1,
+		Entries:        metadata,
+		TarSplitDigest: tarSplitData.Digest,
 	}
 
 	json := jsoniter.ConfigCompatibleWithStandardLibrary

Original file line number	Diff line number	Diff line change
`@@ -897,6 +897,14 @@ func unmarshalToc(manifest []byte) (*internal.TOC, error) {`
`897`	`897`	`toc.Entries = append(toc.Entries, m)`
`898`	`898`	`}`
`899`	`899`
	`900`	`+ case "tarsplitdigest": // strings.ToLower("tarSplitDigest")`
	`901`	`+ s := iter.ReadString()`
	`902`	`+ d, err := digest.Parse(s)`
	`903`	`+ if err != nil {`
	`904`	`+ return nil, fmt.Errorf("Invalid tarSplitDigest %q: %w", s, err)`
	`905`	`+ }`
	`906`	`+ toc.TarSplitDigest = d`
	`907`	`+`
`900`	`908`	`default:`
`901`	`909`	`iter.Skip()`
`902`	`910`	`}`
Original file line number	Diff line number	Diff line change
`@@ -147,12 +147,10 @@ func readZstdChunkedManifest(blobStream ImageSourceSeekable, tocDigest digest.Di`
`147`	`147`	`// The tarSplit… values are valid if tarSplitChunk.Offset > 0`
`148`	`148`	`var tarSplitChunk ImageSourceChunk`
`149`	`149`	`var tarSplitLengthUncompressed uint64`
`150`		`- var tarSplitChecksum string`
`151`	`150`	`if tarSplitInfoKeyAnnotation, found := annotations[internal.TarSplitInfoKey]; found {`
`152`	`151`	`if _, err := fmt.Sscanf(tarSplitInfoKeyAnnotation, "%d:%d:%d", &tarSplitChunk.Offset, &tarSplitChunk.Length, &tarSplitLengthUncompressed); err != nil {`
`153`	`152`	`return nil, nil, nil, 0, err`
`154`	`153`	`}`
`155`		`- tarSplitChecksum = annotations[internal.TarSplitChecksumKey]`
`156`	`154`	`}`
`157`	`155`
`158`	`156`	`if manifestType != internal.ManifestTypeCRFS {`
`@@ -217,7 +215,7 @@ func readZstdChunkedManifest(blobStream ImageSourceSeekable, tocDigest digest.Di`
`217`	`215`	`return nil, nil, nil, 0, err`
`218`	`216`	`}`
`219`	`217`
`220`		`- decodedTarSplit, err = decodeAndValidateBlob(tarSplit, tarSplitLengthUncompressed, tarSplitChecksum)`
	`218`	`+ decodedTarSplit, err = decodeAndValidateBlob(tarSplit, tarSplitLengthUncompressed, toc.TarSplitDigest.String())`
`221`	`219`	`if err != nil {`
`222`	`220`	`return nil, nil, nil, 0, err`
`223`	`221`	`}`