@@ -310,11 +310,11 @@ pub struct StorageManifest {
310310pub struct PrefixManifest {
311311 /// Exact community-scoped listing prefix.
312312 pub prefix : String ,
313- /// Objects under the prefix at enumeration time.
313+ /// Object versions and delete markers under the prefix at enumeration time.
314314 pub object_count : u64 ,
315- /// Total object bytes under the prefix at enumeration time.
315+ /// Total object-version bytes under the prefix at enumeration time.
316316 pub total_bytes : u64 ,
317- /// Hex SHA-256 of the newline-terminated ascending key stream.
317+ /// Hex SHA-256 of the newline-terminated ascending version-entry stream.
318318 pub keys_digest : String ,
319319}
320320
@@ -325,10 +325,88 @@ pub struct ManifestKeyChunk {
325325 pub chunk_no : i64 ,
326326 /// The tenant prefix every key in this chunk lives under.
327327 pub prefix : String ,
328- /// Strictly ascending keys .
328+ /// Strictly ascending serialized manifest entries .
329329 pub keys : Vec < String > ,
330330}
331331
332+ /// One immutable object-store manifest entry.
333+ ///
334+ /// Version 5 storage manifests serialize entries as
335+ /// `key\u{1f}version_id\u{1f}kind`, where kind is `object` or
336+ /// `delete_marker`. Version 4 manifests used bare keys. Keeping the side-table
337+ /// column name unchanged avoids a database migration while making the stream
338+ /// explicitly version-aware.
339+ #[ derive( Debug , Clone , PartialEq , Eq ) ]
340+ pub struct StorageManifestEntry {
341+ /// Object key.
342+ pub key : String ,
343+ /// S3 version id.
344+ pub version_id : String ,
345+ /// Either `object` or `delete_marker`.
346+ pub kind : String ,
347+ }
348+
349+ impl StorageManifestEntry {
350+ /// Create a manifest entry.
351+ pub fn new (
352+ key : impl Into < String > ,
353+ version_id : impl Into < String > ,
354+ kind : impl Into < String > ,
355+ ) -> Self {
356+ Self {
357+ key : key. into ( ) ,
358+ version_id : version_id. into ( ) ,
359+ kind : kind. into ( ) ,
360+ }
361+ }
362+
363+ /// Serialize this entry into the chunk stream.
364+ pub fn encode ( & self ) -> Result < String > {
365+ validate_manifest_component ( "key" , & self . key ) ?;
366+ validate_manifest_component ( "version id" , & self . version_id ) ?;
367+ validate_manifest_component ( "kind" , & self . kind ) ?;
368+ if self . kind != "object" && self . kind != "delete_marker" {
369+ return Err ( DbError :: DeletionSafety ( format ! (
370+ "unsupported storage manifest entry kind {}" ,
371+ self . kind
372+ ) ) ) ;
373+ }
374+ Ok ( format ! (
375+ "{}\u{1f} {}\u{1f} {}" ,
376+ self . key, self . version_id, self . kind
377+ ) )
378+ }
379+
380+ /// Decode a manifest stream entry.
381+ pub fn decode ( value : & str ) -> Result < Self > {
382+ let mut parts = value. split ( '\u{1f}' ) ;
383+ let key = parts. next ( ) . unwrap_or_default ( ) ;
384+ let version_id = parts. next ( ) . ok_or_else ( || {
385+ DbError :: DeletionSafety ( "storage manifest entry is missing version id" . to_string ( ) )
386+ } ) ?;
387+ let kind = parts. next ( ) . ok_or_else ( || {
388+ DbError :: DeletionSafety ( "storage manifest entry is missing kind" . to_string ( ) )
389+ } ) ?;
390+ if parts. next ( ) . is_some ( ) {
391+ return Err ( DbError :: DeletionSafety (
392+ "storage manifest entry has too many fields" . to_string ( ) ,
393+ ) ) ;
394+ }
395+ let entry = Self :: new ( key, version_id, kind) ;
396+ entry. encode ( ) ?;
397+ Ok ( entry)
398+ }
399+ }
400+
401+ fn validate_manifest_component ( name : & str , value : & str ) -> Result < ( ) > {
402+ if value. is_empty ( ) || value. contains ( [ '\n' , '\u{1f}' ] ) {
403+ return Err ( DbError :: DeletionSafety ( format ! (
404+ "storage manifest {name} is empty or contains a reserved delimiter"
405+ ) ) ) ;
406+ }
407+ Ok ( ( ) )
408+ }
409+
332410/// One durable fleet-wide object-store taxonomy sweep record.
333411#[ derive( Debug , Clone , Serialize ) ]
334412pub struct TaxonomySweep {
@@ -358,11 +436,11 @@ type TaxonomySweepRow = (
358436 i64 ,
359437) ;
360438
361- /// Streaming SHA-256 over a strictly ascending key stream.
439+ /// Streaming SHA-256 over a strictly ascending storage manifest stream.
362440///
363441/// The executor's prefix enumeration and the destructive freeze's chunk
364- /// validation both fold keys through this, so "the chunk rows are exactly
365- /// the frozen enumeration" reduces to digest equality. Each key is hashed
442+ /// validation both fold entries through this, so "the chunk rows are exactly
443+ /// the frozen enumeration" reduces to digest equality. Each entry is hashed
366444/// with a trailing newline so concatenation cannot alias two streams.
367445pub struct KeyStreamDigest {
368446 hasher : Sha256 ,
@@ -395,6 +473,17 @@ impl KeyStreamDigest {
395473 "storage key stream is not strictly ascending at {key}"
396474 ) ) ) ;
397475 }
476+ self . fold_unordered ( key)
477+ }
478+
479+ /// Fold an already-canonical manifest entry whose source ordering is owned
480+ /// by the object store, not by key lexicographic order.
481+ ///
482+ /// S3 `ListObjectVersions` sorts by key but orders multiple versions of one
483+ /// key by recency with opaque version ids, so version-aware manifests cannot
484+ /// require strictly ascending serialized entries. Digest equality still
485+ /// binds the exact stream that was listed and chunked.
486+ pub fn fold_unordered ( & mut self , key : & str ) -> Result < ( ) > {
398487 self . hasher . update ( key. as_bytes ( ) ) ;
399488 self . hasher . update ( b"\n " ) ;
400489 self . last = Some ( key. to_owned ( ) ) ;
@@ -2594,7 +2683,7 @@ async fn live_fenced_tables_on(conn: &mut PgConnection) -> Result<BTreeSet<Strin
25942683
25952684/// Fail closed when a community-prefix inventory has an unsafe shape.
25962685pub fn validate_storage_manifest ( manifest : & StorageManifest ) -> Result < ( ) > {
2597- if manifest. version != 4 {
2686+ if ! matches ! ( manifest. version, 4 | 5 ) {
25982687 return Err ( DbError :: DeletionSafety ( format ! (
25992688 "unsupported storage manifest version {}" ,
26002689 manifest. version
@@ -2682,12 +2771,21 @@ fn validate_manifest_key_chunks(
26822771 ) ) ;
26832772 }
26842773 for key in & keys. 0 {
2685- if !key. starts_with ( chunk_prefix. as_str ( ) ) {
2774+ let prefix_key = if manifest. version >= 5 {
2775+ StorageManifestEntry :: decode ( key) ?. key
2776+ } else {
2777+ key. clone ( )
2778+ } ;
2779+ if !prefix_key. starts_with ( chunk_prefix. as_str ( ) ) {
26862780 return Err ( DbError :: DeletionSafety ( format ! (
2687- "frozen key {key } is outside its chunk prefix {chunk_prefix}"
2781+ "frozen key {prefix_key } is outside its chunk prefix {chunk_prefix}"
26882782 ) ) ) ;
26892783 }
2690- digest. fold ( key) ?;
2784+ if manifest. version >= 5 {
2785+ digest. fold_unordered ( key) ?;
2786+ } else {
2787+ digest. fold ( key) ?;
2788+ }
26912789 }
26922790 }
26932791 if let Some ( summary) = current {
@@ -3039,6 +3137,36 @@ mod tests {
30393137 assert ! ( validate_storage_manifest( & malformed_digest) . is_err( ) ) ;
30403138 }
30413139
3140+ #[ test]
3141+ fn frozen_inventory_digest_is_canonical_for_v5_manifest_entries ( ) {
3142+ let entry = StorageManifestEntry :: new ( "_meta/c/a.json" , "null" , "object" )
3143+ . encode ( )
3144+ . expect ( "entry" ) ;
3145+ let mut digest = KeyStreamDigest :: new ( ) ;
3146+ digest. fold_unordered ( & entry) . expect ( "fold entry" ) ;
3147+ let ( keys_digest, object_count) = digest. finish ( ) ;
3148+ let inventory = FrozenInventory {
3149+ schema : SchemaManifest {
3150+ scoped_tables : vec ! [ "events" . to_string( ) ] ,
3151+ row_counts : BTreeMap :: from ( [ ( "events" . to_string ( ) , 1 ) ] ) ,
3152+ fenced_tables : vec ! [ "events" . to_string( ) ] ,
3153+ } ,
3154+ storage : StorageManifest {
3155+ version : 5 ,
3156+ prefixes : vec ! [ PrefixManifest {
3157+ prefix: "_meta/c/" . to_string( ) ,
3158+ object_count,
3159+ total_bytes: 4 ,
3160+ keys_digest,
3161+ } ] ,
3162+ } ,
3163+ } ;
3164+ let digest = inventory. digest ( ) . unwrap ( ) ;
3165+ let round_tripped: FrozenInventory =
3166+ serde_json:: from_slice ( & serde_json:: to_vec ( & inventory) . unwrap ( ) ) . unwrap ( ) ;
3167+ assert_eq ! ( digest, round_tripped. digest( ) . unwrap( ) ) ;
3168+ }
3169+
30423170 #[ test]
30433171 fn key_stream_digest_requires_strict_order_and_is_chunking_invariant ( ) {
30443172 let keys = [ "a/1" , "a/2" , "a/3" ] ;
@@ -3100,6 +3228,49 @@ mod tests {
31003228 assert ! ( validate_manifest_key_chunks( & storage_manifest( ) , & [ ] ) . is_ok( ) ) ;
31013229 }
31023230
3231+ #[ test]
3232+ fn versioned_manifest_entries_decode_and_validate_chunks ( ) {
3233+ let entries = vec ! [
3234+ StorageManifestEntry :: new( "_meta/c/1" , "v2" , "object" )
3235+ . encode( )
3236+ . expect( "entry 1" ) ,
3237+ StorageManifestEntry :: new( "_meta/c/1" , "v1" , "delete_marker" )
3238+ . encode( )
3239+ . expect( "entry 2" ) ,
3240+ ] ;
3241+ let mut digest = KeyStreamDigest :: new ( ) ;
3242+ for entry in & entries {
3243+ digest. fold_unordered ( entry) . expect ( "fold version entry" ) ;
3244+ }
3245+ let ( hex_digest, count) = digest. finish ( ) ;
3246+ let mut manifest = storage_manifest ( ) ;
3247+ manifest. version = 5 ;
3248+ manifest. prefixes [ 0 ] . object_count = count;
3249+ manifest. prefixes [ 0 ] . keys_digest = hex_digest;
3250+
3251+ let chunk = |entries : & [ String ] | {
3252+ vec ! [ (
3253+ 0 ,
3254+ "_meta/c/" . to_string( ) ,
3255+ sqlx:: types:: Json ( entries. to_vec( ) ) ,
3256+ ) ]
3257+ } ;
3258+ // v5 freeze validation is retry-stable: a retried freeze with the
3259+ // same canonical version-entry stream is accepted, while a drifted
3260+ // stream is rejected.
3261+ assert ! ( validate_manifest_key_chunks( & manifest, & chunk( & entries) ) . is_ok( ) ) ;
3262+ assert ! ( validate_manifest_key_chunks( & manifest, & chunk( & entries) ) . is_ok( ) ) ;
3263+
3264+ let foreign = vec ! [ StorageManifestEntry :: new( "_uploads/c/1" , "v1" , "object" )
3265+ . encode( )
3266+ . expect( "foreign entry" ) ] ;
3267+ assert ! ( validate_manifest_key_chunks( & manifest, & chunk( & foreign) ) . is_err( ) ) ;
3268+ assert ! ( StorageManifestEntry :: decode( "_meta/c/1" ) . is_err( ) ) ;
3269+ assert ! ( StorageManifestEntry :: new( "_meta/c/1" , "v1" , "unknown" )
3270+ . encode( )
3271+ . is_err( ) ) ;
3272+ }
3273+
31033274 #[ test]
31043275 fn frozen_inventory_digest_is_stable ( ) {
31053276 let inventory = FrozenInventory {
0 commit comments