Skip to content

Commit d12dea4

Browse files
authored
Support community deletion in versioned media buckets (#6738)
1 parent bb5b935 commit d12dea4

11 files changed

Lines changed: 1584 additions & 101 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎crates/buzz-db/src/deletion.rs‎

Lines changed: 182 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -310,11 +310,11 @@ pub struct StorageManifest {
310310
pub struct PrefixManifest {
311311
/// Exact community-scoped listing prefix.
312312
pub prefix: String,
313-
/// Objects under the prefix at enumeration time.
313+
/// Object versions and delete markers under the prefix at enumeration time.
314314
pub object_count: u64,
315-
/// Total object bytes under the prefix at enumeration time.
315+
/// Total object-version bytes under the prefix at enumeration time.
316316
pub total_bytes: u64,
317-
/// Hex SHA-256 of the newline-terminated ascending key stream.
317+
/// Hex SHA-256 of the newline-terminated ascending version-entry stream.
318318
pub keys_digest: String,
319319
}
320320

@@ -325,10 +325,88 @@ pub struct ManifestKeyChunk {
325325
pub chunk_no: i64,
326326
/// The tenant prefix every key in this chunk lives under.
327327
pub prefix: String,
328-
/// Strictly ascending keys.
328+
/// Strictly ascending serialized manifest entries.
329329
pub keys: Vec<String>,
330330
}
331331

332+
/// One immutable object-store manifest entry.
333+
///
334+
/// Version 5 storage manifests serialize entries as
335+
/// `key\u{1f}version_id\u{1f}kind`, where kind is `object` or
336+
/// `delete_marker`. Version 4 manifests used bare keys. Keeping the side-table
337+
/// column name unchanged avoids a database migration while making the stream
338+
/// explicitly version-aware.
339+
#[derive(Debug, Clone, PartialEq, Eq)]
340+
pub struct StorageManifestEntry {
341+
/// Object key.
342+
pub key: String,
343+
/// S3 version id.
344+
pub version_id: String,
345+
/// Either `object` or `delete_marker`.
346+
pub kind: String,
347+
}
348+
349+
impl StorageManifestEntry {
350+
/// Create a manifest entry.
351+
pub fn new(
352+
key: impl Into<String>,
353+
version_id: impl Into<String>,
354+
kind: impl Into<String>,
355+
) -> Self {
356+
Self {
357+
key: key.into(),
358+
version_id: version_id.into(),
359+
kind: kind.into(),
360+
}
361+
}
362+
363+
/// Serialize this entry into the chunk stream.
364+
pub fn encode(&self) -> Result<String> {
365+
validate_manifest_component("key", &self.key)?;
366+
validate_manifest_component("version id", &self.version_id)?;
367+
validate_manifest_component("kind", &self.kind)?;
368+
if self.kind != "object" && self.kind != "delete_marker" {
369+
return Err(DbError::DeletionSafety(format!(
370+
"unsupported storage manifest entry kind {}",
371+
self.kind
372+
)));
373+
}
374+
Ok(format!(
375+
"{}\u{1f}{}\u{1f}{}",
376+
self.key, self.version_id, self.kind
377+
))
378+
}
379+
380+
/// Decode a manifest stream entry.
381+
pub fn decode(value: &str) -> Result<Self> {
382+
let mut parts = value.split('\u{1f}');
383+
let key = parts.next().unwrap_or_default();
384+
let version_id = parts.next().ok_or_else(|| {
385+
DbError::DeletionSafety("storage manifest entry is missing version id".to_string())
386+
})?;
387+
let kind = parts.next().ok_or_else(|| {
388+
DbError::DeletionSafety("storage manifest entry is missing kind".to_string())
389+
})?;
390+
if parts.next().is_some() {
391+
return Err(DbError::DeletionSafety(
392+
"storage manifest entry has too many fields".to_string(),
393+
));
394+
}
395+
let entry = Self::new(key, version_id, kind);
396+
entry.encode()?;
397+
Ok(entry)
398+
}
399+
}
400+
401+
fn validate_manifest_component(name: &str, value: &str) -> Result<()> {
402+
if value.is_empty() || value.contains(['\n', '\u{1f}']) {
403+
return Err(DbError::DeletionSafety(format!(
404+
"storage manifest {name} is empty or contains a reserved delimiter"
405+
)));
406+
}
407+
Ok(())
408+
}
409+
332410
/// One durable fleet-wide object-store taxonomy sweep record.
333411
#[derive(Debug, Clone, Serialize)]
334412
pub struct TaxonomySweep {
@@ -358,11 +436,11 @@ type TaxonomySweepRow = (
358436
i64,
359437
);
360438

361-
/// Streaming SHA-256 over a strictly ascending key stream.
439+
/// Streaming SHA-256 over a strictly ascending storage manifest stream.
362440
///
363441
/// The executor's prefix enumeration and the destructive freeze's chunk
364-
/// validation both fold keys through this, so "the chunk rows are exactly
365-
/// the frozen enumeration" reduces to digest equality. Each key is hashed
442+
/// validation both fold entries through this, so "the chunk rows are exactly
443+
/// the frozen enumeration" reduces to digest equality. Each entry is hashed
366444
/// with a trailing newline so concatenation cannot alias two streams.
367445
pub struct KeyStreamDigest {
368446
hasher: Sha256,
@@ -395,6 +473,17 @@ impl KeyStreamDigest {
395473
"storage key stream is not strictly ascending at {key}"
396474
)));
397475
}
476+
self.fold_unordered(key)
477+
}
478+
479+
/// Fold an already-canonical manifest entry whose source ordering is owned
480+
/// by the object store, not by key lexicographic order.
481+
///
482+
/// S3 `ListObjectVersions` sorts by key but orders multiple versions of one
483+
/// key by recency with opaque version ids, so version-aware manifests cannot
484+
/// require strictly ascending serialized entries. Digest equality still
485+
/// binds the exact stream that was listed and chunked.
486+
pub fn fold_unordered(&mut self, key: &str) -> Result<()> {
398487
self.hasher.update(key.as_bytes());
399488
self.hasher.update(b"\n");
400489
self.last = Some(key.to_owned());
@@ -2594,7 +2683,7 @@ async fn live_fenced_tables_on(conn: &mut PgConnection) -> Result<BTreeSet<Strin
25942683

25952684
/// Fail closed when a community-prefix inventory has an unsafe shape.
25962685
pub fn validate_storage_manifest(manifest: &StorageManifest) -> Result<()> {
2597-
if manifest.version != 4 {
2686+
if !matches!(manifest.version, 4 | 5) {
25982687
return Err(DbError::DeletionSafety(format!(
25992688
"unsupported storage manifest version {}",
26002689
manifest.version
@@ -2682,12 +2771,21 @@ fn validate_manifest_key_chunks(
26822771
));
26832772
}
26842773
for key in &keys.0 {
2685-
if !key.starts_with(chunk_prefix.as_str()) {
2774+
let prefix_key = if manifest.version >= 5 {
2775+
StorageManifestEntry::decode(key)?.key
2776+
} else {
2777+
key.clone()
2778+
};
2779+
if !prefix_key.starts_with(chunk_prefix.as_str()) {
26862780
return Err(DbError::DeletionSafety(format!(
2687-
"frozen key {key} is outside its chunk prefix {chunk_prefix}"
2781+
"frozen key {prefix_key} is outside its chunk prefix {chunk_prefix}"
26882782
)));
26892783
}
2690-
digest.fold(key)?;
2784+
if manifest.version >= 5 {
2785+
digest.fold_unordered(key)?;
2786+
} else {
2787+
digest.fold(key)?;
2788+
}
26912789
}
26922790
}
26932791
if let Some(summary) = current {
@@ -3039,6 +3137,36 @@ mod tests {
30393137
assert!(validate_storage_manifest(&malformed_digest).is_err());
30403138
}
30413139

3140+
#[test]
3141+
fn frozen_inventory_digest_is_canonical_for_v5_manifest_entries() {
3142+
let entry = StorageManifestEntry::new("_meta/c/a.json", "null", "object")
3143+
.encode()
3144+
.expect("entry");
3145+
let mut digest = KeyStreamDigest::new();
3146+
digest.fold_unordered(&entry).expect("fold entry");
3147+
let (keys_digest, object_count) = digest.finish();
3148+
let inventory = FrozenInventory {
3149+
schema: SchemaManifest {
3150+
scoped_tables: vec!["events".to_string()],
3151+
row_counts: BTreeMap::from([("events".to_string(), 1)]),
3152+
fenced_tables: vec!["events".to_string()],
3153+
},
3154+
storage: StorageManifest {
3155+
version: 5,
3156+
prefixes: vec![PrefixManifest {
3157+
prefix: "_meta/c/".to_string(),
3158+
object_count,
3159+
total_bytes: 4,
3160+
keys_digest,
3161+
}],
3162+
},
3163+
};
3164+
let digest = inventory.digest().unwrap();
3165+
let round_tripped: FrozenInventory =
3166+
serde_json::from_slice(&serde_json::to_vec(&inventory).unwrap()).unwrap();
3167+
assert_eq!(digest, round_tripped.digest().unwrap());
3168+
}
3169+
30423170
#[test]
30433171
fn key_stream_digest_requires_strict_order_and_is_chunking_invariant() {
30443172
let keys = ["a/1", "a/2", "a/3"];
@@ -3100,6 +3228,49 @@ mod tests {
31003228
assert!(validate_manifest_key_chunks(&storage_manifest(), &[]).is_ok());
31013229
}
31023230

3231+
#[test]
3232+
fn versioned_manifest_entries_decode_and_validate_chunks() {
3233+
let entries = vec![
3234+
StorageManifestEntry::new("_meta/c/1", "v2", "object")
3235+
.encode()
3236+
.expect("entry 1"),
3237+
StorageManifestEntry::new("_meta/c/1", "v1", "delete_marker")
3238+
.encode()
3239+
.expect("entry 2"),
3240+
];
3241+
let mut digest = KeyStreamDigest::new();
3242+
for entry in &entries {
3243+
digest.fold_unordered(entry).expect("fold version entry");
3244+
}
3245+
let (hex_digest, count) = digest.finish();
3246+
let mut manifest = storage_manifest();
3247+
manifest.version = 5;
3248+
manifest.prefixes[0].object_count = count;
3249+
manifest.prefixes[0].keys_digest = hex_digest;
3250+
3251+
let chunk = |entries: &[String]| {
3252+
vec![(
3253+
0,
3254+
"_meta/c/".to_string(),
3255+
sqlx::types::Json(entries.to_vec()),
3256+
)]
3257+
};
3258+
// v5 freeze validation is retry-stable: a retried freeze with the
3259+
// same canonical version-entry stream is accepted, while a drifted
3260+
// stream is rejected.
3261+
assert!(validate_manifest_key_chunks(&manifest, &chunk(&entries)).is_ok());
3262+
assert!(validate_manifest_key_chunks(&manifest, &chunk(&entries)).is_ok());
3263+
3264+
let foreign = vec![StorageManifestEntry::new("_uploads/c/1", "v1", "object")
3265+
.encode()
3266+
.expect("foreign entry")];
3267+
assert!(validate_manifest_key_chunks(&manifest, &chunk(&foreign)).is_err());
3268+
assert!(StorageManifestEntry::decode("_meta/c/1").is_err());
3269+
assert!(StorageManifestEntry::new("_meta/c/1", "v1", "unknown")
3270+
.encode()
3271+
.is_err());
3272+
}
3273+
31033274
#[test]
31043275
fn frozen_inventory_digest_is_stable() {
31053276
let inventory = FrozenInventory {

0 commit comments

Comments
 (0)