Skip to content

Commit 2a67653

Browse files
fix(media): finish deferred provenance and access feedback
Signed-off-by: Nikolaj Goodger <[email protected]>
1 parent 42e69e6 commit 2a67653

5 files changed

Lines changed: 231 additions & 18 deletions

File tree

‎crates/buzz-admin/src/main.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -105,13 +105,15 @@ enum Command {
105105
/// Run after every relay pod records shares and
106106
/// before enforcing channel-scoped media reads. Historical references are
107107
/// trusted only before each community's schema-recorded cutoff.
108+
/// Restores uploader proof from surviving server media_uploaded audit rows;
109+
/// files without that evidence still need an upload of the exact bytes.
108110
/// Later references use current permissions, so any run can grant references
109111
/// live recording previously refused if their authors have since gained access.
110112
/// Accept the resulting lasting grants before the first backfill; otherwise
111113
/// defer it. Each invocation restarts the scan and deduplicates existing rows.
112114
BackfillMediaShares {
113115
/// Events read and recorded per transaction.
114-
#[arg(long, default_value_t = 1000)]
116+
#[arg(long, default_value_t = 1000, value_parser = clap::value_parser!(i64).range(1..))]
115117
batch_size: i64,
116118
},
117119
/// Run the partition catalog audit using a read-only database session.

‎crates/buzz-core/src/media.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,8 @@
33
use std::collections::BTreeSet;
44

55
use crate::kind::{
6-
event_is_shared, is_author_only_event_kind, is_shared_gated_kind, P_GATED_KINDS,
6+
event_is_shared, is_author_only_event_kind, is_shared_gated_kind, KIND_AGENT_ENGRAM,
7+
P_GATED_KINDS,
78
};
89

910
/// Hashes of every blob `event` shares into its channel, or with the whole
@@ -17,6 +18,7 @@ pub fn shared_blob_hashes(event: &nostr::Event) -> Vec<String> {
1718
let kind = u32::from(event.kind.as_u16());
1819
if is_author_only_event_kind(event)
1920
|| P_GATED_KINDS.contains(&kind)
21+
|| kind == KIND_AGENT_ENGRAM
2022
|| (is_shared_gated_kind(kind) && !event_is_shared(event))
2123
{
2224
return Vec::new();

‎crates/buzz-db/src/store/media_reference.rs‎

Lines changed: 153 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,10 @@ impl Db {
108108
/// reading `batch_size` events at a time; returns
109109
/// the number of shares added.
110110
///
111+
/// Retained server upload audits restore uploader proof before references
112+
/// are evaluated. Missing, malformed, or unattributed audit rows establish
113+
/// no ownership; those uploads still need the exact bytes uploaded again.
114+
///
111115
/// Shares come from the same rule as live writes. Events received before
112116
/// share recording was installed predate the readability check, so their
113117
/// references are recorded as they stand; later events get the live,
@@ -130,6 +134,7 @@ impl Db {
130134
let mut added = 0;
131135
for (community, installed) in communities {
132136
let community = CommunityId::from_uuid(community);
137+
self.backfill_media_uploaders(community, batch_size).await?;
133138
let mut cursor: Option<(DateTime<Utc>, Vec<u8>)> = None;
134139
loop {
135140
let rows = sqlx::query(
@@ -343,6 +348,44 @@ mod postgres_tests {
343348
expected,
344349
"an uploader may share their own bytes"
345350
);
351+
352+
let excluded = unique_sha();
353+
upload(&db, community, &alice, &excluded).await;
354+
use buzz_core::kind::{
355+
AUTHOR_ONLY_KINDS, KIND_AGENT_ENGRAM, KIND_DELETION, KIND_EVENT_REMINDER,
356+
P_GATED_KINDS, SHARED_GATED_KINDS,
357+
};
358+
let kinds = AUTHOR_ONLY_KINDS
359+
.iter()
360+
.chain(P_GATED_KINDS)
361+
.chain(SHARED_GATED_KINDS)
362+
.copied()
363+
.chain([KIND_AGENT_ENGRAM, KIND_DELETION]);
364+
for kind in kinds {
365+
if (20_000..30_000).contains(&kind) {
366+
continue; // Ephemeral frames have no stored reference to backfill.
367+
}
368+
let mut tags = vec![Tag::parse(["d", "private-media"]).unwrap()];
369+
if kind == KIND_DELETION {
370+
tags.push(Tag::parse(["k", &KIND_EVENT_REMINDER.to_string()]).unwrap());
371+
}
372+
let event = EventBuilder::new(Kind::from(kind as u16), format!("/media/{excluded}"))
373+
.tags(tags)
374+
.sign_with_keys(&alice)
375+
.unwrap();
376+
db.insert_event(community, &event, None).await.unwrap();
377+
assert!(
378+
db.get_event_by_id(community, event.id.as_bytes())
379+
.await
380+
.unwrap()
381+
.is_some(),
382+
"kind {kind} must exercise a stored event"
383+
);
384+
assert!(
385+
shares(&db, community, &excluded).await.is_empty(),
386+
"private kind {kind} must create no grant even for its uploader"
387+
);
388+
}
346389
}
347390

348391
#[tokio::test]
@@ -490,21 +533,7 @@ mod postgres_tests {
490533
db.soft_delete_event_and_update_thread(community, retired.id.as_bytes(), None, None)
491534
.await
492535
.unwrap();
493-
let mut lifecycle = db.pool.begin().await.unwrap();
494-
sqlx::query(
495-
"SELECT set_config('buzz.deletion_executor_community', $1, true), \
496-
set_config('buzz.deletion_fence_generation', '0', true)",
497-
)
498-
.bind(community.as_uuid().to_string())
499-
.execute(&mut *lifecycle)
500-
.await
501-
.unwrap();
502-
sqlx::query("UPDATE communities SET deletion_state = 'quiescing' WHERE id = $1")
503-
.bind(community.as_uuid())
504-
.execute(&mut *lifecycle)
505-
.await
506-
.unwrap();
507-
lifecycle.commit().await.unwrap();
536+
quiesce(&db, community).await;
508537
let (other, other_community) = fixture().await;
509538
upload(&other, other_community, &owner, &sha).await;
510539
let old = attachment(&owner, &sha, None);
@@ -543,6 +572,24 @@ mod postgres_tests {
543572
.unwrap()
544573
}
545574

575+
async fn quiesce(db: &Db, community: CommunityId) {
576+
let mut lifecycle = db.pool.begin().await.unwrap();
577+
sqlx::query(
578+
"SELECT set_config('buzz.deletion_executor_community', $1, true), \
579+
set_config('buzz.deletion_fence_generation', '0', true)",
580+
)
581+
.bind(community.as_uuid().to_string())
582+
.execute(&mut *lifecycle)
583+
.await
584+
.unwrap();
585+
sqlx::query("UPDATE communities SET deletion_state = 'quiescing' WHERE id = $1")
586+
.bind(community.as_uuid())
587+
.execute(&mut *lifecycle)
588+
.await
589+
.unwrap();
590+
lifecycle.commit().await.unwrap();
591+
}
592+
546593
async fn reap_all(db: &Db) -> u64 {
547594
let mut cursor = None;
548595
let mut removed = 0;
@@ -727,6 +774,73 @@ mod postgres_tests {
727774
.expect("forget shares");
728775
tx.commit().await.expect("commit");
729776

777+
// Surviving server upload audits recover legacy ownership; malformed,
778+
// missing, or unrelated audit evidence never creates uploader proof.
779+
let (recovered, unproven) = (unique_sha(), unique_sha());
780+
let (uploader, stranger) = (Keys::generate(), Keys::generate());
781+
db.insert_event(community, &attachment(&uploader, &recovered, None), None)
782+
.await
783+
.unwrap();
784+
db.insert_event(community, &attachment(&stranger, &unproven, None), None)
785+
.await
786+
.unwrap();
787+
let mut tx = db.begin_media_share_transaction(community).await.unwrap();
788+
for (seq, action, actor, object) in [
789+
(
790+
1_i64,
791+
"media_uploaded",
792+
Some(uploader.public_key().to_bytes().to_vec()),
793+
recovered.clone(),
794+
),
795+
(2, "media_uploaded", None, unproven.clone()),
796+
(3, "media_uploaded", Some(vec![1; 31]), unproven.clone()),
797+
(
798+
4,
799+
"event_created",
800+
Some(stranger.public_key().to_bytes().to_vec()),
801+
unproven.clone(),
802+
),
803+
(
804+
5,
805+
"media_uploaded",
806+
Some(stranger.public_key().to_bytes().to_vec()),
807+
"bad-hash".into(),
808+
),
809+
] {
810+
sqlx::query(
811+
"INSERT INTO audit_log (community_id, seq, hash, action, actor_pubkey, object_id) \
812+
VALUES ($1,$2,$3,$4,$5,$6)",
813+
)
814+
.bind(community.as_uuid())
815+
.bind(seq)
816+
.bind(vec![seq as u8; 32])
817+
.bind(action)
818+
.bind(actor)
819+
.bind(object)
820+
.execute(tx.conn())
821+
.await
822+
.unwrap();
823+
}
824+
tx.commit().await.unwrap();
825+
let (fenced, fenced_community) = fixture().await;
826+
let mut tx = fenced
827+
.begin_media_share_transaction(fenced_community)
828+
.await
829+
.unwrap();
830+
sqlx::query(
831+
"INSERT INTO audit_log (community_id, seq, hash, action, actor_pubkey, object_id) \
832+
VALUES ($1,1,$2,'media_uploaded',$3,$4)",
833+
)
834+
.bind(fenced_community.as_uuid())
835+
.bind(vec![1_u8; 32])
836+
.bind(stranger.public_key().to_bytes().as_slice())
837+
.bind(&unproven)
838+
.execute(tx.conn())
839+
.await
840+
.unwrap();
841+
tx.commit().await.unwrap();
842+
quiesce(&fenced, fenced_community).await;
843+
730844
db.backfill_media_shares(1).await.expect("backfill");
731845
assert_eq!(db.backfill_media_shares(1).await.expect("rerun"), 0);
732846
assert_eq!(
@@ -735,6 +849,30 @@ mod postgres_tests {
735849
"a reference live recording refused must stay unshared"
736850
);
737851
assert_eq!(shares(&db, community, &historical).await, vec![None]);
852+
let fenced_proofs: i64 =
853+
sqlx::query_scalar("SELECT count(*) FROM media_uploaders WHERE community_id = $1")
854+
.bind(fenced_community.as_uuid())
855+
.fetch_one(&db.pool)
856+
.await
857+
.unwrap();
858+
assert_eq!(
859+
fenced_proofs, 0,
860+
"audit recovery must not write across a fenced tenant boundary"
861+
);
862+
assert_eq!(
863+
shares(&db, community, &recovered).await,
864+
vec![None],
865+
"retained upload evidence must recover an uploader's intended share"
866+
);
867+
assert!(shares(&db, community, &unproven).await.is_empty());
868+
let proofs: Vec<(String, Vec<u8>)> = sqlx::query_as(
869+
"SELECT sha256, pubkey FROM media_uploaders WHERE community_id = $1 AND sha256 = ANY($2)",
870+
).bind(community.as_uuid()).bind(vec![recovered.clone(), unproven])
871+
.fetch_all(&db.pool).await.unwrap();
872+
assert_eq!(
873+
proofs,
874+
vec![(recovered, uploader.public_key().to_bytes().to_vec())]
875+
);
738876
assert!(
739877
shares(&db, community, &deleted_history).await.is_empty(),
740878
"legacy deleted messages must never gain a share"

‎crates/buzz-db/src/store/media_uploader.rs‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,71 @@ use buzz_datastore_tracing::datastore_span;
44

55
use crate::deletion::ServingWriteLease;
66
use crate::{error::Result, observability, Db};
7+
use buzz_core::CommunityId;
78

89
impl Db {
10+
/// Recover possession proof from retained server-authored upload audits.
11+
/// Scan a finite sequence range using audit_log's tenant-leading primary
12+
/// key; malformed or unattributed records never establish ownership.
13+
#[datastore_span(name = "backfill_media_uploaders", system = "postgresql")]
14+
pub(crate) async fn backfill_media_uploaders(
15+
&self,
16+
community: CommunityId,
17+
batch_size: i64,
18+
) -> Result<u64> {
19+
let mut conn =
20+
observability::acquire_writer(&self.pool, observability::WriterOperation::Maintenance)
21+
.await?;
22+
let upper: Option<i64> =
23+
sqlx::query_scalar("SELECT max(seq) FROM audit_log WHERE community_id = $1")
24+
.bind(community.as_uuid())
25+
.fetch_one(&mut *conn)
26+
.await?;
27+
drop(conn);
28+
let Some(upper) = upper else {
29+
return Ok(0);
30+
};
31+
let (mut cursor, mut added) = (0_i64, 0_u64);
32+
while cursor < upper {
33+
let mut tx = crate::begin_community_write_transaction(
34+
&self.pool,
35+
community,
36+
observability::WriterOperation::Maintenance,
37+
)
38+
.await?;
39+
let page: Vec<i64> = sqlx::query_scalar(
40+
"SELECT seq FROM audit_log WHERE community_id = $1 AND seq > $2 AND seq <= $3 \
41+
ORDER BY seq LIMIT $4",
42+
)
43+
.bind(community.as_uuid())
44+
.bind(cursor)
45+
.bind(upper)
46+
.bind(batch_size)
47+
.fetch_all(tx.conn())
48+
.await?;
49+
let Some(&last) = page.last() else {
50+
break;
51+
};
52+
added += sqlx::query(
53+
"INSERT INTO media_uploaders (community_id, sha256, pubkey, created_at) \
54+
SELECT community_id, object_id, actor_pubkey, created_at FROM audit_log \
55+
WHERE community_id = $1 AND seq > $2 AND seq <= $3 \
56+
AND action = 'media_uploaded' AND octet_length(actor_pubkey) = 32 \
57+
AND object_id ~ '^[0-9a-f]{64}$' \
58+
ORDER BY object_id, actor_pubkey ON CONFLICT DO NOTHING",
59+
)
60+
.bind(community.as_uuid())
61+
.bind(cursor)
62+
.bind(last)
63+
.execute(tx.conn())
64+
.await?
65+
.rows_affected();
66+
tx.commit().await?;
67+
cursor = last;
68+
}
69+
Ok(added)
70+
}
71+
972
/// Record that `pubkey` uploaded the blob `sha256`; repeat uploads are
1073
/// no-ops. The write presents the upload's serving lease, so it may finish
1174
/// while the community quiesces but never after the lease is gone.

‎crates/buzz-relay/src/api/media.rs‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -587,7 +587,15 @@ async fn upload_blob_result(
587587
auth.auth_event.pubkey.as_bytes(),
588588
)
589589
.await
590-
.map_err(|error| serving_lease_lost(error.into()))?;
590+
.map_err(|error| {
591+
let error = anyhow::Error::from(error);
592+
if buzz_deletion::ServingWriteGuard::is_lease_lost(&error) {
593+
serving_lease_lost(error)
594+
} else {
595+
tracing::error!(%error, "media uploader recording failed");
596+
MediaError::ServiceUnavailable
597+
}
598+
})?;
591599

592600
// Normalize MIME to a known set to bound label cardinality.
593601
let mime_label = match descriptor.mime_type.as_str() {

0 commit comments

Comments
 (0)