Skip to content

Commit 5bb3637

Browse files
tlongwell-blocknpub1jh9wn95s0472h86ahapupaf7m6kx4v9sx2n0atj2hltcfer8k06s5n3pyf
andauthored
fix(cli): paginate channel directory queries (#2181)
Signed-off-by: npub1jh9wn95s0472h86ahapupaf7m6kx4v9sx2n0atj2hltcfer8k06s5n3pyf <95cae996907d7cab9f5dbf43c0f53edeac6ab0b032a6feae4abfd784e467b3f5@sprout-oss.stage.blox.sqprod.co> Co-authored-by: npub1jh9wn95s0472h86ahapupaf7m6kx4v9sx2n0atj2hltcfer8k06s5n3pyf <95cae996907d7cab9f5dbf43c0f53edeac6ab0b032a6feae4abfd784e467b3f5@sprout-oss.stage.blox.sqprod.co>
1 parent 33267e4 commit 5bb3637

5 files changed

Lines changed: 234 additions & 84 deletions

File tree

‎crates/buzz-cli/src/client.rs‎

Lines changed: 100 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -335,6 +335,29 @@ mod media_download_tests {
335335
}
336336
}
337337

338+
const QUERY_PAGE_SIZE: u32 = 500;
339+
340+
fn advance_query_cursor(
341+
filter: &mut serde_json::Value,
342+
page: &[serde_json::Value],
343+
) -> Result<(), CliError> {
344+
let last = page
345+
.last()
346+
.expect("a full query page always has a last event");
347+
let created_at = last
348+
.get("created_at")
349+
.and_then(serde_json::Value::as_u64)
350+
.ok_or_else(|| CliError::Other("query event missing created_at".into()))?;
351+
let id = last
352+
.get("id")
353+
.and_then(serde_json::Value::as_str)
354+
.filter(|id| id.len() == 64 && id.chars().all(|c| c.is_ascii_hexdigit()))
355+
.ok_or_else(|| CliError::Other("query event missing valid id".into()))?;
356+
filter["until"] = serde_json::json!(created_at);
357+
filter["before_id"] = serde_json::json!(id);
358+
Ok(())
359+
}
360+
338361
pub struct BuzzClient {
339362
http: reqwest::Client,
340363
relay_url: String, // base URL, no trailing slash, e.g. "https://relay.buzz.place"
@@ -428,6 +451,54 @@ impl BuzzClient {
428451
}
429452
}
430453

454+
async fn query_pages(
455+
&self,
456+
mut filter: serde_json::Value,
457+
limit: Option<u32>,
458+
) -> Result<Vec<serde_json::Value>, CliError> {
459+
let mut events = Vec::new();
460+
461+
while limit.is_none_or(|limit| events.len() < limit as usize) {
462+
let page_limit = limit
463+
.map(|limit| (limit as usize - events.len()).min(QUERY_PAGE_SIZE as usize))
464+
.unwrap_or(QUERY_PAGE_SIZE as usize);
465+
filter["limit"] = serde_json::json!(page_limit);
466+
467+
let raw = self.query(&filter).await?;
468+
let page: Vec<serde_json::Value> = serde_json::from_str(&raw)
469+
.map_err(|e| CliError::Other(format!("failed to parse query response: {e}")))?;
470+
let done = page.len() < page_limit;
471+
472+
if !done {
473+
advance_query_cursor(&mut filter, &page)?;
474+
}
475+
events.extend(page);
476+
if done {
477+
break;
478+
}
479+
}
480+
481+
Ok(events)
482+
}
483+
484+
/// Query up to `limit` historical events, following the relay bridge's
485+
/// composite `(until, before_id)` cursor across bounded result pages.
486+
pub async fn query_paginated(
487+
&self,
488+
filter: serde_json::Value,
489+
limit: u32,
490+
) -> Result<Vec<serde_json::Value>, CliError> {
491+
self.query_pages(filter, Some(limit)).await
492+
}
493+
494+
/// Query every historical event matching a filter across bounded pages.
495+
pub async fn query_all(
496+
&self,
497+
filter: serde_json::Value,
498+
) -> Result<Vec<serde_json::Value>, CliError> {
499+
self.query_pages(filter, None).await
500+
}
501+
431502
/// Execute a one-shot query via the HTTP bridge.
432503
/// `filter` is a Nostr filter object (will be wrapped in an array).
433504
/// Returns the raw JSON response (array of events).
@@ -849,7 +920,35 @@ pub fn normalize_write_response(raw: &str) -> String {
849920

850921
#[cfg(test)]
851922
mod tests {
852-
use super::{create_response_with_id, extract_relay_response_field};
923+
use super::{advance_query_cursor, create_response_with_id, extract_relay_response_field};
924+
925+
#[test]
926+
fn query_cursor_uses_last_events_composite_sort_key() {
927+
let mut filter = serde_json::json!({"kinds": [39000], "limit": 500});
928+
let page = vec![
929+
serde_json::json!({"id": "a".repeat(64), "created_at": 20}),
930+
serde_json::json!({"id": "b".repeat(64), "created_at": 10}),
931+
];
932+
933+
advance_query_cursor(&mut filter, &page).unwrap();
934+
935+
assert_eq!(filter["until"], serde_json::json!(10));
936+
assert_eq!(filter["before_id"], serde_json::json!("b".repeat(64)));
937+
}
938+
939+
#[test]
940+
fn query_cursor_rejects_missing_or_malformed_sort_key() {
941+
let mut filter = serde_json::json!({});
942+
assert!(
943+
advance_query_cursor(&mut filter, &[serde_json::json!({"id": "a".repeat(64)})])
944+
.is_err()
945+
);
946+
assert!(advance_query_cursor(
947+
&mut filter,
948+
&[serde_json::json!({"id": "not-an-event-id", "created_at": 10})]
949+
)
950+
.is_err());
951+
}
853952

854953
#[test]
855954
fn extract_relay_response_field_reads_response_message_json() {

‎crates/buzz-cli/src/commands/channels.rs‎

Lines changed: 29 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -29,17 +29,16 @@ pub async fn cmd_list_channels(
2929
format: &crate::OutputFormat,
3030
) -> Result<(), CliError> {
3131
let effective_limit = limit.unwrap_or(500);
32-
let raw = if member == Some(true) {
32+
let events = if member == Some(true) {
3333
// Step 1: find channel IDs where we're a member (kind:39002)
3434
let my_pk = client.keys().public_key().to_hex();
3535
let member_filter = serde_json::json!({
3636
"kinds": [39002],
3737
"#p": [my_pk],
38-
"limit": effective_limit
3938
});
40-
let member_resp = client.query(&member_filter).await?;
41-
let member_events: Vec<serde_json::Value> =
42-
serde_json::from_str(&member_resp).unwrap_or_default();
39+
let member_events = client
40+
.query_paginated(member_filter, effective_limit)
41+
.await?;
4342
let channel_ids: Vec<String> = member_events
4443
.iter()
4544
.map(extract_d_tag)
@@ -49,22 +48,21 @@ pub async fn cmd_list_channels(
4948
println!("[]");
5049
return Ok(());
5150
}
52-
// Step 2: fetch kind:39000 metadata for those channels
51+
// Step 2: fetch kind:39000 metadata for those channels.
5352
let metadata_filter = serde_json::json!({
5453
"kinds": [39000],
5554
"#d": channel_ids,
56-
"limit": effective_limit
5755
});
58-
client.query(&metadata_filter).await?
56+
client
57+
.query_paginated(metadata_filter, effective_limit)
58+
.await?
5959
} else {
6060
let filter = serde_json::json!({
6161
"kinds": [39000],
62-
"limit": effective_limit
6362
});
64-
client.query(&filter).await?
63+
client.query_paginated(filter, effective_limit).await?
6564
};
6665

67-
let events: Vec<serde_json::Value> = serde_json::from_str(&raw).unwrap_or_default();
6866
let channels: Vec<serde_json::Value> = events
6967
.iter()
7068
.filter(|e| {
@@ -130,16 +128,8 @@ pub async fn cmd_search_channels(
130128

131129
let filter = serde_json::json!({
132130
"kinds": [39000],
133-
"limit": limit,
134131
});
135-
let raw = client.query(&filter).await?;
136-
137-
let events: serde_json::Value = serde_json::from_str(&raw)
138-
.map_err(|e| CliError::Other(format!("failed to parse response: {e}")))?;
139-
let Some(arr) = events.as_array() else {
140-
println!("[]");
141-
return Ok(());
142-
};
132+
let arr = client.query_paginated(filter, limit).await?;
143133

144134
let needle = query.to_ascii_lowercase();
145135
let mut matches: Vec<ChannelSummary> = arr
@@ -337,14 +327,6 @@ pub async fn cmd_create_channel(
337327
Ok(())
338328
}
339329

340-
/// Relay-side cap on a single historical query (`buzz-relay/src/handlers/req.rs`).
341-
/// Never request above this — a clamped response compared against a larger
342-
/// requested size would falsely look like "no more pages."
343-
const RELAY_MAX_HISTORICAL_LIMIT: u32 = 2000;
344-
345-
/// One page of a keyset-paginated kind:30177 scan.
346-
const MANAGED_AGENT_PAGE_SIZE: u32 = RELAY_MAX_HISTORICAL_LIMIT;
347-
348330
/// A resolved live managed-agent instance backing a template persona slug.
349331
#[derive(Debug, Clone, PartialEq, Eq)]
350332
struct ResolvedAgent {
@@ -424,62 +406,29 @@ async fn scan_managed_agents_by_owner(
424406
owner: &str,
425407
slugs: &HashSet<&str>,
426408
) -> Result<Vec<ResolvedAgent>, CliError> {
409+
let filter = serde_json::json!({
410+
"kinds": [KIND_MANAGED_AGENT],
411+
"authors": [owner],
412+
});
413+
let events = client.query_all(filter).await?;
427414
let mut found: Vec<ResolvedAgent> = Vec::new();
428-
let mut seen_event_ids: HashSet<String> = HashSet::new();
429-
let mut cursor: Option<(i64, String)> = None;
430-
431-
loop {
432-
let mut filter = serde_json::json!({
433-
"kinds": [KIND_MANAGED_AGENT],
434-
"authors": [owner],
435-
"limit": MANAGED_AGENT_PAGE_SIZE,
436-
});
437-
if let Some((until, ref before_id)) = cursor {
438-
filter["until"] = serde_json::json!(until);
439-
filter["before_id"] = serde_json::json!(before_id);
440-
}
441-
442-
let raw = client.query(&filter).await?;
443-
let events: Vec<serde_json::Value> = serde_json::from_str(&raw).map_err(|e| {
444-
CliError::Other(format!("failed to parse managed-agent query response: {e}"))
445-
})?;
446-
447-
let page_len = events.len();
448-
for event in &events {
449-
let Some(event_id) = event.get("id").and_then(|v| v.as_str()) else {
450-
continue;
451-
};
452-
if !seen_event_ids.insert(event_id.to_string()) {
453-
continue;
454-
}
455-
let pubkey = extract_d_tag(event);
456-
if pubkey.is_empty() {
457-
continue;
458-
}
459-
let content: ManagedAgentContent = event
460-
.get("content")
461-
.and_then(|c| c.as_str())
462-
.and_then(|s| serde_json::from_str(s).ok())
463-
.unwrap_or(ManagedAgentContent { persona_id: None });
464-
let Some(persona_id) = content.persona_id else {
465-
continue;
466-
};
467-
if slugs.contains(persona_id.as_str()) {
468-
found.push(ResolvedAgent { persona_id, pubkey });
469-
}
470-
}
471415

472-
if (page_len as u32) < MANAGED_AGENT_PAGE_SIZE {
473-
break;
416+
for event in &events {
417+
let pubkey = extract_d_tag(event);
418+
if pubkey.is_empty() {
419+
continue;
474420
}
475-
let Some(last) = events.last() else { break };
476-
let Some(created_at) = last.get("created_at").and_then(|v| v.as_i64()) else {
477-
break;
478-
};
479-
let Some(last_id) = last.get("id").and_then(|v| v.as_str()) else {
480-
break;
421+
let content: ManagedAgentContent = event
422+
.get("content")
423+
.and_then(|c| c.as_str())
424+
.and_then(|s| serde_json::from_str(s).ok())
425+
.unwrap_or(ManagedAgentContent { persona_id: None });
426+
let Some(persona_id) = content.persona_id else {
427+
continue;
481428
};
482-
cursor = Some((created_at, last_id.to_string()));
429+
if slugs.contains(persona_id.as_str()) {
430+
found.push(ResolvedAgent { persona_id, pubkey });
431+
}
483432
}
484433

485434
Ok(found)

‎crates/buzz-db/src/event.rs‎

Lines changed: 57 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -63,8 +63,9 @@ pub struct EventQuery {
6363
/// Restrict results to events with an `e` tag referencing any of these event IDs (hex).
6464
/// Uses JSONB containment (`tags @> ...`) against the `tags` column.
6565
pub e_tags: Option<Vec<String>>,
66-
/// Restrict results to events in any of these channels (multi-channel `IN` pushdown).
67-
/// Used by NIP-45 COUNT to enforce channel access without fetching all rows.
66+
/// Restrict results to events in any of these channels, while retaining
67+
/// channel-less global events. Applied before SQL `LIMIT` so access-filtered
68+
/// historical pages have exact exhaustion semantics.
6869
pub channel_ids: Option<Vec<uuid::Uuid>>,
6970
/// Override the default limit clamp (1000). Used by COUNT fallback path
7071
/// which needs to fetch all matching events for post-filter counting.
@@ -1760,6 +1761,60 @@ mod tests {
17601761
.expect("sign")
17611762
}
17621763

1764+
fn make_event_at(kind: u16, content: &str, created_at: u64) -> nostr::Event {
1765+
EventBuilder::new(Kind::Custom(kind), content)
1766+
.custom_created_at(nostr::Timestamp::from(created_at))
1767+
.sign_with_keys(&Keys::generate())
1768+
.expect("sign timestamped event")
1769+
}
1770+
1771+
#[tokio::test]
1772+
#[ignore = "requires Postgres"]
1773+
async fn access_scope_is_applied_before_historical_page_limit() {
1774+
let pool = setup_pool().await;
1775+
let community_uuid = make_test_community(&pool).await;
1776+
let community = CommunityId::from_uuid(community_uuid);
1777+
let accessible = make_test_channel(&pool, community_uuid, None).await;
1778+
let inaccessible = make_test_channel(&pool, community_uuid, None).await;
1779+
let base = 1_800_000_000;
1780+
1781+
// This is the bridge underfetch shape: newer inaccessible candidates
1782+
// outnumber the requested page, while the visible match is older.
1783+
for offset in 10..13 {
1784+
let event = make_event_at(39_000, "newer inaccessible", base + offset);
1785+
insert_event(&pool, community, &event, Some(inaccessible))
1786+
.await
1787+
.expect("insert inaccessible candidate");
1788+
}
1789+
let global = make_event_at(39_000, "newer global", base + 2);
1790+
insert_event(&pool, community, &global, None)
1791+
.await
1792+
.expect("insert global candidate");
1793+
let older_accessible = make_event_at(39_000, "older accessible", base + 1);
1794+
insert_event(&pool, community, &older_accessible, Some(accessible))
1795+
.await
1796+
.expect("insert accessible candidate");
1797+
1798+
let events = query_events(
1799+
&pool,
1800+
&EventQuery {
1801+
kinds: Some(vec![39_000]),
1802+
channel_ids: Some(vec![accessible]),
1803+
limit: Some(2),
1804+
..EventQuery::for_community(community)
1805+
},
1806+
)
1807+
.await
1808+
.expect("query access-scoped page");
1809+
1810+
assert_eq!(events.len(), 2, "visible page must be filled before EOF");
1811+
assert_eq!(events[0].event.id, global.id, "global rows remain visible");
1812+
assert_eq!(
1813+
events[1].event.id, older_accessible.id,
1814+
"older accessible row must not be hidden behind newer inaccessible rows"
1815+
);
1816+
}
1817+
17631818
fn make_text_event(content: &str) -> nostr::Event {
17641819
let keys = Keys::generate();
17651820
EventBuilder::new(Kind::Custom(9), content)

‎crates/buzz-relay/src/api/bridge.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -955,6 +955,11 @@ pub async fn query_events(
955955
tenant.community(),
956956
)
957957
.await;
958+
crate::handlers::req::apply_access_scope_to_query(
959+
&mut query,
960+
extract_channel_from_filter(filter),
961+
&accessible_channels,
962+
);
958963

959964
match extract_before_id(raw) {
960965
BeforeId::Malformed => {

0 commit comments

Comments
 (0)