Skip to content

Commit e7a35b6

Browse files
Ian Oberstclaude
andcommitted
fix(relay): fast-retry only a catch-all serving the current month
The prompt retry also fired on the routine monthly advance, when only the last lookahead month falls into the catch-all and six months of runway remain. On a busy table each retry queues an ACCESS EXCLUSIVE request that stalls readers and writers for up to the 2 s lock timeout. Retry promptly only while the current month routes to the catch-all and the maintenance planner would replace it under the relay's policy. Layouts the planner refuses, and a disabled policy, stay on the normal audit interval. Co-Authored-By: Claude Opus 5.5 <[email protected]> Signed-off-by: Ian Oberst <[email protected]>
1 parent 90a295b commit e7a35b6

4 files changed

Lines changed: 120 additions & 71 deletions

File tree

‎crates/buzz-db/src/store/partition.rs‎

Lines changed: 0 additions & 52 deletions
Original file line numberDiff line numberDiff line change
@@ -225,23 +225,6 @@ impl PartitionTableAudit {
225225
.count()
226226
}
227227

228-
/// Whether an empty catch-all still serves an audited month, so a
229-
/// maintenance pass with advancement enabled would replace it.
230-
///
231-
/// False once any catch-all holds rows: that case needs an operator, and
232-
/// retrying cannot fix it.
233-
pub fn awaits_catch_all_advance(&self) -> bool {
234-
self.nonempty_catch_all_count() == 0
235-
&& self.children.iter().any(|child| {
236-
child.kind == PartitionChildKind::CatchAll
237-
&& child.catch_all_nonempty == Some(false)
238-
})
239-
&& self
240-
.months
241-
.iter()
242-
.any(|month| month.kind == MonthCoverageKind::CoveredByCatchAll)
243-
}
244-
245228
/// Number of root children whose `DEFAULT` subtree contains at least one row.
246229
pub fn nonempty_default_count(&self) -> usize {
247230
self.children
@@ -398,14 +381,6 @@ impl PartitionAudit {
398381
.any(|leaf| leaf_covers_timestamp(leaf, &timestamp))
399382
})
400383
}
401-
402-
/// Whether any managed parent still has an empty catch-all serving an
403-
/// audited month. See [`PartitionTableAudit::awaits_catch_all_advance`].
404-
pub fn awaits_catch_all_advance(&self) -> bool {
405-
self.tables
406-
.iter()
407-
.any(PartitionTableAudit::awaits_catch_all_advance)
408-
}
409384
}
410385

411386
#[derive(Debug)]
@@ -1988,33 +1963,6 @@ mod tests {
19881963
}
19891964
}
19901965

1991-
#[test]
1992-
fn awaits_catch_all_advance_only_while_an_empty_catch_all_serves_a_month() {
1993-
let before = Utc.with_ymd_and_hms(2026, 8, 15, 12, 0, 0).unwrap();
1994-
assert!(!repaired_incident_audit("events", before).awaits_catch_all_advance());
1995-
1996-
// October's three-month lookahead reaches January, which the catch-all serves.
1997-
let now = Utc.with_ymd_and_hms(2026, 10, 15, 12, 0, 0).unwrap();
1998-
let mut events = repaired_incident_audit("events", now);
1999-
assert!(events.awaits_catch_all_advance());
2000-
let audit = PartitionAudit {
2001-
audited_at: now,
2002-
tables: vec![
2003-
repaired_incident_audit("delivery_log", before),
2004-
events.clone(),
2005-
],
2006-
};
2007-
assert!(audit.awaits_catch_all_advance());
2008-
2009-
// Rows in the catch-all need an operator; retrying cannot advance it.
2010-
for child in &mut events.children {
2011-
if child.kind == PartitionChildKind::CatchAll {
2012-
child.catch_all_nonempty = Some(true);
2013-
}
2014-
}
2015-
assert!(!events.awaits_catch_all_advance());
2016-
}
2017-
20181966
#[test]
20191967
fn repaired_incident_layout_is_historical_once_its_months_have_passed() {
20201968
for (now, degraded, outcome) in [

‎crates/buzz-db/src/store/partition/maintenance.rs‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -315,6 +315,38 @@ fn plan_maintenance(
315315
})
316316
}
317317

318+
impl PartitionTableAudit {
319+
/// Whether rows timestamped now route to an empty catch-all that a
320+
/// maintenance pass under `policy` would replace.
321+
///
322+
/// Only the current month counts: a catch-all that serves just the far
323+
/// end of the lookahead window is the routine monthly advance, which can
324+
/// wait for the next pass. Any layout the planner refuses, such as an
325+
/// occupied catch-all, needs an operator and is false.
326+
pub fn awaits_catch_all_advance(&self, policy: PartitionMaintenancePolicy) -> bool {
327+
self.months
328+
.first()
329+
.is_some_and(|month| month.kind == MonthCoverageKind::CoveredByCatchAll)
330+
&& matches!(
331+
plan_maintenance(self, policy),
332+
MaintenancePlan::Apply(MaintenanceChange {
333+
catch_all: Some(_),
334+
..
335+
})
336+
)
337+
}
338+
}
339+
340+
impl PartitionAudit {
341+
/// Whether any managed parent awaits catch-all advancement. See
342+
/// [`PartitionTableAudit::awaits_catch_all_advance`].
343+
pub fn awaits_catch_all_advance(&self, policy: PartitionMaintenancePolicy) -> bool {
344+
self.tables
345+
.iter()
346+
.any(|table| table.awaits_catch_all_advance(policy))
347+
}
348+
}
349+
318350
/// Audit, then issue only the DDL `policy` permits, one bounded transaction per table.
319351
///
320352
/// A table that is already at the desired layout, or whose required DDL is

‎crates/buzz-db/src/store/partition/maintenance_tests.rs‎

Lines changed: 80 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -126,6 +126,83 @@ fn advancement_requires_its_own_policy_flag() {
126126
);
127127
}
128128

129+
/// The repaired layout once January 2027 arrives: the current month and the
130+
/// whole lookahead route to the empty catch-all, as on a fresh install.
131+
fn catch_all_serving_now_audit() -> PartitionTableAudit {
132+
let mut audit = repaired_audit(false);
133+
audit.months = (0..=PARTITION_MANAGER_MONTHS_AHEAD as i32)
134+
.map(|offset| {
135+
let (year, m) = add_months(2027, 1, offset).unwrap();
136+
MonthCoverage {
137+
start: month(year, m),
138+
kind: MonthCoverageKind::CoveredByCatchAll,
139+
}
140+
})
141+
.collect();
142+
audit
143+
}
144+
145+
fn with_catch_all(
146+
mut audit: PartitionTableAudit,
147+
edit: impl Fn(&mut PartitionChildAudit),
148+
) -> PartitionTableAudit {
149+
for child in &mut audit.children {
150+
if child.kind == PartitionChildKind::CatchAll {
151+
edit(child);
152+
}
153+
}
154+
audit
155+
}
156+
157+
#[test]
158+
fn only_a_catch_all_serving_the_current_month_awaits_advance() {
159+
let serving = catch_all_serving_now_audit();
160+
assert!(serving.awaits_catch_all_advance(ADVANCE));
161+
162+
// The routine monthly advance: the planner would advance, but only the end
163+
// of the window routes to the catch-all, so it waits for the next pass.
164+
let routine = repaired_audit(false);
165+
assert!(matches!(
166+
plan_maintenance(&routine, ADVANCE),
167+
MaintenancePlan::Apply(_)
168+
));
169+
assert!(!routine.awaits_catch_all_advance(ADVANCE));
170+
171+
// Advancement disabled, including by the create kill switch.
172+
for policy in [
173+
CREATE_ONLY,
174+
PartitionMaintenancePolicy {
175+
create_enabled: false,
176+
advance_enabled: true,
177+
},
178+
PartitionMaintenancePolicy::default(),
179+
] {
180+
assert!(!serving.awaits_catch_all_advance(policy), "{policy:?}");
181+
}
182+
183+
// Layouts the planner refuses need an operator, not prompt retries.
184+
let occupied = with_catch_all(serving.clone(), |child| {
185+
child.catch_all_nonempty = Some(true);
186+
});
187+
let child_triggers = with_catch_all(serving.clone(), |child| {
188+
child.extra_triggers = vec!["child_only_probe".to_string()];
189+
});
190+
for refused in [&occupied, &child_triggers] {
191+
assert!(matches!(
192+
plan_maintenance(refused, ADVANCE),
193+
MaintenancePlan::Refuse(_)
194+
));
195+
assert!(!refused.awaits_catch_all_advance(ADVANCE));
196+
}
197+
198+
let catalog = |tables| PartitionAudit {
199+
audited_at: month(2027, 1),
200+
tables,
201+
};
202+
assert!(catalog(vec![routine.clone(), serving]).awaits_catch_all_advance(ADVANCE));
203+
assert!(!catalog(vec![routine, occupied]).awaits_catch_all_advance(ADVANCE));
204+
}
205+
129206
#[test]
130207
fn full_monthly_runway_is_a_noop_even_when_disabled() {
131208
let mut audit = repaired_audit(false);
@@ -1374,14 +1451,14 @@ mod postgres_tests {
13741451
// already serves months, and creation alone cannot replace it.
13751452
for audit in [&passes.migrated, &passes.create_only] {
13761453
assert_eq!(audit.tables.len(), PARTITIONED_TABLES.len());
1377-
assert!(audit.awaits_catch_all_advance(), "{audit:?}");
1454+
assert!(audit.awaits_catch_all_advance(ADVANCE), "{audit:?}");
13781455
for table in &audit.tables {
1379-
assert!(table.awaits_catch_all_advance(), "{}", table.table);
1456+
assert!(table.awaits_catch_all_advance(ADVANCE), "{}", table.table);
13801457
}
13811458
}
13821459

13831460
assert_advanced(&passes.advanced);
1384-
assert!(!passes.advanced.awaits_catch_all_advance());
1461+
assert!(!passes.advanced.awaits_catch_all_advance(ADVANCE));
13851462
assert!(passes.advanced.serving_safe());
13861463
for table in &passes.advanced.tables {
13871464
assert!(!table.degraded(), "{}: {table:?}", table.table);

‎crates/buzz-relay/src/main.rs‎

Lines changed: 8 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -295,23 +295,15 @@ where
295295
}
296296
}
297297

298-
/// Prompt retries allowed while an empty catch-all awaits advancement. With the
299-
/// 5 s to 60 s backoff they span about five minutes, after which the normal
300-
/// period applies until the catch-all no longer awaits advancement.
298+
/// Prompt retries allowed while rows timestamped now route to an empty catch-all
299+
/// awaiting advancement. With the 5 s to 60 s backoff they span about five
300+
/// minutes, after which the normal period applies until that changes.
301301
const PARTITION_ADVANCE_RETRY_LIMIT: u32 = 8;
302302

303-
/// Whether the next maintenance pass would advance an empty catch-all.
304-
fn partition_advance_pending(
305-
policy: buzz_db::partition::PartitionMaintenancePolicy,
306-
audit: &PartitionAudit,
307-
) -> bool {
308-
policy.create_enabled && policy.advance_enabled && audit.awaits_catch_all_advance()
309-
}
310-
311303
/// Retry missing diagnostics promptly, without tying audit recovery to readiness.
312-
/// Also retries promptly, a bounded number of times, while an empty catch-all
313-
/// awaits advancement, so a failed first pass on a fresh install does not wait
314-
/// a full period. Delays start after each attempt finishes, so slow audits
304+
/// Also retries promptly, a bounded number of times, while the current month
305+
/// routes to an empty catch-all awaiting advancement, so a failed first pass on
306+
/// a fresh install does not wait a full period. Delays start after each attempt finishes, so slow audits
315307
/// cannot cause bursts.
316308
struct PartitionAuditSchedule {
317309
period: std::time::Duration,
@@ -928,7 +920,7 @@ async fn run_relay_main(boot: BootTracker) -> anyhow::Result<()> {
928920
let state = Arc::new(app_state);
929921
let startup_partition_attempt = startup_partition_audit
930922
.as_ref()
931-
.map(|audit| partition_advance_pending(partition_policy, audit));
923+
.map(|audit| audit.awaits_catch_all_advance(partition_policy));
932924
if let Some(audit) = startup_partition_audit {
933925
state.record_partition_audit(audit);
934926
}
@@ -967,7 +959,7 @@ async fn run_relay_main(boot: BootTracker) -> anyhow::Result<()> {
967959
result
968960
.as_ref()
969961
.ok()
970-
.map(|audit| partition_advance_pending(partition_policy, audit)),
962+
.map(|audit| audit.awaits_catch_all_advance(partition_policy)),
971963
);
972964
match result {
973965
Ok(audit) => partition_state.record_partition_audit(audit),

0 commit comments

Comments
 (0)