@@ -1221,8 +1221,7 @@ impl ObserverPublishQueue {
12211221 // Pre-trim at enqueue so (a) byte accounting reflects what will ship
12221222 // and (b) one oversized leaf cannot force every frame it touches into
12231223 // whole-envelope elision downstream.
1224- fit_observer_event_to_budget(&mut event);
1225- let bytes = serialized_len(&event);
1224+ let bytes = fit_observer_event_to_budget(&mut event);
12261225 self.pending_bytes += bytes;
12271226 self.events.push_back((bytes, source_events, event));
12281227 }
@@ -1638,44 +1637,55 @@ const OBSERVER_LEAF_RETAIN_BYTES: usize = 3_000;
16381637/// byte-identical.
16391638///
16401639/// The cap is measured in SERIALIZED bytes (JSON escaping makes serialized
1641- /// length differ from raw), so the stop condition is always a full reserialize
1642- /// of the whole frame: that counts the envelope, the variable `Option<String>`
1643- /// IDs, and any elision markers exactly. No separate margin constant is needed.
1640+ /// length differ from raw). For small leaves, subtract the old serialized
1641+ /// size and add the replacement's size. JSON delimiters and the envelope stay
1642+ /// unchanged, so this accounting is exact. For a dominant leaf, recount the
1643+ /// fitted frame instead of traversing the large original leaf again.
16441644///
16451645/// Termination is provable: each iteration elides the largest string leaf that
1646- /// would STRICTLY shrink the serialized frame, then reserializes. Shrinkability
1647- /// is re-evaluated against each leaf's CURRENT value, so a leaf already at its
1648- /// retained floor can never be re-elided — the loop strictly decreases the
1649- /// serialized length each pass and is bounded by the leaf count . When no leaf
1650- /// can shrink the frame and it still overflows, the payload is replaced with a
1651- /// tiny stub, which trivially fits. Monotone decrease, bounded below by the stub .
1646+ /// would STRICTLY shrink the serialized frame, then updates its size.
1647+ /// Shrinkability is re-evaluated against each leaf's CURRENT value: a shorter
1648+ /// removed-byte count can shorten an existing marker once more. Every pass
1649+ /// strictly decreases serialized length, so the loop terminates . When no leaf
1650+ /// can shrink the frame and it still overflows, replace the payload with a
1651+ /// small stub. Callers must still reject an oversized metadata envelope .
16521652///
1653- /// **Signature choice (`&mut`, double-serialize accepted):** on the common
1654- /// under-budget path this serializes the frame once to decide it fits, then
1655- /// `encrypt_observer_payload` serializes it again — one extra `to_string` of an
1656- /// already-small frame. Reusing that string would mean changing buzz-core's
1657- /// `encrypt_observer_payload` signature or adding a parallel encrypt path; both
1658- /// are out of this change's scope (buzz-core stays untouched). The clean `&mut`
1659- /// signature with one cheap redundant serialize is the deliberate tradeoff.
1660- fn fit_observer_event_to_budget(event: &mut observer::ObserverEvent) {
1661- if serialized_len(event) <= OBSERVER_MAX_PLAINTEXT_LEN {
1662- return;
1663- }
1664-
1665- // Raw size of the payload we are about to trim, captured before mutation so
1666- // the stub's `originalBytes` reports source bytes discarded, not serialized
1667- // overflow — consistent with the per-leaf marker's raw byte count.
1668- let original_payload_bytes = serde_json::to_string(&event.payload)
1669- .map(|s| s.len())
1670- .unwrap_or(0);
1671-
1672- // Elide the largest shrinkable leaf, reserialize, repeat. Each successful
1673- // elision strictly shrinks the serialized frame, and a floored leaf can
1674- // never be re-elided, so the loop is bounded by the leaf count.
1653+ /// Returns the fitted serialized size so retention can account for bytes
1654+ /// without another traversal. Counting writes avoid allocating a second copy
1655+ /// of a potentially multi-megabyte source frame just to measure its size.
1656+ fn fit_observer_event_to_budget(event: &mut observer::ObserverEvent) -> usize {
1657+ let mut bytes = serialized_len(event);
1658+ if bytes <= OBSERVER_MAX_PLAINTEXT_LEN {
1659+ return bytes;
1660+ }
1661+
1662+ // Derive the original payload size from the measured frame and its small
1663+ // envelope, avoiding a second traversal of the oversized source payload.
1664+ // JSON null occupies four bytes in the temporary envelope.
1665+ let payload = event.payload.take();
1666+ let envelope_bytes = serialized_len(event).saturating_sub(4);
1667+ event.payload = payload;
1668+ let original_payload_bytes = bytes.saturating_sub(envelope_bytes);
1669+
1670+ // Each successful elision strictly shrinks the serialized frame, so the
1671+ // loop terminates even when a shortened marker permits another elision.
16751672 while let Some(leaf) = largest_shrinkable_leaf(&mut event.payload) {
1676- elide_leaf(leaf);
1677- if serialized_len(event) <= OBSERVER_MAX_PLAINTEXT_LEN {
1678- return;
1673+ // A JSON string uses at most six bytes per source byte (\u0000),
1674+ // plus quotes. Use deltas only when even that upper bound is small
1675+ // relative to the frame; otherwise recount the now-smaller frame.
1676+ let leaf_bound = leaf
1677+ .as_str()
1678+ .map_or(0, |s| s.len().saturating_mul(6).saturating_add(2));
1679+ if leaf_bound < bytes / 2 {
1680+ let old_leaf_bytes = serialized_len(leaf);
1681+ elide_leaf(leaf);
1682+ bytes = bytes - old_leaf_bytes + serialized_len(leaf);
1683+ } else {
1684+ elide_leaf(leaf);
1685+ bytes = serialized_len(event);
1686+ }
1687+ if bytes <= OBSERVER_MAX_PLAINTEXT_LEN {
1688+ return bytes;
16791689 }
16801690 }
16811691
@@ -1685,10 +1695,27 @@ fn fit_observer_event_to_budget(event: &mut observer::ObserverEvent) {
16851695 "elided": format!("{} payload too large", event.kind),
16861696 "originalBytes": original_payload_bytes,
16871697 });
1698+ serialized_len(event)
16881699}
16891700
1690- fn serialized_len(event: &observer::ObserverEvent) -> usize {
1691- serde_json::to_string(event).map(|s| s.len()).unwrap_or(0)
1701+ fn serialized_len(value: &impl serde::Serialize) -> usize {
1702+ let mut counter = ObserverByteCounter(0);
1703+ serde_json::to_writer(&mut counter, value)
1704+ .map(|()| counter.0)
1705+ .unwrap_or(usize::MAX)
1706+ }
1707+
1708+ struct ObserverByteCounter(usize);
1709+
1710+ impl std::io::Write for ObserverByteCounter {
1711+ fn write(&mut self, bytes: &[u8]) -> std::io::Result<usize> {
1712+ self.0 = self.0.saturating_add(bytes.len());
1713+ Ok(bytes.len())
1714+ }
1715+
1716+ fn flush(&mut self) -> std::io::Result<()> {
1717+ Ok(())
1718+ }
16921719}
16931720
16941721/// Find the longest string leaf that would STRICTLY shrink if elided, returning
@@ -13566,6 +13593,38 @@ mod observer_payload_trim_tests {
1356613593 20_000,
1356713594 "the smaller leaf is left untouched once the frame fits"
1356813595 );
13596+
13597+ // Many escaped leaves exercise incremental size accounting; enough
13598+ // shrink to fit while untouched leaves and section order survive.
13599+ for text in ["x", "\n\"\\", "🐝\u{0}"] {
13600+ let leaf = format!(
13601+ "{}{}{}",
13602+ "x".repeat(3_000),
13603+ text.repeat(4_000),
13604+ "y".repeat(3_000)
13605+ );
13606+ let mut event = event_with_payload(
13607+ "acp_read",
13608+ serde_json::json!({"items": vec![leaf.clone(); 9], "small": "keep"}),
13609+ );
13610+ let fitted_bytes = fit_observer_event_to_budget(&mut event);
13611+ assert_eq!(fitted_bytes, serialized(&event).len());
13612+ assert!(fitted_bytes <= OBSERVER_MAX_PLAINTEXT_LEN);
13613+ let items = event.payload["items"]
13614+ .as_array()
13615+ .expect("trimmed sections survive");
13616+ assert_eq!(items.len(), 9);
13617+ assert_eq!(event.payload["small"], "keep");
13618+ let mut saw_untouched = false;
13619+ for item in items {
13620+ if item == &leaf {
13621+ saw_untouched = true;
13622+ } else {
13623+ assert!(!saw_untouched, "equal-size leaves elide in traversal order");
13624+ assert!(item.as_str().unwrap().contains("…[elided"));
13625+ }
13626+ }
13627+ }
1356913628 }
1357013629
1357113630 #[test]
@@ -13617,6 +13676,19 @@ mod observer_payload_trim_tests {
1361713676 "fell back to the stub"
1361813677 );
1361913678 assert!(event.payload.get("originalBytes").is_some());
13679+
13680+ // An escaped leaf just above its retained floor is small relative to
13681+ // a non-shrinkable numeric array. Delta accounting must include the
13682+ // replacement, or it would incorrectly accept this oversized frame.
13683+ let mut event = event_with_payload(
13684+ "acp_read",
13685+ serde_json::json!({"numbers": vec![0; 18_500], "edge": "\u{0}".repeat(6_100)}),
13686+ );
13687+ let original_payload_bytes = serialized_len(&event.payload);
13688+ let fitted_bytes = fit_observer_event_to_budget(&mut event);
13689+ assert_eq!(fitted_bytes, serialized(&event).len());
13690+ assert!(fitted_bytes <= OBSERVER_MAX_PLAINTEXT_LEN);
13691+ assert_eq!(event.payload["originalBytes"], original_payload_bytes);
1362013692 }
1362113693
1362213694 #[test]
0 commit comments