Skip to content

Commit f4e751b

Browse files
committed
fix(claude): decide duplicate members by the typed parse on every line
Lines that needed the nullable-field check were parsed into a UniqueJsonValue tree that rejected any duplicate member, while every other line went straight to the typed parse, which rejects only duplicate known fields. On main the split was "contains null or not"; after the pre-check change it became "contains a guarded null or not". Either way, whether a line with a duplicate unknown member loaded depended on an unrelated null. #1719 added UniqueJsonValue to keep typed Serde's duplicate-field rejection when deserializing from the shared Value, not to reject more. The null check now only validates a last-wins Value (as JSON.parse does in the TypeScript loader it mirrors), and the typed parse of the raw line decides everything else for every line. The duplicate-checking visitor is gone; the second parse only happens on the ~0.1% of lines with a guarded null.
1 parent 87e7d1c commit f4e751b

1 file changed

Lines changed: 40 additions & 112 deletions

File tree

  • rust/adapters/claude/src

‎rust/adapters/claude/src/lib.rs‎

Lines changed: 40 additions & 112 deletions
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,7 @@ use std::{
1515
use jiff::tz::TimeZone as JiffTimeZone;
1616
use memchr::memmem;
1717
use rustc_hash::FxHasher;
18-
use serde::{
19-
Deserialize,
20-
de::{DeserializeOwned, Error as _, MapAccess, SeqAccess, Visitor},
21-
};
18+
use serde::{Deserialize, de::DeserializeOwned};
2219
use smallvec::SmallVec;
2320

2421
use crate::{
@@ -824,17 +821,18 @@ fn has_unsupported_null_field(line: &[u8]) -> bool {
824821
}
825822

826823
/// Deserializes a transcript line while enforcing Claude's nullable-field contract.
827-
/// Lines that may hold an unsupported `null` member are parsed into a value once so
828-
/// structural validation and typed deserialization can share the same parse.
824+
///
825+
/// Every other rejection, such as a duplicate known field, comes from the typed parse
826+
/// of the raw line, so a line is loaded or skipped the same way whether or not it
827+
/// also needed the null check.
829828
pub(crate) fn deserialize_usage_line<T: DeserializeOwned>(line: &[u8]) -> Option<T> {
830-
if !may_have_unsupported_null_field(line) {
831-
return serde_json::from_slice(line).ok();
832-
}
833-
let root = serde_json::from_slice::<UniqueJsonValue>(line).ok()?.0;
834-
if has_unsupported_null_field_in_value(&root) {
835-
return None;
829+
if may_have_unsupported_null_field(line) {
830+
let root = serde_json::from_slice::<serde_json::Value>(line).ok()?;
831+
if has_unsupported_null_field_in_value(&root) {
832+
return None;
833+
}
836834
}
837-
serde_json::from_value(root).ok()
835+
serde_json::from_slice(line).ok()
838836
}
839837

840838
/// Whether a `null` in `line` could be the value of a member that
@@ -869,100 +867,6 @@ fn trim_json_whitespace_end(bytes: &[u8]) -> &[u8] {
869867
&bytes[..end]
870868
}
871869

872-
/// A JSON value parser that preserves typed Serde's rejection of duplicate members.
873-
/// Parsing through `serde_json::Value` normally keeps only the final duplicate, which could
874-
/// make a malformed transcript valid before the in-memory typed deserialization runs.
875-
struct UniqueJsonValue(serde_json::Value);
876-
877-
impl<'de> Deserialize<'de> for UniqueJsonValue {
878-
fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
879-
where
880-
D: serde::Deserializer<'de>,
881-
{
882-
deserializer.deserialize_any(UniqueJsonValueVisitor)
883-
}
884-
}
885-
886-
struct UniqueJsonValueVisitor;
887-
888-
impl<'de> Visitor<'de> for UniqueJsonValueVisitor {
889-
type Value = UniqueJsonValue;
890-
891-
fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
892-
formatter.write_str("a JSON value without duplicate object members")
893-
}
894-
895-
fn visit_bool<E>(self, value: bool) -> std::result::Result<Self::Value, E> {
896-
Ok(UniqueJsonValue(serde_json::Value::Bool(value)))
897-
}
898-
899-
fn visit_i64<E>(self, value: i64) -> std::result::Result<Self::Value, E> {
900-
Ok(UniqueJsonValue(serde_json::Value::Number(value.into())))
901-
}
902-
903-
fn visit_u64<E>(self, value: u64) -> std::result::Result<Self::Value, E> {
904-
Ok(UniqueJsonValue(serde_json::Value::Number(value.into())))
905-
}
906-
907-
fn visit_f64<E: serde::de::Error>(self, value: f64) -> std::result::Result<Self::Value, E> {
908-
serde_json::Number::from_f64(value)
909-
.map(serde_json::Value::Number)
910-
.map(UniqueJsonValue)
911-
.ok_or_else(|| E::custom("non-finite JSON number"))
912-
}
913-
914-
fn visit_str<E>(self, value: &str) -> std::result::Result<Self::Value, E> {
915-
Ok(UniqueJsonValue(serde_json::Value::String(value.to_owned())))
916-
}
917-
918-
fn visit_string<E>(self, value: String) -> std::result::Result<Self::Value, E> {
919-
Ok(UniqueJsonValue(serde_json::Value::String(value)))
920-
}
921-
922-
fn visit_none<E>(self) -> std::result::Result<Self::Value, E> {
923-
Ok(UniqueJsonValue(serde_json::Value::Null))
924-
}
925-
926-
fn visit_unit<E>(self) -> std::result::Result<Self::Value, E> {
927-
Ok(UniqueJsonValue(serde_json::Value::Null))
928-
}
929-
930-
fn visit_some<D>(self, deserializer: D) -> std::result::Result<Self::Value, D::Error>
931-
where
932-
D: serde::Deserializer<'de>,
933-
{
934-
UniqueJsonValue::deserialize(deserializer)
935-
}
936-
937-
fn visit_seq<A>(self, mut sequence: A) -> std::result::Result<Self::Value, A::Error>
938-
where
939-
A: SeqAccess<'de>,
940-
{
941-
let mut values = Vec::new();
942-
while let Some(value) = sequence.next_element::<UniqueJsonValue>()? {
943-
values.push(value.0);
944-
}
945-
Ok(UniqueJsonValue(serde_json::Value::Array(values)))
946-
}
947-
948-
fn visit_map<A>(self, mut object: A) -> std::result::Result<Self::Value, A::Error>
949-
where
950-
A: MapAccess<'de>,
951-
{
952-
let mut fields = serde_json::Map::new();
953-
while let Some(field) = object.next_key::<String>()? {
954-
if fields.contains_key(&field) {
955-
return Err(A::Error::custom(format!(
956-
"duplicate object member `{field}`"
957-
)));
958-
}
959-
let value = object.next_value::<UniqueJsonValue>()?;
960-
fields.insert(field, value.0);
961-
}
962-
Ok(UniqueJsonValue(serde_json::Value::Object(fields)))
963-
}
964-
}
965-
966870
fn has_unsupported_null_field_in_value(root: &serde_json::Value) -> bool {
967871
let iteration_arrays = [
968872
root.pointer("/message/usage/iterations"),
@@ -1291,12 +1195,36 @@ null,"usage":{"input_tokens":1}}}"#
12911195
));
12921196
}
12931197

1198+
/// Duplicate handling is the typed parse's alone, so it must not depend on whether
1199+
/// the line also carries an allowed or exempted `null`.
12941200
#[test]
1295-
fn rejects_duplicate_members_on_lines_with_only_unguarded_nulls() {
1296-
assert!(deserialize_usage_line::<UsageEntry>(
1297-
br#"{"type":"assistant","timestamp":"2026-09-12T04:38:42.296Z","sessionId":"session-a","message":{"id":"msg_1","model":"first","model":"second","stop_sequence":null,"usage":{"input_tokens":2,"output_tokens":289}}}"#
1298-
)
1299-
.is_none());
1201+
fn handles_duplicate_members_the_same_with_or_without_nulls() {
1202+
for nulls in [
1203+
"",
1204+
r#","stop_sequence":null"#,
1205+
r#","iterations":[{"type":"message","model":null}]"#,
1206+
] {
1207+
let (message_nulls, usage_nulls) = if nulls.contains("iterations") {
1208+
("", nulls)
1209+
} else {
1210+
(nulls, "")
1211+
};
1212+
let duplicate_known = format!(
1213+
r#"{{"timestamp":"2026-09-12T04:38:42.296Z","message":{{"id":"msg_1","model":"first","model":"second"{message_nulls},"usage":{{"input_tokens":2,"output_tokens":1{usage_nulls}}}}}}}"#
1214+
);
1215+
let duplicate_unknown = format!(
1216+
r#"{{"timestamp":"2026-09-12T04:38:42.296Z","extra":1,"extra":2,"message":{{"id":"msg_1","model":"m"{message_nulls},"usage":{{"input_tokens":2,"output_tokens":1{usage_nulls}}}}}}}"#
1217+
);
1218+
1219+
assert!(
1220+
deserialize_usage_line::<UsageEntry>(duplicate_known.as_bytes()).is_none(),
1221+
"{duplicate_known}"
1222+
);
1223+
assert!(
1224+
deserialize_usage_line::<UsageEntry>(duplicate_unknown.as_bytes()).is_some(),
1225+
"{duplicate_unknown}"
1226+
);
1227+
}
13001228
}
13011229

13021230
#[test]

0 commit comments

Comments
 (0)