@@ -15,10 +15,7 @@ use std::{
1515use jiff:: tz:: TimeZone as JiffTimeZone ;
1616use memchr:: memmem;
1717use rustc_hash:: FxHasher ;
18- use serde:: {
19- Deserialize ,
20- de:: { DeserializeOwned , Error as _, MapAccess , SeqAccess , Visitor } ,
21- } ;
18+ use serde:: { Deserialize , de:: DeserializeOwned } ;
2219use smallvec:: SmallVec ;
2320
2421use crate :: {
@@ -824,17 +821,18 @@ fn has_unsupported_null_field(line: &[u8]) -> bool {
824821}
825822
826823/// Deserializes a transcript line while enforcing Claude's nullable-field contract.
827- /// Lines that may hold an unsupported `null` member are parsed into a value once so
828- /// structural validation and typed deserialization can share the same parse.
824+ ///
825+ /// Every other rejection, such as a duplicate known field, comes from the typed parse
826+ /// of the raw line, so a line is loaded or skipped the same way whether or not it
827+ /// also needed the null check.
829828pub ( crate ) fn deserialize_usage_line < T : DeserializeOwned > ( line : & [ u8 ] ) -> Option < T > {
830- if !may_have_unsupported_null_field ( line) {
831- return serde_json:: from_slice ( line) . ok ( ) ;
832- }
833- let root = serde_json:: from_slice :: < UniqueJsonValue > ( line) . ok ( ) ?. 0 ;
834- if has_unsupported_null_field_in_value ( & root) {
835- return None ;
829+ if may_have_unsupported_null_field ( line) {
830+ let root = serde_json:: from_slice :: < serde_json:: Value > ( line) . ok ( ) ?;
831+ if has_unsupported_null_field_in_value ( & root) {
832+ return None ;
833+ }
836834 }
837- serde_json:: from_value ( root ) . ok ( )
835+ serde_json:: from_slice ( line ) . ok ( )
838836}
839837
840838/// Whether a `null` in `line` could be the value of a member that
@@ -869,100 +867,6 @@ fn trim_json_whitespace_end(bytes: &[u8]) -> &[u8] {
869867 & bytes[ ..end]
870868}
871869
872- /// A JSON value parser that preserves typed Serde's rejection of duplicate members.
873- /// Parsing through `serde_json::Value` normally keeps only the final duplicate, which could
874- /// make a malformed transcript valid before the in-memory typed deserialization runs.
875- struct UniqueJsonValue ( serde_json:: Value ) ;
876-
877- impl < ' de > Deserialize < ' de > for UniqueJsonValue {
878- fn deserialize < D > ( deserializer : D ) -> std:: result:: Result < Self , D :: Error >
879- where
880- D : serde:: Deserializer < ' de > ,
881- {
882- deserializer. deserialize_any ( UniqueJsonValueVisitor )
883- }
884- }
885-
886- struct UniqueJsonValueVisitor ;
887-
888- impl < ' de > Visitor < ' de > for UniqueJsonValueVisitor {
889- type Value = UniqueJsonValue ;
890-
891- fn expecting ( & self , formatter : & mut std:: fmt:: Formatter < ' _ > ) -> std:: fmt:: Result {
892- formatter. write_str ( "a JSON value without duplicate object members" )
893- }
894-
895- fn visit_bool < E > ( self , value : bool ) -> std:: result:: Result < Self :: Value , E > {
896- Ok ( UniqueJsonValue ( serde_json:: Value :: Bool ( value) ) )
897- }
898-
899- fn visit_i64 < E > ( self , value : i64 ) -> std:: result:: Result < Self :: Value , E > {
900- Ok ( UniqueJsonValue ( serde_json:: Value :: Number ( value. into ( ) ) ) )
901- }
902-
903- fn visit_u64 < E > ( self , value : u64 ) -> std:: result:: Result < Self :: Value , E > {
904- Ok ( UniqueJsonValue ( serde_json:: Value :: Number ( value. into ( ) ) ) )
905- }
906-
907- fn visit_f64 < E : serde:: de:: Error > ( self , value : f64 ) -> std:: result:: Result < Self :: Value , E > {
908- serde_json:: Number :: from_f64 ( value)
909- . map ( serde_json:: Value :: Number )
910- . map ( UniqueJsonValue )
911- . ok_or_else ( || E :: custom ( "non-finite JSON number" ) )
912- }
913-
914- fn visit_str < E > ( self , value : & str ) -> std:: result:: Result < Self :: Value , E > {
915- Ok ( UniqueJsonValue ( serde_json:: Value :: String ( value. to_owned ( ) ) ) )
916- }
917-
918- fn visit_string < E > ( self , value : String ) -> std:: result:: Result < Self :: Value , E > {
919- Ok ( UniqueJsonValue ( serde_json:: Value :: String ( value) ) )
920- }
921-
922- fn visit_none < E > ( self ) -> std:: result:: Result < Self :: Value , E > {
923- Ok ( UniqueJsonValue ( serde_json:: Value :: Null ) )
924- }
925-
926- fn visit_unit < E > ( self ) -> std:: result:: Result < Self :: Value , E > {
927- Ok ( UniqueJsonValue ( serde_json:: Value :: Null ) )
928- }
929-
930- fn visit_some < D > ( self , deserializer : D ) -> std:: result:: Result < Self :: Value , D :: Error >
931- where
932- D : serde:: Deserializer < ' de > ,
933- {
934- UniqueJsonValue :: deserialize ( deserializer)
935- }
936-
937- fn visit_seq < A > ( self , mut sequence : A ) -> std:: result:: Result < Self :: Value , A :: Error >
938- where
939- A : SeqAccess < ' de > ,
940- {
941- let mut values = Vec :: new ( ) ;
942- while let Some ( value) = sequence. next_element :: < UniqueJsonValue > ( ) ? {
943- values. push ( value. 0 ) ;
944- }
945- Ok ( UniqueJsonValue ( serde_json:: Value :: Array ( values) ) )
946- }
947-
948- fn visit_map < A > ( self , mut object : A ) -> std:: result:: Result < Self :: Value , A :: Error >
949- where
950- A : MapAccess < ' de > ,
951- {
952- let mut fields = serde_json:: Map :: new ( ) ;
953- while let Some ( field) = object. next_key :: < String > ( ) ? {
954- if fields. contains_key ( & field) {
955- return Err ( A :: Error :: custom ( format ! (
956- "duplicate object member `{field}`"
957- ) ) ) ;
958- }
959- let value = object. next_value :: < UniqueJsonValue > ( ) ?;
960- fields. insert ( field, value. 0 ) ;
961- }
962- Ok ( UniqueJsonValue ( serde_json:: Value :: Object ( fields) ) )
963- }
964- }
965-
966870fn has_unsupported_null_field_in_value ( root : & serde_json:: Value ) -> bool {
967871 let iteration_arrays = [
968872 root. pointer ( "/message/usage/iterations" ) ,
@@ -1291,12 +1195,36 @@ null,"usage":{"input_tokens":1}}}"#
12911195 ) ) ;
12921196 }
12931197
1198+ /// Duplicate handling is the typed parse's alone, so it must not depend on whether
1199+ /// the line also carries an allowed or exempted `null`.
12941200 #[ test]
1295- fn rejects_duplicate_members_on_lines_with_only_unguarded_nulls ( ) {
1296- assert ! ( deserialize_usage_line:: <UsageEntry >(
1297- br#"{"type":"assistant","timestamp":"2026-09-12T04:38:42.296Z","sessionId":"session-a","message":{"id":"msg_1","model":"first","model":"second","stop_sequence":null,"usage":{"input_tokens":2,"output_tokens":289}}}"#
1298- )
1299- . is_none( ) ) ;
1201+ fn handles_duplicate_members_the_same_with_or_without_nulls ( ) {
1202+ for nulls in [
1203+ "" ,
1204+ r#","stop_sequence":null"# ,
1205+ r#","iterations":[{"type":"message","model":null}]"# ,
1206+ ] {
1207+ let ( message_nulls, usage_nulls) = if nulls. contains ( "iterations" ) {
1208+ ( "" , nulls)
1209+ } else {
1210+ ( nulls, "" )
1211+ } ;
1212+ let duplicate_known = format ! (
1213+ r#"{{"timestamp":"2026-09-12T04:38:42.296Z","message":{{"id":"msg_1","model":"first","model":"second"{message_nulls},"usage":{{"input_tokens":2,"output_tokens":1{usage_nulls}}}}}}}"#
1214+ ) ;
1215+ let duplicate_unknown = format ! (
1216+ r#"{{"timestamp":"2026-09-12T04:38:42.296Z","extra":1,"extra":2,"message":{{"id":"msg_1","model":"m"{message_nulls},"usage":{{"input_tokens":2,"output_tokens":1{usage_nulls}}}}}}}"#
1217+ ) ;
1218+
1219+ assert ! (
1220+ deserialize_usage_line:: <UsageEntry >( duplicate_known. as_bytes( ) ) . is_none( ) ,
1221+ "{duplicate_known}"
1222+ ) ;
1223+ assert ! (
1224+ deserialize_usage_line:: <UsageEntry >( duplicate_unknown. as_bytes( ) ) . is_some( ) ,
1225+ "{duplicate_unknown}"
1226+ ) ;
1227+ }
13001228 }
13011229
13021230 #[ test]
0 commit comments