@@ -50,13 +50,17 @@ pub fn build_query_plan(pattern: &str, case_insensitive: bool) -> Result<QueryPl
5050}
5151
5252/// Build a query plan for a literal (fixed-string) pattern.
53+ ///
54+ /// If regex planning fails, scan all candidates rather than reject a valid
55+ /// literal. This only disables index narrowing; the matcher still verifies it.
5356pub fn build_literal_plan ( literal : & str , case_insensitive : bool ) -> QueryPlan {
54- let text = if case_insensitive {
55- literal. to_lowercase ( )
56- } else {
57- literal. to_string ( )
58- } ;
59- literals_to_query_plan ( text. as_bytes ( ) )
57+ if case_insensitive {
58+ // The index only folds ASCII. Let the regex planner break literal runs
59+ // at Unicode case classes that cannot safely use those folded trigrams.
60+ return build_query_plan ( & regex_syntax:: escape ( literal) , true )
61+ . unwrap_or ( QueryPlan :: MatchAll ) ;
62+ }
63+ literals_to_query_plan ( literal. as_bytes ( ) )
6064}
6165
6266/// Build one plan covering every pattern the user supplied.
@@ -1102,14 +1106,12 @@ mod tests {
11021106
11031107 #[ test]
11041108 fn test_case_insensitive_literal_plan ( ) {
1105- // "class AlertSchema" with case-insensitive should produce trigrams
1106- // from "class alertschema"
1107- let plan = build_literal_plan ( "class AlertSchema" , true ) ;
1109+ let plan = build_literal_plan ( "HeLLo" , true ) ;
11081110 match & plan {
11091111 QueryPlan :: And ( queries) => {
11101112 assert ! ( !queries. is_empty( ) , "should have trigrams" ) ;
11111113 // Verify these are lowercase trigrams
1112- let expected = trigram:: extract_from_literal ( "class alertschema " ) ;
1114+ let expected = trigram:: extract_from_literal ( "hello " ) ;
11131115 let hashes: Vec < TrigramHash > = queries. iter ( ) . map ( |q| q. hash ) . collect ( ) ;
11141116 for tri in & expected {
11151117 assert ! ( hashes. contains( tri) , "missing trigram {tri:#010x}" ) ;
@@ -1119,6 +1121,100 @@ mod tests {
11191121 }
11201122 }
11211123
1124+ #[ test]
1125+ fn case_insensitive_literal_plan_handles_long_literals ( ) {
1126+ let literal = "[(HeLLo.+)\u{e9} ]" . repeat ( 4096 ) ;
1127+ let content = "[(HELLO.+)\u{c9} ]" . repeat ( 4096 ) ;
1128+ let plan = build_literal_plan ( & literal, true ) ;
1129+ if plan. is_match_all ( ) {
1130+ return ;
1131+ }
1132+
1133+ let masks = trigram:: extract_merged_masks ( content. as_bytes ( ) ) ;
1134+ let candidates = execute_plan_with_masks ( & plan, & |tri| {
1135+ masks
1136+ . get ( & tri)
1137+ . map ( |mask| PostingEntry {
1138+ file_id : 0 ,
1139+ loc_mask : mask. loc_mask ,
1140+ next_mask : mask. next_mask ,
1141+ } )
1142+ . into_iter ( )
1143+ . collect ( )
1144+ } ) ;
1145+ assert_eq ! ( candidates, vec![ 0 ] , "{plan:?}" ) ;
1146+ }
1147+
1148+ #[ test]
1149+ fn case_insensitive_literals_preserve_unicode_candidates ( ) {
1150+ let cases: & [ ( & str , & [ & str ] , bool ) ] = & [
1151+ ( "caf\u{e9} " , & [ "CAF\u{c9} " , "caf\u{e9} " ] , true ) ,
1152+ ( "CAF\u{c9} " , & [ "CAF\u{c9} " , "caf\u{e9} " ] , true ) ,
1153+ (
1154+ "\u{ff49} \u{ff44} " ,
1155+ & [ "\u{ff29} \u{ff24} " , "\u{ff49} \u{ff44} " ] ,
1156+ false ,
1157+ ) ,
1158+ (
1159+ "\u{ff29} \u{ff24} " ,
1160+ & [ "\u{ff29} \u{ff24} " , "\u{ff49} \u{ff44} " ] ,
1161+ false ,
1162+ ) ,
1163+ (
1164+ "shellshock" ,
1165+ & [ "SHELLSHOCK" , "\u{17f} hell\u{17f} hoc\u{212a} " ] ,
1166+ true ,
1167+ ) ,
1168+ ( "sks" , & [ "SKS" , "\u{17f} \u{212a} \u{17f} " ] , false ) ,
1169+ (
1170+ "prefix\u{e9} suffix" ,
1171+ & [ "PREFIX\u{c9} SUFFIX" , "prefix\u{e9} suffix" ] ,
1172+ true ,
1173+ ) ,
1174+ ( "a\u{130} bc" , & [ "A\u{130} BC" ] , true ) ,
1175+ ( r"hello[.+](?i)\end" , & [ r"HELLO[.+](?i)\END" ] , true ) ,
1176+ ( "" , & [ "" ] , false ) ,
1177+ ( "ab" , & [ "AB" ] , false ) ,
1178+ ] ;
1179+ for & ( literal, matches, narrows) in cases {
1180+ let mut inverted = std:: collections:: HashMap :: < u32 , Vec < PostingEntry > > :: new ( ) ;
1181+ for ( id, content) in matches. iter ( ) . chain ( [ & "unrelated decoy" ] ) . enumerate ( ) {
1182+ for ( tri, masks) in trigram:: extract_merged_masks ( content. as_bytes ( ) ) {
1183+ inverted. entry ( tri) . or_default ( ) . push ( PostingEntry {
1184+ file_id : id as u32 ,
1185+ loc_mask : masks. loc_mask ,
1186+ next_mask : masks. next_mask ,
1187+ } ) ;
1188+ }
1189+ }
1190+ let plan = build_literal_plan ( literal, true ) ;
1191+ assert_eq ! ( !plan. is_match_all( ) , narrows, "{literal:?}: {plan:?}" ) ;
1192+ let matcher = regex:: RegexBuilder :: new ( & regex_syntax:: escape ( literal) )
1193+ . case_insensitive ( true )
1194+ . build ( )
1195+ . unwrap ( ) ;
1196+ for content in matches {
1197+ assert ! ( matcher. is_match( content) , "{literal:?}: {content:?}" ) ;
1198+ }
1199+ if plan. is_match_all ( ) {
1200+ continue ;
1201+ }
1202+ let lookup = |tri| inverted. get ( & tri) . cloned ( ) . unwrap_or_default ( ) ;
1203+ for candidates in [
1204+ execute_plan ( & plan, & |tri| {
1205+ lookup ( tri) . iter ( ) . map ( |entry| entry. file_id ) . collect ( )
1206+ } ) ,
1207+ execute_plan_with_masks ( & plan, & lookup) ,
1208+ ] {
1209+ assert_eq ! (
1210+ candidates,
1211+ ( 0 ..matches. len( ) as u32 ) . collect:: <Vec <_>>( ) ,
1212+ "{literal:?}: {plan:?}"
1213+ ) ;
1214+ }
1215+ }
1216+ }
1217+
11221218 #[ test]
11231219 fn test_case_insensitive_regex_plan ( ) {
11241220 // Unicode case folds of S include non-ASCII bytes, so only the
0 commit comments