|
1 | | -use memchr::memchr; |
| 1 | +use memchr::{memchr, memmem::Finder}; |
2 | 2 | use smallvec::SmallVec; |
3 | 3 |
|
4 | 4 | pub(crate) type FxHashMap<K, V> = rustc_hash::FxHashMap<K, V>; |
5 | 5 | pub(crate) type FxHashSet<T> = rustc_hash::FxHashSet<T>; |
6 | 6 | pub(crate) type SmallIndexVec = SmallVec<[usize; 1]>; |
7 | 7 |
|
| 8 | +/// Whether a [`LinePrefilter`] requires every marker or just one of them. |
| 9 | +#[derive(Clone, Copy)] |
| 10 | +enum PrefilterMode { |
| 11 | + /// The line must contain every configured marker. |
| 12 | + All, |
| 13 | + /// The line must contain at least one configured marker. |
| 14 | + Any, |
| 15 | +} |
| 16 | + |
| 17 | +/// Reusable byte-substring prefilter for newline-delimited JSON logs. |
| 18 | +/// |
| 19 | +/// JSONL adapters skip lines that cannot contain a usage record before paying |
| 20 | +/// for a full `serde_json` parse. Building the [`Finder`] needles once and |
| 21 | +/// reusing them across every line keeps that skip check on the SIMD-accelerated |
| 22 | +/// `memmem` path instead of allocating a fresh searcher per `str::contains` |
| 23 | +/// call. |
| 24 | +pub(crate) struct LinePrefilter { |
| 25 | + finders: SmallVec<[Finder<'static>; 4]>, |
| 26 | + mode: PrefilterMode, |
| 27 | +} |
| 28 | + |
| 29 | +impl LinePrefilter { |
| 30 | + /// Build a prefilter that only admits lines containing *all* `markers`. |
| 31 | + /// |
| 32 | + /// # Example |
| 33 | + /// |
| 34 | + /// ```ignore |
| 35 | + /// let prefilter = LinePrefilter::all(&[b"\"usage\"", b"\"message\""]); |
| 36 | + /// assert!(prefilter.matches(br#"{"message":{"usage":{}}}"#)); |
| 37 | + /// assert!(!prefilter.matches(br#"{"message":{}}"#)); |
| 38 | + /// ``` |
| 39 | + pub(crate) fn all(markers: &[&[u8]]) -> Self { |
| 40 | + Self::new(markers, PrefilterMode::All) |
| 41 | + } |
| 42 | + |
| 43 | + /// Build a prefilter that admits lines containing *any* of `markers`. |
| 44 | + pub(crate) fn any(markers: &[&[u8]]) -> Self { |
| 45 | + Self::new(markers, PrefilterMode::Any) |
| 46 | + } |
| 47 | + |
| 48 | + fn new(markers: &[&[u8]], mode: PrefilterMode) -> Self { |
| 49 | + // `Finder::new` borrows the needle, so take an owned copy to outlive the |
| 50 | + // caller's marker slice and allow the prefilter to be stored freely. |
| 51 | + let finders = markers |
| 52 | + .iter() |
| 53 | + .map(|marker| Finder::new(marker).into_owned()) |
| 54 | + .collect(); |
| 55 | + Self { finders, mode } |
| 56 | + } |
| 57 | + |
| 58 | + /// Return `true` when `line` passes the filter and is worth parsing. |
| 59 | + pub(crate) fn matches(&self, line: &[u8]) -> bool { |
| 60 | + match self.mode { |
| 61 | + PrefilterMode::All => self |
| 62 | + .finders |
| 63 | + .iter() |
| 64 | + .all(|finder| finder.find(line).is_some()), |
| 65 | + PrefilterMode::Any => self |
| 66 | + .finders |
| 67 | + .iter() |
| 68 | + .any(|finder| finder.find(line).is_some()), |
| 69 | + } |
| 70 | + } |
| 71 | +} |
| 72 | + |
8 | 73 | pub(crate) struct ByteLines<'a> { |
9 | 74 | bytes: &'a [u8], |
10 | 75 | } |
@@ -47,7 +112,25 @@ pub(crate) fn suffix_string(value: &str, suffix: &str) -> String { |
47 | 112 |
|
48 | 113 | #[cfg(test)] |
49 | 114 | mod tests { |
50 | | - use super::{byte_lines, suffix_string}; |
| 115 | + use super::{LinePrefilter, byte_lines, suffix_string}; |
| 116 | + |
| 117 | + #[test] |
| 118 | + fn line_prefilter_all_requires_every_marker() { |
| 119 | + let prefilter = LinePrefilter::all(&[b"\"usage\"", b"\"message\""]); |
| 120 | + |
| 121 | + assert!(prefilter.matches(br#"{"message":{"usage":{"input":1}}}"#)); |
| 122 | + assert!(!prefilter.matches(br#"{"message":{"role":"user"}}"#)); |
| 123 | + assert!(!prefilter.matches(br#"{"usage":{"input":1}}"#)); |
| 124 | + } |
| 125 | + |
| 126 | + #[test] |
| 127 | + fn line_prefilter_any_requires_one_marker() { |
| 128 | + let prefilter = LinePrefilter::any(&[b"\"model_change\"", b"\"usage\""]); |
| 129 | + |
| 130 | + assert!(prefilter.matches(br#"{"type":"model_change"}"#)); |
| 131 | + assert!(prefilter.matches(br#"{"message":{"usage":{}}}"#)); |
| 132 | + assert!(!prefilter.matches(br#"{"type":"message"}"#)); |
| 133 | + } |
51 | 134 |
|
52 | 135 | #[test] |
53 | 136 | fn byte_lines_returns_newline_delimited_slices() { |
|
0 commit comments