2121//! ("Codec diagnostic").
2222
2323use std:: path:: PathBuf ;
24+ use std:: sync:: Arc ;
2425use std:: sync:: LazyLock ;
2526use std:: time:: Duration ;
2627
@@ -29,9 +30,15 @@ use anyhow::bail;
2930use clap:: ValueEnum ;
3031use futures:: StreamExt ;
3132use futures:: TryStreamExt ;
33+ use parquet:: arrow:: ArrowWriter ;
3234use parquet:: arrow:: ParquetRecordBatchStreamBuilder ;
3335use parquet:: arrow:: ProjectionMask ;
36+ use parquet:: arrow:: async_reader;
37+ use parquet:: basic:: Compression ;
38+ use parquet:: basic:: ZstdLevel ;
39+ use parquet:: file:: properties:: WriterProperties ;
3440use tokio:: fs:: File ;
41+ use url:: Url ;
3542use vortex:: VortexSessionDefault ;
3643use vortex:: aggregate_fn:: fns:: uncompressed_size_in_bytes:: UncompressedSizeOpts ;
3744use vortex:: aggregate_fn:: fns:: uncompressed_size_in_bytes:: uncompressed_size_in_bytes;
@@ -50,13 +57,16 @@ use vortex_bench::IdempotentPath;
5057use vortex_bench:: datasets:: Dataset ;
5158use vortex_bench:: datasets:: data_downloads:: download_data;
5259use vortex_bench:: datasets:: tpch_l_comment:: TPCHLCommentCanonical ;
60+ use vortex_bench:: idempotent_async;
61+ use vortex_cloud:: hf:: make_hf_store_from_url;
5362use vortex_fsst:: FSST ;
5463use vortex_onpair:: Config ;
64+ use vortex_onpair:: DEFAULT_CONFIG ;
65+ use vortex_onpair:: MaxDictBits ;
5566use vortex_onpair:: OnPair ;
5667
5768mod codec;
5869pub use codec:: ColumnResult ;
59- pub use codec:: DirectCandidate ;
6070pub use codec:: bench_column;
6171
6272const CLICKBENCH_SHARD_COUNT : u32 = 100 ;
@@ -118,6 +128,52 @@ impl std::fmt::Display for StringEncoder {
118128 }
119129}
120130
131+ /// A fully configured encoder candidate, run by both suites: an encoder family
132+ /// plus the fixed configuration that path needs.
133+ pub enum Candidate {
134+ /// OnPair with a deterministic config, whose dictionary budget is the only
135+ /// thing the benchmark varies.
136+ OnPair ( Config ) ,
137+ /// FSST (no configuration).
138+ Fsst ,
139+ }
140+
141+ impl Candidate {
142+ /// OnPair with a dictionary budget of up to `2^max_dict_bits` tokens,
143+ /// reusing every other default. `max_dict_bits` must be in `9..=16`; OnPair
144+ /// stores codes as `u16`, so all of those fit.
145+ pub fn on_pair ( max_dict_bits : u8 ) -> Result < Self > {
146+ let max_dict_bits = MaxDictBits :: new ( max_dict_bits) . map_err ( |e| {
147+ anyhow:: anyhow!(
148+ "invalid maximum dictionary bit width {max_dict_bits} (want 9..=16): {e}"
149+ )
150+ } ) ?;
151+ Ok ( Self :: OnPair ( Config {
152+ max_dict_bits,
153+ ..DEFAULT_CONFIG
154+ } ) )
155+ }
156+
157+ /// The encoder family, for the post-compression type check.
158+ pub fn family ( & self ) -> StringEncoder {
159+ match self {
160+ Self :: OnPair ( _) => StringEncoder :: OnPair ,
161+ Self :: Fsst => StringEncoder :: Fsst ,
162+ }
163+ }
164+
165+ /// Stable label used in benchmark output, e.g. `onpair-12` or `fsst`.
166+ /// Changing a candidate's config renames its metrics and so restarts their
167+ /// benchmark history, which is intended: neither size nor speed is
168+ /// comparable across dictionary budgets.
169+ pub fn label ( & self ) -> String {
170+ match self {
171+ Self :: OnPair ( config) => onpair_label ( config) ,
172+ Self :: Fsst => StringEncoder :: Fsst . label ( ) . to_string ( ) ,
173+ }
174+ }
175+ }
176+
121177/// Label for an OnPair configuration, e.g. `onpair-12`. The dictionary budget is
122178/// read from the config that was actually used, so a label can never claim a
123179/// budget the measured data was not encoded with.
@@ -291,6 +347,121 @@ pub async fn load_tpch_l_comment(ctx: &mut ExecutionCtx) -> Result<StringColumn>
291347 } )
292348}
293349
350+ /// One Utf8 column of a public parquet file on the Hugging Face Hub.
351+ pub struct HfColumn {
352+ /// Output name used in measurement names and matched by `--columns`.
353+ pub name : & ' static str ,
354+ /// `hf://` URL of the parquet file, pinned to a commit or tag so the input
355+ /// cannot change under the benchmark history.
356+ pub url : & ' static str ,
357+ /// Column to read.
358+ pub column : & ' static str ,
359+ /// Keep only the first `max_rows` rows. The file is read with range
360+ /// requests, so a prefix of a large file fetches only the row groups it
361+ /// needs.
362+ pub max_rows : Option < usize > ,
363+ }
364+
365+ /// The datasets the OnPair paper (Gargiulo & Venturini, "OnPair: Short Strings
366+ /// Compression for Fast Random Access", arXiv:2508.02280) evaluates on, from
367+ /// public parquet mirrors, plus a slice of FineWeb as long-document web text.
368+ ///
369+ /// The paper's fifth dataset, OpenWebText URLs, has no parquet mirror; the
370+ /// ClickBench `URL` column covers that shape.
371+ pub const HF_COLUMNS : & [ HfColumn ] = & [
372+ // Amazon Reviews 2023 (Hou et al. 2024), Books. The paper takes the first
373+ // 500 MiB of reviews; 200k reviews is about 80 MiB.
374+ HfColumn {
375+ name : "amazon/book_reviews" ,
376+ url : "hf://datasets/cogsci13/Amazon-Reviews-2023-Books-Review@df2faf70120558bddedb83c4d798385e5036b821/data/full-00000-of-00033.parquet" ,
377+ column : "text" ,
378+ max_rows : Some ( 200_000 ) ,
379+ } ,
380+ // Amazon Reviews 2023, Books item titles: the first of 28 metadata shards.
381+ HfColumn {
382+ name : "amazon/book_titles" ,
383+ url : "hf://datasets/cogsci13/Amazon-Reviews-2023-Books-Meta@35db8e29f8681b8694da79558a5b86a381e6e260/data/full-00000-of-00028.parquet" ,
384+ column : "title" ,
385+ max_rows : None ,
386+ } ,
387+ // "A Million News Headlines" (ABC News), the whole dataset.
388+ HfColumn {
389+ name : "news/headlines" ,
390+ url : "hf://datasets/rajistics/million-headlines@refs%2Fconvert%2Fparquet/default/train/0000.parquet" ,
391+ column : "headline_text" ,
392+ max_rows : None ,
393+ } ,
394+ // Sentiment140 tweets: the 1.36M-row train split of the 1.6M tweets.
395+ HfColumn {
396+ name : "twitter/sentiment140" ,
397+ url : "hf://datasets/adilbekovich/Sentiment140Twitter@refs%2Fconvert%2Fparquet/default/train/0000.parquet" ,
398+ column : "text" ,
399+ max_rows : None ,
400+ } ,
401+ // FineWeb document text: the first 10k documents (about 30 MiB) of the
402+ // shard the FineWeb query benchmark uses.
403+ HfColumn {
404+ name : "fineweb/text" ,
405+ url : "hf://datasets/HuggingFaceFW/[email protected] /sample/10BT/001_00000.parquet" , 406+ column : "text" ,
407+ max_rows : Some ( 10_000 ) ,
408+ } ,
409+ ] ;
410+
411+ /// Reader for a remote parquet file, used only to fill the local input cache.
412+ #[ expect(
413+ deprecated,
414+ reason = "arrow-rs deprecated this in favour of a hand-rolled AsyncFileReader; a one-time, \
415+ untimed input download does not need one (arrow-rs#10308)"
416+ ) ]
417+ type ObjectReader = async_reader:: ParquetObjectReader ;
418+
419+ /// Load one Hugging Face column, fetching and caching it locally first if it
420+ /// is not already present.
421+ pub async fn load_hf_column ( source : & HfColumn , ctx : & mut ExecutionCtx ) -> Result < StringColumn > {
422+ let path = cache_hf_column ( source) . await ?;
423+ let struct_array = read_parquet_projected ( path, source. column ) . await ?;
424+ Ok ( StringColumn {
425+ name : source. name . to_string ( ) ,
426+ array : to_utf8_column ( struct_array, source. column , ctx) ?,
427+ } )
428+ }
429+
430+ /// Read `source`'s rows over HTTP and cache just that column as a local
431+ /// zstd-compressed parquet file under `vortex-bench/data/string-bench/`.
432+ ///
433+ /// The cache file is named after the column, so changing a source's URL or row
434+ /// cap needs the stale file deleted.
435+ async fn cache_hf_column ( source : & HfColumn ) -> Result < PathBuf > {
436+ let cache_path = format ! ( "string-bench/{}.parquet" , source. name. replace( '/' , "_" ) ) ;
437+ idempotent_async ( cache_path, |tmp| async move {
438+ let ( store, path) = make_hf_store_from_url ( & Url :: parse ( source. url ) ?) ?;
439+ let builder = ParquetRecordBatchStreamBuilder :: new ( ObjectReader :: new ( store, path) ) . await ?;
440+ let col_idx = builder. schema ( ) . index_of ( source. column ) ?;
441+ let mask = ProjectionMask :: roots ( builder. parquet_schema ( ) , [ col_idx] ) ;
442+ let mut builder = builder. with_projection ( mask) ;
443+ if let Some ( max_rows) = source. max_rows {
444+ builder = builder. with_limit ( max_rows) ;
445+ }
446+ let mut reader = builder. build ( ) ?;
447+
448+ let props = WriterProperties :: builder ( )
449+ . set_compression ( Compression :: ZSTD ( ZstdLevel :: try_new ( 3 ) ?) )
450+ . build ( ) ;
451+ let mut writer = ArrowWriter :: try_new (
452+ std:: fs:: File :: create ( & tmp) ?,
453+ Arc :: clone ( reader. schema ( ) ) ,
454+ Some ( props) ,
455+ ) ?;
456+ while let Some ( batch) = reader. try_next ( ) . await ? {
457+ writer. write ( & batch) ?;
458+ }
459+ writer. close ( ) ?;
460+ Ok ( ( ) )
461+ } )
462+ . await
463+ }
464+
294465/// Read a single column from a parquet file (projected at the parquet level to
295466/// avoid decoding the other columns) into a chunked struct array.
296467async fn read_parquet_projected ( path : PathBuf , column : & str ) -> Result < ArrayRef > {
0 commit comments