File tree Expand file tree Collapse file tree
vortex-array/src/builders/dict
vortex-layout/src/layouts/dict Expand file tree Collapse file tree Original file line number Diff line number Diff line change @@ -30,6 +30,11 @@ use crate::dtype::PType;
3030use crate :: dtype:: UnsignedPType ;
3131use crate :: validity:: Validity ;
3232
33+ /// Creates an encoder with a fixed code width based on the value domain and dictionary length limit.
34+ ///
35+ /// The bound includes a possible null entry, so nullable values may require wider codes even when
36+ /// few distinct values occur. [`super::dict_encode_with_constraints`] narrows the codes after
37+ /// encoding; callers using this encoder directly retain the selected width.
3338pub fn primitive_dict_builder < T : NativePType > (
3439 nullability : Nullability ,
3540 constraints : & DictConstraints ,
Original file line number Diff line number Diff line change @@ -70,9 +70,9 @@ pub struct DictLayoutConstraints {
7070 /// Maximum dictionary length. Limited to `u16` because dictionaries with more than 64k unique
7171 /// values provide diminishing compression returns given typical chunk sizes (~8k elements).
7272 ///
73- /// The codes dtype is determined upfront from this constraint:
74- /// - [`PType::U8`] when max_len <= 255
75- /// - [`PType::U16 `] when max_len > 255
73+ /// The codes dtype is chosen upfront and is at most [`PType::U16`]. Primitive dictionaries
74+ /// use [`PType::U8`] when ` max_len <= 256` or the value domain fits in 256 entries including
75+ /// any null entry. String and binary dictionaries use [`PType::U8 `] when ` max_len <= 255`.
7676 ///
7777 /// Vortex encoders must always produce unsigned integer codes; signed codes are only accepted for external compatibility.
7878 pub max_len : u16 ,
You can’t perform that action at this time.
0 commit comments