@@ -12,7 +12,6 @@ use polars_utils::abs_diff::AbsDiff;
1212use polars_utils:: hashing:: { DirtyHash , hash_to_partition} ;
1313use polars_utils:: nulls:: IsNull ;
1414use polars_utils:: total_ord:: { ToTotalOrd , TotalEq , TotalHash } ;
15- use rayon:: prelude:: * ;
1615
1716use super :: * ;
1817use crate :: frame:: join:: { prepare_binary, prepare_keys_multiple} ;
@@ -101,64 +100,56 @@ where
101100 let split_by_right = split_and_flatten ( by_right, n_threads) ;
102101 let offsets = compute_len_offsets ( split_by_left. iter ( ) . map ( |s| s. len ( ) ) ) ;
103102
104- let right_slices = split_by_right
103+ let right_arrays = split_by_right
105104 . iter ( )
106105 . map ( |ca| {
107106 assert_eq ! ( ca. chunks( ) . len( ) , 1 ) ;
108- ca. downcast_iter ( )
109- . next ( )
110- . unwrap ( )
111- . iter ( )
112- . map ( |v| v. copied ( ) )
107+ ca. downcast_iter ( ) . next ( ) . unwrap ( )
113108 } )
114- . collect ( ) ;
115- let hash_tbls = build_tables ( right_slices , false ) ;
109+ . collect :: < Vec < _ > > ( ) ;
110+ let hash_tbls = build_tables_from_arrays :: < S > ( & right_arrays , false ) ;
116111 let n_tables = hash_tbls. len ( ) ;
117112
118113 // Now we probe the right hand side for each left hand side.
119- let out = split_by_left
120- . into_par_iter ( )
121- . zip ( offsets)
122- . map ( |( by_left, offset) | {
123- let mut results = Vec :: with_capacity ( by_left. len ( ) ) ;
124- let mut group_states: PlHashMap < IdxSize , A > =
125- PlHashMap :: with_capacity ( _HASHMAP_INIT_SIZE) ;
126-
127- assert_eq ! ( by_left. chunks( ) . len( ) , 1 ) ;
128- let by_left_chunk = by_left. downcast_iter ( ) . next ( ) . unwrap ( ) ;
129- for ( rel_idx_left, opt_by_left_k) in by_left_chunk. iter ( ) . enumerate ( ) {
130- let Some ( by_left_k) = opt_by_left_k else {
131- results. push ( NullableIdxSize :: null ( ) ) ;
132- continue ;
133- } ;
134- let by_left_k = Some ( * by_left_k) . to_total_ord ( ) ;
135- let idx_left = ( rel_idx_left + offset) as IdxSize ;
136- let Some ( left_val) = left_val_arr. get ( idx_left as usize ) else {
137- results. push ( NullableIdxSize :: null ( ) ) ;
138- continue ;
139- } ;
140-
141- let group_probe_table = unsafe {
142- hash_tbls. get_unchecked ( hash_to_partition ( by_left_k. dirty_hash ( ) , n_tables) )
143- } ;
144- let Some ( right_grp_idxs) = group_probe_table. get ( & by_left_k) else {
145- results. push ( NullableIdxSize :: null ( ) ) ;
146- continue ;
147- } ;
148- let id = asof_in_group :: < T , A , & F > (
149- left_val,
150- right_val_arr,
151- right_grp_idxs. as_slice ( ) ,
152- & mut group_states,
153- & filter,
154- allow_eq,
155- ) ;
156- results. push ( materialize_nullable ( id) ) ;
157- }
158- results
159- } ) ;
160-
161- let bufs = RAYON . install ( || out. collect :: < Vec < _ > > ( ) ) ;
114+ let bufs = par_map_collect ( split_by_left. len ( ) , & |part_idx| {
115+ let by_left = & split_by_left[ part_idx] ;
116+ let offset = offsets[ part_idx] ;
117+ let mut results = Vec :: with_capacity ( by_left. len ( ) ) ;
118+ let mut group_states: PlHashMap < IdxSize , A > = PlHashMap :: with_capacity ( _HASHMAP_INIT_SIZE) ;
119+
120+ assert_eq ! ( by_left. chunks( ) . len( ) , 1 ) ;
121+ let by_left_chunk = by_left. downcast_iter ( ) . next ( ) . unwrap ( ) ;
122+ for ( rel_idx_left, opt_by_left_k) in by_left_chunk. iter ( ) . enumerate ( ) {
123+ let Some ( by_left_k) = opt_by_left_k else {
124+ results. push ( NullableIdxSize :: null ( ) ) ;
125+ continue ;
126+ } ;
127+ let by_left_k = Some ( * by_left_k) . to_total_ord ( ) ;
128+ let idx_left = ( rel_idx_left + offset) as IdxSize ;
129+ let Some ( left_val) = left_val_arr. get ( idx_left as usize ) else {
130+ results. push ( NullableIdxSize :: null ( ) ) ;
131+ continue ;
132+ } ;
133+
134+ let group_probe_table = unsafe {
135+ hash_tbls. get_unchecked ( hash_to_partition ( by_left_k. dirty_hash ( ) , n_tables) )
136+ } ;
137+ let Some ( right_grp_idxs) = group_probe_table. get ( & by_left_k) else {
138+ results. push ( NullableIdxSize :: null ( ) ) ;
139+ continue ;
140+ } ;
141+ let id = asof_in_group :: < T , A , & F > (
142+ left_val,
143+ right_val_arr,
144+ right_grp_idxs. as_slice ( ) ,
145+ & mut group_states,
146+ & filter,
147+ allow_eq,
148+ ) ;
149+ results. push ( materialize_nullable ( id) ) ;
150+ }
151+ results
152+ } ) ;
162153 Ok ( flatten_nullable ( & bufs) )
163154}
164155
@@ -187,41 +178,39 @@ where
187178 let n_tables = hash_tbls. len ( ) ;
188179
189180 // Now we probe the right hand side for each left hand side.
190- let iter = prep_by_left
191- . into_par_iter ( )
192- . zip ( offsets)
193- . map ( |( by_left, offset) | {
194- let mut results = Vec :: with_capacity ( by_left. len ( ) ) ;
195- let mut group_states: PlHashMap < _ , A > = PlHashMap :: with_capacity ( _HASHMAP_INIT_SIZE) ;
196-
197- for ( rel_idx_left, by_left_k) in by_left. iter ( ) . enumerate ( ) {
198- let idx_left = ( rel_idx_left + offset) as IdxSize ;
199- let Some ( left_val) = left_val_arr. get ( idx_left as usize ) else {
200- results. push ( NullableIdxSize :: null ( ) ) ;
201- continue ;
202- } ;
181+ let bufs = par_map_collect ( prep_by_left. len ( ) , & |part_idx| {
182+ let by_left = & prep_by_left[ part_idx] ;
183+ let offset = offsets[ part_idx] ;
184+ let mut results = Vec :: with_capacity ( by_left. len ( ) ) ;
185+ let mut group_states: PlHashMap < _ , A > = PlHashMap :: with_capacity ( _HASHMAP_INIT_SIZE) ;
186+
187+ for ( rel_idx_left, by_left_k) in by_left. iter ( ) . enumerate ( ) {
188+ let idx_left = ( rel_idx_left + offset) as IdxSize ;
189+ let Some ( left_val) = left_val_arr. get ( idx_left as usize ) else {
190+ results. push ( NullableIdxSize :: null ( ) ) ;
191+ continue ;
192+ } ;
193+
194+ let group_probe_table = unsafe {
195+ hash_tbls. get_unchecked ( hash_to_partition ( by_left_k. dirty_hash ( ) , n_tables) )
196+ } ;
197+ let Some ( right_grp_idxs) = group_probe_table. get ( by_left_k) else {
198+ results. push ( NullableIdxSize :: null ( ) ) ;
199+ continue ;
200+ } ;
201+ let id = asof_in_group :: < T , A , & F > (
202+ left_val,
203+ right_val_arr,
204+ right_grp_idxs. as_slice ( ) ,
205+ & mut group_states,
206+ & filter,
207+ allow_eq,
208+ ) ;
203209
204- let group_probe_table = unsafe {
205- hash_tbls. get_unchecked ( hash_to_partition ( by_left_k. dirty_hash ( ) , n_tables) )
206- } ;
207- let Some ( right_grp_idxs) = group_probe_table. get ( by_left_k) else {
208- results. push ( NullableIdxSize :: null ( ) ) ;
209- continue ;
210- } ;
211- let id = asof_in_group :: < T , A , & F > (
212- left_val,
213- right_val_arr,
214- right_grp_idxs. as_slice ( ) ,
215- & mut group_states,
216- & filter,
217- allow_eq,
218- ) ;
219-
220- results. push ( materialize_nullable ( id) ) ;
221- }
222- results
223- } ) ;
224- let bufs = RAYON . install ( || iter. collect :: < Vec < _ > > ( ) ) ;
210+ results. push ( materialize_nullable ( id) ) ;
211+ }
212+ results
213+ } ) ;
225214 flatten_nullable ( & bufs)
226215}
227216
0 commit comments