@@ -4483,6 +4483,210 @@ async fn unnest_dict_encoded_columns() -> Result<()> {
44834483 Ok ( ( ) )
44844484}
44854485
4486+ #[ rstest]
4487+ #[ case:: list( DataType :: List ) ]
4488+ #[ case:: large_list( DataType :: LargeList ) ]
4489+ #[ case:: fixed_size_list( |field| DataType :: FixedSizeList ( field, 2 ) ) ]
4490+ #[ case:: list_view( DataType :: ListView ) ]
4491+ #[ case:: large_list_view( DataType :: LargeListView ) ]
4492+ #[ tokio:: test]
4493+ async fn unnest_list_item_metadata (
4494+ #[ case] list_type : fn ( Arc < Field > ) -> DataType ,
4495+ ) -> Result < ( ) > {
4496+ use arrow:: array:: BinaryArray ;
4497+ use arrow:: buffer:: OffsetBuffer ;
4498+ use arrow:: compute:: cast;
4499+ use datafusion_common:: RecursionUnnestOption ;
4500+
4501+ let item = Arc :: new ( Field :: new ( "wkb" , DataType :: Binary , false ) . with_metadata (
4502+ HashMap :: from ( [
4503+ ( "ARROW:extension:name" . into ( ) , "geoarrow.wkb" . into ( ) ) ,
4504+ (
4505+ "ARROW:extension:metadata" . into ( ) ,
4506+ r#"{"crs":"EPSG:4326"}"# . into ( ) ,
4507+ ) ,
4508+ ( "custom" . into ( ) , "item" . into ( ) ) ,
4509+ ] ) ,
4510+ ) ) ;
4511+ let values: ArrayRef =
4512+ Arc :: new ( BinaryArray :: from_iter_values ( [ b"a" , b"b" , b"c" , b"d" ] ) ) ;
4513+ let lists = ListArray :: new (
4514+ Arc :: clone ( & item) ,
4515+ OffsetBuffer :: new ( vec ! [ 0 , 2 , 4 ] . into ( ) ) ,
4516+ Arc :: clone ( & values) ,
4517+ None ,
4518+ ) ;
4519+ let lists = cast ( & lists, & list_type ( Arc :: clone ( & item) ) ) ?;
4520+ let nested_item = Arc :: new (
4521+ Field :: new ( "lists" , lists. data_type ( ) . clone ( ) , false )
4522+ . with_metadata ( HashMap :: from ( [ ( "custom" . into ( ) , "nested item" . into ( ) ) ] ) ) ,
4523+ ) ;
4524+ let nested = ListArray :: new (
4525+ Arc :: clone ( & nested_item) ,
4526+ OffsetBuffer :: new ( vec ! [ 0 , 2 ] . into ( ) ) ,
4527+ Arc :: clone ( & lists) ,
4528+ None ,
4529+ ) ;
4530+ let nested = cast ( & nested, & list_type ( Arc :: clone ( & nested_item) ) ) ?;
4531+ let schema = Arc :: new ( Schema :: new ( vec ! [
4532+ Field :: new( "input" , nested. data_type( ) . clone( ) , false )
4533+ . with_metadata( HashMap :: from( [ ( "custom" . into( ) , "container" . into( ) ) ] ) ) ,
4534+ ] ) ) ;
4535+ let ctx = SessionContext :: new ( ) ;
4536+ let batch = RecordBatch :: try_new ( schema, vec ! [ nested] ) ?;
4537+ ctx. register_batch ( "t" , batch. clone ( ) ) ?;
4538+ let df = ctx. read_batch ( batch) ?;
4539+
4540+ // Each recursion depth must use that element's metadata, rather than the
4541+ // outer list's metadata. Renaming the result must retain it as well.
4542+ for ( depth, field, expected_values) in
4543+ [ ( 1 , & nested_item, & lists) , ( 2 , & item, & values) ]
4544+ {
4545+ let options = UnnestOptions :: new ( ) . with_recursions ( RecursionUnnestOption {
4546+ input_column : "input" . into ( ) ,
4547+ output_column : "expanded" . into ( ) ,
4548+ depth,
4549+ } ) ;
4550+ let unnested = df
4551+ . clone ( )
4552+ . unnest_columns_with_options ( & [ "input" ] , options) ?
4553+ . select ( vec ! [ col( "expanded" ) . alias( "output" ) ] ) ?;
4554+ let expected_field = field
4555+ . as_ref ( )
4556+ . clone ( )
4557+ . with_name ( "output" )
4558+ . with_nullable ( true ) ;
4559+ let sql = if depth == 1 {
4560+ "SELECT unnest(input) AS output FROM t"
4561+ } else {
4562+ "SELECT unnest(unnest(input)) AS output FROM t"
4563+ } ;
4564+ for unnested in [ unnested, ctx. sql ( sql) . await ?] {
4565+ assert_eq ! ( unnested. schema( ) . field( 0 ) . as_ref( ) , & expected_field) ;
4566+ let batches = unnested. collect ( ) . await ?;
4567+ assert_eq ! ( batches. len( ) , 1 ) ;
4568+ assert_eq ! ( batches[ 0 ] . schema( ) . field( 0 ) , & expected_field) ;
4569+ assert_eq ! ( batches[ 0 ] . column( 0 ) , expected_values) ;
4570+ }
4571+ }
4572+ Ok ( ( ) )
4573+ }
4574+
4575+ #[ tokio:: test]
4576+ async fn unnest_list_item_metadata_in_udf ( ) -> Result < ( ) > {
4577+ use arrow:: buffer:: OffsetBuffer ;
4578+ use datafusion_expr:: {
4579+ ReturnFieldArgs , ScalarFunctionArgs , ScalarUDF , ScalarUDFImpl , Signature ,
4580+ } ;
4581+
4582+ #[ derive( Debug , PartialEq , Eq , Hash ) ]
4583+ struct RequireItemMetadata ( Signature ) ;
4584+
4585+ impl ScalarUDFImpl for RequireItemMetadata {
4586+ fn name ( & self ) -> & str {
4587+ "require_item_metadata"
4588+ }
4589+
4590+ fn signature ( & self ) -> & Signature {
4591+ & self . 0
4592+ }
4593+
4594+ fn return_type ( & self , arg_types : & [ DataType ] ) -> Result < DataType > {
4595+ Ok ( arg_types[ 0 ] . clone ( ) )
4596+ }
4597+
4598+ fn return_field_from_args ( & self , args : ReturnFieldArgs ) -> Result < Arc < Field > > {
4599+ let field = & args. arg_fields [ 0 ] ;
4600+ if field. metadata ( ) . get ( "custom" ) . map ( String :: as_str) != Some ( "item" ) {
4601+ return datafusion_common:: plan_err!( "Missing item metadata" ) ;
4602+ }
4603+ Ok ( Arc :: clone ( field) )
4604+ }
4605+
4606+ fn invoke_with_args ( & self , args : ScalarFunctionArgs ) -> Result < ColumnarValue > {
4607+ Ok ( args. args [ 0 ] . clone ( ) )
4608+ }
4609+ }
4610+
4611+ let item = Arc :: new (
4612+ Field :: new ( "item" , DataType :: Int32 , false )
4613+ . with_metadata ( HashMap :: from ( [ ( "custom" . into ( ) , "item" . into ( ) ) ] ) ) ,
4614+ ) ;
4615+ let values: ArrayRef = Arc :: new ( Int32Array :: from ( vec ! [ 1 , 2 ] ) ) ;
4616+ let list = ListArray :: new (
4617+ Arc :: clone ( & item) ,
4618+ OffsetBuffer :: new ( vec ! [ 0 , 2 ] . into ( ) ) ,
4619+ Arc :: clone ( & values) ,
4620+ None ,
4621+ ) ;
4622+ let schema = Arc :: new ( Schema :: new ( vec ! [ Field :: new(
4623+ "input" ,
4624+ list. data_type( ) . clone( ) ,
4625+ false ,
4626+ ) ] ) ) ;
4627+ let ctx = SessionContext :: new ( ) ;
4628+ ctx. register_batch ( "t" , RecordBatch :: try_new ( schema, vec ! [ Arc :: new( list) ] ) ?) ?;
4629+ ctx. register_udf ( ScalarUDF :: from ( RequireItemMetadata ( Signature :: any (
4630+ 1 ,
4631+ Volatility :: Immutable ,
4632+ ) ) ) ) ;
4633+ let expected_field = item
4634+ . as_ref ( )
4635+ . clone ( )
4636+ . with_name ( "output" )
4637+ . with_nullable ( true ) ;
4638+
4639+ // A UDF can inspect argument metadata before SQL rewrites UNNEST into a plan.
4640+ for sql in [
4641+ "SELECT require_item_metadata(unnest(input)) AS output FROM t" ,
4642+ "SELECT require_item_metadata(output) AS output \
4643+ FROM (SELECT unnest(input) AS output FROM t)",
4644+ ] {
4645+ let df = ctx. sql ( sql) . await ?;
4646+ assert_eq ! ( df. schema( ) . field( 0 ) . as_ref( ) , & expected_field) ;
4647+ let batches = df. collect ( ) . await ?;
4648+ assert_eq ! ( batches. len( ) , 1 ) ;
4649+ assert_eq ! ( batches[ 0 ] . schema( ) . field( 0 ) , & expected_field) ;
4650+ assert_eq ! ( batches[ 0 ] . column( 0 ) , & values) ;
4651+ }
4652+ Ok ( ( ) )
4653+ }
4654+
4655+ #[ tokio:: test]
4656+ async fn unnest_list_item_metadata_with_nulls ( ) -> Result < ( ) > {
4657+ let item = Arc :: new (
4658+ Field :: new ( "item" , DataType :: Int32 , false )
4659+ . with_metadata ( HashMap :: from ( [ ( "custom" . into ( ) , "item" . into ( ) ) ] ) ) ,
4660+ ) ;
4661+ let mut builder = ListBuilder :: new ( Int32Builder :: new ( ) ) . with_field ( Arc :: clone ( & item) ) ;
4662+ builder. values ( ) . append_slice ( & [ 1 , 2 ] ) ;
4663+ builder. append ( true ) ;
4664+ builder. append ( true ) ; // Empty list
4665+ builder. append ( false ) ; // NULL list
4666+ let list = builder. finish ( ) ;
4667+ let schema = Arc :: new ( Schema :: new ( vec ! [ Field :: new(
4668+ "input" ,
4669+ list. data_type( ) . clone( ) ,
4670+ true ,
4671+ ) ] ) ) ;
4672+ let ctx = SessionContext :: new ( ) ;
4673+ let options = UnnestOptions :: new ( )
4674+ . with_null_handling ( datafusion_common:: NullHandling :: PreserveAndExpandEmpty ) ;
4675+ let df = ctx
4676+ . read_batch ( RecordBatch :: try_new ( schema, vec ! [ Arc :: new( list) ] ) ?) ?
4677+ . unnest_columns_with_options ( & [ "input" ] , options) ?;
4678+ let expected_field = item. as_ref ( ) . clone ( ) . with_name ( "input" ) . with_nullable ( true ) ;
4679+ assert_eq ! ( df. schema( ) . field( 0 ) . as_ref( ) , & expected_field) ;
4680+ let batches = df. collect ( ) . await ?;
4681+ assert_eq ! ( batches. len( ) , 1 ) ;
4682+ assert_eq ! ( batches[ 0 ] . schema( ) . field( 0 ) , & expected_field) ;
4683+ assert_eq ! (
4684+ batches[ 0 ] . column( 0 ) . as_ref( ) ,
4685+ & Int32Array :: from( vec![ Some ( 1 ) , Some ( 2 ) , None , None ] ) as & dyn Array
4686+ ) ;
4687+ Ok ( ( ) )
4688+ }
4689+
44864690#[ tokio:: test]
44874691async fn unnest_column_nulls ( ) -> Result < ( ) > {
44884692 let df = table_with_lists_and_nulls ( ) . await ?;
0 commit comments