Skip to content

Commit 49bfc97

Browse files
authored
Merge pull request apache#3 from Kontinuation/st_random_geom
Added UDFs for generating random polygons and linestrings
2 parents c9f85b8 + 2b633f6 commit 49bfc97

7 files changed

Lines changed: 492 additions & 116 deletions

File tree

‎native/core/src/execution/datafusion/expressions/comet_scalar_funcs.rs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ use datafusion_comet_spark_expr::scalar_funcs::{
2727
spark_st_intersects, spark_st_intersects_use_geo, spark_st_intersects_wkb,
2828
spark_st_geomfromwkt, spark_st_geomfromwkb,
2929
spark_st_contains, spark_st_within,
30+
spark_st_random_polygon, spark_st_random_linestring
3031
};
3132
use datafusion_common::{DataFusionError, Result as DataFusionResult};
3233
use datafusion_expr::registry::FunctionRegistry;
@@ -83,6 +84,12 @@ pub fn create_comet_physical_fun(
8384
"st_envelope" => {
8485
make_comet_scalar_udf!("st_envelope", spark_st_envelope, data_type)
8586
}
87+
"st_randompolygon" => {
88+
make_comet_scalar_udf!("st_randompolygon", spark_st_random_polygon, data_type)
89+
}
90+
"st_randomlinestring" => {
91+
make_comet_scalar_udf!("st_randomlinestring", spark_st_random_linestring, data_type)
92+
}
8693
"st_intersects" => {
8794
make_comet_scalar_udf!("st_intersects", spark_st_intersects, data_type)
8895
}

‎native/spark-expr/src/scalar_funcs.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ mod wkb;
5858

5959
pub use st::{spark_st_envelope, spark_st_point, spark_st_points,
6060
spark_st_linestring, spark_st_multilinestring, spark_st_polygon,
61+
spark_st_random_polygon, spark_st_random_linestring,
6162
spark_st_intersects, spark_st_intersects_use_geo, spark_st_intersects_wkb,
6263
spark_st_geomfromwkt, spark_st_geomfromwkb, spark_st_contains, spark_st_within};
6364

‎native/spark-expr/src/scalar_funcs/geo_helpers.rs‎

Lines changed: 88 additions & 66 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,11 @@
1616
// under the License.
1717

1818
use std::sync::Arc;
19-
use arrow_array::{Array};
19+
use arrow_array::Array;
2020
use arrow_array::{Float64Array, ListArray, StructArray};
2121
use datafusion::logical_expr::ColumnarValue;
2222
use datafusion_common::{DataFusionError, ScalarValue};
23-
use geo::Point;
23+
use geo::{Coord, Point};
2424
use geo_types::Geometry;
2525
use crate::scalar_funcs::geometry_helpers::{
2626
GEOMETRY_TYPE_POINT,
@@ -80,7 +80,7 @@ pub fn arrow_to_geo(geom: &ColumnarValue) -> Result<Vec<geo_types::Geometry>, D
8080
.downcast_ref::<Float64Array>()
8181
.unwrap();
8282

83-
let mut geometries = Vec::new();
83+
let mut geometries = Vec::with_capacity(x_array.len());
8484
for i in 0..x_array.len() {
8585
let x = x_array.value(i);
8686
let y = y_array.value(i);
@@ -97,7 +97,7 @@ pub fn arrow_to_geo(geom: &ColumnarValue) -> Result<Vec<geo_types::Geometry>, D
9797
.downcast_ref::<ListArray>()
9898
.unwrap();
9999

100-
let mut geometries = Vec::new();
100+
let mut geometries = Vec::with_capacity(multipoints_array.len());
101101

102102
for i in 0..multipoints_array.len() {
103103
let array_ref = multipoints_array.value(i);
@@ -138,33 +138,38 @@ pub fn arrow_to_geo(geom: &ColumnarValue) -> Result<Vec<geo_types::Geometry>, D
138138
.downcast_ref::<ListArray>()
139139
.unwrap();
140140

141-
let mut geometries = Vec::new();
141+
// Get points data and offsets
142+
let points_array = linestring_array.values().as_any().downcast_ref::<StructArray>().unwrap();
143+
let points_offsets = linestring_array.offsets();
142144

143-
for i in 0..linestring_array.len() {
144-
let array_ref = linestring_array.value(i);
145-
let point_array = array_ref.as_any().downcast_ref::<StructArray>().unwrap();
146-
let x_array = point_array
147-
.column_by_name("x")
148-
.ok_or_else(|| DataFusionError::Internal("Missing 'x' field".to_string()))?
149-
.as_any()
150-
.downcast_ref::<Float64Array>()
151-
.unwrap();
145+
// Get coordinate arrays once
146+
let x_array = points_array
147+
.column_by_name("x")
148+
.ok_or_else(|| DataFusionError::Internal("Missing 'x' field".to_string()))?
149+
.as_any()
150+
.downcast_ref::<Float64Array>()
151+
.unwrap();
152152

153-
let y_array = point_array
154-
.column_by_name("y")
155-
.ok_or_else(|| DataFusionError::Internal("Missing 'y' field".to_string()))?
156-
.as_any()
157-
.downcast_ref::<Float64Array>()
158-
.unwrap();
153+
let y_array = points_array
154+
.column_by_name("y")
155+
.ok_or_else(|| DataFusionError::Internal("Missing 'y' field".to_string()))?
156+
.as_any()
157+
.downcast_ref::<Float64Array>()
158+
.unwrap();
159159

160-
let mut coords = Vec::new();
161-
for j in 0..x_array.len() {
160+
let mut geometries = Vec::with_capacity(linestring_array.len());
161+
162+
for i in 0..linestring_array.len() {
163+
let points_start = points_offsets[i] as usize;
164+
let points_end = points_offsets[i + 1] as usize;
165+
166+
// Create slices for this linestring's coordinates
167+
let mut coords = Vec::with_capacity(points_end - points_start);
168+
for j in points_start..points_end {
162169
let x = x_array.value(j);
163170
let y = y_array.value(j);
164-
coords.push((x, y));
171+
coords.push(Coord::from((x, y)));
165172
}
166-
167-
let coords: Vec<[f64; 2]> = coords.iter().map(|&(x, y)| [x, y]).collect();
168173
let linestring_geom = geo_types::LineString::from(coords);
169174
geometries.push(geo_types::Geometry::LineString(linestring_geom));
170175
}
@@ -179,45 +184,62 @@ pub fn arrow_to_geo(geom: &ColumnarValue) -> Result<Vec<geo_types::Geometry>, D
179184
.downcast_ref::<ListArray>()
180185
.unwrap();
181186

182-
let mut geometries = Vec::new();
187+
// Get all rings data
188+
let rings_array = polygon_array.values();
189+
let rings_offsets = polygon_array.offsets();
183190

184-
for i in 0..polygon_array.len() {
185-
let array_ref = polygon_array.value(i);
186-
let linestring_array = array_ref.as_any().downcast_ref::<ListArray>().unwrap();
191+
// Get all points data
192+
let points_array = rings_array.as_any().downcast_ref::<ListArray>().unwrap();
193+
let points_offsets = points_array.offsets();
187194

188-
let mut coords = Vec::new();
189-
for j in 0..linestring_array.len() {
190-
let array_ref = linestring_array.value(j);
191-
let point_array = array_ref.as_any().downcast_ref::<StructArray>().unwrap();
192-
let x_array = point_array
193-
.column_by_name("x")
194-
.ok_or_else(|| DataFusionError::Internal("Missing 'x' field".to_string()))?
195-
.as_any()
196-
.downcast_ref::<Float64Array>()
197-
.unwrap();
198-
199-
let y_array = point_array
200-
.column_by_name("y")
201-
.ok_or_else(|| DataFusionError::Internal("Missing 'y' field".to_string()))?
202-
.as_any()
203-
.downcast_ref::<Float64Array>()
204-
.unwrap();
205-
206-
let mut line_coords = Vec::new();
207-
for k in 0..x_array.len() {
208-
let x = x_array.value(k);
209-
let y = y_array.value(k);
210-
line_coords.push((x, y));
195+
// Get coordinate arrays
196+
let point_struct_array = points_array.values().as_any().downcast_ref::<StructArray>().unwrap();
197+
let x_array = point_struct_array
198+
.column_by_name("x")
199+
.ok_or_else(|| DataFusionError::Internal("Missing 'x' field".to_string()))?
200+
.as_any()
201+
.downcast_ref::<Float64Array>()
202+
.unwrap();
203+
let y_array = point_struct_array
204+
.column_by_name("y")
205+
.ok_or_else(|| DataFusionError::Internal("Missing 'y' field".to_string()))?
206+
.as_any()
207+
.downcast_ref::<Float64Array>()
208+
.unwrap();
209+
210+
let mut geometries = Vec::with_capacity(polygon_array.len());
211+
212+
// Iterate over polygons
213+
for i in 0..polygon_array.len() {
214+
let rings_start = rings_offsets[i] as usize;
215+
let rings_end = rings_offsets[i + 1] as usize;
216+
let num_rings = rings_end - rings_start;
217+
let num_interior_rings = std::cmp::max(num_rings - 1, 0);
218+
219+
let mut exterior_coords: Vec<Coord> = Vec::new();
220+
let mut interior_rings: Vec<geo_types::LineString> = Vec::with_capacity(num_interior_rings);
221+
222+
// Iterate over rings
223+
for j in 0..num_rings {
224+
let ring_idx = rings_start + j;
225+
let points_start = points_offsets[ring_idx] as usize;
226+
let points_end = points_offsets[ring_idx + 1] as usize;
227+
228+
let mut ring_coords = Vec::with_capacity(points_end - points_start);
229+
230+
// Collect coordinates for this ring
231+
for k in points_start..points_end {
232+
ring_coords.push(Coord::from((x_array.value(k), y_array.value(k))));
211233
}
212234

213-
let line_coords: Vec<[f64; 2]> = line_coords.iter().map(|&(x, y)| [x, y]).collect();
214-
coords.push(line_coords);
235+
if j == 0 {
236+
exterior_coords = ring_coords;
237+
} else {
238+
interior_rings.push(geo_types::LineString::from(ring_coords));
239+
}
215240
}
216241

217-
let exterior_ring = coords.remove(0);
218-
let interior_rings = coords;
219-
220-
let polygon_geom = geo_types::Polygon::new(exterior_ring.into(), interior_rings.into_iter().map(Into::into).collect());
242+
let polygon_geom = geo_types::Polygon::new(geo_types::LineString::from(exterior_coords), interior_rings);
221243
geometries.push(geo_types::Geometry::Polygon(polygon_geom));
222244
}
223245

@@ -425,7 +447,7 @@ pub fn geo_to_arrow(geometries: &[geo::Geometry]) -> Result<ColumnarValue, DataF
425447
x_coords.push(point.x());
426448
y_coords.push(point.y());
427449
}
428-
append_multipoint(&mut geometry_builder, x_coords, y_coords);
450+
append_multipoint(&mut geometry_builder, &x_coords, &y_coords);
429451
}
430452
geo::Geometry::LineString(line_string) => {
431453
let mut x_coords = Vec::with_capacity(line_string.0.len());
@@ -434,7 +456,7 @@ pub fn geo_to_arrow(geometries: &[geo::Geometry]) -> Result<ColumnarValue, DataF
434456
x_coords.push(coord.x);
435457
y_coords.push(coord.y);
436458
}
437-
append_linestring(&mut geometry_builder, x_coords, y_coords);
459+
append_linestring(&mut geometry_builder, &x_coords, &y_coords);
438460
}
439461
geo::Geometry::MultiLineString(multi_line_string) => {
440462
for line_string in &multi_line_string.0 {
@@ -444,7 +466,7 @@ pub fn geo_to_arrow(geometries: &[geo::Geometry]) -> Result<ColumnarValue, DataF
444466
x_coords.push(coord.x);
445467
y_coords.push(coord.y);
446468
}
447-
append_multilinestring(&mut geometry_builder, vec![(x_coords, y_coords)]);
469+
append_multilinestring(&mut geometry_builder, &[(x_coords, y_coords)]);
448470
}
449471
}
450472
geo::Geometry::Polygon(polygon) => {
@@ -455,7 +477,7 @@ pub fn geo_to_arrow(geometries: &[geo::Geometry]) -> Result<ColumnarValue, DataF
455477
x_coords.push(coord.x);
456478
y_coords.push(coord.y);
457479
}
458-
append_polygon(&mut geometry_builder, vec![(x_coords, y_coords)]);
480+
append_polygon(&mut geometry_builder, &[(x_coords, y_coords)]);
459481
}
460482
_ => return Err(DataFusionError::Internal("Unsupported geometry type".to_string())),
461483
}
@@ -493,7 +515,7 @@ pub fn geo_to_arrow_scalar(geometry: &Geometry) -> Result<ColumnarValue, DataFus
493515
x_coords.push(point.x());
494516
y_coords.push(point.y());
495517
}
496-
append_multipoint(&mut geometry_builder, x_coords, y_coords);
518+
append_multipoint(&mut geometry_builder, &x_coords, &y_coords);
497519
}
498520
geo::Geometry::LineString(line_string) => {
499521
let mut x_coords = Vec::with_capacity(line_string.0.len());
@@ -502,7 +524,7 @@ pub fn geo_to_arrow_scalar(geometry: &Geometry) -> Result<ColumnarValue, DataFus
502524
x_coords.push(coord.x);
503525
y_coords.push(coord.y);
504526
}
505-
append_linestring(&mut geometry_builder, x_coords, y_coords);
527+
append_linestring(&mut geometry_builder, &x_coords, &y_coords);
506528
}
507529
geo::Geometry::MultiLineString(multi_line_string) => {
508530
for line_string in &multi_line_string.0 {
@@ -512,7 +534,7 @@ pub fn geo_to_arrow_scalar(geometry: &Geometry) -> Result<ColumnarValue, DataFus
512534
x_coords.push(coord.x);
513535
y_coords.push(coord.y);
514536
}
515-
append_multilinestring(&mut geometry_builder, vec![(x_coords, y_coords)]);
537+
append_multilinestring(&mut geometry_builder, &[(x_coords, y_coords)]);
516538
}
517539
}
518540
geo::Geometry::Polygon(polygon) => {
@@ -523,7 +545,7 @@ pub fn geo_to_arrow_scalar(geometry: &Geometry) -> Result<ColumnarValue, DataFus
523545
x_coords.push(coord.x);
524546
y_coords.push(coord.y);
525547
}
526-
append_polygon(&mut geometry_builder, vec![(x_coords, y_coords)]);
548+
append_polygon(&mut geometry_builder, &[(x_coords, y_coords)]);
527549
}
528550
_ => return Err(DataFusionError::Internal("Unsupported geometry type".to_string())),
529551
}

0 commit comments

Comments
 (0)