Skip to content

Commit 9261829

Browse files
fix: support createDataFrame with list of dicts in Spark API
Port schema inference from duckdb/duckdb#18051 to fix duckdb#183. When calling spark.createDataFrame([{"col": value}, ...]), the Spark API now infers the schema from dict keys, matching PySpark behavior. Changes: - Add _type_mappings, _array_type_mappings, _has_nulltype, _merge_type, _infer_type, and _infer_schema functions to types.py - Update session.py to handle dict rows in _combine_data_and_schema and add schema inference branch in createDataFrame for list[dict] - Add _inferSchemaFromList method to SparkSession - Fix test_struct_column to use inferred field names instead of col0/col1 - Add test_dataframe_from_list_dicts test case
1 parent e8546f8 commit 9261829

4 files changed

Lines changed: 394 additions & 20 deletions

File tree

‎duckdb/experimental/spark/sql/session.py‎

Lines changed: 59 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import uuid
22
from collections.abc import Iterable, Sized
3-
from typing import TYPE_CHECKING, Any, NoReturn, Union
3+
from functools import reduce
4+
from typing import TYPE_CHECKING, Any, List, NoReturn, Optional, Union
45

56
import duckdb
67

@@ -12,13 +13,13 @@
1213

1314
from ..conf import SparkConf
1415
from ..context import SparkContext
15-
from ..errors import PySparkTypeError
16+
from ..errors import PySparkTypeError, PySparkValueError
1617
from ..exception import ContributionsAcceptedError
1718
from .conf import RuntimeConfig
1819
from .dataframe import DataFrame
1920
from .readwriter import DataFrameReader
2021
from .streaming import DataStreamReader
21-
from .types import StructType
22+
from .types import StructType, _has_nulltype, _infer_schema, _merge_type
2223
from .udf import UDFRegistration
2324

2425
# In spark:
@@ -38,7 +39,11 @@ def _combine_data_and_schema(data: Iterable[Any], schema: StructType) -> list[du
3839

3940
new_data = []
4041
for row in data:
41-
new_row = [Value(x, dtype.duckdb_type) for x, dtype in zip(row, [y.dataType for y in schema], strict=False)]
42+
if isinstance(row, dict):
43+
row_values = list(map(row.get, schema.fieldNames()))
44+
else:
45+
row_values = list(row)
46+
new_row = [Value(x, dtype.duckdb_type) for x, dtype in zip(row_values, [y.dataType for y in schema], strict=False)]
4247
new_data.append(new_row)
4348
return new_data
4449

@@ -150,6 +155,9 @@ def createDataFrame( # noqa: D102
150155
types, names = schema.extract_types_and_names()
151156
else:
152157
names = schema
158+
elif isinstance(data, list) and data:
159+
schema = self._inferSchemaFromList(data)
160+
types, names = schema.extract_types_and_names()
153161

154162
try:
155163
import pandas
@@ -188,6 +196,53 @@ def createDataFrame( # noqa: D102
188196
df = df.toDF(*names)
189197
return df
190198

199+
def _inferSchemaFromList(
200+
self, data: Iterable[Any], names: Optional[List[str]] = None
201+
) -> StructType:
202+
"""Infer schema from list of Row, dict, or tuple.
203+
204+
Parameters
205+
----------
206+
data : iterable
207+
list of Row, dict, or tuple
208+
names : list, optional
209+
list of column names
210+
211+
Returns
212+
-------
213+
:class:`duckdb.experimental.spark.sql.types.StructType`
214+
"""
215+
if not data:
216+
raise PySparkValueError(
217+
error_class="CANNOT_INFER_EMPTY_SCHEMA",
218+
message_parameters={},
219+
)
220+
221+
# TODO: These should be configurable
222+
infer_dict_as_struct = False
223+
infer_array_from_first_element = False
224+
prefer_timestamp_ntz = False
225+
226+
schema = reduce(
227+
_merge_type,
228+
(
229+
_infer_schema(
230+
row,
231+
names,
232+
infer_dict_as_struct=infer_dict_as_struct,
233+
infer_array_from_first_element=infer_array_from_first_element,
234+
prefer_timestamp_ntz=prefer_timestamp_ntz,
235+
)
236+
for row in data
237+
),
238+
)
239+
if _has_nulltype(schema):
240+
raise PySparkValueError(
241+
error_class="CANNOT_DETERMINE_TYPE",
242+
message_parameters={},
243+
)
244+
return schema
245+
191246
def newSession(self) -> "SparkSession": # noqa: D102
192247
return SparkSession(self._context)
193248

0 commit comments

Comments
 (0)