Skip to content

Commit f5e4e84

Browse files
dtenedorueshin
authored andcommitted
[SPARK-45746][PYTHON] Return specific error messages if UDTF 'analyze' or 'eval' method accepts or returns wrong values
### What changes were proposed in this pull request? This PR adds checks to return specific error messages if any Python UDTF `analyze` or `eval` method accepts or returns wrong values. Error messages improved include: * If the `__init__` method takes more arguments than `self` and `analyze_result`. * If the UDTF call passes more or fewer arguments than `analyze` or `eval` expects (not using `*args` or `**kwargs`). * If the `analyze` method returns an object besides a `StructType` in the `AnalyzeResult` `schema` field. * If there are extra optional `AnalyzeResult` fields relating to input table arguments (e.g. `with_single_partition`) but the `analyze` method received no input table argument. * If the `analyze` method tries to return a list of strings for the `partition_by` optional field of the `AnalyzeResult` instead of a list of `PartitioningColumn` objects. * If the `AnalyzeResult` is missing the `schema` argument entirely. * If we use keyword arguments in the TVF call but the `analyze` or `eval` method does not accept arguments with those keyword(s) (or `**kwargs`). ### Why are the changes needed? This helps users understand how to easily fix their user-defined table functions if they are malformed. ### Does this PR introduce _any_ user-facing change? Yes, see above. ### How was this patch tested? This PR adds test coverage. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #43611 from dtenedor/fix-more-udtf-errors. Authored-by: Daniel Tenedorio <[email protected]> Signed-off-by: Takuya UESHIN <[email protected]>
1 parent 8bd9c8c commit f5e4e84

12 files changed

Lines changed: 939 additions & 309 deletions

File tree

‎python/pyspark/errors/error_classes.py‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -798,6 +798,21 @@
798798
"Cannot convert the output value of the column '<col_name>' with type '<col_type>' to the specified return type of the column: '<arrow_type>'. Please check if the data types match and try again."
799799
]
800800
},
801+
"UDTF_CONSTRUCTOR_INVALID_IMPLEMENTS_ANALYZE_METHOD" : {
802+
"message" : [
803+
"Failed to evaluate the user-defined table function '<name>' because its constructor is invalid: the function implements the 'analyze' method, but its constructor has more than two arguments (including the 'self' reference). Please update the table function so that its constructor accepts exactly one 'self' argument, or one 'self' argument plus another argument for the result of the 'analyze' method, and try the query again."
804+
]
805+
},
806+
"UDTF_CONSTRUCTOR_INVALID_NO_ANALYZE_METHOD" : {
807+
"message" : [
808+
"Failed to evaluate the user-defined table function '<name>' because its constructor is invalid: the function does not implement the 'analyze' method, and its constructor has more than one argument (including the 'self' reference). Please update the table function so that its constructor accepts exactly one 'self' argument, and try the query again."
809+
]
810+
},
811+
"UDTF_EVAL_METHOD_ARGUMENTS_DO_NOT_MATCH_SIGNATURE" : {
812+
"message" : [
813+
"Failed to evaluate the user-defined table function '<name>' because the function arguments did not match the expected signature of the 'eval' method (<reason>). Please update the query so that this table function call provides arguments matching the expected signature, or else update the table function so that its 'eval' method accepts the provided arguments, and then try the query again."
814+
]
815+
},
801816
"UDTF_EXEC_ERROR" : {
802817
"message" : [
803818
"User defined table function encountered an error in the '<method_name>' method: <error>"

‎python/pyspark/sql/tests/test_udtf.py‎

Lines changed: 18 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -274,20 +274,21 @@ def eval(self, a: int):
274274
df = self.spark.sql("SELECT * FROM testUDTF(null)")
275275
self.assertEqual(df.collect(), [Row(a=None)])
276276

277+
# These are expected error message substrings to be used in test cases below.
278+
tooManyPositionalArguments = "too many positional arguments"
279+
missingARequiredArgument = "missing a required argument"
280+
multipleValuesForArgument = "multiple values for argument"
281+
277282
def test_udtf_with_wrong_num_input(self):
278283
@udtf(returnType="a: int, b: int")
279284
class TestUDTF:
280285
def eval(self, a: int):
281286
yield a, a + 1
282287

283-
with self.assertRaisesRegex(
284-
PythonException, r"eval\(\) missing 1 required positional argument: 'a'"
285-
):
288+
with self.assertRaisesRegex(PythonException, BaseUDTFTestsMixin.missingARequiredArgument):
286289
TestUDTF().collect()
287290

288-
with self.assertRaisesRegex(
289-
PythonException, r"eval\(\) takes 2 positional arguments but 3 were given"
290-
):
291+
with self.assertRaisesRegex(PythonException, BaseUDTFTestsMixin.tooManyPositionalArguments):
291292
TestUDTF(lit(1), lit(2)).collect()
292293

293294
def test_udtf_init_with_additional_args(self):
@@ -299,9 +300,7 @@ def __init__(self, a: int):
299300
def eval(self, a: int):
300301
yield a,
301302

302-
with self.assertRaisesRegex(
303-
PythonException, r"__init__\(\) missing 1 required positional argument: 'a'"
304-
):
303+
with self.assertRaisesRegex(PythonException, r".*constructor has more than one argument.*"):
305304
TestUDTF(lit(1)).show()
306305

307306
def test_udtf_terminate_with_additional_args(self):
@@ -1582,8 +1581,9 @@ def eval(self):
15821581

15831582
with self.assertRaisesRegex(
15841583
AnalysisException,
1585-
"Output of `analyze` static method of Python UDTFs expects "
1586-
"a pyspark.sql.udtf.AnalyzeResult but got: <class 'pyspark.sql.types.StringType'>",
1584+
"'analyze' method expects a result of type pyspark.sql.udtf.AnalyzeResult, "
1585+
"but instead this method returned a value of type: "
1586+
"<class 'pyspark.sql.types.StringType'>",
15871587
):
15881588
func().collect()
15891589

@@ -1622,26 +1622,17 @@ class TestUDTF:
16221622
def analyze(a: AnalyzeArgument, b: AnalyzeArgument) -> AnalyzeResult:
16231623
return AnalyzeResult(StructType().add("a", a.dataType).add("b", b.dataType))
16241624

1625-
def eval(self, a):
1625+
def eval(self, a, b):
16261626
yield a, a + 1
16271627

16281628
func = udtf(TestUDTF)
16291629

1630-
with self.assertRaisesRegex(
1631-
AnalysisException, r"analyze\(\) missing 1 required positional argument: 'b'"
1632-
):
1630+
with self.assertRaisesRegex(AnalysisException, r"arguments"):
16331631
func(lit(1)).collect()
16341632

1635-
with self.assertRaisesRegex(
1636-
AnalysisException, r"analyze\(\) takes 2 positional arguments but 3 were given"
1637-
):
1633+
with self.assertRaisesRegex(AnalysisException, r"arguments"):
16381634
func(lit(1), lit(2), lit(3)).collect()
16391635

1640-
with self.assertRaisesRegex(
1641-
PythonException, r"eval\(\) takes 2 positional arguments but 3 were given"
1642-
):
1643-
func(lit(1), lit(2)).collect()
1644-
16451636
def test_udtf_with_analyze_taking_keyword_arguments(self):
16461637
@udtf
16471638
class TestUDTF:
@@ -1660,12 +1651,12 @@ def eval(self, **kwargs):
16601651
assertDataFrameEqual(self.spark.sql("SELECT * FROM test_udtf(a=>1)"), expected)
16611652

16621653
with self.assertRaisesRegex(
1663-
AnalysisException, r"analyze\(\) takes 0 positional arguments but 1 was given"
1654+
AnalysisException, BaseUDTFTestsMixin.tooManyPositionalArguments
16641655
):
16651656
TestUDTF(lit(1)).collect()
16661657

16671658
with self.assertRaisesRegex(
1668-
AnalysisException, r"analyze\(\) takes 0 positional arguments but 2 were given"
1659+
AnalysisException, BaseUDTFTestsMixin.tooManyPositionalArguments
16691660
):
16701661
self.spark.sql("SELECT * FROM test_udtf(1, 'x')").collect()
16711662

@@ -1924,14 +1915,10 @@ def eval(self, a, b):
19241915
with self.assertRaisesRegex(AnalysisException, "UNEXPECTED_POSITIONAL_ARGUMENT"):
19251916
self.spark.sql("SELECT * FROM test_udtf(a => 10, 'x')").show()
19261917

1927-
with self.assertRaisesRegex(
1928-
PythonException, r"eval\(\) got an unexpected keyword argument 'c'"
1929-
):
1918+
with self.assertRaisesRegex(PythonException, BaseUDTFTestsMixin.missingARequiredArgument):
19301919
self.spark.sql("SELECT * FROM test_udtf(c => 'x')").show()
19311920

1932-
with self.assertRaisesRegex(
1933-
PythonException, r"eval\(\) got multiple values for argument 'a'"
1934-
):
1921+
with self.assertRaisesRegex(PythonException, BaseUDTFTestsMixin.multipleValuesForArgument):
19351922
self.spark.sql("SELECT * FROM test_udtf(10, a => 100)").show()
19361923

19371924
def test_udtf_with_kwargs(self):

‎python/pyspark/sql/worker/analyze_udtf.py‎

Lines changed: 83 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import inspect
1919
import os
2020
import sys
21+
from textwrap import dedent
2122
from typing import Dict, List, IO, Tuple
2223

2324
from pyspark.accumulators import _accumulatorRegistry
@@ -30,7 +31,8 @@
3031
write_with_length,
3132
SpecialLengths,
3233
)
33-
from pyspark.sql.types import _parse_datatype_json_string
34+
from pyspark.sql.functions import PartitioningColumn
35+
from pyspark.sql.types import _parse_datatype_json_string, StructType
3436
from pyspark.sql.udtf import AnalyzeArgument, AnalyzeResult
3537
from pyspark.util import handle_worker_exception
3638
from pyspark.worker_util import (
@@ -113,15 +115,93 @@ def main(infile: IO, outfile: IO) -> None:
113115

114116
_accumulatorRegistry.clear()
115117

118+
udtf_name = utf8_deserializer.loads(infile)
116119
handler = read_udtf(infile)
117120
args, kwargs = read_arguments(infile)
118121

122+
error_prefix = f"Failed to evaluate the user-defined table function '{udtf_name}'"
123+
124+
def format_error(msg: str) -> str:
125+
return dedent(msg).replace("\n", " ")
126+
127+
# Check that the arguments provided to the UDTF call match the expected parameters defined
128+
# in the static 'analyze' method signature.
129+
try:
130+
inspect.signature(handler.analyze).bind(*args, **kwargs) # type: ignore[attr-defined]
131+
except TypeError as e:
132+
# The UDTF call's arguments did not match the expected signature.
133+
raise PySparkValueError(
134+
format_error(
135+
f"""
136+
{error_prefix} because the function arguments did not match the expected
137+
signature of the static 'analyze' method ({e}). Please update the query so that
138+
this table function call provides arguments matching the expected signature, or
139+
else update the table function so that its static 'analyze' method accepts the
140+
provided arguments, and then try the query again."""
141+
)
142+
)
143+
144+
# Invoke the UDTF's 'analyze' method.
119145
result = handler.analyze(*args, **kwargs) # type: ignore[attr-defined]
120146

147+
# Check invariants about the 'analyze' method after running it.
121148
if not isinstance(result, AnalyzeResult):
122149
raise PySparkValueError(
123-
"Output of `analyze` static method of Python UDTFs expects "
124-
f"a pyspark.sql.udtf.AnalyzeResult but got: {type(result)}"
150+
format_error(
151+
f"""
152+
{error_prefix} because the static 'analyze' method expects a result of type
153+
pyspark.sql.udtf.AnalyzeResult, but instead this method returned a value of
154+
type: {type(result)}"""
155+
)
156+
)
157+
elif not isinstance(result.schema, StructType):
158+
raise PySparkValueError(
159+
format_error(
160+
f"""
161+
{error_prefix} because the static 'analyze' method expects a result of type
162+
pyspark.sql.udtf.AnalyzeResult with a 'schema' field comprising a StructType,
163+
but the 'schema' field had the wrong type: {type(result.schema)}"""
164+
)
165+
)
166+
has_table_arg = any(arg.isTable for arg in args) or any(
167+
arg.isTable for arg in kwargs.values()
168+
)
169+
if not has_table_arg and result.withSinglePartition:
170+
raise PySparkValueError(
171+
format_error(
172+
f"""
173+
{error_prefix} because the static 'analyze' method returned an
174+
'AnalyzeResult' object with the 'withSinglePartition' field set to 'true', but
175+
the function call did not provide any table argument. Please update the query so
176+
that it provides a table argument, or else update the table function so that its
177+
'analyze' method returns an 'AnalyzeResult' object with the
178+
'withSinglePartition' field set to 'false', and then try the query again."""
179+
)
180+
)
181+
elif not has_table_arg and len(result.partitionBy) > 0:
182+
raise PySparkValueError(
183+
format_error(
184+
f"""
185+
{error_prefix} because the static 'analyze' method returned an
186+
'AnalyzeResult' object with the 'partitionBy' list set to non-empty, but the
187+
function call did not provide any table argument. Please update the query so
188+
that it provides a table argument, or else update the table function so that its
189+
'analyze' method returns an 'AnalyzeResult' object with the 'partitionBy' list
190+
set to empty, and then try the query again."""
191+
)
192+
)
193+
elif isinstance(result.partitionBy, (list, tuple)) and (
194+
len(result.partitionBy) > 0
195+
and not all([isinstance(val, PartitioningColumn) for val in result.partitionBy])
196+
):
197+
raise PySparkValueError(
198+
format_error(
199+
f"""
200+
{error_prefix} because the static 'analyze' method returned an
201+
'AnalyzeResult' object with the 'partitionBy' field set to a value besides a
202+
list or tuple of 'PartitioningColumn' objects. Please update the table function
203+
and then try the query again."""
204+
)
125205
)
126206

127207
# Return the analyzed schema.

‎python/pyspark/worker.py‎

Lines changed: 31 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@
2222
import sys
2323
import dataclasses
2424
import time
25-
from inspect import getfullargspec
25+
import inspect
2626
import json
2727
from typing import Any, Callable, Iterable, Iterator, Optional
2828
import faulthandler
@@ -616,12 +616,12 @@ def read_single_udf(pickleSer, infile, eval_type, runner_conf, udf_index):
616616
elif eval_type == PythonEvalType.SQL_MAP_ARROW_ITER_UDF:
617617
return args_offsets, wrap_arrow_batch_iter_udf(func, return_type)
618618
elif eval_type == PythonEvalType.SQL_GROUPED_MAP_PANDAS_UDF:
619-
argspec = getfullargspec(chained_func) # signature was lost when wrapping it
619+
argspec = inspect.getfullargspec(chained_func) # signature was lost when wrapping it
620620
return args_offsets, wrap_grouped_map_pandas_udf(func, return_type, argspec, runner_conf)
621621
elif eval_type == PythonEvalType.SQL_GROUPED_MAP_PANDAS_UDF_WITH_STATE:
622622
return args_offsets, wrap_grouped_map_pandas_udf_with_state(func, return_type)
623623
elif eval_type == PythonEvalType.SQL_COGROUPED_MAP_PANDAS_UDF:
624-
argspec = getfullargspec(chained_func) # signature was lost when wrapping it
624+
argspec = inspect.getfullargspec(chained_func) # signature was lost when wrapping it
625625
return args_offsets, wrap_cogrouped_map_pandas_udf(func, return_type, argspec, runner_conf)
626626
elif eval_type == PythonEvalType.SQL_GROUPED_AGG_PANDAS_UDF:
627627
return wrap_grouped_agg_pandas_udf(func, args_offsets, kwargs_offsets, return_type)
@@ -691,7 +691,6 @@ def read_udtf(pickleSer, infile, eval_type):
691691
pickled_analyze_result = None
692692
# Initially we assume that the UDTF __init__ method accepts the pickled AnalyzeResult,
693693
# although we may set this to false later if we find otherwise.
694-
udtf_init_method_accepts_analyze_result = True
695694
handler = read_command(pickleSer, infile)
696695
if not isinstance(handler, type):
697696
raise PySparkRuntimeError(
@@ -704,29 +703,32 @@ def read_udtf(pickleSer, infile, eval_type):
704703
raise PySparkRuntimeError(
705704
f"The return type of a UDTF must be a struct type, but got {type(return_type)}."
706705
)
706+
udtf_name = utf8_deserializer.loads(infile)
707707

708708
# Update the handler that creates a new UDTF instance to first try calling the UDTF constructor
709709
# with one argument containing the previous AnalyzeResult. If that fails, then try a constructor
710710
# with no arguments. In this way each UDTF class instance can decide if it wants to inspect the
711711
# AnalyzeResult.
712+
udtf_init_args = inspect.getfullargspec(handler)
712713
if has_pickled_analyze_result:
713-
prev_handler = handler
714+
if len(udtf_init_args.args) > 2:
715+
raise PySparkRuntimeError(
716+
error_class="UDTF_CONSTRUCTOR_INVALID_IMPLEMENTS_ANALYZE_METHOD",
717+
message_parameters={"name": udtf_name},
718+
)
719+
elif len(udtf_init_args.args) == 2:
720+
prev_handler = handler
714721

715-
def construct_udtf():
716-
nonlocal udtf_init_method_accepts_analyze_result
717-
if not udtf_init_method_accepts_analyze_result:
718-
return prev_handler()
719-
else:
720-
try:
721-
# Here we pass the AnalyzeResult to the UDTF's __init__ method.
722-
return prev_handler(dataclasses.replace(pickled_analyze_result))
723-
except TypeError:
724-
# This means that the UDTF handler does not accept an AnalyzeResult object in
725-
# its __init__ method.
726-
udtf_init_method_accepts_analyze_result = False
727-
return prev_handler()
722+
def construct_udtf():
723+
# Here we pass the AnalyzeResult to the UDTF's __init__ method.
724+
return prev_handler(dataclasses.replace(pickled_analyze_result))
728725

729-
handler = construct_udtf
726+
handler = construct_udtf
727+
elif len(udtf_init_args.args) > 1:
728+
raise PySparkRuntimeError(
729+
error_class="UDTF_CONSTRUCTOR_INVALID_NO_ANALYZE_METHOD",
730+
message_parameters={"name": udtf_name},
731+
)
730732

731733
class UDTFWithPartitions:
732734
"""
@@ -854,6 +856,16 @@ def _remove_partition_by_exprs(self, arg: Any) -> Any:
854856
"the query again."
855857
)
856858

859+
# Check that the arguments provided to the UDTF call match the expected parameters defined
860+
# in the 'eval' method signature.
861+
try:
862+
inspect.signature(udtf.eval).bind(*args_offsets, **kwargs_offsets)
863+
except TypeError as e:
864+
raise PySparkRuntimeError(
865+
error_class="UDTF_EVAL_METHOD_ARGUMENTS_DO_NOT_MATCH_SIGNATURE",
866+
message_parameters={"name": udtf_name, "reason": str(e)},
867+
) from None
868+
857869
def build_null_checker(return_type: StructType) -> Optional[Callable[[Any], None]]:
858870
def raise_(result_column_index):
859871
raise PySparkRuntimeError(

‎sql/core/src/main/scala/org/apache/spark/sql/execution/python/BatchEvalPythonUDTFExec.scala‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,5 +142,7 @@ object PythonUDTFRunner {
142142
PythonWorkerUtils.writePythonFunction(udtf.func, dataOut)
143143
// Write the result schema of the UDTF call.
144144
PythonWorkerUtils.writeUTF(udtf.elementSchema.json, dataOut)
145+
// Write the UDTF name.
146+
PythonWorkerUtils.writeUTF(udtf.name, dataOut)
145147
}
146148
}

‎sql/core/src/main/scala/org/apache/spark/sql/execution/python/UserDefinedPythonFunction.scala‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,7 @@ case class UserDefinedPythonTableFunction(
133133
case _ => false
134134
}
135135
val runAnalyzeInPython = (func: PythonFunction, exprs: Seq[Expression]) => {
136-
val runner = new UserDefinedPythonTableFunctionAnalyzeRunner(func, exprs, tableArgs)
136+
val runner = new UserDefinedPythonTableFunctionAnalyzeRunner(name, func, exprs, tableArgs)
137137
runner.runInPython()
138138
}
139139
UnresolvedPolymorphicPythonUDTF(
@@ -184,6 +184,7 @@ case class UserDefinedPythonTableFunction(
184184
* will be thrown when an exception is raised in Python.
185185
*/
186186
class UserDefinedPythonTableFunctionAnalyzeRunner(
187+
name: String,
187188
func: PythonFunction,
188189
exprs: Seq[Expression],
189190
tableArgs: Seq[Boolean]) extends PythonPlannerRunner[PythonUDTFAnalyzeResult](func) {
@@ -192,6 +193,7 @@ class UserDefinedPythonTableFunctionAnalyzeRunner(
192193

193194
override protected def writeToPython(dataOut: DataOutputStream, pickler: Pickler): Unit = {
194195
// Send Python UDTF
196+
PythonWorkerUtils.writeUTF(name, dataOut)
195197
PythonWorkerUtils.writePythonFunction(func, dataOut)
196198

197199
// Send arguments
@@ -226,6 +228,9 @@ class UserDefinedPythonTableFunctionAnalyzeRunner(
226228
val length = dataIn.readInt()
227229
if (length == SpecialLengths.PYTHON_EXCEPTION_THROWN) {
228230
val msg = PythonWorkerUtils.readUTF(dataIn)
231+
// Remove the leading traceback stack trace from the error message string, if any, since it
232+
// usually only includes the "analyze_udtf.py" filename and a line number.
233+
.split("PySparkValueError:").last.strip()
229234
throw QueryCompilationErrors.tableValuedFunctionFailedToAnalyseInPythonError(msg)
230235
}
231236

0 commit comments

Comments
 (0)