Skip to content

Commit bcf9e01

Browse files
authored
feat(pyamber): support Python UDF UI parameters (apache#5603)
### What changes were proposed in this PR? This PR adds Python runtime support for Python UDF UI parameters. It introduces: | Area | Change | | --- | --- | | PyTexera runtime API | Adds `self.UiParameter(...)` support on Python UDF operator base classes. | | Runtime injection bridge | Adds the `_texera_injected_ui_parameters` base hook that the Scala injector overrides. | | Typed parameter parsing | Converts injected UI parameter values into Python values using `AttributeType`. | | PyTexera exports | Exports `Dict`, `Any`, and `AttributeType` through `from pytexera import *`, so generated code from the Scala injector loads correctly. | | Attribute type compatibility | Adds Python enum aliases for `AttributeType.INTEGER` and `AttributeType.BOOLEAN`, matching the frontend parser’s accepted tokens. | | Follow-up cleanup from PR apache#5141 | Removes the temporary generated-code comment that described this runtime PR as a future dependency. | | Test coverage | Adds PyAmber tests for injected values, parsing, duplicate declarations, unsupported types, enum aliases, and instance-local state. | This PR is stacked after the merged frontend foundation PR apache#5043 and Scala backend injection PR apache#5141. It does not wire UI parameters into operator execution end to end; that integration is handled by the next PR in the stack. ### Any related issues, documentation, discussions? Part of the Python UDF UI parameter feature split from `feat/ui-parameter`. Related tracking issue / stack: apache#5044 Stack order: 1. Frontend UI parameter building blocks: apache#5043 2. Scala backend injection model: apache#5141 3. Python runtime support: this PR 4. End-to-end integration ### How was this PR tested? Commands run: ```bash cd amber ruff check src/main/python/core/models/schema/attribute_type.py src/main/python/pytexera/udf/udf_operator.py src/test/python/pytexera/udf/test_udf_operator.py pytest src/test/python/pytexera/udf/test_udf_operator.py -q pytest src/test/python/pytexera/udf -q ``` ### Was this PR authored or co-authored using generative AI tooling? Generated-by: Me
1 parent 429be11 commit bcf9e01

6 files changed

Lines changed: 566 additions & 11 deletions

File tree

‎amber/src/main/python/core/models/schema/attribute_type.py‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,11 @@ class AttributeType(Enum):
3333

3434
STRING = 1
3535
INT = 2
36+
# Attribute access needs Java names; RAW_TYPE_MAPPING handles raw schema strings.
37+
INTEGER = 2
3638
LONG = 3
3739
BOOL = 4
40+
BOOLEAN = 4
3841
DOUBLE = 5
3942
TIMESTAMP = 6
4043
BINARY = 7
@@ -78,6 +81,47 @@ class AttributeType(Enum):
7881
}
7982

8083

84+
def _is_empty_value(v):
85+
return v is None or (isinstance(v, str) and v.strip() == "")
86+
87+
88+
def _parse_bool(v):
89+
if _is_empty_value(v):
90+
return False
91+
92+
normalized_value = str(v).strip().lower()
93+
if normalized_value == "true":
94+
return True
95+
if normalized_value == "false":
96+
return False
97+
return float(normalized_value) != 0
98+
99+
100+
def _parse_timestamp(v):
101+
if _is_empty_value(v):
102+
return datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc)
103+
104+
normalized_value = str(v)
105+
if normalized_value.endswith("Z"):
106+
normalized_value = normalized_value[:-1] + "+00:00"
107+
parsed_value = datetime.datetime.fromisoformat(normalized_value)
108+
if (
109+
parsed_value.tzinfo is None
110+
or parsed_value.tzinfo.utcoffset(parsed_value) is None
111+
):
112+
return parsed_value.replace(tzinfo=datetime.timezone.utc)
113+
return parsed_value
114+
115+
116+
FROM_STRING_PARSER_MAPPING = {
117+
AttributeType.STRING: str,
118+
AttributeType.INT: lambda v: 0 if _is_empty_value(v) else int(v),
119+
AttributeType.LONG: lambda v: 0 if _is_empty_value(v) else int(v),
120+
AttributeType.DOUBLE: lambda v: 0.0 if _is_empty_value(v) else float(v),
121+
AttributeType.BOOL: _parse_bool,
122+
AttributeType.TIMESTAMP: _parse_timestamp,
123+
}
124+
81125
# Only single-directional mapping.
82126
TO_PYOBJECT_MAPPING = {
83127
AttributeType.STRING: str,

‎amber/src/main/python/pytexera/__init__.py‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717

1818
from loguru import logger
1919
from overrides import overrides
20-
from typing import Iterator, Optional, Union
20+
from typing import Iterator, Optional, Union, Dict, Any
2121

2222
from core.models.operator import LoopStartOperator, LoopEndOperator
2323
from pyamber import *
@@ -31,6 +31,7 @@
3131
UDFSourceOperator,
3232
)
3333
from core.models.type.large_binary import largebinary
34+
from core.models.schema.attribute_type import AttributeType
3435

3536
__all__ = [
3637
"State",
@@ -56,4 +57,7 @@
5657
"Iterator",
5758
"Optional",
5859
"Union",
60+
"Dict",
61+
"Any",
62+
"AttributeType",
5963
]

‎amber/src/main/python/pytexera/udf/udf_operator.py‎

Lines changed: 163 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,170 @@
1616
# under the License.
1717

1818
from abc import abstractmethod
19-
from typing import Iterator, Optional, Union
19+
from dataclasses import dataclass
20+
import functools
21+
from typing import Any, Dict, Iterator, Optional, Set, Union
2022

2123
from pyamber import *
24+
from core.models.schema.attribute_type import AttributeType, FROM_STRING_PARSER_MAPPING
25+
from loguru import logger
2226

2327

24-
class UDFOperatorV2(TupleOperatorV2):
28+
@dataclass(frozen=True)
29+
class _UiParameterValue:
30+
name: str
31+
type: AttributeType
32+
value: Any
33+
34+
35+
class _UiParameterSupport:
36+
_ui_parameter_injected_values: Dict[str, Any]
37+
_ui_parameter_name_types: Dict[str, AttributeType]
38+
_ui_parameter_used_names: Set[str]
39+
_unsupported_ui_parameter_types = {
40+
AttributeType.BINARY,
41+
AttributeType.LARGE_BINARY,
42+
}
43+
44+
# Reserved hook name. Backend injector will generate this in the user's class.
45+
def _texera_injected_ui_parameters(self) -> Dict[str, Any]:
46+
return {}
47+
48+
def _ensure_ui_parameter_state(self) -> None:
49+
if "_ui_parameter_injected_values" not in self.__dict__:
50+
self._ui_parameter_injected_values = {}
51+
if "_ui_parameter_name_types" not in self.__dict__:
52+
self._ui_parameter_name_types = {}
53+
if "_ui_parameter_used_names" not in self.__dict__:
54+
self._ui_parameter_used_names = set()
55+
56+
def _texera_apply_injected_ui_parameters(self) -> None:
57+
self._ensure_ui_parameter_state()
58+
values = self._texera_injected_ui_parameters()
59+
self._ui_parameter_injected_values = dict(values or {})
60+
self._ui_parameter_name_types = {}
61+
self._ui_parameter_used_names = set()
62+
63+
def _warn_unused_injected_ui_parameters(self) -> None:
64+
unused_names = sorted(
65+
set(self._ui_parameter_injected_values) - self._ui_parameter_used_names
66+
)
67+
if unused_names:
68+
logger.warning(
69+
"Injected UI parameter value(s) were not used: {}.",
70+
", ".join(unused_names),
71+
)
72+
73+
def __init_subclass__(cls, **kwargs):
74+
super().__init_subclass__(**kwargs)
75+
76+
original_open = getattr(cls, "open", None)
77+
if original_open is None:
78+
return
79+
80+
if getattr(original_open, "__texera_ui_params_wrapped__", False):
81+
return
82+
83+
@functools.wraps(original_open)
84+
def wrapped_open(self, *args, **kwargs):
85+
"""Apply injected UI parameters once before the outermost open()."""
86+
if getattr(self, "_ui_parameter_open_in_progress", False):
87+
return original_open(self, *args, **kwargs)
88+
89+
self._ui_parameter_open_in_progress = True
90+
try:
91+
self._texera_apply_injected_ui_parameters()
92+
result = original_open(self, *args, **kwargs)
93+
self._warn_unused_injected_ui_parameters()
94+
return result
95+
finally:
96+
self._ui_parameter_open_in_progress = False
97+
98+
setattr(wrapped_open, "__texera_ui_params_wrapped__", True)
99+
cls.open = wrapped_open
100+
101+
def UiParameter(
102+
self, name: str, attr_type: Optional[AttributeType] = None, **kwargs: Any
103+
) -> _UiParameterValue:
104+
"""
105+
Return the current UI parameter value parsed as attr_type.
106+
107+
Re-reading the same name with the same type is idempotent. Reusing a
108+
name with a different type is rejected because the parsed value would be
109+
ambiguous.
110+
"""
111+
if "type" in kwargs:
112+
if attr_type is not None:
113+
raise TypeError("UiParameter.type was provided multiple times.")
114+
attr_type = kwargs.pop("type")
115+
116+
if kwargs:
117+
unexpected_arguments = ", ".join(sorted(kwargs))
118+
raise TypeError(
119+
f"UiParameter got unexpected keyword argument(s): "
120+
f"{unexpected_arguments}."
121+
)
122+
123+
if attr_type is None:
124+
raise TypeError("UiParameter.type is required.")
125+
126+
if not isinstance(attr_type, AttributeType):
127+
raise TypeError(
128+
f"UiParameter.type must be an AttributeType, got {attr_type!r}."
129+
)
130+
131+
self._ensure_ui_parameter_state()
132+
existing_type = self._ui_parameter_name_types.get(name)
133+
if existing_type is not None and existing_type != attr_type:
134+
raise ValueError(
135+
f"Duplicate UiParameter name '{name}' with conflicting types: "
136+
f"{existing_type.name} vs {attr_type.name}."
137+
)
138+
139+
self._ui_parameter_name_types[name] = attr_type
140+
if name in self._ui_parameter_injected_values:
141+
self._ui_parameter_used_names.add(name)
142+
raw_value = self._ui_parameter_injected_values[name]
143+
else:
144+
logger.warning(
145+
"No injected UI parameter value found for name '{}'.",
146+
name,
147+
)
148+
raw_value = None
149+
150+
return _UiParameterValue(
151+
name=name,
152+
type=attr_type,
153+
value=self._parse(raw_value, attr_type),
154+
)
155+
156+
@staticmethod
157+
def _parse(value: Any, attr_type: AttributeType) -> Any:
158+
if attr_type in _UiParameterSupport._unsupported_ui_parameter_types:
159+
raise ValueError(
160+
f"UiParameter does not support {attr_type.name} values. "
161+
"Use a supported type instead."
162+
)
163+
164+
parser = FROM_STRING_PARSER_MAPPING.get(attr_type)
165+
if parser is None:
166+
raise TypeError(
167+
f"UiParameter.type {attr_type!r} is not supported for parsing."
168+
)
169+
170+
if value is None:
171+
return None
172+
173+
try:
174+
return parser(value)
175+
except Exception as e:
176+
raise ValueError(
177+
f"Failed to parse UiParameter value {value!r} as {attr_type.name}. "
178+
f"Please provide a valid {attr_type.name.lower()} value."
179+
) from e
180+
181+
182+
class UDFOperatorV2(_UiParameterSupport, TupleOperatorV2):
25183
"""
26184
Base class for tuple-oriented user-defined operators. A concrete implementation must
27185
be provided upon using.
@@ -65,7 +223,7 @@ def close(self) -> None:
65223
pass
66224

67225

68-
class UDFSourceOperator(SourceOperator):
226+
class UDFSourceOperator(_UiParameterSupport, SourceOperator):
69227
def open(self) -> None:
70228
"""
71229
Open a context of the operator. Usually can be used for loading/initiating some
@@ -90,7 +248,7 @@ def close(self) -> None:
90248
pass
91249

92250

93-
class UDFTableOperator(TableOperator):
251+
class UDFTableOperator(_UiParameterSupport, TableOperator):
94252
"""
95253
Base class for table-oriented user-defined operators. A concrete implementation must
96254
be provided upon using.
@@ -123,7 +281,7 @@ def close(self) -> None:
123281
pass
124282

125283

126-
class UDFBatchOperator(BatchOperator):
284+
class UDFBatchOperator(_UiParameterSupport, BatchOperator):
127285
"""
128286
Base class for batch-oriented user-defined operators. A concrete implementation must
129287
be provided upon using.

0 commit comments

Comments
 (0)