Skip to content

Commit a7f4386

Browse files
authored
feat(udf): enable Python UDF UI parameters end to end (apache#5912)
### What changes were proposed in this PR? This PR wires the existing Python UDF UI parameter support into the end-to-end execution path. It changes: | Area | Change | | --- | --- | | Python UDF descriptors | Adds the `uiParameters` property to Python UDF, dual-input Python UDF, and Python UDF source operators. | | Execution wiring | Applies `PythonUdfUiParameterInjector.inject(...)` before creating Python physical operators. | | Frontend wiring | Connects the existing UI-parameter sync service to the code editor and property editor, and renders `uiParameters` with the existing UI-parameter editor. | | Operator templates | Adds commented `UiParameter` examples to Python UDF templates. | ### Any related issues, documentation, discussions? Part of the Python UDF UI parameter feature split from `feat/ui-parameter`. Related tracking issue / stack: apache#5044 Stack order: 1. Frontend UI parameter building blocks: apache#5043 2. Scala backend injection model: apache#5141 3. Python runtime support: apache#5603 4. End-to-end execution wiring: this PR ### How was this PR tested? Manually + Commands run: ```bash sbt --no-server scalafmtAll scalafixAll sbt --no-server scalafmtCheckAll "scalafixAll --check" cd amber ruff check src/main/python src/test/python ruff format --check src/main/python src/test/python cd .. sbt --no-server "WorkflowExecutionService / Test / testOnly org.apache.texera.amber.engine.architecture.pythonworker.PythonWorkflowWorkerStartupConfigSpec" ``` ### Was this PR authored or co-authored using generative AI tooling? Generated-by: Me
1 parent 8aec229 commit a7f4386

24 files changed

Lines changed: 446 additions & 49 deletions

File tree

‎amber/src/main/python/pytexera/udf/udf_operator.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -170,6 +170,15 @@ def _parse(value: Any, attr_type: AttributeType) -> Any:
170170
if value is None:
171171
return None
172172

173+
if (
174+
attr_type is not AttributeType.STRING
175+
and isinstance(value, str)
176+
and not value.strip()
177+
):
178+
raise ValueError(
179+
f"UiParameter value cannot be empty for type {attr_type.name}."
180+
)
181+
173182
try:
174183
return parser(value)
175184
except Exception as e:

‎amber/src/test/python/pytexera/udf/test_udf_operator.py‎

Lines changed: 14 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -238,20 +238,23 @@ def test_parse_supported_types(self, raw_value, attr_type, expected):
238238
assert _UiParameterSupport._parse(raw_value, attr_type) == expected
239239

240240
@pytest.mark.parametrize(
241-
("raw_value", "attr_type", "expected"),
241+
("raw_value", "attr_type"),
242242
[
243-
("", AttributeType.INT, 0),
244-
(" ", AttributeType.LONG, 0),
245-
("", AttributeType.DOUBLE, 0.0),
246-
(
247-
"",
248-
AttributeType.TIMESTAMP,
249-
datetime.datetime(1970, 1, 1, tzinfo=datetime.timezone.utc),
250-
),
243+
("", AttributeType.INT),
244+
(" ", AttributeType.LONG),
245+
("", AttributeType.DOUBLE),
246+
("\t", AttributeType.BOOL),
247+
("", AttributeType.TIMESTAMP),
251248
],
252249
)
253-
def test_parse_empty_values(self, raw_value, attr_type, expected):
254-
assert _UiParameterSupport._parse(raw_value, attr_type) == expected
250+
def test_parse_empty_non_string_values_raises_value_error(
251+
self, raw_value, attr_type
252+
):
253+
with pytest.raises(ValueError, match="UiParameter value cannot be empty"):
254+
_UiParameterSupport._parse(raw_value, attr_type)
255+
256+
def test_parse_empty_string_value(self):
257+
assert _UiParameterSupport._parse("", AttributeType.STRING) == ""
255258

256259
def test_java_attribute_type_aliases_parse_like_python_names(self):
257260
assert AttributeType.INTEGER is AttributeType.INT
@@ -262,8 +265,6 @@ def test_java_attribute_type_aliases_parse_like_python_names(self):
262265
@pytest.mark.parametrize(
263266
("raw_value", "expected"),
264267
[
265-
("", False),
266-
(" ", False),
267268
("True", True),
268269
("true", True),
269270
("1", True),

‎common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/python/DualInputPortsPythonUDFOpDescV2.scala‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,15 @@ import org.apache.texera.amber.core.workflow._
2929
import org.apache.texera.amber.operator.LogicalOp
3030
import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo}
3131

32-
class DualInputPortsPythonUDFOpDescV2 extends LogicalOp {
32+
class DualInputPortsPythonUDFOpDescV2 extends LogicalOp with PythonUdfUiParameterSupport {
3333
@JsonProperty(
3434
required = true,
3535
defaultValue =
3636
"# Choose from the following templates:\n" +
37+
"# \n" +
38+
"# Define UiParameter inside open() of ProcessTupleOperator, ProcessBatchOperator, or ProcessTableOperator.\n" +
39+
"# Example: self.count = self.UiParameter(\"count\", AttributeType.INT).value\n" +
40+
"# See the Python UDF operator documentation for supported types and behavior.\n" +
3741
"# \n" +
3842
"# from pytexera import *\n" +
3943
"# \n" +
@@ -105,14 +109,15 @@ class DualInputPortsPythonUDFOpDescV2 extends LogicalOp {
105109
)
106110
trimmed
107111
}
112+
val codeWithParameters = injectUiParameters(code)
108113

109114
val physicalOp = if (workers > 1) {
110115
PhysicalOp
111116
.oneToOnePhysicalOp(
112117
workflowId,
113118
executionId,
114119
operatorIdentifier,
115-
OpExecWithCode(code, "python")
120+
OpExecWithCode(codeWithParameters, "python")
116121
)
117122
.withParallelizable(true)
118123
.withSuggestedWorkerNum(workers)
@@ -122,7 +127,7 @@ class DualInputPortsPythonUDFOpDescV2 extends LogicalOp {
122127
workflowId,
123128
executionId,
124129
operatorIdentifier,
125-
OpExecWithCode(code, "python")
130+
OpExecWithCode(codeWithParameters, "python")
126131
)
127132
.withParallelizable(false)
128133
}

‎common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/python/PythonUDFOpDescV2.scala‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,11 +31,15 @@ import org.apache.texera.amber.operator.{LogicalOp, PortDescription, StateTransf
3131

3232
import scala.util.{Success, Try}
3333

34-
class PythonUDFOpDescV2 extends LogicalOp {
34+
class PythonUDFOpDescV2 extends LogicalOp with PythonUdfUiParameterSupport {
3535
@JsonProperty(
3636
required = true,
3737
defaultValue =
3838
"# Choose from the following templates:\n" +
39+
"# \n" +
40+
"# Define UiParameter inside open() of ProcessTupleOperator, ProcessBatchOperator, or ProcessTableOperator.\n" +
41+
"# Example: self.count = self.UiParameter(\"count\", AttributeType.INT).value\n" +
42+
"# See the Python UDF operator documentation for supported types and behavior.\n" +
3943
"# \n" +
4044
"# from pytexera import *\n" +
4145
"# \n" +
@@ -102,6 +106,7 @@ class PythonUDFOpDescV2 extends LogicalOp {
102106
} else {
103107
opInfo.inputPorts.map(_ => None)
104108
}
109+
val codeWithParameters = injectUiParameters(code)
105110

106111
val propagateSchema = (inputSchemas: Map[PortIdentity, Schema]) => {
107112
val inputSchema = inputSchemas(operatorInfo.inputPorts.head.id)
@@ -130,7 +135,7 @@ class PythonUDFOpDescV2 extends LogicalOp {
130135
workflowId,
131136
executionId,
132137
operatorIdentifier,
133-
OpExecWithCode(code, "python")
138+
OpExecWithCode(codeWithParameters, "python")
134139
)
135140
.withParallelizable(true)
136141
.withSuggestedWorkerNum(workers)
@@ -140,7 +145,7 @@ class PythonUDFOpDescV2 extends LogicalOp {
140145
workflowId,
141146
executionId,
142147
operatorIdentifier,
143-
OpExecWithCode(code, "python")
148+
OpExecWithCode(codeWithParameters, "python")
144149
)
145150
.withParallelizable(false)
146151
}

‎common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/python/PythonUdfUiParameterInjector.scala‎

Lines changed: 26 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,14 +38,27 @@ object PythonUdfUiParameterInjector {
3838
private val InjectedUiParametersHookMethodHeader =
3939
s"def $InjectedUiParametersHookMethodName(self) -> Dict[str, Any]:"
4040
private val UnsupportedUiParameterTypes = Set(AttributeType.BINARY, AttributeType.LARGE_BINARY)
41+
private val UiParameterTypesRequiringValues = Set(
42+
AttributeType.INTEGER,
43+
AttributeType.LONG,
44+
AttributeType.DOUBLE,
45+
AttributeType.BOOLEAN,
46+
AttributeType.TIMESTAMP
47+
)
4148

4249
// Keep supported user-facing UDF class names in sync with the frontend parser.
4350
private val SupportedPythonUdfClassHeaderRegex: Regex =
4451
"""(?m)^([ \t]*)class\s+(ProcessTupleOperator|ProcessBatchOperator|ProcessTableOperator|GenerateOperator)\s*\([^)]*\)\s*:\s*(?:#.*)?$""".r
4552

4653
private def validate(uiParameters: List[UiUDFParameter]): Unit = {
47-
val attributes = uiParameters.map(parameterAttribute)
48-
attributes.foreach(validateSupportedType)
54+
val parametersWithAttributes =
55+
uiParameters.map(parameter => parameter -> parameterAttribute(parameter))
56+
val attributes = parametersWithAttributes.map(_._2)
57+
parametersWithAttributes.foreach {
58+
case (parameter, attribute) =>
59+
validateSupportedType(attribute)
60+
validateRequiredValue(parameter, attribute)
61+
}
4962

5063
attributes
5164
.groupBy(_.getName)
@@ -71,6 +84,17 @@ object PythonUdfUiParameterInjector {
7184
}
7285
}
7386

87+
private def validateRequiredValue(parameter: UiUDFParameter, attribute: Attribute): Unit = {
88+
if (
89+
UiParameterTypesRequiringValues.contains(attribute.getType) &&
90+
Option(parameter.value).forall(_.trim.isEmpty)
91+
) {
92+
throw new RuntimeException(
93+
s"UiParameter '${attribute.getName}' requires a value for type '${attribute.getType.name()}'."
94+
)
95+
}
96+
}
97+
7498
private def buildInjectedParameterEntry(parameter: UiUDFParameter): PythonTemplateBuilder = {
7599
pyb"${parameter.attribute.getName}: ${parameter.value}"
76100
}
Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.texera.amber.operator.udf.python
21+
22+
import com.fasterxml.jackson.annotation.{JsonProperty, JsonPropertyDescription}
23+
import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle
24+
25+
/** Shared serialized UI-parameter property and code-injection hook for Python UDF descriptors. */
26+
trait PythonUdfUiParameterSupport {
27+
28+
@JsonProperty
29+
@JsonSchemaTitle("Parameters")
30+
@JsonPropertyDescription(
31+
"Parameters inferred from active self.UiParameter(...) calls in the Python script"
32+
)
33+
var uiParameters: List[UiUDFParameter] = List()
34+
35+
protected final def injectUiParameters(code: String): String =
36+
PythonUdfUiParameterInjector.inject(code, uiParameters)
37+
}

‎common/workflow-operator/src/main/scala/org/apache/texera/amber/operator/udf/python/source/PythonUDFSourceOpDescV2.scala‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,12 +27,17 @@ import org.apache.texera.amber.core.virtualidentity.{ExecutionIdentity, Workflow
2727
import org.apache.texera.amber.core.workflow.{OutputPort, PhysicalOp, SchemaPropagationFunc}
2828
import org.apache.texera.amber.operator.metadata.{OperatorGroupConstants, OperatorInfo}
2929
import org.apache.texera.amber.operator.source.SourceOperatorDescriptor
30+
import org.apache.texera.amber.operator.udf.python.PythonUdfUiParameterSupport
3031

31-
class PythonUDFSourceOpDescV2 extends SourceOperatorDescriptor {
32+
class PythonUDFSourceOpDescV2 extends SourceOperatorDescriptor with PythonUdfUiParameterSupport {
3233

3334
@JsonProperty(
3435
required = true,
35-
defaultValue = "# from pytexera import *\n" +
36+
defaultValue = "# Define UiParameter inside GenerateOperator.open().\n" +
37+
"# Example: self.count = self.UiParameter(\"count\", AttributeType.INT).value\n" +
38+
"# See the Python UDF operator documentation for supported types and behavior.\n" +
39+
"# \n" +
40+
"# from pytexera import *\n" +
3641
"# class GenerateOperator(UDFSourceOperator):\n" +
3742
"# \n" +
3843
"# @overrides\n" +
@@ -82,9 +87,15 @@ class PythonUDFSourceOpDescV2 extends SourceOperatorDescriptor {
8287
)
8388
trimmed
8489
}
90+
val codeWithParameters = injectUiParameters(code)
8591

8692
val physicalOp = PhysicalOp
87-
.sourcePhysicalOp(workflowId, executionId, operatorIdentifier, OpExecWithCode(code, "python"))
93+
.sourcePhysicalOp(
94+
workflowId,
95+
executionId,
96+
operatorIdentifier,
97+
OpExecWithCode(codeWithParameters, "python")
98+
)
8899
.withInputPorts(operatorInfo.inputPorts)
89100
.withOutputPorts(operatorInfo.outputPorts)
90101
.withIsOneToManyOp(true)

‎common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/DualInputPortsPythonUDFOpDescV2Spec.scala‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,13 @@ class DualInputPortsPythonUDFOpDescV2Spec extends AnyFlatSpec with Matchers {
3434
private val workflowId = WorkflowIdentity(1L)
3535
private val executionId = ExecutionIdentity(1L)
3636

37+
private def uiParameter(name: String, value: String): UiUDFParameter = {
38+
val parameter = new UiUDFParameter
39+
parameter.attribute = new Attribute(name, AttributeType.INTEGER)
40+
parameter.value = value
41+
parameter
42+
}
43+
3744
private def twoPortSchemas(dataSchema: Schema): Map[PortIdentity, Schema] =
3845
Map(
3946
PortIdentity() -> Schema().add(new Attribute("model", AttributeType.STRING)),
@@ -87,6 +94,24 @@ class DualInputPortsPythonUDFOpDescV2Spec extends AnyFlatSpec with Matchers {
8794
physical.suggestedWorkerNum shouldBe Some(2)
8895
}
8996

97+
it should "inject configured UI parameters into the generated Python code" in {
98+
val d = new DualInputPortsPythonUDFOpDescV2
99+
d.code = """from pytexera import *
100+
|
101+
|class ProcessTupleOperator(UDFOperatorV2):
102+
| def process_tuple(self, tuple_, port):
103+
| yield tuple_
104+
|""".stripMargin
105+
d.uiParameters = List(uiParameter("count", "7"))
106+
107+
d.getPhysicalOp(workflowId, executionId).opExecInitInfo match {
108+
case OpExecWithCode(code, "python") =>
109+
code should include("def _texera_injected_ui_parameters")
110+
code should include("self.decode_python_template")
111+
case other => fail(s"expected Python OpExecWithCode, got $other")
112+
}
113+
}
114+
90115
it should "reject a blank virtual-environment name when the default env is disabled" in {
91116
val d = new DualInputPortsPythonUDFOpDescV2
92117
d.defaultEnv = false
@@ -154,6 +179,7 @@ class DualInputPortsPythonUDFOpDescV2Spec extends AnyFlatSpec with Matchers {
154179
d.defaultEnv = false
155180
d.envName = "myenv"
156181
d.outputColumns = List(new Attribute("res", AttributeType.INTEGER))
182+
d.uiParameters = List(uiParameter("count", "7"))
157183
val restored = objectMapper.readValue(objectMapper.writeValueAsString(d), classOf[LogicalOp])
158184
restored shouldBe a[DualInputPortsPythonUDFOpDescV2]
159185
val r = restored.asInstanceOf[DualInputPortsPythonUDFOpDescV2]
@@ -163,5 +189,7 @@ class DualInputPortsPythonUDFOpDescV2Spec extends AnyFlatSpec with Matchers {
163189
r.defaultEnv shouldBe false
164190
r.envName shouldBe "myenv"
165191
r.outputColumns shouldBe List(new Attribute("res", AttributeType.INTEGER))
192+
r.uiParameters.map(_.attribute) shouldBe List(new Attribute("count", AttributeType.INTEGER))
193+
r.uiParameters.map(_.value) shouldBe List("7")
166194
}
167195
}

‎common/workflow-operator/src/test/scala/org/apache/texera/amber/operator/udf/python/PythonUDFOpDescV2Spec.scala‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,13 @@ class PythonUDFOpDescV2Spec extends AnyFlatSpec with Matchers {
3434
private val workflowId = WorkflowIdentity(1L)
3535
private val executionId = ExecutionIdentity(1L)
3636

37+
private def uiParameter(name: String, value: String): UiUDFParameter = {
38+
val parameter = new UiUDFParameter
39+
parameter.attribute = new Attribute(name, AttributeType.INTEGER)
40+
parameter.value = value
41+
parameter
42+
}
43+
3744
"PythonUDFOpDescV2.operatorInfo" should
3845
"advertise the name, Python group, dynamic ports, and a default 1-in/1-out shape" in {
3946
val info = (new PythonUDFOpDescV2).operatorInfo
@@ -74,6 +81,24 @@ class PythonUDFOpDescV2Spec extends AnyFlatSpec with Matchers {
7481
intercept[IllegalArgumentException] { d.getPhysicalOp(workflowId, executionId) }
7582
}
7683

84+
it should "inject configured UI parameters into the generated Python code" in {
85+
val d = new PythonUDFOpDescV2
86+
d.code = """from pytexera import *
87+
|
88+
|class ProcessTupleOperator(UDFOperatorV2):
89+
| def process_tuple(self, tuple_, port):
90+
| yield tuple_
91+
|""".stripMargin
92+
d.uiParameters = List(uiParameter("count", "7"))
93+
94+
d.getPhysicalOp(workflowId, executionId).opExecInitInfo match {
95+
case OpExecWithCode(code, "python") =>
96+
code should include("def _texera_injected_ui_parameters")
97+
code should include("self.decode_python_template")
98+
case other => fail(s"expected Python OpExecWithCode, got $other")
99+
}
100+
}
101+
77102
it should "reject a blank virtual-environment name when the default env is disabled" in {
78103
val d = new PythonUDFOpDescV2
79104
d.defaultEnv = false
@@ -125,6 +150,7 @@ class PythonUDFOpDescV2Spec extends AnyFlatSpec with Matchers {
125150
d.defaultEnv = false
126151
d.envName = "myenv"
127152
d.outputColumns = List(new Attribute("res", AttributeType.INTEGER))
153+
d.uiParameters = List(uiParameter("count", "7"))
128154
val restored = objectMapper.readValue(objectMapper.writeValueAsString(d), classOf[LogicalOp])
129155
restored shouldBe a[PythonUDFOpDescV2]
130156
val p = restored.asInstanceOf[PythonUDFOpDescV2]
@@ -134,6 +160,8 @@ class PythonUDFOpDescV2Spec extends AnyFlatSpec with Matchers {
134160
p.defaultEnv shouldBe false
135161
p.envName shouldBe "myenv"
136162
p.outputColumns shouldBe List(new Attribute("res", AttributeType.INTEGER))
163+
p.uiParameters.map(_.attribute) shouldBe List(new Attribute("count", AttributeType.INTEGER))
164+
p.uiParameters.map(_.value) shouldBe List("7")
137165
}
138166

139167
"PythonUDFOpDescV2.getPhysicalOp" should

0 commit comments

Comments
 (0)