Skip to content

Commit 7deed35

Browse files
feat(workflow-operator): add Python UDF UI parameter injection model (apache#5141)
### What changes were proposed in this PR? This PR adds the Scala backend foundation for Python UDF UI parameters. It introduces: | Area | Change | | --- | --- | | UI parameter model | Adds `UiUDFParameter`, containing backend-compatible `attribute` metadata and an editable `value`. | | Python UDF injector | Adds `PythonUdfUiParameterInjector`, which validates UI parameters and injects a reserved hook method into supported Python UDF classes. | | Safe string encoding | Marks `Attribute.getName()` as encodable so UI parameter names are safely rendered through the Python template builder. | | Test coverage | Adds Scala tests for hook injection, validation, unsupported types, reserved method conflicts, and unchanged behavior when no UI parameters exist. | This PR is stacked after the merged frontend foundation PR apache#5043. It does not yet wire the injector into operator execution; that wiring is handled by later PRs in the stack. Existing Python UDF workflow execution remains unchanged in this PR because `PythonUDFOpDescV2`, `PythonUDFSourceOpDescV2`, and `DualInputPortsPythonUDFOpDescV2` are not modified here. ### Any related issues, documentation, discussions? Part of the Python UDF UI parameter feature split from `feat/ui-parameter`. Related tracking issue / stack: apache#5044 Stack order: 1. Frontend UI parameter building blocks: apache#5043 2. Scala backend injection model: this PR 3. Python runtime support 4. End-to-end integration ### How was this PR tested? Commands run: ```bash sbt "WorkflowOperator / Test / testOnly org.apache.texera.amber.operator.udf.python.PythonUdfUiParameterInjectorSpec" sbt scalafmtAll sbt scalafmtCheckAll "scalafixAll --check" ``` Results: - `PythonUdfUiParameterInjectorSpec`: 10 tests passed. - `scalafmtAll`: no file changes. - `scalafmtCheckAll` and `scalafixAll --check`: passed. ### Was this PR authored or co-authored using generative AI tooling? No --------- Co-authored-by: Xiaozhen Liu <[email protected]>
1 parent 7a6c22a commit 7deed35

6 files changed

Lines changed: 609 additions & 0 deletions

File tree

‎common/pybuilder/src/main/scala/org/apache/texera/amber/pybuilder/PythonLexerUtils.scala‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,68 @@ object PythonLexerUtils {
3737
if (lastNewlineIndex >= 0) s.substring(lastNewlineIndex + 1) else s
3838
}
3939

40+
/**
41+
* Update triple-quoted-string state after scanning one physical Python source line.
42+
*
43+
* This is intentionally lightweight. It only tracks whether scanning is inside a `'''` or `"""` string so callers
44+
* that reason about indentation can avoid treating string contents as real Python statements.
45+
*
46+
* Known limitations: escaped delimiters inside an active triple-quoted string are still treated as closing
47+
* delimiters, and delimiter-like runs next to ordinary string boundaries may be detected because this helper does
48+
* not fully parse Python string literal adjacency.
49+
*/
50+
def updateTripleQuotedStringState(
51+
line: String,
52+
activeDelimiter: Option[String]
53+
): Option[String] = {
54+
var delimiter = activeDelimiter
55+
var inSingleQuotedString = false
56+
var inDoubleQuotedString = false
57+
var escaped = false
58+
var index = 0
59+
60+
while (index < line.length) {
61+
delimiter match {
62+
case Some(active) =>
63+
if (line.startsWith(active, index)) {
64+
delimiter = None
65+
index += active.length
66+
} else {
67+
index += 1
68+
}
69+
70+
case None =>
71+
val char = line.charAt(index)
72+
73+
if (escaped) {
74+
escaped = false
75+
index += 1
76+
} else if ((inSingleQuotedString || inDoubleQuotedString) && char == '\\') {
77+
escaped = true
78+
index += 1
79+
} else if (!inSingleQuotedString && !inDoubleQuotedString && char == '#') {
80+
return delimiter
81+
} else if (!inDoubleQuotedString && line.startsWith("'''", index)) {
82+
delimiter = Some("'''")
83+
index += 3
84+
} else if (!inSingleQuotedString && line.startsWith("\"\"\"", index)) {
85+
delimiter = Some("\"\"\"")
86+
index += 3
87+
} else if (!inDoubleQuotedString && char == '\'') {
88+
inSingleQuotedString = !inSingleQuotedString
89+
index += 1
90+
} else if (!inSingleQuotedString && char == '"') {
91+
inDoubleQuotedString = !inDoubleQuotedString
92+
index += 1
93+
} else {
94+
index += 1
95+
}
96+
}
97+
}
98+
99+
delimiter
100+
}
101+
40102
/**
41103
* Detect whether the provided line tail contains an unclosed single or double quote.
42104
*

‎common/pybuilder/src/test/scala/org/apache/texera/amber/pybuilder/PythonLexerUtilsSpec.scala‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,37 @@ class PythonLexerUtilsSpec extends AnyFunSuite {
100100
assert(PythonLexerUtils.lineTail(text) == "")
101101
}
102102

103+
// -------- updateTripleQuotedStringState --------
104+
105+
test("updateTripleQuotedStringState: enters and exits triple single quoted strings") {
106+
val opened = PythonLexerUtils.updateTripleQuotedStringState("sql = '''", None)
107+
assert(opened.contains("'''"))
108+
109+
val stillOpen = PythonLexerUtils.updateTripleQuotedStringState("SELECT * FROM t", opened)
110+
assert(stillOpen.contains("'''"))
111+
112+
val closed = PythonLexerUtils.updateTripleQuotedStringState("'''", stillOpen)
113+
assert(closed.isEmpty)
114+
}
115+
116+
test("updateTripleQuotedStringState: enters and exits triple double quoted strings") {
117+
val opened = PythonLexerUtils.updateTripleQuotedStringState("sql = \"\"\"", None)
118+
assert(opened.contains("\"\"\""))
119+
120+
val closed = PythonLexerUtils.updateTripleQuotedStringState("\"\"\"", opened)
121+
assert(closed.isEmpty)
122+
}
123+
124+
test("updateTripleQuotedStringState: ignores triple quotes in single-line comments") {
125+
val state = PythonLexerUtils.updateTripleQuotedStringState("# ''' not a string", None)
126+
assert(state.isEmpty)
127+
}
128+
129+
test("updateTripleQuotedStringState: ignores triple quotes inside ordinary strings") {
130+
val state = PythonLexerUtils.updateTripleQuotedStringState("value = \"'''\"", None)
131+
assert(state.isEmpty)
132+
}
133+
103134
// -------- hasUnclosedQuote --------
104135

105136
test("hasUnclosedQuote: empty string has no unclosed quote") {

‎common/workflow-core/src/main/scala/org/apache/texera/amber/core/tuple/Attribute.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121

2222
import com.fasterxml.jackson.annotation.JsonCreator;
2323
import com.fasterxml.jackson.annotation.JsonProperty;
24+
import org.apache.texera.amber.pybuilder.EncodableStringAnnotation;
2425

2526
import javax.validation.constraints.NotBlank;
2627
import javax.validation.constraints.NotNull;
@@ -49,6 +50,7 @@ public Attribute(
4950

5051
@JsonProperty(value = "attributeName", required = true)
5152
@NotBlank(message = "Attribute name is required")
53+
@EncodableStringAnnotation
5254
public String getName() {
5355
return attributeName;
5456
}
Lines changed: 205 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,205 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
package org.apache.texera.amber.operator.udf.python
20+
21+
import org.apache.texera.amber.core.tuple.{Attribute, AttributeType}
22+
import org.apache.texera.amber.pybuilder.PythonLexerUtils.updateTripleQuotedStringState
23+
import org.apache.texera.amber.pybuilder.PythonTemplateBuilder
24+
import org.apache.texera.amber.pybuilder.PythonTemplateBuilder.PythonTemplateBuilderStringContext
25+
26+
import scala.util.matching.Regex
27+
28+
/**
29+
* Injects the reserved UI-parameter hook into user-written Python UDF code.
30+
*
31+
* Operator descriptors should call this after loading saved [[UiUDFParameter]] values and before sending Python source
32+
* to runtime execution. The injected hook returns decoded parameter names and values that Python runtime support reads
33+
* before the user's `open()` method runs.
34+
*/
35+
object PythonUdfUiParameterInjector {
36+
37+
private val InjectedUiParametersHookMethodName = "_texera_injected_ui_parameters"
38+
private val InjectedUiParametersHookMethodHeader =
39+
s"def $InjectedUiParametersHookMethodName(self) -> Dict[str, Any]:"
40+
private val UnsupportedUiParameterTypes = Set(AttributeType.BINARY, AttributeType.LARGE_BINARY)
41+
42+
// Keep supported user-facing UDF class names in sync with the frontend parser.
43+
private val SupportedPythonUdfClassHeaderRegex: Regex =
44+
"""(?m)^([ \t]*)class\s+(ProcessTupleOperator|ProcessBatchOperator|ProcessTableOperator|GenerateOperator)\s*\([^)]*\)\s*:\s*(?:#.*)?$""".r
45+
46+
private def validate(uiParameters: List[UiUDFParameter]): Unit = {
47+
val attributes = uiParameters.map(parameterAttribute)
48+
attributes.foreach(validateSupportedType)
49+
50+
attributes
51+
.groupBy(_.getName)
52+
.collectFirst {
53+
case (parameterName, matchingAttributes) if matchingAttributes.size > 1 => parameterName
54+
}
55+
.foreach { duplicateName =>
56+
throw new RuntimeException(s"UiParameter name '$duplicateName' is declared more than once.")
57+
}
58+
}
59+
60+
private def parameterAttribute(parameter: UiUDFParameter): Attribute =
61+
Option(parameter).flatMap(parameter => Option(parameter.attribute)).getOrElse {
62+
throw new RuntimeException("UiParameter attribute is required.")
63+
}
64+
65+
private def validateSupportedType(attribute: Attribute): Unit = {
66+
if (UnsupportedUiParameterTypes.contains(attribute.getType)) {
67+
throw new RuntimeException(
68+
s"UiParameter type '${attribute.getType.name()}' is not supported. " +
69+
"Use string, integer, long, double, boolean, or timestamp instead."
70+
)
71+
}
72+
}
73+
74+
private def buildInjectedParameterEntry(parameter: UiUDFParameter): PythonTemplateBuilder = {
75+
pyb"${parameter.attribute.getName}: ${parameter.value}"
76+
}
77+
78+
private def buildInjectedParametersMap(
79+
uiParameters: List[UiUDFParameter]
80+
): PythonTemplateBuilder = {
81+
val entries = uiParameters.map(buildInjectedParameterEntry)
82+
entries.reduceOption((acc, entry) => acc + pyb", " + entry).getOrElse(pyb"")
83+
}
84+
85+
private def buildInjectedHookMethod(uiParameters: List[UiUDFParameter]): String = {
86+
val injectedParametersMap = buildInjectedParametersMap(uiParameters)
87+
88+
(pyb"""|# Follow-up runtime support exports Dict/Any and defines the base hook that @overrides targets.
89+
|@overrides
90+
|$InjectedUiParametersHookMethodHeader
91+
| return {""" +
92+
injectedParametersMap +
93+
pyb"""}
94+
|""").encode
95+
}
96+
97+
private def indentBlock(block: String, indent: String): String = {
98+
block
99+
.split("\n", -1)
100+
.map { line =>
101+
if (line.nonEmpty) indent + line else line
102+
}
103+
.mkString("\n")
104+
}
105+
106+
private def lineEndIndex(text: String, from: Int): Int = {
107+
val lineEnd = text.indexOf('\n', from)
108+
if (lineEnd < 0) text.length else lineEnd
109+
}
110+
111+
private def detectClassBlockEnd(code: String, classHeaderStart: Int, classIndent: String): Int = {
112+
val classLineEnd = lineEndIndex(code, classHeaderStart)
113+
var lineStart = if (classLineEnd < code.length) classLineEnd + 1 else code.length
114+
var tripleQuotedStringDelimiter: Option[String] = None
115+
116+
while (lineStart < code.length) {
117+
val lineEnd = lineEndIndex(code, lineStart)
118+
val line = code.substring(lineStart, lineEnd)
119+
120+
val trimmed = line.trim
121+
val isBlank = trimmed.isEmpty
122+
123+
val currentIndentLen = line.segmentLength(ch => ch == ' ' || ch == '\t')
124+
val classIndentLen = classIndent.length
125+
126+
if (tripleQuotedStringDelimiter.isEmpty && !isBlank && currentIndentLen <= classIndentLen) {
127+
return lineStart
128+
}
129+
130+
tripleQuotedStringDelimiter = updateTripleQuotedStringState(line, tripleQuotedStringDelimiter)
131+
132+
lineStart = if (lineEnd < code.length) lineEnd + 1 else code.length
133+
}
134+
135+
code.length
136+
}
137+
138+
private def containsReservedHook(classBlock: String): Boolean = {
139+
val hookRegex =
140+
("""(?m)^[ \t]+def\s+""" + Regex.quote(InjectedUiParametersHookMethodName) + """\s*\(""").r
141+
hookRegex.findFirstIn(classBlock).isDefined
142+
}
143+
144+
private def injectHookIntoUserClass(userCode: String, hookMethod: String): String = {
145+
val classHeaderMatch =
146+
SupportedPythonUdfClassHeaderRegex.findFirstMatchIn(userCode).getOrElse {
147+
throw new RuntimeException(
148+
"UiParameters were provided, but no supported Python UDF class was found. " +
149+
"Use one of ProcessTupleOperator, ProcessBatchOperator, ProcessTableOperator, or GenerateOperator."
150+
)
151+
}
152+
153+
val classHeaderStart = classHeaderMatch.start
154+
val classIndent = classHeaderMatch.group(1)
155+
val classBlockEnd = detectClassBlockEnd(userCode, classHeaderStart, classIndent)
156+
157+
val classBlock = userCode.substring(classHeaderStart, classBlockEnd)
158+
159+
if (containsReservedHook(classBlock)) {
160+
throw new RuntimeException(
161+
s"Reserved method '$InjectedUiParametersHookMethodName' is already defined in the UDF class. Please rename your method."
162+
)
163+
}
164+
165+
val bodyIndent = inferClassBodyIndent(classBlock, classIndent).getOrElse(classIndent + " ")
166+
val indentedHook = indentBlock(
167+
(if (classBlock.endsWith("\n")) "" else "\n") + hookMethod.trim + "\n",
168+
bodyIndent
169+
)
170+
171+
userCode.substring(0, classBlockEnd) +
172+
indentedHook +
173+
userCode.substring(classBlockEnd)
174+
}
175+
176+
private def inferClassBodyIndent(classBlock: String, classIndent: String): Option[String] = {
177+
val lines = classBlock.split("\n", -1).toList.drop(1)
178+
179+
lines.collectFirst {
180+
case line if line.trim.nonEmpty =>
181+
val leading = line.takeWhile(ch => ch == ' ' || ch == '\t')
182+
if (leading.length > classIndent.length) leading else classIndent + " "
183+
}
184+
}
185+
186+
/**
187+
* Returns Python code with the UI-parameter hook injected into the supported UDF class.
188+
*
189+
* If `uiParameters` is empty, the code is returned unchanged. Throws [[RuntimeException]] when parameter metadata is
190+
* invalid, the user already defines the reserved hook method, or parameters are provided for an unsupported class.
191+
*/
192+
def inject(code: String, uiParameters: List[UiUDFParameter]): String = {
193+
val parameters = Option(uiParameters).getOrElse(List.empty)
194+
validate(parameters)
195+
196+
val userCode = Option(code).getOrElse("")
197+
198+
if (parameters.isEmpty) {
199+
return userCode
200+
}
201+
202+
val hookMethod = buildInjectedHookMethod(parameters)
203+
injectHookIntoUserClass(userCode, hookMethod)
204+
}
205+
}
Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,47 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one
3+
* or more contributor license agreements. See the NOTICE file
4+
* distributed with this work for additional information
5+
* regarding copyright ownership. The ASF licenses this file
6+
* to you under the Apache License, Version 2.0 (the
7+
* "License"); you may not use this file except in compliance
8+
* with the License. You may obtain a copy of the License at
9+
*
10+
* http://www.apache.org/licenses/LICENSE-2.0
11+
*
12+
* Unless required by applicable law or agreed to in writing,
13+
* software distributed under the License is distributed on an
14+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15+
* KIND, either express or implied. See the License for the
16+
* specific language governing permissions and limitations
17+
* under the License.
18+
*/
19+
20+
package org.apache.texera.amber.operator.udf.python
21+
22+
import com.fasterxml.jackson.annotation.JsonProperty
23+
import com.kjetland.jackson.jsonSchema.annotations.JsonSchemaTitle
24+
import org.apache.texera.amber.core.tuple.Attribute
25+
import org.apache.texera.amber.pybuilder.PyStringTypes.EncodableString
26+
27+
import javax.validation.Valid
28+
import javax.validation.constraints.NotNull
29+
30+
/**
31+
* Serialized operator property for one Python UDF UI parameter.
32+
*
33+
* `attribute` carries the inferred parameter name and type. `value` is user-entered text and is marked as
34+
* [[EncodableString]] so Python code generation decodes it at runtime instead of embedding raw text into generated code.
35+
*/
36+
class UiUDFParameter {
37+
38+
@JsonProperty(required = true)
39+
@JsonSchemaTitle("Attribute")
40+
@Valid
41+
@NotNull(message = "Attribute is required")
42+
var attribute: Attribute = _
43+
44+
@JsonProperty()
45+
@JsonSchemaTitle("Value")
46+
var value: EncodableString = ""
47+
}

0 commit comments

Comments
 (0)