Skip to content

Commit fd46a3f

Browse files
committed
feat: bind $i in generated Python operators' text properties
1 parent b2912c0 commit fd46a3f

15 files changed

Lines changed: 1132 additions & 63 deletions

File tree

‎amber/src/main/python/core/models/operator.py‎

Lines changed: 66 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,16 @@
2020
from functools import lru_cache
2121
from abc import ABC, abstractmethod
2222
from collections import defaultdict
23-
from typing import Iterator, List, Mapping, Optional, Union, MutableMapping, Protocol
23+
from typing import (
24+
Any,
25+
Iterator,
26+
List,
27+
Mapping,
28+
Optional,
29+
Union,
30+
MutableMapping,
31+
Protocol,
32+
)
2433

2534
from . import Table, TableLike, Tuple, TupleLike, Batch, BatchLike
2635
from .state import State
@@ -74,16 +83,68 @@ def _get_template_decoder(self) -> "Operator.PythonTemplateDecoder":
7483
def decode_python_template(self, data: Union[str, bytes]) -> str:
7584
return self._get_template_decoder().decode(data)
7685

86+
def loop_variable_text(self, name: str) -> str:
87+
"""
88+
The text of loop variable ``name`` in the state messages handed to this
89+
operator, the latest one that carried it winning. Inside a control
90+
block, the code generated for an operator calls this where a text
91+
property holds ``$name``, in place of decoding the literal, so the
92+
value is read when the code runs. It is spelled as a JVM operator's
93+
text property binds it (``LateBoundExecutor``): a boolean as ``true``
94+
or ``false``, and only a scalar at all.
95+
96+
:param name: str, the loop variable's name, without the leading ``$``.
97+
:return: str, the variable's value, as text.
98+
"""
99+
if self._loop_variables is None:
100+
raise RuntimeError(
101+
f"loop variable ${name} is read before the iteration's state arrived"
102+
)
103+
if name not in self._loop_variables:
104+
raise RuntimeError(
105+
f"the operator refers to loop variable ${name}, "
106+
"but no state message carried it"
107+
)
108+
value = self._loop_variables[name]
109+
if isinstance(value, bool):
110+
return "true" if value else "false"
111+
if isinstance(value, (str, int, float)):
112+
return str(value)
113+
raise RuntimeError(
114+
f"the operator refers to loop variable ${name}, "
115+
f"but its value {value!r} is not a scalar"
116+
)
117+
118+
def register_loop_state(self, state: State) -> None:
119+
"""
120+
Register ``state`` as the latest state message handed to this operator.
121+
It becomes ``loop_state``, and its values join the loop variables
122+
``loop_variable_text`` reads, replacing those of the same name. The
123+
runtime calls this right before ``process_state``.
124+
125+
:param state: State, the state message that arrived.
126+
"""
127+
self.loop_state = state
128+
self._loop_variables = {**(self._loop_variables or {}), **state}
129+
77130
__internal_is_source: bool = False
78131

79132
# The state message most recently handed to this operator -- inside a
80133
# control block, the iteration's loop variables. The runtime registers it
81-
# right before ``process_state`` runs (see ``DataProcessor.process_state``),
82-
# so the callback and every call after it can consult it. A class-level
83-
# default rather than an ``__init__`` assignment: a subclass ``__init__``
84-
# that skips ``super()`` still reads ``None`` until a state arrives.
134+
# (``register_loop_state``) right before ``process_state`` runs (see
135+
# ``DataProcessor.process_state``), so the callback and every call after
136+
# it can consult it. A class-level default rather than an ``__init__``
137+
# assignment: a subclass ``__init__`` that skips ``super()`` still reads
138+
# ``None`` until a state arrives.
85139
loop_state: Optional[State] = None
86140

141+
# The values of every state message registered so far, a later message's
142+
# winning, as the JVM's LateBoundExecutor merges them: a loop body
143+
# operator's own state, or a nested loop's inner one, arrives after the
144+
# loop's and must not hide its variables. ``None`` until the first; a
145+
# class-level default for the same reason as ``loop_state``.
146+
_loop_variables: Optional[Mapping[str, Any]] = None
147+
87148
@property
88149
@overrides.final
89150
def is_source(self) -> bool:

‎amber/src/main/python/core/runnables/data_processor.py‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -85,8 +85,9 @@ def process_state(self, state: State) -> None:
8585
"""
8686
with self._executor_session() as (executor, port_id):
8787
# Register the state message before the callback runs, so
88-
# process_state (and every call after it) can consult loop_state.
89-
executor.loop_state = state
88+
# process_state (and every call after it) can consult loop_state,
89+
# and read every loop variable the messages so far carried.
90+
executor.register_loop_state(state)
9091
self._set_output_state(executor.process_state(state, port_id))
9192

9293
def process_tuple(self) -> None:

‎amber/src/main/python/core/runnables/main_loop.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -202,7 +202,7 @@ def _consume_pending_loop_state(self, executor: LoopEndOperator) -> None:
202202
):
203203
# This deferred consume bypasses DataProcessor.process_state, so
204204
# register the state here too, before the callback runs.
205-
executor.loop_state = pending
205+
executor.register_loop_state(pending)
206206
executor.process_state(pending, 0)
207207

208208
def _jump_to_loop_start(

‎amber/src/test/integration/org/apache/texera/amber/engine/e2e/LoopIntegrationSpec.scala‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,13 @@ import org.apache.texera.amber.operator.filter.{
4848
}
4949
import org.apache.texera.amber.operator.limit.LimitOpDesc
5050
import org.apache.texera.amber.operator.loop.{LoopEndOpDesc, LoopStartOpDesc}
51+
import org.apache.texera.amber.operator.machineLearning.sklearnAdvanced.SVRTrainer.{
52+
SklearnAdvancedSVRParameters,
53+
SklearnAdvancedSVRTrainerOpDesc
54+
}
55+
import org.apache.texera.amber.operator.machineLearning.sklearnAdvanced.base.HyperParameters
5156
import org.apache.texera.amber.operator.sleep.SleepOpDesc
57+
import org.apache.texera.amber.operator.source.scan.FileAttributeType
5258
import org.apache.texera.amber.operator.source.scan.text.TextInputSourceOpDesc
5359
import org.apache.texera.amber.operator.udf.python.PythonUDFOpDescV2
5460
import org.apache.texera.amber.tags.IntegrationTest
@@ -193,6 +199,22 @@ class LoopIntegrationSpec
193199
op
194200
}
195201

202+
/**
203+
* An SVM regressor fitting `line` on itself, with one hyperparameter row: `C = c`. A Python
204+
* operator whose code the descriptor generates before the loop runs; its `Parameters` output
205+
* column records each fit's hyperparameters as text ("C = 2.0,").
206+
*/
207+
private def svrTrainer(c: String): SklearnAdvancedSVRTrainerOpDesc = {
208+
val op = new SklearnAdvancedSVRTrainerOpDesc()
209+
val row = new HyperParameters[SklearnAdvancedSVRParameters]()
210+
row.parameter = SklearnAdvancedSVRParameters.C
211+
row.value = c
212+
op.paraList = List(row)
213+
op.selectedFeatures = List("line")
214+
op.groundTruthAttribute = "line"
215+
op
216+
}
217+
196218
/** `attribute = value`; a `$name` value refers to a loop variable (bound at the worker). */
197219
private def filterEquals(attribute: String, value: String): SpecializedFilterOpDesc = {
198220
val op = new SpecializedFilterOpDesc()
@@ -356,6 +378,51 @@ class LoopIntegrationSpec
356378
)
357379
}
358380

381+
it should "read a $i loop-variable reference in a Python-generated operator's property on every iteration" in {
382+
// TextInput(1.0, 2.0, 3.0) -> LoopStart(i = 1 / table) -> SVR trainer(C = $i) -> LoopEnd.
383+
//
384+
// The trainer's Python is generated before the loop runs. pyb renders the C value "$i" as an
385+
// expression the operator evaluates when it trains, and because the trainer sits inside the
386+
// block, the compiler had that expression read `i` from the iteration's state message
387+
// (self.loop_variable_text('i')), which the region's input reader replays before the tuples.
388+
// LoopStart hands the whole table to both inputs (training, then parameter) every iteration,
389+
// so each iteration fits once and the LoopEnd accumulates one row per iteration, its
390+
// Parameters naming the C that fit used. The literal "$i" would fail float("$i") and fail
391+
// the run; a reference that never rebinds to the new i would repeat one C.
392+
val src = textInput("1\n2\n3")
393+
src.attributeType = FileAttributeType.DOUBLE
394+
val start = loopStart("i = 1", "table")
395+
val trainer = svrTrainer("$i")
396+
val end = loopEnd("i += 1", "i <= len(table)")
397+
val results = runWorkflowAndReadResults(
398+
system,
399+
buildWorkflow(
400+
List(src, start, trainer, end),
401+
List(
402+
link(src, start),
403+
link(start, trainer),
404+
LogicalLink(
405+
start.operatorIdentifier,
406+
PortIdentity(),
407+
trainer.operatorIdentifier,
408+
PortIdentity(1)
409+
),
410+
link(trainer, end)
411+
),
412+
materializedContext()
413+
),
414+
List(end.operatorIdentifier),
415+
_.get().toList,
416+
Duration.fromSeconds(90)
417+
)
418+
val parameters =
419+
results.getOrElse(end.operatorIdentifier, Nil).map(_.getField[String]("Parameters")).sorted
420+
assert(
421+
parameters == List("C = 1.0,", "C = 2.0,", "C = 3.0,"),
422+
s"each iteration's fit must use its own i as C: got $parameters"
423+
)
424+
}
425+
359426
it should "run a nested loop whose inner body chains multiple JVM operators" in {
360427
// TextInput -> OuterStart -> InnerStart -> Limit -> Sleep(0) -> InnerEnd -> OuterEnd.
361428
//

‎amber/src/test/python/core/models/test_operator.py‎

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -313,6 +313,180 @@ def test_decode_python_template_delegates_to_lazy_decoder(self):
313313
assert op.decode_python_template(encoded) == "payload"
314314

315315

316+
# The shape the backend gives a Python-generated operator's code inside a
317+
# control block: where a text property holds `$name`, the rendered
318+
# `self.decode_python_template('<base64 of "$name">')` is replaced by
319+
# `self.loop_variable_text('name')`. Mirrors the sklearn trainers'
320+
# `<name> = <type> (<value>),` hyperparameter rendering, including the lambda
321+
# the SVC trainer emits as its boolean converter.
322+
_GENERATED_TRAINER_TEMPLATE = """from pytexera import *
323+
324+
class ProcessTableOperator(UDFTableOperator):
325+
326+
@overrides
327+
def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]:
328+
yield dict(C = float (self.loop_variable_text('c')),kernel = str (self.loop_variable_text('kernel')),probability = (lambda value: value.lower() == "true") (self.loop_variable_text('p')),)
329+
"""
330+
331+
332+
class TestLoopVariableText:
333+
def test_returns_the_text_of_an_int_a_float_and_a_string(self):
334+
op = _ConcreteOperator()
335+
op.register_loop_state(State({"n": 3, "rate": 0.5, "ticker": "AAPL"}))
336+
assert op.loop_variable_text("n") == "3"
337+
assert op.loop_variable_text("rate") == "0.5"
338+
assert op.loop_variable_text("ticker") == "AAPL"
339+
340+
def test_spells_a_boolean_as_the_jvm_binds_one(self):
341+
# A JVM operator's text property takes a boolean as true / false
342+
# (LateBoundExecutor), and so does this, not Python's True / False.
343+
op = _ConcreteOperator()
344+
op.register_loop_state(State({"on": True, "off": False}))
345+
assert op.loop_variable_text("on") == "true"
346+
assert op.loop_variable_text("off") == "false"
347+
348+
def test_a_falsy_value_is_still_carried(self):
349+
# Zero, False and the empty string are values the state carried, not
350+
# absent ones: none of them may fall into the missing-name error.
351+
op = _ConcreteOperator()
352+
op.register_loop_state(State({"n": 0, "flag": False, "note": ""}))
353+
assert op.loop_variable_text("n") == "0"
354+
assert op.loop_variable_text("flag") == "false"
355+
assert op.loop_variable_text("note") == ""
356+
357+
@pytest.mark.parametrize("value", [None, [1, 2], {"p": 2}, b"raw"], ids=repr)
358+
def test_raises_for_a_value_that_is_not_a_scalar(self, value):
359+
# The JVM rejects a non-scalar for a text property too, rather than
360+
# pass on a spelling of its own, such as Python's repr.
361+
op = _ConcreteOperator()
362+
op.register_loop_state(State({"m": value}))
363+
with pytest.raises(RuntimeError) as excinfo:
364+
op.loop_variable_text("m")
365+
assert str(excinfo.value) == (
366+
f"the operator refers to loop variable $m, but its value {value!r} "
367+
"is not a scalar"
368+
)
369+
370+
def test_a_value_that_looks_like_a_reference_is_returned_verbatim(self):
371+
# The text is the value itself; it is never resolved again.
372+
op = _ConcreteOperator()
373+
op.register_loop_state(State({"ticker": "$AAPL", "AAPL": "not me"}))
374+
assert op.loop_variable_text("ticker") == "$AAPL"
375+
376+
def test_a_later_state_message_replaces_a_value_it_carries(self):
377+
# Every iteration delivers a fresh state message; the text is read
378+
# from the latest one that carried the name, never cached.
379+
op = _ConcreteOperator()
380+
op.register_loop_state(State({"i": 1}))
381+
assert op.loop_variable_text("i") == "1"
382+
op.register_loop_state(State({"i": 2}))
383+
assert op.loop_variable_text("i") == "2"
384+
385+
def test_a_later_state_message_without_the_name_keeps_its_value(self):
386+
# A loop body operator's own state (here a UDF publishing centroids)
387+
# reaches the operator after the loop's: it must not hide `i`.
388+
op = _ConcreteOperator()
389+
op.register_loop_state(State({"i": 1}))
390+
op.register_loop_state(State({"centroids": 3}))
391+
assert op.loop_variable_text("i") == "1"
392+
assert op.loop_variable_text("centroids") == "3"
393+
394+
def test_an_inner_loops_state_shadows_the_outer_one_and_keeps_the_rest(self):
395+
# A nested loop's body receives the outer loop's state first, then the
396+
# inner Loop Start's, as a JVM operator does.
397+
op = _ConcreteOperator()
398+
op.register_loop_state(State({"j": 5, "i": "outer"}))
399+
op.register_loop_state(State({"i": 0}))
400+
assert op.loop_variable_text("j") == "5"
401+
assert op.loop_variable_text("i") == "0"
402+
403+
def test_registering_makes_the_message_loop_state_and_changes_no_message(self):
404+
op = _ConcreteOperator()
405+
first, second = State({"i": 1}), State({"j": 2})
406+
op.register_loop_state(first)
407+
op.register_loop_state(second)
408+
assert op.loop_state is second
409+
# The merge is the operator's own: neither message gains a key.
410+
assert first == State({"i": 1})
411+
assert second == State({"j": 2})
412+
413+
def test_registering_reaches_the_one_operator_only(self):
414+
# The defaults live on the class, so a registration written there
415+
# would reach every operator.
416+
target, bystander = _ConcreteOperator(), _ConcreteOperator()
417+
target.register_loop_state(State({"i": 1}))
418+
assert target.loop_variable_text("i") == "1"
419+
with pytest.raises(RuntimeError, match="read before the iteration's state"):
420+
bystander.loop_variable_text("i")
421+
assert bystander.loop_state is None
422+
assert _ConcreteOperator.loop_state is None
423+
424+
def test_raises_when_read_before_the_iterations_state_arrived(self):
425+
op = _ConcreteOperator()
426+
with pytest.raises(RuntimeError) as excinfo:
427+
op.loop_variable_text("i")
428+
assert str(excinfo.value) == (
429+
"loop variable $i is read before the iteration's state arrived"
430+
)
431+
432+
def test_raises_for_a_name_no_state_message_carried(self):
433+
op = _ConcreteOperator()
434+
op.register_loop_state(State({"i": 1}))
435+
op.register_loop_state(State({"k": 2}))
436+
with pytest.raises(RuntimeError) as excinfo:
437+
op.loop_variable_text("j")
438+
assert str(excinfo.value) == (
439+
"the operator refers to loop variable $j, but no state message carried it"
440+
)
441+
442+
def test_an_empty_state_has_arrived_but_carries_no_name(self):
443+
# An empty state is a state that arrived: the missing-name error, not
444+
# the not-yet-arrived one.
445+
op = _ConcreteOperator()
446+
op.register_loop_state(State())
447+
with pytest.raises(RuntimeError, match="no state message carried it"):
448+
op.loop_variable_text("i")
449+
450+
def test_a_prefix_of_a_declared_name_is_not_matched(self):
451+
op = _ConcreteOperator()
452+
op.register_loop_state(State({"items": 3}))
453+
with pytest.raises(RuntimeError, match=r"\$item, but no state message"):
454+
op.loop_variable_text("item")
455+
456+
def test_an_extension_of_a_declared_name_is_not_matched(self):
457+
op = _ConcreteOperator()
458+
op.register_loop_state(State({"item": 3}))
459+
with pytest.raises(RuntimeError, match=r"\$items, but no state message"):
460+
op.loop_variable_text("items")
461+
462+
def test_names_are_case_sensitive(self):
463+
op = _ConcreteOperator()
464+
op.register_loop_state(State({"i": 1}))
465+
with pytest.raises(RuntimeError, match=r"\$I, but no state message"):
466+
op.loop_variable_text("I")
467+
468+
def test_generated_code_reads_the_iterations_value_at_run_time(self):
469+
# __name__ so the exec'd class's methods get a real __module__ (the
470+
# @overrides decorator on the generated methods inspects it).
471+
namespace: dict = {"__name__": "generated_trainer_operator"}
472+
exec(_GENERATED_TRAINER_TEMPLATE, namespace)
473+
op = namespace["ProcessTableOperator"]()
474+
475+
# Before the iteration's state arrives, the generated call fails loud
476+
# instead of converting a stand-in value.
477+
with pytest.raises(RuntimeError, match="read before the iteration's state"):
478+
list(op.process_table(Table([Tuple({"a": 1})]), 0))
479+
480+
op.register_loop_state(State({"c": 2, "kernel": "rbf", "p": True}))
481+
(params,) = list(op.process_table(Table([Tuple({"a": 1})]), 0))
482+
assert params == {"C": 2.0, "kernel": "rbf", "probability": True}
483+
484+
# The next iteration's state reaches the same operator instance.
485+
op.register_loop_state(State({"c": 0.25, "kernel": "linear", "p": False}))
486+
(params,) = list(op.process_table(Table([Tuple({"a": 1})]), 0))
487+
assert params == {"C": 0.25, "kernel": "linear", "probability": False}
488+
489+
316490
class TestBatchOperatorValidation:
317491
def test_validate_batch_size_rejects_none(self):
318492
with pytest.raises(ValueError, match="cannot be None"):

0 commit comments

Comments
 (0)