Skip to content

Commit 15ee0b7

Browse files
committed
feat: bind numeric and boolean $i in generated operators
1 parent fd46a3f commit 15ee0b7

6 files changed

Lines changed: 1258 additions & 63 deletions

File tree

‎amber/src/main/python/core/models/operator.py‎

Lines changed: 58 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
from functools import lru_cache
2121
from abc import ABC, abstractmethod
2222
from collections import defaultdict
23+
from decimal import Decimal, InvalidOperation
2324
from typing import (
2425
Any,
2526
Iterator,
@@ -115,12 +116,67 @@ def loop_variable_text(self, name: str) -> str:
115116
f"but its value {value!r} is not a scalar"
116117
)
117118

119+
def loop_variable_value(self, name: str, kind: str) -> Union[int, float, bool]:
120+
"""
121+
The value of loop variable ``name``, its text (``loop_variable_text``)
122+
converted to ``kind``. Inside a control block, the code generated for
123+
an operator calls this where a numeric or boolean property holds
124+
``$name``, in place of the value written into the code, so the value is
125+
read when the code runs. It is converted as a JVM operator's numeric or
126+
boolean property binds it (``LateBoundExecutor``), from its text: an
127+
integer must be integral and fit a signed 64-bit long, a number is any
128+
numeral, and a boolean is ``true`` or ``false`` in any case, never ``1``
129+
or ``0``.
130+
131+
:param name: str, the loop variable's name, without the leading ``$``.
132+
:param kind: str, ``integer``, ``number`` or ``boolean``: the JSON type
133+
of the property that holds ``$name``.
134+
:return: int, float or bool, the variable's value as ``kind``.
135+
"""
136+
if kind not in ("integer", "number", "boolean"):
137+
raise ValueError(
138+
f"loop variable ${name} cannot be read as {kind!r}: "
139+
"the kind is one of 'integer', 'number' and 'boolean'"
140+
)
141+
text = self.loop_variable_text(name).strip()
142+
value = self._loop_variables[name]
143+
144+
def not_a(description: str) -> RuntimeError:
145+
return RuntimeError(
146+
f"the operator refers to loop variable ${name}, "
147+
f"but its value {value!r} is not {description}"
148+
)
149+
150+
if kind == "boolean":
151+
if text.lower() not in ("true", "false"):
152+
raise not_a("a boolean")
153+
return text.lower() == "true"
154+
description = "a number" if kind == "number" else "an integer"
155+
try:
156+
numeral = Decimal(text)
157+
except InvalidOperation:
158+
raise not_a(description) from None
159+
if kind == "number":
160+
try:
161+
return float(numeral)
162+
except ValueError: # a signalling NaN
163+
raise not_a(description) from None
164+
# BigDecimal.longValueExact: integral and within a signed 64-bit long.
165+
if (
166+
not numeral.is_finite()
167+
or numeral != numeral.to_integral_value()
168+
or not -(2**63) <= numeral < 2**63
169+
):
170+
raise not_a(description)
171+
return int(numeral)
172+
118173
def register_loop_state(self, state: State) -> None:
119174
"""
120175
Register ``state`` as the latest state message handed to this operator.
121176
It becomes ``loop_state``, and its values join the loop variables
122-
``loop_variable_text`` reads, replacing those of the same name. The
123-
runtime calls this right before ``process_state``.
177+
``loop_variable_text`` and ``loop_variable_value`` read, replacing
178+
those of the same name. The runtime calls this right before
179+
``process_state``.
124180
125181
:param state: State, the state message that arrived.
126182
"""

‎amber/src/test/python/core/models/test_operator.py‎

Lines changed: 240 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
# under the License.
1717

1818
import base64
19+
import math
1920

2021
import pandas
2122
import pytest
@@ -487,6 +488,245 @@ def test_generated_code_reads_the_iterations_value_at_run_time(self):
487488
assert params == {"C": 0.25, "kernel": "linear", "probability": False}
488489

489490

491+
# The shape the backend gives a Python-generated operator's code inside a
492+
# control block where a numeric or boolean property holds `$name`: the value
493+
# it wrote into the code is replaced by
494+
# `self.loop_variable_value('<name>', '<kind>')`. Mirrors the 2D histogram's
495+
# `nbinsx=$xBins,` rendering, with a number and a boolean keyword beside it.
496+
_GENERATED_HISTOGRAM_TEMPLATE = """from pytexera import *
497+
498+
class ProcessTableOperator(UDFTableOperator):
499+
500+
@overrides
501+
def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]:
502+
yield dict(
503+
nbinsx=self.loop_variable_value('n', 'integer'),
504+
nbinsy=self.loop_variable_value('n', 'integer'),
505+
opacity=self.loop_variable_value('alpha', 'number'),
506+
text_auto=self.loop_variable_value('labels', 'boolean')
507+
)
508+
"""
509+
510+
511+
class TestLoopVariableValue:
512+
@staticmethod
513+
def _read(value, kind):
514+
op = _ConcreteOperator()
515+
op.register_loop_state(State({"v": value}))
516+
return op.loop_variable_value("v", kind)
517+
518+
@staticmethod
519+
def _rejection(value, kind):
520+
op = _ConcreteOperator()
521+
op.register_loop_state(State({"v": value}))
522+
with pytest.raises(RuntimeError) as excinfo:
523+
op.loop_variable_value("v", kind)
524+
return str(excinfo.value)
525+
526+
@pytest.mark.parametrize(
527+
"value, expected",
528+
[
529+
(3, 3),
530+
(-7, -7),
531+
(0, 0),
532+
# An integral float, as BigDecimal("3.0").longValueExact() takes it,
533+
# including one Python spells with an exponent (1e+16).
534+
(3.0, 3),
535+
(1e16, 10**16),
536+
("42", 42),
537+
(" 42 ", 42),
538+
("-5", -5),
539+
("+5", 5),
540+
("4.2e1", 42),
541+
(2**63 - 1, 2**63 - 1),
542+
(-(2**63), -(2**63)),
543+
],
544+
ids=repr,
545+
)
546+
def test_an_integer_is_read_from_every_scalar_that_holds_one(self, value, expected):
547+
result = self._read(value, "integer")
548+
assert result == expected
549+
assert type(result) is int
550+
551+
@pytest.mark.parametrize(
552+
"value",
553+
[
554+
# Not integral.
555+
3.5,
556+
"2.5",
557+
# Outside a signed 64-bit long, which longValueExact rejects.
558+
2**63,
559+
-(2**63) - 1,
560+
1e20,
561+
# Not a number at all, or not a finite one.
562+
"abc",
563+
"",
564+
"inf",
565+
"nan",
566+
"snan",
567+
"Infinity",
568+
float("inf"),
569+
# A boolean is spelled true / false first, as the JVM spells it,
570+
# so it is no integer even though Python's bool is an int.
571+
True,
572+
False,
573+
],
574+
ids=repr,
575+
)
576+
def test_an_integer_rejects_what_the_jvm_rejects(self, value):
577+
assert self._rejection(value, "integer") == (
578+
f"the operator refers to loop variable $v, but its value {value!r} "
579+
"is not an integer"
580+
)
581+
582+
@pytest.mark.parametrize(
583+
"value, expected",
584+
[
585+
(0.25, 0.25),
586+
(0.0, 0.0),
587+
(3, 3.0),
588+
("2.5", 2.5),
589+
(" 7.5 ", 7.5),
590+
("1e-3", 0.001),
591+
("-.5", -0.5),
592+
("1.", 1.0),
593+
("Infinity", math.inf),
594+
("-Infinity", -math.inf),
595+
(float("inf"), math.inf),
596+
(2**63, float(2**63)),
597+
],
598+
ids=repr,
599+
)
600+
def test_a_number_is_read_from_every_scalar_that_holds_one(self, value, expected):
601+
result = self._read(value, "number")
602+
assert result == expected
603+
assert type(result) is float
604+
605+
@pytest.mark.parametrize("value", ["NaN", "+NaN", float("nan")], ids=repr)
606+
def test_a_number_is_nan_where_the_jvm_reads_nan(self, value):
607+
assert math.isnan(self._read(value, "number"))
608+
609+
@pytest.mark.parametrize(
610+
"value",
611+
[
612+
"abc",
613+
"",
614+
"1.2.3",
615+
True,
616+
False,
617+
],
618+
ids=repr,
619+
)
620+
def test_a_number_rejects_what_the_jvm_rejects(self, value):
621+
assert self._rejection(value, "number") == (
622+
f"the operator refers to loop variable $v, but its value {value!r} "
623+
"is not a number"
624+
)
625+
626+
@pytest.mark.parametrize(
627+
"value, expected",
628+
[
629+
(True, True),
630+
(False, False),
631+
("true", True),
632+
("false", False),
633+
# Any case, around any whitespace, as Scala's toBooleanOption.
634+
("TRUE", True),
635+
("False", False),
636+
("tRuE", True),
637+
(" true ", True),
638+
],
639+
ids=repr,
640+
)
641+
def test_a_boolean_is_read_from_every_scalar_that_holds_one(self, value, expected):
642+
assert self._read(value, "boolean") is expected
643+
644+
@pytest.mark.parametrize("value", [1, 0, 1.0, "1", "0", "yes", "t", ""], ids=repr)
645+
def test_a_boolean_rejects_what_the_jvm_rejects(self, value):
646+
# 1 and 0 are no booleans to the JVM's toBooleanOption, so neither
647+
# are they here.
648+
assert self._rejection(value, "boolean") == (
649+
f"the operator refers to loop variable $v, but its value {value!r} "
650+
"is not a boolean"
651+
)
652+
653+
@pytest.mark.parametrize("kind", ["integer", "number", "boolean"])
654+
@pytest.mark.parametrize("value", [None, [1, 2], {"p": 2}, b"raw"], ids=repr)
655+
def test_raises_for_a_value_that_is_not_a_scalar(self, value, kind):
656+
assert self._rejection(value, kind) == (
657+
f"the operator refers to loop variable $v, but its value {value!r} "
658+
"is not a scalar"
659+
)
660+
661+
@pytest.mark.parametrize("kind", ["string", "Integer", "int", "double", ""])
662+
def test_raises_for_a_kind_the_backend_never_writes(self, kind):
663+
op = _ConcreteOperator()
664+
op.register_loop_state(State({"v": 3}))
665+
with pytest.raises(ValueError, match="cannot be read as"):
666+
op.loop_variable_value("v", kind)
667+
668+
def test_an_unknown_kind_is_reported_before_the_state_is_read(self):
669+
# The kind is the generated code's own mistake, so it is reported
670+
# whether or not a state has arrived.
671+
with pytest.raises(ValueError, match="cannot be read as 'int'"):
672+
_ConcreteOperator().loop_variable_value("v", "int")
673+
674+
@pytest.mark.parametrize("kind", ["integer", "number", "boolean"])
675+
def test_raises_when_read_before_the_iterations_state_arrived(self, kind):
676+
with pytest.raises(RuntimeError) as excinfo:
677+
_ConcreteOperator().loop_variable_value("n", kind)
678+
assert str(excinfo.value) == (
679+
"loop variable $n is read before the iteration's state arrived"
680+
)
681+
682+
@pytest.mark.parametrize("kind", ["integer", "number", "boolean"])
683+
def test_raises_for_a_name_no_state_message_carried(self, kind):
684+
op = _ConcreteOperator()
685+
op.register_loop_state(State({"i": 1}))
686+
with pytest.raises(RuntimeError) as excinfo:
687+
op.loop_variable_value("n", kind)
688+
assert str(excinfo.value) == (
689+
"the operator refers to loop variable $n, but no state message carried it"
690+
)
691+
692+
def test_a_later_state_message_wins_and_an_earlier_name_is_kept(self):
693+
op = _ConcreteOperator()
694+
op.register_loop_state(State({"n": 3, "on": True, "rate": 0.5}))
695+
op.register_loop_state(State({"n": 5, "centroids": "2"}))
696+
assert op.loop_variable_value("n", "integer") == 5
697+
assert op.loop_variable_value("on", "boolean") is True
698+
assert op.loop_variable_value("rate", "number") == 0.5
699+
assert op.loop_variable_value("centroids", "integer") == 2
700+
701+
def test_generated_code_reads_the_iterations_value_at_run_time(self):
702+
# __name__ so the exec'd class's methods get a real __module__ (the
703+
# @overrides decorator on the generated methods inspects it).
704+
namespace: dict = {"__name__": "generated_histogram_operator"}
705+
exec(_GENERATED_HISTOGRAM_TEMPLATE, namespace)
706+
op = namespace["ProcessTableOperator"]()
707+
708+
# Before the iteration's state arrives, the generated call fails loud
709+
# instead of running on the value the backend generated the code with.
710+
with pytest.raises(RuntimeError, match="read before the iteration's state"):
711+
list(op.process_table(Table([Tuple({"a": 1})]), 0))
712+
713+
op.register_loop_state(State({"n": 3, "alpha": 0.25, "labels": True}))
714+
(params,) = list(op.process_table(Table([Tuple({"a": 1})]), 0))
715+
assert params == {"nbinsx": 3, "nbinsy": 3, "opacity": 0.25, "text_auto": True}
716+
717+
# The next iteration's state reaches the same operator instance, and
718+
# its values are converted to the kind the code asks for.
719+
op.register_loop_state(State({"n": "90", "alpha": 2, "labels": "false"}))
720+
(params,) = list(op.process_table(Table([Tuple({"a": 1})]), 0))
721+
assert params == {
722+
"nbinsx": 90,
723+
"nbinsy": 90,
724+
"opacity": 2.0,
725+
"text_auto": False,
726+
}
727+
assert [type(v) for v in params.values()] == [int, int, float, bool]
728+
729+
490730
class TestBatchOperatorValidation:
491731
def test_validate_batch_size_rejects_none(self):
492732
with pytest.raises(ValueError, match="cannot be None"):

0 commit comments

Comments
 (0)