Skip to content

Commit 8aa187c

Browse files
committed
refactor: move number and true/false references to their own PR
This reverts 15ee0b7 (feat: bind numeric and boolean $i in generated operators) and the parts of 187de6e that only documented it. The probe values move to feat/generated-op-number-refs, so this PR keeps the text references alone. A number or true/false property of a generated Python operator that refers to a loop variable is rejected again at compile time, naming the property's pointer: "numeric and boolean properties are written into its generated code before the loop runs". That error is reported ahead of the code-generation failure the placeholder may cause, so the compile no longer blames a value the user never entered. WorkflowCompiler.unbindableStateReferences decides every reason a generated operator's reference is unbound, from the code generated once; nothing depends on probe reasons any more. Operator.loop_variable_value and its tests go too; loop_variable_text keeps reading Operator.loop_variables.
1 parent fb2ddbd commit 8aa187c

6 files changed

Lines changed: 62 additions & 1256 deletions

File tree

‎amber/src/main/python/core/models/operator.py‎

Lines changed: 0 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,6 @@
2020
from functools import lru_cache
2121
from abc import ABC, abstractmethod
2222
from collections import defaultdict
23-
from decimal import Decimal, InvalidOperation
2423
from types import MappingProxyType
2524
from typing import (
2625
Any,
@@ -133,60 +132,6 @@ def loop_variable_text(self, name: str) -> str:
133132
f"but its value {value!r} is not a scalar"
134133
)
135134

136-
def loop_variable_value(self, name: str, kind: str) -> Union[int, float, bool]:
137-
"""
138-
The value of loop variable ``name``, its text (``loop_variable_text``)
139-
converted to ``kind``. Inside a control block, the code generated for
140-
an operator calls this where a numeric or boolean property holds
141-
``$name``, in place of the value written into the code, so the value is
142-
read when the code runs. It is converted as a JVM operator's numeric or
143-
boolean property binds it (``StateReferenceBinding``), from its text: an
144-
integer must be integral and fit a signed 64-bit long, a number is any
145-
numeral, and a boolean is ``true`` or ``false`` in any case, never ``1``
146-
or ``0``.
147-
148-
:param name: str, the loop variable's name, without the leading ``$``.
149-
:param kind: str, ``integer``, ``number`` or ``boolean``: the JSON type
150-
of the property that holds ``$name``.
151-
:return: int, float or bool, the variable's value as ``kind``.
152-
"""
153-
if kind not in ("integer", "number", "boolean"):
154-
raise ValueError(
155-
f"loop variable ${name} cannot be read as {kind!r}: "
156-
"the kind is one of 'integer', 'number' and 'boolean'"
157-
)
158-
text = self.loop_variable_text(name).strip()
159-
value = self.loop_variables[name]
160-
161-
def not_a(description: str) -> RuntimeError:
162-
return RuntimeError(
163-
f"the operator refers to loop variable ${name}, "
164-
f"but its value {value!r} is not {description}"
165-
)
166-
167-
if kind == "boolean":
168-
if text.lower() not in ("true", "false"):
169-
raise not_a("a boolean")
170-
return text.lower() == "true"
171-
description = "a number" if kind == "number" else "an integer"
172-
try:
173-
numeral = Decimal(text)
174-
except InvalidOperation:
175-
raise not_a(description) from None
176-
if kind == "number":
177-
try:
178-
return float(numeral)
179-
except ValueError: # a signalling NaN
180-
raise not_a(description) from None
181-
# BigDecimal.longValueExact: integral and within a signed 64-bit long.
182-
if (
183-
not numeral.is_finite()
184-
or numeral != numeral.to_integral_value()
185-
or not -(2**63) <= numeral < 2**63
186-
):
187-
raise not_a(description)
188-
return int(numeral)
189-
190135
__internal_is_source: bool = False
191136

192137
# The loop variables received so far, by name, and the nesting depth of

‎amber/src/test/python/core/models/test_operator.py‎

Lines changed: 0 additions & 240 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,6 @@
1616
# under the License.
1717

1818
import base64
19-
import math
2019

2120
import pandas
2221
import pytest
@@ -573,245 +572,6 @@ def test_generated_code_reads_the_iterations_value_at_run_time(self):
573572
assert params == {"C": 0.25, "kernel": "linear", "probability": False}
574573

575574

576-
# The shape the backend gives a Python-generated operator's code inside a
577-
# control block where a numeric or boolean property holds `$name`: the value
578-
# it wrote into the code is replaced by
579-
# `self.loop_variable_value('<name>', '<kind>')`. Mirrors the 2D histogram's
580-
# `nbinsx=$xBins,` rendering, with a number and a boolean keyword beside it.
581-
_GENERATED_HISTOGRAM_TEMPLATE = """from pytexera import *
582-
583-
class ProcessTableOperator(UDFTableOperator):
584-
585-
@overrides
586-
def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]:
587-
yield dict(
588-
nbinsx=self.loop_variable_value('n', 'integer'),
589-
nbinsy=self.loop_variable_value('n', 'integer'),
590-
opacity=self.loop_variable_value('alpha', 'number'),
591-
text_auto=self.loop_variable_value('labels', 'boolean')
592-
)
593-
"""
594-
595-
596-
class TestLoopVariableValue:
597-
@staticmethod
598-
def _read(value, kind):
599-
op = _ConcreteOperator()
600-
op.register_state(State({"v": value}))
601-
return op.loop_variable_value("v", kind)
602-
603-
@staticmethod
604-
def _rejection(value, kind):
605-
op = _ConcreteOperator()
606-
op.register_state(State({"v": value}))
607-
with pytest.raises(RuntimeError) as excinfo:
608-
op.loop_variable_value("v", kind)
609-
return str(excinfo.value)
610-
611-
@pytest.mark.parametrize(
612-
"value, expected",
613-
[
614-
(3, 3),
615-
(-7, -7),
616-
(0, 0),
617-
# An integral float, as BigDecimal("3.0").longValueExact() takes it,
618-
# including one Python spells with an exponent (1e+16).
619-
(3.0, 3),
620-
(1e16, 10**16),
621-
("42", 42),
622-
(" 42 ", 42),
623-
("-5", -5),
624-
("+5", 5),
625-
("4.2e1", 42),
626-
(2**63 - 1, 2**63 - 1),
627-
(-(2**63), -(2**63)),
628-
],
629-
ids=repr,
630-
)
631-
def test_an_integer_is_read_from_every_scalar_that_holds_one(self, value, expected):
632-
result = self._read(value, "integer")
633-
assert result == expected
634-
assert type(result) is int
635-
636-
@pytest.mark.parametrize(
637-
"value",
638-
[
639-
# Not integral.
640-
3.5,
641-
"2.5",
642-
# Outside a signed 64-bit long, which longValueExact rejects.
643-
2**63,
644-
-(2**63) - 1,
645-
1e20,
646-
# Not a number at all, or not a finite one.
647-
"abc",
648-
"",
649-
"inf",
650-
"nan",
651-
"snan",
652-
"Infinity",
653-
float("inf"),
654-
# A boolean is spelled true / false first, as the JVM spells it,
655-
# so it is no integer even though Python's bool is an int.
656-
True,
657-
False,
658-
],
659-
ids=repr,
660-
)
661-
def test_an_integer_rejects_what_the_jvm_rejects(self, value):
662-
assert self._rejection(value, "integer") == (
663-
f"the operator refers to loop variable $v, but its value {value!r} "
664-
"is not an integer"
665-
)
666-
667-
@pytest.mark.parametrize(
668-
"value, expected",
669-
[
670-
(0.25, 0.25),
671-
(0.0, 0.0),
672-
(3, 3.0),
673-
("2.5", 2.5),
674-
(" 7.5 ", 7.5),
675-
("1e-3", 0.001),
676-
("-.5", -0.5),
677-
("1.", 1.0),
678-
("Infinity", math.inf),
679-
("-Infinity", -math.inf),
680-
(float("inf"), math.inf),
681-
(2**63, float(2**63)),
682-
],
683-
ids=repr,
684-
)
685-
def test_a_number_is_read_from_every_scalar_that_holds_one(self, value, expected):
686-
result = self._read(value, "number")
687-
assert result == expected
688-
assert type(result) is float
689-
690-
@pytest.mark.parametrize("value", ["NaN", "+NaN", float("nan")], ids=repr)
691-
def test_a_number_is_nan_where_the_jvm_reads_nan(self, value):
692-
assert math.isnan(self._read(value, "number"))
693-
694-
@pytest.mark.parametrize(
695-
"value",
696-
[
697-
"abc",
698-
"",
699-
"1.2.3",
700-
True,
701-
False,
702-
],
703-
ids=repr,
704-
)
705-
def test_a_number_rejects_what_the_jvm_rejects(self, value):
706-
assert self._rejection(value, "number") == (
707-
f"the operator refers to loop variable $v, but its value {value!r} "
708-
"is not a number"
709-
)
710-
711-
@pytest.mark.parametrize(
712-
"value, expected",
713-
[
714-
(True, True),
715-
(False, False),
716-
("true", True),
717-
("false", False),
718-
# Any case, around any whitespace, as Scala's toBooleanOption.
719-
("TRUE", True),
720-
("False", False),
721-
("tRuE", True),
722-
(" true ", True),
723-
],
724-
ids=repr,
725-
)
726-
def test_a_boolean_is_read_from_every_scalar_that_holds_one(self, value, expected):
727-
assert self._read(value, "boolean") is expected
728-
729-
@pytest.mark.parametrize("value", [1, 0, 1.0, "1", "0", "yes", "t", ""], ids=repr)
730-
def test_a_boolean_rejects_what_the_jvm_rejects(self, value):
731-
# 1 and 0 are no booleans to the JVM's toBooleanOption, so neither
732-
# are they here.
733-
assert self._rejection(value, "boolean") == (
734-
f"the operator refers to loop variable $v, but its value {value!r} "
735-
"is not a boolean"
736-
)
737-
738-
@pytest.mark.parametrize("kind", ["integer", "number", "boolean"])
739-
@pytest.mark.parametrize("value", [None, [1, 2], {"p": 2}, b"raw"], ids=repr)
740-
def test_raises_for_a_value_that_is_not_a_scalar(self, value, kind):
741-
assert self._rejection(value, kind) == (
742-
f"the operator refers to loop variable $v, but its value {value!r} "
743-
"is not a scalar"
744-
)
745-
746-
@pytest.mark.parametrize("kind", ["string", "Integer", "int", "double", ""])
747-
def test_raises_for_a_kind_the_backend_never_writes(self, kind):
748-
op = _ConcreteOperator()
749-
op.register_state(State({"v": 3}))
750-
with pytest.raises(ValueError, match="cannot be read as"):
751-
op.loop_variable_value("v", kind)
752-
753-
def test_an_unknown_kind_is_reported_before_the_state_is_read(self):
754-
# The kind is the generated code's own mistake, so it is reported
755-
# whether or not a state has arrived.
756-
with pytest.raises(ValueError, match="cannot be read as 'int'"):
757-
_ConcreteOperator().loop_variable_value("v", "int")
758-
759-
@pytest.mark.parametrize("kind", ["integer", "number", "boolean"])
760-
def test_raises_when_read_before_the_iterations_state_arrived(self, kind):
761-
with pytest.raises(RuntimeError) as excinfo:
762-
_ConcreteOperator().loop_variable_value("n", kind)
763-
assert str(excinfo.value) == (
764-
"loop variable $n is read before the iteration's state arrived"
765-
)
766-
767-
@pytest.mark.parametrize("kind", ["integer", "number", "boolean"])
768-
def test_raises_for_a_name_no_state_message_carried(self, kind):
769-
op = _ConcreteOperator()
770-
op.register_state(State({"i": 1}))
771-
with pytest.raises(RuntimeError) as excinfo:
772-
op.loop_variable_value("n", kind)
773-
assert str(excinfo.value) == (
774-
"the operator refers to loop variable $n, but no state message carried it"
775-
)
776-
777-
def test_a_later_state_message_wins_and_an_earlier_name_is_kept(self):
778-
op = _ConcreteOperator()
779-
op.register_state(State({"n": 3, "on": True, "rate": 0.5}))
780-
op.register_state(State({"n": 5, "centroids": "2"}))
781-
assert op.loop_variable_value("n", "integer") == 5
782-
assert op.loop_variable_value("on", "boolean") is True
783-
assert op.loop_variable_value("rate", "number") == 0.5
784-
assert op.loop_variable_value("centroids", "integer") == 2
785-
786-
def test_generated_code_reads_the_iterations_value_at_run_time(self):
787-
# __name__ so the exec'd class's methods get a real __module__ (the
788-
# @overrides decorator on the generated methods inspects it).
789-
namespace: dict = {"__name__": "generated_histogram_operator"}
790-
exec(_GENERATED_HISTOGRAM_TEMPLATE, namespace)
791-
op = namespace["ProcessTableOperator"]()
792-
793-
# Before the iteration's state arrives, the generated call fails loud
794-
# instead of running on the value the backend generated the code with.
795-
with pytest.raises(RuntimeError, match="read before the iteration's state"):
796-
list(op.process_table(Table([Tuple({"a": 1})]), 0))
797-
798-
op.register_state(State({"n": 3, "alpha": 0.25, "labels": True}))
799-
(params,) = list(op.process_table(Table([Tuple({"a": 1})]), 0))
800-
assert params == {"nbinsx": 3, "nbinsy": 3, "opacity": 0.25, "text_auto": True}
801-
802-
# The next iteration's state reaches the same operator instance, and
803-
# its values are converted to the kind the code asks for.
804-
op.register_state(State({"n": "90", "alpha": 2, "labels": "false"}))
805-
(params,) = list(op.process_table(Table([Tuple({"a": 1})]), 0))
806-
assert params == {
807-
"nbinsx": 90,
808-
"nbinsy": 90,
809-
"opacity": 2.0,
810-
"text_auto": False,
811-
}
812-
assert [type(v) for v in params.values()] == [int, int, float, bool]
813-
814-
815575
class TestBatchOperatorValidation:
816576
def test_validate_batch_size_rejects_none(self):
817577
with pytest.raises(ValueError, match="cannot be None"):

0 commit comments

Comments
 (0)