|
16 | 16 | # under the License. |
17 | 17 |
|
18 | 18 | import base64 |
| 19 | +import math |
19 | 20 |
|
20 | 21 | import pandas |
21 | 22 | import pytest |
@@ -487,6 +488,245 @@ def test_generated_code_reads_the_iterations_value_at_run_time(self): |
487 | 488 | assert params == {"C": 0.25, "kernel": "linear", "probability": False} |
488 | 489 |
|
489 | 490 |
|
| 491 | +# The shape the backend gives a Python-generated operator's code inside a |
| 492 | +# control block where a numeric or boolean property holds `$name`: the value |
| 493 | +# it wrote into the code is replaced by |
| 494 | +# `self.loop_variable_value('<name>', '<kind>')`. Mirrors the 2D histogram's |
| 495 | +# `nbinsx=$xBins,` rendering, with a number and a boolean keyword beside it. |
| 496 | +_GENERATED_HISTOGRAM_TEMPLATE = """from pytexera import * |
| 497 | +
|
| 498 | +class ProcessTableOperator(UDFTableOperator): |
| 499 | +
|
| 500 | + @overrides |
| 501 | + def process_table(self, table: Table, port: int) -> Iterator[Optional[TableLike]]: |
| 502 | + yield dict( |
| 503 | + nbinsx=self.loop_variable_value('n', 'integer'), |
| 504 | + nbinsy=self.loop_variable_value('n', 'integer'), |
| 505 | + opacity=self.loop_variable_value('alpha', 'number'), |
| 506 | + text_auto=self.loop_variable_value('labels', 'boolean') |
| 507 | + ) |
| 508 | +""" |
| 509 | + |
| 510 | + |
| 511 | +class TestLoopVariableValue: |
| 512 | + @staticmethod |
| 513 | + def _read(value, kind): |
| 514 | + op = _ConcreteOperator() |
| 515 | + op.register_loop_state(State({"v": value})) |
| 516 | + return op.loop_variable_value("v", kind) |
| 517 | + |
| 518 | + @staticmethod |
| 519 | + def _rejection(value, kind): |
| 520 | + op = _ConcreteOperator() |
| 521 | + op.register_loop_state(State({"v": value})) |
| 522 | + with pytest.raises(RuntimeError) as excinfo: |
| 523 | + op.loop_variable_value("v", kind) |
| 524 | + return str(excinfo.value) |
| 525 | + |
| 526 | + @pytest.mark.parametrize( |
| 527 | + "value, expected", |
| 528 | + [ |
| 529 | + (3, 3), |
| 530 | + (-7, -7), |
| 531 | + (0, 0), |
| 532 | + # An integral float, as BigDecimal("3.0").longValueExact() takes it, |
| 533 | + # including one Python spells with an exponent (1e+16). |
| 534 | + (3.0, 3), |
| 535 | + (1e16, 10**16), |
| 536 | + ("42", 42), |
| 537 | + (" 42 ", 42), |
| 538 | + ("-5", -5), |
| 539 | + ("+5", 5), |
| 540 | + ("4.2e1", 42), |
| 541 | + (2**63 - 1, 2**63 - 1), |
| 542 | + (-(2**63), -(2**63)), |
| 543 | + ], |
| 544 | + ids=repr, |
| 545 | + ) |
| 546 | + def test_an_integer_is_read_from_every_scalar_that_holds_one(self, value, expected): |
| 547 | + result = self._read(value, "integer") |
| 548 | + assert result == expected |
| 549 | + assert type(result) is int |
| 550 | + |
| 551 | + @pytest.mark.parametrize( |
| 552 | + "value", |
| 553 | + [ |
| 554 | + # Not integral. |
| 555 | + 3.5, |
| 556 | + "2.5", |
| 557 | + # Outside a signed 64-bit long, which longValueExact rejects. |
| 558 | + 2**63, |
| 559 | + -(2**63) - 1, |
| 560 | + 1e20, |
| 561 | + # Not a number at all, or not a finite one. |
| 562 | + "abc", |
| 563 | + "", |
| 564 | + "inf", |
| 565 | + "nan", |
| 566 | + "snan", |
| 567 | + "Infinity", |
| 568 | + float("inf"), |
| 569 | + # A boolean is spelled true / false first, as the JVM spells it, |
| 570 | + # so it is no integer even though Python's bool is an int. |
| 571 | + True, |
| 572 | + False, |
| 573 | + ], |
| 574 | + ids=repr, |
| 575 | + ) |
| 576 | + def test_an_integer_rejects_what_the_jvm_rejects(self, value): |
| 577 | + assert self._rejection(value, "integer") == ( |
| 578 | + f"the operator refers to loop variable $v, but its value {value!r} " |
| 579 | + "is not an integer" |
| 580 | + ) |
| 581 | + |
| 582 | + @pytest.mark.parametrize( |
| 583 | + "value, expected", |
| 584 | + [ |
| 585 | + (0.25, 0.25), |
| 586 | + (0.0, 0.0), |
| 587 | + (3, 3.0), |
| 588 | + ("2.5", 2.5), |
| 589 | + (" 7.5 ", 7.5), |
| 590 | + ("1e-3", 0.001), |
| 591 | + ("-.5", -0.5), |
| 592 | + ("1.", 1.0), |
| 593 | + ("Infinity", math.inf), |
| 594 | + ("-Infinity", -math.inf), |
| 595 | + (float("inf"), math.inf), |
| 596 | + (2**63, float(2**63)), |
| 597 | + ], |
| 598 | + ids=repr, |
| 599 | + ) |
| 600 | + def test_a_number_is_read_from_every_scalar_that_holds_one(self, value, expected): |
| 601 | + result = self._read(value, "number") |
| 602 | + assert result == expected |
| 603 | + assert type(result) is float |
| 604 | + |
| 605 | + @pytest.mark.parametrize("value", ["NaN", "+NaN", float("nan")], ids=repr) |
| 606 | + def test_a_number_is_nan_where_the_jvm_reads_nan(self, value): |
| 607 | + assert math.isnan(self._read(value, "number")) |
| 608 | + |
| 609 | + @pytest.mark.parametrize( |
| 610 | + "value", |
| 611 | + [ |
| 612 | + "abc", |
| 613 | + "", |
| 614 | + "1.2.3", |
| 615 | + True, |
| 616 | + False, |
| 617 | + ], |
| 618 | + ids=repr, |
| 619 | + ) |
| 620 | + def test_a_number_rejects_what_the_jvm_rejects(self, value): |
| 621 | + assert self._rejection(value, "number") == ( |
| 622 | + f"the operator refers to loop variable $v, but its value {value!r} " |
| 623 | + "is not a number" |
| 624 | + ) |
| 625 | + |
| 626 | + @pytest.mark.parametrize( |
| 627 | + "value, expected", |
| 628 | + [ |
| 629 | + (True, True), |
| 630 | + (False, False), |
| 631 | + ("true", True), |
| 632 | + ("false", False), |
| 633 | + # Any case, around any whitespace, as Scala's toBooleanOption. |
| 634 | + ("TRUE", True), |
| 635 | + ("False", False), |
| 636 | + ("tRuE", True), |
| 637 | + (" true ", True), |
| 638 | + ], |
| 639 | + ids=repr, |
| 640 | + ) |
| 641 | + def test_a_boolean_is_read_from_every_scalar_that_holds_one(self, value, expected): |
| 642 | + assert self._read(value, "boolean") is expected |
| 643 | + |
| 644 | + @pytest.mark.parametrize("value", [1, 0, 1.0, "1", "0", "yes", "t", ""], ids=repr) |
| 645 | + def test_a_boolean_rejects_what_the_jvm_rejects(self, value): |
| 646 | + # 1 and 0 are no booleans to the JVM's toBooleanOption, so neither |
| 647 | + # are they here. |
| 648 | + assert self._rejection(value, "boolean") == ( |
| 649 | + f"the operator refers to loop variable $v, but its value {value!r} " |
| 650 | + "is not a boolean" |
| 651 | + ) |
| 652 | + |
| 653 | + @pytest.mark.parametrize("kind", ["integer", "number", "boolean"]) |
| 654 | + @pytest.mark.parametrize("value", [None, [1, 2], {"p": 2}, b"raw"], ids=repr) |
| 655 | + def test_raises_for_a_value_that_is_not_a_scalar(self, value, kind): |
| 656 | + assert self._rejection(value, kind) == ( |
| 657 | + f"the operator refers to loop variable $v, but its value {value!r} " |
| 658 | + "is not a scalar" |
| 659 | + ) |
| 660 | + |
| 661 | + @pytest.mark.parametrize("kind", ["string", "Integer", "int", "double", ""]) |
| 662 | + def test_raises_for_a_kind_the_backend_never_writes(self, kind): |
| 663 | + op = _ConcreteOperator() |
| 664 | + op.register_loop_state(State({"v": 3})) |
| 665 | + with pytest.raises(ValueError, match="cannot be read as"): |
| 666 | + op.loop_variable_value("v", kind) |
| 667 | + |
| 668 | + def test_an_unknown_kind_is_reported_before_the_state_is_read(self): |
| 669 | + # The kind is the generated code's own mistake, so it is reported |
| 670 | + # whether or not a state has arrived. |
| 671 | + with pytest.raises(ValueError, match="cannot be read as 'int'"): |
| 672 | + _ConcreteOperator().loop_variable_value("v", "int") |
| 673 | + |
| 674 | + @pytest.mark.parametrize("kind", ["integer", "number", "boolean"]) |
| 675 | + def test_raises_when_read_before_the_iterations_state_arrived(self, kind): |
| 676 | + with pytest.raises(RuntimeError) as excinfo: |
| 677 | + _ConcreteOperator().loop_variable_value("n", kind) |
| 678 | + assert str(excinfo.value) == ( |
| 679 | + "loop variable $n is read before the iteration's state arrived" |
| 680 | + ) |
| 681 | + |
| 682 | + @pytest.mark.parametrize("kind", ["integer", "number", "boolean"]) |
| 683 | + def test_raises_for_a_name_no_state_message_carried(self, kind): |
| 684 | + op = _ConcreteOperator() |
| 685 | + op.register_loop_state(State({"i": 1})) |
| 686 | + with pytest.raises(RuntimeError) as excinfo: |
| 687 | + op.loop_variable_value("n", kind) |
| 688 | + assert str(excinfo.value) == ( |
| 689 | + "the operator refers to loop variable $n, but no state message carried it" |
| 690 | + ) |
| 691 | + |
| 692 | + def test_a_later_state_message_wins_and_an_earlier_name_is_kept(self): |
| 693 | + op = _ConcreteOperator() |
| 694 | + op.register_loop_state(State({"n": 3, "on": True, "rate": 0.5})) |
| 695 | + op.register_loop_state(State({"n": 5, "centroids": "2"})) |
| 696 | + assert op.loop_variable_value("n", "integer") == 5 |
| 697 | + assert op.loop_variable_value("on", "boolean") is True |
| 698 | + assert op.loop_variable_value("rate", "number") == 0.5 |
| 699 | + assert op.loop_variable_value("centroids", "integer") == 2 |
| 700 | + |
| 701 | + def test_generated_code_reads_the_iterations_value_at_run_time(self): |
| 702 | + # __name__ so the exec'd class's methods get a real __module__ (the |
| 703 | + # @overrides decorator on the generated methods inspects it). |
| 704 | + namespace: dict = {"__name__": "generated_histogram_operator"} |
| 705 | + exec(_GENERATED_HISTOGRAM_TEMPLATE, namespace) |
| 706 | + op = namespace["ProcessTableOperator"]() |
| 707 | + |
| 708 | + # Before the iteration's state arrives, the generated call fails loud |
| 709 | + # instead of running on the value the backend generated the code with. |
| 710 | + with pytest.raises(RuntimeError, match="read before the iteration's state"): |
| 711 | + list(op.process_table(Table([Tuple({"a": 1})]), 0)) |
| 712 | + |
| 713 | + op.register_loop_state(State({"n": 3, "alpha": 0.25, "labels": True})) |
| 714 | + (params,) = list(op.process_table(Table([Tuple({"a": 1})]), 0)) |
| 715 | + assert params == {"nbinsx": 3, "nbinsy": 3, "opacity": 0.25, "text_auto": True} |
| 716 | + |
| 717 | + # The next iteration's state reaches the same operator instance, and |
| 718 | + # its values are converted to the kind the code asks for. |
| 719 | + op.register_loop_state(State({"n": "90", "alpha": 2, "labels": "false"})) |
| 720 | + (params,) = list(op.process_table(Table([Tuple({"a": 1})]), 0)) |
| 721 | + assert params == { |
| 722 | + "nbinsx": 90, |
| 723 | + "nbinsy": 90, |
| 724 | + "opacity": 2.0, |
| 725 | + "text_auto": False, |
| 726 | + } |
| 727 | + assert [type(v) for v in params.values()] == [int, int, float, bool] |
| 728 | + |
| 729 | + |
490 | 730 | class TestBatchOperatorValidation: |
491 | 731 | def test_validate_batch_size_rejects_none(self): |
492 | 732 | with pytest.raises(ValueError, match="cannot be None"): |
|
0 commit comments