|
18 | 18 | import numpy as np |
19 | 19 | import pytest |
20 | 20 |
|
| 21 | +from pandas._libs import parsers as libparsers |
21 | 22 | from pandas.compat import WASM |
22 | 23 | from pandas.errors import ( |
| 24 | + DtypeWarning, |
23 | 25 | ParserError, |
24 | 26 | ParserWarning, |
25 | 27 | ) |
@@ -642,3 +644,143 @@ def test_float_precision_options(c_parser_only): |
642 | 644 |
|
643 | 645 | with pytest.raises(ValueError, match=msg): |
644 | 646 | parser.read_csv(StringIO(s), float_precision="junk") |
| 647 | + |
| 648 | + |
| 649 | +# small enough that DEFAULT_BUFFER_HEURISTIC // table_width leaves one line per |
| 650 | +# buffer, so the rows below land in separate low_memory chunks |
| 651 | +_MIXED_DTYPE_HEURISTIC = 2**3 |
| 652 | + |
| 653 | + |
| 654 | +def _mixed_dtype_data(n_leading_names: int = 3) -> str: |
| 655 | + """ |
| 656 | + Build a csv whose middle chunk makes the second of three fields mixed-dtype. |
| 657 | +
|
| 658 | + Naming fewer than the three fields per row leaves an unnamed leading field, |
| 659 | + which the reader takes as an implicit index, shifting which name that field |
| 660 | + carries. |
| 661 | + """ |
| 662 | + names = ["a", "b", "c"][:n_leading_names] |
| 663 | + rows = [f"{i},{i},{i}" for i in range(_MIXED_DTYPE_HEURISTIC - 1)] |
| 664 | + return ",".join(names) + "\n" + "\n".join([*rows, "7,x,7", "8,y,8", *rows]) + "\n" |
| 665 | + |
| 666 | + |
| 667 | +@pytest.mark.parametrize("usecols", [["b"], ["b", "c"], [1], [1, 2]]) |
| 668 | +def test_mixed_dtype_warning_with_usecols(c_parser_only, monkeypatch, usecols): |
| 669 | + # GH#67375: read_low_memory keys its chunks by field position in the source |
| 670 | + # row, so usecols leaves gaps in those keys. The mixed-dtype warning used |
| 671 | + # them to index the column names directly, which raised IndexError for any |
| 672 | + # selected column that was not still at its original position. |
| 673 | + parser = c_parser_only |
| 674 | + data = _mixed_dtype_data() |
| 675 | + |
| 676 | + with monkeypatch.context() as m: |
| 677 | + m.setattr(libparsers, "DEFAULT_BUFFER_HEURISTIC", _MIXED_DTYPE_HEURISTIC) |
| 678 | + if parser.low_memory: |
| 679 | + result = parser.read_csv_check_warnings( |
| 680 | + DtypeWarning, |
| 681 | + r"Columns \(1: b\) have mixed types\. " |
| 682 | + "Specify dtype option on import or set low_memory=False.", |
| 683 | + StringIO(data), |
| 684 | + usecols=usecols, |
| 685 | + ) |
| 686 | + else: |
| 687 | + result = parser.read_csv(StringIO(data), usecols=usecols) |
| 688 | + |
| 689 | + expected = ["b"] if len(usecols) == 1 else ["b", "c"] |
| 690 | + assert result.columns.tolist() == expected |
| 691 | + |
| 692 | + |
| 693 | +def test_mixed_dtype_warning_with_index_col(c_parser_only, monkeypatch): |
| 694 | + # GH#67375: an index column shifts the parsed field positions past the data |
| 695 | + # columns, so the same positional lookup named the column after the mixed |
| 696 | + # one instead of naming "b". |
| 697 | + parser = c_parser_only |
| 698 | + data = _mixed_dtype_data() |
| 699 | + |
| 700 | + with monkeypatch.context() as m: |
| 701 | + m.setattr(libparsers, "DEFAULT_BUFFER_HEURISTIC", _MIXED_DTYPE_HEURISTIC) |
| 702 | + if parser.low_memory: |
| 703 | + result = parser.read_csv_check_warnings( |
| 704 | + DtypeWarning, |
| 705 | + r"Columns \(1: b\) have mixed types\. " |
| 706 | + "Specify dtype option on import or set low_memory=False.", |
| 707 | + StringIO(data), |
| 708 | + index_col=0, |
| 709 | + ) |
| 710 | + else: |
| 711 | + result = parser.read_csv(StringIO(data), index_col=0) |
| 712 | + |
| 713 | + assert result.columns.tolist() == ["b", "c"] |
| 714 | + assert result.index.name == "a" |
| 715 | + |
| 716 | + |
| 717 | +def test_mixed_dtype_warning_with_implicit_index(c_parser_only, monkeypatch): |
| 718 | + # GH#67375: a header shorter than the rows makes the reader take the first |
| 719 | + # field as an implicit index. That field has no name, so the parsed |
| 720 | + # positions run one ahead of the column names and the warning named "b" |
| 721 | + # while the mixed column was "a". |
| 722 | + parser = c_parser_only |
| 723 | + data = _mixed_dtype_data(n_leading_names=2) |
| 724 | + |
| 725 | + with monkeypatch.context() as m: |
| 726 | + m.setattr(libparsers, "DEFAULT_BUFFER_HEURISTIC", _MIXED_DTYPE_HEURISTIC) |
| 727 | + if parser.low_memory: |
| 728 | + result = parser.read_csv_check_warnings( |
| 729 | + DtypeWarning, |
| 730 | + r"Columns \(1: a\) have mixed types\. " |
| 731 | + "Specify dtype option on import or set low_memory=False.", |
| 732 | + StringIO(data), |
| 733 | + ) |
| 734 | + else: |
| 735 | + result = parser.read_csv(StringIO(data)) |
| 736 | + |
| 737 | + assert result.columns.tolist() == ["a", "b"] |
| 738 | + |
| 739 | + |
| 740 | +def test_mixed_dtype_warning_reports_source_positions(c_parser_only, monkeypatch): |
| 741 | + # GH#67375: the number before each name is the column's position in the |
| 742 | + # source row -- as GH#58174 asked for and as the pre-GH#58250 message |
| 743 | + # printed -- not a running count of the mixed columns. With two mixed |
| 744 | + # columns at positions 1 and 3 the two readings disagree. |
| 745 | + parser = c_parser_only |
| 746 | + rows = [f"{i},{i},{i},{i}" for i in range(_MIXED_DTYPE_HEURISTIC - 1)] |
| 747 | + data = "a,b,c,d\n" + "\n".join([*rows, "7,x,7,x", "8,y,8,y", *rows]) + "\n" |
| 748 | + |
| 749 | + with monkeypatch.context() as m: |
| 750 | + m.setattr(libparsers, "DEFAULT_BUFFER_HEURISTIC", _MIXED_DTYPE_HEURISTIC) |
| 751 | + if parser.low_memory: |
| 752 | + result = parser.read_csv_check_warnings( |
| 753 | + DtypeWarning, |
| 754 | + r"Columns \(1: b, 3: d\) have mixed types\. " |
| 755 | + "Specify dtype option on import or set low_memory=False.", |
| 756 | + StringIO(data), |
| 757 | + ) |
| 758 | + else: |
| 759 | + result = parser.read_csv(StringIO(data)) |
| 760 | + |
| 761 | + assert result.columns.tolist() == ["a", "b", "c", "d"] |
| 762 | + |
| 763 | + |
| 764 | +def test_mixed_dtype_warning_with_mixed_implicit_index(c_parser_only, monkeypatch): |
| 765 | + # GH#67375: the implicit index column is parsed like any other field and can |
| 766 | + # itself be mixed, but it has no name to report. Naming it by position, as |
| 767 | + # the message did before GH#58250 added names, beats borrowing the name of |
| 768 | + # the first data column -- which is not the column that is mixed. |
| 769 | + parser = c_parser_only |
| 770 | + rows = [f"{i},{i},{i},{i}" for i in range(_MIXED_DTYPE_HEURISTIC - 1)] |
| 771 | + data = "a,b,c\n" + "\n".join([*rows, "x,7,7,7", "y,8,8,8", *rows]) + "\n" |
| 772 | + |
| 773 | + with monkeypatch.context() as m: |
| 774 | + m.setattr(libparsers, "DEFAULT_BUFFER_HEURISTIC", _MIXED_DTYPE_HEURISTIC) |
| 775 | + if parser.low_memory: |
| 776 | + result = parser.read_csv_check_warnings( |
| 777 | + DtypeWarning, |
| 778 | + r"Columns \(0\) have mixed types\. " |
| 779 | + "Specify dtype option on import or set low_memory=False.", |
| 780 | + StringIO(data), |
| 781 | + ) |
| 782 | + else: |
| 783 | + result = parser.read_csv(StringIO(data)) |
| 784 | + |
| 785 | + assert result.columns.tolist() == ["a", "b", "c"] |
| 786 | + assert result.index.name is None |
0 commit comments