Skip to content

Commit 6193801

Browse files
committed
Speed up reading VOTable TABLEDATA cells
- Check float cells for NaN with math.isnan; np.isnan on a Python float cost about 30 times as much. Some tables have many NaNs - Return empty float cells as masked without raising and catching an error - Read plain in-range integers with one int() call; empty, NaN, hex and out-of-range values still go to the code that warns about them - Skip the attribute checks on TD start tags that have no attributes - In vo_warn, build a warning only when it will be shown - About 1.6x faster on a 75 MB Gaia cone search (astropygh-8191)
1 parent 780a4cf commit 6193801

6 files changed

Lines changed: 144 additions & 13 deletions

File tree

‎astropy/io/votable/converters.py‎

Lines changed: 28 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88
import re
99
import struct
1010
import sys
11-
from math import prod
11+
from math import isnan, prod
1212

1313
# THIRD-PARTY
1414
import numpy as np
@@ -736,9 +736,14 @@ def __init__(self, field, config=None, pos=None):
736736
if self.null is not None:
737737
self._fast_null_param = float(self.null)
738738
self._fast_has_null = True
739+
# Compares in the column's dtype, so a float32 null such as -999.9
740+
# matches the parsed Python float; float(self.null) would not.
741+
self._is_null_scalar = self.is_null
739742
else:
740743
self._fast_null_param = 0.0
741744
self._fast_has_null = False
745+
# np.isnan on a Python float costs about 30 times math.isnan
746+
self._is_null_scalar = isnan
742747

743748
precision = field.precision
744749
width = field.width
@@ -786,12 +791,16 @@ def _parse_pedantic(self, value, config=None, pos=None):
786791
if value.strip() == "":
787792
return self.null, True
788793
f = float(value)
789-
return f, self.is_null(f)
794+
return f, self._is_null_scalar(f)
790795

791796
def _parse_permissive(self, value, config=None, pos=None):
797+
# Same result as the except branch below, without raising: empty
798+
# cells are common in large tables
799+
if value == "":
800+
return self.null, True
792801
try:
793802
f = float(value)
794-
return f, self.is_null(f)
803+
return f, self._is_null_scalar(f)
795804
except ValueError:
796805
# IRSA VOTables use the word 'null' to specify empty values,
797806
# but this is not defined in the VOTable spec.
@@ -880,6 +889,22 @@ def __init__(self, field, config=None, pos=None):
880889
self._fast_has_null = False
881890

882891
def parse(self, value, config=None, pos=None):
892+
# Fast path for plain in-range integers; _parse_general handles the
893+
# rest. Empty cells skip the fast path because a failed int() is slow.
894+
if value != "":
895+
try:
896+
ivalue = int(value)
897+
except (TypeError, ValueError):
898+
pass
899+
else:
900+
low, high = self.val_range
901+
if low <= ivalue <= high:
902+
return ivalue, (
903+
self._fast_has_null and ivalue == self._fast_null_param
904+
)
905+
return self._parse_general(value, config, pos)
906+
907+
def _parse_general(self, value, config=None, pos=None):
883908
if config is None:
884909
config = {}
885910
mask = False

‎astropy/io/votable/exceptions.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -82,12 +82,13 @@ def _format_message(message, name, config=None, pos=None):
8282
return f"{filename}:{pos[0]}:{pos[1]}: {name}: {message}"
8383

8484

85-
def _suppressed_warning(warning, config, stacklevel=2):
86-
warning_class = type(warning)
85+
def _suppressed_warning(warning_class, args, config, pos, stacklevel=2):
8786
config.setdefault("_warning_counts", {}).setdefault(warning_class, 0)
8887
config["_warning_counts"][warning_class] += 1
8988
message_count = config["_warning_counts"][warning_class]
9089
if message_count <= conf.max_warnings:
90+
# Built only when shown: formatting the message is the slow part
91+
warning = warning_class(args, config, pos)
9192
if message_count == conf.max_warnings:
9293
warning.formatted_message += (
9394
" (suppressing further warnings of this type...)"
@@ -152,8 +153,7 @@ def vo_warn(warning_class, args=(), config=None, pos=None, stacklevel=1):
152153
# one would expect that calling warn_or_raise without config should not
153154
# silence the warnings.
154155
if config.get("verify", "warn") != "ignore":
155-
warning = warning_class(args, config, pos)
156-
_suppressed_warning(warning, config, stacklevel=stacklevel + 1)
156+
_suppressed_warning(warning_class, args, config, pos, stacklevel=stacklevel + 1)
157157

158158

159159
def warn_unknown_attrs(element, attrs, config, pos, good_attr=[], stacklevel=1): # ruff: ignore[PLR0917]

‎astropy/io/votable/tests/test_converter.py‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,38 @@ def test_float_mask_permissive():
141141
assert c.parse("null", config=config) == (c.null, True)
142142

143143

144+
def test_float_mask_empty_permissive():
145+
"""
146+
Test that the float converter's parse with verify="ignore", given an empty
147+
string, returns the null value masked.
148+
"""
149+
config = {"verify": "ignore"}
150+
field = tree.Field(None, name="c", datatype="float", config=config)
151+
c = converters.get_converter(field, config=config)
152+
assert c.parse("", config=config) == (c.null, True)
153+
154+
155+
def test_double_parse_zero_number():
156+
"""
157+
Test that the double converter's parse, given the number 0.0 rather than
158+
a string, returns 0.0 unmasked. MIN and MAX values reach parse this way.
159+
"""
160+
field = tree.Field(None, name="c", datatype="double")
161+
c = converters.get_converter(field)
162+
assert c.parse(0.0) == (0.0, False)
163+
164+
165+
def test_float_parse_null_value():
166+
"""
167+
Test that the float converter's parse, given "-999.9" on a field whose
168+
null is -999.9, returns -999.9 masked. -999.9 is not exact in float32.
169+
"""
170+
field = tree.Field(None, name="c", datatype="float")
171+
field.values.null = "-999.9"
172+
c = converters.get_converter(field)
173+
assert c.parse("-999.9") == (-999.9, True)
174+
175+
144176
def test_double_array():
145177
config = {"verify": "exception", "version_1_3_or_later": True}
146178
field = tree.Field(None, name="c", datatype="double", arraysize="3", config=config)
@@ -284,6 +316,39 @@ def test_integer_overflow():
284316
c.parse("-2208988800", config=config)
285317

286318

319+
@pytest.mark.parametrize("value", [np.iinfo(np.int32).min, np.iinfo(np.int32).max])
320+
def test_int_parse_range_limits(value):
321+
"""
322+
Test that the int converter's parse, given the smallest or the largest
323+
32-bit int as a string, returns that value unmasked.
324+
"""
325+
field = tree.Field(None, name="c", datatype="int")
326+
c = converters.get_converter(field)
327+
assert c.parse(str(value)) == (value, False)
328+
329+
330+
def test_int_parse_null_value():
331+
"""
332+
Test that the int converter's parse, given "-1" on a field whose null is
333+
-1, returns -1 masked.
334+
"""
335+
field = tree.Field(None, name="c", datatype="int")
336+
field.values.null = "-1"
337+
c = converters.get_converter(field)
338+
assert c.parse("-1") == (-1, True)
339+
340+
341+
def test_int_parse_empty():
342+
"""
343+
Test that the int converter's parse for VOTable 1.3 or later, given an
344+
empty string, returns 0 masked.
345+
"""
346+
config = {"version_1_3_or_later": True}
347+
field = tree.Field(None, name="c", datatype="int", config=config)
348+
c = converters.get_converter(field, config=config)
349+
assert c.parse("", config=config) == (0, True)
350+
351+
287352
def test_float_default_precision():
288353
config = {"verify": "exception"}
289354

‎astropy/io/votable/tests/test_exception.py‎

Lines changed: 37 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
# Licensed under a 3-clause BSD style license - see LICENSE.rst
2+
import warnings
3+
24
import pytest
35

46
# LOCAL
@@ -41,18 +43,49 @@ def test_parse_vowarning():
4143

4244
def test_suppress_warnings():
4345
cfg = {}
44-
warn = exceptions.W01("foo")
4546

4647
with exceptions.conf.set_temp("max_warnings", 2):
4748
with pytest.warns(exceptions.W01) as record:
48-
exceptions._suppressed_warning(warn, cfg)
49+
exceptions._suppressed_warning(exceptions.W01, "foo", cfg, None)
4950
assert len(record) == 1
5051
assert "suppressing" not in str(record[0].message)
5152

5253
with pytest.warns(exceptions.W01, match="suppressing"):
53-
exceptions._suppressed_warning(warn, cfg)
54+
exceptions._suppressed_warning(exceptions.W01, "foo", cfg, None)
5455

55-
exceptions._suppressed_warning(warn, cfg)
56+
exceptions._suppressed_warning(exceptions.W01, "foo", cfg, None)
5657

5758
assert cfg["_warning_counts"][exceptions.W01] == 3
5859
assert exceptions.conf.max_warnings == 10
60+
61+
62+
def test_vo_warn_counts_suppressed_warnings():
63+
"""
64+
Test that vo_warn called four times with max_warnings set to 2 emits two
65+
W01 warnings and counts all four calls.
66+
"""
67+
cfg = {}
68+
69+
with exceptions.conf.set_temp("max_warnings", 2):
70+
with pytest.warns(exceptions.W01) as record:
71+
for _ in range(4):
72+
exceptions.vo_warn(exceptions.W01, config=cfg)
73+
74+
assert len(record) == 2
75+
assert cfg["_warning_counts"][exceptions.W01] == 4
76+
77+
78+
def test_vo_warn_with_max_warnings_zero():
79+
"""
80+
Test that vo_warn called with W01 and then W02 while max_warnings is 0
81+
emits no warnings and counts each class once.
82+
"""
83+
cfg = {}
84+
85+
with exceptions.conf.set_temp("max_warnings", 0):
86+
with warnings.catch_warnings():
87+
warnings.simplefilter("error")
88+
exceptions.vo_warn(exceptions.W01, config=cfg)
89+
exceptions.vo_warn(exceptions.W02, ("x", "a b"), config=cfg)
90+
91+
assert cfg["_warning_counts"] == {exceptions.W01: 1, exceptions.W02: 1}

‎astropy/io/votable/tree.py‎

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3079,8 +3079,13 @@ def _parse_tabledata(self, iterator, colnumbers, config):
30793079
j = 0 # index of the column being written in array (not necessarily == i)
30803080
for start, tag, data, pos in iterator:
30813081
if start:
3082-
binary = data.get("encoding", None) == "base64"
3083-
warn_unknown_attrs(tag, data.keys(), config, pos, ["encoding"])
3082+
if data:
3083+
binary = data.get("encoding", None) == "base64"
3084+
warn_unknown_attrs(
3085+
tag, data.keys(), config, pos, ["encoding"]
3086+
)
3087+
else:
3088+
binary = False
30843089
else:
30853090
if tag == "TD":
30863091
if i >= len(fields):
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Reading a VOTable stored as TABLEDATA is faster, mostly for numeric and empty
2+
cells: about 1.6 times faster for a 75 MB Gaia cone search of 44,000 rows and
3+
100 columns, and 1.2 to 1.5 times faster for the other tables tried.

0 commit comments

Comments
 (0)