Skip to content

Commit bd5a7bb

Browse files
committed
Stop XML end events from repeating the text of nested elements
- In get_xml_iterator, the end event of an element that contains other elements now has empty text. Before, it repeated the text of the last element inside it. This applies to both the C and pure-Python parsers. This dramatically reduces memory use for reading large XML files. - For a VOTable BINARY or BINARY2 table, the end events of BINARY2, DATA and TABLE elements no longer each hold a full copy of the base64 data stream. - The C parser frees a text buffer that has grown past 1 MB once its text is handed over, instead of keeping it until parsing ends. - VOTable reading decodes the stream with binascii.a2b_base64 directly, which skips a full copy of the stream text. Base64 text is always ASCII, so valid files decode exactly as before. Only the error for a malformed stream with a non-ASCII character changes: it is now ValueError ("string argument should contain only ASCII characters") instead of UnicodeEncodeError, which is a ValueError subclass. - Reading the 2.5 GB file from issue astropy#8946 now needs about 6.8 GB of memory at its peak, down from about 17 GB. - Update the one test that expected the repeated text, which was just a coverage test pinning current behavior from PR astropy#19922. - Add tests for both parsers and for elements holding more than 1 MB of text.
1 parent b9c7d5a commit bd5a7bb

7 files changed

Lines changed: 105 additions & 3 deletions

File tree

‎astropy/io/votable/tree.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33

44
# STDLIB
55
import base64
6+
import binascii
67
import codecs
78
import gzip
89
import io
@@ -3126,7 +3127,9 @@ def _get_binary_data_stream(self, iterator, config):
31263127
break
31273128

31283129
if have_local_stream:
3129-
buffer = base64.b64decode(buffer.encode("ascii"))
3130+
# binascii reads the ASCII str in place without copying it, which
3131+
# keeps memory usage down for large files.
3132+
buffer = binascii.a2b_base64(buffer)
31303133
string_io = io.BytesIO(buffer)
31313134
string_io.seek(0)
31323135
read = string_io.read

‎astropy/utils/xml/iterparser.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -97,6 +97,8 @@ def end(name):
9797
(parser.CurrentLineNumber, parser.CurrentColumnNumber),
9898
)
9999
)
100+
# The text belongs to this element alone, not to its ancestors
101+
del text[:]
100102

101103
parser = expat.ParserCreate()
102104
parser.specified_attributes = True

‎astropy/utils/xml/src/iterparse.c‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -281,6 +281,38 @@ static void text_clear(IterParser *self)
281281
self->text_size = 0;
282282
}
283283

284+
/*
285+
* Size above which the text buffer is freed once an element's text has been
286+
* handed over. Below it, keeping the buffer avoids reallocating it for every
287+
* element.
288+
*/
289+
#define TEXT_KEEP_MAX (1 << 20)
290+
291+
/*
292+
* Erase the text buffer once its content has been handed to Python.
293+
*
294+
* The text of a single element, such as the base64 STREAM of a VOTable
295+
* BINARY table, can grow the buffer to gigabytes. A buffer that has
296+
* grown past TEXT_KEEP_MAX is therefore swapped for a small one,
297+
* rather than being held until the parser is freed.
298+
*/
299+
static void text_release(IterParser *self)
300+
{
301+
XML_Char *small = NULL;
302+
303+
if (self->text_alloc > TEXT_KEEP_MAX && self->text_alloc > self->buffersize) {
304+
small = malloc((size_t)self->buffersize * sizeof(XML_Char));
305+
/* If that fails, keep using the large buffer */
306+
if (small != NULL) {
307+
free(self->text);
308+
self->text = small;
309+
self->text_alloc = self->buffersize;
310+
}
311+
}
312+
313+
text_clear(self);
314+
}
315+
284316
/******************************************************************************
285317
* XML event handling
286318
******************************************************************************/
@@ -509,6 +541,10 @@ static void endElement(IterParser *self, const XML_Char *name)
509541
PyTuple_SetItem(tuple, 2, pytext);
510542
pytext = NULL;
511543

544+
/* The text belongs to this element alone. Clearing it here stops
545+
the end events of its ancestors from each carrying a copy. */
546+
text_release(self);
547+
512548
pos = make_pos(self);
513549
if (pos == NULL) {
514550
goto fail;

‎astropy/utils/xml/tests/test_c_iterparse.py‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ class XMLTestData:
2424
(True, "root", {}, (1, 21)),
2525
(True, "child", {}, (1, 27)),
2626
(False, "child", "text", (1, 34)),
27-
(False, "root", "text", (1, 34)),
27+
(False, "root", "", (1, 34)),
2828
),
2929
),
3030
id="happy_path_standard_xml",
@@ -78,3 +78,32 @@ def test_c_iterparser_malformed_raises_error() -> None:
7878

7979
with pytest.raises(ValueError, match="no element found"):
8080
list(parser)
81+
82+
83+
def test_c_iterparser_two_elements_each_over_1mb_of_text() -> None:
84+
"""
85+
IterParser on two elements in a row, each holding 2 MB of text, gives each
86+
end event its own text in full.
87+
88+
After handing over more than 1 MB of text, the parser frees its buffer and
89+
starts again with a small one, so the second element checks that the
90+
buffer grows again correctly. Only the C parser has a text buffer, so this
91+
calls it directly.
92+
"""
93+
first = "a" * 2**21
94+
second = "b" * 2**21
95+
stream = io.BytesIO(f"<root><one>{first}</one><two>{second}</two></root>".encode())
96+
parser = _iterparser.IterParser(stream.read, buffersize=1024)
97+
98+
events = [event[:3] for event in parser]
99+
100+
# Each end event has exactly its own element's text: "two" has none of the
101+
# "a"s left over from before the buffer was freed, and "root" has no text.
102+
assert events == [
103+
(True, "root", {}),
104+
(True, "one", {}),
105+
(False, "one", first),
106+
(True, "two", {}),
107+
(False, "two", second),
108+
(False, "root", ""),
109+
]

‎astropy/utils/xml/tests/test_iterparse.py‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,9 @@
55
import io
66
import zlib
77

8-
from astropy.utils.xml.iterparser import _fast_iterparse
8+
import pytest
9+
10+
from astropy.utils.xml.iterparser import _fast_iterparse, get_xml_iterator
911

1012
# The C-based XML parser for VOTables previously used fixed-sized
1113
# buffers (allocated at __init__() time). This test will
@@ -130,3 +132,27 @@ def test_iterparser_over_read_simple():
130132
uncompressed_fd = UngzipFileWrapper(fd)
131133
iterable = _fast_iterparse(uncompressed_fd.read, MINIMUM_REQUESTABLE_BUFFER_SIZE)
132134
list(iterable)
135+
136+
137+
@pytest.mark.parametrize("python_parser", [False, True])
138+
def test_parent_end_event_has_no_child_text(python_parser):
139+
"""
140+
get_xml_iterator on a parent element holding one child with text gives
141+
the parent's end event empty text.
142+
143+
The parent's end event used to repeat the child's text, so reading a
144+
VOTable BINARY stream held several copies of the whole table (#8946).
145+
It runs with both the C parser and the pure-Python parser, which must give
146+
the same events.
147+
"""
148+
xml = io.BytesIO(b"<parent><child>text</child></parent>")
149+
150+
with get_xml_iterator(xml, _debug_python_based_parser=python_parser) as iterator:
151+
events = [event[:3] for event in iterator]
152+
153+
assert events == [
154+
(True, "parent", {}),
155+
(True, "child", {}),
156+
(False, "child", "text"),
157+
(False, "parent", ""),
158+
]
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Reading a VOTable with an embedded BINARY or BINARY2 stream uses much less
2+
memory. A 2.5 GB table previously needed about 16 GB at its peak, and now needs
3+
about 6.5 GB, or 4.9 GB when reading only some of its columns.
Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
In the events from ``astropy.utils.xml.iterparser.get_xml_iterator``, the end
2+
event of an element that contains other elements now has empty text, instead
3+
of repeating the text of the last element inside it.

0 commit comments

Comments
 (0)