Skip to content

Commit e8a1c87

Browse files
KRRT7nedbat
andauthored
perf: speed up SQLite numbits aggregation (#2239)
* perf: aggregate numbits with integer bitsets * fix: satisfy pylint in numbits aggregate * fix: satisfy mypy for SQLite aggregate test * fix: place mypy suppression on aggregate argument * docs --------- Co-authored-by: Ned Batchelder <[email protected]>
1 parent 18c2a59 commit e8a1c87

4 files changed

Lines changed: 26 additions & 3 deletions

File tree

‎CHANGES.rst‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,12 @@ Unreleased
2929
dropped. Now a warning is issued, thanks to `Jisang Han <pull 2234_>`_.
3030
Closes `issue 2200`_.
3131

32+
- A performance improvement in the low-level line number bookkeeping when
33+
combining data files, thanks to `Kevin Turcios <pull 2239_>`_.
34+
3235
.. _issue 2200: https://github.com/coveragepy/coveragepy/issues/2200
3336
.. _pull 2234: https://github.com/coveragepy/coveragepy/pull/2234
37+
.. _pull 2239: https://github.com/coveragepy/coveragepy/pull/2239
3438

3539

3640
.. start-releases

‎CONTRIBUTORS.txt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -140,6 +140,7 @@ Karthikeyan Singaravelan
140140
Kassandra Keeton
141141
Ken Schackart
142142
Kevin Brown-Silva
143+
Kevin Turcios
143144
Kjell Braden
144145
Krystian Kichewko
145146
Kumar Aditya

‎coverage/sqldata.py‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -132,15 +132,19 @@ class NumbitsUnionAgg:
132132
"""SQLite aggregate function for computing union of numbits."""
133133

134134
def __init__(self) -> None:
135-
self.result = b""
135+
self.result = 0
136+
self.result_nbytes = 0
136137

137138
def step(self, value: bytes) -> None:
138139
"""Process one value in the aggregation."""
139-
self.result = numbits_union(self.result, value)
140+
self.result |= int.from_bytes(value, "little")
141+
self.result_nbytes = max(self.result_nbytes, len(value))
140142

141143
def finalize(self) -> bytes:
142144
"""Return the final aggregated result."""
143-
return self.result
145+
if not self.result_nbytes:
146+
return b""
147+
return self.result.to_bytes(self.result_nbytes, "little")
144148

145149

146150
class CoverageData:

‎tests/test_numbits.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
num_in_numbits,
2121
register_sqlite_functions,
2222
)
23+
from coverage.sqldata import NumbitsUnionAgg
2324

2425
from tests.coveragetest import CoverageTest
2526
from tests.hypo import example, given, settings
@@ -112,6 +113,11 @@ def setUp(self) -> None:
112113
conn = sqlite3.connect(":memory:")
113114
register_sqlite_functions(conn)
114115
self.cursor = conn.cursor()
116+
conn.create_aggregate(
117+
"numbits_union_agg",
118+
1,
119+
NumbitsUnionAgg, # type: ignore[arg-type]
120+
)
115121
self.cursor.execute("create table data (id int, numbits blob)")
116122
self.cursor.executemany(
117123
"insert into data (id, numbits) values (?, ?)",
@@ -155,6 +161,14 @@ def test_numbits_union(self) -> None:
155161
answer = numbits_to_nums(list(res)[0][0])
156162
assert expected == answer
157163

164+
def test_numbits_union_aggregate(self) -> None:
165+
res = self.cursor.execute(
166+
"select numbits_union_agg(numbits) from data where id in (7, 9)",
167+
)
168+
expected = set(range(7, 100, 7)) | set(range(9, 100, 9))
169+
answer = set(numbits_to_nums(list(res)[0][0]))
170+
assert expected == answer
171+
158172
def test_numbits_intersection(self) -> None:
159173
res = self.cursor.execute(
160174
"select numbits_intersection("

0 commit comments

Comments
 (0)