Skip to content

Commit f46234d

Browse files
committed
[improvement](lance) Push down array label predicates and plan boolean scalar index scans
1 parent 78aa9c9 commit f46234d

43 files changed

Lines changed: 538 additions & 39 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.
Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
"""Build isolated array pushdown fixtures with lance_fixture_requirements.txt.
19+
20+
The standard MinIO mirror publishes these below warehouse/lance/predicate_arrays.
21+
Each dataset uses relative data/index paths and two eight-row fragments. The
22+
partial dataset appends one unindexed fragment after creating its LabelList index.
23+
"""
24+
25+
import argparse
26+
from pathlib import Path
27+
28+
import lance
29+
import pyarrow as pa
30+
31+
32+
LABELS = [["red"], ["blue"], ["red", "blue"], [], None,
33+
["red", "red"], [None, "blue"], ["blue", "red"]] * 2
34+
35+
36+
def build(output):
37+
if not __debug__:
38+
raise RuntimeError("Fixture verification requires assertions; do not use python -O")
39+
if lance.__version__ != "7.0.0":
40+
raise RuntimeError("Use the pinned pylance 7.0.0 fixture writer")
41+
output.mkdir(parents=True, exist_ok=True)
42+
table = pa.table({"id": pa.array(range(16), type=pa.int64()),
43+
"labels": pa.array(LABELS, type=pa.list_(pa.string())),
44+
"category": pa.array([i % 3 for i in range(16)], type=pa.int32())})
45+
for name in ["indexed", "partial", "unindexed"]:
46+
path = output / (name + ".lance")
47+
if path.exists():
48+
raise FileExistsError(path)
49+
first = table.slice(0, 8) if name == "partial" else table
50+
dataset = lance.write_dataset(first, str(path), max_rows_per_file=8, max_rows_per_group=8)
51+
if name != "unindexed":
52+
dataset.create_scalar_index("labels", "LABEL_LIST", name="labels_idx")
53+
dataset.create_scalar_index("category", "BTREE", name="category_idx")
54+
if name == "partial":
55+
lance.write_dataset(table.slice(8), str(path), mode="append", max_rows_per_file=8)
56+
dataset = lance.dataset(str(path))
57+
assert dataset.to_table().to_pydict() == table.to_pydict()
58+
assert len(dataset.get_fragments()) == 2
59+
for predicate, expected in [
60+
("array_contains(labels, 'red')", [0, 2, 5, 7, 8, 10, 13, 15]),
61+
("array_contains(labels, 'red') AND array_contains(labels, 'blue')", [2, 7, 10, 15]),
62+
("array_contains(labels, 'red') OR array_contains(labels, 'blue')",
63+
[0, 1, 2, 5, 6, 7, 8, 9, 10, 13, 14, 15])]:
64+
for use_index in [False, True]:
65+
ids = dataset.to_table(columns=["id"], filter=predicate,
66+
use_scalar_index=use_index)["id"].to_pylist()
67+
assert sorted(ids) == expected, (name, predicate, ids)
68+
assert len(dataset.list_indices()) == (0 if name == "unindexed" else 2)
69+
print("Verified array predicate fixtures (indexed, partial, unindexed)")
70+
71+
72+
if __name__ == "__main__":
73+
parser = argparse.ArgumentParser(description=__doc__)
74+
parser.add_argument("--output", type=Path, default=Path(__file__).parent /
75+
"preinstalled_data/lance/predicate_arrays")
76+
build(parser.parse_args().output)

‎docker/thirdparties/docker-compose/iceberg/scripts/lance_build_preinstalled_catalog.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@
8989
import lance_namespace
9090
import pyarrow as pa
9191
import pyarrow.ipc as ipc
92+
from lance_build_array_predicates import build as build_array_predicates
9293
from lance_build_multivector import build as build_multivector, check as check_multivector
9394
from lance_build_nested_null import build as build_nested_null, check as check_nested_null
9495
from lance_build_time_travel import check as check_time_travel
@@ -1810,6 +1811,7 @@ def main() -> int:
18101811
staging = Path(staging_name) / "lance"
18111812
staging.mkdir()
18121813
build(staging, all_types_source, time_travel_source)
1814+
build_array_predicates(staging / "predicate_arrays")
18131815
check_catalog(staging)
18141816
backup = output.with_name(output.name + ".old")
18151817
if backup.exists():

0 commit comments

Comments
 (0)