Skip to content

Commit a7b9979

Browse files
committed
[test](lance) Cover a legacy vector index and a historical two-phase fetch
A vector index written before Lance 0.20 carries no index details. Lance infers them from the index files when it loads the index, so the FE plans it and the BE searches it through the planned segment rather than flat. legacy_vector_index.lance, built by pylance 0.18.2 and carried over as-is by the catalog build, and test_lance_vector_search_legacy_index check that it is planned with one segment and returns the flat search's rows. The tag two-phase case now fetches row 3, which version 5 still has and version 6 deleted, so a fetch from a later version would return fewer rows, and it asserts the materialization node.
1 parent e55bd49 commit a7b9979

13 files changed

Lines changed: 229 additions & 10 deletions

File tree

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
"""Generate the Lance legacy vector index regression fixture.
19+
20+
legacy_vector_index.lance is a root table of the preinstalled Directory catalog next to
21+
time_travel.lance. Its vector index was written by pylance 0.18.2, which predates index details
22+
in the manifest, so no segment of vec_idx carries details:
23+
24+
version 1 create row_id 1..512 fragment 0
25+
version 2 vector index vec_idx (IVF_PQ, l2, 2 partitions, 2 sub-vectors) over fragment 0
26+
27+
vec = [row_id, row_id + 1, row_id + 2, row_id + 3] as float32. Lance infers the missing details
28+
from the index files when it loads the index, so the FE plans vec_idx like any other index and
29+
the BE searches it through the planned segment instead of falling back to a flat search.
30+
31+
Build it with pylance 0.18.2, not the writer pinned in lance_fixture_requirements.txt; the
32+
catalog build carries it over as-is. --check runs with any reader.
33+
34+
Usage:
35+
pip install pylance==0.18.2 && python3 lance_build_legacy_vector_index.py preinstalled_data/lance
36+
python3 lance_build_legacy_vector_index.py --check preinstalled_data/lance
37+
"""
38+
import argparse
39+
import shutil
40+
from pathlib import Path
41+
42+
import lance
43+
import pyarrow as pa
44+
45+
LEGACY_DIR = "legacy_vector_index.lance"
46+
WRITER_VERSION = "0.18.2"
47+
ROWS = 512
48+
DIMENSION = 4
49+
50+
51+
def build(output: Path) -> None:
52+
if lance.__version__ != WRITER_VERSION:
53+
raise SystemExit(f"build {LEGACY_DIR} with pylance {WRITER_VERSION}, not {lance.__version__}")
54+
if output.exists():
55+
shutil.rmtree(output)
56+
ids = list(range(1, ROWS + 1))
57+
table = pa.table({
58+
"row_id": pa.array(ids, pa.int32()),
59+
"vec": pa.array([[float(i + j) for j in range(DIMENSION)] for i in ids],
60+
pa.list_(pa.float32(), DIMENSION)),
61+
})
62+
uri = str(output)
63+
lance.write_dataset(table, uri, mode="create")
64+
lance.dataset(uri).create_index("vec", "IVF_PQ", name="vec_idx", metric="l2", num_partitions=2,
65+
num_sub_vectors=2, sample_rate=256)
66+
assert lance.dataset(uri).version == 2, lance.dataset(uri).version
67+
68+
69+
def check(output: Path) -> None:
70+
dataset = lance.dataset(str(output))
71+
assert [v["version"] for v in dataset.versions()] == [1, 2], dataset.versions()
72+
row_ids = sorted(dataset.to_table(columns=["row_id"])["row_id"].to_pylist())
73+
assert row_ids == list(range(1, ROWS + 1)), "rows differ"
74+
indexes = dataset.list_indices()
75+
assert [index["name"] for index in indexes] == ["vec_idx"], indexes
76+
assert sorted(indexes[0]["fragment_ids"]) == [0], indexes
77+
manifests = sorted((output / "_versions").glob("*.manifest"))
78+
assert manifests, f"no manifest under {output / '_versions'}"
79+
for manifest in manifests:
80+
content = manifest.read_bytes()
81+
assert b"VectorIndexDetails" not in content, f"{manifest} carries index details"
82+
assert WRITER_VERSION.encode() in content, f"{manifest} was not written by pylance {WRITER_VERSION}"
83+
84+
85+
def main() -> int:
86+
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
87+
parser.add_argument("root", type=Path, help="directory holding the fixture")
88+
parser.add_argument("--check", action="store_true", help="verify the existing fixture")
89+
args = parser.parse_args()
90+
if args.check:
91+
check(args.root / LEGACY_DIR)
92+
else:
93+
build(args.root / LEGACY_DIR)
94+
check(args.root / LEGACY_DIR)
95+
print(f"{LEGACY_DIR} {'checked' if args.check else 'written'}")
96+
return 0
97+
98+
99+
if __name__ == "__main__":
100+
raise SystemExit(main())

‎docker/thirdparties/docker-compose/iceberg/scripts/lance_build_preinstalled_catalog.py‎

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,10 @@
3939
- search_snapshot_evolved.lance
4040
A column added and renamed after the vector index; rebuilt, since
4141
no suite depends on its commit times.
42+
- legacy_vector_index.lance
43+
A vector index written by pylance 0.18.2, without index details;
44+
carried over as-is because this writer cannot rebuild it
45+
(see lance_build_legacy_vector_index.py).
4246
- The `doris` namespace with two full-text-search fixtures, one indexed vector table per cell of the
4347
algorithm x element type x metric matrix (hash-prefixed directories), listed in
4448
VECTOR_TABLES below; BREADTH_TABLE, one table carrying the remaining cells at plan
@@ -96,6 +100,8 @@
96100
import lance_namespace
97101
import pyarrow as pa
98102
import pyarrow.ipc as ipc
103+
from lance_build_legacy_vector_index import LEGACY_DIR as LEGACY_VECTOR_INDEX_DIR
104+
from lance_build_legacy_vector_index import check as check_legacy_vector_index
99105
from lance_build_multivector import build as build_multivector, check as check_multivector
100106
from lance_build_nested_null import build as build_nested_null, check as check_nested_null
101107
from lance_build_search_snapshot import (
@@ -950,12 +956,12 @@ def build_multi_frag(root: Path) -> None:
950956
lance.dataset(location).delete(f"row_id in ({deleted})")
951957

952958

953-
def build(root: Path, all_types_source: Path, time_travel_source: Path, search_snapshot_root: Path) -> None:
959+
def build(root: Path, all_types_source: Path, time_travel_source: Path, carried_root: Path) -> None:
954960
shutil.copytree(all_types_source, root / ALL_TYPES_DIR)
955961
# Not rebuilt: their commit times are hard-coded in the time-travel and search-snapshot suites.
956962
shutil.copytree(time_travel_source, root / TIME_TRAVEL_DIR)
957-
for name in (SEARCH_SNAPSHOT_DIR, SEARCH_SNAPSHOT_PRUNED_DIR):
958-
shutil.copytree(search_snapshot_root / name, root / name)
963+
for name in (SEARCH_SNAPSHOT_DIR, SEARCH_SNAPSHOT_PRUNED_DIR, LEGACY_VECTOR_INDEX_DIR):
964+
shutil.copytree(carried_root / name, root / name)
959965
build_search_snapshot_evolved(root / SEARCH_SNAPSHOT_EVOLVED_DIR)
960966
build_multi_frag(root)
961967
# Recreate this fixture in staging because promotion replaces the entire catalog tree.
@@ -1752,6 +1758,7 @@ def check_catalog(root: Path) -> None:
17521758
check_nested_null(root / NESTED_NULL_DIR)
17531759
check_time_travel(root / TIME_TRAVEL_DIR)
17541760
check_search_snapshot(root)
1761+
check_legacy_vector_index(root / LEGACY_VECTOR_INDEX_DIR)
17551762
check_multivector(root / "multivector.lance")
17561763

17571764
full_fts = namespace.describe_table(DescribeTableRequest(id=[NAMESPACE, FTS_TABLE]))
@@ -1823,9 +1830,9 @@ def main() -> int:
18231830
if not time_travel_source.is_dir():
18241831
print(f"missing time_travel source: {time_travel_source}", file=sys.stderr)
18251832
return 1
1826-
for name in (SEARCH_SNAPSHOT_DIR, SEARCH_SNAPSHOT_PRUNED_DIR):
1833+
for name in (SEARCH_SNAPSHOT_DIR, SEARCH_SNAPSHOT_PRUNED_DIR, LEGACY_VECTOR_INDEX_DIR):
18271834
if not (output / name).is_dir():
1828-
print(f"missing search_snapshot source: {output / name}", file=sys.stderr)
1835+
print(f"missing carried-over source: {output / name}", file=sys.stderr)
18291836
return 1
18301837

18311838
with tempfile.TemporaryDirectory(prefix="lance_fixture_") as staging_name:

‎docker/thirdparties/docker-compose/iceberg/scripts/lance_fixture_requirements.txt‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,10 @@
3838
# it: the Spark and REST catalog suites read and register into the same __manifest this
3939
# writer produces, and they must pass for a fixture rebuild to be acceptable.
4040
#
41+
# One fixture is the exception: legacy_vector_index.lance must carry a vector index without
42+
# index details, which only writers before 0.20 produce, so lance_build_legacy_vector_index.py
43+
# builds it with pylance 0.18.2 and the catalog build carries it over as-is.
44+
#
4145
# The pylance and lance-namespace pins are enforced rather than advisory:
4246
# check_pinned_writer() refuses to build or verify the fixture unless the installed versions
4347
# match them exactly. That check has to exist because the generator's own self-check cannot

‎regression-test/data/external_table_p0/lance/test_lance_search_snapshot.out‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -146,9 +146,9 @@
146146
103 doc 103 doris 36.0
147147

148148
-- !tag_two_phase --
149-
24 doc 24 lance 23104.0
150-
23 doc 23 doris 23716.0
151-
22 doc 22 lance 24336.0
149+
3 doc 3 doris 0.25
150+
4 doc 4 lance 2.25
151+
2 doc 2 lance 6.25
152152

153153
-- !evolved_two_phase --
154154
2 [2, 3, 4, 5] note 2 doc 2 lance 0.25

0 commit comments

Comments
 (0)