Skip to content

Commit 0d630aa

Browse files
committed
[fix](test) Keep Lance Unicode fixture paths portable
Apply the fixture portability fix from #69124 to branch-4.2.
1 parent 7b81582 commit 0d630aa

7 files changed

Lines changed: 15 additions & 5 deletions

File tree

‎docker/thirdparties/docker-compose/iceberg/iceberg.yaml.tpl‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -229,6 +229,8 @@ services:
229229
# an earlier fixture would win over the one just published and point readers at tables
230230
# that no longer exist. This bucket is not recreated between runs, so a plain copy would
231231
# merge the two catalogs. --remove makes the prefix match the committed fixture exactly.
232+
# The Unicode branch uses an ASCII directory in Git for portable checkout. Restore its
233+
# object keys after each mirror: Lance resolves the original name recorded in the manifests.
232234
entrypoint: >
233235
/bin/sh -c "
234236
until (/usr/bin/mc config host add minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done;
@@ -240,7 +242,8 @@ services:
240242
/usr/bin/mc policy set public minio/warehouse;
241243
/usr/bin/mc cp -r /mnt/data/input/minio/warehouse/* minio/warehouse/;
242244
fi;
243-
/usr/bin/mc mirror --overwrite --remove /mnt/preinstalled_data/lance/ minio/warehouse/lance/;
245+
/usr/bin/mc mirror --overwrite --remove /mnt/preinstalled_data/lance/ minio/warehouse/lance/ &&
246+
/usr/bin/mc mv --recursive minio/warehouse/lance/unicode_branch.lance/tree/dev_u1c89/ minio/warehouse/lance/unicode_branch.lance/tree/devᲉ/ || exit 1;
244247
/usr/bin/mc cp -r /mnt/preinstalled_data/iceberg/ minio/warehouse/wh/multi_catalog/;
245248
"
246249

‎docker/thirdparties/docker-compose/iceberg/scripts/lance_build_unicode_branch.py‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -37,12 +37,18 @@
3737
--storage-option secret_access_key=password --storage-option region=us-east-1 \\
3838
--storage-option allow_http=true
3939
# then sync tree/ and _refs/branches/ from that location back
40+
# store the branch under an ASCII name before committing (Bash syntax)
41+
mv preinstalled_data/lance/unicode_branch.lance/tree/$'dev\\u1c89' \\
42+
preinstalled_data/lance/unicode_branch.lance/tree/dev_u1c89
4043
python3 lance_build_unicode_branch.py --check preinstalled_data/lance
4144
4245
Build it with pylance 12.0.0, the Lance the FE uses, not the writer pinned in
4346
lance_fixture_requirements.txt: Lance 7 stores a non-ASCII branch directory percent-encoded
4447
(tree/dev%E1%B2%89/), which Lance 12 does not find, while Lance 12 stores it as tree/dev\u1c89/.
45-
The catalog build carries the fixture over as-is. --check runs with any reader.
48+
The repository stores that directory as tree/dev_u1c89/ so checkout does not depend on
49+
filesystem Unicode support. iceberg.yaml.tpl restores the original object keys after
50+
uploading to MinIO. The catalog build carries the portable fixture over as-is.
51+
--check verifies this stored layout and runs with any reader.
4652
"""
4753
import argparse
4854
import shutil
@@ -52,7 +58,8 @@
5258
import pyarrow as pa
5359

5460
UNICODE_BRANCH_DIR = "unicode_branch.lance"
55-
BRANCH = "devᲉ"
61+
BRANCH = "dev\u1c89"
62+
STORED_BRANCH = "dev_u1c89"
5663
PARENT_URI = "s3://warehouse/lance/unicode_branch.lance"
5764
WRITER_VERSION = "12.0.0"
5865
DIMENSION = 4
@@ -102,8 +109,8 @@ def check(output: Path) -> None:
102109
assert [v["version"] for v in dataset.versions()] == [1], dataset.versions()
103110
assert row_ids(dataset) == list(range(1, 9)), "main rows differ"
104111
assert list(dataset.branches.list()) == [BRANCH], dataset.branches.list()
105-
# The branch directory keeps the name as is; Lance 12 does not find a percent-encoded one.
106-
manifests = sorted((output / "tree" / BRANCH / "_versions").glob("*.manifest"))
112+
# Keep Git paths portable; the upload restores the exact name Lance 12 resolves.
113+
manifests = sorted((output / "tree" / STORED_BRANCH / "_versions").glob("*.manifest"))
107114
assert len(manifests) == 2, f"branch {BRANCH!r} must carry versions 1 and 2: {manifests}"
108115
for manifest in manifests:
109116
assert PARENT_URI.encode() in manifest.read_bytes(), f"{manifest} must reference {PARENT_URI}"

docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/tree/devᲉ/_transactions/1-1be00bd2-8443-4851-9ab8-4bdcee3ffa5e.txn renamed to docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/tree/dev_u1c89/_transactions/1-1be00bd2-8443-4851-9ab8-4bdcee3ffa5e.txn

File renamed without changes.

docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/tree/devᲉ/_transactions/1-c1998c6f-6213-4b7e-ae7c-cc00cde3b7de.txn renamed to docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/tree/dev_u1c89/_transactions/1-c1998c6f-6213-4b7e-ae7c-cc00cde3b7de.txn

File renamed without changes.

docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/tree/devᲉ/_versions/18446744073709551613.manifest renamed to docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/tree/dev_u1c89/_versions/18446744073709551613.manifest

File renamed without changes.

docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/tree/devᲉ/_versions/18446744073709551614.manifest renamed to docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/tree/dev_u1c89/_versions/18446744073709551614.manifest

File renamed without changes.

docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/tree/devᲉ/data/01110100011000100001111040aa57401ab473a6a921298540.lance renamed to docker/thirdparties/docker-compose/iceberg/scripts/preinstalled_data/lance/unicode_branch.lance/tree/dev_u1c89/data/01110100011000100001111040aa57401ab473a6a921298540.lance

File renamed without changes.

0 commit comments

Comments
 (0)