Skip to content

Commit 6c24181

Browse files
d-v-bd-v-b
committed
ci: run byte-order-sensitive tests on s390x under QEMU
A weekly and path-filtered job that runs the data type, codec and metadata tests on an emulated big-endian host, in a Debian sid image that takes numpy and numcodecs from Debian packages so nothing heavy compiles under emulation. Assisted-by: ClaudeCode:claude-opus-5-5
1 parent 764b79c commit 6c24181

2 files changed

Lines changed: 93 additions & 0 deletions

File tree

‎.github/workflows/big-endian.yml‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
# Run the byte-order-sensitive tests on a big-endian host: s390x, emulated with QEMU.
2+
# Emulation makes the full suite take about 40 minutes, so this job runs only the tests
3+
# that exercise data types, codecs and metadata, and only when that code changes (plus
4+
# a weekly run to catch anything the path filter misses).
5+
6+
name: Big-endian
7+
8+
on:
9+
pull_request:
10+
branches: [ main ]
11+
paths:
12+
- "src/zarr/codecs/**"
13+
- "src/zarr/core/buffer/**"
14+
- "src/zarr/core/dtype/**"
15+
- "src/zarr/core/metadata/**"
16+
- "src/zarr/metadata/**"
17+
- "tests/test_codecs/**"
18+
- "tests/test_dtype/**"
19+
- "tests/test_metadata/**"
20+
- "ci/big-endian/**"
21+
- ".github/workflows/big-endian.yml"
22+
schedule:
23+
- cron: "0 3 * * 1"
24+
workflow_dispatch:
25+
26+
permissions:
27+
contents: read
28+
29+
concurrency:
30+
group: ${{ github.workflow }}-${{ github.ref }}
31+
cancel-in-progress: true
32+
33+
jobs:
34+
s390x:
35+
name: s390x (QEMU)
36+
runs-on: ubuntu-latest
37+
timeout-minutes: 90
38+
steps:
39+
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
40+
with:
41+
fetch-depth: 0 # hatch-vcs needs the tags to resolve a version
42+
persist-credentials: false
43+
- uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0
44+
with:
45+
platforms: s390x
46+
- uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1
47+
- name: Build test image
48+
uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0
49+
with:
50+
context: ci/big-endian
51+
platforms: linux/s390x
52+
tags: zarr-big-endian
53+
load: true
54+
cache-from: type=gha,scope=big-endian
55+
cache-to: type=gha,scope=big-endian,mode=max
56+
- name: Run tests
57+
run: |
58+
docker run --rm --platform linux/s390x -v "$PWD":/src:ro zarr-big-endian bash -euc '
59+
cp -r /src /work && cd /work
60+
pip install -q --no-deps --no-build-isolation .
61+
python -c "import sys; assert sys.byteorder == \"big\", sys.byteorder"
62+
python -m pytest -p no:cacheprovider --hypothesis-profile ci -n auto \
63+
tests/test_dtype tests/test_dtype_registry.py \
64+
tests/test_codecs tests/test_metadata \
65+
tests/test_array.py tests/test_v2.py tests/test_info.py \
66+
tests/test_pipeline_parity.py tests/test_cli/test_migrate_v3.py
67+
'

‎ci/big-endian/Dockerfile‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
# Test image for a big-endian host (s390x, run under QEMU by .github/workflows/big-endian.yml).
2+
# Debian packages supply numpy and numcodecs, so nothing heavy compiles under emulation;
3+
# this is also the environment Debian tests zarr in. Debian does not package Google's
4+
# crc32c library, so it is built here for the google-crc32c C extension.
5+
FROM debian:sid
6+
7+
RUN apt-get update -qq && apt-get install -y -qq --no-install-recommends \
8+
build-essential cmake git python3 python3-dev python3-pip python3-venv \
9+
python3-crc32c python3-donfig python3-fsspec python3-hatch-vcs python3-hatchling \
10+
python3-hypothesis python3-numcodecs python3-numpy python3-numpydoc python3-packaging \
11+
python3-pytest python3-pytest-asyncio python3-pytest-benchmark python3-pytest-xdist \
12+
python3-tomlkit python3-typer python3-typing-extensions \
13+
&& rm -rf /var/lib/apt/lists/*
14+
15+
RUN git clone -q --depth 1 --branch 1.1.2 https://github.com/google/crc32c /tmp/crc32c \
16+
&& cmake -S /tmp/crc32c -B /tmp/crc32c/build -DBUILD_SHARED_LIBS=1 \
17+
-DCRC32C_BUILD_TESTS=0 -DCRC32C_BUILD_BENCHMARKS=0 -DCRC32C_USE_GLOG=0 \
18+
-DCMAKE_POLICY_VERSION_MINIMUM=3.5 \
19+
&& cmake --build /tmp/crc32c/build -j"$(nproc)" \
20+
&& cmake --install /tmp/crc32c/build \
21+
&& ldconfig \
22+
&& rm -rf /tmp/crc32c
23+
24+
RUN python3 -m venv --system-site-packages /venv
25+
ENV PATH=/venv/bin:$PATH
26+
RUN pip install -q --no-binary google-crc32c google-crc32c msgspec pytest-accept

0 commit comments

Comments
 (0)