Skip to content

Commit e86a498

Browse files
ODH Feature Server Power Support (feast-dev#44)
* Updating dockerfile and prebuild script for power arch * replace CentOS Stream repo * add Power support for feature server * Supports building Power-specific wheels * Addresses coderabbit review comment * Addressing coderabbit review comment * remove setuptools version pin * feature-server: dynamically detect package versions in Power prebuild * feature-server: fix error message to show correct release version * Addressing coderabbit review comment * feature-server: use Python-version variable in fallback URL for Power prebuild
1 parent 6b0206b commit e86a498

2 files changed

Lines changed: 164 additions & 0 deletions

File tree

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,27 @@
11
FROM registry.access.redhat.com/ubi9/python-311:1
22

3+
# OS Packages needs to be installed as root
4+
USER 0
5+
6+
# Copy Power prebuild script
7+
COPY prebuild-power-base.sh /prebuild-power.sh
8+
RUN chmod +x /prebuild-power.sh
9+
310
COPY requirements.txt requirements.txt
11+
12+
# Run prebuild script conditionally (Power only)
13+
RUN if [ "$(uname -m)" = "ppc64le" ]; then \
14+
echo "Power architecture detected. Running prebuild script..."; \
15+
/prebuild-power.sh && pip install /wheelhouse/*.whl; \
16+
echo "Listing built wheels:" && ls -lh /wheelhouse; \
17+
else \
18+
echo "Non-Power architecture detected; skipping Power prebuild"; \
19+
fi
20+
421
RUN pip install -r requirements.txt
522

623
# modify permissions to support running with a random uid
724
RUN chmod g+w $(python -c "import feast.ui as ui; print(ui.__path__)" | tr -d "[']")/build/projects-list.json
25+
26+
# Switch back to the default non-root user
27+
USER 1001
Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
#!/bin/bash
2+
set -Eeuo pipefail
3+
trap 'echo "[prebuild-power] failed at line $LINENO"; exit 1' ERR
4+
shopt -s dotglob nullglob
5+
PYTHON_VERSION=3.11
6+
WORKDIR=$(pwd)
7+
8+
echo "[prebuild-power] Starting prebuild script..."
9+
10+
# Detect release version from requirements.txt if not already provided
11+
if [[ -z "${RELEASE_VERSION:-}" && -f "requirements.txt" ]]; then
12+
RELEASE_VERSION=$(grep -E '^feast\[minimal\]' requirements.txt | sed -E 's/.*==\s*([0-9]+\.[0-9]+\.[0-9]+).*/v\1/')
13+
echo "[prebuild-power] Detected RELEASE_VERSION=$RELEASE_VERSION from requirements.txt"
14+
fi
15+
16+
# URL to requirements file (using the release version)
17+
REQ_FILE_URL="https://raw.githubusercontent.com/opendatahub-io/feast/${RELEASE_VERSION}/sdk/python/requirements/py${PYTHON_VERSION}-ci-requirements.txt"
18+
19+
# Fetch the file
20+
echo "[prebuild-power] Fetching package versions from $REQ_FILE_URL ..."
21+
if ! REQ_CONTENT=$(curl -fsSL "$REQ_FILE_URL"); then
22+
echo "[prebuild-power] Failed to fetch from '$RELEASE_VERSION'. Falling back to master..."
23+
REQ_FILE_URL="https://raw.githubusercontent.com/opendatahub-io/feast/master/sdk/python/requirements/py${PYTHON_VERSION}-ci-requirements.txt"
24+
REQ_CONTENT=$(curl -fsSL "$REQ_FILE_URL")
25+
fi
26+
27+
# Extract package versions
28+
DUCKDB_VER=$(echo "$REQ_CONTENT" | grep -E "^duckdb==" | head -n1 | sed -E 's/.*==([0-9a-zA-Z\.\-]+).*/\1/')
29+
GRPCIO_VER=$(echo "$REQ_CONTENT" | grep -E "^grpcio==" | head -n1 | sed -E 's/.*==([0-9a-zA-Z\.\-]+).*/\1/')
30+
PYARROW_VER=$(echo "$REQ_CONTENT" | grep -E "^pyarrow==" | head -n1 | sed -E 's/.*==([0-9a-zA-Z\.\-]+).*/\1/')
31+
MILVUS_VER=$(echo "$REQ_CONTENT" | grep -E "^milvus-lite==" | head -n1 | sed -E 's/.*==([0-9a-zA-Z\.\-]+).*/\1/')
32+
33+
echo "[prebuild-power] Detected versions:"
34+
echo " duckdb=$DUCKDB_VER"
35+
echo " grpcio=$GRPCIO_VER"
36+
echo " pyarrow=$PYARROW_VER"
37+
echo " milvus-lite=$MILVUS_VER"
38+
39+
# Ensure all versions were detected
40+
if [[ -z "$DUCKDB_VER" || -z "$GRPCIO_VER" || -z "$PYARROW_VER" || -z "$MILVUS_VER" ]]; then
41+
echo "[prebuild-power] Error: One or more package versions could not be detected."
42+
exit 1
43+
fi
44+
45+
dnf install -y gcc-toolset-13 make cmake ninja-build libomp-devel \
46+
git python${PYTHON_VERSION} python${PYTHON_VERSION}-devel python${PYTHON_VERSION}-pip \
47+
openssl openssl-devel zlib-devel libuuid-devel
48+
49+
# Enable GCC toolset
50+
source /opt/rh/gcc-toolset-13/enable
51+
export CXX=/opt/rh/gcc-toolset-13/root/usr/bin/g++
52+
53+
# Ensure CXXFLAGS and LINKFLAGS are initialized
54+
: "${CMAKE_ARGS:=""}"
55+
: "${CXXFLAGS:=""}"
56+
: "${CFLAGS:=""}"
57+
: "${LINKFLAGS:=""}"
58+
59+
# Installing Python build dependencies
60+
python${PYTHON_VERSION} -m pip install build wheel setuptools ninja pybind11 numpy setuptools_scm Cython==3.0.8
61+
62+
# Directory to collect built wheels
63+
mkdir -p /wheelhouse
64+
65+
#######################################################
66+
# Build DuckDB (Python package)
67+
#######################################################
68+
echo "[prebuild-power] Building duckdb==$DUCKDB_VER ..."
69+
git clone https://github.com/duckdb/duckdb.git
70+
cd duckdb
71+
git checkout "v${DUCKDB_VER}"
72+
cd tools/pythonpkg
73+
python${PYTHON_VERSION} -m build --wheel --no-isolation
74+
ls dist/*.whl >/dev/null
75+
cp -v dist/*.whl /wheelhouse/
76+
cd $WORKDIR
77+
78+
#######################################################
79+
# Build gRPC (Python package)
80+
#######################################################
81+
echo "[prebuild-power] Building grpcio==$GRPCIO_VER ..."
82+
GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 python${PYTHON_VERSION} -m pip install --no-binary=:all: "grpcio==${GRPCIO_VER}"
83+
84+
#######################################################
85+
# Build Pyarrow (Python package)
86+
#######################################################
87+
echo "[prebuild-power] Building pyarrow==$PYARROW_VER ..."
88+
git clone https://github.com/apache/arrow.git
89+
cd arrow
90+
git checkout "apache-arrow-${PYARROW_VER}"
91+
git submodule update --init --recursive
92+
cd cpp
93+
mkdir -p release && cd release
94+
cmake -DCMAKE_BUILD_TYPE=Release \
95+
-DCMAKE_INSTALL_PREFIX=/usr/local \
96+
-DARROW_PYTHON=ON \
97+
-DARROW_PARQUET=ON \
98+
-DARROW_ORC=ON \
99+
-DARROW_FILESYSTEM=ON \
100+
-DARROW_WITH_LZ4=ON \
101+
-DARROW_WITH_ZSTD=ON \
102+
-DARROW_WITH_SNAPPY=ON \
103+
-DARROW_JSON=ON \
104+
-DARROW_CSV=ON \
105+
-DARROW_DATASET=ON \
106+
-DARROW_S3=ON \
107+
-DARROW_BUILD_TESTS=OFF \
108+
-DARROW_SUBSTRAIT=ON \
109+
-DProtobuf_SOURCE=BUNDLED \
110+
-DARROW_DEPENDENCY_SOURCE=BUNDLED \
111+
..
112+
make -j"$(nproc)"
113+
make install
114+
cd ../../python
115+
export BUILD_TYPE=release
116+
python${PYTHON_VERSION} setup.py build_ext --build-type=$BUILD_TYPE --bundle-arrow-cpp bdist_wheel
117+
ls dist/*.whl >/dev/null
118+
cp -v dist/*.whl /wheelhouse/
119+
cd $WORKDIR
120+
121+
#######################################################
122+
# Build Milvus-Lite (Python package)
123+
#######################################################
124+
echo "[prebuild-power] Building milvus-lite==$MILVUS_VER ..."
125+
# Remove gcc-toolset-13; Milvus-Lite build (via Conan) requires standard gcc
126+
dnf remove -y gcc-toolset-13
127+
128+
dnf install -y perl ncurses-devel wget openblas-devel cargo gcc gcc-c++ libstdc++-static which libaio \
129+
libtool m4 autoconf automake zlib-devel libffi-devel scl-utils xz
130+
131+
export CC=gcc
132+
export CXX=g++
133+
export CXXFLAGS="-std=c++17"
134+
135+
python${PYTHON_VERSION} -m pip install conan==1.64.1
136+
137+
git clone https://github.com/milvus-io/milvus-lite
138+
cd milvus-lite/python
139+
git checkout "v${MILVUS_VER}"
140+
git submodule update --init --recursive
141+
python${PYTHON_VERSION} -m pip install -v -e .
142+
cd $WORKDIR
143+
144+
echo "[prebuild-power] All packages built successfully."

0 commit comments

Comments
 (0)