|
| 1 | +#!/bin/bash |
| 2 | +set -Eeuo pipefail |
| 3 | +trap 'echo "[prebuild-power] failed at line $LINENO"; exit 1' ERR |
| 4 | +shopt -s dotglob nullglob |
| 5 | +PYTHON_VERSION=3.11 |
| 6 | +WORKDIR=$(pwd) |
| 7 | + |
| 8 | +echo "[prebuild-power] Starting prebuild script..." |
| 9 | + |
| 10 | +# Detect release version from requirements.txt if not already provided |
| 11 | +if [[ -z "${RELEASE_VERSION:-}" && -f "requirements.txt" ]]; then |
| 12 | + RELEASE_VERSION=$(grep -E '^feast\[minimal\]' requirements.txt | sed -E 's/.*==\s*([0-9]+\.[0-9]+\.[0-9]+).*/v\1/') |
| 13 | + echo "[prebuild-power] Detected RELEASE_VERSION=$RELEASE_VERSION from requirements.txt" |
| 14 | +fi |
| 15 | + |
| 16 | +# URL to requirements file (using the release version) |
| 17 | +REQ_FILE_URL="https://raw.githubusercontent.com/opendatahub-io/feast/${RELEASE_VERSION}/sdk/python/requirements/py${PYTHON_VERSION}-ci-requirements.txt" |
| 18 | + |
| 19 | +# Fetch the file |
| 20 | +echo "[prebuild-power] Fetching package versions from $REQ_FILE_URL ..." |
| 21 | +if ! REQ_CONTENT=$(curl -fsSL "$REQ_FILE_URL"); then |
| 22 | + echo "[prebuild-power] Failed to fetch from '$RELEASE_VERSION'. Falling back to master..." |
| 23 | + REQ_FILE_URL="https://raw.githubusercontent.com/opendatahub-io/feast/master/sdk/python/requirements/py${PYTHON_VERSION}-ci-requirements.txt" |
| 24 | + REQ_CONTENT=$(curl -fsSL "$REQ_FILE_URL") |
| 25 | +fi |
| 26 | + |
| 27 | +# Extract package versions |
| 28 | +DUCKDB_VER=$(echo "$REQ_CONTENT" | grep -E "^duckdb==" | head -n1 | sed -E 's/.*==([0-9a-zA-Z\.\-]+).*/\1/') |
| 29 | +GRPCIO_VER=$(echo "$REQ_CONTENT" | grep -E "^grpcio==" | head -n1 | sed -E 's/.*==([0-9a-zA-Z\.\-]+).*/\1/') |
| 30 | +PYARROW_VER=$(echo "$REQ_CONTENT" | grep -E "^pyarrow==" | head -n1 | sed -E 's/.*==([0-9a-zA-Z\.\-]+).*/\1/') |
| 31 | +MILVUS_VER=$(echo "$REQ_CONTENT" | grep -E "^milvus-lite==" | head -n1 | sed -E 's/.*==([0-9a-zA-Z\.\-]+).*/\1/') |
| 32 | + |
| 33 | +echo "[prebuild-power] Detected versions:" |
| 34 | +echo " duckdb=$DUCKDB_VER" |
| 35 | +echo " grpcio=$GRPCIO_VER" |
| 36 | +echo " pyarrow=$PYARROW_VER" |
| 37 | +echo " milvus-lite=$MILVUS_VER" |
| 38 | + |
| 39 | +# Ensure all versions were detected |
| 40 | +if [[ -z "$DUCKDB_VER" || -z "$GRPCIO_VER" || -z "$PYARROW_VER" || -z "$MILVUS_VER" ]]; then |
| 41 | + echo "[prebuild-power] Error: One or more package versions could not be detected." |
| 42 | + exit 1 |
| 43 | +fi |
| 44 | + |
| 45 | +dnf install -y gcc-toolset-13 make cmake ninja-build libomp-devel \ |
| 46 | + git python${PYTHON_VERSION} python${PYTHON_VERSION}-devel python${PYTHON_VERSION}-pip \ |
| 47 | + openssl openssl-devel zlib-devel libuuid-devel |
| 48 | + |
| 49 | +# Enable GCC toolset |
| 50 | +source /opt/rh/gcc-toolset-13/enable |
| 51 | +export CXX=/opt/rh/gcc-toolset-13/root/usr/bin/g++ |
| 52 | + |
| 53 | +# Ensure CXXFLAGS and LINKFLAGS are initialized |
| 54 | +: "${CMAKE_ARGS:=""}" |
| 55 | +: "${CXXFLAGS:=""}" |
| 56 | +: "${CFLAGS:=""}" |
| 57 | +: "${LINKFLAGS:=""}" |
| 58 | + |
| 59 | +# Installing Python build dependencies |
| 60 | +python${PYTHON_VERSION} -m pip install build wheel setuptools ninja pybind11 numpy setuptools_scm Cython==3.0.8 |
| 61 | + |
| 62 | +# Directory to collect built wheels |
| 63 | +mkdir -p /wheelhouse |
| 64 | + |
| 65 | +####################################################### |
| 66 | +# Build DuckDB (Python package) |
| 67 | +####################################################### |
| 68 | +echo "[prebuild-power] Building duckdb==$DUCKDB_VER ..." |
| 69 | +git clone https://github.com/duckdb/duckdb.git |
| 70 | +cd duckdb |
| 71 | +git checkout "v${DUCKDB_VER}" |
| 72 | +cd tools/pythonpkg |
| 73 | +python${PYTHON_VERSION} -m build --wheel --no-isolation |
| 74 | +ls dist/*.whl >/dev/null |
| 75 | +cp -v dist/*.whl /wheelhouse/ |
| 76 | +cd $WORKDIR |
| 77 | + |
| 78 | +####################################################### |
| 79 | +# Build gRPC (Python package) |
| 80 | +####################################################### |
| 81 | +echo "[prebuild-power] Building grpcio==$GRPCIO_VER ..." |
| 82 | +GRPC_PYTHON_BUILD_SYSTEM_OPENSSL=1 python${PYTHON_VERSION} -m pip install --no-binary=:all: "grpcio==${GRPCIO_VER}" |
| 83 | + |
| 84 | +####################################################### |
| 85 | +# Build Pyarrow (Python package) |
| 86 | +####################################################### |
| 87 | +echo "[prebuild-power] Building pyarrow==$PYARROW_VER ..." |
| 88 | +git clone https://github.com/apache/arrow.git |
| 89 | +cd arrow |
| 90 | +git checkout "apache-arrow-${PYARROW_VER}" |
| 91 | +git submodule update --init --recursive |
| 92 | +cd cpp |
| 93 | +mkdir -p release && cd release |
| 94 | +cmake -DCMAKE_BUILD_TYPE=Release \ |
| 95 | + -DCMAKE_INSTALL_PREFIX=/usr/local \ |
| 96 | + -DARROW_PYTHON=ON \ |
| 97 | + -DARROW_PARQUET=ON \ |
| 98 | + -DARROW_ORC=ON \ |
| 99 | + -DARROW_FILESYSTEM=ON \ |
| 100 | + -DARROW_WITH_LZ4=ON \ |
| 101 | + -DARROW_WITH_ZSTD=ON \ |
| 102 | + -DARROW_WITH_SNAPPY=ON \ |
| 103 | + -DARROW_JSON=ON \ |
| 104 | + -DARROW_CSV=ON \ |
| 105 | + -DARROW_DATASET=ON \ |
| 106 | + -DARROW_S3=ON \ |
| 107 | + -DARROW_BUILD_TESTS=OFF \ |
| 108 | + -DARROW_SUBSTRAIT=ON \ |
| 109 | + -DProtobuf_SOURCE=BUNDLED \ |
| 110 | + -DARROW_DEPENDENCY_SOURCE=BUNDLED \ |
| 111 | + .. |
| 112 | +make -j"$(nproc)" |
| 113 | +make install |
| 114 | +cd ../../python |
| 115 | +export BUILD_TYPE=release |
| 116 | +python${PYTHON_VERSION} setup.py build_ext --build-type=$BUILD_TYPE --bundle-arrow-cpp bdist_wheel |
| 117 | +ls dist/*.whl >/dev/null |
| 118 | +cp -v dist/*.whl /wheelhouse/ |
| 119 | +cd $WORKDIR |
| 120 | + |
| 121 | +####################################################### |
| 122 | +# Build Milvus-Lite (Python package) |
| 123 | +####################################################### |
| 124 | +echo "[prebuild-power] Building milvus-lite==$MILVUS_VER ..." |
| 125 | +# Remove gcc-toolset-13; Milvus-Lite build (via Conan) requires standard gcc |
| 126 | +dnf remove -y gcc-toolset-13 |
| 127 | + |
| 128 | +dnf install -y perl ncurses-devel wget openblas-devel cargo gcc gcc-c++ libstdc++-static which libaio \ |
| 129 | + libtool m4 autoconf automake zlib-devel libffi-devel scl-utils xz |
| 130 | + |
| 131 | +export CC=gcc |
| 132 | +export CXX=g++ |
| 133 | +export CXXFLAGS="-std=c++17" |
| 134 | + |
| 135 | +python${PYTHON_VERSION} -m pip install conan==1.64.1 |
| 136 | + |
| 137 | +git clone https://github.com/milvus-io/milvus-lite |
| 138 | +cd milvus-lite/python |
| 139 | +git checkout "v${MILVUS_VER}" |
| 140 | +git submodule update --init --recursive |
| 141 | +python${PYTHON_VERSION} -m pip install -v -e . |
| 142 | +cd $WORKDIR |
| 143 | + |
| 144 | +echo "[prebuild-power] All packages built successfully." |
0 commit comments