Skip to content

Commit 8740049

Browse files
authored
[third-party](faiss) Enable FAISS integration in Doris. (#49644)
### What problem does this PR solve? Enable FAISS integration in Doris. Dependency of faiss is OpenMP, BLAS and LAPACK. OpenMP is distributed with gcc/llvm. OpenBLAS could supply BLAS & LAPACK impl, so we introduced OpenBLAS. If you are using ldb-toolchain, and version is before https://github.com/amosbird/ldb_toolchain_gen/releases/tag/v0.24, gcc should be used to compile openblas and faiss, since libopm.a is missing. Build new thirdparty: ``` sh build-thirdparty.sh openblas sh build-thirdparty.sh faiss ``` `export ENABLE_BUILD_FAISS=ON` to make doris link with faiss.
1 parent b1108d1 commit 8740049

7 files changed

Lines changed: 170 additions & 0 deletions

File tree

‎be/CMakeLists.txt‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,6 +181,9 @@ endif()
181181

182182
set(GPERFTOOLS_HOME "${THIRDPARTY_DIR}/gperftools")
183183

184+
option(BUILD_FAISS "Link doris with faiss for vector similarity search" OFF)
185+
message(STATUS "build faiss: ${BUILD_FAISS}")
186+
184187
include (cmake/thirdparty.cmake)
185188

186189
find_program(THRIFT_COMPILER thrift ${CMAKE_SOURCE_DIR}/bin)

‎be/cmake/thirdparty.cmake‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -175,3 +175,8 @@ endif()
175175
add_thirdparty(icuuc LIB64)
176176
add_thirdparty(icui18n LIB64)
177177
add_thirdparty(icudata LIB64)
178+
179+
if (BUILD_FAISS)
180+
add_thirdparty(openblas LIB64)
181+
add_thirdparty(faiss LIB64)
182+
endif()

‎build.sh‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ Usage: $0 <options>
7070
DISABLE_BE_JAVA_EXTENSIONS If set DISABLE_BE_JAVA_EXTENSIONS=ON, we will do not build binary with java-udf,hudi-scanner,jdbc-scanner and so on Default is OFF.
7171
DISABLE_JAVA_CHECK_STYLE If set DISABLE_JAVA_CHECK_STYLE=ON, it will skip style check of java code in FE.
7272
DISABLE_BUILD_AZURE If set DISABLE_BUILD_AZURE=ON, it will not build azure into BE.
73+
ENABLE_BUILD_FAISS If set BUILD_FAISS=ON, it will link BE with faiss.
7374
7475
Eg.
7576
$0 build all
@@ -173,6 +174,7 @@ PARAMETER_COUNT="$#"
173174
PARAMETER_FLAG=0
174175
DENABLE_CLANG_COVERAGE='OFF'
175176
BUILD_AZURE='ON'
177+
BUILD_FAISS='OFF'
176178
BUILD_UI=1
177179
if [[ "$#" == 1 ]]; then
178180
# default
@@ -472,6 +474,10 @@ if [[ -n "${DISABLE_BUILD_AZURE}" ]]; then
472474
BUILD_AZURE='OFF'
473475
fi
474476

477+
if [[ -n "${ENABLE_BUILD_FAISS}" ]]; then
478+
BUILD_FAISS='ON'
479+
fi
480+
475481
if [[ -z "${ENABLE_INJECTION_POINT}" ]]; then
476482
ENABLE_INJECTION_POINT='OFF'
477483
fi
@@ -640,6 +646,7 @@ if [[ "${BUILD_BE}" -eq 1 ]]; then
640646
-DENABLE_CLANG_COVERAGE="${DENABLE_CLANG_COVERAGE}" \
641647
-DDORIS_JAVA_HOME="${JAVA_HOME}" \
642648
-DBUILD_AZURE="${BUILD_AZURE}" \
649+
-DBUILD_FAISS="${BUILD_FAISS}" \
643650
"${DORIS_HOME}/be"
644651

645652
if [[ "${OUTPUT_BE_BINARY}" -eq 1 ]]; then
@@ -681,6 +688,7 @@ if [[ "${BUILD_CLOUD}" -eq 1 ]]; then
681688
-DEXTRA_CXX_FLAGS="${EXTRA_CXX_FLAGS}" \
682689
-DBUILD_AZURE="${BUILD_AZURE}" \
683690
-DBUILD_CHECK_META="${BUILD_CHECK_META:-OFF}" \
691+
-DBUILD_FAISS="${BUILD_FAISS}" \
684692
"${DORIS_HOME}/cloud/"
685693
"${BUILD_SYSTEM}" -j "${PARALLEL}"
686694
"${BUILD_SYSTEM}" install

‎thirdparty/build-thirdparty.sh‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1885,6 +1885,66 @@ build_pugixml() {
18851885
cp "${TP_SOURCE_DIR}/${PUGIXML_SOURCE}/src/pugiconfig.hpp" "${TP_INSTALL_DIR}/include/"
18861886
}
18871887

1888+
build_openblas() {
1889+
check_if_source_exist "${OPENBLAS_SOURCE}"
1890+
cd "${TP_SOURCE_DIR}/${OPENBLAS_SOURCE}"
1891+
1892+
rm -rf "${BUILD_DIR}"
1893+
mkdir -p "${BUILD_DIR}"
1894+
cd "${BUILD_DIR}"
1895+
OPENBLAS_CMAKE_OPTIONS=(
1896+
"-DCMAKE_PREFIX_PATH=${TP_INSTALL_DIR}"
1897+
"-DCMAKE_INSTALL_PREFIX=${TP_INSTALL_DIR}"
1898+
"-DCMAKE_BUILD_TYPE=Release"
1899+
"-DBUILD_WITHOUT_LAPACK=OFF"
1900+
"-DNO_SHARED=TRUE"
1901+
"-DNO_AVX512=TRUE"
1902+
"-DC_LAPACK=TRUE"
1903+
"-DUSE_OPENMP=TRUE"
1904+
"-DBUILD_STATIC_LIBS=ON"
1905+
"-DNOFORTRAN=TRUE"
1906+
"-DBUILD_TESTING=OFF"
1907+
"-DBUILD_RELAPACK=ON"
1908+
"-DBUILD_BENCHMARKS=OFF"
1909+
)
1910+
1911+
echo "Building openblas at $(pwd) with cmake parameters: ${OPENBLAS_CMAKE_OPTIONS[*]}"
1912+
1913+
"${CMAKE_CMD}" -G "${GENERATOR}" "${OPENBLAS_CMAKE_OPTIONS[@]}" ..
1914+
"${BUILD_SYSTEM}" -j "${PARALLEL}"
1915+
"${BUILD_SYSTEM}" install
1916+
}
1917+
1918+
build_faiss() {
1919+
check_if_source_exist "${FAISS_SOURCE}"
1920+
echo "Building faiss ${FAISS_SOURCE}"
1921+
cd "${TP_SOURCE_DIR}"
1922+
# if faiss dir not exists, create a symlink to faiss source dir
1923+
# this symlink is necessary since faiss source code must be compiled in a directory named faiss.
1924+
if [[ ! -d "${TP_SOURCE_DIR}/faiss" ]]; then
1925+
ln -s "${FAISS_SOURCE}" faiss
1926+
fi
1927+
cd "${TP_SOURCE_DIR}/faiss"
1928+
1929+
rm -rf "${BUILD_DIR}"
1930+
mkdir -p "${BUILD_DIR}"
1931+
cd "${BUILD_DIR}"
1932+
1933+
FAISS_CMAKE_OPTIONS=(
1934+
"-DDORIS_THIRD_LIB_INSTALL_DIR=${TP_INSTALL_DIR}"
1935+
"-DCMAKE_INSTALL_PREFIX=${TP_INSTALL_DIR}"
1936+
"-DCMAKE_BUILD_TYPE=Release"
1937+
"-DFAISS_ENABLE_GPU=OFF"
1938+
"-DFAISS_ENABLE_PYTHON=OFF"
1939+
)
1940+
1941+
echo "Building faiss at $(pwd) with cmake parameters: ${FAISS_CMAKE_OPTIONS[*]}"
1942+
1943+
"${CMAKE_CMD}" -G "${GENERATOR}" "${FAISS_CMAKE_OPTIONS[@]}" ..
1944+
"${BUILD_SYSTEM}" -j "${PARALLEL}"
1945+
"${BUILD_SYSTEM}" install
1946+
}
1947+
18881948
if [[ "${#packages[@]}" -eq 0 ]]; then
18891949
packages=(
18901950
jindofs

‎thirdparty/download-thirdparty.sh‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -590,5 +590,18 @@ if [[ " ${TP_ARCHIVES[*]} " =~ " THRIFT " ]]; then
590590
echo "Finished patching ${THRIFT_SOURCE}"
591591
fi
592592

593+
# patch faiss cmake so that we can use openblas
594+
if [[ " ${TP_ARCHIVES[*]} " =~ " FAISS " ]]; then
595+
if [[ "${FAISS_SOURCE}" = "faiss-1.10.0" ]]; then
596+
cd "${TP_SOURCE_DIR}/${FAISS_SOURCE}"
597+
if [[ ! -f "${PATCHED_MARK}" ]]; then
598+
patch -p2 <"${TP_PATCH_DIR}/faiss-1.10.0.patch"
599+
touch "${PATCHED_MARK}"
600+
fi
601+
cd -
602+
fi
603+
echo "Finished patching ${FAISS_SOURCE}"
604+
fi
605+
593606

594607
# vim: ts=4 sw=4 ts=4 tw=100:
Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
--- src/faiss-1.10.0/faiss/CMakeLists.txt 2025-02-01 05:52:00.000000000 +0800
2+
+++ src/faiss-1.10.0/faiss/CMakeLists.txt.new 2025-03-28 19:45:37.513624103 +0800
3+
@@ -381,19 +381,51 @@
4+
target_link_libraries(faiss_avx512 PRIVATE ${MKL_LIBRARIES})
5+
target_link_libraries(faiss_avx512_spr PRIVATE ${MKL_LIBRARIES})
6+
else()
7+
- find_package(BLAS REQUIRED)
8+
- target_link_libraries(faiss PRIVATE ${BLAS_LIBRARIES})
9+
- target_link_libraries(faiss_avx2 PRIVATE ${BLAS_LIBRARIES})
10+
- target_link_libraries(faiss_avx512 PRIVATE ${BLAS_LIBRARIES})
11+
- target_link_libraries(faiss_avx512_spr PRIVATE ${BLAS_LIBRARIES})
12+
- target_link_libraries(faiss_sve PRIVATE ${BLAS_LIBRARIES})
13+
+ # If not found through find_package, look in the DORIS_THIRD_LIB_INSTALL_DIR
14+
+ if(DEFINED DORIS_THIRD_LIB_INSTALL_DIR)
15+
+ set(OpenBLAS_ROOT ${DORIS_THIRD_LIB_INSTALL_DIR})
16+
+
17+
+ # Check if libopenblas exists in DORIS_THIRD_LIB_INSTALL_DIR
18+
+ if(EXISTS "${DORIS_THIRD_LIB_INSTALL_DIR}/lib/libopenblas.a")
19+
+ set(OpenBLAS_LIB "${DORIS_THIRD_LIB_INSTALL_DIR}/lib/libopenblas.a")
20+
+ endif()
21+
+ # Terminate if OpenBLAS_LIB is not found
22+
+ if(NOT OpenBLAS_LIB)
23+
+ message(WARNING "OpenBLAS not found in DORIS_THIRD_LIB_INSTALL_DIR: ${DORIS_THIRD_LIB_INSTALL_DIR}")
24+
+ endif()
25+
26+
- find_package(LAPACK REQUIRED)
27+
- target_link_libraries(faiss PRIVATE ${LAPACK_LIBRARIES})
28+
- target_link_libraries(faiss_avx2 PRIVATE ${LAPACK_LIBRARIES})
29+
- target_link_libraries(faiss_avx512 PRIVATE ${LAPACK_LIBRARIES})
30+
- target_link_libraries(faiss_avx512_spr PRIVATE ${LAPACK_LIBRARIES})
31+
- target_link_libraries(faiss_sve PRIVATE ${LAPACK_LIBRARIES})
32+
+ if(OpenBLAS_LIB)
33+
+ set(OpenBLAS_LIBRARIES ${OpenBLAS_LIB})
34+
+ set(OpenBLAS_FOUND TRUE)
35+
+ message(STATUS "Found OpenBLAS in DORIS_THIRD_LIB_INSTALL_DIR: ${OpenBLAS_LIB}")
36+
+ endif()
37+
+ else()
38+
+ message(WARNING "DORIS_THIRD_LIB_INSTALL_DIR is not defined. Please set it to the directory where OpenBLAS is installed.")
39+
+ endif()
40+
+
41+
+ if(OpenBLAS_FOUND)
42+
+ message(STATUS "Using OpenBLAS: ${OpenBLAS_LIBRARIES}")
43+
+ target_link_libraries(faiss PRIVATE ${OpenBLAS_LIBRARIES})
44+
+ target_link_libraries(faiss_avx2 PRIVATE ${OpenBLAS_LIBRARIES})
45+
+ target_link_libraries(faiss_avx512 PRIVATE ${OpenBLAS_LIBRARIES})
46+
+ target_link_libraries(faiss_avx512_spr PRIVATE ${OpenBLAS_LIBRARIES})
47+
+ target_link_libraries(faiss_sve PRIVATE ${OpenBLAS_LIBRARIES})
48+
+ else()
49+
+ # Fall back to separate BLAS and LAPACK if OpenBLAS is not found
50+
+ find_package(BLAS REQUIRED)
51+
+ target_link_libraries(faiss PRIVATE ${BLAS_LIBRARIES})
52+
+ target_link_libraries(faiss_avx2 PRIVATE ${BLAS_LIBRARIES})
53+
+ target_link_libraries(faiss_avx512 PRIVATE ${BLAS_LIBRARIES})
54+
+ target_link_libraries(faiss_avx512_spr PRIVATE ${BLAS_LIBRARIES})
55+
+ target_link_libraries(faiss_sve PRIVATE ${BLAS_LIBRARIES})
56+
+
57+
+ find_package(LAPACK REQUIRED)
58+
+ target_link_libraries(faiss PRIVATE ${LAPACK_LIBRARIES})
59+
+ target_link_libraries(faiss_avx2 PRIVATE ${LAPACK_LIBRARIES})
60+
+ target_link_libraries(faiss_avx512 PRIVATE ${LAPACK_LIBRARIES})
61+
+ target_link_libraries(faiss_avx512_spr PRIVATE ${LAPACK_LIBRARIES})
62+
+ target_link_libraries(faiss_sve PRIVATE ${LAPACK_LIBRARIES})
63+
+ endif()
64+
endif()
65+
66+
install(TARGETS faiss

‎thirdparty/vars.sh‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -538,6 +538,19 @@ PUGIXML_NAME=pugixml-1.15.tar.gz
538538
PUGIXML_SOURCE=pugixml-1.15
539539
PUGIXML_MD5SUM="3b894c29455eb33a40b165c6e2de5895"
540540

541+
# openblas
542+
OPENBLAS_DOWNLOAD="https://github.com/OpenMathLib/OpenBLAS/releases/download/v0.3.29/OpenBLAS-0.3.29.tar.gz"
543+
OPENBLAS_NAME="OpenBLAS-0.3.29.tar.gz"
544+
OPENBLAS_SOURCE="OpenBLAS-0.3.29"
545+
OPENBLAS_MD5SUM="853a0c5c0747c5943e7ef4bbb793162d"
546+
547+
# faiss
548+
FAISS_DOWNLOAD="https://github.com/facebookresearch/faiss/archive/refs/tags/v1.10.0.tar.gz"
549+
FAISS_NAME="faiss-1.10.0.tar.gz"
550+
FAISS_SOURCE="faiss-1.10.0"
551+
FAISS_MD5SUM="f31edf2492808b27cc963d0ab316a205"
552+
553+
541554
# all thirdparties which need to be downloaded is set in array TP_ARCHIVES
542555
export TP_ARCHIVES=(
543556
'LIBEVENT'
@@ -618,6 +631,8 @@ export TP_ARCHIVES=(
618631
'ICU'
619632
'JINDOFS'
620633
'PUGIXML'
634+
'OPENBLAS'
635+
'FAISS'
621636
)
622637

623638
if [[ "$(uname -s)" == 'Darwin' ]]; then

0 commit comments

Comments
 (0)