Skip to content

Commit 4ecdfe0

Browse files
authored
GH-54902: [C++] Move GCS and google-cloud-cpp to its own libarrow_gcs.so (#54905)
### Rationale for this change Trying to reduce the size of libarrow.so and remove google-cloud-cpp SDK on some builds. Allow for users to plug and play based on requirements and divide our functionality into cleaner modules. ### What changes are included in this PR? Unconditionally build GCS and the google-cloud-cpp SDK into a different module libarrow_gcs.so outside of libarrow.so. Update bindings to link against the new libarrow_gcs.so library. Update the Linux Package jobs to have the new module into a different package. ### Are these changes tested? Yes via CI ### Are there any user-facing changes? Yes, users will need to either link against libarrow_s3.so or register using LoadFileSystemFactories ### Was AI used for this PR? In accordance to the [AI generation guidelines](https://arrow.apache.org/docs/dev/developers/overview.html#ai-generated-code), please disclose below whether and how AI was used in this PR. **PR code and description written by:** - [x] Human - [x] AI **Reviewed before submission by:** - [x] Human - [ ] AI - [ ] Not reviewed * GitHub Issue: #54902 Authored-by: Raúl Cumplido <[email protected]> Signed-off-by: Raúl Cumplido <[email protected]>
1 parent 5f75a12 commit 4ecdfe0

24 files changed

Lines changed: 462 additions & 74 deletions

‎ci/scripts/python_wheel_xlinux_build.sh‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,16 @@
2020
set -ex
2121

2222
function check_arrow_visibility {
23-
nm --demangle --dynamic /tmp/arrow-dist/lib/libarrow.so > nm_arrow.log
23+
local libs=(libarrow.so)
24+
if [[ "${ARROW_S3}" == "ON" ]]; then
25+
libs+=(libarrow_s3.so)
26+
fi
27+
if [[ "${ARROW_GCS}" == "ON" ]]; then
28+
libs+=(libarrow_gcs.so)
29+
fi
30+
for lib in "${libs[@]}"; do
31+
nm --demangle --dynamic "/tmp/arrow-dist/lib/${lib}"
32+
done > nm_arrow.log
2433

2534
# Filter out Arrow symbols and see if anything remains.
2635
# '_init' and '_fini' symbols may or not be present, we don't care.
@@ -37,7 +46,7 @@ function check_arrow_visibility {
3746
if [[ -f visible_symbols.log && ! -s visible_symbols.log ]]; then
3847
return 0
3948
else
40-
echo "== Unexpected symbols exported by libarrow.so =="
49+
echo "== Unexpected symbols exported by ${libs[*]} =="
4150
cat visible_symbols.log
4251
echo "================================================"
4352

‎cpp/CMakeLists.txt‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -484,6 +484,15 @@ if(ARROW_BUILD_STATIC)
484484
endif()
485485
set(ARROW_FLIGHT_PC_REQUIRES_PRIVATE "")
486486

487+
# For arrow-gcs.pc.
488+
set(ARROW_GCS_PC_CFLAGS "")
489+
set(ARROW_GCS_PC_CFLAGS_PRIVATE "")
490+
if(ARROW_BUILD_STATIC)
491+
string(APPEND ARROW_GCS_PC_CFLAGS_PRIVATE " -DARROW_GCS_STATIC")
492+
endif()
493+
set(ARROW_GCS_PC_LIBS_PRIVATE "")
494+
set(ARROW_GCS_PC_REQUIRES_PRIVATE "")
495+
487496
# For arrow-s3.pc.
488497
set(ARROW_S3_PC_CFLAGS "")
489498
set(ARROW_S3_PC_CFLAGS_PRIVATE "")

‎cpp/cmake_modules/ThirdpartyToolchain.cmake‎

Lines changed: 26 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1442,7 +1442,11 @@ endmacro()
14421442
# pkg-config
14431443

14441444
# SDK libraries (on macOS) may be available without .pc files
1445-
macro(arrow_append_pc_system_library PC_PACKAGE PC_PREFIX FALLBACK)
1445+
macro(arrow_append_pc_system_library
1446+
PC_PACKAGE
1447+
PC_PREFIX
1448+
FALLBACK
1449+
ARROW_PC_TARGET)
14461450
find_package(PkgConfig QUIET)
14471451
if(PkgConfig_FOUND)
14481452
pkg_check_modules(${PC_PREFIX}
@@ -1452,10 +1456,10 @@ macro(arrow_append_pc_system_library PC_PACKAGE PC_PREFIX FALLBACK)
14521456
QUIET)
14531457
endif()
14541458
if(PkgConfig_FOUND AND ${PC_PREFIX}_FOUND)
1455-
string(APPEND ARROW_PC_REQUIRES_PRIVATE " ${PC_PACKAGE}")
1459+
string(APPEND ${ARROW_PC_TARGET}_PC_REQUIRES_PRIVATE " ${PC_PACKAGE}")
14561460
else()
1457-
message(STATUS "No .pc for ${PC_PACKAGE}. Using ${FALLBACK} in arrow.pc")
1458-
string(APPEND ARROW_PC_LIBS_PRIVATE " ${FALLBACK}")
1461+
message(STATUS "No .pc for ${PC_PACKAGE}. Using ${FALLBACK} for ${ARROW_PC_TARGET}")
1462+
string(APPEND ${ARROW_PC_TARGET}_PC_LIBS_PRIVATE " ${FALLBACK}")
14591463
endif()
14601464
endmacro()
14611465

@@ -3915,8 +3919,14 @@ if(ARROW_WITH_GOOGLE_CLOUD_CPP)
39153919

39163920
# curl is required on all platforms. We always use system curl to
39173921
# avoid conflict.
3918-
find_curl(ARROW)
3919-
resolve_dependency(google_cloud_cpp_storage PC_PACKAGE_NAMES google_cloud_cpp_storage)
3922+
find_curl(ARROW_GCS)
3923+
resolve_dependency(google_cloud_cpp_storage
3924+
ARROW_CMAKE_PACKAGE_NAME
3925+
ArrowGcs
3926+
ARROW_PC_PACKAGE_NAME
3927+
arrow-gcs
3928+
PC_PACKAGE_NAMES
3929+
google_cloud_cpp_storage)
39203930
get_target_property(google_cloud_cpp_storage_INCLUDE_DIR google-cloud-cpp::storage
39213931
INTERFACE_INCLUDE_DIRECTORIES)
39223932
message(STATUS "Found google-cloud-cpp::storage headers: ${google_cloud_cpp_storage_INCLUDE_DIR}"
@@ -4480,16 +4490,18 @@ if(ARROW_WITH_AZURE_SDK)
44804490
endif()
44814491

44824492
if(ARROW_BUILD_STATIC)
4483-
if((ARROW_GCS AND google_cloud_cpp_storage_SOURCE STREQUAL "BUNDLED")
4484-
OR (ARROW_AZURE
4485-
AND AZURE_SDK_VENDORED
4486-
AND NOT WIN32))
4487-
arrow_append_pc_system_library("libcurl" ARROW_CURL_PC "-lcurl")
4493+
if(ARROW_GCS AND google_cloud_cpp_storage_SOURCE STREQUAL "BUNDLED")
4494+
arrow_append_pc_system_library("libcurl" ARROW_GCS_CURL_PC "-lcurl" ARROW_GCS)
4495+
endif()
4496+
if(ARROW_AZURE
4497+
AND AZURE_SDK_VENDORED
4498+
AND NOT WIN32)
4499+
arrow_append_pc_system_library("libcurl" ARROW_CURL_PC "-lcurl" ARROW)
44884500
endif()
44894501
if(ARROW_AZURE
44904502
AND AZURE_SDK_VENDORED
44914503
AND NOT WIN32)
4492-
arrow_append_pc_system_library("libxml-2.0" ARROW_LIBXML2_PC "-lxml2")
4504+
arrow_append_pc_system_library("libxml-2.0" ARROW_LIBXML2_PC "-lxml2" ARROW)
44934505
endif()
44944506
if(ARROW_GCS
44954507
AND google_cloud_cpp_storage_SOURCE STREQUAL "BUNDLED"
@@ -4515,14 +4527,14 @@ if(ARROW_BUILD_STATIC)
45154527
NO_CMAKE_ENVIRONMENT_PATH
45164528
QUIET)
45174529
if(ARROW_GCS_${ARROW_GCS_ABSL_PC_PACKAGE}_PC_FOUND)
4518-
string(APPEND ARROW_PC_REQUIRES_PRIVATE " ${ARROW_GCS_ABSL_PC_PACKAGE}")
4530+
string(APPEND ARROW_GCS_PC_REQUIRES_PRIVATE " ${ARROW_GCS_ABSL_PC_PACKAGE}")
45194531
else()
45204532
message(STATUS "No .pc for ${ARROW_GCS_ABSL_PC_PACKAGE}; "
45214533
"static pkg-config metadata may be incomplete. Consider CMake")
45224534
endif()
45234535
endforeach()
45244536
else()
4525-
message(STATUS "PkgConfig not available. Skipping Abseil dependencies from arrow.pc"
4537+
message(STATUS "PkgConfig not available. Skipping Abseil dependencies from arrow-gcs.pc"
45264538
)
45274539
endif()
45284540
endif()
Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
#
18+
# This config sets the following variables in your project::
19+
#
20+
# ArrowGcs_FOUND - true if Arrow GCS found on the system
21+
#
22+
# This config sets the following targets in your project::
23+
#
24+
# ArrowGcs::arrow_gcs_shared - for linked as shared library if shared library is built
25+
# ArrowGcs::arrow_gcs_static - for linked as static library if static library is built
26+
27+
@PACKAGE_INIT@
28+
29+
set(ARROW_GCS_SYSTEM_DEPENDENCIES "@ARROW_GCS_SYSTEM_DEPENDENCIES@")
30+
31+
include(CMakeFindDependencyMacro)
32+
find_dependency(Arrow CONFIG)
33+
34+
if(ARROW_BUILD_STATIC)
35+
arrow_find_dependencies("${ARROW_GCS_SYSTEM_DEPENDENCIES}")
36+
endif()
37+
38+
include("${CMAKE_CURRENT_LIST_DIR}/ArrowGcsTargets.cmake")
39+
40+
arrow_keep_backward_compatibility(ArrowGcs arrow_gcs)
41+
42+
check_required_components(ArrowGcs)
43+
44+
arrow_show_details(ArrowGcs ARROW_GCS)

‎cpp/src/arrow/CMakeLists.txt‎

Lines changed: 68 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -33,29 +33,6 @@ set(ARROW_SHARED_PRIVATE_LINK_LIBS)
3333
set(ARROW_SHARED_INSTALL_INTERFACE_LIBS)
3434
set(ARROW_STATIC_INSTALL_INTERFACE_LIBS)
3535

36-
if(ARROW_GCS)
37-
if(google_cloud_cpp_storage_SOURCE STREQUAL "SYSTEM")
38-
list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS google-cloud-cpp::storage)
39-
elseif(google_cloud_cpp_storage_SOURCE STREQUAL "BUNDLED")
40-
list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS CURL::libcurl)
41-
if(absl_SOURCE STREQUAL "SYSTEM")
42-
# Bundled google-cloud-cpp with system Abseil needs these for static consumers.
43-
list(APPEND
44-
ARROW_STATIC_INSTALL_INTERFACE_LIBS
45-
absl::base
46-
absl::cord
47-
absl::crc32c
48-
absl::memory
49-
absl::optional
50-
absl::span
51-
absl::str_format
52-
absl::strings
53-
absl::time
54-
absl::variant)
55-
endif()
56-
endif()
57-
endif()
58-
5936
if(ARROW_AZURE)
6037
if(Azure_SOURCE STREQUAL "SYSTEM")
6138
list(APPEND ARROW_STATIC_INSTALL_INTERFACE_LIBS ${AZURE_SDK_LINK_LIBRARIES})
@@ -992,8 +969,31 @@ if(ARROW_FILESYSTEM)
992969
PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON)
993970
endif()
994971
if(ARROW_GCS)
995-
list(APPEND ARROW_FILESYSTEM_SRCS filesystem/gcsfs.cc filesystem/gcsfs_internal.cc)
996-
set_source_files_properties(filesystem/gcsfs.cc filesystem/gcsfs_internal.cc
972+
set(ARROW_GCS_STATIC_INSTALL_INTERFACE_LIBS Arrow::arrow_static)
973+
if(google_cloud_cpp_storage_SOURCE STREQUAL "SYSTEM")
974+
list(APPEND ARROW_GCS_STATIC_INSTALL_INTERFACE_LIBS google-cloud-cpp::storage)
975+
elseif(google_cloud_cpp_storage_SOURCE STREQUAL "BUNDLED")
976+
list(APPEND ARROW_GCS_STATIC_INSTALL_INTERFACE_LIBS CURL::libcurl)
977+
if(absl_SOURCE STREQUAL "SYSTEM")
978+
# Bundled google-cloud-cpp with system Abseil needs these for static consumers.
979+
list(APPEND
980+
ARROW_GCS_STATIC_INSTALL_INTERFACE_LIBS
981+
absl::base
982+
absl::cord
983+
absl::crc32c
984+
absl::memory
985+
absl::optional
986+
absl::span
987+
absl::str_format
988+
absl::strings
989+
absl::time
990+
absl::variant)
991+
endif()
992+
endif()
993+
set(ARROW_GCS_LIB_SRCS filesystem/gcsfs_module.cc filesystem/gcsfs.cc
994+
filesystem/gcsfs_internal.cc)
995+
set_source_files_properties(filesystem/gcsfs_module.cc filesystem/gcsfs.cc
996+
filesystem/gcsfs_internal.cc
997997
PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON)
998998
# Suppress documentation warnings from google-cloud-cpp headers
999999
if(CMAKE_CXX_COMPILER_ID MATCHES "Clang|AppleClang")
@@ -1009,6 +1009,49 @@ if(ARROW_FILESYSTEM)
10091009
APPEND
10101010
PROPERTY COMPILE_OPTIONS "-Wno-error=deprecated-declarations")
10111011
endif()
1012+
# If libarrow_gcs.a is only built, "pkg-config --cflags --libs
1013+
# arrow-gcs" outputs build flags for static linking not shared
1014+
# linking. ARROW_GCS_PC_* except ARROW_GCS_PC_*_PRIVATE are for the
1015+
# static linking case.
1016+
if(NOT ARROW_BUILD_SHARED AND ARROW_BUILD_STATIC)
1017+
string(APPEND ARROW_GCS_PC_CFLAGS "${ARROW_GCS_PC_CFLAGS_PRIVATE}")
1018+
set(ARROW_GCS_PC_CFLAGS_PRIVATE "")
1019+
set(ARROW_GCS_PC_LIBS "${ARROW_GCS_PC_LIBS_PRIVATE}")
1020+
set(ARROW_GCS_PC_LIBS_PRIVATE "")
1021+
set(ARROW_GCS_PC_REQUIRES "${ARROW_GCS_PC_REQUIRES_PRIVATE}")
1022+
set(ARROW_GCS_PC_REQUIRES_PRIVATE "")
1023+
else()
1024+
set(ARROW_GCS_PC_LIBS "")
1025+
set(ARROW_GCS_PC_REQUIRES "")
1026+
endif()
1027+
add_arrow_lib(arrow_gcs
1028+
CMAKE_PACKAGE_NAME
1029+
ArrowGcs
1030+
PKG_CONFIG_NAME
1031+
arrow-gcs
1032+
SOURCES
1033+
${ARROW_GCS_LIB_SRCS}
1034+
SHARED_LINK_LIBS
1035+
arrow_shared
1036+
SHARED_PRIVATE_LINK_LIBS
1037+
google-cloud-cpp::storage
1038+
SHARED_INSTALL_INTERFACE_LIBS
1039+
Arrow::arrow_shared
1040+
STATIC_LINK_LIBS
1041+
arrow_static
1042+
google-cloud-cpp::storage
1043+
STATIC_INSTALL_INTERFACE_LIBS
1044+
${ARROW_GCS_STATIC_INSTALL_INTERFACE_LIBS}
1045+
SHARED_LINK_FLAGS
1046+
${ARROW_VERSION_SCRIPT_FLAGS}
1047+
OUTPUTS
1048+
ARROW_GCS_LIBRARIES)
1049+
foreach(LIB_TARGET ${ARROW_GCS_LIBRARIES})
1050+
target_compile_definitions(${LIB_TARGET} PRIVATE ARROW_GCS_EXPORTING)
1051+
endforeach()
1052+
if(ARROW_BUILD_STATIC AND WIN32)
1053+
target_compile_definitions(arrow_gcs_static PUBLIC ARROW_GCS_STATIC)
1054+
endif()
10121055
endif()
10131056
if(ARROW_HDFS)
10141057
list(APPEND ARROW_FILESYSTEM_SRCS filesystem/hdfs.cc filesystem/hdfs_internal.cc)
@@ -1021,11 +1064,6 @@ if(ARROW_FILESYSTEM)
10211064
PRIVATE ${AZURE_SDK_LINK_LIBRARIES})
10221065
endforeach()
10231066
endif()
1024-
if(ARROW_GCS)
1025-
foreach(ARROW_FILESYSTEM_TARGET ${ARROW_FILESYSTEM_TARGETS})
1026-
target_link_libraries(${ARROW_FILESYSTEM_TARGET} PRIVATE google-cloud-cpp::storage)
1027-
endforeach()
1028-
endif()
10291067
if(ARROW_HDFS)
10301068
foreach(ARROW_FILESYSTEM_TARGET ${ARROW_FILESYSTEM_TARGETS})
10311069
target_link_libraries(${ARROW_FILESYSTEM_TARGET} PRIVATE arrow::hadoop)

‎cpp/src/arrow/arrow-gcs.pc.in‎

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
prefix=@CMAKE_INSTALL_PREFIX@
19+
includedir=@ARROW_PKG_CONFIG_INCLUDEDIR@
20+
libdir=@ARROW_PKG_CONFIG_LIBDIR@
21+
22+
Name: Apache Arrow GCS
23+
Description: Apache Arrow's GCS filesystem implementation.
24+
Version: @ARROW_VERSION@
25+
Requires: arrow@ARROW_GCS_PC_REQUIRES@
26+
Requires.private:@ARROW_GCS_PC_REQUIRES_PRIVATE@
27+
Libs: -L${libdir} -larrow_gcs@ARROW_GCS_PC_LIBS@
28+
Libs.private:@ARROW_GCS_PC_LIBS_PRIVATE@
29+
Cflags:@ARROW_GCS_PC_CFLAGS@
30+
Cflags.private:@ARROW_GCS_PC_CFLAGS_PRIVATE@

‎cpp/src/arrow/filesystem/CMakeLists.txt‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,17 +43,35 @@ if(ARROW_BUILD_BENCHMARKS)
4343
endif()
4444

4545
if(ARROW_GCS)
46+
set(ARROW_GCS_TEST_EXTRA_LINK_LIBS)
47+
if(ARROW_TEST_LINKAGE STREQUAL "shared")
48+
list(APPEND ARROW_GCS_TEST_EXTRA_LINK_LIBS arrow_gcs_shared)
49+
else()
50+
list(APPEND ARROW_GCS_TEST_EXTRA_LINK_LIBS arrow_gcs_static)
51+
endif()
52+
list(APPEND ARROW_GCS_TEST_EXTRA_LINK_LIBS google-cloud-cpp::storage)
4653
add_arrow_test(gcsfs_test
4754
EXTRA_LABELS
4855
filesystem
4956
EXTRA_LINK_LIBS
50-
google-cloud-cpp::storage)
57+
${ARROW_GCS_TEST_EXTRA_LINK_LIBS})
5158
if(CMAKE_CXX_COMPILER_ID MATCHES "Clang|AppleClang")
5259
if(TARGET arrow-gcsfs-test)
5360
target_compile_options(arrow-gcsfs-test PRIVATE -Wno-documentation
5461
-Wno-documentation-deprecated-sync)
5562
endif()
5663
endif()
64+
if(ARROW_BUILD_TESTS AND ARROW_BUILD_SHARED)
65+
add_arrow_test(gcsfs_module_test
66+
SOURCES
67+
gcsfs_module_test.cc
68+
EXTRA_LABELS
69+
filesystem
70+
DEFINITIONS
71+
ARROW_GCS_LIBPATH="$<TARGET_FILE:arrow_gcs_shared>"
72+
EXTRA_DEPENDENCIES
73+
arrow_gcs_shared)
74+
endif()
5775
endif()
5876

5977
if(ARROW_AZURE)

‎cpp/src/arrow/filesystem/filesystem.cc‎

Lines changed: 0 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,6 @@
2828
#ifdef ARROW_AZURE
2929
# include "arrow/filesystem/azurefs.h"
3030
#endif
31-
#ifdef ARROW_GCS
32-
# include "arrow/filesystem/gcsfs.h"
33-
#endif
3431
#ifdef ARROW_HDFS
3532
# include "arrow/filesystem/hdfs.h"
3633
#endif
@@ -920,16 +917,6 @@ Result<std::shared_ptr<FileSystem>> FileSystemFromUriReal(
920917
return Status::NotImplemented(
921918
"Got Azure Blob File System URI but Arrow compiled without Azure Blob File "
922919
"System support");
923-
#endif
924-
}
925-
if (scheme == "gs" || scheme == "gcs") {
926-
#ifdef ARROW_GCS
927-
ARROW_ASSIGN_OR_RAISE(auto options, GcsOptions::FromUri(uri, out_path));
928-
return GcsFileSystem::Make(options, io_context);
929-
#else
930-
return Status::NotImplemented(
931-
"Got GCS URI but Arrow compiled "
932-
"without GCS support");
933920
#endif
934921
}
935922
if (scheme == "hdfs" || scheme == "viewfs") {

0 commit comments

Comments
 (0)