Skip to content

Commit 2443fb3

Browse files
committed
apacheGH-50194: [C++] Move S3 and AWS-SDK to its own libarrow_s3.so
1 parent d6bba6b commit 2443fb3

9 files changed

Lines changed: 109 additions & 41 deletions

File tree

‎ci/docker/conda-cpp.dockerfile‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -73,7 +73,6 @@ ENV ARROW_ACERO=ON \
7373
ARROW_ORC=ON \
7474
ARROW_PARQUET=ON \
7575
ARROW_S3=ON \
76-
ARROW_S3_MODULE=ON \
7776
ARROW_SUBSTRAIT=ON \
7877
ARROW_USE_CCACHE=ON \
7978
ARROW_USE_MOLD=ON \

‎ci/scripts/cpp_build.sh‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,6 @@ if [ "${ARROW_ENABLE_THREADING:-ON}" = "OFF" ]; then
7070
ARROW_JEMALLOC=OFF
7171
ARROW_MIMALLOC=OFF
7272
ARROW_S3=OFF
73-
ARROW_S3_MODULE=OFF
7473
ARROW_WITH_OPENTELEMETRY=OFF
7574
fi
7675

@@ -230,7 +229,6 @@ else
230229
-DARROW_PARQUET=${ARROW_PARQUET:-OFF} \
231230
-DARROW_RUNTIME_SIMD_LEVEL=${ARROW_RUNTIME_SIMD_LEVEL:-MAX} \
232231
-DARROW_S3=${ARROW_S3:-OFF} \
233-
-DARROW_S3_MODULE=${ARROW_S3_MODULE:-OFF} \
234232
-DARROW_SIMD_LEVEL=${ARROW_SIMD_LEVEL:-DEFAULT} \
235233
-DARROW_SUBSTRAIT=${ARROW_SUBSTRAIT:-OFF} \
236234
-DARROW_TEST_LINKAGE=${ARROW_TEST_LINKAGE:-shared} \

‎cpp/cmake_modules/DefineOptions.cmake‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -390,17 +390,11 @@ takes precedence over ccache if a storage backend is configured" ON)
390390
ARROW_JSON)
391391

392392
define_option(ARROW_S3
393-
"Build Arrow with S3 support (requires the AWS SDK for C++)"
393+
"Build Arrow S3 Module (requires the AWS SDK for C++)"
394394
OFF
395395
DEPENDS
396396
ARROW_FILESYSTEM)
397397

398-
define_option(ARROW_S3_MODULE
399-
"Build the Arrow S3 filesystem as a dynamic module"
400-
OFF
401-
DEPENDS
402-
ARROW_S3)
403-
404398
define_option(ARROW_SUBSTRAIT
405399
"Build the Arrow Substrait Consumer Module"
406400
OFF
Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
#
18+
# This config sets the following variables in your project::
19+
#
20+
# ArrowS3_FOUND - true if Arrow S3 found on the system
21+
#
22+
# This config sets the following targets in your project::
23+
#
24+
# ArrowS3::arrow_s3_shared - for linked as shared library if shared library is built
25+
# ArrowS3::arrow_s3_static - for linked as static library if static library is built
26+
27+
@PACKAGE_INIT@
28+
29+
include(CMakeFindDependencyMacro)
30+
find_dependency(Arrow CONFIG)
31+
32+
include("${CMAKE_CURRENT_LIST_DIR}/ArrowS3Targets.cmake")
33+
34+
arrow_keep_backward_compatibility(ArrowS3 arrow_s3)
35+
36+
check_required_components(ArrowS3)
37+
38+
arrow_show_details(ArrowS3 ARROW_S3)

‎cpp/src/arrow/CMakeLists.txt‎

Lines changed: 22 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -960,11 +960,6 @@ if(ARROW_FILESYSTEM)
960960
if(ARROW_HDFS)
961961
list(APPEND ARROW_FILESYSTEM_SRCS filesystem/hdfs.cc)
962962
endif()
963-
if(ARROW_S3)
964-
list(APPEND ARROW_FILESYSTEM_SRCS filesystem/s3fs.cc)
965-
set_source_files_properties(filesystem/s3fs.cc PROPERTIES SKIP_UNITY_BUILD_INCLUSION
966-
ON)
967-
endif()
968963

969964
arrow_add_object_library(ARROW_FILESYSTEM ${ARROW_FILESYSTEM_SRCS})
970965
if(ARROW_AZURE)
@@ -984,21 +979,29 @@ if(ARROW_FILESYSTEM)
984979
endforeach()
985980
endif()
986981
if(ARROW_S3)
987-
foreach(ARROW_FILESYSTEM_TARGET ${ARROW_FILESYSTEM_TARGETS})
988-
target_link_libraries(${ARROW_FILESYSTEM_TARGET} PRIVATE ${AWSSDK_LINK_LIBRARIES})
989-
endforeach()
990-
991-
if(ARROW_S3_MODULE)
992-
if(NOT ARROW_BUILD_SHARED)
993-
message(FATAL_ERROR "ARROW_S3_MODULE without shared libarrow (-DARROW_BUILD_SHARED=ON) is not supported"
994-
)
995-
endif()
996-
997-
add_library(arrow_s3fs MODULE filesystem/s3fs_module.cc filesystem/s3fs.cc)
998-
target_link_libraries(arrow_s3fs PRIVATE ${AWSSDK_LINK_LIBRARIES} arrow_shared)
999-
set_source_files_properties(filesystem/s3fs.cc filesystem/s3fs_module.cc
1000-
PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON)
982+
if(NOT ARROW_BUILD_SHARED AND ARROW_BUILD_STATIC)
983+
string(APPEND ARROW_S3_PC_CFLAGS "${ARROW_S3_PC_CFLAGS_PRIVATE}")
984+
set(ARROW_S3_PC_CFLAGS_PRIVATE "")
1001985
endif()
986+
list(APPEND ARROW_S3_LIB_SRCS filesystem/s3fs_module.cc filesystem/s3fs.cc)
987+
add_arrow_lib(arrow_s3
988+
CMAKE_PACKAGE_NAME
989+
ArrowS3
990+
PKG_CONFIG_NAME
991+
arrow-s3
992+
SOURCES
993+
${ARROW_S3_LIB_SRCS}
994+
SHARED_LINK_LIBS
995+
arrow_shared
996+
SHARED_PRIVATE_LINK_LIBS
997+
${AWSSDK_LINK_LIBRARIES}
998+
SHARED_INSTALL_INTERFACE_LIBS
999+
${ARROW_S3_SHARED_INSTALL_INTERFACE_LIBS}
1000+
STATIC_LINK_LIBS
1001+
arrow_static
1002+
${AWSSDK_LINK_LIBRARIES})
1003+
set_source_files_properties(filesystem/s3fs.cc filesystem/s3fs_module.cc
1004+
PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON)
10021005
endif()
10031006

10041007
list(APPEND ARROW_TESTING_SHARED_LINK_LIBS ${ARROW_GTEST_GMOCK})

‎cpp/src/arrow/arrow-s3.pc.in‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
prefix=@CMAKE_INSTALL_PREFIX@
19+
includedir=@ARROW_PKG_CONFIG_INCLUDEDIR@
20+
libdir=@ARROW_PKG_CONFIG_LIBDIR@
21+
22+
Name: Apache Arrow S3
23+
Description: Apache Arrow's S3 filesystem implementation.
24+
Version: @ARROW_VERSION@
25+
Requires: arrow
26+
Libs: -L${libdir} -larrow_s3
27+
Cflags:@ARROW_S3_PC_CFLAGS@
28+
Cflags.private:@ARROW_S3_PC_CFLAGS_PRIVATE@

‎cpp/src/arrow/filesystem/CMakeLists.txt‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -73,9 +73,9 @@ if(ARROW_S3)
7373
# static variables storage of AWS SDK for C++ in libaws*.a may be
7474
# mixed with one in libarrow.
7575
if(ARROW_TEST_LINKAGE STREQUAL "shared")
76-
list(APPEND ARROW_S3_TEST_EXTRA_LINK_LIBS arrow_shared)
76+
list(APPEND ARROW_S3_TEST_EXTRA_LINK_LIBS arrow_s3_shared)
7777
else()
78-
list(APPEND ARROW_S3_TEST_EXTRA_LINK_LIBS arrow_static)
78+
list(APPEND ARROW_S3_TEST_EXTRA_LINK_LIBS arrow_s3_static)
7979
endif()
8080
list(APPEND ARROW_S3_TEST_EXTRA_LINK_LIBS ${AWSSDK_LINK_LIBRARIES})
8181
add_arrow_test(s3fs_test
@@ -107,7 +107,7 @@ if(ARROW_S3)
107107
if(ARROW_BUILD_TESTS)
108108
add_executable(arrow-s3fs-narrative-test s3fs_narrative_test.cc)
109109
target_link_libraries(arrow-s3fs-narrative-test ${ARROW_TEST_LINK_LIBS}
110-
${GFLAGS_LIBRARIES})
110+
${ARROW_S3_TEST_EXTRA_LINK_LIBS} ${GFLAGS_LIBRARIES})
111111
add_dependencies(arrow-tests arrow-s3fs-narrative-test)
112112
endif()
113113

@@ -119,6 +119,7 @@ if(ARROW_S3)
119119
s3fs_benchmark.cc
120120
s3_test_util.cc
121121
STATIC_LINK_LIBS
122+
${ARROW_S3_TEST_EXTRA_LINK_LIBS}
122123
${AWSSDK_LINK_LIBRARIES}
123124
${ARROW_BENCHMARK_LINK_LIBS})
124125
if(ARROW_TEST_LINKAGE STREQUAL "static")
@@ -128,19 +129,19 @@ if(ARROW_S3)
128129
endif()
129130
endif()
130131

131-
if(ARROW_S3_MODULE AND ARROW_BUILD_TESTS)
132+
if(ARROW_BUILD_TESTS)
132133
add_arrow_test(s3fs_module_test
133134
SOURCES
134135
s3fs_module_test.cc
135136
s3_test_util.cc
136137
EXTRA_LABELS
137138
filesystem
138139
DEFINITIONS
139-
ARROW_S3_LIBPATH="$<TARGET_FILE:arrow_s3fs>"
140+
ARROW_S3_LIBPATH="$<TARGET_FILE:arrow_s3_shared>"
140141
EXTRA_LINK_LIBS
141142
Boost::filesystem)
142143
target_compile_definitions(arrow-filesystem-test
143-
PUBLIC ARROW_S3_LIBPATH="$<TARGET_FILE:arrow_s3fs>")
144+
PUBLIC ARROW_S3_LIBPATH="$<TARGET_FILE:arrow_s3_shared>")
144145
target_sources(arrow-filesystem-test PUBLIC s3fs_module_test.cc s3_test_util.cc)
145146
target_link_libraries(arrow-filesystem-test PUBLIC Boost::filesystem)
146147
endif()

‎cpp/src/arrow/filesystem/s3fs_module_test.cc‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -57,12 +57,7 @@ MinioTestEnvironment* GetMinioEnv() {
5757

5858
class RegistrationTestEnvironment : public ::testing::Environment {
5959
public:
60-
void SetUp() override {
61-
// Unregister the s3 filesystem factory so that we can be sure the module loading and
62-
// the factories from the module are actually working
63-
ASSERT_OK(internal::UnregisterFileSystemFactory("s3"));
64-
ASSERT_OK(LoadFileSystemFactories(ARROW_S3_LIBPATH));
65-
}
60+
void SetUp() override { ASSERT_OK(LoadFileSystemFactories(ARROW_S3_LIBPATH)); }
6661
void TearDown() override { EnsureFinalized(); }
6762
};
6863

‎python/CMakeLists.txt‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -742,6 +742,14 @@ if(PYARROW_BUILD_S3)
742742
if(NOT ARROW_S3)
743743
message(FATAL_ERROR "You must build Arrow C++ with ARROW_S3=ON")
744744
endif()
745+
find_package(ArrowS3 REQUIRED)
746+
if(PYARROW_BUNDLE_ARROW_CPP)
747+
bundle_arrow_lib(${ARROW_S3_SHARED_LIB} SO_VERSION ${ARROW_SO_VERSION})
748+
if(MSVC)
749+
bundle_arrow_import_lib(${ARROW_S3_IMPORT_LIB})
750+
endif()
751+
endif()
752+
set(S3_LINK_LIBS ArrowS3::arrow_s3_shared)
745753
list(APPEND CYTHON_EXTENSIONS _s3fs)
746754
endif()
747755

@@ -1038,6 +1046,10 @@ if(PYARROW_BUILD_PARQUET)
10381046
endif()
10391047
endif()
10401048

1049+
if(PYARROW_BUILD_S3)
1050+
target_link_libraries(_s3fs PRIVATE ${S3_LINK_LIBS})
1051+
endif()
1052+
10411053
#
10421054
# Type stubs with docstring injection
10431055
#

0 commit comments

Comments
 (0)