Skip to content

Commit cbd8567

Browse files
committed
apacheGH-50194: [C++] Move S3 and AWS-SDK to its own libarrow_s3.so and add
required ARROW_EXPORTs
1 parent 6287272 commit cbd8567

13 files changed

Lines changed: 182 additions & 52 deletions

File tree

‎ci/docker/conda-cpp.dockerfile‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,6 @@ ENV ARROW_ACERO=ON \
7171
ARROW_ORC=ON \
7272
ARROW_PARQUET=ON \
7373
ARROW_S3=ON \
74-
ARROW_S3_MODULE=ON \
7574
ARROW_SUBSTRAIT=ON \
7675
ARROW_USE_CCACHE=ON \
7776
ARROW_USE_MOLD=ON \

‎ci/scripts/cpp_build.sh‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,6 @@ if [ "${ARROW_ENABLE_THREADING:-ON}" = "OFF" ]; then
7676
ARROW_JEMALLOC=OFF
7777
ARROW_MIMALLOC=OFF
7878
ARROW_S3=OFF
79-
ARROW_S3_MODULE=OFF
8079
ARROW_WITH_OPENTELEMETRY=OFF
8180
fi
8281

@@ -237,7 +236,6 @@ else
237236
-DARROW_PARQUET="${ARROW_PARQUET:-OFF}" \
238237
-DARROW_RUNTIME_SIMD_LEVEL="${ARROW_RUNTIME_SIMD_LEVEL:-MAX}" \
239238
-DARROW_S3="${ARROW_S3:-OFF}" \
240-
-DARROW_S3_MODULE="${ARROW_S3_MODULE:-OFF}" \
241239
-DARROW_SIMD_LEVEL="${ARROW_SIMD_LEVEL:-DEFAULT}" \
242240
-DARROW_SUBSTRAIT="${ARROW_SUBSTRAIT:-OFF}" \
243241
-DARROW_TEST_LINKAGE="${ARROW_TEST_LINKAGE:-shared}" \

‎cpp/cmake_modules/DefineOptions.cmake‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -390,17 +390,11 @@ takes precedence over ccache if a storage backend is configured" ON)
390390
ARROW_JSON)
391391

392392
define_option(ARROW_S3
393-
"Build Arrow with S3 support (requires the AWS SDK for C++)"
393+
"Build Arrow S3 Module (requires the AWS SDK for C++)"
394394
OFF
395395
DEPENDS
396396
ARROW_FILESYSTEM)
397397

398-
define_option(ARROW_S3_MODULE
399-
"Build the Arrow S3 filesystem as a dynamic module"
400-
OFF
401-
DEPENDS
402-
ARROW_S3)
403-
404398
define_option(ARROW_SUBSTRAIT
405399
"Build the Arrow Substrait Consumer Module"
406400
OFF
Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
#
18+
# This config sets the following variables in your project::
19+
#
20+
# ArrowS3_FOUND - true if Arrow S3 found on the system
21+
#
22+
# This config sets the following targets in your project::
23+
#
24+
# ArrowS3::arrow_s3_shared - for linked as shared library if shared library is built
25+
# ArrowS3::arrow_s3_static - for linked as static library if static library is built
26+
27+
@PACKAGE_INIT@
28+
29+
include(CMakeFindDependencyMacro)
30+
find_dependency(Arrow CONFIG)
31+
32+
include("${CMAKE_CURRENT_LIST_DIR}/ArrowS3Targets.cmake")
33+
34+
arrow_keep_backward_compatibility(ArrowS3 arrow_s3)
35+
36+
check_required_components(ArrowS3)
37+
38+
arrow_show_details(ArrowS3 ARROW_S3)

‎cpp/src/arrow/CMakeLists.txt‎

Lines changed: 30 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -960,11 +960,6 @@ if(ARROW_FILESYSTEM)
960960
if(ARROW_HDFS)
961961
list(APPEND ARROW_FILESYSTEM_SRCS filesystem/hdfs.cc)
962962
endif()
963-
if(ARROW_S3)
964-
list(APPEND ARROW_FILESYSTEM_SRCS filesystem/s3fs.cc)
965-
set_source_files_properties(filesystem/s3fs.cc PROPERTIES SKIP_UNITY_BUILD_INCLUSION
966-
ON)
967-
endif()
968963

969964
arrow_add_object_library(ARROW_FILESYSTEM ${ARROW_FILESYSTEM_SRCS})
970965
if(ARROW_AZURE)
@@ -984,21 +979,38 @@ if(ARROW_FILESYSTEM)
984979
endforeach()
985980
endif()
986981
if(ARROW_S3)
987-
foreach(ARROW_FILESYSTEM_TARGET ${ARROW_FILESYSTEM_TARGETS})
988-
target_link_libraries(${ARROW_FILESYSTEM_TARGET} PRIVATE ${AWSSDK_LINK_LIBRARIES})
982+
if(NOT ARROW_BUILD_SHARED AND ARROW_BUILD_STATIC)
983+
string(APPEND ARROW_S3_PC_CFLAGS "${ARROW_S3_PC_CFLAGS_PRIVATE}")
984+
set(ARROW_S3_PC_CFLAGS_PRIVATE "")
985+
endif()
986+
list(APPEND ARROW_S3_LIB_SRCS filesystem/s3fs_module.cc filesystem/s3fs.cc)
987+
add_arrow_lib(arrow_s3
988+
CMAKE_PACKAGE_NAME
989+
ArrowS3
990+
PKG_CONFIG_NAME
991+
arrow-s3
992+
SOURCES
993+
${ARROW_S3_LIB_SRCS}
994+
SHARED_LINK_LIBS
995+
arrow_shared
996+
SHARED_PRIVATE_LINK_LIBS
997+
${AWSSDK_LINK_LIBRARIES}
998+
SHARED_INSTALL_INTERFACE_LIBS
999+
${ARROW_S3_SHARED_INSTALL_INTERFACE_LIBS}
1000+
STATIC_LINK_LIBS
1001+
arrow_static
1002+
${AWSSDK_LINK_LIBRARIES}
1003+
OUTPUTS
1004+
ARROW_S3_LIBRARIES)
1005+
foreach(LIB_TARGET ${ARROW_S3_LIBRARIES})
1006+
target_compile_definitions(${LIB_TARGET} PRIVATE ARROW_S3_EXPORTING)
9891007
endforeach()
990-
991-
if(ARROW_S3_MODULE)
992-
if(NOT ARROW_BUILD_SHARED)
993-
message(FATAL_ERROR "ARROW_S3_MODULE without shared libarrow (-DARROW_BUILD_SHARED=ON) is not supported"
994-
)
995-
endif()
996-
997-
add_library(arrow_s3fs MODULE filesystem/s3fs_module.cc filesystem/s3fs.cc)
998-
target_link_libraries(arrow_s3fs PRIVATE ${AWSSDK_LINK_LIBRARIES} arrow_shared)
999-
set_source_files_properties(filesystem/s3fs.cc filesystem/s3fs_module.cc
1000-
PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON)
1008+
if(ARROW_BUILD_STATIC AND WIN32)
1009+
target_compile_definitions(arrow_s3_static PUBLIC ARROW_S3_STATIC)
10011010
endif()
1011+
1012+
set_source_files_properties(filesystem/s3fs.cc filesystem/s3fs_module.cc
1013+
PROPERTIES SKIP_UNITY_BUILD_INCLUSION ON)
10021014
endif()
10031015

10041016
list(APPEND ARROW_TESTING_SHARED_LINK_LIBS ${ARROW_GTEST_GMOCK})

‎cpp/src/arrow/arrow-s3.pc.in‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
18+
prefix=@CMAKE_INSTALL_PREFIX@
19+
includedir=@ARROW_PKG_CONFIG_INCLUDEDIR@
20+
libdir=@ARROW_PKG_CONFIG_LIBDIR@
21+
22+
Name: Apache Arrow S3
23+
Description: Apache Arrow's S3 filesystem implementation.
24+
Version: @ARROW_VERSION@
25+
Requires: arrow
26+
Libs: -L${libdir} -larrow_s3
27+
Cflags:@ARROW_S3_PC_CFLAGS@
28+
Cflags.private:@ARROW_S3_PC_CFLAGS_PRIVATE@

‎cpp/src/arrow/filesystem/CMakeLists.txt‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -73,9 +73,9 @@ if(ARROW_S3)
7373
# static variables storage of AWS SDK for C++ in libaws*.a may be
7474
# mixed with one in libarrow.
7575
if(ARROW_TEST_LINKAGE STREQUAL "shared")
76-
list(APPEND ARROW_S3_TEST_EXTRA_LINK_LIBS arrow_shared)
76+
list(APPEND ARROW_S3_TEST_EXTRA_LINK_LIBS arrow_s3_shared)
7777
else()
78-
list(APPEND ARROW_S3_TEST_EXTRA_LINK_LIBS arrow_static)
78+
list(APPEND ARROW_S3_TEST_EXTRA_LINK_LIBS arrow_s3_static)
7979
endif()
8080
list(APPEND ARROW_S3_TEST_EXTRA_LINK_LIBS ${AWSSDK_LINK_LIBRARIES})
8181
add_arrow_test(s3fs_test
@@ -107,7 +107,7 @@ if(ARROW_S3)
107107
if(ARROW_BUILD_TESTS)
108108
add_executable(arrow-s3fs-narrative-test s3fs_narrative_test.cc)
109109
target_link_libraries(arrow-s3fs-narrative-test ${ARROW_TEST_LINK_LIBS}
110-
${GFLAGS_LIBRARIES})
110+
${ARROW_S3_TEST_EXTRA_LINK_LIBS} ${GFLAGS_LIBRARIES})
111111
add_dependencies(arrow-tests arrow-s3fs-narrative-test)
112112
endif()
113113

@@ -119,6 +119,7 @@ if(ARROW_S3)
119119
s3fs_benchmark.cc
120120
s3_test_util.cc
121121
STATIC_LINK_LIBS
122+
${ARROW_S3_TEST_EXTRA_LINK_LIBS}
122123
${AWSSDK_LINK_LIBRARIES}
123124
${ARROW_BENCHMARK_LINK_LIBS})
124125
if(ARROW_TEST_LINKAGE STREQUAL "static")
@@ -128,19 +129,19 @@ if(ARROW_S3)
128129
endif()
129130
endif()
130131

131-
if(ARROW_S3_MODULE AND ARROW_BUILD_TESTS)
132+
if(ARROW_BUILD_TESTS)
132133
add_arrow_test(s3fs_module_test
133134
SOURCES
134135
s3fs_module_test.cc
135136
s3_test_util.cc
136137
EXTRA_LABELS
137138
filesystem
138139
DEFINITIONS
139-
ARROW_S3_LIBPATH="$<TARGET_FILE:arrow_s3fs>"
140+
ARROW_S3_LIBPATH="$<TARGET_FILE:arrow_s3_shared>"
140141
EXTRA_LINK_LIBS
141142
Boost::filesystem)
142143
target_compile_definitions(arrow-filesystem-test
143-
PUBLIC ARROW_S3_LIBPATH="$<TARGET_FILE:arrow_s3fs>")
144+
PUBLIC ARROW_S3_LIBPATH="$<TARGET_FILE:arrow_s3_shared>")
144145
target_sources(arrow-filesystem-test PUBLIC s3fs_module_test.cc s3_test_util.cc)
145146
target_link_libraries(arrow-filesystem-test PUBLIC Boost::filesystem)
146147
endif()

‎cpp/src/arrow/filesystem/filesystem.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -560,6 +560,7 @@ class ARROW_EXPORT SlowFileSystem : public FileSystem {
560560
/// will fail with an error.
561561
///
562562
/// The user is responsible for synchronization of calls to this function.
563+
ARROW_EXPORT
563564
void EnsureFinalized();
564565

565566
/// \defgroup filesystem-factories Functions for creating FileSystem instances
Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,49 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
#pragma once
19+
20+
#if defined(_WIN32) || defined(__CYGWIN__)
21+
# if defined(_MSC_VER)
22+
# pragma warning(push)
23+
# pragma warning(disable : 4251)
24+
# else
25+
# pragma GCC diagnostic ignored "-Wattributes"
26+
# endif
27+
28+
# ifdef ARROW_S3_STATIC
29+
# define ARROW_S3_EXPORT
30+
# elif defined(ARROW_S3_EXPORTING)
31+
# define ARROW_S3_EXPORT __declspec(dllexport)
32+
# else
33+
# define ARROW_S3_EXPORT __declspec(dllimport)
34+
# endif
35+
36+
# define ARROW_S3_NO_EXPORT
37+
38+
# if defined(_MSC_VER)
39+
# pragma warning(pop)
40+
# endif
41+
42+
#else // Not Windows
43+
# ifndef ARROW_S3_EXPORT
44+
# define ARROW_S3_EXPORT __attribute__((visibility("default")))
45+
# endif
46+
# ifndef ARROW_S3_NO_EXPORT
47+
# define ARROW_S3_NO_EXPORT __attribute__((visibility("hidden")))
48+
# endif
49+
#endif

‎cpp/src/arrow/filesystem/s3fs.h‎

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
#include <vector>
2424

2525
#include "arrow/filesystem/filesystem.h"
26+
#include "arrow/filesystem/s3_visibility.h"
2627
#include "arrow/util/macros.h"
2728
#include "arrow/util/uri.h"
2829

@@ -38,7 +39,7 @@ class STSClient;
3839
namespace arrow::fs {
3940

4041
/// Options for using a proxy for S3
41-
struct ARROW_EXPORT S3ProxyOptions {
42+
struct ARROW_S3_EXPORT S3ProxyOptions {
4243
std::string scheme;
4344
std::string host;
4445
int port = -1;
@@ -67,7 +68,7 @@ enum class S3CredentialsKind : int8_t {
6768
};
6869

6970
/// Pure virtual class for describing custom S3 retry strategies
70-
class ARROW_EXPORT S3RetryStrategy {
71+
class ARROW_S3_EXPORT S3RetryStrategy {
7172
public:
7273
virtual ~S3RetryStrategy() = default;
7374

@@ -96,7 +97,7 @@ class ARROW_EXPORT S3RetryStrategy {
9697
};
9798

9899
/// Options for the S3FileSystem implementation.
99-
struct ARROW_EXPORT S3Options {
100+
struct ARROW_S3_EXPORT S3Options {
100101
/// \brief Smart defaults for option values
101102
///
102103
/// The possible values for this setting are explained in the AWS docs:
@@ -315,7 +316,7 @@ struct ARROW_EXPORT S3Options {
315316
/// Some implementation notes:
316317
/// - buckets are special and the operations available on them may be limited
317318
/// or more expensive than desired.
318-
class ARROW_EXPORT S3FileSystem : public FileSystem {
319+
class ARROW_S3_EXPORT S3FileSystem : public FileSystem {
319320
public:
320321
~S3FileSystem() override;
321322

@@ -409,7 +410,7 @@ class ARROW_EXPORT S3FileSystem : public FileSystem {
409410

410411
enum class S3LogLevel : int8_t { Off, Fatal, Error, Warn, Info, Debug, Trace };
411412

412-
struct ARROW_EXPORT S3GlobalOptions {
413+
struct ARROW_S3_EXPORT S3GlobalOptions {
413414
/// The log level for S3-originating messages.
414415
S3LogLevel log_level;
415416

@@ -444,21 +445,21 @@ struct ARROW_EXPORT S3GlobalOptions {
444445
///
445446
/// Once this function is called you MUST call FinalizeS3 before the end of the
446447
/// application in order to avoid a segmentation fault at shutdown.
447-
ARROW_EXPORT
448+
ARROW_S3_EXPORT
448449
Status InitializeS3(const S3GlobalOptions& options);
449450

450451
/// \brief Ensure the S3 APIs are initialized, but only if not already done.
451452
///
452453
/// If necessary, this will call InitializeS3() with some default options.
453-
ARROW_EXPORT
454+
ARROW_S3_EXPORT
454455
Status EnsureS3Initialized();
455456

456457
/// Whether S3 was initialized, and not finalized.
457-
ARROW_EXPORT
458+
ARROW_S3_EXPORT
458459
bool IsS3Initialized();
459460

460461
/// Whether S3 was finalized.
461-
ARROW_EXPORT
462+
ARROW_S3_EXPORT
462463
bool IsS3Finalized();
463464

464465
/// \brief Shutdown the S3 APIs.
@@ -470,16 +471,16 @@ bool IsS3Finalized();
470471
/// Calls to InitializeS3() and FinalizeS3() should be serialized by the
471472
/// application (this also applies to EnsureS3Initialized() and
472473
/// EnsureS3Finalized()).
473-
ARROW_EXPORT
474+
ARROW_S3_EXPORT
474475
Status FinalizeS3();
475476

476477
/// \brief Ensure the S3 APIs are shutdown, but only if not already done.
477478
///
478479
/// If necessary, this will call FinalizeS3().
479-
ARROW_EXPORT
480+
ARROW_S3_EXPORT
480481
Status EnsureS3Finalized();
481482

482-
ARROW_EXPORT
483+
ARROW_S3_EXPORT
483484
Result<std::string> ResolveS3BucketRegion(const std::string& bucket);
484485

485486
} // namespace arrow::fs

0 commit comments

Comments
 (0)