Skip to content

Commit 669009e

Browse files
marsupialtailZiheng Wangpitrou
authored
ARROW-16521 [C++][Python] Configure curl timeout policy for S3 (#13385)
added timeout options to c++ and python s3fs Lead-authored-by: Ziheng Wang <[email protected]> Co-authored-by: Ziheng Wang <[email protected]> Co-authored-by: Ziheng Wang <[email protected]> Co-authored-by: Antoine Pitrou <[email protected]> Signed-off-by: Joris Van den Bossche <[email protected]>
1 parent ee874d6 commit 669009e

5 files changed

Lines changed: 53 additions & 8 deletions

File tree

‎cpp/src/arrow/filesystem/s3fs.cc‎

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -386,10 +386,11 @@ bool S3Options::Equals(const S3Options& other) const {
386386
default_metadata_size
387387
? (other.default_metadata && other.default_metadata->Equals(*default_metadata))
388388
: (!other.default_metadata || other.default_metadata->size() == 0);
389-
return (region == other.region && endpoint_override == other.endpoint_override &&
390-
scheme == other.scheme && role_arn == other.role_arn &&
391-
session_name == other.session_name && external_id == other.external_id &&
392-
load_frequency == other.load_frequency &&
389+
return (region == other.region && connect_timeout == other.connect_timeout &&
390+
request_timeout == other.request_timeout &&
391+
endpoint_override == other.endpoint_override && scheme == other.scheme &&
392+
role_arn == other.role_arn && session_name == other.session_name &&
393+
external_id == other.external_id && load_frequency == other.load_frequency &&
393394
proxy_options.Equals(other.proxy_options) &&
394395
credentials_kind == other.credentials_kind &&
395396
background_writes == other.background_writes &&
@@ -718,6 +719,14 @@ class ClientBuilder {
718719
if (!options_.region.empty()) {
719720
client_config_.region = ToAwsString(options_.region);
720721
}
722+
if (options_.request_timeout > 0) {
723+
// Use ceil() to avoid setting it to 0 as that probably means no timeout.
724+
client_config_.requestTimeoutMs = ceil(options_.request_timeout * 1000);
725+
}
726+
if (options_.connect_timeout > 0) {
727+
client_config_.connectTimeoutMs = ceil(options_.connect_timeout * 1000);
728+
}
729+
721730
client_config_.endpointOverride = ToAwsString(options_.endpoint_override);
722731
if (options_.scheme == "http") {
723732
client_config_.scheme = Aws::Http::Scheme::HTTP;

‎cpp/src/arrow/filesystem/s3fs.h‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,17 @@ struct ARROW_EXPORT S3Options {
103103
/// server).
104104
std::string region;
105105

106+
/// \brief Socket connection timeout, in seconds
107+
///
108+
/// If negative, the AWS SDK default value is used (typically 1 second).
109+
double connect_timeout = -1;
110+
111+
/// \brief Socket read timeout on Windows and macOS, in seconds
112+
///
113+
/// If negative, the AWS SDK default value is used (typically 3 seconds).
114+
/// This option is ignored on non-Windows, non-macOS systems.
115+
double request_timeout = -1;
116+
106117
/// If non-empty, override region with a connect string such as "localhost:9000"
107118
// XXX perhaps instead take a URL like "http://localhost:9000"?
108119
std::string endpoint_override;

‎python/pyarrow/_s3fs.pyx‎

Lines changed: 18 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -137,6 +137,13 @@ cdef class S3FileSystem(FileSystem):
137137
assumed role session will be refreshed.
138138
region : str, default 'us-east-1'
139139
AWS region to connect to.
140+
request_timeout : double, default None
141+
Socket read timeouts on Windows and macOS, in seconds.
142+
If omitted, the AWS SDK default value is used (typically 3 seconds).
143+
This option is ignored on non-Windows, non-macOS systems.
144+
connect_timeout : double, default None
145+
Socket connection timeout, in seconds.
146+
If omitted, the AWS SDK default value is used (typically 1 second).
140147
scheme : str, default 'https'
141148
S3 connection transport scheme.
142149
endpoint_override : str, default None
@@ -183,10 +190,11 @@ cdef class S3FileSystem(FileSystem):
183190
CS3FileSystem* s3fs
184191

185192
def __init__(self, *, access_key=None, secret_key=None, session_token=None,
186-
bint anonymous=False, region=None, scheme=None,
187-
endpoint_override=None, bint background_writes=True,
188-
default_metadata=None, role_arn=None, session_name=None,
189-
external_id=None, load_frequency=900, proxy_options=None,
193+
bint anonymous=False, region=None, request_timeout=None,
194+
connect_timeout=None, scheme=None, endpoint_override=None,
195+
bint background_writes=True, default_metadata=None,
196+
role_arn=None, session_name=None, external_id=None,
197+
load_frequency=900, proxy_options=None,
190198
allow_bucket_creation=False, allow_bucket_deletion=False):
191199
cdef:
192200
CS3Options options
@@ -254,6 +262,10 @@ cdef class S3FileSystem(FileSystem):
254262

255263
if region is not None:
256264
options.region = tobytes(region)
265+
if request_timeout is not None:
266+
options.request_timeout = request_timeout
267+
if connect_timeout is not None:
268+
options.connect_timeout = connect_timeout
257269
if scheme is not None:
258270
options.scheme = tobytes(scheme)
259271
if endpoint_override is not None:
@@ -324,6 +336,8 @@ cdef class S3FileSystem(FileSystem):
324336
CS3CredentialsKind_Anonymous),
325337
region=frombytes(opts.region),
326338
scheme=frombytes(opts.scheme),
339+
connect_timeout=opts.connect_timeout,
340+
request_timeout=opts.request_timeout,
327341
endpoint_override=frombytes(opts.endpoint_override),
328342
role_arn=frombytes(opts.role_arn),
329343
session_name=frombytes(opts.session_name),

‎python/pyarrow/includes/libarrow_fs.pxd‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,8 @@ cdef extern from "arrow/filesystem/api.h" namespace "arrow::fs" nogil:
152152

153153
cdef cppclass CS3Options "arrow::fs::S3Options":
154154
c_string region
155+
double connect_timeout
156+
double request_timeout
155157
c_string endpoint_override
156158
c_string scheme
157159
c_bool background_writes

‎python/pyarrow/tests/test_fs.py‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1131,6 +1131,15 @@ def test_s3_options():
11311131
assert isinstance(fs, S3FileSystem)
11321132
assert pickle.loads(pickle.dumps(fs)) == fs
11331133

1134+
fs = S3FileSystem(request_timeout=0.5, connect_timeout=0.25)
1135+
assert isinstance(fs, S3FileSystem)
1136+
assert pickle.loads(pickle.dumps(fs)) == fs
1137+
1138+
fs2 = S3FileSystem(request_timeout=0.25, connect_timeout=0.5)
1139+
assert isinstance(fs2, S3FileSystem)
1140+
assert pickle.loads(pickle.dumps(fs2)) == fs2
1141+
assert fs2 != fs
1142+
11341143
with pytest.raises(ValueError):
11351144
S3FileSystem(access_key='access')
11361145
with pytest.raises(ValueError):

0 commit comments

Comments
 (0)