Skip to content

Commit 69e69f3

Browse files
committed
fix: virtual-host S3 bucket names that contain -- and no dots
The AWS SDK's endpoint rules check a bucket without dots first, and that check allows adjacent hyphens, so a bucket such as a--b on a custom endpoint is addressed as http://a--b.host. Comet rejected any adjacent separators and sent it path-style. The IPv4 and adjacent-separator checks now apply only to dotted names, which the SDK accepts over plain HTTP and still rejects with --, so a--b.c stays path-style.
1 parent eced565 commit 69e69f3

1 file changed

Lines changed: 35 additions & 4 deletions

File tree

  • native/core/src/parquet/objectstore

‎native/core/src/parquet/objectstore/s3.rs‎

Lines changed: 35 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -403,9 +403,11 @@ fn extract_s3_config_options(
403403
s3_configs
404404
}
405405

406-
/// Mirrors the AWS SDK's `isVirtualHostableS3Bucket`: 3 to 63 lowercase letters, digits and
407-
/// hyphens that start and end with a letter or digit. Dots only over plain HTTP, since a dotted
408-
/// host falls outside S3's wildcard certificate, and then no IPv4 shape or adjacent separators.
406+
/// Mirrors how the AWS SDK's S3 endpoint rules apply `isVirtualHostableS3Bucket`: the check
407+
/// without dots first, then the dotted one over plain HTTP. A name is 3 to 63 lowercase letters,
408+
/// digits and hyphens that start and end with a letter or digit, and may contain `--`. Dots are
409+
/// allowed only over plain HTTP, since a dotted host falls outside S3's wildcard certificate, and
410+
/// then the name must not look like an IPv4 address or have two separators (`.` or `-`) in a row.
409411
fn is_virtual_hostable_bucket(bucket: &str, allow_dots: bool) -> bool {
410412
let bytes = bucket.as_bytes();
411413
let edge = |b: &u8| b.is_ascii_lowercase() || b.is_ascii_digit();
@@ -424,7 +426,7 @@ fn is_virtual_hostable_bucket(bucket: &str, allow_dots: bool) -> bool {
424426
let separators_touch = bytes
425427
.windows(2)
426428
.any(|pair| pair.iter().all(|b| *b == b'.' || *b == b'-'));
427-
!(allow_dots && (ipv4_shaped || separators_touch))
429+
!(bytes.contains(&b'.') && (ipv4_shaped || separators_touch))
428430
}
429431

430432
/// An endpoint and the addressing mode it was shaped for, so the two never disagree.
@@ -1472,6 +1474,30 @@ mod tests {
14721474
assert_eq!(actual, expected);
14731475
}
14741476

1477+
#[test]
1478+
fn test_is_virtual_hostable_bucket() {
1479+
let cases = [
1480+
("a--b", false, true),
1481+
("a--b", true, true),
1482+
("a-b.c", true, true),
1483+
("a.-b", true, false),
1484+
("a-.b", true, false),
1485+
("a..b", true, false),
1486+
("a--b.c", true, false),
1487+
];
1488+
let mut actual = Vec::new();
1489+
let mut expected = Vec::new();
1490+
for (bucket, allow_dots, hostable) in cases {
1491+
let label = format!("{bucket} allow_dots={allow_dots}");
1492+
actual.push(format!(
1493+
"{label}: {}",
1494+
is_virtual_hostable_bucket(bucket, allow_dots)
1495+
));
1496+
expected.push(format!("{label}: {hostable}"));
1497+
}
1498+
assert_eq!(actual, expected);
1499+
}
1500+
14751501
#[test]
14761502
fn test_get_config_trimmed() {
14771503
let configs = TestConfigBuilder::new()
@@ -2895,6 +2921,11 @@ mod tests {
28952921
"test-bucket",
28962922
"http://minio.internal:9000/test-bucket/object",
28972923
),
2924+
(
2925+
vec![http],
2926+
"a--b",
2927+
"http://a--b.storage.example.test/object",
2928+
),
28982929
// Plain HTTP allows dots, but not an IPv4 shape, `-.` or uppercase
28992930
(
29002931
vec![http],

0 commit comments

Comments
 (0)