Skip to content

Commit 8bf7a62

Browse files
kalvinnchaunpub122y0pqkertljmedu303rl0aqrj3w8pvu43t6jxm6875lzg6f2pwqegc3xc
andauthored
feat: configure S3 URL addressing style (#3400)
## Summary - add one strict `BUZZ_S3_ADDRESSING_STYLE=path|virtual` setting shared by media and Git/CAS storage - preserve path-style defaults for bundled Compose/Helm MinIO while supporting Railway's virtual-hosted bucket contract - fail startup on invalid or non-Unicode values before dependency connection, and validate the Helm value with the same two choices - document operator mappings and why endpoint and bucket remain separate for routing and SigV4 signing ## Best-practice rationale AWS documents both URL forms and favors virtual-hosted addressing for S3, while compatibility endpoints such as the bundled MinIO deployment can require path style. `rust-s3` defaults to virtual/subdomain addressing and provides `with_path_style()` for the explicit compatibility case. Some providers buckets only support as virtual-hosted bucket styles. This PR therefore uses one explicit, provider-neutral switch rather than endpoint heuristics or fallback behavior, while retaining `path` as Buzz's backward-compatible default. Sources: - https://docs.aws.amazon.com/AmazonS3/latest/userguide/VirtualHosting.html - https://docs.rs/rust-s3/0.37.0/s3/bucket/struct.Bucket.html - https://docs.railway.com/storage-buckets#url-style - https://github.com/minio/minio/blob/master/docs/config/README.md#domain ## Validation - `cargo fmt --all` - `cargo check --workspace --all-targets` - targeted `buzz-media` and `buzz-relay` parsing/client-construction tests for defaults, strict errors, and both URL styles - Helm unittest: 45/45 passed - Compose config/render validation passed - local MinIO path-mode relay startup passed the Git A3 conformance probe and became ready - unreachable object storage failed startup and readiness never opened - push hooks completed the broader Rust and desktop suites successfully --------- Signed-off-by: npub122y0pqkertljmedu303rl0aqrj3w8pvu43t6jxm6875lzg6f2pwqegc3xc <5288f082d91aff2de5bc8be23fbfa01ca2e3859cac57a91b7a3fa9f12349505c@buzz.block.builderlab.xyz> Signed-off-by: Kalvin Chau <[email protected]> Co-authored-by: npub122y0pqkertljmedu303rl0aqrj3w8pvu43t6jxm6875lzg6f2pwqegc3xc <5288f082d91aff2de5bc8be23fbfa01ca2e3859cac57a91b7a3fa9f12349505c@buzz.block.builderlab.xyz>
1 parent c1553f3 commit 8bf7a62

28 files changed

Lines changed: 471 additions & 60 deletions

‎.env.example‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,19 @@ RELAY_URL=ws://localhost:3000
8282
# BUZZ_GIT_PACK_CACHE_MAX_BYTES=5368709120
8383
# BUZZ_GIT_PACK_CACHE_MAX_CONCURRENT_POPULATIONS=2
8484

85+
# -----------------------------------------------------------------------------
86+
# S3-Compatible Object Storage (media + Git/CAS)
87+
# -----------------------------------------------------------------------------
88+
# The local MinIO container is reachable from host processes at localhost:9000.
89+
# Path style keeps the bucket in the URL path and is required by this local DNS
90+
# setup. Use `virtual` only when the provider requires bucket-as-subdomain URLs.
91+
BUZZ_S3_ENDPOINT=http://localhost:9000
92+
BUZZ_S3_ACCESS_KEY=buzz_dev
93+
BUZZ_S3_SECRET_KEY=buzz_dev_secret
94+
BUZZ_S3_BUCKET=buzz-media
95+
BUZZ_S3_REGION=us-east-1
96+
BUZZ_S3_ADDRESSING_STYLE=path
97+
8598
# -----------------------------------------------------------------------------
8699
# Media Upload Admission
87100
# -----------------------------------------------------------------------------

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎crates/buzz-media/src/config.rs‎

Lines changed: 68 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,38 @@
11
//! Media storage configuration.
22
3+
use std::str::FromStr;
4+
5+
/// S3 URL addressing style shared by media and Git/CAS storage.
6+
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, serde::Deserialize)]
7+
#[serde(rename_all = "lowercase")]
8+
pub enum S3AddressingStyle {
9+
/// Put the bucket in the request path (`https://endpoint/bucket/key`).
10+
///
11+
/// This preserves compatibility with the bundled MinIO deployments, whose
12+
/// internal DNS only resolves the endpoint hostname.
13+
#[default]
14+
Path,
15+
/// Put the bucket in the hostname (`https://bucket.endpoint/key`).
16+
///
17+
/// This is the standard S3 form and is required by providers such as new
18+
/// Railway Storage Buckets.
19+
Virtual,
20+
}
21+
22+
impl FromStr for S3AddressingStyle {
23+
type Err = String;
24+
25+
fn from_str(value: &str) -> Result<Self, Self::Err> {
26+
match value {
27+
"path" => Ok(Self::Path),
28+
"virtual" => Ok(Self::Virtual),
29+
_ => Err(format!(
30+
"BUZZ_S3_ADDRESSING_STYLE must be 'path' or 'virtual', got {value:?}"
31+
)),
32+
}
33+
}
34+
}
35+
336
fn default_max_video_bytes() -> u64 {
437
524_288_000 // 500 MB
538
}
@@ -31,6 +64,9 @@ pub struct MediaConfig {
3164
/// the value is not meaningfully checked.
3265
#[serde(default = "default_s3_region")]
3366
pub s3_region: String,
67+
/// S3 URL addressing style. Defaults to path style for MinIO compatibility.
68+
#[serde(default)]
69+
pub s3_addressing_style: S3AddressingStyle,
3470
/// Maximum upload size for images (bytes). Default: 50 MB.
3571
pub max_image_bytes: u64,
3672
/// Maximum upload size for animated GIFs (bytes). Default: 10 MB.
@@ -123,7 +159,8 @@ impl MediaConfig {
123159

124160
#[cfg(test)]
125161
mod tests {
126-
use super::MediaConfig;
162+
use super::{MediaConfig, S3AddressingStyle};
163+
use std::str::FromStr;
127164

128165
fn valid_config() -> MediaConfig {
129166
MediaConfig {
@@ -132,6 +169,7 @@ mod tests {
132169
s3_secret_key: "s".to_string(),
133170
s3_bucket: "buzz-media".to_string(),
134171
s3_region: "us-east-1".to_string(),
172+
s3_addressing_style: S3AddressingStyle::Path,
135173
max_image_bytes: 1,
136174
max_gif_bytes: 1,
137175
max_video_bytes: 1,
@@ -143,6 +181,35 @@ mod tests {
143181
}
144182
}
145183

184+
#[test]
185+
fn addressing_style_parses_supported_values() {
186+
assert_eq!(
187+
S3AddressingStyle::from_str("path"),
188+
Ok(S3AddressingStyle::Path)
189+
);
190+
assert_eq!(
191+
S3AddressingStyle::from_str("virtual"),
192+
Ok(S3AddressingStyle::Virtual)
193+
);
194+
}
195+
196+
#[test]
197+
fn addressing_style_defaults_to_path() {
198+
assert_eq!(S3AddressingStyle::default(), S3AddressingStyle::Path);
199+
}
200+
201+
#[test]
202+
fn addressing_style_rejects_unknown_or_ambiguous_values() {
203+
for invalid in ["", "auto", "PATH", "virtual-hosted"] {
204+
let error =
205+
S3AddressingStyle::from_str(invalid).expect_err("must reject invalid style");
206+
assert!(
207+
error.contains("BUZZ_S3_ADDRESSING_STYLE must be 'path' or 'virtual'"),
208+
"unexpected error for {invalid:?}: {error}"
209+
);
210+
}
211+
}
212+
146213
#[test]
147214
fn upload_record_knobs_default_off_and_validate() {
148215
assert!(valid_config().validate().is_ok());

‎crates/buzz-media/src/lib.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ pub use bucket_index::{
1717
classify_key, fold_bucket_listing, BucketAggregate, BucketSnapshot, CommunityStorage, KeyClass,
1818
Page, SweepError,
1919
};
20-
pub use config::MediaConfig;
20+
pub use config::{MediaConfig, S3AddressingStyle};
2121
pub use error::MediaError;
2222
pub use storage::{BlobHeadMeta, BlobMeta, ByteStream, MediaStorage};
2323
pub use types::BlobDescriptor;

‎crates/buzz-media/src/storage.rs‎

Lines changed: 24 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ use std::pin::Pin;
55

66
use buzz_core::tenant::{CommunityId, TenantContext};
77

8-
use crate::config::MediaConfig;
8+
use crate::config::{MediaConfig, S3AddressingStyle};
99
use crate::error::MediaError;
1010
use bytes::Bytes;
1111
use s3::creds::Credentials;
@@ -61,8 +61,11 @@ impl MediaStorage {
6161
}
6262
.map_err(|e| MediaError::StorageError(e.to_string()))?;
6363
let bucket = Bucket::new(&config.s3_bucket, region, creds)
64-
.map_err(|e| MediaError::StorageError(e.to_string()))?
65-
.with_path_style();
64+
.map_err(|e| MediaError::StorageError(e.to_string()))?;
65+
let bucket = match config.s3_addressing_style {
66+
S3AddressingStyle::Path => bucket.with_path_style(),
67+
S3AddressingStyle::Virtual => bucket,
68+
};
6669
Ok(Self { bucket })
6770
}
6871

@@ -285,6 +288,7 @@ mod tests {
285288
s3_secret_key: secret.to_string(),
286289
s3_bucket: "buzz-media".to_string(),
287290
s3_region: "us-west-2".to_string(),
291+
s3_addressing_style: S3AddressingStyle::Path,
288292
max_image_bytes: 50 * 1024 * 1024,
289293
max_gif_bytes: 10 * 1024 * 1024,
290294
max_video_bytes: 524_288_000,
@@ -309,6 +313,23 @@ mod tests {
309313
}
310314
}
311315

316+
#[test]
317+
fn client_constructor_applies_both_addressing_styles() {
318+
let path = MediaStorage::new(&storage_config("buzz_dev", "buzz_dev_secret"))
319+
.expect("path-style client");
320+
assert!(path.bucket.is_path_style());
321+
assert_eq!(path.bucket.url(), "http://localhost:9000/buzz-media");
322+
323+
let mut virtual_config = storage_config("buzz_dev", "buzz_dev_secret");
324+
virtual_config.s3_addressing_style = S3AddressingStyle::Virtual;
325+
let virtual_hosted = MediaStorage::new(&virtual_config).expect("virtual-hosted client");
326+
assert!(virtual_hosted.bucket.is_subdomain_style());
327+
assert_eq!(
328+
virtual_hosted.bucket.url(),
329+
"http://buzz-media.localhost:9000"
330+
);
331+
}
332+
312333
#[test]
313334
fn partial_static_keys_are_rejected() {
314335
let err = match MediaStorage::new(&storage_config("buzz_dev", "")) {

‎crates/buzz-media/src/upload.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -570,6 +570,7 @@ mod tests {
570570
s3_secret_key: String::new(),
571571
s3_bucket: String::new(),
572572
s3_region: "us-east-1".to_string(),
573+
s3_addressing_style: crate::config::S3AddressingStyle::Path,
573574
max_image_bytes: 50 * 1024 * 1024,
574575
max_gif_bytes: 10 * 1024 * 1024,
575576
max_video_bytes: 524_288_000,

‎crates/buzz-media/src/validation.rs‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -949,6 +949,7 @@ mod tests {
949949
s3_secret_key: String::new(),
950950
s3_bucket: String::new(),
951951
s3_region: "us-east-1".to_string(),
952+
s3_addressing_style: crate::config::S3AddressingStyle::Path,
952953
max_image_bytes: 50 * 1024 * 1024,
953954
max_gif_bytes: 10 * 1024 * 1024,
954955
max_video_bytes: 524_288_000,

‎crates/buzz-media/tests/static_creds_minio.rs‎

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
1-
//! Live round-trip test for the **static-credentials** S3 path against a local
2-
//! MinIO, guarded by `#[ignore]`.
1+
//! Live round-trip test for the **static-credentials** S3 path against an
2+
//! S3-compatible service. It is guarded by `#[ignore]`.
33
//!
44
//! This is the path local/dev and any static-key deployment uses
55
//! (`s3_access_key`/`s3_secret_key` both non-empty -> `Credentials::new`). It
@@ -15,7 +15,8 @@
1515
//! ```
1616
//!
1717
//! Overridable via `BUZZ_S3_ENDPOINT` / `BUZZ_S3_ACCESS_KEY` /
18-
//! `BUZZ_S3_SECRET_KEY` / `BUZZ_S3_BUCKET`.
18+
//! `BUZZ_S3_SECRET_KEY` / `BUZZ_S3_BUCKET` / `BUZZ_S3_REGION` /
19+
//! `BUZZ_S3_ADDRESSING_STYLE`. The default remains `path` for MinIO.
1920
2021
use buzz_media::config::MediaConfig;
2122
use buzz_media::storage::MediaStorage;
@@ -29,7 +30,11 @@ fn minio_config() -> MediaConfig {
2930
s3_secret_key: std::env::var("BUZZ_S3_SECRET_KEY")
3031
.unwrap_or_else(|_| "buzz_dev_secret".to_string()),
3132
s3_bucket: std::env::var("BUZZ_S3_BUCKET").unwrap_or_else(|_| "buzz-media".to_string()),
32-
s3_region: "us-east-1".to_string(),
33+
s3_region: std::env::var("BUZZ_S3_REGION").unwrap_or_else(|_| "us-east-1".to_string()),
34+
s3_addressing_style: std::env::var("BUZZ_S3_ADDRESSING_STYLE")
35+
.unwrap_or_else(|_| "path".to_string())
36+
.parse()
37+
.expect("BUZZ_S3_ADDRESSING_STYLE must be path or virtual"),
3338
max_image_bytes: 50 * 1024 * 1024,
3439
max_gif_bytes: 10 * 1024 * 1024,
3540
max_video_bytes: 524_288_000,

‎crates/buzz-relay/src/api/git/cas_publish.rs‎

Lines changed: 9 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1583,22 +1583,15 @@ mod tests {
15831583
}
15841584

15851585
fn live_store() -> GitStore {
1586-
let endpoint = std::env::var("BUZZ_GIT_S3_ENDPOINT")
1587-
.or_else(|_| std::env::var("BUZZ_S3_ENDPOINT"))
1588-
.unwrap_or_else(|_| "http://localhost:9000".into());
1589-
let access_key = std::env::var("BUZZ_GIT_S3_ACCESS_KEY")
1590-
.or_else(|_| std::env::var("BUZZ_S3_ACCESS_KEY"))
1591-
.unwrap_or_else(|_| "buzz_dev".into());
1592-
let secret_key = std::env::var("BUZZ_GIT_S3_SECRET_KEY")
1593-
.or_else(|_| std::env::var("BUZZ_S3_SECRET_KEY"))
1594-
.unwrap_or_else(|_| "buzz_dev_secret".into());
1595-
let bucket = std::env::var("BUZZ_GIT_S3_BUCKET")
1596-
.or_else(|_| std::env::var("BUZZ_S3_BUCKET"))
1597-
.unwrap_or_else(|_| "buzz-media".into());
1598-
let region = std::env::var("BUZZ_GIT_S3_REGION")
1599-
.or_else(|_| std::env::var("BUZZ_S3_REGION"))
1600-
.unwrap_or_else(|_| "us-east-1".into());
1601-
GitStore::new(&endpoint, &access_key, &secret_key, &bucket, &region).expect("connect minio")
1586+
GitStore::new(
1587+
"http://localhost:9000",
1588+
"buzz_dev",
1589+
"buzz_dev_secret",
1590+
"buzz-media",
1591+
"us-east-1",
1592+
buzz_media::config::S3AddressingStyle::Path,
1593+
)
1594+
.expect("connect local MinIO")
16021595
}
16031596

16041597
fn tenant() -> TenantContext {

‎crates/buzz-relay/src/api/git/hydrate.rs‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -543,8 +543,15 @@ mod tests {
543543
#[tokio::test]
544544
async fn materialized_repo_is_created_under_configured_scratch_dir() {
545545
let scratch = TempDir::new().unwrap();
546-
let store = GitStore::new("http://localhost:9000", "x", "x", "x", "us-east-1")
547-
.expect("construct store");
546+
let store = GitStore::new(
547+
"http://localhost:9000",
548+
"x",
549+
"x",
550+
"x",
551+
"us-east-1",
552+
buzz_media::config::S3AddressingStyle::Path,
553+
)
554+
.expect("construct store");
548555
let manifest = Manifest {
549556
version: 1,
550557
head: "refs/heads/main".into(),
@@ -587,8 +594,9 @@ mod tests {
587594
"buzz_dev_secret",
588595
"buzz-git",
589596
"us-east-1",
597+
buzz_media::config::S3AddressingStyle::Path,
590598
)
591-
.expect("connect minio")
599+
.expect("connect local MinIO")
592600
}
593601

594602
/// Build a tiny on-disk repo, return (pack bytes, head_oid).

0 commit comments

Comments
 (0)