Skip to main content

s3

41 configuration options · available in Rust, Java, Python, Node.js, Ruby, Go

Quick start
use opendal::Operator;

let operator = Operator::via_iter("s3", [
("bucket".to_string(), "my-bucket".to_string()),
])?;
All configuration options (copy & trim)
Full reference
use opendal::Operator;

let operator = Operator::via_iter("s3", [
// --- General ---
// root of this backend.
//
// All operations will happen under this root.
//
// default to `/` if not set.
// ("root".to_string(), "/".to_string()),

// bucket name of this backend.
//
// required.
("bucket".to_string(), "my-bucket".to_string()),

// endpoint of this backend.
//
// Endpoint must be full uri, e.g.
//
// - AWS S3: `https://s3.amazonaws.com` or `https://s3.{region}.amazonaws.com`
// - Cloudflare R2: `https://<ACCOUNT_ID>.r2.cloudflarestorage.com`
// - Aliyun OSS: `https://{region}.aliyuncs.com`
// - Tencent COS: `https://cos.{region}.myqcloud.com`
// - Minio: `http://127.0.0.1:9000`
//
// If user inputs endpoint without scheme like "s3.amazonaws.com", we
// will prepend "https://" before it.
//
// - If endpoint is set, we will take user's input first.
// - If not, we will try to load it from environment.
// - If still not set, default to `https://s3.amazonaws.com`.
// ("endpoint".to_string(), "https://s3.amazonaws.com".to_string()),

// Region represent the signing region of this endpoint. This is required
// if you are using the default AWS S3 endpoint.
//
// If using a custom endpoint,
// - If region is set, we will take user's input first.
// - If not, we will try to load it from environment.
// ("region".to_string(), "us-east-1".to_string()),

// Fallback signing region when neither `region`, `AWS_REGION`, nor
// `AWS_DEFAULT_REGION` supplies one.
// ("default_region".to_string(), "...".to_string()),
// --- Credentials ---
// AWS profile.
//
// By default, reqsign which is the default credential provider, supplies profile in order:
// - explicit option
// - `AWS_PROFILE` environment variable, from which reqsign reads profile from:
// - `~/.aws/credentials` (or the path specified by `AWS_SHARED_CREDENTIALS_FILE`)
// - `~/.aws/config` (or the path specified by `AWS_CONFIG_FILE`)
// ("profile".to_string(), "development".to_string()),

// access_key_id of this backend.
//
// - If access_key_id is set, we will take user's input first.
// - If not, we will try to load it from environment.
// ("access_key_id".to_string(), "...".to_string()),

// secret_access_key of this backend.
//
// - If secret_access_key is set, we will take user's input first.
// - If not, we will try to load it from environment.
// ("secret_access_key".to_string(), "...".to_string()),

// session_token (aka, security token) of this backend.
//
// This token will expire after sometime, it's recommended to set session_token
// by hand.
// ("session_token".to_string(), "...".to_string()),

// Relative URI for AWS container credential metadata.
//
// An explicit full endpoint takes precedence over this value. This value
// takes precedence over the ambient full and relative container credential
// URIs. OpenDAL joins it to `container_metadata_uri_override` when set,
// otherwise to `http://169.254.170.2`.
//
// Setting any container credential field replaces the ECS slot in the
// default credential chain. It does not move that slot ahead of earlier
// default providers.
// ("container_credentials_relative_uri".to_string(), "...".to_string()),

// Full URI for AWS container credential metadata.
//
// This endpoint takes precedence over the relative container credential
// URI and both ambient container credential URI forms. It does not change
// the position or priority of the default ECS credential provider.
// ("container_credentials_endpoint".to_string(), "...".to_string()),

// Authorization token sent to the AWS container credential endpoint.
//
// This token takes precedence over the configured token file and the
// ambient direct and file token forms.
// ("container_authorization_token".to_string(), "...".to_string()),

// File containing the authorization token sent to the AWS container
// credential endpoint.
//
// This file takes precedence over both ambient token forms. The credential
// provider trims surrounding whitespace after reading the file.
// ("container_authorization_token_file".to_string(), "...".to_string()),

// Base URI used to resolve a relative AWS container credential URI.
//
// This override affects relative URIs only. It does not alter a configured
// or ambient full URI. The override alone does not select an endpoint.
// ("container_metadata_uri_override".to_string(), "...".to_string()),

// Disable automatic endpoint loading and the environment/profile
// credential providers.
//
// This skips endpoint environment variables and removes the static
// environment and shared-profile credential slots. Region environment
// lookup and other credential providers, including AWS container
// credentials and EC2 instance metadata, remain enabled.
// ("disable_config_load".to_string(), "true".to_string()),

// Disable load credential from ec2 metadata.
//
// This option is used to disable the default behavior of opendal
// to load credential from ec2 metadata, a.k.a., IMDSv2
// ("disable_ec2_metadata".to_string(), "true".to_string()),

// Skip signature will skip loading credentials and signing requests.
// ("skip_signature".to_string(), "true".to_string()),
// --- Assume role ---
// role_arn for this backend.
//
// If `role_arn` is set, we will use already known config as source
// credential to assume role with `role_arn`.
// ("role_arn".to_string(), "...".to_string()),

// external_id for this backend.
// ("external_id".to_string(), "...".to_string()),

// role_session_name for this backend.
// ("role_session_name".to_string(), "...".to_string()),

// assume_role_duration_seconds for this backend.
// ("assume_role_duration_seconds".to_string(), "1000".to_string()),

// assume_role_session_tags for this backend.
// ("assume_role_session_tags".to_string(), "...".to_string()),
// --- Encryption ---
// server_side_encryption for this backend.
//
// Available values: `AES256`, `aws:kms`.
// ("server_side_encryption".to_string(), "...".to_string()),

// server_side_encryption_aws_kms_key_id for this backend
//
// - If `server_side_encryption` set to `aws:kms`, and `server_side_encryption_aws_kms_key_id`
// is not set, S3 will use aws managed kms key to encrypt data.
// - If `server_side_encryption` set to `aws:kms`, and `server_side_encryption_aws_kms_key_id`
// is a valid kms key id, S3 will use the provided kms key to encrypt data.
// - If the `server_side_encryption_aws_kms_key_id` is invalid or not found, an error will be
// returned.
// - If `server_side_encryption` is not `aws:kms`, setting `server_side_encryption_aws_kms_key_id`
// is a noop.
// ("server_side_encryption_aws_kms_key_id".to_string(), "...".to_string()),

// Control S3 Bucket Keys for SSE-KMS writes and copies.
//
// Omit this option to inherit the bucket setting. `false` disables Bucket
// Keys for destination objects, and `true` enables them.
// ("server_side_encryption_bucket_key_enabled".to_string(), "true".to_string()),

// server_side_encryption_customer_algorithm for this backend.
//
// Available values: `AES256`.
// ("server_side_encryption_customer_algorithm".to_string(), "...".to_string()),

// server_side_encryption_customer_key for this backend.
//
// Value: BASE64-encoded key that matches algorithm specified in
// `server_side_encryption_customer_algorithm`.
// ("server_side_encryption_customer_key".to_string(), "...".to_string()),

// Set server_side_encryption_customer_key_md5 for this backend.
//
// Value: MD5 digest of key specified in `server_side_encryption_customer_key`.
// ("server_side_encryption_customer_key_md5".to_string(), "...".to_string()),
// --- Behavior ---
// default storage_class for this backend.
//
// Available values:
// - `DEEP_ARCHIVE`
// - `GLACIER`
// - `GLACIER_IR`
// - `INTELLIGENT_TIERING`
// - `ONEZONE_IA`
// - `EXPRESS_ONEZONE`
// - `OUTPOSTS`
// - `REDUCED_REDUNDANCY`
// - `STANDARD`
// - `STANDARD_IA`
//
// S3 compatible services don't support all of them
// ("default_storage_class".to_string(), "...".to_string()),

// Enable virtual host style so that opendal will send API requests
// in virtual host style instead of path style.
//
// - By default, opendal will send API to `https://s3.us-east-1.amazonaws.com/bucket_name`
// - Enabled, opendal will send API to `https://bucket_name.s3.us-east-1.amazonaws.com`
// ("enable_virtual_host_style".to_string(), "true".to_string()),

// Checksum Algorithm to use when sending checksums in HTTP headers.
// This is necessary when writing to AWS S3 Buckets with Object Lock enabled for example.
//
// Available options:
// - "crc32c"
// - "md5"
// ("checksum_algorithm".to_string(), "...".to_string()),

// OpenDAL uses List Objects V2 by default to list objects.
// However, some legacy services do not yet support V2.
// This option allows users to switch back to the older List Objects V1.
// ("disable_list_objects_v2".to_string(), "true".to_string()),

// Indicates whether the client agrees to pay for the requests made to the S3 bucket.
// ("enable_request_payer".to_string(), "true".to_string()),

// Default ACL for new objects.
// Note that some s3 services like minio do not support this option.
// ("default_acl".to_string(), "...".to_string()),
])?;

Every option is passed as a string key; OpenDAL parses it into the right type. Some services may require building the binding with the matching services-* feature enabled.

Configuration reference

General5
KeyTypeRequiredDescription
rootdefault /stringnoroot of this backend. All operations will happen under this root. default to / if not set.
buckete.g. my-bucketstringyesbucket name of this backend. required.
endpointdefault https://s3.amazonaws.comstringnoendpoint of this backend. Endpoint must be full uri, e.g. - AWS S3: https://s3.amazonaws.com or https://s3.{region}.amazonaws.com - Cloudflare R2: https://<ACCOUNT_ID>.r2.cloudflarestorage.com - Aliyun OSS: https://{region}.aliyuncs.com - Tencent COS: https://cos.{region}.myqcloud.com - Minio: http://127.0.0.1:9000 If user inputs endpoint without scheme like "s3.amazonaws.com", we will prepend "https://" before it. - If endpoint is set, we will take user's input first. - If not, we will try to load it from environment. - If still not set, default to https://s3.amazonaws.com.
regione.g. us-east-1stringnoRegion represent the signing region of this endpoint. This is required if you are using the default AWS S3 endpoint. If using a custom endpoint, - If region is set, we will take user's input first. - If not, we will try to load it from environment.
default_regionstringnoFallback signing region when neither region, AWS_REGION, nor AWS_DEFAULT_REGION supplies one.
Credentials12
KeyTypeRequiredDescription
profilee.g. developmentstringnoAWS profile. By default, reqsign which is the default credential provider, supplies profile in order: - explicit option - AWS_PROFILE environment variable, from which reqsign reads profile from: - ~/.aws/credentials (or the path specified by AWS_SHARED_CREDENTIALS_FILE) - ~/.aws/config (or the path specified by AWS_CONFIG_FILE)
access_key_idstringnoaccess_key_id of this backend. - If access_key_id is set, we will take user's input first. - If not, we will try to load it from environment.
secret_access_keystringnosecret_access_key of this backend. - If secret_access_key is set, we will take user's input first. - If not, we will try to load it from environment.
session_tokenstringnosession_token (aka, security token) of this backend. This token will expire after sometime, it's recommended to set session_token by hand.
container_credentials_relative_uristringnoRelative URI for AWS container credential metadata. An explicit full endpoint takes precedence over this value. This value takes precedence over the ambient full and relative container credential URIs. OpenDAL joins it to container_metadata_uri_override when set, otherwise to http://169.254.170.2. Setting any container credential field replaces the ECS slot in the default credential chain. It does not move that slot ahead of earlier default providers.
container_credentials_endpointstringnoFull URI for AWS container credential metadata. This endpoint takes precedence over the relative container credential URI and both ambient container credential URI forms. It does not change the position or priority of the default ECS credential provider.
container_authorization_tokenstringnoAuthorization token sent to the AWS container credential endpoint. This token takes precedence over the configured token file and the ambient direct and file token forms.
container_authorization_token_filestringnoFile containing the authorization token sent to the AWS container credential endpoint. This file takes precedence over both ambient token forms. The credential provider trims surrounding whitespace after reading the file.
container_metadata_uri_overridestringnoBase URI used to resolve a relative AWS container credential URI. This override affects relative URIs only. It does not alter a configured or ambient full URI. The override alone does not select an endpoint.
disable_config_loadboolnoDisable automatic endpoint loading and the environment/profile credential providers. This skips endpoint environment variables and removes the static environment and shared-profile credential slots. Region environment lookup and other credential providers, including AWS container credentials and EC2 instance metadata, remain enabled.
disable_ec2_metadataboolnoDisable load credential from ec2 metadata. This option is used to disable the default behavior of opendal to load credential from ec2 metadata, a.k.a., IMDSv2
skip_signatureboolnoSkip signature will skip loading credentials and signing requests.
Assume role5
KeyTypeRequiredDescription
role_arnstringnorole_arn for this backend. If role_arn is set, we will use already known config as source credential to assume role with role_arn.
external_idstringnoexternal_id for this backend.
role_session_namestringnorole_session_name for this backend.
assume_role_duration_secondsintegernoassume_role_duration_seconds for this backend.
assume_role_session_tagsmapnoassume_role_session_tags for this backend.
Encryption6
KeyTypeRequiredDescription
server_side_encryptionstringnoserver_side_encryption for this backend. Available values: AES256, aws:kms.
server_side_encryption_aws_kms_key_idstringnoserver_side_encryption_aws_kms_key_id for this backend - If server_side_encryption set to aws:kms, and server_side_encryption_aws_kms_key_id is not set, S3 will use aws managed kms key to encrypt data. - If server_side_encryption set to aws:kms, and server_side_encryption_aws_kms_key_id is a valid kms key id, S3 will use the provided kms key to encrypt data. - If the server_side_encryption_aws_kms_key_id is invalid or not found, an error will be returned. - If server_side_encryption is not aws:kms, setting server_side_encryption_aws_kms_key_id is a noop.
server_side_encryption_bucket_key_enabledboolnoControl S3 Bucket Keys for SSE-KMS writes and copies. Omit this option to inherit the bucket setting. false disables Bucket Keys for destination objects, and true enables them.
server_side_encryption_customer_algorithmstringnoserver_side_encryption_customer_algorithm for this backend. Available values: AES256.
server_side_encryption_customer_keystringnoserver_side_encryption_customer_key for this backend. Value: BASE64-encoded key that matches algorithm specified in server_side_encryption_customer_algorithm.
server_side_encryption_customer_key_md5stringnoSet server_side_encryption_customer_key_md5 for this backend. Value: MD5 digest of key specified in server_side_encryption_customer_key.
Behavior6
KeyTypeRequiredDescription
default_storage_classstringnodefault storage_class for this backend. Available values: - DEEP_ARCHIVE - GLACIER - GLACIER_IR - INTELLIGENT_TIERING - ONEZONE_IA - EXPRESS_ONEZONE - OUTPOSTS - REDUCED_REDUNDANCY - STANDARD - STANDARD_IA S3 compatible services don't support all of them
enable_virtual_host_styleboolnoEnable virtual host style so that opendal will send API requests in virtual host style instead of path style. - By default, opendal will send API to https://s3.us-east-1.amazonaws.com/bucket_name - Enabled, opendal will send API to https://bucket_name.s3.us-east-1.amazonaws.com
checksum_algorithmstringnoChecksum Algorithm to use when sending checksums in HTTP headers. This is necessary when writing to AWS S3 Buckets with Object Lock enabled for example. Available options: - "crc32c" - "md5"
disable_list_objects_v2boolnoOpenDAL uses List Objects V2 by default to list objects. However, some legacy services do not yet support V2. This option allows users to switch back to the older List Objects V1.
enable_request_payerboolnoIndicates whether the client agrees to pay for the requests made to the S3 bucket.
default_aclstringnoDefault ACL for new objects. Note that some s3 services like minio do not support this option.
Deprecated7
KeyTypeRequiredDescription
enable_versioningdeprecatedboolnoDeprecated: S3 versioning capability is enabled by default.
Deprecated since 0.57.0: S3 versioning capability is enabled by default and this option is no longer needed.
allow_anonymousdeprecatedboolnoAllow anonymous will allow opendal to send request without signing when credential is not loaded.
Deprecated since 0.57.0: Please use skip_signature instead of allow_anonymous
batch_max_operationsdeprecatedintegernoDeprecated: S3 delete batch capability is enabled by default.
Deprecated since 0.57.0: S3 delete batch capability is enabled by default. Use CapabilityOverrideLayer to override delete_max_size for specific endpoints.
delete_max_sizedeprecatedintegernoDeprecated: S3 delete batch capability is enabled by default.
Deprecated since 0.57.0: S3 delete batch capability is enabled by default. Use CapabilityOverrideLayer to override delete_max_size for specific endpoints.
disable_stat_with_overridedeprecatedboolnoDeprecated: S3 stat override capabilities are enabled by default.
Deprecated since 0.57.0: S3 stat override capabilities are enabled by default. Use CapabilityOverrideLayer to override them for specific endpoints.
disable_write_with_if_matchdeprecatedboolnoDeprecated: S3 write with If-Match capability is enabled by default.
Deprecated since 0.57.0: S3 write with If-Match capability is enabled by default and this option is no longer needed.
enable_write_with_appenddeprecatedboolnoDeprecated: S3 append capability is enabled by default.
Deprecated since 0.57.0: S3 append capability is enabled by default and this option is no longer needed.