-
Notifications
You must be signed in to change notification settings - Fork 715
Open
Labels
good first issueGood for newcomersGood for newcomershelp wantedExtra attention is neededExtra attention is neededrustPull requests that update Rust codePull requests that update Rust code
Description
#5562 has implemented all the fundamental elements for Write Returns Metadata. Now, we need to extend this feature for every service. This issue will be used to track the progress.
The modification is straightforward -- we just need to extract metadata from all write responses.
for example:
diff --git a/core/src/services/azblob/backend.rs b/core/src/services/azblob/backend.rs
index eafe32ddb..331d9b954 100644
--- a/core/src/services/azblob/backend.rs
+++ b/core/src/services/azblob/backend.rs
@@ -32,6 +32,7 @@ use sha2::Digest;
use sha2::Sha256;
use super::core::constants::X_MS_META_PREFIX;
+use super::core::constants::X_MS_VERSION_ID;
use super::core::AzblobCore;
use super::delete::AzblobDeleter;
use super::error::parse_error;
@@ -567,6 +568,9 @@ impl Access for AzblobBackend {
StatusCode::OK => {
let headers = resp.headers();
let mut meta = parse_into_metadata(path, headers)?;
+ if let Some(version_id) = parse_header_to_str(headers, X_MS_VERSION_ID)? {
+ meta.set_version(version_id);
+ }
let user_meta = parse_prefixed_headers(headers, X_MS_META_PREFIX);
if !user_meta.is_empty() {
diff --git a/core/src/services/azblob/core.rs b/core/src/services/azblob/core.rs
index 87fd4da11..7f664e64b 100644
--- a/core/src/services/azblob/core.rs
+++ b/core/src/services/azblob/core.rs
@@ -47,6 +47,7 @@ use crate::raw::*;
use crate::*;
pub mod constants {
+ // Indicates the Blob Storage version that was used to execute the request
pub const X_MS_VERSION: &str = "x-ms-version";
pub const X_MS_BLOB_TYPE: &str = "x-ms-blob-type";
@@ -55,6 +56,9 @@ pub mod constants {
pub const X_MS_BLOB_CONDITION_APPENDPOS: &str = "x-ms-blob-condition-appendpos";
pub const X_MS_META_PREFIX: &str = "x-ms-meta-";
+ // indicates the version of the blob, and it can be used in subsequent requests to access the blob.
+ pub const X_MS_VERSION_ID: &str = "x-ms-version-id";
+
// Server-side encryption with customer-provided headers
pub const X_MS_ENCRYPTION_KEY: &str = "x-ms-encryption-key";
pub const X_MS_ENCRYPTION_KEY_SHA256: &str = "x-ms-encryption-key-sha256";
diff --git a/core/src/services/azblob/writer.rs b/core/src/services/azblob/writer.rs
index 17cf8b676..d67b43e5a 100644
--- a/core/src/services/azblob/writer.rs
+++ b/core/src/services/azblob/writer.rs
@@ -20,6 +20,7 @@ use std::sync::Arc;
use http::StatusCode;
use uuid::Uuid;
+use super::core::constants::X_MS_VERSION_ID;
use super::core::AzblobCore;
use super::error::parse_error;
use crate::raw::*;
@@ -40,6 +41,26 @@ impl AzblobWriter {
pub fn new(core: Arc<AzblobCore>, op: OpWrite, path: String) -> Self {
AzblobWriter { core, op, path }
}
+
+ // skip extracting `content-md5` here, as it pertains to the content of the request rather than
+ // the content of the block itself for the `append` and `complete put block list` operations.
+ fn parse_metadata(headers: &http::HeaderMap) -> Result<Metadata> {
+ let mut metadata = Metadata::default();
+
+ if let Some(last_modified) = parse_last_modified(headers)? {
+ metadata.set_last_modified(last_modified);
+ }
+ let etag = parse_etag(headers)?;
+ if let Some(etag) = etag {
+ metadata.set_etag(etag);
+ }
+ let version_id = parse_header_to_str(headers, X_MS_VERSION_ID)?;
+ if let Some(version_id) = version_id {
+ metadata.set_version(version_id);
+ }
+
+ Ok(metadata)
+ }
}
impl oio::AppendWrite for AzblobWriter {
@@ -97,9 +118,10 @@ impl oio::AppendWrite for AzblobWriter {
let resp = self.core.send(req).await?;
+ let meta = AzblobWriter::parse_metadata(resp.headers())?;
let status = resp.status();
match status {
- StatusCode::CREATED => Ok(Metadata::default()),
+ StatusCode::CREATED => Ok(meta),
_ => Err(parse_error(resp)),
}
}
@@ -116,8 +138,13 @@ impl oio::BlockWrite for AzblobWriter {
let status = resp.status();
+ let mut meta = AzblobWriter::parse_metadata(resp.headers())?;
+ let md5 = parse_content_md5(resp.headers())?;
+ if let Some(md5) = md5 {
+ meta.set_content_md5(md5);
+ }
match status {
- StatusCode::CREATED | StatusCode::OK => Ok(Metadata::default()),
+ StatusCode::CREATED | StatusCode::OK => Ok(meta),
_ => Err(parse_error(resp)),
}
}
@@ -141,9 +168,10 @@ impl oio::BlockWrite for AzblobWriter {
.azblob_complete_put_block_list(&self.path, block_ids, &self.op)
.await?;
+ let meta = AzblobWriter::parse_metadata(resp.headers())?;
let status = resp.status();
match status {
- StatusCode::CREATED | StatusCode::OK => Ok(Metadata::default()),
+ StatusCode::CREATED | StatusCode::OK => Ok(meta),
_ => Err(parse_error(resp)),
}
}The whole change could be seen at: #5958
Note: we only need to extract metadata that represents the entire object, not just a portion of it or the request. For example: the content_length in HTTP always indicates the size of response, so we should't include it. Please review the documentations carefully and include only relevant metadata.
Services
- s3 feat(core)!: implement write returns metadata #5562
- aliyun_drive
- alluxio
- atomicserver
- azblob feat(services/azblob): implement write returns metadata #5958
- azdls feat(services/azdls): Implement write returns metadata #6368
- azfile
- b2
- comfs
- cos services/cos: Implement Write Returns Metadata for cos #5777
- dbfs
- dropbox feat(services/dropbox): Implement write returns metadata #6673
- foundationdb
- fs feat(core)!: implement write returns metadata #5562
- ftp
- gcs feat(services/gcs): implement write returns metadata #5933
- gdrive feat(services/gdrive): Implement write returns metadata #6683
- ghac
- github
- gridfs
- hdfs
- hdfs_native
- http
- icloud
- ipmfs
- koofr
- lakefs feat(services/lakefs): Implement write returns metadata #6770
- monoiofs feat(core)!: implement write returns metadata #5562
- obs refactor(services/obs): Implement Write Returns Metadata for obs #5912
- onedrive feat(services/onedrive): implement additional OneDrive features #5784
- oss feat(services/oss): Implement
Write Returns Metadatafor oss #5688 - seafile
- sftp
- swift feat(services/swift): Implement write returns metadata #6669
- webdav feat(services/webdav): Implement write returns metadata #6660
- webhdfs
- yandex_disk
Reactions are currently unavailable
Metadata
Metadata
Assignees
Labels
good first issueGood for newcomersGood for newcomershelp wantedExtra attention is neededExtra attention is neededrustPull requests that update Rust codePull requests that update Rust code