Skip to content

Commit 311f324

Browse files
authored
fix(services): improve stat implementation (#7857)
* fix: avoid full value reads in kv stat * harden sql services and improve d1 and gridfs * Fix linting
1 parent 3695235 commit 311f324

16 files changed

Lines changed: 380 additions & 84 deletions

File tree

‎core/services/d1/src/backend.rs‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -249,10 +249,9 @@ impl Service for D1Backend {
249249
if p == build_abs_path(&self.root, "") {
250250
Ok(RpStat::new(Metadata::new(EntryMode::DIR)))
251251
} else {
252-
let bs = self.core.get(ctx, &p).await?;
253-
match bs {
254-
Some(bs) => Ok(RpStat::new(
255-
Metadata::new(EntryMode::FILE).with_content_length(bs.len() as u64),
252+
match self.core.get_length(ctx, &p).await? {
253+
Some(length) => Ok(RpStat::new(
254+
Metadata::new(EntryMode::FILE).with_content_length(length as u64),
256255
)),
257256
None => Err(Error::new(ErrorKind::NotFound, "kv not found in d1")),
258257
}

‎core/services/d1/src/core.rs‎

Lines changed: 38 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,8 @@ impl Debug for D1Core {
4747
}
4848
}
4949

50+
const CLOUDFLARE_API_BASE_URL: &str = "https://api.cloudflare.com/client/v4";
51+
5052
impl D1Core {
5153
fn create_d1_query_request(
5254
&self,
@@ -59,11 +61,7 @@ impl D1Core {
5961
"/accounts/{}/d1/database/{}/query",
6062
self.account_id, self.database_id
6163
);
62-
let url: String = format!(
63-
"{}{}",
64-
"https://api.cloudflare.com/client/v4",
65-
percent_encode_path(&p)
66-
);
64+
let url: String = format!("{}{}", CLOUDFLARE_API_BASE_URL, percent_encode_path(&p));
6765

6866
let mut req = Request::post(&url);
6967
if let Some(auth) = &self.authorization {
@@ -84,8 +82,13 @@ impl D1Core {
8482
}
8583

8684
pub async fn get(&self, ctx: &OperationContext, path: &str) -> Result<Option<Buffer>> {
85+
// d1 follows SQLite SQL syntax, we use `"` for identifier quote. A quoted identifier is case-sensitive and
86+
// can contain special characters.
87+
// Read more https://www.sqlite.org/quirks.html#double_quoted_string_literals
88+
//
89+
// We uses identifier quote for trusted table and field configuration to ensure correctness.
8790
let query = format!(
88-
"SELECT {} FROM {} WHERE {} = ? LIMIT 1",
91+
r#"SELECT "{}" FROM "{}" WHERE "{}" = ? LIMIT 1"#,
8992
self.value_field, self.table, self.key_field
9093
);
9194
let req =
@@ -104,15 +107,36 @@ impl D1Core {
104107
}
105108
}
106109

110+
pub async fn get_length(&self, ctx: &OperationContext, path: &str) -> Result<Option<usize>> {
111+
let query = format!(
112+
r#"SELECT LENGTH(CAST("{}" AS BLOB)) AS "content_length" FROM "{}" WHERE "{}" = ? LIMIT 1"#,
113+
self.value_field, self.table, self.key_field
114+
);
115+
let req =
116+
self.create_d1_query_request(&query, vec![path.into()], Operation::Stat, "Stat")?;
117+
118+
let resp = ctx.http_transport().send(req).await?;
119+
let status = resp.status();
120+
match status {
121+
StatusCode::OK | StatusCode::PARTIAL_CONTENT => {
122+
let body = resp.into_body();
123+
let bs = body.to_bytes();
124+
let d1_response = D1Response::parse(&bs)?;
125+
d1_response.get_usize_result("content_length")
126+
}
127+
_ => Err(parse_error(resp)),
128+
}
129+
}
130+
107131
pub async fn set(&self, ctx: &OperationContext, path: &str, value: Buffer) -> Result<()> {
108132
let table = &self.table;
109133
let key_field = &self.key_field;
110134
let value_field = &self.value_field;
111135
let query = format!(
112-
"INSERT INTO {table} ({key_field}, {value_field}) \
113-
VALUES (?, ?) \
114-
ON CONFLICT ({key_field}) \
115-
DO UPDATE SET {value_field} = EXCLUDED.{value_field}",
136+
r#"INSERT INTO "{table}" ("{key_field}", "{value_field}") \
137+
VALUES ('?', ?) \
138+
ON CONFLICT ("{key_field}") \
139+
DO UPDATE SET "{value_field}" = EXCLUDED."{value_field}""#,
116140
);
117141

118142
let params = vec![path.into(), value.to_vec().into()];
@@ -127,7 +151,10 @@ impl D1Core {
127151
}
128152

129153
pub async fn delete(&self, ctx: &OperationContext, path: &str) -> Result<()> {
130-
let query = format!("DELETE FROM {} WHERE {} = ?", self.table, self.key_field);
154+
let query = format!(
155+
r#"DELETE FROM "{}" WHERE "{}" = ?"#,
156+
self.table, self.key_field
157+
);
131158
let req =
132159
self.create_d1_query_request(&query, vec![path.into()], Operation::Delete, "Delete")?;
133160

‎core/services/d1/src/model.rs‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,38 @@ impl D1Response {
7171
_ => None,
7272
}
7373
}
74+
75+
pub fn get_usize_result(&self, key: &str) -> Result<Option<usize>, Error> {
76+
if self.result.is_empty() || self.result[0].results.is_empty() {
77+
return Ok(None);
78+
}
79+
let result = &self.result[0].results[0];
80+
let Some(value) = result.get(key) else {
81+
return Ok(None);
82+
};
83+
84+
match value {
85+
Value::Number(n) => {
86+
let value = n.as_u64().ok_or_else(|| {
87+
Error::new(
88+
opendal_core::ErrorKind::Unexpected,
89+
"d1 value length is invalid",
90+
)
91+
})?;
92+
value.try_into().map(Some).map_err(|err| {
93+
Error::new(
94+
opendal_core::ErrorKind::Unexpected,
95+
"d1 value length is invalid",
96+
)
97+
.set_source(err)
98+
})
99+
}
100+
_ => Err(Error::new(
101+
opendal_core::ErrorKind::Unexpected,
102+
"d1 value length is invalid",
103+
)),
104+
}
105+
}
74106
}
75107

76108
#[derive(Deserialize, Debug)]
@@ -121,4 +153,32 @@ mod test {
121153
let response: D1Response = serde_json::from_str(data).unwrap();
122154
println!("{:?}", response.result[0].results[0]);
123155
}
156+
157+
#[test]
158+
fn test_get_usize_result() {
159+
let data = r#"
160+
{
161+
"result": [
162+
{
163+
"results": [
164+
{
165+
"content_length": 6
166+
}
167+
],
168+
"success": true,
169+
"meta": {}
170+
}
171+
],
172+
"success": true,
173+
"errors": [],
174+
"messages": []
175+
}"#;
176+
177+
let response: D1Response = serde_json::from_str(data).unwrap();
178+
assert_eq!(
179+
response.get_usize_result("content_length").unwrap(),
180+
Some(6)
181+
);
182+
assert_eq!(response.get_usize_result("missing").unwrap(), None);
183+
}
124184
}

‎core/services/gridfs/src/backend.rs‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -223,10 +223,9 @@ impl Service for GridfsBackend {
223223
if p == build_abs_path(&self.root, "") {
224224
Ok(RpStat::new(Metadata::new(EntryMode::DIR)))
225225
} else {
226-
let bs = self.core.get(&p).await?;
227-
match bs {
228-
Some(bs) => Ok(RpStat::new(
229-
Metadata::new(EntryMode::FILE).with_content_length(bs.len() as u64),
226+
match self.core.get_length(&p).await? {
227+
Some(len) => Ok(RpStat::new(
228+
Metadata::new(EntryMode::FILE).with_content_length(len as u64),
230229
)),
231230
None => Err(Error::new(ErrorKind::NotFound, "kv not found in gridfs")),
232231
}

‎core/services/gridfs/src/core.rs‎

Lines changed: 23 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ use futures::AsyncReadExt;
2121
use futures::AsyncWriteExt;
2222
use mea::once::OnceCell;
2323
use mongodb::bson::doc;
24-
use mongodb::gridfs::GridFsBucket;
24+
use mongodb::gridfs::{FilesCollectionDocument, GridFsBucket};
2525
use mongodb::options::ClientOptions;
2626
use mongodb::options::GridFsBucketOptions;
2727
use opendal_core::raw::*;
@@ -67,10 +67,18 @@ impl GridfsCore {
6767
.await
6868
}
6969

70+
async fn get_one_doc(
71+
bucket: &GridFsBucket,
72+
path: &str,
73+
) -> Result<Option<FilesCollectionDocument>> {
74+
let filter = doc! { "filename": path };
75+
let doc = bucket.find_one(filter).await.map_err(parse_mongodb_error)?;
76+
Ok(doc)
77+
}
78+
7079
pub async fn get(&self, path: &str) -> Result<Option<Buffer>> {
7180
let bucket = self.get_bucket().await?;
72-
let filter = doc! { "filename": path };
73-
let Some(doc) = bucket.find_one(filter).await.map_err(parse_mongodb_error)? else {
81+
let Some(doc) = Self::get_one_doc(bucket, path).await? else {
7482
return Ok(None);
7583
};
7684

@@ -87,12 +95,21 @@ impl GridfsCore {
8795
Ok(Some(Buffer::from(destination)))
8896
}
8997

98+
/// Get the byte length of the file.
99+
pub async fn get_length(&self, path: &str) -> Result<Option<usize>> {
100+
let bucket = self.get_bucket().await?;
101+
let Some(doc) = Self::get_one_doc(bucket, path).await? else {
102+
return Ok(None);
103+
};
104+
105+
Ok(Some(doc.length as usize))
106+
}
107+
90108
pub async fn set(&self, path: &str, value: Buffer) -> Result<()> {
91109
let bucket = self.get_bucket().await?;
92110

93111
// delete old file if exists
94-
let filter = doc! { "filename": path };
95-
if let Some(doc) = bucket.find_one(filter).await.map_err(parse_mongodb_error)? {
112+
if let Some(doc) = Self::get_one_doc(bucket, path).await? {
96113
let file_id = doc.id;
97114
bucket.delete(file_id).await.map_err(parse_mongodb_error)?;
98115
};
@@ -113,8 +130,7 @@ impl GridfsCore {
113130

114131
pub async fn delete(&self, path: &str) -> Result<()> {
115132
let bucket = self.get_bucket().await?;
116-
let filter = doc! { "filename": path };
117-
let Some(doc) = bucket.find_one(filter).await.map_err(parse_mongodb_error)? else {
133+
let Some(doc) = Self::get_one_doc(bucket, path).await? else {
118134
return Ok(());
119135
};
120136

‎core/services/mongodb/src/backend.rs‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -241,10 +241,9 @@ impl Service for MongodbBackend {
241241
if p == build_abs_path(&self.root, "") {
242242
Ok(RpStat::new(Metadata::new(EntryMode::DIR)))
243243
} else {
244-
let bs = self.core.get(&p).await?;
245-
match bs {
246-
Some(bs) => Ok(RpStat::new(
247-
Metadata::new(EntryMode::FILE).with_content_length(bs.len() as u64),
244+
match self.core.get_length(&p).await? {
245+
Some(length) => Ok(RpStat::new(
246+
Metadata::new(EntryMode::FILE).with_content_length(length as u64),
248247
)),
249248
None => Err(Error::new(ErrorKind::NotFound, "kv not found in mongodb")),
250249
}

‎core/services/mongodb/src/core.rs‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ use std::fmt::Debug;
1919

2020
use mea::once::OnceCell;
2121
use mongodb::bson::Binary;
22+
use mongodb::bson::Bson;
2223
use mongodb::bson::Document;
2324
use mongodb::bson::doc;
2425
use mongodb::options::ClientOptions;
@@ -80,6 +81,32 @@ impl MongodbCore {
8081
}
8182
}
8283

84+
pub async fn get_length(&self, path: &str) -> Result<Option<usize>> {
85+
let collection = self.get_collection().await?;
86+
let mut cursor = collection
87+
.aggregate(vec![
88+
doc! { "$match": { self.key_field.as_str(): path } },
89+
doc! { "$limit": 1 },
90+
doc! {
91+
"$project": {
92+
"_id": 0,
93+
"content_length": {
94+
"$binarySize": format!("${}", self.value_field)
95+
}
96+
}
97+
},
98+
])
99+
.await
100+
.map_err(parse_mongodb_error)?;
101+
102+
if !cursor.advance().await.map_err(parse_mongodb_error)? {
103+
return Ok(None);
104+
}
105+
106+
let doc: Document = cursor.deserialize_current().map_err(parse_mongodb_error)?;
107+
parse_bson_usize(doc.get("content_length"))
108+
}
109+
83110
pub async fn set(&self, path: &str, value: Buffer) -> Result<()> {
84111
let collection = self.get_collection().await?;
85112
let filter = doc! { self.key_field.as_str(): path };
@@ -111,3 +138,36 @@ fn parse_mongodb_error(err: mongodb::error::Error) -> Error {
111138
fn parse_bson_error(err: mongodb::bson::document::ValueAccessError) -> Error {
112139
Error::new(ErrorKind::Unexpected, "bson error").set_source(err)
113140
}
141+
142+
fn parse_bson_usize(value: Option<&Bson>) -> Result<Option<usize>> {
143+
let Some(value) = value else {
144+
return Ok(None);
145+
};
146+
147+
match value {
148+
Bson::Int32(v) => (*v).try_into().map(Some).map_err(|err| {
149+
Error::new(ErrorKind::Unexpected, "mongodb value length is invalid").set_source(err)
150+
}),
151+
Bson::Int64(v) => (*v).try_into().map(Some).map_err(|err| {
152+
Error::new(ErrorKind::Unexpected, "mongodb value length is invalid").set_source(err)
153+
}),
154+
_ => Err(Error::new(
155+
ErrorKind::Unexpected,
156+
"mongodb value length is invalid",
157+
)),
158+
}
159+
}
160+
161+
#[cfg(test)]
162+
mod tests {
163+
use super::*;
164+
165+
#[test]
166+
fn test_parse_bson_usize() {
167+
assert_eq!(parse_bson_usize(Some(&Bson::Int32(6))).unwrap(), Some(6));
168+
assert_eq!(parse_bson_usize(Some(&Bson::Int64(6))).unwrap(), Some(6));
169+
assert_eq!(parse_bson_usize(None).unwrap(), None);
170+
assert!(parse_bson_usize(Some(&Bson::Int32(-1))).is_err());
171+
assert!(parse_bson_usize(Some(&Bson::String("6".to_string()))).is_err());
172+
}
173+
}

‎core/services/mysql/src/backend.rs‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -222,10 +222,9 @@ impl Service for MysqlBackend {
222222
if p == build_abs_path(&self.root, "") {
223223
Ok(RpStat::new(Metadata::new(EntryMode::DIR)))
224224
} else {
225-
let bs = self.core.get(&p).await?;
226-
match bs {
227-
Some(bs) => Ok(RpStat::new(
228-
Metadata::new(EntryMode::FILE).with_content_length(bs.len() as u64),
225+
match self.core.get_length(&p).await? {
226+
Some(length) => Ok(RpStat::new(
227+
Metadata::new(EntryMode::FILE).with_content_length(length as u64),
229228
)),
230229
None => Err(Error::new(ErrorKind::NotFound, "kv not found in mysql")),
231230
}

0 commit comments

Comments
 (0)