Skip to content

Commit f44c7f9

Browse files
antiguruclaude
andcommitted
persist: fail Azure gets without a blob length
A first range response without a length left the get unable to tell which ranges remain, and it returned only the first partition. Fail such gets instead. The remaining ranges now start after the bytes the first response delivered, so a response that ignores the requested range is not read twice. Also spell out that the emulator account key is not a secret. Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 4c1f19b commit f44c7f9

1 file changed

Lines changed: 96 additions & 19 deletions

File tree

‎src/persist/src/azure.rs‎

Lines changed: 96 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,11 @@ const AZURE_FEDERATED_TOKEN_FILE: &str = "AZURE_FEDERATED_TOKEN_FILE";
6464
/// The account name of the Azurite emulator.
6565
const EMULATOR_ACCOUNT: &str = "devstoreaccount1";
6666

67-
/// The publicly documented account key of the Azurite emulator's
68-
/// [EMULATOR_ACCOUNT].
67+
/// The account key of the Azurite emulator's [EMULATOR_ACCOUNT].
68+
///
69+
/// NOTE: This is not a secret. Azurite ships with this key, Microsoft
70+
/// publishes it in the Azurite documentation, and it grants access only to
71+
/// local emulator instances.
6972
const EMULATOR_ACCOUNT_KEY: &str =
7073
"Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==";
7174

@@ -800,19 +803,23 @@ impl Blob for AzureBlob {
800803
let Some(first) = download_range(&blob, 0, None).await? else {
801804
return Ok(None);
802805
};
803-
let content_length = download_total_len(&first.headers);
806+
// Without the blob's length there is no telling which ranges remain,
807+
// and returning only the first would silently truncate the blob.
808+
let content_length = download_total_len(&first.headers)
809+
.ok_or_else(|| anyhow!("azure blob get error: no length for blob {path}"))?;
804810
// Pin the remaining ranges to the version the first range read.
805811
let etag = first.properties.etag.clone();
806-
let partition_size = u64::cast_from(GET_PARTITION_SIZE.get());
807-
let offsets =
808-
(partition_size..content_length.unwrap_or(0)).step_by(GET_PARTITION_SIZE.get());
809812

810813
let mut segments = SegmentedBytes::new();
811814
let mut total_len: u64 = 0;
812815
for part in read_body(first.body).await? {
813816
total_len += u64::cast_from(part.len());
814817
segments.push(part);
815818
}
819+
// Continue after the bytes the first response delivered rather than
820+
// after one partition, so a response that ignored the requested range
821+
// and returned more is not read twice.
822+
let offsets = (total_len..content_length).step_by(GET_PARTITION_SIZE.get());
816823
let mut rest = futures_util::stream::iter(offsets)
817824
.map(|offset| {
818825
let (blob, etag) = (&blob, etag.clone());
@@ -836,7 +843,7 @@ impl Blob for AzureBlob {
836843

837844
// Report if the content-length header didn't match the number of
838845
// bytes we read from the network.
839-
if content_length != Some(total_len) {
846+
if content_length != total_len {
840847
self.metrics.get_invalid_resp.inc();
841848
}
842849

@@ -991,17 +998,8 @@ mod tests {
991998
}
992999
}
9931000

994-
/// `HedgedBlob` cancels the losing leg of a race by dropping its get, so
995-
/// dropping a get must drop every request it has in flight.
996-
#[mz_ore::test(tokio::test)]
997-
#[cfg_attr(miri, ignore)] // too slow
998-
async fn azure_blob_get_drop_cancels_requests() {
999-
let started = Arc::new(AtomicUsize::new(0));
1000-
let dropped = Arc::new(AtomicUsize::new(0));
1001-
let http_client = HangingRangesClient {
1002-
started: Arc::clone(&started),
1003-
dropped: Arc::clone(&dropped),
1004-
};
1001+
/// Returns an [AzureBlob] whose requests `http_client` serves.
1002+
fn mock_blob(http_client: impl HttpClient + 'static) -> AzureBlob {
10051003
let client = BlobContainerClient::new(
10061004
Url::parse("https://account.blob.core.windows.net/container").expect("valid url"),
10071005
None,
@@ -1014,11 +1012,90 @@ mod tests {
10141012
}),
10151013
)
10161014
.expect("valid client");
1017-
let blob = AzureBlob {
1015+
AzureBlob {
10181016
metrics: S3BlobMetrics::new(&MetricsRegistry::new()),
10191017
client: Arc::new(client),
10201018
prefix: "prefix".to_string(),
1019+
}
1020+
}
1021+
1022+
/// An [HttpClient] that ignores the requested range and answers every
1023+
/// request with the whole blob of three partitions.
1024+
#[derive(Debug)]
1025+
struct FullBodyClient {
1026+
/// Whether responses carry a `Content-Length`.
1027+
with_length: bool,
1028+
/// Number of requests served.
1029+
requests: Arc<AtomicUsize>,
1030+
}
1031+
1032+
#[async_trait]
1033+
impl HttpClient for FullBodyClient {
1034+
async fn execute_request(
1035+
&self,
1036+
_request: &Request,
1037+
) -> azure_core::Result<AsyncRawResponse> {
1038+
self.requests.fetch_add(1, Ordering::SeqCst);
1039+
let len = 3 * GET_PARTITION_SIZE.get();
1040+
let mut headers = Headers::new();
1041+
if self.with_length {
1042+
headers.insert("content-length", len.to_string());
1043+
}
1044+
headers.insert("etag", "\"v1\"");
1045+
Ok(AsyncRawResponse::from_bytes(
1046+
StatusCode::Ok,
1047+
headers,
1048+
vec![7u8; len],
1049+
))
1050+
}
1051+
}
1052+
1053+
#[mz_ore::test(tokio::test)]
1054+
#[cfg_attr(miri, ignore)] // too slow
1055+
async fn azure_blob_get_full_body_response() {
1056+
let requests = Arc::new(AtomicUsize::new(0));
1057+
let blob = mock_blob(FullBodyClient {
1058+
with_length: true,
1059+
requests: Arc::clone(&requests),
1060+
});
1061+
let value = blob
1062+
.get("key")
1063+
.await
1064+
.expect("get")
1065+
.expect("blob exists")
1066+
.into_contiguous();
1067+
// Compared piecewise so a failure does not print megabytes.
1068+
assert_eq!(value.len(), 3 * GET_PARTITION_SIZE.get());
1069+
assert!(value.iter().all(|b| *b == 7), "unexpected blob contents");
1070+
assert_eq!(requests.load(Ordering::SeqCst), 1);
1071+
assert_eq!(blob.metrics.get_invalid_resp.get(), 0);
1072+
}
1073+
1074+
#[mz_ore::test(tokio::test)]
1075+
#[cfg_attr(miri, ignore)] // too slow
1076+
async fn azure_blob_get_without_length_fails() {
1077+
let blob = mock_blob(FullBodyClient {
1078+
with_length: false,
1079+
requests: Arc::new(AtomicUsize::new(0)),
1080+
});
1081+
let Err(err) = blob.get("key").await else {
1082+
panic!("get without length succeeded");
1083+
};
1084+
assert!(err.to_string().contains("no length"), "{err}");
1085+
}
1086+
1087+
/// `HedgedBlob` cancels the losing leg of a race by dropping its get, so
1088+
/// dropping a get must drop every request it has in flight.
1089+
#[mz_ore::test(tokio::test)]
1090+
#[cfg_attr(miri, ignore)] // too slow
1091+
async fn azure_blob_get_drop_cancels_requests() {
1092+
let started = Arc::new(AtomicUsize::new(0));
1093+
let dropped = Arc::new(AtomicUsize::new(0));
1094+
let http_client = HangingRangesClient {
1095+
started: Arc::clone(&started),
1096+
dropped: Arc::clone(&dropped),
10211097
};
1098+
let blob = mock_blob(http_client);
10221099

10231100
// Drive the get until both hanging partitions are requested, then
10241101
// drop it.

0 commit comments

Comments
 (0)