@@ -64,8 +64,11 @@ const AZURE_FEDERATED_TOKEN_FILE: &str = "AZURE_FEDERATED_TOKEN_FILE";
6464/// The account name of the Azurite emulator.
6565const EMULATOR_ACCOUNT : & str = "devstoreaccount1" ;
6666
67- /// The publicly documented account key of the Azurite emulator's
68- /// [EMULATOR_ACCOUNT].
67+ /// The account key of the Azurite emulator's [EMULATOR_ACCOUNT].
68+ ///
69+ /// NOTE: This is not a secret. Azurite ships with this key, Microsoft
70+ /// publishes it in the Azurite documentation, and it grants access only to
71+ /// local emulator instances.
6972const EMULATOR_ACCOUNT_KEY : & str =
7073 "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" ;
7174
@@ -800,19 +803,23 @@ impl Blob for AzureBlob {
800803 let Some ( first) = download_range ( & blob, 0 , None ) . await ? else {
801804 return Ok ( None ) ;
802805 } ;
803- let content_length = download_total_len ( & first. headers ) ;
806+ // Without the blob's length there is no telling which ranges remain,
807+ // and returning only the first would silently truncate the blob.
808+ let content_length = download_total_len ( & first. headers )
809+ . ok_or_else ( || anyhow ! ( "azure blob get error: no length for blob {path}" ) ) ?;
804810 // Pin the remaining ranges to the version the first range read.
805811 let etag = first. properties . etag . clone ( ) ;
806- let partition_size = u64:: cast_from ( GET_PARTITION_SIZE . get ( ) ) ;
807- let offsets =
808- ( partition_size..content_length. unwrap_or ( 0 ) ) . step_by ( GET_PARTITION_SIZE . get ( ) ) ;
809812
810813 let mut segments = SegmentedBytes :: new ( ) ;
811814 let mut total_len: u64 = 0 ;
812815 for part in read_body ( first. body ) . await ? {
813816 total_len += u64:: cast_from ( part. len ( ) ) ;
814817 segments. push ( part) ;
815818 }
819+ // Continue after the bytes the first response delivered rather than
820+ // after one partition, so a response that ignored the requested range
821+ // and returned more is not read twice.
822+ let offsets = ( total_len..content_length) . step_by ( GET_PARTITION_SIZE . get ( ) ) ;
816823 let mut rest = futures_util:: stream:: iter ( offsets)
817824 . map ( |offset| {
818825 let ( blob, etag) = ( & blob, etag. clone ( ) ) ;
@@ -836,7 +843,7 @@ impl Blob for AzureBlob {
836843
837844 // Report if the content-length header didn't match the number of
838845 // bytes we read from the network.
839- if content_length != Some ( total_len) {
846+ if content_length != total_len {
840847 self . metrics . get_invalid_resp . inc ( ) ;
841848 }
842849
@@ -991,17 +998,8 @@ mod tests {
991998 }
992999 }
9931000
994- /// `HedgedBlob` cancels the losing leg of a race by dropping its get, so
995- /// dropping a get must drop every request it has in flight.
996- #[ mz_ore:: test( tokio:: test) ]
997- #[ cfg_attr( miri, ignore) ] // too slow
998- async fn azure_blob_get_drop_cancels_requests ( ) {
999- let started = Arc :: new ( AtomicUsize :: new ( 0 ) ) ;
1000- let dropped = Arc :: new ( AtomicUsize :: new ( 0 ) ) ;
1001- let http_client = HangingRangesClient {
1002- started : Arc :: clone ( & started) ,
1003- dropped : Arc :: clone ( & dropped) ,
1004- } ;
1001+ /// Returns an [AzureBlob] whose requests `http_client` serves.
1002+ fn mock_blob ( http_client : impl HttpClient + ' static ) -> AzureBlob {
10051003 let client = BlobContainerClient :: new (
10061004 Url :: parse ( "https://account.blob.core.windows.net/container" ) . expect ( "valid url" ) ,
10071005 None ,
@@ -1014,11 +1012,90 @@ mod tests {
10141012 } ) ,
10151013 )
10161014 . expect ( "valid client" ) ;
1017- let blob = AzureBlob {
1015+ AzureBlob {
10181016 metrics : S3BlobMetrics :: new ( & MetricsRegistry :: new ( ) ) ,
10191017 client : Arc :: new ( client) ,
10201018 prefix : "prefix" . to_string ( ) ,
1019+ }
1020+ }
1021+
1022+ /// An [HttpClient] that ignores the requested range and answers every
1023+ /// request with the whole blob of three partitions.
1024+ #[ derive( Debug ) ]
1025+ struct FullBodyClient {
1026+ /// Whether responses carry a `Content-Length`.
1027+ with_length : bool ,
1028+ /// Number of requests served.
1029+ requests : Arc < AtomicUsize > ,
1030+ }
1031+
1032+ #[ async_trait]
1033+ impl HttpClient for FullBodyClient {
1034+ async fn execute_request (
1035+ & self ,
1036+ _request : & Request ,
1037+ ) -> azure_core:: Result < AsyncRawResponse > {
1038+ self . requests . fetch_add ( 1 , Ordering :: SeqCst ) ;
1039+ let len = 3 * GET_PARTITION_SIZE . get ( ) ;
1040+ let mut headers = Headers :: new ( ) ;
1041+ if self . with_length {
1042+ headers. insert ( "content-length" , len. to_string ( ) ) ;
1043+ }
1044+ headers. insert ( "etag" , "\" v1\" " ) ;
1045+ Ok ( AsyncRawResponse :: from_bytes (
1046+ StatusCode :: Ok ,
1047+ headers,
1048+ vec ! [ 7u8 ; len] ,
1049+ ) )
1050+ }
1051+ }
1052+
1053+ #[ mz_ore:: test( tokio:: test) ]
1054+ #[ cfg_attr( miri, ignore) ] // too slow
1055+ async fn azure_blob_get_full_body_response ( ) {
1056+ let requests = Arc :: new ( AtomicUsize :: new ( 0 ) ) ;
1057+ let blob = mock_blob ( FullBodyClient {
1058+ with_length : true ,
1059+ requests : Arc :: clone ( & requests) ,
1060+ } ) ;
1061+ let value = blob
1062+ . get ( "key" )
1063+ . await
1064+ . expect ( "get" )
1065+ . expect ( "blob exists" )
1066+ . into_contiguous ( ) ;
1067+ // Compared piecewise so a failure does not print megabytes.
1068+ assert_eq ! ( value. len( ) , 3 * GET_PARTITION_SIZE . get( ) ) ;
1069+ assert ! ( value. iter( ) . all( |b| * b == 7 ) , "unexpected blob contents" ) ;
1070+ assert_eq ! ( requests. load( Ordering :: SeqCst ) , 1 ) ;
1071+ assert_eq ! ( blob. metrics. get_invalid_resp. get( ) , 0 ) ;
1072+ }
1073+
1074+ #[ mz_ore:: test( tokio:: test) ]
1075+ #[ cfg_attr( miri, ignore) ] // too slow
1076+ async fn azure_blob_get_without_length_fails ( ) {
1077+ let blob = mock_blob ( FullBodyClient {
1078+ with_length : false ,
1079+ requests : Arc :: new ( AtomicUsize :: new ( 0 ) ) ,
1080+ } ) ;
1081+ let Err ( err) = blob. get ( "key" ) . await else {
1082+ panic ! ( "get without length succeeded" ) ;
1083+ } ;
1084+ assert ! ( err. to_string( ) . contains( "no length" ) , "{err}" ) ;
1085+ }
1086+
1087+ /// `HedgedBlob` cancels the losing leg of a race by dropping its get, so
1088+ /// dropping a get must drop every request it has in flight.
1089+ #[ mz_ore:: test( tokio:: test) ]
1090+ #[ cfg_attr( miri, ignore) ] // too slow
1091+ async fn azure_blob_get_drop_cancels_requests ( ) {
1092+ let started = Arc :: new ( AtomicUsize :: new ( 0 ) ) ;
1093+ let dropped = Arc :: new ( AtomicUsize :: new ( 0 ) ) ;
1094+ let http_client = HangingRangesClient {
1095+ started : Arc :: clone ( & started) ,
1096+ dropped : Arc :: clone ( & dropped) ,
10211097 } ;
1098+ let blob = mock_blob ( http_client) ;
10221099
10231100 // Drive the get until both hanging partitions are requested, then
10241101 // drop it.
0 commit comments