1+ // @effect -diagnostics nodeBuiltinImport:off -- Builds the zip Azure returns and stands in for az writing it.
2+ import * as NodeFS from "node:fs" ;
3+ import * as NodeZlib from "node:zlib" ;
4+
5+ import * as NodeServices from "@effect/platform-node/NodeServices" ;
16import { afterEach , assert , expect , it , vi } from "@effect/vitest" ;
27import * as Effect from "effect/Effect" ;
38import * as Layer from "effect/Layer" ;
@@ -16,6 +21,7 @@ const layer = it.layer(
1621 execute : mockedExecute ,
1722 } ) ,
1823 ) ,
24+ Layer . provide ( NodeServices . layer ) ,
1925 ) ,
2026) ;
2127
@@ -81,6 +87,52 @@ function pullRequests(count: number, firstNumber: number): string {
8187 return JSON . stringify ( pullRequestRows ( count , firstNumber ) ) ;
8288}
8389
90+ /** Entries as Azure streams them: data descriptors, with sizes only in the central directory. */
91+ function streamedZip (
92+ entries : ReadonlyArray < {
93+ readonly name : string ;
94+ readonly data : Uint8Array ;
95+ readonly stored ?: true ;
96+ } > ,
97+ ) : Buffer {
98+ const parts : Buffer [ ] = [ ] ;
99+ const directory : Buffer [ ] = [ ] ;
100+ let offset = 0 ;
101+ for ( const entry of entries ) {
102+ const name = Buffer . from ( entry . name ) ;
103+ const method = entry . stored ? 0 : 8 ;
104+ const body = entry . stored ? Buffer . from ( entry . data ) : NodeZlib . deflateRawSync ( entry . data ) ;
105+ const local = Buffer . alloc ( 30 ) ;
106+ local . writeUInt32LE ( 0x04034b50 , 0 ) ;
107+ local . writeUInt16LE ( 0x0008 , 6 ) ;
108+ local . writeUInt16LE ( method , 8 ) ;
109+ local . writeUInt16LE ( name . length , 26 ) ;
110+ const descriptor = Buffer . alloc ( 16 ) ;
111+ descriptor . writeUInt32LE ( 0x08074b50 , 0 ) ;
112+ descriptor . writeUInt32LE ( body . length , 8 ) ;
113+ descriptor . writeUInt32LE ( entry . data . length , 12 ) ;
114+ const central = Buffer . alloc ( 46 ) ;
115+ central . writeUInt32LE ( 0x02014b50 , 0 ) ;
116+ central . writeUInt16LE ( 0x0008 , 8 ) ;
117+ central . writeUInt16LE ( method , 10 ) ;
118+ central . writeUInt32LE ( body . length , 20 ) ;
119+ central . writeUInt32LE ( entry . data . length , 24 ) ;
120+ central . writeUInt16LE ( name . length , 28 ) ;
121+ central . writeUInt32LE ( offset , 42 ) ;
122+ parts . push ( local , name , body , descriptor ) ;
123+ directory . push ( central , name ) ;
124+ offset += local . length + name . length + body . length + descriptor . length ;
125+ }
126+ const centralDirectory = Buffer . concat ( directory ) ;
127+ const end = Buffer . alloc ( 22 ) ;
128+ end . writeUInt32LE ( 0x06054b50 , 0 ) ;
129+ end . writeUInt16LE ( entries . length , 8 ) ;
130+ end . writeUInt16LE ( entries . length , 10 ) ;
131+ end . writeUInt32LE ( centralDirectory . length , 12 ) ;
132+ end . writeUInt32LE ( offset , 16 ) ;
133+ return Buffer . concat ( [ ...parts , centralDirectory , end ] ) ;
134+ }
135+
84136function argsOfCall ( index : number ) : ReadonlyArray < string > {
85137 const call = mockedExecute . mock . calls [ index ] ;
86138 assert . isDefined ( call ) ;
@@ -850,6 +902,80 @@ layer("AzureDevOpsPullRequestCli.layer", (it) => {
850902 } ) ,
851903 ) ;
852904
905+ it . effect (
906+ "reads a batch of files from one blobs zip, keeping only what Azure's read would say" ,
907+ ( ) =>
908+ Effect . gen ( function * ( ) {
909+ const id = ( digit : string ) => digit . repeat ( 40 ) ;
910+ const utf8 = ( text : string ) => new TextEncoder ( ) . encode ( text ) ;
911+ let sent = "" ;
912+ mockedExecute . mockImplementationOnce ( ( input ) =>
913+ Effect . sync ( ( ) => {
914+ const flag = ( name : string ) => input . args [ input . args . indexOf ( name ) + 1 ] ?? "" ;
915+ sent = NodeFS . readFileSync ( flag ( "--in-file" ) , "utf8" ) ;
916+ NodeFS . writeFileSync (
917+ flag ( "--out-file" ) ,
918+ streamedZip ( [
919+ { name : id ( "a" ) , data : utf8 ( "plain text\n" ) } ,
920+ { name : id ( "b" ) , data : utf8 ( "\uFEFFcafé ✓\n" ) } ,
921+ { name : id ( "c" ) , data : Uint8Array . of ( 0x89 , 0x50 , 0x00 , 0x01 ) } ,
922+ // Windows-1252 "café" and UTF-16: Azure decodes these itself.
923+ { name : id ( "d" ) , data : Uint8Array . of ( 0x63 , 0x61 , 0x66 , 0xe9 ) } ,
924+ { name : id ( "e" ) , data : Uint8Array . of ( 0xff , 0xfe , 0x68 , 0x00 ) } ,
925+ { name : id ( "f" ) , data : utf8 ( "stored\n" ) , stored : true } ,
926+ { name : id ( "9" ) , data : utf8 ( "never asked for\n" ) } ,
927+ ] ) ,
928+ ) ;
929+ return output ( "" ) ;
930+ } ) ,
931+ ) ;
932+ const cli = yield * AzureDevOpsPullRequestCli . AzureDevOpsPullRequestCli ;
933+
934+ const blobs = yield * cli . readBlobs ( {
935+ cwd : "/w" ,
936+ location : { project : "platform" , repository : "web" } ,
937+ objectIds : [ id ( "A" ) , id ( "b" ) , id ( "C" ) , id ( "d" ) , id ( "e" ) , id ( "f" ) , id ( "a" ) , "../etc" ] ,
938+ } ) ;
939+
940+ expect ( JSON . parse ( sent ) ) . toEqual ( [ id ( "A" ) , id ( "b" ) , id ( "C" ) , id ( "d" ) , id ( "e" ) , id ( "f" ) ] ) ;
941+ expect ( argsOfCall ( 0 ) ) . toEqual (
942+ expect . arrayContaining ( [ "blobs" , "POST" , "application/zip" , "project=platform" ] ) ,
943+ ) ;
944+ expect ( Object . fromEntries ( blobs ) ) . toEqual ( {
945+ [ id ( "a" ) ] : { contents : "plain text\n" , isBinary : false } ,
946+ [ id ( "b" ) ] : { contents : "café ✓\n" , isBinary : false } ,
947+ [ id ( "c" ) ] : { contents : "" , isBinary : true } ,
948+ [ id ( "f" ) ] : { contents : "stored\n" , isBinary : false } ,
949+ } ) ;
950+ } ) ,
951+ ) ;
952+
953+ it . effect ( "leaves blobs past the batch's total size to per-file reads" , ( ) =>
954+ Effect . gen ( function * ( ) {
955+ const ids = [ "1" , "2" , "3" , "4" , "5" , "6" ] . map ( ( digit ) => digit . repeat ( 40 ) ) ;
956+ // Compresses to a few KiB each, so only the inflated total can bound memory.
957+ const blob = new Uint8Array ( 3 * 1024 * 1024 ) . fill ( 0x61 ) ;
958+ mockedExecute . mockImplementationOnce ( ( input ) =>
959+ Effect . sync ( ( ) => {
960+ NodeFS . writeFileSync (
961+ input . args [ input . args . indexOf ( "--out-file" ) + 1 ] ?? "" ,
962+ streamedZip ( ids . map ( ( name ) => ( { name, data : blob } ) ) ) ,
963+ ) ;
964+ return output ( "" ) ;
965+ } ) ,
966+ ) ;
967+ const cli = yield * AzureDevOpsPullRequestCli . AzureDevOpsPullRequestCli ;
968+
969+ const blobs = yield * cli . readBlobs ( {
970+ cwd : "/w" ,
971+ location : { project : "platform" , repository : "web" } ,
972+ objectIds : ids ,
973+ } ) ;
974+
975+ expect ( [ ...blobs . keys ( ) ] ) . toEqual ( ids . slice ( 0 , 5 ) ) ;
976+ } ) ,
977+ ) ;
978+
853979 it . effect ( "escapes characters in a file path that would cut the query string" , ( ) =>
854980 Effect . gen ( function * ( ) {
855981 mockedExecute . mockReturnValueOnce ( Effect . succeed ( output ( json ( { content : "x" } ) ) ) ) ;
0 commit comments