Skip to content

Commit d3beb60

Browse files
stephentoubcarlossanlop
stephentoub
and
carlossanlop
authored
Implement ZLibStream and fix SocketsHttpHandler deflate support (#42717)
* Implement ZLibStream and fix SocketsHttpHandler deflate support - Implements ZLibStream, exposes it in the ref, and add tests - Fixes SocketsHttpHandler to use ZLibStream instead of DeflateStream * Add comment about deflate content encoding * Apply suggestions from code review Co-authored-by: Carlos Sanchez <[email protected]> * Fix netfx build Co-authored-by: Carlos Sanchez <[email protected]>
1 parent c5b6881 commit d3beb60

File tree

16 files changed

+520
-234
lines changed

16 files changed

+520
-234
lines changed

‎src/libraries/Common/tests/System/IO/Compression/CompressionStreamUnitTestBase.cs‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1254,6 +1254,59 @@ await Task.WhenAll(Enumerable.Range(0, ParallelOperations).Select(_ => Task.Run(
12541254
Assert.Equal(sourceData, decompressedStream.ToArray());
12551255
})));
12561256
}
1257+
1258+
[Fact]
1259+
public void Precancellation()
1260+
{
1261+
var ms = new MemoryStream();
1262+
using (Stream compressor = CreateStream(ms, CompressionMode.Compress, leaveOpen: true))
1263+
{
1264+
Assert.True(compressor.WriteAsync(new byte[1], 0, 1, new CancellationToken(true)).IsCanceled);
1265+
Assert.True(compressor.FlushAsync(new CancellationToken(true)).IsCanceled);
1266+
}
1267+
using (Stream decompressor = CreateStream(ms, CompressionMode.Decompress, leaveOpen: true))
1268+
{
1269+
Assert.True(decompressor.ReadAsync(new byte[1], 0, 1, new CancellationToken(true)).IsCanceled);
1270+
}
1271+
}
1272+
1273+
[Theory]
1274+
[InlineData(false)]
1275+
[InlineData(true)]
1276+
public async Task DisposeAsync_Flushes(bool leaveOpen)
1277+
{
1278+
var ms = new MemoryStream();
1279+
var cs = CreateStream(ms, CompressionMode.Compress, leaveOpen);
1280+
cs.WriteByte(1);
1281+
await cs.FlushAsync();
1282+
1283+
long pos = ms.Position;
1284+
cs.WriteByte(1);
1285+
Assert.Equal(pos, ms.Position);
1286+
1287+
await cs.DisposeAsync();
1288+
Assert.InRange(ms.ToArray().Length, pos + 1, int.MaxValue);
1289+
if (leaveOpen)
1290+
{
1291+
Assert.InRange(ms.Position, pos + 1, int.MaxValue);
1292+
}
1293+
else
1294+
{
1295+
Assert.Throws<ObjectDisposedException>(() => ms.Position);
1296+
}
1297+
}
1298+
1299+
[Theory]
1300+
[InlineData(false)]
1301+
[InlineData(true)]
1302+
public async Task DisposeAsync_MultipleCallsAllowed(bool leaveOpen)
1303+
{
1304+
using (var cs = CreateStream(new MemoryStream(), CompressionMode.Compress, leaveOpen))
1305+
{
1306+
await cs.DisposeAsync();
1307+
await cs.DisposeAsync();
1308+
}
1309+
}
12571310
}
12581311

12591312
internal sealed class BadWrappedStream : MemoryStream

‎src/libraries/Common/tests/System/Net/Http/HttpClientHandlerTest.Decompression.cs‎

Lines changed: 65 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -33,46 +33,55 @@ public static IEnumerable<object[]> RemoteServersAndCompressionUris()
3333
foreach (Configuration.Http.RemoteServer remoteServer in Configuration.Http.RemoteServers)
3434
{
3535
yield return new object[] { remoteServer, remoteServer.GZipUri };
36-
yield return new object[] { remoteServer, remoteServer.DeflateUri };
36+
37+
// Remote deflate endpoint isn't correctly following the deflate protocol.
38+
//yield return new object[] { remoteServer, remoteServer.DeflateUri };
3739
}
3840
}
3941

40-
public static IEnumerable<object[]> DecompressedResponse_MethodSpecified_DecompressedContentReturned_MemberData()
42+
[Theory]
43+
[InlineData("gzip", false)]
44+
[InlineData("gzip", true)]
45+
[InlineData("deflate", false)]
46+
[InlineData("deflate", true)]
47+
[InlineData("br", false)]
48+
[InlineData("br", true)]
49+
public async Task DecompressedResponse_MethodSpecified_DecompressedContentReturned(string encodingName, bool all)
4150
{
42-
foreach (bool specifyAllMethods in new[] { false, true })
51+
Func<Stream, Stream> compress;
52+
DecompressionMethods methods;
53+
switch (encodingName)
4354
{
44-
yield return new object[]
45-
{
46-
"deflate",
47-
new Func<Stream, Stream>(s => new DeflateStream(s, CompressionLevel.Optimal, leaveOpen: true)),
48-
specifyAllMethods ? DecompressionMethods.Deflate : _all
49-
};
50-
yield return new object[]
51-
{
52-
"gzip",
53-
new Func<Stream, Stream>(s => new GZipStream(s, CompressionLevel.Optimal, leaveOpen: true)),
54-
specifyAllMethods ? DecompressionMethods.GZip : _all
55-
};
55+
case "gzip":
56+
compress = s => new GZipStream(s, CompressionLevel.Optimal, leaveOpen: true);
57+
methods = all ? DecompressionMethods.GZip : _all;
58+
break;
59+
5660
#if !NETFRAMEWORK
57-
yield return new object[]
58-
{
59-
"br",
60-
new Func<Stream, Stream>(s => new BrotliStream(s, CompressionLevel.Optimal, leaveOpen: true)),
61-
specifyAllMethods ? DecompressionMethods.Brotli : _all
62-
};
61+
case "br":
62+
if (IsWinHttpHandler)
63+
{
64+
// Brotli only supported on SocketsHttpHandler.
65+
return;
66+
}
67+
68+
compress = s => new BrotliStream(s, CompressionLevel.Optimal, leaveOpen: true);
69+
methods = all ? DecompressionMethods.Brotli : _all;
70+
break;
71+
72+
case "deflate":
73+
// WinHttpHandler continues to use DeflateStream as it doesn't have a newer build than netstandard2.0
74+
// and doesn't have access to ZLibStream.
75+
compress = IsWinHttpHandler ?
76+
new Func<Stream, Stream>(s => new DeflateStream(s, CompressionLevel.Optimal, leaveOpen: true)) :
77+
new Func<Stream, Stream>(s => new ZLibStream(s, CompressionLevel.Optimal, leaveOpen: true));
78+
methods = all ? DecompressionMethods.Deflate : _all;
79+
break;
6380
#endif
64-
}
65-
}
6681

67-
[Theory]
68-
[MemberData(nameof(DecompressedResponse_MethodSpecified_DecompressedContentReturned_MemberData))]
69-
public async Task DecompressedResponse_MethodSpecified_DecompressedContentReturned(
70-
string encodingName, Func<Stream, Stream> compress, DecompressionMethods methods)
71-
{
72-
// Brotli only supported on SocketsHttpHandler.
73-
if (IsWinHttpHandler && encodingName == "br")
74-
{
75-
return;
82+
default:
83+
Assert.Contains(encodingName, new[] { "br", "deflate", "gzip" });
84+
return;
7685
}
7786

7887
var expectedContent = new byte[12345];
@@ -104,15 +113,15 @@ public static IEnumerable<object[]> DecompressedResponse_MethodNotSpecified_Orig
104113
{
105114
yield return new object[]
106115
{
107-
"deflate",
108-
new Func<Stream, Stream>(s => new DeflateStream(s, CompressionLevel.Optimal, leaveOpen: true)),
116+
"gzip",
117+
new Func<Stream, Stream>(s => new GZipStream(s, CompressionLevel.Optimal, leaveOpen: true)),
109118
DecompressionMethods.None
110119
};
111120
#if !NETFRAMEWORK
112121
yield return new object[]
113122
{
114-
"gzip",
115-
new Func<Stream, Stream>(s => new GZipStream(s, CompressionLevel.Optimal, leaveOpen: true)),
123+
"deflate",
124+
new Func<Stream, Stream>(s => new ZLibStream(s, CompressionLevel.Optimal, leaveOpen: true)),
116125
DecompressionMethods.Brotli
117126
};
118127
yield return new object[]
@@ -186,6 +195,26 @@ public async Task GetAsync_SetAutomaticDecompression_ContentDecompressed(Configu
186195
}
187196
}
188197

198+
// The remote server endpoint was written to use DeflateStream, which isn't actually a correct
199+
// implementation of the deflate protocol (the deflate protocol requires the zlib wrapper around
200+
// deflate). Until we can get that updated (and deal with previous releases still testing it
201+
// via a DeflateStream-based implementation), we utilize httpbin.org to help validate behavior.
202+
[OuterLoop("Uses external servers")]
203+
[Theory]
204+
[InlineData("http://httpbin.org/deflate", "\"deflated\": true")]
205+
[InlineData("https://httpbin.org/deflate", "\"deflated\": true")]
206+
[InlineData("http://httpbin.org/gzip", "\"gzipped\": true")]
207+
[InlineData("https://httpbin.org/gzip", "\"gzipped\": true")]
208+
public async Task GetAsync_SetAutomaticDecompression_ContentDecompressed(string uri, string expectedContent)
209+
{
210+
HttpClientHandler handler = CreateHttpClientHandler();
211+
handler.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
212+
using (HttpClient client = CreateHttpClient(handler))
213+
{
214+
Assert.Contains(expectedContent, await client.GetStringAsync(uri));
215+
}
216+
}
217+
189218
[OuterLoop("Uses external server")]
190219
[Theory, MemberData(nameof(RemoteServersAndCompressionUris))]
191220
public async Task GetAsync_SetAutomaticDecompression_HeadersRemoved(Configuration.Http.RemoteServer remoteServer, Uri uri)

‎src/libraries/System.IO.Compression.Brotli/tests/CompressionStreamUnitTests.Brotli.cs‎

Lines changed: 0 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -23,42 +23,6 @@ public class BrotliStreamUnitTests : CompressionStreamUnitTestBase
2323

2424
protected override string CompressedTestFile(string uncompressedPath) => Path.Combine("BrotliTestData", Path.GetFileName(uncompressedPath) + ".br");
2525

26-
[Fact]
27-
public void Precancellation()
28-
{
29-
var ms = new MemoryStream();
30-
using (Stream compressor = new BrotliStream(ms, CompressionMode.Compress, leaveOpen: true))
31-
{
32-
Assert.True(compressor.WriteAsync(new byte[1], 0, 1, new CancellationToken(true)).IsCanceled);
33-
Assert.True(compressor.FlushAsync(new CancellationToken(true)).IsCanceled);
34-
}
35-
using (Stream decompressor = CreateStream(ms, CompressionMode.Decompress, leaveOpen: true))
36-
{
37-
Assert.True(decompressor.ReadAsync(new byte[1], 0, 1, new CancellationToken(true)).IsCanceled);
38-
}
39-
}
40-
41-
[Theory]
42-
[InlineData(false)]
43-
[InlineData(true)]
44-
public async Task DisposeAsync_Flushes(bool leaveOpen)
45-
{
46-
var ms = new MemoryStream();
47-
var bs = new BrotliStream(ms, CompressionMode.Compress, leaveOpen);
48-
bs.WriteByte(1);
49-
Assert.Equal(0, ms.Position);
50-
await bs.DisposeAsync();
51-
Assert.InRange(ms.ToArray().Length, 1, int.MaxValue);
52-
if (leaveOpen)
53-
{
54-
Assert.InRange(ms.Position, 1, int.MaxValue);
55-
}
56-
else
57-
{
58-
Assert.Throws<ObjectDisposedException>(() => ms.Position);
59-
}
60-
}
61-
6226
[Fact]
6327
[OuterLoop("Test takes ~6 seconds to run")]
6428
public override void FlushAsync_DuringWriteAsync() { base.FlushAsync_DuringWriteAsync(); }

‎src/libraries/System.IO.Compression/ref/System.IO.Compression.cs‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,4 +121,39 @@ public enum ZipArchiveMode
121121
Create = 1,
122122
Update = 2,
123123
}
124+
public sealed partial class ZLibStream : System.IO.Stream
125+
{
126+
public ZLibStream(System.IO.Stream stream, System.IO.Compression.CompressionLevel compressionLevel) { }
127+
public ZLibStream(System.IO.Stream stream, System.IO.Compression.CompressionLevel compressionLevel, bool leaveOpen) { }
128+
public ZLibStream(System.IO.Stream stream, System.IO.Compression.CompressionMode mode) { }
129+
public ZLibStream(System.IO.Stream stream, System.IO.Compression.CompressionMode mode, bool leaveOpen) { }
130+
public System.IO.Stream BaseStream { get { throw null; } }
131+
public override bool CanRead { get { throw null; } }
132+
public override bool CanSeek { get { throw null; } }
133+
public override bool CanWrite { get { throw null; } }
134+
public override long Length { get { throw null; } }
135+
public override long Position { get { throw null; } set { } }
136+
public override System.IAsyncResult BeginRead(byte[] array, int offset, int count, System.AsyncCallback? asyncCallback, object? asyncState) { throw null; }
137+
public override System.IAsyncResult BeginWrite(byte[] array, int offset, int count, System.AsyncCallback? asyncCallback, object? asyncState) { throw null; }
138+
public override void CopyTo(System.IO.Stream destination, int bufferSize) { }
139+
public override System.Threading.Tasks.Task CopyToAsync(System.IO.Stream destination, int bufferSize, System.Threading.CancellationToken cancellationToken) { throw null; }
140+
protected override void Dispose(bool disposing) { }
141+
public override System.Threading.Tasks.ValueTask DisposeAsync() { throw null; }
142+
public override int EndRead(System.IAsyncResult asyncResult) { throw null; }
143+
public override void EndWrite(System.IAsyncResult asyncResult) { }
144+
public override void Flush() { }
145+
public override System.Threading.Tasks.Task FlushAsync(System.Threading.CancellationToken cancellationToken) { throw null; }
146+
public override int Read(byte[] array, int offset, int count) { throw null; }
147+
public override int Read(System.Span<byte> buffer) { throw null; }
148+
public override System.Threading.Tasks.Task<int> ReadAsync(byte[] array, int offset, int count, System.Threading.CancellationToken cancellationToken) { throw null; }
149+
public override System.Threading.Tasks.ValueTask<int> ReadAsync(System.Memory<byte> buffer, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
150+
public override int ReadByte() { throw null; }
151+
public override long Seek(long offset, System.IO.SeekOrigin origin) { throw null; }
152+
public override void SetLength(long value) { }
153+
public override void Write(byte[] array, int offset, int count) { }
154+
public override void Write(System.ReadOnlySpan<byte> buffer) { }
155+
public override void WriteByte(byte value) { }
156+
public override System.Threading.Tasks.Task WriteAsync(byte[] array, int offset, int count, System.Threading.CancellationToken cancellationToken) { throw null; }
157+
public override System.Threading.Tasks.ValueTask WriteAsync(System.ReadOnlyMemory<byte> buffer, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
158+
}
124159
}

‎src/libraries/System.IO.Compression/src/System.IO.Compression.csproj‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
<Project Sdk="Microsoft.NET.Sdk">
1+
<Project Sdk="Microsoft.NET.Sdk">
22
<PropertyGroup>
33
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
44
<TargetFrameworks>$(NetCoreAppCurrent)-Windows_NT;$(NetCoreAppCurrent)-Unix;$(NetCoreAppCurrent)-Browser</TargetFrameworks>
@@ -33,6 +33,7 @@
3333
<Compile Include="System\IO\Compression\Crc32Helper.ZLib.cs" />
3434
<Compile Include="System\IO\Compression\GZipStream.cs" />
3535
<Compile Include="System\IO\Compression\PositionPreservingWriteOnlyStreamWrapper.cs" />
36+
<Compile Include="System\IO\Compression\ZLibStream.cs" />
3637
<Compile Include="$(CommonPath)System\IO\StreamHelpers.CopyValidation.cs"
3738
Link="Common\System\IO\StreamHelpers.CopyValidation.cs" />
3839
<Compile Include="$(CommonPath)System\Threading\Tasks\TaskToApm.cs"

‎src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateManaged/DeflateManagedStream.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,7 @@ private void EnsureNotDisposed()
162162

163163
private static void ThrowStreamClosedException()
164164
{
165-
throw new ObjectDisposedException(null, SR.ObjectDisposed_StreamClosed);
165+
throw new ObjectDisposedException(nameof(DeflateStream), SR.ObjectDisposed_StreamClosed);
166166
}
167167

168168
public override IAsyncResult BeginRead(byte[] buffer, int offset, int count, AsyncCallback? asyncCallback, object? asyncState) =>

‎src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateZLib/DeflateStream.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -320,7 +320,7 @@ private void EnsureNotDisposed()
320320

321321
private static void ThrowStreamClosedException()
322322
{
323-
throw new ObjectDisposedException(null, SR.ObjectDisposed_StreamClosed);
323+
throw new ObjectDisposedException(nameof(DeflateStream), SR.ObjectDisposed_StreamClosed);
324324
}
325325

326326
private void EnsureDecompressionMode()

‎src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateZLib/ZLibNative.cs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,14 @@ public enum CompressionMethod : int
114114
public const int Deflate_DefaultWindowBits = -15; // Legal values are 8..15 and -8..-15. 15 is the window size,
115115
// negative val causes deflate to produce raw deflate data (no zlib header).
116116

117+
/// <summary>
118+
/// <p><strong>From the ZLib manual:</strong></p>
119+
/// <p>ZLib's <code>windowBits</code> parameter is the base two logarithm of the window size (the size of the history buffer).
120+
/// It should be in the range 8..15 for this version of the library. Larger values of this parameter result in better compression
121+
/// at the expense of memory usage. The default value is 15 if deflateInit is used instead.<br /></p>
122+
/// </summary>
123+
public const int ZLib_DefaultWindowBits = 15;
124+
117125
/// <summary>
118126
/// <p>Zlib's <code>windowBits</code> parameter is the base two logarithm of the window size (the size of the history buffer).
119127
/// For GZip header encoding, <code>windowBits</code> should be equal to a value between 8..15 (to specify Window Size) added to

‎src/libraries/System.IO.Compression/src/System/IO/Compression/GZipStream.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -234,7 +234,7 @@ private void CheckDeflateStream()
234234

235235
private static void ThrowStreamClosedException()
236236
{
237-
throw new ObjectDisposedException(null, SR.ObjectDisposed_StreamClosed);
237+
throw new ObjectDisposedException(nameof(GZipStream), SR.ObjectDisposed_StreamClosed);
238238
}
239239
}
240240
}

0 commit comments

Comments
 (0)