Implement ZLibStream and fix SocketsHttpHandler deflate support

src/libraries/Common/tests/System/IO/Compression/CompressionStreamUnitTestBase.cs

@@ -1254,6 +1254,59 @@ await Task.WhenAll(Enumerable.Range(0, ParallelOperations).Select(_ => Task.Run(
  Assert.Equal(sourceData, decompressedStream.ToArray());
  })));
  }
+
+ [Fact]
+ public void Precancellation()
+ {
+ var ms = new MemoryStream();
+ using (Stream compressor = CreateStream(ms, CompressionMode.Compress, leaveOpen: true))
+ {
+ Assert.True(compressor.WriteAsync(new byte[1], 0, 1, new CancellationToken(true)).IsCanceled);
+ Assert.True(compressor.FlushAsync(new CancellationToken(true)).IsCanceled);
+ }
+ using (Stream decompressor = CreateStream(ms, CompressionMode.Decompress, leaveOpen: true))
+ {
+ Assert.True(decompressor.ReadAsync(new byte[1], 0, 1, new CancellationToken(true)).IsCanceled);
+ }
+ }
+
+ [Theory]
+ [InlineData(false)]
+ [InlineData(true)]
+ public async Task DisposeAsync_Flushes(bool leaveOpen)
+ {
+ var ms = new MemoryStream();
+ var cs = CreateStream(ms, CompressionMode.Compress, leaveOpen);
+ cs.WriteByte(1);
+ await cs.FlushAsync();
+
+ long pos = ms.Position;
+ cs.WriteByte(1);
+ Assert.Equal(pos, ms.Position);
+
+ await cs.DisposeAsync();
+ Assert.InRange(ms.ToArray().Length, pos + 1, int.MaxValue);
+ if (leaveOpen)
+ {
+ Assert.InRange(ms.Position, pos + 1, int.MaxValue);
+ }
+ else
+ {
+ Assert.Throws<ObjectDisposedException>(() => ms.Position);
+ }
+ }
+
+ [Theory]
+ [InlineData(false)]
+ [InlineData(true)]
+ public async Task DisposeAsync_MultipleCallsAllowed(bool leaveOpen)
+ {
+ using (var cs = CreateStream(new MemoryStream(), CompressionMode.Compress, leaveOpen))
+ {
+ await cs.DisposeAsync();
+ await cs.DisposeAsync();
+ }
+ }
  }
 
  internal sealed class BadWrappedStream : MemoryStream

src/libraries/Common/tests/System/Net/Http/HttpClientHandlerTest.Decompression.cs

@@ -33,46 +33,55 @@ public static IEnumerable<object[]> RemoteServersAndCompressionUris()
  foreach (Configuration.Http.RemoteServer remoteServer in Configuration.Http.RemoteServers)
  {
  yield return new object[] { remoteServer, remoteServer.GZipUri };
- yield return new object[] { remoteServer, remoteServer.DeflateUri };
+
+ // Remote deflate endpoint isn't correctly following the deflate protocol.
+ //yield return new object[] { remoteServer, remoteServer.DeflateUri };
  }
  }
 
- public static IEnumerable<object[]> DecompressedResponse_MethodSpecified_DecompressedContentReturned_MemberData()
+ [Theory]
+ [InlineData("gzip", false)]
+ [InlineData("gzip", true)]
+ [InlineData("deflate", false)]
+ [InlineData("deflate", true)]
+ [InlineData("br", false)]
+ [InlineData("br", true)]
+ public async Task DecompressedResponse_MethodSpecified_DecompressedContentReturned(string encodingName, bool all)
  {
- foreach (bool specifyAllMethods in new[] { false, true })
+ Func<Stream, Stream> compress;
+ DecompressionMethods methods;
+ switch (encodingName)
  {
- yield return new object[]
- {
- "deflate",
- new Func<Stream, Stream>(s => new DeflateStream(s, CompressionLevel.Optimal, leaveOpen: true)),
- specifyAllMethods ? DecompressionMethods.Deflate : _all
- };
- yield return new object[]
- {
- "gzip",
- new Func<Stream, Stream>(s => new GZipStream(s, CompressionLevel.Optimal, leaveOpen: true)),
- specifyAllMethods ? DecompressionMethods.GZip : _all
- };
+ case "gzip":
+ compress = s => new GZipStream(s, CompressionLevel.Optimal, leaveOpen: true);
+ methods = all ? DecompressionMethods.GZip : _all;
+ break;
+
 #if !NETFRAMEWORK
- yield return new object[]
- {
- "br",
- new Func<Stream, Stream>(s => new BrotliStream(s, CompressionLevel.Optimal, leaveOpen: true)),
- specifyAllMethods ? DecompressionMethods.Brotli : _all
- };
+ case "br":
+ if (IsWinHttpHandler)
+ {
+ // Brotli only supported on SocketsHttpHandler.
+ return;
+ }
+
+ compress = s => new BrotliStream(s, CompressionLevel.Optimal, leaveOpen: true);
+ methods = all ? DecompressionMethods.Brotli : _all;
+ break;
+
+ case "deflate":
+ // WinHttpHandler continues to use DeflateStream as it doesn't have a newer build than netstandard2.0
+ // and doesn't have access to ZLibStream.
+ compress = IsWinHttpHandler ?
+ new Func<Stream, Stream>(s => new DeflateStream(s, CompressionLevel.Optimal, leaveOpen: true)) :
+ new Func<Stream, Stream>(s => new ZLibStream(s, CompressionLevel.Optimal, leaveOpen: true));
+ methods = all ? DecompressionMethods.Deflate : _all;
+ break;
 #endif
- }
- }
 
- [Theory]
- [MemberData(nameof(DecompressedResponse_MethodSpecified_DecompressedContentReturned_MemberData))]
- public async Task DecompressedResponse_MethodSpecified_DecompressedContentReturned(
- string encodingName, Func<Stream, Stream> compress, DecompressionMethods methods)
- {
- // Brotli only supported on SocketsHttpHandler.
- if (IsWinHttpHandler && encodingName == "br")
- {
- return;
+ default:
+ Assert.Contains(encodingName, new[] { "br", "deflate", "gzip" });
+ return;
  }
 
  var expectedContent = new byte[12345];
@@ -104,15 +113,15 @@ public static IEnumerable<object[]> DecompressedResponse_MethodNotSpecified_Orig
  {
  yield return new object[]
  {
- "deflate",
- new Func<Stream, Stream>(s => new DeflateStream(s, CompressionLevel.Optimal, leaveOpen: true)),
+ "gzip",
+ new Func<Stream, Stream>(s => new GZipStream(s, CompressionLevel.Optimal, leaveOpen: true)),
  DecompressionMethods.None
  };
 #if !NETFRAMEWORK
  yield return new object[]
  {
- "gzip",
- new Func<Stream, Stream>(s => new GZipStream(s, CompressionLevel.Optimal, leaveOpen: true)),
+ "deflate",
+ new Func<Stream, Stream>(s => new ZLibStream(s, CompressionLevel.Optimal, leaveOpen: true)),
  DecompressionMethods.Brotli
  };
  yield return new object[]
@@ -186,6 +195,26 @@ public async Task GetAsync_SetAutomaticDecompression_ContentDecompressed(Configu
  }
  }
 
+ // The remote server endpoint was written to use DeflateStream, which isn't actually a correct
+ // implementation of the deflate protocol (the deflate protocol requires the zlib wrapper around
+ // deflate). Until we can get that updated (and deal with previous releases still testing it
+ // via a DeflateStream-based implementation), we utilize httpbin.org to help validate behavior.
+ [OuterLoop("Uses external servers")]
+ [Theory]
+ [InlineData("http://httpbin.org/deflate", "\"deflated\": true")]
+ [InlineData("https://httpbin.org/deflate", "\"deflated\": true")]
+ [InlineData("http://httpbin.org/gzip", "\"gzipped\": true")]
+ [InlineData("https://httpbin.org/gzip", "\"gzipped\": true")]
+ public async Task GetAsync_SetAutomaticDecompression_ContentDecompressed(string uri, string expectedContent)
+ {
+ HttpClientHandler handler = CreateHttpClientHandler();
+ handler.AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate;
+ using (HttpClient client = CreateHttpClient(handler))
+ {
+ Assert.Contains(expectedContent, await client.GetStringAsync(uri));
+ }
+ }
+
  [OuterLoop("Uses external server")]
  [Theory, MemberData(nameof(RemoteServersAndCompressionUris))]
  public async Task GetAsync_SetAutomaticDecompression_HeadersRemoved(Configuration.Http.RemoteServer remoteServer, Uri uri)

src/libraries/System.IO.Compression.Brotli/tests/CompressionStreamUnitTests.Brotli.cs

@@ -23,42 +23,6 @@ public class BrotliStreamUnitTests : CompressionStreamUnitTestBase
 
  protected override string CompressedTestFile(string uncompressedPath) => Path.Combine("BrotliTestData", Path.GetFileName(uncompressedPath) + ".br");
 
- [Fact]
- public void Precancellation()
- {
- var ms = new MemoryStream();
- using (Stream compressor = new BrotliStream(ms, CompressionMode.Compress, leaveOpen: true))
- {
- Assert.True(compressor.WriteAsync(new byte[1], 0, 1, new CancellationToken(true)).IsCanceled);
- Assert.True(compressor.FlushAsync(new CancellationToken(true)).IsCanceled);
- }
- using (Stream decompressor = CreateStream(ms, CompressionMode.Decompress, leaveOpen: true))
- {
- Assert.True(decompressor.ReadAsync(new byte[1], 0, 1, new CancellationToken(true)).IsCanceled);
- }
- }
-
- [Theory]
- [InlineData(false)]
- [InlineData(true)]
- public async Task DisposeAsync_Flushes(bool leaveOpen)
- {
- var ms = new MemoryStream();
- var bs = new BrotliStream(ms, CompressionMode.Compress, leaveOpen);
- bs.WriteByte(1);
- Assert.Equal(0, ms.Position);
- await bs.DisposeAsync();
- Assert.InRange(ms.ToArray().Length, 1, int.MaxValue);
- if (leaveOpen)
- {
- Assert.InRange(ms.Position, 1, int.MaxValue);
- }
- else
- {
- Assert.Throws<ObjectDisposedException>(() => ms.Position);
- }
- }
-
  [Fact]
  [OuterLoop("Test takes ~6 seconds to run")]
  public override void FlushAsync_DuringWriteAsync() { base.FlushAsync_DuringWriteAsync(); }

src/libraries/System.IO.Compression/ref/System.IO.Compression.cs

@@ -121,4 +121,39 @@ public enum ZipArchiveMode
  Create = 1,
  Update = 2,
  }
+ public sealed partial class ZLibStream : System.IO.Stream
+ {
+ public ZLibStream(System.IO.Stream stream, System.IO.Compression.CompressionLevel compressionLevel) { }
+ public ZLibStream(System.IO.Stream stream, System.IO.Compression.CompressionLevel compressionLevel, bool leaveOpen) { }
+ public ZLibStream(System.IO.Stream stream, System.IO.Compression.CompressionMode mode) { }
+ public ZLibStream(System.IO.Stream stream, System.IO.Compression.CompressionMode mode, bool leaveOpen) { }
+ public System.IO.Stream BaseStream { get { throw null; } }
+ public override bool CanRead { get { throw null; } }
+ public override bool CanSeek { get { throw null; } }
+ public override bool CanWrite { get { throw null; } }
+ public override long Length { get { throw null; } }
+ public override long Position { get { throw null; } set { } }
+ public override System.IAsyncResult BeginRead(byte[] array, int offset, int count, System.AsyncCallback? asyncCallback, object? asyncState) { throw null; }
+ public override System.IAsyncResult BeginWrite(byte[] array, int offset, int count, System.AsyncCallback? asyncCallback, object? asyncState) { throw null; }
+ public override void CopyTo(System.IO.Stream destination, int bufferSize) { }
+ public override System.Threading.Tasks.Task CopyToAsync(System.IO.Stream destination, int bufferSize, System.Threading.CancellationToken cancellationToken) { throw null; }
+ protected override void Dispose(bool disposing) { }
+ public override System.Threading.Tasks.ValueTask DisposeAsync() { throw null; }
+ public override int EndRead(System.IAsyncResult asyncResult) { throw null; }
+ public override void EndWrite(System.IAsyncResult asyncResult) { }
+ public override void Flush() { }
+ public override System.Threading.Tasks.Task FlushAsync(System.Threading.CancellationToken cancellationToken) { throw null; }
+ public override int Read(byte[] array, int offset, int count) { throw null; }
+ public override int Read(System.Span<byte> buffer) { throw null; }
+ public override System.Threading.Tasks.Task<int> ReadAsync(byte[] array, int offset, int count, System.Threading.CancellationToken cancellationToken) { throw null; }
+ public override System.Threading.Tasks.ValueTask<int> ReadAsync(System.Memory<byte> buffer, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
+ public override int ReadByte() { throw null; }
+ public override long Seek(long offset, System.IO.SeekOrigin origin) { throw null; }
+ public override void SetLength(long value) { }
+ public override void Write(byte[] array, int offset, int count) { }
+ public override void Write(System.ReadOnlySpan<byte> buffer) { }
+ public override void WriteByte(byte value) { }
+ public override System.Threading.Tasks.Task WriteAsync(byte[] array, int offset, int count, System.Threading.CancellationToken cancellationToken) { throw null; }
+ public override System.Threading.Tasks.ValueTask WriteAsync(System.ReadOnlyMemory<byte> buffer, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
+ }
 }

src/libraries/System.IO.Compression/src/System.IO.Compression.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
  <PropertyGroup>
  <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
  <TargetFrameworks>$(NetCoreAppCurrent)-Windows_NT;$(NetCoreAppCurrent)-Unix;$(NetCoreAppCurrent)-Browser</TargetFrameworks>
@@ -33,6 +33,7 @@
  <Compile Include="System\IO\Compression\Crc32Helper.ZLib.cs" />
  <Compile Include="System\IO\Compression\GZipStream.cs" />
  <Compile Include="System\IO\Compression\PositionPreservingWriteOnlyStreamWrapper.cs" />
+ <Compile Include="System\IO\Compression\ZLibStream.cs" />
  <Compile Include="$(CommonPath)System\IO\StreamHelpers.CopyValidation.cs"
  Link="Common\System\IO\StreamHelpers.CopyValidation.cs" />
  <Compile Include="$(CommonPath)System\Threading\Tasks\TaskToApm.cs"

src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateManaged/DeflateManagedStream.cs

@@ -162,7 +162,7 @@ private void EnsureNotDisposed()
 
  private static void ThrowStreamClosedException()
  {
- throw new ObjectDisposedException(null, SR.ObjectDisposed_StreamClosed);
+ throw new ObjectDisposedException(nameof(DeflateStream), SR.ObjectDisposed_StreamClosed);
  }
 
  public override IAsyncResult BeginRead(byte[] buffer, int offset, int count, AsyncCallback? asyncCallback, object? asyncState) =>

src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateZLib/DeflateStream.cs

@@ -320,7 +320,7 @@ private void EnsureNotDisposed()
 
  private static void ThrowStreamClosedException()
  {
- throw new ObjectDisposedException(null, SR.ObjectDisposed_StreamClosed);
+ throw new ObjectDisposedException(nameof(DeflateStream), SR.ObjectDisposed_StreamClosed);
  }
 
  private void EnsureDecompressionMode()

src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateZLib/ZLibNative.cs

@@ -114,6 +114,14 @@ public enum CompressionMethod : int
  public const int Deflate_DefaultWindowBits = -15; // Legal values are 8..15 and -8..-15. 15 is the window size,
  // negative val causes deflate to produce raw deflate data (no zlib header).
 
+ /// <summary>
+ /// <p><strong>From the ZLib manual:</strong></p>
+ /// <p>ZLib's <code>windowBits</code> parameter is the base two logarithm of the window size (the size of the history buffer).
+ /// It should be in the range 8..15 for this version of the library. Larger values of this parameter result in better compression
+ /// at the expense of memory usage. The default value is 15 if deflateInit is used instead.<br /></p>
+ /// </summary>
+ public const int ZLib_DefaultWindowBits = 15;
+
  /// <summary>
  /// <p>Zlib's <code>windowBits</code> parameter is the base two logarithm of the window size (the size of the history buffer).
  /// For GZip header encoding, <code>windowBits</code> should be equal to a value between 8..15 (to specify Window Size) added to

src/libraries/System.IO.Compression/src/System/IO/Compression/GZipStream.cs

@@ -234,7 +234,7 @@ private void CheckDeflateStream()
 
  private static void ThrowStreamClosedException()
  {
- throw new ObjectDisposedException(null, SR.ObjectDisposed_StreamClosed);
+ throw new ObjectDisposedException(nameof(GZipStream), SR.ObjectDisposed_StreamClosed);
  }
  }
 }