Skip to content

Make DeflateStream write out headers and footers on empty streams. #118570

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Oct 21, 2025
Merged

Make DeflateStream write out headers and footers on empty streams. #118570

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
71e16e6
Make DeflateStream write out headers and footers
rzikm Aug 11, 2025
297311e
Remove semicolons on empty line
rzikm Aug 11, 2025
657a961
Code review feedback
rzikm Oct 14, 2025
1aead9e
Second attempt on the implementation
rzikm Oct 14, 2025
1188234
Improve comments
rzikm Oct 14, 2025
b84b42b
Apply suggestion from @ericstj
rzikm Oct 17, 2025
3c24650
Add tests for decompressing empty streams
rzikm Oct 21, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
123 changes: 37 additions & 86 deletions src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateZLib/DeflateStream.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ public partial class DeflateStream : Stream
private Deflater? _deflater;
private byte[]? _buffer;
private volatile bool _activeAsyncOperation;
private bool _wroteBytes;

internal DeflateStream(Stream stream, CompressionMode mode, long uncompressedSize) : this(stream, mode, leaveOpen: false, ZLibNative.Deflate_DefaultWindowBits, uncompressedSize)
{
Expand Down Expand Up @@ -62,7 +61,7 @@ internal DeflateStream(Stream stream, ZLibCompressionOptions compressionOptions,
ArgumentNullException.ThrowIfNull(stream);
ArgumentNullException.ThrowIfNull(compressionOptions);

InitializeDeflater(stream, (ZLibNative.CompressionLevel)compressionOptions.CompressionLevel, (CompressionStrategy)compressionOptions.CompressionStrategy, leaveOpen, windowBits);
InitializeDeflater(stream, (ZLibNative.CompressionLevel)compressionOptions.CompressionLevel, (CompressionStrategy)compressionOptions.CompressionStrategy, leaveOpen, windowBits);
}

/// <summary>
Expand Down Expand Up @@ -565,7 +564,6 @@ internal void WriteCore(ReadOnlySpan<byte> buffer)
{
_deflater.SetInput(bufferPtr, buffer.Length);
WriteDeflaterOutput();
_wroteBytes = true;
}
}
}
Expand All @@ -586,25 +584,22 @@ private void WriteDeflaterOutput()
// This is called by Flush:
private void FlushBuffers()
{
if (_wroteBytes)
{
// Compress any bytes left:
WriteDeflaterOutput();
// Compress any bytes left:
WriteDeflaterOutput();

Debug.Assert(_deflater != null && _buffer != null);
// Pull out any bytes left inside deflater:
bool flushSuccessful;
do
Debug.Assert(_deflater != null && _buffer != null);
// Pull out any bytes left inside deflater:
bool flushSuccessful;
do
{
int compressedBytes;
flushSuccessful = _deflater.Flush(_buffer, out compressedBytes);
if (flushSuccessful)
{
int compressedBytes;
flushSuccessful = _deflater.Flush(_buffer, out compressedBytes);
if (flushSuccessful)
{
_stream.Write(_buffer, 0, compressedBytes);
}
Debug.Assert(flushSuccessful == (compressedBytes > 0));
} while (flushSuccessful);
}
_stream.Write(_buffer, 0, compressedBytes);
}
Debug.Assert(flushSuccessful == (compressedBytes > 0));
} while (flushSuccessful);

// Always flush on the underlying stream
_stream.Flush();
Expand All @@ -623,40 +618,19 @@ private void PurgeBuffers(bool disposing)
return;

Debug.Assert(_deflater != null && _buffer != null);
// Some deflaters (e.g. ZLib) write more than zero bytes for zero byte inputs.
// This round-trips and we should be ok with this, but our legacy managed deflater
// always wrote zero output for zero input and upstack code (e.g. ZipArchiveEntry)
// took dependencies on it. Thus, make sure to only "flush" when we actually had
// some input:
if (_wroteBytes)
{
// Compress any bytes left
WriteDeflaterOutput();

// Pull out any bytes left inside deflater:
bool finished;
do
{
int compressedBytes;
finished = _deflater.Finish(_buffer, out compressedBytes);
// Compress any bytes left
WriteDeflaterOutput();

if (compressedBytes > 0)
_stream.Write(_buffer, 0, compressedBytes);
} while (!finished);
}
else
// Pull out any bytes left inside deflater:
bool finished;
do
{
// In case of zero length buffer, we still need to clean up the native created stream before
// the object get disposed because eventually ZLibNative.ReleaseHandle will get called during
// the dispose operation and although it frees the stream but it return error code because the
// stream state was still marked as in use. The symptoms of this problem will not be seen except
// if running any diagnostic tools which check for disposing safe handle objects
bool finished;
do
{
finished = _deflater.Finish(_buffer, out _);
} while (!finished);
}
int compressedBytes;
finished = _deflater.Finish(_buffer, out compressedBytes);

if (compressedBytes > 0)
_stream.Write(_buffer, 0, compressedBytes);
} while (!finished);
}

private async ValueTask PurgeBuffersAsync()
Expand All @@ -670,40 +644,19 @@ private async ValueTask PurgeBuffersAsync()
return;

Debug.Assert(_deflater != null && _buffer != null);
// Some deflaters (e.g. ZLib) write more than zero bytes for zero byte inputs.
// This round-trips and we should be ok with this, but our legacy managed deflater
// always wrote zero output for zero input and upstack code (e.g. ZipArchiveEntry)
// took dependencies on it. Thus, make sure to only "flush" when we actually had
// some input.
if (_wroteBytes)
{
// Compress any bytes left
await WriteDeflaterOutputAsync(default).ConfigureAwait(false);

// Pull out any bytes left inside deflater:
bool finished;
do
{
int compressedBytes;
finished = _deflater.Finish(_buffer, out compressedBytes);
// Compress any bytes left
await WriteDeflaterOutputAsync(default).ConfigureAwait(false);

if (compressedBytes > 0)
await _stream.WriteAsync(new ReadOnlyMemory<byte>(_buffer, 0, compressedBytes)).ConfigureAwait(false);
} while (!finished);
}
else
// Pull out any bytes left inside deflater:
bool finished;
do
{
// In case of zero length buffer, we still need to clean up the native created stream before
// the object get disposed because eventually ZLibNative.ReleaseHandle will get called during
// the dispose operation and although it frees the stream, it returns an error code because the
// stream state was still marked as in use. The symptoms of this problem will not be seen except
// if running any diagnostic tools which check for disposing safe handle objects.
bool finished;
do
{
finished = _deflater.Finish(_buffer, out _);
} while (!finished);
}
int compressedBytes;
finished = _deflater.Finish(_buffer, out compressedBytes);

if (compressedBytes > 0)
await _stream.WriteAsync(new ReadOnlyMemory<byte>(_buffer, 0, compressedBytes)).ConfigureAwait(false);
} while (!finished);
}

protected override void Dispose(bool disposing)
Expand Down Expand Up @@ -857,8 +810,6 @@ async ValueTask Core(ReadOnlyMemory<byte> buffer, CancellationToken cancellation
_deflater.SetInput(buffer);

await WriteDeflaterOutputAsync(cancellationToken).ConfigureAwait(false);

_wroteBytes = true;
}
finally
{
Expand Down
25 changes: 17 additions & 8 deletions src/libraries/System.IO.Compression/src/System/IO/Compression/ZipArchiveEntry.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -702,30 +702,40 @@ private CheckSumAndSizeWriteStream GetDataCompressor(Stream backingStream, bool
{
// stream stack: backingStream -> DeflateStream -> CheckSumWriteStream

// By default we compress with deflate, except if compression level is set to NoCompression then stored is used.
// Stored is also used for empty files, but we don't actually call through this function for that - we just write the stored value in the header
// Deflate64 is not supported on all platforms
// By default we compress with deflate, except if compression level
// is set to NoCompression then stored is used.
//
// Stored is also used for empty files, but we can't know at this
// point if user will write anything to the stream or not. For that
// reason, we defer the instantiation of the compression stream
// until the first write to the CheckSumAndSizeWriteStream happens.
// If the user never writes anything, this will be detected during
// saving and the compression method in the file header will be
// changed to Stored.
//
// Note: Deflate64 is not supported on all platforms
Debug.Assert(CompressionMethod == CompressionMethodValues.Deflate
|| CompressionMethod == CompressionMethodValues.Stored);
Func<Stream> compressorStreamFactory;

bool isIntermediateStream = true;
Stream compressorStream;

switch (CompressionMethod)
{
case CompressionMethodValues.Stored:
compressorStream = backingStream;
compressorStreamFactory = () => backingStream;
isIntermediateStream = false;
break;
case CompressionMethodValues.Deflate:
case CompressionMethodValues.Deflate64:
default:
compressorStream = new DeflateStream(backingStream, _compressionLevel, leaveBackingStreamOpen);
compressorStreamFactory = () => new DeflateStream(backingStream, _compressionLevel, leaveBackingStreamOpen);
break;

}
bool leaveCompressorStreamOpenOnClose = leaveBackingStreamOpen && !isIntermediateStream;
var checkSumStream = new CheckSumAndSizeWriteStream(
compressorStream,
compressorStreamFactory,
backingStream,
leaveCompressorStreamOpenOnClose,
this,
Expand Down Expand Up @@ -975,7 +985,6 @@ private bool WriteLocalFileHeaderInitialize(bool isEmptyFile, bool forceWrite, o
CompressionMethod = CompressionMethodValues.Stored;
compressedSizeTruncated = 0;
uncompressedSizeTruncated = 0;
Debug.Assert(_compressedSize == 0);
Debug.Assert(_uncompressedSize == 0);
Debug.Assert(_crc32 == 0);
}
Expand Down
32 changes: 24 additions & 8 deletions src/libraries/System.IO.Compression/src/System/IO/Compression/ZipCustomStreams.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -433,7 +433,8 @@ protected override void Dispose(bool disposing)

internal sealed class CheckSumAndSizeWriteStream : Stream
{
private readonly Stream _baseStream;
private readonly Func<Stream> _baseStreamFactory;
private Stream? _baseStream;
private readonly Stream _baseBaseStream;
private long _position;
private uint _checksum;
Expand All @@ -459,11 +460,11 @@ internal sealed class CheckSumAndSizeWriteStream : Stream
// baseBaseStream it's a backingStream, passed here so as to avoid closure allocation,
// zipArchiveEntry passed here so as to avoid closure allocation,
// onClose handler passed here so as to avoid closure allocation
public CheckSumAndSizeWriteStream(Stream baseStream, Stream baseBaseStream, bool leaveOpenOnClose,
public CheckSumAndSizeWriteStream(Func<Stream> baseStreamFactory, Stream baseBaseStream, bool leaveOpenOnClose,
ZipArchiveEntry entry, EventHandler? onClose,
Action<long, long, uint, Stream, ZipArchiveEntry, EventHandler?> saveCrcAndSizes)
{
_baseStream = baseStream;
_baseStreamFactory = baseStreamFactory;
_baseBaseStream = baseBaseStream;
_position = 0;
_checksum = 0;
Expand Down Expand Up @@ -555,10 +556,15 @@ public override void Write(byte[] buffer, int offset, int count)

if (!_everWritten)
{
Debug.Assert(_baseStream == null);
_baseStream = _baseStreamFactory();

_initialPosition = _baseBaseStream.Position;
_everWritten = true;
}

Debug.Assert(_baseStream != null);

_checksum = Crc32Helper.UpdateCrc32(_checksum, buffer, offset, count);
_baseStream.Write(buffer, offset, count);
_position += count;
Expand All @@ -575,10 +581,15 @@ public override void Write(ReadOnlySpan<byte> source)

if (!_everWritten)
{
Debug.Assert(_baseStream == null);
_baseStream = _baseStreamFactory();

_initialPosition = _baseBaseStream.Position;
_everWritten = true;
}

Debug.Assert(_baseStream != null);

_checksum = Crc32Helper.UpdateCrc32(_checksum, source);
_baseStream.Write(source);
_position += source.Length;
Expand Down Expand Up @@ -606,10 +617,15 @@ async ValueTask Core(ReadOnlyMemory<byte> buffer, CancellationToken cancellation
{
if (!_everWritten)
{
Debug.Assert(_baseStream == null);
_baseStream = _baseStreamFactory();

_initialPosition = _baseBaseStream.Position;
_everWritten = true;
}

Debug.Assert(_baseStream != null);

_checksum = Crc32Helper.UpdateCrc32(_checksum, buffer.Span);

await _baseStream.WriteAsync(buffer, cancellationToken).ConfigureAwait(false);
Expand All @@ -624,13 +640,13 @@ public override void Flush()
// assume writable if not disposed
Debug.Assert(CanWrite);

_baseStream.Flush();
_baseStream?.Flush();
}

public override Task FlushAsync(CancellationToken cancellationToken)
{
ThrowIfDisposed();
return _baseStream.FlushAsync(cancellationToken);
return _baseStream?.FlushAsync(cancellationToken) ?? Task.CompletedTask;
}

protected override void Dispose(bool disposing)
Expand All @@ -641,7 +657,7 @@ protected override void Dispose(bool disposing)
if (!_everWritten)
_initialPosition = _baseBaseStream.Position;
if (!_leaveOpenOnClose)
_baseStream.Dispose(); // Close my super-stream (flushes the last data)
_baseStream?.Dispose(); // Close my super-stream (flushes the last data if we ever wrote any)
_saveCrcAndSizes?.Invoke(_initialPosition, Position, _checksum, _baseBaseStream, _zipArchiveEntry, _onClose);
_isDisposed = true;
}
Expand All @@ -655,8 +671,8 @@ public override async ValueTask DisposeAsync()
// if we never wrote through here, save the position
if (!_everWritten)
_initialPosition = _baseBaseStream.Position;
if (!_leaveOpenOnClose)
await _baseStream.DisposeAsync().ConfigureAwait(false); // Close my super-stream (flushes the last data)
if (!_leaveOpenOnClose && _baseStream != null)
await _baseStream.DisposeAsync().ConfigureAwait(false); // Close my super-stream (flushes the last data if we ever wrote any)
_saveCrcAndSizes?.Invoke(_initialPosition, Position, _checksum, _baseBaseStream, _zipArchiveEntry, _onClose);
_isDisposed = true;
}
Expand Down
36 changes: 34 additions & 2 deletions src/libraries/System.IO.Compression/tests/CompressionStreamUnitTests.Deflate.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,11 +156,11 @@ public void StreamTruncation_IsDetected(TestScenario testScenario)
break;

case TestScenario.Read:
while (ZipFileTestBase.ReadAllBytes(decompressor, buffer, 0, buffer.Length) != 0) { };
while (ZipFileTestBase.ReadAllBytes(decompressor, buffer, 0, buffer.Length) != 0) { }
break;

case TestScenario.ReadAsync:
while (await ZipFileTestBase.ReadAllBytesAsync(decompressor, buffer, 0, buffer.Length) != 0) { };
while (await ZipFileTestBase.ReadAllBytesAsync(decompressor, buffer, 0, buffer.Length) != 0) { }
break;

case TestScenario.ReadByte:
Expand Down Expand Up @@ -219,5 +219,37 @@ public override Task WriteAsync(byte[] buffer, int offset, int count, Cancellati
return base.WriteAsync(buffer, offset, count, cancellationToken);
}
}

[Fact]
public void EmptyDeflateStream_WritesOutput()
{
using (var ms = new MemoryStream())
{
using (var deflateStream = new DeflateStream(ms, CompressionMode.Compress, leaveOpen: true))
{
// Write nothing
}

// DeflateStream should now write output even for empty streams
Assert.True(ms.Length > 0, "Empty DeflateStream should write finalization data");
}
}

[Fact]
public void EmptyStream_CanBeDecompressed()
{
// for compatibility reasons, an empty stream should be decompressible back to an empty stream
using (var ms = new MemoryStream())
{
ms.Position = 0;

using (var deflateStream = new DeflateStream(ms, CompressionMode.Decompress))
using (var reader = new StreamReader(deflateStream))
{
string result = reader.ReadToEnd();
Assert.Equal(string.Empty, result);
}
}
}
}
}
Loading