[Foundation] Remove Content-Encoding and Content-Length headers for auto-decompressed responses in NSUrlSessionHandler. Fixes #23958. (#24924)

NSURLSession automatically decompresses content for supported encodings (gzip,
deflate, br, zstd, etc.) but leaves the original Content-Encoding and
Content-Length headers in the response. This causes a mismatch where
Content-Length refers to the compressed size while the body is decompressed.

All other HTTP handlers (SocketsHttpHandler, WinHttpHandler, Android's
handler) remove these headers after automatic decompression. This change makes
NSUrlSessionHandler consistent with them.

Fixes #23958

---------

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: rolfbjarne

src/Foundation/NSUrlSessionHandler.cs

@@ -112,6 +112,8 @@ public static string GetHeaderValue (this NSHttpCookie cookie)
 	public partial class NSUrlSessionHandler : HttpMessageHandler {
 		private const string SetCookie = "Set-Cookie";
 		private const string Cookie = "Cookie";
+		private const string ContentEncodingHeaderName = "Content-Encoding";
+		private const string ContentLengthHeaderName = "Content-Length";
 		private CookieContainer? cookieContainer;
 		readonly Dictionary<string, string> headerSeparators = new Dictionary<string, string> {
 			["User-Agent"] = " ",
@@ -869,6 +871,24 @@ public bool UseProxy {
 			}
 		}
 
+		static bool HasCompressedEncoding (string headerValue)
+		{
+			foreach (var encoding in headerValue.Split (',')) {
+				if (IsCompressedEncoding (encoding.Trim ()))
+					return true;
+			}
+			return false;
+		}
+
+		static bool IsCompressedEncoding (string encoding)
+		{
+			return string.Equals (encoding, "gzip", StringComparison.OrdinalIgnoreCase)
+				|| string.Equals (encoding, "deflate", StringComparison.OrdinalIgnoreCase)
+				|| string.Equals (encoding, "br", StringComparison.OrdinalIgnoreCase)
+				|| string.Equals (encoding, "compress", StringComparison.OrdinalIgnoreCase)
+				|| string.Equals (encoding, "zstd", StringComparison.OrdinalIgnoreCase);
+		}
+
 		partial class NSUrlSessionHandlerDelegate : NSUrlSessionDataDelegate {
 			readonly NSUrlSessionHandler sessionHandler;
 
@@ -959,6 +979,17 @@ void DidReceiveResponseImpl (NSUrlSession session, NSUrlSessionDataTask dataTask
 					if (wasRedirected)
 						httpResponse.RequestMessage.RequestUri = absoluteUri;
 
+					// NSURLSession automatically decompresses content for all supported
+					// encodings (gzip, deflate, br, zstd, etc.), and there's no way to
+					// turn it off. After decompression, Content-Encoding and Content-Length
+					// are stale (Content-Length refers to compressed size), so we need to
+					// remove them to match the behavior of other HTTP handlers:
+					// - SocketsHttpHandler:    https://github.com/dotnet/runtime/blob/b2974279efd059efaa17f359ed4b266b1c705721/src/libraries/System.Net.Http/src/System/Net/Http/SocketsHttpHandler/DecompressionHandler.cs#L122-L123
+					// - AndroidMessageHandler: https://github.com/dotnet/android/pull/7785
+					// Ref: https://github.com/dotnet/macios/issues/23958
+					string? contentEncodingValue = null;
+					string? contentLengthValue = null;
+
 					foreach (var v in urlResponse.AllHeaderFields) {
 						var key = v.Key?.ToString ();
 						var value = v.Value?.ToString ();
@@ -968,10 +999,31 @@ void DidReceiveResponseImpl (NSUrlSession session, NSUrlSessionDataTask dataTask
 						// NSUrlSession tries to be smart with cookies, we will not use the raw value but the ones provided by the cookie storage
 						if (key == SetCookie) continue;
 
+						if (string.Equals (key, ContentEncodingHeaderName, StringComparison.OrdinalIgnoreCase)) {
+							contentEncodingValue = value;
+							continue;
+						}
+						if (string.Equals (key, ContentLengthHeaderName, StringComparison.OrdinalIgnoreCase)) {
+							contentLengthValue = value;
+							continue;
+						}
+
 						httpResponse.Headers.TryAddWithoutValidation (key, value);
 						httpResponse.Content.Headers.TryAddWithoutValidation (key, value);
 					}
 
+					var contentWasDecompressed = contentEncodingValue is not null && HasCompressedEncoding (contentEncodingValue);
+					if (!contentWasDecompressed) {
+						if (contentEncodingValue is not null) {
+							httpResponse.Headers.TryAddWithoutValidation (ContentEncodingHeaderName, contentEncodingValue);
+							httpResponse.Content.Headers.TryAddWithoutValidation (ContentEncodingHeaderName, contentEncodingValue);
+						}
+						if (contentLengthValue is not null) {
+							httpResponse.Headers.TryAddWithoutValidation (ContentLengthHeaderName, contentLengthValue);
+							httpResponse.Content.Headers.TryAddWithoutValidation (ContentLengthHeaderName, contentLengthValue);
+						}
+					}
+
 					// it might be confusing that we are not using the managed CookieStore here, this is ONLY for those cookies that have been retrieved from
 					// the server via a Set-Cookie header, the managed container does not know a thing about this and apple is storing them in the native
 					// cookie container. Once we have the cookies from the response, we need to update the managed cookie container

tests/monotouch-test/System.Net.Http/NSUrlSessionHandlerTest.cs

@@ -14,6 +14,85 @@ namespace MonoTests.System.Net.Http {
 	[Preserve (AllMembers = true)]
 	public class NSUrlSessionHandlerTest {
 
+		// https://github.com/dotnet/macios/issues/23958
+		[Test]
+		public void DecompressedResponseDoesNotHaveContentEncodingOrContentLength ()
+		{
+			bool noContentEncoding = false;
+			bool noContentLength = false;
+			string body = "";
+
+			var done = TestRuntime.TryRunAsync (TimeSpan.FromSeconds (30), async () => {
+				using var handler = new NSUrlSessionHandler ();
+				using var client = new HttpClient (handler);
+				// Explicitly request gzip to ensure the server compresses the response.
+				using var request = new HttpRequestMessage (HttpMethod.Get, $"{NetworkResources.Httpbin.Url}/gzip");
+				request.Headers.TryAddWithoutValidation ("Accept-Encoding", "gzip");
+				// Use ResponseHeadersRead so that the response content is not buffered,
+				// which would cause HttpContent to compute Content-Length from the buffer.
+				var response = await client.SendAsync (request, HttpCompletionOption.ResponseHeadersRead);
+
+				if (!response.IsSuccessStatusCode) {
+					Assert.Inconclusive ($"Request failed with status {response.StatusCode}");
+					return;
+				}
+
+				noContentEncoding = response.Content.Headers.ContentEncoding.Count == 0;
+				noContentLength = response.Content.Headers.ContentLength is null;
+				body = await response.Content.ReadAsStringAsync ();
+			}, out var ex);
+
+			if (!done) {
+				TestRuntime.IgnoreInCI ("Transient network failure - ignore in CI");
+				Assert.Inconclusive ("Request timed out.");
+			}
+			TestRuntime.IgnoreInCIIfBadNetwork (ex);
+			Assert.IsNull (ex, $"Exception: {ex}");
+			Assert.IsTrue (noContentEncoding, "Content-Encoding header should be removed for decompressed content");
+			Assert.IsTrue (noContentLength, "Content-Length header should be removed for decompressed content");
+			Assert.IsTrue (body.Contains ("\"gzipped\"", StringComparison.OrdinalIgnoreCase), "Response body should contain decompressed gzip data");
+		}
+
+		// https://github.com/dotnet/macios/issues/23958
+		[Test]
+		public void NonCompressedResponseHasContentLength ()
+		{
+			bool noContentEncoding = false;
+			long? contentLength = null;
+			string body = "";
+
+			var done = TestRuntime.TryRunAsync (TimeSpan.FromSeconds (30), async () => {
+				using var handler = new NSUrlSessionHandler ();
+				using var client = new HttpClient (handler);
+				// Request identity encoding to ensure no compression is applied.
+				using var request = new HttpRequestMessage (HttpMethod.Get, $"{NetworkResources.Httpbin.Url}/html");
+				request.Headers.TryAddWithoutValidation ("Accept-Encoding", "identity");
+				// Use ResponseHeadersRead so that the response content is not buffered,
+				// which would cause HttpContent to compute Content-Length from the buffer.
+				var response = await client.SendAsync (request, HttpCompletionOption.ResponseHeadersRead);
+
+				if (!response.IsSuccessStatusCode) {
+					Assert.Inconclusive ($"Request failed with status {response.StatusCode}");
+					return;
+				}
+
+				noContentEncoding = response.Content.Headers.ContentEncoding.Count == 0;
+				contentLength = response.Content.Headers.ContentLength;
+				body = await response.Content.ReadAsStringAsync ();
+			}, out var ex);
+
+			if (!done) {
+				TestRuntime.IgnoreInCI ("Transient network failure - ignore in CI");
+				Assert.Inconclusive ("Request timed out.");
+			}
+			TestRuntime.IgnoreInCIIfBadNetwork (ex);
+			Assert.IsNull (ex, $"Exception: {ex}");
+			Assert.IsTrue (noContentEncoding, "Content-Encoding should not be present for non-compressed content");
+			Assert.IsNotNull (contentLength, "Content-Length header should be present for non-compressed content");
+			Assert.IsTrue (contentLength > 0, "Content-Length should be greater than zero");
+			Assert.IsTrue (body.Length > 0, "Response body should not be empty");
+		}
+
 		// https://github.com/dotnet/macios/issues/24376
 		[Test]
 		public void DisposeAndRecreateBackgroundSessionHandler ()