Skip to content

Commit e867cfb

Browse files
Fix managed SNI MARS receive teardown race (#4680)
* Fix managed SNI MARS receive teardown race Handle disposed TCP and named-pipe streams through SNI errors and preserve MARS packet ownership during receive re-arm failure. Add deterministic and SQL-backed regression coverage for #4679. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Fix MARS teardown callbacks and regression test hangs Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Address MARS teardown review feedback Keep consumed-packet cleanup outside the receive lock, add a lock-scope regression, and remove redundant manual-test collection isolation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com>
1 parent 84953ea commit e867cfb

9 files changed

Lines changed: 840 additions & 16 deletions

File tree

‎src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/TdsParserStateObject.netcore.cs‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -364,7 +364,13 @@ public void ReadAsyncCallback(IntPtr key, PacketHandle packet, uint error)
364364
bool processFinallyBlock = true;
365365
try
366366
{
367-
Debug.Assert(CheckPacket(packet, source) && source != null, "AsyncResult null on callback");
367+
// Closing a MARS connection can clear a session's task before its error callback.
368+
// The late callback must still retire the pending callback count.
369+
Debug.Assert(
370+
(CheckPacket(packet, source) && source != null) ||
371+
(source == null && error != 0 && _parser.MARSOn &&
372+
(_parser.State == TdsParserState.Closed || _parser.State == TdsParserState.Broken)),
373+
"AsyncResult null on callback");
368374

369375
if (_parser.MARSOn)
370376
{

‎src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ManagedSni/SniMarsConnection.netcore.cs‎

Lines changed: 31 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -128,22 +128,35 @@ public uint ReceiveAsync(ref SniPacket packet)
128128
{
129129
using (TrySNIEventScope.Create(nameof(SniMarsConnection)))
130130
{
131-
if (packet != null)
131+
SniPacket previousPacket = packet;
132+
packet = null;
133+
try
132134
{
133-
ReturnPacket(packet);
135+
lock (DemuxerSync)
136+
{
137+
var response = _lowerHandle.ReceiveAsync(ref packet);
138+
if (response != TdsEnums.SNI_SUCCESS_IO_PENDING && packet == null)
139+
{
140+
// Immediate transport errors release the new packet. Keep the consumed
141+
// packet alive so HandleReceiveError can notify all sessions and release it.
142+
packet = previousPacket;
143+
previousPacket = null;
144+
}
134145
#if DEBUG
135-
SqlClientEventSource.Log.TrySNITraceEvent(nameof(SniMarsConnection), EventType.INFO, "MARS Session Id {0}, Packet {1} returned", args0: ConnectionId, args1: packet?._id);
146+
SqlClientEventSource.Log.TrySNITraceEvent(nameof(SniMarsConnection), EventType.INFO, "MARS Session Id {0}, Received new packet {1}", args0: ConnectionId, args1: packet?._id);
136147
#endif
137-
packet = null;
148+
return response;
149+
}
138150
}
139-
140-
lock (DemuxerSync)
151+
finally
141152
{
142-
var response = _lowerHandle.ReceiveAsync(ref packet);
153+
if (previousPacket != null)
154+
{
155+
ReturnPacket(previousPacket);
143156
#if DEBUG
144-
SqlClientEventSource.Log.TrySNITraceEvent(nameof(SniMarsConnection), EventType.INFO, "MARS Session Id {0}, Received new packet {1}", args0: ConnectionId, args1: packet?._id);
157+
SqlClientEventSource.Log.TrySNITraceEvent(nameof(SniMarsConnection), EventType.INFO, "MARS Session Id {0}, Packet {1} returned", args0: ConnectionId, args1: previousPacket._id);
145158
#endif
146-
return response;
159+
}
147160
}
148161
}
149162
}
@@ -168,7 +181,15 @@ public uint CheckConnection()
168181
/// </summary>
169182
public void HandleReceiveError(SniPacket packet)
170183
{
171-
Debug.Assert(Monitor.IsEntered(this), "HandleReceiveError was called without being locked.");
184+
Debug.Assert(Monitor.IsEntered(DemuxerSync), "HandleReceiveError was called without being locked.");
185+
if (_dataBytesLeft > 0 && _currentPacket != null)
186+
{
187+
SqlClientEventSource.Log.TrySNITraceEvent(nameof(SniMarsConnection), EventType.ERR, "MARS Session Id {0}, Returning incomplete packet after receive error, _dataBytesLeft {1}", args0: ConnectionId, args1: _dataBytesLeft);
188+
ReturnPacket(_currentPacket);
189+
_currentPacket = null;
190+
_dataBytesLeft = 0;
191+
}
192+
172193
foreach (SniMarsHandle handle in _sessions.Values)
173194
{
174195
if (packet.HasAsyncIOCompletionCallback)

‎src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ManagedSni/SniMarsHandle.netcore.cs‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -343,14 +343,24 @@ public void HandleReceiveError(SniPacket packet)
343343
// which should handle ownership of the packet because the individual mars handles are not aware of
344344
// each other and cannot know if they are the last one in the list and that it is safe to return the packet
345345

346+
bool notifyAsyncReceive;
346347
lock (_receivedPacketQueue)
347348
{
348349
_connectionError = SniLoadHandle.SingletonInstance.LastError;
349350
SqlClientEventSource.Log.TrySNITraceEvent(nameof(SniMarsHandle), EventType.ERR, "MARS Session Id {0}, _connectionError to be handled: {1}", args0: ConnectionId, args1: _connectionError);
350351
_packetEvent.Set();
352+
notifyAsyncReceive = _asyncReceives > 0;
353+
if (notifyAsyncReceive)
354+
{
355+
_asyncReceives--;
356+
}
351357
}
352358

353-
((TdsParserStateObject)_callbackObject).ReadAsyncCallback(PacketHandle.FromManagedPacket(packet), 1);
359+
// Sync and idle sessions observe _connectionError without an async callback.
360+
if (notifyAsyncReceive)
361+
{
362+
((TdsParserStateObject)_callbackObject).ReadAsyncCallback(PacketHandle.FromManagedPacket(packet), 1);
363+
}
354364
}
355365
}
356366

‎src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ManagedSni/SniNpHandle.netcore.cs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -194,7 +194,7 @@ public override uint Receive(out SniPacket packet, int timeout)
194194
try
195195
{
196196
packet = RentPacket(headerSize: 0, dataSize: _bufferSize);
197-
packet.ReadFromStream(_stream);
197+
packet.ReadFromStream(_stream ?? throw new ObjectDisposedException(nameof(SniNpHandle)));
198198
SqlClientEventSource.Log.TrySNITraceEvent(nameof(SniNpHandle), EventType.INFO, "Connection Id {0}, Rented and read packet, dataLeft {1}", args0: _connectionId, args1: packet?.DataLeft);
199199

200200
if (packet.Length == 0)
@@ -234,7 +234,9 @@ public override uint ReceiveAsync(ref SniPacket packet)
234234
packet.SetAsyncIOCompletionCallback(_receiveCallback);
235235
try
236236
{
237-
packet.ReadFromStreamAsync(_stream);
237+
// Capture once: Dispose can clear the field while a MARS receive is re-armed.
238+
Stream stream = _stream ?? throw new ObjectDisposedException(nameof(SniNpHandle));
239+
packet.ReadFromStreamAsync(stream);
238240
SqlClientEventSource.Log.TrySNITraceEvent(nameof(SniNpHandle), EventType.INFO, "Connection Id {0}, Rented and read packet asynchronously, dataLeft {1}", args0: _connectionId, args1: packet?.DataLeft);
239241
return TdsEnums.SNI_SUCCESS_IO_PENDING;
240242
}

‎src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ManagedSni/SniTcpHandle.netcore.cs‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -890,6 +890,11 @@ public override uint Receive(out SniPacket packet, int timeoutInMilliseconds)
890890
return ReportTcpSNIError(0, SniCommon.ConnOpenFailedError, Strings.SNI_ERROR_10);
891891
}
892892

893+
if (_stream == null)
894+
{
895+
return ReportTcpSNIError(new ObjectDisposedException(nameof(SniTcpHandle)));
896+
}
897+
893898
try
894899
{
895900
if (timeoutInMilliseconds > 0)
@@ -1019,7 +1024,9 @@ public override uint ReceiveAsync(ref SniPacket packet)
10191024
packet.SetAsyncIOCompletionCallback(_receiveCallback);
10201025
try
10211026
{
1022-
packet.ReadFromStreamAsync(_stream);
1027+
// Capture once: Dispose can clear the field while a MARS receive is re-armed.
1028+
Stream stream = _stream ?? throw new ObjectDisposedException(nameof(SniTcpHandle));
1029+
packet.ReadFromStreamAsync(stream);
10231030
SqlClientEventSource.Log.TrySNITraceEvent(nameof(SniTcpHandle), EventType.INFO, "Connection Id {0}, Data received from stream asynchronously", args0: _connectionId);
10241031
return TdsEnums.SNI_SUCCESS_IO_PENDING;
10251032
}

‎src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/TdsParserStateObject.cs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2602,7 +2602,10 @@ private void CancelWritePacket()
26022602
internal TdsOperationStatus TryReadNetworkPacket()
26032603
{
26042604
#if DEBUG
2605-
Debug.Assert(!_shouldHaveEnoughData || _attentionSent, "Caller said there should be enough data, but we are currently reading a packet");
2605+
// Teardown can invalidate the reader's buffered-data estimate before this call.
2606+
Debug.Assert(!_shouldHaveEnoughData || _attentionSent ||
2607+
_parser.State == TdsParserState.Closed || _parser.State == TdsParserState.Broken,
2608+
"Caller said there should be enough data, but we are currently reading a packet");
26062609
#endif
26072610
TdsOperationStatus result = TdsOperationStatus.InvalidData;
26082611
if (_snapshot != null)

‎src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTesting.Tests.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@
8181
<Compile Include="SQL\AsyncTest\BeginExecReaderAsyncTest.cs" />
8282
<Compile Include="SQL\AsyncTest\XmlReaderAsyncTest.cs" />
8383
<Compile Include="SQL\AsyncTest\AsyncCancelledConnectionsTest.cs" />
84+
<Compile Include="SQL\AsyncTest\MarsReceiveTeardownTest.cs" />
8485
<Compile Include="SQL\ConnectionTestWithSSLCert\HostNameInCertificateTest.cs" />
8586
</ItemGroup>
8687
<ItemGroup Condition="'$(TestSet)' == '' OR '$(TestSet)' == '2'">
Lines changed: 194 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
#if NET
6+
7+
#nullable enable
8+
9+
using System;
10+
using System.Collections.Concurrent;
11+
using System.IO;
12+
using System.Threading.Tasks;
13+
using Xunit;
14+
15+
namespace Microsoft.Data.SqlClient.ManualTesting.Tests
16+
{
17+
/// <summary>
18+
/// Exercises GH#4679 with a streaming MARS session and a connection-terminating error
19+
/// on a second session. Requires a test SQL Server and a sysadmin login; each case
20+
/// writes a severity-20 error to the server log.
21+
/// </summary>
22+
[Trait("Set", "1")]
23+
public sealed class MarsReceiveTeardownTest
24+
{
25+
/// <summary>
26+
/// Physical teardown must report normal command errors rather than faulting an
27+
/// unobserved MARS receive continuation, with or without connection pooling.
28+
/// Each delay is a separate case rather than a long-running stress loop.
29+
/// </summary>
30+
[ConditionalTheory(typeof(MarsReceiveTeardownTest), nameof(CanRun))]
31+
[InlineData(false, false, 1)]
32+
[InlineData(false, true, 1)]
33+
[InlineData(true, false, 1)]
34+
[InlineData(true, true, 1)]
35+
[InlineData(false, false, 29)]
36+
[InlineData(false, true, 29)]
37+
[InlineData(true, false, 29)]
38+
[InlineData(true, true, 29)]
39+
public async Task FatalError_WhileMarsReaderStreams_DoesNotFaultReceivePump(bool async, bool pooling, int delay)
40+
{
41+
string connectionString = new SqlConnectionStringBuilder(DataTestUtility.TCPConnectionString)
42+
{
43+
MultipleActiveResultSets = true,
44+
Pooling = pooling,
45+
ConnectRetryCount = 0,
46+
ApplicationName = nameof(MarsReceiveTeardownTest),
47+
}.ConnectionString;
48+
ConcurrentQueue<Exception> unobserved = new();
49+
EventHandler<UnobservedTaskExceptionEventArgs> handler = (_, args) =>
50+
{
51+
foreach (Exception exception in args.Exception.Flatten().InnerExceptions)
52+
{
53+
if (exception.StackTrace?.Contains("ManagedSni.SniMarsConnection") == true)
54+
{
55+
unobserved.Enqueue(exception);
56+
args.SetObserved();
57+
}
58+
}
59+
};
60+
61+
TaskScheduler.UnobservedTaskException += handler;
62+
try
63+
{
64+
await TerminateStreamingConnection(connectionString, async, delay);
65+
66+
for (int attempt = 0; attempt < 10; attempt++)
67+
{
68+
CollectReceiveContinuations();
69+
await Task.Delay(50);
70+
}
71+
72+
Assert.Empty(unobserved);
73+
}
74+
finally
75+
{
76+
TaskScheduler.UnobservedTaskException -= handler;
77+
using SqlConnection poolKey = new(connectionString);
78+
SqlConnection.ClearPool(poolKey);
79+
}
80+
}
81+
82+
/// <summary>
83+
/// Requires managed SNI and a non-Azure SQL Server login allowed to raise fatal errors.
84+
/// </summary>
85+
/// <returns>Whether the configured login is a sysadmin on a supported test server.</returns>
86+
public static bool CanRun()
87+
{
88+
if (!DataTestUtility.AreConnStringsSetup() ||
89+
!DataTestUtility.IsNotAzureServer() || !DataTestUtility.IsUsingManagedSNI())
90+
{
91+
return false;
92+
}
93+
94+
using SqlConnection connection = new(DataTestUtility.TCPConnectionString);
95+
using SqlCommand command = new("SELECT IS_SRVROLEMEMBER('sysadmin')", connection);
96+
connection.Open();
97+
return command.ExecuteScalar() is int member && member == 1;
98+
}
99+
100+
/// <summary>
101+
/// Starts two MARS sessions, terminates only their connection, and observes all
102+
/// application tasks so any unobserved exception belongs to the driver's receive pump.
103+
/// </summary>
104+
/// <param name="connectionString">Connection string for the test server.</param>
105+
/// <param name="async">Whether to use asynchronous command and reader APIs.</param>
106+
/// <param name="delay">Delay before terminating the streaming connection, in milliseconds.</param>
107+
/// <returns>A task that completes after both sessions and the connection are cleaned up.</returns>
108+
private static async Task TerminateStreamingConnection(string connectionString, bool async, int delay)
109+
{
110+
SqlConnection connection = new(connectionString);
111+
SqlDataReader? reader = null;
112+
Task<Exception?>? consumer = null;
113+
using SqlCommand stream = new(
114+
"SELECT TOP (200000) a.object_id, b.name, REPLICATE('x', 200) AS pad " +
115+
"FROM sys.all_objects a CROSS JOIN sys.all_columns b", connection);
116+
stream.CommandTimeout = 60;
117+
try
118+
{
119+
if (async)
120+
{
121+
await connection.OpenAsync();
122+
}
123+
else
124+
{
125+
connection.Open();
126+
}
127+
128+
reader = async ? await stream.ExecuteReaderAsync() : stream.ExecuteReader();
129+
Assert.True(async ? await reader.ReadAsync() : reader.Read());
130+
consumer = Task.Run(() => Record.ExceptionAsync(async () =>
131+
{
132+
while (async ? await reader.ReadAsync() : reader.Read())
133+
{
134+
}
135+
}));
136+
await Task.Delay(delay);
137+
138+
using SqlCommand terminate = new(
139+
"RAISERROR('SqlClient MARS receive teardown regression', 20, 1) WITH LOG", connection);
140+
SqlException error;
141+
if (async)
142+
{
143+
error = await Assert.ThrowsAsync<SqlException>(() => terminate.ExecuteNonQueryAsync());
144+
}
145+
else
146+
{
147+
error = Assert.Throws<SqlException>(() => terminate.ExecuteNonQuery());
148+
}
149+
Assert.True(error.Class >= 20, "The second session must terminate the physical connection.");
150+
}
151+
finally
152+
{
153+
Exception? closeError;
154+
Exception? readerError;
155+
try
156+
{
157+
Exception? consumeError = consumer is null
158+
? null
159+
: await consumer.WaitAsync(TimeSpan.FromSeconds(65));
160+
AssertExpectedTeardownError(consumeError);
161+
}
162+
finally
163+
{
164+
closeError = Record.Exception(connection.Dispose);
165+
readerError = reader is null ? null : Record.Exception(reader.Dispose);
166+
}
167+
AssertExpectedTeardownError(closeError);
168+
AssertExpectedTeardownError(readerError);
169+
}
170+
}
171+
172+
/// <summary>
173+
/// Allows expected failures of commands on a broken connection, but rejects programming errors.
174+
/// </summary>
175+
/// <param name="exception">The observed application-side teardown exception, if any.</param>
176+
private static void AssertExpectedTeardownError(Exception? exception)
177+
{
178+
Assert.True(exception is null or SqlException or InvalidOperationException or IOException,
179+
$"Unexpected application-side teardown error: {exception}");
180+
}
181+
182+
/// <summary>
183+
/// Finalizes abandoned continuation tasks so their failures reach the event handler.
184+
/// </summary>
185+
private static void CollectReceiveContinuations()
186+
{
187+
GC.Collect();
188+
GC.WaitForPendingFinalizers();
189+
GC.Collect();
190+
}
191+
}
192+
}
193+
194+
#endif

0 commit comments

Comments
 (0)