Skip to content

Commit 1e0aa45

Browse files
github-actions[bot]cheenamalhotraCopilotpaulmedynski
authored
[6.1.9 Cherry-pick] Fix managed SNI MARS receive teardown race (#4949)
* Cherry-pick of #4680 requires manual resolution To resolve, run: git cherry-pick 266a32e * Fix managed SNI MARS receive teardown race (#4680) * Fix managed SNI MARS receive teardown race Handle disposed TCP and named-pipe streams through SNI errors and preserve MARS packet ownership during receive re-arm failure. Add deterministic and SQL-backed regression coverage for #4679. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Fix MARS teardown callbacks and regression test hangs Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * Address MARS teardown review feedback Keep consumed-packet cleanup outside the receive lock, add a lock-scope regression, and remove redundant manual-test collection isolation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> * [6.1] Adapt MARS teardown tests to parser namespace refactor Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> * Fix managed SNI regression prerequisites on Unix (#4949) Restrict the Windows managed-SNI configuration check to Windows so Unix can run the MARS teardown regression when SQL prerequisites are met. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --------- Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Cheena Malhotra <13396919+cheenamalhotra@users.noreply.github.com> Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Co-authored-by: Paul Medynski <31868385+paulmedynski@users.noreply.github.com>
1 parent 1025102 commit 1e0aa45

9 files changed

Lines changed: 842 additions & 16 deletions

File tree

‎src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/Parser/TdsParserStateObject.netcore.cs‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -365,7 +365,13 @@ public void ReadAsyncCallback(IntPtr key, PacketHandle packet, uint error)
365365
bool processFinallyBlock = true;
366366
try
367367
{
368-
Debug.Assert(CheckPacket(packet, source) && source != null, "AsyncResult null on callback");
368+
// Closing a MARS connection can clear a session's task before its error callback.
369+
// The late callback must still retire the pending callback count.
370+
Debug.Assert(
371+
(CheckPacket(packet, source) && source != null) ||
372+
(source == null && error != 0 && _parser.MARSOn &&
373+
(_parser.State == TdsParserState.Closed || _parser.State == TdsParserState.Broken)),
374+
"AsyncResult null on callback");
369375

370376
if (_parser.MARSOn)
371377
{

‎src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ManagedSni/SniMarsConnection.netcore.cs‎

Lines changed: 31 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -130,22 +130,35 @@ public uint ReceiveAsync(ref SniPacket packet)
130130
{
131131
using (TrySNIEventScope.Create(nameof(SniMarsConnection)))
132132
{
133-
if (packet != null)
133+
SniPacket previousPacket = packet;
134+
packet = null;
135+
try
134136
{
135-
ReturnPacket(packet);
137+
lock (DemuxerSync)
138+
{
139+
var response = _lowerHandle.ReceiveAsync(ref packet);
140+
if (response != TdsEnums.SNI_SUCCESS_IO_PENDING && packet == null)
141+
{
142+
// Immediate transport errors release the new packet. Keep the consumed
143+
// packet alive so HandleReceiveError can notify all sessions and release it.
144+
packet = previousPacket;
145+
previousPacket = null;
146+
}
136147
#if DEBUG
137-
SqlClientEventSource.Log.TrySNITraceEvent(nameof(SniMarsConnection), EventType.INFO, "MARS Session Id {0}, Packet {1} returned", args0: ConnectionId, args1: packet?._id);
148+
SqlClientEventSource.Log.TrySNITraceEvent(nameof(SniMarsConnection), EventType.INFO, "MARS Session Id {0}, Received new packet {1}", args0: ConnectionId, args1: packet?._id);
138149
#endif
139-
packet = null;
150+
return response;
151+
}
140152
}
141-
142-
lock (DemuxerSync)
153+
finally
143154
{
144-
var response = _lowerHandle.ReceiveAsync(ref packet);
155+
if (previousPacket != null)
156+
{
157+
ReturnPacket(previousPacket);
145158
#if DEBUG
146-
SqlClientEventSource.Log.TrySNITraceEvent(nameof(SniMarsConnection), EventType.INFO, "MARS Session Id {0}, Received new packet {1}", args0: ConnectionId, args1: packet?._id);
159+
SqlClientEventSource.Log.TrySNITraceEvent(nameof(SniMarsConnection), EventType.INFO, "MARS Session Id {0}, Packet {1} returned", args0: ConnectionId, args1: previousPacket._id);
147160
#endif
148-
return response;
161+
}
149162
}
150163
}
151164
}
@@ -170,7 +183,15 @@ public uint CheckConnection()
170183
/// </summary>
171184
public void HandleReceiveError(SniPacket packet)
172185
{
173-
Debug.Assert(Monitor.IsEntered(this), "HandleReceiveError was called without being locked.");
186+
Debug.Assert(Monitor.IsEntered(DemuxerSync), "HandleReceiveError was called without being locked.");
187+
if (_dataBytesLeft > 0 && _currentPacket != null)
188+
{
189+
SqlClientEventSource.Log.TrySNITraceEvent(nameof(SniMarsConnection), EventType.ERR, "MARS Session Id {0}, Returning incomplete packet after receive error, _dataBytesLeft {1}", args0: ConnectionId, args1: _dataBytesLeft);
190+
ReturnPacket(_currentPacket);
191+
_currentPacket = null;
192+
_dataBytesLeft = 0;
193+
}
194+
174195
foreach (SniMarsHandle handle in _sessions.Values)
175196
{
176197
if (packet.HasAsyncIOCompletionCallback)

‎src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ManagedSni/SniMarsHandle.netcore.cs‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -345,14 +345,24 @@ public void HandleReceiveError(SniPacket packet)
345345
// which should handle ownership of the packet because the individual mars handles are not aware of
346346
// each other and cannot know if they are the last one in the list and that it is safe to return the packet
347347

348+
bool notifyAsyncReceive;
348349
lock (_receivedPacketQueue)
349350
{
350351
_connectionError = SniLoadHandle.SingletonInstance.LastError;
351352
SqlClientEventSource.Log.TrySNITraceEvent(nameof(SniMarsHandle), EventType.ERR, "MARS Session Id {0}, _connectionError to be handled: {1}", args0: ConnectionId, args1: _connectionError);
352353
_packetEvent.Set();
354+
notifyAsyncReceive = _asyncReceives > 0;
355+
if (notifyAsyncReceive)
356+
{
357+
_asyncReceives--;
358+
}
353359
}
354360

355-
((TdsParserStateObject)_callbackObject).ReadAsyncCallback(PacketHandle.FromManagedPacket(packet), 1);
361+
// Sync and idle sessions observe _connectionError without an async callback.
362+
if (notifyAsyncReceive)
363+
{
364+
((TdsParserStateObject)_callbackObject).ReadAsyncCallback(PacketHandle.FromManagedPacket(packet), 1);
365+
}
356366
}
357367
}
358368

‎src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ManagedSni/SniNpHandle.netcore.cs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -196,7 +196,7 @@ public override uint Receive(out SniPacket packet, int timeout)
196196
try
197197
{
198198
packet = RentPacket(headerSize: 0, dataSize: _bufferSize);
199-
packet.ReadFromStream(_stream);
199+
packet.ReadFromStream(_stream ?? throw new ObjectDisposedException(nameof(SniNpHandle)));
200200
SqlClientEventSource.Log.TrySNITraceEvent(nameof(SniNpHandle), EventType.INFO, "Connection Id {0}, Rented and read packet, dataLeft {1}", args0: _connectionId, args1: packet?.DataLeft);
201201

202202
if (packet.Length == 0)
@@ -236,7 +236,9 @@ public override uint ReceiveAsync(ref SniPacket packet)
236236
packet.SetAsyncIOCompletionCallback(_receiveCallback);
237237
try
238238
{
239-
packet.ReadFromStreamAsync(_stream);
239+
// Capture once: Dispose can clear the field while a MARS receive is re-armed.
240+
Stream stream = _stream ?? throw new ObjectDisposedException(nameof(SniNpHandle));
241+
packet.ReadFromStreamAsync(stream);
240242
SqlClientEventSource.Log.TrySNITraceEvent(nameof(SniNpHandle), EventType.INFO, "Connection Id {0}, Rented and read packet asynchronously, dataLeft {1}", args0: _connectionId, args1: packet?.DataLeft);
241243
return TdsEnums.SNI_SUCCESS_IO_PENDING;
242244
}

‎src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/ManagedSni/SniTcpHandle.netcore.cs‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -892,6 +892,11 @@ public override uint Receive(out SniPacket packet, int timeoutInMilliseconds)
892892
return ReportTcpSNIError(0, SniCommon.ConnOpenFailedError, Strings.SNI_ERROR_10);
893893
}
894894

895+
if (_stream == null)
896+
{
897+
return ReportTcpSNIError(new ObjectDisposedException(nameof(SniTcpHandle)));
898+
}
899+
895900
try
896901
{
897902
if (timeoutInMilliseconds > 0)
@@ -1021,7 +1026,9 @@ public override uint ReceiveAsync(ref SniPacket packet)
10211026
packet.SetAsyncIOCompletionCallback(_receiveCallback);
10221027
try
10231028
{
1024-
packet.ReadFromStreamAsync(_stream);
1029+
// Capture once: Dispose can clear the field while a MARS receive is re-armed.
1030+
Stream stream = _stream ?? throw new ObjectDisposedException(nameof(SniTcpHandle));
1031+
packet.ReadFromStreamAsync(stream);
10251032
SqlClientEventSource.Log.TrySNITraceEvent(nameof(SniTcpHandle), EventType.INFO, "Connection Id {0}, Data received from stream asynchronously", args0: _connectionId);
10261033
return TdsEnums.SNI_SUCCESS_IO_PENDING;
10271034
}

‎src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/Parser/TdsParserStateObject.cs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2593,7 +2593,10 @@ private void CancelWritePacket()
25932593
internal TdsOperationStatus TryReadNetworkPacket()
25942594
{
25952595
#if DEBUG
2596-
Debug.Assert(!_shouldHaveEnoughData || _attentionSent, "Caller said there should be enough data, but we are currently reading a packet");
2596+
// Teardown can invalidate the reader's buffered-data estimate before this call.
2597+
Debug.Assert(!_shouldHaveEnoughData || _attentionSent ||
2598+
_parser.State == TdsParserState.Closed || _parser.State == TdsParserState.Broken,
2599+
"Caller said there should be enough data, but we are currently reading a packet");
25972600
#endif
25982601
TdsOperationStatus result = TdsOperationStatus.InvalidData;
25992602
if (_snapshot != null)

‎src/Microsoft.Data.SqlClient/tests/ManualTests/Microsoft.Data.SqlClient.ManualTesting.Tests.csproj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,7 @@
8181
<Compile Include="SQL\AsyncTest\BeginExecReaderAsyncTest.cs" />
8282
<Compile Include="SQL\AsyncTest\XmlReaderAsyncTest.cs" />
8383
<Compile Include="SQL\AsyncTest\AsyncCancelledConnectionsTest.cs" />
84+
<Compile Include="SQL\AsyncTest\MarsReceiveTeardownTest.cs" />
8485
<Compile Include="SQL\ConnectionTestWithSSLCert\HostNameInCertificateTest.cs" />
8586
</ItemGroup>
8687
<ItemGroup Condition="'$(TestSet)' == '' OR '$(TestSet)' == '2'">
Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
// Licensed to the .NET Foundation under one or more agreements.
2+
// The .NET Foundation licenses this file to you under the MIT license.
3+
// See the LICENSE file in the project root for more information.
4+
5+
#if NET
6+
7+
#nullable enable
8+
9+
using System;
10+
using System.Collections.Concurrent;
11+
using System.IO;
12+
using System.Threading.Tasks;
13+
using Xunit;
14+
15+
namespace Microsoft.Data.SqlClient.ManualTesting.Tests
16+
{
17+
/// <summary>
18+
/// Exercises GH#4679 with a streaming MARS session and a connection-terminating error
19+
/// on a second session. Requires a test SQL Server and a sysadmin login; each case
20+
/// writes a severity-20 error to the server log.
21+
/// </summary>
22+
[Trait("Set", "1")]
23+
public sealed class MarsReceiveTeardownTest
24+
{
25+
/// <summary>
26+
/// Physical teardown must report normal command errors rather than faulting an
27+
/// unobserved MARS receive continuation, with or without connection pooling.
28+
/// Each delay is a separate case rather than a long-running stress loop.
29+
/// </summary>
30+
[ConditionalTheory(typeof(MarsReceiveTeardownTest), nameof(CanRun))]
31+
[InlineData(false, false, 1)]
32+
[InlineData(false, true, 1)]
33+
[InlineData(true, false, 1)]
34+
[InlineData(true, true, 1)]
35+
[InlineData(false, false, 29)]
36+
[InlineData(false, true, 29)]
37+
[InlineData(true, false, 29)]
38+
[InlineData(true, true, 29)]
39+
public async Task FatalError_WhileMarsReaderStreams_DoesNotFaultReceivePump(bool async, bool pooling, int delay)
40+
{
41+
string connectionString = new SqlConnectionStringBuilder(DataTestUtility.TCPConnectionString)
42+
{
43+
MultipleActiveResultSets = true,
44+
Pooling = pooling,
45+
ConnectRetryCount = 0,
46+
ApplicationName = nameof(MarsReceiveTeardownTest),
47+
}.ConnectionString;
48+
ConcurrentQueue<Exception> unobserved = new();
49+
EventHandler<UnobservedTaskExceptionEventArgs> handler = (_, args) =>
50+
{
51+
foreach (Exception exception in args.Exception.Flatten().InnerExceptions)
52+
{
53+
if (exception.StackTrace?.Contains("ManagedSni.SniMarsConnection") == true)
54+
{
55+
unobserved.Enqueue(exception);
56+
args.SetObserved();
57+
}
58+
}
59+
};
60+
61+
TaskScheduler.UnobservedTaskException += handler;
62+
try
63+
{
64+
await TerminateStreamingConnection(connectionString, async, delay);
65+
66+
for (int attempt = 0; attempt < 10; attempt++)
67+
{
68+
CollectReceiveContinuations();
69+
await Task.Delay(50);
70+
}
71+
72+
Assert.Empty(unobserved);
73+
}
74+
finally
75+
{
76+
TaskScheduler.UnobservedTaskException -= handler;
77+
using SqlConnection poolKey = new(connectionString);
78+
SqlConnection.ClearPool(poolKey);
79+
}
80+
}
81+
82+
/// <summary>
83+
/// Requires managed SNI and a non-Azure SQL Server login allowed to raise fatal errors.
84+
/// </summary>
85+
/// <returns>Whether the configured login is a sysadmin on a supported test server.</returns>
86+
public static bool CanRun()
87+
{
88+
if (!DataTestUtility.AreConnStringsSetup() ||
89+
!DataTestUtility.IsNotAzureServer() ||
90+
(OperatingSystem.IsWindows() && !DataTestUtility.IsUsingManagedSNI()))
91+
{
92+
return false;
93+
}
94+
95+
using SqlConnection connection = new(DataTestUtility.TCPConnectionString);
96+
using SqlCommand command = new("SELECT IS_SRVROLEMEMBER('sysadmin')", connection);
97+
connection.Open();
98+
return command.ExecuteScalar() is int member && member == 1;
99+
}
100+
101+
/// <summary>
102+
/// Starts two MARS sessions, terminates only their connection, and observes all
103+
/// application tasks so any unobserved exception belongs to the driver's receive pump.
104+
/// </summary>
105+
/// <param name="connectionString">Connection string for the test server.</param>
106+
/// <param name="async">Whether to use asynchronous command and reader APIs.</param>
107+
/// <param name="delay">Delay before terminating the streaming connection, in milliseconds.</param>
108+
/// <returns>A task that completes after both sessions and the connection are cleaned up.</returns>
109+
private static async Task TerminateStreamingConnection(string connectionString, bool async, int delay)
110+
{
111+
SqlConnection connection = new(connectionString);
112+
SqlDataReader? reader = null;
113+
Task<Exception?>? consumer = null;
114+
using SqlCommand stream = new(
115+
"SELECT TOP (200000) a.object_id, b.name, REPLICATE('x', 200) AS pad " +
116+
"FROM sys.all_objects a CROSS JOIN sys.all_columns b", connection);
117+
stream.CommandTimeout = 60;
118+
try
119+
{
120+
if (async)
121+
{
122+
await connection.OpenAsync();
123+
}
124+
else
125+
{
126+
connection.Open();
127+
}
128+
129+
reader = async ? await stream.ExecuteReaderAsync() : stream.ExecuteReader();
130+
Assert.True(async ? await reader.ReadAsync() : reader.Read());
131+
consumer = Task.Run(() => Record.ExceptionAsync(async () =>
132+
{
133+
while (async ? await reader.ReadAsync() : reader.Read())
134+
{
135+
}
136+
}));
137+
await Task.Delay(delay);
138+
139+
using SqlCommand terminate = new(
140+
"RAISERROR('SqlClient MARS receive teardown regression', 20, 1) WITH LOG", connection);
141+
SqlException error;
142+
if (async)
143+
{
144+
error = await Assert.ThrowsAsync<SqlException>(() => terminate.ExecuteNonQueryAsync());
145+
}
146+
else
147+
{
148+
error = Assert.Throws<SqlException>(() => terminate.ExecuteNonQuery());
149+
}
150+
Assert.True(error.Class >= 20, "The second session must terminate the physical connection.");
151+
}
152+
finally
153+
{
154+
Exception? closeError;
155+
Exception? readerError;
156+
try
157+
{
158+
Exception? consumeError = consumer is null
159+
? null
160+
: await consumer.WaitAsync(TimeSpan.FromSeconds(65));
161+
AssertExpectedTeardownError(consumeError);
162+
}
163+
finally
164+
{
165+
closeError = Record.Exception(connection.Dispose);
166+
readerError = reader is null ? null : Record.Exception(reader.Dispose);
167+
}
168+
AssertExpectedTeardownError(closeError);
169+
AssertExpectedTeardownError(readerError);
170+
}
171+
}
172+
173+
/// <summary>
174+
/// Allows expected failures of commands on a broken connection, but rejects programming errors.
175+
/// </summary>
176+
/// <param name="exception">The observed application-side teardown exception, if any.</param>
177+
private static void AssertExpectedTeardownError(Exception? exception)
178+
{
179+
Assert.True(exception is null or SqlException or InvalidOperationException or IOException,
180+
$"Unexpected application-side teardown error: {exception}");
181+
}
182+
183+
/// <summary>
184+
/// Finalizes abandoned continuation tasks so their failures reach the event handler.
185+
/// </summary>
186+
private static void CollectReceiveContinuations()
187+
{
188+
GC.Collect();
189+
GC.WaitForPendingFinalizers();
190+
GC.Collect();
191+
}
192+
}
193+
}
194+
195+
#endif

0 commit comments

Comments
 (0)