From 88c22d406d20207cc4001f36bea3ed68d66812f1 Mon Sep 17 00:00:00 2001 From: Cheena Malhotra Date: Mon, 16 Nov 2026 13:08:34 -0700 Subject: [PATCH 1/5] Async AE: async enclave key loading and package retry routing Generate enclave packages with the async key store APIs after the describe round trip, revalidate the connection after each await, and route retryable package faults into the bounded enclave retry without transferring an already released reservation. Final layer of the #4585 split. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Data/SqlClient/EnclaveDelegate.Crypto.cs | 82 ++++- .../Data/SqlClient/EnclaveDelegate.cs | 102 +++++-- .../Data/SqlClient/SqlCommand.Reader.cs | 285 +++++++++++++----- .../Microsoft/Data/SqlClient/SqlCommand.cs | 13 +- .../ManualTests/AlwaysEncrypted/ApiShould.cs | 20 +- .../Data/SqlClient/EnclaveDelegateTests.cs | 253 ++++++++++++++++ .../Data/SqlClient/SqlCommandTests.cs | 221 +++++++++++++- .../SqlCommandEnclaveAsyncTests.cs | 147 ++++++++- 8 files changed, 991 insertions(+), 132 deletions(-) create mode 100644 src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/EnclaveDelegateTests.cs diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/EnclaveDelegate.Crypto.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/EnclaveDelegate.Crypto.cs index 2ece42d648..bab162cad2 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/EnclaveDelegate.Crypto.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/EnclaveDelegate.Crypto.cs @@ -248,11 +248,89 @@ internal EnclavePackage GenerateEnclavePackage(SqlConnectionAttestationProtocol throw new RetryableEnclaveQueryExecutionException(e.Message, e); } - List decryptedKeysToBeSentToEnclave = GetDecryptedKeysToBeSentToEnclave(keysToBeSentToEnclave, enclaveSessionParameters.ServerName, connection, command); + List decryptedKeysToBeSentToEnclave = GetDecryptedKeysToBeSentToEnclave(keysToBeSentToEnclave, connection, command); + return BuildEnclavePackage(decryptedKeysToBeSentToEnclave, queryText, counter, sqlEnclaveSession, enclaveSessionParameters.ServerName); + } + + /// + /// Generates an enclave package with asynchronous column encryption key decryption. + /// + /// + /// Async counterpart of . Column encryption key decryption is + /// awaited. The enclave session lookup stays synchronous: it reads the session cached by attestation + /// and does not attest, but a cache miss can wait on the provider's session lock. Package + /// cryptography is performed inline. + /// + /// attestation protocol + /// Keys to be sent to enclave + /// Text of the query being executed + /// enclave type + /// The set of parameters required for enclave session. + /// connection executing the query + /// command executing the query + /// Token used to request cancellation of the operation + internal async Task GenerateEnclavePackageAsync( + SqlConnectionAttestationProtocol attestationProtocol, + ConcurrentDictionary keysToBeSentToEnclave, + string queryText, + string enclaveType, + EnclaveSessionParameters enclaveSessionParameters, + SqlConnection connection, + SqlCommand command, + CancellationToken cancellationToken) + { + SqlEnclaveSession sqlEnclaveSession; + long counter; + + try + { + // Reads the session cached by attestation; a miss can wait on the provider's session lock. + GetEnclaveSession( + attestationProtocol, + enclaveType, + enclaveSessionParameters, + generateCustomData: false, + isRetry: false, + sqlEnclaveSession: out sqlEnclaveSession, + counter: out counter, + customData: out _, + customDataLength: out _, + throwIfNull: true + ); + } + catch (Exception e) + { + throw new RetryableEnclaveQueryExecutionException(e.Message, e); + } + + List decryptedKeysToBeSentToEnclave = await GetDecryptedKeysToBeSentToEnclaveAsync( + keysToBeSentToEnclave, + connection, + command, + cancellationToken) + .ConfigureAwait(false); + + return BuildEnclavePackage(decryptedKeysToBeSentToEnclave, queryText, counter, sqlEnclaveSession, enclaveSessionParameters.ServerName); + } + + /// + /// Assembles the enclave package from already-decrypted column encryption keys. + /// + /// + /// Shared by the synchronous and asynchronous package generation paths. All work here is local + /// cryptography, so there is no asynchronous counterpart. + /// + private EnclavePackage BuildEnclavePackage( + List decryptedKeysToBeSentToEnclave, + string queryText, + long counter, + SqlEnclaveSession sqlEnclaveSession, + string serverName) + { byte[] queryStringHashBytes = ComputeQueryStringHash(queryText); byte[] keyBytePackage = GenerateBytePackageForKeys(counter, queryStringHashBytes, decryptedKeysToBeSentToEnclave); byte[] sessionKey = sqlEnclaveSession.GetSessionKey(); - byte[] encryptedBytePackage = EncryptBytePackage(keyBytePackage, sessionKey, enclaveSessionParameters.ServerName); + byte[] encryptedBytePackage = EncryptBytePackage(keyBytePackage, sessionKey, serverName); byte[] enclaveSessionHandle = BitConverter.GetBytes(sqlEnclaveSession.SessionId); byte[] byteArrayToBeSentToEnclave = CombineByteArrays(enclaveSessionHandle, encryptedBytePackage); return new EnclavePackage(byteArrayToBeSentToEnclave, sqlEnclaveSession); diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/EnclaveDelegate.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/EnclaveDelegate.cs index 32bccd0873..409047c572 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/EnclaveDelegate.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/EnclaveDelegate.cs @@ -8,6 +8,8 @@ using System.Collections.Generic; using System.Security.Cryptography; using System.Text; +using System.Threading; +using System.Threading.Tasks; namespace Microsoft.Data.SqlClient { @@ -46,46 +48,90 @@ private byte[] GetUintBytes(string enclaveType, int intValue, string variableNam /// Decrypt the keys that need to be sent to the enclave /// /// Keys that need to sent to the enclave - /// - /// - /// + /// Connection executing the query + /// Command executing the query /// - private List GetDecryptedKeysToBeSentToEnclave(ConcurrentDictionary keysTobeSentToEnclave, string serverName, SqlConnection connection, SqlCommand command) + internal List GetDecryptedKeysToBeSentToEnclave(ConcurrentDictionary keysTobeSentToEnclave, SqlConnection connection, SqlCommand command) { - List decryptedKeysToBeSentToEnclave = new List(); + List decryptedKeysToBeSentToEnclave = new List(keysTobeSentToEnclave.Count); foreach (SqlTceCipherInfoEntry cipherInfo in keysTobeSentToEnclave.Values) { SqlSecurityUtility.DecryptSymmetricKey(cipherInfo, out SymmetricKey sqlClientSymmetricKey, out SqlEncryptionKeyInfo encryptionkeyInfoChosen, connection, command); - if (sqlClientSymmetricKey == null) - { - throw SQL.NullArgumentInternal(nameof(sqlClientSymmetricKey), nameof(EnclaveDelegate), nameof(GetDecryptedKeysToBeSentToEnclave)); - } - if (cipherInfo.ColumnEncryptionKeyValues == null) - { - throw SQL.NullArgumentInternal(nameof(cipherInfo.ColumnEncryptionKeyValues), nameof(EnclaveDelegate), nameof(GetDecryptedKeysToBeSentToEnclave)); - } - if (!(cipherInfo.ColumnEncryptionKeyValues.Count > 0)) - { - throw SQL.ColumnEncryptionKeysNotFound(); - } + decryptedKeysToBeSentToEnclave.Add(CreateColumnEncryptionKeyInfo(cipherInfo, sqlClientSymmetricKey)); + } + return decryptedKeysToBeSentToEnclave; + } - //cipherInfo.CekId is always 0, hence used cipherInfo.ColumnEncryptionKeyValues[0].cekId. Even when cek has multiple ColumnEncryptionKeyValues - //the cekid and the plaintext value will remain the same, what varies is the encrypted cek value, since the cek can be encrypted by - //multiple CMKs - decryptedKeysToBeSentToEnclave.Add( - new ColumnEncryptionKeyInfo( - sqlClientSymmetricKey.RootKey, - cipherInfo.ColumnEncryptionKeyValues[0].databaseId, - cipherInfo.ColumnEncryptionKeyValues[0].cekMdVersion, - cipherInfo.ColumnEncryptionKeyValues[0].cekId - ) - ); + /// + /// Asynchronously decrypts the keys that need to be sent to the enclave. + /// + /// + /// Async counterpart of . Each column encryption key is + /// resolved through + /// so that key store providers performing network I/O (for example Azure Key Vault) do not block a + /// thread while the enclave package is being assembled. + /// + /// Keys that need to sent to the enclave + /// Connection executing the query + /// Command executing the query + /// Token used to request cancellation of the operation + internal async Task> GetDecryptedKeysToBeSentToEnclaveAsync( + ConcurrentDictionary keysTobeSentToEnclave, + SqlConnection connection, + SqlCommand command, + CancellationToken cancellationToken) + { + List decryptedKeysToBeSentToEnclave = new List(keysTobeSentToEnclave.Count); + + foreach (SqlTceCipherInfoEntry cipherInfo in keysTobeSentToEnclave.Values) + { + (SymmetricKey sqlClientSymmetricKey, SqlEncryptionKeyInfo _) = + await SqlSecurityUtility.DecryptSymmetricKeyAsync(cipherInfo, connection, command, cancellationToken) + .ConfigureAwait(false); + + decryptedKeysToBeSentToEnclave.Add(CreateColumnEncryptionKeyInfo(cipherInfo, sqlClientSymmetricKey)); } + return decryptedKeysToBeSentToEnclave; } + /// + /// Validates a decrypted column encryption key and projects it into the shape the enclave expects. + /// + /// + /// Shared by the synchronous and asynchronous decryption paths so that both apply identical validation + /// and produce identical instances. + /// + private static ColumnEncryptionKeyInfo CreateColumnEncryptionKeyInfo( + SqlTceCipherInfoEntry cipherInfo, + SymmetricKey sqlClientSymmetricKey) + { + if (sqlClientSymmetricKey == null) + { + throw SQL.NullArgumentInternal(nameof(sqlClientSymmetricKey), nameof(EnclaveDelegate), nameof(CreateColumnEncryptionKeyInfo)); + } + if (cipherInfo.ColumnEncryptionKeyValues == null) + { + throw SQL.NullArgumentInternal(nameof(cipherInfo.ColumnEncryptionKeyValues), nameof(EnclaveDelegate), nameof(CreateColumnEncryptionKeyInfo)); + } + if (!(cipherInfo.ColumnEncryptionKeyValues.Count > 0)) + { + throw SQL.ColumnEncryptionKeysNotFound(); + } + + //cipherInfo.CekId is always 0, hence used cipherInfo.ColumnEncryptionKeyValues[0].cekId. Even when cek has multiple ColumnEncryptionKeyValues + //the cekid and the plaintext value will remain the same, what varies is the encrypted cek value, since the cek can be encrypted by + //multiple CMKs + return new ColumnEncryptionKeyInfo( + sqlClientSymmetricKey.RootKey, + cipherInfo.ColumnEncryptionKeyValues[0].databaseId, + cipherInfo.ColumnEncryptionKeyValues[0].cekMdVersion, + cipherInfo.ColumnEncryptionKeyValues[0].cekId + ); + } + /// /// Generate a byte package consisting of decrypted keys and some headers expected by the enclave /// diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.Reader.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.Reader.cs index 98e26bf62a..3004019909 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.Reader.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.Reader.cs @@ -904,10 +904,111 @@ private void FinishExecuteReader( private void GenerateEnclavePackage() { + if (!TryPrepareEnclavePackageGeneration( + out string enclaveType, + out SqlConnectionAttestationProtocol attestationProtocol)) + { + return; + } + + // Generate the enclave package + try + { + ThrowIfForcedRetryableEnclaveQueryExecutionException(); + + enclavePackage = EnclaveDelegate.Instance.GenerateEnclavePackage( + attestationProtocol, + keysToBeSentToEnclave, + CommandText, + enclaveType, + GetEnclaveSessionParameters(), + _activeConnection, + command: this); + } + catch (EnclaveDelegate.RetryableEnclaveQueryExecutionException) + { + throw; + } + catch (Exception e) + { + throw SQL.ExceptionWhenGeneratingEnclavePackage(e); + } + } + + /// + /// Asynchronously generates the enclave package for the current command. + /// + /// + /// Async counterpart of . Column encryption keys destined for the + /// enclave are decrypted through the asynchronous key store provider APIs, so a key store that performs + /// network I/O does not block a thread while the package is generated. + /// + /// Token used to request cancellation of the operation + private async Task GenerateEnclavePackageAsync(CancellationToken cancellationToken) + { + if (!TryPrepareEnclavePackageGeneration( + out string enclaveType, + out SqlConnectionAttestationProtocol attestationProtocol)) + { + return; + } + + // Generate the enclave package + try + { + ThrowIfForcedRetryableEnclaveQueryExecutionException(); + + enclavePackage = await EnclaveDelegate.Instance.GenerateEnclavePackageAsync( + attestationProtocol, + keysToBeSentToEnclave, + CommandText, + enclaveType, + GetEnclaveSessionParameters(), + _activeConnection, + command: this, + cancellationToken) + .ConfigureAwait(false); + } + catch (EnclaveDelegate.RetryableEnclaveQueryExecutionException) + { + throw; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + // Propagate cancellation unwrapped so the returned Task is cancelled rather than faulted + // with an enclave-specific exception that would suggest an attestation problem. + throw; + } + catch (Exception e) + { + throw SQL.ExceptionWhenGeneratingEnclavePackage(e); + } + } + + /// + /// Validates the connection state required to generate an enclave package. + /// + /// + /// Shared by the synchronous and asynchronous enclave package generation paths so that both perform + /// identical validation and throw identical exceptions. + /// + /// The validated enclave type reported by the server + /// The validated attestation protocol configured on the connection + /// + /// false when there are no keys to send to the enclave and package generation must be skipped; + /// otherwise true. + /// + private bool TryPrepareEnclavePackageGeneration( + out string enclaveType, + out SqlConnectionAttestationProtocol attestationProtocol) + { + enclaveType = null; + attestationProtocol = SqlConnectionAttestationProtocol.NotSpecified; + // Skip processing if there are no keys to send to enclave if (keysToBeSentToEnclave is null || keysToBeSentToEnclave.IsEmpty) { - return; + return false; } // Validate attestation url is provided when necessary @@ -919,49 +1020,42 @@ private void GenerateEnclavePackage() } // Validate enclave type - string enclaveType = _activeConnection.Parser.EnclaveType; + enclaveType = _activeConnection.Parser.EnclaveType; if (string.IsNullOrWhiteSpace(enclaveType)) { throw SQL.EnclaveTypeNullForEnclaveBasedQuery(); } // Validate protocol type - SqlConnectionAttestationProtocol attestationProtocol = _activeConnection.AttestationProtocol; + attestationProtocol = _activeConnection.AttestationProtocol; if (attestationProtocol is SqlConnectionAttestationProtocol.NotSpecified) { throw SQL.AttestationProtocolNotSpecifiedForGeneratingEnclavePackage(); } - // Generate the enclave package - try - { - #if DEBUG - // @TODO: These should be wrapped with something other than DEBUG since we don't even run tests in debug mode - // Test-only code for forcing a retryable exception to occur - if (_forceRetryableEnclaveQueryExecutionExceptionDuringGenerateEnclavePackage) - { - _forceRetryableEnclaveQueryExecutionExceptionDuringGenerateEnclavePackage = false; - throw new EnclaveDelegate.RetryableEnclaveQueryExecutionException("testing", null); - } - #endif + return true; + } - enclavePackage = EnclaveDelegate.Instance.GenerateEnclavePackage( - attestationProtocol, - keysToBeSentToEnclave, - CommandText, - enclaveType, - GetEnclaveSessionParameters(), - _activeConnection, - command: this); - } - catch (EnclaveDelegate.RetryableEnclaveQueryExecutionException) - { - throw; - } - catch (Exception e) + /// + /// Test-only failpoint that forces a retryable enclave failure during package generation. + /// + /// + /// Both guards are required and are not redundant: elides the + /// call sites in non-DEBUG builds, while the #if DEBUG is needed because the body still + /// compiles there and the field it reads is itself DEBUG-only. + /// + // @TODO: These should be wrapped with something other than DEBUG since we don't even run tests in debug mode + [Conditional("DEBUG")] + private void ThrowIfForcedRetryableEnclaveQueryExecutionException() + { + #if DEBUG + // Test-only code for forcing a retryable exception to occur + if (_forceRetryableEnclaveQueryExecutionExceptionDuringGenerateEnclavePackage) { - throw SQL.ExceptionWhenGeneratingEnclavePackage(e); + _forceRetryableEnclaveQueryExecutionExceptionDuringGenerateEnclavePackage = false; + throw new EnclaveDelegate.RetryableEnclaveQueryExecutionException("testing", null); } + #endif } private Task InternalExecuteReaderAsync( @@ -1840,57 +1934,29 @@ private SqlDataReader RunExecuteReaderTdsWithTransparentParameterEncryption( // @TODO: I guess this means async execution? Using tasks as the primary means of determining async vs sync is clunky. It would be better to have separate async vs sync pathways. long parameterEncryptionStart = ADP.TimerCurrent(); - // @TODO: This can totally be a non-generic TCS - // @TODO: This is a prime candidate for proper async-await execution - TaskCompletionSource completion = new TaskCompletionSource(); + // The remainder of the execution issues blocking TDS writes, so it must never run inline on + // the caller's thread (nor on a network callback thread). Task.Run reproduces the thread pool + // hand-off that the previous ContinueWith-based continuation provided. + // Capture the caller's cancellation token here, on the thread that started the + // execution. The body below runs after a thread pool hand-off and may observe the field + // already cleared by the execution's cleanup continuation. + CancellationToken cancellationToken = isAsync ? _asyncExecutionCancellationToken : CancellationToken.None; SqlConnectionInternal retryConnection = _enclaveRetryConnection; - AsyncHelper.ContinueTaskWithState( - taskToContinue: describeParameterEncryptionTask, - taskCompletionSource: completion, - state: this, - onSuccess: sqlCommand => - { - sqlCommand.ValidateConnectionForEnclaveExecution(retryConnection); - sqlCommand.GenerateEnclavePackage(); - sqlCommand.RunExecuteReaderTds( - cmdBehavior, - runBehavior, - returnStream, - isAsync, - TdsParserStaticMethods.GetRemainingTimeout(timeout, parameterEncryptionStart), - out Task subTask, - asyncWrite, - isRetry, - ds); - - if (subTask is null) - { - // @TODO: Why would this ever be the case? We should structure this so that it doesn't need to be checked. - completion.SetResult(null); - } - else - { - AsyncHelper.ContinueTaskWithState( - taskToContinue: subTask, - taskCompletionSource: completion, - state: completion, - onSuccess: static state => state.SetResult(null)); - } - }, - onFailure: static (sqlCommand, exception) => - { - sqlCommand.CachedAsyncState?.ResetAsyncState(); - if (exception is not null) - { - throw exception; - } - }, - onCancellation: static sqlCommand => - { - sqlCommand.CachedAsyncState?.ResetAsyncState(); - }); - task = completion.Task; + task = Task.Run(() => ContinueRunExecuteReaderTdsAsync( + describeParameterEncryptionTask, + cmdBehavior, + runBehavior, + returnStream, + isAsync, + timeout, + parameterEncryptionStart, + asyncWrite, + isRetry, + ds, + cancellationToken, + retryConnection)); + return ds; } else @@ -1910,6 +1976,67 @@ private SqlDataReader RunExecuteReaderTdsWithTransparentParameterEncryption( } } + /// + /// Completes transparent parameter encryption processing and then executes the command. + /// + /// + /// + /// Awaits the describe-parameter-encryption round trip, generates the enclave package using the + /// asynchronous key store provider APIs, and then issues the actual command execution. This replaces a + /// callback-based continuation chain so that key store network I/O yields the thread instead of + /// blocking it. + /// + /// + /// Error semantics match the previous continuation chain: a failure or cancellation of the + /// describe-parameter-encryption round trip resets the cached async state, while a failure that occurs + /// after that point is surfaced without resetting it. + /// + /// + private async Task ContinueRunExecuteReaderTdsAsync( + Task describeParameterEncryptionTask, + CommandBehavior cmdBehavior, + RunBehavior runBehavior, + bool returnStream, + bool isAsync, + int timeout, + long parameterEncryptionStart, + bool asyncWrite, + bool isRetry, + SqlDataReader ds, + CancellationToken cancellationToken, + SqlConnectionInternal retryConnection) + { + try + { + await describeParameterEncryptionTask.ConfigureAwait(false); + } + catch + { + CachedAsyncState?.ResetAsyncState(); + throw; + } + + ValidateConnectionForEnclaveExecution(retryConnection); + await GenerateEnclavePackageAsync(cancellationToken).ConfigureAwait(false); + + ValidateConnectionForEnclaveExecution(retryConnection); + RunExecuteReaderTds( + cmdBehavior, + runBehavior, + returnStream, + isAsync, + TdsParserStaticMethods.GetRemainingTimeout(timeout, parameterEncryptionStart), + out Task subTask, + asyncWrite, + isRetry, + ds); + + if (subTask is not null) + { + await subTask.ConfigureAwait(false); + } + } + private SqlDataReader RunExecuteReaderWithRetry( CommandBehavior cmdBehavior, RunBehavior runBehavior, diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.cs index c2aa34fc90..8c91142a62 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.cs @@ -2417,7 +2417,8 @@ private void CreateLocalCompletionTask( localCompletion.Task.ContinueWith( async task => { - if (task.IsFaulted) + if (task.IsFaulted && + task.Exception.InnerException is not EnclaveDelegate.RetryableEnclaveQueryExecutionException) { globalCompletion.TrySetException(task.Exception.InnerException); return; @@ -2431,6 +2432,9 @@ private void CreateLocalCompletionTask( try { + // Package generation can fail before the read stage; use the same bounded retry. + task.GetAwaiter().GetResult(); + // Mark that we initiated the internal EndExecute. This should always be false // until we set it here. Debug.Assert(!_internalEndExecuteInitiated); @@ -2495,8 +2499,11 @@ private void CreateLocalCompletionTask( // Remove the entry from the cache since it was inconsistent SqlQueryMetadataCache.GetInstance().InvalidateCacheEntry(this); - // Kick off the retry - SqlConnectionInternal retryConnection = _activeConnection.InnerConnection as SqlConnectionInternal; + // Kick off the retry. A faulted write released its reservation, so only a + // read-stage failure has one to transfer to the retry. + SqlConnectionInternal retryConnection = task.IsFaulted + ? null + : _activeConnection.InnerConnection as SqlConnectionInternal; Task retryTask = null; try { diff --git a/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/ApiShould.cs b/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/ApiShould.cs index 4aea616c06..2d655cf5c3 100644 --- a/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/ApiShould.cs +++ b/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/ApiShould.cs @@ -2761,6 +2761,10 @@ private async Task ExecuteEnclaveQueryAsync(string connectionString, int return session; } + /// + /// Sync and async enclave queries must retry stale cached sessions, including retryable + /// package-generation failures on warmed metadata cache entries in debug builds. + /// [ConditionalTheory(typeof(DataTestUtility), nameof(DataTestUtility.AreConnStringSetupForAE), nameof(DataTestUtility.EnclaveEnabled))] [ClassData(typeof(AEConnectionStringProvider))] public void TestRetryWhenAEEnclaveCacheIsStale(string connectionString) @@ -2821,21 +2825,6 @@ public void TestRetryWhenAEEnclaveCacheIsStale(string connectionString) Task readAsyncTask = ReadAsync(cmd, values, CommandBehavior.Default); readAsyncTask.GetAwaiter().GetResult(); - // TODO(GH-3604): This section fails on Linux: - // - // Failed Microsoft.Data.SqlClient.ManualTesting.Tests.AlwaysEncrypted.ApiShould.TestRetryWhenAEEnclaveCacheIsStale(connectionString: "Data Source=tcp:10.0.0.4;Database=NORTHWIND;UID=sa"ยทยทยท) [45 s] - // EXEC : error Message: [/mnt/vss/_work/1/s/build.proj] - // Microsoft.Data.SqlClient.EnclaveDelegate+RetryableEnclaveQueryExecutionException : testing - // Stack Trace: - // at Microsoft.Data.SqlClient.SqlCommand.EndExecuteReaderAsync(IAsyncResult asyncResult) in /_/src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/SqlCommand.netcore.cs:line 1025 - // at Microsoft.Data.SqlClient.SqlCommand.<>c.b__233_1(IAsyncResult asyncResult) in /_/src/Microsoft.Data.SqlClient/netcore/src/Microsoft/Data/SqlClient/SqlCommand.netcore.cs:line 1569 - // at System.Threading.Tasks.TaskFactory`1.FromAsyncCoreLogic(IAsyncResult iar, Func`2 endFunction, Action`1 endAction, Task`1 promise, Boolean requiresSynchronization) - // --- End of stack trace from previous location --- - // at Microsoft.Data.SqlClient.ManualTesting.Tests.AlwaysEncrypted.ApiShould.ReadAsync(SqlCommand sqlCommand, IList`1 values, CommandBehavior commandBehavior) in /_/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/ApiShould.cs:line 2776 - // at Microsoft.Data.SqlClient.ManualTesting.Tests.AlwaysEncrypted.ApiShould.TestRetryWhenAEEnclaveCacheIsStale(String connectionString) in /_/src/Microsoft.Data.SqlClient/tests/ManualTests/AlwaysEncrypted/ApiShould.cs:line 2563 - // at System.RuntimeMethodHandle.InvokeMethod(Object target, Void** arguments, Signature sig, Boolean isConstructor) - // at System.Reflection.MethodBaseInvoker.InvokeDirectByRefWithFewArgs(Object obj, Span`1 copyOfArgs, BindingFlags invokeAttr) - /* #if DEBUG CommandHelper.ForceThrowDuringGenerateEnclavePackage(cmd); @@ -2855,7 +2844,6 @@ public void TestRetryWhenAEEnclaveCacheIsStale(string connectionString) Task readAsyncTask2 = ReadAsync(cmd, values, CommandBehavior.Default); readAsyncTask2.GetAwaiter().GetResult(); #endif - */ // revert the CEK change to the CustomerId column cmd.Parameters.Clear(); diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/EnclaveDelegateTests.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/EnclaveDelegateTests.cs new file mode 100644 index 0000000000..63cb778217 --- /dev/null +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/EnclaveDelegateTests.cs @@ -0,0 +1,253 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +// Internal enclave signatures predate nullable annotations and accept null commands. +#nullable disable + +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using Microsoft.Data.SqlClient.UnitTests.Utilities; +using Xunit; + +namespace Microsoft.Data.SqlClient.UnitTests; + +/// +/// Verifies the column encryption key resolution used to build an enclave package. The package +/// itself needs an attested enclave session, so these tests cover the step that does key store +/// I/O. +/// +public class EnclaveDelegateTests +{ + private const string ProviderName = "TEST_ENCLAVE_ASYNC_PROVIDER"; + private const int DatabaseId = 42; + private const int CekId = 7; + private const ulong CekMdVersion = 3; + + /// + /// The async path must use the provider's async API so key store I/O does not block a pooled + /// thread, and must return the decrypted key with its identifiers. + /// + [Fact] + public async Task GetDecryptedKeysToBeSentToEnclaveAsync_OneKey_UsesAsyncProviderApi() + { + // Arrange + byte[] expectedKey = QueryMetadataCacheTestCommands.NewPlaintextKey(seed: 11); + TestKeyStoreProvider provider = new() { PlaintextKey = expectedKey }; + using SqlConnection connection = NewConnection(provider); + + // Act + List keys = + await EnclaveDelegate.Instance.GetDecryptedKeysToBeSentToEnclaveAsync( + NewKeyTable(NewCipherInfoEntry()), + connection, + command: null, + CancellationToken.None); + + // Assert + ColumnEncryptionKeyInfo key = Assert.Single(keys); + Assert.Equal(expectedKey, key.DecryptedKeyBytes); + Assert.Equal(BitConverter.GetBytes((uint)DatabaseId), key.DatabaseIdBytes); + Assert.Equal(BitConverter.GetBytes((ushort)CekId), key.KeyIdBytes); + Assert.Equal(CekMdVersion, key.KeyMetadataVersion); + Assert.Equal(1, provider.DecryptAsyncCallCount); + Assert.Equal(0, provider.DecryptCallCount); + } + + /// Every requested key is resolved, each through the async provider API. + [Fact] + public async Task GetDecryptedKeysToBeSentToEnclaveAsync_ThreeKeys_ResolvesEveryKey() + { + // Arrange + TestKeyStoreProvider provider = new() + { + PlaintextKey = QueryMetadataCacheTestCommands.NewPlaintextKey(seed: 23), + }; + using SqlConnection connection = NewConnection(provider); + ConcurrentDictionary table = NewKeyTable( + NewCipherInfoEntry(ordinal: 0), + NewCipherInfoEntry(ordinal: 1), + NewCipherInfoEntry(ordinal: 2)); + + // Act + List keys = + await EnclaveDelegate.Instance.GetDecryptedKeysToBeSentToEnclaveAsync( + table, connection, command: null, CancellationToken.None); + + // Assert + Assert.Equal(3, keys.Count); + Assert.Equal(3, provider.DecryptAsyncCallCount); + Assert.Equal(0, provider.DecryptCallCount); + } + + /// + /// Async key loads run one at a time, so a provider never sees overlapping calls for one + /// package. + /// + [Fact] + public async Task GetDecryptedKeysToBeSentToEnclaveAsync_ThreeKeys_LoadsSequentially() + { + // Arrange + int active = 0; + int maxActive = 0; + TestKeyStoreProvider provider = new() + { + DecryptAsyncCallback = async _ => + { + int current = Interlocked.Increment(ref active); + maxActive = Math.Max(maxActive, current); + await Task.Yield(); + Interlocked.Decrement(ref active); + return QueryMetadataCacheTestCommands.NewPlaintextKey(seed: 29); + }, + }; + using SqlConnection connection = NewConnection(provider); + ConcurrentDictionary table = NewKeyTable( + NewCipherInfoEntry(ordinal: 0), + NewCipherInfoEntry(ordinal: 1), + NewCipherInfoEntry(ordinal: 2)); + + // Act + List keys = + await EnclaveDelegate.Instance.GetDecryptedKeysToBeSentToEnclaveAsync( + table, connection, command: null, CancellationToken.None); + + // Assert + Assert.Equal(3, keys.Count); + Assert.Equal(1, maxActive); + } + + /// + /// The sync path keeps using the sync provider API, so providers that only override the sync + /// members keep working. + /// + [Fact] + public void GetDecryptedKeysToBeSentToEnclave_OneKey_UsesSyncProviderApi() + { + // Arrange + TestKeyStoreProvider provider = new() + { + PlaintextKey = QueryMetadataCacheTestCommands.NewPlaintextKey(seed: 37), + }; + using SqlConnection connection = NewConnection(provider); + + // Act + List keys = + EnclaveDelegate.Instance.GetDecryptedKeysToBeSentToEnclave( + NewKeyTable(NewCipherInfoEntry()), connection, command: null); + + // Assert + Assert.Single(keys); + Assert.Equal(1, provider.DecryptCallCount); + Assert.Equal(0, provider.DecryptAsyncCallCount); + } + + /// + /// A canceled token surfaces as cancellation, not as an Always Encrypted exception, and no + /// provider call is made. + /// + [Fact] + public async Task GetDecryptedKeysToBeSentToEnclaveAsync_CanceledToken_ThrowsOperationCanceled() + { + // Arrange + TestKeyStoreProvider provider = new() + { + PlaintextKey = QueryMetadataCacheTestCommands.NewPlaintextKey(seed: 53), + }; + using SqlConnection connection = NewConnection(provider); + using CancellationTokenSource source = new(); + source.Cancel(); + + // Act + Task action() => EnclaveDelegate.Instance.GetDecryptedKeysToBeSentToEnclaveAsync( + NewKeyTable(NewCipherInfoEntry()), connection, command: null, source.Token); + + // Assert + await Assert.ThrowsAnyAsync(action); + Assert.Equal(0, provider.DecryptAsyncCallCount); + } + + /// + /// A provider failure is wrapped as the sync path wraps it, instead of returning a partial key + /// list. + /// + [Fact] + public async Task GetDecryptedKeysToBeSentToEnclaveAsync_ProviderFails_ThrowsSqlException() + { + // Arrange + TestKeyStoreProvider provider = new() + { + DecryptAsyncCallback = + _ => throw new InvalidOperationException("key store unavailable"), + }; + using SqlConnection connection = NewConnection(provider); + + // Act + Task action() => EnclaveDelegate.Instance.GetDecryptedKeysToBeSentToEnclaveAsync( + NewKeyTable(NewCipherInfoEntry()), + connection, + command: null, + CancellationToken.None); + + // Assert + SqlException exception = await Assert.ThrowsAsync(action); + InvalidOperationException inner = Assert.IsType( + exception.InnerException); + Assert.Equal("key store unavailable", inner.Message); + } + + /// + /// Registers a provider on an unopened connection without global registration. + /// + /// Provider used to resolve this test's enclave keys. + /// An unopened connection that the caller must dispose. + private static SqlConnection NewConnection(SqlColumnEncryptionKeyStoreProvider provider) + { + SqlConnection connection = new( + "Data Source=async-ae-enclave-unit-test;Column Encryption Setting=Enabled"); + connection.RegisterColumnEncryptionKeyStoreProvidersOnConnection( + new Dictionary + { + [ProviderName] = provider + }); + return connection; + } + + /// + /// Creates a cipher entry with unique encrypted bytes, so the process-wide key cache cannot + /// return another test's key. + /// + /// Ordinal assigned to the entry. + /// A cipher entry with one encrypted key. + private static SqlTceCipherInfoEntry NewCipherInfoEntry(int ordinal = 0) + { + SqlTceCipherInfoEntry entry = new(ordinal); + entry.Add( + encryptedKey: Guid.NewGuid().ToByteArray(), + databaseId: DatabaseId, + cekId: CekId, + cekVersion: 1, + cekMdVersion: CekMdVersion, + keyPath: "test-enclave-key-path", + keyStoreName: ProviderName, + algorithmName: "RSA_OAEP"); + return entry; + } + + /// Builds an enclave key table with sequential ordinals. + /// Cipher metadata for the keys the enclave needs. + /// A key table for sync or async key resolution. + private static ConcurrentDictionary NewKeyTable( + params SqlTceCipherInfoEntry[] entries) + { + ConcurrentDictionary table = new(); + for (int i = 0; i < entries.Length; i++) + { + table[i] = entries[i]; + } + return table; + } +} diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlCommandTests.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlCommandTests.cs index 15d13392e9..02ed06f31e 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlCommandTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlCommandTests.cs @@ -4,9 +4,11 @@ using System; using System.Collections.Generic; +using System.Data; using System.Reflection; using System.Threading; using System.Threading.Tasks; +using Microsoft.Data.Common; using Microsoft.Data.SqlClient.Parser; using Microsoft.Data.SqlClient.UnitTests.Utilities; using Xunit; @@ -15,7 +17,8 @@ namespace Microsoft.Data.SqlClient.UnitTests; /// /// Verifies batch command lookup for SQL errors and info messages, asynchronous execution -/// cancellation state cleanup, and asynchronous query metadata cache dispatch. +/// cancellation state cleanup, asynchronous query metadata cache dispatch, and bounded retry +/// routing for enclave package failures. /// public class SqlCommandTests { @@ -337,6 +340,222 @@ public async Task PrepareForTransparentEncryption_AsyncCacheHitWithStaleKey_Fall Assert.False(SqlQueryMetadataCache.GetInstance().TryGetCachedQueryMetadata(command, out _)); } + /// + /// A retryable enclave package fault before the read stage must invalidate cached metadata and + /// retry once, completing with the retry's result. + /// + [Fact] + public async Task + CreateLocalCompletionTask_RetryableWriteFaultRetrySucceeds_ReturnsRetryResult() + { + // Arrange + using SqlCommand command = NewCachedRetryCommand(); + using SqlConnection connection = command.Connection; + object expectedResult = new(); + RetryObserver retry = new(command, Task.FromResult(expectedResult)); + TaskCompletionSource localCompletion = new(); + TaskCompletionSource globalCompletion = new(); + AttachCachedExecutionRetry(command, localCompletion, globalCompletion, retry.Retry); + + // Act + localCompletion.SetException(NewPackageFault()); + + // Assert + Assert.Same(expectedResult, await globalCompletion.Task); + Assert.Equal(1, retry.Calls); + } + + /// + /// A retryable enclave package fault retries once; a failed retry faults the caller's + /// completion with the retry's own exception instead of retrying again. + /// + [Fact] + public async Task CreateLocalCompletionTask_RetryableWriteFaultRetryFails_ThrowsRetryFailure() + { + // Arrange + using SqlCommand command = NewCachedRetryCommand(); + using SqlConnection connection = command.Connection; + EnclaveDelegate.RetryableEnclaveQueryExecutionException retryFailure = + new("retry failed", null); + RetryObserver retry = new(command, Task.FromException(retryFailure)); + TaskCompletionSource localCompletion = new(); + TaskCompletionSource globalCompletion = new(); + AttachCachedExecutionRetry(command, localCompletion, globalCompletion, retry.Retry); + + // Act + localCompletion.SetException(NewPackageFault()); + + // Assert + var exception = await Assert.ThrowsAsync< + EnclaveDelegate.RetryableEnclaveQueryExecutionException>(() => globalCompletion.Task); + Assert.Same(retryFailure, exception); + Assert.Equal(1, retry.Calls); + } + + /// + /// A retryable enclave package fault retries once; a canceled retry cancels the caller's + /// completion. + /// + [Fact] + public async Task CreateLocalCompletionTask_RetryableWriteFaultRetryCanceled_Cancels() + { + // Arrange + using SqlCommand command = NewCachedRetryCommand(); + using SqlConnection connection = command.Connection; + RetryObserver retry = new(command, Task.FromCanceled(new CancellationToken(true))); + TaskCompletionSource localCompletion = new(); + TaskCompletionSource globalCompletion = new(); + AttachCachedExecutionRetry(command, localCompletion, globalCompletion, retry.Retry); + + // Act + localCompletion.SetException(NewPackageFault()); + + // Assert + await Assert.ThrowsAnyAsync(() => globalCompletion.Task); + Assert.True(globalCompletion.Task.IsCanceled); + Assert.Equal(1, retry.Calls); + } + + /// + /// An ordinary write failure keeps its exception and does not trigger a retry. + /// + [Fact] + public async Task CreateLocalCompletionTask_NonRetryableWriteFault_ThrowsWithoutRetry() + { + // Arrange + using SqlConnection connection = new( + "Data Source=async-ae-unit-test;Column Encryption Setting=Enabled"); + using SqlCommand command = new("SELECT @p", connection); + RetryObserver retry = new(command, Task.FromResult(command)); + InvalidOperationException failure = new("write failed"); + TaskCompletionSource localCompletion = new(); + TaskCompletionSource globalCompletion = new(); + AttachCachedExecutionRetry(command, localCompletion, globalCompletion, retry.Retry); + + // Act + localCompletion.SetException(failure); + + // Assert + InvalidOperationException exception = + await Assert.ThrowsAsync(() => globalCompletion.Task); + Assert.Same(failure, exception); + Assert.Equal(0, retry.Calls); + } + + /// + /// A canceled write cancels the caller's completion without triggering a retry. + /// + [Fact] + public async Task CreateLocalCompletionTask_CanceledWrite_CancelsWithoutRetry() + { + // Arrange + using SqlConnection connection = new( + "Data Source=async-ae-unit-test;Column Encryption Setting=Enabled"); + using SqlCommand command = new("SELECT @p", connection); + RetryObserver retry = new(command, Task.FromResult(command)); + TaskCompletionSource localCompletion = new(); + TaskCompletionSource globalCompletion = new(); + AttachCachedExecutionRetry(command, localCompletion, globalCompletion, retry.Retry); + + // Act + localCompletion.SetCanceled(); + + // Assert + await Assert.ThrowsAnyAsync(() => globalCompletion.Task); + Assert.True(globalCompletion.Task.IsCanceled); + Assert.Equal(0, retry.Calls); + } + + /// + /// Creates a command whose query metadata is cached, so a retry must invalidate it. + /// + /// A cached command with an unopened connection; the caller disposes both. + private static SqlCommand NewCachedRetryCommand() + { + SqlCommand command = + QueryMetadataCacheTestCommands.NewCachedCommand(new TestKeyStoreProvider()); + Assert.True(SqlQueryMetadataCache.GetInstance().TryGetCachedQueryMetadata(command, out _)); + return command; + } + + /// Creates the fault raised when enclave package generation must be retried. + /// A retryable enclave exception. + private static EnclaveDelegate.RetryableEnclaveQueryExecutionException NewPackageFault() => + new("package generation failed", null); + + /// + /// Wires the command's cache-hit completion path with an End delegate that fails the test if a + /// failed write reaches the read stage. + /// + /// Command whose cached metadata may need invalidating. + /// Completion representing the initial write stage. + /// Completion returned to the caller. + /// Delegate observing the retry attempt. + private static void AttachCachedExecutionRetry( + SqlCommand command, + TaskCompletionSource localCompletion, + TaskCompletionSource globalCompletion, + Func + retry) + { + MethodInfo method = typeof(SqlCommand).GetMethod( + "CreateLocalCompletionTask", BindingFlags.Instance | BindingFlags.NonPublic) + ?? throw new InvalidOperationException("Async retry method not found."); + Func end = (_, _, _, _) => + throw new InvalidOperationException("A failed write must not enter the read stage."); + method.Invoke(command, new object?[] + { + CommandBehavior.Default, null, 30, true, true, + globalCompletion, localCompletion, end, retry, nameof(SqlCommand.EndExecuteReader), + ADP.TimerCurrent(), CancellationToken.None, + }); + } + + /// + /// Records retry attempts and checks that each one is an async-write retry of the original + /// command that starts only after its cached metadata was invalidated. + /// + private sealed class RetryObserver + { + private readonly SqlCommand _command; + private readonly Task _result; + + /// Creates an observer that completes every retry with the given task. + /// The command expected to be retried. + /// The task returned by the retry. + internal RetryObserver(SqlCommand command, Task result) + { + _command = command; + _result = result; + } + + /// Gets the number of retry attempts. + internal int Calls { get; private set; } + + /// Validates and records one retry attempt. + internal IAsyncResult Retry( + SqlCommand command, + CommandBehavior behavior, + AsyncCallback? callback, + object? state, + int timeout, + bool isRetry, + bool asyncWrite) + { + Calls++; + Assert.Same(_command, command); + Assert.Equal(CommandBehavior.Default, behavior); + Assert.Null(callback); + Assert.Null(state); + Assert.InRange(timeout, 0, 30); + Assert.True(isRetry); + Assert.True(asyncWrite); + Assert.False( + SqlQueryMetadataCache.GetInstance().TryGetCachedQueryMetadata(command, out _)); + return _result; + } + } + /// Creates RPC entries without requiring a network connection. /// The number of entries to add. /// A command owned by the test. diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SqlCommandEnclaveAsyncTests.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SqlCommandEnclaveAsyncTests.cs index 236641937c..ee4c2c2553 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SqlCommandEnclaveAsyncTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SqlCommandEnclaveAsyncTests.cs @@ -6,6 +6,7 @@ using System; using System.Collections.Concurrent; +using System.Collections.Generic; using System.Data; using System.Reflection; using System.Security.Cryptography; @@ -91,6 +92,116 @@ public async Task Execute_QueuedPreparation_RevalidatesConnection() Assert.Equal(-1, await other.ExecuteNonQueryAsync()); } + /// Package key loading must finish before queued execution checks for a competing reader. + [Fact] + public async Task Execute_PendingPackageKey_RevalidatesConnection() + { + // Arrange + // A warm session lets package generation reach the pending key load. + using Fixture fixture = new Fixture(); + using SqlCommand command = fixture.CreateCommand(); + fixture.Provider.SyncCacheHit = true; + PendingKeyStoreProvider keyProvider = new PendingKeyStoreProvider(); + fixture.Connection.RegisterColumnEncryptionKeyStoreProvidersOnConnection( + new Dictionary { ["PENDING_PACKAGE_KEY"] = keyProvider }); + SqlTceCipherInfoEntry entry = new SqlTceCipherInfoEntry(0); + entry.Add(Guid.NewGuid().ToByteArray(), 42, 7, 1, 3, + "pending-package-key", "PENDING_PACKAGE_KEY", "RSA_OAEP"); + command.keysToBeSentToEnclave = new ConcurrentDictionary(); + Assert.True(command.keysToBeSentToEnclave.TryAdd(0, entry)); + object[] arguments = + { + CommandBehavior.Default, RunBehavior.ReturnImmediately, true, true, 30, null, + true, false, null, Task.CompletedTask + }; + + // Act + using SqlDataReader pendingReader = (SqlDataReader)typeof(SqlCommand) + .GetMethod("RunExecuteReaderTdsWithTransparentParameterEncryption", BindingFlags.NonPublic | BindingFlags.Instance) + .Invoke(command, arguments); + Task execution = (Task)arguments[5]; + try + { + await await Task.WhenAny(keyProvider.Entered.Task, execution); + Assert.False(execution.IsCompleted); + + // Assert + // A reader opened while the key loads must stop the queued execution. + using SqlCommand other = new SqlCommand("/* competing package reader */", fixture.Connection); + using (SqlDataReader reader = other.ExecuteReader()) + { + keyProvider.Completion.SetResult(new byte[32]); + await Assert.ThrowsAsync(() => execution); + Assert.NotNull(GetField(command, "enclavePackage")); + Assert.Null(GetField(command, "_stateObj")); + Assert.False(reader.IsClosed); + } + Assert.Equal(0, GetAsyncCount((SqlConnectionInternal)fixture.Connection.InnerConnection)); + Assert.Equal(-1, await other.ExecuteNonQueryAsync()); + } + finally + { + // Cleanup + keyProvider.Completion.TrySetResult(new byte[32]); + } + } + + /// + /// A package fault happens after the failed write released its reservation, so the retry + /// must not inherit one: it revalidates the connection, and a failed retry does not + /// release a reservation twice. + /// + [Fact] + public async Task Retry_PackageFault_DoesNotTransferReservation() + { + // Arrange + using Fixture fixture = new Fixture(); + using SqlCommand command = fixture.CreateCommand(); + SqlConnectionInternal inner = (SqlConnectionInternal)fixture.Connection.InnerConnection; + var local = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + var global = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + InvalidOperationException retryFailure = new InvalidOperationException("retry failed"); + object retryConnection = inner; + int retries = 0; + Func end = + (cmd, result, internalEnd, name) => + throw new InvalidOperationException("A failed write must not be read."); + Func + retry = + (cmd, behavior, callback, state, timeout, isRetry, asyncWrite) => + { + retries++; + retryConnection = GetField(cmd, "_enclaveRetryConnection"); + return Task.FromException(retryFailure); + }; + typeof(SqlCommand) + .GetMethod( + "CreateLocalCompletionTask", + BindingFlags.NonPublic | BindingFlags.Instance) + .Invoke(command, new object[] + { + CommandBehavior.Default, null, 30, false, true, global, local, end, retry, + nameof(SqlCommand.EndExecuteReader), ADP.TimerCurrent(), CancellationToken.None + }); + + // Act + local.SetException( + new EnclaveDelegate.RetryableEnclaveQueryExecutionException("Package", null)); + + // Assert + Assert.Same( + retryFailure, + await Assert.ThrowsAsync(() => global.Task)); + Assert.Equal(1, retries); + Assert.Null(retryConnection); + Assert.Equal(0, GetAsyncCount(inner)); + using SqlCommand reusable = + new SqlCommand("/* reuse after package retry */", fixture.Connection); + Assert.Equal(-1, await reusable.ExecuteNonQueryAsync()); + } + /// Revalidation must not mistake an internal retry's own reservation for a competing command. [Theory] [InlineData(false)] @@ -923,15 +1034,37 @@ public void Dispose() } } - /// - /// Suspends one provider operation until explicitly faulted or canceled by its caller. - /// + /// Suspends enclave-package key loading without blocking the command caller. + private sealed class PendingKeyStoreProvider : SqlColumnEncryptionKeyStoreProvider + { + internal readonly TaskCompletionSource Entered = + new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + internal readonly TaskCompletionSource Completion = + new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + /// Signals that package generation reached asynchronous key loading. + public override Task DecryptColumnEncryptionKeyAsync( + string path, string algorithm, byte[] key, CancellationToken cancellationToken = default) + { + Entered.SetResult(true); + return Completion.Task; + } + + public override byte[] DecryptColumnEncryptionKey(string path, string algorithm, byte[] key) => + throw new InvalidOperationException("Package key loading must use the asynchronous provider API."); + + public override byte[] EncryptColumnEncryptionKey(string path, string algorithm, byte[] key) => + throw new NotSupportedException(); + } + + /// Suspends one provider operation until explicitly faulted or canceled by its caller. private sealed class PendingProvider : SqlColumnEncryptionEnclaveProvider { internal readonly InvalidOperationException Error = new InvalidOperationException("Provider failure"); internal bool Attestation; internal bool SessionOperation; internal bool CacheHit; + internal bool SyncCacheHit; internal Action OnCacheHit; internal readonly SqlEnclaveSession Session = new SqlEnclaveSession(new byte[32], 42); internal readonly byte[] CustomData = { 2, 3 }; @@ -1029,6 +1162,14 @@ internal override void GetEnclaveSession(EnclaveSessionParameters parameters, bo bool isRetry, out SqlEnclaveSession session, out long counter, out byte[] customData, out int customDataLength) { SyncCalls++; + if (SyncCacheHit) + { + session = Session; + counter = 42; + customData = CustomData; + customDataLength = CustomData.Length; + return; + } throw Error; } From aa9fadbf3455be64f273d120eaa08fb5a309a52b Mon Sep 17 00:00:00 2001 From: Cheena Malhotra Date: Mon, 16 Nov 2026 15:18:03 -0700 Subject: [PATCH 2/5] Return retry task from async XML enclave execution BeginExecuteXmlReader now attaches its callback to and returns the global completion task, like Reader and NonQuery, so callers observe the bounded enclave retry instead of the faulted first attempt. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Data/SqlClient/SqlCommand.Xml.cs | 8 +-- .../SqlCommandEnclaveAsyncTests.cs | 51 +++++++++++++++++++ 2 files changed, 55 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.Xml.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.Xml.cs index 9715f0a144..5444bce122 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.Xml.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.Xml.cs @@ -320,19 +320,19 @@ private IAsyncResult BeginExecuteXmlReaderInternal( endMethod: nameof(EndExecuteXmlReader), cancellationToken)) { - // @TODO: globalCompletion isn't being used past here! Why are we doing this?? globalCompletion = localCompletion; } - // Add callback after work is done to avoid overlapping Begin/End methods + // Add callback after work is done to avoid overlapping Begin/End methods. + // Callers must observe the retry's outcome, not a retried first attempt. if (callback is not null) { - localCompletion.Task.ContinueWith( + globalCompletion.Task.ContinueWith( static (task, state) => ((AsyncCallback)state)(task), state: callback); } - return localCompletion.Task; + return globalCompletion.Task; } finally { diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SqlCommandEnclaveAsyncTests.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SqlCommandEnclaveAsyncTests.cs index 17f1055614..97d6e4a756 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SqlCommandEnclaveAsyncTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SqlCommandEnclaveAsyncTests.cs @@ -942,6 +942,57 @@ public async Task ExecuteReaderAsync_InlineEnclaveRetry_AbortsCleanly(bool retry Assert.Equal(0, GetAsyncCount(inner)); } + /// + /// An XML execution whose first attempt faults with a retryable package failure stays + /// pending until the retry finishes, and then reports the retry's outcome. + /// + [Fact] + public async Task BeginExecuteXmlReader_RetryablePackageFault_CompletesWithRetry() + { + // Arrange + // Cached enclave keys send execution through async package generation, where the + // missing enclave session faults the first attempt with a retryable failure. + using Fixture fixture = new Fixture(); + using SqlCommand command = QueryMetadataCacheTestCommands.NewCommandWithCipherMetadata( + fixture.Connection, + new TestKeyStoreProvider(), + SqlCommandColumnEncryptionSetting.Enabled); + command.requiresEnclaveComputations = true; + command.keysToBeSentToEnclave = new ConcurrentDictionary(); + Assert.True(command.keysToBeSentToEnclave.TryAdd(0, new SqlTceCipherInfoEntry(0))); + SqlQueryMetadataCache.GetInstance().AddQueryMetadata( + command, ignoreQueriesWithReturnValueParams: true); + SqlConnectionInternal inner = (SqlConnectionInternal)fixture.Connection.InnerConnection; + + // Act + // The retry skips the cache and waits on its own attestation lookup. + IAsyncResult result = command.BeginExecuteXmlReader(); + await Task.WhenAny(fixture.Provider.RetryLookupEntered.Task, (Task)result); + bool completedDuringRetry = result.IsCompleted; + fixture.Provider.Fail(); + + // Assert + // The caller observes only the retry's outcome. + Assert.False(completedDuringRetry); + Assert.Equal(1, fixture.Provider.SyncCalls); + Assert.Same(fixture.Provider.Error, + await Assert.ThrowsAsync(() => (Task)result)); + Assert.Same( + fixture.Provider.Error, + Assert.Throws( + () => command.EndExecuteXmlReader(result))); + + // Command and connection state are released, so both remain usable. + Assert.Null(GetField(command, "_stateObj")); + Assert.Null(GetField(command, "_enclaveRetryConnection")); + Assert.False((bool)GetField(command, "_enclaveAttestationInProgress")); + Assert.False((bool)GetField(command, "_internalEndExecuteInitiated")); + Assert.Equal(0, GetAsyncCount(inner)); + using SqlCommand reusable = + new SqlCommand("/* reuse after XML enclave retry */", fixture.Connection); + Assert.Equal(-1, await reusable.ExecuteNonQueryAsync()); + } + /// /// The command retry cannot start before invalidation completes, and must not start after failure or cancellation. /// From d5269ee46a505230d1d3995a871289bece75740c Mon Sep 17 00:00:00 2001 From: Cheena Malhotra Date: Tue, 17 Nov 2026 13:40:47 -0700 Subject: [PATCH 3/5] Keep async package continuation when enclave keys are pending A completed preparation task was dropped even with enclave keys pending, so async execution generated the package synchronously. Keep the task in that case so key loading stays on the async reader continuation. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Data/SqlClient/SqlCommand.Encryption.cs | 6 ++- .../SqlCommandEnclaveAsyncTests.cs | 52 ++++++++++++++++++- 2 files changed, 54 insertions(+), 4 deletions(-) diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.Encryption.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.Encryption.cs index 115f228045..5758bcb978 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.Encryption.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.Encryption.cs @@ -734,8 +734,10 @@ private void PrepareForTransparentEncryptionCore( serializedAttestationParameters: null, cancellationToken); } - // Completed preparation must not queue otherwise unreserved TDS setup. - if (returnTask?.Status == TaskStatus.RanToCompletion) + // Completed preparation must not queue otherwise unreserved TDS setup. Keep the task + // when enclave keys are pending, so the reader continuation loads them asynchronously. + if (returnTask?.Status == TaskStatus.RanToCompletion + && (keysToBeSentToEnclave is null || keysToBeSentToEnclave.IsEmpty)) { returnTask = null; diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SqlCommandEnclaveAsyncTests.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SqlCommandEnclaveAsyncTests.cs index aa5cac8b4b..5f9277257b 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SqlCommandEnclaveAsyncTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SqlCommandEnclaveAsyncTests.cs @@ -510,6 +510,53 @@ public void Prepare_ParameterlessEnclaveBatch_WarmSessionCompletesInline() Assert.Equal(0, fixture.Provider.SyncCalls); } + /// + /// Completed preparation with pending enclave keys must keep its task, so package keys load + /// on the asynchronous reader continuation instead of the caller's thread. + /// + [Fact] + public void Prepare_CompletedWithPendingEnclaveKeys_KeepsAsyncContinuation() + { + // Arrange + // Seed a pending package key; a warm, parameterless batch makes preparation complete. + using Fixture fixture = new Fixture(); + using SqlCommand command = fixture.CreateCommand(); + fixture.Provider.CacheHit = true; + command.SetBatchRPCMode(true, 2); + command.AddBatchCommand(new SqlBatchCommand("SELECT 1", + columnEncryptionSetting: SqlCommandColumnEncryptionSetting.Enabled)); + command.AddBatchCommand(new SqlBatchCommand("SELECT 2", + columnEncryptionSetting: SqlCommandColumnEncryptionSetting.Enabled)); + command.SetBatchRPCModeReadyToExecute(); + ConcurrentDictionary keys = + new ConcurrentDictionary(); + keys.TryAdd(0, new SqlTceCipherInfoEntry(ordinal: 0)); + SetField(command, "keysToBeSentToEnclave", keys); + object[] arguments = + { + true, 30, new TaskCompletionSource(), null, true, false, false, + "ExecuteReader", CancellationToken.None + }; + + // Act + typeof(SqlCommand) + .GetMethod( + "PrepareForTransparentEncryption", + BindingFlags.NonPublic | BindingFlags.Instance) + .Invoke(command, arguments); + + // Assert + // The completed task and the in-progress flag both pass to the reader continuation. + Task prepareTask = Assert.IsAssignableFrom(arguments[3]); + Assert.Equal(TaskStatus.RanToCompletion, prepareTask.Status); + Assert.True((bool)GetField(command, "_enclaveAttestationInProgress")); + Assert.Equal(0, fixture.Provider.SyncCalls); + + // Cleanup + // Release the flag that the reader continuation would otherwise clear. + SetField(command, "_enclaveAttestationInProgress", false); + } + /// Independent fixtures must not share mutated options even when their server endpoint matches. [Fact] public void Fixture_SharedEndpoint_IsolatesConnectionOptions() @@ -915,7 +962,7 @@ public async Task PrepareEnclaveAttestationAsync_CacheHit_PreservesCustomData() } /// - /// Inline package failures await invalidation before retrying and release command state on either failure path. + /// Cached-key package failures await invalidation before retrying and release command state on either failure path. /// [Theory] [InlineData(false, false)] @@ -942,7 +989,8 @@ public async Task ExecuteReaderAsync_InlineEnclaveRetry_AbortsCleanly(bool retry SqlConnectionInternal inner = (SqlConnectionInternal)fixture.Connection.InnerConnection; Task operation = command.ExecuteReaderAsync(cancellation.Token); - Assert.True(fixture.Provider.Entered.Task.IsCompleted); + // Package generation runs on the reader continuation, so wait for invalidation. + await await Task.WhenAny(fixture.Provider.Entered.Task, operation); Assert.False(operation.IsCompleted); Assert.Equal(1, fixture.Provider.AsyncCalls); Assert.Equal(1, fixture.Provider.SyncCalls); From 9bc0d1987931d5079e5b02fa821d562783b23084 Mon Sep 17 00:00:00 2001 From: Cheena Malhotra Date: Tue, 17 Nov 2026 13:41:04 -0700 Subject: [PATCH 4/5] Wrap retry test summary Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../SimulatedServerTests/SqlCommandEnclaveAsyncTests.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SqlCommandEnclaveAsyncTests.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SqlCommandEnclaveAsyncTests.cs index 5f9277257b..a1c8bcaa76 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SqlCommandEnclaveAsyncTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SqlCommandEnclaveAsyncTests.cs @@ -962,7 +962,8 @@ public async Task PrepareEnclaveAttestationAsync_CacheHit_PreservesCustomData() } /// - /// Cached-key package failures await invalidation before retrying and release command state on either failure path. + /// Cached-key package failures await invalidation before retrying and release command state + /// on either failure path. /// [Theory] [InlineData(false, false)] From 6e231f7c5893a3eeb88f1b30f03dea61720ac207 Mon Sep 17 00:00:00 2001 From: Cheena Malhotra Date: Tue, 17 Nov 2026 14:15:46 -0700 Subject: [PATCH 5/5] Pass execution token explicitly to enclave retry Remove the per-command _asyncExecutionCancellationToken field. A rejected overlapping execution cleared it while the first execution was active, so a later retry lost cancellation. The RunExecuteReader token is now passed to RunExecuteReaderTdsWithTransparentParameterEncryption directly. Drop the field-only setup tests and add a regression test for the overlap case. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- .../Data/SqlClient/SqlCommand.NonQuery.cs | 5 - .../Data/SqlClient/SqlCommand.Reader.cs | 11 +-- .../Data/SqlClient/SqlCommand.Xml.cs | 5 - .../Microsoft/Data/SqlClient/SqlCommand.cs | 22 ----- .../Data/SqlClient/SqlCommandTests.cs | 75 --------------- .../SqlCommandEnclaveAsyncTests.cs | 95 ++++++++++++++++++- 6 files changed, 93 insertions(+), 120 deletions(-) diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.NonQuery.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.NonQuery.cs index 7cb9fd1a5f..a31f743ee0 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.NonQuery.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.NonQuery.cs @@ -336,8 +336,6 @@ private void BeginExecuteNonQueryInternalReadStage(TaskCompletionSource private void CleanupAfterExecuteNonQueryAsync(Task task, TaskCompletionSource source, Guid operationId) { - _asyncExecutionCancellationToken = default; - if (task.IsFaulted) { Exception e = task.Exception?.InnerException; @@ -711,8 +709,6 @@ private Task InternalExecuteNonQueryAsync(CancellationToken cancellationTok registration = cancellationToken.Register(callback: s_cancelIgnoreFailure, state: this); } - _asyncExecutionCancellationToken = cancellationToken; - Task returnedTask = source.Task; ExecuteNonQueryAsyncCallContext context = null; try @@ -756,7 +752,6 @@ private Task InternalExecuteNonQueryAsync(CancellationToken cancellationTok } catch (Exception e) { - _asyncExecutionCancellationToken = default; s_diagnosticListener.WriteCommandError(operationId, this, _transaction, e); if (context is not null) diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.Reader.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.Reader.cs index 93771345cf..4854f89fbb 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.Reader.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.Reader.cs @@ -601,8 +601,6 @@ private void CleanupExecuteReaderAsync( TaskCompletionSource source, Guid operationId) { - _asyncExecutionCancellationToken = default; - if (task.IsFaulted) { Exception e = task.Exception.InnerException; @@ -1103,8 +1101,6 @@ private Task InternalExecuteReaderAsync( registration = cancellationToken.Register(s_cancelIgnoreFailure, state: this); } - _asyncExecutionCancellationToken = cancellationToken; - Task returnedTask = source.Task; ExecuteReaderAsyncCallContext context = null; try @@ -1154,7 +1150,6 @@ private Task InternalExecuteReaderAsync( } catch (Exception e) { - _asyncExecutionCancellationToken = default; if (!_parentOperationStarted) { s_diagnosticListener.WriteCommandError(operationId, this, _transaction, e); @@ -1298,6 +1293,7 @@ private SqlDataReader RunExecuteReader( asyncWrite: asyncWrite && isAsync, isRetry, method, + isAsync ? cancellationToken : CancellationToken.None, ds: null, describeParameterEncryptionTask: prepareEncryptionTask); } @@ -1922,6 +1918,7 @@ private SqlDataReader RunExecuteReaderTdsWithTransparentParameterEncryption( bool asyncWrite, bool isRetry, string method, + CancellationToken cancellationToken, SqlDataReader ds = null, Task describeParameterEncryptionTask = null) // @TODO: This task should likely come from this method otherwise this is just setting up a continuation { @@ -1940,10 +1937,6 @@ private SqlDataReader RunExecuteReaderTdsWithTransparentParameterEncryption( // The remainder of the execution issues blocking TDS writes, so it must never run inline on // the caller's thread (nor on a network callback thread). Task.Run reproduces the thread pool // hand-off that the previous ContinueWith-based continuation provided. - // Capture the caller's cancellation token here, on the thread that started the - // execution. The body below runs after a thread pool hand-off and may observe the field - // already cleared by the execution's cleanup continuation. - CancellationToken cancellationToken = isAsync ? _asyncExecutionCancellationToken : CancellationToken.None; SqlConnectionInternal retryConnection = _enclaveRetryConnection; SqlConnection preparedConnection = _activeConnection; int preparedCloseCount = preparedConnection.CloseCount; diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.Xml.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.Xml.cs index 5444bce122..d83c446542 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.Xml.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.Xml.cs @@ -371,8 +371,6 @@ private void CleanupAfterExecuteXmlReaderAsync( TaskCompletionSource source, Guid operationId) { - _asyncExecutionCancellationToken = default; - if (task.IsFaulted) { Exception e = task.Exception?.InnerException; @@ -513,8 +511,6 @@ private Task InternalExecuteXmlReaderAsync(CancellationToken cancella registration = cancellationToken.Register(callback: s_cancelIgnoreFailure, state: this); } - _asyncExecutionCancellationToken = cancellationToken; - // @TODO: This can be cleaned up to lines if InnerConnection is always SqlInternalConnection ExecuteXmlReaderAsyncCallContext context = null; Task returnedTask = source.Task; @@ -564,7 +560,6 @@ private Task InternalExecuteXmlReaderAsync(CancellationToken cancella } catch (Exception e) { - _asyncExecutionCancellationToken = default; s_diagnosticListener.WriteCommandError(operationId, this, _transaction, e); if (context is not null) diff --git a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.cs b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.cs index c6cd4de840..e2ffaf569e 100644 --- a/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.cs +++ b/src/Microsoft.Data.SqlClient/src/Microsoft/Data/SqlClient/SqlCommand.cs @@ -129,28 +129,6 @@ private static class ProcParamsColumns /// private AsyncState _cachedAsyncState = null; - /// - /// Cancellation token supplied by the caller of the asynchronous execution that is currently in - /// flight, or when the command is executing synchronously. - /// - /// - /// - /// The asynchronous execution entry points reach the Always Encrypted machinery through several - /// layers of synchronous, callback-driven code that carry no cancellation token. Rather than - /// threading a token through every one of those signatures, including the synchronous paths that - /// would only ever pass , the token is recorded here for the - /// duration of the operation. It is intended only for the asynchronous Always Encrypted key - /// store calls, which are the sole cancellable I/O in that machinery. - /// - /// - /// Read this field only on the thread that started the execution, and pass the captured value on as - /// an explicit parameter. The Always Encrypted work is handed to the thread pool and can still be - /// running when the execution's cleanup continuation resets this field, so a read taken after that - /// hand-off may silently observe . - /// - /// - private CancellationToken _asyncExecutionCancellationToken; - private int _currentlyExecutingBatch; /// diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlCommandTests.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlCommandTests.cs index 6599769583..56f4263ff6 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlCommandTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/Microsoft/Data/SqlClient/SqlCommandTests.cs @@ -186,72 +186,6 @@ public void EndExecuteXmlReaderAsync_CanceledTask_ThrowsOperationCanceled() Assert.IsAssignableFrom(inner); } - /// - /// A missing or closed connection faults reader setup without retaining the caller's token, - /// even though setup fails before the cleanup continuation is registered. - /// - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task ExecuteReaderAsync_FailedSetup_ClearsCancellationToken(bool hasConnection) - { - // Arrange - using SqlConnection connection = new(); - using SqlCommand command = new("SELECT 1", hasConnection ? connection : null); - using CancellationTokenSource source = new(); - - // Act - await Assert.ThrowsAsync( - () => command.ExecuteReaderAsync(source.Token)); - - // Assert - Assert.Equal(CancellationToken.None, GetAsyncExecutionCancellationToken(command)); - } - - /// - /// A missing or closed connection faults non-query setup without retaining the caller's token, - /// even though setup fails before the cleanup continuation is registered. - /// - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task ExecuteNonQueryAsync_FailedSetup_ClearsCancellationToken(bool hasConnection) - { - // Arrange - using SqlConnection connection = new(); - using SqlCommand command = new("SELECT 1", hasConnection ? connection : null); - using CancellationTokenSource source = new(); - - // Act - await Assert.ThrowsAsync( - () => command.ExecuteNonQueryAsync(source.Token)); - - // Assert - Assert.Equal(CancellationToken.None, GetAsyncExecutionCancellationToken(command)); - } - - /// - /// A missing or closed connection faults XML reader setup without retaining the caller's - /// token, even though setup fails before the cleanup continuation is registered. - /// - [Theory] - [InlineData(false)] - [InlineData(true)] - public async Task ExecuteXmlReaderAsync_FailedSetup_ClearsCancellationToken(bool hasConnection) - { - // Arrange - using SqlConnection connection = new(); - using SqlCommand command = new("SELECT 1", hasConnection ? connection : null); - using CancellationTokenSource source = new(); - - // Act - await Assert.ThrowsAsync( - () => command.ExecuteXmlReaderAsync(source.Token)); - - // Assert - Assert.Equal(CancellationToken.None, GetAsyncExecutionCancellationToken(command)); - } - /// /// An async cache hit whose provider completes synchronously must still load the key on a /// worker, not inline on the caller's thread, and must publish the cached metadata. @@ -665,15 +599,6 @@ private static List<_SqlRPC> GetBatchEntries(SqlCommand command) => return exception.InnerException; } - /// Reads the private token captured for the in-flight async execution. - /// The command to inspect. - /// The captured cancellation token. - private static CancellationToken GetAsyncExecutionCancellationToken(SqlCommand command) => - Assert.IsType(typeof(SqlCommand) - .GetField( - "_asyncExecutionCancellationToken", BindingFlags.Instance | BindingFlags.NonPublic)! - .GetValue(command)); - /// /// Invokes the private asynchronous encryption preparation entry point for a first attempt, /// as async execution does, without opening a connection. diff --git a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SqlCommandEnclaveAsyncTests.cs b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SqlCommandEnclaveAsyncTests.cs index a1c8bcaa76..2b0ae6a82c 100644 --- a/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SqlCommandEnclaveAsyncTests.cs +++ b/src/Microsoft.Data.SqlClient/tests/UnitTests/SimulatedServerTests/SqlCommandEnclaveAsyncTests.cs @@ -76,7 +76,7 @@ public async Task Execute_QueuedPreparation_RevalidatesConnection() object[] arguments = { CommandBehavior.Default, RunBehavior.ReturnImmediately, true, true, 30, null, - true, false, "ExecuteReader", null, preparation.Task + true, false, "ExecuteReader", CancellationToken.None, null, preparation.Task }; using SqlDataReader pendingReader = (SqlDataReader)typeof(SqlCommand) .GetMethod("RunExecuteReaderTdsWithTransparentParameterEncryption", BindingFlags.NonPublic | BindingFlags.Instance) @@ -113,7 +113,7 @@ public async Task Execute_PendingPackageKey_RevalidatesConnection() object[] arguments = { CommandBehavior.Default, RunBehavior.ReturnImmediately, true, true, 30, null, - true, false, "ExecuteReader", null, Task.CompletedTask + true, false, "ExecuteReader", CancellationToken.None, null, Task.CompletedTask }; // Act @@ -373,7 +373,7 @@ private static Task StartPendingPackageExecution( object[] arguments = { CommandBehavior.Default, RunBehavior.ReturnImmediately, true, true, 30, null, - true, false, "ExecuteReader", null, Task.CompletedTask + true, false, "ExecuteReader", CancellationToken.None, null, Task.CompletedTask }; typeof(SqlCommand) .GetMethod( @@ -1041,6 +1041,87 @@ public async Task ExecuteReaderAsync_InlineEnclaveRetry_AbortsCleanly(bool retry Assert.Equal(0, GetAsyncCount(inner)); } + /// + /// An overlapping execution rejected while the bounded retry prepares does not detach the + /// original execution's token, so canceling it still releases the retry's package key load. + /// + [Fact] + public async Task ExecuteReaderAsync_RejectedOverlapDuringRetry_RetryKeepsCancellation() + { + // Arrange + // Cached key metadata whose session disappears faults the first attempt. + using Fixture fixture = new Fixture(); + using SqlCommand command = fixture.CreateCommand(); + using CancellationTokenSource cancellation = new CancellationTokenSource(); + command.SetBatchRPCMode(true, 1); + command.AddBatchCommand(new SqlBatchCommand("SELECT 1", + columnEncryptionSetting: SqlCommandColumnEncryptionSetting.Enabled)); + command.SetBatchRPCModeReadyToExecute(); + fixture.Provider.CacheHit = true; + fixture.Provider.OnCacheHit = () => + { + command.keysToBeSentToEnclave = + new ConcurrentDictionary(); + Assert.True(command.keysToBeSentToEnclave.TryAdd(0, new SqlTceCipherInfoEntry())); + SetField(command, "enclavePackage", + new EnclavePackage(Array.Empty(), fixture.Provider.Session)); + }; + SqlConnectionInternal inner = (SqlConnectionInternal)fixture.Connection.InnerConnection; + Task operation = command.ExecuteReaderAsync(cancellation.Token); + await await Task.WhenAny(fixture.Provider.Entered.Task, operation); + + // The retry finds a warm session, and its package needs a key held pending. + PendingKeyStoreProvider keyProvider = new PendingKeyStoreProvider(); + fixture.Connection.RegisterColumnEncryptionKeyStoreProvidersOnConnection( + new Dictionary + { + ["PENDING_PACKAGE_KEY"] = keyProvider + }); + SqlTceCipherInfoEntry entry = new SqlTceCipherInfoEntry(0); + entry.Add(Guid.NewGuid().ToByteArray(), 42, 7, 1, 3, + "pending-package-key", "PENDING_PACKAGE_KEY", "RSA_OAEP"); + Task second = null; + fixture.Provider.SyncCacheHit = true; + fixture.Provider.OnCacheHit = () => + { + command.keysToBeSentToEnclave = + new ConcurrentDictionary(); + Assert.True(command.keysToBeSentToEnclave.TryAdd(0, entry)); + + // Act + // Start an overlapping execution while the retry prepares. + second = command.ExecuteReaderAsync(); + }; + try + { + fixture.Provider.CompleteSessionOperation(); + await await Task.WhenAny(keyProvider.Entered.Task, operation); + CancellationToken keyToken = keyProvider.Token; + cancellation.Cancel(); + + // Assert + // The overlap is rejected, and the retry's key load still observes the original + // token, so canceling it releases the execution. + await Assert.ThrowsAsync(() => second); + Assert.Equal(cancellation.Token, keyToken); + await Assert.ThrowsAnyAsync(() => operation); + } + finally + { + // Cleanup + keyProvider.Completion.TrySetResult(new byte[32]); + } + + // Command and connection state are released, so the connection remains usable. + Assert.Null(GetField(command, "_stateObj")); + Assert.Null(GetField(command, "_enclaveRetryConnection")); + Assert.False((bool)GetField(command, "_enclaveAttestationInProgress")); + Assert.Equal(0, GetAsyncCount(inner)); + using SqlCommand reusable = + new SqlCommand("/* reuse after canceled retry */", fixture.Connection); + Assert.Equal(-1, await reusable.ExecuteNonQueryAsync()); + } + /// /// An XML execution whose first attempt faults with a retryable package failure stays /// pending until the retry finishes, and then reports the retry's outcome. @@ -1382,11 +1463,17 @@ private sealed class PendingKeyStoreProvider : SqlColumnEncryptionKeyStoreProvid new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); internal readonly TaskCompletionSource Completion = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + internal CancellationToken Token; - /// Signals that package generation reached asynchronous key loading. + /// + /// Signals that package generation reached asynchronous key loading, and cancels the + /// pending load when the caller's token is canceled. + /// public override Task DecryptColumnEncryptionKeyAsync( string path, string algorithm, byte[] key, CancellationToken cancellationToken = default) { + Token = cancellationToken; + cancellationToken.Register(() => Completion.TrySetCanceled(cancellationToken)); Entered.SetResult(true); return Completion.Task; }