diff --git a/src/Grpc/orchestrator_service.proto b/src/Grpc/orchestrator_service.proto index 0c34d986..cbcd648d 100644 --- a/src/Grpc/orchestrator_service.proto +++ b/src/Grpc/orchestrator_service.proto @@ -182,6 +182,7 @@ message EntityOperationSignaledEvent { google.protobuf.Timestamp scheduledTime = 3; google.protobuf.StringValue input = 4; google.protobuf.StringValue targetInstanceId = 5; // used only within histories, null in messages + TraceContext parentTraceContext = 6; } message EntityOperationCalledEvent { @@ -192,6 +193,7 @@ message EntityOperationCalledEvent { google.protobuf.StringValue parentInstanceId = 5; // used only within messages, null in histories google.protobuf.StringValue parentExecutionId = 6; // used only within messages, null in histories google.protobuf.StringValue targetInstanceId = 7; // used only within histories, null in messages + TraceContext parentTraceContext = 8; } message EntityLockRequestedEvent { @@ -318,6 +320,8 @@ message SendEntityMessageAction { EntityLockRequestedEvent entityLockRequested = 3; EntityUnlockSentEvent entityUnlockSent = 4; } + + TraceContext parentTraceContext = 5; } message OrchestratorAction { diff --git a/src/Shared/Grpc/ProtoUtils.cs b/src/Shared/Grpc/ProtoUtils.cs index 2412fac3..6bac27c5 100644 --- a/src/Shared/Grpc/ProtoUtils.cs +++ b/src/Shared/Grpc/ProtoUtils.cs @@ -415,6 +415,7 @@ internal static P.OrchestratorResponse ConstructOrchestratorResponse( out string requestId); entityConversionState.EntityRequestIds.Add(requestId); + sendAction.ParentTraceContext = CreateTraceContext(); switch (sendAction.EntityMessageTypeCase) { @@ -636,6 +637,9 @@ internal static void ToEntityBatchRequest( Id = Guid.Parse(op.EntityOperationSignaled.RequestId), Operation = op.EntityOperationSignaled.Operation, Input = op.EntityOperationSignaled.Input, + TraceContext = op.EntityOperationSignaled.ParentTraceContext is { } signalTc + ? new DistributedTraceContext(signalTc.TraceParent, signalTc.TraceState) + : null, }); operationInfos.Add(new P.OperationInfo { @@ -650,6 +654,9 @@ internal static void ToEntityBatchRequest( Id = Guid.Parse(op.EntityOperationCalled.RequestId), Operation = op.EntityOperationCalled.Operation, Input = op.EntityOperationCalled.Input, + TraceContext = op.EntityOperationCalled.ParentTraceContext is { } calledTc + ? new DistributedTraceContext(calledTc.TraceParent, calledTc.TraceState) + : null, }); operationInfos.Add(new P.OperationInfo { diff --git a/src/Shared/Grpc/Tracing/Schema.cs b/src/Shared/Grpc/Tracing/Schema.cs index 37777817..65741114 100644 --- a/src/Shared/Grpc/Tracing/Schema.cs +++ b/src/Shared/Grpc/Tracing/Schema.cs @@ -60,5 +60,10 @@ public static class Task /// The time at which the timer is scheduled to fire. /// public const string FireAt = "durabletask.fire_at"; + + /// + /// The name of the entity operation being executed. + /// + public const string EntityOperationName = "durabletask.entity.operation"; } } diff --git a/src/Shared/Grpc/Tracing/TraceActivityConstants.cs b/src/Shared/Grpc/Tracing/TraceActivityConstants.cs index cba4bf52..60472b1a 100644 --- a/src/Shared/Grpc/Tracing/TraceActivityConstants.cs +++ b/src/Shared/Grpc/Tracing/TraceActivityConstants.cs @@ -24,6 +24,11 @@ static class TraceActivityConstants /// public const string Event = "event"; + /// + /// The name of the activity that represents entity operation execution. + /// + public const string EntityOperation = "entity_operation"; + /// /// The name of the activity that represents timer operations. /// diff --git a/src/Shared/Grpc/Tracing/TraceHelper.cs b/src/Shared/Grpc/Tracing/TraceHelper.cs index 1283ff12..9886b46b 100644 --- a/src/Shared/Grpc/Tracing/TraceHelper.cs +++ b/src/Shared/Grpc/Tracing/TraceHelper.cs @@ -156,6 +156,58 @@ static class TraceHelper return newActivity; } + /// + /// Starts a new trace activity for executing an entity operation. + /// + /// The name of the entity. + /// The name of the operation being executed. + /// The instance ID of the entity. + /// The W3C traceparent header value from the parent orchestration. + /// The W3C tracestate header value from the parent orchestration. + /// + /// Returns a newly started with entity operation metadata, or null if tracing is not enabled. + /// + public static Activity? StartTraceActivityForEntityOperation( + string entityName, + string? operationName, + string instanceId, + string? traceParent, + string? traceState) + { + if (traceParent is null || !ActivityContext.TryParse( + traceParent, + traceState, + out ActivityContext activityContext)) + { + return null; + } + + string spanName = string.IsNullOrEmpty(operationName) + ? $"{TraceActivityConstants.EntityOperation}:{entityName}" + : $"{TraceActivityConstants.EntityOperation}:{entityName}:{operationName}"; + + Activity? newActivity = ActivityTraceSource.StartActivity( + spanName, + kind: ActivityKind.Server, + parentContext: activityContext); + + if (newActivity == null) + { + return null; + } + + newActivity.SetTag(Schema.Task.Type, TraceActivityConstants.EntityOperation); + newActivity.SetTag(Schema.Task.Name, entityName); + newActivity.SetTag(Schema.Task.InstanceId, instanceId); + + if (!string.IsNullOrEmpty(operationName)) + { + newActivity.SetTag(Schema.Task.EntityOperationName, operationName); + } + + return newActivity; + } + /// /// Emits a new trace activity for a (task) activity that successfully completes. /// @@ -202,6 +254,51 @@ public static void EmitTraceActivityForTaskFailed( activity?.Dispose(); } + /// + /// Emits a new trace activity for an entity operation that successfully completes. + /// + /// The ID of the associated orchestration. + /// The associated . + /// The associated . + public static void EmitTraceActivityForEntityOperationCompleted( + string? instanceId, + P.HistoryEvent? historyEvent, + P.EntityOperationCalledEvent? calledEvent) + { + Activity? activity = StartTraceActivityForSchedulingEntityOperation(instanceId, historyEvent, calledEvent); + + activity?.Dispose(); + } + + /// + /// Emits a new trace activity for an entity operation that fails. + /// + /// The ID of the associated orchestration. + /// The associated . + /// The associated . + /// The associated . + public static void EmitTraceActivityForEntityOperationFailed( + string? instanceId, + P.HistoryEvent? historyEvent, + P.EntityOperationCalledEvent? calledEvent, + P.EntityOperationFailedEvent? failedEvent) + { + Activity? activity = StartTraceActivityForSchedulingEntityOperation(instanceId, historyEvent, calledEvent); + + if (activity is null) + { + return; + } + + if (failedEvent != null) + { + string statusDescription = failedEvent.FailureDetails?.ErrorMessage ?? "Unspecified entity operation failure"; + activity.SetStatus(ActivityStatusCode.Error, statusDescription); + } + + activity.Dispose(); + } + /// /// Emits a new trace activity for sub-orchestration execution when the sub-orchestration /// completes successfully. @@ -409,6 +506,74 @@ static string CreateSpanName(string spanDescription, string? taskName, string? t return newActivity; } + /// + /// Starts a new trace activity for scheduling an entity operation. Represents the time between + /// enqueuing the entity operation message and it completing. + /// + /// The ID of the associated orchestration. + /// The associated . + /// The associated . + /// + /// Returns a newly started with entity operation metadata. + /// + static Activity? StartTraceActivityForSchedulingEntityOperation( + string? instanceId, + P.HistoryEvent? historyEvent, + P.EntityOperationCalledEvent? calledEvent) + { + if (calledEvent == null) + { + return null; + } + + string targetInstanceId = historyEvent?.EntityOperationCalled?.TargetInstanceId ?? string.Empty; + + // Extract entity name from instance ID (format: "@name@key") + string entityName = targetInstanceId; + if (targetInstanceId.Length > 1 && targetInstanceId[0] == '@') + { + int secondAt = targetInstanceId.IndexOf('@', 1); + if (secondAt > 1) + { + entityName = targetInstanceId.Substring(1, secondAt - 1); + } + } + string spanName = string.IsNullOrEmpty(calledEvent.Operation) + ? $"{TraceActivityConstants.EntityOperation}:{entityName}" + : $"{TraceActivityConstants.EntityOperation}:{entityName}:{calledEvent.Operation}"; + + Activity? newActivity = ActivityTraceSource.StartActivity( + spanName, + kind: ActivityKind.Client, + startTime: historyEvent?.Timestamp?.ToDateTimeOffset() ?? default, + parentContext: Activity.Current?.Context ?? default); + + if (newActivity == null) + { + return null; + } + + if (calledEvent.ParentTraceContext != null + && ActivityContext.TryParse( + calledEvent.ParentTraceContext.TraceParent, + calledEvent.ParentTraceContext?.TraceState, + out ActivityContext parentContext)) + { + newActivity.SetSpanId(parentContext.SpanId.ToString()); + } + + newActivity.AddTag(Schema.Task.Type, TraceActivityConstants.EntityOperation); + newActivity.AddTag(Schema.Task.Name, entityName); + newActivity.AddTag(Schema.Task.InstanceId, instanceId); + + if (!string.IsNullOrEmpty(calledEvent.Operation)) + { + newActivity.AddTag(Schema.Task.EntityOperationName, calledEvent.Operation); + } + + return newActivity; + } + /// /// Starts a new trace activity for sub-orchestrations. Represents the time between enqueuing /// the sub-orchestration message and it completing. diff --git a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs index a3aa3dab..1e1d9e05 100644 --- a/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs +++ b/src/Worker/Grpc/GrpcDurableTaskWorker.Processor.cs @@ -502,6 +502,14 @@ async Task OnRunOrchestratorAsync( return taskScheduledEvent; } + P.HistoryEvent? GetEntityOperationCalledEvent(string requestId) + { + return request + .PastEvents + .Where(x => x.EventTypeCase == P.HistoryEvent.EventTypeOneofCase.EntityOperationCalled) + .FirstOrDefault(x => x.EntityOperationCalled.RequestId == requestId); + } + foreach (var newEvent in request.NewEvents) { switch (newEvent.EventTypeCase) @@ -565,6 +573,33 @@ async Task OnRunOrchestratorAsync( newEvent.Timestamp.ToDateTime(), newEvent.TimerFired); break; + + case P.HistoryEvent.EventTypeOneofCase.EntityOperationCompleted: + { + P.HistoryEvent? entityCalledEvent = + GetEntityOperationCalledEvent( + newEvent.EntityOperationCompleted.RequestId); + + TraceHelper.EmitTraceActivityForEntityOperationCompleted( + request.InstanceId, + entityCalledEvent, + entityCalledEvent?.EntityOperationCalled); + break; + } + + case P.HistoryEvent.EventTypeOneofCase.EntityOperationFailed: + { + P.HistoryEvent? entityCalledEvent = + GetEntityOperationCalledEvent( + newEvent.EntityOperationFailed.RequestId); + + TraceHelper.EmitTraceActivityForEntityOperationFailed( + request.InstanceId, + entityCalledEvent, + entityCalledEvent?.EntityOperationCalled, + newEvent.EntityOperationFailed); + break; + } } } } @@ -865,6 +900,20 @@ async Task OnRunEntityBatchAsync( var coreEntityId = DTCore.Entities.EntityId.FromString(batchRequest.InstanceId!); EntityId entityId = new(coreEntityId.Name, coreEntityId.Key); + // Start a Server trace span for each entity operation in the batch. + // Each operation may come from a different orchestration with its own trace context, + // so we create individual spans to preserve correct parent-child relationships. + List traceActivities = batchRequest.Operations? + .Select(op => TraceHelper.StartTraceActivityForEntityOperation( + entityId.Name, + op.Operation, + batchRequest.InstanceId!, + op.TraceContext?.TraceParent, + op.TraceContext?.TraceState)) + .OfType() + .ToList() + ?? []; + TaskName name = new(entityId.Name); EntityBatchResult? batchResult; @@ -885,6 +934,9 @@ async Task OnRunEntityBatchAsync( { // we could not find the entity. This is considered an application error, // so we return a non-retriable error-OperationResult for each operation in the batch. + string errorMessage = $"No entity task named '{name}' was found."; + traceActivities.ForEach(a => a.SetStatus(ActivityStatusCode.Error, errorMessage)); + batchResult = new EntityBatchResult() { Actions = [], // no actions @@ -894,7 +946,7 @@ async Task OnRunEntityBatchAsync( { FailureDetails = new FailureDetails( errorType: "EntityTaskNotFound", - errorMessage: $"No entity task named '{name}' was found.", + errorMessage: errorMessage, stackTrace: null, innerFailure: null, isNonRetriable: true), @@ -913,6 +965,12 @@ async Task OnRunEntityBatchAsync( { FailureDetails = new FailureDetails(frameworkException), }; + + traceActivities.ForEach(a => a.SetStatus(ActivityStatusCode.Error, frameworkException.Message)); + } + finally + { + traceActivities.ForEach(a => a.Dispose()); } P.EntityBatchResult response = batchResult.ToEntityBatchResult( diff --git a/test/Worker/Grpc.Tests/ProtoUtilsTraceContextTests.cs b/test/Worker/Grpc.Tests/ProtoUtilsTraceContextTests.cs new file mode 100644 index 00000000..d130c54a --- /dev/null +++ b/test/Worker/Grpc.Tests/ProtoUtilsTraceContextTests.cs @@ -0,0 +1,378 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +using System.Diagnostics; +using DurableTask.Core; +using DurableTask.Core.Command; +using DurableTask.Core.Entities.OperationFormat; +using Newtonsoft.Json; +using P = Microsoft.DurableTask.Protobuf; + +namespace Microsoft.DurableTask.Worker.Grpc.Tests; + +public class ProtoUtilsTraceContextTests +{ + static readonly ActivitySource TestSource = new(nameof(ProtoUtilsTraceContextTests)); + + [Fact] + public void SendEntityMessage_SignalEntity_SetsParentTraceContext() + { + // Arrange + using ActivityListener listener = CreateListener(); + using Activity? orchestrationActivity = TestSource.StartActivity("TestOrchestration"); + orchestrationActivity.Should().NotBeNull(); + + string requestId = Guid.NewGuid().ToString(); + string entityInstanceId = "@counter@myKey"; + string eventData = JsonConvert.SerializeObject(new + { + op = "increment", + signal = true, + id = requestId, + }); + + SendEventOrchestratorAction sendEventAction = new() + { + Id = 1, + Instance = new OrchestrationInstance { InstanceId = entityInstanceId }, + EventName = "op", + EventData = eventData, + }; + + ProtoUtils.EntityConversionState entityConversionState = new(insertMissingEntityUnlocks: false); + + // Act + P.OrchestratorResponse response = ProtoUtils.ConstructOrchestratorResponse( + instanceId: "test-orchestration", + executionId: "exec-1", + customStatus: null, + actions: [sendEventAction], + completionToken: "token", + entityConversionState: entityConversionState, + orchestrationActivity: orchestrationActivity); + + // Assert + response.Actions.Should().ContainSingle(); + P.OrchestratorAction action = response.Actions[0]; + action.SendEntityMessage.Should().NotBeNull(); + action.SendEntityMessage.EntityOperationSignaled.Should().NotBeNull(); + action.SendEntityMessage.ParentTraceContext.Should().NotBeNull(); + action.SendEntityMessage.ParentTraceContext.TraceParent.Should().NotBeNullOrEmpty(); + action.SendEntityMessage.ParentTraceContext.TraceParent.Should().Contain( + orchestrationActivity!.TraceId.ToString()); + } + + [Fact] + public void SendEntityMessage_CallEntity_SetsParentTraceContext() + { + // Arrange + using ActivityListener listener = CreateListener(); + using Activity? orchestrationActivity = TestSource.StartActivity("TestOrchestration"); + orchestrationActivity.Should().NotBeNull(); + + string requestId = Guid.NewGuid().ToString(); + string entityInstanceId = "@counter@myKey"; + string eventData = JsonConvert.SerializeObject(new + { + op = "get", + signal = false, + id = requestId, + parent = "parent-instance", + }); + + SendEventOrchestratorAction sendEventAction = new() + { + Id = 1, + Instance = new OrchestrationInstance { InstanceId = entityInstanceId }, + EventName = "op", + EventData = eventData, + }; + + ProtoUtils.EntityConversionState entityConversionState = new(insertMissingEntityUnlocks: false); + + // Act + P.OrchestratorResponse response = ProtoUtils.ConstructOrchestratorResponse( + instanceId: "test-orchestration", + executionId: "exec-1", + customStatus: null, + actions: [sendEventAction], + completionToken: "token", + entityConversionState: entityConversionState, + orchestrationActivity: orchestrationActivity); + + // Assert + response.Actions.Should().ContainSingle(); + P.OrchestratorAction action = response.Actions[0]; + action.SendEntityMessage.Should().NotBeNull(); + action.SendEntityMessage.EntityOperationCalled.Should().NotBeNull(); + action.SendEntityMessage.ParentTraceContext.Should().NotBeNull(); + action.SendEntityMessage.ParentTraceContext.TraceParent.Should().NotBeNullOrEmpty(); + action.SendEntityMessage.ParentTraceContext.TraceParent.Should().Contain( + orchestrationActivity!.TraceId.ToString()); + } + + [Fact] + public void SendEntityMessage_NoOrchestrationActivity_DoesNotSetParentTraceContext() + { + // Arrange + string requestId = Guid.NewGuid().ToString(); + string entityInstanceId = "@counter@myKey"; + string eventData = JsonConvert.SerializeObject(new + { + op = "increment", + signal = true, + id = requestId, + }); + + SendEventOrchestratorAction sendEventAction = new() + { + Id = 1, + Instance = new OrchestrationInstance { InstanceId = entityInstanceId }, + EventName = "op", + EventData = eventData, + }; + + ProtoUtils.EntityConversionState entityConversionState = new(insertMissingEntityUnlocks: false); + + // Act + P.OrchestratorResponse response = ProtoUtils.ConstructOrchestratorResponse( + instanceId: "test-orchestration", + executionId: "exec-1", + customStatus: null, + actions: [sendEventAction], + completionToken: "token", + entityConversionState: entityConversionState, + orchestrationActivity: null); + + // Assert + response.Actions.Should().ContainSingle(); + P.OrchestratorAction action = response.Actions[0]; + action.SendEntityMessage.Should().NotBeNull(); + action.SendEntityMessage.ParentTraceContext.Should().BeNull(); + } + + [Fact] + public void SendEntityMessage_NoEntityConversionState_SendsAsSendEvent() + { + // Arrange + using ActivityListener listener = CreateListener(); + using Activity? orchestrationActivity = TestSource.StartActivity("TestOrchestration"); + + string requestId = Guid.NewGuid().ToString(); + string entityInstanceId = "@counter@myKey"; + string eventData = JsonConvert.SerializeObject(new + { + op = "increment", + signal = true, + id = requestId, + }); + + SendEventOrchestratorAction sendEventAction = new() + { + Id = 1, + Instance = new OrchestrationInstance { InstanceId = entityInstanceId }, + EventName = "op", + EventData = eventData, + }; + + // Act - no entityConversionState means entity events are NOT converted + P.OrchestratorResponse response = ProtoUtils.ConstructOrchestratorResponse( + instanceId: "test-orchestration", + executionId: "exec-1", + customStatus: null, + actions: [sendEventAction], + completionToken: "token", + entityConversionState: null, + orchestrationActivity: orchestrationActivity); + + // Assert - should be a SendEvent, not SendEntityMessage + response.Actions.Should().ContainSingle(); + P.OrchestratorAction action = response.Actions[0]; + action.SendEvent.Should().NotBeNull(); + action.SendEntityMessage.Should().BeNull(); + } + + [Fact] + public void SendEntityMessage_TraceContextHasUniqueSpanId() + { + // Arrange + using ActivityListener listener = CreateListener(); + using Activity? orchestrationActivity = TestSource.StartActivity("TestOrchestration"); + orchestrationActivity.Should().NotBeNull(); + + string entityInstanceId = "@counter@myKey"; + string eventData1 = JsonConvert.SerializeObject(new + { + op = "increment", + signal = true, + id = Guid.NewGuid().ToString(), + }); + + string eventData2 = JsonConvert.SerializeObject(new + { + op = "increment", + signal = true, + id = Guid.NewGuid().ToString(), + }); + + SendEventOrchestratorAction action1 = new() + { + Id = 1, + Instance = new OrchestrationInstance { InstanceId = entityInstanceId }, + EventName = "op", + EventData = eventData1, + }; + + SendEventOrchestratorAction action2 = new() + { + Id = 2, + Instance = new OrchestrationInstance { InstanceId = entityInstanceId }, + EventName = "op", + EventData = eventData2, + }; + + ProtoUtils.EntityConversionState entityConversionState = new(insertMissingEntityUnlocks: false); + + // Act + P.OrchestratorResponse response = ProtoUtils.ConstructOrchestratorResponse( + instanceId: "test-orchestration", + executionId: "exec-1", + customStatus: null, + actions: [action1, action2], + completionToken: "token", + entityConversionState: entityConversionState, + orchestrationActivity: orchestrationActivity); + + // Assert - each entity message should get a unique span ID + response.Actions.Should().HaveCount(2); + string traceParent1 = response.Actions[0].SendEntityMessage.ParentTraceContext.TraceParent; + string traceParent2 = response.Actions[1].SendEntityMessage.ParentTraceContext.TraceParent; + traceParent1.Should().NotBeNullOrEmpty(); + traceParent2.Should().NotBeNullOrEmpty(); + + // Same trace ID (from orchestration activity) + traceParent1.Should().Contain(orchestrationActivity!.TraceId.ToString()); + traceParent2.Should().Contain(orchestrationActivity.TraceId.ToString()); + + // Different span IDs + traceParent1.Should().NotBe(traceParent2); + } + + static ActivityListener CreateListener() + { + ActivityListener listener = new() + { + ShouldListenTo = _ => true, + Sample = (ref ActivityCreationOptions _) => ActivitySamplingResult.AllDataAndRecorded, + }; + + ActivitySource.AddActivityListener(listener); + return listener; + } + + [Fact] + public void ToEntityBatchRequest_SignalEntity_ExtractsTraceContext() + { + // Arrange + string traceParent = "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"; + string traceState = "vendor=value"; + + P.EntityRequest entityRequest = new() + { + InstanceId = "@counter@myKey", + OperationRequests = + { + new P.HistoryEvent + { + EntityOperationSignaled = new P.EntityOperationSignaledEvent + { + RequestId = Guid.NewGuid().ToString(), + Operation = "increment", + ParentTraceContext = new P.TraceContext + { + TraceParent = traceParent, + TraceState = traceState, + }, + }, + }, + }, + }; + + // Act + entityRequest.ToEntityBatchRequest(out EntityBatchRequest batchRequest, out _); + + // Assert + batchRequest.Operations.Should().ContainSingle(); + batchRequest.Operations[0].TraceContext.Should().NotBeNull(); + batchRequest.Operations[0].TraceContext!.TraceParent.Should().Be(traceParent); + batchRequest.Operations[0].TraceContext!.TraceState.Should().Be(traceState); + } + + [Fact] + public void ToEntityBatchRequest_CallEntity_ExtractsTraceContext() + { + // Arrange + string traceParent = "00-0af7651916cd43dd8448eb211c80319c-b7ad6b7169203331-01"; + string traceState = "vendor=value"; + + P.EntityRequest entityRequest = new() + { + InstanceId = "@counter@myKey", + OperationRequests = + { + new P.HistoryEvent + { + EntityOperationCalled = new P.EntityOperationCalledEvent + { + RequestId = Guid.NewGuid().ToString(), + Operation = "get", + ParentInstanceId = "parent-instance", + ParentExecutionId = "parent-exec", + ParentTraceContext = new P.TraceContext + { + TraceParent = traceParent, + TraceState = traceState, + }, + }, + }, + }, + }; + + // Act + entityRequest.ToEntityBatchRequest(out EntityBatchRequest batchRequest, out _); + + // Assert + batchRequest.Operations.Should().ContainSingle(); + batchRequest.Operations[0].TraceContext.Should().NotBeNull(); + batchRequest.Operations[0].TraceContext!.TraceParent.Should().Be(traceParent); + batchRequest.Operations[0].TraceContext!.TraceState.Should().Be(traceState); + } + + [Fact] + public void ToEntityBatchRequest_NoTraceContext_LeavesTraceContextNull() + { + // Arrange + P.EntityRequest entityRequest = new() + { + InstanceId = "@counter@myKey", + OperationRequests = + { + new P.HistoryEvent + { + EntityOperationSignaled = new P.EntityOperationSignaledEvent + { + RequestId = Guid.NewGuid().ToString(), + Operation = "increment", + }, + }, + }, + }; + + // Act + entityRequest.ToEntityBatchRequest(out EntityBatchRequest batchRequest, out _); + + // Assert + batchRequest.Operations.Should().ContainSingle(); + batchRequest.Operations[0].TraceContext.Should().BeNull(); + } +}