diff --git a/src/Adaptive.Cluster.Tests/Client/AeronClusterAsyncConnectTest.cs b/src/Adaptive.Cluster.Tests/Client/AeronClusterAsyncConnectTest.cs index 0f490238..bdf70eca 100644 --- a/src/Adaptive.Cluster.Tests/Client/AeronClusterAsyncConnectTest.cs +++ b/src/Adaptive.Cluster.Tests/Client/AeronClusterAsyncConnectTest.cs @@ -172,6 +172,7 @@ public void ShouldCloseIngressPublicationsOnMembers() A.CallTo(() => _aeron.AsyncAddSubscription(_context.EgressChannel(), _context.EgressStreamId())) .Returns(subscriptionId); var subscription = A.Fake(); + A.CallTo(() => subscription.TryResolveChannelEndpointPort()).Returns("something"); A.CallTo(() => _aeron.GetSubscription(subscriptionId)).Returns(subscription); const int ingressStreamId = 878; @@ -206,24 +207,28 @@ public void ShouldCloseIngressPublicationsOnMembers() for (int i = 0; i < iterations; i++) { Assert.IsNull(asyncConnect.Poll()); - Assert.AreEqual(AsyncConnectState.CREATE_INGRESS_PUBLICATIONS, asyncConnect.State()); + Assert.AreEqual(AsyncConnectState.AWAIT_PUBLICATION_CONNECTED, asyncConnect.State()); } A.CallTo(() => _aeron.AsyncAddExclusivePublication("aeron:udp?endpoint=localhost:20000", ingressStreamId)) .MustHaveHappenedANumberOfTimesMatching(n => n <= 1); A.CallTo(() => _aeron.AsyncAddExclusivePublication("aeron:udp?endpoint=localhost:20001", ingressStreamId)) - .MustHaveHappened(iterations, Times.Exactly); + .MustHaveHappened(iterations - 1, Times.Exactly); A.CallTo(() => _aeron.AsyncAddExclusivePublication("aeron:udp?endpoint=localhost:20002", ingressStreamId)) .MustHaveHappenedANumberOfTimesMatching(n => n <= 1); A.CallTo(() => _aeron.GetExclusivePublication(publicationId1)).MustHaveHappened(2, Times.Exactly); - A.CallTo(() => _aeron.GetExclusivePublication(publicationId2)).MustHaveHappened(iterations, Times.Exactly); - A.CallTo(() => _aeron.GetExclusivePublication(publicationId3)).MustHaveHappened(iterations, Times.Exactly); + A.CallTo(() => _aeron.GetExclusivePublication(publicationId2)) + .MustHaveHappened(iterations - 1, Times.Exactly); + A.CallTo(() => _aeron.GetExclusivePublication(publicationId3)) + .MustHaveHappened(iterations - 1, Times.Exactly); asyncConnect.Dispose(); - A.CallTo(() => subscription.Dispose()).MustHaveHappened(); - A.CallTo(subscription).MustHaveHappenedOnceExactly(); - A.CallTo(() => publication1.Dispose()).MustHaveHappened(); + A.CallTo(() => subscription.TryResolveChannelEndpointPort()) + .MustHaveHappened(iterations - 1, Times.Exactly); + A.CallTo(() => subscription.Dispose()).MustHaveHappenedOnceExactly(); + A.CallTo(() => publication1.IsConnected).MustHaveHappened(iterations - 2, Times.Exactly); + A.CallTo(() => publication1.Dispose()).MustHaveHappenedOnceExactly(); A.CallTo(() => _context.Dispose()).MustHaveHappenedOnceExactly(); A.CallTo(() => _aeron.AsyncRemovePublication(publicationId3)) .MustHaveHappenedANumberOfTimesMatching(n => n <= 1); diff --git a/src/Adaptive.Cluster.Tests/Client/AeronClusterTest.cs b/src/Adaptive.Cluster.Tests/Client/AeronClusterTest.cs index bcad1030..6e2617ee 100644 --- a/src/Adaptive.Cluster.Tests/Client/AeronClusterTest.cs +++ b/src/Adaptive.Cluster.Tests/Client/AeronClusterTest.cs @@ -17,6 +17,7 @@ using System; using System.Collections.Generic; using Adaptive.Aeron; +using Adaptive.Aeron.Exceptions; using Adaptive.Aeron.LogBuffer; using Adaptive.Aeron.Protocol; using Adaptive.Agrona; @@ -214,6 +215,37 @@ public void ShouldStayConnectedAfterSuccessfulFailover(bool withIngressDisconnec Assert.IsFalse(_aeronCluster.Closed); } + [Test] + public void ShouldRetryNewLeaderIngressPublicationWhenSendChannelEndpointIsClosing() + { + long registrationId = _ingressPublication.RegistrationId; + A.CallTo(() => _aeron.GetExclusivePublication(registrationId)) + .Throws( + new RegistrationException( + registrationId, + (int)ErrorCode.RESOURCE_TEMPORARILY_UNAVAILABLE, + ErrorCode.RESOURCE_TEMPORARILY_UNAVAILABLE, + "send_channel_endpoint found in CLOSING state, please retry" + ) + ) + .Once() + .Then.Returns(_ingressPublication); + + MakeEgressSubscriptionDeliverNewLeaderEvent(); + + Assert.AreEqual(1, _aeronCluster.PollEgress()); + A.CallTo(() => + _egressListener.OnNewLeader( + ClusterSessionId, + _leadershipTermId, + _leaderMemberId, + IngressEndpoints + ) + ) + .MustHaveHappened(); + Assert.IsFalse(_aeronCluster.Closed); + } + [TestCase(false)] [TestCase(true)] public void ShouldCloseItselfWhenUnableToSendMessageForLongerThanNewLeaderConnectionTimeout( diff --git a/src/Adaptive.Cluster/Client/AeronCluster.cs b/src/Adaptive.Cluster/Client/AeronCluster.cs index 4a744959..96ab1d14 100644 --- a/src/Adaptive.Cluster/Client/AeronCluster.cs +++ b/src/Adaptive.Cluster/Client/AeronCluster.cs @@ -1087,11 +1087,28 @@ private Publication AddNewLeaderIngressPublication(Context ctx, string channel, long deadlineNs = _nanoClock.NanoTime() + ctx.MessageTimeoutNs(); do { - Publication publication = GetIngressPublication(ctx, registrationId); - if (null != publication) + if (NULL_VALUE == registrationId) { - return publication; + registrationId = AsyncAddIngressPublication(ctx, channel, streamId); } + + try + { + Publication publication = GetIngressPublication(ctx, registrationId); + if (null != publication) + { + return publication; + } + } + catch (RegistrationException ex) + { + registrationId = NULL_VALUE; + if (ErrorCode.RESOURCE_TEMPORARILY_UNAVAILABLE != ex.ErrorCode()) + { + throw; + } + } + _idleStrategy.Idle(ctx.RunAgentInvokers()); } while (_nanoClock.NanoTime() < deadlineNs); @@ -2447,10 +2464,13 @@ private void CreateIngressPublications() { _ingressPublication = GetIngressPublication(_ctx, _ingressRegistrationId); } - catch (RegistrationException) + catch (RegistrationException ex) { _ingressRegistrationId = NULL_VALUE; - throw; + if (ErrorCode.RESOURCE_TEMPORARILY_UNAVAILABLE != ex.ErrorCode()) + { + throw; + } } } else @@ -2461,27 +2481,7 @@ private void CreateIngressPublications() } else { - int count = 0; - foreach (MemberIngress member in _memberByIdMap.Values) - { - if (null != member._publication || null != member._publicationException) - { - count++; - } - else - { - if (NULL_VALUE == member._registrationId) - { - member.AsyncAddPublication(); - } - member.AsyncGetPublication(); - } - } - - if (_memberByIdMap.Count == count) - { - State(AWAIT_PUBLICATION_CONNECTED); - } + State(AWAIT_PUBLICATION_CONNECTED); } } @@ -2494,8 +2494,12 @@ private void AwaitPublicationConnected() { foreach (MemberIngress member in _memberByIdMap.Values) { - if (null == member._publication && NULL_VALUE != member._registrationId) + if (null == member._publication && null == member._publicationException) { + if (NULL_VALUE == member._registrationId) + { + member.AsyncAddPublication(); + } member.AsyncGetPublication(); } @@ -2728,7 +2732,11 @@ internal void AsyncGetPublication() } catch (RegistrationException ex) { - _publicationException = ex; + if (ErrorCode.RESOURCE_TEMPORARILY_UNAVAILABLE != ex.ErrorCode()) + { + _publicationException = ex; + } + _registrationId = NULL_VALUE; } }