diff --git a/packages/client-runtime/src/connection/supervisor.test.ts b/packages/client-runtime/src/connection/supervisor.test.ts index a925859049f..64ce6c04c05 100644 --- a/packages/client-runtime/src/connection/supervisor.test.ts +++ b/packages/client-runtime/src/connection/supervisor.test.ts @@ -949,6 +949,89 @@ describe("EnvironmentSupervisor", () => { }).pipe(Effect.provide(TestClock.layer())), ); + it.effect("reconnects only after consecutive foreground liveness probe timeouts", () => + Effect.gen(function* () { + const firstProbeStarted = yield* Deferred.make(); + const secondProbeStarted = yield* Deferred.make(); + const probeCount = yield* Ref.make(0); + const harness = yield* makeHarness({ + probe: () => + Ref.updateAndGet(probeCount, (count) => count + 1).pipe( + Effect.tap((count) => + Deferred.succeed(count === 1 ? firstProbeStarted : secondProbeStarted, undefined), + ), + Effect.andThen(Effect.never), + ), + }); + const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { + initiallyDesired: true, + }).pipe(Effect.provide(harness.dependencies)); + + yield* awaitState(supervisor.state, (state) => state.phase === "connected"); + yield* harness.wake("application-active"); + yield* Deferred.await(firstProbeStarted); + yield* TestClock.adjust("15 seconds"); + yield* Effect.yieldNow; + + expect((yield* SubscriptionRef.get(supervisor.state)).phase).toBe("connected"); + expect(yield* Ref.get(harness.sessionCount)).toBe(1); + expect(yield* Ref.get(harness.releaseCount)).toBe(0); + + yield* harness.wake("application-active"); + yield* Deferred.await(secondProbeStarted); + yield* TestClock.adjust("15 seconds"); + yield* awaitState(supervisor.state, (state) => state.phase === "backoff"); + + expect(yield* Ref.get(harness.sessionCount)).toBe(1); + expect(yield* Ref.get(harness.releaseCount)).toBe(1); + }).pipe(Effect.provide(TestClock.layer())), + ); + + it.effect("resets the foreground probe timeout count after a successful probe", () => + Effect.gen(function* () { + const firstProbeStarted = yield* Deferred.make(); + const secondProbeStarted = yield* Deferred.make(); + const thirdProbeStarted = yield* Deferred.make(); + const probeCount = yield* Ref.make(0); + const harness = yield* makeHarness({ + probe: () => + Ref.updateAndGet(probeCount, (count) => count + 1).pipe( + Effect.tap((count) => + count === 1 + ? Deferred.succeed(firstProbeStarted, undefined) + : count === 2 + ? Deferred.succeed(secondProbeStarted, undefined) + : count === 3 + ? Deferred.succeed(thirdProbeStarted, undefined) + : Effect.void, + ), + Effect.flatMap((count) => (count === 2 ? Effect.void : Effect.never)), + ), + }); + const supervisor = yield* EnvironmentSupervisor.make(TARGET_ENTRY, { + initiallyDesired: true, + }).pipe(Effect.provide(harness.dependencies)); + + yield* awaitState(supervisor.state, (state) => state.phase === "connected"); + yield* harness.wake("application-active"); + yield* Deferred.await(firstProbeStarted); + yield* TestClock.adjust("15 seconds"); + + yield* harness.wake("application-active"); + yield* Deferred.await(secondProbeStarted); + yield* Effect.yieldNow; + + yield* harness.wake("application-active"); + yield* Deferred.await(thirdProbeStarted); + yield* TestClock.adjust("15 seconds"); + yield* Effect.yieldNow; + + expect((yield* SubscriptionRef.get(supervisor.state)).phase).toBe("connected"); + expect(yield* Ref.get(harness.sessionCount)).toBe(1); + expect(yield* Ref.get(harness.releaseCount)).toBe(0); + }).pipe(Effect.provide(TestClock.layer())), + ); + it.effect("quickly times out a stalled mobile foreground liveness probe", () => Effect.gen(function* () { const harness = yield* makeHarness({ diff --git a/packages/client-runtime/src/connection/supervisor.ts b/packages/client-runtime/src/connection/supervisor.ts index 2a9c7519072..578559811c3 100644 --- a/packages/client-runtime/src/connection/supervisor.ts +++ b/packages/client-runtime/src/connection/supervisor.ts @@ -33,6 +33,7 @@ const RETRY_DELAYS_MS = [1_000, 2_000, 4_000, 8_000, 16_000] as const; const CONNECTION_ESTABLISHMENT_TIMEOUT = "15 seconds"; const CONNECTION_PROBE_TIMEOUT = "15 seconds"; const MOBILE_CONNECTION_PROBE_TIMEOUT = "3 seconds"; +const FOREGROUND_PROBE_TIMEOUTS_BEFORE_RECONNECT = 2; const BACKOFF_RESET_AFTER_MS = 30_000; interface SupervisorIntent { @@ -392,6 +393,7 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( const monitorConnectedLease = Effect.fnUntraced(function* ( lease: ConnectionDriver.EnvironmentConnectionLease, ) { + let foregroundProbeTimeoutCount = 0; for (;;) { const next = yield* Queue.take(signals); switch (next._tag) { @@ -415,19 +417,23 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( return true; } if (next.reason === "application-active" || next.reason === "application-active-probe") { + const tolerateProbeTimeout = next.reason === "application-active"; const probe = yield* lease.session.probe.pipe( + Effect.as(true), Effect.timeoutOrElse({ duration: next.reason === "application-active-probe" ? MOBILE_CONNECTION_PROBE_TIMEOUT : CONNECTION_PROBE_TIMEOUT, orElse: () => - Effect.fail( - new ConnectionTransientError({ - reason: "timeout", - detail: `${target.label} did not respond to a connection health check.`, - }), - ), + tolerateProbeTimeout + ? Effect.succeed(false) + : Effect.fail( + new ConnectionTransientError({ + reason: "timeout", + detail: `${target.label} did not respond to a connection health check.`, + }), + ), }), Effect.forkChild, ); @@ -441,7 +447,26 @@ export const make = Effect.fn("EnvironmentSupervisor.make")(function* ( ), ); if (probeEvent._tag === "ProbeCompleted") { - yield* probeEvent.exit; + const responded = yield* probeEvent.exit; + if (responded) { + foregroundProbeTimeoutCount = 0; + break; + } + foregroundProbeTimeoutCount += 1; + if (foregroundProbeTimeoutCount >= FOREGROUND_PROBE_TIMEOUTS_BEFORE_RECONNECT) { + return yield* new ConnectionTransientError({ + reason: "timeout", + detail: `${target.label} did not respond to a connection health check.`, + }); + } + yield* Effect.logWarning( + "Foreground connection health check timed out; keeping the existing session.", + ).pipe( + Effect.annotateLogs({ + "environment.id": target.environmentId, + "environment.label": target.label, + }), + ); break; } switch (probeEvent.signal._tag) {