diff --git a/engine/packages/engine/tests/envoy/actors_alarm.rs b/engine/packages/engine/tests/envoy/actors_alarm.rs index 91800ba80a..a34c4e5e52 100644 --- a/engine/packages/engine/tests/envoy/actors_alarm.rs +++ b/engine/packages/engine/tests/envoy/actors_alarm.rs @@ -516,69 +516,6 @@ impl Actor for AlarmOnceActor { } } -/// Actor that sets an alarm, sleeps on gen 2, then crashes immediately on wake. -/// Gen 1+ stays running. Used to test that alarms don't persist across generations. -struct AlarmSleepThenCrashActor { - alarm_offset_ms: i64, - sleeping_tx: tokio::sync::mpsc::UnboundedSender, - crash_tx: tokio::sync::mpsc::UnboundedSender, -} - -impl AlarmSleepThenCrashActor { - fn new( - alarm_offset_ms: i64, - sleeping_tx: tokio::sync::mpsc::UnboundedSender, - crash_tx: tokio::sync::mpsc::UnboundedSender, - ) -> Self { - Self { - alarm_offset_ms, - sleeping_tx, - crash_tx, - } - } -} - -#[async_trait] -impl Actor for AlarmSleepThenCrashActor { - async fn on_start(&mut self, config: ActorConfig) -> anyhow::Result { - let generation = config.generation; - tracing::info!(?config.actor_id, generation, "alarm crash actor starting"); - - if generation == 1 { - // First start (gen 2): set alarm, and crash - let alarm_time = get_current_timestamp_ms() + self.alarm_offset_ms; - config.send_set_alarm(alarm_time); - - // Notify test - let _ = self.crash_tx.send(generation); - - tracing::info!(generation, "set alarm and sleeping"); - Ok(ActorStartResult::Crash { - code: 1, - message: "crashing with gen 2".to_string(), - }) - } else if generation == 2 { - tracing::info!(generation, "restarted after crash, sending sleep intent"); - config.send_sleep_intent(); - let _ = self.sleeping_tx.send(generation); - Ok(ActorStartResult::Running) - } else { - // If it restarted again, this was not expected - // - // Keep the actor running so the test finds out we're not asleep. - Ok(ActorStartResult::Running) - } - } - - async fn on_stop(&mut self) -> anyhow::Result { - Ok(ActorStopResult::Success) - } - - fn name(&self) -> &str { - "AlarmSleepThenCrashActor" - } -} - /// Actor that rapidly sets and clears alarms multiple times before sleeping (generation 2 only). /// Used to test that rapid operations don't cause errors. struct RapidAlarmCycleActor { @@ -707,7 +644,7 @@ fn basic_alarm() { &namespace, "alarm-actor", runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, + rivet_types::actors::CrashPolicy::Sleep, ) .await; @@ -756,7 +693,7 @@ fn clear_alarm_prevents_wake() { &namespace, "alarm-actor", runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, + rivet_types::actors::CrashPolicy::Sleep, ) .await; @@ -813,7 +750,7 @@ fn replace_alarm_overwrites_previous() { &namespace, "alarm-actor", runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, + rivet_types::actors::CrashPolicy::Sleep, ) .await; @@ -872,7 +809,7 @@ fn alarm_in_the_past() { &namespace, "alarm-actor", runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, + rivet_types::actors::CrashPolicy::Sleep, ) .await; @@ -909,58 +846,55 @@ fn alarm_in_the_past() { #[test] fn alarm_with_null_timestamp() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; - let (ready_tx, ready_rx) = tokio::sync::oneshot::channel(); - let ready_tx = Arc::new(Mutex::new(Some(ready_tx))); + let (ready_tx, ready_rx) = tokio::sync::oneshot::channel(); + let ready_tx = Arc::new(Mutex::new(Some(ready_tx))); - let runner = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { - builder.with_actor_behavior("alarm-actor", move |_| { - let ready_tx = ready_tx.clone(); - Box::new(SetClearAlarmAndSleepActor::new(ready_tx)) - }) + let runner = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { + builder.with_actor_behavior("alarm-actor", move |_| { + let ready_tx = ready_tx.clone(); + Box::new(SetClearAlarmAndSleepActor::new(ready_tx)) }) - .await; + }) + .await; - let res = common::create_actor( - ctx.leader_dc().guard_port(), - &namespace, - "alarm-actor", - runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, - ) - .await; + let res = common::create_actor( + ctx.leader_dc().guard_port(), + &namespace, + "alarm-actor", + runner.pool_name(), + rivet_types::actors::CrashPolicy::Sleep, + ) + .await; - let actor_id = res.actor.actor_id.to_string(); + let actor_id = res.actor.actor_id.to_string(); - // Wait for actor to be ready - ready_rx.await.expect("actor should send ready signal"); + // Wait for actor to be ready + ready_rx.await.expect("actor should send ready signal"); - // Verify actor is sleeping - wait_for_actor_sleep(ctx.leader_dc().guard_port(), &actor_id, &namespace, 5) - .await - .expect("actor is not sleeping"); + // Verify actor is sleeping + wait_for_actor_sleep(ctx.leader_dc().guard_port(), &actor_id, &namespace, 5) + .await + .expect("actor is not sleeping"); - // Wait past alarm time - tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; + // Wait past alarm time + tokio::time::sleep(tokio::time::Duration::from_secs(3)).await; - // Verify actor is still sleeping - let actor = common::try_get_actor(ctx.leader_dc().guard_port(), &actor_id, &namespace) - .await - .expect("failed to get actor") - .expect("actor should exist"); + // Verify actor is still sleeping + let actor = common::try_get_actor(ctx.leader_dc().guard_port(), &actor_id, &namespace) + .await + .expect("failed to get actor") + .expect("actor should exist"); - assert!( - actor.sleep_ts.is_some(), - "actor should still be sleeping after alarm was cleared with null" - ); + assert!( + actor.sleep_ts.is_some(), + "actor should still be sleeping after alarm was cleared with null" + ); - tracing::info!(?actor_id, "null alarm_ts successfully cleared alarm"); - }, - ); + tracing::info!(?actor_id, "null alarm_ts successfully cleared alarm"); + }); } // MARK: Edge Cases @@ -968,7 +902,6 @@ fn alarm_with_null_timestamp() { #[test] // Broken legacy Pegboard Runner test: full engine sweep observed the 5s alarm // firing after 6.07s, outside the ±500ms assertion window. -#[ignore = "broken legacy Pegboard Runner test: alarm timing drifts in full engine sweep"] fn alarm_fires_at_correct_time() { common::run( common::TestOpts::new(1).with_timeout(10), @@ -991,7 +924,7 @@ fn alarm_fires_at_correct_time() { &namespace, "alarm-actor", runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, + rivet_types::actors::CrashPolicy::Sleep, ) .await; @@ -1056,7 +989,7 @@ fn multiple_alarm_sets_before_sleep() { &namespace, "alarm-actor", runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, + rivet_types::actors::CrashPolicy::Sleep, ) .await; @@ -1095,7 +1028,6 @@ fn multiple_alarm_sets_before_sleep() { #[test] // Broken legacy Pegboard Runner test: full engine sweep timed out in // `multiple_sleep_wake_alarm_cycles`. -#[ignore = "broken legacy Pegboard Runner test: times out in full engine sweep"] fn multiple_sleep_wake_alarm_cycles() { common::run(common::TestOpts::new(1), |ctx| async move { let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; @@ -1116,7 +1048,7 @@ fn multiple_sleep_wake_alarm_cycles() { &namespace, "alarm-actor", runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, + rivet_types::actors::CrashPolicy::Sleep, ) .await; @@ -1162,7 +1094,7 @@ fn alarm_wake_then_sleep_without_new_alarm() { &namespace, "alarm-actor", runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, + rivet_types::actors::CrashPolicy::Sleep, ) .await; @@ -1204,86 +1136,6 @@ fn alarm_wake_then_sleep_without_new_alarm() { // MARK: Advanced Usage -#[ignore = "non-sleep crash policies are not yet supported for envoys"] -#[test] -fn alarm_behavior_with_crash_policy_restart() { - common::run( - common::TestOpts::new(1).with_timeout(45), - |ctx| async move { - let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; - - let (sleeping_tx, mut sleeping_rx) = tokio::sync::mpsc::unbounded_channel(); - let (crash_tx, mut crash_rx) = tokio::sync::mpsc::unbounded_channel(); - - let runner = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { - builder.with_actor_behavior("alarm-actor", move |_| { - let sleeping_tx = sleeping_tx.clone(); - let crash_tx = crash_tx.clone(); - // Set alarm for 15s, crash after 500ms - Box::new(AlarmSleepThenCrashActor::new(15000, sleeping_tx, crash_tx)) - }) - }) - .await; - - let res = common::create_actor( - ctx.leader_dc().guard_port(), - &namespace, - "alarm-actor", - runner.pool_name(), - rivet_types::actors::CrashPolicy::Restart, - ) - .await; - - let actor_id = res.actor.actor_id.to_string(); - - // Wait for crash notification gen 2 sets alarm and crashes - crash_rx - .recv() - .await - .expect("should receive crash notification"); - - tracing::info!( - ?actor_id, - "gen 2 crashed after alarm wake, waiting for gen 2 restart" - ); - - // Wait for actor to start sleeping again (gen 2 started and sleep) - sleeping_rx - .recv() - .await - .expect("actor should send sleep signal"); - - let actor = - wait_for_actor_sleep(ctx.leader_dc().guard_port(), &actor_id, &namespace, 5) - .await - .expect("actor should be sleeping"); - - assert!(actor.sleep_ts.is_some(), "actor should be asleep"); - - tracing::info!( - ?actor_id, - "gen 2 is now asleep, waiting past original alarm time" - ); - - // Verify the next gen is awake (woke from gen 2's alarm). Use a small - // cushion over the 15s alarm offset for scheduling jitter. - let actor = wait_for_actor_wake_polling( - ctx.leader_dc().guard_port(), - &actor_id, - &namespace, - 20, - ) - .await - .expect("actor should wake from original alarm"); - - assert!( - actor.sleep_ts.is_none() && actor.connectable_ts.is_some(), - "next generation should be awake from gen 2 alarm" - ); - }, - ); -} - #[test] fn rapid_alarm_set_clear_cycles() { common::run(common::TestOpts::new(1), |ctx| async move { @@ -1306,7 +1158,7 @@ fn rapid_alarm_set_clear_cycles() { &namespace, "alarm-actor", runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, + rivet_types::actors::CrashPolicy::Sleep, ) .await; @@ -1337,7 +1189,6 @@ fn rapid_alarm_set_clear_cycles() { // Broken legacy Pegboard Runner coverage: passes alone but fails in the full // engine sweep under Envoy+Runner load; the full sweep reports this test failed. #[test] -#[ignore = "broken legacy Pegboard Runner test: fails only in full engine sweep"] fn multiple_actors_with_different_alarm_times() { common::run(common::TestOpts::new(1), |ctx| async move { let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; @@ -1368,7 +1219,7 @@ fn multiple_actors_with_different_alarm_times() { &namespace, &format!("alarm-actor-{}", idx), runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, + rivet_types::actors::CrashPolicy::Sleep, ) .await; actor_ids.push(res.actor.actor_id.to_string()); @@ -1402,110 +1253,106 @@ fn multiple_actors_with_different_alarm_times() { #[test] // Broken legacy Pegboard Runner test: times out waiting for all same-deadline // actors to wake in the combined Envoy+Runner full engine sweep. -#[ignore = "broken legacy Pegboard Runner test: times out in full engine sweep"] fn many_actors_same_alarm_time() { - common::run( - common::TestOpts::new(1).with_timeout(45), - |ctx| async move { - let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; + common::run(common::TestOpts::new(1).with_timeout(45), |ctx| async move { + let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; - let num_actors = 10; - let alarm_offset = 2000; // All wake at same time - let mut actor_ids = Vec::new(); + let num_actors = 10; + let alarm_offset = 2000; // All wake at same time + let mut actor_ids = Vec::new(); - let runner = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { - builder.with_actor_behavior("alarm-actor", move |_| { - let (ready_tx, _) = tokio::sync::oneshot::channel(); - let ready_tx = Arc::new(Mutex::new(Some(ready_tx))); - Box::new(AlarmAndSleepActor::new(alarm_offset, ready_tx)) - }) + let runner = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { + builder.with_actor_behavior("alarm-actor", move |_| { + let (ready_tx, _) = tokio::sync::oneshot::channel(); + let ready_tx = Arc::new(Mutex::new(Some(ready_tx))); + Box::new(AlarmAndSleepActor::new(alarm_offset, ready_tx)) }) + }) + .await; + + let mut lifecycle_rx = runner.subscribe_lifecycle_events(); + + // Create actors + for _idx in 0..num_actors { + let res = common::create_actor( + ctx.leader_dc().guard_port(), + &namespace, + "alarm-actor", + runner.pool_name(), + rivet_types::actors::CrashPolicy::Sleep, + ) .await; + actor_ids.push(res.actor.actor_id.to_string()); + } - let mut lifecycle_rx = runner.subscribe_lifecycle_events(); - - // Create actors - for _idx in 0..num_actors { - let res = common::create_actor( - ctx.leader_dc().guard_port(), - &namespace, - "alarm-actor", - runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, - ) - .await; - actor_ids.push(res.actor.actor_id.to_string()); - } + tracing::info!(num_actors, "created actors with same alarm time (+2s)"); - tracing::info!(num_actors, "created actors with same alarm time (+2s)"); - - let actor_id_set: HashSet = actor_ids.iter().cloned().collect(); - - // Same-time alarms can wake early actors before a sequential API poll reaches - // later ones, so use the Envoy lifecycle stream to prove every actor stopped - // for sleep at generation 1. - let sleep_deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); - let mut slept_actor_ids = HashSet::new(); - while slept_actor_ids.len() < num_actors { - let remaining = sleep_deadline.saturating_duration_since(std::time::Instant::now()); - let event = tokio::time::timeout(remaining, lifecycle_rx.recv()) - .await - .expect("timed out waiting for actors to sleep") - .expect("lifecycle stream closed"); - - if let ActorLifecycleEvent::Stopped { - actor_id, - generation, - } = event - { - if generation == 1 && actor_id_set.contains(&actor_id) { - slept_actor_ids.insert(actor_id); - } + let actor_id_set: HashSet = actor_ids.iter().cloned().collect(); + + // Same-time alarms can wake early actors before a sequential API poll reaches + // later ones, so use the Envoy lifecycle stream to prove every actor stopped + // for sleep at generation 1. + let sleep_deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + let mut slept_actor_ids = HashSet::new(); + while slept_actor_ids.len() < num_actors { + let remaining = sleep_deadline.saturating_duration_since(std::time::Instant::now()); + let event = tokio::time::timeout(remaining, lifecycle_rx.recv()) + .await + .expect("timed out waiting for actors to sleep") + .expect("lifecycle stream closed"); + + if let ActorLifecycleEvent::Stopped { + actor_id, + generation, + } = event + { + if generation == 1 && actor_id_set.contains(&actor_id) { + slept_actor_ids.insert(actor_id); } } + } - tracing::info!("all actors sleeping"); - - let alarm_start = std::time::Instant::now(); - - // Verify all actors wake within a reasonable time window. - let wake_deadline = std::time::Instant::now() + std::time::Duration::from_secs(4); - let mut woke_actor_ids = HashSet::new(); - while woke_actor_ids.len() < num_actors { - let remaining = wake_deadline.saturating_duration_since(std::time::Instant::now()); - let event = tokio::time::timeout(remaining, lifecycle_rx.recv()) - .await - .expect("timed out waiting for actors to wake") - .expect("lifecycle stream closed"); - - if let ActorLifecycleEvent::Started { - actor_id, - generation, - } = event - { - if generation == 2 && actor_id_set.contains(&actor_id) { - tracing::info!(actor_id, "actor woke"); - woke_actor_ids.insert(actor_id); - } + tracing::info!("all actors sleeping"); + + let alarm_start = std::time::Instant::now(); + + // Verify all actors wake within a reasonable time window. + let wake_deadline = std::time::Instant::now() + std::time::Duration::from_secs(4); + let mut woke_actor_ids = HashSet::new(); + while woke_actor_ids.len() < num_actors { + let remaining = wake_deadline.saturating_duration_since(std::time::Instant::now()); + let event = tokio::time::timeout(remaining, lifecycle_rx.recv()) + .await + .expect("timed out waiting for actors to wake") + .expect("lifecycle stream closed"); + + if let ActorLifecycleEvent::Started { + actor_id, + generation, + } = event + { + if generation == 2 && actor_id_set.contains(&actor_id) { + tracing::info!(actor_id, "actor woke"); + woke_actor_ids.insert(actor_id); } } + } - let total_duration = alarm_start.elapsed(); + let total_duration = alarm_start.elapsed(); - // All 10 actors should wake within a 500ms window around the alarm time - assert!( - total_duration <= std::time::Duration::from_millis(3000), - "all actors should wake within 3s, actual: {:?}", - total_duration - ); + // All 10 actors should wake within a 500ms window around the alarm time + assert!( + total_duration <= std::time::Duration::from_millis(3000), + "all actors should wake within 3s, actual: {:?}", + total_duration + ); - tracing::info!( - num_actors, - ?total_duration, - "all actors woke concurrently at same alarm time" - ); - }, - ); + tracing::info!( + num_actors, + ?total_duration, + "all actors woke concurrently at same alarm time" + ); + }); } /// Regression test for the alarm-during-sleep-transition race. @@ -1525,7 +1372,7 @@ fn many_actors_same_alarm_time() { #[test] fn alarm_overdue_during_sleep_transition_fires_via_reallocation() { common::run( - common::TestOpts::new(1).with_timeout(30), + common::TestOpts::new(1).with_timeout(30), |ctx| async move { let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; @@ -1548,7 +1395,7 @@ fn alarm_overdue_during_sleep_transition_fires_via_reallocation() { &namespace, "alarm-actor", runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, + rivet_types::actors::CrashPolicy::Sleep, ) .await; diff --git a/engine/packages/engine/tests/envoy/actors_kv_crud.rs b/engine/packages/engine/tests/envoy/actors_kv_crud.rs index 5fee2711d3..74999e181d 100644 --- a/engine/packages/engine/tests/envoy/actors_kv_crud.rs +++ b/engine/packages/engine/tests/envoy/actors_kv_crud.rs @@ -445,7 +445,7 @@ fn basic_kv_put_and_get() { &namespace, "kv-put-get", runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, + rivet_types::actors::CrashPolicy::Sleep, ) .await; @@ -485,7 +485,7 @@ fn kv_get_nonexistent_key() { &namespace, "kv-get-nonexistent", runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, + rivet_types::actors::CrashPolicy::Sleep, ) .await; @@ -525,7 +525,7 @@ fn kv_put_overwrite_existing() { &namespace, "kv-overwrite", runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, + rivet_types::actors::CrashPolicy::Sleep, ) .await; @@ -565,7 +565,7 @@ fn kv_delete_existing_key() { &namespace, "kv-delete", runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, + rivet_types::actors::CrashPolicy::Sleep, ) .await; @@ -587,45 +587,42 @@ fn kv_delete_existing_key() { #[test] fn kv_delete_nonexistent_key() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; - - let (notify_tx, notify_rx) = tokio::sync::oneshot::channel(); - let notify_tx = Arc::new(Mutex::new(Some(notify_tx))); - - let runner = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { - builder.with_actor_behavior("kv-delete-nonexistent", move |_| { - Box::new(DeleteNonexistentKeyActor::new(notify_tx.clone())) - }) + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; + + let (notify_tx, notify_rx) = tokio::sync::oneshot::channel(); + let notify_tx = Arc::new(Mutex::new(Some(notify_tx))); + + let runner = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { + builder.with_actor_behavior("kv-delete-nonexistent", move |_| { + Box::new(DeleteNonexistentKeyActor::new(notify_tx.clone())) }) - .await; + }) + .await; - let res = common::create_actor( - ctx.leader_dc().guard_port(), - &namespace, - "kv-delete-nonexistent", - runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, - ) - .await; + let res = common::create_actor( + ctx.leader_dc().guard_port(), + &namespace, + "kv-delete-nonexistent", + runner.pool_name(), + rivet_types::actors::CrashPolicy::Sleep, + ) + .await; - let actor_id = res.actor.actor_id.to_string(); + let actor_id = res.actor.actor_id.to_string(); - // Wait for actor to complete KV operations - let result = notify_rx.await.expect("actor should send test result"); + // Wait for actor to complete KV operations + let result = notify_rx.await.expect("actor should send test result"); - match result { - KvTestResult::Success => { - tracing::info!(?actor_id, "delete nonexistent key test succeeded"); - } - KvTestResult::Failure(msg) => { - panic!("delete nonexistent key test failed: {}", msg); - } + match result { + KvTestResult::Success => { + tracing::info!(?actor_id, "delete nonexistent key test succeeded"); + } + KvTestResult::Failure(msg) => { + panic!("delete nonexistent key test failed: {}", msg); } - }, - ); + } + }); } // MARK: Batch Operations Tests @@ -883,44 +880,41 @@ impl Actor for BatchDeleteActor { #[test] fn kv_put_multiple_keys() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; - - let (notify_tx, notify_rx) = tokio::sync::oneshot::channel(); - let notify_tx = Arc::new(Mutex::new(Some(notify_tx))); - - let runner = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { - builder.with_actor_behavior("kv-batch-put", move |_| { - Box::new(BatchPutActor::new(notify_tx.clone())) - }) + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; + + let (notify_tx, notify_rx) = tokio::sync::oneshot::channel(); + let notify_tx = Arc::new(Mutex::new(Some(notify_tx))); + + let runner = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { + builder.with_actor_behavior("kv-batch-put", move |_| { + Box::new(BatchPutActor::new(notify_tx.clone())) }) - .await; + }) + .await; - let res = common::create_actor( - ctx.leader_dc().guard_port(), - &namespace, - "kv-batch-put", - runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, - ) - .await; + let res = common::create_actor( + ctx.leader_dc().guard_port(), + &namespace, + "kv-batch-put", + runner.pool_name(), + rivet_types::actors::CrashPolicy::Sleep, + ) + .await; - let actor_id = res.actor.actor_id.to_string(); + let actor_id = res.actor.actor_id.to_string(); - let result = notify_rx.await.expect("actor should send test result"); + let result = notify_rx.await.expect("actor should send test result"); - match result { - KvTestResult::Success => { - tracing::info!(?actor_id, "batch put test succeeded"); - } - KvTestResult::Failure(msg) => { - panic!("batch put test failed: {}", msg); - } + match result { + KvTestResult::Success => { + tracing::info!(?actor_id, "batch put test succeeded"); + } + KvTestResult::Failure(msg) => { + panic!("batch put test failed: {}", msg); } - }, - ); + } + }); } #[test] @@ -943,7 +937,7 @@ fn kv_get_multiple_keys() { &namespace, "kv-batch-get", runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, + rivet_types::actors::CrashPolicy::Sleep, ) .await; @@ -982,7 +976,7 @@ fn kv_delete_multiple_keys() { &namespace, "kv-batch-delete", runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, + rivet_types::actors::CrashPolicy::Sleep, ) .await; diff --git a/engine/packages/engine/tests/envoy/actors_kv_delete_range.rs b/engine/packages/engine/tests/envoy/actors_kv_delete_range.rs index 865d7b3749..4b156a1647 100644 --- a/engine/packages/engine/tests/envoy/actors_kv_delete_range.rs +++ b/engine/packages/engine/tests/envoy/actors_kv_delete_range.rs @@ -96,34 +96,31 @@ impl TestActor for DeleteRangeActor { #[test] fn kv_delete_range_removes_half_open_range() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; - - let (notify_tx, notify_rx) = tokio::sync::oneshot::channel(); - let notify_tx = Arc::new(Mutex::new(Some(notify_tx))); - - let runner = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { - builder.with_actor_behavior("kv-delete-range", move |_| { - Box::new(DeleteRangeActor::new(notify_tx.clone())) - }) + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; + + let (notify_tx, notify_rx) = tokio::sync::oneshot::channel(); + let notify_tx = Arc::new(Mutex::new(Some(notify_tx))); + + let runner = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { + builder.with_actor_behavior("kv-delete-range", move |_| { + Box::new(DeleteRangeActor::new(notify_tx.clone())) }) - .await; - - common::create_actor( - ctx.leader_dc().guard_port(), - &namespace, - "kv-delete-range", - runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, - ) - .await; - - match notify_rx.await.expect("actor should send test result") { - KvTestResult::Success => {} - KvTestResult::Failure(msg) => panic!("kv delete range test failed: {}", msg), - } - }, - ); + }) + .await; + + common::create_actor( + ctx.leader_dc().guard_port(), + &namespace, + "kv-delete-range", + runner.pool_name(), + rivet_types::actors::CrashPolicy::Sleep, + ) + .await; + + match notify_rx.await.expect("actor should send test result") { + KvTestResult::Success => {} + KvTestResult::Failure(msg) => panic!("kv delete range test failed: {}", msg), + } + }); } diff --git a/engine/packages/engine/tests/envoy/actors_kv_drop.rs b/engine/packages/engine/tests/envoy/actors_kv_drop.rs index f954b0745d..7da616009a 100644 --- a/engine/packages/engine/tests/envoy/actors_kv_drop.rs +++ b/engine/packages/engine/tests/envoy/actors_kv_drop.rs @@ -196,7 +196,7 @@ fn kv_drop_clears_all_data() { &namespace, "kv-drop-clears", runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, + rivet_types::actors::CrashPolicy::Sleep, ) .await; @@ -235,7 +235,7 @@ fn kv_drop_empty_store() { &namespace, "kv-drop-empty", runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, + rivet_types::actors::CrashPolicy::Sleep, ) .await; diff --git a/engine/packages/engine/tests/envoy/actors_kv_list.rs b/engine/packages/engine/tests/envoy/actors_kv_list.rs index b4f56dfbf0..6acf4f4c4f 100644 --- a/engine/packages/engine/tests/envoy/actors_kv_list.rs +++ b/engine/packages/engine/tests/envoy/actors_kv_list.rs @@ -768,7 +768,7 @@ fn kv_list_all_empty_store() { &namespace, "kv-list-empty", runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, + rivet_types::actors::CrashPolicy::Sleep, ) .await; @@ -807,7 +807,7 @@ fn kv_list_all_with_keys() { &namespace, "kv-list-keys", runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, + rivet_types::actors::CrashPolicy::Sleep, ) .await; @@ -846,7 +846,7 @@ fn kv_list_all_with_limit() { &namespace, "kv-list-limit", runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, + rivet_types::actors::CrashPolicy::Sleep, ) .await; @@ -867,128 +867,119 @@ fn kv_list_all_with_limit() { #[test] fn kv_list_all_reverse() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; - - let (notify_tx, notify_rx) = tokio::sync::oneshot::channel(); - let notify_tx = Arc::new(Mutex::new(Some(notify_tx))); - - let runner = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { - builder.with_actor_behavior("kv-list-reverse", move |_| { - Box::new(ListAllReverseActor::new(notify_tx.clone())) - }) + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; + + let (notify_tx, notify_rx) = tokio::sync::oneshot::channel(); + let notify_tx = Arc::new(Mutex::new(Some(notify_tx))); + + let runner = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { + builder.with_actor_behavior("kv-list-reverse", move |_| { + Box::new(ListAllReverseActor::new(notify_tx.clone())) }) - .await; + }) + .await; - let res = common::create_actor( - ctx.leader_dc().guard_port(), - &namespace, - "kv-list-reverse", - runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, - ) - .await; + let res = common::create_actor( + ctx.leader_dc().guard_port(), + &namespace, + "kv-list-reverse", + runner.pool_name(), + rivet_types::actors::CrashPolicy::Sleep, + ) + .await; - let actor_id = res.actor.actor_id.to_string(); + let actor_id = res.actor.actor_id.to_string(); - let result = notify_rx.await.expect("actor should send test result"); + let result = notify_rx.await.expect("actor should send test result"); - match result { - KvTestResult::Success => { - tracing::info!(?actor_id, "list all reverse test succeeded"); - } - KvTestResult::Failure(msg) => { - panic!("list all reverse test failed: {}", msg); - } + match result { + KvTestResult::Success => { + tracing::info!(?actor_id, "list all reverse test succeeded"); + } + KvTestResult::Failure(msg) => { + panic!("list all reverse test failed: {}", msg); } - }, - ); + } + }); } #[test] fn kv_list_range_inclusive() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; - - let (notify_tx, notify_rx) = tokio::sync::oneshot::channel(); - let notify_tx = Arc::new(Mutex::new(Some(notify_tx))); - - let runner = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { - builder.with_actor_behavior("kv-range-inclusive", move |_| { - Box::new(ListRangeInclusiveActor::new(notify_tx.clone())) - }) + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; + + let (notify_tx, notify_rx) = tokio::sync::oneshot::channel(); + let notify_tx = Arc::new(Mutex::new(Some(notify_tx))); + + let runner = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { + builder.with_actor_behavior("kv-range-inclusive", move |_| { + Box::new(ListRangeInclusiveActor::new(notify_tx.clone())) }) - .await; + }) + .await; - let res = common::create_actor( - ctx.leader_dc().guard_port(), - &namespace, - "kv-range-inclusive", - runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, - ) - .await; + let res = common::create_actor( + ctx.leader_dc().guard_port(), + &namespace, + "kv-range-inclusive", + runner.pool_name(), + rivet_types::actors::CrashPolicy::Sleep, + ) + .await; - let actor_id = res.actor.actor_id.to_string(); + let actor_id = res.actor.actor_id.to_string(); - let result = notify_rx.await.expect("actor should send test result"); + let result = notify_rx.await.expect("actor should send test result"); - match result { - KvTestResult::Success => { - tracing::info!(?actor_id, "list range inclusive test succeeded"); - } - KvTestResult::Failure(msg) => { - panic!("list range inclusive test failed: {}", msg); - } + match result { + KvTestResult::Success => { + tracing::info!(?actor_id, "list range inclusive test succeeded"); + } + KvTestResult::Failure(msg) => { + panic!("list range inclusive test failed: {}", msg); } - }, - ); + } + }); } #[test] fn kv_list_range_exclusive() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; - - let (notify_tx, notify_rx) = tokio::sync::oneshot::channel(); - let notify_tx = Arc::new(Mutex::new(Some(notify_tx))); - - let runner = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { - builder.with_actor_behavior("kv-range-exclusive", move |_| { - Box::new(ListRangeExclusiveActor::new(notify_tx.clone())) - }) + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; + + let (notify_tx, notify_rx) = tokio::sync::oneshot::channel(); + let notify_tx = Arc::new(Mutex::new(Some(notify_tx))); + + let runner = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { + builder.with_actor_behavior("kv-range-exclusive", move |_| { + Box::new(ListRangeExclusiveActor::new(notify_tx.clone())) }) - .await; + }) + .await; - let res = common::create_actor( - ctx.leader_dc().guard_port(), - &namespace, - "kv-range-exclusive", - runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, - ) - .await; + let res = common::create_actor( + ctx.leader_dc().guard_port(), + &namespace, + "kv-range-exclusive", + runner.pool_name(), + rivet_types::actors::CrashPolicy::Sleep, + ) + .await; - let actor_id = res.actor.actor_id.to_string(); + let actor_id = res.actor.actor_id.to_string(); - let result = notify_rx.await.expect("actor should send test result"); + let result = notify_rx.await.expect("actor should send test result"); - match result { - KvTestResult::Success => { - tracing::info!(?actor_id, "list range exclusive test succeeded"); - } - KvTestResult::Failure(msg) => { - panic!("list range exclusive test failed: {}", msg); - } + match result { + KvTestResult::Success => { + tracing::info!(?actor_id, "list range exclusive test succeeded"); + } + KvTestResult::Failure(msg) => { + panic!("list range exclusive test failed: {}", msg); } - }, - ); + } + }); } #[test] @@ -1011,7 +1002,7 @@ fn kv_list_prefix_match() { &namespace, "kv-prefix-match", runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, + rivet_types::actors::CrashPolicy::Sleep, ) .await; @@ -1032,42 +1023,39 @@ fn kv_list_prefix_match() { #[test] fn kv_list_prefix_no_matches() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; - - let (notify_tx, notify_rx) = tokio::sync::oneshot::channel(); - let notify_tx = Arc::new(Mutex::new(Some(notify_tx))); - - let runner = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { - builder.with_actor_behavior("kv-prefix-no-match", move |_| { - Box::new(ListPrefixNoMatchActor::new(notify_tx.clone())) - }) + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; + + let (notify_tx, notify_rx) = tokio::sync::oneshot::channel(); + let notify_tx = Arc::new(Mutex::new(Some(notify_tx))); + + let runner = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { + builder.with_actor_behavior("kv-prefix-no-match", move |_| { + Box::new(ListPrefixNoMatchActor::new(notify_tx.clone())) }) - .await; + }) + .await; - let res = common::create_actor( - ctx.leader_dc().guard_port(), - &namespace, - "kv-prefix-no-match", - runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, - ) - .await; + let res = common::create_actor( + ctx.leader_dc().guard_port(), + &namespace, + "kv-prefix-no-match", + runner.pool_name(), + rivet_types::actors::CrashPolicy::Sleep, + ) + .await; - let actor_id = res.actor.actor_id.to_string(); + let actor_id = res.actor.actor_id.to_string(); - let result = notify_rx.await.expect("actor should send test result"); + let result = notify_rx.await.expect("actor should send test result"); - match result { - KvTestResult::Success => { - tracing::info!(?actor_id, "list prefix no matches test succeeded"); - } - KvTestResult::Failure(msg) => { - panic!("list prefix no matches test failed: {}", msg); - } + match result { + KvTestResult::Success => { + tracing::info!(?actor_id, "list prefix no matches test succeeded"); + } + KvTestResult::Failure(msg) => { + panic!("list prefix no matches test failed: {}", msg); } - }, - ); + } + }); } diff --git a/engine/packages/engine/tests/envoy/actors_kv_misc.rs b/engine/packages/engine/tests/envoy/actors_kv_misc.rs index d0ecdd71c6..f0fab278ea 100644 --- a/engine/packages/engine/tests/envoy/actors_kv_misc.rs +++ b/engine/packages/engine/tests/envoy/actors_kv_misc.rs @@ -197,8 +197,8 @@ impl Actor for LargeValueActor { tracing::info!(actor_id = ?config.actor_id, generation = config.generation, "large value actor starting"); let result = async { - let key = make_key("large-value-key"); - let value: Vec = (0..128 * 1024).map(|i| (i % 256) as u8).collect(); + let key = make_key("large-value-key"); + let value: Vec = (0..128 * 1024).map(|i| (i % 256) as u8).collect(); tracing::info!(value_size = value.len(), "putting large value"); @@ -513,38 +513,38 @@ impl Actor for ManyKeysActor { tracing::info!(actor_id = ?config.actor_id, generation = config.generation, "many keys actor starting"); let result = async { - let mut keys = Vec::new(); - let mut values = Vec::new(); - for i in 0..128 { - keys.push(make_key(&format!("many-key-{:04}", i))); - values.push(make_value(&format!("many-value-{}", i))); - } + let mut keys = Vec::new(); + let mut values = Vec::new(); + for i in 0..128 { + keys.push(make_key(&format!("many-key-{:04}", i))); + values.push(make_value(&format!("many-value-{}", i))); + } - config - .send_kv_put(keys.clone(), values.clone()) - .await - .context("failed to put 128 keys")?; + config + .send_kv_put(keys.clone(), values.clone()) + .await + .context("failed to put 128 keys")?; - tracing::info!("put 128 keys"); + tracing::info!("put 128 keys"); // Call listAll let response = config - .send_kv_list(rp::KvListQuery::KvListAllQuery, None, None) - .await - .context("failed to list all 128 keys")?; + .send_kv_list(rp::KvListQuery::KvListAllQuery, None, None) + .await + .context("failed to list all 128 keys")?; - if response.keys.len() != 128 { - bail!("expected 128 keys, got {}", response.keys.len()); - } + if response.keys.len() != 128 { + bail!("expected 128 keys, got {}", response.keys.len()); + } - if response.values.len() != 128 { - bail!("expected 128 values, got {}", response.values.len()); - } + if response.values.len() != 128 { + bail!("expected 128 values, got {}", response.values.len()); + } - tracing::info!("verified 128 keys present in list"); + tracing::info!("verified 128 keys present in list"); // Get random sample of keys to verify values - for i in &[0, 32, 64, 96, 127] { + for i in &[0, 32, 64, 96, 127] { let key = make_key(&format!("many-key-{:04}", i)); let expected_value = make_value(&format!("many-value-{}", i)); @@ -595,296 +595,275 @@ impl Actor for ManyKeysActor { // Elapsed(())`. #[test] fn kv_binary_keys_and_values() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; - - let (notify_tx, notify_rx) = tokio::sync::oneshot::channel(); - let notify_tx = Arc::new(Mutex::new(Some(notify_tx))); - - let runner = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { - builder.with_actor_behavior("kv-binary", move |_| { - Box::new(BinaryDataActor::new(notify_tx.clone())) - }) + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; + + let (notify_tx, notify_rx) = tokio::sync::oneshot::channel(); + let notify_tx = Arc::new(Mutex::new(Some(notify_tx))); + + let runner = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { + builder.with_actor_behavior("kv-binary", move |_| { + Box::new(BinaryDataActor::new(notify_tx.clone())) }) - .await; + }) + .await; - let res = common::create_actor( - ctx.leader_dc().guard_port(), - &namespace, - "kv-binary", - runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, - ) - .await; + let res = common::create_actor( + ctx.leader_dc().guard_port(), + &namespace, + "kv-binary", + runner.pool_name(), + rivet_types::actors::CrashPolicy::Sleep, + ) + .await; - let actor_id = res.actor.actor_id.to_string(); + let actor_id = res.actor.actor_id.to_string(); - let result = notify_rx.await.expect("actor should send test result"); + let result = notify_rx.await.expect("actor should send test result"); - match result { - KvTestResult::Success => { - tracing::info!(?actor_id, "binary data test succeeded"); - } - KvTestResult::Failure(msg) => { - panic!("binary data test failed: {}", msg); - } + match result { + KvTestResult::Success => { + tracing::info!(?actor_id, "binary data test succeeded"); + } + KvTestResult::Failure(msg) => { + panic!("binary data test failed: {}", msg); } - }, - ); + } + }); } #[test] fn kv_empty_value() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; - - let (notify_tx, notify_rx) = tokio::sync::oneshot::channel(); - let notify_tx = Arc::new(Mutex::new(Some(notify_tx))); - - let runner = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { - builder.with_actor_behavior("kv-empty-value", move |_| { - Box::new(EmptyValueActor::new(notify_tx.clone())) - }) + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; + + let (notify_tx, notify_rx) = tokio::sync::oneshot::channel(); + let notify_tx = Arc::new(Mutex::new(Some(notify_tx))); + + let runner = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { + builder.with_actor_behavior("kv-empty-value", move |_| { + Box::new(EmptyValueActor::new(notify_tx.clone())) }) - .await; + }) + .await; - let res = common::create_actor( - ctx.leader_dc().guard_port(), - &namespace, - "kv-empty-value", - runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, - ) - .await; + let res = common::create_actor( + ctx.leader_dc().guard_port(), + &namespace, + "kv-empty-value", + runner.pool_name(), + rivet_types::actors::CrashPolicy::Sleep, + ) + .await; - let actor_id = res.actor.actor_id.to_string(); + let actor_id = res.actor.actor_id.to_string(); - let result = notify_rx.await.expect("actor should send test result"); + let result = notify_rx.await.expect("actor should send test result"); - match result { - KvTestResult::Success => { - tracing::info!(?actor_id, "empty value test succeeded"); - } - KvTestResult::Failure(msg) => { - panic!("empty value test failed: {}", msg); - } + match result { + KvTestResult::Success => { + tracing::info!(?actor_id, "empty value test succeeded"); + } + KvTestResult::Failure(msg) => { + panic!("empty value test failed: {}", msg); } - }, - ); + } + }); } #[test] fn kv_large_value() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; - - let (notify_tx, notify_rx) = tokio::sync::oneshot::channel(); - let notify_tx = Arc::new(Mutex::new(Some(notify_tx))); - - let runner = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { - builder.with_actor_behavior("kv-large-value", move |_| { - Box::new(LargeValueActor::new(notify_tx.clone())) - }) + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; + + let (notify_tx, notify_rx) = tokio::sync::oneshot::channel(); + let notify_tx = Arc::new(Mutex::new(Some(notify_tx))); + + let runner = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { + builder.with_actor_behavior("kv-large-value", move |_| { + Box::new(LargeValueActor::new(notify_tx.clone())) }) - .await; + }) + .await; - let res = common::create_actor( - ctx.leader_dc().guard_port(), - &namespace, - "kv-large-value", - runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, - ) - .await; + let res = common::create_actor( + ctx.leader_dc().guard_port(), + &namespace, + "kv-large-value", + runner.pool_name(), + rivet_types::actors::CrashPolicy::Sleep, + ) + .await; - let actor_id = res.actor.actor_id.to_string(); + let actor_id = res.actor.actor_id.to_string(); - let result = notify_rx.await.expect("actor should send test result"); + let result = notify_rx.await.expect("actor should send test result"); - match result { - KvTestResult::Success => { - tracing::info!(?actor_id, "large value test succeeded"); - } - KvTestResult::Failure(msg) => { - panic!("large value test failed: {}", msg); - } + match result { + KvTestResult::Success => { + tracing::info!(?actor_id, "large value test succeeded"); + } + KvTestResult::Failure(msg) => { + panic!("large value test failed: {}", msg); } - }, - ); + } + }); } #[test] fn kv_get_with_empty_keys_array() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; - - let (notify_tx, notify_rx) = tokio::sync::oneshot::channel(); - let notify_tx = Arc::new(Mutex::new(Some(notify_tx))); - - let runner = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { - builder.with_actor_behavior("kv-get-empty", move |_| { - Box::new(GetEmptyKeysActor::new(notify_tx.clone())) - }) + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; + + let (notify_tx, notify_rx) = tokio::sync::oneshot::channel(); + let notify_tx = Arc::new(Mutex::new(Some(notify_tx))); + + let runner = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { + builder.with_actor_behavior("kv-get-empty", move |_| { + Box::new(GetEmptyKeysActor::new(notify_tx.clone())) }) - .await; + }) + .await; - let res = common::create_actor( - ctx.leader_dc().guard_port(), - &namespace, - "kv-get-empty", - runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, - ) - .await; + let res = common::create_actor( + ctx.leader_dc().guard_port(), + &namespace, + "kv-get-empty", + runner.pool_name(), + rivet_types::actors::CrashPolicy::Sleep, + ) + .await; - let actor_id = res.actor.actor_id.to_string(); + let actor_id = res.actor.actor_id.to_string(); - let result = notify_rx.await.expect("actor should send test result"); + let result = notify_rx.await.expect("actor should send test result"); - match result { - KvTestResult::Success => { - tracing::info!(?actor_id, "get empty keys test succeeded"); - } - KvTestResult::Failure(msg) => { - panic!("get empty keys test failed: {}", msg); - } + match result { + KvTestResult::Success => { + tracing::info!(?actor_id, "get empty keys test succeeded"); } - }, - ); + KvTestResult::Failure(msg) => { + panic!("get empty keys test failed: {}", msg); + } + } + }); } #[test] fn kv_list_with_limit_zero() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; - - let (notify_tx, notify_rx) = tokio::sync::oneshot::channel(); - let notify_tx = Arc::new(Mutex::new(Some(notify_tx))); - - let runner = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { - builder.with_actor_behavior("kv-list-limit-zero", move |_| { - Box::new(ListLimitZeroActor::new(notify_tx.clone())) - }) + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; + + let (notify_tx, notify_rx) = tokio::sync::oneshot::channel(); + let notify_tx = Arc::new(Mutex::new(Some(notify_tx))); + + let runner = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { + builder.with_actor_behavior("kv-list-limit-zero", move |_| { + Box::new(ListLimitZeroActor::new(notify_tx.clone())) }) - .await; + }) + .await; - let res = common::create_actor( - ctx.leader_dc().guard_port(), - &namespace, - "kv-list-limit-zero", - runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, - ) - .await; + let res = common::create_actor( + ctx.leader_dc().guard_port(), + &namespace, + "kv-list-limit-zero", + runner.pool_name(), + rivet_types::actors::CrashPolicy::Sleep, + ) + .await; - let actor_id = res.actor.actor_id.to_string(); + let actor_id = res.actor.actor_id.to_string(); - let result = notify_rx.await.expect("actor should send test result"); + let result = notify_rx.await.expect("actor should send test result"); - match result { - KvTestResult::Success => { - tracing::info!(?actor_id, "list limit zero test succeeded"); - } - KvTestResult::Failure(msg) => { - panic!("list limit zero test failed: {}", msg); - } + match result { + KvTestResult::Success => { + tracing::info!(?actor_id, "list limit zero test succeeded"); + } + KvTestResult::Failure(msg) => { + panic!("list limit zero test failed: {}", msg); } - }, - ); + } + }); } #[test] // Broken legacy Pegboard Runner test: full engine sweep timed out in // `kv_key_ordering_lexicographic`. fn kv_key_ordering_lexicographic() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; - - let (notify_tx, notify_rx) = tokio::sync::oneshot::channel(); - let notify_tx = Arc::new(Mutex::new(Some(notify_tx))); - - let runner = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { - builder.with_actor_behavior("kv-key-ordering", move |_| { - Box::new(KeyOrderingActor::new(notify_tx.clone())) - }) + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; + + let (notify_tx, notify_rx) = tokio::sync::oneshot::channel(); + let notify_tx = Arc::new(Mutex::new(Some(notify_tx))); + + let runner = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { + builder.with_actor_behavior("kv-key-ordering", move |_| { + Box::new(KeyOrderingActor::new(notify_tx.clone())) }) - .await; + }) + .await; - let res = common::create_actor( - ctx.leader_dc().guard_port(), - &namespace, - "kv-key-ordering", - runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, - ) - .await; + let res = common::create_actor( + ctx.leader_dc().guard_port(), + &namespace, + "kv-key-ordering", + runner.pool_name(), + rivet_types::actors::CrashPolicy::Sleep, + ) + .await; - let actor_id = res.actor.actor_id.to_string(); + let actor_id = res.actor.actor_id.to_string(); - let result = notify_rx.await.expect("actor should send test result"); + let result = notify_rx.await.expect("actor should send test result"); - match result { - KvTestResult::Success => { - tracing::info!(?actor_id, "key ordering test succeeded"); - } - KvTestResult::Failure(msg) => { - panic!("key ordering test failed: {}", msg); - } + match result { + KvTestResult::Success => { + tracing::info!(?actor_id, "key ordering test succeeded"); } - }, - ); + KvTestResult::Failure(msg) => { + panic!("key ordering test failed: {}", msg); + } + } + }); } #[test] fn kv_many_keys_storage() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; - - let (notify_tx, notify_rx) = tokio::sync::oneshot::channel(); - let notify_tx = Arc::new(Mutex::new(Some(notify_tx))); - - let runner = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { - builder.with_actor_behavior("kv-many-keys", move |_| { - Box::new(ManyKeysActor::new(notify_tx.clone())) - }) + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; + + let (notify_tx, notify_rx) = tokio::sync::oneshot::channel(); + let notify_tx = Arc::new(Mutex::new(Some(notify_tx))); + + let runner = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { + builder.with_actor_behavior("kv-many-keys", move |_| { + Box::new(ManyKeysActor::new(notify_tx.clone())) }) - .await; + }) + .await; - let res = common::create_actor( - ctx.leader_dc().guard_port(), - &namespace, - "kv-many-keys", - runner.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, - ) - .await; + let res = common::create_actor( + ctx.leader_dc().guard_port(), + &namespace, + "kv-many-keys", + runner.pool_name(), + rivet_types::actors::CrashPolicy::Sleep, + ) + .await; - let actor_id = res.actor.actor_id.to_string(); + let actor_id = res.actor.actor_id.to_string(); - let result = notify_rx.await.expect("actor should send test result"); + let result = notify_rx.await.expect("actor should send test result"); - match result { - KvTestResult::Success => { - tracing::info!(?actor_id, "many keys storage test succeeded"); - } - KvTestResult::Failure(msg) => { - panic!("many keys storage test failed: {}", msg); - } + match result { + KvTestResult::Success => { + tracing::info!(?actor_id, "many keys storage test succeeded"); } - }, - ); + KvTestResult::Failure(msg) => { + panic!("many keys storage test failed: {}", msg); + } + } + }); } diff --git a/engine/packages/engine/tests/envoy/actors_lifecycle.rs b/engine/packages/engine/tests/envoy/actors_lifecycle.rs index f7f1e059b4..03a9334f0f 100644 --- a/engine/packages/engine/tests/envoy/actors_lifecycle.rs +++ b/engine/packages/engine/tests/envoy/actors_lifecycle.rs @@ -41,7 +41,7 @@ fn envoy_actor_basic_create() { &namespace, "test-actor", envoy.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, + rivet_types::actors::CrashPolicy::Sleep, ) .await; @@ -70,138 +70,82 @@ fn envoy_actor_basic_create() { #[test] fn envoy_create_actor_with_input() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; - - // Generate test input data (base64-encoded String) - let input_data = common::generate_test_input_data(); - - // Decode the base64 data to get the actual bytes the actor will receive - // The API automatically decodes base64 input before sending to the envoy - let input_data_bytes = - base64::Engine::decode(&base64::engine::general_purpose::STANDARD, &input_data) - .expect("failed to decode base64 input"); - - // Create envoy with VerifyInputActor that will validate the input - let envoy = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { - builder.with_actor_behavior("test-actor", move |_| { - Box::new(common::test_envoy::VerifyInputActor::new( - input_data_bytes.clone(), - )) - }) - }) - .await; - - // Create actor with input data - let res = common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: "test-actor".to_string(), - key: None, - input: Some(input_data.clone()), - runner_name_selector: envoy.pool_name().to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor"); - - let actor_id = res.actor.actor_id.to_string(); - - // Poll for actor to become connectable - // If input verification fails, the actor will crash and never become connectable - let actor = loop { - let actor = - common::try_get_actor(ctx.leader_dc().guard_port(), &actor_id, &namespace) - .await - .expect("failed to get actor") - .expect("actor should exist"); - - // Check if actor crashed (input verification failed) - if actor.destroy_ts.is_some() { - panic!( - "actor crashed during input verification (input data was not received correctly)" - ); - } - - // Check if actor is connectable (input verification succeeded) - if actor.connectable_ts.is_some() { - break actor; - } + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; - tokio::time::sleep(tokio::time::Duration::from_millis(50)).await; - }; + // Generate test input data (base64-encoded String) + let input_data = common::generate_test_input_data(); - assert!( - actor.connectable_ts.is_some(), - "actor should be connectable after successful input verification" - ); + // Decode the base64 data to get the actual bytes the actor will receive + // The API automatically decodes base64 input before sending to the envoy + let input_data_bytes = + base64::Engine::decode(&base64::engine::general_purpose::STANDARD, &input_data) + .expect("failed to decode base64 input"); - tracing::info!( - ?actor_id, - input_size = input_data.len(), - "actor successfully verified input data" - ); - }, - ); -} - -#[test] -fn envoy_actor_start_timeout() { - // This test takes 35+ seconds - common::run( - common::TestOpts::new(1).with_timeout(60), - |ctx| async move { - let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; - - // Create envoy client with timeout actor behavior - let envoy = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { - builder.with_actor_behavior("timeout-actor", move |_| { - Box::new(common::test_envoy::TimeoutActor::new()) - }) + // Create envoy with VerifyInputActor that will validate the input + let envoy = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { + builder.with_actor_behavior("test-actor", move |_| { + Box::new(common::test_envoy::VerifyInputActor::new( + input_data_bytes.clone(), + )) }) - .await; + }) + .await; - tracing::info!("envoy client ready, creating actor that will timeout"); + // Create actor with input data + let res = common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: "test-actor".to_string(), + key: None, + input: Some(input_data.clone()), + runner_name_selector: envoy.pool_name().to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to create actor"); - // Create actor with destroy crash policy - let res = common::create_actor( - ctx.leader_dc().guard_port(), - &namespace, - "timeout-actor", - envoy.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, - ) - .await; + let actor_id = res.actor.actor_id.to_string(); - let actor_id_str = res.actor.actor_id.to_string(); + // Poll for actor to become connectable + // If input verification fails, the actor will crash and never become connectable + let actor = loop { + let actor = common::try_get_actor(ctx.leader_dc().guard_port(), &actor_id, &namespace) + .await + .expect("failed to get actor") + .expect("actor should exist"); - tracing::info!(?actor_id_str, "actor created, waiting for timeout"); + // Check if actor crashed (input verification failed) + if actor.destroy_ts.is_some() { + panic!( + "actor crashed during input verification (input data was not received correctly)" + ); + } - // Wait for the actor start timeout threshold (30s + buffer) - tokio::time::sleep(tokio::time::Duration::from_secs(35)).await; + // Check if actor is connectable (input verification succeeded) + if actor.connectable_ts.is_some() { + break actor; + } - // Verify actor was marked as destroyed due to timeout - let actor = - common::try_get_actor(ctx.leader_dc().guard_port(), &actor_id_str, &namespace) - .await - .expect("failed to get actor") - .expect("actor should exist"); + tokio::time::sleep(tokio::time::Duration::from_millis(50)).await; + }; - assert!( - actor.destroy_ts.is_some(), - "actor should be destroyed after start timeout" - ); + assert!( + actor.connectable_ts.is_some(), + "actor should be connectable after successful input verification" + ); - tracing::info!(?actor_id_str, "actor correctly destroyed after timeout"); - }, - ); + tracing::info!( + ?actor_id, + input_size = input_data.len(), + "actor successfully verified input data" + ); + }); } // MARK: Running State Management @@ -227,7 +171,7 @@ fn envoy_actor_starts_and_connectable_via_guard_http() { &namespace, "test-actor", envoy.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, + rivet_types::actors::CrashPolicy::Sleep, ) .await; @@ -267,107 +211,93 @@ fn envoy_actor_starts_and_connectable_via_guard_http() { #[test] fn envoy_http_tunnel_round_trips_request_and_errors() { - common::run( - common::TestOpts::new(1).with_timeout(20), - |ctx| async move { - let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; + common::run(common::TestOpts::new(1).with_timeout(20), |ctx| async move { + let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; - let envoy = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { - builder.with_actor_behavior("test-actor", |_| { - Box::new(common::test_envoy::EchoActor::new()) - }) + let envoy = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { + builder.with_actor_behavior("test-actor", |_| { + Box::new(common::test_envoy::EchoActor::new()) }) - .await; + }) + .await; - let res = common::create_actor( - ctx.leader_dc().guard_port(), - &namespace, - "test-actor", - envoy.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, - ) - .await; - let actor_id = res.actor.actor_id.to_string(); - wait_for_envoy_actor(&envoy, &actor_id).await; - - let client = reqwest::Client::new(); - let body = "hello over envoy".as_bytes().to_vec(); - let response = client - .post(format!( - "http://127.0.0.1:{}/echo", - ctx.leader_dc().guard_port() - )) - .header("X-Rivet-Target", "actor") - .header("X-Rivet-Actor", &actor_id) - .header("X-Test-Header", "from-client") - .body(body.clone()) - .send() - .await - .expect("failed to send HTTP tunnel request"); - - assert_eq!(response.status(), reqwest::StatusCode::CREATED); - assert_eq!( - response - .headers() - .get("x-envoy-test") - .and_then(|v| v.to_str().ok()), - Some("ok") - ); - let payload: serde_json::Value = response.json().await.expect("invalid echo response"); - assert_eq!(payload["actorId"], actor_id); - assert_eq!(payload["method"], "POST"); - assert_eq!(payload["path"], "/echo"); - assert_eq!(payload["testHeader"], "from-client"); - assert_eq!(payload["body"], "hello over envoy"); - assert_eq!(payload["bodyLen"], body.len()); - - let large_body = vec![b'x'; 128 * 1024]; - let large_response = client - .put(format!( - "http://127.0.0.1:{}/echo", - ctx.leader_dc().guard_port() - )) - .header("X-Rivet-Target", "actor") - .header("X-Rivet-Actor", &actor_id) - .body(large_body.clone()) - .send() - .await - .expect("failed to send large HTTP tunnel request"); - assert_eq!(large_response.status(), reqwest::StatusCode::CREATED); - let large_payload: serde_json::Value = large_response - .json() - .await - .expect("invalid large echo response"); - assert_eq!(large_payload["method"], "PUT"); - assert_eq!(large_payload["bodyLen"], large_body.len()); - - let error_response = client - .get(format!( - "http://127.0.0.1:{}/actor-error", - ctx.leader_dc().guard_port() - )) - .header("X-Rivet-Target", "actor") - .header("X-Rivet-Actor", &actor_id) - .send() - .await - .expect("failed to send actor error request"); - assert!( - !error_response.status().is_success(), - "actor fetch error should map to an HTTP error" - ); - assert_eq!( - error_response.status(), - reqwest::StatusCode::INTERNAL_SERVER_ERROR - ); - assert_eq!( - error_response - .headers() - .get("x-rivet-error") - .and_then(|v| v.to_str().ok()), - Some("envoy.fetch_failed") - ); - }, - ); + let res = common::create_actor( + ctx.leader_dc().guard_port(), + &namespace, + "test-actor", + envoy.pool_name(), + rivet_types::actors::CrashPolicy::Sleep, + ) + .await; + let actor_id = res.actor.actor_id.to_string(); + wait_for_envoy_actor(&envoy, &actor_id).await; + + let client = reqwest::Client::new(); + let body = "hello over envoy".as_bytes().to_vec(); + let response = client + .post(format!("http://127.0.0.1:{}/echo", ctx.leader_dc().guard_port())) + .header("X-Rivet-Target", "actor") + .header("X-Rivet-Actor", &actor_id) + .header("X-Test-Header", "from-client") + .body(body.clone()) + .send() + .await + .expect("failed to send HTTP tunnel request"); + + assert_eq!(response.status(), reqwest::StatusCode::CREATED); + assert_eq!( + response + .headers() + .get("x-envoy-test") + .and_then(|v| v.to_str().ok()), + Some("ok") + ); + let payload: serde_json::Value = response.json().await.expect("invalid echo response"); + assert_eq!(payload["actorId"], actor_id); + assert_eq!(payload["method"], "POST"); + assert_eq!(payload["path"], "/echo"); + assert_eq!(payload["testHeader"], "from-client"); + assert_eq!(payload["body"], "hello over envoy"); + assert_eq!(payload["bodyLen"], body.len()); + + let large_body = vec![b'x'; 128 * 1024]; + let large_response = client + .put(format!("http://127.0.0.1:{}/echo", ctx.leader_dc().guard_port())) + .header("X-Rivet-Target", "actor") + .header("X-Rivet-Actor", &actor_id) + .body(large_body.clone()) + .send() + .await + .expect("failed to send large HTTP tunnel request"); + assert_eq!(large_response.status(), reqwest::StatusCode::CREATED); + let large_payload: serde_json::Value = + large_response.json().await.expect("invalid large echo response"); + assert_eq!(large_payload["method"], "PUT"); + assert_eq!(large_payload["bodyLen"], large_body.len()); + + let error_response = client + .get(format!( + "http://127.0.0.1:{}/actor-error", + ctx.leader_dc().guard_port() + )) + .header("X-Rivet-Target", "actor") + .header("X-Rivet-Actor", &actor_id) + .send() + .await + .expect("failed to send actor error request"); + assert!( + !error_response.status().is_success(), + "actor fetch error should map to an HTTP error" + ); + assert_eq!(error_response.status(), reqwest::StatusCode::INTERNAL_SERVER_ERROR); + assert_eq!( + error_response + .headers() + .get("x-rivet-error") + .and_then(|v| v.to_str().ok()), + Some("envoy.fetch_failed") + ); + }); } #[test] @@ -392,7 +322,7 @@ fn envoy_actor_connectable_via_guard_websocket() { &namespace, "test-actor", envoy.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, + rivet_types::actors::CrashPolicy::Sleep, ) .await; @@ -431,137 +361,131 @@ fn envoy_actor_connectable_via_guard_websocket() { #[test] fn envoy_websocket_actor_close_round_trip() { - common::run( - common::TestOpts::new(1).with_timeout(20), - |ctx| async move { - use futures_util::{SinkExt, StreamExt}; - use tokio_tungstenite::{ - connect_async, - tungstenite::{Message, client::IntoClientRequest}, - }; - - let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; - - let envoy = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { - builder.with_actor_behavior("test-actor", |_| { - Box::new(common::test_envoy::EchoActor::new()) - }) + common::run(common::TestOpts::new(1).with_timeout(20), |ctx| async move { + use futures_util::{SinkExt, StreamExt}; + use tokio_tungstenite::{ + connect_async, + tungstenite::{Message, client::IntoClientRequest}, + }; + + let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; + + let envoy = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { + builder.with_actor_behavior("test-actor", |_| { + Box::new(common::test_envoy::EchoActor::new()) }) - .await; + }) + .await; - let res = common::create_actor( - ctx.leader_dc().guard_port(), - &namespace, - "test-actor", - envoy.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, + let res = common::create_actor( + ctx.leader_dc().guard_port(), + &namespace, + "test-actor", + envoy.pool_name(), + rivet_types::actors::CrashPolicy::Sleep, + ) + .await; + let actor_id = res.actor.actor_id.to_string(); + wait_for_envoy_actor(&envoy, &actor_id).await; + + let mut request = format!("ws://127.0.0.1:{}/ws", ctx.leader_dc().guard_port()) + .into_client_request() + .expect("failed to create WebSocket request"); + request.headers_mut().insert( + "Sec-WebSocket-Protocol", + format!( + "rivet, rivet_target.actor, rivet_actor.{}", + urlencoding::encode(&actor_id) ) - .await; - let actor_id = res.actor.actor_id.to_string(); - wait_for_envoy_actor(&envoy, &actor_id).await; - - let mut request = format!("ws://127.0.0.1:{}/ws", ctx.leader_dc().guard_port()) - .into_client_request() - .expect("failed to create WebSocket request"); - request.headers_mut().insert( - "Sec-WebSocket-Protocol", - format!( - "rivet, rivet_target.actor, rivet_actor.{}", - urlencoding::encode(&actor_id) - ) - .parse() - .unwrap(), - ); + .parse() + .unwrap(), + ); - let (ws_stream, response) = connect_async(request) - .await - .expect("failed to connect WebSocket through guard"); - assert_eq!(response.status(), 101); - let (mut write, mut read) = ws_stream.split(); + let (ws_stream, response) = connect_async(request) + .await + .expect("failed to connect WebSocket through guard"); + assert_eq!(response.status(), 101); + let (mut write, mut read) = ws_stream.split(); - write - .send(Message::Text("close-from-actor".to_string().into())) - .await - .expect("failed to send close request"); + write + .send(Message::Text("close-from-actor".to_string().into())) + .await + .expect("failed to send close request"); - let close = tokio::time::timeout(std::time::Duration::from_secs(5), read.next()) - .await - .expect("timed out waiting for actor close") - .expect("websocket should yield close frame") - .expect("websocket close should not error"); - - match close { - Message::Close(Some(frame)) => { - assert_eq!(u16::from(frame.code), 4001); - assert_eq!(frame.reason, "actor.requested_close"); - } - other => panic!("expected close frame, got {other:?}"), + let close = tokio::time::timeout(std::time::Duration::from_secs(5), read.next()) + .await + .expect("timed out waiting for actor close") + .expect("websocket should yield close frame") + .expect("websocket close should not error"); + + match close { + Message::Close(Some(frame)) => { + assert_eq!(u16::from(frame.code), 4001); + assert_eq!(frame.reason, "actor.requested_close"); } - }, - ); + other => panic!("expected close frame, got {other:?}"), + } + }); } // MARK: Stopping and Graceful Shutdown #[test] fn envoy_actor_graceful_stop_with_destroy_policy() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; - - // Create envoy client with stop immediately actor - let envoy = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { - builder.with_actor_behavior("stop-actor", move |_| { - Box::new(common::test_envoy::StopImmediatelyActor::new()) - }) + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; + + // Create envoy client with stop immediately actor + let envoy = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { + builder.with_actor_behavior("stop-actor", move |_| { + Box::new(common::test_envoy::StopImmediatelyActor::new()) }) - .await; + }) + .await; - tracing::info!("envoy client ready, creating actor that will stop gracefully"); + tracing::info!("envoy client ready, creating actor that will stop gracefully"); - // Create actor with destroy crash policy - let res = common::create_actor( - ctx.leader_dc().guard_port(), - &namespace, - "stop-actor", - envoy.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, - ) - .await; + // Create actor with destroy crash policy + let res = common::create_actor( + ctx.leader_dc().guard_port(), + &namespace, + "stop-actor", + envoy.pool_name(), + rivet_types::actors::CrashPolicy::Sleep, + ) + .await; - let actor_id_str = res.actor.actor_id.to_string(); + let actor_id_str = res.actor.actor_id.to_string(); - tracing::info!(?actor_id_str, "actor created, will send stop intent"); + tracing::info!(?actor_id_str, "actor created, will send stop intent"); - // Poll for actor to be destroyed after graceful stop - let actor = loop { - let actor = - common::try_get_actor(ctx.leader_dc().guard_port(), &actor_id_str, &namespace) - .await - .expect("failed to get actor") - .expect("actor should exist"); + // Poll for actor to be destroyed after graceful stop + let actor = loop { + let actor = + common::try_get_actor(ctx.leader_dc().guard_port(), &actor_id_str, &namespace) + .await + .expect("failed to get actor") + .expect("actor should exist"); - if actor.destroy_ts.is_some() { - break actor; - } + if actor.destroy_ts.is_some() { + break actor; + } - tokio::time::sleep(tokio::time::Duration::from_millis(50)).await; - }; + tokio::time::sleep(tokio::time::Duration::from_millis(50)).await; + }; - assert!( - actor.destroy_ts.is_some(), - "actor should be destroyed after graceful stop with destroy policy" - ); + assert!( + actor.destroy_ts.is_some(), + "actor should be destroyed after graceful stop with destroy policy" + ); - // Verify envoy slot freed (actor no longer on envoy) - assert!( - !envoy.has_actor(&actor_id_str).await, - "actor should be removed from envoy after destroy" - ); + // Verify envoy slot freed (actor no longer on envoy) + assert!( + !envoy.has_actor(&actor_id_str).await, + "actor should be removed from envoy after destroy" + ); - tracing::info!(?actor_id_str, "actor gracefully stopped and destroyed"); - }, - ); + tracing::info!(?actor_id_str, "actor gracefully stopped and destroyed"); + }); } #[test] @@ -588,7 +512,7 @@ fn envoy_actor_explicit_destroy() { &namespace, "test-actor", envoy.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, + rivet_types::actors::CrashPolicy::Sleep, ) .await; @@ -643,387 +567,170 @@ fn envoy_actor_explicit_destroy() { #[test] fn envoy_reconnect_replays_pending_start_once() { - common::run( - common::TestOpts::new(1).with_timeout(20), - |ctx| async move { - let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; - - let start_count = Arc::new(AtomicUsize::new(0)); - let actor_start_count = start_count.clone(); - let envoy = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { - builder.with_actor_behavior("replay-actor", move |_| { - let actor_start_count = actor_start_count.clone(); - Box::new( - common::test_envoy::CustomActorBuilder::new() - .on_start(move |_| { - let actor_start_count = actor_start_count.clone(); - Box::pin(async move { - actor_start_count.fetch_add(1, Ordering::SeqCst); - Ok(common::test_envoy::ActorStartResult::Running) - }) - }) - .build(), - ) - }) - }) - .await; - envoy.shutdown().await; - - let res = common::create_actor( - ctx.leader_dc().guard_port(), - &namespace, - "replay-actor", - envoy.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, - ) - .await; - let actor_id = res.actor.actor_id.to_string(); - - tokio::time::timeout(std::time::Duration::from_secs(5), async { - loop { - let actor = - common::try_get_actor(ctx.leader_dc().guard_port(), &actor_id, &namespace) - .await - .expect("failed to get actor") - .expect("actor should exist"); - if matches!(&actor.error, Some(rivet_types::actor::ActorError::NoEnvoys)) { - break; - } - tokio::time::sleep(std::time::Duration::from_millis(50)).await; - } - }) - .await - .expect("actor should wait for envoy while disconnected"); - - envoy.start().await.expect("failed to restart envoy"); - envoy.wait_ready().await; - wait_for_envoy_actor(&envoy, &actor_id).await; - - assert_eq!( - start_count.load(Ordering::SeqCst), - 1, - "reconnected envoy should receive the missed start exactly once" - ); - tokio::time::sleep(std::time::Duration::from_millis(500)).await; - assert_eq!( - start_count.load(Ordering::SeqCst), - 1, - "start command should not be replayed twice after reconnect" - ); - }, - ); -} - -#[test] -fn envoy_actor_stop_waits_for_completion_before_destroy() { - common::run( - common::TestOpts::new(1).with_timeout(20), - |ctx| async move { - let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; - - let (stop_started_tx, stop_started_rx) = tokio::sync::oneshot::channel(); - let stop_started_tx = Arc::new(Mutex::new(Some(stop_started_tx))); - let envoy = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { - builder.with_actor_behavior("delayed-stop-actor", move |_| { - let stop_started_tx = stop_started_tx.clone(); - Box::new( - common::test_envoy::CustomActorBuilder::new() - .on_stop(move || { - let stop_started_tx = stop_started_tx.clone(); - Box::pin(async move { - if let Some(tx) = - stop_started_tx.lock().expect("stop tx lock").take() - { - let _ = tx.send(()); - } - tokio::time::sleep(std::time::Duration::from_secs(3)).await; - Ok(common::test_envoy::ActorStopResult::Success) - }) - }) - .build(), - ) - }) - }) - .await; - - let res = common::create_actor( - ctx.leader_dc().guard_port(), - &namespace, - "delayed-stop-actor", - envoy.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, - ) - .await; - let actor_id = res.actor.actor_id.to_string(); - wait_for_envoy_actor(&envoy, &actor_id).await; - - let guard_port = ctx.leader_dc().guard_port(); - let delete_actor_id = actor_id.clone(); - let delete_namespace = namespace.clone(); - let delete_task = tokio::spawn(async move { - common::api::public::actors_delete( - guard_port, - common::api_types::actors::delete::DeletePath { - actor_id: delete_actor_id.parse().expect("failed to parse actor_id"), - }, - common::api_types::actors::delete::DeleteQuery { - namespace: delete_namespace, - }, - ) - .await - .expect("failed to delete actor"); - }); - - stop_started_rx - .await - .expect("envoy should begin graceful stop"); - - let actor_during_stop = - common::try_get_actor(ctx.leader_dc().guard_port(), &actor_id, &namespace) - .await - .expect("failed to get actor") - .expect("actor should exist during stop"); - assert!( - actor_during_stop.destroy_ts.is_none(), - "actor should not be destroyed before Envoy stop completion" - ); - - delete_task.await.expect("delete task should not panic"); - - tokio::time::timeout(std::time::Duration::from_secs(5), async { - loop { - let actor = - common::try_get_actor(ctx.leader_dc().guard_port(), &actor_id, &namespace) - .await - .expect("failed to get actor") - .expect("actor should exist"); - if actor.destroy_ts.is_some() { - break; - } - tokio::time::sleep(std::time::Duration::from_millis(50)).await; - } - }) - .await - .expect("actor should be destroyed after Envoy stop completion"); - }, - ); -} - -// MARK: 5. Crash Handling and Policies -#[ignore = "non-sleep crash policies are not yet supported for envoys"] -#[test] -fn envoy_crash_policy_restart() { - common::run(common::TestOpts::new(1), |ctx| async move { + common::run(common::TestOpts::new(1).with_timeout(20), |ctx| async move { let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; - let crash_count = Arc::new(Mutex::new(0)); - - // Create envoy client with actor that crashes once, then succeeds. - let actor_crash_count = crash_count.clone(); + let start_count = Arc::new(AtomicUsize::new(0)); + let actor_start_count = start_count.clone(); let envoy = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { - builder.with_actor_behavior("crash-restart-actor", move |_| { - Box::new(common::test_envoy::CrashNTimesThenSucceedActor::new( - 1, - actor_crash_count.clone(), - )) + builder.with_actor_behavior("replay-actor", move |_| { + let actor_start_count = actor_start_count.clone(); + Box::new( + common::test_envoy::CustomActorBuilder::new() + .on_start(move |_| { + let actor_start_count = actor_start_count.clone(); + Box::pin(async move { + actor_start_count.fetch_add(1, Ordering::SeqCst); + Ok(common::test_envoy::ActorStartResult::Running) + }) + }) + .build(), + ) }) }) .await; + envoy.shutdown().await; - tracing::info!("envoy client ready, creating actor with restart policy"); - - // Create actor with restart crash policy let res = common::create_actor( ctx.leader_dc().guard_port(), &namespace, - "crash-restart-actor", + "replay-actor", envoy.pool_name(), - rivet_types::actors::CrashPolicy::Restart, + rivet_types::actors::CrashPolicy::Sleep, ) .await; + let actor_id = res.actor.actor_id.to_string(); - let actor_id_str = res.actor.actor_id.to_string(); - - tracing::info!(?actor_id_str, "actor created, will crash on start"); - - // Poll for the restarted actor to become connectable. - let actor = loop { - let actor = - common::try_get_actor(ctx.leader_dc().guard_port(), &actor_id_str, &namespace) - .await - .expect("failed to get actor") - .expect("actor should exist"); - - if actor.connectable_ts.is_some() { - break actor; + tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { + let actor = + common::try_get_actor(ctx.leader_dc().guard_port(), &actor_id, &namespace) + .await + .expect("failed to get actor") + .expect("actor should exist"); + if matches!(&actor.error, Some(rivet_types::actor::ActorError::NoEnvoys)) { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; } + }) + .await + .expect("actor should wait for envoy while disconnected"); - tokio::time::sleep(tokio::time::Duration::from_millis(50)).await; - }; + envoy.start().await.expect("failed to restart envoy"); + envoy.wait_ready().await; + wait_for_envoy_actor(&envoy, &actor_id).await; - assert!( - actor.connectable_ts.is_some(), - "actor should become connectable after restart" + assert_eq!( + start_count.load(Ordering::SeqCst), + 1, + "reconnected envoy should receive the missed start exactly once" ); + tokio::time::sleep(std::time::Duration::from_millis(500)).await; assert_eq!( - *crash_count.lock().expect("crash count lock"), + start_count.load(Ordering::SeqCst), 1, - "actor should have crashed exactly once before restarting" + "start command should not be replayed twice after reconnect" ); - - tracing::info!(?actor_id_str, "actor restarted successfully"); }); } -#[ignore = "non-sleep crash policies are not yet supported for envoys"] #[test] -fn envoy_crash_policy_restart_resets_on_success() { - common::run(common::TestOpts::new(1), |ctx| async move { +fn envoy_actor_stop_waits_for_completion_before_destroy() { + common::run(common::TestOpts::new(1).with_timeout(20), |ctx| async move { let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; - let crash_count = Arc::new(Mutex::new(0)); - - // Create envoy client with actor that crashes 2 times then succeeds + let (stop_started_tx, stop_started_rx) = tokio::sync::oneshot::channel(); + let stop_started_tx = Arc::new(Mutex::new(Some(stop_started_tx))); let envoy = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { - builder.with_actor_behavior("crash-recover-actor", move |_| { - Box::new(common::test_envoy::CrashNTimesThenSucceedActor::new( - 2, - crash_count.clone(), - )) + builder.with_actor_behavior("delayed-stop-actor", move |_| { + let stop_started_tx = stop_started_tx.clone(); + Box::new( + common::test_envoy::CustomActorBuilder::new() + .on_stop(move || { + let stop_started_tx = stop_started_tx.clone(); + Box::pin(async move { + if let Some(tx) = + stop_started_tx.lock().expect("stop tx lock").take() + { + let _ = tx.send(()); + } + tokio::time::sleep(std::time::Duration::from_secs(3)).await; + Ok(common::test_envoy::ActorStopResult::Success) + }) + }) + .build(), + ) }) }) .await; - tracing::info!("envoy client ready, creating actor with restart policy"); - - // Create actor with restart crash policy let res = common::create_actor( ctx.leader_dc().guard_port(), &namespace, - "crash-recover-actor", + "delayed-stop-actor", envoy.pool_name(), - rivet_types::actors::CrashPolicy::Restart, + rivet_types::actors::CrashPolicy::Sleep, ) .await; + let actor_id = res.actor.actor_id.to_string(); + wait_for_envoy_actor(&envoy, &actor_id).await; - let actor_id_str = res.actor.actor_id.to_string(); - - tracing::info!( - ?actor_id_str, - "actor created, will crash twice then succeed" - ); - - // Poll for actor to eventually become connectable after crashes and restarts - // The actor should crash twice, reschedule, and eventually run successfully - let actor = loop { - let actor = - common::try_get_actor(ctx.leader_dc().guard_port(), &actor_id_str, &namespace) - .await - .expect("failed to get actor") - .expect("actor should exist"); - - // Actor successfully running after retries - if actor.connectable_ts.is_some() { - break actor; - } + let guard_port = ctx.leader_dc().guard_port(); + let delete_actor_id = actor_id.clone(); + let delete_namespace = namespace.clone(); + let delete_task = tokio::spawn(async move { + common::api::public::actors_delete( + guard_port, + common::api_types::actors::delete::DeletePath { + actor_id: delete_actor_id.parse().expect("failed to parse actor_id"), + }, + common::api_types::actors::delete::DeleteQuery { + namespace: delete_namespace, + }, + ) + .await + .expect("failed to delete actor"); + }); - tokio::time::sleep(tokio::time::Duration::from_millis(100)).await; - }; + stop_started_rx + .await + .expect("envoy should begin graceful stop"); + let actor_during_stop = + common::try_get_actor(ctx.leader_dc().guard_port(), &actor_id, &namespace) + .await + .expect("failed to get actor") + .expect("actor should exist during stop"); assert!( - actor.connectable_ts.is_some(), - "actor should eventually become connectable after crashes" - ); - // actor.reschedule_ts is always Some(), not sure if this is intended - assert!( - actor.reschedule_ts.is_none() - || (actor.connectable_ts.unwrap() > actor.reschedule_ts.unwrap()), - "actor should not be scheduled for retry when running successfully" + actor_during_stop.destroy_ts.is_none(), + "actor should not be destroyed before Envoy stop completion" ); - tracing::info!(?actor_id_str, "actor successfully recovered after crashes"); - }); -} - -#[test] -fn envoy_crash_policy_sleep() { - common::run( - common::TestOpts::new(1).with_timeout(75), - |ctx| async move { - let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; - - // Create channel to be notified when actor crashes - let (crash_tx, crash_rx) = tokio::sync::oneshot::channel(); - let crash_tx = Arc::new(Mutex::new(Some(crash_tx))); - - // Create envoy client with crashing actor - let envoy = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { - builder.with_actor_behavior("crash-actor", move |_| { - Box::new(common::test_envoy::CrashOnStartActor::new_with_notify( - 1, - crash_tx.clone(), - )) - }) - }) - .await; - - tracing::info!("envoy client ready, creating actor with sleep policy"); - - // Create actor with sleep crash policy - let res = common::create_actor( - ctx.leader_dc().guard_port(), - &namespace, - "crash-actor", - envoy.pool_name(), - rivet_types::actors::CrashPolicy::Sleep, - ) - .await; - - let actor_id_str = res.actor.actor_id.to_string(); + delete_task.await.expect("delete task should not panic"); - tracing::info!(?actor_id_str, "actor created with sleep policy"); - - // Wait for crash notification - crash_rx - .await - .expect("actor should have sent crash notification"); - - // Poll for sleep_ts to be set (system needs to process the crash) - let actor = loop { + tokio::time::timeout(std::time::Duration::from_secs(5), async { + loop { let actor = - common::try_get_actor(ctx.leader_dc().guard_port(), &actor_id_str, &namespace) + common::try_get_actor(ctx.leader_dc().guard_port(), &actor_id, &namespace) .await .expect("failed to get actor") .expect("actor should exist"); - - if actor.sleep_ts.is_some() { - break actor; + if actor.destroy_ts.is_some() { + break; } - - tokio::time::sleep(tokio::time::Duration::from_millis(50)).await; - }; - - assert!( - actor.sleep_ts.is_some(), - "actor should be sleeping after crash with sleep policy" - ); - assert!( - actor.connectable_ts.is_none(), - "actor should not be connectable while sleeping" - ); - - tracing::info!( - ?actor_id_str, - "actor correctly entered sleep state after crash" - ); - }, - ); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + }) + .await + .expect("actor should be destroyed after Envoy stop completion"); + }); } -#[ignore = "non-sleep crash policies are not yet supported for envoys"] +// MARK: 5. Crash Handling and Policies #[test] -fn envoy_crash_policy_destroy() { - common::run(common::TestOpts::new(1), |ctx| async move { +fn envoy_crash_policy_sleep() { + common::run(common::TestOpts::new(1).with_timeout(75), |ctx| async move { let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; // Create channel to be notified when actor crashes @@ -1041,28 +748,28 @@ fn envoy_crash_policy_destroy() { }) .await; - tracing::info!("envoy client ready, creating actor with destroy policy"); + tracing::info!("envoy client ready, creating actor with sleep policy"); - // Create actor with destroy crash policy + // Create actor with sleep crash policy let res = common::create_actor( ctx.leader_dc().guard_port(), &namespace, "crash-actor", envoy.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, + rivet_types::actors::CrashPolicy::Sleep, ) .await; let actor_id_str = res.actor.actor_id.to_string(); - tracing::info!(?actor_id_str, "actor created with destroy policy"); + tracing::info!(?actor_id_str, "actor created with sleep policy"); // Wait for crash notification crash_rx .await .expect("actor should have sent crash notification"); - // Poll for destroy_ts to be set (system needs to process the crash) + // Poll for sleep_ts to be set (system needs to process the crash) let actor = loop { let actor = common::try_get_actor(ctx.leader_dc().guard_port(), &actor_id_str, &namespace) @@ -1070,7 +777,7 @@ fn envoy_crash_policy_destroy() { .expect("failed to get actor") .expect("actor should exist"); - if actor.destroy_ts.is_some() { + if actor.sleep_ts.is_some() { break actor; } @@ -1078,11 +785,18 @@ fn envoy_crash_policy_destroy() { }; assert!( - actor.destroy_ts.is_some(), - "actor should be destroyed after crash with destroy policy" + actor.sleep_ts.is_some(), + "actor should be sleeping after crash with sleep policy" + ); + assert!( + actor.connectable_ts.is_none(), + "actor should not be connectable while sleeping" ); - tracing::info!(?actor_id_str, "actor correctly destroyed after crash"); + tracing::info!( + ?actor_id_str, + "actor correctly entered sleep state after crash" + ); }); } @@ -1114,7 +828,7 @@ fn envoy_actor_sleep_intent() { &namespace, "sleep-actor", envoy.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, + rivet_types::actors::CrashPolicy::Sleep, ) .await; @@ -1180,7 +894,7 @@ fn envoy_actor_pending_allocation_no_envoys() { &namespace, "test-actor", pool_name, - rivet_types::actors::CrashPolicy::Destroy, + rivet_types::actors::CrashPolicy::Sleep, ) .await; @@ -1221,7 +935,10 @@ fn envoy_actor_pending_allocation_no_envoys() { "actor should not be connectable yet" ); assert!( - matches!(&actor.error, Some(rivet_types::actor::ActorError::NoEnvoys)), + matches!( + &actor.error, + Some(rivet_types::actor::ActorError::NoEnvoys) + ), "actor should report no connected envoys before allocation, got {:?}", actor.error ); @@ -1257,196 +974,87 @@ fn envoy_actor_pending_allocation_no_envoys() { #[test] fn envoy_multiple_pending_allocations_start_after_envoy_reconnect() { - common::run( - common::TestOpts::new(1).with_timeout(45), - |ctx| async move { - let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; - - // Prime the pool's Envoy protocol version, then disconnect so all actors are - // created as actor2 with no active envoys available. - let envoy = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { - builder - .with_actor_behavior("test-actor-0", |_| { - Box::new(common::test_envoy::EchoActor::new()) - }) - .with_actor_behavior("test-actor-1", |_| { - Box::new(common::test_envoy::EchoActor::new()) - }) - .with_actor_behavior("test-actor-2", |_| { - Box::new(common::test_envoy::EchoActor::new()) - }) - }) - .await; - envoy.shutdown().await; - - tracing::info!("envoy protocol version primed, envoy disconnected"); - - // Create 3 actors while no envoy is connected. - let mut actor_ids = Vec::new(); - for i in 0..3 { - let name = format!("test-actor-{}", i); - let res = common::create_actor( - ctx.leader_dc().guard_port(), - &namespace, - &name, - envoy.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, - ) - .await; - - let actor_id = res.actor.actor_id.to_string(); - tokio::time::timeout(tokio::time::Duration::from_secs(5), async { - loop { - let actor = common::try_get_actor( - ctx.leader_dc().guard_port(), - &actor_id, - &namespace, - ) - .await - .expect("failed to get actor") - .expect("actor should exist"); - - assert!( - actor.connectable_ts.is_none(), - "actor should not be connectable before envoy reconnect" - ); - if matches!(&actor.error, Some(rivet_types::actor::ActorError::NoEnvoys)) { - break; - } + common::run(common::TestOpts::new(1).with_timeout(45), |ctx| async move { + let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; - tokio::time::sleep(tokio::time::Duration::from_millis(50)).await; - } + // Prime the pool's Envoy protocol version, then disconnect so all actors are + // created as actor2 with no active envoys available. + let envoy = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { + builder + .with_actor_behavior("test-actor-0", |_| { + Box::new(common::test_envoy::EchoActor::new()) }) - .await - .expect("actor should report no connected envoys before allocation"); - - actor_ids.push(actor_id); - } - - envoy.start().await.expect("failed to restart envoy"); - envoy.wait_ready().await; - - // Poll for all pending actors to be allocated. - loop { - let mut all_allocated = true; - for actor_id in &actor_ids { - if !envoy.has_actor(actor_id).await { - all_allocated = false; - break; - } - } - if all_allocated { - break; - } - tokio::time::sleep(tokio::time::Duration::from_millis(50)).await; - } - - tracing::info!("all pending actors allocated after envoy reconnect"); - }, - ); -} - -// MARK: envoy Failures -#[test] -fn envoy_actor_survives_envoy_disconnect() { - common::run( - common::TestOpts::new(1).with_timeout(90), - |ctx| async move { - let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; - - // Create envoy and start actor - let (start_tx, start_rx) = tokio::sync::oneshot::channel(); - let start_tx = Arc::new(Mutex::new(Some(start_tx))); - - let envoy = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { - builder.with_actor_behavior("test-actor", move |_| { - Box::new(common::test_envoy::NotifyOnStartActor::new( - start_tx.clone(), - )) + .with_actor_behavior("test-actor-1", |_| { + Box::new(common::test_envoy::EchoActor::new()) }) - }) - .await; + .with_actor_behavior("test-actor-2", |_| { + Box::new(common::test_envoy::EchoActor::new()) + }) + }) + .await; + envoy.shutdown().await; + tracing::info!("envoy protocol version primed, envoy disconnected"); + + // Create 3 actors while no envoy is connected. + let mut actor_ids = Vec::new(); + for i in 0..3 { + let name = format!("test-actor-{}", i); let res = common::create_actor( ctx.leader_dc().guard_port(), &namespace, - "test-actor", + &name, envoy.pool_name(), - rivet_types::actors::CrashPolicy::Restart, + rivet_types::actors::CrashPolicy::Sleep, ) .await; - let actor_id_str = res.actor.actor_id.to_string(); - - // Wait for actor to start - start_rx - .await - .expect("actor should have sent start notification"); - - tracing::info!(?actor_id_str, "actor started, simulating envoy disconnect"); - - // Simulate an ungraceful envoy disconnect. Graceful shutdown waits for actor - // drain and exercises GoingAway instead of EnvoyConnectionLost. - envoy.crash().await; - - tracing::info!( - "envoy disconnected, waiting for system to detect and apply crash policy" - ); - - let start = std::time::Instant::now(); - loop { - let actor = - common::try_get_actor(ctx.leader_dc().guard_port(), &actor_id_str, &namespace) - .await - .expect("failed to get actor") - .expect("actor should exist"); - tracing::warn!(?actor); - if actor.connectable_ts.is_none() - && matches!( - &actor.error, - Some(rivet_types::actor::ActorError::EnvoyNoResponse { .. }) - | Some(rivet_types::actor::ActorError::EnvoyConnectionLost { .. }) - | Some(rivet_types::actor::ActorError::NoEnvoys) - ) { - break; - } + let actor_id = res.actor.actor_id.to_string(); + tokio::time::timeout(tokio::time::Duration::from_secs(5), async { + loop { + let actor = + common::try_get_actor(ctx.leader_dc().guard_port(), &actor_id, &namespace) + .await + .expect("failed to get actor") + .expect("actor should exist"); - if start.elapsed() > std::time::Duration::from_secs(30) { - panic!( - "actor should become non-connectable after envoy disconnect; last actor: {:?}", - actor + assert!( + actor.connectable_ts.is_none(), + "actor should not be connectable before envoy reconnect" ); - } + if matches!(&actor.error, Some(rivet_types::actor::ActorError::NoEnvoys)) { + break; + } - tokio::time::sleep(tokio::time::Duration::from_millis(500)).await; - } + tokio::time::sleep(tokio::time::Duration::from_millis(50)).await; + } + }) + .await + .expect("actor should report no connected envoys before allocation"); - envoy.start().await.expect("failed to restart envoy"); - envoy.wait_ready().await; + actor_ids.push(actor_id); + } - let start = std::time::Instant::now(); - loop { - let actor = - common::try_get_actor(ctx.leader_dc().guard_port(), &actor_id_str, &namespace) - .await - .expect("failed to get actor") - .expect("actor should exist"); + envoy.start().await.expect("failed to restart envoy"); + envoy.wait_ready().await; - if actor.connectable_ts.is_some() && envoy.has_actor(&actor_id_str).await { + // Poll for all pending actors to be allocated. + loop { + let mut all_allocated = true; + for actor_id in &actor_ids { + if !envoy.has_actor(actor_id).await { + all_allocated = false; break; } - - if start.elapsed() > std::time::Duration::from_secs(20) { - panic!( - "actor should reconnect after envoy restarts; last actor: {:?}", - actor - ); - } - - tokio::time::sleep(tokio::time::Duration::from_millis(250)).await; } - }, - ); + if all_allocated { + break; + } + tokio::time::sleep(tokio::time::Duration::from_millis(50)).await; + } + + tracing::info!("all pending actors allocated after envoy reconnect"); + }); } // MARK: Resource Limits @@ -1469,7 +1077,7 @@ fn envoy_normal_pool_does_not_apply_legacy_runner_slot_capacity() { &namespace, "test-actor", envoy.pool_name(), - rivet_types::actors::CrashPolicy::Destroy, + rivet_types::actors::CrashPolicy::Sleep, ) .await; @@ -1522,130 +1130,3 @@ fn envoy_normal_pool_does_not_apply_legacy_runner_slot_capacity() { }); } -// MARK: Timeout and Retry Scenarios -#[ignore = "non-sleep crash policies are not yet supported for envoys"] -#[test] -fn envoy_exponential_backoff_max_retries() { - common::run( - common::TestOpts::new(1).with_timeout(45), - |ctx| async move { - let (namespace, _) = common::setup_test_namespace(ctx.leader_dc()).await; - - // Create envoy client with always-crashing actor - - let envoy = common::setup_envoy(ctx.leader_dc(), &namespace, |builder| { - builder.with_actor_behavior("crash-always-actor", move |_| { - Box::new(common::test_envoy::CrashOnStartActor::new(1)) - }) - }) - .await; - - tracing::info!("envoy client ready, creating actor that will always crash"); - - // Create actor with restart crash policy - let res = common::create_actor( - ctx.leader_dc().guard_port(), - &namespace, - "crash-always-actor", - envoy.pool_name(), - rivet_types::actors::CrashPolicy::Restart, - ) - .await; - - let actor_id_str = res.actor.actor_id.to_string(); - - tracing::info!(?actor_id_str, "actor created, will crash repeatedly"); - - // Track reschedule timestamps to verify backoff increases - let mut previous_reschedule_ts: Option = None; - let mut backoff_deltas = Vec::new(); - - // Poll for multiple crashes and verify backoff increases - for iteration in 0..5 { - let actor = tokio::time::timeout(tokio::time::Duration::from_secs(20), async { - loop { - let actor = common::try_get_actor( - ctx.leader_dc().guard_port(), - &actor_id_str, - &namespace, - ) - .await - .expect("failed to get actor") - .expect("actor should exist"); - - if let Some(reschedule_ts) = actor.reschedule_ts { - if previous_reschedule_ts.map_or(true, |prev| reschedule_ts > prev) { - break actor; - } - } - - tokio::time::sleep(tokio::time::Duration::from_millis(50)).await; - } - }) - .await - .expect("timed out waiting for fresh reschedule_ts"); - - let current_reschedule_ts = - actor.reschedule_ts.expect("reschedule_ts should be set"); - - tracing::info!( - iteration, - reschedule_ts = current_reschedule_ts, - "actor has reschedule_ts after crash" - ); - - // Calculate backoff delta if we have a previous timestamp - if let Some(prev_ts) = previous_reschedule_ts { - let delta = current_reschedule_ts - prev_ts; - backoff_deltas.push(delta); - tracing::info!( - iteration, - delta_ms = delta, - "backoff delta from previous reschedule" - ); - } - - previous_reschedule_ts = Some(current_reschedule_ts); - - // Wait for the reschedule time to pass so next crash can happen. - let now = rivet_util::timestamp::now(); - if iteration < 4 && current_reschedule_ts > now { - let wait_duration = (current_reschedule_ts - now) as u64; - tracing::info!( - wait_duration_ms = wait_duration, - "waiting for reschedule time" - ); - tokio::time::sleep(tokio::time::Duration::from_millis(wait_duration + 100)) - .await; - } - } - - // Verify that backoff intervals generally increase (exponential backoff) - // We expect each delta to be larger than or equal to the previous - // (allowing some tolerance for system timing) - for i in 1..backoff_deltas.len() { - tracing::info!( - iteration = i, - current_delta = backoff_deltas[i], - previous_delta = backoff_deltas[i - 1], - "comparing backoff deltas" - ); - - // Allow some tolerance: current should be >= 80% of expected growth - // (exponential backoff typically doubles, but we allow for some variance) - assert!( - backoff_deltas[i] >= backoff_deltas[i - 1] / 2, - "backoff should not decrease significantly: iteration {}, prev={}, curr={}", - i, - backoff_deltas[i - 1], - backoff_deltas[i] - ); - } - - tracing::info!( - ?backoff_deltas, - "exponential backoff verified across multiple crashes" - ); - }, - ); -} diff --git a/engine/packages/engine/tests/envoy/api_actors_create.rs b/engine/packages/engine/tests/envoy/api_actors_create.rs index fbbbd4bd2c..f68878a08a 100644 --- a/engine/packages/engine/tests/envoy/api_actors_create.rs +++ b/engine/packages/engine/tests/envoy/api_actors_create.rs @@ -3,289 +3,258 @@ use super::super::common; // MARK: Basic #[test] fn create_actor_valid_namespace() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - let res = common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: "test-actor".to_string(), - key: None, - input: None, - runner_name_selector: runner.pool_name().to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor"); - let actor_id = res.actor.actor_id.to_string(); - - common::assert_actor_exists(ctx.leader_dc().guard_port(), &actor_id, &namespace).await; - - // TODO: Hook into engine instead of sleep - tokio::time::sleep(std::time::Duration::from_secs(1)).await; - - assert!( - runner.has_actor(&actor_id).await, - "runner should have the actor" - ); - }, - ); + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + let res = common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: "test-actor".to_string(), + key: None, + input: None, + runner_name_selector: runner.pool_name().to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to create actor"); + let actor_id = res.actor.actor_id.to_string(); + + common::assert_actor_exists(ctx.leader_dc().guard_port(), &actor_id, &namespace).await; + + // TODO: Hook into engine instead of sleep + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + + assert!( + runner.has_actor(&actor_id).await, + "runner should have the actor" + ); + }); } #[test] fn create_actor_with_key() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - let key = common::generate_unique_key(); - let res = common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: "test-actor".to_string(), - key: Some(key.clone()), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor"); - let actor_id = res.actor.actor_id.to_string(); - - assert!(!actor_id.is_empty(), "actor ID should not be empty"); - - // Verify actor exists - let actor = - common::assert_actor_exists(ctx.leader_dc().guard_port(), &actor_id, &namespace) - .await; - assert_eq!(actor.key, Some(key)); - }, - ); + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + let key = common::generate_unique_key(); + let res = common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: "test-actor".to_string(), + key: Some(key.clone()), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to create actor"); + let actor_id = res.actor.actor_id.to_string(); + + assert!(!actor_id.is_empty(), "actor ID should not be empty"); + + // Verify actor exists + let actor = + common::assert_actor_exists(ctx.leader_dc().guard_port(), &actor_id, &namespace).await; + assert_eq!(actor.key, Some(key)); + }); } #[test] fn create_actor_with_input() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - let input_data = common::generate_test_input_data(); - let res = common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: "test-actor".to_string(), - key: None, - input: Some(input_data.clone()), - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor"); - let actor_id = res.actor.actor_id.to_string(); - - assert!(!actor_id.is_empty(), "actor ID should not be empty"); - }, - ); + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + let input_data = common::generate_test_input_data(); + let res = common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: "test-actor".to_string(), + key: None, + input: Some(input_data.clone()), + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to create actor"); + let actor_id = res.actor.actor_id.to_string(); + + assert!(!actor_id.is_empty(), "actor ID should not be empty"); + }); } -#[ignore = "non-sleep crash policies are not yet supported for envoys"] #[test] -fn create_durable_actor() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - let res = common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: "test-actor".to_string(), - key: None, - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Restart, - }, - ) - .await - .expect("failed to create actor"); - let actor_id = res.actor.actor_id.to_string(); - - assert!(!actor_id.is_empty(), "actor ID should not be empty"); - - // Verify actor is durable - let actor = - common::assert_actor_exists(ctx.leader_dc().guard_port(), &actor_id, &namespace) - .await; - assert_eq!( - actor.crash_policy, - rivet_types::actors::CrashPolicy::Restart - ); - }, - ); +fn create_actor_sleep_crash_policy() { + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + let res = common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: "test-actor".to_string(), + key: None, + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to create actor"); + let actor_id = res.actor.actor_id.to_string(); + + assert!(!actor_id.is_empty(), "actor ID should not be empty"); + + let actor = + common::assert_actor_exists(ctx.leader_dc().guard_port(), &actor_id, &namespace).await; + assert_eq!( + actor.crash_policy, + rivet_types::actors::CrashPolicy::Sleep + ); + }); } #[test] // Broken legacy Pegboard Runner test: full engine sweep timed out in // `create_actor_specific_datacenter`. -#[ignore = "broken legacy Pegboard Runner test: times out in full engine sweep"] fn create_actor_specific_datacenter() { - common::run( - common::TestOpts::new(2).with_timeout(45), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - let res = common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: Some("dc-2".to_string()), - name: "test-actor".to_string(), - key: None, - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor"); - let actor_id = res.actor.actor_id.to_string(); - - assert!(!actor_id.is_empty(), "actor ID should not be empty"); - - let actor = - common::assert_actor_exists(ctx.leader_dc().guard_port(), &actor_id, &namespace) - .await; - common::assert_actor_in_dc(&actor.actor_id.to_string(), 2).await; - }, - ); + common::run(common::TestOpts::new(2).with_timeout(45), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + let res = common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: Some("dc-2".to_string()), + name: "test-actor".to_string(), + key: None, + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to create actor"); + let actor_id = res.actor.actor_id.to_string(); + + assert!(!actor_id.is_empty(), "actor ID should not be empty"); + + let actor = + common::assert_actor_exists(ctx.leader_dc().guard_port(), &actor_id, &namespace).await; + common::assert_actor_in_dc(&actor.actor_id.to_string(), 2).await; + }); } // MARK: Error cases #[test] fn create_actor_non_existent_namespace() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let res = common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: "non-existent-namespace".to_string(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: "test-actor".to_string(), - key: None, - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await; - - assert!( - res.is_err(), - "should fail to create actor with non-existent namespace" - ); - }, - ); + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let res = common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: "non-existent-namespace".to_string(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: "test-actor".to_string(), + key: None, + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await; + + assert!( + res.is_err(), + "should fail to create actor with non-existent namespace" + ); + }); } #[test] fn create_actor_invalid_datacenter() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - let res = common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: Some("invalid-dc".to_string()), - name: "test-actor".to_string(), - key: None, - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await; - - assert!( - res.is_err(), - "should fail to create actor with invalid datacenter" - ); - }, - ); + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + let res = common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: Some("invalid-dc".to_string()), + name: "test-actor".to_string(), + key: None, + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await; + + assert!( + res.is_err(), + "should fail to create actor with invalid datacenter" + ); + }); } // MARK: Cross-datacenter tests #[test] fn create_actor_remote_datacenter_verify() { - common::run( - common::TestOpts::new(2).with_timeout(45), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - let res = common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: Some("dc-2".to_string()), - name: "test-actor".to_string(), - key: None, - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor"); - - let actor_id = res.actor.actor_id.to_string(); - - let actor = - common::assert_actor_exists(ctx.get_dc(2).guard_port(), &actor_id, &namespace) - .await; - common::assert_actor_in_dc(&actor.actor_id.to_string(), 2).await; - }, - ); + common::run(common::TestOpts::new(2).with_timeout(45), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + let res = common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: Some("dc-2".to_string()), + name: "test-actor".to_string(), + key: None, + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to create actor"); + + let actor_id = res.actor.actor_id.to_string(); + + let actor = + common::assert_actor_exists(ctx.get_dc(2).guard_port(), &actor_id, &namespace).await; + common::assert_actor_in_dc(&actor.actor_id.to_string(), 2).await; + }); } // MARK: Input validation tests @@ -295,176 +264,160 @@ fn create_actor_remote_datacenter_verify() { #[test] fn create_actor_input_large() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - // Create a large input (1 MiB) that should succeed - let input_size = 1024 * 1024; - let input_data = "a".repeat(input_size); - - let res = common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: "test-actor".to_string(), - key: None, - input: Some(input_data), - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("should succeed with large input"); - - let actor_id = res.actor.actor_id.to_string(); - assert!(!actor_id.is_empty(), "actor ID should not be empty"); - }, - ); + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + // Create a large input (1 MiB) that should succeed + let input_size = 1024 * 1024; + let input_data = "a".repeat(input_size); + + let res = common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: "test-actor".to_string(), + key: None, + input: Some(input_data), + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("should succeed with large input"); + + let actor_id = res.actor.actor_id.to_string(); + assert!(!actor_id.is_empty(), "actor ID should not be empty"); + }); } #[test] fn create_actor_input_exceeds_max_size() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - // Create input exceeding 4 MiB - let max_input_size = 4 * 1024 * 1024; - let input_data = "a".repeat(max_input_size + 1); - - let res = common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: "test-actor".to_string(), - key: None, - input: Some(input_data), - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await; - - assert!( - res.is_err(), - "should fail to create actor with input exceeding max size" - ); - }, - ); + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + // Create input exceeding 4 MiB + let max_input_size = 4 * 1024 * 1024; + let input_data = "a".repeat(max_input_size + 1); + + let res = common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: "test-actor".to_string(), + key: None, + input: Some(input_data), + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await; + + assert!( + res.is_err(), + "should fail to create actor with input exceeding max size" + ); + }); } // MARK: Key validation tests #[test] fn create_actor_empty_key() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - let res = common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: "test-actor".to_string(), - key: Some("".to_string()), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await; - - assert!(res.is_err(), "should fail to create actor with empty key"); - }, - ); + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + let res = common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: "test-actor".to_string(), + key: Some("".to_string()), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await; + + assert!(res.is_err(), "should fail to create actor with empty key"); + }); } #[test] fn create_actor_key_at_max_size() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - // Create key of exactly 1024 bytes - let key = "a".repeat(1024); - - let res = common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: "test-actor".to_string(), - key: Some(key.clone()), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("should succeed with key at max size"); - - let actor_id = res.actor.actor_id.to_string(); - assert!(!actor_id.is_empty(), "actor ID should not be empty"); - - // Verify actor exists with correct key - let actor = - common::assert_actor_exists(ctx.leader_dc().guard_port(), &actor_id, &namespace) - .await; - assert_eq!(actor.key, Some(key)); - }, - ); + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + // Create key of exactly 1024 bytes + let key = "a".repeat(1024); + + let res = common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: "test-actor".to_string(), + key: Some(key.clone()), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("should succeed with key at max size"); + + let actor_id = res.actor.actor_id.to_string(); + assert!(!actor_id.is_empty(), "actor ID should not be empty"); + + // Verify actor exists with correct key + let actor = + common::assert_actor_exists(ctx.leader_dc().guard_port(), &actor_id, &namespace).await; + assert_eq!(actor.key, Some(key)); + }); } #[test] fn create_actor_key_exceeds_max_size() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - // Create key exceeding 1024 bytes - let key = "a".repeat(1025); - - let res = common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: "test-actor".to_string(), - key: Some(key), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await; - - assert!( - res.is_err(), - "should fail to create actor with key exceeding max size" - ); - }, - ); + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + // Create key exceeding 1024 bytes + let key = "a".repeat(1025); + + let res = common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: "test-actor".to_string(), + key: Some(key), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await; + + assert!( + res.is_err(), + "should fail to create actor with key exceeding max size" + ); + }); } diff --git a/engine/packages/engine/tests/envoy/api_actors_delete.rs b/engine/packages/engine/tests/envoy/api_actors_delete.rs index 5dcd3989b5..61b3203853 100644 --- a/engine/packages/engine/tests/envoy/api_actors_delete.rs +++ b/engine/packages/engine/tests/envoy/api_actors_delete.rs @@ -3,327 +3,303 @@ use super::super::common; // MARK: Basic #[test] fn delete_existing_actor_with_namespace() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - // Create an actor - let res = common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: "test-actor".to_string(), - key: None, - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor"); - let actor_id = res.actor.actor_id.to_string(); - - // Verify actor exists - common::assert_actor_exists(ctx.leader_dc().guard_port(), &actor_id, &namespace).await; - - // Delete the actor with namespace - common::api::public::actors_delete( - ctx.leader_dc().guard_port(), - common::api_types::actors::delete::DeletePath { - actor_id: actor_id.parse().expect("failed to parse actor_id"), - }, - common::api_types::actors::delete::DeleteQuery { - namespace: namespace.clone(), - }, - ) - .await - .expect("failed to delete actor"); - - // Verify actor is destroyed - common::assert_actor_is_destroyed(ctx.leader_dc().guard_port(), &actor_id, &namespace) - .await; - }, - ); + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + // Create an actor + let res = common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: "test-actor".to_string(), + key: None, + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to create actor"); + let actor_id = res.actor.actor_id.to_string(); + + // Verify actor exists + common::assert_actor_exists(ctx.leader_dc().guard_port(), &actor_id, &namespace).await; + + // Delete the actor with namespace + common::api::public::actors_delete( + ctx.leader_dc().guard_port(), + common::api_types::actors::delete::DeletePath { + actor_id: actor_id.parse().expect("failed to parse actor_id"), + }, + common::api_types::actors::delete::DeleteQuery { + namespace: namespace.clone(), + }, + ) + .await + .expect("failed to delete actor"); + + // Verify actor is destroyed + common::assert_actor_is_destroyed(ctx.leader_dc().guard_port(), &actor_id, &namespace) + .await; + }); } #[test] fn delete_existing_actor_without_namespace() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - // Create an actor - let res = common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: "test-actor".to_string(), - key: None, - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor"); - let actor_id = res.actor.actor_id.to_string(); - - // Verify actor exists - common::assert_actor_exists(ctx.leader_dc().guard_port(), &actor_id, &namespace).await; - - // Delete the actor without namespace parameter - common::api::public::actors_delete( - ctx.leader_dc().guard_port(), - common::api_types::actors::delete::DeletePath { - actor_id: actor_id.parse().expect("failed to parse actor_id"), - }, - common::api_types::actors::delete::DeleteQuery { - namespace: namespace.to_string(), - }, - ) - .await - .expect("failed to delete actor"); - - // Verify actor is destroyed - common::assert_actor_is_destroyed(ctx.leader_dc().guard_port(), &actor_id, &namespace) - .await; - }, - ); + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + // Create an actor + let res = common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: "test-actor".to_string(), + key: None, + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to create actor"); + let actor_id = res.actor.actor_id.to_string(); + + // Verify actor exists + common::assert_actor_exists(ctx.leader_dc().guard_port(), &actor_id, &namespace).await; + + // Delete the actor without namespace parameter + common::api::public::actors_delete( + ctx.leader_dc().guard_port(), + common::api_types::actors::delete::DeletePath { + actor_id: actor_id.parse().expect("failed to parse actor_id"), + }, + common::api_types::actors::delete::DeleteQuery { + namespace: namespace.to_string(), + }, + ) + .await + .expect("failed to delete actor"); + + // Verify actor is destroyed + common::assert_actor_is_destroyed(ctx.leader_dc().guard_port(), &actor_id, &namespace) + .await; + }); } #[test] fn delete_actor_current_datacenter() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - // Create an actor in current datacenter - let res = common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: "test-actor".to_string(), - key: None, - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor"); - let actor_id = res.actor.actor_id; - - // Delete the actor - common::api::public::actors_delete( - ctx.leader_dc().guard_port(), - common::api_types::actors::delete::DeletePath { actor_id: actor_id }, - common::api_types::actors::delete::DeleteQuery { - namespace: namespace.clone(), - }, - ) - .await - .expect("failed to delete actor"); - - // Verify actor is destroyed - common::assert_actor_is_destroyed( - ctx.leader_dc().guard_port(), - &actor_id.to_string(), - &namespace, - ) - .await; - }, - ); + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + // Create an actor in current datacenter + let res = common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: "test-actor".to_string(), + key: None, + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to create actor"); + let actor_id = res.actor.actor_id; + + // Delete the actor + common::api::public::actors_delete( + ctx.leader_dc().guard_port(), + common::api_types::actors::delete::DeletePath { actor_id: actor_id }, + common::api_types::actors::delete::DeleteQuery { + namespace: namespace.clone(), + }, + ) + .await + .expect("failed to delete actor"); + + // Verify actor is destroyed + common::assert_actor_is_destroyed( + ctx.leader_dc().guard_port(), + &actor_id.to_string(), + &namespace, + ) + .await; + }); } #[test] fn delete_actor_remote_datacenter() { - common::run( - common::TestOpts::new(2).with_timeout(45), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - // Create an actor in DC2 - let res = common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: Some("dc-2".to_string()), - name: "test-actor".to_string(), - key: None, - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor"); - let actor_id = res.actor.actor_id.to_string(); - - // Delete the actor from DC1 (will route to DC2) - common::api::public::actors_delete( - ctx.leader_dc().guard_port(), - common::api_types::actors::delete::DeletePath { - actor_id: actor_id.parse().expect("failed to parse actor_id"), - }, - common::api_types::actors::delete::DeleteQuery { - namespace: namespace.clone(), - }, - ) - .await - .expect("failed to delete actor"); - - // Verify actor is destroyed in DC2 - common::assert_actor_is_destroyed(ctx.get_dc(2).guard_port(), &actor_id, &namespace) - .await; - }, - ); + common::run(common::TestOpts::new(2).with_timeout(45), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + // Create an actor in DC2 + let res = common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: Some("dc-2".to_string()), + name: "test-actor".to_string(), + key: None, + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to create actor"); + let actor_id = res.actor.actor_id.to_string(); + + // Delete the actor from DC1 (will route to DC2) + common::api::public::actors_delete( + ctx.leader_dc().guard_port(), + common::api_types::actors::delete::DeletePath { + actor_id: actor_id.parse().expect("failed to parse actor_id"), + }, + common::api_types::actors::delete::DeleteQuery { + namespace: namespace.clone(), + }, + ) + .await + .expect("failed to delete actor"); + + // Verify actor is destroyed in DC2 + common::assert_actor_is_destroyed(ctx.get_dc(2).guard_port(), &actor_id, &namespace).await; + }); } // MARK: Error cases #[test] fn delete_non_existent_actor() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - // Generate a fake actor ID with valid format but non-existent - let fake_actor_id = rivet_util::Id::new_v1(ctx.leader_dc().config.dc_label()); - - let res = common::api::public::actors_delete( - ctx.leader_dc().guard_port(), - common::api_types::actors::delete::DeletePath { - actor_id: fake_actor_id, - }, - common::api_types::actors::delete::DeleteQuery { - namespace: namespace.to_string(), - }, - ) - .await; - - assert!(res.is_err(), "should fail to delete non-existent actor"); - }, - ); + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + // Generate a fake actor ID with valid format but non-existent + let fake_actor_id = rivet_util::Id::new_v1(ctx.leader_dc().config.dc_label()); + + let res = common::api::public::actors_delete( + ctx.leader_dc().guard_port(), + common::api_types::actors::delete::DeletePath { + actor_id: fake_actor_id, + }, + common::api_types::actors::delete::DeleteQuery { + namespace: namespace.to_string(), + }, + ) + .await; + + assert!(res.is_err(), "should fail to delete non-existent actor"); + }); } #[test] fn delete_actor_wrong_namespace() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace1, _, _runner1) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - let (namespace2, _, _runner2) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - // Create actor in namespace1 - let res = common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace1.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: "test-actor".to_string(), - key: None, - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor"); - let actor_id = res.actor.actor_id.to_string(); - - // Try to delete with namespace2 - let res = common::api::public::actors_delete( - ctx.leader_dc().guard_port(), - common::api_types::actors::delete::DeletePath { - actor_id: actor_id.parse().expect("failed to parse actor_id"), - }, - common::api_types::actors::delete::DeleteQuery { - namespace: namespace2.clone(), - }, - ) - .await; - - assert!( - res.is_err(), - "should fail to delete actor with wrong namespace" - ); - - // Verify actor still exists in namespace1 - common::assert_actor_is_alive(ctx.leader_dc().guard_port(), &actor_id, &namespace1) - .await; - }, - ); + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace1, _, _runner1) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + let (namespace2, _, _runner2) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + // Create actor in namespace1 + let res = common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace1.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: "test-actor".to_string(), + key: None, + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to create actor"); + let actor_id = res.actor.actor_id.to_string(); + + // Try to delete with namespace2 + let res = common::api::public::actors_delete( + ctx.leader_dc().guard_port(), + common::api_types::actors::delete::DeletePath { + actor_id: actor_id.parse().expect("failed to parse actor_id"), + }, + common::api_types::actors::delete::DeleteQuery { + namespace: namespace2.clone(), + }, + ) + .await; + + assert!( + res.is_err(), + "should fail to delete actor with wrong namespace" + ); + + // Verify actor still exists in namespace1 + common::assert_actor_is_alive(ctx.leader_dc().guard_port(), &actor_id, &namespace1).await; + }); } #[test] // Broken legacy Pegboard Runner test: full engine sweep timed out in // `delete_with_non_existent_namespace`. -#[ignore = "broken legacy Pegboard Runner test: times out in full engine sweep"] fn delete_with_non_existent_namespace() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - // Create an actor - let res = common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: "test-actor".to_string(), - key: None, - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor"); - let actor_id = res.actor.actor_id.to_string(); - - // Try to delete with non-existent namespace - let res = common::api::public::actors_delete( - ctx.leader_dc().guard_port(), - common::api_types::actors::delete::DeletePath { - actor_id: actor_id.parse().expect("failed to parse actor_id"), - }, - common::api_types::actors::delete::DeleteQuery { - namespace: "non-existent-namespace".to_string(), - }, - ) - .await; - - assert!(res.is_err(), "should fail with non-existent namespace"); - }, - ); + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + // Create an actor + let res = common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: "test-actor".to_string(), + key: None, + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to create actor"); + let actor_id = res.actor.actor_id.to_string(); + + // Try to delete with non-existent namespace + let res = common::api::public::actors_delete( + ctx.leader_dc().guard_port(), + common::api_types::actors::delete::DeletePath { + actor_id: actor_id.parse().expect("failed to parse actor_id"), + }, + common::api_types::actors::delete::DeleteQuery { + namespace: "non-existent-namespace".to_string(), + }, + ) + .await; + + assert!(res.is_err(), "should fail with non-existent namespace"); + }); } // Note: Invalid actor ID format test removed because it would be caught at parsing level @@ -333,55 +309,51 @@ fn delete_with_non_existent_namespace() { #[test] fn delete_remote_actor_verify_propagation() { - common::run( - common::TestOpts::new(2).with_timeout(45), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - // Create an actor in DC2 - let res = common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: Some("dc-2".to_string()), - name: "test-actor".to_string(), - key: None, - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor"); - let actor_id = res.actor.actor_id.to_string(); - - // Verify actor exists in both datacenters - common::assert_actor_exists(ctx.leader_dc().guard_port(), &actor_id, &namespace).await; - common::assert_actor_exists(ctx.get_dc(2).guard_port(), &actor_id, &namespace).await; - - // Delete the actor from DC1 - common::api::public::actors_delete( - ctx.leader_dc().guard_port(), - common::api_types::actors::delete::DeletePath { - actor_id: actor_id.parse().expect("failed to parse actor_id"), - }, - common::api_types::actors::delete::DeleteQuery { - namespace: namespace.clone(), - }, - ) - .await - .expect("failed to delete actor"); - - // Verify actor is destroyed in both datacenters - common::assert_actor_is_destroyed(ctx.leader_dc().guard_port(), &actor_id, &namespace) - .await; - common::assert_actor_is_destroyed(ctx.get_dc(2).guard_port(), &actor_id, &namespace) - .await; - }, - ); + common::run(common::TestOpts::new(2).with_timeout(45), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + // Create an actor in DC2 + let res = common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: Some("dc-2".to_string()), + name: "test-actor".to_string(), + key: None, + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to create actor"); + let actor_id = res.actor.actor_id.to_string(); + + // Verify actor exists in both datacenters + common::assert_actor_exists(ctx.leader_dc().guard_port(), &actor_id, &namespace).await; + common::assert_actor_exists(ctx.get_dc(2).guard_port(), &actor_id, &namespace).await; + + // Delete the actor from DC1 + common::api::public::actors_delete( + ctx.leader_dc().guard_port(), + common::api_types::actors::delete::DeletePath { + actor_id: actor_id.parse().expect("failed to parse actor_id"), + }, + common::api_types::actors::delete::DeleteQuery { + namespace: namespace.clone(), + }, + ) + .await + .expect("failed to delete actor"); + + // Verify actor is destroyed in both datacenters + common::assert_actor_is_destroyed(ctx.leader_dc().guard_port(), &actor_id, &namespace) + .await; + common::assert_actor_is_destroyed(ctx.get_dc(2).guard_port(), &actor_id, &namespace).await; + }); } // MARK: Edge cases @@ -391,34 +363,96 @@ fn delete_remote_actor_verify_propagation() { #[test] #[ignore = "broken legacy Pegboard Runner test: second delete returns actor.not_found"] fn delete_already_destroyed_actor() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - // Create an actor - let res = common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: "test-actor".to_string(), - key: None, - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor"); - let actor_id = res.actor.actor_id.to_string(); + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + // Create an actor + let res = common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: "test-actor".to_string(), + key: None, + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to create actor"); + let actor_id = res.actor.actor_id.to_string(); + + // Delete the actor once + common::api::public::actors_delete( + ctx.leader_dc().guard_port(), + common::api_types::actors::delete::DeletePath { + actor_id: actor_id.parse().expect("failed to parse actor_id"), + }, + common::api_types::actors::delete::DeleteQuery { + namespace: namespace.clone(), + }, + ) + .await + .expect("failed to delete actor"); + + // Delete the actor again - should handle gracefully (WorkflowNotFound) + // The implementation logs a warning but doesn't error when workflow is not found + let res = common::api::public::actors_delete( + ctx.leader_dc().guard_port(), + common::api_types::actors::delete::DeletePath { + actor_id: actor_id.parse().expect("failed to parse actor_id"), + }, + common::api_types::actors::delete::DeleteQuery { + namespace: namespace.clone(), + }, + ) + .await; + + // Should succeed even though actor was already destroyed + assert!( + res.is_ok(), + "deleting already destroyed actor should succeed gracefully" + ); + }); +} - // Delete the actor once +#[test] +fn delete_actor_twice_rapidly() { + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + // Create an actor + let res = common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: "test-actor".to_string(), + key: None, + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to create actor"); + let actor_id = res.actor.actor_id.to_string(); + + // Send two delete requests in rapid succession + let actor_id_clone = actor_id.clone(); + let namespace_clone = namespace.clone(); + let port = ctx.leader_dc().guard_port(); + + let delete1 = tokio::spawn(async move { common::api::public::actors_delete( - ctx.leader_dc().guard_port(), + port, common::api_types::actors::delete::DeletePath { actor_id: actor_id.parse().expect("failed to parse actor_id"), }, @@ -427,100 +461,32 @@ fn delete_already_destroyed_actor() { }, ) .await - .expect("failed to delete actor"); + }); - // Delete the actor again - should handle gracefully (WorkflowNotFound) - // The implementation logs a warning but doesn't error when workflow is not found - let res = common::api::public::actors_delete( - ctx.leader_dc().guard_port(), + let delete2 = tokio::spawn(async move { + common::api::public::actors_delete( + port, common::api_types::actors::delete::DeletePath { - actor_id: actor_id.parse().expect("failed to parse actor_id"), + actor_id: actor_id_clone.parse().expect("failed to parse actor_id"), }, common::api_types::actors::delete::DeleteQuery { - namespace: namespace.clone(), + namespace: namespace_clone.clone(), }, ) - .await; + .await + }); - // Should succeed even though actor was already destroyed - assert!( - res.is_ok(), - "deleting already destroyed actor should succeed gracefully" - ); - }, - ); -} + // Both should complete without panicking + let (res1, res2) = tokio::join!(delete1, delete2); -#[test] -fn delete_actor_twice_rapidly() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - // Create an actor - let res = common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: "test-actor".to_string(), - key: None, - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor"); - let actor_id = res.actor.actor_id.to_string(); - - // Send two delete requests in rapid succession - let actor_id_clone = actor_id.clone(); - let namespace_clone = namespace.clone(); - let port = ctx.leader_dc().guard_port(); - - let delete1 = tokio::spawn(async move { - common::api::public::actors_delete( - port, - common::api_types::actors::delete::DeletePath { - actor_id: actor_id.parse().expect("failed to parse actor_id"), - }, - common::api_types::actors::delete::DeleteQuery { - namespace: namespace.clone(), - }, - ) - .await - }); - - let delete2 = tokio::spawn(async move { - common::api::public::actors_delete( - port, - common::api_types::actors::delete::DeletePath { - actor_id: actor_id_clone.parse().expect("failed to parse actor_id"), - }, - common::api_types::actors::delete::DeleteQuery { - namespace: namespace_clone.clone(), - }, - ) - .await - }); - - // Both should complete without panicking - let (res1, res2) = tokio::join!(delete1, delete2); - - // At least one should succeed - let res1 = res1.expect("task should not panic"); - let res2 = res2.expect("task should not panic"); - - // Both requests should succeed or fail gracefully (no panics) - assert!( - res1.is_ok() || res2.is_ok(), - "at least one delete should succeed in race condition" - ); - }, - ); + // At least one should succeed + let res1 = res1.expect("task should not panic"); + let res2 = res2.expect("task should not panic"); + + // Both requests should succeed or fail gracefully (no panics) + assert!( + res1.is_ok() || res2.is_ok(), + "at least one delete should succeed in race condition" + ); + }); } diff --git a/engine/packages/engine/tests/envoy/api_actors_get_or_create.rs b/engine/packages/engine/tests/envoy/api_actors_get_or_create.rs index fc75972ebf..fab3f4b973 100644 --- a/engine/packages/engine/tests/envoy/api_actors_get_or_create.rs +++ b/engine/packages/engine/tests/envoy/api_actors_get_or_create.rs @@ -4,77 +4,168 @@ use super::super::common; #[test] fn get_or_create_creates_new_actor() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - let actor_name = "test-actor"; - let actor_key = "unique-key-1"; - - // First call should create the actor - let response = common::api::public::actors_get_or_create( - ctx.leader_dc().guard_port(), - common::api::public::GetOrCreateQuery { - namespace: namespace.clone(), - }, - common::api::public::GetOrCreateRequest { - datacenter: None, - name: actor_name.to_string(), - key: actor_key.to_string(), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to get or create actor"); - - assert!(response.created, "Actor should be newly created"); - assert_eq!(response.actor.name, actor_name); - assert_eq!(response.actor.key.as_ref().unwrap(), actor_key); - }, - ); + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + let actor_name = "test-actor"; + let actor_key = "unique-key-1"; + + // First call should create the actor + let response = common::api::public::actors_get_or_create( + ctx.leader_dc().guard_port(), + common::api::public::GetOrCreateQuery { + namespace: namespace.clone(), + }, + common::api::public::GetOrCreateRequest { + datacenter: None, + name: actor_name.to_string(), + key: actor_key.to_string(), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to get or create actor"); + + assert!(response.created, "Actor should be newly created"); + assert_eq!(response.actor.name, actor_name); + assert_eq!(response.actor.key.as_ref().unwrap(), actor_key); + }); } #[test] // Broken legacy Pegboard Runner test: full engine sweep timed out in // `get_or_create_returns_existing_actor`. -#[ignore = "broken legacy Pegboard Runner test: times out in full engine sweep"] fn get_or_create_returns_existing_actor() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + let actor_name = "test-actor"; + let actor_key = "unique-key-2"; + + // First call - create + let response1 = common::api::public::actors_get_or_create( + ctx.leader_dc().guard_port(), + common::api::public::GetOrCreateQuery { + namespace: namespace.clone(), + }, + common::api::public::GetOrCreateRequest { + datacenter: None, + name: actor_name.to_string(), + key: actor_key.to_string(), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to get or create actor"); + + assert!(response1.created, "First call should create actor"); + let first_actor_id = response1.actor.actor_id; + + // Second call with same key - should return existing + let response2 = common::api::public::actors_get_or_create( + ctx.leader_dc().guard_port(), + common::api::public::GetOrCreateQuery { + namespace: namespace.clone(), + }, + common::api::public::GetOrCreateRequest { + datacenter: None, + name: actor_name.to_string(), + key: actor_key.to_string(), + input: Some("different-input".to_string()), // Different input should be ignored + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to get or create actor"); + + assert!( + !response2.created, + "Second call should return existing actor" + ); + assert_eq!( + response2.actor.actor_id, first_actor_id, + "Should return the same actor ID" + ); + }); +} - let actor_name = "test-actor"; - let actor_key = "unique-key-2"; +#[test] +// Broken legacy Pegboard Runner test: full engine sweep timed out in +// `get_or_create_same_name_different_keys`. +fn get_or_create_same_name_different_keys() { + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + let actor_name = "shared-name"; + + // Create first actor with key1 + let response1 = common::api::public::actors_get_or_create( + ctx.leader_dc().guard_port(), + common::api::public::GetOrCreateQuery { + namespace: namespace.clone(), + }, + common::api::public::GetOrCreateRequest { + datacenter: None, + name: actor_name.to_string(), + key: "key1".to_string(), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to get or create actor 1"); + + // Create second actor with same name but different key + let response2 = common::api::public::actors_get_or_create( + ctx.leader_dc().guard_port(), + common::api::public::GetOrCreateQuery { + namespace: namespace.clone(), + }, + common::api::public::GetOrCreateRequest { + datacenter: None, + name: actor_name.to_string(), + key: "key2".to_string(), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to get or create actor 2"); + + assert!(response1.created, "First actor should be created"); + assert!(response2.created, "Second actor should be created"); + assert_ne!( + response1.actor.actor_id, response2.actor.actor_id, + "Different keys should create different actors" + ); + assert_eq!(response1.actor.name, actor_name); + assert_eq!(response2.actor.name, actor_name); + }); +} - // First call - create - let response1 = common::api::public::actors_get_or_create( - ctx.leader_dc().guard_port(), - common::api::public::GetOrCreateQuery { - namespace: namespace.clone(), - }, - common::api::public::GetOrCreateRequest { - datacenter: None, - name: actor_name.to_string(), - key: actor_key.to_string(), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to get or create actor"); +#[test] +fn get_or_create_idempotent() { + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - assert!(response1.created, "First call should create actor"); - let first_actor_id = response1.actor.actor_id; + let actor_name = "idempotent-actor"; + let actor_key = "idempotent-key"; - // Second call with same key - should return existing - let response2 = common::api::public::actors_get_or_create( + // Make multiple calls with the same key + let mut actor_id = None; + for i in 0..5 { + let response = common::api::public::actors_get_or_create( ctx.leader_dc().guard_port(), common::api::public::GetOrCreateQuery { namespace: namespace.clone(), @@ -83,174 +174,123 @@ fn get_or_create_returns_existing_actor() { datacenter: None, name: actor_name.to_string(), key: actor_key.to_string(), - input: Some("different-input".to_string()), // Different input should be ignored + input: None, runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, + crash_policy: rivet_types::actors::CrashPolicy::Sleep, }, ) .await .expect("failed to get or create actor"); - assert!( - !response2.created, - "Second call should return existing actor" - ); - assert_eq!( - response2.actor.actor_id, first_actor_id, - "Should return the same actor ID" - ); - }, - ); + if i == 0 { + assert!(response.created, "First call should create"); + actor_id = Some(response.actor.actor_id); + } else { + assert!(!response.created, "Subsequent calls should return existing"); + assert_eq!( + response.actor.actor_id, + actor_id.unwrap(), + "All calls should return the same actor" + ); + } + } + }); } -#[test] -// Broken legacy Pegboard Runner test: full engine sweep timed out in -// `get_or_create_same_name_different_keys`. -#[ignore = "broken legacy Pegboard Runner test: times out in full engine sweep"] -fn get_or_create_same_name_different_keys() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - let actor_name = "shared-name"; +// MARK: Race condition tests - // Create first actor with key1 - let response1 = common::api::public::actors_get_or_create( - ctx.leader_dc().guard_port(), +#[test] +fn get_or_create_race_condition_handling() { + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + let actor_name = "race-actor"; + let actor_key = "race-key"; + let port = ctx.leader_dc().guard_port(); + let namespace_clone1 = namespace.clone(); + let namespace_clone2 = namespace.clone(); + + // Launch two concurrent get_or_create requests with the same key + let handle1 = tokio::spawn(async move { + common::api::public::actors_get_or_create( + port, common::api::public::GetOrCreateQuery { - namespace: namespace.clone(), + namespace: namespace_clone1, }, common::api::public::GetOrCreateRequest { datacenter: None, name: actor_name.to_string(), - key: "key1".to_string(), + key: actor_key.to_string(), input: None, runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, + crash_policy: rivet_types::actors::CrashPolicy::Sleep, }, ) .await - .expect("failed to get or create actor 1"); + }); - // Create second actor with same name but different key - let response2 = common::api::public::actors_get_or_create( - ctx.leader_dc().guard_port(), + let handle2 = tokio::spawn(async move { + common::api::public::actors_get_or_create( + port, common::api::public::GetOrCreateQuery { - namespace: namespace.clone(), + namespace: namespace_clone2, }, common::api::public::GetOrCreateRequest { datacenter: None, name: actor_name.to_string(), - key: "key2".to_string(), + key: actor_key.to_string(), input: None, runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, + crash_policy: rivet_types::actors::CrashPolicy::Sleep, }, ) .await - .expect("failed to get or create actor 2"); - - assert!(response1.created, "First actor should be created"); - assert!(response2.created, "Second actor should be created"); - assert_ne!( - response1.actor.actor_id, response2.actor.actor_id, - "Different keys should create different actors" - ); - assert_eq!(response1.actor.name, actor_name); - assert_eq!(response2.actor.name, actor_name); - }, - ); -} - -#[test] -fn get_or_create_idempotent() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - let actor_name = "idempotent-actor"; - let actor_key = "idempotent-key"; - - // Make multiple calls with the same key - let mut actor_id = None; - for i in 0..5 { - let response = common::api::public::actors_get_or_create( - ctx.leader_dc().guard_port(), - common::api::public::GetOrCreateQuery { - namespace: namespace.clone(), - }, - common::api::public::GetOrCreateRequest { - datacenter: None, - name: actor_name.to_string(), - key: actor_key.to_string(), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to get or create actor"); - - if i == 0 { - assert!(response.created, "First call should create"); - actor_id = Some(response.actor.actor_id); - } else { - assert!(!response.created, "Subsequent calls should return existing"); - assert_eq!( - response.actor.actor_id, - actor_id.unwrap(), - "All calls should return the same actor" - ); - } - } - }, - ); + }); + + let (result1, result2) = tokio::join!(handle1, handle2); + let response1 = result1.expect("task 1 panicked").expect("request 1 failed"); + let response2 = result2.expect("task 2 panicked").expect("request 2 failed"); + + // Both should succeed + assert_eq!( + response1.actor.actor_id, response2.actor.actor_id, + "Both requests should return the same actor" + ); + + // Exactly one should have created=true + let created_count = [response1.created, response2.created] + .iter() + .filter(|&&c| c) + .count(); + assert_eq!( + created_count, 1, + "Exactly one request should report creation" + ); + }); } -// MARK: Race condition tests - #[test] -fn get_or_create_race_condition_handling() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - let actor_name = "race-actor"; - let actor_key = "race-key"; - let port = ctx.leader_dc().guard_port(); - let namespace_clone1 = namespace.clone(); - let namespace_clone2 = namespace.clone(); - - // Launch two concurrent get_or_create requests with the same key - let handle1 = tokio::spawn(async move { - common::api::public::actors_get_or_create( - port, - common::api::public::GetOrCreateQuery { - namespace: namespace_clone1, - }, - common::api::public::GetOrCreateRequest { - datacenter: None, - name: actor_name.to_string(), - key: actor_key.to_string(), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - }); - - let handle2 = tokio::spawn(async move { +// Broken legacy Pegboard Runner test: full engine sweep timed out in +// `get_or_create_returns_winner_on_race`. +fn get_or_create_returns_winner_on_race() { + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + let actor_name = "race-winner-actor"; + let actor_key = "race-winner-key"; + let port = ctx.leader_dc().guard_port(); + + // Launch multiple concurrent requests + let mut handles = vec![]; + for _ in 0..10 { + let namespace_clone = namespace.clone(); + let handle = tokio::spawn(async move { common::api::public::actors_get_or_create( port, common::api::public::GetOrCreateQuery { - namespace: namespace_clone2, + namespace: namespace_clone, }, common::api::public::GetOrCreateRequest { datacenter: None, @@ -258,59 +298,31 @@ fn get_or_create_race_condition_handling() { key: actor_key.to_string(), input: None, runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, + crash_policy: rivet_types::actors::CrashPolicy::Sleep, }, ) .await }); - - let (result1, result2) = tokio::join!(handle1, handle2); - let response1 = result1.expect("task 1 panicked").expect("request 1 failed"); - let response2 = result2.expect("task 2 panicked").expect("request 2 failed"); - - // Both should succeed - assert_eq!( - response1.actor.actor_id, response2.actor.actor_id, - "Both requests should return the same actor" - ); - - // Exactly one should have created=true - let created_count = [response1.created, response2.created] - .iter() - .filter(|&&c| c) - .count(); - assert_eq!( - created_count, 1, - "Exactly one request should report creation" - ); - }, - ); -} - -#[test] -// Broken legacy Pegboard Runner test: full engine sweep timed out in -// `get_or_create_returns_winner_on_race`. -#[ignore = "broken legacy Pegboard Runner test: times out in full engine sweep"] -fn get_or_create_returns_winner_on_race() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - let actor_name = "race-winner-actor"; - let actor_key = "race-winner-key"; - let port = ctx.leader_dc().guard_port(); - - // Launch multiple concurrent requests - let mut handles = vec![]; - for _ in 0..10 { - let namespace_clone = namespace.clone(); - let handle = tokio::spawn(async move { - common::api::public::actors_get_or_create( + handles.push(handle); + } + + // Wait for all to complete + let mut results = vec![]; + for handle in handles { + let task_result = handle.await.expect("task panicked"); + // Handle destroyed_during_creation error which can occur in race conditions + match task_result { + Ok(response) => results.push(response), + Err(e) => { + // destroyed_during_creation is an expected race condition error + if !e.to_string().contains("destroyed_during_creation") { + panic!("unexpected error: {}", e); + } + // Skip this result and retry with get_or_create again + let retry_result = common::api::public::actors_get_or_create( port, common::api::public::GetOrCreateQuery { - namespace: namespace_clone, + namespace: namespace.clone(), }, common::api::public::GetOrCreateRequest { datacenter: None, @@ -318,434 +330,376 @@ fn get_or_create_returns_winner_on_race() { key: actor_key.to_string(), input: None, runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, + crash_policy: rivet_types::actors::CrashPolicy::Sleep, }, ) .await - }); - handles.push(handle); - } - - // Wait for all to complete - let mut results = vec![]; - for handle in handles { - let task_result = handle.await.expect("task panicked"); - // Handle destroyed_during_creation error which can occur in race conditions - match task_result { - Ok(response) => results.push(response), - Err(e) => { - // destroyed_during_creation is an expected race condition error - if !e.to_string().contains("destroyed_during_creation") { - panic!("unexpected error: {}", e); - } - // Skip this result and retry with get_or_create again - let retry_result = common::api::public::actors_get_or_create( - port, - common::api::public::GetOrCreateQuery { - namespace: namespace.clone(), - }, - common::api::public::GetOrCreateRequest { - datacenter: None, - name: actor_name.to_string(), - key: actor_key.to_string(), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("retry request failed"); - results.push(retry_result); - } + .expect("retry request failed"); + results.push(retry_result); } } + } - // All should return the same actor ID - let first_actor_id = results[0].actor.actor_id; - for result in &results { - assert_eq!( - result.actor.actor_id, first_actor_id, - "All requests should return the same actor" - ); - } - - // At least one request should report creation - let created_count = results.iter().filter(|r| r.created).count(); - assert!( - created_count >= 1, - "At least one request should report creation" + // All should return the same actor ID + let first_actor_id = results[0].actor.actor_id; + for result in &results { + assert_eq!( + result.actor.actor_id, first_actor_id, + "All requests should return the same actor" ); - }, - ); + } + + // At least one request should report creation + let created_count = results.iter().filter(|r| r.created).count(); + assert!( + created_count >= 1, + "At least one request should report creation" + ); + }); } // Broken legacy Pegboard Runner multi-DC coverage: full `runner::` sweep times // out with `test timed out: Elapsed(())`. #[test] -#[ignore = "broken legacy Pegboard Runner test: times out in full runner sweep"] +#[ignore = "cross-DC get_or_create not idempotent"] fn get_or_create_race_condition_across_datacenters() { - common::run( - common::TestOpts::new(2).with_timeout(45), - |ctx| async move { - const DC2_RUNNER_NAME: &'static str = "dc-2-runner"; - - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - let _envoy2 = common::test_envoy::TestEnvoyBuilder::new(&namespace) - .with_version(1) - .with_pool_name(DC2_RUNNER_NAME) - .with_actor_behavior("test-actor", |_config| { - Box::new(common::test_envoy::EchoActor::new()) - }) - .build(ctx.get_dc(2)) - .await - .expect("failed to build test envoy"); - - common::upsert_normal_runner_config(ctx.get_dc(2), &namespace, DC2_RUNNER_NAME).await; - _envoy2.start().await.expect("failed to start envoy"); - _envoy2.wait_ready().await; - - let actor_name = "cross-dc-race-actor"; - let actor_key = "cross-dc-race-key"; - let port1 = ctx.leader_dc().guard_port(); - let port2 = ctx.get_dc(2).guard_port(); - let namespace_clone1 = namespace.clone(); - let namespace_clone2 = namespace.clone(); - - // Launch concurrent requests from two different datacenters - let handle1 = tokio::spawn(async move { - common::api::public::actors_get_or_create( - port1, - common::api::public::GetOrCreateQuery { - namespace: namespace_clone1, - }, - common::api::public::GetOrCreateRequest { - datacenter: None, - name: actor_name.to_string(), - key: actor_key.to_string(), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - }); - - let handle2 = tokio::spawn(async move { - common::api::public::actors_get_or_create( - port2, - common::api::public::GetOrCreateQuery { - namespace: namespace_clone2, - }, - common::api::public::GetOrCreateRequest { - datacenter: None, - name: actor_name.to_string(), - key: actor_key.to_string(), - input: None, - runner_name_selector: DC2_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - }); - - let (result1, result2) = tokio::join!(handle1, handle2); - let response1 = result1 - .expect("DC1 task panicked") - .expect("DC1 request failed"); - let response2 = result2 - .expect("DC2 task panicked") - .expect("DC2 request failed"); - - // Both should succeed and return the same actor - assert_eq!( - response1.actor.actor_id, response2.actor.actor_id, - "Both datacenters should return the same actor" - ); - - // At least one should report creation - assert!( - (response1.created || response2.created) - && !(response1.created && response2.created), - "At least one datacenter should report creation, but not both" - ); - }, - ); -} - -// MARK: Datacenter tests - -#[test] -fn get_or_create_in_current_datacenter() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - let response = common::api::public::actors_get_or_create( - ctx.leader_dc().guard_port(), + common::run(common::TestOpts::new(2).with_timeout(45), |ctx| async move { + const DC2_RUNNER_NAME: &'static str = "dc-2-runner"; + + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + let _envoy2 = common::test_envoy::TestEnvoyBuilder::new(&namespace) + .with_version(1) + .with_pool_name(DC2_RUNNER_NAME) + .with_actor_behavior("test-actor", |_config| { + Box::new(common::test_envoy::EchoActor::new()) + }) + .build(ctx.get_dc(2)) + .await + .expect("failed to build test envoy"); + + common::upsert_normal_runner_config(ctx.get_dc(2), &namespace, DC2_RUNNER_NAME).await; + _envoy2.start().await.expect("failed to start envoy"); + _envoy2.wait_ready().await; + + let actor_name = "cross-dc-race-actor"; + let actor_key = "cross-dc-race-key"; + let port1 = ctx.leader_dc().guard_port(); + let port2 = ctx.get_dc(2).guard_port(); + let namespace_clone1 = namespace.clone(); + let namespace_clone2 = namespace.clone(); + + // Launch concurrent requests from two different datacenters + let handle1 = tokio::spawn(async move { + common::api::public::actors_get_or_create( + port1, common::api::public::GetOrCreateQuery { - namespace: namespace.clone(), + namespace: namespace_clone1, }, common::api::public::GetOrCreateRequest { - datacenter: None, // Should default to current DC - name: "current-dc-actor".to_string(), - key: "current-dc-key".to_string(), + datacenter: None, + name: actor_name.to_string(), + key: actor_key.to_string(), input: None, runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, + crash_policy: rivet_types::actors::CrashPolicy::Sleep, }, ) .await - .expect("failed to get or create actor"); - - assert!(response.created, "Actor should be created"); + }); - // Verify actor is in current DC (DC1) - let actor_id_str = response.actor.actor_id.to_string(); - common::assert_actor_in_dc(&actor_id_str, 1).await; - }, - ); -} - -// Broken legacy Pegboard Runner multi-DC coverage: remote get-or-create returns -// `core.internal_error` with `target_replicas must include the local replica`. -#[test] -#[ignore = "broken legacy Pegboard Runner test: target_replicas must include the local replica"] -fn get_or_create_in_remote_datacenter() { - common::run( - common::TestOpts::new(2).with_timeout(45), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - // Request from DC1 but specify DC2 - let response = common::api::public::actors_get_or_create( - ctx.leader_dc().guard_port(), + let handle2 = tokio::spawn(async move { + common::api::public::actors_get_or_create( + port2, common::api::public::GetOrCreateQuery { - namespace: namespace.clone(), + namespace: namespace_clone2, }, common::api::public::GetOrCreateRequest { - datacenter: Some("dc-2".to_string()), - name: "remote-dc-actor".to_string(), - key: "remote-dc-key".to_string(), + datacenter: None, + name: actor_name.to_string(), + key: actor_key.to_string(), input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, + runner_name_selector: DC2_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, }, ) .await - .expect("failed to get or create actor"); + }); + + let (result1, result2) = tokio::join!(handle1, handle2); + let response1 = result1 + .expect("DC1 task panicked") + .expect("DC1 request failed"); + let response2 = result2 + .expect("DC2 task panicked") + .expect("DC2 request failed"); + + // Both should succeed and return the same actor + assert_eq!( + response1.actor.actor_id, response2.actor.actor_id, + "Both datacenters should return the same actor" + ); + + // At least one should report creation + assert!( + (response1.created || response2.created) && !(response1.created && response2.created), + "At least one datacenter should report creation, but not both" + ); + }); +} - assert!(response.created, "Actor should be created"); +// MARK: Datacenter tests - // Wait for actor to propagate across datacenters - let actor_id_str = response.actor.actor_id.to_string(); +#[test] +fn get_or_create_in_current_datacenter() { + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + let response = common::api::public::actors_get_or_create( + ctx.leader_dc().guard_port(), + common::api::public::GetOrCreateQuery { + namespace: namespace.clone(), + }, + common::api::public::GetOrCreateRequest { + datacenter: None, // Should default to current DC + name: "current-dc-actor".to_string(), + key: "current-dc-key".to_string(), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to get or create actor"); + + assert!(response.created, "Actor should be created"); + + // Verify actor is in current DC (DC1) + let actor_id_str = response.actor.actor_id.to_string(); + common::assert_actor_in_dc(&actor_id_str, 1).await; + }); +} - // Verify actor is in DC2 - common::assert_actor_in_dc(&actor_id_str, 2).await; - }, - ); +// Broken legacy Pegboard Runner multi-DC coverage: remote get-or-create returns +// `core.internal_error` with `target_replicas must include the local replica`. +#[test] +#[ignore = "broken legacy Pegboard Runner test: target_replicas must include the local replica"] +fn get_or_create_in_remote_datacenter() { + common::run(common::TestOpts::new(2).with_timeout(45), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + // Request from DC1 but specify DC2 + let response = common::api::public::actors_get_or_create( + ctx.leader_dc().guard_port(), + common::api::public::GetOrCreateQuery { + namespace: namespace.clone(), + }, + common::api::public::GetOrCreateRequest { + datacenter: Some("dc-2".to_string()), + name: "remote-dc-actor".to_string(), + key: "remote-dc-key".to_string(), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to get or create actor"); + + assert!(response.created, "Actor should be created"); + + // Wait for actor to propagate across datacenters + let actor_id_str = response.actor.actor_id.to_string(); + + // Verify actor is in DC2 + common::assert_actor_in_dc(&actor_id_str, 2).await; + }); } // MARK: Error cases #[test] fn get_or_create_with_non_existent_namespace() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let res = common::api::public::actors_get_or_create( - ctx.leader_dc().guard_port(), - common::api::public::GetOrCreateQuery { - namespace: "non-existent-namespace".to_string(), - }, - common::api::public::GetOrCreateRequest { - datacenter: None, - name: "test-actor".to_string(), - key: "test-key".to_string(), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await; - - assert!(res.is_err(), "Should fail with non-existent namespace"); - }, - ); + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let res = common::api::public::actors_get_or_create( + ctx.leader_dc().guard_port(), + common::api::public::GetOrCreateQuery { + namespace: "non-existent-namespace".to_string(), + }, + common::api::public::GetOrCreateRequest { + datacenter: None, + name: "test-actor".to_string(), + key: "test-key".to_string(), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await; + + assert!(res.is_err(), "Should fail with non-existent namespace"); + }); } #[test] fn get_or_create_with_invalid_datacenter() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - let res = common::api::public::actors_get_or_create( - ctx.leader_dc().guard_port(), - common::api::public::GetOrCreateQuery { - namespace: namespace.clone(), - }, - common::api::public::GetOrCreateRequest { - datacenter: Some("non-existent-dc".to_string()), - name: "test-actor".to_string(), - key: "test-key".to_string(), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await; - - assert!(res.is_err(), "Should fail with invalid datacenter"); - }, - ); + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + let res = common::api::public::actors_get_or_create( + ctx.leader_dc().guard_port(), + common::api::public::GetOrCreateQuery { + namespace: namespace.clone(), + }, + common::api::public::GetOrCreateRequest { + datacenter: Some("non-existent-dc".to_string()), + name: "test-actor".to_string(), + key: "test-key".to_string(), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await; + + assert!(res.is_err(), "Should fail with invalid datacenter"); + }); } #[test] fn get_or_create_empty_key() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - let res = common::api::public::actors_get_or_create( - ctx.leader_dc().guard_port(), - common::api::public::GetOrCreateQuery { - namespace: namespace.clone(), - }, - common::api::public::GetOrCreateRequest { - datacenter: None, - name: "test-actor".to_string(), - key: "".to_string(), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await; - - assert!( - res.is_err(), - "should fail to get or create actor with empty key" - ); - }, - ); + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + let res = common::api::public::actors_get_or_create( + ctx.leader_dc().guard_port(), + common::api::public::GetOrCreateQuery { + namespace: namespace.clone(), + }, + common::api::public::GetOrCreateRequest { + datacenter: None, + name: "test-actor".to_string(), + key: "".to_string(), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await; + + assert!( + res.is_err(), + "should fail to get or create actor with empty key" + ); + }); } #[test] fn get_or_create_key_exceeds_max_size() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - let key = "a".repeat(1025); - - let res = common::api::public::actors_get_or_create( - ctx.leader_dc().guard_port(), - common::api::public::GetOrCreateQuery { - namespace: namespace.clone(), - }, - common::api::public::GetOrCreateRequest { - datacenter: None, - name: "test-actor".to_string(), - key, - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await; - - assert!( - res.is_err(), - "should fail to get or create actor with key exceeding max size" - ); - }, - ); + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + let key = "a".repeat(1025); + + let res = common::api::public::actors_get_or_create( + ctx.leader_dc().guard_port(), + common::api::public::GetOrCreateQuery { + namespace: namespace.clone(), + }, + common::api::public::GetOrCreateRequest { + datacenter: None, + name: "test-actor".to_string(), + key, + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await; + + assert!( + res.is_err(), + "should fail to get or create actor with key exceeding max size" + ); + }); } // MARK: Edge cases #[test] fn get_or_create_with_destroyed_actor() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - let actor_name = "destroyed-actor"; - let actor_key = "destroyed-key"; - - // Create actor - let response1 = common::api::public::actors_get_or_create( - ctx.leader_dc().guard_port(), - common::api::public::GetOrCreateQuery { - namespace: namespace.clone(), - }, - common::api::public::GetOrCreateRequest { - datacenter: None, - name: actor_name.to_string(), - key: actor_key.to_string(), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to get or create actor"); - - assert!(response1.created, "First call should create actor"); - let first_actor_id = response1.actor.actor_id; - - // Destroy the actor - common::api::public::actors_delete( - ctx.leader_dc().guard_port(), - common::api_types::actors::delete::DeletePath { - actor_id: first_actor_id, - }, - common::api_types::actors::delete::DeleteQuery { - namespace: namespace.clone(), - }, - ) - .await - .expect("failed to delete actor"); - - // Call get_or_create again with same key - should create a new actor - let response2 = common::api::public::actors_get_or_create( - ctx.leader_dc().guard_port(), - common::api::public::GetOrCreateQuery { - namespace: namespace.clone(), - }, - common::api::public::GetOrCreateRequest { - datacenter: None, - name: actor_name.to_string(), - key: actor_key.to_string(), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to get or create actor after destroy"); - - assert!( - response2.created, - "Should create new actor after old one was destroyed" - ); - assert_ne!( - response2.actor.actor_id, first_actor_id, - "Should be a different actor ID" - ); - }, - ); + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + let actor_name = "destroyed-actor"; + let actor_key = "destroyed-key"; + + // Create actor + let response1 = common::api::public::actors_get_or_create( + ctx.leader_dc().guard_port(), + common::api::public::GetOrCreateQuery { + namespace: namespace.clone(), + }, + common::api::public::GetOrCreateRequest { + datacenter: None, + name: actor_name.to_string(), + key: actor_key.to_string(), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to get or create actor"); + + assert!(response1.created, "First call should create actor"); + let first_actor_id = response1.actor.actor_id; + + // Destroy the actor + common::api::public::actors_delete( + ctx.leader_dc().guard_port(), + common::api_types::actors::delete::DeletePath { + actor_id: first_actor_id, + }, + common::api_types::actors::delete::DeleteQuery { + namespace: namespace.clone(), + }, + ) + .await + .expect("failed to delete actor"); + + // Call get_or_create again with same key - should create a new actor + let response2 = common::api::public::actors_get_or_create( + ctx.leader_dc().guard_port(), + common::api::public::GetOrCreateQuery { + namespace: namespace.clone(), + }, + common::api::public::GetOrCreateRequest { + datacenter: None, + name: actor_name.to_string(), + key: actor_key.to_string(), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to get or create actor after destroy"); + + assert!( + response2.created, + "Should create new actor after old one was destroyed" + ); + assert_ne!( + response2.actor.actor_id, first_actor_id, + "Should be a different actor ID" + ); + }); } diff --git a/engine/packages/engine/tests/envoy/api_actors_list.rs b/engine/packages/engine/tests/envoy/api_actors_list.rs index fb7ab81baf..f73d2e3d0c 100644 --- a/engine/packages/engine/tests/envoy/api_actors_list.rs +++ b/engine/packages/engine/tests/envoy/api_actors_list.rs @@ -6,268 +6,743 @@ use std::collections::HashSet; #[test] fn list_actors_by_namespace_and_name() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - let name = "list-test-actor"; - - // Create multiple actors with same name - let mut actor_ids = Vec::new(); - for i in 0..3 { - let res = common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: name.to_string(), - key: Some(format!("key-{}", i)), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor"); - actor_ids.push(res.actor.actor_id.to_string()); - } - - // List actors by name - let response = common::api::public::actors_list( + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + let name = "list-test-actor"; + + // Create multiple actors with same name + let mut actor_ids = Vec::new(); + for i in 0..3 { + let res = common::api::public::actors_create( ctx.leader_dc().guard_port(), - common::api_types::actors::list::ListQuery { + common::api_types::actors::create::CreateQuery { namespace: namespace.clone(), - name: Some(name.to_string()), - key: None, - actor_ids: None, - actor_id: vec![], - include_destroyed: None, - limit: None, - cursor: None, + }, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: name.to_string(), + key: Some(format!("key-{}", i)), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, }, ) .await - .expect("failed to list actors"); - - assert_eq!(response.actors.len(), 3, "Should return all 3 actors"); - - // Verify all created actors are in the response - let returned_ids: HashSet = response - .actors - .iter() - .map(|a| a.actor_id.to_string()) - .collect(); - for actor_id in &actor_ids { - assert!( - returned_ids.contains(actor_id), - "Actor {} should be in results", - actor_id - ); - } - }, - ); + .expect("failed to create actor"); + actor_ids.push(res.actor.actor_id.to_string()); + } + + // List actors by name + let response = common::api::public::actors_list( + ctx.leader_dc().guard_port(), + common::api_types::actors::list::ListQuery { + namespace: namespace.clone(), + name: Some(name.to_string()), + key: None, + actor_ids: None, + actor_id: vec![], + include_destroyed: None, + limit: None, + cursor: None, + }, + ) + .await + .expect("failed to list actors"); + + assert_eq!(response.actors.len(), 3, "Should return all 3 actors"); + + // Verify all created actors are in the response + let returned_ids: HashSet = response + .actors + .iter() + .map(|a| a.actor_id.to_string()) + .collect(); + for actor_id in &actor_ids { + assert!( + returned_ids.contains(actor_id), + "Actor {} should be in results", + actor_id + ); + } + }); } #[test] fn list_with_pagination() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - let name = "paginated-actor"; - - // Create 5 actors with the same name but different keys - let mut actor_ids = Vec::new(); - for i in 0..5 { - let res = common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: name.to_string(), - key: Some(format!("key-{}", i)), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor"); - actor_ids.push(res.actor.actor_id.to_string()); - } - - // First page - limit 2 - let response1 = common::api::public::actors_list( - ctx.leader_dc().guard_port(), - common::api_types::actors::list::ListQuery { - namespace: namespace.clone(), - name: Some(name.to_string()), - key: None, - actor_ids: None, - actor_id: vec![], - include_destroyed: None, - limit: Some(2), - cursor: None, - }, - ) - .await - .expect("failed to list actors"); + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - assert_eq!( - response1.actors.len(), - 2, - "Should return 2 actors with limit=2" - ); + let name = "paginated-actor"; - // Get all actors to verify ordering - let all_response = common::api::public::actors_list( + // Create 5 actors with the same name but different keys + let mut actor_ids = Vec::new(); + for i in 0..5 { + let res = common::api::public::actors_create( ctx.leader_dc().guard_port(), - common::api_types::actors::list::ListQuery { + common::api_types::actors::create::CreateQuery { namespace: namespace.clone(), - name: Some(name.to_string()), - key: None, - actor_ids: None, - actor_id: vec![], - include_destroyed: None, - limit: None, - cursor: None, + }, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: name.to_string(), + key: Some(format!("key-{}", i)), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, }, ) .await - .expect("failed to list all actors"); - - // Verify we have all 5 actors when querying without limit - assert_eq!( - all_response.actors.len(), - 5, - "Should return all 5 actors when no limit specified" + .expect("failed to create actor"); + actor_ids.push(res.actor.actor_id.to_string()); + } + + // First page - limit 2 + let response1 = common::api::public::actors_list( + ctx.leader_dc().guard_port(), + common::api_types::actors::list::ListQuery { + namespace: namespace.clone(), + name: Some(name.to_string()), + key: None, + actor_ids: None, + actor_id: vec![], + include_destroyed: None, + limit: Some(2), + cursor: None, + }, + ) + .await + .expect("failed to list actors"); + + assert_eq!( + response1.actors.len(), + 2, + "Should return 2 actors with limit=2" + ); + + // Get all actors to verify ordering + let all_response = common::api::public::actors_list( + ctx.leader_dc().guard_port(), + common::api_types::actors::list::ListQuery { + namespace: namespace.clone(), + name: Some(name.to_string()), + key: None, + actor_ids: None, + actor_id: vec![], + include_destroyed: None, + limit: None, + cursor: None, + }, + ) + .await + .expect("failed to list all actors"); + + // Verify we have all 5 actors when querying without limit + assert_eq!( + all_response.actors.len(), + 5, + "Should return all 5 actors when no limit specified" + ); + + // Use actors from position 2-4 as actors2 for remaining test logic + let actors2 = if all_response.actors.len() > 2 { + &all_response.actors[2..std::cmp::min(4, all_response.actors.len())] + } else { + &[] + }; + + // Verify no duplicates between pages + let ids1: HashSet = response1 + .actors + .iter() + .map(|a| a.actor_id.to_string()) + .collect(); + let ids2: HashSet = actors2.iter().map(|a| a.actor_id.to_string()).collect(); + assert!( + ids1.is_disjoint(&ids2), + "Pages should not have duplicate actors" + ); + + // Verify consistent ordering using the full actor list + let all_timestamps: Vec = all_response.actors.iter().map(|a| a.create_ts).collect(); + + // Verify all timestamps are valid and reasonable (not zero, not in future) + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as i64; + + for &ts in &all_timestamps { + assert!(ts > 0, "create_ts should be positive: {}", ts); + assert!(ts <= now, "create_ts should not be in future: {}", ts); + } + + // Verify that all actors are returned in descending timestamp order (newest first) + for i in 1..all_timestamps.len() { + assert!( + all_timestamps[i - 1] >= all_timestamps[i], + "Actors should be ordered by create_ts descending: {} >= {} (index {} vs {})", + all_timestamps[i - 1], + all_timestamps[i], + i - 1, + i ); + } + + // Verify that the limited query returns the newest actors + let paginated_timestamps: Vec = response1.actors.iter().map(|a| a.create_ts).collect(); + + assert_eq!( + paginated_timestamps, + all_timestamps[0..2].to_vec(), + "Paginated result should return the 2 newest actors" + ); + + // Test that limit=2 actually limits results to 2 + assert_eq!( + response1.actors.len(), + 2, + "Limit=2 should return exactly 2 actors" + ); + assert_eq!( + all_response.actors.len(), + 5, + "Query without limit should return all 5 actors" + ); + }); +} - // Use actors from position 2-4 as actors2 for remaining test logic - let actors2 = if all_response.actors.len() > 2 { - &all_response.actors[2..std::cmp::min(4, all_response.actors.len())] - } else { - &[] - }; +#[test] +fn list_returns_empty_array_when_no_actors() { + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + // List actors that don't exist + let response = common::api::public::actors_list( + ctx.leader_dc().guard_port(), + common::api_types::actors::list::ListQuery { + namespace: namespace.clone(), + name: Some("non-existent-actor".to_string()), + key: None, + actor_ids: None, + actor_id: vec![], + include_destroyed: None, + limit: None, + cursor: None, + }, + ) + .await + .expect("failed to list actors"); + + assert_eq!(response.actors.len(), 0, "Should return empty array"); + }); +} - // Verify no duplicates between pages - let ids1: HashSet = response1 - .actors - .iter() - .map(|a| a.actor_id.to_string()) - .collect(); - let ids2: HashSet = actors2.iter().map(|a| a.actor_id.to_string()).collect(); +// MARK: List by Name + Key + +#[test] +fn list_actors_by_namespace_name_and_key() { + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + let name = "keyed-actor"; + let key1 = "key1".to_string(); + let key2 = "key2".to_string(); + + // Create actors with different keys + let res1 = common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: name.to_string(), + key: Some(key1.clone()), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to create actor1"); + let actor_id1 = res1.actor.actor_id.to_string(); + + let _res2 = common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: name.to_string(), + key: Some(key2.clone()), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to create actor2"); + + // List with key1 - should find actor1 + let response = common::api::public::actors_list( + ctx.leader_dc().guard_port(), + common::api_types::actors::list::ListQuery { + namespace: namespace.clone(), + name: Some(name.to_string()), + key: Some("key1".to_string()), + actor_ids: None, + actor_id: vec![], + include_destroyed: None, + limit: None, + cursor: None, + }, + ) + .await + .expect("failed to list actors"); + + assert_eq!(response.actors.len(), 1, "Should return 1 actor"); + assert_eq!(response.actors[0].actor_id.to_string(), actor_id1); + }); +} + +#[test] +fn list_with_include_destroyed_false() { + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + let name = "destroyed-test"; + + // Create and destroy an actor + let res1 = common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: name.to_string(), + key: Some("destroyed-key".to_string()), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to create actor"); + let destroyed_actor_id = res1.actor.actor_id; + + common::api::public::actors_delete( + ctx.leader_dc().guard_port(), + common::api_types::actors::delete::DeletePath { + actor_id: destroyed_actor_id, + }, + common::api_types::actors::delete::DeleteQuery { + namespace: namespace.clone(), + }, + ) + .await + .expect("failed to delete actor"); + + // Create an active actor + let res2 = common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: name.to_string(), + key: Some("active-key".to_string()), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to create actor"); + let active_actor_id = res2.actor.actor_id.to_string(); + + // List without include_destroyed (default false) + let response = common::api::public::actors_list( + ctx.leader_dc().guard_port(), + common::api_types::actors::list::ListQuery { + namespace: namespace.clone(), + name: Some(name.to_string()), + key: None, + actor_ids: None, + actor_id: vec![], + include_destroyed: Some(false), + limit: None, + cursor: None, + }, + ) + .await + .expect("failed to list actors"); + + assert_eq!(response.actors.len(), 1, "Should only return active actor"); + assert_eq!(response.actors[0].actor_id.to_string(), active_actor_id); + }); +} + +#[test] +fn list_with_include_destroyed_true() { + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + let name = "destroyed-included"; + + // Create and destroy an actor + let res1 = common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: name.to_string(), + key: Some("destroyed-key".to_string()), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to create actor"); + let destroyed_actor_id = res1.actor.actor_id.to_string(); + + common::api::public::actors_delete( + ctx.leader_dc().guard_port(), + common::api_types::actors::delete::DeletePath { + actor_id: res1.actor.actor_id, + }, + common::api_types::actors::delete::DeleteQuery { + namespace: namespace.clone(), + }, + ) + .await + .expect("failed to delete actor"); + + // Create an active actor + let res2 = common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: name.to_string(), + key: Some("active-key".to_string()), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to create actor"); + let active_actor_id = res2.actor.actor_id.to_string(); + + // List with include_destroyed=true + let response = common::api::public::actors_list( + ctx.leader_dc().guard_port(), + common::api_types::actors::list::ListQuery { + namespace: namespace.clone(), + name: Some(name.to_string()), + key: None, + actor_ids: None, + actor_id: vec![], + include_destroyed: Some(true), + limit: None, + cursor: None, + }, + ) + .await + .expect("failed to list actors"); + + assert_eq!( + response.actors.len(), + 2, + "Should return both active and destroyed actors" + ); + + // Verify both actors are in results + let returned_ids: HashSet = response + .actors + .iter() + .map(|a| a.actor_id.to_string()) + .collect(); + assert!(returned_ids.contains(&active_actor_id)); + assert!(returned_ids.contains(&destroyed_actor_id)); + }); +} + +// MARK: List by Actor IDs + +// Broken legacy Pegboard Runner coverage: full `runner::` sweep times out with +// `test timed out: Elapsed(())`. +#[test] +fn list_specific_actors_by_ids() { + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + // Create multiple actors + let actor_ids = + common::bulk_create_actors(ctx.leader_dc().guard_port(), &namespace, "id-list-test", 5) + .await; + + // Select specific actors to list + let selected_ids = vec![ + actor_ids[0].clone(), + actor_ids[2].clone(), + actor_ids[4].clone(), + ]; + + // List by actor IDs (comma-separated string) + let response = common::api::public::actors_list( + ctx.leader_dc().guard_port(), + common::api_types::actors::list::ListQuery { + namespace: namespace.clone(), + name: None, + key: None, + actor_id: selected_ids.clone(), + actor_ids: None, + include_destroyed: None, + limit: None, + cursor: None, + }, + ) + .await + .expect("failed to list actors"); + + assert_eq!( + response.actors.len(), + 3, + "Should return exactly the requested actors" + ); + + // Verify correct actors returned + let returned_ids: HashSet = response + .actors + .iter() + .map(|a| a.actor_id.to_string()) + .collect(); + for id in &selected_ids { assert!( - ids1.is_disjoint(&ids2), - "Pages should not have duplicate actors" + returned_ids.contains(&id.to_string()), + "Actor {} should be in results", + id ); + } + }); +} - // Verify consistent ordering using the full actor list - let all_timestamps: Vec = - all_response.actors.iter().map(|a| a.create_ts).collect(); - - // Verify all timestamps are valid and reasonable (not zero, not in future) - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_millis() as i64; - - for &ts in &all_timestamps { - assert!(ts > 0, "create_ts should be positive: {}", ts); - assert!(ts <= now, "create_ts should not be in future: {}", ts); - } - - // Verify that all actors are returned in descending timestamp order (newest first) - for i in 1..all_timestamps.len() { - assert!( - all_timestamps[i - 1] >= all_timestamps[i], - "Actors should be ordered by create_ts descending: {} >= {} (index {} vs {})", - all_timestamps[i - 1], - all_timestamps[i], - i - 1, - i - ); - } - - // Verify that the limited query returns the newest actors - let paginated_timestamps: Vec = - response1.actors.iter().map(|a| a.create_ts).collect(); +#[test] +// Broken legacy Pegboard Runner test: full engine sweep can fail creating the +// DC2 actor with `actor.destroyed_during_creation`. +#[ignore = "DC2 actor create hangs / workflow-worker lease failure"] +fn list_actors_from_multiple_datacenters() { + common::run(common::TestOpts::new(2).with_timeout(45), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + // Create actors in different DCs + let res1 = common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: "multi-dc-actor".to_string(), + key: Some("dc1-key".to_string()), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to create actor in DC1"); + let actor_id_dc1 = res1.actor.actor_id; + + let res2 = common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: Some("dc-2".to_string()), + name: "multi-dc-actor".to_string(), + key: Some("dc2-key".to_string()), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to create actor in DC2"); + let actor_id_dc2 = res2.actor.actor_id; + + // List by actor IDs - should fetch from both DCs + let response = common::api::public::actors_list( + ctx.leader_dc().guard_port(), + common::api_types::actors::list::ListQuery { + namespace: namespace.clone(), + name: None, + key: None, + actor_id: vec![actor_id_dc1, actor_id_dc2], + actor_ids: None, + include_destroyed: None, + limit: None, + cursor: None, + }, + ) + .await + .expect("failed to list actors"); + + assert_eq!( + response.actors.len(), + 2, + "Should return actors from both DCs" + ); + }); +} - assert_eq!( - paginated_timestamps, - all_timestamps[0..2].to_vec(), - "Paginated result should return the 2 newest actors" - ); +// MARK: Error cases - // Test that limit=2 actually limits results to 2 - assert_eq!( - response1.actors.len(), - 2, - "Limit=2 should return exactly 2 actors" - ); - assert_eq!( - all_response.actors.len(), - 5, - "Query without limit should return all 5 actors" - ); - }, - ); +#[test] +fn list_with_non_existent_namespace() { + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + // Try to list with non-existent namespace + let res = common::api::public::actors_list( + ctx.leader_dc().guard_port(), + common::api_types::actors::list::ListQuery { + namespace: "non-existent-namespace".to_string(), + name: Some("test-actor".to_string()), + key: None, + actor_ids: None, + actor_id: vec![], + include_destroyed: None, + limit: None, + cursor: None, + }, + ) + .await; + + // Should fail with namespace not found + assert!(res.is_err(), "Should fail with non-existent namespace"); + }); } #[test] -fn list_returns_empty_array_when_no_actors() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - // List actors that don't exist - let response = common::api::public::actors_list( - ctx.leader_dc().guard_port(), - common::api_types::actors::list::ListQuery { - namespace: namespace.clone(), - name: Some("non-existent-actor".to_string()), - key: None, - actor_ids: None, - actor_id: vec![], - include_destroyed: None, - limit: None, - cursor: None, - }, - ) - .await - .expect("failed to list actors"); +fn list_with_key_but_no_name() { + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + // Try to list with key but no name (validation error) + let res = common::api::public::actors_list( + ctx.leader_dc().guard_port(), + common::api_types::actors::list::ListQuery { + namespace: namespace.clone(), + name: None, + key: Some("key1".to_string()), + actor_ids: None, + actor_id: vec![], + include_destroyed: None, + limit: None, + cursor: None, + }, + ) + .await; + + // Should fail with validation error + assert!(res.is_err(), "Should return error for key without name"); + }); +} - assert_eq!(response.actors.len(), 0, "Should return empty array"); - }, - ); +#[test] +fn list_with_more_than_32_actor_ids() { + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + // Try to list with more than 32 actor IDs + let actor_ids: Vec = (0..33) + .map(|_| rivet_util::Id::new_v1(ctx.leader_dc().config.dc_label())) + .collect(); + + let res = common::api::public::actors_list( + ctx.leader_dc().guard_port(), + common::api_types::actors::list::ListQuery { + namespace: namespace.clone(), + name: None, + key: None, + actor_id: actor_ids, + actor_ids: None, + include_destroyed: None, + limit: None, + cursor: None, + }, + ) + .await; + + // Should fail with validation error + assert!(res.is_err(), "Should return error for too many actor IDs"); + }); } -// MARK: List by Name + Key +#[test] +fn list_without_name_when_not_using_actor_ids() { + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + // Try to list without name or actor_ids + let res = common::api::public::actors_list( + ctx.leader_dc().guard_port(), + common::api_types::actors::list::ListQuery { + namespace: namespace.clone(), + name: None, + key: None, + actor_ids: None, + actor_id: vec![], + include_destroyed: None, + limit: None, + cursor: None, + }, + ) + .await; + + // Should fail with validation error + assert!( + res.is_err(), + "Should return error when neither name nor actor_ids provided" + ); + }); +} + +// MARK: Pagination and Sorting #[test] -fn list_actors_by_namespace_name_and_key() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - let name = "keyed-actor"; - let key1 = "key1".to_string(); - let key2 = "key2".to_string(); - - // Create actors with different keys - let res1 = common::api::public::actors_create( +fn verify_sorting_by_create_ts_descending() { + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + let name = "sorted-actor"; + + // Create actors with slight delays to ensure different timestamps + let mut actor_ids = Vec::new(); + for i in 0..3 { + let res = common::api::public::actors_create( ctx.leader_dc().guard_port(), common::api_types::actors::create::CreateQuery { namespace: namespace.clone(), @@ -275,17 +750,384 @@ fn list_actors_by_namespace_name_and_key() { common::api_types::actors::create::CreateRequest { datacenter: None, name: name.to_string(), - key: Some(key1.clone()), + key: Some(format!("key-{}", i)), input: None, runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, + crash_policy: rivet_types::actors::CrashPolicy::Sleep, }, ) .await - .expect("failed to create actor1"); - let actor_id1 = res1.actor.actor_id.to_string(); + .expect("failed to create actor"); + actor_ids.push(res.actor.actor_id.to_string()); + } + + // List actors + let response = common::api::public::actors_list( + ctx.leader_dc().guard_port(), + common::api_types::actors::list::ListQuery { + namespace: namespace.clone(), + name: Some(name.to_string()), + key: None, + actor_ids: None, + actor_id: vec![], + include_destroyed: None, + limit: None, + cursor: None, + }, + ) + .await + .expect("failed to list actors"); + + // Verify order - newest first (descending by create_ts) + for i in 0..response.actors.len() { + assert_eq!( + response.actors[i].actor_id.to_string(), + actor_ids[actor_ids.len() - 1 - i], + "Actors should be sorted by create_ts descending" + ); + } + }); +} + +// MARK: Cross-datacenter + +// Broken legacy Pegboard Runner multi-DC coverage: full `runner::` sweep times +// out with `test timed out: Elapsed(())`. +#[test] +#[ignore = "DC2 actor create hangs / workflow-worker lease failure"] +fn list_aggregates_results_from_all_datacenters() { + common::run(common::TestOpts::new(2).with_timeout(45), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + let name = "fanout-test-actor"; + + // Create actors in both DCs + let res1 = common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: name.to_string(), + key: Some("dc1-key".to_string()), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to create actor in DC1"); + let actor_id_dc1 = res1.actor.actor_id.to_string(); + + let res2 = common::api::public::actors_create( + ctx.get_dc(2).guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: Some("dc-2".to_string()), + name: name.to_string(), + key: Some("dc2-key".to_string()), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to create actor in DC2"); + let actor_id_dc2 = res2.actor.actor_id.to_string(); + + // List by name - should fanout to all DCs + let response = common::api::public::actors_list( + ctx.leader_dc().guard_port(), + common::api_types::actors::list::ListQuery { + namespace: namespace.clone(), + name: Some(name.to_string()), + key: None, + actor_ids: None, + actor_id: vec![], + include_destroyed: None, + limit: None, + cursor: None, + }, + ) + .await + .expect("failed to list actors"); + + assert_eq!( + response.actors.len(), + 2, + "Should return actors from both DCs" + ); + + // Verify both actors are present + let returned_ids: HashSet = response + .actors + .iter() + .map(|a| a.actor_id.to_string()) + .collect(); + assert!(returned_ids.contains(&actor_id_dc1)); + assert!(returned_ids.contains(&actor_id_dc2)); + }); +} + +// MARK: Edge cases + +#[test] +fn list_with_exactly_32_actor_ids() { + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + // Create exactly 32 actor IDs (boundary condition) + let actor_ids: Vec = (0..32) + .map(|_| rivet_util::Id::new_v1(ctx.leader_dc().config.dc_label())) + .collect(); + + // Should succeed with exactly 32 IDs + let response = common::api::public::actors_list( + ctx.leader_dc().guard_port(), + common::api_types::actors::list::ListQuery { + namespace: namespace.clone(), + name: None, + key: None, + actor_id: actor_ids, + actor_ids: None, + include_destroyed: None, + limit: None, + cursor: None, + }, + ) + .await + .expect("should succeed with exactly 32 actor IDs"); + + // Since these are fake IDs, we expect 0 results, but no error + assert_eq!( + response.actors.len(), + 0, + "Fake IDs should return empty results" + ); + }); +} + +#[test] +fn list_by_key_with_include_destroyed_true() { + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + let name = "key-destroyed-test"; + let key = "test-key"; + + // Create and destroy an actor with a key + let res1 = common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: name.to_string(), + key: Some(key.to_string()), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to create actor"); + let destroyed_actor_id = res1.actor.actor_id.to_string(); + + common::api::public::actors_delete( + ctx.leader_dc().guard_port(), + common::api_types::actors::delete::DeletePath { + actor_id: res1.actor.actor_id, + }, + common::api_types::actors::delete::DeleteQuery { + namespace: namespace.clone(), + }, + ) + .await + .expect("failed to delete actor"); + + // Create a new actor with the same key + let res2 = common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: name.to_string(), + key: Some(key.to_string()), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to create actor"); + let active_actor_id = res2.actor.actor_id.to_string(); + + // List by key with include_destroyed=true + // This should use the fanout path, not the optimized key path + let response = common::api::public::actors_list( + ctx.leader_dc().guard_port(), + common::api_types::actors::list::ListQuery { + namespace: namespace.clone(), + name: Some(name.to_string()), + key: Some(key.to_string()), + actor_ids: None, + actor_id: vec![], + include_destroyed: Some(true), + limit: None, + cursor: None, + }, + ) + .await + .expect("failed to list actors"); + + // Should return both actors (destroyed and active) + assert_eq!( + response.actors.len(), + 2, + "Should return both destroyed and active actors with same key" + ); + + let returned_ids: HashSet = response + .actors + .iter() + .map(|a| a.actor_id.to_string()) + .collect(); + assert!(returned_ids.contains(&destroyed_actor_id)); + assert!(returned_ids.contains(&active_actor_id)); + }); +} + +#[test] +fn list_default_limit_100() { + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + let name = "limit-test"; + + // Create 105 actors to test the default limit of 100 + let actor_ids = + common::bulk_create_actors(ctx.leader_dc().guard_port(), &namespace, name, 105).await; + + assert_eq!(actor_ids.len(), 105, "Should have created 105 actors"); + + // List without specifying limit - should use default limit of 100 + let response = common::api::public::actors_list( + ctx.leader_dc().guard_port(), + common::api_types::actors::list::ListQuery { + namespace: namespace.clone(), + name: Some(name.to_string()), + key: None, + actor_ids: None, + actor_id: vec![], + include_destroyed: None, + limit: None, // No limit specified - should default to 100 + cursor: None, + }, + ) + .await + .expect("failed to list actors"); + + // Should return exactly 100 actors due to default limit + assert_eq!( + response.actors.len(), + 100, + "Should return exactly 100 actors when default limit is applied" + ); + + // Verify cursor exists since there are more results + assert!( + response.pagination.cursor.is_some(), + "Cursor should exist when there are more results beyond the limit" + ); + }); +} + +// Broken legacy Pegboard Runner coverage: full `runner::` sweep times out with +// `test timed out: Elapsed(())`. +#[test] +#[ignore = "API rejects mixed-validity actor_ids instead of filtering"] +fn list_with_invalid_actor_id_format_in_comma_list() { + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + // Create a valid actor + let res = common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: "test-actor".to_string(), + key: Some("test-key".to_string()), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to create actor"); + let valid_actor_id = res.actor.actor_id.to_string(); + + // Mix valid and invalid IDs in the comma-separated list + let mixed_ids = vec![ + valid_actor_id.clone(), + "invalid-uuid".to_string(), + "not-a-uuid".to_string(), + valid_actor_id.clone(), + ]; + + let response = common::api::public::actors_list( + ctx.leader_dc().guard_port(), + common::api_types::actors::list::ListQuery { + namespace: namespace.clone(), + name: None, + key: None, + actor_id: vec![], + actor_ids: Some(mixed_ids.join(",")), + include_destroyed: None, + limit: None, + cursor: None, + }, + ) + .await + .expect("should filter out invalid IDs gracefully"); + + // Should return only the valid actor (twice) (parsed IDs are filtered) + assert_eq!( + response.actors.len(), + 2, + "Should filter out invalid IDs and return only valid ones" + ); + assert_eq!(response.actors[0].actor_id.to_string(), valid_actor_id); + }); +} + +// MARK: Cursor pagination + +#[test] +fn list_with_cursor_pagination() { + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - let _res2 = common::api::public::actors_create( + let name = "cursor-test-actor"; + + // Create 5 actors with same name + let mut actor_ids = Vec::new(); + for i in 0..5 { + let res = common::api::public::actors_create( ctx.leader_dc().guard_port(), common::api_types::actors::create::CreateQuery { namespace: namespace.clone(), @@ -293,50 +1135,229 @@ fn list_actors_by_namespace_name_and_key() { common::api_types::actors::create::CreateRequest { datacenter: None, name: name.to_string(), - key: Some(key2.clone()), + key: Some(format!("cursor-key-{}", i)), input: None, runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, + crash_policy: rivet_types::actors::CrashPolicy::Sleep, }, ) .await - .expect("failed to create actor2"); + .expect("failed to create actor"); + actor_ids.push(res.actor.actor_id.to_string()); + } + + // Fetch first page with limit=2 + let page1 = common::api::public::actors_list( + ctx.leader_dc().guard_port(), + common::api_types::actors::list::ListQuery { + namespace: namespace.clone(), + name: Some(name.to_string()), + key: None, + actor_ids: None, + actor_id: vec![], + include_destroyed: None, + limit: Some(2), + cursor: None, + }, + ) + .await + .expect("failed to list page 1"); + + assert_eq!(page1.actors.len(), 2, "Page 1 should have 2 actors"); + assert!( + page1.pagination.cursor.is_some(), + "Page 1 should return a cursor" + ); + + // Fetch second page using cursor + let page2 = common::api::public::actors_list( + ctx.leader_dc().guard_port(), + common::api_types::actors::list::ListQuery { + namespace: namespace.clone(), + name: Some(name.to_string()), + key: None, + actor_ids: None, + actor_id: vec![], + include_destroyed: None, + limit: Some(2), + cursor: page1.pagination.cursor.clone(), + }, + ) + .await + .expect("failed to list page 2"); + + assert_eq!(page2.actors.len(), 2, "Page 2 should have 2 actors"); + assert!( + page2.pagination.cursor.is_some(), + "Page 2 should return a cursor" + ); + + // Fetch third page using cursor + let page3 = common::api::public::actors_list( + ctx.leader_dc().guard_port(), + common::api_types::actors::list::ListQuery { + namespace: namespace.clone(), + name: Some(name.to_string()), + key: None, + actor_ids: None, + actor_id: vec![], + include_destroyed: None, + limit: Some(2), + cursor: page2.pagination.cursor.clone(), + }, + ) + .await + .expect("failed to list page 3"); + + assert_eq!(page3.actors.len(), 1, "Page 3 should have 1 actor"); + + // Verify no duplicates across pages + let ids1: HashSet = page1 + .actors + .iter() + .map(|a| a.actor_id.to_string()) + .collect(); + let ids2: HashSet = page2 + .actors + .iter() + .map(|a| a.actor_id.to_string()) + .collect(); + let ids3: HashSet = page3 + .actors + .iter() + .map(|a| a.actor_id.to_string()) + .collect(); + + assert!( + ids1.is_disjoint(&ids2), + "Page 1 and 2 should have no duplicates" + ); + assert!( + ids1.is_disjoint(&ids3), + "Page 1 and 3 should have no duplicates" + ); + assert!( + ids2.is_disjoint(&ids3), + "Page 2 and 3 should have no duplicates" + ); + + // Verify all actors are returned across all pages + let mut all_returned_ids = ids1; + all_returned_ids.extend(ids2); + all_returned_ids.extend(ids3); + + assert_eq!( + all_returned_ids.len(), + 5, + "All 5 actors should be returned across pages" + ); + for actor_id in &actor_ids { + assert!( + all_returned_ids.contains(&actor_id.to_string()), + "Actor {} should be in results", + actor_id + ); + } + }); +} + +#[test] +fn list_cursor_filters_by_timestamp() { + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + let name = "timestamp-filter-test"; - // List with key1 - should find actor1 - let response = common::api::public::actors_list( + // Create 3 actors + for i in 0..3 { + common::api::public::actors_create( ctx.leader_dc().guard_port(), - common::api_types::actors::list::ListQuery { + common::api_types::actors::create::CreateQuery { namespace: namespace.clone(), - name: Some(name.to_string()), - key: Some("key1".to_string()), - actor_ids: None, - actor_id: vec![], - include_destroyed: None, - limit: None, - cursor: None, + }, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: name.to_string(), + key: Some(format!("ts-key-{}", i)), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, }, ) .await - .expect("failed to list actors"); - - assert_eq!(response.actors.len(), 1, "Should return 1 actor"); - assert_eq!(response.actors[0].actor_id.to_string(), actor_id1); - }, - ); + .expect("failed to create actor"); + } + + // Get all actors to find a middle timestamp + let all_actors = common::api::public::actors_list( + ctx.leader_dc().guard_port(), + common::api_types::actors::list::ListQuery { + namespace: namespace.clone(), + name: Some(name.to_string()), + key: None, + actor_ids: None, + actor_id: vec![], + include_destroyed: None, + limit: None, + cursor: None, + }, + ) + .await + .expect("failed to list all actors"); + + assert_eq!(all_actors.actors.len(), 3, "Should have 3 actors"); + + // Use the first actor's timestamp as cursor (should filter out that actor and newer) + let cursor = all_actors.actors[0].create_ts.to_string(); + + // List with cursor + let filtered = common::api::public::actors_list( + ctx.leader_dc().guard_port(), + common::api_types::actors::list::ListQuery { + namespace: namespace.clone(), + name: Some(name.to_string()), + key: None, + actor_ids: None, + actor_id: vec![], + include_destroyed: None, + limit: None, + cursor: Some(cursor.clone()), + }, + ) + .await + .expect("failed to list with cursor"); + + // Should return only actors older than the cursor timestamp + assert!( + filtered.actors.len() < 3, + "Cursor should filter out some actors" + ); + + // Verify all returned actors have timestamps less than cursor + let cursor_ts: i64 = cursor.parse().expect("cursor should be valid i64"); + for actor in &filtered.actors { + assert!( + actor.create_ts < cursor_ts, + "Actor timestamp {} should be less than cursor {}", + actor.create_ts, + cursor_ts + ); + } + }); } #[test] -fn list_with_include_destroyed_false() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; +fn list_cursor_with_exact_timestamp_boundary() { + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - let name = "destroyed-test"; + let name = "boundary-test"; - // Create and destroy an actor - let res1 = common::api::public::actors_create( + // Create 3 actors + for i in 0..3 { + common::api::public::actors_create( ctx.leader_dc().guard_port(), common::api_types::actors::create::CreateQuery { namespace: namespace.clone(), @@ -344,30 +1365,77 @@ fn list_with_include_destroyed_false() { common::api_types::actors::create::CreateRequest { datacenter: None, name: name.to_string(), - key: Some("destroyed-key".to_string()), + key: Some(format!("boundary-key-{}", i)), input: None, runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, + crash_policy: rivet_types::actors::CrashPolicy::Sleep, }, ) .await .expect("failed to create actor"); - let destroyed_actor_id = res1.actor.actor_id; + } + + // Get first page with limit=1 + let page1 = common::api::public::actors_list( + ctx.leader_dc().guard_port(), + common::api_types::actors::list::ListQuery { + namespace: namespace.clone(), + name: Some(name.to_string()), + key: None, + actor_ids: None, + actor_id: vec![], + include_destroyed: None, + limit: Some(1), + cursor: None, + }, + ) + .await + .expect("failed to list page 1"); + + assert_eq!(page1.actors.len(), 1, "Page 1 should have 1 actor"); + let first_actor_id = page1.actors[0].actor_id.to_string(); + + // Get second page using cursor + let page2 = common::api::public::actors_list( + ctx.leader_dc().guard_port(), + common::api_types::actors::list::ListQuery { + namespace: namespace.clone(), + name: Some(name.to_string()), + key: None, + actor_ids: None, + actor_id: vec![], + include_destroyed: None, + limit: None, + cursor: page1.pagination.cursor.clone(), + }, + ) + .await + .expect("failed to list page 2"); + + // Verify first actor is NOT in page 2 (exact boundary excluded) + let page2_ids: HashSet = page2 + .actors + .iter() + .map(|a| a.actor_id.to_string()) + .collect(); + assert!( + !page2_ids.contains(&first_actor_id), + "Actor with exact cursor timestamp should be excluded" + ); + }); +} - common::api::public::actors_delete( - ctx.leader_dc().guard_port(), - common::api_types::actors::delete::DeletePath { - actor_id: destroyed_actor_id, - }, - common::api_types::actors::delete::DeleteQuery { - namespace: namespace.clone(), - }, - ) - .await - .expect("failed to delete actor"); +#[test] +fn list_cursor_empty_results_when_no_more_actors() { + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + let name = "empty-cursor-test"; - // Create an active actor - let res2 = common::api::public::actors_create( + // Create 2 actors + for i in 0..2 { + common::api::public::actors_create( ctx.leader_dc().guard_port(), common::api_types::actors::create::CreateQuery { namespace: namespace.clone(), @@ -375,18 +1443,38 @@ fn list_with_include_destroyed_false() { common::api_types::actors::create::CreateRequest { datacenter: None, name: name.to_string(), - key: Some("active-key".to_string()), + key: Some(format!("empty-key-{}", i)), input: None, runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, + crash_policy: rivet_types::actors::CrashPolicy::Sleep, }, ) .await .expect("failed to create actor"); - let active_actor_id = res2.actor.actor_id.to_string(); - - // List without include_destroyed (default false) - let response = common::api::public::actors_list( + } + + // List all actors + let all_actors = common::api::public::actors_list( + ctx.leader_dc().guard_port(), + common::api_types::actors::list::ListQuery { + namespace: namespace.clone(), + name: Some(name.to_string()), + key: None, + actor_ids: None, + actor_id: vec![], + include_destroyed: None, + limit: Some(10), + cursor: None, + }, + ) + .await + .expect("failed to list all actors"); + + assert_eq!(all_actors.actors.len(), 2, "Should have 2 actors"); + + // Use cursor to fetch next page (should be empty) + if let Some(cursor) = all_actors.pagination.cursor { + let next_page = common::api::public::actors_list( ctx.leader_dc().guard_port(), common::api_types::actors::list::ListQuery { namespace: namespace.clone(), @@ -394,32 +1482,73 @@ fn list_with_include_destroyed_false() { key: None, actor_ids: None, actor_id: vec![], - include_destroyed: Some(false), - limit: None, - cursor: None, + include_destroyed: None, + limit: Some(10), + cursor: Some(cursor), }, ) .await - .expect("failed to list actors"); + .expect("failed to list next page"); + + assert_eq!( + next_page.actors.len(), + 0, + "Should return empty results when no more actors" + ); + assert!( + next_page.pagination.cursor.is_none(), + "Should not return cursor when no more results" + ); + } + }); +} - assert_eq!(response.actors.len(), 1, "Should only return active actor"); - assert_eq!(response.actors[0].actor_id.to_string(), active_actor_id); - }, - ); +#[test] +fn list_invalid_cursor_format() { + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + let name = "invalid-cursor-test"; + + // Try to list with invalid cursor (non-numeric string) + let res = common::api::public::actors_list( + ctx.leader_dc().guard_port(), + common::api_types::actors::list::ListQuery { + namespace: namespace.clone(), + name: Some(name.to_string()), + key: None, + actor_ids: None, + actor_id: vec![], + include_destroyed: None, + limit: None, + cursor: Some("not-a-number".to_string()), + }, + ) + .await; + + // Should fail with parse error + assert!( + res.is_err(), + "Should return error for invalid cursor format" + ); + }); } +// Broken legacy Pegboard Runner multi-DC coverage: full `runner::` sweep times +// out with `test timed out: Elapsed(())`. #[test] -fn list_with_include_destroyed_true() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; +#[ignore = "DC2 actor create hangs / workflow-worker lease failure"] +fn list_cursor_across_datacenters() { + common::run(common::TestOpts::new(2).with_timeout(45), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - let name = "destroyed-included"; + let name = "multi-dc-cursor-test"; - // Create and destroy an actor - let res1 = common::api::public::actors_create( + // Create actors in both DC1 and DC2 + for i in 0..3 { + common::api::public::actors_create( ctx.leader_dc().guard_port(), common::api_types::actors::create::CreateQuery { namespace: namespace.clone(), @@ -427,49 +1556,60 @@ fn list_with_include_destroyed_true() { common::api_types::actors::create::CreateRequest { datacenter: None, name: name.to_string(), - key: Some("destroyed-key".to_string()), + key: Some(format!("dc1-cursor-key-{}", i)), input: None, runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor"); - let destroyed_actor_id = res1.actor.actor_id.to_string(); - - common::api::public::actors_delete( - ctx.leader_dc().guard_port(), - common::api_types::actors::delete::DeletePath { - actor_id: res1.actor.actor_id, - }, - common::api_types::actors::delete::DeleteQuery { - namespace: namespace.clone(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, }, ) .await - .expect("failed to delete actor"); + .expect("failed to create actor in DC1"); + } - // Create an active actor - let res2 = common::api::public::actors_create( - ctx.leader_dc().guard_port(), + for i in 0..3 { + common::api::public::actors_create( + ctx.get_dc(2).guard_port(), common::api_types::actors::create::CreateQuery { namespace: namespace.clone(), }, common::api_types::actors::create::CreateRequest { - datacenter: None, + datacenter: Some("dc-2".to_string()), name: name.to_string(), - key: Some("active-key".to_string()), + key: Some(format!("dc2-cursor-key-{}", i)), input: None, runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, + crash_policy: rivet_types::actors::CrashPolicy::Sleep, }, ) .await - .expect("failed to create actor"); - let active_actor_id = res2.actor.actor_id.to_string(); - - // List with include_destroyed=true - let response = common::api::public::actors_list( + .expect("failed to create actor in DC2"); + } + + // Fetch first page with limit=3 + let page1 = common::api::public::actors_list( + ctx.leader_dc().guard_port(), + common::api_types::actors::list::ListQuery { + namespace: namespace.clone(), + name: Some(name.to_string()), + key: None, + actor_ids: None, + actor_id: vec![], + include_destroyed: None, + limit: Some(3), + cursor: None, + }, + ) + .await + .expect("failed to list page 1"); + + assert!( + page1.actors.len() <= 3, + "Page 1 should have at most 3 actors" + ); + + // Fetch second page using cursor + if let Some(cursor) = page1.pagination.cursor { + let page2 = common::api::public::actors_list( ctx.leader_dc().guard_port(), common::api_types::actors::list::ListQuery { namespace: namespace.clone(), @@ -477,1395 +1617,172 @@ fn list_with_include_destroyed_true() { key: None, actor_ids: None, actor_id: vec![], - include_destroyed: Some(true), - limit: None, - cursor: None, + include_destroyed: None, + limit: Some(3), + cursor: Some(cursor), }, ) .await - .expect("failed to list actors"); - - assert_eq!( - response.actors.len(), - 2, - "Should return both active and destroyed actors" - ); + .expect("failed to list page 2"); - // Verify both actors are in results - let returned_ids: HashSet = response + // Verify no duplicates between pages + let ids1: HashSet = page1 + .actors + .iter() + .map(|a| a.actor_id.to_string()) + .collect(); + let ids2: HashSet = page2 .actors .iter() .map(|a| a.actor_id.to_string()) .collect(); - assert!(returned_ids.contains(&active_actor_id)); - assert!(returned_ids.contains(&destroyed_actor_id)); - }, - ); -} -// MARK: List by Actor IDs + assert!( + ids1.is_disjoint(&ids2), + "Pages should have no duplicate actors across DCs" + ); + } + }); +} // Broken legacy Pegboard Runner coverage: full `runner::` sweep times out with // `test timed out: Elapsed(())`. #[test] -#[ignore = "broken legacy Pegboard Runner test: times out in full runner sweep"] -fn list_specific_actors_by_ids() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - // Create multiple actors - let actor_ids = common::bulk_create_actors( - ctx.leader_dc().guard_port(), - &namespace, - "id-list-test", - 5, - ) - .await; - - // Select specific actors to list - let selected_ids = vec![ - actor_ids[0].clone(), - actor_ids[2].clone(), - actor_ids[4].clone(), - ]; - - // List by actor IDs (comma-separated string) - let response = common::api::public::actors_list( - ctx.leader_dc().guard_port(), - common::api_types::actors::list::ListQuery { - namespace: namespace.clone(), - name: None, - key: None, - actor_id: selected_ids.clone(), - actor_ids: None, - include_destroyed: None, - limit: None, - cursor: None, - }, - ) - .await - .expect("failed to list actors"); - - assert_eq!( - response.actors.len(), - 3, - "Should return exactly the requested actors" - ); - - // Verify correct actors returned - let returned_ids: HashSet = response - .actors - .iter() - .map(|a| a.actor_id.to_string()) - .collect(); - for id in &selected_ids { - assert!( - returned_ids.contains(&id.to_string()), - "Actor {} should be in results", - id - ); - } - }, - ); -} - -#[test] -// Broken legacy Pegboard Runner test: full engine sweep can fail creating the -// DC2 actor with `actor.destroyed_during_creation`. -#[ignore = "broken legacy Pegboard Runner test: actor.destroyed_during_creation in full engine sweep"] -fn list_actors_from_multiple_datacenters() { - common::run( - common::TestOpts::new(2).with_timeout(45), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - // Create actors in different DCs - let res1 = common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: "multi-dc-actor".to_string(), - key: Some("dc1-key".to_string()), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor in DC1"); - let actor_id_dc1 = res1.actor.actor_id; - - let res2 = common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: Some("dc-2".to_string()), - name: "multi-dc-actor".to_string(), - key: Some("dc2-key".to_string()), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor in DC2"); - let actor_id_dc2 = res2.actor.actor_id; - - // List by actor IDs - should fetch from both DCs - let response = common::api::public::actors_list( - ctx.leader_dc().guard_port(), - common::api_types::actors::list::ListQuery { - namespace: namespace.clone(), - name: None, - key: None, - actor_id: vec![actor_id_dc1, actor_id_dc2], - actor_ids: None, - include_destroyed: None, - limit: None, - cursor: None, - }, - ) - .await - .expect("failed to list actors"); - - assert_eq!( - response.actors.len(), - 2, - "Should return actors from both DCs" - ); - }, - ); -} - -// MARK: Error cases - -#[test] -fn list_with_non_existent_namespace() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - // Try to list with non-existent namespace - let res = common::api::public::actors_list( - ctx.leader_dc().guard_port(), - common::api_types::actors::list::ListQuery { - namespace: "non-existent-namespace".to_string(), - name: Some("test-actor".to_string()), - key: None, - actor_ids: None, - actor_id: vec![], - include_destroyed: None, - limit: None, - cursor: None, - }, - ) - .await; - - // Should fail with namespace not found - assert!(res.is_err(), "Should fail with non-existent namespace"); - }, - ); -} - -#[test] -fn list_with_key_but_no_name() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - // Try to list with key but no name (validation error) - let res = common::api::public::actors_list( - ctx.leader_dc().guard_port(), - common::api_types::actors::list::ListQuery { - namespace: namespace.clone(), - name: None, - key: Some("key1".to_string()), - actor_ids: None, - actor_id: vec![], - include_destroyed: None, - limit: None, - cursor: None, - }, - ) - .await; - - // Should fail with validation error - assert!(res.is_err(), "Should return error for key without name"); - }, - ); -} - -#[test] -fn list_with_more_than_32_actor_ids() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - // Try to list with more than 32 actor IDs - let actor_ids: Vec = (0..33) - .map(|_| rivet_util::Id::new_v1(ctx.leader_dc().config.dc_label())) - .collect(); - - let res = common::api::public::actors_list( - ctx.leader_dc().guard_port(), - common::api_types::actors::list::ListQuery { - namespace: namespace.clone(), - name: None, - key: None, - actor_id: actor_ids, - actor_ids: None, - include_destroyed: None, - limit: None, - cursor: None, - }, - ) - .await; - - // Should fail with validation error - assert!(res.is_err(), "Should return error for too many actor IDs"); - }, - ); -} - -#[test] -fn list_without_name_when_not_using_actor_ids() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - // Try to list without name or actor_ids - let res = common::api::public::actors_list( - ctx.leader_dc().guard_port(), - common::api_types::actors::list::ListQuery { - namespace: namespace.clone(), - name: None, - key: None, - actor_ids: None, - actor_id: vec![], - include_destroyed: None, - limit: None, - cursor: None, - }, - ) - .await; - - // Should fail with validation error - assert!( - res.is_err(), - "Should return error when neither name nor actor_ids provided" - ); - }, - ); -} - -// MARK: Pagination and Sorting - -#[test] -fn verify_sorting_by_create_ts_descending() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - let name = "sorted-actor"; - - // Create actors with slight delays to ensure different timestamps - let mut actor_ids = Vec::new(); - for i in 0..3 { - let res = common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: name.to_string(), - key: Some(format!("key-{}", i)), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor"); - actor_ids.push(res.actor.actor_id.to_string()); - } - - // List actors - let response = common::api::public::actors_list( - ctx.leader_dc().guard_port(), - common::api_types::actors::list::ListQuery { - namespace: namespace.clone(), - name: Some(name.to_string()), - key: None, - actor_ids: None, - actor_id: vec![], - include_destroyed: None, - limit: None, - cursor: None, - }, - ) - .await - .expect("failed to list actors"); - - // Verify order - newest first (descending by create_ts) - for i in 0..response.actors.len() { - assert_eq!( - response.actors[i].actor_id.to_string(), - actor_ids[actor_ids.len() - 1 - i], - "Actors should be sorted by create_ts descending" - ); - } - }, - ); -} - -// MARK: Cross-datacenter - -// Broken legacy Pegboard Runner multi-DC coverage: full `runner::` sweep times -// out with `test timed out: Elapsed(())`. -#[test] -#[ignore = "broken legacy Pegboard Runner test: times out in full runner sweep"] -fn list_aggregates_results_from_all_datacenters() { - common::run( - common::TestOpts::new(2).with_timeout(45), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - let name = "fanout-test-actor"; - - // Create actors in both DCs - let res1 = common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: name.to_string(), - key: Some("dc1-key".to_string()), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor in DC1"); - let actor_id_dc1 = res1.actor.actor_id.to_string(); - - let res2 = common::api::public::actors_create( - ctx.get_dc(2).guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: Some("dc-2".to_string()), - name: name.to_string(), - key: Some("dc2-key".to_string()), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor in DC2"); - let actor_id_dc2 = res2.actor.actor_id.to_string(); - - // List by name - should fanout to all DCs - let response = common::api::public::actors_list( - ctx.leader_dc().guard_port(), - common::api_types::actors::list::ListQuery { - namespace: namespace.clone(), - name: Some(name.to_string()), - key: None, - actor_ids: None, - actor_id: vec![], - include_destroyed: None, - limit: None, - cursor: None, - }, - ) - .await - .expect("failed to list actors"); - - assert_eq!( - response.actors.len(), - 2, - "Should return actors from both DCs" - ); - - // Verify both actors are present - let returned_ids: HashSet = response - .actors - .iter() - .map(|a| a.actor_id.to_string()) - .collect(); - assert!(returned_ids.contains(&actor_id_dc1)); - assert!(returned_ids.contains(&actor_id_dc2)); - }, - ); -} - -// MARK: Edge cases - -#[test] -fn list_with_exactly_32_actor_ids() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - // Create exactly 32 actor IDs (boundary condition) - let actor_ids: Vec = (0..32) - .map(|_| rivet_util::Id::new_v1(ctx.leader_dc().config.dc_label())) - .collect(); - - // Should succeed with exactly 32 IDs - let response = common::api::public::actors_list( - ctx.leader_dc().guard_port(), - common::api_types::actors::list::ListQuery { - namespace: namespace.clone(), - name: None, - key: None, - actor_id: actor_ids, - actor_ids: None, - include_destroyed: None, - limit: None, - cursor: None, - }, - ) - .await - .expect("should succeed with exactly 32 actor IDs"); - - // Since these are fake IDs, we expect 0 results, but no error - assert_eq!( - response.actors.len(), - 0, - "Fake IDs should return empty results" - ); - }, - ); -} - -#[test] -fn list_by_key_with_include_destroyed_true() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - let name = "key-destroyed-test"; - let key = "test-key"; - - // Create and destroy an actor with a key - let res1 = common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: name.to_string(), - key: Some(key.to_string()), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor"); - let destroyed_actor_id = res1.actor.actor_id.to_string(); - - common::api::public::actors_delete( - ctx.leader_dc().guard_port(), - common::api_types::actors::delete::DeletePath { - actor_id: res1.actor.actor_id, - }, - common::api_types::actors::delete::DeleteQuery { - namespace: namespace.clone(), - }, - ) - .await - .expect("failed to delete actor"); - - // Create a new actor with the same key - let res2 = common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: name.to_string(), - key: Some(key.to_string()), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor"); - let active_actor_id = res2.actor.actor_id.to_string(); - - // List by key with include_destroyed=true - // This should use the fanout path, not the optimized key path - let response = common::api::public::actors_list( - ctx.leader_dc().guard_port(), - common::api_types::actors::list::ListQuery { - namespace: namespace.clone(), - name: Some(name.to_string()), - key: Some(key.to_string()), - actor_ids: None, - actor_id: vec![], - include_destroyed: Some(true), - limit: None, - cursor: None, - }, - ) - .await - .expect("failed to list actors"); - - // Should return both actors (destroyed and active) - assert_eq!( - response.actors.len(), - 2, - "Should return both destroyed and active actors with same key" - ); - - let returned_ids: HashSet = response - .actors - .iter() - .map(|a| a.actor_id.to_string()) - .collect(); - assert!(returned_ids.contains(&destroyed_actor_id)); - assert!(returned_ids.contains(&active_actor_id)); - }, - ); -} - -#[test] -fn list_default_limit_100() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - let name = "limit-test"; - - // Create 105 actors to test the default limit of 100 - let actor_ids = - common::bulk_create_actors(ctx.leader_dc().guard_port(), &namespace, name, 105) - .await; - - assert_eq!(actor_ids.len(), 105, "Should have created 105 actors"); - - // List without specifying limit - should use default limit of 100 - let response = common::api::public::actors_list( - ctx.leader_dc().guard_port(), - common::api_types::actors::list::ListQuery { - namespace: namespace.clone(), - name: Some(name.to_string()), - key: None, - actor_ids: None, - actor_id: vec![], - include_destroyed: None, - limit: None, // No limit specified - should default to 100 - cursor: None, - }, - ) - .await - .expect("failed to list actors"); - - // Should return exactly 100 actors due to default limit - assert_eq!( - response.actors.len(), - 100, - "Should return exactly 100 actors when default limit is applied" - ); - - // Verify cursor exists since there are more results - assert!( - response.pagination.cursor.is_some(), - "Cursor should exist when there are more results beyond the limit" - ); - }, - ); -} - -// Broken legacy Pegboard Runner coverage: full `runner::` sweep times out with -// `test timed out: Elapsed(())`. -#[test] -#[ignore = "broken legacy Pegboard Runner test: times out in full runner sweep"] -fn list_with_invalid_actor_id_format_in_comma_list() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - // Create a valid actor - let res = common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: "test-actor".to_string(), - key: Some("test-key".to_string()), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor"); - let valid_actor_id = res.actor.actor_id.to_string(); - - // Mix valid and invalid IDs in the comma-separated list - let mixed_ids = vec![ - valid_actor_id.clone(), - "invalid-uuid".to_string(), - "not-a-uuid".to_string(), - valid_actor_id.clone(), - ]; - - let response = common::api::public::actors_list( - ctx.leader_dc().guard_port(), - common::api_types::actors::list::ListQuery { - namespace: namespace.clone(), - name: None, - key: None, - actor_id: vec![], - actor_ids: Some(mixed_ids.join(",")), - include_destroyed: None, - limit: None, - cursor: None, - }, - ) - .await - .expect("should filter out invalid IDs gracefully"); - - // Should return only the valid actor (twice) (parsed IDs are filtered) - assert_eq!( - response.actors.len(), - 2, - "Should filter out invalid IDs and return only valid ones" - ); - assert_eq!(response.actors[0].actor_id.to_string(), valid_actor_id); - }, - ); -} - -// MARK: Cursor pagination - -#[test] -fn list_with_cursor_pagination() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - let name = "cursor-test-actor"; - - // Create 5 actors with same name - let mut actor_ids = Vec::new(); - for i in 0..5 { - let res = common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: name.to_string(), - key: Some(format!("cursor-key-{}", i)), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor"); - actor_ids.push(res.actor.actor_id.to_string()); - } - - // Fetch first page with limit=2 - let page1 = common::api::public::actors_list( - ctx.leader_dc().guard_port(), - common::api_types::actors::list::ListQuery { - namespace: namespace.clone(), - name: Some(name.to_string()), - key: None, - actor_ids: None, - actor_id: vec![], - include_destroyed: None, - limit: Some(2), - cursor: None, - }, - ) - .await - .expect("failed to list page 1"); - - assert_eq!(page1.actors.len(), 2, "Page 1 should have 2 actors"); - assert!( - page1.pagination.cursor.is_some(), - "Page 1 should return a cursor" - ); - - // Fetch second page using cursor - let page2 = common::api::public::actors_list( - ctx.leader_dc().guard_port(), - common::api_types::actors::list::ListQuery { - namespace: namespace.clone(), - name: Some(name.to_string()), - key: None, - actor_ids: None, - actor_id: vec![], - include_destroyed: None, - limit: Some(2), - cursor: page1.pagination.cursor.clone(), - }, - ) - .await - .expect("failed to list page 2"); - - assert_eq!(page2.actors.len(), 2, "Page 2 should have 2 actors"); - assert!( - page2.pagination.cursor.is_some(), - "Page 2 should return a cursor" - ); - - // Fetch third page using cursor - let page3 = common::api::public::actors_list( - ctx.leader_dc().guard_port(), - common::api_types::actors::list::ListQuery { - namespace: namespace.clone(), - name: Some(name.to_string()), - key: None, - actor_ids: None, - actor_id: vec![], - include_destroyed: None, - limit: Some(2), - cursor: page2.pagination.cursor.clone(), - }, - ) - .await - .expect("failed to list page 3"); - - assert_eq!(page3.actors.len(), 1, "Page 3 should have 1 actor"); - - // Verify no duplicates across pages - let ids1: HashSet = page1 - .actors - .iter() - .map(|a| a.actor_id.to_string()) - .collect(); - let ids2: HashSet = page2 - .actors - .iter() - .map(|a| a.actor_id.to_string()) - .collect(); - let ids3: HashSet = page3 - .actors - .iter() - .map(|a| a.actor_id.to_string()) - .collect(); - - assert!( - ids1.is_disjoint(&ids2), - "Page 1 and 2 should have no duplicates" - ); - assert!( - ids1.is_disjoint(&ids3), - "Page 1 and 3 should have no duplicates" - ); - assert!( - ids2.is_disjoint(&ids3), - "Page 2 and 3 should have no duplicates" - ); - - // Verify all actors are returned across all pages - let mut all_returned_ids = ids1; - all_returned_ids.extend(ids2); - all_returned_ids.extend(ids3); - - assert_eq!( - all_returned_ids.len(), - 5, - "All 5 actors should be returned across pages" - ); - for actor_id in &actor_ids { - assert!( - all_returned_ids.contains(&actor_id.to_string()), - "Actor {} should be in results", - actor_id - ); - } - }, - ); -} - -#[test] -fn list_cursor_filters_by_timestamp() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - let name = "timestamp-filter-test"; - - // Create 3 actors - for i in 0..3 { - common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: name.to_string(), - key: Some(format!("ts-key-{}", i)), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor"); - } - - // Get all actors to find a middle timestamp - let all_actors = common::api::public::actors_list( - ctx.leader_dc().guard_port(), - common::api_types::actors::list::ListQuery { - namespace: namespace.clone(), - name: Some(name.to_string()), - key: None, - actor_ids: None, - actor_id: vec![], - include_destroyed: None, - limit: None, - cursor: None, - }, - ) - .await - .expect("failed to list all actors"); - - assert_eq!(all_actors.actors.len(), 3, "Should have 3 actors"); - - // Use the first actor's timestamp as cursor (should filter out that actor and newer) - let cursor = all_actors.actors[0].create_ts.to_string(); - - // List with cursor - let filtered = common::api::public::actors_list( - ctx.leader_dc().guard_port(), - common::api_types::actors::list::ListQuery { - namespace: namespace.clone(), - name: Some(name.to_string()), - key: None, - actor_ids: None, - actor_id: vec![], - include_destroyed: None, - limit: None, - cursor: Some(cursor.clone()), - }, - ) - .await - .expect("failed to list with cursor"); - - // Should return only actors older than the cursor timestamp - assert!( - filtered.actors.len() < 3, - "Cursor should filter out some actors" - ); - - // Verify all returned actors have timestamps less than cursor - let cursor_ts: i64 = cursor.parse().expect("cursor should be valid i64"); - for actor in &filtered.actors { - assert!( - actor.create_ts < cursor_ts, - "Actor timestamp {} should be less than cursor {}", - actor.create_ts, - cursor_ts - ); - } - }, - ); -} - -#[test] -fn list_cursor_with_exact_timestamp_boundary() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - let name = "boundary-test"; - - // Create 3 actors - for i in 0..3 { - common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: name.to_string(), - key: Some(format!("boundary-key-{}", i)), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor"); - } - - // Get first page with limit=1 - let page1 = common::api::public::actors_list( - ctx.leader_dc().guard_port(), - common::api_types::actors::list::ListQuery { - namespace: namespace.clone(), - name: Some(name.to_string()), - key: None, - actor_ids: None, - actor_id: vec![], - include_destroyed: None, - limit: Some(1), - cursor: None, - }, - ) - .await - .expect("failed to list page 1"); - - assert_eq!(page1.actors.len(), 1, "Page 1 should have 1 actor"); - let first_actor_id = page1.actors[0].actor_id.to_string(); - - // Get second page using cursor - let page2 = common::api::public::actors_list( - ctx.leader_dc().guard_port(), - common::api_types::actors::list::ListQuery { - namespace: namespace.clone(), - name: Some(name.to_string()), - key: None, - actor_ids: None, - actor_id: vec![], - include_destroyed: None, - limit: None, - cursor: page1.pagination.cursor.clone(), - }, - ) - .await - .expect("failed to list page 2"); - - // Verify first actor is NOT in page 2 (exact boundary excluded) - let page2_ids: HashSet = page2 - .actors - .iter() - .map(|a| a.actor_id.to_string()) - .collect(); - assert!( - !page2_ids.contains(&first_actor_id), - "Actor with exact cursor timestamp should be excluded" - ); - }, - ); -} - -#[test] -fn list_cursor_empty_results_when_no_more_actors() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - let name = "empty-cursor-test"; - - // Create 2 actors - for i in 0..2 { - common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: name.to_string(), - key: Some(format!("empty-key-{}", i)), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor"); - } - - // List all actors - let all_actors = common::api::public::actors_list( - ctx.leader_dc().guard_port(), - common::api_types::actors::list::ListQuery { - namespace: namespace.clone(), - name: Some(name.to_string()), - key: None, - actor_ids: None, - actor_id: vec![], - include_destroyed: None, - limit: Some(10), - cursor: None, - }, - ) - .await - .expect("failed to list all actors"); - - assert_eq!(all_actors.actors.len(), 2, "Should have 2 actors"); - - // Use cursor to fetch next page (should be empty) - if let Some(cursor) = all_actors.pagination.cursor { - let next_page = common::api::public::actors_list( - ctx.leader_dc().guard_port(), - common::api_types::actors::list::ListQuery { - namespace: namespace.clone(), - name: Some(name.to_string()), - key: None, - actor_ids: None, - actor_id: vec![], - include_destroyed: None, - limit: Some(10), - cursor: Some(cursor), - }, - ) - .await - .expect("failed to list next page"); - - assert_eq!( - next_page.actors.len(), - 0, - "Should return empty results when no more actors" - ); - assert!( - next_page.pagination.cursor.is_none(), - "Should not return cursor when no more results" - ); - } - }, - ); -} - -#[test] -fn list_invalid_cursor_format() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - let name = "invalid-cursor-test"; - - // Try to list with invalid cursor (non-numeric string) - let res = common::api::public::actors_list( - ctx.leader_dc().guard_port(), - common::api_types::actors::list::ListQuery { - namespace: namespace.clone(), - name: Some(name.to_string()), - key: None, - actor_ids: None, - actor_id: vec![], - include_destroyed: None, - limit: None, - cursor: Some("not-a-number".to_string()), - }, - ) - .await; - - // Should fail with parse error - assert!( - res.is_err(), - "Should return error for invalid cursor format" - ); - }, - ); -} - -// Broken legacy Pegboard Runner multi-DC coverage: full `runner::` sweep times -// out with `test timed out: Elapsed(())`. -#[test] -#[ignore = "broken legacy Pegboard Runner test: times out in full runner sweep"] -fn list_cursor_across_datacenters() { - common::run( - common::TestOpts::new(2).with_timeout(45), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - let name = "multi-dc-cursor-test"; - - // Create actors in both DC1 and DC2 - for i in 0..3 { - common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: name.to_string(), - key: Some(format!("dc1-cursor-key-{}", i)), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor in DC1"); - } - - for i in 0..3 { - common::api::public::actors_create( - ctx.get_dc(2).guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: Some("dc-2".to_string()), - name: name.to_string(), - key: Some(format!("dc2-cursor-key-{}", i)), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor in DC2"); - } - - // Fetch first page with limit=3 - let page1 = common::api::public::actors_list( - ctx.leader_dc().guard_port(), - common::api_types::actors::list::ListQuery { - namespace: namespace.clone(), - name: Some(name.to_string()), - key: None, - actor_ids: None, - actor_id: vec![], - include_destroyed: None, - limit: Some(3), - cursor: None, - }, - ) - .await - .expect("failed to list page 1"); - - assert!( - page1.actors.len() <= 3, - "Page 1 should have at most 3 actors" - ); - - // Fetch second page using cursor - if let Some(cursor) = page1.pagination.cursor { - let page2 = common::api::public::actors_list( - ctx.leader_dc().guard_port(), - common::api_types::actors::list::ListQuery { - namespace: namespace.clone(), - name: Some(name.to_string()), - key: None, - actor_ids: None, - actor_id: vec![], - include_destroyed: None, - limit: Some(3), - cursor: Some(cursor), - }, - ) - .await - .expect("failed to list page 2"); - - // Verify no duplicates between pages - let ids1: HashSet = page1 - .actors - .iter() - .map(|a| a.actor_id.to_string()) - .collect(); - let ids2: HashSet = page2 - .actors - .iter() - .map(|a| a.actor_id.to_string()) - .collect(); - - assert!( - ids1.is_disjoint(&ids2), - "Pages should have no duplicate actors across DCs" - ); - } - }, - ); -} - -// Broken legacy Pegboard Runner coverage: full `runner::` sweep times out with -// `test timed out: Elapsed(())`. -#[test] -#[ignore = "broken legacy Pegboard Runner test: times out in full runner sweep"] +#[ignore = "cursor pagination off-by-one on final page"] fn list_actor_ids_with_cursor_pagination() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - let name = "actor-ids-cursor-test"; - - // Create 5 actors - let actor_ids = - common::bulk_create_actors(ctx.leader_dc().guard_port(), &namespace, name, 5).await; - - // List by actor_ids with limit=2 - let page1 = common::api::public::actors_list( - ctx.leader_dc().guard_port(), - common::api_types::actors::list::ListQuery { - namespace: namespace.clone(), - name: None, - key: None, - actor_id: actor_ids.clone(), - actor_ids: None, - include_destroyed: None, - limit: Some(2), - cursor: None, - }, - ) - .await - .expect("failed to list page 1"); - - assert_eq!( - page1.actors.len(), - 2, - "Page 1 should return exactly 2 actors" - ); - assert!( - page1.pagination.cursor.is_some(), - "Page 1 should return a cursor" - ); - - // Fetch second page using cursor - let page2 = common::api::public::actors_list( - ctx.leader_dc().guard_port(), - common::api_types::actors::list::ListQuery { - namespace: namespace.clone(), - name: None, - key: None, - actor_id: actor_ids.clone(), - actor_ids: None, - include_destroyed: None, - limit: Some(2), - cursor: page1.pagination.cursor.clone(), - }, - ) - .await - .expect("failed to list page 2"); - - assert_eq!( - page2.actors.len(), - 2, - "Page 2 should return exactly 2 actors" - ); - assert!( - page2.pagination.cursor.is_some(), - "Page 2 should return a cursor" - ); - - // Fetch third page using cursor - let page3 = common::api::public::actors_list( - ctx.leader_dc().guard_port(), - common::api_types::actors::list::ListQuery { - namespace: namespace.clone(), - name: None, - key: None, - actor_id: actor_ids.clone(), - actor_ids: None, - include_destroyed: None, - limit: Some(2), - cursor: page2.pagination.cursor.clone(), - }, - ) - .await - .expect("failed to list page 3"); - - assert_eq!( - page3.actors.len(), - 1, - "Page 3 should return 1 remaining actor" - ); - - // Verify no duplicates across pages - let ids1: HashSet = page1 - .actors - .iter() - .map(|a| a.actor_id.to_string()) - .collect(); - let ids2: HashSet = page2 - .actors - .iter() - .map(|a| a.actor_id.to_string()) - .collect(); - let ids3: HashSet = page3 - .actors - .iter() - .map(|a| a.actor_id.to_string()) - .collect(); - - assert!( - ids1.is_disjoint(&ids2), - "Page 1 and 2 should have no duplicates" - ); - assert!( - ids1.is_disjoint(&ids3), - "Page 1 and 3 should have no duplicates" - ); + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + let name = "actor-ids-cursor-test"; + + // Create 5 actors + let actor_ids = + common::bulk_create_actors(ctx.leader_dc().guard_port(), &namespace, name, 5).await; + + // List by actor_ids with limit=2 + let page1 = common::api::public::actors_list( + ctx.leader_dc().guard_port(), + common::api_types::actors::list::ListQuery { + namespace: namespace.clone(), + name: None, + key: None, + actor_id: actor_ids.clone(), + actor_ids: None, + include_destroyed: None, + limit: Some(2), + cursor: None, + }, + ) + .await + .expect("failed to list page 1"); + + assert_eq!( + page1.actors.len(), + 2, + "Page 1 should return exactly 2 actors" + ); + assert!( + page1.pagination.cursor.is_some(), + "Page 1 should return a cursor" + ); + + // Fetch second page using cursor + let page2 = common::api::public::actors_list( + ctx.leader_dc().guard_port(), + common::api_types::actors::list::ListQuery { + namespace: namespace.clone(), + name: None, + key: None, + actor_id: actor_ids.clone(), + actor_ids: None, + include_destroyed: None, + limit: Some(2), + cursor: page1.pagination.cursor.clone(), + }, + ) + .await + .expect("failed to list page 2"); + + assert_eq!( + page2.actors.len(), + 2, + "Page 2 should return exactly 2 actors" + ); + assert!( + page2.pagination.cursor.is_some(), + "Page 2 should return a cursor" + ); + + // Fetch third page using cursor + let page3 = common::api::public::actors_list( + ctx.leader_dc().guard_port(), + common::api_types::actors::list::ListQuery { + namespace: namespace.clone(), + name: None, + key: None, + actor_id: actor_ids.clone(), + actor_ids: None, + include_destroyed: None, + limit: Some(2), + cursor: page2.pagination.cursor.clone(), + }, + ) + .await + .expect("failed to list page 3"); + + assert_eq!( + page3.actors.len(), + 1, + "Page 3 should return 1 remaining actor" + ); + + // Verify no duplicates across pages + let ids1: HashSet = page1 + .actors + .iter() + .map(|a| a.actor_id.to_string()) + .collect(); + let ids2: HashSet = page2 + .actors + .iter() + .map(|a| a.actor_id.to_string()) + .collect(); + let ids3: HashSet = page3 + .actors + .iter() + .map(|a| a.actor_id.to_string()) + .collect(); + + assert!( + ids1.is_disjoint(&ids2), + "Page 1 and 2 should have no duplicates" + ); + assert!( + ids1.is_disjoint(&ids3), + "Page 1 and 3 should have no duplicates" + ); + assert!( + ids2.is_disjoint(&ids3), + "Page 2 and 3 should have no duplicates" + ); + + // Verify all actors are returned across all pages + let mut all_returned_ids = ids1; + all_returned_ids.extend(ids2); + all_returned_ids.extend(ids3); + + assert_eq!( + all_returned_ids.len(), + 5, + "All 5 actors should be returned across pages" + ); + for actor_id in &actor_ids { assert!( - ids2.is_disjoint(&ids3), - "Page 2 and 3 should have no duplicates" - ); - - // Verify all actors are returned across all pages - let mut all_returned_ids = ids1; - all_returned_ids.extend(ids2); - all_returned_ids.extend(ids3); - - assert_eq!( - all_returned_ids.len(), - 5, - "All 5 actors should be returned across pages" + all_returned_ids.contains(&actor_id.to_string()), + "Actor {} should be in results", + actor_id ); - for actor_id in &actor_ids { - assert!( - all_returned_ids.contains(&actor_id.to_string()), - "Actor {} should be in results", - actor_id - ); - } - }, - ); + } + }); } diff --git a/engine/packages/engine/tests/envoy/api_actors_list_names.rs b/engine/packages/engine/tests/envoy/api_actors_list_names.rs index 99f25db3d7..9f7ce62b38 100644 --- a/engine/packages/engine/tests/envoy/api_actors_list_names.rs +++ b/engine/packages/engine/tests/envoy/api_actors_list_names.rs @@ -6,212 +6,201 @@ use std::collections::HashSet; #[test] fn list_all_actor_names_in_namespace() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - // Create actors with different names - let names = vec!["actor-alpha", "actor-beta", "actor-gamma"]; - for name in &names { - common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: name.to_string(), - key: Some(common::generate_unique_key()), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor"); - } - - // Create multiple actors with same name (should deduplicate) - for i in 0..3 { - common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: "actor-alpha".to_string(), - key: Some(format!("key-{}", i)), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor"); - } - - // List actor names - let response = common::api::public::actors_list_names( + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let names = vec!["actor-alpha", "actor-beta", "actor-gamma"]; + let (namespace, _, _runner) = common::setup_test_namespace_with_envoy_for_names( + ctx.leader_dc(), + names.iter().map(|s| s.to_string()).collect(), + ) + .await; + for name in &names { + common::api::public::actors_create( ctx.leader_dc().guard_port(), - common::api_types::actors::list_names::ListNamesQuery { + common::api_types::actors::create::CreateQuery { namespace: namespace.clone(), - limit: None, - cursor: None, }, - ) - .await - .expect("failed to list actor names"); - - // Should return unique names only (HashMap automatically deduplicates) - assert_eq!(response.names.len(), 3, "Should return 3 unique names"); - - // Verify all names are present in the HashMap keys - let returned_names: HashSet = response.names.keys().cloned().collect(); - for name in &names { - assert!( - returned_names.contains(*name), - "Name {} should be in results", - name - ); - } - }, - ); -} - -#[test] -// Broken legacy Pegboard Runner test: full engine sweep timed out in -// `list_names_with_pagination`. -#[ignore = "broken legacy Pegboard Runner test: times out in full engine sweep"] -fn list_names_with_pagination() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - // Create actors with many different names - for i in 0..9 { - common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: format!("actor-{:02}", i), - key: Some(common::generate_unique_key()), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor"); - } - - // First page - limit 5 - let response1 = common::api::public::actors_list_names( - ctx.leader_dc().guard_port(), - common::api_types::actors::list_names::ListNamesQuery { - namespace: namespace.clone(), - limit: Some(5), - cursor: None, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: name.to_string(), + key: Some(common::generate_unique_key()), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, }, ) .await - .expect("failed to list actor names"); - - assert_eq!( - response1.names.len(), - 5, - "Should return 5 names with limit=5" - ); - - let cursor = response1 - .pagination - .cursor - .as_ref() - .expect("Should have cursor for pagination"); + .expect("failed to create actor"); + } - // Second page - use cursor - let response2 = common::api::public::actors_list_names( + // Create multiple actors with same name (should deduplicate) + for i in 0..3 { + common::api::public::actors_create( ctx.leader_dc().guard_port(), - common::api_types::actors::list_names::ListNamesQuery { + common::api_types::actors::create::CreateQuery { namespace: namespace.clone(), - limit: Some(5), - cursor: Some(cursor.clone()), + }, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: "actor-alpha".to_string(), + key: Some(format!("key-{}", i)), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, }, ) .await - .expect("failed to list actor names page 2"); - - assert_eq!(response2.names.len(), 4, "Should return remaining 4 names"); - - // Verify no duplicates between pages - let set1: HashSet = response1.names.keys().cloned().collect(); - let set2: HashSet = response2.names.keys().cloned().collect(); + .expect("failed to create actor"); + } + + // List actor names + let response = common::api::public::actors_list_names( + ctx.leader_dc().guard_port(), + common::api_types::actors::list_names::ListNamesQuery { + namespace: namespace.clone(), + limit: None, + cursor: None, + }, + ) + .await + .expect("failed to list actor names"); + + // Should return unique names only (HashMap automatically deduplicates) + assert_eq!(response.names.len(), 3, "Should return 3 unique names"); + + // Verify all names are present in the HashMap keys + let returned_names: HashSet = response.names.keys().cloned().collect(); + for name in &names { assert!( - set1.is_disjoint(&set2), - "Pages should not have duplicate names" + returned_names.contains(*name), + "Name {} should be in results", + name ); - }, - ); + } + }); } #[test] -fn list_names_returns_empty_for_empty_namespace() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; +// Broken legacy Pegboard Runner test: full engine sweep timed out in +// `list_names_with_pagination`. +#[ignore = "list_names ignores limit param"] +fn list_names_with_pagination() { + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - // List names in empty namespace - let response = common::api::public::actors_list_names( + // Create actors with many different names + for i in 0..9 { + common::api::public::actors_create( ctx.leader_dc().guard_port(), - common::api_types::actors::list_names::ListNamesQuery { + common::api_types::actors::create::CreateQuery { namespace: namespace.clone(), - limit: None, - cursor: None, + }, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: format!("actor-{:02}", i), + key: Some(common::generate_unique_key()), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, }, ) .await - .expect("failed to list actor names"); + .expect("failed to create actor"); + } + + // First page - limit 5 + let response1 = common::api::public::actors_list_names( + ctx.leader_dc().guard_port(), + common::api_types::actors::list_names::ListNamesQuery { + namespace: namespace.clone(), + limit: Some(5), + cursor: None, + }, + ) + .await + .expect("failed to list actor names"); + + assert_eq!( + response1.names.len(), + 5, + "Should return 5 names with limit=5" + ); + + let cursor = response1 + .pagination + .cursor + .as_ref() + .expect("Should have cursor for pagination"); + + // Second page - use cursor + let response2 = common::api::public::actors_list_names( + ctx.leader_dc().guard_port(), + common::api_types::actors::list_names::ListNamesQuery { + namespace: namespace.clone(), + limit: Some(5), + cursor: Some(cursor.clone()), + }, + ) + .await + .expect("failed to list actor names page 2"); + + assert_eq!(response2.names.len(), 4, "Should return remaining 4 names"); + + // Verify no duplicates between pages + let set1: HashSet = response1.names.keys().cloned().collect(); + let set2: HashSet = response2.names.keys().cloned().collect(); + assert!( + set1.is_disjoint(&set2), + "Pages should not have duplicate names" + ); + }); +} - assert_eq!( - response.names.len(), - 0, - "Should return empty HashMap for empty namespace" - ); - }, - ); +#[test] +fn list_names_returns_empty_for_empty_namespace() { + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy_for_names(ctx.leader_dc(), vec![]).await; + + // List names in empty namespace + let response = common::api::public::actors_list_names( + ctx.leader_dc().guard_port(), + common::api_types::actors::list_names::ListNamesQuery { + namespace: namespace.clone(), + limit: None, + cursor: None, + }, + ) + .await + .expect("failed to list actor names"); + + assert_eq!( + response.names.len(), + 0, + "Should return empty HashMap for empty namespace" + ); + }); } // MARK: Error cases #[test] fn list_names_with_non_existent_namespace() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - // Try to list names with non-existent namespace - let res = common::api::public::actors_list_names( - ctx.leader_dc().guard_port(), - common::api_types::actors::list_names::ListNamesQuery { - namespace: "non-existent-namespace".to_string(), - limit: None, - cursor: None, - }, - ) - .await; - - // Should fail with namespace not found - assert!(res.is_err(), "Should fail with non-existent namespace"); - }, - ); + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + // Try to list names with non-existent namespace + let res = common::api::public::actors_list_names( + ctx.leader_dc().guard_port(), + common::api_types::actors::list_names::ListNamesQuery { + namespace: "non-existent-namespace".to_string(), + limit: None, + cursor: None, + }, + ) + .await; + + // Should fail with namespace not found + assert!(res.is_err(), "Should fail with non-existent namespace"); + }); } // MARK: Cross-datacenter tests @@ -219,15 +208,156 @@ fn list_names_with_non_existent_namespace() { // Broken legacy Pegboard Runner multi-DC coverage: full engine sweep returns // `actor.destroyed_during_creation` while creating the DC2 actor. #[test] -#[ignore = "broken legacy Pegboard Runner test: actor.destroyed_during_creation"] +#[ignore = "DC2 actor create hangs / workflow-worker lease failure"] fn list_names_fanout_to_all_datacenters() { - common::run( - common::TestOpts::new(2).with_timeout(45), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + common::run(common::TestOpts::new(2).with_timeout(45), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + // Create actors with different names in different DCs + common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: "dc1-actor".to_string(), + key: Some(common::generate_unique_key()), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to create actor in DC1"); + + common::api::public::actors_create( + ctx.get_dc(2).guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: Some("dc-2".to_string()), + name: "dc2-actor".to_string(), + key: Some(common::generate_unique_key()), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to create actor in DC2"); + + // List names from DC 1 - should fanout to all DCs + let response = common::api::public::actors_list_names( + ctx.leader_dc().guard_port(), + common::api_types::actors::list_names::ListNamesQuery { + namespace: namespace.clone(), + limit: None, + cursor: None, + }, + ) + .await + .expect("failed to list actor names"); + + // Should return names from both DCs + let returned_names: HashSet = response.names.keys().cloned().collect(); + assert!( + returned_names.contains("dc1-actor"), + "Should contain DC1 actor name" + ); + assert!( + returned_names.contains("dc2-actor"), + "Should contain DC2 actor name" + ); + }); +} + +#[test] +// Broken legacy Pegboard Runner test: full engine sweep timed out in +// `list_names_deduplication_across_datacenters`. +#[ignore = "DC2 actor create hangs / workflow-worker lease failure"] +fn list_names_deduplication_across_datacenters() { + common::run(common::TestOpts::new(2).with_timeout(45), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + // Create actors with same name in different DCs + let shared_name = "shared-name-actor"; + + common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: shared_name.to_string(), + key: Some("dc1-key".to_string()), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to create actor in DC1"); + + common::api::public::actors_create( + ctx.get_dc(2).guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: Some("dc-2".to_string()), + name: shared_name.to_string(), + key: Some("dc2-key".to_string()), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to create actor in DC2"); + + // List names - should deduplicate + let response = common::api::public::actors_list_names( + ctx.leader_dc().guard_port(), + common::api_types::actors::list_names::ListNamesQuery { + namespace: namespace.clone(), + limit: None, + cursor: None, + }, + ) + .await + .expect("failed to list actor names"); + + // Should return only one instance of the name (HashMap deduplicates) + assert!( + response.names.contains_key(shared_name), + "Should contain the shared name" + ); + + // Count occurrences - should be exactly 1 in the HashMap + let name_count = response + .names + .keys() + .filter(|n| n.as_str() == shared_name) + .count(); + assert_eq!(name_count, 1, "Should deduplicate names across datacenters"); + }); +} - // Create actors with different names in different DCs +#[test] +fn list_names_alphabetical_sorting() { + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let unsorted_names = vec!["zebra-actor", "alpha-actor", "beta-actor", "gamma-actor"]; + let (namespace, _, _runner) = common::setup_test_namespace_with_envoy_for_names( + ctx.leader_dc(), + unsorted_names.iter().map(|s| s.to_string()).collect(), + ) + .await; + for name in &unsorted_names { common::api::public::actors_create( ctx.leader_dc().guard_port(), common::api_types::actors::create::CreateQuery { @@ -235,73 +365,200 @@ fn list_names_fanout_to_all_datacenters() { }, common::api_types::actors::create::CreateRequest { datacenter: None, - name: "dc1-actor".to_string(), + name: name.to_string(), key: Some(common::generate_unique_key()), input: None, runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, + crash_policy: rivet_types::actors::CrashPolicy::Sleep, }, ) .await - .expect("failed to create actor in DC1"); + .expect("failed to create actor"); + } + + // List names + let response = common::api::public::actors_list_names( + ctx.leader_dc().guard_port(), + common::api_types::actors::list_names::ListNamesQuery { + namespace: namespace.clone(), + limit: None, + cursor: None, + }, + ) + .await + .expect("failed to list actor names"); + + // Convert HashMap keys to sorted vector + let mut returned_names: Vec = response.names.keys().cloned().collect(); + returned_names.sort(); + + // Verify alphabetical order + assert_eq!(returned_names.len(), 4, "Should return all 4 unique names"); + assert_eq!(returned_names[0], "alpha-actor"); + assert_eq!(returned_names[1], "beta-actor"); + assert_eq!(returned_names[2], "gamma-actor"); + assert_eq!(returned_names[3], "zebra-actor"); + }); +} +// MARK: Edge cases + +#[test] +// Broken legacy Pegboard Runner test: full engine sweep timed out in +// `list_names_default_limit_100`. +#[ignore = "list_names default limit not applied (returns 1)"] +fn list_names_default_limit_100() { + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + + // Create 105 actors with different names to test the default limit of 100 + for i in 0..105 { common::api::public::actors_create( - ctx.get_dc(2).guard_port(), + ctx.leader_dc().guard_port(), common::api_types::actors::create::CreateQuery { namespace: namespace.clone(), }, common::api_types::actors::create::CreateRequest { - datacenter: Some("dc-2".to_string()), - name: "dc2-actor".to_string(), + datacenter: None, + name: format!("actor-{:03}", i), key: Some(common::generate_unique_key()), input: None, runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor in DC2"); - - // List names from DC 1 - should fanout to all DCs - let response = common::api::public::actors_list_names( - ctx.leader_dc().guard_port(), - common::api_types::actors::list_names::ListNamesQuery { - namespace: namespace.clone(), - limit: None, - cursor: None, + crash_policy: rivet_types::actors::CrashPolicy::Sleep, }, ) .await - .expect("failed to list actor names"); + .expect("failed to create actor"); + } + + // List without specifying limit - should use default limit of 100 + let response = common::api::public::actors_list_names( + ctx.leader_dc().guard_port(), + common::api_types::actors::list_names::ListNamesQuery { + namespace: namespace.clone(), + limit: None, // No limit specified - should default to 100 + cursor: None, + }, + ) + .await + .expect("failed to list actor names"); + + // Should return exactly 100 names due to default limit + assert_eq!( + response.names.len(), + 100, + "Should return exactly 100 names when default limit is applied" + ); + + // Verify cursor exists since there are more results + assert!( + response.pagination.cursor.is_some(), + "Cursor should exist when there are more results beyond the limit" + ); + }); +} - // Should return names from both DCs - let returned_names: HashSet = response.names.keys().cloned().collect(); - assert!( - returned_names.contains("dc1-actor"), - "Should contain DC1 actor name" - ); - assert!( - returned_names.contains("dc2-actor"), - "Should contain DC2 actor name" - ); - }, - ); +#[test] +fn list_names_with_metadata() { + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let actor_name = "test-actor-with-metadata"; + let (namespace, _, _runner) = common::setup_test_namespace_with_envoy_for_names( + ctx.leader_dc(), + vec![actor_name.to_string()], + ) + .await; + + // Create an actor + common::api::public::actors_create( + ctx.leader_dc().guard_port(), + common::api_types::actors::create::CreateQuery { + namespace: namespace.clone(), + }, + common::api_types::actors::create::CreateRequest { + datacenter: None, + name: actor_name.to_string(), + key: Some(common::generate_unique_key()), + input: None, + runner_name_selector: common::TEST_RUNNER_NAME.to_string(), + crash_policy: rivet_types::actors::CrashPolicy::Sleep, + }, + ) + .await + .expect("failed to create actor"); + + // List names + let response = common::api::public::actors_list_names( + ctx.leader_dc().guard_port(), + common::api_types::actors::list_names::ListNamesQuery { + namespace: namespace.clone(), + limit: None, + cursor: None, + }, + ) + .await + .expect("failed to list actor names"); + + // Verify the name exists and has metadata + assert!( + response.names.contains_key(actor_name), + "Should contain the actor name" + ); + + let _actor_name_info = response + .names + .get(actor_name) + .expect("Should have actor name info"); + + // Verify ActorName exists - the fact that we got it from the HashMap means + // it has the expected structure with metadata field + // No need to assert further on the metadata since it's always present as a Map + }); } #[test] -// Broken legacy Pegboard Runner test: full engine sweep timed out in -// `list_names_deduplication_across_datacenters`. -#[ignore = "broken legacy Pegboard Runner test: times out in full engine sweep"] -fn list_names_deduplication_across_datacenters() { - common::run( - common::TestOpts::new(2).with_timeout(45), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; +fn list_names_empty_response_no_cursor() { + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = + common::setup_test_namespace_with_envoy_for_names(ctx.leader_dc(), vec![]).await; + + // List names in empty namespace + let response = common::api::public::actors_list_names( + ctx.leader_dc().guard_port(), + common::api_types::actors::list_names::ListNamesQuery { + namespace: namespace.clone(), + limit: None, + cursor: None, + }, + ) + .await + .expect("failed to list actor names"); + + // Empty response should have no cursor + assert_eq!(response.names.len(), 0, "Should return empty HashMap"); + assert!( + response.pagination.cursor.is_none(), + "Empty response should not have a cursor" + ); + }); +} - // Create actors with same name in different DCs - let shared_name = "shared-name-actor"; +// MARK: Comprehensive pagination tests +/// This test exhaustively checks that pagination works correctly by iterating +/// through all pages and verifying no duplicates appear across pages. +/// This is a regression test for the cursor being inclusive instead of exclusive. +#[test] +fn list_names_pagination_no_duplicates_comprehensive() { + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let (namespace, _, _runner) = common::setup_test_namespace_with_envoy_for_names( + ctx.leader_dc(), + (0..15).map(|i| format!("paginate-actor-{:02}", i)).collect(), + ) + .await; + + // Create actors with sequential names + for i in 0..15 { common::api::public::actors_create( ctx.leader_dc().guard_port(), common::api_types::actors::create::CreateQuery { @@ -309,189 +566,88 @@ fn list_names_deduplication_across_datacenters() { }, common::api_types::actors::create::CreateRequest { datacenter: None, - name: shared_name.to_string(), - key: Some("dc1-key".to_string()), + name: format!("paginate-actor-{:02}", i), + key: Some(common::generate_unique_key()), input: None, runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, + crash_policy: rivet_types::actors::CrashPolicy::Sleep, }, ) .await - .expect("failed to create actor in DC1"); + .expect("failed to create actor"); + } - common::api::public::actors_create( - ctx.get_dc(2).guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: Some("dc-2".to_string()), - name: shared_name.to_string(), - key: Some("dc2-key".to_string()), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor in DC2"); + // Paginate through all results with small page size + let mut all_names: HashSet = HashSet::new(); + let mut cursor: Option = None; + let mut page_count = 0; - // List names - should deduplicate + loop { let response = common::api::public::actors_list_names( ctx.leader_dc().guard_port(), common::api_types::actors::list_names::ListNamesQuery { namespace: namespace.clone(), - limit: None, - cursor: None, + limit: Some(4), + cursor: cursor.clone(), }, ) .await .expect("failed to list actor names"); - // Should return only one instance of the name (HashMap deduplicates) - assert!( - response.names.contains_key(shared_name), - "Should contain the shared name" - ); - - // Count occurrences - should be exactly 1 in the HashMap - let name_count = response - .names - .keys() - .filter(|n| n.as_str() == shared_name) - .count(); - assert_eq!(name_count, 1, "Should deduplicate names across datacenters"); - }, - ); -} + page_count += 1; -#[test] -fn list_names_alphabetical_sorting() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - // Create actors with names that need sorting - let unsorted_names = vec!["zebra-actor", "alpha-actor", "beta-actor", "gamma-actor"]; - for name in &unsorted_names { - common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: name.to_string(), - key: Some(common::generate_unique_key()), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor"); + // Check for duplicates - this is the key assertion for the bug fix + for name in response.names.keys() { + assert!( + !all_names.contains(name), + "DUPLICATE FOUND: '{}' appeared on page {} but was already seen. \ + This indicates the cursor is inclusive instead of exclusive. \ + All names so far: {:?}", + name, + page_count, + all_names + ); + all_names.insert(name.clone()); } - // List names - let response = common::api::public::actors_list_names( - ctx.leader_dc().guard_port(), - common::api_types::actors::list_names::ListNamesQuery { - namespace: namespace.clone(), - limit: None, - cursor: None, - }, - ) - .await - .expect("failed to list actor names"); - - // Convert HashMap keys to sorted vector - let mut returned_names: Vec = response.names.keys().cloned().collect(); - returned_names.sort(); - - // Verify alphabetical order - assert_eq!(returned_names.len(), 4, "Should return all 4 unique names"); - assert_eq!(returned_names[0], "alpha-actor"); - assert_eq!(returned_names[1], "beta-actor"); - assert_eq!(returned_names[2], "gamma-actor"); - assert_eq!(returned_names[3], "zebra-actor"); - }, - ); -} - -// MARK: Edge cases - -#[test] -// Broken legacy Pegboard Runner test: full engine sweep timed out in -// `list_names_default_limit_100`. -#[ignore = "broken legacy Pegboard Runner test: times out in full engine sweep"] -fn list_names_default_limit_100() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - // Create 105 actors with different names to test the default limit of 100 - for i in 0..105 { - common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: format!("actor-{:03}", i), - key: Some(common::generate_unique_key()), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor"); + // Move to next page or break + if response.pagination.cursor.is_none() || response.names.is_empty() { + break; } + cursor = response.pagination.cursor; - // List without specifying limit - should use default limit of 100 - let response = common::api::public::actors_list_names( - ctx.leader_dc().guard_port(), - common::api_types::actors::list_names::ListNamesQuery { - namespace: namespace.clone(), - limit: None, // No limit specified - should default to 100 - cursor: None, - }, - ) - .await - .expect("failed to list actor names"); - - // Should return exactly 100 names due to default limit - assert_eq!( - response.names.len(), - 100, - "Should return exactly 100 names when default limit is applied" - ); - - // Verify cursor exists since there are more results - assert!( - response.pagination.cursor.is_some(), - "Cursor should exist when there are more results beyond the limit" - ); - }, - ); + // Safety limit to prevent infinite loops + if page_count > 20 { + panic!("Too many pages, possible infinite loop"); + } + } + + // Should have found all actor names + assert_eq!( + all_names.len(), + 15, + "Should find all 15 actor names across pages, found: {}. Names: {:?}", + all_names.len(), + all_names + ); + }); } +/// Tests that the cursor correctly advances past boundary conditions. +/// Creates actors with names that test edge cases in lexicographic ordering. #[test] -fn list_names_with_metadata() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - let actor_name = "test-actor-with-metadata"; - - // Create an actor +fn list_names_pagination_boundary_cases() { + common::run(common::TestOpts::new(1).with_timeout(30), |ctx| async move { + let names = vec![ + "test-a", "test-aa", "test-aaa", "test-ab", "test-b", "test-ba", + ]; + let (namespace, _, _runner) = common::setup_test_namespace_with_envoy_for_names( + ctx.leader_dc(), + names.iter().map(|s| s.to_string()).collect(), + ) + .await; + + for name in &names { common::api::public::actors_create( ctx.leader_dc().guard_port(), common::api_types::actors::create::CreateQuery { @@ -499,241 +655,57 @@ fn list_names_with_metadata() { }, common::api_types::actors::create::CreateRequest { datacenter: None, - name: actor_name.to_string(), + name: name.to_string(), key: Some(common::generate_unique_key()), input: None, runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, + crash_policy: rivet_types::actors::CrashPolicy::Sleep, }, ) .await .expect("failed to create actor"); + } - // List names - let response = common::api::public::actors_list_names( - ctx.leader_dc().guard_port(), - common::api_types::actors::list_names::ListNamesQuery { - namespace: namespace.clone(), - limit: None, - cursor: None, - }, - ) - .await - .expect("failed to list actor names"); - - // Verify the name exists and has metadata - assert!( - response.names.contains_key(actor_name), - "Should contain the actor name" - ); - - let _actor_name_info = response - .names - .get(actor_name) - .expect("Should have actor name info"); - - // Verify ActorName exists - the fact that we got it from the HashMap means - // it has the expected structure with metadata field - // No need to assert further on the metadata since it's always present as a Map - }, - ); -} - -#[test] -fn list_names_empty_response_no_cursor() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; + // Page through with limit=2 + let mut collected_names: Vec = Vec::new(); + let mut cursor: Option = None; - // List names in empty namespace + loop { let response = common::api::public::actors_list_names( ctx.leader_dc().guard_port(), common::api_types::actors::list_names::ListNamesQuery { namespace: namespace.clone(), - limit: None, - cursor: None, + limit: Some(2), + cursor: cursor.clone(), }, ) .await .expect("failed to list actor names"); - // Empty response should have no cursor - assert_eq!(response.names.len(), 0, "Should return empty HashMap"); - assert!( - response.pagination.cursor.is_none(), - "Empty response should not have a cursor" - ); - }, - ); -} - -// MARK: Comprehensive pagination tests - -/// This test exhaustively checks that pagination works correctly by iterating -/// through all pages and verifying no duplicates appear across pages. -/// This is a regression test for the cursor being inclusive instead of exclusive. -#[test] -fn list_names_pagination_no_duplicates_comprehensive() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - // Create actors with sequential names - for i in 0..15 { - common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: format!("paginate-actor-{:02}", i), - key: Some(common::generate_unique_key()), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor"); - } - - // Paginate through all results with small page size - let mut all_names: HashSet = HashSet::new(); - let mut cursor: Option = None; - let mut page_count = 0; - - loop { - let response = common::api::public::actors_list_names( - ctx.leader_dc().guard_port(), - common::api_types::actors::list_names::ListNamesQuery { - namespace: namespace.clone(), - limit: Some(4), - cursor: cursor.clone(), - }, - ) - .await - .expect("failed to list actor names"); - - page_count += 1; - - // Check for duplicates - this is the key assertion for the bug fix - for name in response.names.keys() { - assert!( - !all_names.contains(name), - "DUPLICATE FOUND: '{}' appeared on page {} but was already seen. \ - This indicates the cursor is inclusive instead of exclusive. \ - All names so far: {:?}", - name, - page_count, - all_names - ); - all_names.insert(name.clone()); - } - - // Move to next page or break - if response.pagination.cursor.is_none() || response.names.is_empty() { - break; - } - cursor = response.pagination.cursor; - - // Safety limit to prevent infinite loops - if page_count > 20 { - panic!("Too many pages, possible infinite loop"); - } - } - - // Should have found all actor names - assert_eq!( - all_names.len(), - 15, - "Should find all 15 actor names across pages, found: {}. Names: {:?}", - all_names.len(), - all_names - ); - }, - ); -} - -/// Tests that the cursor correctly advances past boundary conditions. -/// Creates actors with names that test edge cases in lexicographic ordering. -#[test] -fn list_names_pagination_boundary_cases() { - common::run( - common::TestOpts::new(1).with_timeout(30), - |ctx| async move { - let (namespace, _, _runner) = - common::setup_test_namespace_with_envoy(ctx.leader_dc()).await; - - // Create actors with names that have similar prefixes to test boundary conditions - let names = vec![ - "test-a", "test-aa", "test-aaa", "test-ab", "test-b", "test-ba", - ]; - - for name in &names { - common::api::public::actors_create( - ctx.leader_dc().guard_port(), - common::api_types::actors::create::CreateQuery { - namespace: namespace.clone(), - }, - common::api_types::actors::create::CreateRequest { - datacenter: None, - name: name.to_string(), - key: Some(common::generate_unique_key()), - input: None, - runner_name_selector: common::TEST_RUNNER_NAME.to_string(), - crash_policy: rivet_types::actors::CrashPolicy::Destroy, - }, - ) - .await - .expect("failed to create actor"); - } + // Collect names from this page + let page_names: Vec<_> = response.names.keys().cloned().collect(); + collected_names.extend(page_names); - // Page through with limit=2 - let mut collected_names: Vec = Vec::new(); - let mut cursor: Option = None; - - loop { - let response = common::api::public::actors_list_names( - ctx.leader_dc().guard_port(), - common::api_types::actors::list_names::ListNamesQuery { - namespace: namespace.clone(), - limit: Some(2), - cursor: cursor.clone(), - }, - ) - .await - .expect("failed to list actor names"); - - // Collect names from this page - let page_names: Vec<_> = response.names.keys().cloned().collect(); - collected_names.extend(page_names); - - if response.pagination.cursor.is_none() || response.names.is_empty() { - break; - } - cursor = response.pagination.cursor; + if response.pagination.cursor.is_none() || response.names.is_empty() { + break; } - - // Filter to just our test names - let test_names: HashSet<_> = collected_names - .iter() - .filter(|n| names.contains(&n.as_str())) - .cloned() - .collect(); - - // All names should be present exactly once - assert_eq!( - test_names.len(), - names.len(), - "All test names should be present. Expected {:?}, got {:?}", - names, - test_names - ); - }, - ); + cursor = response.pagination.cursor; + } + + // Filter to just our test names + let test_names: HashSet<_> = collected_names + .iter() + .filter(|n| names.contains(&n.as_str())) + .cloned() + .collect(); + + // All names should be present exactly once + assert_eq!( + test_names.len(), + names.len(), + "All test names should be present. Expected {:?}, got {:?}", + names, + test_names + ); + }); }