diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index 5033983a..e58aa05b 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -96,6 +96,16 @@ jobs: DATABASE_URL: postgres://postgres:postgres@localhost:5432/gitlawb_test run: cargo test --workspace + # The real-node deny harness is behind the `test-harness` feature (so its + # spawn surface never compiles into the production binary), which means + # `--workspace` above skips it. Run it explicitly so the trust-boundary + # regression cases (INV-1/INV-2/INV-8) execute on every PR. Uses the same + # Postgres service; `#[sqlx::test]` provisions an isolated DB per test. + - name: cargo test (real-node deny harness) + env: + DATABASE_URL: postgres://postgres:postgres@localhost:5432/gitlawb_test + run: cargo test -p gitlawb-node --features test-harness --test deny_harness + build-release: name: build --release runs-on: ubuntu-latest diff --git a/crates/gitlawb-node/Cargo.toml b/crates/gitlawb-node/Cargo.toml index 7cc3727f..822624a3 100644 --- a/crates/gitlawb-node/Cargo.toml +++ b/crates/gitlawb-node/Cargo.toml @@ -6,10 +6,28 @@ edition.workspace = true rust-version.workspace = true license.workspace = true +[lib] +name = "gitlawb_node" +path = "src/lib.rs" + [[bin]] name = "gitlawb-node" path = "src/main.rs" +# Exposes the `test_harness` spawn surface (src/test_harness.rs) to the +# real-node deny-harness integration crate. Off by default so the production +# binary never compiles test-only boot code. Enable with +# `cargo test -p gitlawb-node --features test-harness`. +[features] +test-harness = [] + +# Only builds when `test-harness` is enabled; otherwise skipped (not an error), +# so `cargo test --workspace` without the feature stays green. +[[test]] +name = "deny_harness" +path = "tests/deny_harness.rs" +required-features = ["test-harness"] + [dependencies] gitlawb-core = { path = "../gitlawb-core" } ed25519-dalek = { workspace = true } @@ -77,3 +95,12 @@ libp2p-dns = { version = "0.44.0", features = ["tokio"] } [dev-dependencies] mockito = "1" tempfile = "3" +# test-util enables tokio::test(start_paused): the delayed claim re-sweep +# test drives the real CLAIM_SWEEP_MIN_AGE-plus-margin sleep on paused time. +tokio = { workspace = true, features = ["test-util"] } +# Used by the deny-harness integration crate (tests/deny_harness.rs): the +# #[sqlx::test] macro for an ephemeral per-test pool, and reqwest as the real +# HTTP client that drives deny paths over the socket. +sqlx = { version = "0.8", features = ["postgres", "runtime-tokio-rustls", "macros", "migrate"] } +reqwest = { workspace = true } +hex = "0.4" diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index fc2247d9..2a8f8cc1 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -162,8 +162,10 @@ pub struct Config { pub metrics_addr: String, /// Maximum time to wait for in-flight requests to drain on shutdown, in - /// seconds. After this elapses, the server returns 503 to anything still - /// in flight and exits. Default: 30s. + /// seconds. On the shutdown signal the server stops accepting new + /// connections and gives in-flight requests this long to complete; at + /// expiry it abandons whatever remains (clients observe a closed + /// connection, not an error response) and exits. Default: 30s. #[arg(long, env = "GITLAWB_SHUTDOWN_GRACE_SECS", default_value_t = 30)] pub shutdown_grace_secs: u64, diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 33f5bd67..e6a3c0eb 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -255,7 +255,7 @@ impl Db { &self.pool } - #[cfg(test)] + #[cfg(any(test, feature = "test-harness"))] pub fn for_testing(pool: PgPool) -> Self { Self { pool } } @@ -264,7 +264,7 @@ impl Db { /// provisions an empty per-test database, so DB-backed tests must run this /// before seeding. Reuses the production `migrate()` path (the advisory lock /// is harmless on an isolated test DB and migrations are idempotent). - #[cfg(test)] + #[cfg(any(test, feature = "test-harness"))] pub(crate) async fn run_migrations(&self) -> Result<()> { self.migrate().await } diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index a5c367e9..73d42fa1 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -33,7 +33,7 @@ pub struct RepoStore { } impl RepoStore { - #[cfg(test)] + #[cfg(any(test, feature = "test-harness"))] pub fn for_testing(repos_dir: PathBuf, pool: PgPool) -> Self { Self { repos_dir, diff --git a/crates/gitlawb-node/src/lib.rs b/crates/gitlawb-node/src/lib.rs new file mode 100644 index 00000000..bbcfd484 --- /dev/null +++ b/crates/gitlawb-node/src/lib.rs @@ -0,0 +1,7593 @@ +// Crate builds as both a library (`gitlawb_node`) and the `gitlawb-node` binary. +// The library exposes the boot surface (build_router, AppState, Config, +// migrations) so the out-of-crate deny-harness integration crate can spawn a +// real node; `src/main.rs` is a thin shim over `run()`. Modules the harness +// reaches are `pub`; the rest stay crate-private. +pub mod api; +mod arweave; +pub mod auth; +mod bootstrap; +mod cert; +pub mod config; +pub mod db; +mod encrypted_pin; +pub mod error; +pub mod git; +mod graphql; +mod icaptcha; +mod ipfs_pin; +mod metrics; +mod operator; +mod p2p; +mod pinata; +mod rate_limit; +pub mod server; +pub mod state; +mod sync; +#[cfg(feature = "test-harness")] +#[doc(hidden)] +pub mod test_harness; +#[cfg(test)] +mod test_support; +mod visibility; +mod webhooks; + +use anyhow::{anyhow, Context, Result}; +use axum::extract::State; +use axum::http::StatusCode; +use axum::response::IntoResponse; +use axum::routing::get; +use axum::{Json, Router}; +use clap::Parser; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use tokio::net::TcpListener; +use tokio::sync::watch; +use tracing::{info, warn}; + +use gitlawb_core::http_sig::sign_request; +use gitlawb_core::identity::Keypair; + +pub use config::Config; +use db::Db; +pub use state::AppState; + +#[derive(Clone)] +struct DegradedState { + node_did: String, + db_startup: Arc, +} + +/// Two independent counters with no cross-field invariant — atomics, not a +/// lock, so the retry loop and the degraded handlers never contend. +#[derive(Default)] +struct DbStartupStatus { + attempts: AtomicU64, + next_retry_secs: AtomicU64, +} + +/// Boot and run the node to completion. The `gitlawb-node` binary is a thin +/// `#[tokio::main]` shim over this; keeping the body here (not in `main.rs`) +/// lets the library own the full module tree so integration tests can link it. +pub async fn run() -> Result<()> { + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::from_default_env() + .add_directive("gitlawb_node=debug".parse().unwrap()) + .add_directive("tower_http=info".parse().unwrap()), + ) + .init(); + + let mut config = Config::parse(); + + // Merge the embedded seed list of public network nodes into the runtime + // bootstrap peers. Operators can opt out via GITLAWB_BOOTSTRAP_DISABLE_SEEDS. + bootstrap::merge_seeds(&mut config); + + if !config.public_read { + warn!( + "GITLAWB_PUBLIC_READ=false is reserved; per-repository private-read enforcement is not wired in alpha" + ); + } + + // Load or generate the node's identity keypair + let keypair = load_or_create_keypair(&config)?; + // The boot sweep spares young `.recovering.` claims (they may gate a + // live round); this one delayed re-sweep clears any spared orphan once + // it has aged past the liveness floor, ~12s into uptime, so aged claim + // residue cannot linger beside a healthy final (F2). Fire-and-forget. + spawn_delayed_claim_resweep(config.resolved_key_path()); + let node_did = keypair.did(); + + // One-time metrics init. Must run before any handler that calls into + // `metrics::record_*` so the registry exists when the first event fires. + // Safe to call even when GITLAWB_METRICS_ADDR is unset — those helpers + // are simply no-ops until something reads from the registry. + metrics::init(env!("CARGO_PKG_VERSION"), &node_did.to_string()); + + info!("╔══════════════════════════════════════════╗"); + info!( + "║ gitlawb node v{} ║", + env!("CARGO_PKG_VERSION") + ); + info!("╚══════════════════════════════════════════╝"); + // Process-wide shutdown signal. One sender lives in AppState (cloned + // into every handler); main() keeps a clone and flips it on SIGINT + // or SIGTERM. Tasks that hold a watch::Receiver get notified at + // their next await point. + let (shutdown_tx, _shutdown_rx_for_main) = watch::channel(false); + spawn_shutdown_signal(shutdown_tx.clone()); + + info!(did = %node_did, "node identity"); + info!(addr = %config.bind_addr(), "binding HTTP listener"); + + // Bind HTTP once, before dependency initialization, and keep this socket + // for the life of the process. The degraded server accepts on a dup of the + // same socket, so the degraded→full handoff never closes the port: while + // the full server initializes, connections queue in the shared backlog + // instead of being refused. + let listener = TcpListener::bind(config.bind_addr()) + .await + .with_context(|| format!("failed to bind to {}", config.bind_addr()))?; + let full_std_listener = listener.into_std()?; + let degraded_listener = TcpListener::from_std( + full_std_listener + .try_clone() + .context("failed to clone HTTP listener for degraded server")?, + )?; + + // Metrics must stay observable during a database outage — the degraded + // window is exactly when dashboards need data — so this listener starts + // before the DB connects. + let metrics_handle = if !config.metrics_addr.is_empty() { + match spawn_metrics_server(&config.metrics_addr, shutdown_tx.subscribe()).await { + Ok(handle) => { + info!(addr = %config.metrics_addr, "metrics endpoint listening"); + Some(handle) + } + Err(e) => { + warn!(err = %e, addr = %config.metrics_addr, "failed to start metrics endpoint — continuing without"); + None + } + } + } else { + info!("metrics endpoint disabled (GITLAWB_METRICS_ADDR not set)"); + None + }; + + let db_startup = Arc::new(DbStartupStatus::default()); + let (db_ready_tx, db_ready_rx) = watch::channel(false); + let mut degraded_handle = tokio::spawn(run_degraded_server( + degraded_listener, + node_did.to_string(), + Arc::clone(&db_startup), + db_ready_rx, + shutdown_tx.subscribe(), + )); + + // Connect to PostgreSQL database. A transient outage or bad secret should + // not crash-loop the process and hammer the database provider; permanent + // misconfiguration surfaces through error-level logs and the /ready check. + let db = tokio::select! { + db = connect_db_with_retry(&config, Arc::clone(&db_startup), shutdown_tx.subscribe()) => { + match db { + Some(db) => db, + None => { + // Shutdown requested while waiting for the database. The + // degraded server only serves one-shot 503s — abort it + // rather than drain, so a slow client can't stall exit. + degraded_handle.abort(); + return Ok(()); + } + } + } + degraded = &mut degraded_handle => { + if *shutdown_tx.borrow() { + return Ok(()); + } + return match degraded { + Ok(Ok(())) => Err(anyhow!("degraded HTTP server stopped before database became ready")), + Ok(Err(err)) => Err(err.context("degraded HTTP server failed")), + Err(err) => Err(anyhow!("degraded HTTP server task failed: {err}")), + }; + } + }; + + // Flip the degraded server into graceful shutdown, but do NOT await the + // drain: one slow in-flight request must not delay the full server, and + // the shared socket means there is no port gap to cover. The drain + // finishes (and logs) in the background. + db_ready_tx.send(true).ok(); + tokio::spawn(async move { + match degraded_handle.await { + Ok(Ok(())) => {} + Ok(Err(err)) => warn!(err = %err, "degraded HTTP server exited with error"), + Err(err) => warn!(err = %err, "degraded HTTP server task failed"), + } + }); + info!(addr = %config.bind_addr(), "database ready; starting full HTTP server"); + + // Prune peer rows that point back at this node (stale self-loop entries) + if let Some(public_url) = config.public_url.as_deref() { + match db.prune_self_peers(public_url).await { + Ok(0) => {} + Ok(n) => info!(removed = n, public_url, "pruned self-loop peer rows"), + Err(e) => warn!(err = %e, "prune_self_peers failed (non-fatal)"), + } + } + + // Prune peer rows with non-public hosts (loopback/private/internal) that + // were injected via the unauthenticated announce route — they poison the + // sync-notify fan-out (SSRF + crowding out real peers). + match db.prune_non_public_peers().await { + Ok(0) => {} + Ok(n) => info!(removed = n, "pruned non-public (poisoned) peer rows"), + Err(e) => warn!(err = %e, "prune_non_public_peers failed (non-fatal)"), + } + + // Ensure repos directory exists + std::fs::create_dir_all(&config.repos_dir).context("failed to create repos directory")?; + + // Start libp2p swarm (if p2p_port > 0) + let p2p_handle = if config.p2p_port > 0 { + let bootstrap_addrs = config + .p2p_bootstrap + .iter() + .filter_map(|s| s.parse().ok()) + .collect(); + let shutdown_rx = shutdown_tx.subscribe(); + match p2p::start( + &node_did.to_string(), + config.p2p_port, + bootstrap_addrs, + Arc::clone(&db), + config.auto_sync, + shutdown_rx, + ) + .await + { + Ok(handle) => { + info!(port = config.p2p_port, peer_id = %handle.local_peer_id, "libp2p swarm started"); + Some(Arc::new(handle)) + } + Err(e) => { + tracing::warn!(err = %e, "failed to start libp2p swarm — continuing without p2p"); + None + } + } + } else { + info!("p2p disabled (p2p_port = 0)"); + None + }; + + // Shared no-redirect HTTP client. See build_http_client for the SSRF rationale. + let http_client = Arc::new(build_http_client()?); + + let (ref_update_tx, _) = tokio::sync::broadcast::channel::(256); + let (task_event_tx, _) = tokio::sync::broadcast::channel::(256); + + let graphql_schema = Arc::new(graphql::build_schema( + Arc::clone(&db), + ref_update_tx.clone(), + task_event_tx.clone(), + )); + + let machine_id = std::env::var("FLY_MACHINE_ID").ok(); + if let Some(ref mid) = machine_id { + info!(" fly machine: {mid}"); + } + + // Initialize Tigris S3 client if bucket is configured + let tigris = if !config.tigris_bucket.is_empty() { + match git::tigris::TigrisClient::new(&config.tigris_bucket).await { + Ok(client) => { + info!(bucket = %config.tigris_bucket, "tigris storage enabled"); + Some(client) + } + Err(e) => { + tracing::warn!(err = %e, "failed to initialize Tigris client — using local-only storage"); + None + } + } + } else { + info!("tigris storage disabled (no bucket configured)"); + None + }; + + let repo_store = + git::repo_store::RepoStore::new(config.repos_dir.clone(), tigris, db.pool().clone()); + + // Per-DID limiter for the creation endpoints. Keyed on the authenticated + // DID (attacker-varied), so bound its key set to cap memory. + let rate_limiter = + rate_limit::RateLimiter::new_bounded(10, std::time::Duration::from_secs(3600), 200_000); + + // Per-client-IP flood brake for the creation endpoints. The per-DID limiter + // above is bypassed by a DID farm (one throwaway did:key per repo), which is + // exactly how the recurring spam-repo floods get past both it and the + // iCaptcha gate. Keyed on the resolved client IP so a single-source flood is + // capped regardless of how many identities it mints. Sized well above any + // legitimate per-IP creation rate; GITLAWB_CREATE_RATE_LIMIT overrides, 0 + // disables. Bounded key set — the key is a client-influenced IP. + let create_limit = std::env::var("GITLAWB_CREATE_RATE_LIMIT") + .ok() + .and_then(|v| v.trim().parse::().ok()) + .unwrap_or(120); + let create_ip_rate_limiter = rate_limit::RateLimiter::new_bounded( + create_limit, + std::time::Duration::from_secs(3600), + 200_000, + ); + if create_limit == 0 { + tracing::warn!("GITLAWB_CREATE_RATE_LIMIT=0 — per-IP creation rate limiting disabled"); + } + + // Push-path flood brake: max git-receive-pack requests per client IP per + // hour (counts both the info/refs advertisement and the push POST). Sized + // for heavy agent automation while still stopping flood traffic (the June + // 2026 attack pushed several times per second per IP). GITLAWB_PUSH_RATE_LIMIT + // overrides; 0 disables. Bounded key set — the key is a client-influenced IP. + let push_limit = std::env::var("GITLAWB_PUSH_RATE_LIMIT") + .ok() + .and_then(|v| v.trim().parse::().ok()) + .unwrap_or(600); + let push_rate_limiter = rate_limit::RateLimiter::new_bounded( + push_limit, + std::time::Duration::from_secs(3600), + 200_000, + ); + if push_limit == 0 { + tracing::warn!("GITLAWB_PUSH_RATE_LIMIT=0 — per-IP push rate limiting disabled"); + } + + // Which forwarded header the edge is trusted to set. Default None (trust + // nothing, key on the socket peer). Fly nodes set GITLAWB_TRUSTED_PROXY=fly; + // a node behind Caddy/NGINX sets it to x-forwarded-for. + let push_limiter_trust = rate_limit::TrustedProxy::from_env_value( + &std::env::var("GITLAWB_TRUSTED_PROXY").unwrap_or_default(), + ); + tracing::info!(trust = ?push_limiter_trust, push_limit, "push rate limiter configured"); + + // Peer-sync flood brakes, keyed on the resolved client IP (per-DID is useless + // here — a did:key farm self-registers). Two buckets so an unsigned notify + // flood can't drain the signed trigger caller's quota (#82). Bounded key sets + // (the key is a client-influenced IP); 0 disables each. + let sync_trigger_rate_limiter = rate_limit::RateLimiter::new_bounded( + config.sync_trigger_rate_limit, + std::time::Duration::from_secs(3600), + 200_000, + ); + let peer_write_rate_limiter = rate_limit::RateLimiter::new_bounded( + config.peer_write_rate_limit, + std::time::Duration::from_secs(3600), + 200_000, + ); + if config.sync_trigger_rate_limit == 0 { + tracing::warn!( + "GITLAWB_SYNC_TRIGGER_RATE_LIMIT=0 — /sync/trigger IP rate limiting disabled" + ); + } + if config.peer_write_rate_limit == 0 { + tracing::warn!("GITLAWB_PEER_WRITE_RATE_LIMIT=0 — peer-write IP rate limiting disabled"); + } + + // Initialize the iCaptcha proof gate (inert unless ICAPTCHA_MODE is set). + icaptcha::init().await; + + let state = AppState { + config: Arc::new(config.clone()), + db, + node_did: node_did.clone(), + node_keypair: Arc::new(keypair), + p2p: p2p_handle, + http_client, + ref_update_tx, + task_event_tx, + graphql_schema, + machine_id, + repo_store, + rate_limiter, + create_ip_rate_limiter, + push_rate_limiter, + push_limiter_trust, + sync_trigger_rate_limiter, + peer_write_rate_limiter, + shutdown_tx: shutdown_tx.clone(), + }; + + // Periodic peer-count poll for the metrics gauge. If p2p is disabled + // we still set the gauge to 0 so dashboards don't show "no data". + { + let p2p_for_metrics = state.p2p.clone(); + let mut shutdown_rx = state.subscribe_shutdown(); + tokio::spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_secs(15)); + loop { + tokio::select! { + _ = interval.tick() => { + let count = match &p2p_for_metrics { + Some(h) => h.status().await.map(|s| s.connected_peers).unwrap_or(0), + None => 0, + }; + metrics::set_peers_connected(count as i64); + } + _ = shutdown_rx.changed() => { + if *shutdown_rx.borrow() { + return; + } + } + } + } + }); + } + + // Periodic cleanup of expired rate limit entries + consumed-proof ledger + { + let rl = state.rate_limiter.clone(); + let create_ip_rl = state.create_ip_rate_limiter.clone(); + let push_rl = state.push_rate_limiter.clone(); + let sync_trigger_rl = state.sync_trigger_rate_limiter.clone(); + let peer_write_rl = state.peer_write_rate_limiter.clone(); + let db = state.db.clone(); + let mut shutdown_rx = state.subscribe_shutdown(); + tokio::spawn(async move { + loop { + tokio::select! { + _ = tokio::time::sleep(std::time::Duration::from_secs(300)) => { + rl.cleanup().await; + create_ip_rl.cleanup().await; + push_rl.cleanup().await; + sync_trigger_rl.cleanup().await; + peer_write_rl.cleanup().await; + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or(0); + if let Err(e) = db.sweep_expired_proofs(now).await { + tracing::warn!(err = %e, "failed to sweep expired iCaptcha proofs"); + } + } + _ = shutdown_rx.changed() => { + if *shutdown_rx.borrow() { + break; + } + } + } + } + }); + } + + let router = server::build_router(state.clone()); + // Re-register the socket bound at startup — same fd, so there was never a + // moment with the port closed between the degraded and full servers. + let listener = TcpListener::from_std(full_std_listener) + .context("failed to re-register HTTP listener with the runtime")?; + + info!("✓ node started — did:{}", node_did); + info!(" repos dir: {}", config.repos_dir.display()); + info!( + " database: PostgreSQL ({})", + &config.database_url.split('@').next_back().unwrap_or("?") + ); + + // Publish our DID record to the Kademlia DHT shortly after startup + if let Some(p2p) = &state.p2p { + let did_record = p2p::DidRecord { + did: node_did.to_string(), + http_url: config.public_url.clone().unwrap_or_default(), + peer_id: p2p.local_peer_id.to_string(), + p2p_port: config.p2p_port, + timestamp: chrono::Utc::now().to_rfc3339(), + }; + let p2p_clone = Arc::clone(p2p); + let mut shutdown_rx = state.subscribe_shutdown(); + tokio::spawn(async move { + // Small delay so Kademlia can find peers first + tokio::select! { + _ = tokio::time::sleep(std::time::Duration::from_secs(5)) => {} + _ = shutdown_rx.changed() => return, + } + p2p_clone.put_did(did_record).await; + info!("DID record published to Kademlia DHT"); + }); + } + + // Spawn background gossip: announce to bootstrap peers, then ping known peers periodically + { + let gossip_state = state.clone(); + let bootstrap_peers = config.bootstrap_peers.clone(); + let shutdown_rx = state.subscribe_shutdown(); + tokio::spawn(async move { + gossip_task(gossip_state, bootstrap_peers, shutdown_rx).await; + }); + } + + // Start multi-node sync worker if auto_sync is enabled + if config.auto_sync { + sync::start( + Arc::clone(&state.db), + Arc::clone(&state.config), + Arc::clone(&state.node_keypair), + state.subscribe_shutdown(), + ); + info!("auto-sync worker started"); + } + + // On-chain operator setup: verify stake + spawn heartbeat loop + if !state.config.contract_node_staking.is_empty() + && !state.config.operator_private_key.is_empty() + { + match build_operator_client(&state.config, &state.node_did.to_string()) { + Ok(client) => match operator::startup_check(&client).await { + Ok(_) => { + let arc_client = Arc::new(client); + arc_client.spawn_heartbeat_loop(state.subscribe_shutdown()); + } + Err(e) => { + if state.config.operator_strict_mode { + return Err(e.context("strict-mode operator check failed")); + } + tracing::warn!(err = %e, "operator startup check failed — continuing without heartbeat loop"); + } + }, + Err(e) => { + if state.config.operator_strict_mode { + return Err(e.context("strict-mode: failed to build operator client")); + } + tracing::warn!(err = %e, "operator client could not be built — continuing without PoS"); + } + } + } else { + info!("on-chain PoS disabled (GITLAWB_CONTRACT_NODE_STAKING or GITLAWB_OPERATOR_PRIVATE_KEY unset)"); + } + + // axum's `with_graceful_shutdown` begins draining in-flight requests once + // the shutdown watch flips. That drain is otherwise unbounded, so we bound + // it by `grace`: the closure fires `armed_tx` the instant it observes the + // signal, and `drive_serve_with_grace` abandons the drain if it has not + // finished `grace` after that moment. The clock starts at the signal, not at + // server start, so total uptime is never bounded. + let shutdown_signal_for_axum = state.subscribe_shutdown(); + let grace = std::time::Duration::from_secs(config.shutdown_grace_secs); + info!(grace_secs = config.shutdown_grace_secs, "axum server ready"); + + let (armed_tx, armed_rx) = tokio::sync::oneshot::channel::<()>(); + + // `into_make_service_with_connect_info` exposes the socket peer address as + // `ConnectInfo` so the push limiter can key on the real client + // when no trusted proxy header applies (see `rate_limit::client_key`). + let serve = axum::serve( + listener, + router.into_make_service_with_connect_info::(), + ) + .with_graceful_shutdown(async move { + let mut rx = shutdown_signal_for_axum; + // Wait until the watcher flips to true, then return so axum + // can begin draining. + while !*rx.borrow_and_update() { + if rx.changed().await.is_err() { + // Sender dropped — treat as shutdown. + break; + } + } + // Start the grace clock at the signal, right before axum drains. + let _ = armed_tx.send(()); + }); + + // Race the drain against the grace deadline (see `drive_serve_with_grace`): + // the clock starts when the closure above arms `armed_rx`, i.e. at the + // signal, and the drain is abandoned if it outlasts `grace`. + let (serve_result, grace_expired) = drive_serve_with_grace(serve, armed_rx, grace).await; + + if grace_expired { + warn!( + grace_secs = config.shutdown_grace_secs, + "shutdown grace expired; abandoning in-flight requests" + ); + } + + // Server has stopped accepting new connections and drained in-flight + // requests (or the grace deadline fired). Tear the rest of the system down. + info!("HTTP server stopped, beginning process shutdown"); + if let Some(h) = metrics_handle { + h.abort(); + } + serve_result?; + info!("clean exit"); + Ok(()) +} + +/// Drive the HTTP `serve` future to completion, bounding the post-signal drain +/// by `grace`. `armed` resolves the instant the shutdown signal fires (the serve +/// future's graceful-shutdown closure sends on it), so the grace clock starts at +/// the signal, not at server start — total uptime is never bounded. Returns the +/// serve result plus whether the deadline fired (which abandons in-flight +/// requests so teardown can proceed). +async fn drive_serve_with_grace( + serve: F, + armed: tokio::sync::oneshot::Receiver<()>, + grace: std::time::Duration, +) -> (std::io::Result<()>, bool) +where + F: std::future::IntoFuture>, +{ + let serve = serve.into_future(); + tokio::select! { + result = serve => (result, false), + _ = async { + // Park until the signal arms the clock, then bound the drain. + let _ = armed.await; + tokio::time::sleep(grace).await; + } => (Ok(()), true), + } +} + +fn spawn_shutdown_signal(tx: watch::Sender) { + tokio::spawn(async move { + #[cfg(unix)] + { + use tokio::signal::unix::{signal as unix_signal, SignalKind}; + let mut sigterm = + unix_signal(SignalKind::terminate()).expect("install SIGTERM handler"); + let mut sigint = unix_signal(SignalKind::interrupt()).expect("install SIGINT handler"); + tokio::select! { + _ = sigterm.recv() => info!("SIGTERM received, shutting down"), + _ = sigint.recv() => info!("SIGINT received, shutting down"), + } + } + #[cfg(not(unix))] + { + use tokio::signal; + let _ = signal::ctrl_c().await; + info!("Ctrl-C received, shutting down"); + } + tx.send(true).ok(); + }); +} + +async fn connect_db_with_retry( + config: &Config, + db_startup: Arc, + mut shutdown_rx: watch::Receiver, +) -> Option> { + let initial_retry_secs = config.db_retry_initial_secs; + let max_retry_secs = config.db_retry_max_secs.max(initial_retry_secs); + let acquire_timeout = std::time::Duration::from_secs(config.db_acquire_timeout_secs); + let attempt_timeout = std::time::Duration::from_secs(config.db_connect_timeout_secs); + let mut attempts = 0_u64; + + loop { + if *shutdown_rx.borrow() { + return None; + } + + attempts = attempts.saturating_add(1); + db_startup.attempts.store(attempts, Ordering::Relaxed); + + // Bound the whole attempt, not just the pool connect: migrations + // block on a cross-instance advisory lock, and an unbounded wait + // there would wedge this loop — no retries, no logs, no recovery. + // Timing out and retrying is safe; migrations are idempotent. + let attempt = match tokio::time::timeout( + attempt_timeout, + Db::connect( + &config.database_url, + config.db_max_connections, + acquire_timeout, + ), + ) + .await + { + Ok(result) => result, + Err(_) => Err(anyhow!( + "connect + migrate attempt exceeded {}s (GITLAWB_DB_CONNECT_TIMEOUT_SECS); \ + is another instance holding the migration lock?", + attempt_timeout.as_secs() + )), + }; + + match attempt { + Ok(db) => { + info!(attempts, "database connection established"); + return Some(Arc::new(db)); + } + Err(err) => { + // A bad DATABASE_URL or rejected credentials won't heal on + // their own. Still retry (exiting would crash-loop and hammer + // the provider — and take liveness down with it), but log at + // error level and skip straight to the maximum backoff; the + // /ready health check is what surfaces this to deploys. + let permanent = is_likely_permanent_db_error(&err); + let retry_secs = if permanent { + max_retry_secs + } else { + database_retry_delay_secs(initial_retry_secs, max_retry_secs, attempts) + }; + db_startup + .next_retry_secs + .store(retry_secs, Ordering::Relaxed); + if permanent { + tracing::error!( + attempts, + retry_secs, + err = %err, + "database rejected our configuration (bad DATABASE_URL or credentials?) — retrying, but operator action is likely required" + ); + } else { + warn!( + attempts, + retry_secs, + err = %err, + "database unavailable during startup; retrying" + ); + } + + tokio::select! { + _ = tokio::time::sleep(std::time::Duration::from_secs(retry_secs)) => {} + changed = shutdown_rx.changed() => { + if changed.is_err() || *shutdown_rx.borrow() { + return None; + } + } + } + } + } + } +} + +/// Errors that indicate misconfiguration rather than a transient outage: a +/// malformed DATABASE_URL, or a server that answered and rejected us — +/// Postgres error class 28xxx (invalid authorization) or 3D000 (database +/// does not exist). Best-effort: an error that anyhow can't downcast back to +/// sqlx just counts as transient. +fn is_likely_permanent_db_error(err: &anyhow::Error) -> bool { + match err.downcast_ref::() { + Some(sqlx::Error::Configuration(_)) => true, + Some(sqlx::Error::Database(db)) => db + .code() + .map(|c| c.starts_with("28") || c.starts_with("3D")) + .unwrap_or(false), + _ => false, + } +} + +fn database_retry_delay_secs(initial_secs: u64, max_secs: u64, attempts: u64) -> u64 { + // The exponent bound only keeps the u32 cast safe — max_secs is the real + // (operator-configurable) cap, and saturating math handles overflow. + let exponent = attempts.saturating_sub(1).min(63) as u32; + initial_secs + .saturating_mul(2_u64.saturating_pow(exponent)) + .min(max_secs) +} + +async fn run_degraded_server( + listener: TcpListener, + node_did: String, + db_startup: Arc, + mut db_ready_rx: watch::Receiver, + mut shutdown_rx: watch::Receiver, +) -> Result<()> { + let addr = listener.local_addr().ok(); + let router = build_degraded_router(node_did, db_startup); + info!(?addr, "degraded HTTP server ready"); + + axum::serve(listener, router) + .with_graceful_shutdown(async move { + // wait_for resolves on predicate-true or sender-drop; either way + // this phase is over. + tokio::select! { + _ = db_ready_rx.wait_for(|ready| *ready) => {} + _ = shutdown_rx.wait_for(|stop| *stop) => {} + } + }) + .await?; + + Ok(()) +} + +fn build_degraded_router(node_did: String, db_startup: Arc) -> Router { + let state = DegradedState { + node_did, + db_startup, + }; + // Everything answers 503 with the same body — including /health and + // /ready, so peer pings (which treat any 2xx /health as alive) and + // uptime monitors correctly see a node that cannot serve traffic. + // `/` additionally carries the node identity for probing peers. + Router::new() + .route("/", get(degraded_node_info)) + .fallback(degraded_unavailable) + .with_state(state) +} + +/// One source of truth for the degraded 503 body, sharing the error +/// vocabulary with error.rs so clients see the same code/message for +/// "database unavailable" regardless of which phase produced it. +fn degraded_body(db_startup: &DbStartupStatus) -> serde_json::Value { + serde_json::json!({ + "status": "degraded", + "database": "initializing", + "error": error::DB_UNAVAILABLE_CODE, + "message": error::DB_UNAVAILABLE_MESSAGE, + "db_attempts": db_startup.attempts.load(Ordering::Relaxed), + "db_next_retry_secs": db_startup.next_retry_secs.load(Ordering::Relaxed), + }) +} + +async fn degraded_node_info(State(state): State) -> impl IntoResponse { + let mut body = degraded_body(&state.db_startup); + if let Some(obj) = body.as_object_mut() { + obj.insert("name".into(), "gitlawb-node".into()); + obj.insert("version".into(), env!("CARGO_PKG_VERSION").into()); + obj.insert("did".into(), state.node_did.clone().into()); + } + (StatusCode::SERVICE_UNAVAILABLE, Json(body)) +} + +async fn degraded_unavailable(State(state): State) -> impl IntoResponse { + ( + StatusCode::SERVICE_UNAVAILABLE, + Json(degraded_body(&state.db_startup)), + ) +} + +/// Spawn a small axum router that exposes only `GET /metrics` on its own +/// listener. Returns the JoinHandle so `main()` can abort it on shutdown. +/// This is deliberately separate from the main router so the metrics port +/// can be firewalled differently from the API port — bind to localhost +/// or a private interface only. +async fn spawn_metrics_server( + addr: &str, + mut shutdown_rx: watch::Receiver, +) -> Result> { + use axum::{response::IntoResponse, routing::get, Router}; + + async fn metrics_handler() -> impl IntoResponse { + match metrics::encode() { + Ok(body) => ( + axum::http::StatusCode::OK, + [( + axum::http::header::CONTENT_TYPE, + "text/plain; version=0.0.4; charset=utf-8", + )], + body, + ), + Err(e) => ( + axum::http::StatusCode::INTERNAL_SERVER_ERROR, + [( + axum::http::header::CONTENT_TYPE, + "text/plain; charset=utf-8", + )], + format!("metrics encode error: {e}"), + ), + } + } + + let listener = TcpListener::bind(addr) + .await + .with_context(|| format!("failed to bind metrics listener to {addr}"))?; + let app = Router::new().route("/metrics", get(metrics_handler)); + + let handle = tokio::spawn(async move { + if let Err(e) = axum::serve(listener, app) + .with_graceful_shutdown(async move { + while !*shutdown_rx.borrow_and_update() { + if shutdown_rx.changed().await.is_err() { + break; + } + } + }) + .await + { + warn!(err = %e, "metrics server exited with error"); + } + }); + Ok(handle) +} + +fn build_operator_client( + config: &config::Config, + node_did: &str, +) -> Result { + use alloy::primitives::Address; + use std::str::FromStr; + + let contract_address = Address::from_str(&config.contract_node_staking) + .with_context(|| format!("invalid contract address: {}", config.contract_node_staking))?; + + let cfg = operator::OperatorConfig { + rpc_url: config.chain_rpc_url.clone(), + private_key: config.operator_private_key.clone(), + contract_address, + node_did: node_did.to_string(), + heartbeat_interval: std::time::Duration::from_secs(config.heartbeat_interval_hours * 3600), + strict_mode: config.operator_strict_mode, + }; + Ok(operator::OperatorClient::new(cfg)) +} + +/// Announce to bootstrap peers on startup, then periodically ping all known peers. +async fn gossip_task( + state: AppState, + bootstrap_peers: Vec, + mut shutdown_rx: tokio::sync::watch::Receiver, +) { + // If shutdown arrives during the initial delay, exit before announcing. + tokio::select! { + _ = tokio::time::sleep(std::time::Duration::from_secs(2)) => {} + _ = shutdown_rx.changed() => { + if *shutdown_rx.borrow() { + info!("gossip: shutdown during startup delay, exiting"); + return; + } + } + } + + // Reuse the shared no-redirect client for every gossip outbound call (the + // bootstrap announce POST and the periodic peer /health ping). Peer URLs are + // attacker-influenceable, so a 3xx to a private address must not be followed. + // Do NOT fall back to reqwest::Client::new(): its default follows redirects + // and would reintroduce the SSRF closed here (#93). + let client = state.http_client.clone(); + let my_did = state.node_did.to_string(); + let my_url = state.config.public_url.clone().unwrap_or_default(); + + // Announce ourselves to each bootstrap peer + for peer_url in &bootstrap_peers { + // Cooperative shutdown between peers — a slow peer shouldn't + // block the node exiting. + if *shutdown_rx.borrow() { + info!("gossip: shutdown signalled during peer announce, exiting"); + return; + } + let path = "/api/v1/peers/announce"; + let announce_url = format!("{}{}", peer_url.trim_end_matches('/'), path); + let body = serde_json::json!({ + "did": my_did.clone(), + "http_url": my_url.clone(), + }); + let body_bytes = match serde_json::to_vec(&body) { + Ok(bytes) => bytes, + Err(e) => { + tracing::warn!(err = %e, "failed to serialize peer announce body"); + continue; + } + }; + let signed = sign_request(state.node_keypair.as_ref(), "POST", path, &body_bytes); + // Per-request timeout inside the loop; do not let one hung peer + // block others. The request itself is a normal tokio future so + // it's cancel-safe on shutdown. + match tokio::time::timeout( + std::time::Duration::from_secs(5), + client + .post(&announce_url) + .header("Content-Type", "application/json") + .header("Content-Digest", signed.content_digest) + .header("Signature-Input", signed.signature_input) + .header("Signature", signed.signature) + .body(body_bytes) + .send(), + ) + .await + { + Ok(Ok(resp)) => { + if resp.status().is_success() { + if let Ok(json) = resp.json::().await { + // Add them back to our peer list + if let (Some(their_did), Some(their_url)) = ( + json.get("node_did").and_then(|v| v.as_str()), + json.get("node_url").and_then(|v| v.as_str()), + ) { + if !their_url.is_empty() { + let _ = state.db.upsert_peer(their_did, their_url).await; + tracing::info!(did = %their_did, url = %their_url, "bootstrap peer added"); + } + } + } + } + } + Ok(Err(e)) => { + tracing::warn!(url = %announce_url, err = %e, "failed to announce to bootstrap peer") + } + Err(_) => tracing::warn!(url = %announce_url, "bootstrap peer announce timed out (5s)"), + } + } + + // Periodic ping every 5 minutes — exit on shutdown. + let mut interval = tokio::time::interval(std::time::Duration::from_secs(300)); + loop { + tokio::select! { + _ = interval.tick() => { + let peers = match state.db.list_peers().await { + Ok(p) => p, + Err(_) => continue, + }; + for peer in peers { + let ok = ping_peer_health(&client, &peer.http_url).await; + let _ = state.db.mark_peer_ping(&peer.did, ok).await; + } + } + _ = shutdown_rx.changed() => { + if *shutdown_rx.borrow() { + info!("gossip task: shutdown signal received, exiting"); + return; + } + } + } + } +} + +/// Build the shared node HTTP client used for every outbound fan-out (sync +/// trigger, profile/repo fetches, gossip announce + peer pings). +/// +/// No redirects: peer URLs are attacker-influenceable, so a `3xx` to a private +/// address must not be followed (SSRF guard, #78/#93). Do NOT replace with +/// `reqwest::Client::new()` — its default follows redirects. Kept as a named +/// builder so tests bind the redirect guarantee to the real client the node +/// runs, not a hand-rolled equivalent. +fn build_http_client() -> reqwest::Result { + reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(10)) + .redirect(reqwest::redirect::Policy::none()) + .build() +} + +/// Ping a peer's `/health` endpoint and report whether it answered 2xx. +/// +/// Takes the client by reference so callers supply the shared, no-redirect +/// `state.http_client`. Peer URLs are attacker-influenceable, so a `3xx` to a +/// private address must not be followed. Do NOT call this with a bare +/// `reqwest::Client::new()`: its default follows redirects and would +/// reintroduce the SSRF this guards against (#93). +async fn ping_peer_health(client: &reqwest::Client, http_url: &str) -> bool { + let url = format!("{}/health", http_url.trim_end_matches('/')); + client + .get(&url) + .send() + .await + .map(|r| r.status().is_success()) + .unwrap_or(false) +} + +/// Persist a just-created identity key, removing the file if the write failed +/// (#194, F1). `create_new` makes the inode appear before the PEM is flushed, so a +/// failed `write_all` (ENOSPC/EIO/quota) leaves an empty or partial PEM behind; +/// every later start would then take the `exists()` branch and re-parse that +/// corrupt file forever, exiting `invalid PEM key` instead of regenerating. Remove +/// it on failure so the next start starts clean. +fn write_key_or_cleanup(path: &std::path::Path, write_result: std::io::Result<()>) -> Result<()> { + write_result.map_err(|e| { + // Best-effort: if removal also fails there is nothing more to do, and the + // original write error is the one worth surfacing. + let _ = std::fs::remove_file(path); + anyhow::Error::new(e).context(format!("failed to write key to {}", path.display())) + }) +} + +/// Verify an identity key file is not world/group-readable (#194, F2). A `chmod` +/// that failed or silently no-op'd (read-only mount, ACL mismatch) must not leave +/// a readable private key in use, so this is checked after any tightening attempt +/// and fails closed rather than logging a normal "loaded identity" path. +#[cfg(unix)] +fn ensure_key_mode_0600(path: &std::path::Path) -> Result<()> { + use std::os::unix::fs::PermissionsExt; + let mode = std::fs::metadata(path) + .with_context(|| format!("stat identity key {}", path.display()))? + .permissions() + .mode() + & 0o777; + if mode != 0o600 { + anyhow::bail!( + "identity key {} has mode {mode:o}, expected 0600 — refusing to use a \ + world/group-readable private key", + path.display() + ); + } + Ok(()) +} + +/// Secure a just-created, still-EMPTY key file before any PEM byte is written +/// (#194, U2): tighten to 0600, then verify with `ensure_key_mode_0600`. The +/// pair matters: `create_new`'s requested 0600 is narrowed by the process +/// umask (0277 lands 0400), so the tighten repairs that first; the verify +/// alone would falsely fail closed. On a mode-ignoring mount (vfat: chmod +/// silently no-ops returning Ok) the tighten changes nothing and the verify +/// fails CLOSED, so the private key never hits a group/other-readable file. +/// On failure the empty file is removed so a retry is not wedged on +/// `AlreadyExists`; removal is safe precisely because nothing has been +/// written yet (the split-phase rule protects only files holding PEM bytes). +/// `verify_fault` is the fault-injection hook (`Ok` in production). +#[cfg(unix)] +fn tighten_and_verify_created( + path: &std::path::Path, + verify_fault: fn() -> std::io::Result<()>, +) -> Result<()> { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) + .map_err(anyhow::Error::new) + .and_then(|()| verify_fault().map_err(anyhow::Error::new)) + .and_then(|()| ensure_key_mode_0600(path)) + .map_err(|e| { + let _ = std::fs::remove_file(path); + e.context(format!( + "could not secure just-created identity key {} to 0600", + path.display() + )) + }) +} + +/// Fsync the parent directory of `final_path` so the namespace change (a link +/// or a create into it) is durable before success is reported (#194). Unix-only: +/// opening a directory as a file is not portable. An empty or missing parent +/// resolves to `"."`. +#[cfg(unix)] +fn fsync_parent_dir(final_path: &std::path::Path) -> anyhow::Result<()> { + let dir = final_path + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or_else(|| std::path::Path::new(".")); + std::fs::File::open(dir) + .with_context(|| format!("open key directory {} to fsync", dir.display()))? + .sync_all() + .with_context(|| format!("fsync key directory {}", dir.display()))?; + Ok(()) +} + +/// Outcome of an atomic key publish. +enum KeyPublish { + /// We created the key file; it now holds the complete PEM. + Won, + /// Another concurrent start already created it; ours was discarded. + Lost, +} + +/// Wall-clock budget for reading a key another start may still be publishing. +/// On the atomic hard-link path (`publish_key_atomically`) a *present* key file +/// is always complete, so a read there succeeds on the first try. On the +/// hard-link fallback (`publish_key_fallback`) the final NAME becomes visible +/// while the PEM is still being written, so a losing/concurrent reader can +/// observe an empty/partial final; this deadline is the window it rides out +/// until the winner's write completes. Cross-host cache lag is a secondary +/// reason a present key may not read on the first attempt. It is a wall-clock +/// budget, NOT a fixed retry count, so a slow/stalled filesystem cannot starve a +/// losing start after an arbitrarily short (~100ms) window (#194). +const KEY_RACE_DEADLINE: std::time::Duration = std::time::Duration::from_secs(5); + +/// Minimum age (metadata mtime) before a `.recovering.` claim is sweepable +/// by a healthy boot (#194, G1). LIVENESS CONTRACT: a live recovery +/// HEARTBEATS every claim it holds (`spawn_claim_heartbeat`), refreshing +/// each mtime every quarter of this floor until release, so a live round's +/// claims NEVER age out, however long the round runs on a slow shared +/// volume. An AGED claim therefore means its recoverer stopped heartbeating +/// (crashed or killed): orphaned residue, claimable and sweepable (the +/// demotion waiter, by contrast, keys on claim-file ABSENCE, not age; see +/// `wait_for_recovery_claims`). A YOUNG +/// claim means live or recently dead, an ambiguity bounded by one floor +/// (the price of never stealing a live round). The 2x margin over +/// `KEY_RACE_DEADLINE` absorbs clock skew between writers on network +/// mounts. The gate matters because live claims are load-bearing: a +/// demoted publisher's `wait_for_recovery_claims` waits on them, and a third +/// start sweeping a live claim would cancel that wait mid-recovery, letting +/// the publisher re-load the pre-quarantine key while the recoverer +/// quarantines and republishes: two-sided divergence. An unreadable mtime is +/// treated as YOUNG (skip removal; fail safe). +/// +/// The same floor also gates which `.recovering.` claims are CLAIMABLE +/// (`list_recovery_claimables`), so a claim-crash state (a content-bad +/// final whose only claimable is a claim) becomes recoverable only after +/// this floor passes. That delay is bounded (~2x the deadline) and is the +/// price of never stealing a live recovery round. +const CLAIM_SWEEP_MIN_AGE: std::time::Duration = KEY_RACE_DEADLINE.saturating_mul(2); + +/// Retention window for forensic residue beside a healthy final: both +/// `.{stem}.superseded.` files and plain `.{stem}.quarantined.` files. A +/// superseded quarantine is kept for operator inspection after a lost +/// round, and a plain quarantine lingers as crash forensics (unparseable +/// bytes) or as aged history (a parseable key already past the adoption +/// floor, never resurrected); with no expiry either class accumulates +/// forever (every lost round or crash adds one file beside the final). 24 +/// hours is a generous post-mortem window while keeping growth bounded: +/// once a file in either class is older than this, the healthy-boot sweep +/// removes it (behind the same durability gate as every other removal in +/// `sweep_stale_markers`). The window is orders of magnitude above the +/// adoption floor (`CLAIM_SWEEP_MIN_AGE`), so no quarantine this sweep can +/// touch was still adoptable. An unreadable mtime reads as young (kept; +/// fail safe). +const SUPERSEDED_RETENTION: std::time::Duration = std::time::Duration::from_secs(24 * 60 * 60); + +/// How long the crash signature (a content-bad final beside a claimable) +/// must persist across consecutive observations before boot classifies a +/// crash (#194, G3). A single ~2ms poll sits well inside a LIVE fallback +/// publisher's mid-write window once chmod/stat/scheduling latency on shared +/// volumes is real, so a shorter grace lets a concurrent starter classify a +/// live publish as a crash and churn quarantines and identities. 250ms is +/// far above that latency yet far under the <2s recovery budget the tests +/// pin (250ms of persistence plus claim, quarantine, and regenerate stays +/// well under 2s). Any successful load or signature-free observation resets +/// the clock. +/// +/// Honest trade-off: on mounts where a live first publish's pre-write window +/// (the fallback's `create_new` through `write_all`) can itself exceed this +/// floor, a concurrent starter can still classify that live publish as +/// crashed. Every such path converges (the demotion commit check, the +/// post-claim re-parse, the no-clobber restore, the quarantine adoption), so +/// the cost is churn, not divergence; raising the floor trades that +/// probability against recovery latency after a real crash. +const CRASH_SIGNATURE_MIN_PERSIST: std::time::Duration = std::time::Duration::from_millis(250); + +/// Bounded number of temp-file names a publish will try before giving up. +/// Attempt 0 is the deterministic `.{stem}.tmp.{pid}.0`, so a stale temp left +/// by a crashed prior start with the same PID (a restarted container is +/// commonly PID 1 again) is skipped rather than wedging the restart (#194). +/// 64 names is far more than any plausible pile-up of stale temps plus live +/// concurrent publishers. Also bounds the fallback publish-marker names, +/// which follow the same stale-name discipline (#194, U1). +const KEY_TEMP_ATTEMPTS: u32 = 64; + +/// A key-load failure caused by the file's CONTENT (an empty or unparseable +/// PEM) rather than IO or permissions (#194, U2). Used as the anyhow error's +/// root so the boot loop can classify a failure as crash-recoverable via +/// `is_key_content_error` WITHOUT changing the message text operators and +/// tests rely on ("invalid PEM key"). Every other load failure (stat, +/// tighten, mode verification, read IO) stays a plain anyhow error and is +/// never recovered from. +#[derive(Debug)] +struct KeyContentError(String); + +impl std::fmt::Display for KeyContentError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +impl std::error::Error for KeyContentError {} + +/// True when `err` is a content-class load failure (see `KeyContentError`). +fn is_key_content_error(err: &anyhow::Error) -> bool { + err.downcast_ref::().is_some() +} + +/// Load an already-provisioned identity key. On Unix, defensively tighten looser +/// permissions to 0600 (do NOT reject a loose key — that would break existing +/// deployments; just narrow them), then verify the mode is actually 0600. +fn load_existing_key(path: &std::path::Path) -> Result { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + if let Ok(meta) = std::fs::metadata(path) { + if meta.permissions().mode() & 0o777 != 0o600 { + // Do NOT silently ignore a failed tighten (#194, F2): surface it + // so a key we cannot secure is not read and used exposed. + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) + .with_context(|| { + format!("could not tighten identity key {} to 0600", path.display()) + })?; + } + } + // Verify the key is actually 0600 before using it — a chmod that + // succeeded-but-no-op'd (some mounts) still leaves it exposed. + ensure_key_mode_0600(path)?; + } + let pem = std::fs::read_to_string(path) + .with_context(|| format!("failed to read key from {}", path.display()))?; + // Content-class failure (#194, U2): an empty file reads Ok("") and is + // rejected here too, so this one arm classifies both the zero-length and + // the unparseable case. The message text is unchanged. + let kp = Keypair::from_pem(&pem) + .map_err(|e| anyhow::Error::new(KeyContentError(format!("invalid PEM key: {e}"))))?; + info!(path = %path.display(), "loaded existing identity"); + Ok(kp) +} + +/// `".{stem}.publishing."`: the single source of truth for the publish +/// marker name class (#194, U1). Every production site that constructs a +/// marker name or prefix goes through here; the test module keeps raw +/// literals on purpose, as a guard against prefix drift. +fn publishing_prefix(stem: &str) -> String { + format!(".{stem}.publishing.") +} + +/// `".{stem}.recovering."`: the single source of truth for the recovery +/// claim name class (#194, F3). Same policy as `publishing_prefix`. +fn recovering_prefix(stem: &str) -> String { + format!(".{stem}.recovering.") +} + +/// `".{stem}.quarantined."`: the single source of truth for the quarantine +/// name class. Same policy as `publishing_prefix`. Class distinction (N2/N4): +/// a `.quarantined.` file is possibly-current residue, the crashed round's +/// own key, adoptable while young; a `.superseded.` file +/// (`superseded_prefix`) is provably outcompeted and forensics only. +fn quarantined_prefix(stem: &str) -> String { + format!(".{stem}.quarantined.") +} + +/// `".{stem}.superseded."`: the single source of truth for the SUPERSEDED +/// quarantine class (N2/N4). A quarantine moves into this class when its key +/// provably LOST its round: a restore republish returned Lost (a concurrent +/// winner durably published meanwhile), or a sibling quarantine won the +/// adoption. Superseded files are forensics only: never adoptable +/// (`find_adoptable_quarantine` scans `.quarantined.` alone), never part of +/// the boot crash signature, and swept only once older than +/// `SUPERSEDED_RETENTION` (bounded forensics); every consumer matches its +/// own prefix class, so this class is inert by construction. +fn superseded_prefix(stem: &str) -> String { + format!(".{stem}.superseded.") +} + +/// Rename `quarantine` into the superseded class +/// (`.{stem}.superseded.{pid}.{n}`), probing bounded fresh names per the +/// KEY_TEMP_ATTEMPTS discipline (skip an existing name, never clobber it). +/// A NotFound source means the quarantine resolved concurrently and there is +/// nothing left to reclassify. Best-effort: if every attempt fails the file +/// stays a plain quarantine, still adoptable while young, and a warn names +/// that hazard. +fn supersede_quarantine(final_path: &std::path::Path, quarantine: &std::path::Path) { + let dir = final_path + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or_else(|| std::path::Path::new(".")); + let stem = final_path + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or("identity.pem"); + let prefix = superseded_prefix(stem); + for n in 0..KEY_TEMP_ATTEMPTS { + let dest = dir.join(format!("{prefix}{}.{n}", std::process::id())); + if dest.exists() { + continue; + } + match std::fs::rename(quarantine, &dest) { + Ok(()) => return, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return, + Err(_) => continue, + } + } + warn!( + quarantine = %quarantine.display(), + "could not rename an outcompeted quarantine into the superseded class; \ + it remains adoptable while young" + ); +} + +/// Rename every YOUNG parseable `.{stem}.quarantined.*` sibling other than +/// `adopted` into the superseded class (N4). The adoption decided this round +/// in favor of the adopted key, so any other young parseable quarantine is +/// provably outcompeted; left adoptable, it would resurrect a different +/// historical DID the next time the final goes missing. Unparseable entries +/// stay plain quarantines (crash forensics, never adoptable anyway), and so +/// do AGED parseable ones (already outside the adoption bound; unreadable +/// mtime reads as aged, matching `find_adoptable_quarantine`). Best-effort +/// throughout. +fn supersede_sibling_quarantines(final_path: &std::path::Path, adopted: &std::path::Path) { + let dir = final_path + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or_else(|| std::path::Path::new(".")); + let stem = final_path + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or("identity.pem"); + let prefix = quarantined_prefix(stem); + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + let now = std::time::SystemTime::now(); + for entry in entries.filter_map(|e| e.ok()) { + if !entry + .file_name() + .to_str() + .is_some_and(|n| n.starts_with(&prefix)) + { + continue; + } + let path = entry.path(); + if path == *adopted { + continue; + } + let Ok(mtime) = entry.metadata().and_then(|m| m.modified()) else { + continue; + }; + let age = now.duration_since(mtime).unwrap_or_default(); + if age >= CLAIM_SWEEP_MIN_AGE { + continue; + } + let parseable = + std::fs::read_to_string(&path).is_ok_and(|pem| Keypair::from_pem(&pem).is_ok()); + if parseable { + supersede_quarantine(final_path, &path); + } + } +} + +/// The newest `.{stem}.quarantined.*` entry beside `final_path` YOUNGER than +/// `CLAIM_SWEEP_MIN_AGE` whose bytes parse as a keypair, with those bytes: a +/// COMPLETED key the CURRENT round's crashed or failed recovery left +/// stranded, adoptable by the generate arm of `load_or_create_keypair_with` +/// (F1). The age bound is the point: a stranded current round's quarantine +/// is seconds old, so continuity across a just-crashed restore is preserved, +/// while a HISTORICAL parseable quarantine (forensics kept after a Lost +/// restore, or any prior round) is never resurrected; without the bound, a +/// final that goes missing later silently revives an old identity, and the +/// operator procedure "delete identity.pem for a fresh DID" is defeated by +/// whatever quarantine still sits beside it. Once the floor passes, that +/// procedure mints a genuinely fresh identity. An unreadable mtime reads as +/// too old (skip; fail safe against resurrection). Candidates are tried +/// newest mtime first (the most recent quarantine is the most recently live +/// identity); unparseable entries are skipped, staying on disk as forensics. +/// Directory-read and non-UTF-8 policy match `list_publish_markers`. +fn find_adoptable_quarantine( + final_path: &std::path::Path, +) -> Option<(std::path::PathBuf, String, Keypair)> { + let dir = final_path + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or_else(|| std::path::Path::new(".")); + let stem = final_path + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or("identity.pem"); + let prefix = quarantined_prefix(stem); + let entries = std::fs::read_dir(dir).ok()?; + let now = std::time::SystemTime::now(); + let mut quarantines: Vec<(std::path::PathBuf, std::time::SystemTime)> = entries + .filter_map(|e| e.ok()) + .filter(|e| { + e.file_name() + .to_str() + .is_some_and(|n| n.starts_with(&prefix)) + }) + .filter_map(|e| { + // The adoption age bound (see the doc comment): an unreadable + // mtime is skipped outright, and a future mtime (clock skew) + // reads as age zero, matching the sweep's young side. + let mtime = e.metadata().and_then(|m| m.modified()).ok()?; + let age = now.duration_since(mtime).unwrap_or_default(); + (age < CLAIM_SWEEP_MIN_AGE).then_some((e.path(), mtime)) + }) + .collect(); + // Newest first. + quarantines.sort_by_key(|q| std::cmp::Reverse(q.1)); + for (path, _) in quarantines { + let Ok(pem) = std::fs::read_to_string(&path) else { + continue; + }; + if let Ok(kp) = Keypair::from_pem(&pem) { + return Some((path, pem, kp)); + } + } + None +} + +/// The publish markers (`.{stem}.publishing.*`) beside `final_path`, sorted: +/// U1's crash signature for a fallback publish interrupted mid-window (#194, +/// U2). An unreadable directory reads as no markers, so recovery stays off +/// and the load error surfaces unchanged (the loud-fail default); entries +/// with non-UTF-8 names cannot be our markers and are skipped. +fn list_publish_markers(final_path: &std::path::Path) -> Vec { + let dir = final_path + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or_else(|| std::path::Path::new(".")); + let stem = final_path + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or("identity.pem"); + let prefix = publishing_prefix(stem); + let Ok(entries) = std::fs::read_dir(dir) else { + return Vec::new(); + }; + let mut markers: Vec<_> = entries + .filter_map(|e| e.ok()) + .filter(|e| { + e.file_name() + .to_str() + .is_some_and(|n| n.starts_with(&prefix)) + }) + .map(|e| e.path()) + .collect(); + markers.sort(); + markers +} + +/// The publish markers AND recovery claims (`.{stem}.publishing.*` plus +/// `.{stem}.recovering.*`) beside `final_path`, sorted: the full claimable +/// set for crash recovery (#194, F3). Claims count because a recoverer's +/// claim IS a renamed marker: a recoverer that crashes between its claim +/// rename and the quarantine leaves a content-bad final with a claim and +/// ZERO markers, and a signature keyed on markers alone would leave that +/// state unrecoverable forever. Directory-read and non-UTF-8 policy match +/// `list_publish_markers`. +/// +/// A `.recovering.` claim counts only once older than `CLAIM_SWEEP_MIN_AGE` +/// (same age gate and rationale as the G1 sweep): a fresh claim may gate a +/// LIVE recovery round, and a second starter that claims it mid-round steals +/// the round from the recoverer that owns it. Because both the crash +/// signature (`boot_load_key`) and the claim loop +/// (`recover_crashed_publish`) read this one list, the gate holds at a +/// single point. An unreadable claim mtime reads as young, so the claim is +/// skipped (fail safe, matching the sweep). `.publishing.` markers stay +/// unconditionally claimable: claiming a live publisher's marker only fails +/// its commit check, a safe demotion. +fn list_recovery_claimables(final_path: &std::path::Path) -> Vec { + let dir = final_path + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or_else(|| std::path::Path::new(".")); + let stem = final_path + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or("identity.pem"); + let publishing = publishing_prefix(stem); + let recovering = recovering_prefix(stem); + let Ok(entries) = std::fs::read_dir(dir) else { + return Vec::new(); + }; + let mut claimables: Vec<_> = entries + .filter_map(|e| e.ok()) + .filter(|e| { + let name = e.file_name(); + let Some(name) = name.to_str() else { + return false; + }; + if name.starts_with(&publishing) { + return true; + } + if !name.starts_with(&recovering) { + return false; + } + // F2 age gate (see the doc comment): only an ORPHANED claim is + // claimable; a fresh one may gate a live round. + e.metadata() + .and_then(|m| m.modified()) + .ok() + .and_then(|mtime| std::time::SystemTime::now().duration_since(mtime).ok()) + .is_some_and(|age| age >= CLAIM_SWEEP_MIN_AGE) + }) + .map(|e| e.path()) + .collect(); + claimables.sort(); + claimables +} + +/// Load a key another concurrent start may still be publishing, polling until it +/// parses or `KEY_RACE_DEADLINE` elapses. An already-provisioned key parses on +/// the first attempt with no sleep. +fn load_racing_key(path: &std::path::Path) -> Result { + match boot_load_key(path, false, || Ok(())) { + BootLoad::Loaded(kp) => Ok(*kp), + BootLoad::CrashSignature(e) | BootLoad::Failed(e) => Err(e), + } +} + +/// Outcome of `boot_load_key` (#194, U2). The tri-state exists so the boot +/// loop can tell "content failure WITH the crash signature observed" (a +/// recovery candidate) from every other failure; folding both into one error +/// would let a concurrent recoverer's marker sweep turn a recoverable boot +/// into a loud failure. +enum BootLoad { + /// Boxed to keep the variants' sizes comparable (clippy: + /// large_enum_variant); the enum lives only across a boot-time match. + Loaded(Box), + /// Content-class failure observed while a `.publishing.` marker or a + /// `.recovering.` claim was present, continuously for + /// `CRASH_SIGNATURE_MIN_PERSIST`; returned without riding the rest of + /// the deadline. + CrashSignature(anyhow::Error), + /// Any other failure, after the full deadline. + Failed(anyhow::Error), +} + +/// `load_racing_key` for the boot loop (#194, U2). With `crash_exit` set, a +/// content-class failure (`KeyContentError`) observed while a `.publishing.` +/// marker or a `.recovering.` claim is present, continuously for +/// `CRASH_SIGNATURE_MIN_PERSIST` of consecutive observations, returns +/// `CrashSignature` instead of riding the deadline: claimable plus +/// unreadable content is the crash signature recovery exists for, and the +/// caller's atomic claim renames arbitrate the (rare) race against a live +/// fallback publisher still inside its window. Claims count toward the +/// signature for F3 (see `list_recovery_claimables`). The persistence window +/// (#194, G3) exists so a reader that catches a live publisher's mid-write +/// window is not classified as a crash: chmod/stat/scheduling latency on +/// shared volumes routinely stretches that window past any small fixed poll +/// count, while a real crash state persists across every poll, so the +/// signature clock starts at its first observation and any successful load +/// or signature-free observation resets it. Every other failure keeps the +/// full wall-clock deadline. `load_fault` is the injection hook (a no-op +/// `Ok` in production); an injected `Err` is taken as that attempt's result +/// and is NOT content-class. +fn boot_load_key( + path: &std::path::Path, + crash_exit: bool, + load_fault: fn() -> std::io::Result<()>, +) -> BootLoad { + let deadline = std::time::Instant::now() + KEY_RACE_DEADLINE; + let mut last_err; + let mut signature_since: Option = None; + loop { + match load_fault() + .map_err(anyhow::Error::new) + .and_then(|()| load_existing_key(path)) + { + Ok(kp) => return BootLoad::Loaded(Box::new(kp)), + Err(e) => last_err = e, + } + let signature = crash_exit + && is_key_content_error(&last_err) + && !list_recovery_claimables(path).is_empty(); + if signature { + let since = *signature_since.get_or_insert_with(std::time::Instant::now); + if since.elapsed() >= CRASH_SIGNATURE_MIN_PERSIST { + return BootLoad::CrashSignature(last_err); + } + } else { + signature_since = None; + } + if std::time::Instant::now() >= deadline { + return BootLoad::Failed(last_err); + } + std::thread::sleep(std::time::Duration::from_millis(2)); + } +} + +/// Wait until NO `.{stem}.recovering.*` claim FILE remains beside +/// `final_path`, polling on `load_racing_key`'s cadence (~2ms), bounded by +/// `CLAIM_SWEEP_MIN_AGE` plus a one-second margin (#194, U3; F3; N3). Why +/// the wait exists: a fallback winner that failed its commit check was +/// demoted by a recovering peer, and that peer is mid-round — it +/// quarantines the (old) final, republishes a fresh key, and only then +/// clears its claim. A demoted winner that re-loaded before the claim +/// cleared could observe the pre-quarantine key and diverge from what the +/// peer ends up publishing; waiting for claim clearance makes the follow-up +/// load observe the settled state. +/// +/// The waiter keys purely on ABSENCE, never on age. Age stays the liveness +/// rule for the SWEEP (`sweep_stale_markers`) and CLAIMABILITY +/// (`list_recovery_claimables`), but an aged claim is not proof its +/// recoverer is gone: with the heartbeat spawn failed AND both mtime stamp +/// rungs failing on a hostile volume, a LIVE recoverer's claim can sit +/// mtime-frozen while the round is still settling disk, and a waiter that +/// read it as orphaned residue would let the demoted publisher load a +/// pre-settlement key while that recoverer settles disk differently: +/// two-sided divergence under compounded failure. A file that is GONE is +/// unambiguous: the round settled (release) or a sweep decided the claim +/// was orphaned; either way the follow-up load observes a settled state. +/// +/// So: `Ok` only when the claim files are gone. Bound expiry with ANY claim +/// file still present (young or aged) returns an `Err` naming the file(s): +/// the round cannot be proven settled, and the caller must fail closed +/// instead of loading. Consequence: an aged ORPHAN claim (its recoverer +/// long dead) now fails a concurrently-demoted publisher loudly until a +/// healthy boot's sweep or the delayed re-sweep +/// (`spawn_delayed_claim_resweep`) clears it — a bounded, +/// operator-visible degradation, priced against the silent divergence the +/// age rule risked. An unreadable directory reads as no claims, matching +/// `list_publish_markers`' policy. +fn wait_for_recovery_claims(final_path: &std::path::Path) -> Result<()> { + let dir = final_path + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or_else(|| std::path::Path::new(".")); + let stem = final_path + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or("identity.pem"); + let prefix = recovering_prefix(stem); + let bound = CLAIM_SWEEP_MIN_AGE + std::time::Duration::from_secs(1); + let deadline = std::time::Instant::now() + bound; + loop { + let present: Vec = std::fs::read_dir(dir) + .map(|entries| { + entries + .filter_map(|e| e.ok()) + .filter(|e| { + e.file_name() + .to_str() + .is_some_and(|n| n.starts_with(&prefix)) + }) + .map(|e| e.file_name().to_string_lossy().into_owned()) + .collect() + }) + .unwrap_or_default(); + if present.is_empty() { + return Ok(()); + } + if std::time::Instant::now() >= deadline { + anyhow::bail!( + "recovery round did not settle within the {bound:?} wait bound: \ + recovery claim(s) {present:?} still present beside {}", + final_path.display() + ); + } + std::thread::sleep(std::time::Duration::from_millis(2)); + } +} + +/// Best-effort sweep of stale `.{stem}.publishing.*` markers, +/// `.{stem}.recovering.*` claims, and expired `.{stem}.superseded.*` / +/// `.{stem}.quarantined.*` forensics beside a healthy final, run on both +/// healthy-boot arms of `load_or_create_keypair_with` (#194, U4). Without it +/// a marker orphaned by a crash between its durability fsync and the final's +/// `create_new`, or a claim left by a crashed recovery, outlives every +/// healthy boot and later misclassifies a real corruption of a good key as +/// an interrupted first write (quarantine-and-regenerate instead of the loud +/// error a corrupted good key deserves). +/// +/// DURABILITY PRECONDITION (critical, the reason this is not a bare unlink +/// loop): a loadable final is not necessarily durable — a racing fallback +/// winner's `write_all` is visible via the page cache before its `sync_all`, +/// so sweeping on loadability alone could remove a marker ahead of the +/// content it vouches for and let a power loss recreate exactly the +/// marker-less partial final the bracket exists to prevent. The sweeper +/// therefore makes the content durable ITSELF before touching any marker: +/// open the final read-only, `sync_all` it, then `fsync_parent_dir` (Unix). +/// If either step fails the sweep is skipped entirely — every marker stays +/// in place, at most a warn. Only after both succeed is every matching entry +/// removed (each best-effort), followed by a best-effort dir fsync. On +/// non-Unix the dir-fsync halves are skipped like every other dir fsync here +/// (reasoned, not run: no non-Unix target is exercised by these tests). +/// +/// Sweeping a FOREIGN marker is safe once durability holds: the swept +/// publisher's own disposal tolerates NotFound on every arm; a live +/// publisher still before its final `create_new` hits AlreadyExists and +/// routes to Lost regardless of its marker; and the content the marker +/// vouched for is durable by the sweeper's own hand. `sweep_sync` is the +/// `RecoverySeam` test hook, run before the final's `sync_all` with an `Err` +/// taken as that sync's result; production passes a no-op `Ok`. +/// +/// SWEEP ASYMMETRY (#194, G1): `.publishing.` markers are removed +/// unconditionally, but `.recovering.` claims only once older than +/// `CLAIM_SWEEP_MIN_AGE`. Sweeping a LIVE publisher's marker is safe (it +/// only fails that publisher's commit check, forcing a safe +/// demotion-and-reload), while sweeping a LIVE recoverer's claim is not: a +/// demoted publisher's `wait_for_recovery_claims` waits on that claim until +/// the recovery resolves, and removing it early lets the publisher re-load +/// the pre-quarantine key while the recoverer quarantines and republishes: +/// two-sided divergence. An unreadable claim mtime reads as young, so the +/// claim is kept (fail safe; the next boot past the age floor sweeps it). +fn sweep_stale_markers(final_path: &std::path::Path, sweep_sync: fn() -> std::io::Result<()>) { + let durable = std::fs::File::open(final_path) + .and_then(|f| sweep_sync().and_then(|()| f.sync_all())) + .map_err(anyhow::Error::new); + #[cfg(unix)] + let durable = durable.and_then(|()| fsync_parent_dir(final_path)); + if let Err(e) = durable { + warn!( + path = %final_path.display(), + err = %e, + "skipping stale publish-marker sweep: could not make the final key durable" + ); + return; + } + let dir = final_path + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or_else(|| std::path::Path::new(".")); + let stem = final_path + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or("identity.pem"); + let publishing = publishing_prefix(stem); + let recovering = recovering_prefix(stem); + let superseded = superseded_prefix(stem); + let quarantined = quarantined_prefix(stem); + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for entry in entries.filter_map(|e| e.ok()) { + let name = entry.file_name(); + let Some(name) = name.to_str() else { continue }; + if name.starts_with(&publishing) { + let _ = std::fs::remove_file(entry.path()); + } else if name.starts_with(&recovering) { + // G1 age gate: only an orphaned claim (older than + // CLAIM_SWEEP_MIN_AGE) is sweepable; see the doc comment. + let orphaned = entry + .metadata() + .and_then(|m| m.modified()) + .ok() + .and_then(|mtime| std::time::SystemTime::now().duration_since(mtime).ok()) + .is_some_and(|age| age >= CLAIM_SWEEP_MIN_AGE); + if orphaned { + let _ = std::fs::remove_file(entry.path()); + } + } else if name.starts_with(&superseded) || name.starts_with(&quarantined) { + // Forensic residue expires after SUPERSEDED_RETENTION (the + // operator's inspection window); without an expiry every lost + // round's superseded file, and every crash's plain quarantine + // (unparseable forensics, or parseable history already far past + // the adoption floor), accumulates forever. Unreadable mtime + // reads as young (kept; fail safe), matching the claim gate + // above. + let expired = entry + .metadata() + .and_then(|m| m.modified()) + .ok() + .and_then(|mtime| std::time::SystemTime::now().duration_since(mtime).ok()) + .is_some_and(|age| age >= SUPERSEDED_RETENTION); + if expired { + let _ = std::fs::remove_file(entry.path()); + } + } + } + #[cfg(unix)] + { + let _ = fsync_parent_dir(final_path); + } +} + +/// Fire-and-forget companion to the boot sweep's young-claim sparing (F2): +/// one delayed re-sweep of `key_path`, `CLAIM_SWEEP_MIN_AGE` plus a +/// two-second margin into uptime. The boot-time sweep must SPARE a young +/// `.recovering.` claim (it may gate a live round), but a spared orphan is +/// never re-examined in-process, and once it ages on disk a LATER content +/// failure of the good final would match the crash signature (content-bad +/// beside an aged claimable) and silently regenerate where marker-less +/// corruption must fail loudly. By the time this fires, any claim the boot +/// sweep spared has aged past the liveness floor, so `sweep_stale_markers` +/// clears it and aged residue cannot coexist with a long-healthy final. The +/// node never waits on this task; failures stay at warn (the sweep itself +/// warns internally when it skips). +fn spawn_delayed_claim_resweep(key_path: std::path::PathBuf) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + tokio::time::sleep(CLAIM_SWEEP_MIN_AGE + std::time::Duration::from_secs(2)).await; + let sweep_path = key_path.clone(); + if let Err(e) = + tokio::task::spawn_blocking(move || sweep_stale_markers(&sweep_path, || Ok(()))).await + { + warn!( + path = %key_path.display(), + err = %e, + "delayed identity-claim re-sweep task failed" + ); + } + }) +} + +/// Compile-time fault-injection seam for `publish_key_atomically`, extending +/// the `before_link` closure precedent: every hook is a no-op `Ok` in +/// production (`PublishFaults::NONE`); tests swap in a hook returning `Err` to +/// force that step to fail deterministically. Plain `fn` pointers, not a +/// runtime/env knob: production code only ever passes `NONE`. +#[derive(Clone, Copy)] +struct PublishFaults { + /// Runs before the `hard_link`; an `Err` is taken as the link result and + /// the real link is never attempted. + link: fn() -> std::io::Result<()>, + /// Runs before the fallback's `create_new` open of the FINAL path; an `Err` + /// is taken as the open result (a non-`AlreadyExists` open error), so the + /// publish surfaces it chained with `link_err`. + fallback_create: fn() -> std::io::Result<()>, + /// Runs before the fallback's publish-marker `create_new` (each name + /// attempt); an `Err` is taken as the create result (a non-`AlreadyExists` + /// open error), so the publish surfaces it chained with `link_err` (#194, + /// U1). + marker_create: fn() -> std::io::Result<()>, + /// Runs before the fsync of the marker's parent directory; an `Err` is + /// taken as that fsync's result. This fsync is warn-and-continue, so an + /// injected `Err` must NOT fail the publish (#194, U1). + #[cfg(unix)] + marker_fsync: fn() -> std::io::Result<()>, + /// Runs before the fallback's `write_all`; an `Err` is taken as the write + /// result. + fallback_write: fn() -> std::io::Result<()>, + /// Runs before the fallback's file `sync_all`; an `Err` is taken as the + /// fsync result. + fallback_fsync: fn() -> std::io::Result<()>, + /// Runs between the tighten and the 0600 verification of the just-created + /// TEMP file; an `Err` is taken as the verification result. Simulates a + /// mode-ignoring mount (vfat: chmod no-ops returning Ok), which cannot be + /// reproduced on ext4. + #[cfg(unix)] + temp_mode_verify: fn() -> std::io::Result<()>, + /// Same, for the FINAL file created on the hard-link fallback path. + #[cfg(unix)] + fallback_mode_verify: fn() -> std::io::Result<()>, +} + +impl PublishFaults { + /// The production value: no fault injected at any step. + const NONE: Self = Self { + link: || Ok(()), + fallback_create: || Ok(()), + marker_create: || Ok(()), + #[cfg(unix)] + marker_fsync: || Ok(()), + fallback_write: || Ok(()), + fallback_fsync: || Ok(()), + #[cfg(unix)] + temp_mode_verify: || Ok(()), + #[cfg(unix)] + fallback_mode_verify: || Ok(()), + }; +} + +/// Publish `pem` to `final_path` atomically: write the full bytes to a sibling +/// temp file, then `hard_link` the temp into place. `hard_link` is atomic and +/// fails if `final_path` already exists, which on the hard-link path gives +/// these guarantees at once: +/// (a) the final path only ever appears with COMPLETE content — a concurrent +/// reader never observes an empty/half-written key, unlike a +/// `create_new`+`write_all` that exposes an empty inode before the PEM is +/// flushed (#194); +/// (b) a lost race never clobbers the winner — exactly one publisher links; +/// (c) a crashed publisher leaves only the temp, never a partial final that +/// would wedge every later start on `invalid PEM key`; +/// (d) durability: the PEM bytes are fsynced before the name appears and the +/// namespace change is fsynced (on Unix) before success is reported, so a +/// power crash cannot leave a durable-but-truncated final key (#194). +/// +/// If the link fails for any reason other than `AlreadyExists` (a filesystem +/// without hard-link support, or a transient link error), the publish falls +/// back to `create_new(0o600)` directly on the final path +/// (`publish_key_fallback`). The fallback keeps (b): `create_new` still admits +/// exactly one winner and a lost race loads the existing key. It weakens (a) +/// and (d): the final name is visible while the PEM is being written, a +/// window concurrent readers ride out via `load_racing_key`'s wall-clock +/// deadline; and the final name can become durable before the PEM bytes are +/// fsynced, so a power loss can additionally leave a durable empty/truncated +/// final. Guarantee (c) now holds on the fallback too, via the marker +/// protocol (#194, U2/U4): a crash inside the non-atomic window leaves +/// marker+final together, and the next boot claims the marker, quarantines +/// the partial final, and regenerates instead of wedging. On +/// dir-fsync-hostile mounts the marker dir fsync is warn-and-continue (see +/// `publish_key_fallback`), so there the recovery guarantee narrows to +/// non-power-loss (SIGKILL-class) crashes: a power loss can drop the +/// undurable marker name and leave the partial final to the loud +/// invalid-PEM error instead. Honest residual on +/// top of the U3 list below: that recovery runs at the next boot, not at +/// crash time, so the partial final persists until then. In every unrecovered +/// case the failure mode is the same LOUD invalid-PEM (or mode) error at the +/// next start, never a silently used key. +/// +/// Against the boot-time crash RECOVERY race (#194, U3) the fallback's marker +/// protocol restores (b)-style convergence: removing the publish marker is +/// the winner's commit check, and recovery claims a round by renaming EVERY +/// marker and stale claim beside the final (a live winner's own marker is +/// necessarily among them), so exactly one side takes the round. A winner +/// that loses its marker demotes itself, waits for every `.recovering.` +/// claim FILE to clear (bounded by `CLAIM_SWEEP_MIN_AGE` plus a margin; see +/// `wait_for_recovery_claims`), and re-loads the settled key as Lost; if +/// any claim file survives the bound the publish fails closed with a loud +/// error instead of loading. A recovery that finds the final completed +/// after claiming re-loads it instead of quarantining. Honest residuals: a +/// winner stalled past `KEY_RACE_DEADLINE` that loads in the sub-ms window +/// before a recoverer's claim exists degrades to a re-load, and a +/// crash-interrupted recovery whose orphaned claim outlives the bounded +/// wait fails the demoted publish loudly until a sweep clears the claim: +/// loud error or re-load, never silent two-sided divergence. +/// +/// `before_link` is a no-op in production; tests use it to widen the +/// post-write / pre-link window deterministically. `before_commit` runs +/// between the fallback final's dir fsync and the marker-removal commit +/// check — a no-op in production; tests use it to interleave a concurrent +/// recovery into that window (#194, U3). `faults` is the compile-time +/// fault-injection seam (`PublishFaults::NONE` in production). +fn publish_key_atomically( + final_path: &std::path::Path, + pem: &[u8], + before_link: &dyn Fn(), + before_commit: &dyn Fn(), + faults: PublishFaults, +) -> Result { + use std::io::Write; + let dir = final_path + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or_else(|| std::path::Path::new(".")); + let stem = final_path + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or("identity.pem"); + + // Pick the temp name with a bounded per-call retry, not a process-global + // counter: a counter is only unique within one process (two containers + // sharing a volume can both run as PID 1), and a crash between the temp + // write and its cleanup leaves the old name behind, so a single + // deterministic name would fail `create_new` on every restart until an + // operator deleted it (#194). `AlreadyExists` means either such a stale + // temp or a live concurrent publisher's temp; the two are indistinguishable, + // so never delete a temp this call did not create (unlinking a live one + // between its write and its hard_link would fail that healthy start), just + // skip to the next name. Consequence: a crashed start leaks at most one + // 0600 temp file, which later starts skip. + let mut opened = None; + for attempt in 0..KEY_TEMP_ATTEMPTS { + let tmp = dir.join(format!(".{stem}.tmp.{}.{attempt}", std::process::id())); + let mut opts = std::fs::OpenOptions::new(); + opts.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + opts.mode(0o600); + } + match opts.open(&tmp) { + Ok(f) => { + opened = Some((tmp, f)); + break; + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(e) => return Err(e).with_context(|| format!("create temp key {}", tmp.display())), + } + } + let Some((tmp, mut f)) = opened else { + anyhow::bail!( + "all {KEY_TEMP_ATTEMPTS} temp key names .{stem}.tmp.{}.* in {} are taken; \ + remove the stale temp files and restart", + std::process::id(), + dir.display() + ); + }; + // Secure the empty temp BEFORE the PEM bytes hit it: on a mode-ignoring + // mount this fails closed with nothing sensitive on disk, and the + // hard_link later carries the verified 0600 inode to the final name + // (#194, U2). + #[cfg(unix)] + tighten_and_verify_created(&tmp, faults.temp_mode_verify)?; + // On a failed write OR fsync, remove the temp so nothing partial is ever + // linked (#194). The fsync makes the PEM bytes durable BEFORE the name can + // appear: without it a power crash after the link could leave a durable + // final name over truncated bytes, wedging every later start. + write_key_or_cleanup(&tmp, f.write_all(pem).and_then(|()| f.sync_all()))?; + drop(f); + + before_link(); + + let linked = (faults.link)().and_then(|()| std::fs::hard_link(&tmp, final_path)); + let _ = std::fs::remove_file(&tmp); + match linked { + Ok(()) => { + // Fsync the parent directory so the link (and the temp unlink) are + // durable before success is reported (#194). Skipped on non-Unix, + // where opening a directory fails. On fsync failure do NOT remove + // the final file: it is complete and data-synced, a concurrent + // loser may already have loaded it, and a later start loads it via + // the exists() path. + #[cfg(unix)] + fsync_parent_dir(final_path)?; + Ok(KeyPublish::Won) + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => Ok(KeyPublish::Lost), + Err(e) => publish_key_fallback(final_path, pem, e, before_commit, faults), + } +} + +/// Fallback publish for a key directory whose filesystem cannot hard-link +/// (Unsupported/EPERM on some network and overlay mounts, or a transient link +/// error): create the FINAL path directly with `create_new(true).mode(0o600)` +/// (the INV-23 creation pattern) and write the PEM into it. `AlreadyExists` +/// still routes to the lost-race path, so no-clobber holds. Error handling +/// splits at the first PEM byte. PRE-write failures (marker create, final +/// create, tighten/mode-verify) remove the just-created EMPTY final and +/// dispose the marker: nothing recoverable exists yet, and those arms +/// complete within `CRASH_SIGNATURE_MIN_PERSIST` of the final's creation, +/// so no recovery can have claimed the round. POST-write failures +/// (`write_all` or the file `sync_all`) remove NOTHING: the partial final +/// and its marker are left in place together and the error surfaces naming +/// the handoff. That pair is exactly the crash state boot-time recovery +/// handles (claim, quarantine, regenerate), while an unlink here is a +/// stat-then-unlink race against a recovery that already claimed this round +/// and republished a live key at the name, and a marker disposed beside a +/// kept partial final recreates the marker-less wedge the bracket below +/// exists to prevent. The complete-but-unsynced `sync_all`-failure state is +/// safe too: a later healthy load fsyncs the final ITSELF before sweeping +/// the marker (`sweep_stale_markers`' durability gate), and a crash before +/// then re-enters recovery via the surviving marker. A failed parent-DIR +/// fsync also keeps the final but DOES dispose the marker (the content is +/// complete and data-synced, and a lost directory entry just disappears -> +/// clean regen, no wedge). Every error context chains `link_err` so a +/// two-step failure is diagnosable. +/// +/// The whole non-atomic window is bracketed by a durable publish marker +/// (`.{stem}.publishing.{pid}.{attempt}`, empty, same dir as the final): the +/// marker is created and its NAME fsynced BEFORE the final can exist, and it +/// is removed only once the final is complete-and-durable (success) or the +/// exit leaves nothing needing vouching (pre-write error / lost race / +/// dir-fsync failure); on a post-write error the marker intentionally +/// OUTLIVES the attempt to vouch for the partial final beside it. On success +/// the removal doubles as the COMMIT CHECK (#194, U3): a removal that fails +/// means a recovering peer claimed this round, and the publish demotes +/// itself to Lost (see the success arm). A crash inside the window therefore +/// leaves marker+final together, so a later boot can tell "partial final from +/// a crashed fallback publish" from a key an operator corrupted (#194, U1). +/// Two constraints: +/// - Disposal ORDER on the pre-write arms is pinned: the empty final's +/// removal is ISSUED BEFORE the marker's. Cleanup is deliberately NOT a +/// Drop guard: a drop guard can run ahead of the final's disposal on an +/// early exit, and it would also fire on the post-write error exits, +/// whose whole point is that the marker survives. +/// - The marker dir fsync is warn-and-continue, not a hard gate: on +/// dir-fsync-hostile mounts a hard failure would turn "wedge once, then +/// recover" into "never provisions". Without that durability the marker +/// still survives SIGKILL-class crashes (the fs state persists); only the +/// power-loss protection narrows. +fn publish_key_fallback( + final_path: &std::path::Path, + pem: &[u8], + link_err: std::io::Error, + before_commit: &dyn Fn(), + faults: PublishFaults, +) -> Result { + use std::io::Write; + warn!( + path = %final_path.display(), + err = %link_err, + "hard_link failed; publishing identity key via direct create_new fallback" + ); + let dir = final_path + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or_else(|| std::path::Path::new(".")); + let stem = final_path + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or("identity.pem"); + + // Create the publish marker BEFORE the final name can exist, probing + // bounded per-call names exactly like the temp-name loop and for the same + // reasons: a stale marker left by a crashed same-PID start must not wedge + // the restart, and a stale marker is indistinguishable from a live + // concurrent publisher's, so never delete a name this call did not + // create; just skip to the next one (#194, U1). + let marker_prefix = publishing_prefix(stem); + let mut marker = None; + for attempt in 0..KEY_TEMP_ATTEMPTS { + let candidate = dir.join(format!("{marker_prefix}{}.{attempt}", std::process::id())); + match (faults.marker_create)().and_then(|()| { + std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&candidate) + }) { + Ok(f) => { + drop(f); + marker = Some(candidate); + break; + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(e) => { + return Err(e).with_context(|| { + format!( + "create publish marker {} in hard_link fallback (link failed: {link_err})", + candidate.display() + ) + }) + } + } + } + let Some(marker) = marker else { + anyhow::bail!( + "all {KEY_TEMP_ATTEMPTS} publish marker names {marker_prefix}{}.* in {} are \ + taken; remove the stale marker files and restart (link failed: {link_err})", + std::process::id(), + dir.display() + ); + }; + // Marker disposal is explicit at every disposing exit, never a Drop + // guard; see the order constraint in the doc comment (the post-write + // error exits deliberately keep the marker). + let dispose_marker = || { + let _ = std::fs::remove_file(&marker); + }; + // Make the marker NAME durable before the final can exist, so no crash + // can leave a partial final without its marker. Warn-and-continue on + // failure per the doc comment's degradation constraint. + #[cfg(unix)] + { + if let Err(e) = (faults.marker_fsync)() + .map_err(anyhow::Error::new) + .and_then(|()| fsync_parent_dir(final_path)) + { + warn!( + marker = %marker.display(), + err = %e, + "publish marker dir fsync failed; continuing with reduced power-loss protection" + ); + } + } + + let mut opts = std::fs::OpenOptions::new(); + opts.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + opts.mode(0o600); + } + let mut f = match (faults.fallback_create)().and_then(|()| opts.open(final_path)) { + Ok(f) => f, + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { + // Lost race: the existing final is the winner's complete key, not + // ours to bracket; drop our marker and defer to it. + dispose_marker(); + return Ok(KeyPublish::Lost); + } + Err(e) => { + // The final was never created here, so there is no final removal + // to order ahead of the marker's. + dispose_marker(); + return Err(e).with_context(|| { + format!( + "create identity key {} in hard_link fallback (link failed: {link_err})", + final_path.display() + ) + }); + } + }; + // Secure the empty final BEFORE the PEM bytes hit it (#194, U2). The + // helper's fail-closed removal is what keeps a retry from wedging on + // AlreadyExists, and is safe here for the same reason as on every + // pre-write arm: the file is still empty, so the post-write + // keep-everything rule below does not apply yet. The helper removes the + // final before returning Err, so the pinned final-before-marker disposal + // order holds. By-name removal stays right on this and the other + // pre-write arms (create/mode-verify): they run within + // CRASH_SIGNATURE_MIN_PERSIST of the final's creation, so no recovery + // can have claimed this round yet. + #[cfg(unix)] + { + if let Err(e) = tighten_and_verify_created(final_path, faults.fallback_mode_verify) + .with_context(|| { + format!("secure identity key in hard_link fallback (link failed: {link_err})") + }) + { + dispose_marker(); + return Err(e); + } + } + // A failed write LEAVES the partial final and its marker in place and + // only surfaces the error: marker+final together is exactly the crash + // state the next boot recovers from (claim, quarantine, regenerate). + // Any removal here would be a stat-then-unlink race against a recovery + // that already claimed this round and republished a live key at the + // name, and disposing the marker while the partial final stays would + // recreate the marker-less wedge the bracket exists to prevent, so this + // exit removes nothing and disposes nothing. Not write_key_or_cleanup, + // whose by-name removal stays right for the primary path's TEMP file + // (a private name no recovery can race). + if let Err(e) = (faults.fallback_write)().and_then(|()| f.write_all(pem)) { + return Err(anyhow::Error::new(e) + .context(format!( + "failed to write key to {}; the partial final and its publish marker \ + are left in place for boot-time crash recovery", + final_path.display() + )) + .context(format!( + "hard_link fallback publish to {} (link failed: {link_err})", + final_path.display() + ))); + } + // A failed file fsync takes the same keep-everything exit as the failed + // write: the bytes were accepted but may not be durable, and marker+ + // final left together is the recoverable crash state (the next boot's + // load either parses the complete content, after which the sweep's + // durability gate fsyncs the final BEFORE removing the marker, or + // content-fails beside the marker and recovers). A DISTINCT context is + // kept so an operator debugging ENOSPC/EIO can tell "durability failed" + // (bytes accepted, not synced) from the write-rejected case above. Only + // the parent-DIR fsync below keeps the final while disposing the marker + // (a lost directory entry just disappears -> clean regen). + if let Err(e) = (faults.fallback_fsync)().and_then(|()| f.sync_all()) { + return Err(anyhow::Error::new(e).context(format!( + "fsync identity key {} in hard_link fallback (durability failed; the \ + final and its publish marker are left in place for boot-time crash \ + recovery; link failed: {link_err})", + final_path.display() + ))); + } + drop(f); + // Fsync the parent directory so the new name is durable before success is + // reported, as on the hard-link path. Same policy on failure: the final is + // complete and data-synced, so it is NOT removed; the marker is still + // disposed of best-effort. + #[cfg(unix)] + { + if let Err(e) = fsync_parent_dir(final_path) { + dispose_marker(); + return Err(e); + } + } + // Success: the final is complete and durable. Removing our OWN marker is + // the COMMIT CHECK (#194, U3): recovery claims a round by renaming EVERY + // marker beside the final, so our marker is necessarily among any + // claimed round and a successful removal proves no recovery can ever + // claim it, linearizing this winner ahead of any recoverer. Any failure + // (NotFound after a peer's claim rename, or any other error) means + // ownership of the round cannot be proven: a recovering peer may have + // quarantined our final and republished, so returning Won could report + // key A while disk holds key B. Demote to Lost instead. Demotion is safe + // ONLY for a SETTLED round (N3): the caller's Lost arm re-loads the + // final, and that load observes the recoverer's outcome only once the + // recoverer's claims have cleared, so the demoted publish first waits + // them out. An UNSETTLED round past the wait bound (any claim file + // still present) fails the publish closed with a loud error instead: + // loading there could return a pre-settlement key while the still-live + // recoverer settles disk on a different identity. + before_commit(); + match std::fs::remove_file(&marker) { + Ok(()) => { + // Best-effort fsync so the marker's removal tends to be durable. + #[cfg(unix)] + { + let _ = fsync_parent_dir(final_path); + } + Ok(KeyPublish::Won) + } + Err(e) => { + warn!( + marker = %marker.display(), + err = %e, + "fallback publish lost its marker before commit; a recovering peer owns \ + this round, demoting the publish to lost" + ); + wait_for_recovery_claims(final_path).with_context(|| { + format!( + "fallback publish of {} was demoted by a recovering peer, and the \ + recovery round did not settle within the wait bound; failing the \ + publish closed instead of loading a possibly pre-settlement key \ + (marker removal failed: {e})", + final_path.display() + ) + })?; + Ok(KeyPublish::Lost) + } + } +} + +/// Compile-time test seam for the load-side crash recovery in +/// `load_or_create_keypair_with` (#194, U2), following the `PublishFaults` +/// precedent: production passes `RecoverySeam::NONE` (every hook a no-op). +#[derive(Clone, Copy)] +struct RecoverySeam<'a> { + /// Runs before every load attempt of the boot loader; an `Err` is taken as + /// that attempt's result. The injected failure is NOT content-class, so it + /// deterministically exercises the transient-failure (never-recover) policy. + load_fault: fn() -> std::io::Result<()>, + /// Runs at the top of each generate+publish pass, so a test can mutate + /// on-disk state between a completed recovery and the retry publish. + before_publish: &'a dyn Fn(), + /// Runs inside `sweep_stale_markers` before the final's `sync_all`; an + /// `Err` is taken as that sync's result, so a test can fail the sweep's + /// durability gate deterministically (#194, U4). + sweep_sync: fn() -> std::io::Result<()>, + /// Runs inside `recover_crashed_publish` between the claim renames and + /// the post-claim re-parse of the final, so a test can interleave a + /// concurrent publisher's resolution into exactly that window (#194, F2). + before_reparse: &'a dyn Fn(), + /// Runs immediately before the post-claim RE-parse load inside + /// `recover_crashed_publish`; an `Err` is taken as that load's result. + /// The injected failure is NOT content-class, so it deterministically + /// exercises the transient-reparse (release-and-fail-loudly) policy + /// (#194, G2a). Narrower than `load_fault`, which only reaches + /// `boot_load_key`'s attempts, never this re-parse. + reparse_fault: fn() -> std::io::Result<()>, + /// Runs inside `recover_crashed_publish` between the quarantine's + /// completed-parse and the restore of the completed key to the final + /// name, so a test can interleave a concurrent starter's publish into + /// exactly the quarantine-to-restore window (#194, G2b follow-up). + before_restore: &'a dyn Fn(), + /// Faults threaded into the restore's republish of a completed + /// quarantined key, so a test can force the restore onto its link-hostile + /// degradation tier (production: `PublishFaults::NONE`). + restore_faults: PublishFaults, + /// Runs inside `recover_crashed_publish` between the post-claim re-parse + /// and the quarantine rename, so a test can land a publisher's completed + /// write in exactly the re-parse-to-rename window (#194, G2b). + before_quarantine: &'a dyn Fn(), +} + +impl RecoverySeam<'_> { + /// The production value: no fault, no hook. + const NONE: RecoverySeam<'static> = RecoverySeam { + load_fault: || Ok(()), + before_publish: &|| {}, + sweep_sync: || Ok(()), + before_reparse: &|| {}, + reparse_fault: || Ok(()), + before_restore: &|| {}, + restore_faults: PublishFaults::NONE, + before_quarantine: &|| {}, + }; +} + +fn load_or_create_keypair(config: &Config) -> Result { + load_or_create_keypair_with( + &config.resolved_key_path(), + &|| {}, + &|| {}, + PublishFaults::NONE, + RecoverySeam::NONE, + ) +} + +/// Outcome of `recover_crashed_publish` (#194, U2). +enum Recovery { + /// The round was claimed and the corrupt final quarantined (or removed); + /// the caller regenerates. Holds every claim file this recovery created + /// plus their heartbeat (`ClaimSet`), released once the follow-up + /// publish resolves (or on any error exit). + Claimed(ClaimSet), + /// The marker vanished before it could be claimed (the publisher — or a + /// competing recoverer — resolved the window concurrently): the single + /// follow-up load's result is final, success or error, no recovery. + /// Boxed to keep the variants' sizes comparable (clippy: + /// large_enum_variant); the enum lives only across a boot-time match. + Reloaded(Box>), +} + +/// The per-process nonce embedded in recovery-claim names +/// (`.{stem}.recovering.{pid}.{nonce}.{n}`). The pid alone is NOT unique +/// across recoverers on a shared volume: two containers each running as +/// PID 1 (the same restart topology `KEY_TEMP_ATTEMPTS`' stale-name +/// discipline already handles) can recover the same directory concurrently +/// and derive identical destination names, and `rename` REPLACES an +/// existing destination, so the second recoverer's claim rename would +/// clobber the first's LIVE claim inode (the claim loop's `exists()` probe +/// is check-then-act, not an arbiter). Both would then track the same +/// path, and the first release would unlink the other's live claim, +/// early-waking the demoted publisher that claim gates. The nonce makes +/// the two name sequences disjoint. Std-only seeding: boot-time nanos +/// XORed with this static's address (ASLR-shifted per process), folded to +/// 32 bits; uniqueness is probabilistic-by-construction, which suffices +/// because a collision needs the same pid AND the same nonce against the +/// same directory at the same time. Threads within one process share the +/// nonce; intra-process claim arbitration remains the per-source rename +/// (production calls this once, at boot). +fn claim_nonce() -> u32 { + static NONCE: std::sync::OnceLock = std::sync::OnceLock::new(); + *NONCE.get_or_init(|| { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos() as u64) + .unwrap_or(0); + let mixed = nanos ^ (&NONCE as *const _ as u64); + (mixed ^ (mixed >> 32)) as u32 + }) +} + +/// Production primary stamp for `refresh_claim_mtime`: set the file's mtime +/// to now via `set_modified`. +fn set_mtime_now(path: &std::path::Path) -> std::io::Result<()> { + std::fs::File::options() + .write(true) + .open(path)? + .set_modified(std::time::SystemTime::now()) +} + +/// Stamp `claim`'s mtime fresh, for the G1 age gate (see the call site in +/// `recover_crashed_publish`). `primary_stamp` is `set_mtime_now` in +/// production (injectable, the file's fn-pointer seam style; some +/// filesystems reject explicit timestamps while honoring write-driven +/// updates). If the primary fails, fall back to appending one byte, which +/// updates the mtime on effectively every filesystem, plus a best-effort +/// data sync. The claim is then no longer empty, which is safe: claims +/// vouch for no content and every consumer matches them by NAME only +/// (`list_recovery_claimables`, `sweep_stale_markers`, +/// `wait_for_recovery_claims`); nothing reads a claim's bytes. Errs only +/// when BOTH rungs fail, carrying both failures in the message. +fn refresh_claim_mtime( + claim: &std::path::Path, + primary_stamp: fn(&std::path::Path) -> std::io::Result<()>, +) -> std::io::Result<()> { + let primary_err = match primary_stamp(claim) { + Ok(()) => return Ok(()), + Err(e) => e, + }; + std::fs::File::options() + .append(true) + .open(claim) + .and_then(|mut f| { + use std::io::Write; + f.write_all(b".")?; + let _ = f.sync_data(); + Ok(()) + }) + .map_err(|fallback_err| { + std::io::Error::new( + fallback_err.kind(), + format!( + "set_modified failed ({primary_err}) and the write fallback \ + failed ({fallback_err})" + ), + ) + }) +} + +/// Handle to a live recovery's claim-heartbeat thread. Dropping it stops +/// the beat and joins the thread, so every exit path (release, error, or +/// unwind) reclaims it; the join is bounded because the thread checks the +/// stop flag at least every ~250ms (the sleep granule in +/// `spawn_claim_heartbeat`). +#[derive(Debug)] +struct ClaimHeartbeat { + stop: std::sync::Arc, + handle: Option>, +} + +impl Drop for ClaimHeartbeat { + fn drop(&mut self) { + self.stop.store(true, std::sync::atomic::Ordering::SeqCst); + if let Some(handle) = self.handle.take() { + let _ = handle.join(); + } + } +} + +/// Spawn the claim heartbeat: a thread refreshing every claim's mtime every +/// `CLAIM_SWEEP_MIN_AGE / 4` until stopped, sleeping in ~250ms granules so +/// the joining release is prompt. This is what makes the G1 age-as-orphan +/// rule sound (see `CLAIM_SWEEP_MIN_AGE`'s liveness contract): without it a +/// recoverer stalled longer than the floor between claim and settlement +/// reads as an orphan to the sweep and to claimability, so a healthy boot +/// can remove the still-live claim (early-releasing the demoted waiter to +/// a pre-settlement load) or a competing recoverer can steal the round: +/// divergence returns on slow shared volumes. The demotion waiter itself is +/// age-blind (it keys on claim-file absence; see +/// `wait_for_recovery_claims`), so a frozen mtime alone fails a demoted +/// publisher closed rather than releasing it early. Best-effort: +/// a failed spawn warns and degrades to the single claim-time stamp (the +/// pre-heartbeat exposure, churn not divergence); a failed refresh warns and +/// retries at the next beat. +fn spawn_claim_heartbeat(files: Vec) -> Option { + let stop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let thread_stop = std::sync::Arc::clone(&stop); + let spawned = std::thread::Builder::new() + .name("identity-claim-heartbeat".into()) + .spawn(move || { + let beat = CLAIM_SWEEP_MIN_AGE / 4; + let granule = std::time::Duration::from_millis(250); + 'beat: loop { + let mut slept = std::time::Duration::ZERO; + while slept < beat { + if thread_stop.load(std::sync::atomic::Ordering::SeqCst) { + break 'beat; + } + std::thread::sleep(granule); + slept += granule; + } + for claim in &files { + if thread_stop.load(std::sync::atomic::Ordering::SeqCst) { + break 'beat; + } + if let Err(e) = refresh_claim_mtime(claim, set_mtime_now) { + warn!( + claim = %claim.display(), + err = %e, + "claim heartbeat could not refresh a live claim's mtime; if \ + this persists past the liveness floor the claim reads as \ + orphaned (bounded to churn, not divergence)" + ); + } + } + } + }); + match spawned { + Ok(handle) => Some(ClaimHeartbeat { + stop, + handle: Some(handle), + }), + Err(e) => { + warn!( + err = %e, + "could not spawn the claim heartbeat; falling back to the single \ + claim-time stamp: a recovery stalled past the liveness floor may \ + read as orphaned (bounded to churn, not divergence)" + ); + None + } + } +} + +/// The claim files a recovery holds, plus the heartbeat keeping them young +/// (see `spawn_claim_heartbeat`). Carried through `Recovery::Claimed` so the +/// boot loop's follow-up publish keeps the round visibly live until +/// `release_recovery_claims`; dropping the set (any path) stops and joins +/// the heartbeat via `ClaimHeartbeat`'s `Drop`. +#[derive(Debug)] +struct ClaimSet { + files: Vec, + heartbeat: Option, +} + +impl ClaimSet { + /// No claims, no heartbeat: the boot loop's initial state. + fn empty() -> Self { + ClaimSet { + files: Vec::new(), + heartbeat: None, + } + } + + /// Wrap freshly created claim files and start their heartbeat + /// (best-effort; `None` on spawn failure). + fn with_heartbeat(files: Vec) -> Self { + let heartbeat = spawn_claim_heartbeat(files.clone()); + ClaimSet { files, heartbeat } + } +} + +/// Best-effort removal of the claim files a recovery created, plus a +/// best-effort dir fsync. Stops and joins the claim heartbeat FIRST (via +/// `ClaimHeartbeat`'s `Drop`; bounded by its ~250ms flag granule), so no +/// beat can restamp a claim after its removal. Claims vouch for NO on-disk +/// content (unlike +/// publish markers, whose disposal order against the final is pinned): they +/// only gate a demoted publisher's bounded wait and recovery arbitration. +/// Release therefore needs no ordering against the final, the markers, or +/// anything else: any exit may drop the claims at any point once their owner +/// stops recovering, and the worst a lost claim can cause is a waiter +/// re-loading early, which the load path copes with (#194, F1/F3). +fn release_recovery_claims(final_path: &std::path::Path, claims: &mut ClaimSet) { + drop(claims.heartbeat.take()); + for claim in &claims.files { + let _ = std::fs::remove_file(claim); + } + claims.files.clear(); + #[cfg(unix)] + { + let _ = fsync_parent_dir(final_path); + } +} + +/// Claim-and-quarantine for a boot whose load returned `CrashSignature`: a +/// content-class failure beside a `.publishing.` marker or `.recovering.` +/// claim, the signature of a fallback publish (or a recovery) crashed +/// inside its window (#194, U2/F2/F3). Claims the WHOLE round: every marker +/// and every pre-existing claim is renamed to a fresh claim of our own +/// (`.{stem}.recovering.{pid}.{n}`, bounded fresh-name probing per item, the +/// KEY_TEMP_ATTEMPTS discipline; a NotFound on an individual rename means +/// that item resolved concurrently and is skipped). Claiming everything is +/// what makes the publisher's commit check sound: a live fallback publisher +/// still inside its window necessarily has its own marker among the +/// claimables, so its marker removal fails and it demotes (F2); claiming +/// only one marker would let a stale marker's claim steal a live round. +/// +/// If NO rename lands, the round resolved concurrently: one immediate +/// re-load decides, with no recovery. If at least one lands, this recovery +/// owns the round, and it RE-PARSES the final once before touching it: a +/// publisher that completed inside the claim window has already resolved the +/// round, so a parseable final is returned as `Reloaded` and the claims are +/// released, never quarantining a good key. A TRANSIENT (non-content) +/// re-parse failure releases the claims and surfaces the error loudly with +/// nothing destroyed (#194, G2a). Only if the final still content-fails does +/// the recovery quarantine it by rename (preserving bytes and mode for +/// post-mortem; falling back to removal only if every quarantine name +/// fails); a quarantine whose bytes then PARSE is a publisher that completed +/// in the re-parse-to-rename window and is restored to the final name +/// (#194, G2b; see the inline comment). Otherwise recovery consumes any +/// remaining markers best-effort and hands control back to regenerate. +/// Every error exit after claiming releases +/// the claims first (see `release_recovery_claims`; a leaked claim would +/// ride later demoted publishers through the full wait bound and fail +/// their publish closed until a sweep cleared it, F1). `seam` carries +/// the test hooks for the claim-to-re-parse and re-parse-to-quarantine +/// windows (`before_reparse`, `reparse_fault`, `before_quarantine`; all +/// no-ops in production). +fn recover_crashed_publish( + final_path: &std::path::Path, + load_err: anyhow::Error, + seam: RecoverySeam<'_>, +) -> Result { + let dir = final_path + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or_else(|| std::path::Path::new(".")); + let stem = final_path + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or("identity.pem"); + let claimables = list_recovery_claimables(final_path); + if claimables.is_empty() { + return Ok(Recovery::Reloaded(Box::new(load_racing_key(final_path)))); + } + + // CLAIM every marker and foreign claim by rename. Each rename admits + // exactly one winner; NotFound means that item resolved concurrently (a + // publisher committed, or a competing recoverer claimed it) and is + // skipped. Names are never reused or clobbered: the monotonic counter + // plus the exists() probe keeps every destination fresh, bounded per + // item like the temp/marker/quarantine probes. + let claim_prefix = recovering_prefix(stem); + let mut claims: Vec = Vec::new(); + let mut next_name = 0u32; + for item in &claimables { + for _ in 0..KEY_TEMP_ATTEMPTS { + // The name carries a per-process nonce beside the pid: pid alone + // is not unique across recoverers (two PID-1 containers on one + // shared volume), and a colliding destination would let one + // recoverer's rename clobber the other's live claim; see + // `claim_nonce`. + let dest = dir.join(format!( + "{claim_prefix}{}.{:08x}.{next_name}", + std::process::id(), + claim_nonce() + )); + next_name = next_name.wrapping_add(1); + if dest.exists() { + continue; + } + match std::fs::rename(item, &dest) { + Ok(()) => { + // The rename preserves the SOURCE's mtime, but the G1 + // sweep gate reads the CLAIM file's mtime: a claim made + // from an aged stale marker would be born already past + // CLAIM_SWEEP_MIN_AGE, and a concurrent healthy boot + // could sweep this LIVE claim mid-round, early-waking + // the demoted publisher it gates. The age gate needs + // claim-time freshness, so stamp it now (two rungs; see + // refresh_claim_mtime). Only if both rungs fail is the + // live claim left exposed to the age-gated sweep, so + // warn naming that hazard and continue: the exposure is + // bounded to churn, not divergence, by the post-claim + // re-parse and the restore's no-clobber republish. + if let Err(e) = refresh_claim_mtime(&dest, set_mtime_now) { + warn!( + claim = %dest.display(), + err = %e, + "could not refresh a recovery claim's mtime by any rung: a \ + concurrent healthy boot's age-gated sweep may remove this \ + LIVE claim mid-round and early-wake the demoted publisher \ + it gates (bounded to churn, not divergence, by the \ + post-claim re-parse, the restore's no-clobber republish, \ + and the demoted publisher's waiter failing closed on an \ + unsettled round)" + ); + } + claims.push(dest); + break; + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => break, + // Any other rename failure: probe the next fresh name, up to + // the per-item bound; an unclaimable item is left for the + // best-effort marker consumption below (or a later sweep). + Err(_) => continue, + } + } + } + if claims.is_empty() { + // Every item vanished before it could be claimed: the round resolved + // concurrently; one immediate re-load decides, with no recovery. + return Ok(Recovery::Reloaded(Box::new(load_racing_key(final_path)))); + } + // CLAIM HEARTBEAT: from here until release these claims are load-bearing + // (they gate demoted publishers and arbitrate the round), and the G1 age + // rule reads aged-out as orphaned, so keep them visibly live for as long + // as this recovery actually runs; see spawn_claim_heartbeat. + let mut claims = ClaimSet::with_heartbeat(claims); + + // POST-CLAIM RE-PARSE (#194, F2): between the crash-signature + // observation and the renames above, a live publisher may have completed + // its final. Now that the round is claimed no publisher can commit + // anymore, so one parse decides: a readable final means the round is + // already resolved, so release the claims and defer to it, never + // quarantining a good key. Only a CONTENT-class failure proceeds to the + // quarantine (#194, G2a): a transient (IO/permissions) failure of a + // PRESENT final proves nothing about the content, and quarantining on it + // would destroy a possibly good key, so the claims are released and the + // error surfaces loudly with nothing destroyed. A NotFound is neither: a + // vanished final means a competing recoverer disposed of it + // concurrently, there is nothing a quarantine could destroy, and the + // quarantine loop's own NotFound arm resolves it into a clean + // regenerate, so it falls through like the content class. + (seam.before_reparse)(); + match (seam.reparse_fault)() + .map_err(anyhow::Error::new) + .and_then(|()| load_existing_key(final_path)) + { + Ok(kp) => { + release_recovery_claims(final_path, &mut claims); + return Ok(Recovery::Reloaded(Box::new(Ok(kp)))); + } + Err(e) + if is_key_content_error(&e) + || e.downcast_ref::() + .is_some_and(|io| io.kind() == std::io::ErrorKind::NotFound) => + { + // Still the crashed content, or vanished concurrently: fall + // through to the quarantine. + } + Err(e) => { + release_recovery_claims(final_path, &mut claims); + return Err(e.context(format!( + "transient failure re-parsing identity key {} after claiming crash \ + recovery; claims released, nothing quarantined", + final_path.display() + ))); + } + } + + // QUARANTINE the final under a bounded name probe (the KEY_TEMP_ATTEMPTS + // discipline: a stale quarantine from a crashed same-PID start must not + // wedge this one). The warn precedes the move and carries stable + // operator-greppable fields; the destination it names is the first free + // candidate, which the rename below tries first. `before_quarantine` is + // the test seam for the re-parse-to-rename window (#194, G2b; a no-op in + // production). + (seam.before_quarantine)(); + let dest_for = |n: u32| { + dir.join(format!( + "{}{}.{n}", + quarantined_prefix(stem), + std::process::id() + )) + }; + let start = (0..KEY_TEMP_ATTEMPTS) + .find(|&n| !dest_for(n).exists()) + .unwrap_or(0); + warn!( + path = %final_path.display(), + claimables = ?claimables, + claims = ?claims.files, + quarantine = %dest_for(start).display(), + "crash-interrupted identity key publish: quarantining the unreadable key and \ + regenerating the identity" + ); + let mut quarantined = false; + let mut quarantined_at: Option = None; + for n in start..KEY_TEMP_ATTEMPTS { + let dest = dest_for(n); + if dest.exists() { + continue; + } + match std::fs::rename(final_path, &dest) { + Ok(()) => { + quarantined = true; + quarantined_at = Some(dest); + break; + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + // The final vanished concurrently; nothing left to quarantine. + quarantined = true; + break; + } + Err(_) => continue, + } + } + if !quarantined { + // Every rename failed: removal still unblocks the regeneration + // (mirroring the publish paths' removal policy); only if even that + // fails is the boot left to its loud error, with the claims + // released first, so the dead round cannot stall later publishers + // (#194, F1). + if let Err(e) = std::fs::remove_file(final_path) { + if e.kind() != std::io::ErrorKind::NotFound { + release_recovery_claims(final_path, &mut claims); + return Err(load_err.context(format!( + "could not quarantine or remove corrupt identity key {} during crash \ + recovery: {e}", + final_path.display() + ))); + } + } + } + + // POST-QUARANTINE COMPLETED-WRITE CHECK (#194, G2b): the re-parse above + // and the quarantine rename are not atomic, so a publisher that + // completed its write inside the re-parse-to-rename window has its + // VALID key swept into the quarantine. The rename preserved the bytes + // (and the 0600 inode), so parse them: a parseable quarantine means that + // publisher already resolved the round, and its key is restored to the + // final name instead of being replaced by a regeneration, which would + // diverge from the Won the publisher already reported. The restore is a + // REPUBLISH of the quarantined PEM through publish_key_atomically, which + // holds no-clobber on EVERY tier: the hard-link tier refuses an existing + // final, and so does the fallback's create_new on link-hostile mounts (a + // plain rename here would clobber a final a fresh start published into + // the freed name on exactly those mounts). Won means the completed key + // is durably back at the final; the quarantine is then removed, because + // its bytes are an exact copy of what now lives at the final, so keeping + // it buys no forensics. The keypair returned is the one already parsed + // from the quarantined bytes, not a re-read of the final, which could + // race a writer that lands after our republish. Won also supersedes + // every OTHER young parseable quarantine sibling (they lost this round + // to the restored key, the N4 rule adoption-Won already applies); left + // adoptable, a later missing final would resurrect a different + // historical DID. Lost means someone else + // durably published meanwhile: the quarantined key provably LOST its + // round, so it is renamed into the superseded class (bytes preserved as + // forensics, never adoptable; a young parseable quarantine left in the + // adoptable class here would let the operator's delete-final-for-a- + // fresh-DID procedure resurrect the losing key, N2) and the boot defers + // to the settled final. A publish error keeps the quarantine ADOPTABLE + // (the next boot's F1 adoption is the remedy for a failed restore), + // releases the claims, and surfaces loudly, nothing destroyed. An + // unparseable quarantine is the expected crash state and falls through + // to regenerate. + if let Some(quarantine) = &quarantined_at { + let completed = std::fs::read_to_string(quarantine) + .ok() + .and_then(|pem| Keypair::from_pem(&pem).ok().map(|kp| (pem, kp))); + if let Some((pem, kp)) = completed { + (seam.before_restore)(); + match publish_key_atomically( + final_path, + pem.as_bytes(), + &|| {}, + &|| {}, + seam.restore_faults, + ) { + Ok(KeyPublish::Won) => { + // Mirror-arm invariant (N4): the restore decided this + // round in favor of the restored key, so any OTHER + // young parseable quarantine sibling provably lost it; + // supersede each, exactly like adoption-Won, or a later + // missing final resurrects a different historical DID. + // ORDER: siblings first, THEN consume the chosen + // quarantine, so a crash in the gap leaves the siblings + // already superseded, never adoptable beside a final + // that already holds the winner. + supersede_sibling_quarantines(final_path, quarantine); + let _ = std::fs::remove_file(quarantine); + #[cfg(unix)] + { + let _ = fsync_parent_dir(final_path); + } + release_recovery_claims(final_path, &mut claims); + return Ok(Recovery::Reloaded(Box::new(Ok(kp)))); + } + Ok(KeyPublish::Lost) => { + supersede_quarantine(final_path, quarantine); + release_recovery_claims(final_path, &mut claims); + return Ok(Recovery::Reloaded(Box::new(load_racing_key(final_path)))); + } + Err(e) => { + release_recovery_claims(final_path, &mut claims); + return Err(e.context(format!( + "could not restore the completed identity key to {} during crash \ + recovery; quarantine kept at {}", + final_path.display(), + quarantine.display() + ))); + } + } + } + } + + // Consume any remaining markers (a multi-crash pile-up, or an item the + // claim probe could not rename) best-effort, and best-effort fsync the + // directory so the transition tends to be durable. + for marker in list_publish_markers(final_path) { + let _ = std::fs::remove_file(&marker); + } + #[cfg(unix)] + { + let _ = fsync_parent_dir(final_path); + } + Ok(Recovery::Claimed(claims)) +} + +/// Body of `load_or_create_keypair` with the test seams threaded in (#194, +/// U2): `before_link`, `before_commit`, and `faults` pass through to +/// `publish_key_atomically` (production: no-op / `NONE`), `seam` is the +/// load-side recovery seam (production: `RecoverySeam::NONE`). +/// +/// Load-or-generate with AT MOST ONE crash recovery per start: a load +/// failure recovers (claim the round, quarantine the final, regenerate) +/// only when it is content-class (`KeyContentError`), a `.publishing.` +/// marker or `.recovering.` claim exists (the crash signature, persisting +/// for `CRASH_SIGNATURE_MIN_PERSIST`), and no recovery has run yet this start. Every +/// other failure surfaces unchanged. The Lost arm participates in the same +/// policy; a recovery fired there loops back for one retry publish, and any +/// failure on that second pass surfaces unchanged. +fn load_or_create_keypair_with( + key_path: &std::path::Path, + before_link: &dyn Fn(), + before_commit: &dyn Fn(), + faults: PublishFaults, + seam: RecoverySeam<'_>, +) -> Result { + // The claim set (files plus heartbeat) of a fired recovery. Held while + // the follow-up + // generate+publish runs, so a stalled winner our claims demoted can still + // observe them while waiting (U3 relies on this), and released on EVERY + // exit after that, success, Lost, and error alike (#194, F1): a leaked + // claim rides every later demoted publisher through the full wait bound + // and fails its publish closed until a sweep clears the orphan. + // Release order against the final or the markers does not matter; see + // release_recovery_claims. + let mut claims = ClaimSet::empty(); + let mut recovery_allowed = true; + + // N1 invariant: EVERY arm that returns a successfully resolved keypair + // (loaded, reloaded, adopted, or generated) must run the durability-gated + // stale-marker sweep before returning, by funneling through this closure. + // A healthy return that skips the sweep leaves unclaimable markers or + // aged orphan claims beside a good final, and a LATER content failure of + // that final would pair with the leftover residue and misclassify plain + // corruption as a recoverable crash. Route any future healthy return + // through here too, so no arm can forget the sweep. + let finish_healthy = |kp: Keypair| -> Result { + sweep_stale_markers(key_path, seam.sweep_sync); + Ok(kp) + }; + + // At most two passes: the second exists only so a Lost-arm recovery can + // retry the generate+publish once. + for _pass in 0..2 { + // Fast path for the common already-provisioned case (still race-safe: a + // concurrently-publishing winner is handled by the retry in + // boot_load_key). + if key_path.exists() { + match boot_load_key(key_path, recovery_allowed, seam.load_fault) { + BootLoad::Loaded(kp) => { + release_recovery_claims(key_path, &mut claims); + return finish_healthy(*kp); + } + // Only emitted while recovery is still allowed (crash_exit is + // gated on it). + BootLoad::CrashSignature(e) => { + match recover_crashed_publish(key_path, e, seam)? { + Recovery::Claimed(c) => { + claims = c; + recovery_allowed = false; + // Fall through to regenerate and publish. + } + Recovery::Reloaded(result) => return (*result).and_then(&finish_healthy), + } + } + BootLoad::Failed(e) => { + release_recovery_claims(key_path, &mut claims); + return Err(e); + } + } + } + + (seam.before_publish)(); + + // ADOPTION SCAN (F1): with no loadable final, a COMPLETED key may be + // stranded in a quarantine: the recoverer's restore republish + // errored, or the recoverer crashed between its quarantine rename + // and the restore. Generating here would mint a fresh DID over a + // still-live identity, so a YOUNG parseable quarantine (the current + // round's; see find_adoptable_quarantine's age bound) is republished + // and adopted instead: adoption preserves identity continuity across + // a crashed or failed restore. An unparseable quarantine is the + // expected crash state and stays on disk as forensics, as does an + // AGED parseable one (history, never resurrected); only when no + // quarantine qualifies does the boot fall through to generate. The republish holds no-clobber on every tier, so Lost + // means a concurrent starter durably published meanwhile: the + // quarantine provably LOST its round, so it moves to the superseded + // class and the boot defers to the winner, exactly like the G2b + // restore's Lost arm. A + // republish error surfaces loudly with the quarantine kept: minting + // a fresh identity over an adoptable one is the failure F1 exists to + // prevent. + if let Some((quarantine, pem, kp)) = find_adoptable_quarantine(key_path) { + match publish_key_atomically(key_path, pem.as_bytes(), &|| {}, &|| {}, faults) { + Ok(KeyPublish::Won) => { + // N4: any OTHER young parseable quarantine sibling lost + // this round to the adopted key; move each into the + // superseded class so a later missing final cannot + // resurrect a different historical DID. Unparseable + // siblings stay plain quarantines as crash forensics. + // ORDER: siblings first, THEN consume the chosen + // quarantine, so a crash in the gap leaves the siblings + // already superseded, never adoptable beside a final + // that already holds the winner. + supersede_sibling_quarantines(key_path, &quarantine); + // The quarantine's bytes are an exact copy of what now + // durably lives at the final, so keeping it buys no + // forensics (the G2b restore's removal rationale). + let _ = std::fs::remove_file(&quarantine); + #[cfg(unix)] + { + let _ = fsync_parent_dir(key_path); + } + release_recovery_claims(key_path, &mut claims); + info!( + path = %key_path.display(), + did = %kp.did(), + "adopted a completed identity stranded in quarantine" + ); + return finish_healthy(kp); + } + Ok(KeyPublish::Lost) => { + // Mirror-arm invariant (N2): EVERY Lost outcome + // supersedes the losing bytes. This quarantine just + // provably LOST its round to the concurrent winner; + // left adoptable, the operator's delete-final-for-a- + // fresh-DID procedure would resurrect it within the age + // floor, exactly the hazard the restore's Lost arm + // already closes. + supersede_quarantine(key_path, &quarantine); + let result = load_racing_key(key_path); + release_recovery_claims(key_path, &mut claims); + return result.and_then(&finish_healthy); + } + Err(e) => { + release_recovery_claims(key_path, &mut claims); + return Err(e.context(format!( + "could not republish the completed identity key stranded at {}; \ + quarantine kept", + quarantine.display() + ))); + } + } + } + + let kp = Keypair::generate(); + // Every error exit below this point may hold a fired recovery's + // claims; release them before propagating (#194, F1). The bare `?` + // this replaces leaked the claim files on a failed retry publish. + let pem = match kp.to_pem() { + Ok(pem) => pem, + Err(e) => { + release_recovery_claims(key_path, &mut claims); + return Err(anyhow::anyhow!("failed to serialize key: {e}")); + } + }; + + if let Some(parent) = key_path.parent() { + if let Err(e) = std::fs::create_dir_all(parent) { + release_recovery_claims(key_path, &mut claims); + return Err(e.into()); + } + } + + // Publish atomically: the final path only ever appears complete, and a + // lost race loads the winner's key rather than overwriting it. + let published = match publish_key_atomically( + key_path, + pem.as_bytes(), + before_link, + before_commit, + faults, + ) { + Ok(published) => published, + Err(e) => { + release_recovery_claims(key_path, &mut claims); + return Err(e); + } + }; + match published { + KeyPublish::Won => { + release_recovery_claims(key_path, &mut claims); + info!(path = %key_path.display(), did = %kp.did(), "generated new node identity"); + return finish_healthy(kp); + } + KeyPublish::Lost => { + match boot_load_key(key_path, recovery_allowed, seam.load_fault) { + BootLoad::Loaded(kp) => { + release_recovery_claims(key_path, &mut claims); + return finish_healthy(*kp); + } + // Only emitted while recovery is still allowed (crash_exit + // is gated on it). + BootLoad::CrashSignature(e) => { + match recover_crashed_publish(key_path, e, seam) { + Ok(Recovery::Claimed(c)) => { + claims = c; + recovery_allowed = false; + continue; // Second pass: regenerate and publish. + } + Ok(Recovery::Reloaded(result)) => { + release_recovery_claims(key_path, &mut claims); + return (*result).and_then(&finish_healthy); + } + Err(err) => { + release_recovery_claims(key_path, &mut claims); + return Err(err); + } + } + } + BootLoad::Failed(e) => { + release_recovery_claims(key_path, &mut claims); + return Err(e); + } + } + } + } + } + unreachable!("the boot loop returns within two passes"); +} + +#[cfg(test)] +mod gossip_ssrf_tests { + use super::ping_peer_health; + + // Build the client exactly as production does (super::build_http_client) so + // these tests bind the redirect guarantee to the real shared client the + // node runs. A regression that makes build_http_client follow redirects + // fails ping_peer_health_does_not_follow_redirect. + fn production_http_client() -> reqwest::Client { + super::build_http_client().expect("failed to build production http client") + } + + // A peer answering `/health` with a 302 toward an internal address must not + // be followed: the redirect target must never be requested (#93). + #[tokio::test] + async fn ping_peer_health_does_not_follow_redirect() { + let mut server = mockito::Server::new_async().await; + let internal = server + .mock("GET", "/internal-metadata") + .with_status(200) + .expect(0) + .create_async() + .await; + let _health = server + .mock("GET", "/health") + .with_status(302) + .with_header("location", &format!("{}/internal-metadata", server.url())) + .create_async() + .await; + + let ok = ping_peer_health(&production_http_client(), &server.url()).await; + + assert!(!ok, "a 302 must not count as a healthy peer"); + // expect(0) is enforced only at assert time; this fails if the redirect + // was followed to the internal target. + internal.assert_async().await; + } + + #[tokio::test] + async fn ping_peer_health_reports_success_on_200() { + let mut server = mockito::Server::new_async().await; + let _health = server + .mock("GET", "/health") + .with_status(200) + .create_async() + .await; + + let ok = ping_peer_health(&production_http_client(), &server.url()).await; + + assert!(ok, "a 200 /health must count as a healthy peer"); + } + + // A transport error (nothing listening) must map to unhealthy, never a + // spurious healthy — the .unwrap_or(false) arm. + #[tokio::test] + async fn ping_peer_health_reports_unhealthy_on_connection_error() { + let ok = ping_peer_health(&production_http_client(), "http://127.0.0.1:1").await; + assert!(!ok, "a connection error must count as an unhealthy peer"); + } +} + +#[cfg(all(test, unix))] +mod identity_key_tests { + use super::{ + load_or_create_keypair, load_or_create_keypair_with, load_racing_key, + publish_key_atomically, Config, KeyPublish, Keypair, PublishFaults, RecoverySeam, + KEY_TEMP_ATTEMPTS, + }; + use clap::Parser; + use std::os::unix::fs::PermissionsExt; + + // #194 (P2): the racing loader must wait out a SLOW winner on a meaningful + // wall-clock deadline, not give up after an arbitrary ~100ms window. Simulate + // a winner that only publishes the key after 250ms (past the old 100ms budget) + // and assert the loader waits it out instead of failing with `invalid PEM key`. + // RED with the old `for _ in 0..50` (2ms) fixed-count loop. + #[test] + fn load_racing_key_waits_out_a_slow_winner() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let pem = Keypair::generate().to_pem().expect("pem"); + + let writer_path = key_path.clone(); + let writer = std::thread::spawn(move || { + std::thread::sleep(std::time::Duration::from_millis(250)); + publish_key_atomically( + &writer_path, + pem.as_bytes(), + &|| {}, + &|| {}, + PublishFaults::NONE, + ) + .expect("publish"); + }); + + let started = std::time::Instant::now(); + let kp = load_racing_key(&key_path) + .expect("racing load must wait out the slow winner, not fail"); + let waited = started.elapsed(); + writer.join().expect("writer joins"); + + assert!( + waited >= std::time::Duration::from_millis(200), + "loader should have waited for the ~250ms-slow winner, only waited {waited:?}" + ); + assert!( + !format!("{}", kp.did()).is_empty(), + "loaded the published key" + ); + } + + // #194 (P2): the atomic publish preserves the single-winner no-clobber + // guarantee — a second publish of a DIFFERENT key must lose and leave the + // winner's key untouched — and it cleans up its temp files. + #[test] + fn publish_wins_then_second_publish_loses_without_clobbering() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let pem_a = Keypair::generate().to_pem().expect("pem a"); + let pem_b = Keypair::generate().to_pem().expect("pem b"); + assert_ne!(pem_a.as_str(), pem_b.as_str(), "fixtures must differ"); + + let out = publish_key_atomically( + &key_path, + pem_a.as_bytes(), + &|| {}, + &|| {}, + PublishFaults::NONE, + ) + .expect("publish a"); + assert!(matches!(out, KeyPublish::Won), "first publish wins"); + assert_eq!( + std::fs::read_to_string(&key_path).unwrap().as_str(), + pem_a.as_str(), + "final holds the full PEM" + ); + + let out = publish_key_atomically( + &key_path, + pem_b.as_bytes(), + &|| {}, + &|| {}, + PublishFaults::NONE, + ) + .expect("publish b"); + assert!(matches!(out, KeyPublish::Lost), "second publish loses"); + assert_eq!( + std::fs::read_to_string(&key_path).unwrap().as_str(), + pem_a.as_str(), + "the winner's key must NOT be clobbered by a losing publish" + ); + + let leftovers: Vec<_> = std::fs::read_dir(dir.path()) + .unwrap() + .filter_map(|e| e.ok()) + .filter(|e| e.file_name().to_string_lossy().contains(".tmp.")) + .collect(); + assert!( + leftovers.is_empty(), + "temp files must be cleaned up, found {leftovers:?}" + ); + } + + // #194 (P2, the core of jatmn's option b): while a winner is between writing + // its temp and linking it into place, a reader watching the FINAL path must + // see it absent or COMPLETE — never empty/partial. RED with a + // create_new(final)+write approach, which exposes an empty final for the whole + // write window. + #[test] + fn publish_never_exposes_a_partial_final() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = std::sync::Arc::new(dir.path().join("identity.pem")); + let pem = Keypair::generate().to_pem().expect("pem"); + + let wp = key_path.clone(); + let writer = std::thread::spawn(move || { + // Hold the post-write / pre-link window open for 200ms. + publish_key_atomically( + &wp, + pem.as_bytes(), + &|| { + std::thread::sleep(std::time::Duration::from_millis(200)); + }, + &|| {}, + PublishFaults::NONE, + ) + .expect("publish"); + }); + + // Poll until the final path appears (bounded by a generous deadline: + // the publish now fsyncs the temp and the directory, so under parallel + // suite load the link can land well after the writer's 200ms hold). + // Every observation of an existing final must parse; the first + // complete observation ends the test, since a linked-complete final + // cannot become partial afterwards. The partial assertion still runs + // BEFORE the break, so a publish that exposes an empty/partial final + // during the write window is caught on the first poll that sees it. + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); + let mut saw_complete = false; + while std::time::Instant::now() < deadline { + if key_path.exists() { + let body = std::fs::read_to_string(&*key_path).unwrap_or_default(); + assert!( + Keypair::from_pem(&body).is_ok(), + "final key observed in a partial/empty state: {body:?}" + ); + saw_complete = true; + break; + } + std::thread::sleep(std::time::Duration::from_millis(1)); + } + writer.join().expect("writer joins"); + assert!( + saw_complete, + "reader should have observed the completed key within the deadline" + ); + } + + // #194: a temp file left by a crash between the temp write and its cleanup + // must not wedge the restart. A restarted container commonly runs as PID 1 + // again, so the first name tried collides with the stale one; the publish + // must skip it and still win with the complete PEM. RED with a single + // deterministic temp name: `create_new` fails AlreadyExists and the node + // never starts until an operator deletes the temp. + #[test] + fn publish_recovers_from_a_stale_crashed_temp() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let pem = Keypair::generate().to_pem().expect("pem"); + + // The crashed prior start's leftover: the deterministic attempt-0 name, + // 0600, holding a partial write. + let stale = dir + .path() + .join(format!(".identity.pem.tmp.{}.0", std::process::id())); + std::fs::write(&stale, b"partial-").expect("seed stale temp"); + std::fs::set_permissions(&stale, std::fs::Permissions::from_mode(0o600)) + .expect("stale temp perms"); + + let out = publish_key_atomically( + &key_path, + pem.as_bytes(), + &|| {}, + &|| {}, + PublishFaults::NONE, + ) + .expect("publish must recover from a stale temp, not wedge the start"); + assert!(matches!(out, KeyPublish::Won), "publish wins"); + assert_eq!( + std::fs::read_to_string(&key_path).unwrap().as_str(), + pem.as_str(), + "final holds the complete PEM, not the stale partial" + ); + } + + // #194: temp-name probing is bounded. With every candidate name taken the + // publish must return a clear error naming the directory and prefix (not + // hang or spin) and must not create the final path. Together with the + // stale-temp test this runs the exhaustion branch both ways. + #[test] + fn publish_errors_when_all_temp_names_are_taken() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let pem = Keypair::generate().to_pem().expect("pem"); + + for attempt in 0..KEY_TEMP_ATTEMPTS { + let tmp = dir.path().join(format!( + ".identity.pem.tmp.{}.{attempt}", + std::process::id() + )); + std::fs::write(&tmp, b"taken").expect("seed taken temp"); + } + + let err = match publish_key_atomically( + &key_path, + pem.as_bytes(), + &|| {}, + &|| {}, + PublishFaults::NONE, + ) { + Err(e) => e, + Ok(_) => panic!("publish must fail when every temp name is taken"), + }; + let msg = format!("{err:#}"); + assert!( + msg.contains(&format!(".identity.pem.tmp.{}", std::process::id())) + && msg.contains(&dir.path().display().to_string()), + "the error must name the temp prefix and directory: {msg}" + ); + assert!( + !key_path.exists(), + "an exhausted publish must not create or modify the final key" + ); + } + + // A freshly created identity key must be 0600 immediately, with no + // world-readable disclosure window (the atomic create_new(...).mode(0o600) + // guarantee). This is the RED-then-GREEN anchor for the perms fix: the old + // fs::write + set_permissions sequence left a 0644 window. + #[test] + fn created_key_is_mode_0600() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let config = Config::parse_from([ + "gitlawb-node", + "--key-path", + key_path.to_str().expect("utf8 path"), + ]); + + let _kp = load_or_create_keypair(&config).expect("create keypair"); + + let mode = std::fs::metadata(&key_path) + .expect("key file exists") + .permissions() + .mode(); + assert_eq!( + mode & 0o777, + 0o600, + "identity key must be 0600, got {:o}", + mode & 0o777 + ); + } + + // A lost create race (file already present) must load the existing key + // rather than overwrite it, and must not error on AlreadyExists. Loading an + // existing loose-permission key tightens it to 0600 defensively. + #[test] + fn existing_key_is_loaded_and_tightened() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let config = Config::parse_from([ + "gitlawb-node", + "--key-path", + key_path.to_str().expect("utf8 path"), + ]); + + let first = load_or_create_keypair(&config).expect("create keypair"); + // Loosen perms to simulate a legacy/loose on-disk key. + std::fs::set_permissions(&key_path, std::fs::Permissions::from_mode(0o644)) + .expect("loosen perms"); + + let second = load_or_create_keypair(&config).expect("load keypair"); + assert_eq!( + first.did(), + second.did(), + "reloading must return the same identity, not a new one" + ); + + let mode = std::fs::metadata(&key_path) + .expect("key file exists") + .permissions() + .mode(); + assert_eq!( + mode & 0o777, + 0o600, + "load path must tighten loose perms to 0600, got {:o}", + mode & 0o777 + ); + } + + // The create-race arm the single-threaded tests skip (they enter via the + // `exists()` fast path). N concurrent starts on ONE fresh path must converge + // on a single identity: the atomic publish links exactly one winner and the + // losers hit `AlreadyExists` and load the winner's key rather than overwriting + // it. RED: replace the atomic publish with a plain `fs::write` and each thread + // returns its own freshly generated key, so the DIDs diverge. + #[test] + fn concurrent_starts_converge_on_one_identity() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let config = std::sync::Arc::new(Config::parse_from([ + "gitlawb-node", + "--key-path", + key_path.to_str().expect("utf8 path"), + ])); + + let n = 8; + let barrier = std::sync::Arc::new(std::sync::Barrier::new(n)); + let handles: Vec<_> = (0..n) + .map(|_| { + let c = config.clone(); + let b = barrier.clone(); + std::thread::spawn(move || { + // Release all threads at once to maximize the race. + b.wait(); + format!("{}", load_or_create_keypair(&c).expect("keypair").did()) + }) + }) + .collect(); + let dids: Vec = handles + .into_iter() + .map(|h| h.join().expect("thread joins")) + .collect(); + + let first = &dids[0]; + assert!( + dids.iter().all(|d| d == first), + "all concurrent starts must converge on one identity, got: {dids:?}" + ); + } + + /// #194 (F1): a failed write of a just-created key file removes it, so a later + /// start regenerates instead of re-parsing an empty/partial PEM forever and + /// wedging on `invalid PEM key`. RED without the `remove_file` in + /// `write_key_or_cleanup`: the partial file survives the error. + #[test] + fn failed_write_removes_the_partial_key_file() { + use super::write_key_or_cleanup; + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + std::fs::write(&key_path, b"partial-pem").expect("seed partial file"); + assert!(key_path.exists()); + + let err = std::io::Error::other("simulated ENOSPC"); + let r = write_key_or_cleanup(&key_path, Err(err)); + assert!(r.is_err(), "the write error must propagate"); + assert!( + !key_path.exists(), + "a failed first write must not leave a partial key file to wedge later starts" + ); + } + + /// A successful write leaves the file in place (the common case). + #[test] + fn successful_write_keeps_the_key_file() { + use super::write_key_or_cleanup; + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + std::fs::write(&key_path, b"good-pem").expect("seed file"); + assert!(write_key_or_cleanup(&key_path, Ok(())).is_ok()); + assert!( + key_path.exists(), + "a successful write leaves the file in place" + ); + } + + /// #194 (F2): a loose (world/group-readable) key that cannot be tightened is + /// rejected fail-closed rather than read and used exposed; a 0600 key is + /// accepted. RED without `ensure_key_mode_0600`: a 0644 key is used silently. + #[test] + fn loose_key_mode_is_rejected_not_used() { + use super::ensure_key_mode_0600; + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + std::fs::write(&key_path, b"key").expect("seed file"); + + std::fs::set_permissions(&key_path, std::fs::Permissions::from_mode(0o644)) + .expect("loosen perms"); + let err = ensure_key_mode_0600(&key_path) + .expect_err("a world/group-readable key must be rejected") + .to_string(); + assert!( + err.contains("644"), + "the failure must name the exposed mode: {err}" + ); + + std::fs::set_permissions(&key_path, std::fs::Permissions::from_mode(0o600)) + .expect("tighten perms"); + assert!( + ensure_key_mode_0600(&key_path).is_ok(), + "a 0600 key must be accepted" + ); + } + + /// Shared body for the link-failure fallback tests: with the link step + /// forced to fail with `kind`, the publish must fall back to a direct + /// `create_new(0o600)` on the final path and win; the key must be 0600 and + /// round-trip through the normal loader. RED without the fallback: the + /// link error propagates and the start fails. + fn assert_link_failure_falls_back(faults: PublishFaults) { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let kp = Keypair::generate(); + let pem = kp.to_pem().expect("pem"); + + let out = publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, &|| {}, faults) + .expect("a failed hard_link must fall back to a direct publish, not fail the start"); + assert!(matches!(out, KeyPublish::Won), "fallback publish wins"); + + let mode = std::fs::metadata(&key_path) + .expect("key file exists") + .permissions() + .mode(); + assert_eq!( + mode & 0o777, + 0o600, + "fallback-published key must be 0600, got {:o}", + mode & 0o777 + ); + let loaded = super::load_existing_key(&key_path) + .expect("fallback-published key loads via the normal loader"); + assert_eq!( + format!("{}", loaded.did()), + format!("{}", kp.did()), + "fallback publish must round-trip the same identity" + ); + } + + // A key directory whose filesystem rejects hard links with EPERM (some + // network/overlay mounts) must not brick a new node. + #[test] + fn permission_denied_link_falls_back_to_direct_create() { + assert_link_failure_falls_back(PublishFaults { + link: || Err(std::io::ErrorKind::PermissionDenied.into()), + ..PublishFaults::NONE + }); + } + + // A filesystem without hard-link support at all (Unsupported). + #[test] + fn unsupported_link_falls_back_to_direct_create() { + assert_link_failure_falls_back(PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + ..PublishFaults::NONE + }); + } + + // Link failure with the final ALREADY present must route to the lost-race + // path: the fallback's create_new hits AlreadyExists, the existing key is + // loaded, and it is never clobbered. + #[test] + fn link_failure_with_existing_final_loses_without_clobbering() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let winner = Keypair::generate(); + let pem_winner = winner.to_pem().expect("pem winner"); + let out = publish_key_atomically( + &key_path, + pem_winner.as_bytes(), + &|| {}, + &|| {}, + PublishFaults::NONE, + ) + .expect("winner publishes"); + assert!(matches!(out, KeyPublish::Won), "winner publish wins"); + + let pem_loser = Keypair::generate().to_pem().expect("pem loser"); + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::PermissionDenied.into()), + ..PublishFaults::NONE + }; + let out = publish_key_atomically(&key_path, pem_loser.as_bytes(), &|| {}, &|| {}, faults) + .expect("an existing final must read as a lost race, not an error"); + assert!( + matches!(out, KeyPublish::Lost), + "fallback loses to the existing key" + ); + assert_eq!( + std::fs::read_to_string(&key_path).unwrap().as_str(), + pem_winner.as_str(), + "the existing key must NOT be clobbered by the fallback" + ); + let loaded = load_racing_key(&key_path).expect("lost race loads the winner"); + assert_eq!( + format!("{}", loaded.did()), + format!("{}", winner.did()), + "the lost race must return the winner's identity" + ); + } + + // Two concurrent publishers, both with the link step failing: the + // fallback's create_new must admit exactly one winner and the loser must + // converge on the winner's key (no-clobber survives the fallback path). + #[test] + fn concurrent_fallback_publishers_converge_on_one_winner() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = std::sync::Arc::new(dir.path().join("identity.pem")); + let barrier = std::sync::Arc::new(std::sync::Barrier::new(2)); + let handles: Vec<_> = (0..2) + .map(|_| { + let kp = key_path.clone(); + let b = barrier.clone(); + std::thread::spawn(move || { + let me = Keypair::generate(); + let pem = me.to_pem().expect("pem"); + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + ..PublishFaults::NONE + }; + // Release both threads at once to maximize the race. + b.wait(); + match publish_key_atomically(&kp, pem.as_bytes(), &|| {}, &|| {}, faults) + .expect("fallback publish") + { + KeyPublish::Won => (true, format!("{}", me.did())), + KeyPublish::Lost => ( + false, + format!( + "{}", + load_racing_key(&kp).expect("loser loads winner").did() + ), + ), + } + }) + }) + .collect(); + let results: Vec<(bool, String)> = handles + .into_iter() + .map(|h| h.join().expect("thread joins")) + .collect(); + + let winners = results.iter().filter(|(won, _)| *won).count(); + assert_eq!( + winners, 1, + "exactly one fallback create_new must win, got {results:?}" + ); + assert_eq!( + results[0].1, results[1].1, + "the loser must converge on the winner's identity: {results:?}" + ); + } + + // A failed fallback WRITE leaves the partial final AND its marker in + // place: the pair is exactly the crash state boot-time recovery handles + // (claim, quarantine, regenerate), while an unlink here is a + // stat-then-unlink race against a recovery that already claimed this + // round and republished a live key at the name, and a marker disposed + // beside a kept partial final recreates the marker-less wedge the + // bracket exists to prevent. The error must surface the write failure + // AND the original link error, so a two-step failure is diagnosable. + #[test] + fn failed_fallback_write_leaves_the_partial_final_for_recovery() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let pem = Keypair::generate().to_pem().expect("pem"); + + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + fallback_write: || Err(std::io::Error::other("injected write failure")), + ..PublishFaults::NONE + }; + let err = match publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, &|| {}, faults) { + Err(e) => e, + Ok(_) => panic!("a failed fallback write must error the start"), + }; + let msg = format!("{err:#}"); + assert!( + msg.contains("injected write failure"), + "the error must surface the write failure: {msg}" + ); + let link_display = std::io::Error::from(std::io::ErrorKind::Unsupported).to_string(); + assert!( + msg.contains(&link_display), + "the error must chain the original link failure ({link_display}): {msg}" + ); + assert!( + key_path.exists(), + "a failed fallback write must leave the partial final for boot-time recovery" + ); + assert!( + std::fs::read(&key_path) + .expect("read partial final") + .is_empty(), + "the write fault fires before any byte lands, so the partial final is empty" + ); + } + + // A non-AlreadyExists error from the fallback's create_new open of the + // final path must surface, chained with the original link error, so a + // two-step failure (link failed, then the direct create also failed) is + // diagnosable. RED without the gate wired: the injected open error never + // reaches the real open, so the publish succeeds instead of erroring. + #[test] + fn failed_fallback_create_surfaces_the_open_error_and_chains_link_err() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let pem = Keypair::generate().to_pem().expect("pem"); + + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + fallback_create: || Err(std::io::Error::other("injected fallback create failure")), + ..PublishFaults::NONE + }; + let err = match publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, &|| {}, faults) { + Err(e) => e, + Ok(_) => panic!("a failed fallback create must error the start"), + }; + let msg = format!("{err:#}"); + assert!( + msg.contains("injected fallback create failure"), + "the error must surface the fallback create failure: {msg}" + ); + let link_display = std::io::Error::from(std::io::ErrorKind::Unsupported).to_string(); + assert!( + msg.contains(&link_display), + "the error must chain the original link failure ({link_display}): {msg}" + ); + assert!( + !key_path.exists(), + "a failed fallback create must not leave the final key" + ); + } + + // A failed fallback FILE fsync takes the same keep-everything exit as + // the failed write: the final (complete but possibly non-durable bytes) + // and its marker stay in place for boot-time recovery. The + // complete-but-unsynced state is safe because a later healthy load + // fsyncs the final ITSELF before sweeping the marker + // (sweep_stale_markers' durability gate), and a crash before then + // re-enters recovery via the surviving marker. + #[test] + fn failed_fallback_fsync_leaves_the_final_for_recovery() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let pem = Keypair::generate().to_pem().expect("pem"); + + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + fallback_fsync: || Err(std::io::Error::other("injected fsync failure")), + ..PublishFaults::NONE + }; + let err = match publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, &|| {}, faults) { + Err(e) => e, + Ok(_) => panic!("a failed fallback fsync must error the start"), + }; + let msg = format!("{err:#}"); + assert!( + msg.contains("injected fsync failure"), + "the error must surface the fsync failure: {msg}" + ); + assert!( + key_path.exists(), + "an fsync failure must leave the final in place for boot-time recovery" + ); + assert_eq!( + std::fs::read(&key_path).expect("read final"), + pem.as_bytes(), + "the fsync fault fires after write_all, so the final holds the full PEM" + ); + } + + // #194: the fallback path can leave a durable empty/truncated final if a + // crash lands between the name becoming durable and the PEM fsync (the + // reconditioned KEY_RACE_DEADLINE / publish_key_atomically doc). This + // observes the promised behavior at the next start: such a residue must + // fail LOUDLY (an invalid-PEM parse error, or a mode rejection), never + // parse as Ok and never be replaced by a freshly generated key. This is a + // characterization/observation test of behavior that already exists, not a + // RED-first guard: it passes on current code. + #[test] + fn fallback_crash_residue_fails_loudly_not_silently() { + use super::{ensure_key_mode_0600, load_existing_key}; + let dir = tempfile::tempdir().expect("tempdir"); + + // (1) an empty final at 0600 (durable name, no PEM bytes yet). + let empty = dir.path().join("empty.pem"); + std::fs::write(&empty, b"").expect("seed empty final"); + std::fs::set_permissions(&empty, std::fs::Permissions::from_mode(0o600)) + .expect("0600 empty"); + let err = match load_existing_key(&empty) { + Err(e) => e.to_string(), + Ok(_) => panic!("an empty final must fail loudly, not parse as Ok"), + }; + assert!( + err.contains("invalid PEM"), + "an empty final must surface a parse error, got: {err}" + ); + + // (2) a truncated/garbage non-PEM final at 0600. + let garbage = dir.path().join("garbage.pem"); + std::fs::write(&garbage, b"-----BEGIN nonsense truncated").expect("seed garbage final"); + std::fs::set_permissions(&garbage, std::fs::Permissions::from_mode(0o600)) + .expect("0600 garbage"); + let err = match load_existing_key(&garbage) { + Err(e) => e.to_string(), + Ok(_) => panic!("a truncated final must fail loudly, not parse as Ok"), + }; + assert!( + err.contains("invalid PEM"), + "a truncated final must surface a parse error, got: {err}" + ); + + // (3) mode-rejection path: an empty final at 0400. The loader tightens a + // loose mode where the mount allows it, so the reachable rejection is + // `ensure_key_mode_0600` failing closed on a mode it cannot narrow (a + // mode-ignoring mount); assert that branch directly on a 0400 file. + let loose = dir.path().join("loose.pem"); + std::fs::write(&loose, b"").expect("seed loose final"); + std::fs::set_permissions(&loose, std::fs::Permissions::from_mode(0o400)) + .expect("0400 loose"); + let err = ensure_key_mode_0600(&loose) + .expect_err("a non-0600 final must be rejected, not used exposed") + .to_string(); + assert!( + err.contains("400"), + "the mode rejection must name the exposed mode, got: {err}" + ); + } + + /// Env var carrying `mask:site` for `umask_worker_publish_lands_0600`. + const UMASK_WORKER_ENV: &str = "GITLAWB_TEST_UMASK_CASE"; + + /// Spawns this test binary again, running only the ignored umask worker + /// with `mask:site` in the environment. The umask is process-global, so + /// setting it in the shared test process narrows every concurrent test's + /// file and tempdir creation modes (observed: a parallel test's fresh + /// tempdir landing dr-x------ and its file create failing EACCES). A + /// child process that runs nothing but the one case confines the mask. + fn run_umask_case(mask: &str, site: &str) { + let exe = std::env::current_exe().expect("test binary path"); + let out = std::process::Command::new(exe) + .args([ + "identity_key_tests::umask_worker_publish_lands_0600", + "--exact", + "--ignored", + ]) + .env(UMASK_WORKER_ENV, format!("{mask}:{site}")) + .output() + .expect("spawn umask worker"); + let stdout = String::from_utf8_lossy(&out.stdout); + // status alone is not enough: a drifted worker name would match zero + // tests and still exit 0, a vacuous green. + assert!( + out.status.success() && stdout.contains("1 passed"), + "umask {mask} {site} worker did not pass:\nstdout:\n{stdout}\nstderr:\n{}", + String::from_utf8_lossy(&out.stderr) + ); + } + + /// In-child body of the umask tests: never runs in the shared suite + /// process (ignored without the spawning tests' env var). Publishes under + /// the given umask and asserts the key lands 0600 and round-trips. The + /// mode assertion runs BEFORE the loader call, which would otherwise + /// tighten a loose key itself and mask a publish-site regression. + #[test] + #[ignore = "umask worker: spawned in a child process by the umask_* tests"] + fn umask_worker_publish_lands_0600() { + let Ok(spec) = std::env::var(UMASK_WORKER_ENV) else { + // A manual `--ignored` sweep without the env var: nothing to do. + return; + }; + let (mask_str, site) = spec.split_once(':').expect("spec is mask:site"); + let mask = libc::mode_t::from_str_radix(mask_str, 8).expect("octal mask"); + let faults = match site { + "hardlink" => PublishFaults::NONE, + "fallback" => PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + ..PublishFaults::NONE + }, + other => panic!("unknown umask worker site {other:?}"), + }; + + // The tempdir is created BEFORE the mask narrows, as a deploy's key + // directory predates the process umask; 0277 would land the dir + // itself 0400. No restore: this process exists only for this case. + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let kp = Keypair::generate(); + let pem = kp.to_pem().expect("pem"); + unsafe { libc::umask(mask) }; + + let out = publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, &|| {}, faults) + .expect("publish must succeed: the tighten repairs the umask-narrowed create mode"); + assert!(matches!(out, KeyPublish::Won), "publish wins"); + + let mode = std::fs::metadata(&key_path) + .expect("key file exists") + .permissions() + .mode(); + assert_eq!( + mode & 0o777, + 0o600, + "under umask {mask:o} the published key must land 0600, got {:o}", + mode & 0o777 + ); + let loaded = super::load_existing_key(&key_path).expect("loader accepts the key"); + assert_eq!( + format!("{}", loaded.did()), + format!("{}", kp.did()), + "publish under umask {mask:o} must round-trip the same identity" + ); + } + + /// #194 (U2): `create_new`'s requested 0600 is narrowed by the process + /// umask, so 0277 lands the file 0400; the tighten half of the + /// tighten-then-verify pair must repair it to 0600 (the verify alone + /// would falsely fail closed). Hard-link site. Pre-state note: before + /// U2 this publish already SUCCEEDED but left the key 0400 on disk, so + /// the mode assertion is what was RED. + #[test] + fn umask_0277_hardlink_publish_is_repaired_to_0600() { + run_umask_case("277", "hardlink"); + } + + /// Same repair, at the fallback creation site (link forced to fail). + #[test] + fn umask_0277_fallback_publish_is_repaired_to_0600() { + run_umask_case("277", "fallback"); + } + + /// #194 (U2): a permissive umask stays healthy: 0600 is already 0600 under + /// 0077, the tighten is a no-op, and the verify must not disturb a clean + /// publish. (`created_key_is_mode_0600` pins the suite's ambient umask; + /// these pin the boundary values explicitly.) Split per-umask so a + /// 0000-specific regression is not masked by a 0077 failure. + #[test] + fn permissive_umask_0077_still_publishes_0600() { + run_umask_case("077", "hardlink"); + } + + /// #194 (U2): the fully permissive umask 0000 stays healthy too — 0600 is + /// already 0600, the tighten is a no-op, and the verify must not disturb a + /// clean publish. + #[test] + fn permissive_umask_0000_still_publishes_0600() { + run_umask_case("000", "hardlink"); + } + + /// #194 (U2): on a mode-ignoring mount (vfat: chmod silently no-ops) the + /// 0600 verification of the just-created TEMP file must fail the publish + /// CLOSED, BEFORE any PEM byte is written. Afterwards the key directory + /// must be empty: the final never appeared, and the empty temp was + /// removed so a retry is not wedged. The error must name the temp file + /// and surface the verify failure. + #[test] + fn forced_temp_mode_verify_failure_fails_closed_before_any_pem_write() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let pem = Keypair::generate().to_pem().expect("pem"); + + let faults = PublishFaults { + temp_mode_verify: || Err(std::io::Error::other("injected mode-ignoring mount")), + ..PublishFaults::NONE + }; + let err = match publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, &|| {}, faults) { + Err(e) => e, + Ok(_) => panic!("an unverifiable temp mode must fail the publish closed"), + }; + let msg = format!("{err:#}"); + assert!( + msg.contains("injected mode-ignoring mount") && msg.contains(".identity.pem.tmp."), + "the error must surface the verify failure and name the temp file: {msg}" + ); + assert!(!key_path.exists(), "the final key must never appear"); + let leftovers: Vec<_> = std::fs::read_dir(dir.path()) + .unwrap() + .filter_map(|e| e.ok().map(|e| e.path())) + .collect(); + assert!( + leftovers.is_empty(), + "no PEM byte may hit disk and the empty temp must be removed: {leftovers:?}" + ); + } + + /// #194 (U2): the same fail-closed check at the FALLBACK creation site + /// (link fault + verify fault). The just-created EMPTY final must be + /// removed, no PEM byte may have hit it, the error must chain the + /// original link failure, and a subsequent retry on a healthy mount must + /// publish cleanly rather than wedge on AlreadyExists. + #[test] + fn forced_fallback_mode_verify_failure_removes_the_empty_final() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let kp = Keypair::generate(); + let pem = kp.to_pem().expect("pem"); + + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + fallback_mode_verify: || Err(std::io::Error::other("injected mode-ignoring mount")), + ..PublishFaults::NONE + }; + let err = match publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, &|| {}, faults) { + Err(e) => e, + Ok(_) => panic!("an unverifiable final mode must fail the publish closed"), + }; + let msg = format!("{err:#}"); + assert!( + msg.contains("injected mode-ignoring mount") && msg.contains("identity.pem"), + "the error must surface the verify failure and name the final: {msg}" + ); + let link_display = std::io::Error::from(std::io::ErrorKind::Unsupported).to_string(); + assert!( + msg.contains(&link_display), + "the error must chain the original link failure ({link_display}): {msg}" + ); + assert!( + !key_path.exists(), + "the empty final must be removed so a retry is not wedged" + ); + let leftovers: Vec<_> = std::fs::read_dir(dir.path()) + .unwrap() + .filter_map(|e| e.ok().map(|e| e.path())) + .collect(); + assert!( + leftovers.is_empty(), + "nothing may remain on disk after the fail-closed publish: {leftovers:?}" + ); + + // The removal is what un-wedges the retry: after a remount (or on a + // start whose chmod works) the same path must publish cleanly. + let retry_faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + ..PublishFaults::NONE + }; + let out = publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, &|| {}, retry_faults) + .expect("a retry after the fail-closed publish must not be wedged"); + assert!(matches!(out, KeyPublish::Won), "retry wins cleanly"); + let loaded = super::load_existing_key(&key_path).expect("retried key loads"); + assert_eq!( + format!("{}", loaded.did()), + format!("{}", kp.did()), + "the retry must publish the intended identity" + ); + } + + /// Lists leftover `.{stem}.publishing.*` marker files in `dir` (#194, U1). + fn publishing_markers(dir: &std::path::Path) -> Vec { + std::fs::read_dir(dir) + .unwrap() + .filter_map(|e| e.ok().map(|e| e.path())) + .filter(|p| { + p.file_name() + .map(|n| n.to_string_lossy().contains(".publishing.")) + .unwrap_or(false) + }) + .collect() + } + + /// #194 (U1): a successful fallback publish (link forced to fail) must win, + /// round-trip the key through the normal loader, and leave NO publish + /// marker behind: the marker exists only for the non-atomic window. + #[test] + fn fallback_publish_succeeds_and_leaves_no_marker() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let kp = Keypair::generate(); + let pem = kp.to_pem().expect("pem"); + + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + ..PublishFaults::NONE + }; + let out = publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, &|| {}, faults) + .expect("fallback publish"); + assert!(matches!(out, KeyPublish::Won), "fallback publish wins"); + let loaded = super::load_existing_key(&key_path).expect("published key loads"); + assert_eq!( + format!("{}", loaded.did()), + format!("{}", kp.did()), + "fallback publish must round-trip the same identity" + ); + let markers = publishing_markers(dir.path()); + assert!( + markers.is_empty(), + "a successful fallback publish must remove its marker, found {markers:?}" + ); + } + + /// #194 (U1): a fallback that loses the create race (final already present) + /// must still behave as today (existing key loaded, never clobbered) and + /// must remove its own marker on the lost-race exit. + #[test] + fn lost_fallback_race_leaves_no_marker() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let winner = Keypair::generate(); + let pem_winner = winner.to_pem().expect("pem winner"); + let out = publish_key_atomically( + &key_path, + pem_winner.as_bytes(), + &|| {}, + &|| {}, + PublishFaults::NONE, + ) + .expect("winner publishes"); + assert!(matches!(out, KeyPublish::Won), "winner publish wins"); + + let pem_loser = Keypair::generate().to_pem().expect("pem loser"); + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::PermissionDenied.into()), + ..PublishFaults::NONE + }; + let out = publish_key_atomically(&key_path, pem_loser.as_bytes(), &|| {}, &|| {}, faults) + .expect("an existing final must read as a lost race, not an error"); + assert!(matches!(out, KeyPublish::Lost), "fallback loses"); + let loaded = load_racing_key(&key_path).expect("lost race loads the winner"); + assert_eq!( + format!("{}", loaded.did()), + format!("{}", winner.did()), + "the lost race must return the winner's identity" + ); + let markers = publishing_markers(dir.path()); + assert!( + markers.is_empty(), + "a lost fallback race must remove its own marker, found {markers:?}" + ); + } + + /// Shared body for the PRE-write fallback error arms (#194, U1): with the + /// link fault plus one injected fallback fault, the error must surface as + /// today, the final must be absent (nothing recoverable exists before the + /// first PEM byte, so those arms remove it), and no publish marker may + /// remain. The POST-write arms (write/fsync) have the opposite contract; + /// see `assert_post_write_failure_leaves_marker_and_final`. + fn assert_fallback_error_leaves_no_marker_or_final(faults: PublishFaults, injected: &str) { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let pem = Keypair::generate().to_pem().expect("pem"); + + let err = match publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, &|| {}, faults) { + Err(e) => e, + Ok(_) => panic!("the injected fallback fault must error the publish"), + }; + let msg = format!("{err:#}"); + assert!( + msg.contains(injected), + "the error must surface the injected fault: {msg}" + ); + assert!( + !key_path.exists(), + "the final must be absent per the existing removal policy" + ); + let markers = publishing_markers(dir.path()); + assert!( + markers.is_empty(), + "an errored fallback must dispose its marker, found {markers:?}" + ); + } + + #[test] + fn failed_fallback_create_leaves_no_marker() { + assert_fallback_error_leaves_no_marker_or_final( + PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + fallback_create: || Err(std::io::Error::other("injected fallback create failure")), + ..PublishFaults::NONE + }, + "injected fallback create failure", + ); + } + + /// Shared body for the POST-write fallback error arms (write/fsync): the + /// error must surface the injected fault AND name the boot-time recovery + /// handoff, the final must SURVIVE, and its `.publishing.` marker must + /// SURVIVE beside it: marker+final together is the recoverable crash + /// state, and marker-less partial final is the permanent wedge. + fn assert_post_write_failure_leaves_marker_and_final(faults: PublishFaults, injected: &str) { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let pem = Keypair::generate().to_pem().expect("pem"); + + let err = match publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, &|| {}, faults) { + Err(e) => e, + Ok(_) => panic!("the injected fallback fault must error the publish"), + }; + let msg = format!("{err:#}"); + assert!( + msg.contains(injected), + "the error must surface the injected fault: {msg}" + ); + assert!( + msg.contains("left in place for boot-time crash recovery"), + "the error must name the recovery handoff: {msg}" + ); + assert!( + key_path.exists(), + "the final must survive a post-write failure for boot-time recovery" + ); + let markers = publishing_markers(dir.path()); + assert_eq!( + markers.len(), + 1, + "the marker must survive to vouch for the partial final, found {markers:?}" + ); + } + + #[test] + fn failed_fallback_write_leaves_the_marker_and_final() { + assert_post_write_failure_leaves_marker_and_final( + PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + fallback_write: || Err(std::io::Error::other("injected write failure")), + ..PublishFaults::NONE + }, + "injected write failure", + ); + } + + #[test] + fn failed_fallback_fsync_leaves_the_marker_and_final() { + assert_post_write_failure_leaves_marker_and_final( + PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + fallback_fsync: || Err(std::io::Error::other("injected fsync failure")), + ..PublishFaults::NONE + }, + "injected fsync failure", + ); + } + + // The handoff the post-write arms promise must actually work: take the + // exact state a failed fallback write leaves behind (partial final plus + // its marker) and run a fresh boot over it. The boot must classify the + // crash signature, claim the marker, quarantine the partial final, and + // regenerate: no wedge, no leftover marker or claim. + #[test] + fn post_write_failure_state_is_recoverable() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let pem = Keypair::generate().to_pem().expect("pem"); + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + fallback_write: || Err(std::io::Error::other("injected write failure")), + ..PublishFaults::NONE + }; + if publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, &|| {}, faults).is_ok() { + panic!("the injected write fault must error the publish"); + } + assert!( + key_path.exists() && publishing_markers(dir.path()).len() == 1, + "precondition: the failed publish leaves final plus marker" + ); + + let kp = load_or_create_keypair(&key_config(&key_path)) + .expect("the left-behind crash state must recover, not wedge the start"); + + let on_disk = super::load_existing_key(&key_path).expect("regenerated final parses"); + assert_eq!( + format!("{}", on_disk.did()), + format!("{}", kp.did()), + "the returned identity must match the regenerated on-disk key" + ); + let quarantined = names_containing(dir.path(), ".quarantined."); + assert_eq!( + quarantined.len(), + 1, + "the partial final must be quarantined: {quarantined:?}" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "recovery must consume the publish marker" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "the recovery claim must be released" + ); + } + + #[test] + fn failed_fallback_mode_verify_leaves_no_marker() { + assert_fallback_error_leaves_no_marker_or_final( + PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + fallback_mode_verify: || Err(std::io::Error::other("injected mode-ignoring mount")), + ..PublishFaults::NONE + }, + "injected mode-ignoring mount", + ); + } + + /// #194 (U1): marker-name probing is bounded like the temp names. With + /// every candidate marker name taken the fallback must fail loudly, naming + /// the stale prefix and directory, and must never create the final. + #[test] + fn marker_name_exhaustion_errors_loudly_and_never_creates_the_final() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let pem = Keypair::generate().to_pem().expect("pem"); + + for attempt in 0..KEY_TEMP_ATTEMPTS { + let stale = dir.path().join(format!( + ".identity.pem.publishing.{}.{attempt}", + std::process::id() + )); + std::fs::write(&stale, b"").expect("seed stale marker"); + } + + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + ..PublishFaults::NONE + }; + let err = match publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, &|| {}, faults) { + Err(e) => e, + Ok(_) => panic!("the fallback must fail when every marker name is taken"), + }; + let msg = format!("{err:#}"); + assert!( + msg.contains(&format!(".identity.pem.publishing.{}", std::process::id())) + && msg.contains(&dir.path().display().to_string()), + "the error must name the marker prefix and directory: {msg}" + ); + assert!( + !key_path.exists(), + "an exhausted marker probe must not create the final key" + ); + } + + /// #194 (U1): a failed marker DIR fsync must NOT fail the publish + /// (warn-and-continue): on dir-fsync-hostile mounts a hard gate would turn + /// "wedge once then recover" into "never provisions". The publish must + /// still win, the key must load, and the marker must still be removed. + #[test] + fn marker_fsync_failure_does_not_fail_the_publish() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let kp = Keypair::generate(); + let pem = kp.to_pem().expect("pem"); + + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + marker_fsync: || Err(std::io::Error::other("injected marker dir fsync failure")), + ..PublishFaults::NONE + }; + let out = publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, &|| {}, faults) + .expect("a marker dir fsync failure must not fail the publish"); + assert!(matches!(out, KeyPublish::Won), "publish still wins"); + let loaded = super::load_existing_key(&key_path).expect("published key loads"); + assert_eq!( + format!("{}", loaded.did()), + format!("{}", kp.did()), + "publish must round-trip the same identity" + ); + let markers = publishing_markers(dir.path()); + assert!( + markers.is_empty(), + "the marker must still be removed, found {markers:?}" + ); + } + + /// #194 (U1): the primary hard-link path must never touch the marker + /// machinery. The `marker_create` hook is poisoned, so if the primary path + /// ever created a marker the publish would error; it must win untouched + /// and leave no `.publishing.` file. + #[test] + fn primary_hardlink_publish_never_creates_a_marker() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let kp = Keypair::generate(); + let pem = kp.to_pem().expect("pem"); + + let faults = PublishFaults { + marker_create: || Err(std::io::Error::other("marker created on the primary path")), + ..PublishFaults::NONE + }; + let out = publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, &|| {}, faults) + .expect("the primary path must never reach the marker machinery"); + assert!(matches!(out, KeyPublish::Won), "primary publish wins"); + let loaded = super::load_existing_key(&key_path).expect("published key loads"); + assert_eq!( + format!("{}", loaded.did()), + format!("{}", kp.did()), + "primary publish must round-trip the same identity" + ); + let markers = publishing_markers(dir.path()); + assert!( + markers.is_empty(), + "the primary path must never create a marker, found {markers:?}" + ); + } + + /// Seed the on-disk signature of a fallback publish crashed mid-write + /// (#194, U2): a 0600 final holding `content` (empty or truncated) beside + /// a `.publishing.` marker carrying a foreign (crashed) process's pid. + /// Returns the final's path. + fn seed_crash_state(dir: &std::path::Path, content: &[u8]) -> std::path::PathBuf { + let key_path = dir.join("identity.pem"); + std::fs::write(&key_path, content).expect("seed crashed final"); + std::fs::set_permissions(&key_path, std::fs::Permissions::from_mode(0o600)) + .expect("0600 crashed final"); + std::fs::write(dir.join(".identity.pem.publishing.99999.0"), b"") + .expect("seed publish marker"); + key_path + } + + /// Set `path`'s mtime `spec` into the past via GNU `touch -d` (std::fs + /// cannot set arbitrary mtimes; spawning coreutils is test-only). + fn set_mtime_past(path: &std::path::Path, spec: &str) { + let status = std::process::Command::new("touch") + .arg("-d") + .arg(spec) + .arg(path) + .status() + .expect("run touch -d"); + assert!(status.success(), "touch -d must succeed to age {path:?}"); + } + + /// Set `path`'s mtime an hour into the past, far beyond + /// `CLAIM_SWEEP_MIN_AGE`. + fn age_beyond_claim_sweep(path: &std::path::Path) { + set_mtime_past(path, "1 hour ago"); + } + + /// File names in `dir` containing `needle` (marker/claim/quarantine sweeps). + fn names_containing(dir: &std::path::Path, needle: &str) -> Vec { + let mut names: Vec = std::fs::read_dir(dir) + .expect("read_dir") + .filter_map(|e| e.ok()) + .map(|e| e.file_name().to_string_lossy().into_owned()) + .filter(|n| n.contains(needle)) + .collect(); + names.sort(); + names + } + + fn key_config(key_path: &std::path::Path) -> Config { + Config::parse_from([ + "gitlawb-node", + "--key-path", + key_path.to_str().expect("utf8 path"), + ]) + } + + // #194 (U2): a `.publishing.` marker beside an EMPTY final is the crash + // signature of an interrupted fallback publish; boot must claim the + // marker, quarantine the empty final, and regenerate instead of wedging + // on `invalid PEM key` forever. RED before U2: load_or_create_keypair + // errors with `invalid PEM key`. + #[test] + fn recovery_regenerates_after_crash_leaves_empty_final() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + + let started = std::time::Instant::now(); + let kp = load_or_create_keypair(&key_config(&key_path)) + .expect("a crash-marked empty final must recover, not wedge the start"); + let elapsed = started.elapsed(); + // The crash signature short-circuits after two ~2ms polls; 2s is + // generous for CI machines yet far under the 5s KEY_RACE_DEADLINE a + // deadline-riding load would burn before recovery could even start. + assert!( + elapsed < std::time::Duration::from_secs(2), + "recovery must short-circuit on the crash signature, not ride \ + KEY_RACE_DEADLINE: took {elapsed:?}" + ); + + let mode = std::fs::metadata(&key_path) + .expect("regenerated final exists") + .permissions() + .mode(); + assert_eq!(mode & 0o777, 0o600, "regenerated final must be 0600"); + let on_disk = super::load_existing_key(&key_path).expect("regenerated final parses"); + assert_eq!( + format!("{}", on_disk.did()), + format!("{}", kp.did()), + "the returned identity must match the regenerated on-disk key" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "recovery must consume the publish marker(s)" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "the recovery claim must be removed after the publish resolves" + ); + let quarantined = names_containing(dir.path(), ".quarantined."); + assert_eq!( + quarantined.len(), + 1, + "exactly one quarantine file must exist: {quarantined:?}" + ); + let bytes = std::fs::read(dir.path().join(&quarantined[0])).expect("read quarantine"); + assert!( + bytes.is_empty(), + "the quarantine must preserve the crashed final's (empty) content" + ); + } + + // #194 (U2): same recovery for a final holding a truncated PEM, and the + // quarantine (a rename) must preserve the truncated bytes exactly for + // post-mortem inspection. + #[test] + fn recovery_quarantines_truncated_final_preserving_bytes() { + let dir = tempfile::tempdir().expect("tempdir"); + let pem = Keypair::generate().to_pem().expect("pem"); + let truncated = &pem.as_bytes()[..pem.len() / 2]; + let key_path = seed_crash_state(dir.path(), truncated); + + let kp = load_or_create_keypair(&key_config(&key_path)) + .expect("a crash-marked truncated final must recover, not wedge the start"); + + let on_disk = super::load_existing_key(&key_path).expect("regenerated final parses"); + assert_eq!( + format!("{}", on_disk.did()), + format!("{}", kp.did()), + "the returned identity must match the regenerated on-disk key" + ); + let quarantined = names_containing(dir.path(), ".quarantined."); + assert_eq!( + quarantined.len(), + 1, + "exactly one quarantine file must exist: {quarantined:?}" + ); + let bytes = std::fs::read(dir.path().join(&quarantined[0])).expect("read quarantine"); + assert_eq!( + bytes, truncated, + "the quarantine must hold exactly the truncated input bytes" + ); + } + + // #194 (U2): quarantine-name probing is bounded like the temp and marker + // names. With EVERY candidate quarantine name taken, recovery must fall + // back to REMOVING the corrupt final (not rename it, not clobber a stale + // quarantine) and still regenerate successfully. Sentinel bytes in each + // stale quarantine prove no rename landed. Load-bearing by mutation + // check: with the remove_file fallback disabled, the corrupt final + // survives, the retry publish loses to it, and the boot fails loudly. + #[test] + fn quarantine_exhaustion_falls_back_to_removal() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + for n in 0..KEY_TEMP_ATTEMPTS { + let stale = dir.path().join(format!( + ".identity.pem.quarantined.{}.{n}", + std::process::id() + )); + std::fs::write(&stale, b"sentinel").expect("seed stale quarantine"); + } + + let kp = load_or_create_keypair(&key_config(&key_path)) + .expect("an exhausted quarantine probe must fall back to removal, not wedge"); + + let on_disk = super::load_existing_key(&key_path).expect("regenerated final parses"); + assert_eq!( + format!("{}", on_disk.did()), + format!("{}", kp.did()), + "the returned identity must match the regenerated on-disk key" + ); + let quarantined = names_containing(dir.path(), ".quarantined."); + assert_eq!( + quarantined.len(), + KEY_TEMP_ATTEMPTS as usize, + "the final must be removed, not quarantined under a fresh name: {quarantined:?}" + ); + for name in &quarantined { + let bytes = std::fs::read(dir.path().join(name)).expect("read stale quarantine"); + assert_eq!( + bytes, b"sentinel", + "no stale quarantine may be clobbered by a rename: {name}" + ); + } + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "recovery must consume the publish marker(s)" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "the recovery claim must be released after the publish resolves" + ); + } + + // #194 (U2/F1): with every quarantine name taken AND removal blocked, + // recovery must surface the loud chained error, never a silent success. + // The dir turns read-only via the before_reparse seam (after the claim + // rename lands; a dir made read-only up front would fail the claim step + // and never reach the quarantine arm at all). The same read-only dir + // necessarily blocks the claim release too, so phase two restores + // permissions and proves the leftover claim is itself recoverable (F3): + // the next boot claims it, removes the final, and regenerates cleanly. + #[test] + fn quarantine_and_removal_both_blocked_errors_loudly() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + for n in 0..KEY_TEMP_ATTEMPTS { + let stale = dir.path().join(format!( + ".identity.pem.quarantined.{}.{n}", + std::process::id() + )); + std::fs::write(&stale, b"sentinel").expect("seed stale quarantine"); + } + let lock_path = dir.path().to_path_buf(); + let lock_dir = move || { + std::fs::set_permissions(&lock_path, std::fs::Permissions::from_mode(0o555)) + .expect("read-only key dir inside the claim window"); + }; + let seam = RecoverySeam { + before_reparse: &lock_dir, + ..RecoverySeam::NONE + }; + + let result = + load_or_create_keypair_with(&key_path, &|| {}, &|| {}, PublishFaults::NONE, seam); + + std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o755)) + .expect("restore key dir"); + let err = match result { + Err(e) => e, + Ok(_) => panic!("a blocked quarantine AND removal must fail loudly, not resolve"), + }; + let msg = format!("{err:#}"); + assert!( + msg.contains("could not quarantine or remove") + && msg.contains(&key_path.display().to_string()) + && msg.contains("invalid PEM key"), + "the error must chain the recovery failure and the load failure: {msg}" + ); + assert!( + key_path.exists(), + "the corrupt final must survive for post-mortem when nothing can dispose of it" + ); + + // Phase two: the claim left behind (the read-only dir blocked its + // release too) must make the state recoverable, not wedge it. F2 + // makes a fresh claim invisible to recovery (it could gate a LIVE + // round), so age the leftovers past the floor first, standing in for + // the bounded wait a real next boot would incur. + for name in names_containing(dir.path(), ".recovering.") { + age_beyond_claim_sweep(&dir.path().join(name)); + } + let kp = load_or_create_keypair(&key_config(&key_path)) + .expect("the leftover claim must be claimable by the next boot, not a wedge"); + let on_disk = super::load_existing_key(&key_path).expect("regenerated final parses"); + assert_eq!( + format!("{}", on_disk.did()), + format!("{}", kp.did()), + "the returned identity must match the regenerated on-disk key" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "the follow-up boot must consume and release every claim" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "no publish markers may remain" + ); + let quarantined = names_containing(dir.path(), ".quarantined."); + assert_eq!( + quarantined.len(), + KEY_TEMP_ATTEMPTS as usize, + "the follow-up boot must also fall back to removal: {quarantined:?}" + ); + } + + // MUST-NOT (#194, U2): an unparseable final WITHOUT a marker is operator + // corruption, not a crash signature; it must keep today's loud fail + // bit-for-bit: same `invalid PEM key` message, final left on disk with + // identical bytes, no quarantine, no regeneration. Rides the full + // KEY_RACE_DEADLINE (~5s), as today. Pre-implementation this must PASS; + // after GREEN it is the mutation guard for the marker-presence check. + #[test] + fn bad_final_without_marker_still_fails_loudly() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let garbage: &[u8] = b"-----BEGIN nonsense truncated"; + std::fs::write(&key_path, garbage).expect("seed garbage final"); + std::fs::set_permissions(&key_path, std::fs::Permissions::from_mode(0o600)) + .expect("0600 garbage final"); + + let err = match load_or_create_keypair(&key_config(&key_path)) { + Err(e) => e, + Ok(_) => panic!("a marker-less corrupt final must fail loudly, never regenerate"), + }; + assert!( + err.to_string().contains("invalid PEM key"), + "the load failure must surface unchanged: {err:#}" + ); + assert_eq!( + std::fs::read(&key_path).expect("final still on disk"), + garbage, + "the corrupt final must be left untouched" + ); + assert!( + names_containing(dir.path(), ".quarantined.").is_empty(), + "no quarantine may be created without a marker" + ); + } + + // MUST-NOT (#194, U2): a VALID final beside a stale marker loads + // unchanged: same identity, same bytes, no quarantine. (U4 adds marker + // hygiene; U2 may leave the stale marker in place.) + #[test] + fn valid_final_with_stale_marker_loads_unchanged() { + let dir = tempfile::tempdir().expect("tempdir"); + let existing = Keypair::generate(); + let pem = existing.to_pem().expect("pem"); + let key_path = seed_crash_state(dir.path(), pem.as_bytes()); + + let kp = load_or_create_keypair(&key_config(&key_path)) + .expect("a valid final with a stale marker must load"); + + assert_eq!( + format!("{}", kp.did()), + format!("{}", existing.did()), + "the pre-existing identity must be preserved, not regenerated" + ); + assert_eq!( + std::fs::read(&key_path).expect("final still on disk"), + pem.as_bytes(), + "the valid final's bytes must be untouched" + ); + assert!( + names_containing(dir.path(), ".quarantined.").is_empty(), + "a valid final must never be quarantined" + ); + } + + // MUST-NOT (#194, U2): a TRANSIENT (non-content) load failure must never + // recover, even with a marker present: loud error, no quarantine, final + // and marker untouched. The seam's load fault injects the transient class + // deterministically. Rides the full KEY_RACE_DEADLINE (~5s). RED if + // recovery keys on any-error instead of content-class. + #[test] + fn transient_error_with_marker_never_recovers() { + let dir = tempfile::tempdir().expect("tempdir"); + let pem = Keypair::generate().to_pem().expect("pem"); + let key_path = seed_crash_state(dir.path(), pem.as_bytes()); + let seam = RecoverySeam { + load_fault: || Err(std::io::Error::other("injected transient load failure")), + ..RecoverySeam::NONE + }; + + let err = + match load_or_create_keypair_with(&key_path, &|| {}, &|| {}, PublishFaults::NONE, seam) + { + Err(e) => e, + Ok(_) => panic!("a transient-class load failure must stay loud, never recover"), + }; + assert!( + format!("{err:#}").contains("injected transient load failure"), + "the transient failure must surface: {err:#}" + ); + assert_eq!( + std::fs::read(&key_path).expect("final still on disk"), + pem.as_bytes(), + "the final must be untouched by a transient failure" + ); + assert_eq!( + names_containing(dir.path(), ".publishing."), + vec![".identity.pem.publishing.99999.0".to_string()], + "the marker must be untouched" + ); + assert!( + names_containing(dir.path(), ".quarantined.").is_empty(), + "a transient failure must never quarantine" + ); + } + + // #194 (U2): recovery runs AT MOST ONCE per start. After the first + // recovery completes, the seam re-plants a crash state (marker + empty + // final) before the retry publish; the second load failure must surface + // as an error, not recover again. Exactly one quarantine, from the first + // pass. Rides the full KEY_RACE_DEADLINE (~5s) on the second load. + #[test] + fn recovery_runs_at_most_once_per_start() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + let replanted = std::sync::atomic::AtomicBool::new(false); + let replant_path = key_path.clone(); + let replant_marker = dir.path().join(".identity.pem.publishing.88888.0"); + let before_publish = move || { + if !replanted.swap(true, std::sync::atomic::Ordering::SeqCst) { + std::fs::write(&replant_marker, b"").expect("re-plant marker"); + std::fs::write(&replant_path, b"").expect("re-plant empty final"); + std::fs::set_permissions(&replant_path, std::fs::Permissions::from_mode(0o600)) + .expect("0600 re-planted final"); + } + }; + let seam = RecoverySeam { + before_publish: &before_publish, + ..RecoverySeam::NONE + }; + + let err = + match load_or_create_keypair_with(&key_path, &|| {}, &|| {}, PublishFaults::NONE, seam) + { + Err(e) => e, + Ok(_) => { + panic!("a second crash state in the same start must error, not recover again") + } + }; + assert!( + err.to_string().contains("invalid PEM key"), + "the second load failure must surface unchanged: {err:#}" + ); + let quarantined = names_containing(dir.path(), ".quarantined."); + assert_eq!( + quarantined.len(), + 1, + "exactly one quarantine, from the first recovery: {quarantined:?}" + ); + } + + // #194 (U2): two concurrent boots against the same crash state must admit + // exactly ONE recoverer (the atomic claim rename arbitrates); both end on + // the SAME identity, one quarantine, no markers or claims left behind. + // Modeled on concurrent_starts_converge_on_one_identity. + #[test] + fn claim_race_admits_exactly_one_recoverer() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + let config = std::sync::Arc::new(key_config(&key_path)); + + let barrier = std::sync::Arc::new(std::sync::Barrier::new(2)); + let handles: Vec<_> = (0..2) + .map(|_| { + let c = config.clone(); + let b = barrier.clone(); + std::thread::spawn(move || { + b.wait(); + format!( + "{}", + load_or_create_keypair(&c).expect("recovering start").did() + ) + }) + }) + .collect(); + let dids: Vec = handles + .into_iter() + .map(|h| h.join().expect("thread joins")) + .collect(); + + assert_eq!( + dids[0], dids[1], + "both concurrent recovering starts must converge on one identity" + ); + let quarantined = names_containing(dir.path(), ".quarantined."); + assert_eq!( + quarantined.len(), + 1, + "exactly one recoverer must quarantine: {quarantined:?}" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "no publish markers may remain" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "no recovery claims may remain" + ); + } + + // #194 (U2): the Lost arm participates in recovery. The publisher's link + // is forced to fail (fallback path) and the crash state (foreign marker + + // empty final) appears between the temp write and the link, so the + // publish loses to the crashed final; the Lost-arm load must then take + // the recovery path and regenerate. + #[test] + fn lost_race_load_failure_takes_recovery_path() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let planted = std::sync::atomic::AtomicBool::new(false); + let plant_path = key_path.clone(); + let plant_marker = dir.path().join(".identity.pem.publishing.77777.0"); + let before_link = move || { + if !planted.swap(true, std::sync::atomic::Ordering::SeqCst) { + std::fs::write(&plant_marker, b"").expect("plant marker"); + std::fs::write(&plant_path, b"").expect("plant empty final"); + std::fs::set_permissions(&plant_path, std::fs::Permissions::from_mode(0o600)) + .expect("0600 planted final"); + } + }; + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + ..PublishFaults::NONE + }; + + let kp = load_or_create_keypair_with( + &key_path, + &before_link, + &|| {}, + faults, + RecoverySeam::NONE, + ) + .expect("a lost race onto a crash-state final must recover, not fail the start"); + + let on_disk = super::load_existing_key(&key_path).expect("regenerated final parses"); + assert_eq!( + format!("{}", on_disk.did()), + format!("{}", kp.did()), + "the returned identity must match the regenerated on-disk key" + ); + let quarantined = names_containing(dir.path(), ".quarantined."); + assert_eq!( + quarantined.len(), + 1, + "exactly one quarantine file must exist: {quarantined:?}" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "no publish markers may remain" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "no recovery claims may remain" + ); + } + + // #194 (U3): THE divergence race. A fallback winner whose final is already + // durable but which stalls before removing its marker can have its round + // stolen by a recovering peer: the peer claims the winner's marker, + // quarantines the winner's final, and republishes key B. The winner's + // OWN-MARKER removal is the commit check; losing it must demote the winner + // to Lost so it converges on B, never returning Won with key A while disk + // holds B. RED before U3: the publish returns Won with the quarantined key + // A while the final holds B (two-sided divergence). + #[test] + fn stalled_winner_demotes_when_recovery_claims_its_round() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let kp_b = Keypair::generate(); + let pem_b = kp_b.to_pem().expect("pem b"); + + let fired = std::sync::atomic::AtomicBool::new(false); + let steal_dir = dir.path().to_path_buf(); + let steal_path = key_path.clone(); + let before_commit = move || { + if fired.swap(true, std::sync::atomic::Ordering::SeqCst) { + return; + } + // A full concurrent recovery inside A's stall window (post + // final-fsync, pre commit check), step for step what + // recover_crashed_publish does: claim A's marker by rename, + // quarantine A's final, republish key B, clear the claim. + let markers = names_containing(&steal_dir, ".publishing."); + assert_eq!( + markers.len(), + 1, + "A's window must be bracketed: {markers:?}" + ); + let claim = steal_dir.join(".identity.pem.recovering.55555"); + std::fs::rename(steal_dir.join(&markers[0]), &claim).expect("claim A's marker"); + std::fs::rename( + &steal_path, + steal_dir.join(".identity.pem.quarantined.55555.0"), + ) + .expect("quarantine A's final"); + let out = publish_key_atomically( + &steal_path, + pem_b.as_bytes(), + &|| {}, + &|| {}, + PublishFaults::NONE, + ) + .expect("peer republishes key B"); + assert!(matches!(out, KeyPublish::Won), "peer's republish wins"); + std::fs::remove_file(&claim).expect("clear the claim"); + }; + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + ..PublishFaults::NONE + }; + + let kp = load_or_create_keypair_with( + &key_path, + &|| {}, + &before_commit, + faults, + RecoverySeam::NONE, + ) + .expect("a demoted winner must converge, not fail the start"); + + let on_disk = super::load_existing_key(&key_path).expect("final parses"); + assert_eq!( + format!("{}", kp.did()), + format!("{}", kp_b.did()), + "the demoted winner must converge on the recovering peer's key B" + ); + assert_eq!( + format!("{}", on_disk.did()), + format!("{}", kp_b.did()), + "the final must hold key B" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "no publish markers may remain" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "no recovery claims may remain" + ); + let quarantined = names_containing(dir.path(), ".quarantined."); + assert_eq!( + quarantined.len(), + 1, + "exactly one live identity: A's old final stays quarantined: {quarantined:?}" + ); + } + + // #194 (U3): with no interference the commit check is invisible. The + // winner removes its own marker, returns Won, and behaves exactly as U1's + // success test: the key round-trips and no marker is left. Also pins that + // the pre-commit seam runs on the fallback success path. + #[test] + fn winner_commits_first_without_interference() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let kp = Keypair::generate(); + let pem = kp.to_pem().expect("pem"); + + let fired = std::sync::atomic::AtomicBool::new(false); + let before_commit = || { + fired.store(true, std::sync::atomic::Ordering::SeqCst); + }; + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + ..PublishFaults::NONE + }; + let out = publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, &before_commit, faults) + .expect("uncontended fallback publish"); + assert!( + matches!(out, KeyPublish::Won), + "the uncontended winner commits and wins" + ); + assert!( + fired.load(std::sync::atomic::Ordering::SeqCst), + "the pre-commit seam must run on the fallback success path" + ); + let loaded = super::load_existing_key(&key_path).expect("published key loads"); + assert_eq!( + format!("{}", loaded.did()), + format!("{}", kp.did()), + "the committed publish must round-trip the same identity" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "no marker may remain after commit" + ); + } + + // #194 (U3): a recovery attempt arriving AFTER the winner committed finds + // no marker to claim and must abort into a plain re-load: no quarantine, + // the winner's identity returned. Honest scope: recover_crashed_publish + // is invoked directly with a synthetic load error, the way the boot loop + // would after observing a (by now resolved) crash signature; the final on + // disk is GOOD here, so the vanished-marker arm's re-load succeeds. + #[test] + fn late_recovery_aborts_after_winner_commit() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let kp = Keypair::generate(); + let pem = kp.to_pem().expect("pem"); + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + ..PublishFaults::NONE + }; + let out = publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, &|| {}, faults) + .expect("winner publishes and commits"); + assert!(matches!(out, KeyPublish::Won), "winner commits"); + + let recovery = super::recover_crashed_publish( + &key_path, + anyhow::anyhow!("synthetic post-commit crash signature"), + RecoverySeam::NONE, + ) + .expect("a late recovery must not error"); + let loaded = match recovery { + super::Recovery::Reloaded(result) => { + result.expect("the vanished-marker arm re-loads the winner's key") + } + super::Recovery::Claimed(claims) => { + panic!("no marker exists to claim after commit, got claims {claims:?}") + } + }; + assert_eq!( + format!("{}", loaded.did()), + format!("{}", kp.did()), + "late recovery must resolve to the committed winner's identity" + ); + assert!( + names_containing(dir.path(), ".quarantined.").is_empty(), + "a committed final must never be quarantined" + ); + } + + // #194 (U3, reshaped by F3 and the absence-keyed waiter): a demoted + // winner whose round is SETTLED (no `.recovering.` claim file remains, + // the recoverer released before the demoted publish reached its wait) + // must resolve Lost immediately: the waiter keys purely on claim-file + // absence, so an empty directory clears it on the first poll. The + // marker is stolen WITHOUT a real recovery (final left in place), so + // the follow-up load resolves the winner's OWN key: the + // demotion-is-safe property. waiter_holds_for_live_young_claims covers + // the hold side; frozen_claim_fails_the_waiter_closed covers a claim + // that never clears. + #[test] + fn demoted_winner_resolves_fast_when_claims_are_gone() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + + let fired = std::sync::atomic::AtomicBool::new(false); + let steal_dir = dir.path().to_path_buf(); + let before_commit = move || { + if fired.swap(true, std::sync::atomic::Ordering::SeqCst) { + return; + } + let markers = names_containing(&steal_dir, ".publishing."); + assert_eq!( + markers.len(), + 1, + "A's window must be bracketed: {markers:?}" + ); + // The demoting peer already resolved AND released its claims: + // only the stolen marker remains as evidence of the demotion. + std::fs::remove_file(steal_dir.join(&markers[0])).expect("steal A's marker"); + }; + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + ..PublishFaults::NONE + }; + + let started = std::time::Instant::now(); + let kp = load_or_create_keypair_with( + &key_path, + &|| {}, + &before_commit, + faults, + RecoverySeam::NONE, + ) + .expect("a demoted winner with no claims left must still resolve a key"); + let waited = started.elapsed(); + + assert!( + waited < std::time::Duration::from_secs(4), + "an absent claim must not hold the demoted winner, waited {waited:?}" + ); + let on_disk = super::load_existing_key(&key_path).expect("final parses"); + assert_eq!( + format!("{}", kp.did()), + format!("{}", on_disk.did()), + "with the final untouched the demoted winner must resolve its own key" + ); + } + + // Q3: a demoted winner facing a claim whose mtime is FROZEN (its + // recoverer's heartbeat never span up and both stamp rungs failed on a + // hostile volume, or it is a long-crashed recovery's orphan) must fail + // the publish CLOSED, never resolve a key: the waiter keys purely on + // claim-file PRESENCE, so a claim that never leaves the directory rides + // the full bound and errors, whatever its age. Before this fix the + // waiter read an aged claim as orphaned residue and returned Ok, and + // the demoted publisher loaded a possibly pre-settlement key while an + // mtime-frozen LIVE recoverer could still settle disk differently. RED + // before the fix: the aged claim clears the wait as Ok and the demoted + // winner resolves its own key. + #[test] + fn frozen_claim_fails_the_waiter_closed() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + + let fired = std::sync::atomic::AtomicBool::new(false); + let steal_dir = dir.path().to_path_buf(); + let final_bytes = std::sync::Arc::new(std::sync::Mutex::new(Vec::::new())); + let final_at_demotion = final_bytes.clone(); + let observe_path = key_path.clone(); + let before_commit = move || { + if fired.swap(true, std::sync::atomic::Ordering::SeqCst) { + return; + } + let markers = names_containing(&steal_dir, ".publishing."); + assert_eq!( + markers.len(), + 1, + "A's window must be bracketed: {markers:?}" + ); + std::fs::remove_file(steal_dir.join(&markers[0])).expect("steal A's marker"); + let claim = steal_dir.join(".identity.pem.recovering.44444"); + std::fs::write(&claim, b"").expect("plant frozen claim"); + // Freeze the claim's mtime far in the past: no heartbeat, no + // stamp, only the file itself, which never leaves the directory. + age_beyond_claim_sweep(&claim); + *final_at_demotion.lock().expect("final bytes lock") = + std::fs::read(&observe_path).expect("final is complete at the commit check"); + }; + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + ..PublishFaults::NONE + }; + + let result = load_or_create_keypair_with( + &key_path, + &|| {}, + &before_commit, + faults, + RecoverySeam::NONE, + ); + + let err = match result { + Err(e) => e, + Ok(_) => { + panic!("a claim still present at the wait bound must fail closed, not resolve") + } + }; + let msg = format!("{err:#}"); + assert!( + msg.contains(".identity.pem.recovering.44444"), + "the error must name the surviving claim file: {msg}" + ); + assert!( + msg.contains("settle"), + "the error must name the unsettled round: {msg}" + ); + assert_eq!( + std::fs::read(&key_path).expect("final still on disk"), + *final_bytes.lock().expect("final bytes lock"), + "the failed-closed publish must leave the final untouched" + ); + } + + // F3: the waiter's hold must follow the CLAIM_SWEEP_MIN_AGE-plus-margin + // bound, not KEY_RACE_DEADLINE: a live claim's round is allowed the + // full bound to finish, so a demoted winner that gives up at the 5s + // deadline can re-load the pre-quarantine key while a slow-mount + // recovery is still mid-round. Plant a claim, hold it in place for ~7s + // (past the old deadline), then remove it mid-wait: the waiter must + // still be holding past 5s (the file is present) and return shortly + // after the removal (absence is the release), well under the bound. RED + // before F3: the waiter expires at KEY_RACE_DEADLINE and the elapsed + // floor fails. + #[test] + fn waiter_holds_for_live_young_claims() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + + let fired = std::sync::atomic::AtomicBool::new(false); + let steal_dir = dir.path().to_path_buf(); + let before_commit = move || { + if fired.swap(true, std::sync::atomic::Ordering::SeqCst) { + return; + } + let markers = names_containing(&steal_dir, ".publishing."); + assert_eq!( + markers.len(), + 1, + "A's window must be bracketed: {markers:?}" + ); + std::fs::remove_file(steal_dir.join(&markers[0])).expect("steal A's marker"); + std::fs::write(steal_dir.join(".identity.pem.recovering.55555"), b"") + .expect("plant fresh (live) claim"); + }; + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + ..PublishFaults::NONE + }; + + // Model the live round resolving mid-wait: once the claim appears, + // keep it live for 7s (past the old 5s deadline, under the ~11s + // bound), then clear it as a finishing recoverer would. + let claim_path = dir.path().join(".identity.pem.recovering.55555"); + let remover_path = claim_path.clone(); + let remover = std::thread::spawn(move || { + while !remover_path.exists() { + std::thread::sleep(std::time::Duration::from_millis(5)); + } + std::thread::sleep(std::time::Duration::from_secs(7)); + std::fs::remove_file(&remover_path).expect("clear the live claim mid-wait"); + }); + + let started = std::time::Instant::now(); + let kp = load_or_create_keypair_with( + &key_path, + &|| {}, + &before_commit, + faults, + RecoverySeam::NONE, + ) + .expect("a demoted winner behind a live claim must still resolve a key"); + let waited = started.elapsed(); + remover.join().expect("remover joins"); + + assert!( + waited > super::KEY_RACE_DEADLINE + std::time::Duration::from_millis(1500), + "a young claim must hold the waiter past the old deadline, waited {waited:?}" + ); + assert!( + waited < std::time::Duration::from_secs(10), + "the waiter must release on the claim's removal, not ride the full \ + bound: waited {waited:?}" + ); + let on_disk = super::load_existing_key(&key_path).expect("final parses"); + assert_eq!( + format!("{}", kp.did()), + format!("{}", on_disk.did()), + "with the final untouched the demoted winner must resolve its own key" + ); + } + + // N3: a demoted fallback winner whose wait bound expires while a claim + // still persists has NOT observed a settled round: loading the final + // there can return a pre-settlement key into run() while the + // still-live recoverer settles disk on a different identity, the exact + // divergence the protocol forbids. The publish must fail closed (a loud + // Err naming the unsettled round), never resolve Lost and load. A + // refresher thread keeps the claim young past the full wait bound, + // modeling a recovery round still heartbeating (live but stuck) beyond + // it; frozen_claim_fails_the_waiter_closed covers the mtime-frozen + // side, which the absence-keyed waiter treats identically. RED before + // N3: the waiter expires silently and the demoted winner returns its + // own key. + #[test] + fn unsettled_round_fails_closed() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + + let fired = std::sync::atomic::AtomicBool::new(false); + let steal_dir = dir.path().to_path_buf(); + let final_bytes = std::sync::Arc::new(std::sync::Mutex::new(Vec::::new())); + let final_at_demotion = final_bytes.clone(); + let observe_path = key_path.clone(); + let before_commit = move || { + if fired.swap(true, std::sync::atomic::Ordering::SeqCst) { + return; + } + let markers = names_containing(&steal_dir, ".publishing."); + assert_eq!( + markers.len(), + 1, + "A's window must be bracketed: {markers:?}" + ); + std::fs::remove_file(steal_dir.join(&markers[0])).expect("steal A's marker"); + std::fs::write(steal_dir.join(".identity.pem.recovering.55555"), b"") + .expect("plant live claim"); + *final_at_demotion.lock().expect("final bytes lock") = + std::fs::read(&observe_path).expect("final is complete at the commit check"); + }; + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + ..PublishFaults::NONE + }; + + // Keep the claim YOUNG for the waiter's whole bound, modeling a + // recovery round that heartbeats but never settles. The + // absence-keyed waiter fails closed on presence regardless of age + // (frozen_claim_fails_the_waiter_closed pins the frozen-mtime + // side); the refresher keeps this test honest about the LIVE-round + // shape it models. + let claim_path = dir.path().join(".identity.pem.recovering.55555"); + let stop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let stop_refresher = stop.clone(); + let refresher_path = claim_path.clone(); + let refresher = std::thread::spawn(move || { + while !refresher_path.exists() { + if stop_refresher.load(std::sync::atomic::Ordering::SeqCst) { + return; + } + std::thread::sleep(std::time::Duration::from_millis(5)); + } + while !stop_refresher.load(std::sync::atomic::Ordering::SeqCst) { + let _ = std::fs::File::options() + .write(true) + .open(&refresher_path) + .and_then(|f| f.set_modified(std::time::SystemTime::now())); + std::thread::sleep(std::time::Duration::from_millis(500)); + } + }); + + let result = load_or_create_keypair_with( + &key_path, + &|| {}, + &before_commit, + faults, + RecoverySeam::NONE, + ); + stop.store(true, std::sync::atomic::Ordering::SeqCst); + refresher.join().expect("refresher joins"); + + let err = match result { + Err(e) => e, + Ok(_) => { + panic!("an unsettled round past the wait bound must fail closed, not resolve a key") + } + }; + let msg = format!("{err:#}"); + assert!( + msg.contains(".identity.pem.recovering.55555"), + "the error must name the unsettled claim file: {msg}" + ); + assert!( + msg.contains("settle"), + "the error must name the unsettled round: {msg}" + ); + assert_eq!( + std::fs::read(&key_path).expect("final still on disk"), + *final_bytes.lock().expect("final bytes lock"), + "the failed-closed publish must leave the final untouched" + ); + } + + // #194 (U4): a HEALTHY load must sweep stale `.publishing.` markers and + // `.recovering.` claims, or they linger to misclassify a much-later real + // corruption of a good key as an interrupted first write. The claim is + // AGED past CLAIM_SWEEP_MIN_AGE: only orphaned claims are sweepable + // (#194, G1; healthy_load_spares_young_claims covers the young side). + // RED before U4: all three files survive the load. + #[test] + fn healthy_load_sweeps_stale_markers_and_claims() { + let dir = tempfile::tempdir().expect("tempdir"); + let existing = Keypair::generate(); + let pem = existing.to_pem().expect("pem"); + let key_path = seed_crash_state(dir.path(), pem.as_bytes()); + std::fs::write(dir.path().join(".identity.pem.publishing.88888.1"), b"") + .expect("seed second stale marker"); + let claim = dir.path().join(".identity.pem.recovering.77777"); + std::fs::write(&claim, b"").expect("seed stale recovery claim"); + age_beyond_claim_sweep(&claim); + + let kp = load_or_create_keypair(&key_config(&key_path)) + .expect("a valid final with stale markers must load"); + + assert_eq!( + format!("{}", kp.did()), + format!("{}", existing.did()), + "the pre-existing identity must be preserved, not regenerated" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "a healthy load must sweep stale publish markers" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "a healthy load must sweep stale recovery claims" + ); + } + + // #194 (G1) MUST-NOT: the healthy-boot sweep must SPARE a young + // `.recovering.` claim. A live recoverer's claims exist only within one + // bounded recovery round, and a demoted publisher's + // wait_for_recovery_claims waits on them; a third start sweeping a live + // claim cancels that wait early, so the publisher re-loads the + // pre-quarantine key while the recoverer proceeds to quarantine and + // republish: two-sided divergence. Only a claim older than + // CLAIM_SWEEP_MIN_AGE (an orphan of a crashed recovery) may go. + // `.publishing.` markers keep the unconditional sweep: sweeping a live + // publisher's marker only fails its commit check, a safe demotion. RED + // before G1: the young claim is swept too. + #[test] + fn healthy_load_spares_young_claims() { + let dir = tempfile::tempdir().expect("tempdir"); + let existing = Keypair::generate(); + let pem = existing.to_pem().expect("pem"); + let key_path = seed_crash_state(dir.path(), pem.as_bytes()); + let young = dir.path().join(".identity.pem.recovering.11111"); + std::fs::write(&young, b"").expect("seed young (possibly live) claim"); + let old = dir.path().join(".identity.pem.recovering.22222"); + std::fs::write(&old, b"").expect("seed orphaned claim"); + age_beyond_claim_sweep(&old); + + let kp = load_or_create_keypair(&key_config(&key_path)) + .expect("a valid final must load regardless of claim ages"); + + assert_eq!( + format!("{}", kp.did()), + format!("{}", existing.did()), + "the pre-existing identity must be preserved" + ); + assert!( + young.exists(), + "a young claim may belong to a LIVE recovery and must survive the sweep" + ); + assert!( + !old.exists(), + "an orphaned claim past CLAIM_SWEEP_MIN_AGE must be swept" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "publish markers keep the unconditional sweep" + ); + } + + // F2 (function level): the boot sweep SPARES a young claim (it may gate + // a live round), so orphaned residue can outlive a healthy boot; once + // the claim ages past CLAIM_SWEEP_MIN_AGE a SECOND sweep over the same + // final must clear it, or the residue later pairs with a real content + // failure of the good final and misclassifies marker-less corruption as + // a crash. GREEN at introduction (the sweep is already age-correct); + // load-bearing by mutation check: inverting the sweep's age gate turns + // it RED. spawn_delayed_claim_resweep is the boot wiring that provides + // the second sweep in-process. + #[test] + fn second_sweep_clears_aged_orphan_claims() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let pem = Keypair::generate().to_pem().expect("pem"); + std::fs::write(&key_path, pem.as_bytes()).expect("seed valid final"); + std::fs::set_permissions(&key_path, std::fs::Permissions::from_mode(0o600)) + .expect("0600 final"); + let claim = dir.path().join(".identity.pem.recovering.33333"); + std::fs::write(&claim, b"").expect("seed young claim"); + + super::sweep_stale_markers(&key_path, || Ok(())); + assert!( + claim.exists(), + "the first sweep must spare the young (possibly live) claim" + ); + + age_beyond_claim_sweep(&claim); + super::sweep_stale_markers(&key_path, || Ok(())); + assert!( + !claim.exists(), + "a second sweep must clear the claim once it ages into an orphan" + ); + } + + // M4: `.superseded.` forensic files never expired, so every lost round + // accumulated one beside the final forever. The healthy sweep must + // remove a superseded file older than SUPERSEDED_RETENTION (the + // operator's inspection window) while a YOUNG one survives; the sweep's + // durability gate already ran before any removal. RED before M4: the + // aged file survives every healthy boot. + #[test] + fn superseded_residue_is_retention_swept() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let pem = Keypair::generate().to_pem().expect("pem"); + std::fs::write(&key_path, pem.as_bytes()).expect("seed valid final"); + std::fs::set_permissions(&key_path, std::fs::Permissions::from_mode(0o600)) + .expect("0600 final"); + let aged = dir.path().join(".identity.pem.superseded.99999.0"); + std::fs::write(&aged, b"old forensics").expect("seed aged superseded file"); + set_mtime_past(&aged, "2 days ago"); + let young = dir.path().join(".identity.pem.superseded.99999.1"); + std::fs::write(&young, b"fresh forensics").expect("seed young superseded file"); + + load_or_create_keypair(&key_config(&key_path)).expect("healthy load"); + + assert!( + !aged.exists(), + "a superseded file past the retention window must be swept" + ); + assert!( + young.exists(), + "a superseded file inside the retention window must survive" + ); + } + + // Q2: plain `.quarantined.` residue also never expired: an unparseable + // quarantine (crash forensics) and an aged parseable one (past the + // adoption floor, so pure history) sat beside a healthy final forever. + // The healthy sweep must remove a `.quarantined.` entry older than + // SUPERSEDED_RETENTION, while a YOUNG unparseable one (fresh crash + // forensics) and a parseable one past the adoption floor but inside + // retention (recent history, an operator may still want it) survive. + // RED before Q2: the aged unparseable quarantine survives every + // healthy boot. + #[test] + fn old_quarantine_forensics_are_retention_swept() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let pem = Keypair::generate().to_pem().expect("pem"); + std::fs::write(&key_path, pem.as_bytes()).expect("seed valid final"); + std::fs::set_permissions(&key_path, std::fs::Permissions::from_mode(0o600)) + .expect("0600 final"); + let aged_garbage = dir.path().join(".identity.pem.quarantined.99999.0"); + std::fs::write(&aged_garbage, b"-----BEGIN nonsense").expect("seed aged forensics"); + set_mtime_past(&aged_garbage, "2 days ago"); + let young_garbage = dir.path().join(".identity.pem.quarantined.99999.1"); + std::fs::write(&young_garbage, b"-----BEGIN nonsense").expect("seed young forensics"); + let recent_parseable = dir.path().join(".identity.pem.quarantined.99999.2"); + std::fs::write( + &recent_parseable, + Keypair::generate().to_pem().expect("pem").as_bytes(), + ) + .expect("seed recent parseable quarantine"); + // Past the adoption floor (CLAIM_SWEEP_MIN_AGE) but well inside the + // 24h retention window. + set_mtime_past(&recent_parseable, "1 hour ago"); + + load_or_create_keypair(&key_config(&key_path)).expect("healthy load"); + + assert!( + !aged_garbage.exists(), + "a quarantine past the retention window must be swept" + ); + assert!( + young_garbage.exists(), + "young crash forensics must survive the healthy sweep" + ); + assert!( + recent_parseable.exists(), + "a quarantine past the adoption floor but inside retention must survive" + ); + } + + // F2 (wiring): the delayed boot re-sweep must actually fire and sweep. + // Paused tokio time drives the real CLAIM_SWEEP_MIN_AGE-plus-margin + // sleep instantly; the claim is pre-aged on disk because paused time + // advances the tokio clock, not SystemTime mtimes. This covers "the + // spawned task runs the sweep after the delay"; + // second_sweep_clears_aged_orphan_claims covers the age semantics. + #[tokio::test(start_paused = true)] + async fn delayed_resweep_clears_aged_orphan_claims() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let pem = Keypair::generate().to_pem().expect("pem"); + std::fs::write(&key_path, pem.as_bytes()).expect("seed valid final"); + std::fs::set_permissions(&key_path, std::fs::Permissions::from_mode(0o600)) + .expect("0600 final"); + let claim = dir.path().join(".identity.pem.recovering.66666"); + std::fs::write(&claim, b"").expect("seed claim"); + age_beyond_claim_sweep(&claim); + + super::spawn_delayed_claim_resweep(key_path.clone()) + .await + .expect("the re-sweep task must complete"); + + assert!( + !claim.exists(), + "the delayed re-sweep must clear the aged orphan claim" + ); + assert!( + super::load_existing_key(&key_path).is_ok(), + "the final must survive the re-sweep intact" + ); + } + + // The G1 age gate reads the CLAIM file's mtime, but a claim is made by + // RENAMING a marker, and rename preserves the source's mtime: a claim + // made from an hours-old stale marker would be born already past + // CLAIM_SWEEP_MIN_AGE, so a concurrent healthy boot's sweep could remove + // the LIVE claim mid-round and early-wake the demoted publisher it + // gates. Recovery must therefore stamp each claim's mtime fresh at + // claim time. RED before the fix: the claim inherits the aged mtime. + #[test] + fn claim_freshness_is_stamped_at_claim_time() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + let marker = dir.path().join(".identity.pem.publishing.99999.0"); + age_beyond_claim_sweep(&marker); + + let recovery = super::recover_crashed_publish( + &key_path, + anyhow::anyhow!("synthetic crash signature"), + RecoverySeam::NONE, + ) + .expect("recovery over an aged marker must not error"); + let mut claims = match recovery { + super::Recovery::Claimed(claims) => claims, + super::Recovery::Reloaded(_) => { + panic!("an aged marker beside an empty final must be claimed") + } + }; + assert!( + !claims.files.is_empty(), + "the aged marker must produce a claim" + ); + for claim in &claims.files { + let mtime = std::fs::metadata(claim) + .expect("stat claim") + .modified() + .expect("claim mtime"); + let age = std::time::SystemTime::now() + .duration_since(mtime) + .unwrap_or_default(); + assert!( + age < super::CLAIM_SWEEP_MIN_AGE, + "a claim must be born fresh, not inherit the aged marker mtime \ + through the rename: age {age:?} at {claim:?}" + ); + } + super::release_recovery_claims(&key_path, &mut claims); + } + + // The G1 freshness stamp must survive a failing set_modified: some + // filesystems reject explicit timestamps while honoring write-driven + // mtime updates, and a claim left with its inherited aged mtime for the + // whole round is exposed to a concurrent healthy boot's age-gated sweep. + // The ladder's fallback appends a byte, which bumps the mtime on + // effectively every filesystem (claims are matched by name only, so a + // non-empty claim is safe). RED with the fallback rung absent: the + // claim keeps the aged mtime. + #[test] + fn stamp_fallback_bumps_mtime_by_write() { + let dir = tempfile::tempdir().expect("tempdir"); + let claim = dir.path().join(".identity.pem.recovering.99999.deadbeef.0"); + std::fs::write(&claim, b"").expect("seed claim"); + age_beyond_claim_sweep(&claim); + + super::refresh_claim_mtime(&claim, |_| Err(std::io::ErrorKind::Unsupported.into())) + .expect("the write fallback must stamp the mtime when the primary fails"); + + let mtime = std::fs::metadata(&claim) + .expect("stat claim") + .modified() + .expect("claim mtime"); + let age = std::time::SystemTime::now() + .duration_since(mtime) + .unwrap_or_default(); + assert!( + age < super::CLAIM_SWEEP_MIN_AGE, + "the fallback write must leave the claim fresh, age {age:?}" + ); + } + + // M3: claim liveness is age < CLAIM_SWEEP_MIN_AGE, but a single + // claim-time stamp lets a recoverer stalled longer than the floor + // between claim and settlement read as an orphan to the sweep and to + // claimability: the delayed re-sweep or a healthy boot removes the + // still-live claim (early-releasing any demoted publisher's + // absence-keyed wait to a pre-settlement load), or a competing + // recoverer steals the round: divergence returns on slow shared + // volumes. A live + // recovery must HEARTBEAT its claims so they never age out while it + // runs. The park holds recovery inside the claim-to-settlement window + // (before_quarantine) just past the floor while the test samples the + // claim's age throughout. RED before M3: the single stamp ages past the + // floor mid-park. The ~12s wall cost is the test's point: the floor is + // real time. + #[test] + fn heartbeat_keeps_live_claims_young() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + + let park_dir = dir.path().to_path_buf(); + let before_quarantine = move || { + let deadline = std::time::Instant::now() + + super::CLAIM_SWEEP_MIN_AGE + + std::time::Duration::from_secs(2); + while std::time::Instant::now() < deadline { + std::thread::sleep(std::time::Duration::from_millis(900)); + let claims = names_containing(&park_dir, ".recovering."); + assert!(!claims.is_empty(), "the parked round's claim must persist"); + for name in &claims { + let mtime = std::fs::metadata(park_dir.join(name)) + .and_then(|m| m.modified()) + .expect("claim mtime"); + let age = std::time::SystemTime::now() + .duration_since(mtime) + .unwrap_or_default(); + assert!( + age < super::CLAIM_SWEEP_MIN_AGE, + "a LIVE recovery's claim must stay young mid-round: age {age:?}" + ); + } + } + }; + let seam = RecoverySeam { + before_quarantine: &before_quarantine, + ..RecoverySeam::NONE + }; + + let started = std::time::Instant::now(); + let kp = load_or_create_keypair_with(&key_path, &|| {}, &|| {}, PublishFaults::NONE, seam) + .expect("the parked recovery must still resolve a key"); + let elapsed = started.elapsed(); + + // The recovery finished: claims released (the on-disk leak + // evidence) and the heartbeat joined (the elapsed ceiling bounds + // release plus join after the ~12s park). + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "the finished recovery must release its claims" + ); + assert!( + elapsed < super::CLAIM_SWEEP_MIN_AGE + std::time::Duration::from_secs(8), + "release and heartbeat join must be prompt after the park: took {elapsed:?}" + ); + let on_disk = super::load_existing_key(&key_path).expect("regenerated final parses"); + assert_eq!( + format!("{}", on_disk.did()), + format!("{}", kp.did()), + "the returned identity must match the on-disk key" + ); + } + + // Claim destinations must carry a per-process nonce beyond the pid: + // `.{stem}.recovering.{pid}.{nonce}.{n}`. The pid alone is not unique + // across recoverers on a shared volume (two PID-1 containers), so two + // concurrent recoverers could derive the SAME destination name, and the + // second's claim rename would replace the first's live claim inode (the + // exists() probe is a check-then-act, not an arbiter); the first release + // would then unlink the other's LIVE claim and early-wake the demoted + // publisher it gates. The cross-process collision itself cannot be + // executed from one test process (one pid, one nonce), so this test + // fences the mechanism: every claim name must carry the process nonce + // field between the pid and the counter. RED under pid-only naming. + #[test] + fn same_pid_recoverers_get_unique_claim_names() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + + let recovery = super::recover_crashed_publish( + &key_path, + anyhow::anyhow!("synthetic crash signature"), + RecoverySeam::NONE, + ) + .expect("recovery over a crash state must not error"); + let mut claims = match recovery { + super::Recovery::Claimed(claims) => claims, + super::Recovery::Reloaded(_) => { + panic!("a marker beside an empty final must be claimed") + } + }; + assert!(!claims.files.is_empty(), "the marker must produce a claim"); + let pid_prefix = format!(".identity.pem.recovering.{}.", std::process::id()); + for claim in &claims.files { + let name = claim + .file_name() + .and_then(|n| n.to_str()) + .expect("claim name is utf8"); + let rest = name + .strip_prefix(&pid_prefix) + .unwrap_or_else(|| panic!("claim {name:?} must start with {pid_prefix:?}")); + let fields: Vec<&str> = rest.split('.').collect(); + assert_eq!( + fields.len(), + 2, + "claim {name:?} must end in {{nonce}}.{{counter}}, got {rest:?}" + ); + assert_eq!( + fields[0], + format!("{:08x}", super::claim_nonce()), + "the field between pid and counter must be the process nonce" + ); + assert!( + fields[1].parse::().is_ok(), + "the final field must be the claim counter, got {:?}", + fields[1] + ); + } + super::release_recovery_claims(&key_path, &mut claims); + } + + // A final that VANISHES between the claim renames and the post-claim + // re-parse (a competing recoverer disposed of it concurrently) must join + // the content class and regenerate, not surface as a loud transient + // error: there is nothing a quarantine could destroy, and the quarantine + // loop's own NotFound arm resolves it cleanly. Locks the NotFound arm of + // the re-parse match (anyhow's downcast_ref:: sees the + // NotFound through the with_context layers). RED with NotFound routed + // into the transient arm (mutation check): the boot fails loudly. + #[test] + fn vanished_final_at_reparse_still_regenerates() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + let vanish_path = key_path.clone(); + let before_reparse = move || { + std::fs::remove_file(&vanish_path).expect("vanish the final inside the claim window"); + }; + + let kp = load_or_create_keypair_with( + &key_path, + &|| {}, + &|| {}, + PublishFaults::NONE, + RecoverySeam { + before_reparse: &before_reparse, + ..RecoverySeam::NONE + }, + ) + .expect("a final vanished at the re-parse must regenerate, not fail the start"); + + let on_disk = super::load_existing_key(&key_path).expect("regenerated final parses"); + assert_eq!( + format!("{}", on_disk.did()), + format!("{}", kp.did()), + "the returned identity must match the regenerated on-disk key" + ); + assert!( + names_containing(dir.path(), ".quarantined.").is_empty(), + "a vanished final leaves nothing to quarantine" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "no publish markers may remain" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "no recovery claims may remain" + ); + } + + // #194 (U4): the generate path must sweep too. A crash between the + // marker's durability fsync and the final's `create_new` leaves a durable + // marker with NO final; only the Won arm of a later boot can ever clean + // that state, since no load will ever succeed beside it. RED before U4 + // (the marker survives); RED again under a load-arm-only implementation + // (mutation check: with the Won-arm call site removed this test must + // fail while healthy_load_sweeps_stale_markers_and_claims stays green). + #[test] + fn generate_path_sweeps_stale_markers() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + std::fs::write(dir.path().join(".identity.pem.publishing.66666.0"), b"") + .expect("seed stale marker without a final"); + + let kp = load_or_create_keypair(&key_config(&key_path)) + .expect("a stale marker with no final must not block generation"); + + let on_disk = super::load_existing_key(&key_path).expect("published key loads"); + assert_eq!( + format!("{}", on_disk.did()), + format!("{}", kp.did()), + "the generated identity must be on disk" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "a Won publish on a healthy boot must sweep the stale marker" + ); + } + + // #194 (U4) MUST-NOT: the sweep must never remove a marker ahead of the + // content it vouches for. With the seam failing the sweep's durability + // sync, the load still succeeds but the marker SURVIVES. Vacuously green + // before U4 (no sweep exists, so the marker trivially survives); its + // load-bearing proof is the mutation check: with the durability gate + // removed (sweep unconditionally) this test must fail. + #[test] + fn sweep_durability_gate_leaves_markers_on_sync_failure() { + let dir = tempfile::tempdir().expect("tempdir"); + let existing = Keypair::generate(); + let pem = existing.to_pem().expect("pem"); + let key_path = seed_crash_state(dir.path(), pem.as_bytes()); + let seam = RecoverySeam { + sweep_sync: || Err(std::io::Error::other("injected sweep sync failure")), + ..RecoverySeam::NONE + }; + + let kp = load_or_create_keypair_with(&key_path, &|| {}, &|| {}, PublishFaults::NONE, seam) + .expect("a failed sweep durability sync must not fail the load"); + + assert_eq!( + format!("{}", kp.did()), + format!("{}", existing.did()), + "the identity must still load when the sweep is skipped" + ); + assert_eq!( + names_containing(dir.path(), ".publishing."), + vec![".identity.pem.publishing.99999.0".to_string()], + "the marker must survive when the sweep cannot make the final durable" + ); + } + + // #194 (U4): the sweep's removals are best-effort; unremovable markers + // must never fail an otherwise healthy load. The key directory is made + // read-only (0555): the durability gate still passes (the file sync_all + // and the dir fsync both open read-only) but every remove_file fails + // EACCES. Vacuously green before U4 (no sweep exists); load-bearing once + // the sweep runs, since a sweep that surfaced removal errors would fail + // this load. + #[test] + fn sweep_failure_tolerated() { + let dir = tempfile::tempdir().expect("tempdir"); + let existing = Keypair::generate(); + let pem = existing.to_pem().expect("pem"); + let key_path = seed_crash_state(dir.path(), pem.as_bytes()); + std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o555)) + .expect("read-only key dir"); + + let result = load_or_create_keypair(&key_config(&key_path)); + + std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o755)) + .expect("restore key dir"); + let kp = result.expect("unremovable markers must not fail the load"); + assert_eq!( + format!("{}", kp.did()), + format!("{}", existing.did()), + "the identity must load despite the failed sweep removals" + ); + assert_eq!( + names_containing(dir.path(), ".publishing."), + vec![".identity.pem.publishing.99999.0".to_string()], + "the unremovable marker survives, harmlessly" + ); + } + + // #194 (F1): a claimed recovery whose retry publish FAILS must release + // its `.recovering.` claim before surfacing the error. A leaked claim + // makes every later demoted fallback publisher ride the full wait + // bound in wait_for_recovery_claims and fail its publish closed until + // some healthy boot sweeps it. RED before F1: the bare `?` on the + // publish call returns with the claim still on disk. + #[test] + fn claim_released_when_post_recovery_publish_fails() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + fallback_create: || Err(std::io::Error::other("injected fallback create failure")), + ..PublishFaults::NONE + }; + + let err = match load_or_create_keypair_with( + &key_path, + &|| {}, + &|| {}, + faults, + RecoverySeam::NONE, + ) { + Err(e) => e, + Ok(_) => panic!("the forced publish failure must surface, not resolve a key"), + }; + assert!( + format!("{err:#}").contains("injected fallback create failure"), + "the publish failure must propagate: {err:#}" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "a failed retry publish must release the recovery claim, not leak it" + ); + } + + // #194 (F2): a stale marker must not let recovery steal a LIVE fallback + // publisher's round. Deterministic construction: publisher A parks at + // its commit check (final complete and durable, own marker M present, + // stale marker S beside it); the real recovery runs, and its + // before-reparse seam (inside the window between the claim step and the + // post-claim re-parse) unparks A and waits for A to finish, landing A's + // commit exactly in the claim window. Under F2 recovery claims EVERY + // claimable, so A's removal of M fails and A demotes; the re-parse then + // sees A's completed key and recovery aborts to Reloaded. Everyone + // converges on one on-disk identity. RED before F2: recovery claims only + // the lexicographically-first stale S, A's commit on M succeeds, and A + // returns Won with key A while recovery quarantines A's final and + // republishes key B: the two-sided divergence R4 forbids. + #[test] + fn stale_marker_cannot_steal_a_live_publishers_round() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + // Sorts before any real-pid marker (pids never start with '0'). + std::fs::write(dir.path().join(".identity.pem.publishing.00000.0"), b"") + .expect("seed stale marker"); + + let (parked_tx, parked_rx) = std::sync::mpsc::channel::<()>(); + let (unpark_tx, unpark_rx) = std::sync::mpsc::channel::<()>(); + let (done_tx, done_rx) = std::sync::mpsc::channel::(); + let p_path = key_path.clone(); + let publisher = std::thread::spawn(move || { + let before_commit = move || { + parked_tx.send(()).expect("signal parked"); + unpark_rx + .recv() + .expect("wait inside the recovery's claim window"); + }; + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + ..PublishFaults::NONE + }; + let kp = load_or_create_keypair_with( + &p_path, + &|| {}, + &before_commit, + faults, + RecoverySeam::NONE, + ) + .expect("the publisher must resolve a key, demoted or not"); + done_tx + .send(format!("{}", kp.did())) + .expect("report the publisher's identity"); + }); + parked_rx + .recv() + .expect("publisher reaches its commit check"); + + // The concurrent recovery, via the REAL recovery path. Its + // before-reparse hook unparks A inside the window between the claim + // step and the post-claim re-parse: A's marker is already claimed, + // so A's commit check fails and A demotes, then waits out the LIVE + // claims until this recovery releases them. (The hook must not park + // recovery until A finishes: A's waiter holds while the claim + // files are present, until release, so that cycle would correctly + // fail A closed; the pre-heartbeat, age-keyed version only + // unblocked because the live claims aged out mid-round, the exact + // divergence window M3 and the absence-keyed waiter close.) + let before_reparse = || { + unpark_tx.send(()).expect("unpark the publisher"); + }; + let recovery = super::recover_crashed_publish( + &key_path, + anyhow::anyhow!("synthetic crash signature"), + RecoverySeam { + before_reparse: &before_reparse, + ..RecoverySeam::NONE + }, + ) + .expect("recovery must not error"); + let recovered_did = match recovery { + super::Recovery::Reloaded(result) => format!( + "{}", + result + .expect("the post-claim re-parse resolves the completed key") + .did() + ), + super::Recovery::Claimed(mut recovery_claims) => { + // The boot loop's follow-through: regenerate, publish, release. + let kp_b = Keypair::generate(); + let pem_b = kp_b.to_pem().expect("pem b"); + let out = publish_key_atomically( + &key_path, + pem_b.as_bytes(), + &|| {}, + &|| {}, + PublishFaults::NONE, + ) + .expect("recovery's republish"); + assert!(matches!(out, KeyPublish::Won), "recovery's republish wins"); + super::release_recovery_claims(&key_path, &mut recovery_claims); + format!("{}", kp_b.did()) + } + }; + publisher.join().expect("publisher joins"); + + let p_did = done_rx.recv().expect("the publisher reported its identity"); + let disk_did = format!( + "{}", + super::load_existing_key(&key_path) + .expect("final parses") + .did() + ); + assert_eq!( + p_did, disk_did, + "the publisher must never resolve an identity that diverges from disk" + ); + assert_eq!( + recovered_did, disk_did, + "recovery must converge on the same on-disk identity" + ); + assert!( + names_containing(dir.path(), ".quarantined.").is_empty(), + "a completed publisher's final must not be quarantined" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "no publish markers may remain" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "no recovery claims may remain" + ); + } + + // #194 (F3): a recoverer crashed between its claim rename and the + // quarantine leaves a content-bad final beside a stale `.recovering.` + // claim and ZERO `.publishing.` markers: the claim IS the renamed + // marker. The claim must count toward the crash signature and be + // claimable itself, or no later boot can ever recover: the permanent + // wedge this protocol exists to eliminate. RED before F3: the signature + // counts only `.publishing.` names, so the boot rides the deadline and + // fails loudly forever. + #[test] + fn claim_crash_state_is_recoverable() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + std::fs::write(&key_path, b"").expect("seed crashed final"); + std::fs::set_permissions(&key_path, std::fs::Permissions::from_mode(0o600)) + .expect("0600 crashed final"); + let claim = dir.path().join(".identity.pem.recovering.99999.0"); + std::fs::write(&claim, b"").expect("seed stale recovery claim"); + // F2 age gate: only a claim past CLAIM_SWEEP_MIN_AGE is claimable; a + // fresh one may gate a LIVE round (live_claims_are_not_claimable is + // the must-not side). Age the orphan so this stays the positive case. + age_beyond_claim_sweep(&claim); + + let kp = load_or_create_keypair(&key_config(&key_path)) + .expect("a claim-crash state must recover, not wedge every later boot"); + + let on_disk = super::load_existing_key(&key_path).expect("regenerated final parses"); + assert_eq!( + format!("{}", on_disk.did()), + format!("{}", kp.did()), + "the returned identity must match the regenerated on-disk key" + ); + let quarantined = names_containing(dir.path(), ".quarantined."); + assert_eq!( + quarantined.len(), + 1, + "exactly one quarantine file must exist: {quarantined:?}" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "the stale claim must be consumed and the fresh claim released" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "no publish markers may remain" + ); + } + + // F2 MUST-NOT: a FRESH `.recovering.` claim may gate a LIVE recovery + // round (claim, quarantine, republish), and a second starter claiming it + // mid-round steals that round: the recoverer's own re-parse and restore + // then race the thief's quarantine-and-regenerate. A fresh claim must + // therefore never count as claimable: with no `.publishing.` marker + // beside it the claimable set is empty, no crash signature fires, and + // the load fails loudly with the claim untouched. The positive case (an + // ORPHANED claim past CLAIM_SWEEP_MIN_AGE) is + // claim_crash_state_is_recoverable. Rides the full KEY_RACE_DEADLINE + // (~5s), as every loud-fail load does. RED before F2: the fresh claim is + // claimed and recovery proceeds. + #[test] + fn live_claims_are_not_claimable() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + std::fs::write(&key_path, b"").expect("seed crashed final"); + std::fs::set_permissions(&key_path, std::fs::Permissions::from_mode(0o600)) + .expect("0600 crashed final"); + let claim = dir.path().join(".identity.pem.recovering.99999.0"); + std::fs::write(&claim, b"").expect("seed fresh (possibly live) claim"); + + let err = match load_or_create_keypair(&key_config(&key_path)) { + Err(e) => e, + Ok(_) => panic!("a fresh claim must never be stolen; the load must fail loudly"), + }; + + assert!( + err.to_string().contains("invalid PEM key"), + "the load failure must surface unchanged: {err:#}" + ); + assert!( + claim.exists(), + "the possibly-live claim must survive untouched" + ); + assert!( + names_containing(dir.path(), ".quarantined.").is_empty(), + "no quarantine may be created off a live claim" + ); + } + + // F1: a COMPLETED key stranded in a quarantine (the recoverer's restore + // republish errored, or the recoverer crashed between its quarantine + // rename and the restore) must be ADOPTED by the next boot: the final is + // absent, so without adoption the boot mints a fresh DID and the node's + // durable identity silently changes. The generate arm scans for + // parseable quarantines and republishes the stranded bytes instead. RED + // before F1: a fresh identity is minted and the quarantine is stranded + // forever. + #[test] + fn stranded_quarantine_is_adopted_not_replaced() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let stranded = Keypair::generate(); + let pem = stranded.to_pem().expect("pem"); + std::fs::write( + dir.path().join(".identity.pem.quarantined.99999.0"), + pem.as_bytes(), + ) + .expect("seed stranded quarantine"); + + let kp = load_or_create_keypair(&key_config(&key_path)) + .expect("a stranded completed key must be adopted, not fail the boot"); + + assert_eq!( + format!("{}", kp.did()), + format!("{}", stranded.did()), + "the stranded identity must be adopted, never replaced by a fresh DID" + ); + let on_disk = super::load_existing_key(&key_path).expect("republished final parses"); + assert_eq!( + format!("{}", on_disk.did()), + format!("{}", stranded.did()), + "the final must hold the adopted key" + ); + assert!( + names_containing(dir.path(), ".quarantined.").is_empty(), + "the quarantine must be consumed once its bytes are durably republished" + ); + } + + // F1 MUST-NOT (age bound): adoption exists for the CURRENT round only (a + // recoverer that crashed or failed seconds ago), so a parseable + // quarantine AGED past CLAIM_SWEEP_MIN_AGE is history, not a stranded + // round: kept forensics after a Lost restore, or an old identity the + // operator already moved past. Adopting it would resurrect a dead DID + // whenever the final later goes missing. The aged quarantine must + // survive untouched as forensics while boot generates fresh. RED before + // the age bound: the aged quarantine is adopted. + #[test] + fn aged_quarantine_is_never_adopted() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let stranded = Keypair::generate(); + let pem = stranded.to_pem().expect("pem"); + let quarantine = dir.path().join(".identity.pem.quarantined.99999.0"); + std::fs::write(&quarantine, pem.as_bytes()).expect("seed aged quarantine"); + age_beyond_claim_sweep(&quarantine); + + let kp = load_or_create_keypair(&key_config(&key_path)) + .expect("an aged quarantine must not block a fresh generation"); + + assert_ne!( + format!("{}", kp.did()), + format!("{}", stranded.did()), + "an aged quarantine is history and must never be resurrected" + ); + assert_eq!( + std::fs::read(&quarantine).expect("quarantine still on disk"), + pem.as_bytes(), + "the aged quarantine must survive byte-for-byte as forensics" + ); + } + + // F1 (operator procedure): deleting identity.pem is the documented way + // to mint a fresh DID. With an old parseable quarantine sitting beside + // the final as forensics, the post-delete boot must GENERATE a new + // identity, not adopt the quarantined one; unbounded adoption silently + // defeats the procedure. RED before the age bound: the old quarantine's + // DID comes back. + #[test] + fn operator_fresh_identity_procedure_works() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let first = load_or_create_keypair(&key_config(&key_path)).expect("provision identity"); + let old = Keypair::generate(); + let old_pem = old.to_pem().expect("pem"); + let quarantine = dir.path().join(".identity.pem.quarantined.88888.0"); + std::fs::write(&quarantine, old_pem.as_bytes()).expect("seed forensic quarantine"); + age_beyond_claim_sweep(&quarantine); + + // The operator action: delete the final to get a fresh DID. + std::fs::remove_file(&key_path).expect("operator deletes the final"); + + let kp = load_or_create_keypair(&key_config(&key_path)) + .expect("the post-delete boot must provision a fresh identity"); + + assert_ne!( + format!("{}", kp.did()), + format!("{}", old.did()), + "the old quarantined identity must not be resurrected" + ); + assert_ne!( + format!("{}", kp.did()), + format!("{}", first.did()), + "the operator must get a genuinely fresh DID" + ); + } + + // N2: a quarantine kept by a LOST restore is provably outcompeted (a + // concurrent winner durably published while its key sat quarantined), + // yet it is young and parseable: exactly what adoption prefers. Left in + // the `.quarantined.` class, the operator's + // delete-identity.pem-for-a-fresh-DID procedure resurrects the LOSING + // key instead of minting fresh. The Lost restore must rename its kept + // quarantine into the `.superseded.` class: bytes preserved as + // forensics, never adoptable. RED before N2: the post-delete boot + // adopts and returns the losing key A. + #[test] + fn lost_restore_quarantine_is_never_adopted() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + let completed = Keypair::generate(); + let pem_a = completed.to_pem().expect("pem a"); + let kp_b = Keypair::generate(); + let pem_b = kp_b.to_pem().expect("pem b"); + + // restore_never_clobbers_a_concurrent_winner's construction: A's + // completed key is swept into quarantine, B takes the freed name + // inside the quarantine-to-restore window, and the link-hostile + // restore republish of A loses to B. + let heal_path = key_path.clone(); + let pem_a_disk = pem_a.clone(); + let before_quarantine = move || { + std::fs::write(&heal_path, pem_a_disk.as_bytes()) + .expect("complete the final inside the re-parse-to-rename window"); + }; + let steal_path = key_path.clone(); + let pem_b_disk = pem_b.clone(); + let before_restore = move || { + let out = publish_key_atomically( + &steal_path, + pem_b_disk.as_bytes(), + &|| {}, + &|| {}, + PublishFaults::NONE, + ) + .expect("concurrent starter republishes key B at the freed name"); + assert!( + matches!(out, KeyPublish::Won), + "B's publish wins the free name" + ); + }; + let seam = RecoverySeam { + before_quarantine: &before_quarantine, + before_restore: &before_restore, + restore_faults: PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + ..PublishFaults::NONE + }, + ..RecoverySeam::NONE + }; + let kp = load_or_create_keypair_with(&key_path, &|| {}, &|| {}, PublishFaults::NONE, seam) + .expect("the losing restore must converge on B"); + assert_eq!( + format!("{}", kp.did()), + format!("{}", kp_b.did()), + "the recovering boot must converge on the concurrent winner's key B" + ); + + // The operator action: delete the final for a fresh DID. + std::fs::remove_file(&key_path).expect("operator deletes the final"); + let fresh = load_or_create_keypair(&key_config(&key_path)) + .expect("the post-delete boot must provision an identity"); + assert_ne!( + format!("{}", fresh.did()), + format!("{}", completed.did()), + "the key that LOST its round must never be resurrected by adoption" + ); + assert_ne!( + format!("{}", fresh.did()), + format!("{}", kp_b.did()), + "the operator must get a genuinely fresh DID" + ); + // A's bytes survive as forensics, in the never-adoptable class. + assert!( + names_containing(dir.path(), ".quarantined.").is_empty(), + "no adoptable quarantine may survive a lost restore" + ); + let superseded = names_containing(dir.path(), ".superseded."); + assert_eq!( + superseded.len(), + 1, + "the losing key must be kept once, superseded: {superseded:?}" + ); + assert_eq!( + std::fs::read(dir.path().join(&superseded[0])).expect("superseded readable"), + pem_a.as_bytes(), + "the superseded file must preserve the losing key's bytes as forensics" + ); + } + + // N4: adoption decides its round in favor of ONE quarantine, but a + // crashed pile-up can leave several young parseable quarantines side by + // side. Adopting the newest while leaving the siblings adoptable means + // a later missing final resurrects a DIFFERENT historical DID. On Won, + // adoption must rename every other young parseable sibling into the + // `.superseded.` class (bytes intact, never adoptable). RED before N4: + // the sibling stays `.quarantined.` and the post-delete boot adopts it. + #[test] + fn adoption_supersedes_sibling_quarantines() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let older = Keypair::generate(); + let older_pem = older.to_pem().expect("pem"); + let q_old = dir.path().join(".identity.pem.quarantined.99999.0"); + std::fs::write(&q_old, older_pem.as_bytes()).expect("seed older quarantine"); + // A strictly newer mtime on the second quarantine, so newest-first + // adoption is deterministic. + std::thread::sleep(std::time::Duration::from_millis(20)); + let newer = Keypair::generate(); + let q_new = dir.path().join(".identity.pem.quarantined.99999.1"); + std::fs::write(&q_new, newer.to_pem().expect("pem").as_bytes()) + .expect("seed newer quarantine"); + + let kp = load_or_create_keypair(&key_config(&key_path)) + .expect("adoption must resolve the stranded round"); + assert_eq!( + format!("{}", kp.did()), + format!("{}", newer.did()), + "adoption must prefer the newest quarantine" + ); + assert!( + names_containing(dir.path(), ".quarantined.").is_empty(), + "the adopted quarantine is consumed and the sibling must be superseded" + ); + let superseded = names_containing(dir.path(), ".superseded."); + assert_eq!( + superseded.len(), + 1, + "exactly the outcompeted sibling is superseded: {superseded:?}" + ); + assert_eq!( + std::fs::read(dir.path().join(&superseded[0])).expect("superseded readable"), + older_pem.as_bytes(), + "the superseded sibling must preserve its bytes as forensics" + ); + + // The sibling must not come back as a different historical DID once + // the final goes missing: the post-delete boot mints fresh. + std::fs::remove_file(&key_path).expect("operator deletes the final"); + let fresh = load_or_create_keypair(&key_config(&key_path)) + .expect("the post-delete boot must provision an identity"); + assert_ne!( + format!("{}", fresh.did()), + format!("{}", older.did()), + "the superseded sibling must never be resurrected" + ); + assert_ne!( + format!("{}", fresh.did()), + format!("{}", newer.did()), + "the operator must get a genuinely fresh DID" + ); + } + + // Q1: both Won arms now supersede the sibling quarantines BEFORE + // consuming the chosen one, so a crash in the gap leaves the + // POST-SUPERSEDE image: final present, siblings already superseded, + // only the chosen quarantine (the winner's own bytes) still adoptable. + // No seam reaches the reordered gap, so this test pins the two crash + // images directly rather than interposing. First the post-fix image: a + // later missing final may re-adopt only the winner's own key, never a + // different historical DID. Then the PRE-fix image (chosen consumed + // first, sibling still plain): a concrete demonstration that the old + // ordering's crash gap left the sibling adoptable, resurrecting a + // different DID after delete-final. The ordering itself is not + // runtime-observable without a production seam; see the fix report. + #[test] + fn crash_between_won_and_sibling_supersede_leaves_no_adoptable() { + // POST-fix crash image: crash fell between the sibling supersede + // and the chosen quarantine's removal. + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let winner = Keypair::generate(); + let winner_pem = winner.to_pem().expect("pem"); + let sibling = Keypair::generate(); + let sibling_pem = sibling.to_pem().expect("pem"); + std::fs::write(&key_path, winner_pem.as_bytes()).expect("seed final"); + std::fs::set_permissions(&key_path, std::fs::Permissions::from_mode(0o600)) + .expect("0600 final"); + std::fs::write( + dir.path().join(".identity.pem.superseded.99999.0"), + sibling_pem.as_bytes(), + ) + .expect("seed superseded sibling"); + let chosen = dir.path().join(".identity.pem.quarantined.99999.1"); + std::fs::write(&chosen, winner_pem.as_bytes()).expect("seed un-consumed chosen"); + + let kp = load_or_create_keypair(&key_config(&key_path)) + .expect("the post-supersede crash image must boot healthy"); + assert_eq!( + format!("{}", kp.did()), + format!("{}", winner.did()), + "the final's key wins the crash-image boot" + ); + + std::fs::remove_file(&key_path).expect("final goes missing"); + let readopted = load_or_create_keypair(&key_config(&key_path)) + .expect("the post-delete boot must provision an identity"); + assert_ne!( + format!("{}", readopted.did()), + format!("{}", sibling.did()), + "a crash in the supersede-then-consume gap must leave no sibling adoptable" + ); + assert_eq!( + format!("{}", readopted.did()), + format!("{}", winner.did()), + "only the winner's own quarantined bytes may be re-adopted" + ); + + // PRE-fix crash image (the old ordering's gap): chosen already + // consumed, sibling still in the plain adoptable class. This is the + // hazard: the sibling IS adopted once the final goes missing. + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + std::fs::write(&key_path, winner_pem.as_bytes()).expect("seed final"); + std::fs::set_permissions(&key_path, std::fs::Permissions::from_mode(0o600)) + .expect("0600 final"); + std::fs::write( + dir.path().join(".identity.pem.quarantined.99999.0"), + sibling_pem.as_bytes(), + ) + .expect("seed still-plain sibling"); + std::fs::remove_file(&key_path).expect("final goes missing"); + let resurrected = load_or_create_keypair(&key_config(&key_path)) + .expect("the post-delete boot must provision an identity"); + assert_eq!( + format!("{}", resurrected.did()), + format!("{}", sibling.did()), + "the pre-fix image demonstrably resurrects the sibling's DID" + ); + } + + // M1, the N2 mirror on the ADOPTION arm: a quarantine whose adoption + // republish LOSES (a concurrent winner durably published first) is + // provably outcompeted, yet the adoption-Lost arm used to leave it in + // the adoptable class while restore-Lost superseded its kept + // quarantine. Within the age floor, the operator's + // delete-identity.pem-for-a-fresh-DID procedure then resurrects the key + // that just LOST its round. EVERY Lost outcome must supersede the + // losing bytes. RED before M1: A survives as `.quarantined.` and the + // post-delete boot adopts A. + #[test] + fn adoption_lost_supersedes_the_losing_quarantine() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let stranded = Keypair::generate(); + let pem_a = stranded.to_pem().expect("pem a"); + let quarantine = dir.path().join(".identity.pem.quarantined.99999.0"); + std::fs::write(&quarantine, pem_a.as_bytes()).expect("seed young quarantine A"); + + // A concurrent starter durably publishes C at the free final name + // before the adoption scan (before_publish runs at the top of the + // pass), so the adoption republish of A returns Lost. + let winner = Keypair::generate(); + let winner_pem = winner.to_pem().expect("pem c"); + let race_path = key_path.clone(); + let before_publish = move || { + if race_path.exists() { + return; + } + let out = publish_key_atomically( + &race_path, + winner_pem.as_bytes(), + &|| {}, + &|| {}, + PublishFaults::NONE, + ) + .expect("concurrent winner publishes C"); + assert!( + matches!(out, KeyPublish::Won), + "C's publish takes the free name" + ); + }; + let seam = RecoverySeam { + before_publish: &before_publish, + ..RecoverySeam::NONE + }; + let kp = load_or_create_keypair_with(&key_path, &|| {}, &|| {}, PublishFaults::NONE, seam) + .expect("the adoption-Lost arm must defer to the winner"); + assert_eq!( + format!("{}", kp.did()), + format!("{}", winner.did()), + "the boot must converge on the concurrent winner's key C" + ); + + // A's bytes survive as forensics, in the never-adoptable class. + assert!( + names_containing(dir.path(), ".quarantined.").is_empty(), + "no adoptable quarantine may survive a lost adoption" + ); + let superseded = names_containing(dir.path(), ".superseded."); + assert_eq!( + superseded.len(), + 1, + "the losing quarantine must be superseded: {superseded:?}" + ); + assert_eq!( + std::fs::read(dir.path().join(&superseded[0])).expect("superseded readable"), + pem_a.as_bytes(), + "the superseded file must preserve A's bytes as forensics" + ); + + // The operator action: delete the final for a fresh DID; A must not + // come back. + std::fs::remove_file(&key_path).expect("operator deletes the final"); + let fresh = load_or_create_keypair(&key_config(&key_path)) + .expect("the post-delete boot must provision an identity"); + assert_ne!( + format!("{}", fresh.did()), + format!("{}", stranded.did()), + "the key that LOST its adoption round must never be resurrected" + ); + assert_ne!( + format!("{}", fresh.did()), + format!("{}", winner.did()), + "the operator must get a genuinely fresh DID" + ); + } + + // F1 MUST-NOT: an UNPARSEABLE quarantine is the expected crash state and + // stays as forensics; boot must fall through to a fresh generation, not + // wedge on it or delete it. + #[test] + fn unparseable_quarantine_falls_through_to_generate() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let garbage: &[u8] = b"-----BEGIN nonsense truncated"; + let quarantine = dir.path().join(".identity.pem.quarantined.99999.0"); + std::fs::write(&quarantine, garbage).expect("seed unparseable quarantine"); + + let kp = load_or_create_keypair(&key_config(&key_path)) + .expect("an unparseable quarantine must not block generation"); + + let on_disk = super::load_existing_key(&key_path).expect("generated final parses"); + assert_eq!( + format!("{}", on_disk.did()), + format!("{}", kp.did()), + "the generated identity must be on disk" + ); + assert_eq!( + std::fs::read(&quarantine).expect("quarantine still on disk"), + garbage, + "the unparseable quarantine must survive byte-for-byte as forensics" + ); + } + + // #194 (grace): the crash signature must persist beyond a single + // observation. A reader that catches a live fallback publisher's + // mid-write window sees content-bad-plus-marker ONCE; if the state heals + // before the next poll, no CrashSignature may fire, since a single + // observation classifying a crash would let a crash-free boot hijack the + // publisher's round and churn a quarantine. The load-fault hook heals + // the state on the second attempt; the boot must LOAD the healed key. + // RED before the grace: the first observation returns CrashSignature + // immediately. (G3 widened the grace from two polls to a + // CRASH_SIGNATURE_MIN_PERSIST window; signature_requires_sustained_observation + // covers the widened floor.) + #[test] + fn mid_write_reader_does_not_hijack_before_grace() { + use std::sync::atomic::{AtomicU32, Ordering}; + use std::sync::OnceLock; + static CALLS: AtomicU32 = AtomicU32::new(0); + static STATE: OnceLock<(std::path::PathBuf, std::path::PathBuf, String)> = OnceLock::new(); + fn heal_on_second_attempt() -> std::io::Result<()> { + if CALLS.fetch_add(1, Ordering::SeqCst) == 1 { + let (marker, final_path, pem) = STATE.get().expect("state seeded"); + std::fs::remove_file(marker)?; + std::fs::write(final_path, pem.as_bytes())?; + } + Ok(()) + } + + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + let marker = dir.path().join(".identity.pem.publishing.99999.0"); + let pem = Keypair::generate().to_pem().expect("pem"); + let expected = format!("{}", Keypair::from_pem(&pem).expect("fixture parses").did()); + STATE + .set((marker, key_path.clone(), pem.to_string())) + .expect("state seeded once"); + + match super::boot_load_key(&key_path, true, heal_on_second_attempt) { + super::BootLoad::Loaded(kp) => { + assert_eq!( + format!("{}", kp.did()), + expected, + "the healed key must be the one loaded" + ); + } + super::BootLoad::CrashSignature(e) => { + panic!("a single mid-write observation must not classify a crash: {e:#}") + } + super::BootLoad::Failed(e) => panic!("the healed key must load: {e:#}"), + } + } + + // #194 (G3): the crash signature must persist for + // CRASH_SIGNATURE_MIN_PERSIST of consecutive observations before boot + // classifies a crash. A few-millisecond glimpse of claimable+content-bad + // is exactly what a reader sees inside a LIVE publisher's mid-write + // window once chmod/stat/scheduling latency on shared volumes exceeds + // one ~2ms poll, and classifying on it hijacks a live round into a + // quarantine churn. Construction: the crash state persists for THREE + // observations (~6ms, far under the persistence floor) and then heals; + // boot must LOAD the healed key. RED before G3: the two-poll grace fires + // CrashSignature on the second observation, before the heal. + #[test] + fn signature_requires_sustained_observation() { + use std::sync::atomic::{AtomicU32, Ordering}; + use std::sync::OnceLock; + static CALLS: AtomicU32 = AtomicU32::new(0); + static STATE: OnceLock<(std::path::PathBuf, std::path::PathBuf, String)> = OnceLock::new(); + fn heal_before_persistence() -> std::io::Result<()> { + if CALLS.fetch_add(1, Ordering::SeqCst) == 3 { + let (marker, final_path, pem) = STATE.get().expect("state seeded"); + std::fs::remove_file(marker)?; + std::fs::write(final_path, pem.as_bytes())?; + } + Ok(()) + } + + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + let marker = dir.path().join(".identity.pem.publishing.99999.0"); + let pem = Keypair::generate().to_pem().expect("pem"); + let expected = format!("{}", Keypair::from_pem(&pem).expect("fixture parses").did()); + STATE + .set((marker, key_path.clone(), pem.to_string())) + .expect("state seeded once"); + + match super::boot_load_key(&key_path, true, heal_before_persistence) { + super::BootLoad::Loaded(kp) => { + assert_eq!( + format!("{}", kp.did()), + expected, + "the healed key must be the one loaded" + ); + } + super::BootLoad::CrashSignature(e) => panic!( + "a signature cleared inside the persistence window must not classify \ + a crash: {e:#}" + ), + super::BootLoad::Failed(e) => panic!("the healed key must load: {e:#}"), + } + } + + // #194 (F2, window-1): a publisher that completes between recovery's + // claim step and its re-parse must win the round retroactively. The + // post-claim re-parse sees the completed key, so recovery releases its + // claims and aborts to Reloaded: no quarantine of a good key, no claim + // left behind, the completed identity returned. RED before F2: no + // re-parse exists, so recovery quarantines the just-completed key and + // regenerates over it. + #[test] + fn post_claim_reparse_aborts_to_reload() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + let completed = Keypair::generate(); + let pem = completed.to_pem().expect("pem"); + let heal_path = key_path.clone(); + let before_reparse = move || { + std::fs::write(&heal_path, pem.as_bytes()) + .expect("complete the final inside the claim window"); + }; + let seam = RecoverySeam { + before_reparse: &before_reparse, + ..RecoverySeam::NONE + }; + + let kp = load_or_create_keypair_with(&key_path, &|| {}, &|| {}, PublishFaults::NONE, seam) + .expect("recovery must abort to the completed key"); + + assert_eq!( + format!("{}", kp.did()), + format!("{}", completed.did()), + "the identity completed inside the window must be the one returned" + ); + assert!( + names_containing(dir.path(), ".quarantined.").is_empty(), + "a key completed before the re-parse must never be quarantined" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "the aborting recovery must release its claims" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "the claimed markers must not reappear" + ); + } + + // N1: EVERY arm of load_or_create_keypair_with that returns a + // successfully resolved keypair must run the stale-marker sweep first, + // including the post-claim-reparse Reloaded arm and the adoption-Lost + // re-load. Residue that lands after the claim step (or beside a missing + // final) otherwise persists past a healthy return, and a LATER content + // failure of the good final would pair with it and misclassify plain + // corruption as a recoverable crash. RED before N1: both arms return + // without sweeping and the planted marker and aged claim survive. + #[test] + fn reloaded_and_adoption_paths_sweep_before_returning() { + // Scenario 1: the post-claim-reparse Reloaded arm + // (post_claim_reparse_aborts_to_reload's construction), with fresh + // residue planted inside the claim window. + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + let completed = Keypair::generate(); + let pem = completed.to_pem().expect("pem"); + let heal_path = key_path.clone(); + let residue_dir = dir.path().to_path_buf(); + let before_reparse = move || { + std::fs::write(&heal_path, pem.as_bytes()) + .expect("complete the final inside the claim window"); + std::fs::write(residue_dir.join(".identity.pem.publishing.31337.0"), b"") + .expect("plant stale marker inside the claim window"); + let claim = residue_dir.join(".identity.pem.recovering.31337"); + std::fs::write(&claim, b"").expect("plant orphan claim"); + age_beyond_claim_sweep(&claim); + }; + let seam = RecoverySeam { + before_reparse: &before_reparse, + ..RecoverySeam::NONE + }; + let kp = load_or_create_keypair_with(&key_path, &|| {}, &|| {}, PublishFaults::NONE, seam) + .expect("the healed final must load"); + assert_eq!( + format!("{}", kp.did()), + format!("{}", completed.did()), + "the reloaded identity must be the healed key" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "the Reloaded return must sweep stale publish markers" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "the Reloaded return must sweep aged orphan claims" + ); + + // Scenario 2: the adoption-Lost re-load. A young adoptable + // quarantine loses its republish to a final that appears + // concurrently (before_publish), and the follow-up load returns + // the winner; the same residue must not survive that return. + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let stranded = Keypair::generate(); + std::fs::write( + dir.path().join(".identity.pem.quarantined.99999.0"), + stranded.to_pem().expect("pem").as_bytes(), + ) + .expect("seed young quarantine"); + std::fs::write(dir.path().join(".identity.pem.publishing.31337.0"), b"") + .expect("seed stale marker"); + let claim = dir.path().join(".identity.pem.recovering.31337"); + std::fs::write(&claim, b"").expect("seed orphan claim"); + age_beyond_claim_sweep(&claim); + let winner = Keypair::generate(); + let winner_pem = winner.to_pem().expect("pem"); + let race_path = key_path.clone(); + let before_publish = move || { + if race_path.exists() { + return; + } + let out = publish_key_atomically( + &race_path, + winner_pem.as_bytes(), + &|| {}, + &|| {}, + PublishFaults::NONE, + ) + .expect("concurrent winner publishes"); + assert!(matches!(out, KeyPublish::Won), "the winner takes the name"); + }; + let seam = RecoverySeam { + before_publish: &before_publish, + ..RecoverySeam::NONE + }; + let kp = load_or_create_keypair_with(&key_path, &|| {}, &|| {}, PublishFaults::NONE, seam) + .expect("the adoption-Lost re-load must resolve the winner"); + assert_eq!( + format!("{}", kp.did()), + format!("{}", winner.did()), + "the adoption-Lost arm must defer to the concurrent winner" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "the adoption-Lost return must sweep stale publish markers" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "the adoption-Lost return must sweep aged orphan claims" + ); + } + + // #194 (G2a) MUST-NOT: a TRANSIENT (non-content) failure of the + // post-claim RE-parse proves nothing about the final's content, so + // recovery must release its claims and surface the error loudly, + // destroying nothing: quarantining on it would destroy a possibly good + // key, the protocol's own must-not. The reparse_fault seam injects the + // transient class deterministically (load_fault only reaches + // boot_load_key's attempts, never this re-parse). RED before G2a: any + // re-parse error is treated as still-crashed and the final is + // quarantined. + #[test] + fn transient_reparse_error_releases_claims_and_fails_loudly() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + let seam = RecoverySeam { + reparse_fault: || Err(std::io::Error::other("injected transient reparse failure")), + ..RecoverySeam::NONE + }; + + let err = + match load_or_create_keypair_with(&key_path, &|| {}, &|| {}, PublishFaults::NONE, seam) + { + Err(e) => e, + Ok(_) => panic!("a transient re-parse failure must stay loud, never resolve"), + }; + assert!( + format!("{err:#}").contains("injected transient reparse failure"), + "the transient failure must surface: {err:#}" + ); + assert_eq!( + std::fs::read(&key_path).expect("final still on disk"), + b"", + "the final's bytes must be untouched" + ); + assert!( + names_containing(dir.path(), ".quarantined.").is_empty(), + "a transient re-parse failure must never quarantine" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "the failing recovery must release its claims, not leak them" + ); + } + + // #194 (G2b): a publisher that completes its write AFTER the re-parse + // observation and BEFORE the quarantine rename has its VALID key swept + // into quarantine; the rename preserves the bytes, so recovery must + // parse the quarantined bytes, detect the completed key, restore it to + // the final name, release its claims, and return that identity: no + // regeneration, no divergence from the Won the publisher already + // reported. The before_quarantine seam lands the completed write in + // exactly that window (before_reparse fires too early: the re-parse + // would catch it). RED before G2b: the completed key stays quarantined + // and a fresh identity is regenerated over it. + #[test] + fn completed_key_is_restored_from_quarantine() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + let completed = Keypair::generate(); + let pem = completed.to_pem().expect("pem"); + let heal_path = key_path.clone(); + let before_quarantine = move || { + std::fs::write(&heal_path, pem.as_bytes()) + .expect("complete the final inside the re-parse-to-rename window"); + }; + let seam = RecoverySeam { + before_quarantine: &before_quarantine, + ..RecoverySeam::NONE + }; + + let kp = load_or_create_keypair_with(&key_path, &|| {}, &|| {}, PublishFaults::NONE, seam) + .expect("recovery must restore the completed key, not fail or regenerate"); + + assert_eq!( + format!("{}", kp.did()), + format!("{}", completed.did()), + "the completed identity must be the one returned, never a regeneration" + ); + let on_disk = super::load_existing_key(&key_path).expect("restored final parses"); + assert_eq!( + format!("{}", on_disk.did()), + format!("{}", completed.did()), + "the final must hold the restored key" + ); + assert!( + names_containing(dir.path(), ".quarantined.").is_empty(), + "the quarantine must be removed once its key is republished at the final" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "the aborting recovery must release its claims" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "the claimed markers must not reappear" + ); + } + + // M2, the N4 mirror on the RESTORE arm: the G2b restore decides its + // round in favor of the restored key, but it used to remove only the + // quarantine it consumed. A second young parseable quarantine (an + // earlier crash's residue) stayed adoptable, so a later missing final + // resurrected a DIFFERENT historical DID. On Won the restore must + // supersede every other young parseable sibling, exactly like + // adoption-Won. RED before M2: Q_old stays `.quarantined.` and the + // post-delete boot adopts it. + #[test] + fn restore_won_supersedes_siblings() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + let older = Keypair::generate(); + let older_pem = older.to_pem().expect("pem"); + let q_old = dir.path().join(".identity.pem.quarantined.11111.0"); + std::fs::write(&q_old, older_pem.as_bytes()).expect("seed sibling quarantine"); + + // completed_key_is_restored_from_quarantine's construction: the + // publisher completes inside the re-parse-to-rename window, so its + // key is swept into a fresh quarantine and restored from there. + let completed = Keypair::generate(); + let pem = completed.to_pem().expect("pem"); + let heal_path = key_path.clone(); + let before_quarantine = move || { + std::fs::write(&heal_path, pem.as_bytes()) + .expect("complete the final inside the re-parse-to-rename window"); + }; + let seam = RecoverySeam { + before_quarantine: &before_quarantine, + ..RecoverySeam::NONE + }; + let kp = load_or_create_keypair_with(&key_path, &|| {}, &|| {}, PublishFaults::NONE, seam) + .expect("recovery must restore the completed key"); + assert_eq!( + format!("{}", kp.did()), + format!("{}", completed.did()), + "the restored identity must be the completed key" + ); + + assert!( + names_containing(dir.path(), ".quarantined.").is_empty(), + "the restore-Won round must leave no adoptable quarantine behind" + ); + let superseded = names_containing(dir.path(), ".superseded."); + assert_eq!( + superseded.len(), + 1, + "exactly the outcompeted sibling is superseded: {superseded:?}" + ); + assert_eq!( + std::fs::read(dir.path().join(&superseded[0])).expect("superseded readable"), + older_pem.as_bytes(), + "the superseded sibling must preserve its bytes as forensics" + ); + + // The sibling must not come back once the final goes missing. + std::fs::remove_file(&key_path).expect("operator deletes the final"); + let fresh = load_or_create_keypair(&key_config(&key_path)) + .expect("the post-delete boot must provision an identity"); + assert_ne!( + format!("{}", fresh.did()), + format!("{}", older.did()), + "the superseded sibling must never be resurrected" + ); + assert_ne!( + format!("{}", fresh.did()), + format!("{}", completed.did()), + "the operator must get a genuinely fresh DID" + ); + } + + // R1: the post-quarantine restore must hold no-clobber on EVERY tier. A + // completed key A is swept into quarantine (G2b), a concurrent starter + // publishes a fresh key B at the freed final name inside the + // quarantine-to-restore window (before_restore), and the restore's link + // tier is forced onto its link-hostile degradation (restore_faults.link). + // The restore must defer to B (keep A's bytes as forensics, converge + // on B), never replace B with A: B's owner already holds B in memory, so + // a clobber leaves that node's memory and disk diverged. RED before the + // fix: the old restore ladder degraded to a plain rename(quarantine -> + // final), which clobbered B with A. Since N2, the kept forensics live in + // the SUPERSEDED class (A provably lost the round, so it must never be + // adoptable); lost_restore_quarantine_is_never_adopted covers that side. + #[test] + fn restore_never_clobbers_a_concurrent_winner() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + let completed = Keypair::generate(); + let pem_a = completed.to_pem().expect("pem a"); + let kp_b = Keypair::generate(); + let pem_b = kp_b.to_pem().expect("pem b"); + + let heal_path = key_path.clone(); + let before_quarantine = move || { + std::fs::write(&heal_path, pem_a.as_bytes()) + .expect("complete the final inside the re-parse-to-rename window"); + }; + let steal_path = key_path.clone(); + let pem_b_disk = pem_b.clone(); + let before_restore = move || { + let out = publish_key_atomically( + &steal_path, + pem_b_disk.as_bytes(), + &|| {}, + &|| {}, + PublishFaults::NONE, + ) + .expect("concurrent starter republishes key B at the freed name"); + assert!( + matches!(out, KeyPublish::Won), + "B's publish wins the free name" + ); + }; + let seam = RecoverySeam { + before_quarantine: &before_quarantine, + before_restore: &before_restore, + restore_faults: PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + ..PublishFaults::NONE + }, + ..RecoverySeam::NONE + }; + + let kp = load_or_create_keypair_with(&key_path, &|| {}, &|| {}, PublishFaults::NONE, seam) + .expect("the restore must converge on B, not fail the start"); + + assert_eq!( + format!("{}", kp.did()), + format!("{}", kp_b.did()), + "the recovering boot must converge on the concurrent winner's key B" + ); + let on_disk = super::load_existing_key(&key_path).expect("final parses"); + assert_eq!( + format!("{}", on_disk.did()), + format!("{}", kp_b.did()), + "key B must survive at the final name on every restore tier" + ); + let superseded = names_containing(dir.path(), ".superseded."); + assert_eq!( + superseded.len(), + 1, + "the losing restore must keep A's bytes as superseded forensics: {superseded:?}" + ); + assert!( + names_containing(dir.path(), ".quarantined.").is_empty(), + "the losing key must not stay in the adoptable quarantine class" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "the recovery must release its claims" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "no publish markers may remain" + ); + } + + // Regression guard: a fallback publisher whose round was claimed by a + // recovery (its inode quarantined away, a live key B republished at the + // name) must NOT let its own failure cleanup unlink B. The post-write + // error arms now remove nothing at all (any unlink there is a + // stat-then-unlink race against the recoverer), which this interleave + // pins. It runs inside the fallback_fsync fault (before_commit never + // fires on the error arms), step for step what a claiming recovery does: + // claim A's marker, quarantine A's final by rename, republish B, clear + // the claim; then the injected Err forces A's fsync-failure arm. RED if + // any by-name removal returns to that arm: A deletes B, destroying the + // live identity whose owner already returned Won. + #[test] + fn failed_publisher_cleanup_spares_a_republished_final() { + thread_local! { + static STEAL: std::cell::RefCell< + Option<(std::path::PathBuf, std::path::PathBuf, String)>, + > = const { std::cell::RefCell::new(None) }; + } + fn steal_round_then_fail_fsync() -> std::io::Result<()> { + STEAL.with(|s| { + if let Some((dir, key_path, pem_b)) = s.borrow_mut().take() { + let markers = names_containing(&dir, ".publishing."); + assert_eq!( + markers.len(), + 1, + "A's window must be bracketed: {markers:?}" + ); + let claim = dir.join(".identity.pem.recovering.55555"); + std::fs::rename(dir.join(&markers[0]), &claim).expect("claim A's marker"); + std::fs::rename(&key_path, dir.join(".identity.pem.quarantined.55555.0")) + .expect("quarantine A's inode away"); + let out = publish_key_atomically( + &key_path, + pem_b.as_bytes(), + &|| {}, + &|| {}, + PublishFaults::NONE, + ) + .expect("recoverer republishes key B"); + assert!(matches!(out, KeyPublish::Won), "recoverer's republish wins"); + std::fs::remove_file(&claim).expect("clear the claim"); + } + }); + Err(std::io::Error::other("injected fsync failure")) + } + + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let kp_b = Keypair::generate(); + let pem_b = kp_b.to_pem().expect("pem b"); + STEAL.with(|s| { + *s.borrow_mut() = Some((dir.path().to_path_buf(), key_path.clone(), (*pem_b).clone())); + }); + let pem_a = Keypair::generate().to_pem().expect("pem a"); + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + fallback_fsync: steal_round_then_fail_fsync, + ..PublishFaults::NONE + }; + + let err = match publish_key_atomically(&key_path, pem_a.as_bytes(), &|| {}, &|| {}, faults) + { + Err(e) => e, + Ok(_) => panic!("A's failed fsync must still error A's publish"), + }; + assert!( + format!("{err:#}").contains("injected fsync failure"), + "the fsync failure must surface: {err:#}" + ); + let on_disk = super::load_existing_key(&key_path) + .expect("the republished final must survive A's cleanup"); + assert_eq!( + format!("{}", on_disk.did()), + format!("{}", kp_b.did()), + "A's error-arm cleanup must spare the recoverer's key B" + ); + let quarantined = names_containing(dir.path(), ".quarantined."); + assert_eq!( + quarantined.len(), + 1, + "A's quarantined inode stays where the recoverer put it: {quarantined:?}" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "no recovery claims may remain" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "no publish markers may remain" + ); + } +} + +#[cfg(test)] +mod shutdown_grace_tests { + use super::drive_serve_with_grace; + use std::time::{Duration, Instant}; + + // The must-not case: the drain (serve future) never completes but the signal + // has fired. The grace deadline must fire, report `grace_expired = true`, and + // return promptly so teardown proceeds — not hang forever. RED: a helper that + // just awaits `serve` (ignoring grace) hangs this test past its bound. + #[tokio::test] + async fn hung_drain_is_abandoned_after_grace() { + let (armed_tx, armed_rx) = tokio::sync::oneshot::channel::<()>(); + let _ = armed_tx.send(()); // signal already fired + let serve = std::future::pending::>(); // never drains + let start = Instant::now(); + + let (result, grace_expired) = + drive_serve_with_grace(serve, armed_rx, Duration::from_millis(50)).await; + + assert!(grace_expired, "a drain outlasting grace must be abandoned"); + assert!( + result.is_ok(), + "abandon path returns Ok so teardown proceeds" + ); + assert!( + start.elapsed() < Duration::from_secs(5), + "must not wait indefinitely for the hung drain" + ); + } + + // Normal drain: the serve future completes before grace, so the real serve + // result is propagated and `grace_expired` is false. + #[tokio::test] + async fn completed_drain_keeps_result_and_does_not_expire() { + let (_armed_tx, armed_rx) = tokio::sync::oneshot::channel::<()>(); + let serve = std::future::ready(Ok::<(), std::io::Error>(())); + + let (result, grace_expired) = + drive_serve_with_grace(serve, armed_rx, Duration::from_secs(3600)).await; + + assert!( + !grace_expired, + "a completed drain must not report grace expiry" + ); + assert!(result.is_ok()); + } + + // The grace clock starts at the SIGNAL, not at call time. With the signal + // unsent, a finite drain longer than `grace` still completes normally — total + // uptime is never bounded by grace. RED: drop `armed.await` from the grace + // branch and the deadline fires at 10ms, expiring before the 80ms drain. + #[tokio::test] + async fn grace_clock_starts_at_signal_not_call() { + let (_armed_tx, armed_rx) = tokio::sync::oneshot::channel::<()>(); + // Signal never fires; drain finishes after > grace. + let serve = async { + tokio::time::sleep(Duration::from_millis(80)).await; + Ok::<(), std::io::Error>(()) + }; + + let (result, grace_expired) = + drive_serve_with_grace(serve, armed_rx, Duration::from_millis(10)).await; + + assert!( + !grace_expired, + "grace must not fire while the shutdown signal is unsent" + ); + assert!(result.is_ok()); + } + + // Real-socket contract: at grace expiry the server abandons a request still + // in flight; the client never receives a 503 (or any late error response), + // it just observes the connection going away or no response at all. Wires a + // real `axum::serve` future through `drive_serve_with_grace` exactly as the + // production path does. RED: asserting the client reads back an + // "HTTP/1.1 503" status line fails (no response bytes ever arrive). + #[tokio::test] + async fn real_socket_in_flight_request_abandoned_without_503() { + use axum::routing::get; + use std::sync::Arc; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + // The slow handler signals once the request is in flight, then outlasts + // the grace by a wide margin. + let entered = Arc::new(tokio::sync::Notify::new()); + let entered_tx = entered.clone(); + let app = axum::Router::new().route( + "/slow", + get(move || { + let entered_tx = entered_tx.clone(); + async move { + entered_tx.notify_one(); + tokio::time::sleep(Duration::from_secs(10)).await; + "done" + } + }), + ); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + + // Mirror the production wiring: axum drains on a watch flip, and the + // graceful-shutdown closure arms the grace clock at the signal. + let (shutdown_tx, mut shutdown_rx) = tokio::sync::watch::channel(false); + let (armed_tx, armed_rx) = tokio::sync::oneshot::channel::<()>(); + let serve = axum::serve(listener, app).with_graceful_shutdown(async move { + while !*shutdown_rx.borrow_and_update() { + if shutdown_rx.changed().await.is_err() { + break; + } + } + let _ = armed_tx.send(()); + }); + + let driver = tokio::spawn(drive_serve_with_grace( + serve, + armed_rx, + Duration::from_millis(250), + )); + + // Put a request in flight on the slow route and wait until the handler + // is actually running, so the drain has something to abandon. + let mut client = tokio::net::TcpStream::connect(addr).await.unwrap(); + client + .write_all(b"GET /slow HTTP/1.1\r\nhost: test\r\nconnection: close\r\n\r\n") + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(5), entered.notified()) + .await + .expect("slow handler must start before the shutdown signal fires"); + + // Fire the shutdown signal; the in-flight request now outlasts grace. + let signal_at = Instant::now(); + shutdown_tx.send(true).unwrap(); + + let (result, grace_expired) = tokio::time::timeout(Duration::from_secs(5), driver) + .await + .expect("driver must return soon after grace expiry, not hang on the drain") + .expect("driver task must not panic"); + assert!( + grace_expired, + "an in-flight request outlasting grace must be abandoned" + ); + assert!( + result.is_ok(), + "abandon path returns Ok so teardown proceeds" + ); + assert!( + signal_at.elapsed() < Duration::from_secs(5), + "must return within a bounded window past the grace" + ); + + // The client must never see a 503 status line: the request is abandoned + // at the transport, not answered. Read whatever arrives before a bounded + // window; a clean close (EOF) and no bytes at all both satisfy the + // contract. + let mut buf = Vec::new(); + let _ = tokio::time::timeout(Duration::from_secs(2), client.read_to_end(&mut buf)).await; + assert!( + !buf.starts_with(b"HTTP/1.1 503"), + "abandoned in-flight request must not be answered with a 503; got: {:?}", + String::from_utf8_lossy(&buf) + ); + } +} diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index aa0483db..efd80ec4 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -1,1086 +1,7 @@ -mod api; -mod arweave; -mod auth; -mod bootstrap; -mod cert; -mod config; -mod db; -mod encrypted_pin; -mod error; -mod git; -mod graphql; -mod icaptcha; -mod ipfs_pin; -mod metrics; -mod operator; -mod p2p; -mod pinata; -mod rate_limit; -mod server; -mod state; -mod sync; -#[cfg(test)] -mod test_support; -mod visibility; -mod webhooks; - -use anyhow::{anyhow, Context, Result}; -use axum::extract::State; -use axum::http::StatusCode; -use axum::response::IntoResponse; -use axum::routing::get; -use axum::{Json, Router}; -use clap::Parser; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::Arc; -use tokio::net::TcpListener; -use tokio::sync::watch; -use tracing::{info, warn}; - -use gitlawb_core::http_sig::sign_request; -use gitlawb_core::identity::Keypair; - -use config::Config; -use db::Db; -use state::AppState; - -#[derive(Clone)] -struct DegradedState { - node_did: String, - db_startup: Arc, -} - -/// Two independent counters with no cross-field invariant — atomics, not a -/// lock, so the retry loop and the degraded handlers never contend. -#[derive(Default)] -struct DbStartupStatus { - attempts: AtomicU64, - next_retry_secs: AtomicU64, -} +//! Thin binary entry point. All boot logic lives in the library crate +//! (`src/lib.rs`) so out-of-crate integration tests can link the same code. #[tokio::main] -async fn main() -> Result<()> { - tracing_subscriber::fmt() - .with_env_filter( - tracing_subscriber::EnvFilter::from_default_env() - .add_directive("gitlawb_node=debug".parse().unwrap()) - .add_directive("tower_http=info".parse().unwrap()), - ) - .init(); - - let mut config = Config::parse(); - - // Merge the embedded seed list of public network nodes into the runtime - // bootstrap peers. Operators can opt out via GITLAWB_BOOTSTRAP_DISABLE_SEEDS. - bootstrap::merge_seeds(&mut config); - - if !config.public_read { - warn!( - "GITLAWB_PUBLIC_READ=false is reserved; per-repository private-read enforcement is not wired in alpha" - ); - } - - // Load or generate the node's identity keypair - let keypair = load_or_create_keypair(&config)?; - let node_did = keypair.did(); - - // One-time metrics init. Must run before any handler that calls into - // `metrics::record_*` so the registry exists when the first event fires. - // Safe to call even when GITLAWB_METRICS_ADDR is unset — those helpers - // are simply no-ops until something reads from the registry. - metrics::init(env!("CARGO_PKG_VERSION"), &node_did.to_string()); - - info!("╔══════════════════════════════════════════╗"); - info!( - "║ gitlawb node v{} ║", - env!("CARGO_PKG_VERSION") - ); - info!("╚══════════════════════════════════════════╝"); - // Process-wide shutdown signal. One sender lives in AppState (cloned - // into every handler); main() keeps a clone and flips it on SIGINT - // or SIGTERM. Tasks that hold a watch::Receiver get notified at - // their next await point. - let (shutdown_tx, _shutdown_rx_for_main) = watch::channel(false); - spawn_shutdown_signal(shutdown_tx.clone()); - - info!(did = %node_did, "node identity"); - info!(addr = %config.bind_addr(), "binding HTTP listener"); - - // Bind HTTP once, before dependency initialization, and keep this socket - // for the life of the process. The degraded server accepts on a dup of the - // same socket, so the degraded→full handoff never closes the port: while - // the full server initializes, connections queue in the shared backlog - // instead of being refused. - let listener = TcpListener::bind(config.bind_addr()) - .await - .with_context(|| format!("failed to bind to {}", config.bind_addr()))?; - let full_std_listener = listener.into_std()?; - let degraded_listener = TcpListener::from_std( - full_std_listener - .try_clone() - .context("failed to clone HTTP listener for degraded server")?, - )?; - - // Metrics must stay observable during a database outage — the degraded - // window is exactly when dashboards need data — so this listener starts - // before the DB connects. - let metrics_handle = if !config.metrics_addr.is_empty() { - match spawn_metrics_server(&config.metrics_addr, shutdown_tx.subscribe()).await { - Ok(handle) => { - info!(addr = %config.metrics_addr, "metrics endpoint listening"); - Some(handle) - } - Err(e) => { - warn!(err = %e, addr = %config.metrics_addr, "failed to start metrics endpoint — continuing without"); - None - } - } - } else { - info!("metrics endpoint disabled (GITLAWB_METRICS_ADDR not set)"); - None - }; - - let db_startup = Arc::new(DbStartupStatus::default()); - let (db_ready_tx, db_ready_rx) = watch::channel(false); - let mut degraded_handle = tokio::spawn(run_degraded_server( - degraded_listener, - node_did.to_string(), - Arc::clone(&db_startup), - db_ready_rx, - shutdown_tx.subscribe(), - )); - - // Connect to PostgreSQL database. A transient outage or bad secret should - // not crash-loop the process and hammer the database provider; permanent - // misconfiguration surfaces through error-level logs and the /ready check. - let db = tokio::select! { - db = connect_db_with_retry(&config, Arc::clone(&db_startup), shutdown_tx.subscribe()) => { - match db { - Some(db) => db, - None => { - // Shutdown requested while waiting for the database. The - // degraded server only serves one-shot 503s — abort it - // rather than drain, so a slow client can't stall exit. - degraded_handle.abort(); - return Ok(()); - } - } - } - degraded = &mut degraded_handle => { - if *shutdown_tx.borrow() { - return Ok(()); - } - return match degraded { - Ok(Ok(())) => Err(anyhow!("degraded HTTP server stopped before database became ready")), - Ok(Err(err)) => Err(err.context("degraded HTTP server failed")), - Err(err) => Err(anyhow!("degraded HTTP server task failed: {err}")), - }; - } - }; - - // Flip the degraded server into graceful shutdown, but do NOT await the - // drain: one slow in-flight request must not delay the full server, and - // the shared socket means there is no port gap to cover. The drain - // finishes (and logs) in the background. - db_ready_tx.send(true).ok(); - tokio::spawn(async move { - match degraded_handle.await { - Ok(Ok(())) => {} - Ok(Err(err)) => warn!(err = %err, "degraded HTTP server exited with error"), - Err(err) => warn!(err = %err, "degraded HTTP server task failed"), - } - }); - info!(addr = %config.bind_addr(), "database ready; starting full HTTP server"); - - // Prune peer rows that point back at this node (stale self-loop entries) - if let Some(public_url) = config.public_url.as_deref() { - match db.prune_self_peers(public_url).await { - Ok(0) => {} - Ok(n) => info!(removed = n, public_url, "pruned self-loop peer rows"), - Err(e) => warn!(err = %e, "prune_self_peers failed (non-fatal)"), - } - } - - // Prune peer rows with non-public hosts (loopback/private/internal) that - // were injected via the unauthenticated announce route — they poison the - // sync-notify fan-out (SSRF + crowding out real peers). - match db.prune_non_public_peers().await { - Ok(0) => {} - Ok(n) => info!(removed = n, "pruned non-public (poisoned) peer rows"), - Err(e) => warn!(err = %e, "prune_non_public_peers failed (non-fatal)"), - } - - // Ensure repos directory exists - std::fs::create_dir_all(&config.repos_dir).context("failed to create repos directory")?; - - // Start libp2p swarm (if p2p_port > 0) - let p2p_handle = if config.p2p_port > 0 { - let bootstrap_addrs = config - .p2p_bootstrap - .iter() - .filter_map(|s| s.parse().ok()) - .collect(); - let shutdown_rx = shutdown_tx.subscribe(); - match p2p::start( - &node_did.to_string(), - config.p2p_port, - bootstrap_addrs, - Arc::clone(&db), - config.auto_sync, - shutdown_rx, - ) - .await - { - Ok(handle) => { - info!(port = config.p2p_port, peer_id = %handle.local_peer_id, "libp2p swarm started"); - Some(Arc::new(handle)) - } - Err(e) => { - tracing::warn!(err = %e, "failed to start libp2p swarm — continuing without p2p"); - None - } - } - } else { - info!("p2p disabled (p2p_port = 0)"); - None - }; - - // Shared no-redirect HTTP client. See build_http_client for the SSRF rationale. - let http_client = Arc::new(build_http_client()?); - - let (ref_update_tx, _) = tokio::sync::broadcast::channel::(256); - let (task_event_tx, _) = tokio::sync::broadcast::channel::(256); - - let graphql_schema = Arc::new(graphql::build_schema( - Arc::clone(&db), - ref_update_tx.clone(), - task_event_tx.clone(), - )); - - let machine_id = std::env::var("FLY_MACHINE_ID").ok(); - if let Some(ref mid) = machine_id { - info!(" fly machine: {mid}"); - } - - // Initialize Tigris S3 client if bucket is configured - let tigris = if !config.tigris_bucket.is_empty() { - match git::tigris::TigrisClient::new(&config.tigris_bucket).await { - Ok(client) => { - info!(bucket = %config.tigris_bucket, "tigris storage enabled"); - Some(client) - } - Err(e) => { - tracing::warn!(err = %e, "failed to initialize Tigris client — using local-only storage"); - None - } - } - } else { - info!("tigris storage disabled (no bucket configured)"); - None - }; - - let repo_store = - git::repo_store::RepoStore::new(config.repos_dir.clone(), tigris, db.pool().clone()); - - // Per-DID limiter for the creation endpoints. Keyed on the authenticated - // DID (attacker-varied), so bound its key set to cap memory. - let rate_limiter = - rate_limit::RateLimiter::new_bounded(10, std::time::Duration::from_secs(3600), 200_000); - - // Per-client-IP flood brake for the creation endpoints. The per-DID limiter - // above is bypassed by a DID farm (one throwaway did:key per repo), which is - // exactly how the recurring spam-repo floods get past both it and the - // iCaptcha gate. Keyed on the resolved client IP so a single-source flood is - // capped regardless of how many identities it mints. Sized well above any - // legitimate per-IP creation rate; GITLAWB_CREATE_RATE_LIMIT overrides, 0 - // disables. Bounded key set — the key is a client-influenced IP. - let create_limit = std::env::var("GITLAWB_CREATE_RATE_LIMIT") - .ok() - .and_then(|v| v.trim().parse::().ok()) - .unwrap_or(120); - let create_ip_rate_limiter = rate_limit::RateLimiter::new_bounded( - create_limit, - std::time::Duration::from_secs(3600), - 200_000, - ); - if create_limit == 0 { - tracing::warn!("GITLAWB_CREATE_RATE_LIMIT=0 — per-IP creation rate limiting disabled"); - } - - // Push-path flood brake: max git-receive-pack requests per client IP per - // hour (counts both the info/refs advertisement and the push POST). Sized - // for heavy agent automation while still stopping flood traffic (the June - // 2026 attack pushed several times per second per IP). GITLAWB_PUSH_RATE_LIMIT - // overrides; 0 disables. Bounded key set — the key is a client-influenced IP. - let push_limit = std::env::var("GITLAWB_PUSH_RATE_LIMIT") - .ok() - .and_then(|v| v.trim().parse::().ok()) - .unwrap_or(600); - let push_rate_limiter = rate_limit::RateLimiter::new_bounded( - push_limit, - std::time::Duration::from_secs(3600), - 200_000, - ); - if push_limit == 0 { - tracing::warn!("GITLAWB_PUSH_RATE_LIMIT=0 — per-IP push rate limiting disabled"); - } - - // Which forwarded header the edge is trusted to set. Default None (trust - // nothing, key on the socket peer). Fly nodes set GITLAWB_TRUSTED_PROXY=fly; - // a node behind Caddy/NGINX sets it to x-forwarded-for. - let push_limiter_trust = rate_limit::TrustedProxy::from_env_value( - &std::env::var("GITLAWB_TRUSTED_PROXY").unwrap_or_default(), - ); - tracing::info!(trust = ?push_limiter_trust, push_limit, "push rate limiter configured"); - - // Peer-sync flood brakes, keyed on the resolved client IP (per-DID is useless - // here — a did:key farm self-registers). Two buckets so an unsigned notify - // flood can't drain the signed trigger caller's quota (#82). Bounded key sets - // (the key is a client-influenced IP); 0 disables each. - let sync_trigger_rate_limiter = rate_limit::RateLimiter::new_bounded( - config.sync_trigger_rate_limit, - std::time::Duration::from_secs(3600), - 200_000, - ); - let peer_write_rate_limiter = rate_limit::RateLimiter::new_bounded( - config.peer_write_rate_limit, - std::time::Duration::from_secs(3600), - 200_000, - ); - if config.sync_trigger_rate_limit == 0 { - tracing::warn!( - "GITLAWB_SYNC_TRIGGER_RATE_LIMIT=0 — /sync/trigger IP rate limiting disabled" - ); - } - if config.peer_write_rate_limit == 0 { - tracing::warn!("GITLAWB_PEER_WRITE_RATE_LIMIT=0 — peer-write IP rate limiting disabled"); - } - - // Initialize the iCaptcha proof gate (inert unless ICAPTCHA_MODE is set). - icaptcha::init().await; - - let state = AppState { - config: Arc::new(config.clone()), - db, - node_did: node_did.clone(), - node_keypair: Arc::new(keypair), - p2p: p2p_handle, - http_client, - ref_update_tx, - task_event_tx, - graphql_schema, - machine_id, - repo_store, - rate_limiter, - create_ip_rate_limiter, - push_rate_limiter, - push_limiter_trust, - sync_trigger_rate_limiter, - peer_write_rate_limiter, - shutdown_tx: shutdown_tx.clone(), - }; - - // Periodic peer-count poll for the metrics gauge. If p2p is disabled - // we still set the gauge to 0 so dashboards don't show "no data". - { - let p2p_for_metrics = state.p2p.clone(); - let mut shutdown_rx = state.subscribe_shutdown(); - tokio::spawn(async move { - let mut interval = tokio::time::interval(std::time::Duration::from_secs(15)); - loop { - tokio::select! { - _ = interval.tick() => { - let count = match &p2p_for_metrics { - Some(h) => h.status().await.map(|s| s.connected_peers).unwrap_or(0), - None => 0, - }; - metrics::set_peers_connected(count as i64); - } - _ = shutdown_rx.changed() => { - if *shutdown_rx.borrow() { - return; - } - } - } - } - }); - } - - // Periodic cleanup of expired rate limit entries + consumed-proof ledger - { - let rl = state.rate_limiter.clone(); - let create_ip_rl = state.create_ip_rate_limiter.clone(); - let push_rl = state.push_rate_limiter.clone(); - let sync_trigger_rl = state.sync_trigger_rate_limiter.clone(); - let peer_write_rl = state.peer_write_rate_limiter.clone(); - let db = state.db.clone(); - let mut shutdown_rx = state.subscribe_shutdown(); - tokio::spawn(async move { - loop { - tokio::select! { - _ = tokio::time::sleep(std::time::Duration::from_secs(300)) => { - rl.cleanup().await; - create_ip_rl.cleanup().await; - push_rl.cleanup().await; - sync_trigger_rl.cleanup().await; - peer_write_rl.cleanup().await; - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_secs() as i64) - .unwrap_or(0); - if let Err(e) = db.sweep_expired_proofs(now).await { - tracing::warn!(err = %e, "failed to sweep expired iCaptcha proofs"); - } - } - _ = shutdown_rx.changed() => { - if *shutdown_rx.borrow() { - break; - } - } - } - } - }); - } - - let router = server::build_router(state.clone()); - // Re-register the socket bound at startup — same fd, so there was never a - // moment with the port closed between the degraded and full servers. - let listener = TcpListener::from_std(full_std_listener) - .context("failed to re-register HTTP listener with the runtime")?; - - info!("✓ node started — did:{}", node_did); - info!(" repos dir: {}", config.repos_dir.display()); - info!( - " database: PostgreSQL ({})", - &config.database_url.split('@').next_back().unwrap_or("?") - ); - - // Publish our DID record to the Kademlia DHT shortly after startup - if let Some(p2p) = &state.p2p { - let did_record = p2p::DidRecord { - did: node_did.to_string(), - http_url: config.public_url.clone().unwrap_or_default(), - peer_id: p2p.local_peer_id.to_string(), - p2p_port: config.p2p_port, - timestamp: chrono::Utc::now().to_rfc3339(), - }; - let p2p_clone = Arc::clone(p2p); - let mut shutdown_rx = state.subscribe_shutdown(); - tokio::spawn(async move { - // Small delay so Kademlia can find peers first - tokio::select! { - _ = tokio::time::sleep(std::time::Duration::from_secs(5)) => {} - _ = shutdown_rx.changed() => return, - } - p2p_clone.put_did(did_record).await; - info!("DID record published to Kademlia DHT"); - }); - } - - // Spawn background gossip: announce to bootstrap peers, then ping known peers periodically - { - let gossip_state = state.clone(); - let bootstrap_peers = config.bootstrap_peers.clone(); - let shutdown_rx = state.subscribe_shutdown(); - tokio::spawn(async move { - gossip_task(gossip_state, bootstrap_peers, shutdown_rx).await; - }); - } - - // Start multi-node sync worker if auto_sync is enabled - if config.auto_sync { - sync::start( - Arc::clone(&state.db), - Arc::clone(&state.config), - Arc::clone(&state.node_keypair), - state.subscribe_shutdown(), - ); - info!("auto-sync worker started"); - } - - // On-chain operator setup: verify stake + spawn heartbeat loop - if !state.config.contract_node_staking.is_empty() - && !state.config.operator_private_key.is_empty() - { - match build_operator_client(&state.config, &state.node_did.to_string()) { - Ok(client) => match operator::startup_check(&client).await { - Ok(_) => { - let arc_client = Arc::new(client); - arc_client.spawn_heartbeat_loop(state.subscribe_shutdown()); - } - Err(e) => { - if state.config.operator_strict_mode { - return Err(e.context("strict-mode operator check failed")); - } - tracing::warn!(err = %e, "operator startup check failed — continuing without heartbeat loop"); - } - }, - Err(e) => { - if state.config.operator_strict_mode { - return Err(e.context("strict-mode: failed to build operator client")); - } - tracing::warn!(err = %e, "operator client could not be built — continuing without PoS"); - } - } - } else { - info!("on-chain PoS disabled (GITLAWB_CONTRACT_NODE_STAKING or GITLAWB_OPERATOR_PRIVATE_KEY unset)"); - } - - // axum's `with_graceful_shutdown` waits for in-flight requests to - // complete (up to the configured grace) once the future resolves. - let shutdown_signal_for_axum = state.subscribe_shutdown(); - let grace = std::time::Duration::from_secs(config.shutdown_grace_secs); - info!(grace_secs = config.shutdown_grace_secs, "axum server ready"); - - // `into_make_service_with_connect_info` exposes the socket peer address as - // `ConnectInfo` so the push limiter can key on the real client - // when no trusted proxy header applies (see `rate_limit::client_key`). - let serve_result = axum::serve( - listener, - router.into_make_service_with_connect_info::(), - ) - .with_graceful_shutdown(async move { - let mut rx = shutdown_signal_for_axum; - // Wait until the watcher flips to true, then return so axum - // can begin draining. - while !*rx.borrow_and_update() { - if rx.changed().await.is_err() { - // Sender dropped — treat as shutdown. - break; - } - } - }) - .await; - - // Server has stopped accepting new connections and drained in-flight - // requests. Tear the rest of the system down. - info!("HTTP server stopped, beginning process shutdown"); - if let Some(h) = metrics_handle { - h.abort(); - } - let _ = grace; // recorded for operators in the log above; not enforced - serve_result?; - info!("clean exit"); - Ok(()) -} - -fn spawn_shutdown_signal(tx: watch::Sender) { - tokio::spawn(async move { - #[cfg(unix)] - { - use tokio::signal::unix::{signal as unix_signal, SignalKind}; - let mut sigterm = - unix_signal(SignalKind::terminate()).expect("install SIGTERM handler"); - let mut sigint = unix_signal(SignalKind::interrupt()).expect("install SIGINT handler"); - tokio::select! { - _ = sigterm.recv() => info!("SIGTERM received, shutting down"), - _ = sigint.recv() => info!("SIGINT received, shutting down"), - } - } - #[cfg(not(unix))] - { - use tokio::signal; - let _ = signal::ctrl_c().await; - info!("Ctrl-C received, shutting down"); - } - tx.send(true).ok(); - }); -} - -async fn connect_db_with_retry( - config: &Config, - db_startup: Arc, - mut shutdown_rx: watch::Receiver, -) -> Option> { - let initial_retry_secs = config.db_retry_initial_secs; - let max_retry_secs = config.db_retry_max_secs.max(initial_retry_secs); - let acquire_timeout = std::time::Duration::from_secs(config.db_acquire_timeout_secs); - let attempt_timeout = std::time::Duration::from_secs(config.db_connect_timeout_secs); - let mut attempts = 0_u64; - - loop { - if *shutdown_rx.borrow() { - return None; - } - - attempts = attempts.saturating_add(1); - db_startup.attempts.store(attempts, Ordering::Relaxed); - - // Bound the whole attempt, not just the pool connect: migrations - // block on a cross-instance advisory lock, and an unbounded wait - // there would wedge this loop — no retries, no logs, no recovery. - // Timing out and retrying is safe; migrations are idempotent. - let attempt = match tokio::time::timeout( - attempt_timeout, - Db::connect( - &config.database_url, - config.db_max_connections, - acquire_timeout, - ), - ) - .await - { - Ok(result) => result, - Err(_) => Err(anyhow!( - "connect + migrate attempt exceeded {}s (GITLAWB_DB_CONNECT_TIMEOUT_SECS); \ - is another instance holding the migration lock?", - attempt_timeout.as_secs() - )), - }; - - match attempt { - Ok(db) => { - info!(attempts, "database connection established"); - return Some(Arc::new(db)); - } - Err(err) => { - // A bad DATABASE_URL or rejected credentials won't heal on - // their own. Still retry (exiting would crash-loop and hammer - // the provider — and take liveness down with it), but log at - // error level and skip straight to the maximum backoff; the - // /ready health check is what surfaces this to deploys. - let permanent = is_likely_permanent_db_error(&err); - let retry_secs = if permanent { - max_retry_secs - } else { - database_retry_delay_secs(initial_retry_secs, max_retry_secs, attempts) - }; - db_startup - .next_retry_secs - .store(retry_secs, Ordering::Relaxed); - if permanent { - tracing::error!( - attempts, - retry_secs, - err = %err, - "database rejected our configuration (bad DATABASE_URL or credentials?) — retrying, but operator action is likely required" - ); - } else { - warn!( - attempts, - retry_secs, - err = %err, - "database unavailable during startup; retrying" - ); - } - - tokio::select! { - _ = tokio::time::sleep(std::time::Duration::from_secs(retry_secs)) => {} - changed = shutdown_rx.changed() => { - if changed.is_err() || *shutdown_rx.borrow() { - return None; - } - } - } - } - } - } -} - -/// Errors that indicate misconfiguration rather than a transient outage: a -/// malformed DATABASE_URL, or a server that answered and rejected us — -/// Postgres error class 28xxx (invalid authorization) or 3D000 (database -/// does not exist). Best-effort: an error that anyhow can't downcast back to -/// sqlx just counts as transient. -fn is_likely_permanent_db_error(err: &anyhow::Error) -> bool { - match err.downcast_ref::() { - Some(sqlx::Error::Configuration(_)) => true, - Some(sqlx::Error::Database(db)) => db - .code() - .map(|c| c.starts_with("28") || c.starts_with("3D")) - .unwrap_or(false), - _ => false, - } -} - -fn database_retry_delay_secs(initial_secs: u64, max_secs: u64, attempts: u64) -> u64 { - // The exponent bound only keeps the u32 cast safe — max_secs is the real - // (operator-configurable) cap, and saturating math handles overflow. - let exponent = attempts.saturating_sub(1).min(63) as u32; - initial_secs - .saturating_mul(2_u64.saturating_pow(exponent)) - .min(max_secs) -} - -async fn run_degraded_server( - listener: TcpListener, - node_did: String, - db_startup: Arc, - mut db_ready_rx: watch::Receiver, - mut shutdown_rx: watch::Receiver, -) -> Result<()> { - let addr = listener.local_addr().ok(); - let router = build_degraded_router(node_did, db_startup); - info!(?addr, "degraded HTTP server ready"); - - axum::serve(listener, router) - .with_graceful_shutdown(async move { - // wait_for resolves on predicate-true or sender-drop; either way - // this phase is over. - tokio::select! { - _ = db_ready_rx.wait_for(|ready| *ready) => {} - _ = shutdown_rx.wait_for(|stop| *stop) => {} - } - }) - .await?; - - Ok(()) -} - -fn build_degraded_router(node_did: String, db_startup: Arc) -> Router { - let state = DegradedState { - node_did, - db_startup, - }; - // Everything answers 503 with the same body — including /health and - // /ready, so peer pings (which treat any 2xx /health as alive) and - // uptime monitors correctly see a node that cannot serve traffic. - // `/` additionally carries the node identity for probing peers. - Router::new() - .route("/", get(degraded_node_info)) - .fallback(degraded_unavailable) - .with_state(state) -} - -/// One source of truth for the degraded 503 body, sharing the error -/// vocabulary with error.rs so clients see the same code/message for -/// "database unavailable" regardless of which phase produced it. -fn degraded_body(db_startup: &DbStartupStatus) -> serde_json::Value { - serde_json::json!({ - "status": "degraded", - "database": "initializing", - "error": error::DB_UNAVAILABLE_CODE, - "message": error::DB_UNAVAILABLE_MESSAGE, - "db_attempts": db_startup.attempts.load(Ordering::Relaxed), - "db_next_retry_secs": db_startup.next_retry_secs.load(Ordering::Relaxed), - }) -} - -async fn degraded_node_info(State(state): State) -> impl IntoResponse { - let mut body = degraded_body(&state.db_startup); - if let Some(obj) = body.as_object_mut() { - obj.insert("name".into(), "gitlawb-node".into()); - obj.insert("version".into(), env!("CARGO_PKG_VERSION").into()); - obj.insert("did".into(), state.node_did.clone().into()); - } - (StatusCode::SERVICE_UNAVAILABLE, Json(body)) -} - -async fn degraded_unavailable(State(state): State) -> impl IntoResponse { - ( - StatusCode::SERVICE_UNAVAILABLE, - Json(degraded_body(&state.db_startup)), - ) -} - -/// Spawn a small axum router that exposes only `GET /metrics` on its own -/// listener. Returns the JoinHandle so `main()` can abort it on shutdown. -/// This is deliberately separate from the main router so the metrics port -/// can be firewalled differently from the API port — bind to localhost -/// or a private interface only. -async fn spawn_metrics_server( - addr: &str, - mut shutdown_rx: watch::Receiver, -) -> Result> { - use axum::{response::IntoResponse, routing::get, Router}; - - async fn metrics_handler() -> impl IntoResponse { - match metrics::encode() { - Ok(body) => ( - axum::http::StatusCode::OK, - [( - axum::http::header::CONTENT_TYPE, - "text/plain; version=0.0.4; charset=utf-8", - )], - body, - ), - Err(e) => ( - axum::http::StatusCode::INTERNAL_SERVER_ERROR, - [( - axum::http::header::CONTENT_TYPE, - "text/plain; charset=utf-8", - )], - format!("metrics encode error: {e}"), - ), - } - } - - let listener = TcpListener::bind(addr) - .await - .with_context(|| format!("failed to bind metrics listener to {addr}"))?; - let app = Router::new().route("/metrics", get(metrics_handler)); - - let handle = tokio::spawn(async move { - if let Err(e) = axum::serve(listener, app) - .with_graceful_shutdown(async move { - while !*shutdown_rx.borrow_and_update() { - if shutdown_rx.changed().await.is_err() { - break; - } - } - }) - .await - { - warn!(err = %e, "metrics server exited with error"); - } - }); - Ok(handle) -} - -fn build_operator_client( - config: &config::Config, - node_did: &str, -) -> Result { - use alloy::primitives::Address; - use std::str::FromStr; - - let contract_address = Address::from_str(&config.contract_node_staking) - .with_context(|| format!("invalid contract address: {}", config.contract_node_staking))?; - - let cfg = operator::OperatorConfig { - rpc_url: config.chain_rpc_url.clone(), - private_key: config.operator_private_key.clone(), - contract_address, - node_did: node_did.to_string(), - heartbeat_interval: std::time::Duration::from_secs(config.heartbeat_interval_hours * 3600), - strict_mode: config.operator_strict_mode, - }; - Ok(operator::OperatorClient::new(cfg)) -} - -/// Announce to bootstrap peers on startup, then periodically ping all known peers. -async fn gossip_task( - state: AppState, - bootstrap_peers: Vec, - mut shutdown_rx: tokio::sync::watch::Receiver, -) { - // If shutdown arrives during the initial delay, exit before announcing. - tokio::select! { - _ = tokio::time::sleep(std::time::Duration::from_secs(2)) => {} - _ = shutdown_rx.changed() => { - if *shutdown_rx.borrow() { - info!("gossip: shutdown during startup delay, exiting"); - return; - } - } - } - - // Reuse the shared no-redirect client for every gossip outbound call (the - // bootstrap announce POST and the periodic peer /health ping). Peer URLs are - // attacker-influenceable, so a 3xx to a private address must not be followed. - // Do NOT fall back to reqwest::Client::new(): its default follows redirects - // and would reintroduce the SSRF closed here (#93). - let client = state.http_client.clone(); - let my_did = state.node_did.to_string(); - let my_url = state.config.public_url.clone().unwrap_or_default(); - - // Announce ourselves to each bootstrap peer - for peer_url in &bootstrap_peers { - // Cooperative shutdown between peers — a slow peer shouldn't - // block the node exiting. - if *shutdown_rx.borrow() { - info!("gossip: shutdown signalled during peer announce, exiting"); - return; - } - let path = "/api/v1/peers/announce"; - let announce_url = format!("{}{}", peer_url.trim_end_matches('/'), path); - let body = serde_json::json!({ - "did": my_did.clone(), - "http_url": my_url.clone(), - }); - let body_bytes = match serde_json::to_vec(&body) { - Ok(bytes) => bytes, - Err(e) => { - tracing::warn!(err = %e, "failed to serialize peer announce body"); - continue; - } - }; - let signed = sign_request(state.node_keypair.as_ref(), "POST", path, &body_bytes); - // Per-request timeout inside the loop; do not let one hung peer - // block others. The request itself is a normal tokio future so - // it's cancel-safe on shutdown. - match tokio::time::timeout( - std::time::Duration::from_secs(5), - client - .post(&announce_url) - .header("Content-Type", "application/json") - .header("Content-Digest", signed.content_digest) - .header("Signature-Input", signed.signature_input) - .header("Signature", signed.signature) - .body(body_bytes) - .send(), - ) - .await - { - Ok(Ok(resp)) => { - if resp.status().is_success() { - if let Ok(json) = resp.json::().await { - // Add them back to our peer list - if let (Some(their_did), Some(their_url)) = ( - json.get("node_did").and_then(|v| v.as_str()), - json.get("node_url").and_then(|v| v.as_str()), - ) { - if !their_url.is_empty() { - let _ = state.db.upsert_peer(their_did, their_url).await; - tracing::info!(did = %their_did, url = %their_url, "bootstrap peer added"); - } - } - } - } - } - Ok(Err(e)) => { - tracing::warn!(url = %announce_url, err = %e, "failed to announce to bootstrap peer") - } - Err(_) => tracing::warn!(url = %announce_url, "bootstrap peer announce timed out (5s)"), - } - } - - // Periodic ping every 5 minutes — exit on shutdown. - let mut interval = tokio::time::interval(std::time::Duration::from_secs(300)); - loop { - tokio::select! { - _ = interval.tick() => { - let peers = match state.db.list_peers().await { - Ok(p) => p, - Err(_) => continue, - }; - for peer in peers { - let ok = ping_peer_health(&client, &peer.http_url).await; - let _ = state.db.mark_peer_ping(&peer.did, ok).await; - } - } - _ = shutdown_rx.changed() => { - if *shutdown_rx.borrow() { - info!("gossip task: shutdown signal received, exiting"); - return; - } - } - } - } -} - -/// Build the shared node HTTP client used for every outbound fan-out (sync -/// trigger, profile/repo fetches, gossip announce + peer pings). -/// -/// No redirects: peer URLs are attacker-influenceable, so a `3xx` to a private -/// address must not be followed (SSRF guard, #78/#93). Do NOT replace with -/// `reqwest::Client::new()` — its default follows redirects. Kept as a named -/// builder so tests bind the redirect guarantee to the real client the node -/// runs, not a hand-rolled equivalent. -fn build_http_client() -> reqwest::Result { - reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(10)) - .redirect(reqwest::redirect::Policy::none()) - .build() -} - -/// Ping a peer's `/health` endpoint and report whether it answered 2xx. -/// -/// Takes the client by reference so callers supply the shared, no-redirect -/// `state.http_client`. Peer URLs are attacker-influenceable, so a `3xx` to a -/// private address must not be followed. Do NOT call this with a bare -/// `reqwest::Client::new()`: its default follows redirects and would -/// reintroduce the SSRF this guards against (#93). -async fn ping_peer_health(client: &reqwest::Client, http_url: &str) -> bool { - let url = format!("{}/health", http_url.trim_end_matches('/')); - client - .get(&url) - .send() - .await - .map(|r| r.status().is_success()) - .unwrap_or(false) -} - -fn load_or_create_keypair(config: &Config) -> Result { - let key_path = config.resolved_key_path(); - - if key_path.exists() { - let pem = std::fs::read_to_string(&key_path) - .with_context(|| format!("failed to read key from {}", key_path.display()))?; - let kp = Keypair::from_pem(&pem).map_err(|e| anyhow::anyhow!("invalid PEM key: {e}"))?; - info!(path = %key_path.display(), "loaded existing identity"); - Ok(kp) - } else { - let kp = Keypair::generate(); - let pem = kp - .to_pem() - .map_err(|e| anyhow::anyhow!("failed to serialize key: {e}"))?; - - if let Some(parent) = key_path.parent() { - std::fs::create_dir_all(parent)?; - } - - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - std::fs::write(&key_path, pem.as_bytes())?; - std::fs::set_permissions(&key_path, std::fs::Permissions::from_mode(0o600))?; - } - #[cfg(not(unix))] - std::fs::write(&key_path, pem.as_bytes())?; - - info!(path = %key_path.display(), did = %kp.did(), "generated new node identity"); - Ok(kp) - } -} - -#[cfg(test)] -mod gossip_ssrf_tests { - use super::ping_peer_health; - - // Build the client exactly as production does (super::build_http_client) so - // these tests bind the redirect guarantee to the real shared client the - // node runs. A regression that makes build_http_client follow redirects - // fails ping_peer_health_does_not_follow_redirect. - fn production_http_client() -> reqwest::Client { - super::build_http_client().expect("failed to build production http client") - } - - // A peer answering `/health` with a 302 toward an internal address must not - // be followed: the redirect target must never be requested (#93). - #[tokio::test] - async fn ping_peer_health_does_not_follow_redirect() { - let mut server = mockito::Server::new_async().await; - let internal = server - .mock("GET", "/internal-metadata") - .with_status(200) - .expect(0) - .create_async() - .await; - let _health = server - .mock("GET", "/health") - .with_status(302) - .with_header("location", &format!("{}/internal-metadata", server.url())) - .create_async() - .await; - - let ok = ping_peer_health(&production_http_client(), &server.url()).await; - - assert!(!ok, "a 302 must not count as a healthy peer"); - // expect(0) is enforced only at assert time; this fails if the redirect - // was followed to the internal target. - internal.assert_async().await; - } - - #[tokio::test] - async fn ping_peer_health_reports_success_on_200() { - let mut server = mockito::Server::new_async().await; - let _health = server - .mock("GET", "/health") - .with_status(200) - .create_async() - .await; - - let ok = ping_peer_health(&production_http_client(), &server.url()).await; - - assert!(ok, "a 200 /health must count as a healthy peer"); - } - - // A transport error (nothing listening) must map to unhealthy, never a - // spurious healthy — the .unwrap_or(false) arm. - #[tokio::test] - async fn ping_peer_health_reports_unhealthy_on_connection_error() { - let ok = ping_peer_health(&production_http_client(), "http://127.0.0.1:1").await; - assert!(!ok, "a connection error must count as an unhealthy peer"); - } +async fn main() -> anyhow::Result<()> { + gitlawb_node::run().await } diff --git a/crates/gitlawb-node/src/test_harness.rs b/crates/gitlawb-node/src/test_harness.rs new file mode 100644 index 00000000..6c80e171 --- /dev/null +++ b/crates/gitlawb-node/src/test_harness.rs @@ -0,0 +1,513 @@ +//! Test-only spawn surface for the real-node deny harness (see +//! `tests/deny_harness.rs`). Feature-gated behind `test-harness` so it never +//! compiles into the production binary. It exposes a single boot constructor so +//! the out-of-crate integration test can bring up a real node over a bound TCP +//! socket without the integration crate needing access to `graphql`, +//! `rate_limit`, `repo_store`, or the other internals `AppState` is built from. +//! +//! The node is built the same way `test_support::build_state` builds it (p2p +//! disabled, real migrated pool), but bound on `127.0.0.1:0` and served through +//! the real `axum::serve` stack with connect-info, so the middleware order, +//! body limits, and per-IP rate limiters all run exactly as in production. + +use std::net::SocketAddr; +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use chrono::Utc; +use clap::Parser; +use sqlx::PgPool; +use tokio::net::TcpListener; +use tokio::sync::watch; +use uuid::Uuid; + +use gitlawb_core::identity::Keypair; + +use crate::config::Config; +use crate::db::{Db, PullRequest, RepoRecord, VisibilityMode}; +use crate::rate_limit::{RateLimiter, TrustedProxy}; +use crate::state::AppState; + +/// A running node bound to an ephemeral port. End every test with +/// [`TestNode::shutdown`]: it joins the serve task and closes the pool, so +/// `#[sqlx::test]`'s `DROP DATABASE` cleanup (which runs the moment the test +/// future returns) never races the server's still-open connections. Dropping +/// without it signals shutdown, aborts any remaining serve task, and removes +/// the temp repository directory, a fallback for tests that return or panic +/// before reaching teardown. +pub struct TestNode { + /// Base URL of the running node, e.g. `http://127.0.0.1:54321`. + pub base_url: String, + /// The node's own DID (for building requests that reference it). + pub node_did: String, + shutdown_tx: watch::Sender, + repos_dir: PathBuf, + /// Seeding handle over the same pool the node serves from. Kept private so + /// the integration crate seeds through the methods below rather than + /// naming `Db`/`RepoRecord` directly. + db: Arc, + /// The detached `axum::serve` task, joined by [`Self::shutdown`]. `None` + /// once taken, so the `Drop` fallback never overlaps a completed teardown. + /// Carries `serve`'s own `io::Result<()>` so `shutdown()` can observe it + /// rather than discard it (see the coverage note there). + server: Option>>, + /// A clone of the served pool, kept so [`Self::shutdown`] can close it + /// (sqlx pool clones share one inner, so closing this closes the clones + /// inside `AppState`/`Db` too). + pool: PgPool, + /// Test-only: when set, `Drop` skips the graceful watch signal so ONLY + /// `handle.abort()` tears the serve task down. Defaults false, so every + /// other test keeps both redundant teardown mechanisms. Set via + /// [`Self::force_abort_only_teardown`] to fence that the abort alone + /// releases the database when the graceful chain is broken. + suppress_graceful_shutdown_on_drop: bool, +} + +impl Drop for TestNode { + fn drop(&mut self) { + // Fallback for tests that return or panic before reaching + // [`TestNode::shutdown`]: flip the shared shutdown signal so the serve + // task exits gracefully, abort whatever is left of it, and remove the + // temp repos dir. Drop cannot await, so it cannot join the server or + // close the pool; a test that already failed should not panic again in + // teardown. All three actions are idempotent or no-ops after + // `shutdown()` (which takes the handle), so running afterwards is + // harmless. + // + // The abort matters when the graceful chain does not run to completion + // before sqlx's `DROP DATABASE` cleanup: `JoinHandle::abort()` is a + // synchronous cancellation request callable from non-async Drop, and + // the aborted task's future (with the `PgPool` clones inside its + // router/state) is dropped on a subsequent scheduler tick, which sqlx + // cleanup's own awaits provide before the drop statement runs. The + // race is closed by those scheduling points, not by the abort alone. + // + // `suppress_graceful_shutdown_on_drop` (test-only) forces the abort to + // carry teardown by itself, so a test can fence that the abort line is + // load-bearing; it is false everywhere else. + if !self.suppress_graceful_shutdown_on_drop { + let _ = self.shutdown_tx.send(true); + } + if let Some(handle) = self.server.take() { + handle.abort(); + } + let _ = std::fs::remove_dir_all(&self.repos_dir); + } +} + +/// Outcome of joining the serve task during teardown, split so the wedged-task +/// path is observable to a test (which injects a short timeout) without waiting +/// the production teardown window. +enum ServeTeardown { + /// The task returned (carrying `serve`'s `io::Result<()>`) — either within + /// the timeout, or in the race between the timeout firing and the abort + /// landing. A task that actually finished is never a wedge. + Graceful(std::io::Result<()>), + /// The task did not finish within the timeout and was aborted (cancelled) + /// and reaped. + TimedOutAborted, +} + +/// Join `handle` within `timeout`; on expiry `abort()` it and await the aborted +/// handle so the serve task (and the `PgPool` clones inside its router/state) is +/// reaped BEFORE the caller returns and `#[sqlx::test]` runs `DROP DATABASE`. +/// Takes `&mut` (a `JoinHandle` is `Unpin`) so ownership is retained across the +/// timeout and the handle can be aborted on expiry — the previous code moved the +/// handle into `timeout` and, on elapse, dropped it, which DETACHES the still +/// running task rather than aborting it, leaking the test database (#194 F2). +async fn join_or_abort( + handle: &mut tokio::task::JoinHandle>, + timeout: Duration, +) -> ServeTeardown { + match tokio::time::timeout(timeout, &mut *handle).await { + Ok(join_result) => ServeTeardown::Graceful(join_result.expect("serve task must not panic")), + Err(_elapsed) => { + handle.abort(); + match handle.await { + // The task finished in the race between the timeout firing and + // the abort landing — honor its result, not a false wedge. + Ok(result) => ServeTeardown::Graceful(result), + Err(e) if e.is_cancelled() => ServeTeardown::TimedOutAborted, + // A genuine panic in the serve task surfaces here, mirroring the + // in-time arm's `.expect("serve task must not panic")`. + Err(e) => std::panic::resume_unwind(e.into_panic()), + } + } + } +} + +/// Allocate a process-unique temp directory for a spawned node's repositories +/// without pulling in the `tempfile` dev-dependency (which is unavailable to +/// the library crate under `--features test-harness`). +fn unique_repos_dir() -> PathBuf { + static COUNTER: AtomicU64 = AtomicU64::new(0); + let n = COUNTER.fetch_add(1, Ordering::Relaxed); + std::env::temp_dir().join(format!("gitlawb-deny-harness-{}-{}", std::process::id(), n)) +} + +/// Build an [`AppState`] over the given migrated pool, mirroring +/// `test_support::build_state` but with a real on-disk repository directory so +/// git smart-HTTP routes can serve. P2P is always disabled. +fn build_state(db: Arc, pool: PgPool, repos_dir: PathBuf) -> AppState { + let keypair = Keypair::generate(); + let node_did = keypair.did(); + let (ref_tx, _) = tokio::sync::broadcast::channel(1); + let (task_tx, _) = tokio::sync::broadcast::channel(1); + let schema = Arc::new(crate::graphql::build_schema( + db.clone(), + ref_tx.clone(), + task_tx.clone(), + )); + AppState { + config: Arc::new(Config::parse_from(["gitlawb-node"])), + db, + node_did, + node_keypair: Arc::new(keypair), + p2p: None, + http_client: Arc::new(reqwest::Client::new()), + ref_update_tx: ref_tx, + task_event_tx: task_tx, + graphql_schema: schema, + machine_id: None, + repo_store: crate::git::repo_store::RepoStore::for_testing(repos_dir, pool), + rate_limiter: RateLimiter::new(100, Duration::from_secs(60)), + create_ip_rate_limiter: RateLimiter::new(1000, Duration::from_secs(3600)), + push_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + push_limiter_trust: TrustedProxy::None, + sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), + peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + shutdown_tx: watch::channel(false).0, + } +} + +/// Spawn a real node bound to `127.0.0.1:0` over the given (already-created, +/// empty) test pool. Runs the schema migrations, builds the router, and serves +/// it on a background task through the production `axum::serve` stack with +/// connect-info so per-IP layers key on the real peer. Returns once the socket +/// is bound and accepting connections. +pub async fn spawn_node(pool: PgPool) -> TestNode { + let db = Arc::new(crate::db::Db::for_testing(pool.clone())); + db.run_migrations() + .await + .expect("test schema migrations should apply"); + + let repos_dir = unique_repos_dir(); + std::fs::create_dir_all(&repos_dir).expect("create temp repos dir"); + + let state = build_state(db.clone(), pool.clone(), repos_dir.clone()); + let node_did = state.node_did.to_string(); + let shutdown_tx = state.shutdown_tx.clone(); + let mut shutdown_rx = shutdown_tx.subscribe(); + + let router = crate::server::build_router(state); + + let listener = TcpListener::bind(("127.0.0.1", 0)) + .await + .expect("bind ephemeral port"); + let addr = listener.local_addr().expect("read bound addr"); + + let server = tokio::spawn(async move { + axum::serve( + listener, + router.into_make_service_with_connect_info::(), + ) + .with_graceful_shutdown(async move { + let _ = shutdown_rx.changed().await; + }) + .await + }); + + TestNode { + base_url: format!("http://{addr}"), + node_did, + shutdown_tx, + repos_dir, + db, + server: Some(server), + pool, + suppress_graceful_shutdown_on_drop: false, + } +} + +impl TestNode { + /// Test-only: make `Drop` tear the serve task down through `handle.abort()` + /// alone by suppressing the graceful watch signal. Used to prove the abort + /// is load-bearing (that the database is released even when the graceful + /// chain is broken); no production or normal-teardown path calls this. + pub fn force_abort_only_teardown(&mut self) { + self.suppress_graceful_shutdown_on_drop = true; + } + + /// Graceful async teardown: signal shutdown, join the serve task, close + /// the pool, remove the temp repos dir. Call this at the end of every + /// test. `#[sqlx::test]` issues `DROP DATABASE` as soon as the test + /// future returns, so the server (which owns pool clones inside + /// `AppState`/`Db`) must be gone and the pool closed before then, or the + /// cleanup fails and leaks per-test databases. A serve task that does not + /// exit within 10s panics loudly: a wedged graceful shutdown must fail + /// the test, not hang CI. + /// + /// When `self` drops on return, `Drop` re-sends the signal and re-removes + /// the dir; both are idempotent, and the serve handle was already taken, + /// so the overlap is harmless. + pub async fn shutdown(self) { + self.shutdown_with_timeout(Duration::from_secs(10)).await; + } + + /// Body of [`Self::shutdown`] with an injectable teardown timeout so a test + /// can drive the wedged-shutdown path without waiting the production 10s. + /// Production always calls this with 10s. + async fn shutdown_with_timeout(mut self, timeout: Duration) { + let _ = self.shutdown_tx.send(true); + let outcome = match self.server.take() { + Some(mut handle) => Some(join_or_abort(&mut handle, timeout).await), + None => None, + }; + // Release the pool and temp dir on EVERY path before returning OR + // panicking. A wedged graceful shutdown is exactly when in-flight + // per-connection tasks still hold `PgPool` clones; aborting the serve + // future alone does not reliably drop them, so the explicit + // `pool.close()` (which closes the shared inner) is what frees them + // before `#[sqlx::test]` runs `DROP DATABASE`. Panicking before this + // (the previous behavior) skipped the close on the one path that needs + // it most (#194 F2). + self.pool.close().await; + let _ = std::fs::remove_dir_all(&self.repos_dir); + match outcome { + // Honest coverage contract: on axum 0.8.8, + // `WithGracefulShutdown::into_future` always resolves `Ok(())` (the + // shutdown signal is treated as success and `accept()` errors are + // retried forever rather than returned), so this `Err` arm is + // unreachable today - every existing harness test exercises only the + // `Ok` path here. Forward-compat insurance for a future axum where + // `serve` can fail: that should surface as a clear panic, not be + // silently discarded. + Some(ServeTeardown::Graceful(result)) => { + result.expect("axum::serve exited with an error"); + } + Some(ServeTeardown::TimedOutAborted) => { + panic!( + "serve task did not exit within {timeout:?} of the shutdown \ + signal (wedged graceful shutdown)" + ); + } + None => {} + } + } + + /// Insert a repo owned by `owner_did` and return its repo id. Mirrors + /// `test_support::seed_repo` (the `disk_path` field is unused by the + /// integration path — `RepoStore` computes the on-disk path from + /// `repos_dir`/owner/name, see [`Self::seed_bare_repo`]). + pub async fn seed_repo(&self, owner_did: &str, name: &str, is_public: bool) -> String { + let now = Utc::now(); + let record = RepoRecord { + id: Uuid::new_v4().to_string(), + name: name.to_string(), + owner_did: owner_did.to_string(), + description: None, + is_public, + default_branch: "main".to_string(), + created_at: now, + updated_at: now, + disk_path: format!("/tmp/{name}"), + forked_from: None, + machine_id: None, + }; + self.db.create_repo(&record).await.expect("seed repo"); + record.id + } + + /// Mark `branch` as protected on `repo_id`, so a non-owner push to it is + /// rejected by the branch-protection gate (used by the deny-prober's + /// protected-push probe — #195, F3). + pub async fn seed_protected_branch(&self, repo_id: &str, branch: &str, created_by: &str) { + self.db + .protect_branch(repo_id, branch, created_by) + .await + .expect("seed protected branch"); + } + + /// Seed an open PR `number` in `repo_id`, authored by `author_did`. The + /// author-or-owner close gate (`close_pr`) loads the PR *before* it runs, so a + /// deny-prober row for `.../pulls/{number}/close` needs the PR to exist or a + /// stranger gets a 404 (absent entity) instead of the 403 the gate emits. + pub async fn seed_pr(&self, repo_id: &str, number: i64, author_did: &str) { + let now = Utc::now().to_rfc3339(); + let pr = PullRequest { + id: Uuid::new_v4().to_string(), + repo_id: repo_id.to_string(), + number, + title: "seed pr".to_string(), + body: None, + author_did: author_did.to_string(), + source_branch: "feature".to_string(), + target_branch: "main".to_string(), + status: "open".to_string(), + merged_by_did: None, + merged_at: None, + created_at: now.clone(), + updated_at: now, + }; + self.db.create_pr(&pr).await.expect("seed pr"); + } + + /// Seed issue `issue_id` (a git-ref JSON blob authored by `author_did`) in the + /// repo's bare on-disk store. Like [`Self::seed_pr`], the `close_issue` + /// owner-or-author gate loads the issue before it runs, so the deny-prober row + /// for `.../issues/{id}/close` needs the issue present to reach the 403. + pub fn seed_issue(&self, owner_did: &str, name: &str, issue_id: &str, author_did: &str) { + let slug = owner_did.replace([':', '/'], "_"); + let bare = self.repos_dir.join(&slug).join(format!("{name}.git")); + let json = format!(r#"{{"author":"{author_did}"}}"#); + crate::git::issues::create_issue(&bare, issue_id, &json).expect("seed issue"); + } + + /// Add a path-scoped visibility rule restricting `path_glob` to + /// `reader_dids` (empty = only the owner). Mode B keeps object SHAs intact + /// and withholds blob content, which is what the read/replication gates + /// enforce. + pub async fn withhold_path( + &self, + repo_id: &str, + path_glob: &str, + reader_dids: &[String], + created_by: &str, + ) { + self.db + .set_visibility_rule( + repo_id, + path_glob, + VisibilityMode::B, + reader_dids, + created_by, + ) + .await + .expect("set visibility rule"); + } + + /// Create a real bare git repository on disk at the exact path + /// `RepoStore::acquire` reads (`//.git`), with + /// one commit on `main` containing `files` (`(path, contents)`). Returns the + /// blob OID of each path so callers can assert a withheld OID never appears + /// in served bytes (U8). Shells out to `git`, mirroring + /// `test_support`'s served-content seam. + /// `object_format` is `"sha1"` for the git smart-HTTP surfaces and + /// `"sha256"` for the content-addressed `/ipfs/{cid}` surface (whose CIDs + /// are the sha2-256 object ids). + pub fn seed_bare_repo( + &self, + owner_did: &str, + name: &str, + files: &[(&str, &str)], + object_format: &str, + ) -> std::collections::HashMap { + let run = |args: &[&str], cwd: &std::path::Path| { + // Test-harness-only fixture seeding, feature-gated out of the production + // binary; runs local git against a tempdir inside a test's own lifetime. + // allow-unbounded-git: test-only seeding, never on a request path holding a permit + let out = std::process::Command::new("git") + .args(args) + .current_dir(cwd) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + + // Build a source work tree, then bare-clone it into the served path. + let src = self.repos_dir.join(format!("src-{name}")); + std::fs::create_dir_all(&src).expect("create src dir"); + let fmt_arg = format!("--object-format={object_format}"); + run(&["init", "-q", "-b", "main", &fmt_arg], &src); + run(&["config", "user.email", "t@t"], &src); + run(&["config", "user.name", "t"], &src); + for (path, contents) in files { + let full = src.join(path); + if let Some(parent) = full.parent() { + std::fs::create_dir_all(parent).expect("create file parent dir"); + } + std::fs::write(&full, contents).expect("write seed file"); + run(&["add", path], &src); + } + run(&["commit", "-q", "-m", "seed"], &src); + + let mut oids = std::collections::HashMap::new(); + for (path, _) in files { + let oid = run(&["rev-parse", &format!("HEAD:{path}")], &src); + oids.insert((*path).to_string(), oid); + } + // The HEAD commit oid, used as the `want` when driving upload-pack. + oids.insert("HEAD".to_string(), run(&["rev-parse", "HEAD"], &src)); + + let slug = owner_did.replace([':', '/'], "_"); + let bare = self.repos_dir.join(&slug).join(format!("{name}.git")); + std::fs::create_dir_all(bare.parent().unwrap()).expect("create bare parent"); + run( + &[ + "clone", + "--bare", + "-q", + src.to_str().unwrap(), + bare.to_str().unwrap(), + ], + &self.repos_dir, + ); + + oids + } +} + +#[cfg(test)] +mod tests { + use super::{join_or_abort, ServeTeardown}; + use std::time::Duration; + + /// A wedged serve task (never returns within the window) must be ABORTED and + /// reaped on timeout, not detached. The previous teardown moved the + /// `JoinHandle` into `timeout` and dropped it on elapse, which detaches the + /// still-running task and leaks the test DB across `DROP DATABASE` (#194 F2). + /// Load-bearing: neutralize the abort in `join_or_abort` (return without + /// abort+await) and this flips RED — `handle.is_finished()` stays false + /// because the task was left running. + #[tokio::test] + async fn join_or_abort_aborts_a_wedged_task_on_timeout() { + let mut handle: tokio::task::JoinHandle> = tokio::spawn(async { + loop { + tokio::time::sleep(Duration::from_secs(3600)).await; + } + }); + assert!( + matches!( + join_or_abort(&mut handle, Duration::from_millis(50)).await, + ServeTeardown::TimedOutAborted + ), + "a task that never returns must be reported timed-out, not graceful" + ); + assert!( + handle.is_finished(), + "the wedged serve task must be aborted and reaped, not detached" + ); + } + + /// A task that returns within the window is reported graceful, carrying its + /// `io::Result` — the normal teardown path. + #[tokio::test] + async fn join_or_abort_reports_graceful_when_task_exits_in_time() { + let mut handle: tokio::task::JoinHandle> = + tokio::spawn(async { Ok(()) }); + assert!( + matches!( + join_or_abort(&mut handle, Duration::from_secs(5)).await, + ServeTeardown::Graceful(Ok(())) + ), + "a prompt clean exit must be reported graceful" + ); + } +} diff --git a/crates/gitlawb-node/tests/deny_harness.rs b/crates/gitlawb-node/tests/deny_harness.rs new file mode 100644 index 00000000..c14203a9 --- /dev/null +++ b/crates/gitlawb-node/tests/deny_harness.rs @@ -0,0 +1,2484 @@ +//! Real-node deny harness: boots a real gitlawb-node over a bound TCP socket +//! and drives trust-boundary DENY paths through a real reqwest client, +//! asserting both the refusal status and that no withheld data leaks +//! (INV-1/INV-2/INV-8). Requires `--features test-harness`. +//! +//! Each `#[sqlx::test]` gets an ephemeral per-test database; `spawn_node` runs +//! the schema migrations and serves the real router on `127.0.0.1:0`. Every +//! test ends with `node.shutdown().await` so the serve task is joined and the +//! pool closed before sqlx's `DROP DATABASE` cleanup runs (except the Drop +//! regression test at the bottom, which deliberately skips it). + +mod support; + +use std::process::Command; + +use support::assert::assert_denied; +use support::probe::{probes_for, Expect, Fixture, Probe, Signer}; +use support::routes::{deny_bearing_routes, GateClass}; +use support::signing::signed_request; + +use gitlawb_core::cid::Cid; +use gitlawb_core::identity::Keypair; +use gitlawb_node::test_harness::spawn_node; + +/// Build the `/ipfs/{cid}` CID for a 64-hex sha2-256 git object id, matching the +/// node's own `Cid::from_sha256_bytes` (the value `get_by_cid` decodes back). +fn cid_for_oid(oid_hex: &str) -> String { + let bytes = hex::decode(oid_hex).expect("hex oid"); + let arr: [u8; 32] = bytes.as_slice().try_into().expect("32-byte sha256 oid"); + Cid::from_sha256_bytes(&arr).to_string() +} + +/// A reqwest client with a bounded timeout so a wedged node route fails the test +/// fast rather than hanging until the CI job timeout. +fn bounded_client() -> reqwest::Client { + reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(30)) + .build() + .expect("client builds") +} + +// ── U2: the signing client produces signatures require_signature accepts ───── + +/// A validly signed receive-pack request clears `require_signature` (it does not +/// get a 401): it proceeds past the signature layer and is denied later for a +/// different reason (the repo does not exist). This proves the signing client +/// is producing signatures the real verifier accepts over the socket. +#[sqlx::test] +async fn signed_receive_pack_clears_signature_layer(pool: sqlx::PgPool) { + let node = spawn_node(pool).await; + let client = bounded_client(); + let kp = Keypair::generate(); + + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + "/alice/repo/git-receive-pack", + b"0000".to_vec(), + &kp, + ) + .send() + .await + .expect("request sends"); + + assert_ne!( + resp.status().as_u16(), + 401, + "a valid signature must clear require_signature; got 401" + ); + + node.shutdown().await; +} + +/// Tampering the body after signing invalidates the content-digest, so the +/// server rejects with 400 `content_digest_mismatch` (distinct from the 401 a +/// missing/invalid signature gets). Proves the signature actually covers the +/// body: the digest is signed and the server re-checks it against the bytes it +/// received. +#[sqlx::test] +async fn tampered_body_after_signing_is_rejected(pool: sqlx::PgPool) { + let node = spawn_node(pool).await; + let client = bounded_client(); + let kp = Keypair::generate(); + + // Sign one body, then replace it before sending. + let mut req = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + "/alice/repo/git-receive-pack", + b"0000".to_vec(), + &kp, + ) + .build() + .expect("build request"); + *req.body_mut() = Some(reqwest::Body::from(b"tampered".to_vec())); + + let resp = client.execute(req).await.expect("request sends"); + assert_eq!( + resp.status().as_u16(), + 400, + "a body that no longer matches its content-digest must be rejected" + ); + + node.shutdown().await; +} + +// ── U5(a): INV-8 — an unsigned push is denied and leaks nothing ────────────── + +/// An unauthenticated git-receive-pack (no signature headers) is rejected with +/// 401 before any handler runs, and the denial body carries no repo internals. +#[sqlx::test] +async fn unsigned_receive_pack_is_denied(pool: sqlx::PgPool) { + let node = spawn_node(pool).await; + let client = bounded_client(); + + let url = format!("{}/alice/repo/git-receive-pack", node.base_url); + let resp = client + .post(&url) + .header("content-type", "application/x-git-receive-pack-request") + .body(b"0000".to_vec()) + .send() + .await + .expect("request sends"); + + // No repo was seeded, so there are no OIDs to leak; the assertion still + // enforces the 4xx-and-not-empty-200 INV-8 shape. + assert_denied(resp, 401, &[]).await; + + node.shutdown().await; +} + +/// Build a minimal git-receive-pack request body: one ref-update pkt-line for +/// `refs/heads/` (dummy 40-hex old/new SHAs — the branch-protection gate +/// only reads the ref NAME) plus a flush. Enough to reach the ref-update parse +/// and the protection gate, not a real pack. +fn receive_pack_update_body(branch: &str) -> Vec { + let old = "0".repeat(40); + let new = "1".repeat(40); + let line = format!("{old} {new} refs/heads/{branch}\0report-status\n"); + let pkt = format!("{:04x}{line}", line.len() + 4); + let mut body = pkt.into_bytes(); + body.extend_from_slice(b"0000"); + body +} + +/// #195 (F3): a signed NON-OWNER pushing to a PROTECTED branch is forbidden (403) +/// by the branch-protection gate (`repos.rs`), and the owner pushing to the same +/// branch is NOT blocked (control). git_receive_pack's registry row only drives the +/// unsigned-401 signature path, so without this probe inverting the 403 leaves the +/// sweep AND the completeness scan green. Drives the 403 so the gate can't rot. +#[sqlx::test] +async fn signed_stranger_protected_branch_push_is_forbidden(pool: sqlx::PgPool) { + let node = spawn_node(pool).await; + // #195 (F1): a bounded timeout so a wedged git-receive-pack fails the suite + // rather than hanging it until CI kills the job. + let client = support::bounded_client(); + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let stranger = Keypair::generate(); + + let repo_id = node.seed_repo(&owner_did, "protrepo", true).await; + node.seed_protected_branch(&repo_id, "main", &owner_did) + .await; + + let path = format!("/{owner_did}/protrepo/git-receive-pack"); + let body = receive_pack_update_body("main"); + + // Signed non-owner -> 403 branch protection; the denial leaks no repo internals. + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &path, + body.clone(), + &stranger, + ) + .send() + .await + .expect("request sends"); + assert_eq!( + resp.status().as_u16(), + 403, + "a signed non-owner push to a protected branch must be forbidden (403)" + ); + assert_denied(resp, 403, &[repo_id.as_str()]).await; + + // Owner control: the owner is NOT blocked by branch protection (it may fail + // later on the dummy pack, but must not be the 403 the stranger got). + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &path, + body, + &owner, + ) + .send() + .await + .expect("request sends"); + assert_ne!( + resp.status().as_u16(), + 403, + "the owner must not be blocked by their own branch protection (control)" + ); + + node.shutdown().await; +} + +// ── U5(b): INV-8/INV-2 — anonymous /ipfs/{cid} of a withheld blob is denied ── + +/// A public repo with a `/secret/**` withhold rule (readers = one allowed DID). +/// An anonymous content-addressed read of the withheld blob's CID is denied +/// (404) and leaks neither the secret bytes nor its OID; the sibling public +/// blob's CID is served anonymously, proving the withhold is blob-scoped. +#[sqlx::test] +async fn anon_ipfs_read_of_withheld_blob_is_denied(pool: sqlx::PgPool) { + let node = spawn_node(pool).await; + let client = bounded_client(); + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let reader = Keypair::generate(); + + let repo_id = node.seed_repo(&owner_did, "u5b-repo", true).await; + // sha256 object format: the /ipfs CID is the sha2-256 object id. + let oids = node.seed_bare_repo( + &owner_did, + "u5b-repo", + &[ + ("public/a.txt", "public bytes U5b"), + ("secret/b.txt", "TOPSECRET-U5b"), + ], + "sha256", + ); + node.withhold_path( + &repo_id, + "/secret/**", + &[reader.did().to_string()], + &owner_did, + ) + .await; + + let secret_oid = oids["secret/b.txt"].clone(); + let secret_cid = cid_for_oid(&secret_oid); + let public_cid = cid_for_oid(&oids["public/a.txt"]); + + // Anonymous read of the withheld blob's CID: denied, no leak of content or OID. + let resp = client + .get(format!("{}/ipfs/{secret_cid}", node.base_url)) + .send() + .await + .expect("request sends"); + assert_denied( + resp, + 404, + &["TOPSECRET-U5b", &secret_oid, &secret_oid[..12]], + ) + .await; + + // The sibling public blob's CID is served to anon (withhold is blob-scoped). + let resp = client + .get(format!("{}/ipfs/{public_cid}", node.base_url)) + .send() + .await + .expect("request sends"); + assert_eq!( + resp.status().as_u16(), + 200, + "public blob CID must be served to anon" + ); + assert!( + resp.text().await.unwrap().contains("public bytes U5b"), + "the public blob content is returned" + ); + + node.shutdown().await; +} + +// ── U6: INV-1 — a validly signed NON-owner mutation is owner-gated (403) ────── + +/// The case the in-crate `oneshot` suite cannot reach over the full stack: a +/// fully valid RFC-9421 signature from a non-owner DID hits a mutation and is +/// rejected by `require_owner` with 403, not merely "not authenticated". The +/// non-owner sends no `x-ucan` header, so `require_ucan_chain` passes through +/// and the request reaches the owner gate. +#[sqlx::test] +async fn wrong_owner_visibility_put_is_forbidden(pool: sqlx::PgPool) { + let node = spawn_node(pool).await; + let client = bounded_client(); + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let stranger = Keypair::generate(); + + node.seed_repo(&owner_did, "harness-repo", true).await; + + let path = format!("/api/v1/repos/{owner_did}/harness-repo/visibility"); + let body = br#"{"path_glob":"/","reader_dids":[]}"#.to_vec(); + + // Non-owner: valid signature, wrong identity -> 403 from the owner gate. + let resp = signed_request( + &client, + reqwest::Method::PUT, + &node.base_url, + &path, + body.clone(), + &stranger, + ) + .header("content-type", "application/json") + .send() + .await + .expect("request sends"); + assert_denied(resp, 403, &[]).await; + + // Reachability proof: the same request signed by the OWNER is not 403 (it + // reaches the handler). Without this, a 403 produced by an earlier layer or + // a mis-seeded repo would masquerade as a passing INV-1 case. + let resp = signed_request( + &client, + reqwest::Method::PUT, + &node.base_url, + &path, + body, + &owner, + ) + .header("content-type", "application/json") + .send() + .await + .expect("request sends"); + assert!( + resp.status().is_success(), + "owner's signed visibility PUT must reach the handler, got {}", + resp.status() + ); + + node.shutdown().await; +} + +// ── U7: INV-2 — a read over a withheld path is denied and leaks nothing ─────── + +/// A public repo with a path-scoped withhold rule on `/secret/**`. An anonymous +/// blob read of the withheld path is denied (404 RepoNotFound) and the body +/// carries neither the secret content nor its blob OID; a read of a sibling +/// public path succeeds, proving the gate is path-scoped, not a blanket refusal. +#[sqlx::test] +async fn withheld_path_blob_read_is_denied(pool: sqlx::PgPool) { + let node = spawn_node(pool).await; + let client = bounded_client(); + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + + let repo_id = node.seed_repo(&owner_did, "u7-repo", true).await; + let oids = node.seed_bare_repo( + &owner_did, + "u7-repo", + &[ + ("public.txt", "hello public"), + ("secret/data.txt", "TOPSECRET-CONTENT-U7"), + ], + "sha1", + ); + node.withhold_path(&repo_id, "/secret/**", &[], &owner_did) + .await; + + let secret_oid = oids["secret/data.txt"].clone(); + let short_oid = &secret_oid[..12]; + + // Withheld path: denied, and neither the content nor the OID (full or short) + // may appear in the denial body. + let resp = client + .get(format!( + "{}/api/v1/repos/{owner_did}/u7-repo/blob/secret/data.txt", + node.base_url + )) + .send() + .await + .expect("request sends"); + assert_denied(resp, 404, &["TOPSECRET-CONTENT-U7", &secret_oid, short_oid]).await; + + // Sibling public path: served, proving the withhold is path-scoped. + let resp = client + .get(format!( + "{}/api/v1/repos/{owner_did}/u7-repo/blob/public.txt", + node.base_url + )) + .send() + .await + .expect("request sends"); + assert_eq!( + resp.status().as_u16(), + 200, + "public sibling path must be served" + ); + assert!( + resp.text().await.unwrap().contains("hello public"), + "the public blob content is returned" + ); + + node.shutdown().await; +} + +// ── U3: get_pr_diff SECOND (per-path) gate — a diff touching a withheld path ── + +/// Build a git v0 receive-pack body creating `refs/heads/` at a fresh +/// commit that adds `files` on top of the existing `main`, force-updating main to +/// a deterministic base so `` shares history with main. Returns the body +/// bytes plus a map of each pushed file's blob OID (full sha1), so the caller can +/// withhold the OID of a pushed secret path (#195, F2). Shells out to the local +/// `git`. +fn build_branch_push_body( + server_main_tip: &str, + branch: &str, + files: &[(&str, &str)], +) -> (Vec, std::collections::HashMap) { + use std::io::Write; + use std::process::{Command, Stdio}; + + let work = std::env::temp_dir().join(format!( + "gl-u3-prdiff-{}-{}", + std::process::id(), + server_main_tip + )); + let _ = std::fs::remove_dir_all(&work); + std::fs::create_dir_all(&work).unwrap(); + let run = |args: &[&str]| -> String { + let out = Command::new("git") + .args(args) + .current_dir(&work) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + // Pin sha1 to match the served fixture repo; init.defaultObjectFormat=sha256 breaks the ref update. + run(&["init", "-q", "-b", "main", "--object-format=sha1"]); + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + std::fs::write(work.join("base.txt"), "prdiff base").unwrap(); + run(&["add", "base.txt"]); + run(&["commit", "-q", "-m", "base"]); + let new_main = run(&["rev-parse", "HEAD"]); + run(&["checkout", "-q", "-b", branch]); + for (p, c) in files { + let full = work.join(p); + if let Some(parent) = full.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(&full, c).unwrap(); + run(&["add", p]); + } + run(&["commit", "-q", "-m", "feature"]); + let feature_tip = run(&["rev-parse", "HEAD"]); + + // #195 (F2): capture each pushed file's blob OID (full sha1). seed_bare_repo + // never seeds these pushed paths, so this is the only place their OID is known; + // the PR-diff denial withholds the secret path's OID (full + [..12] short form). + let mut pushed_oids = std::collections::HashMap::new(); + for (p, _) in files { + pushed_oids.insert((*p).to_string(), run(&["rev-parse", &format!("HEAD:{p}")])); + } + + let child = Command::new("git") + .args(["pack-objects", "--stdout", "--revs"]) + .current_dir(&work) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + child + .stdin + .as_ref() + .unwrap() + .write_all(format!("{new_main}\n{feature_tip}\n").as_bytes()) + .unwrap(); + let pack = child.wait_with_output().unwrap().stdout; + + let zero = "0".repeat(server_main_tip.len()); + let l1 = format!("{server_main_tip} {new_main} refs/heads/main\0report-status\n"); + let l2 = format!("{zero} {feature_tip} refs/heads/{branch}\n"); + let mut body: Vec = Vec::new(); + body.extend(format!("{:04x}{l1}", l1.len() + 4).into_bytes()); + body.extend(format!("{:04x}{l2}", l2.len() + 4).into_bytes()); + body.extend_from_slice(b"0000"); + body.extend_from_slice(&pack); + let _ = std::fs::remove_dir_all(&work); + (body, pushed_oids) +} + +/// get_pr_diff has a SECOND, path-scoped gate after the repo-root read: it +/// iterates the diff's touched paths and 404s if any is withheld (pulls.rs +/// visibility_check loop). On a PUBLIC repo with a `/secret/**` withhold rule and a +/// PR whose diff touches `secret/x.txt`, an anon caller PASSES the public repo-root +/// gate but must be DENIED by the per-path gate (404), leaking neither the withheld +/// content nor its OID; the owner sees the full diff. This drives the per-path gate +/// the private-repo registry row cannot reach (there, the root gate fires first for +/// every non-owner), so reverting the visibility_check Deny-return turns this RED. +#[sqlx::test] +async fn get_pr_diff_withheld_path_is_denied(pool: sqlx::PgPool) { + let node = spawn_node(pool).await; + // #195 (F1): a bounded timeout so a wedged git subprocess push fails the suite + // rather than hanging it until CI kills the job. + let client = support::bounded_client(); + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + + let repo_id = node.seed_repo(&owner_did, "prdiff-repo", true).await; + let oids = node.seed_bare_repo( + &owner_did, + "prdiff-repo", + &[("public/a.txt", "public seed")], + "sha1", + ); + node.withhold_path(&repo_id, "/secret/**", &[], &owner_did) + .await; + + // Push a `feature` branch whose diff vs main touches the withheld secret path. + let (body, pushed_oids) = build_branch_push_body( + &oids["HEAD"], + "feature", + &[("secret/x.txt", "TOPSECRET-PRDIFF-PATH")], + ); + let push = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &format!("/{owner_did}/prdiff-repo/git-receive-pack"), + body, + &owner, + ) + .header("content-type", "application/x-git-receive-pack-request") + .send() + .await + .expect("push sends"); + assert_eq!(push.status().as_u16(), 200, "owner push must land"); + + // Open a PR main <- feature (as owner, a reader of the public repo). + let create = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &format!("/api/v1/repos/{owner_did}/prdiff-repo/pulls"), + br#"{"title":"prdiff","source_branch":"feature","target_branch":"main"}"#.to_vec(), + &owner, + ) + .header("content-type", "application/json") + .send() + .await + .expect("create PR sends"); + assert_eq!(create.status().as_u16(), 201, "PR create must return 201"); + let pr: serde_json::Value = create.json().await.unwrap(); + let number = pr["number"].as_i64().expect("PR number"); + + // Anon PASSES the public repo-root gate, then the per-path gate DENIES the diff + // (it touches the withheld secret path): 404, leaking no content or OID. + // #195 (F2): withhold the pushed secret blob's OID in both forms (the full + // sha1 and `oid[..12]`), matching the private_blob_oid_short convention + // (probe.rs:200), NOT `git rev-parse --short` (variable-length). + let secret_oid_full = pushed_oids["secret/x.txt"].clone(); + let secret_oid_short = secret_oid_full[..12].to_string(); + let resp = client + .get(format!( + "{}/api/v1/repos/{owner_did}/prdiff-repo/pulls/{number}/diff", + node.base_url + )) + .send() + .await + .expect("anon diff read sends"); + let withheld = vec![ + "TOPSECRET-PRDIFF-PATH", + secret_oid_full.as_str(), + secret_oid_short.as_str(), + ]; + assert_denied(resp, 404, &withheld).await; + + // Owner sees the full diff (the per-path gate admits the reader). + let resp = signed_request( + &client, + reqwest::Method::GET, + &node.base_url, + &format!("/api/v1/repos/{owner_did}/prdiff-repo/pulls/{number}/diff"), + Vec::new(), + &owner, + ) + .send() + .await + .expect("owner diff read sends"); + assert_eq!( + resp.status().as_u16(), + 200, + "owner must see the full PR diff (per-path gate admits the reader)" + ); + assert!( + resp.text().await.unwrap().contains("TOPSECRET-PRDIFF-PATH"), + "the owner's diff returns the touched file content" + ); + + node.shutdown().await; +} + +// ── U8: INV-2 — an anonymous clone/fetch excludes withheld subtree blobs ────── + +/// The replication/clone surface, the one `oneshot` cannot serve: a public repo +/// with a `/secret/**` withhold rule. A real anonymous `git clone` over +/// git-upload-pack must yield a packfile that omits the withheld blob's object +/// while keeping the sibling public blob. The assertion is packfile-aware: git +/// parsed the served pack, and `cat-file -e` checks object presence in it. +#[sqlx::test] +async fn anon_clone_excludes_withheld_subtree_blobs(pool: sqlx::PgPool) { + let node = spawn_node(pool).await; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + + let repo_id = node.seed_repo(&owner_did, "u8-repo", true).await; + let oids = node.seed_bare_repo( + &owner_did, + "u8-repo", + &[ + ("public/a.txt", "public bytes U8"), + ("secret/b.txt", "TOPSECRET-U8"), + ], + "sha1", + ); + node.withhold_path(&repo_id, "/secret/**", &[], &owner_did) + .await; + + let secret_oid = oids["secret/b.txt"].clone(); + let public_oid = oids["public/a.txt"].clone(); + let head = oids["HEAD"].clone(); + + // Drive git-upload-pack directly (v0 stateless-RPC): a want for HEAD, a flush, + // then done. No side-band capability, so the response is a raw packfile after + // the NAK pkt-line. Vanilla `git clone` negotiates protocol v2 and deadlocks + // against the node's v0 server; the POST is the real replication surface and, + // via a bounded reqwest timeout, cannot wedge the suite. + let pkt = |s: &str| format!("{:04x}{}", s.len() + 4, s).into_bytes(); + let mut req_body: Vec = Vec::new(); + req_body.extend(pkt(&format!("want {head}\n"))); + req_body.extend_from_slice(b"0000"); // flush after wants + req_body.extend(pkt("done\n")); + + let client = bounded_client(); + let resp = client + .post(format!( + "{}/{owner_did}/u8-repo/git-upload-pack", + node.base_url + )) + .header("content-type", "application/x-git-upload-pack-request") + .body(req_body) + .send() + .await + .expect("upload-pack POST sends"); + assert_eq!( + resp.status().as_u16(), + 200, + "anon upload-pack POST must serve a pack" + ); + let bytes = resp.bytes().await.expect("read pack response"); + + // The packfile starts at the "PACK" magic (after the NAK pkt-line). + let pack_start = bytes + .windows(4) + .position(|w| w == b"PACK") + .expect("response must contain a packfile"); + let pack = &bytes[pack_start..]; + + // Index the served pack and list its objects (packfile-aware: a raw byte scan + // could not see an OID inside the zlib-compressed stream). + let dir = std::env::temp_dir().join(format!("gl-u8-pack-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let pack_path = dir.join("deny.pack"); + std::fs::write(&pack_path, pack).unwrap(); + let idx = Command::new("git") + .args(["index-pack", pack_path.to_str().unwrap()]) + .output() + .expect("git index-pack runs"); + assert!( + idx.status.success(), + "the served pack must index cleanly: {}", + String::from_utf8_lossy(&idx.stderr) + ); + let verify = Command::new("git") + .args([ + "verify-pack", + "-v", + pack_path.with_extension("idx").to_str().unwrap(), + ]) + .output() + .expect("git verify-pack runs"); + let listing = String::from_utf8_lossy(&verify.stdout).to_string(); + let _ = std::fs::remove_dir_all(&dir); + + assert!( + !listing.contains(&secret_oid), + "withheld blob {secret_oid} must be absent from the served pack; listing:\n{listing}" + ); + assert!( + listing.contains(&public_oid), + "public blob {public_oid} must be present (withhold is subtree-scoped); listing:\n{listing}" + ); + + node.shutdown().await; +} + +// ── U3: drive the whole deny-bearing route registry over the real stack ─────── + +/// Send one [`Probe`] against the running node, signing as the probe's identity. +/// Anon requests are unsigned; Owner/Stranger requests carry a real RFC-9421 +/// signature for the fixture's owner / stranger keypair. +async fn send_probe( + client: &reqwest::Client, + base_url: &str, + fixture: &Fixture, + probe: &Probe, +) -> reqwest::Response { + let rb = match probe.signer { + Signer::Anon => client + .request(probe.method.clone(), format!("{base_url}{}", probe.path)) + .body(probe.body.clone()), + Signer::Owner => signed_request( + client, + probe.method.clone(), + base_url, + &probe.path, + probe.body.clone(), + &fixture.owner, + ), + Signer::Stranger => signed_request( + client, + probe.method.clone(), + base_url, + &probe.path, + probe.body.clone(), + &fixture.stranger, + ), + // #195 (F1): each multi-principal arm signs with its own distinct fixture + // identity, so reverting one arm in the handler turns only that arm's twin + // RED. + Signer::Author => signed_request( + client, + probe.method.clone(), + base_url, + &probe.path, + probe.body.clone(), + &fixture.author, + ), + Signer::Creator => signed_request( + client, + probe.method.clone(), + base_url, + &probe.path, + probe.body.clone(), + &fixture.creator, + ), + Signer::Claimant => signed_request( + client, + probe.method.clone(), + base_url, + &probe.path, + probe.body.clone(), + &fixture.claimant, + ), + // #195 (F3): the assignee actor arm of complete_task / fail_task signs with + // the fixture's assignee identity (the one that claimed the seeded task), so + // reverting the assignee gate turns that arm's Not403 twin RED. + Signer::Assignee => signed_request( + client, + probe.method.clone(), + base_url, + &probe.path, + probe.body.clone(), + &fixture.assignee, + ), + }; + let rb = if probe.json { + rb.header("content-type", "application/json") + } else { + rb + }; + rb.send().await.unwrap_or_else(|e| { + panic!( + "probe failed to send: {} [{} {}]: {e}", + probe.label, probe.method, probe.path + ) + }) +} + +/// Walk every deny-bearing route (U1 registry), expand each into its hostile +/// probe plus positive twin (U2), and drive them against a real node: the +/// hostile request must return the exact deny status and leak nothing, and the +/// twin must reach the handler (owner-gate: not 403; read-gate: 2xx). This is +/// the runtime discharge of INV-1/INV-2/INV-8 across the whole registry, not one +/// hand-written case at a time. +/// +/// Terminal anti-vacuous-green invariant: exactly one hostile probe per row is +/// driven and the count equals the registry size, so a row that silently +/// produced no probe fails here instead of the sweep passing by testing nothing. +#[sqlx::test] +async fn deny_bearing_registry_denies_hostile_and_admits_authorized(pool: sqlx::PgPool) { + let node = spawn_node(pool).await; + // A bounded timeout so a wedged route fails the suite rather than hanging it. + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(30)) + .build() + .unwrap(); + let fixture = Fixture::seed(&node).await; + + let rows = deny_bearing_routes(); + assert!( + !rows.is_empty(), + "deny-bearing route registry is empty — nothing to sweep" + ); + + // #195 (F1): every read-gate row is now probed by a signed non-reader too, so a + // dropped stranger probe must fail loudly rather than being absorbed into the + // total. Count the read-gate rows and the stranger hostiles separately. + let readgate_rows = rows + .iter() + .filter(|r| matches!(r.gate, GateClass::ReadGate)) + .count(); + + let mut hostile_driven = 0usize; + let mut readgate_stranger_driven = 0usize; + let mut twins_driven = 0usize; + + for row in rows { + let probes = probes_for(row, &fixture); + assert!( + !probes.is_empty(), + "row {} {} produced no probes", + row.method, + row.path + ); + for probe in &probes { + let ctx = format!("{} [{} {}]", probe.label, probe.method, probe.path); + let resp = send_probe(&client, &node.base_url, &fixture, probe).await; + let status = resp.status(); + match &probe.expect { + Expect::Deny(code) => { + hostile_driven += 1; + // A signed-stranger 404 is the read-gate signed-non-reader probe + // (owner-gate strangers are 403, signature hostiles are anon 401). + if probe.signer == Signer::Stranger && *code == 404 { + readgate_stranger_driven += 1; + } + assert_eq!( + status.as_u16(), + *code, + "hostile probe must deny with {code}, got {status}: {ctx}" + ); + // INV-8 shape guard: rechecks the status and that the body is + // neither an empty-200-as-success nor carrying withheld data. The + // probe's `withheld` tokens (the private repo's secret bytes and + // blob OID on read-gate probes) are passed through, so a 404 that + // spilled the private content fails here instead of being counted + // as a clean denial. + let tokens: Vec<&str> = probe.withheld.iter().map(String::as_str).collect(); + assert_denied(resp, *code, &tokens).await; + } + Expect::Not403 => { + twins_driven += 1; + assert_ne!( + status.as_u16(), + 403, + "owner-reachability twin must reach the handler (not 403): {ctx}" + ); + } + Expect::Ok2xx => { + twins_driven += 1; + assert!( + status.is_success(), + "read-reachability twin must be 2xx, got {status}: {ctx}" + ); + // A 2xx with an empty body is a denial rendered as success: + // the authorized read must actually return the resource. + let body = resp.bytes().await.unwrap_or_default(); + assert!( + !body.is_empty(), + "read-reachability twin returned an empty 2xx body (denial-as-success?): {ctx}" + ); + } + } + } + } + + // One hostile per row (a signed stranger for the 403 gates, including the + // #195 N2 status-gated bounty rows; anon for the signature and read-gate + // rows), plus the EXTRA signed-stranger 404 each read-gate row drives. + assert_eq!( + hostile_driven, + rows.len() + readgate_rows, + "expected one hostile per row ({} rows) plus one signed-stranger hostile per read-gate row ({readgate_rows}), drove {hostile_driven}", + rows.len() + ); + // #195 (F1): every read-gate row must be probed by a signed non-reader — a + // dropped stranger probe fails HERE rather than silently reducing coverage. + assert_eq!( + readgate_stranger_driven, readgate_rows, + "every read-gate row must drive a signed-non-reader 404 probe ({readgate_rows} rows), drove {readgate_stranger_driven}" + ); + assert!( + twins_driven >= 1, + "no positive twins were driven — the reachability proof is missing" + ); + + node.shutdown().await; +} + +// ── Additional INV-1 owner-gates over the real stack (fan-out of U6) ────────── + +/// The remaining high-blast-radius owner-gated mutations that previously had +/// only the source-level authz-table guard and no runtime deny test: branch +/// protection, webhook create/delete, and visibility removal. Each rejects a +/// validly-signed non-owner with 403, and the owner reaches the handler (not +/// 403), proving the 403 is the owner gate. All share `did_matches` as the root +/// gate (see the mutation check in the harness commit). +#[sqlx::test] +async fn additional_owner_gates_reject_non_owner(pool: sqlx::PgPool) { + let node = spawn_node(pool).await; + let client = bounded_client(); + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let stranger = Keypair::generate(); + + node.seed_repo(&owner_did, "gated-repo", true).await; + let base = format!("/api/v1/repos/{owner_did}/gated-repo"); + + // (method, path, body, needs-json-content-type) + let cases: Vec<(reqwest::Method, String, Vec, bool)> = vec![ + ( + reqwest::Method::POST, + format!("{base}/branches/main/protect"), + Vec::new(), + false, + ), + ( + reqwest::Method::DELETE, + format!("{base}/branches/main/protect"), + Vec::new(), + false, + ), + ( + reqwest::Method::POST, + format!("{base}/hooks"), + br#"{"url":"https://example.com/hook","events":["*"]}"#.to_vec(), + true, + ), + ( + reqwest::Method::DELETE, + format!("{base}/hooks/deadbeef"), + Vec::new(), + false, + ), + ( + reqwest::Method::DELETE, + format!("{base}/visibility"), + br#"{"path_glob":"/"}"#.to_vec(), + true, + ), + ]; + + let send = |m: reqwest::Method, path: String, body: Vec, json: bool, kp: &Keypair| { + let mut rb = signed_request(&client, m, &node.base_url, &path, body, kp); + if json { + rb = rb.header("content-type", "application/json"); + } + rb.send() + }; + + for (method, path, body, json) in cases { + // Non-owner: valid signature, wrong identity -> 403 owner gate, no leak. + let resp = send(method.clone(), path.clone(), body.clone(), json, &stranger) + .await + .unwrap_or_else(|e| panic!("{method} {path} sends: {e}")); + assert_denied(resp, 403, &[]).await; + + // Owner reaches the handler: NOT 403 (proves the 403 above was the gate, + // not an earlier layer or a 415/404 masquerade). + let resp = send(method.clone(), path.clone(), body, json, &owner) + .await + .unwrap_or_else(|e| panic!("{method} {path} owner sends: {e}")); + assert_ne!( + resp.status().as_u16(), + 403, + "owner must reach {method} {path} (got 403)" + ); + } + + node.shutdown().await; +} + +// ── Teardown: shutdown() joins the serve task and closes the pool ───────────── + +/// `shutdown()` must join the detached serve task and close the pool before +/// returning. RED condition: with drop-only teardown the pool clones held by +/// the still-running server are open when the test future returns, so +/// `#[sqlx::test]`'s `DROP DATABASE` cleanup fails (sqlx prints and ignores +/// it), leaking per-test databases and flaking under parallel execution. The +/// `observer` clone shares the pool's inner, so `is_closed()` here is exactly +/// the state sqlx cleanup will see. +#[sqlx::test] +async fn shutdown_joins_server_and_closes_pool(pool: sqlx::PgPool) { + let observer = pool.clone(); + + let node = spawn_node(pool).await; + let base_url = node.base_url.clone(); + let client = bounded_client(); + + // One real request so at least one server connection (and its keep-alive) + // has existed; graceful shutdown must close it, not wait on it. + client + .get(format!("{base_url}/health")) + .send() + .await + .expect("probe request sends"); + + node.shutdown().await; + + assert!( + observer.is_closed(), + "shutdown() must close the pool before the test returns, or sqlx's \ + DROP DATABASE cleanup races the server's open connections" + ); + // The serve task was joined, not merely signalled: a fresh request to the + // old address must fail to connect. + let after = client.get(format!("{base_url}/health")).send().await; + assert!( + after.is_err(), + "the server must be gone after shutdown(); got {:?} instead", + after.map(|r| r.status()) + ); +} + +// ── Teardown: Drop without shutdown() must still release the database ───────── + +/// A test that returns WITHOUT calling `shutdown()` (early return, forgotten +/// teardown) must not leave the detached serve task holding pool clones open +/// against sqlx's cleanup: `#[sqlx::test]` issues a plain `DROP DATABASE` (no +/// FORCE) the moment the test future returns, a still-connected session makes +/// the drop fail, and sqlx only prints the error, silently leaking the +/// per-test database. `TestNode`'s `Drop` tears the server down two redundant +/// ways: the graceful watch signal, and an abort of the remaining serve +/// handle (which drops the task's future, and the pool clones inside its +/// router/state, on a subsequent scheduler tick without depending on the +/// graceful chain of watcher/connection tasks completing). This test fails if +/// BOTH are broken (verified by neutering them in turn: with signal and abort +/// both removed the probe sees the leaked sessions for the full bound; either +/// mechanism alone drains them by the first probe), so it fences the Drop +/// teardown contract as a whole rather than either mechanism individually. +/// +/// Two guards against a vacuous pass: the node pool is built with ambient +/// reaping disabled (`idle_timeout`/`max_lifetime` `None`; sqlx's default test +/// pool options reap idle connections after ~1s, which would release the +/// database by itself and mask a broken teardown), and the probe runs over a +/// standalone connection rather than a clone of the node pool, so observing +/// does not itself keep the pool alive. +#[sqlx::test] +async fn drop_without_shutdown_unblocks_database( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, +) { + use sqlx::{ConnectOptions as _, Connection as _}; + + let pool = pool_opts + .idle_timeout(None) + .max_lifetime(None) + .connect_with(connect_opts.clone()) + .await + .expect("node pool connects"); + + // Inner scope so the node (and the client's keep-alive connection) drop + // exactly as they would when a test body returns, with no shutdown() call. + { + let node = spawn_node(pool).await; + let client = bounded_client(); + let resp = client + .get(format!("{}/health", node.base_url)) + .send() + .await + .expect("probe request sends"); + assert!( + resp.status().is_success(), + "the server must be live before the drop" + ); + } + + // Standalone observer: count OTHER sessions connected to the per-test + // database. Zero means the only session left is the probe itself, i.e. the + // database is droppable the instant the probe disconnects, which is + // exactly the state sqlx's cleanup needs. Bounded poll: the abort takes + // effect on a scheduler tick, not synchronously in Drop. + let mut probe = connect_opts + .connect() + .await + .expect("standalone probe connects"); + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(5); + loop { + let others: i64 = sqlx::query_scalar( + "SELECT count(*) FROM pg_stat_activity \ + WHERE datname = current_database() AND pid <> pg_backend_pid()", + ) + .fetch_one(&mut probe) + .await + .expect("session count query runs"); + if others == 0 { + break; + } + assert!( + tokio::time::Instant::now() < deadline, + "dropped-without-shutdown node still holds {others} session(s) \ + against the per-test database after 5s; sqlx's DROP DATABASE \ + cleanup would fail and leak the database" + ); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + probe.close().await.expect("probe connection closes"); +} + +/// Isolates the `handle.abort()` half of `TestNode`'s `Drop` teardown. The +/// existing `drop_without_shutdown_unblocks_database` fences both redundant +/// mechanisms together and stays GREEN if only the abort is removed (the +/// graceful watch signal alone drains the serve task). This test suppresses the +/// graceful signal (`force_abort_only_teardown`) so ONLY the abort runs, making +/// the abort line load-bearing: with the abort removed from `Drop` neither +/// mechanism runs, the serve task keeps its pool clones open, and the probe sees +/// the leaked sessions for the full bound (RED); with the abort present the +/// aborted task's future is dropped on a scheduler tick and the sessions drain +/// (GREEN). Same vacuous-pass guards as the sibling test: node pool built with +/// reaping disabled, and a standalone observer connection. +#[sqlx::test] +async fn drop_with_broken_graceful_chain_still_unblocks_via_abort( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, +) { + use sqlx::{ConnectOptions as _, Connection as _}; + + let pool = pool_opts + .idle_timeout(None) + .max_lifetime(None) + .connect_with(connect_opts.clone()) + .await + .expect("node pool connects"); + + // Inner scope so the node drops as a test body's would, with no shutdown(). + { + let mut node = spawn_node(pool).await; + // Break the graceful chain: Drop will not send the watch signal, so + // only handle.abort() can tear the serve task down. + node.force_abort_only_teardown(); + let client = bounded_client(); + let resp = client + .get(format!("{}/health", node.base_url)) + .send() + .await + .expect("probe request sends"); + assert!( + resp.status().is_success(), + "the server must be live before the drop" + ); + } + + // Standalone observer: with only the abort running, the serve task's future + // (and its pool clones) must still be dropped, leaving zero other sessions. + let mut probe = connect_opts + .connect() + .await + .expect("standalone probe connects"); + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(5); + loop { + let others: i64 = sqlx::query_scalar( + "SELECT count(*) FROM pg_stat_activity \ + WHERE datname = current_database() AND pid <> pg_backend_pid()", + ) + .fetch_one(&mut probe) + .await + .expect("session count query runs"); + if others == 0 { + break; + } + assert!( + tokio::time::Instant::now() < deadline, + "abort-only teardown still holds {others} session(s) against the \ + per-test database after 5s; handle.abort() is not releasing the \ + serve task's pool clones and sqlx's DROP DATABASE cleanup would fail" + ); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + probe.close().await.expect("probe connection closes"); +} + +// ── U4: completeness cross-check (no DB) — the registry cannot silently drift ── +// +// The runtime sweep (U3) only proves the routes it is HANDED. This guard, a pure +// source scrape, keeps that hand-off honest against `server.rs`: no registry row +// points at a route that no longer mounts (stale row), and no owner-gated mount +// escapes the registry (orphan). It complements — does not duplicate — the +// in-crate `authz_guard` egress guard (`src/api/mod.rs`), which proves every +// repo-scoped API handler is *gated*; this proves the deny-bearing ones are +// *driven*, and it reaches the non-API git mounts `authz_guard` never sees. +mod completeness { + use std::collections::HashSet; + + use super::deny_bearing_routes; + use crate::support::routes::{GateClass, Principal}; + + /// The CLOSED set of structural reasons a read-gate handler may sit out of the + /// runtime 404 sweep (#195, U4/R4). This replaces the free-text `reason` + /// strings the old allowlist carried: a "deferred / needs seeding" prose excuse + /// no longer type-checks. Every `READ_GATE_NOT_DRIVEN` entry must name one of + /// these variants, so a reviewer sees a STRUCTURAL class (why the read is not a + /// drivable 404-deny GET), never a soft deferral that hides an undriven private + /// read. A drivable GET (the nine U3 reads) cannot be parked here because none + /// of these classes fits it — and the driven-not-relabeled assertion below + /// makes that a hard failure rather than a judgment call. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + enum NotDrivenReason { + /// A create/write path that read-gates the caller before mutating, not a + /// 404-deny GET. Its owner/author behaviour is covered by the mutation + /// authz guard in `src/api/mod.rs`, not this GET-read sweep. + ReadGatingMutation, + /// A git smart-HTTP read (`info/refs`, `git-upload-pack`): its + /// withheld-subtree 404/exclusion is driven by the U7/U8 real-clone cases, + /// not the API GET sweep. + GitSmartHttpRead, + /// A content-addressed read (`/ipfs/{cid}`): its 404 deny + no-leak is + /// driven by the U5b/U8 anon_ipfs/clone cases in this file. + ContentAddressedRead, + /// A global list-FILTER: returns 200 with unreadable rows removed, never a + /// 404, so it is not a deny-bearing read at all. + GlobalListFilter, + /// The owner-2xx twin needs a live external dependency the harness lacks: + /// `get_encrypted_blob` reads through `ipfs_pin::cat` with no harness stub, + /// so it cannot be driven as a 2xx twin here. An honest exclusion, not a + /// deferral (#195, KTD-5). + ExternalDependencyUnavailable, + } + + /// Read-gate handlers NOT driven as a `ReadGate` row, each carrying a CLOSED + /// structural reason ([`NotDrivenReason`]) — never free-text. This is the + /// enforced form of the prose that used to live in routes.rs; a handler leaving + /// the sweep must be moved here under a structural class, never with a soft + /// "deferred / needs seeding" excuse (which no longer type-checks). + /// + /// Single source of truth: both the marker-scan guard + /// (`every_read_gate_handler_is_driven_or_explicitly_allowlisted`) and the + /// route-table cross-check (`mounted_repo_gets_are_driven_excused_or_declared_public`) + /// read this, so the two cannot drift. + /// + /// get_issue / get_pr / get_pr_diff / list_issue_comments / list_reviews / + /// list_comments / get_cert / get_bounty / get_tree are DRIVEN as real ReadGate + /// rows in deny_bearing_routes() (#195, U3); the driven-not-relabeled assertion + /// bars any of them from being quietly re-parked here. + const READ_GATE_NOT_DRIVEN: &[(&str, NotDrivenReason)] = &[ + // Owner-2xx twin needs a live IPFS backend (`ipfs_pin::cat`) the harness has + // no stub for — an honest external-dependency exclusion (#195, KTD-5). + ( + "get_encrypted_blob", + NotDrivenReason::ExternalDependencyUnavailable, + ), + // Read-gates but is a mutation, not a 404-deny GET. + ("create_review", NotDrivenReason::ReadGatingMutation), + ("create_comment", NotDrivenReason::ReadGatingMutation), + ("create_pr", NotDrivenReason::ReadGatingMutation), + ("create_issue", NotDrivenReason::ReadGatingMutation), + ("create_issue_comment", NotDrivenReason::ReadGatingMutation), + ("create_bounty", NotDrivenReason::ReadGatingMutation), + ("claim_bounty", NotDrivenReason::ReadGatingMutation), + ("fork_repo", NotDrivenReason::ReadGatingMutation), + ("star_repo", NotDrivenReason::ReadGatingMutation), + ("unstar_repo", NotDrivenReason::ReadGatingMutation), + // Content-addressed read: driven by the U5b/U8 anon_ipfs/clone cases. + ("get_by_cid", NotDrivenReason::ContentAddressedRead), + // Git smart-HTTP reads: driven by the U7/U8 real-clone cases. + ("git_info_refs", NotDrivenReason::GitSmartHttpRead), + ("git_upload_pack", NotDrivenReason::GitSmartHttpRead), + // Global list-FILTER: 200 with unreadable rows removed, never a 404. + ("list_all_bounties", NotDrivenReason::GlobalListFilter), + ]; + + /// Every `.route("", ()…)` mount in `src`, as + /// (METHOD, path). Multiline-aware: most mounts put the path on the line after + /// `.route(`, so a per-line scan false-greens. Walks balanced parens from each + /// `.route(` so chained and multi-method (`put().delete().get()`) mounts are + /// all captured. + fn scrape_mounts(src: &str) -> Vec<(String, String)> { + let bytes = src.as_bytes(); + let mut out = Vec::new(); + let mut i = 0; + while let Some(rel) = src[i..].find(".route(") { + let open = i + rel + ".route(".len(); + let mut depth = 1i32; + let mut j = open; + while j < src.len() && depth > 0 { + match bytes[j] { + b'(' => depth += 1, + b')' => depth -= 1, + _ => {} + } + j += 1; + } + let call = &src[open..j.saturating_sub(1)]; + i = j; + // The path is the first string literal in the call. + let Some(qs) = call.find('"') else { continue }; + let Some(qe) = call[qs + 1..].find('"') else { + continue; + }; + let path = &call[qs + 1..qs + 1 + qe]; + for m in ["get", "post", "put", "delete", "patch"] { + let needle = format!("{m}("); + let mut k = 0; + while let Some(mrel) = call[k..].find(&needle) { + let at = k + mrel; + // Reject a match that is the tail of a longer ident (e.g. the + // `get(` inside `budget(`): the char before must be a boundary. + let boundary = call[..at] + .chars() + .last() + .map(|c| !(c.is_alphanumeric() || c == '_')) + .unwrap_or(true); + if boundary { + out.push((m.to_uppercase(), path.to_string())); + } + k = at + needle.len(); + } + } + } + out + } + + /// Names of `fn`s in `src` whose body contains any `marker`. The body is the + /// slice from a fn's `(` to the next top-level fn declaration — the same + /// boundary set `authz_guard::fn_body` uses — so a marker can't leak across + /// into the next handler. + fn handlers_with_marker(src: &str, markers: &[&str]) -> HashSet { + let decls = [ + "\npub async fn ", + "\npub(crate) async fn ", + "\nasync fn ", + "\npub(crate) fn ", + "\npub fn ", + "\nfn ", + ]; + // Ordered start offsets of every fn declaration. + let mut starts: Vec = Vec::new(); + for d in decls { + let mut k = 0; + while let Some(r) = src[k..].find(d) { + starts.push(k + r + 1); // +1: skip the leading '\n' + k = k + r + d.len(); + } + } + starts.sort_unstable(); + let mut out = HashSet::new(); + for (idx, &s) in starts.iter().enumerate() { + let end = starts.get(idx + 1).copied().unwrap_or(src.len()); + let seg = &src[s..end]; + // The name is the ident after this decl's `fn ` (handles `pub(crate)`, + // whose own `(` would otherwise be mistaken for the arg list). + let Some(fnpos) = seg.find("fn ") else { + continue; + }; + let name: String = seg[fnpos + 3..] + .chars() + .take_while(|c| c.is_alphanumeric() || *c == '_') + .collect(); + if name.is_empty() { + continue; + } + if markers.iter().any(|m| seg.contains(m)) { + out.insert(name); + } + } + out + } + + /// Like [`handlers_with_marker`] but selects handlers whose body satisfies a + /// PREDICATE rather than a substring — used for the inline owner-gate + /// did_matches idiom (F3), whose "first arg is the caller, second is + /// owner_did" shape a plain substring cannot express. Same fn-segmentation as + /// `handlers_with_marker` so a match cannot leak across into the next handler. + fn handlers_matching(src: &str, pred: fn(&str) -> bool) -> HashSet { + let decls = [ + "\npub async fn ", + "\npub(crate) async fn ", + "\nasync fn ", + "\npub(crate) fn ", + "\npub fn ", + "\nfn ", + ]; + let mut starts: Vec = Vec::new(); + for d in decls { + let mut k = 0; + while let Some(r) = src[k..].find(d) { + starts.push(k + r + 1); + k = k + r + d.len(); + } + } + starts.sort_unstable(); + let mut out = HashSet::new(); + for (idx, &s) in starts.iter().enumerate() { + let end = starts.get(idx + 1).copied().unwrap_or(src.len()); + let seg = &src[s..end]; + let Some(fnpos) = seg.find("fn ") else { + continue; + }; + let name: String = seg[fnpos + 3..] + .chars() + .take_while(|c| c.is_alphanumeric() || *c == '_') + .collect(); + if name.is_empty() { + continue; + } + if pred(seg) { + out.insert(name); + } + } + out + } + + fn short_handler(h: &str) -> &str { + h.rsplit("::").next().unwrap_or(h) + } + + /// Like [`handlers_with_marker`] but ignores markers that appear only inside a + /// full-line `//` comment. The owner-gate markers never show up in prose, but + /// the read-gate markers (`authorize_repo_read(` / `visibility_check(`) do: + /// several docstrings and a test comment name them (repos.rs's + /// `owner_push_rejection` / `dedupe_canonical_repos` regions, the fork + /// docstring), and a raw `contains` would misattribute those to the nearest + /// preceding fn and force a phantom non-handler onto the read-gate allowlist. + /// Mirrors `authz_guard::fn_body`'s comment-stripping so the scan sees code, + /// not prose. + fn handlers_with_code_marker(src: &str, markers: &[&str]) -> HashSet { + let stripped: String = src + .lines() + .map(|l| { + if l.trim_start().starts_with("//") { + "" + } else { + l + } + }) + .collect::>() + .join("\n"); + handlers_with_marker(&stripped, markers) + } + + /// True when `body` contains an INLINE owner-gate `did_matches` call: one + /// whose FIRST arg is the caller (`caller` / `&caller` / `&auth.0`) AND whose + /// SECOND arg is `&record.owner_did`. That two-arg shape is the discriminator + /// (F3): `require_owner`/`require_repo_owner` are the named markers, but + /// protect_branch / unprotect_branch owner-gate with this raw idiom instead, so + /// a plain marker misses them and a future owner-only handler using the same + /// pattern could slip past the registry while U4 stayed green. + /// + /// It must NOT fire on the signer-self / author forms that share the helper: + /// register_replica's `did_matches(replica_did, &record.owner_did)` (first arg + /// is the replica, not the caller — signer-self by design), close_pr's + /// `did_matches(&auth.0, &pr.author_did)`, and the bounty/task self forms + /// (`&bounty.creator_did`, delegator/assignee dids). Whitespace and newlines + /// inside the arg list are tolerated so a reformatted call still matches. + fn has_owner_did_matches(body: &str) -> bool { + let bytes = body.as_bytes(); + let mut i = 0; + while let Some(rel) = body[i..].find("did_matches(") { + let open = i + rel + "did_matches(".len(); + // Walk to the matching close paren, splitting the top-level args on the + // first depth-0 comma (there are exactly two args). + let mut depth = 1i32; + let mut comma: Option = None; + let mut j = open; + while j < body.len() && depth > 0 { + match bytes[j] { + b'(' => depth += 1, + b')' => depth -= 1, + b',' if depth == 1 && comma.is_none() => comma = Some(j), + _ => {} + } + j += 1; + } + i = j; + let close = j.saturating_sub(1); + let Some(comma) = comma else { continue }; + let arg1 = body[open..comma].split_whitespace().collect::(); + let arg2 = body[comma + 1..close] + .split_whitespace() + .collect::(); + let caller_first = arg1 == "caller" || arg1 == "&caller" || arg1 == "&auth.0"; + let owner_second = arg2 == "&record.owner_did"; + if caller_first && owner_second { + return true; + } + } + false + } + + /// The CLOSED set of caller-self REST gates deliberately NOT driven as a + /// registry row. INTENTIONALLY EMPTY (#195, F3/R4): every caller-self + /// `did_matches(caller, X)` gate in `src/api` is driven by a real-node + /// hostile+control probe (a `SignerSelfGate` field-binding row or a + /// `MultiPrincipalGate` assignee-arm row), so no caller-self REST gate is + /// un-drivable. A new classifier-marked handler that is not a driven row must + /// fail `signer_self_gates_are_all_driven`; it is not parked here. + const SIGNER_SELF_NOT_DRIVEN: &[&str] = &[]; + + /// True when `body` contains a CALLER-SELF `did_matches` call: one whose FIRST + /// arg is the caller (`caller` / `&caller` / `&auth.0`) AND whose SECOND arg is + /// NOT `&record.owner_did`. Sibling to [`has_owner_did_matches`] (the owner form + /// is guarded there); this is the field-binding / actor idiom the task and + /// register handlers use: create_task (`&body.delegator_did`), claim_task + /// (`&body.assignee_did`), complete_task / fail_task (the stored `assignee_did`, + /// spelled across lines), register (`agent_did.as_str()`). The arg1 set matches + /// its owner sibling's (NOT just `&auth.0`), so a `let caller = &auth.0; + /// did_matches(caller, X)` gate cannot escape. The paren-walk is multi-line-robust + /// because complete_task / fail_task spell the call across lines. + fn has_signer_self_did_matches(body: &str) -> bool { + let bytes = body.as_bytes(); + let mut i = 0; + while let Some(rel) = body[i..].find("did_matches(") { + let open = i + rel + "did_matches(".len(); + let mut depth = 1i32; + let mut comma: Option = None; + let mut j = open; + while j < body.len() && depth > 0 { + match bytes[j] { + b'(' => depth += 1, + b')' => depth -= 1, + b',' if depth == 1 && comma.is_none() => comma = Some(j), + _ => {} + } + j += 1; + } + i = j; + let close = j.saturating_sub(1); + let Some(comma) = comma else { continue }; + let arg1 = body[open..comma].split_whitespace().collect::(); + let arg2 = body[comma + 1..close] + .split_whitespace() + .collect::(); + let caller_first = arg1 == "caller" || arg1 == "&caller" || arg1 == "&auth.0"; + let owner_second = arg2 == "&record.owner_did"; + if caller_first && !owner_second { + return true; + } + } + false + } + + #[test] + fn deny_registry_is_not_stale_and_owner_gates_are_all_driven() { + let server = include_str!("../src/server.rs"); + let mounts: HashSet<(String, String)> = scrape_mounts(server).into_iter().collect(); + + // Scraper-integrity floor: if the parser silently stopped finding mounts, + // the anti-stale check below would pass vacuously. The tree has ~95 mounts. + assert!( + mounts.len() >= 90, + "mount scrape found only {} routes — the parser likely broke (floor 90)", + mounts.len() + ); + + // ANTI-STALE: every deny-bearing row must still point at a live mount, so a + // renamed/removed route can't leave a row that the sweep drives against a + // 404 and calls a passing deny. + for row in deny_bearing_routes() { + assert!( + mounts.contains(&(row.method.to_string(), row.path.to_string())), + "stale registry row: {} {} is not mounted in server.rs (renamed or removed?)", + row.method, + row.path + ); + } + + // ORPHAN GUARD: every handler that owner-gates must be a registry owner-gate + // row, so a newly added owner-gated mutation cannot escape the runtime + // sweep. The api dir is read at test time (like authz_guard's completeness + // scan) so a brand-new module is covered too. Two owner-gate shapes are + // recognized: + // (1) the named markers `require_repo_owner(` / `require_owner(`, and + // (2) the INLINE `did_matches(caller, &record.owner_did)` idiom (F3) that + // protect_branch / unprotect_branch use instead of a named helper. + // Only the SECOND-arg-is-owner_did form of did_matches is treated as an + // owner gate; the signer-self / author forms (register_replica's + // did_matches(replica_did, …), close_pr's did_matches(&auth.0, &author_did)) + // are deliberately NOT matched, so they don't false-orphan. + // A handler satisfies the owner marker if it is a plain OwnerGate row OR a + // MultiPrincipalGate row that declares the Owner arm (#195, F1: close_pr / + // close_issue owner-gate via require_repo_owner but are owner-OR-author, so + // they register as MultiPrincipalGate — their owner arm is still driven). + let registry_owner_handlers: HashSet<&str> = deny_bearing_routes() + .iter() + .filter(|r| { + r.gate == GateClass::OwnerGate + || (r.gate == GateClass::MultiPrincipalGate + && r.principals.contains(&Principal::Owner)) + }) + .map(|r| short_handler(r.handler)) + .collect(); + + let api_dir = concat!(env!("CARGO_MANIFEST_DIR"), "/src/api"); + let mut owner_marked: HashSet = HashSet::new(); + for entry in std::fs::read_dir(api_dir).expect("read api dir") { + let path = entry.expect("dir entry").path(); + if path.extension().is_some_and(|e| e == "rs") { + let src = std::fs::read_to_string(&path).expect("read api file"); + owner_marked.extend(handlers_with_marker( + &src, + &["require_repo_owner(", "require_owner("], + )); + // Fold in the inline did_matches-owner idiom (F3). + owner_marked.extend(handlers_matching(&src, has_owner_did_matches)); + } + } + // The gate helpers' own definitions carry the owner-gate idiom (the named + // marker string, and visibility::require_owner's body is itself a + // `did_matches(caller, &record.owner_did)` check); they are helpers, not + // mounted handlers, so drop them. + owner_marked.remove("require_owner"); + owner_marked.remove("require_repo_owner"); + // git_receive_pack also carries the inline did_matches-owner idiom, but for + // a SECONDARY gate: its branch-protection push check (repos.rs), not a + // whole-handler owner gate. Its 403 is DRIVEN by the dedicated + // `signed_stranger_protected_branch_push_is_forbidden` test (#195, F3) — a + // signed non-owner push to a protected branch, with an owner control — so + // inverting the gate goes RED there. It is therefore not an owner-gate + // orphan (the OwnerGate registry rows can't carry the protected-branch + // substrate); drop it so the owner-orphan assert does not misflag a route + // whose owner gate is already driven. + owner_marked.remove("git_receive_pack"); + // Non-vacuous floor: if the marker scan silently found nothing (a parser + // regression), the orphan loop below would pass by checking zero handlers. + // The tree has 10 owner-marker handlers today; 6 trips only on a real break. + assert!( + owner_marked.len() >= 6, + "owner-gate marker scan found only {} handlers — the scan likely broke", + owner_marked.len() + ); + + for h in &owner_marked { + assert!( + registry_owner_handlers.contains(h.as_str()), + "handler `{h}` carries an owner-gate marker but is not an owner-gate \ + row in deny_bearing_routes() — add it so the runtime sweep drives its 403" + ); + } + } + + /// CALLER-SELF ORPHAN GUARD (#195, F3/R4, INV-21b). Sibling to the owner-gate + /// and read-gate orphan guards above: every handler carrying a caller-self + /// `did_matches(caller, X)` gate (X != `&record.owner_did`) must be a DRIVEN + /// registry row (ANY gate class: a `SignerSelfGate` field-binding row or a + /// `MultiPrincipalGate` assignee-arm row) OR listed in the (empty) + /// `SIGNER_SELF_NOT_DRIVEN`. Derived from a SOURCE scan of the `src/api` `.rs` + /// files at test time (KTD-2): a caller-self gate spelled with the inline + /// `did_matches(caller|&caller|&auth.0, X)` idiom in an existing OR new api file + /// is caught and forced to be a driven row. It does NOT catch two shapes (the + /// same limits the owner/read sibling guards have): a check hidden behind a + /// helper (the owner side tracks `require_owner` / `require_repo_owner` as markers + /// for exactly this reason -- add a marker if a caller-self helper is introduced), + /// and a gate in a new SUBDIRECTORY module (the scan is flat, not recursive). The + /// parallel GraphQL mutation gates (`src/graphql/mutation.rs`) are out of this + /// scan's scope, fenced separately (#219). + #[test] + fn signer_self_gates_are_all_driven() { + // A caller-self gate may be driven as a SignerSelfGate row (body-field bind) + // OR a MultiPrincipalGate row (the complete/fail assignee actor gate), so the + // required set is EVERY registry handler regardless of class. + let registry_handlers: HashSet<&str> = deny_bearing_routes() + .iter() + .map(|r| short_handler(r.handler)) + .collect(); + + let api_dir = concat!(env!("CARGO_MANIFEST_DIR"), "/src/api"); + let mut signer_self_marked: HashSet = HashSet::new(); + for entry in std::fs::read_dir(api_dir).expect("read api dir") { + let path = entry.expect("dir entry").path(); + if path.extension().is_some_and(|e| e == "rs") { + let src = std::fs::read_to_string(&path).expect("read api file"); + signer_self_marked.extend(handlers_matching(&src, has_signer_self_did_matches)); + } + } + // Drop the gate helpers' own definitions (mirror of the owner scan dropping + // require_owner / require_repo_owner): none of these is a mounted handler. + signer_self_marked.remove("did_matches"); + signer_self_marked.remove("require_owner"); + signer_self_marked.remove("require_repo_owner"); + + // Non-vacuous floor PINNED to the exact caller-self handler count (close_pr, + // close_issue, dispute/submit/approve/cancel_bounty, create/claim/complete/ + // fail_task, register = 11). Pinned, not loose, so a PARTIAL classifier + // regression (dropping even one) trips here rather than being absorbed by the + // runtime sweep; bump it when the caller-self surface legitimately changes. + assert!( + signer_self_marked.len() >= 11, + "caller-self marker scan found only {} handlers (expected 11); the scan broke or the caller-self surface changed without bumping this floor", + signer_self_marked.len() + ); + + for h in &signer_self_marked { + assert!( + registry_handlers.contains(h.as_str()) + || SIGNER_SELF_NOT_DRIVEN.contains(&h.as_str()), + "handler `{h}` carries a caller-self did_matches gate but is neither a \ + driven registry row nor in SIGNER_SELF_NOT_DRIVEN; drive its 403" + ); + } + + // Staleness: every SIGNER_SELF_NOT_DRIVEN entry must still be a real + // caller-self handler. Vacuous while the list is empty, but it keeps a future + // exemption honest (the mirror of the owner/read staleness checks). + for name in SIGNER_SELF_NOT_DRIVEN { + assert!( + signer_self_marked.contains(*name), + "SIGNER_SELF_NOT_DRIVEN lists `{name}`, which no longer carries a \ + caller-self did_matches gate; update the list" + ); + } + } + + /// READ-GATE ORPHAN GUARD (F1). The owner-gate orphan guard above stops an + /// owner-gated mount from escaping the sweep; this is its read-gate twin. Every + /// handler carrying a read-gate marker (`authorize_repo_read(` / + /// `visibility_check(` — the two markers repos.rs/issues.rs/pulls.rs/ + /// bounties.rs/certs.rs/labels.rs/changelog.rs/events.rs/encrypted.rs/stars.rs/ + /// visibility.rs all use, verified there is no third) must be EITHER a + /// `ReadGate` row in `deny_bearing_routes()` (driven for its 404) OR an explicit + /// entry in `READ_GATE_NOT_DRIVEN` with a reason. That enumerated allowlist + /// REPLACES the old free-text "DEFERRED / EXCLUDED" prose in routes.rs: it is + /// now enforced code, so removing or bypassing the read gate on any of these + /// endpoints (or adding a brand-new read-gated handler) trips this instead of + /// leaving the real-node sweep silently green. The api dir is read at test time, + /// like the owner scan, so a new module is covered too. + #[test] + fn every_read_gate_handler_is_driven_or_explicitly_allowlisted() { + // The allowlist is the module-level `READ_GATE_NOT_DRIVEN` (single source of + // truth, shared with the route-table cross-check). Its entries carry a closed + // `NotDrivenReason`, so a soft "deferred" excuse cannot be added. + + let driven_read_handlers: HashSet<&str> = deny_bearing_routes() + .iter() + .filter(|r| r.gate == GateClass::ReadGate) + .map(|r| short_handler(r.handler)) + .collect(); + let allowlisted: HashSet<&str> = READ_GATE_NOT_DRIVEN.iter().map(|(h, _)| *h).collect(); + + let api_dir = concat!(env!("CARGO_MANIFEST_DIR"), "/src/api"); + let mut read_marked: HashSet = HashSet::new(); + for entry in std::fs::read_dir(api_dir).expect("read api dir") { + let path = entry.expect("dir entry").path(); + // Skip api/mod.rs: it holds the gate HELPERS themselves + // (`authorize_repo_read`, `visibility_check`, `require_repo_owner`, …), + // not mounted route handlers — every mounted read handler lives in a + // sibling module (repos.rs, issues.rs, …). Scanning mod.rs would attach + // the marker to helper definitions and force phantom allowlist entries. + if path.file_name().is_some_and(|f| f == "mod.rs") { + continue; + } + if path.extension().is_some_and(|e| e == "rs") { + let src = std::fs::read_to_string(&path).expect("read api file"); + // Comment-stripped scan: the read-gate markers appear in several + // docstrings, and a raw scan would allowlist phantom non-handlers. + read_marked.extend(handlers_with_code_marker( + &src, + &["authorize_repo_read(", "visibility_check("], + )); + } + } + // The read-gate helper `visibility_check` is re-declared/aliased in + // visibility.rs's own use-path; drop the helper name if the scan attaches + // the marker to it (it is a helper, not a mounted handler) — the mirror of + // the owner scan dropping require_owner / require_repo_owner. + read_marked.remove("authorize_repo_read"); + read_marked.remove("visibility_check"); + + // Non-vacuous floor: the tree has 41 read-gate-marked handlers today. If the + // marker scan silently found far fewer, the orphan loop below would pass by + // checking almost nothing; 30 trips only on a real parser break. + assert!( + read_marked.len() >= 30, + "read-gate marker scan found only {} handlers — the scan likely broke (floor 30)", + read_marked.len() + ); + + for h in &read_marked { + let name = h.as_str(); + assert!( + driven_read_handlers.contains(name) || allowlisted.contains(name), + "handler `{name}` carries a read-gate marker but is neither a ReadGate \ + row in deny_bearing_routes() nor in READ_GATE_NOT_DRIVEN — drive its \ + 404 or add it to the allowlist with a reason" + ); + } + + // Staleness: every allowlist entry must still be a real read-gate handler, + // so a rename/removal can't leave a dead exemption that masks a future gap. + for (name, _reason) in READ_GATE_NOT_DRIVEN { + assert!( + read_marked.contains(*name), + "READ_GATE_NOT_DRIVEN lists `{name}`, which no longer carries a \ + read-gate marker (renamed, removed, or gate dropped?) — update the list" + ); + } + + // DRIVEN-NOT-RELABELED (#195, U4/R5). The nine reads U3 drove must each be a + // driven `ReadGate` row AND absent from `READ_GATE_NOT_DRIVEN`, so none can + // be quietly re-parked under a mislabeled structural reason. A structural + // class (mutation / git / content-addressed / list-filter / external-dep) + // does not fit any of these nine, but a reviewer could still mistype one in; + // this makes that a hard failure rather than a judgment call. Together with + // the closed `NotDrivenReason` enum, a drivable private read cannot leave the + // sweep: either the class does not compile, or this assertion fires. + const U3_DRIVEN_READS: &[&str] = &[ + "get_issue", + "get_pr", + "get_pr_diff", + "list_issue_comments", + "list_reviews", + "list_comments", + "get_cert", + "get_bounty", + "get_tree", + ]; + for name in U3_DRIVEN_READS { + assert!( + driven_read_handlers.contains(name), + "U3 drove `{name}` as a private read, but it is no longer a ReadGate \ + row in deny_bearing_routes() — a driven read was dropped or renamed" + ); + assert!( + !allowlisted.contains(name), + "`{name}` is a U3-driven private read but was re-parked in \ + READ_GATE_NOT_DRIVEN — a drivable 404-deny GET may not be excused \ + under a structural reason; drive it, do not relabel it" + ); + } + } + + /// The repo-scoped GET mounts the [`scrape_mounts`] route-table walk does NOT + /// treat as read-gated — genuinely public repo listings that fetch the repo and + /// return metadata WITHOUT an `authorize_repo_read` / `visibility_check` call + /// (verified in their handlers). They carry no read gate, so the cross-check + /// below must not demand they be driven or structurally excused. Keyed by path + /// (the unit `scrape_mounts` yields) so a new gate added to one of these later + /// still surfaces via the marker scan; this list only says "no gate today". + /// + /// Currently EMPTY: `/hooks`, `/branches/protected`, and `/replicas` were listed + /// here as ungated public GETs, but #113 added an `authorize_repo_read` gate to + /// all three. They are now DRIVEN as real `ReadGate` rows in + /// `deny_bearing_routes()` (list_webhooks also as an OwnerGate row for its owner + /// layer), so they are classified via the driven-path set, not this list. The + /// mechanism is kept (not deleted) so the next genuinely-public repo-GET has a + /// home — the marker scan is the source of truth that keeps it honest. + const PUBLIC_REPO_GETS: &[&str] = &[]; + + /// True for a mounted path that is scoped to a single repo — the API form + /// `/api/v1/repos/{owner}/{repo}/…` and the bare git form `/{owner}/{repo}/…`. + /// These are the paths whose GET, if read-gated, must be driven or excused; a + /// collection/global path (`/api/v1/repos`, `/api/v1/bounties`, `/api/v1/agents` + /// …) is out of scope for the repo-read cross-check. + fn is_repo_scoped_path(path: &str) -> bool { + if let Some(rest) = path.strip_prefix("/api/v1/repos/") { + // `.../{owner}/{repo}/…` — at least owner + repo + a trailing segment, + // and NOT the `/api/v1/repos/federated` or bare `/api/v1/repos` forms + // (those have no `{owner}` placeholder). + return rest.starts_with("{owner}/{repo}/"); + } + // Bare git form: `/{owner}/{repo}/info/refs` etc. It is repo-scoped when the + // first two segments are the owner/repo placeholders. + let segs: Vec<&str> = path.trim_start_matches('/').split('/').collect(); + segs.len() >= 3 && segs[0] == "{owner}" && segs[1] == "{repo}" + } + + /// Extract the handler name from a single `.route("path", …get(handler)…)` mount + /// body: the last `::`-tail ident inside the FIRST routing-method call for the + /// requested HTTP method. Returns `None` if the method is not mounted on this + /// route. Reuses the same balanced-paren mount slice `scrape_mounts` walks. + fn handler_for_mount(call: &str, method: &str) -> Option { + let needle = format!("{}(", method.to_lowercase()); + let mut k = 0; + while let Some(rel) = call[k..].find(&needle) { + let at = k + rel; + let boundary = call[..at] + .chars() + .last() + .map(|c| !(c.is_alphanumeric() || c == '_')) + .unwrap_or(true); + if boundary { + // The handler is the first path-expression after `get(`; take up to + // the closing `)` and keep the final `::`-segment ident. + let inner_start = at + needle.len(); + let inner = &call[inner_start..]; + let end = inner.find([')', ',']).unwrap_or(inner.len()); + let expr = inner[..end].trim(); + let name: String = expr + .rsplit("::") + .next() + .unwrap_or(expr) + .chars() + .take_while(|c| c.is_alphanumeric() || *c == '_') + .collect(); + if !name.is_empty() { + return Some(name); + } + } + k = at + needle.len(); + } + None + } + + /// Like [`scrape_mounts`] but also captures the handler name, so a mounted + /// (METHOD, path) can be resolved to its handler for the read-gate cross-check. + /// Same balanced-paren walk as `scrape_mounts`. + fn scrape_mounts_with_handler(src: &str) -> Vec<(String, String, String)> { + let bytes = src.as_bytes(); + let mut out = Vec::new(); + let mut i = 0; + while let Some(rel) = src[i..].find(".route(") { + let open = i + rel + ".route(".len(); + let mut depth = 1i32; + let mut j = open; + while j < src.len() && depth > 0 { + match bytes[j] { + b'(' => depth += 1, + b')' => depth -= 1, + _ => {} + } + j += 1; + } + let call = &src[open..j.saturating_sub(1)]; + i = j; + let Some(qs) = call.find('"') else { continue }; + let Some(qe) = call[qs + 1..].find('"') else { + continue; + }; + let path = &call[qs + 1..qs + 1 + qe]; + for m in ["get", "post", "put", "delete", "patch"] { + let needle = format!("{m}("); + let mut k = 0; + while let Some(mrel) = call[k..].find(&needle) { + let at = k + mrel; + let boundary = call[..at] + .chars() + .last() + .map(|c| !(c.is_alphanumeric() || c == '_')) + .unwrap_or(true); + if boundary { + if let Some(h) = handler_for_mount(call, m) { + out.push((m.to_uppercase(), path.to_string(), h)); + } + } + k = at + needle.len(); + } + } + } + out + } + + /// SCRAPE-MOUNTS CROSS-CHECK (#195, U4/R5, INV-21b). The read-gate orphan guard + /// above derives its required set from a SOURCE-MARKER scan of `src/api`; this + /// derives it instead from the MOUNTED ROUTE TABLE (`server.rs`). Every mounted + /// repo-scoped GET must be EITHER a driven `ReadGate` row, OR a deny-bearing GET + /// of another class already in the registry (`list_visibility` is an owner-gated + /// GET), OR structurally excused in `READ_GATE_NOT_DRIVEN`, OR a declared public + /// repo-GET (`PUBLIC_REPO_GETS`). A mounted repo-GET that is none of these fails + /// HERE — which catches a read that gates via a helper the marker scan does not + /// recognize: it would still be mounted, so it must be classified rather than + /// slipping past silently. + #[test] + fn mounted_repo_gets_are_driven_excused_or_declared_public() { + let server = include_str!("../src/server.rs"); + let mounts = scrape_mounts_with_handler(server); + + // Integrity floor: the handler-aware walk must find roughly as many mounts + // as the plain scrape; if it collapsed, the cross-check would pass vacuously. + assert!( + mounts.len() >= 90, + "handler-aware mount scrape found only {} routes — the parser likely broke (floor 90)", + mounts.len() + ); + + // Every path that is a driven ReadGate row, and every deny-bearing GET path + // of any class (so the owner-gated `list_visibility` GET is accounted for). + let driven_read_paths: HashSet<&str> = deny_bearing_routes() + .iter() + .filter(|r| r.gate == GateClass::ReadGate) + .map(|r| r.path) + .collect(); + let deny_bearing_get_paths: HashSet<&str> = deny_bearing_routes() + .iter() + .filter(|r| r.method == "GET") + .map(|r| r.path) + .collect(); + let excused_handlers: HashSet<&str> = + READ_GATE_NOT_DRIVEN.iter().map(|(h, _)| *h).collect(); + let public_paths: HashSet<&str> = PUBLIC_REPO_GETS.iter().copied().collect(); + + let mut repo_gets_seen = 0usize; + for (method, path, handler) in &mounts { + if method != "GET" || !is_repo_scoped_path(path) { + continue; + } + repo_gets_seen += 1; + let classified = driven_read_paths.contains(path.as_str()) + || deny_bearing_get_paths.contains(path.as_str()) + || excused_handlers.contains(handler.as_str()) + || public_paths.contains(path.as_str()); + assert!( + classified, + "mounted repo-scoped GET `{path}` (handler `{handler}`) is neither a \ + driven ReadGate row, a deny-bearing GET row, structurally excused in \ + READ_GATE_NOT_DRIVEN, nor a declared public repo-GET — classify it \ + (it may gate via a helper the source-marker scan does not recognize)" + ); + } + + // Non-vacuous floor: the tree mounts well over a dozen repo-scoped GETs; if + // the filter found almost none, the loop above proved nothing. + assert!( + repo_gets_seen >= 15, + "found only {repo_gets_seen} mounted repo-scoped GETs — the path filter \ + or the scrape likely broke (floor 15)" + ); + } + + // ── F3 unit tests: the inline owner-gate did_matches discriminator ─────────── + // + // These pin `has_owner_did_matches` against the EXACT real bodies (positive and + // negative), so the two-arg discriminator can't silently regress into matching + // the signer-self / author forms that share the helper. + + #[test] + fn has_owner_did_matches_catches_the_protect_branch_owner_idiom() { + // protect.rs:28 form: first arg the caller, second `&record.owner_did`. + let body = "let caller = &auth.0;\n\ + if !crate::api::did_matches(caller, &record.owner_did) {\n\ + return Err(AppError::Forbidden(\"only the owner\".into()));\n\ + }"; + assert!( + has_owner_did_matches(body), + "the protect_branch owner idiom must be recognized" + ); + } + + #[test] + fn has_owner_did_matches_catches_the_auth0_owner_form() { + // The `&auth.0` first-arg spelling (repos.rs's branch-protection check), + // tolerating a newline before the second arg. + let body = + "if x\n && !crate::api::did_matches(&auth.0,\n &record.owner_did)\n {"; + assert!( + has_owner_did_matches(body), + "the &auth.0 / owner_did form must be recognized across a newline" + ); + } + + #[test] + fn has_owner_did_matches_ignores_register_replica_signer_self() { + // replicas.rs:52: FIRST arg is `replica_did`, NOT the caller — signer-self + // by design. The second arg is owner_did, so a naive "second arg is + // owner_did" check would wrongly flag it. It must NOT match. + let body = "if crate::api::did_matches(replica_did, &record.owner_did) {\n\ + // the signer is registering itself as a replica\n\ + }"; + assert!( + !has_owner_did_matches(body), + "register_replica's signer-self did_matches must NOT be treated as an owner gate" + ); + } + + #[test] + fn has_owner_did_matches_ignores_close_pr_author_form() { + // pulls.rs:277: caller-first but SECOND arg is `&pr.author_did`, not + // owner_did — the owner-or-author close gate. Must NOT match as owner-only. + let body = "let is_author = crate::api::did_matches(&auth.0, &pr.author_did);"; + assert!( + !has_owner_did_matches(body), + "close_pr's author did_matches must NOT be treated as an owner gate" + ); + } + + #[test] + fn has_owner_did_matches_ignores_bounty_and_task_self_forms() { + // bounties.rs / tasks.rs: caller-first, but the second arg is a + // creator/delegator/assignee did, never owner_did. + for body in [ + "if !crate::api::did_matches(&auth.0, &bounty.creator_did) {", + "if !crate::api::did_matches(&auth.0, &body.delegator_did) {", + "if !crate::api::did_matches(&auth.0, &body.assignee_did) {", + ] { + assert!( + !has_owner_did_matches(body), + "self/author did_matches form must NOT be treated as an owner gate: {body}" + ); + } + } + + // ── F3 unit tests: the caller-self did_matches discriminator ───────────────── + + #[test] + fn has_signer_self_did_matches_fires_on_caller_self_and_ignores_owner_form() { + // Fires: caller-spelled first arg (bound via `let caller = &auth.0;`), a + // non-owner second arg, the field-binding form the task handlers use. Proves + // the arg1 set is honored (a `&auth.0`-only classifier would miss this). + let self_form = "let caller = &auth.0;\n\ + if !crate::api::did_matches(caller, &x.foo_did) {\n\ + return Err(forbidden(\"...\"));\n\ + }"; + assert!( + has_signer_self_did_matches(self_form), + "the caller-self field-binding did_matches must be recognized" + ); + // The `&auth.0` spelling with a non-owner second arg fires too. + let auth0_form = "if !crate::api::did_matches(&auth.0, &body.assignee_did) {"; + assert!( + has_signer_self_did_matches(auth0_form), + "the &auth.0 caller-self form must be recognized" + ); + // Does NOT fire on the owner form (that is has_owner_did_matches' job), so a + // caller-self classifier that matched it would double-count. + let owner_form = "if !crate::api::did_matches(caller, &record.owner_did) {"; + assert!( + !has_signer_self_did_matches(owner_form), + "the owner form must NOT be treated as a caller-self gate (no double-count)" + ); + } + + // ── Shutdown-completeness guard: every spawn_node test must shutdown() ──────── + + /// Blank the CONTENT of comments and double-quoted / raw string literals in `src` + /// (length-preserving, newlines kept) so the brace slicer and the marker scans see + /// CODE only. Without it this guard carries two vacuity holes of exactly the class + /// it exists to close (INV-21 on the guard itself): a `{` inside a format string + /// unbalances the body slice and can swallow a later test's `shutdown()`, and a + /// `// node.shutdown()` comment satisfies the presence check while the test still + /// returns through `Drop`. Handles `//` and nested `/* */` comments, `"..."` (with + /// `\` escapes), and hash-delimited raw strings (`r#"..."#`, `br#"..."#`). Char + /// literals and lifetimes are left as code: they carry neither braces-of-interest + /// nor the shutdown marker, and telling a `'a` lifetime from a `'a'` char would add + /// hazard for no coverage. + fn code_only(src: &str) -> String { + fn blank(out: &mut [u8], from: usize, to: usize) { + for c in &mut out[from..to] { + if *c != b'\n' { + *c = b' '; + } + } + } + let b = src.as_bytes(); + let n = b.len(); + let mut out = b.to_vec(); + let mut i = 0; + while i < n { + // Line comment: `//` … EOL. + if b[i] == b'/' && i + 1 < n && b[i + 1] == b'/' { + let start = i; + while i < n && b[i] != b'\n' { + i += 1; + } + blank(&mut out, start, i); + continue; + } + // Block comment `/* … */`, nesting allowed (Rust nests them). + if b[i] == b'/' && i + 1 < n && b[i + 1] == b'*' { + let start = i; + let mut depth = 1i32; + i += 2; + while i < n && depth > 0 { + if b[i] == b'/' && i + 1 < n && b[i + 1] == b'*' { + depth += 1; + i += 2; + } else if b[i] == b'*' && i + 1 < n && b[i + 1] == b'/' { + depth -= 1; + i += 2; + } else { + i += 1; + } + } + blank(&mut out, start, i.min(n)); + continue; + } + // Raw string `r"…"` / `r#"…"#` (also the b-prefixed byte form, entered at + // the `r`). A raw ident `r#foo` has no `"` after the hashes and falls through. + if b[i] == b'r' && i + 1 < n && (b[i + 1] == b'"' || b[i + 1] == b'#') { + let mut h = i + 1; + let mut hashes = 0usize; + while h < n && b[h] == b'#' { + hashes += 1; + h += 1; + } + if h < n && b[h] == b'"' { + let content = h + 1; + let mut k = content; + let mut close = None; + while k < n { + if b[k] == b'"' { + let mut extra = 0; + while k + 1 + extra < n && extra < hashes && b[k + 1 + extra] == b'#' { + extra += 1; + } + if extra == hashes { + close = Some(k); + break; + } + } + k += 1; + } + blank(&mut out, content, close.unwrap_or(n)); + i = close.map(|c| c + 1 + hashes).unwrap_or(n); + continue; + } + } + // Normal string `"…"`, honoring `\` escapes (so `\"` does not close it). + if b[i] == b'"' { + let content = i + 1; + let mut k = content; + while k < n { + if b[k] == b'\\' { + k += 2; + continue; + } + if b[k] == b'"' { + break; + } + k += 1; + } + blank(&mut out, content, k.min(n)); + i = if k >= n { n } else { k + 1 }; + continue; + } + i += 1; + } + String::from_utf8(out).expect("code_only blanks whole literal spans, so utf-8 survives") + } + + /// The spawn_node tests that fail the shutdown contract: they spawn a node, are not + /// allowlisted, and their (code-only) body carries no `node.shutdown(` call. Pure so + /// the adversarial unit tests below can drive synthetic sources; the `#[test]` guard + /// runs it over this file's real, sanitized source. + fn spawn_node_tests_missing_shutdown( + tests: &[(String, String)], + allow: &[&str], + ) -> Vec { + tests + .iter() + .filter(|(name, body)| { + body.contains("spawn_node") + && !allow.contains(&name.as_str()) + && !body.contains("node.shutdown(") + }) + .map(|(name, _)| name.clone()) + .collect() + } + + /// `(name, body)` for every real `#[sqlx::test]` in `src`. `src` is expected to be + /// [`code_only`]-sanitized so string/comment content cannot unbalance the body + /// slice. The attribute is only counted when nothing but whitespace precedes it on + /// its line (belt-and-suspenders with sanitization, which already blanks any + /// backtick mention inside a doc comment, e.g. the module header naming the + /// attribute in prose, so it is not mistaken for a test). The needle is spelled in + /// two pieces so this + /// scanner does not match its own source when it scrapes this file. The body is + /// brace-matched from the signature's opening `{` to its close, so a marker cannot + /// leak across into the next item. + fn sqlx_test_bodies(src: &str) -> Vec<(String, String)> { + // Split so the literal attribute never appears contiguously in this fn's own + // source (belt-and-suspenders with the whitespace-before-on-line check below). + let attr = concat!("#[sqlx", "::test]"); + let bytes = src.as_bytes(); + let mut out = Vec::new(); + let mut search = 0; + while let Some(rel) = src[search..].find(attr) { + let at = search + rel; + search = at + attr.len(); + // Real attribute only: the text between the line start and the attribute + // must be blank (rejects `//! … #[sqlx::test] …` and any string mention). + let line_start = src[..at].rfind('\n').map(|n| n + 1).unwrap_or(0); + if !src[line_start..at].trim().is_empty() { + continue; + } + let Some(fnrel) = src[search..].find("fn ") else { + break; + }; + let name_start = search + fnrel + "fn ".len(); + let name: String = src[name_start..] + .chars() + .take_while(|c| c.is_alphanumeric() || *c == '_') + .collect(); + if name.is_empty() { + continue; + } + let Some(brel) = src[name_start..].find('{') else { + break; + }; + let open = name_start + brel; + let mut depth = 0i32; + let mut j = open; + while j < src.len() { + match bytes[j] { + b'{' => depth += 1, + b'}' => { + depth -= 1; + if depth == 0 { + j += 1; + break; + } + } + _ => {} + } + j += 1; + } + out.push((name, src[open..j].to_string())); + search = j; + } + out + } + + /// Every `#[sqlx::test]` that spawns a node must end it with + /// `node.shutdown().await`, so the serve task's pool clones are released before + /// `#[sqlx::test]`'s synchronous `DROP DATABASE` cleanup fires. A test that returns + /// via `TestNode`'s async `Drop` instead only *signals* teardown (it drains on a + /// later scheduler tick — the Drop-regression tests poll up to 5s for it), so it + /// races the cleanup and can leak the per-test database / flake under parallelism. + /// The module header states this contract in prose; this makes it load-bearing — + /// removing any one `shutdown()` turns this RED. + /// + /// The two Drop-regression tests are the CLOSED allowlist: they deliberately + /// return WITHOUT `shutdown()` to exercise the Drop teardown, so they must NOT call + /// it. The allowlist is checked both ways — an allowlisted name that no longer + /// exists (rename/removal) or that grows a `shutdown()` call fails here — so it + /// cannot silently rot into covering a test it no longer describes. + #[test] + fn every_spawn_node_test_shuts_the_node_down() { + // The ONLY tests allowed to return without shutdown(): they test Drop itself. + const DROP_REGRESSION_ALLOW: &[&str] = &[ + "drop_without_shutdown_unblocks_database", + "drop_with_broken_graceful_chain_still_unblocks_via_abort", + ]; + + let src = code_only(include_str!("deny_harness.rs")); + let tests = sqlx_test_bodies(&src); + + // Non-vacuous floor: if the scraper silently found nothing, every loop below + // would pass by checking zero tests. The file has 14 sqlx::test fns today. + assert!( + tests.len() >= 12, + "sqlx::test scrape found only {} tests — the parser likely broke (floor 12)", + tests.len() + ); + + // Each allowlisted name must resolve to a real spawn_node test that (correctly) + // does NOT call shutdown — so the allowlist cannot cover a renamed/deleted test + // or one that later grew a shutdown() call and no longer exercises Drop. + for allowed in DROP_REGRESSION_ALLOW { + let (_, body) = tests + .iter() + .find(|(name, _)| name == allowed) + .unwrap_or_else(|| { + panic!( + "allowlisted Drop-regression test `{allowed}` not found — renamed or \ + removed? update DROP_REGRESSION_ALLOW" + ) + }); + assert!( + body.contains("spawn_node"), + "allowlisted test `{allowed}` no longer spawns a node — it does not belong \ + on the shutdown allowlist" + ); + assert!( + !body.contains("node.shutdown("), + "allowlisted Drop-regression test `{allowed}` now calls shutdown() — it no \ + longer exercises Drop-only teardown; remove it from DROP_REGRESSION_ALLOW" + ); + } + + // Every other spawn_node test must shut its node down. + let missing = spawn_node_tests_missing_shutdown(&tests, DROP_REGRESSION_ALLOW); + assert!( + missing.is_empty(), + "these tests spawn a node but never call node.shutdown().await — they return \ + via TestNode::Drop, racing sqlx's DROP DATABASE cleanup (add the awaited \ + shutdown, or add to DROP_REGRESSION_ALLOW if they deliberately test Drop): \ + {missing:?}" + ); + } + + // ── Adversarial unit tests for the guard itself (INV-21 on the guard) ──────── + // These drive synthetic sources so the two vacuity holes an external review + // flagged are proven CLOSED by execution: a commented-out shutdown, and an + // unbalanced brace inside a string that would otherwise swallow a sibling's + // shutdown. Each passes (false green) against a raw substring/brace scan; + // sanitizing via `code_only` first is what makes them fail as they must. + + #[test] + fn shutdown_guard_flags_a_commented_out_shutdown() { + // The ONLY node.shutdown( occurrence is a comment: the node is never torn + // down, so the test must be flagged despite the textual match. + let synth = concat!( + "#[sqlx", + "::test]\n", + "async fn foo(pool: PgPool) {\n", + " let node = spawn_node(pool).await;\n", + " // node.shutdown().await; (deleted, left as a comment)\n", + "}\n" + ); + let tests = sqlx_test_bodies(&code_only(synth)); + assert_eq!( + spawn_node_tests_missing_shutdown(&tests, &[]), + vec!["foo".to_string()], + "a commented-out shutdown must NOT satisfy the guard" + ); + } + + #[test] + fn shutdown_guard_is_not_fooled_by_a_brace_inside_a_string() { + // `a` has no shutdown and an unbalanced `{` inside a string; `b` follows and + // does shut down. A brace-blind slicer would extend `a` into `b`, borrow b's + // shutdown, and clear `a` — sanitization keeps the two bodies separate. + let synth = concat!( + "#[sqlx", + "::test]\n", + "async fn a(pool: PgPool) {\n", + " let node = spawn_node(pool).await;\n", + " let _ = \"oops {\";\n", + "}\n", + "#[sqlx", + "::test]\n", + "async fn b(pool: PgPool) {\n", + " let node = spawn_node(pool).await;\n", + " node.shutdown().await;\n", + "}\n" + ); + let tests = sqlx_test_bodies(&code_only(synth)); + let missing = spawn_node_tests_missing_shutdown(&tests, &[]); + assert!( + missing.contains(&"a".to_string()), + "test `a` (no shutdown; unbalanced brace in a string) must be flagged, not \ + masked by test `b`'s shutdown: got {missing:?}" + ); + assert!( + !missing.contains(&"b".to_string()), + "test `b` shuts down and must not be flagged: {missing:?}" + ); + } + + #[test] + fn shutdown_guard_passes_clean_and_exempts_only_via_allowlist() { + let synth = concat!( + "#[sqlx", + "::test]\n", + "async fn good(pool: PgPool) {\n", + " let node = spawn_node(pool).await;\n", + " node.shutdown().await;\n", + "}\n", + "#[sqlx", + "::test]\n", + "async fn drop_probe(pool: PgPool) {\n", + " let node = spawn_node(pool).await;\n", + " // deliberately no shutdown: exercises Drop\n", + "}\n" + ); + let tests = sqlx_test_bodies(&code_only(synth)); + // Clean test + allowlisted Drop test -> no offenders. + assert!( + spawn_node_tests_missing_shutdown(&tests, &["drop_probe"]).is_empty(), + "a clean test plus an allowlisted Drop test must produce no offenders" + ); + // Without the allowlist the Drop test IS an offender, proving the exemption + // comes from the allowlist, not from an accident of the scan. + assert_eq!( + spawn_node_tests_missing_shutdown(&tests, &[]), + vec!["drop_probe".to_string()], + "un-allowlisted, the no-shutdown test must be flagged" + ); + } +} diff --git a/crates/gitlawb-node/tests/support/assert.rs b/crates/gitlawb-node/tests/support/assert.rs new file mode 100644 index 00000000..57333c8e --- /dev/null +++ b/crates/gitlawb-node/tests/support/assert.rs @@ -0,0 +1,125 @@ +//! INV-8 deny assertion. A denial must be an actual refusal (4xx) whose body +//! carries none of the data it is withholding, and must never be a 2xx (an +//! empty-200 rendered as success is the denial-as-success bug this guards). +//! +//! The check is split into a pure `check_denied` (unit-tested below with the +//! self-check scenarios) and an async `assert_denied` wrapper that reads a real +//! `reqwest::Response`. + +/// Pure deny check. Returns `Err(reason)` when the response is not a clean +/// denial. `withheld` are exact tokens (OIDs, paths, private slugs) that must +/// not appear anywhere in the body; register short-OID prefixes and encoded +/// path forms as separate tokens at the call site (a raw substring scan cannot +/// normalize them). +pub fn check_denied( + status: u16, + body: &str, + expected: u16, + withheld: &[&str], +) -> Result<(), String> { + // The harness only ever expects denials; a non-4xx expectation is a + // programming error in the test, not a node behavior under test. + if !(400..500).contains(&expected) { + return Err(format!( + "test bug: assert_denied expected status {expected} is not a 4xx" + )); + } + // INV-8: a denial rendered as success is the headline bug. Catch it + // explicitly so the failure message names it, rather than only failing the + // status-equality check below. + if (200..300).contains(&status) { + return Err(format!( + "denial rendered as success: got {status}, expected denial {expected}. \ + body={body:?}" + )); + } + if status != expected { + return Err(format!( + "expected denial status {expected}, got {status}. body={body:?}" + )); + } + for token in withheld { + if !token.is_empty() && body.contains(token) { + return Err(format!( + "withheld token {token:?} leaked in denial body: {body:?}" + )); + } + } + Ok(()) +} + +/// Drive a real response through the deny check and panic on failure. Reads the +/// full body once. +pub async fn assert_denied(resp: reqwest::Response, expected: u16, withheld: &[&str]) { + let status = resp.status().as_u16(); + // A body-read failure must NOT fold to "" — an empty body passes the withheld + // scan vacuously, so an unreadable denial (a mid-body reset/truncation) would be + // certified leak-free without ever inspecting the bytes. Fail loud instead. A + // legitimately empty body is `Ok("")`, not `Err`, so the honest empty-401 denials + // still pass. + let body = resp + .text() + .await + .expect("read denial body: an unreadable denial cannot be certified leak-free"); + if let Err(reason) = check_denied(status, &body, expected, withheld) { + panic!("{reason}"); + } +} + +#[cfg(test)] +mod tests { + use super::check_denied; + + #[test] + fn clean_403_with_no_leak_passes() { + let r = check_denied( + 403, + r#"{"error":"only the repo owner can do this"}"#, + 403, + &["a1b2c3"], + ); + assert!(r.is_ok(), "{r:?}"); + } + + #[test] + fn empty_200_is_flagged_as_denial_rendered_as_success() { + let r = check_denied(200, "", 403, &[]); + assert!(r.is_err()); + assert!(r.unwrap_err().contains("rendered as success")); + } + + #[test] + fn leaking_403_fails_and_names_the_token() { + let withheld = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"; + let body = format!("not found near object {withheld}"); + let r = check_denied(403, &body, 403, &[withheld]); + assert!(r.is_err()); + assert!(r.unwrap_err().contains(withheld)); + } + + #[test] + fn wrong_status_fails() { + // A 404 when the owner gate's 403 was expected must not pass: it means + // an earlier layer denied, so the gate under test was never exercised. + let r = check_denied(404, "not found", 403, &[]); + assert!(r.is_err()); + assert!(r.unwrap_err().contains("expected denial status 403")); + } + + #[test] + fn non_4xx_expected_is_a_test_bug() { + // Guards the harness itself: expecting a non-denial status is a test + // authoring error, not a node behavior, and must be rejected. + let r = check_denied(403, "denied", 200, &[]); + assert!(r.is_err()); + assert!(r.unwrap_err().contains("not a 4xx")); + } + + #[test] + fn empty_withheld_token_never_false_positives() { + // An empty token must be skipped, never treated as "contained in every + // body" — otherwise every denial would spuriously fail as a leak. + let r = check_denied(403, "any body", 403, &[""]); + assert!(r.is_ok(), "{r:?}"); + } +} diff --git a/crates/gitlawb-node/tests/support/mod.rs b/crates/gitlawb-node/tests/support/mod.rs new file mode 100644 index 00000000..8b5bf093 --- /dev/null +++ b/crates/gitlawb-node/tests/support/mod.rs @@ -0,0 +1,24 @@ +//! Shared support for the real-node deny harness: the deny assertion (U4), the +//! RFC-9421 signing client (U2), and (via `gitlawb_node::test_harness`) the +//! node spawner (U3). + +pub mod assert; +// U2 fixture state matrix + per-gate-class probe generators; driven by U3. +#[allow(dead_code)] +pub mod probe; +// U1 deny-bearing route registry; fields consumed by U2/U3 as they land. +#[allow(dead_code)] +pub mod routes; +pub mod signing; + +/// A reqwest client with a bounded request timeout, so a wedged git subprocess or +/// route fails the suite rather than hanging it until CI kills the job (#195, F1). +/// The two real-node probes that drive git subprocesses use this; the inline +/// twins at the upload-pack and registry-sweep sites use the same pattern. +#[allow(dead_code)] +pub fn bounded_client() -> reqwest::Client { + reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(30)) + .build() + .unwrap() +} diff --git a/crates/gitlawb-node/tests/support/probe.rs b/crates/gitlawb-node/tests/support/probe.rs new file mode 100644 index 00000000..a449d77c --- /dev/null +++ b/crates/gitlawb-node/tests/support/probe.rs @@ -0,0 +1,1420 @@ +//! U2: fixture state matrix + per-gate-class probe generators. +//! +//! The fixture is a STATE MATRIX, not one repo: owner-gate rows need a repo the +//! signed stranger can REACH (so the request hits the owner gate, not an +//! existence-hiding 404), and read-gate rows need a repo the caller CANNOT see. +//! A single repo cannot be both, so we seed a public repo (owner-gate substrate) +//! and a private repo with content (read-gate substrate). +//! +//! Each deny-bearing row expands to probes: the hostile request (asserting the +//! exact deny) plus a positive twin (owner-reachability for owner-gate, +//! reader/sibling-public read for read-gate) so a 403/404 for the wrong reason +//! cannot false-pass. + +use gitlawb_core::identity::Keypair; +use gitlawb_node::test_harness::TestNode; + +use super::routes::{AnonRead, GateClass, IdSource, Principal, Reach, Row}; + +/// A seeded two-repo state matrix plus the owner and stranger identities. +pub struct Fixture { + pub owner: Keypair, + pub stranger: Keypair, + /// The PR/issue author — distinct from owner and stranger — so the + /// owner-OR-author close gates (#195, F1) drive their author arm with an + /// identity that is NOT the owner (the original bug seeded author == owner). + pub author: Keypair, + /// The bounty creator, distinct from every other identity. + pub creator: Keypair, + /// The bounty claimant, distinct from every other identity. + pub claimant: Keypair, + /// The task assignee, distinct from every other identity (#195, F3). The + /// complete/fail actor gates drive their Assignee arm as this identity, so it + /// must differ from owner and stranger or the arm cannot false-pass on an + /// owner fallback. + pub assignee: Keypair, + pub owner_did: String, + pub author_did: String, + pub creator_did: String, + pub claimant_did: String, + pub assignee_did: String, + /// The id (UUID) of the seeded disputable bounty, filled into the + /// `dispute_bounty` row's `{id}` placeholder (IdSource::BountyId). + pub bounty_id: String, + /// #195 (N2): one bounty per status-gated bounty mutation row. submit/approve/ + /// cancel check the bounty's STATUS before their 403 auth gate, so the stranger + /// hostile only exercises the gate against a bounty in the gate-reaching + /// status, and each row's authorized twin CONSUMES that status; the rows + /// cannot share a bounty. This one stays "claimed" (create + claim) for the + /// submit_bounty row (IdSource::ClaimedBountyId). + pub claimed_bounty_id: String, + /// Driven to "submitted" (create + claim + submit) for the approve_bounty row + /// (IdSource::SubmittedBountyId). + pub submitted_bounty_id: String, + /// Left "open" (create only) for the cancel_bounty row (IdSource::OpenBountyId). + pub open_bounty_id: String, + /// #195 (F3): a task seeded "pending" (delegated by the owner) for the + /// claim_task caller-self row (IdSource::ClaimableTaskId). + pub claimable_task_id: String, + /// A task seeded "claimed" by the assignee for the complete_task actor row + /// (IdSource::CompletableTaskId). + pub completable_task_id: String, + /// A second assignee-claimed task for the fail_task actor row + /// (IdSource::FailableTaskId). + pub failable_task_id: String, + /// The id (UUID) of the seeded issue (authored by `author`), filled into the + /// `close_issue` row's `{id}` placeholder (IdSource::IssueId). + pub issue_id: String, + /// The id (UUID) of an issue seeded in the PRIVATE repo (marker in its title), + /// filled into the get_issue / list_issue_comments read-gate rows + /// (IdSource::PrivIssueId). Distinct from `issue_id` (the public close issue). + pub priv_issue_id: String, + /// The id (UUID) of a bounty seeded against the PRIVATE repo (marker in its + /// title), filled into the get_bounty read-gate row (IdSource::PrivBountyId). + pub priv_bounty_id: String, + /// The id (UUID) of a ref-certificate issued by a real owner push to the + /// private repo, filled into the get_cert read-gate row (IdSource::CertId). + pub cert_id: String, + /// Per-read secret markers seeded into the private sub-entities (issue title, + /// PR title, PR diff, bounty title). Each read's own marker is added to + /// that read's withheld set so a 404 leaking THAT read's private content fails + /// (#195, R3/KTD-4). A union is used so every seeded marker is withheld from + /// every read-gate hostile — a per-read withheld token, never a status-only + /// 404 check. + pub priv_markers: Vec, + /// Public repo: owner-gate rows run against this so the stranger reaches the + /// owner gate rather than a hidden-repo 404. + pub public_repo: String, + /// The public repo's id, needed to seed the PR the close-gate rows require. + pub public_repo_id: String, + /// Private repo (is_public=false) with seeded content: read-gate rows run + /// against this so an anon caller gets the existence-hiding 404 and the owner + /// twin gets 2xx. + pub private_repo: String, + /// The private repo's internal id (UUID). A read-gate denial body must never + /// echo it: a 404 that serializes the private record's internal id has leaked + /// repository metadata just as much as the blob content (#195, F4). + pub private_repo_id: String, + /// A seeded blob path present in both repos, for `{*path}` reads. + pub content_path: String, + /// The DISTINCTIVE secret bytes seeded into the private repo's blob. A + /// read-gate denial body must never echo this (INV-8: the sweep claims the + /// denial "leaks nothing", so it must actually check for the private content). + pub private_secret: String, + /// The private blob's full sha1 object id, and its short prefix. Neither may + /// appear in a read-gate denial body — a 404 that names the withheld object's + /// OID has leaked its existence. + pub private_blob_oid: String, + pub private_blob_oid_short: String, +} + +impl Fixture { + pub async fn seed(node: &TestNode) -> Fixture { + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let stranger = Keypair::generate(); + // #195 (F1): distinct identities for each authorizing arm of the + // multi-principal gates. All four must differ from owner and stranger and + // from each other, or an arm silently collapses onto another and reverting + // it stays invisible (the original author == owner bug). + let author = Keypair::generate(); + let author_did = author.did().to_string(); + let creator = Keypair::generate(); + let creator_did = creator.did().to_string(); + let claimant = Keypair::generate(); + let claimant_did = claimant.did().to_string(); + // #195 (F3): a distinct assignee identity (!= owner, != stranger) for the + // complete/fail actor gates, so their Assignee arm cannot false-pass on an + // owner fallback. + let assignee = Keypair::generate(); + let assignee_did = assignee.did().to_string(); + let content_path = "public/a.txt".to_string(); + + let pub_repo = "prober-pub"; + let public_repo_id = node.seed_repo(&owner_did, pub_repo, true).await; + node.seed_bare_repo( + &owner_did, + pub_repo, + &[("public/a.txt", "pub content")], + "sha1", + ); + // The author-or-owner close gates load the PR/issue before they run, so + // the close rows need the entity to exist. Seed both authored by `author` + // (NOT the owner) so the author arm is granted by a non-owner identity and + // reverting either arm is observable. The PR seeds via the DB (author_did + // is a real column). The issue is seeded over HTTP as `author` (below): + // close_issue deserializes the WHOLE git-JSON IssueRecord before reading + // its author, so a bare `{"author":..}` DB-seed fails to deserialize and + // the author falls back to None — the author arm would then 403. + node.seed_pr(&public_repo_id, 1, &author_did).await; + let issue_id = seed_authored_issue(node, &owner_did, pub_repo, &author).await; + + // Seed a DISPUTABLE bounty (status "claimed", creator + claimant set) so the + // dispute_bounty creator/claimant arms reach their auth check. dispute_bounty + // gates on creator/claimant, NOT the repo, so the bounty is created on the + // public repo (both signers can read it). Seeded over HTTP through the real + // handlers — create (creator) then claim (claimant) — because the DB seam is + // private to TestNode and a test-only seed method would be a src/ change. + let bounty_id = seed_bounty_at_stage( + node, + &owner_did, + pub_repo, + &creator, + &claimant, + "prober dispute bounty", + BountyStage::Claimed, + ) + .await; + + // #195 (N2): submit/approve/cancel check the bounty's STATUS before their + // 403 auth gate, so each row gets its OWN bounty in the gate-reaching + // status (claimed / submitted / open); the authorized twins consume that + // status, so the rows cannot share a bounty with each other or with the + // disputable one above. + let claimed_bounty_id = seed_bounty_at_stage( + node, + &owner_did, + pub_repo, + &creator, + &claimant, + "prober submit bounty", + BountyStage::Claimed, + ) + .await; + let submitted_bounty_id = seed_bounty_at_stage( + node, + &owner_did, + pub_repo, + &creator, + &claimant, + "prober approve bounty", + BountyStage::Submitted, + ) + .await; + let open_bounty_id = seed_bounty_at_stage( + node, + &owner_did, + pub_repo, + &creator, + &claimant, + "prober cancel bounty", + BountyStage::Open, + ) + .await; + + let priv_repo = "prober-priv"; + // Seed the private blob with an UNMISTAKABLE marker (not "priv content"), + // so a read-gate denial body that echoes the content is caught as a leak + // rather than blending into generic prose. The OID is captured so the OID + // (full + short) is withheld too — a 404 naming the object's id has leaked + // its existence just as much as its bytes. + let private_secret = "TOPSECRET-PRIVREAD-U3".to_string(); + let private_repo_id = node.seed_repo(&owner_did, priv_repo, false).await; + let priv_oids = node.seed_bare_repo( + &owner_did, + priv_repo, + &[("public/a.txt", &private_secret)], + "sha1", + ); + let private_blob_oid = priv_oids["public/a.txt"].clone(); + let private_blob_oid_short = private_blob_oid[..12].to_string(); + // The private repo's current `main` commit tip, needed as the `old` oid of + // the ref-update that force-points main at the deterministic push below. + let private_main_tip = priv_oids["HEAD"].clone(); + + // #195 (F2/U3): seed the private sub-entities the deferred reads return, each + // carrying its OWN distinctive marker so the per-read leak assertion is + // load-bearing. All are seeded as the OWNER (the only reader of the private + // repo), so anon/stranger get the existence-hiding 404 and the owner twin + // gets a 2xx that actually returns the entity. + // + // A real owner push to the private repo advances a `feature` branch with a + // marker file (so get_pr_diff's branch_diff_names(main, feature) is + // NON-EMPTY) AND issues a ref-certificate (get_cert). Do it first so the PR's + // source branch exists before create_pr and the cert id is captured. + let pr_diff_marker = "TOPSECRET-PRDIFF-U3".to_string(); + let cert_id = seed_private_push_and_cert( + node, + &owner, + &owner_did, + priv_repo, + &private_main_tip, + &private_secret, + &private_blob_oid, + &pr_diff_marker, + ) + .await; + + let issue_marker = "TOPSECRET-ISSUE-U3".to_string(); + let priv_issue_id = + seed_private_issue(node, &owner, &owner_did, priv_repo, &issue_marker).await; + + let pr_marker = "TOPSECRET-PRTITLE-U3".to_string(); + seed_private_pr(node, &owner, &owner_did, priv_repo, &pr_marker).await; + + let bounty_marker = "TOPSECRET-BOUNTY-U3".to_string(); + let priv_bounty_id = + seed_private_bounty(node, &owner, &owner_did, priv_repo, &bounty_marker).await; + + let priv_markers = vec![issue_marker, pr_marker, pr_diff_marker, bounty_marker]; + + // #195 (F3): seed the caller-self task entities through the real task write + // routes (behind add_auth_layers only, no icaptcha). The claimable task + // stays "pending" so its claim control (the owner) can claim it; the + // completable and failable tasks are claimed by the assignee, so + // complete_task / fail_task reach their assignee actor gate against a + // "claimed" task. + let claimable_task_id = seed_pending_task(node, &owner).await; + let completable_task_id = seed_claimed_task(node, &owner, &assignee).await; + let failable_task_id = seed_claimed_task(node, &owner, &assignee).await; + + Fixture { + owner, + stranger, + author, + creator, + claimant, + assignee, + owner_did, + author_did, + creator_did, + claimant_did, + assignee_did, + bounty_id, + claimed_bounty_id, + submitted_bounty_id, + open_bounty_id, + claimable_task_id, + completable_task_id, + failable_task_id, + issue_id, + priv_issue_id, + priv_bounty_id, + cert_id, + priv_markers, + public_repo: pub_repo.to_string(), + public_repo_id, + private_repo: priv_repo.to_string(), + private_repo_id, + content_path, + private_secret, + private_blob_oid, + private_blob_oid_short, + } + } +} + +/// Seed an issue authored by `author` through the real `create_issue` handler +/// and return its minted UUID id. Written as a full git-JSON `IssueRecord` (the +/// handler serializes the whole record), so `close_issue`'s deserialize-then-read +/// author path finds the author — which a bare-field DB seed cannot satisfy. +async fn seed_authored_issue( + node: &TestNode, + owner_did: &str, + repo: &str, + author: &Keypair, +) -> String { + use super::signing::signed_request; + + let client = super::bounded_client(); + let path = format!("/api/v1/repos/{owner_did}/{repo}/issues"); + let body = br#"{"title":"prober close issue"}"#.to_vec(); + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &path, + body, + author, + ) + .header("content-type", "application/json") + .send() + .await + .expect("create issue sends"); + assert_eq!( + resp.status().as_u16(), + 201, + "seeding: issue create must return 201 (author can read the public repo)" + ); + let created: serde_json::Value = resp.json().await.expect("issue create returns JSON"); + created["id"] + .as_str() + .expect("created issue carries an id") + .to_string() +} + +/// Create a "pending" task delegated by `delegator` through the real `create_task` +/// handler and return its minted id. `create_task` binds `delegator_did` to the +/// signer, so the body names the signer's DID. Task write routes sit behind +/// `add_auth_layers` only (no icaptcha), so a signed create reaches the handler. +async fn seed_pending_task(node: &TestNode, delegator: &Keypair) -> String { + use super::signing::signed_request; + + let client = super::bounded_client(); + let delegator_did = delegator.did().to_string(); + let body = + format!(r#"{{"kind":"review","capability":"read","delegator_did":"{delegator_did}"}}"#) + .into_bytes(); + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + "/api/v1/tasks", + body, + delegator, + ) + .header("content-type", "application/json") + .send() + .await + .expect("create task sends"); + assert_eq!( + resp.status().as_u16(), + 201, + "seeding: task create must return 201 (delegator_did binds to the signer)" + ); + let created: serde_json::Value = resp.json().await.expect("task create returns JSON"); + created["id"] + .as_str() + .expect("created task carries an id") + .to_string() +} + +/// Create a "pending" task (delegated by `owner`), then claim it as `assignee` +/// (`claim_task` binds `assignee_did` to the signer), leaving it "claimed" so +/// `complete_task` / `fail_task` reach their assignee actor gate. Returns the id. +async fn seed_claimed_task(node: &TestNode, owner: &Keypair, assignee: &Keypair) -> String { + use super::signing::signed_request; + + let id = seed_pending_task(node, owner).await; + + let client = super::bounded_client(); + let assignee_did = assignee.did().to_string(); + let claim_path = format!("/api/v1/tasks/{id}/claim"); + let body = format!(r#"{{"assignee_did":"{assignee_did}"}}"#).into_bytes(); + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &claim_path, + body, + assignee, + ) + .header("content-type", "application/json") + .send() + .await + .expect("claim task sends"); + assert_eq!( + resp.status().as_u16(), + 200, + "seeding: task claim must return 200 (assignee_did binds to the signer)" + ); + id +} + +/// How far along its lifecycle a seeded bounty is driven, expressed as the real +/// API transitions that get it there. Each status-gated bounty row needs the +/// status its handler's pre-auth check demands (#195, N2): cancel needs "open", +/// submit (and dispute) need "claimed", approve needs "submitted". +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum BountyStage { + /// `create_bounty` only: status "open". + Open, + /// create + claim: status "claimed" (creator + claimant set). + Claimed, + /// create + claim + submit: status "submitted". + Submitted, +} + +/// Seed a bounty on the (public) repo, drive it to `stage`, and return its id. +/// +/// Driven through the real HTTP handlers rather than the DB: `create_bounty` +/// (creator) mints a UUID id we read back from the response, `claim_bounty` +/// (claimant) sets the claimant, `submit_bounty` (claimant) records a PR id. +/// All read-gate the (public) repo, which the creator/claimant can read, and the +/// claim/submit transitions pass their own auth gates as the claimant. A Claimed +/// bounty's deadline has not been exceeded, so a creator/claimant dispute returns +/// 400 (not 403) after passing auth, which is still a Not403 twin, while a +/// stranger is denied 403 at the auth check first. +async fn seed_bounty_at_stage( + node: &TestNode, + owner_did: &str, + repo: &str, + creator: &Keypair, + claimant: &Keypair, + title: &str, + stage: BountyStage, +) -> String { + use super::signing::signed_request; + + let client = super::bounded_client(); + + // create_bounty (creator) -> 201 with the minted BountyRecord (carries the id). + let create_path = format!("/api/v1/repos/{owner_did}/{repo}/bounties"); + let body = format!(r#"{{"title":"{title}","amount":1}}"#).into_bytes(); + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &create_path, + body, + creator, + ) + .header("content-type", "application/json") + .send() + .await + .expect("create bounty sends"); + assert_eq!( + resp.status().as_u16(), + 201, + "seeding: bounty create must return 201 (creator can read the public repo)" + ); + let created: serde_json::Value = resp.json().await.expect("bounty create returns JSON"); + let bounty_id = created["id"] + .as_str() + .expect("created bounty carries an id") + .to_string(); + let creator_did = creator.did().to_string(); + assert_eq!( + created["creator_did"].as_str(), + Some(creator_did.as_str()), + "seeded bounty creator must be the fixture creator" + ); + if stage == BountyStage::Open { + return bounty_id; + } + + // claim_bounty (claimant) -> 200, status becomes "claimed", claimant recorded. + let claim_path = format!("/api/v1/bounties/{bounty_id}/claim"); + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &claim_path, + br#"{}"#.to_vec(), + claimant, + ) + .header("content-type", "application/json") + .send() + .await + .expect("claim bounty sends"); + assert_eq!( + resp.status().as_u16(), + 200, + "seeding: bounty claim must return 200 (claimant can read the public repo)" + ); + if stage == BountyStage::Claimed { + return bounty_id; + } + + // submit_bounty (claimant) -> 200, status becomes "submitted". pr_id is a + // free-text column; the seed value only needs to clear serde. + let submit_path = format!("/api/v1/bounties/{bounty_id}/submit"); + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &submit_path, + br#"{"pr_id":"1"}"#.to_vec(), + claimant, + ) + .header("content-type", "application/json") + .send() + .await + .expect("submit bounty sends"); + assert_eq!( + resp.status().as_u16(), + 200, + "seeding: bounty submit must return 200 (the claimant passes the submit gate)" + ); + + bounty_id +} + +/// Seed an issue in the PRIVATE repo (as the owner, the only reader) whose title +/// carries `marker`, and return its id. get_issue returns the whole IssueRecord, +/// so a 404 that echoes the title has leaked private content; the marker is added +/// to the withheld set. +async fn seed_private_issue( + node: &TestNode, + owner: &Keypair, + owner_did: &str, + repo: &str, + marker: &str, +) -> String { + use super::signing::signed_request; + + let client = super::bounded_client(); + let path = format!("/api/v1/repos/{owner_did}/{repo}/issues"); + let body = format!(r#"{{"title":"{marker}"}}"#).into_bytes(); + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &path, + body, + owner, + ) + .header("content-type", "application/json") + .send() + .await + .expect("create private issue sends"); + assert_eq!( + resp.status().as_u16(), + 201, + "seeding: private issue create must return 201 (owner reads its own private repo)" + ); + let created: serde_json::Value = resp.json().await.expect("issue create returns JSON"); + created["id"] + .as_str() + .expect("created private issue carries an id") + .to_string() +} + +/// Seed a PR in the PRIVATE repo (as the owner) with `marker` in its title and +/// `feature` as its source branch (which the prior push created). get_pr returns +/// the whole PullRequest and get_pr_diff a real non-empty diff, so both the title +/// marker and the diff marker are withheld. Lands at PR number 1 (first PR). +async fn seed_private_pr( + node: &TestNode, + owner: &Keypair, + owner_did: &str, + repo: &str, + marker: &str, +) { + use super::signing::signed_request; + + let client = super::bounded_client(); + let path = format!("/api/v1/repos/{owner_did}/{repo}/pulls"); + let body = + format!(r#"{{"title":"{marker}","source_branch":"feature","target_branch":"main"}}"#) + .into_bytes(); + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &path, + body, + owner, + ) + .header("content-type", "application/json") + .send() + .await + .expect("create private PR sends"); + assert_eq!( + resp.status().as_u16(), + 201, + "seeding: private PR create must return 201 (owner reads its own private repo)" + ); + let created: serde_json::Value = resp.json().await.expect("PR create returns JSON"); + assert_eq!( + created["number"].as_i64(), + Some(1), + "seeded private PR must be number 1 (the read-gate rows fill {{number}} = 1)" + ); +} + +/// Seed a bounty against the PRIVATE repo (as the owner) with `marker` in its +/// title, and return its id. get_bounty read-gates the bounty's own repo, so +/// anon/stranger get a 404 while the owner gets the bounty JSON (title marker +/// withheld from the 404). +async fn seed_private_bounty( + node: &TestNode, + owner: &Keypair, + owner_did: &str, + repo: &str, + marker: &str, +) -> String { + use super::signing::signed_request; + + let client = super::bounded_client(); + let path = format!("/api/v1/repos/{owner_did}/{repo}/bounties"); + let body = format!(r#"{{"title":"{marker}","amount":1}}"#).into_bytes(); + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &path, + body, + owner, + ) + .header("content-type", "application/json") + .send() + .await + .expect("create private bounty sends"); + assert_eq!( + resp.status().as_u16(), + 201, + "seeding: private bounty create must return 201 (owner reads its own private repo)" + ); + let created: serde_json::Value = resp.json().await.expect("bounty create returns JSON"); + created["id"] + .as_str() + .expect("created private bounty carries an id") + .to_string() +} + +/// Point the PRIVATE repo's `main` at a deterministic commit and add a `feature` +/// branch (a marker file on top of it) via a real owner git-receive-pack push, +/// which also makes the node issue a ref-certificate. Returns the issued cert's +/// id, read back from the owner's list_certs. +/// +/// seed_bare_repo's `main` tip is non-deterministic (its commit dates are ambient), +/// so we cannot build a `feature` that descends from it without the parent objects. +/// Instead we push a full (non-thin) history: a fresh `main` commit and `feature` +/// on top, force-updating the served `main` (the ref-update carries the server's +/// current tip as its `old` oid; receive.denyNonFastForwards is off by default, so +/// the non-fast-forward update applies). After the push, `feature` descends from +/// the new `main`, so get_pr_diff's `git diff main...feature` has a merge base and +/// returns the marker file. The push is signed as the owner (require_signature) and, +/// on a non-protected branch of an owner-owned repo, lands cleanly and issues a cert. +#[allow(clippy::too_many_arguments)] +async fn seed_private_push_and_cert( + node: &TestNode, + owner: &Keypair, + owner_did: &str, + repo: &str, + server_main_tip: &str, + blob_contents: &str, + expected_blob_oid: &str, + marker_file_contents: &str, +) -> String { + use super::signing::signed_request; + use std::process::Command; + + let work = std::env::temp_dir().join(format!( + "gl-u3-certpush-{}-{}", + std::process::id(), + server_main_tip + )); + let _ = std::fs::remove_dir_all(&work); + std::fs::create_dir_all(&work).expect("create cert-push workdir"); + let run = |args: &[&str], cwd: &std::path::Path| -> String { + let out = Command::new("git") + .args(args) + .current_dir(cwd) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + + // Pin sha1 to match the served fixture repo; init.defaultObjectFormat=sha256 skews the OIDs. + run(&["init", "-q", "-b", "main", "--object-format=sha1"], &work); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + // Keep `public/a.txt` (the seeded blob) on the new main so the get_blob / + // get_tree owner twins still resolve it — the blob OID depends only on content, + // so preserving the bytes preserves `private_blob_oid`. + std::fs::create_dir_all(work.join("public")).expect("mk public dir"); + std::fs::write(work.join("public/a.txt"), blob_contents).expect("seed blob file"); + run(&["add", "public/a.txt"], &work); + run(&["commit", "-q", "-m", "u3 main"], &work); + let new_main = run(&["rev-parse", "HEAD"], &work); + let new_blob = run(&["rev-parse", "HEAD:public/a.txt"], &work); + assert_eq!( + new_blob, expected_blob_oid, + "the pushed public/a.txt blob OID must equal the fixture's private_blob_oid" + ); + + // feature = new main + a marker file. + run(&["checkout", "-q", "-b", "feature"], &work); + std::fs::write(work.join("feature.txt"), marker_file_contents).expect("seed marker file"); + run(&["add", "feature.txt"], &work); + run(&["commit", "-q", "-m", "feature commit"], &work); + let feature_tip = run(&["rev-parse", "HEAD"], &work); + + // Full (non-thin) pack of both branches' objects — the server keeps its old + // main objects but needs all of the new ones since feature does not descend + // from the server's current main. + let pack = { + let out = Command::new("git") + .args(["pack-objects", "--stdout", "--revs"]) + .current_dir(&work) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .expect("spawn pack-objects"); + use std::io::Write; + out.stdin + .as_ref() + .expect("pack-objects stdin") + .write_all(format!("{new_main}\n{feature_tip}\n").as_bytes()) + .expect("write revs"); + let done = out.wait_with_output().expect("pack-objects completes"); + assert!(done.status.success(), "pack-objects must succeed"); + done.stdout + }; + + // git v0 receive-pack body: two ref-update pkt-lines — force-update main + // (old = the server's current tip) and create feature — the first carrying the + // capabilities after a NUL, a flush, then the packfile. + let zero = "0".repeat(server_main_tip.len()); + let l1 = format!("{server_main_tip} {new_main} refs/heads/main\0report-status\n"); + let l2 = format!("{zero} {feature_tip} refs/heads/feature\n"); + let mut body: Vec = Vec::new(); + body.extend(format!("{:04x}{l1}", l1.len() + 4).into_bytes()); + body.extend(format!("{:04x}{l2}", l2.len() + 4).into_bytes()); + body.extend_from_slice(b"0000"); + body.extend_from_slice(&pack); + + let client = super::bounded_client(); + let push_path = format!("/{owner_did}/{repo}/git-receive-pack"); + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &push_path, + body, + owner, + ) + .header("content-type", "application/x-git-receive-pack-request") + .send() + .await + .expect("receive-pack push sends"); + assert_eq!( + resp.status().as_u16(), + 200, + "seeding: owner push to a non-protected feature branch must return 200 so a cert issues" + ); + + let _ = std::fs::remove_dir_all(&work); + + // Read the issued cert id back from the owner's list_certs on the private repo. + let certs_path = format!("/api/v1/repos/{owner_did}/{repo}/certs"); + let resp = signed_request( + &client, + reqwest::Method::GET, + &node.base_url, + &certs_path, + Vec::new(), + owner, + ) + .send() + .await + .expect("list_certs sends"); + assert_eq!( + resp.status().as_u16(), + 200, + "seeding: owner list_certs must return 200" + ); + let listed: serde_json::Value = resp.json().await.expect("list_certs returns JSON"); + listed["certificates"][0]["id"] + .as_str() + .expect("a ref-certificate must have been issued by the push") + .to_string() +} + +/// Who signs a probe request. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Signer { + Anon, + Owner, + Stranger, + /// The PR/issue author arm of a multi-principal close gate. + Author, + /// The bounty creator arm of dispute_bounty. + Creator, + /// The bounty claimant arm of dispute_bounty. + Claimant, + /// The task assignee arm of complete_task / fail_task (#195, F3). + Assignee, +} + +/// The signer identity that grants a given multi-principal arm. The runtime sweep +/// resolves each to its fixture keypair; the structural consistency test uses it +/// to confirm every declared arm has a twin signed by ITS identity. +pub fn signer_for_principal(p: Principal) -> Signer { + match p { + Principal::Owner => Signer::Owner, + Principal::Author => Signer::Author, + Principal::Creator => Signer::Creator, + Principal::Claimant => Signer::Claimant, + Principal::Assignee => Signer::Assignee, + } +} + +/// What a probe expects. +#[derive(Debug, Clone)] +pub enum Expect { + /// Hostile request: the exact deny status, body leaking none of `withheld`. + Deny(u16), + /// Owner-gate twin: the owner reaches the handler, so NOT 403. + Not403, + /// Read-gate twin: an authorized read returns a non-empty 2xx (an empty 2xx + /// would be a denial rendered as success). + Ok2xx, +} + +/// A single request to drive against the node. +#[derive(Debug, Clone)] +pub struct Probe { + pub label: String, + pub method: reqwest::Method, + /// Path used both for the URL and for RFC-9421 `@path` signing. + pub path: String, + pub body: Vec, + pub signer: Signer, + pub json: bool, + pub expect: Expect, + /// Private tokens (secret content, blob OID) that the DENY body must not echo. + /// Non-empty only for read-gate hostile probes, where a private repo/blob is + /// actually seeded; the owner-gate (403) and signature (401) hostile probes + /// fire on NO_ENTITY repos before any entity lookup, so there is genuinely + /// nothing seeded to leak and an empty list is the honest assertion there. + pub withheld: Vec, +} + +/// Substitute path-template placeholders from the fixture. `repo` is the public +/// repo for owner-gate rows and the private repo for read-gate rows. +/// +/// `id_source` selects what `{id}` resolves to: `Fixed` uses the static seed id +/// `"1"` (entities seeded at #1), while a per-row source such as `BountyId` +/// injects a UUID minted at seed time. This is the general id-threading hook U3 +/// reuses for the other id-keyed reads (a cert id, etc.): add an `IdSource` +/// variant + fixture field and map it here. +fn fill(path: &str, fixture: &Fixture, repo: &str, id_source: IdSource) -> String { + let id = match id_source { + IdSource::Fixed => "1", + IdSource::BountyId => fixture.bounty_id.as_str(), + IdSource::IssueId => fixture.issue_id.as_str(), + IdSource::PrivIssueId => fixture.priv_issue_id.as_str(), + IdSource::PrivBountyId => fixture.priv_bounty_id.as_str(), + IdSource::CertId => fixture.cert_id.as_str(), + // #195 (N2): the status-gated bounty mutations, each keyed to the bounty + // seeded in its gate-reaching status. + IdSource::ClaimedBountyId => fixture.claimed_bounty_id.as_str(), + IdSource::SubmittedBountyId => fixture.submitted_bounty_id.as_str(), + IdSource::OpenBountyId => fixture.open_bounty_id.as_str(), + // #195 (F3): the caller-self task rows, each keyed to a seeded task in the + // state its gate needs (pending for claim, claimed for complete/fail). + IdSource::ClaimableTaskId => fixture.claimable_task_id.as_str(), + IdSource::CompletableTaskId => fixture.completable_task_id.as_str(), + IdSource::FailableTaskId => fixture.failable_task_id.as_str(), + }; + path.replace("{owner}", &fixture.owner_did) + .replace("{repo}", repo) + .replace("{number}", "1") + .replace("{id}", id) + .replace("{label}", "bug") + .replace("{branch}", "main") + .replace("{*path}", &fixture.content_path) + .replace("{path}", &fixture.content_path) +} + +/// Expand one deny-bearing row into its hostile probe plus positive twin. +pub fn probes_for(row: &Row, fixture: &Fixture) -> Vec { + let method: reqwest::Method = row.method.parse().expect("valid method"); + let body = row.body.map(|b| b.as_bytes().to_vec()).unwrap_or_default(); + let json = row.body.is_some(); + + match row.gate { + GateClass::OwnerGate => { + let path = fill(row.path, fixture, &fixture.public_repo, row.id_source); + vec![ + Probe { + label: format!("{} owner-gate hostile", row.handler), + method: method.clone(), + path: path.clone(), + body: body.clone(), + signer: Signer::Stranger, + json, + expect: Expect::Deny(403), + // No entity is seeded on the public owner-gate substrate before + // the 403 fires, so there is nothing private to leak here. + withheld: Vec::new(), + }, + Probe { + label: format!("{} owner-reachability twin", row.handler), + method, + path, + body, + signer: Signer::Owner, + json, + expect: Expect::Not403, + withheld: Vec::new(), + }, + ] + } + GateClass::MultiPrincipalGate => { + // #195 (F1): a single stranger-403 hostile plus one Not403 twin PER + // declared arm. Each arm is signed by ITS distinct fixture identity, so + // reverting any single arm in the handler turns that arm's twin RED + // while the others (and the stranger 403) stay green. + let path = fill(row.path, fixture, &fixture.public_repo, row.id_source); + let mut v = vec![Probe { + label: format!("{} multi-principal hostile (stranger)", row.handler), + method: method.clone(), + path: path.clone(), + body: body.clone(), + signer: Signer::Stranger, + json, + expect: Expect::Deny(403), + // The gate fires before returning any entity; nothing private is + // seeded to leak on this substrate. + withheld: Vec::new(), + }]; + for arm in row.principals { + let signer = signer_for_principal(*arm); + v.push(Probe { + label: format!("{} arm twin ({:?})", row.handler, arm), + method: method.clone(), + path: path.clone(), + body: body.clone(), + signer, + json, + expect: Expect::Not403, + withheld: Vec::new(), + }); + } + v + } + GateClass::ReadGate => { + let path = fill(row.path, fixture, &fixture.private_repo, row.id_source); + // INV-8: the private repo's blob and its OID are actually seeded here, so + // the 404 body must echo neither the secret content nor the blob OID + // (full or short) nor the private repo's internal id (#195, F4). An empty + // withheld list would let a 404 that spilled the private content pass as + // a clean denial. + // #195 (R3/KTD-4): the fixed blob-scoped withheld set PLUS every seeded + // sub-entity marker. The new reads (get_issue/get_pr/get_pr_diff/get_cert + // /get_bounty and the list rows off them) return their OWN private + // content, whose markers are NOT in the blob set — a 404 that leaks an + // issue title or PR diff must fail, so the per-read markers are withheld + // from every read-gate hostile. A status-only 404 check would be vacuous. + let mut withheld = vec![ + fixture.private_secret.clone(), + fixture.private_blob_oid.clone(), + fixture.private_blob_oid_short.clone(), + fixture.private_repo_id.clone(), + ]; + withheld.extend(fixture.priv_markers.iter().cloned()); + // The anon leg: an `Option`-auth read existence-hides with a 404, but an + // auth-required read (`AnonRead::Deny401`, e.g. list_webhooks) rejects a + // headerless caller 401 BEFORE any lookup. The signed-non-reader 404 + // below is the existence-hiding gate either way; only this leg differs. + let anon_status = match row.anon_read { + AnonRead::Deny404 => 404, + AnonRead::Deny401 => 401, + }; + let mut v = vec![ + Probe { + label: format!("{} read-gate hostile (anon -> {anon_status})", row.handler), + method: method.clone(), + path: path.clone(), + body: body.clone(), + signer: Signer::Anon, + json, + expect: Expect::Deny(anon_status), + withheld: withheld.clone(), + }, + // #195 (F1): a signed NON-READER must also get the existence-hiding + // 404 — a signature is not access (INV-12). Without this probe, a + // regression that treats any valid signature as authorized would leak + // the private repo while the anon probe stayed green. + Probe { + label: format!("{} read-gate hostile (signed non-reader)", row.handler), + method: method.clone(), + path: path.clone(), + body: body.clone(), + signer: Signer::Stranger, + json, + expect: Expect::Deny(404), + withheld: withheld.clone(), + }, + ]; + // Positive twin: prove the 404 is the gate, not an absent entity. + let twin = match row.reach { + Reach::ReaderReads => Probe { + label: format!("{} read-reachability twin (owner)", row.handler), + method, + path, + body, + signer: Signer::Owner, + json, + expect: Expect::Ok2xx, + // The twin is a GRANTED read; it is expected to return the + // content, so nothing is withheld from it. + withheld: Vec::new(), + }, + Reach::SiblingPublic(sibling) => Probe { + label: format!("{} read-reachability twin (sibling public)", row.handler), + method, + path: fill(sibling, fixture, &fixture.private_repo, row.id_source), + body, + signer: Signer::Anon, + json, + expect: Expect::Ok2xx, + withheld: Vec::new(), + }, + Reach::None => panic!( + "read-gate row {} {} has no Reach twin (U1 consistency should have caught this)", + row.method, row.path + ), + }; + v.push(twin); + v + } + GateClass::SignatureRequired => { + let path = fill(row.path, fixture, &fixture.public_repo, row.id_source); + vec![Probe { + label: format!("{} signature-required hostile", row.handler), + method, + path, + body, + signer: Signer::Anon, + json, + expect: Expect::Deny(401), + // The 401 fires at the signature layer before any repo/entity is + // looked up (NO_ENTITY substrate), so nothing private is in scope. + withheld: Vec::new(), + }] + } + GateClass::SignerSelfGate => { + // #195 (F3, KTD-1): a body-field caller-self gate. The shared body runs + // through `fill()` so `{owner}` resolves to the real owner DID; hostile + // and control differ ONLY in signer. Hostile: a Stranger signs but the + // body names the owner -> `did_matches(stranger, owner) == false` -> 403. + // Control: the Owner signs and the body names the owner -> match -> + // Not403. (The repo arg to `fill` is inert here: task/register paths + // carry no `{repo}` placeholder, but the signature requires it.) + let path = fill(row.path, fixture, &fixture.public_repo, row.id_source); + let filled_body = row + .body + .map(|b| fill(b, fixture, &fixture.public_repo, row.id_source).into_bytes()) + .unwrap_or_default(); + vec![ + Probe { + label: format!("{} signer-self hostile (stranger)", row.handler), + method: method.clone(), + path: path.clone(), + body: filled_body.clone(), + signer: Signer::Stranger, + json, + expect: Expect::Deny(403), + // Each gate returns a fixed forbidden() message before serializing + // any entity, and nothing private is seeded on this substrate, so + // an empty withheld list is the honest assertion (KTD-4). + withheld: Vec::new(), + }, + Probe { + label: format!("{} signer-self control (owner)", row.handler), + method, + path, + body: filled_body, + signer: Signer::Owner, + json, + expect: Expect::Not403, + withheld: Vec::new(), + }, + ] + } + } +} + +/// A DB-less `Fixture` for the pure-generator unit tests (here and the structural +/// consistency test in `routes.rs`). No node is seeded, so the identities are +/// generated keypairs with placeholder repo/secret metadata; only the arm DIDs +/// and the injected `{id}` matter to `probes_for`. Shared so both test modules +/// build the same shape. +#[cfg(test)] +pub mod tests_support { + use super::*; + + pub fn fx() -> Fixture { + let author = Keypair::generate(); + let creator = Keypair::generate(); + let claimant = Keypair::generate(); + let assignee = Keypair::generate(); + Fixture { + author_did: author.did().to_string(), + creator_did: creator.did().to_string(), + claimant_did: claimant.did().to_string(), + assignee_did: assignee.did().to_string(), + author, + creator, + claimant, + assignee, + bounty_id: "bounty-uuid-1234".to_string(), + claimed_bounty_id: "claimed-bounty-uuid-1234".to_string(), + submitted_bounty_id: "submitted-bounty-uuid-1234".to_string(), + open_bounty_id: "open-bounty-uuid-1234".to_string(), + claimable_task_id: "claimable-task-uuid-1234".to_string(), + completable_task_id: "completable-task-uuid-1234".to_string(), + failable_task_id: "failable-task-uuid-1234".to_string(), + issue_id: "issue-uuid-1234".to_string(), + priv_issue_id: "priv-issue-uuid-1234".to_string(), + priv_bounty_id: "priv-bounty-uuid-1234".to_string(), + cert_id: "cert-uuid-1234".to_string(), + priv_markers: vec![ + "TOPSECRET-ISSUE-U3".to_string(), + "TOPSECRET-PRTITLE-U3".to_string(), + "TOPSECRET-PRDIFF-U3".to_string(), + "TOPSECRET-BOUNTY-U3".to_string(), + ], + owner: Keypair::generate(), + stranger: Keypair::generate(), + owner_did: "did:key:zOWNER".to_string(), + public_repo: "prober-pub".to_string(), + public_repo_id: "pub-id".to_string(), + private_repo: "prober-priv".to_string(), + private_repo_id: "priv-repo-uuid-0BADF00D".to_string(), + content_path: "public/a.txt".to_string(), + private_secret: "TOPSECRET-PRIVREAD-U3".to_string(), + private_blob_oid: "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef".to_string(), + private_blob_oid_short: "deadbeefdead".to_string(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::support::routes::{GateClass, Reach, Row}; + + use super::tests_support::fx; + + #[test] + fn owner_gate_row_yields_stranger_403_and_owner_twin() { + let row = Row { + method: "PUT", + path: "/api/v1/repos/{owner}/{repo}/visibility", + gate: GateClass::OwnerGate, + handler: "visibility::set_visibility", + body: Some(r#"{"path_glob":"/"}"#), + needs: &[], + reach: Reach::None, + principals: &[], + id_source: crate::support::routes::IdSource::Fixed, + anon_read: crate::support::routes::AnonRead::Deny404, + }; + let ps = probes_for(&row, &fx()); + assert_eq!(ps.len(), 2); + assert_eq!(ps[0].signer, Signer::Stranger); + assert!(ps[0].json); + assert!(matches!(ps[0].expect, Expect::Deny(403))); + assert!( + ps[0].path.contains("prober-pub"), + "owner-gate uses the public repo" + ); + assert_eq!(ps[1].signer, Signer::Owner); + assert!(matches!(ps[1].expect, Expect::Not403)); + // Owner-gate hostile fires on a NO_ENTITY public repo before any lookup, so + // there is nothing private seeded to leak: an empty withheld list is honest. + assert!( + ps[0].withheld.is_empty(), + "owner-gate hostile probe seeds no private entity, so withholds nothing" + ); + } + + #[test] + fn read_gate_row_yields_anon_and_stranger_404_and_positive_twin() { + let row = Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/blob/{*path}", + gate: GateClass::ReadGate, + handler: "repos::get_blob", + body: None, + needs: &[], + reach: Reach::ReaderReads, + principals: &[], + id_source: crate::support::routes::IdSource::Fixed, + anon_read: crate::support::routes::AnonRead::Deny404, + }; + let f = fx(); + let ps = probes_for(&row, &f); + // anon hostile, signed-stranger hostile (#195 F1), owner twin. + assert_eq!(ps.len(), 3); + assert_eq!(ps[0].signer, Signer::Anon); + assert!(matches!(ps[0].expect, Expect::Deny(404))); + // #195 (F1): the signed non-reader hostile must also expect a 404 no-leak. + assert_eq!(ps[1].signer, Signer::Stranger); + assert!(matches!(ps[1].expect, Expect::Deny(404))); + assert!( + !ps[1].withheld.is_empty(), + "the signed-stranger hostile must carry the same withheld tokens" + ); + assert!( + ps[0].path.contains("prober-priv"), + "read-gate uses the private repo" + ); + assert!(ps[0].path.contains("public/a.txt"), "{{*path}} substituted"); + // F2 belt-and-suspenders: the read-gate hostile probe MUST carry the private + // tokens, so a future refactor that drops them (reverting F2 to a vacuous + // empty-withheld 404 check) fails loudly here. + assert!( + !ps[0].withheld.is_empty(), + "read-gate hostile probe must carry withheld private tokens" + ); + assert!( + ps[0].withheld.contains(&f.private_secret), + "the seeded secret content must be a withheld token" + ); + assert!( + ps[0].withheld.contains(&f.private_blob_oid), + "the private blob OID must be a withheld token" + ); + assert!( + ps[0].withheld.contains(&f.private_blob_oid_short), + "the short blob OID prefix must be a withheld token" + ); + assert_eq!(ps[2].signer, Signer::Owner); + assert!(matches!(ps[2].expect, Expect::Ok2xx)); + // The granted twin returns the content, so it withholds nothing. + assert!( + ps[2].withheld.is_empty(), + "the granted read twin must not carry withheld tokens" + ); + } + + #[test] + fn signature_row_yields_unsigned_401_no_twin() { + let row = Row { + method: "POST", + path: "/{owner}/{repo}/git-receive-pack", + gate: GateClass::SignatureRequired, + handler: "repos::git_receive_pack", + body: None, + needs: &[], + reach: Reach::None, + principals: &[], + id_source: crate::support::routes::IdSource::Fixed, + anon_read: crate::support::routes::AnonRead::Deny404, + }; + let ps = probes_for(&row, &fx()); + assert_eq!(ps.len(), 1); + assert_eq!(ps[0].signer, Signer::Anon); + assert!(matches!(ps[0].expect, Expect::Deny(401))); + // The 401 fires before any entity lookup, so nothing private is in scope. + assert!( + ps[0].withheld.is_empty(), + "signature-required hostile probe withholds nothing (fires pre-lookup)" + ); + } + + #[test] + fn read_gate_sibling_public_twin_reads_the_sibling_path_anon() { + let row = Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/blob/secret/x.txt", + gate: GateClass::ReadGate, + handler: "repos::get_blob", + body: None, + needs: &[], + reach: Reach::SiblingPublic("/api/v1/repos/{owner}/{repo}/blob/{*path}"), + principals: &[], + id_source: crate::support::routes::IdSource::Fixed, + anon_read: crate::support::routes::AnonRead::Deny404, + }; + let ps = probes_for(&row, &fx()); + // anon hostile, signed-stranger hostile (#195 F1), sibling-public twin. + assert_eq!(ps.len(), 3); + // Hostile: anon on the withheld path -> 404. + assert_eq!(ps[0].signer, Signer::Anon); + assert!(matches!(ps[0].expect, Expect::Deny(404))); + assert!(ps[0].path.ends_with("/blob/secret/x.txt")); + // Signed non-reader on the withheld path -> 404 too. + assert_eq!(ps[1].signer, Signer::Stranger); + assert!(matches!(ps[1].expect, Expect::Deny(404))); + assert!(ps[1].path.ends_with("/blob/secret/x.txt")); + // Twin: anon on the sibling PUBLIC path -> non-empty 2xx (proves the 404 + // is path-scoped withholding, not a blanket refusal). + assert_eq!(ps[2].signer, Signer::Anon); + assert!(matches!(ps[2].expect, Expect::Ok2xx)); + assert!( + ps[2].path.contains("public/a.txt"), + "sibling template's {{*path}} is filled from the fixture content path" + ); + } + + // F2 RED proof at the check_denied level (no DB): feed a synthetic denial body + // that DOES contain the private secret through the SAME withheld tokens the + // read-gate probe carries. If the tokens are truly threaded and non-empty, the + // deny check must reject it as a leak. This is the fail case a real node's + // clean 404 avoids; it proves the sweep would actually catch a leaking 404. + #[test] + fn read_gate_withheld_tokens_reject_a_leaking_denial_body() { + use crate::support::assert::check_denied; + + let row = Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}", + gate: GateClass::ReadGate, + handler: "repos::get_repo", + body: None, + needs: &[], + reach: Reach::ReaderReads, + principals: &[], + id_source: crate::support::routes::IdSource::Fixed, + anon_read: crate::support::routes::AnonRead::Deny404, + }; + let f = fx(); + let ps = probes_for(&row, &f); + let tokens: Vec<&str> = ps[0].withheld.iter().map(String::as_str).collect(); + assert!( + !tokens.is_empty(), + "precondition: read-gate probe carries tokens" + ); + + // A hostile 404 that spilled the secret content: must be rejected as a leak. + let leaking = format!( + r#"{{"error":"blob {} at {}"}}"#, + f.private_secret, f.private_blob_oid + ); + let r = check_denied(404, &leaking, 404, &tokens); + assert!( + r.is_err(), + "a 404 body echoing the secret must be flagged: {r:?}" + ); + assert!( + r.unwrap_err().contains(&f.private_secret), + "the failure must name the leaked secret token" + ); + + // #195 (F4): a 404 that serialized the private repo's internal id (UUID) is + // a repository-metadata leak and must be flagged too — proving the UUID is a + // load-bearing withheld token, not just the blob content. + let leaking_meta = format!(r#"{{"error":"repo {} not found"}}"#, f.private_repo_id); + let rm = check_denied(404, &leaking_meta, 404, &tokens); + assert!( + rm.is_err(), + "a 404 body echoing the private repo id must be flagged: {rm:?}" + ); + assert!( + rm.unwrap_err().contains(&f.private_repo_id), + "the failure must name the leaked private repo id token" + ); + + // A clean 404 (no private tokens) with the same tokens passes: the tokens do + // not spuriously trip on an honest denial. + let clean = check_denied(404, r#"{"error":"repository not found"}"#, 404, &tokens); + assert!( + clean.is_ok(), + "a clean 404 must pass the withheld check: {clean:?}" + ); + } + + #[test] + #[should_panic(expected = "has no Reach twin")] + fn read_gate_row_without_a_reach_twin_panics() { + // A read-gate row with Reach::None is a registry bug the U1 consistency + // test rejects; if one slips through, the probe generator must fail loud + // rather than silently emit a hostile probe with no positive twin. + let row = Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/issues", + gate: GateClass::ReadGate, + handler: "issues::list_issues", + body: None, + needs: &[], + reach: Reach::None, + principals: &[], + id_source: crate::support::routes::IdSource::Fixed, + anon_read: crate::support::routes::AnonRead::Deny404, + }; + let _ = probes_for(&row, &fx()); + } +} diff --git a/crates/gitlawb-node/tests/support/routes.rs b/crates/gitlawb-node/tests/support/routes.rs new file mode 100644 index 00000000..c898361b --- /dev/null +++ b/crates/gitlawb-node/tests/support/routes.rs @@ -0,0 +1,1210 @@ +//! U1: the deny-bearing route set for the invariant deny-prober. +//! +//! Every route that carries a runtime deny to assert lives here: owner-gate +//! (403), multi-principal (403), read-gate (404), signature-required (401), and +//! caller-self / actor (403): `SignerSelfGate` for the request-body field +//! bindings (create_task delegator, claim_task assignee, register did) and +//! `MultiPrincipalGate` with the assignee arm for the complete/fail actor gates. +//! The completeness cross-check in `deny_harness.rs` derives the caller-self set +//! from `src/api` and requires each `did_matches(caller, X)` handler to be a +//! driven row, so a new one cannot silently escape the sweep. +//! +//! Scope: this doc claims the REST/api surface only. The parallel GraphQL +//! mutation caller-self gates (`src/graphql/mutation.rs`) are out of this +//! harness and fenced separately (#219). +//! +//! Each row is classified by READING its handler, never inferred from the route +//! name, and records the handler fn name so a reviewer can spot-check. Guessing +//! is how a false test enters the sweep (the register_replica lesson: it looks +//! like an owner-gate, it is signer-self; and list_visibility is a 403 +//! owner-gate despite being a GET). +//! +//! Rows are declarative (gate class + entities + reachability), not prebuilt +//! requests, so Flavor B (the cross-surface differential prober) can reuse them +//! (R9). + +/// The gate class of a deny-bearing route and the exact status its deny emits. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum GateClass { + /// `require_repo_owner` / `require_owner` / inline `did_matches(caller, owner)` + /// -> `AppError::Forbidden` == 403. Probed with a validly-signed non-owner. + OwnerGate, + /// A 403 gate with ONE OR MORE NAMED authorizing principals (owner-OR-author, + /// creator-OR-claimant, claimant-only, …). Probed with a validly-signed + /// stranger (403) plus one Not403 twin per declared arm (`Row.principals`), + /// so reverting ANY single arm turns that arm's twin RED (#195, F1). + /// `OwnerGate` is the owner-arm special case; a single-arm gate whose + /// principal is NOT the repo owner (submit_bounty's claimant-only check, + /// approve/cancel's creator-only check) belongs here, because the point is + /// that the arms are DECLARED and each one gets its own twin (#195, N2). + MultiPrincipalGate, + /// `authorize_repo_read` / `visibility_check` -> `RepoNotFound` == 404 + /// (existence-hiding). Probed with a non-reader against a private/withheld + /// target. A 404 alone is ambiguous (a missing entity also 404s), so every + /// read-gate row carries a `Reach` positive twin. + ReadGate, + /// `require_signature` layer -> 401 on the git write routes. Probed unsigned. + SignatureRequired, + /// A caller-self 403 gate that binds a DID field in the request BODY to the + /// authenticated signer (`did_matches(caller, &body._did)`): create_task + /// (delegator_did), claim_task (assignee_did), register (did). Probed with a + /// validly-signed stranger whose body names the OWNER (mismatch -> 403) plus an + /// owner control whose body names the owner (match -> Not403). Distinct from + /// `OwnerGate`: the bound identity is a body field, not the stored repo owner, + /// and the completeness classifier keys on exactly that (#195, F3, KTD-1). The + /// actor-vs-stored caller-self gates (complete_task/fail_task) are the + /// `MultiPrincipalGate` assignee-arm shape instead, not this class. + SignerSelfGate, +} + +impl GateClass { + pub fn expected_status(self) -> u16 { + match self { + GateClass::OwnerGate | GateClass::MultiPrincipalGate | GateClass::SignerSelfGate => 403, + GateClass::ReadGate => 404, + GateClass::SignatureRequired => 401, + } + } +} + +/// An authorizing principal (arm) of a multi-principal 403 gate. Each maps to a +/// distinct fixture identity in `probe.rs` so reverting one arm cannot silently +/// re-collapse onto another (the original bug seeded `author == owner`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Principal { + /// The repo owner (`require_repo_owner` / `did_matches(caller, owner)`). + Owner, + /// The PR/issue author (`did_matches(caller, author_did)`). + Author, + /// The bounty creator (`did_matches(caller, creator_did)`). + Creator, + /// The bounty claimant (`did_matches(caller, claimant_did)`). + Claimant, + /// The task assignee (`did_matches(caller, stored assignee_did)`): the actor + /// arm of complete_task / fail_task (#195, F3). Unlike the field-binding + /// `SignerSelfGate`, the identity is the task's STORED assignee, so the arm is + /// driven against a seeded claimed task, not a body field. + Assignee, +} + +/// How a read-gate row's positive twin proves the 404 is the gate and not a +/// merely-absent entity/repo. Owner-gate and signature rows use `None` +/// (the owner-gate twin is the same request re-signed as the owner, handled by +/// the probe generator from the class, not a path). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Reach { + /// Not a read-gate row; no read-twin. + None, + /// The authorized reader/owner issues the same request and gets 2xx. + ReaderReads, + /// A sibling PUBLIC path on the same repo returns 2xx-with-content, proving + /// the 404 is path-scoped withholding, not a blanket/absent 404. Mirrors the + /// existing U7/U8/anon_ipfs cases in `tests/deny_harness.rs`. + SiblingPublic(&'static str), +} + +/// How a row's `{id}` placeholder is filled. Most id-keyed paths use a fixed +/// seed id (`"1"`), but some entities are keyed by a UUID minted at seed time +/// (a bounty, a cert), so the fixture must inject the captured id per row. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum IdSource { + /// `{id}` -> the fixed seed id `"1"` (static-template entities seeded at #1). + Fixed, + /// `{id}` -> the fixture's seeded disputable bounty id (UUID). + BountyId, + /// `{id}` -> the fixture's seeded issue id (UUID). The issue is stored as a + /// full git-JSON `IssueRecord` (authored by the fixture author), which a + /// bare `{"author":..}` seed cannot satisfy — close_issue deserializes the + /// whole record before reading its author, so the author arm needs the + /// complete record. + IssueId, + /// `{id}` -> the fixture's seeded PRIVATE-repo issue id (UUID). Distinct from + /// `IssueId` (the public-repo close-gate issue): the get_issue / + /// list_issue_comments read-gate rows run against the PRIVATE repo, so their + /// `{id}` must be an issue that actually exists there (#195, F2/U3). + PrivIssueId, + /// `{id}` -> the fixture's seeded PRIVATE-repo bounty id (UUID). get_bounty is + /// NOT repo-scoped in its path (`/api/v1/bounties/{id}`) but read-gates on the + /// bounty's own repo; the bounty is seeded against the private repo so anon / + /// stranger get the existence-hiding 404 and the owner gets 2xx (#195, U3). + PrivBountyId, + /// `{id}` -> the fixture's seeded ref-certificate id (UUID), issued by a real + /// owner push to the private repo. get_cert read-gates the private repo on "/" + /// then fetches the cert by id, so the owner-2xx twin needs a real cert whose + /// repo_id is the private repo's (#195, U3). + CertId, + /// `{id}` -> the fixture's bounty seeded in status "claimed" (#195, N2). + /// submit_bounty checks status BEFORE auth (bounties.rs:294), so the stranger + /// only reaches the claimant 403 gate on a bounty that is already "claimed"; + /// the claimant twin then consumes that status (claimed -> submitted), so the + /// row needs its own bounty. + ClaimedBountyId, + /// `{id}` -> the fixture's bounty driven to status "submitted" (#195, N2). + /// approve_bounty's status check fires before its creator gate + /// (bounties.rs:334), so only a "submitted" bounty reaches the 403; the + /// creator twin consumes it (submitted -> completed). + SubmittedBountyId, + /// `{id}` -> the fixture's bounty left in status "open" (#195, N2). + /// cancel_bounty's status check fires before its creator gate + /// (bounties.rs:381), so only an "open" bounty reaches the 403; the creator + /// twin consumes it (open -> cancelled). + OpenBountyId, + /// `{id}` -> the fixture's task seeded "pending" (#195, F3). The claim_task + /// caller-self gate fires before the DB claim, so the stranger 403 does not + /// consume it; the owner control then claims it (pending -> claimed). + ClaimableTaskId, + /// `{id}` -> the fixture's task seeded "claimed" by the assignee (#195, F3). + /// complete_task's assignee actor gate reaches it; the assignee twin then + /// finishes it (claimed -> completed). + CompletableTaskId, + /// `{id}` -> a second assignee-claimed task (#195, F3), so fail_task's twin + /// (claimed -> failed) does not race complete_task for one entity. + FailableTaskId, +} + +/// One deny-bearing route. `path` is a template with `{owner}`/`{repo}`/`{id}` +/// /`{number}`/`{*path}` placeholders the probe generator (U2) fills from the +/// fixture. `needs` names the seeded sub-entities the path requires (empty for +/// owner-gate rows: the gate fires before the entity lookup). +#[derive(Debug, Clone, Copy)] +pub struct Row { + pub method: &'static str, + pub path: &'static str, + pub gate: GateClass, + /// Handler fn name, recorded for review spot-check (per-row verification). + pub handler: &'static str, + /// Sample request body (JSON) or `None`. JSON bodies drive the + /// `Content-Type: application/json` the probe attaches to clear the extractor. + pub body: Option<&'static str>, + /// Seeded sub-entities the path template consumes. + pub needs: &'static [&'static str], + /// Positive-twin strategy for read-gate rows. + pub reach: Reach, + /// For `MultiPrincipalGate` rows, the authorizing arms this gate must drive + /// (one Not403 twin each). Empty for every other class — only meaningful for + /// `MultiPrincipalGate`, and the consistency test enforces that pairing. + pub principals: &'static [Principal], + /// How the `{id}` placeholder is filled for this row (see [`IdSource`]). + pub id_source: IdSource, + /// For `ReadGate` rows, the status an ANONYMOUS (headerless) caller gets on + /// the withheld target. Most read GETs take `Option` auth and existence-hide + /// with a 404 (`Deny404`), but a read GET that *requires* auth rejects a + /// headerless caller with 401 BEFORE any lookup (`Deny401`) — e.g. + /// `list_webhooks`, whose callback URLs are owner-secret config with no anon + /// form. The signed-non-reader 404 (existence-hiding) is unchanged either way; + /// only the anon leg differs. Meaningless on non-`ReadGate` rows, which set + /// `Deny404` as the not-applicable sentinel (mirrors `reach: Reach::None`). + /// A new auth-required read added as a `ReadGate` row that forgets `Deny401` + /// fails loudly (its anon probe gets 401, not the expected 404), so the field + /// is self-correcting rather than a silent default. + pub anon_read: AnonRead, +} + +/// The status class an anonymous caller gets from a `ReadGate` row's withheld +/// target — 404 existence-hiding (the default) or a 401 auth-required rejection. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AnonRead { + /// `Option`-auth read: anon gets the existence-hiding 404, same as a signed + /// non-reader. The default for every read GET that has an anonymous form. + Deny404, + /// Auth-required read: a headerless caller is rejected 401 before any lookup + /// (no anonymous form). The signed-non-reader still gets the 404. + Deny401, +} + +const NO_ENTITY: &[&str] = &[]; +const NO_PRINCIPAL: &[Principal] = &[]; + +/// The deny-bearing route set. Owner-gate and signature-required tranches are +/// fully verified against their handlers this session; the read-gate tranche is +/// populated as each handler's deny path (404-deny vs list-filter) is verified. +pub fn deny_bearing_routes() -> &'static [Row] { + &[ + // ── Owner-gate (403) — verified: each calls require_repo_owner / + // require_owner / inline did_matches against the repo owner. ────────── + Row { + method: "POST", + path: "/api/v1/repos/{owner}/{repo}/pulls/{number}/merge", + gate: GateClass::OwnerGate, + handler: "pulls::merge_pr", + body: None, + needs: &["pr_number"], + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + // #195 (F1): close_pr / close_issue are owner-OR-author gates (pulls.rs:276, + // issues.rs:255). A plain OwnerGate tests only the owner arm, so reverting + // the author arm is invisible. MultiPrincipalGate drives BOTH arms. + Row { + method: "POST", + path: "/api/v1/repos/{owner}/{repo}/pulls/{number}/close", + gate: GateClass::MultiPrincipalGate, + handler: "pulls::close_pr", + body: None, + needs: &["pr_number"], + reach: Reach::None, + principals: &[Principal::Owner, Principal::Author], + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "POST", + path: "/api/v1/repos/{owner}/{repo}/issues/{id}/close", + gate: GateClass::MultiPrincipalGate, + handler: "issues::close_issue", + body: None, + needs: &["issue_id"], + reach: Reach::None, + principals: &[Principal::Owner, Principal::Author], + id_source: IdSource::IssueId, + anon_read: AnonRead::Deny404, + }, + // #195 (F1): dispute_bounty is creator-OR-claimant (bounties.rs:425). Not + // repo-scoped: it gates on the bounty's creator/claimant, so the row is + // id-keyed by the seeded disputable bounty's UUID (IdSource::BountyId). + Row { + method: "POST", + path: "/api/v1/bounties/{id}/dispute", + gate: GateClass::MultiPrincipalGate, + handler: "bounties::dispute_bounty", + body: None, + needs: &["bounty_id"], + reach: Reach::None, + principals: &[Principal::Creator, Principal::Claimant], + id_source: IdSource::BountyId, + anon_read: AnonRead::Deny404, + }, + // #195 (N2): submit/approve/cancel each 403 a non-claimant/non-creator, but + // their STATUS check fires before the auth check, so each row gets its own + // bounty seeded in the gate-reaching status (claimed / submitted / open). + // The sweep drives probes in emission order (hostile first, probes_for), + // so the stranger 403 fires while the bounty still holds that status; the + // arm twin then consumes it (submit: claimed -> submitted, approve: + // submitted -> completed, cancel: open -> cancelled), which is why the + // rows cannot share a bounty with each other or with dispute_bounty. + Row { + method: "POST", + path: "/api/v1/bounties/{id}/submit", + gate: GateClass::MultiPrincipalGate, + handler: "bounties::submit_bounty", + // pr_id is a free-text column (no FK); any string clears serde and + // the UPDATE. The gate fires before the value is used. + body: Some(r#"{"pr_id":"1"}"#), + needs: &["claimed_bounty_id"], + reach: Reach::None, + principals: &[Principal::Claimant], + id_source: IdSource::ClaimedBountyId, + anon_read: AnonRead::Deny404, + }, + Row { + method: "POST", + path: "/api/v1/bounties/{id}/approve", + gate: GateClass::MultiPrincipalGate, + handler: "bounties::approve_bounty", + // ApproveBountyRequest's only field (tx_hash) is Option, so an empty + // JSON object clears the Json extractor; a missing body would 4xx at + // the extractor, before the gate under test. + body: Some(r#"{}"#), + needs: &["submitted_bounty_id"], + reach: Reach::None, + principals: &[Principal::Creator], + id_source: IdSource::SubmittedBountyId, + anon_read: AnonRead::Deny404, + }, + Row { + method: "POST", + path: "/api/v1/bounties/{id}/cancel", + gate: GateClass::MultiPrincipalGate, + handler: "bounties::cancel_bounty", + body: None, + needs: &["open_bounty_id"], + reach: Reach::None, + principals: &[Principal::Creator], + id_source: IdSource::OpenBountyId, + anon_read: AnonRead::Deny404, + }, + Row { + method: "POST", + path: "/api/v1/repos/{owner}/{repo}/hooks", + gate: GateClass::OwnerGate, + handler: "webhooks::create_webhook", + body: Some(r#"{"url":"https://e.example/h","events":["*"]}"#), + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "DELETE", + path: "/api/v1/repos/{owner}/{repo}/hooks/{id}", + gate: GateClass::OwnerGate, + handler: "webhooks::delete_webhook", + body: None, + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + // #113: the OWNER layer of list_webhooks (GET /hooks). On the PUBLIC repo a + // signed stranger passes authorize_repo_read (public) and is then denied 403 + // by require_repo_owner — this row drives that 403. The READ layer (404 on + // the private repo, anon 401) is the sibling ReadGate row; the two share the + // path, admitted by the (method, path, gate) dedup key. A GET owner-gate + // like list_visibility below. + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/hooks", + gate: GateClass::OwnerGate, + handler: "webhooks::list_webhooks", + body: None, + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "POST", + path: "/api/v1/repos/{owner}/{repo}/labels", + gate: GateClass::OwnerGate, + handler: "labels::add_label", + body: Some(r#"{"label":"bug"}"#), + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "DELETE", + path: "/api/v1/repos/{owner}/{repo}/labels/{label}", + gate: GateClass::OwnerGate, + handler: "labels::remove_label", + body: None, + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "POST", + path: "/api/v1/repos/{owner}/{repo}/branches/{branch}/protect", + gate: GateClass::OwnerGate, + handler: "protect::protect_branch", + body: None, + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "DELETE", + path: "/api/v1/repos/{owner}/{repo}/branches/{branch}/protect", + gate: GateClass::OwnerGate, + handler: "protect::unprotect_branch", + body: None, + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "PUT", + path: "/api/v1/repos/{owner}/{repo}/visibility", + gate: GateClass::OwnerGate, + handler: "visibility::set_visibility", + body: Some(r#"{"path_glob":"/","reader_dids":[]}"#), + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "DELETE", + path: "/api/v1/repos/{owner}/{repo}/visibility", + gate: GateClass::OwnerGate, + handler: "visibility::remove_visibility", + body: Some(r#"{"path_glob":"/"}"#), + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + // list_visibility is a 403 owner-gate despite being a GET (calls + // require_owner); the /visibility mount chains put+delete+get, all gated. + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/visibility", + gate: GateClass::OwnerGate, + handler: "visibility::list_visibility", + body: None, + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + // ── Signature-required (401) — verified: git write route wrapped by the + // require_signature layer (add_auth_layers in server.rs). ───────────── + Row { + method: "POST", + path: "/{owner}/{repo}/git-receive-pack", + gate: GateClass::SignatureRequired, + handler: "repos::git_receive_pack", + body: None, + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + // ── Read-gate (404) — verified: each handler calls + // authorize_repo_read / visibility_check on "/" which returns + // AppError::RepoNotFound (404) when a non-reader hits a private repo + // (api/mod.rs:44-62). The Reach twin is the owner issuing the same + // request against the private repo and getting 2xx, proving the 404 is + // the gate and not a merely-absent entity/repo. Every row here gates on + // the whole repo ("/"), so ReaderReads (owner re-read) is the twin; + // the fully-private fixture repo needs no per-row sub-entity seeding + // because these reads either return the seeded repo/blob or an empty + // 2xx list. + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}", + gate: GateClass::ReadGate, + handler: "repos::get_repo", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + // #195 (F2): repo-root reads that gate on "/" — driven as real ReadGate rows + // rather than source-only exemptions, so a runtime bypass that keeps the + // authorize_repo_read/visibility_check marker but leaks is caught. + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/star", + gate: GateClass::ReadGate, + handler: "stars::get_star_status", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/icaptcha-proof", + gate: GateClass::ReadGate, + handler: "repos::get_icaptcha_proof", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/encrypted-blobs/replicate", + gate: GateClass::ReadGate, + handler: "encrypted::replicate_encrypted_blobs", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/commits", + gate: GateClass::ReadGate, + handler: "repos::list_commits", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/tree", + gate: GateClass::ReadGate, + handler: "repos::get_tree_root", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + // Path-scoped read: get_blob gates on "/{path}" (repos.rs:390). The + // fully-private fixture repo denies any path to anon (404); the owner + // reads the seeded blob at content_path and gets 2xx bytes. (Path-scoped + // subtree WITHHOLDING on a *public* repo — the SiblingPublic twin — is + // already covered by the U7/U8 cases in deny_harness.rs.) + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/blob/{*path}", + gate: GateClass::ReadGate, + handler: "repos::get_blob", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/refs", + gate: GateClass::ReadGate, + handler: "repos::list_refs", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/issues", + gate: GateClass::ReadGate, + handler: "issues::list_issues", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/labels", + gate: GateClass::ReadGate, + handler: "labels::list_labels", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/certs", + gate: GateClass::ReadGate, + handler: "certs::list_certs", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/events", + gate: GateClass::ReadGate, + handler: "events::list_repo_events", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/pulls", + gate: GateClass::ReadGate, + handler: "pulls::list_prs", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/changelog", + gate: GateClass::ReadGate, + handler: "changelog::get_changelog", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/bounties", + gate: GateClass::ReadGate, + handler: "bounties::list_repo_bounties", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/withheld-paths", + gate: GateClass::ReadGate, + handler: "visibility::withheld_paths", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/encrypted-blobs", + gate: GateClass::ReadGate, + handler: "encrypted::list_encrypted_blobs", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + // #195 (F2, U3): sub-entity reads that gate on "/" (the private repo) but + // require the entity to EXIST at the request path for the owner-2xx twin. + // Each carries its own distinctive marker (seeded in probe.rs) added to the + // per-read withheld set, so a 404 that echoes THAT read's private content + // (issue title, PR title, PR diff, bounty title) fails — a status- + // only 404 check would be vacuous. These were source-only exemptions in + // READ_GATE_NOT_DRIVEN before; they are now driven as real ReadGate rows. + // get_issue: id-keyed by the seeded PRIVATE issue (marker in its title). + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/issues/{id}", + gate: GateClass::ReadGate, + handler: "issues::get_issue", + body: None, + needs: &["priv_issue_id"], + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::PrivIssueId, + anon_read: AnonRead::Deny404, + }, + // list_issue_comments: parent issue must exist (private); child list may be + // empty — an empty `{"comments":[]}` is a non-empty 2xx body. + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/issues/{id}/comments", + gate: GateClass::ReadGate, + handler: "issues::list_issue_comments", + body: None, + needs: &["priv_issue_id"], + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::PrivIssueId, + anon_read: AnonRead::Deny404, + }, + // get_pr: the seeded PRIVATE PR #1 (marker in its title). + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/pulls/{number}", + gate: GateClass::ReadGate, + handler: "pulls::get_pr", + body: None, + needs: &["priv_pr"], + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + // get_pr_diff: the PRIVATE PR #1 has a real `feature` source branch with a + // marker file, so branch_diff_names(main, feature) is NON-EMPTY and the + // owner twin returns a real diff (a bail!/500 if the source ref were + // missing). The per-path visibility_check Deny-return is additionally + // driven by the hand-written get_pr_diff_withheld_path_is_denied test. + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/pulls/{number}/diff", + gate: GateClass::ReadGate, + handler: "pulls::get_pr_diff", + body: None, + needs: &["priv_pr"], + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + // list_reviews / list_comments: parent PR #1 must exist (private); child + // lists may be empty. + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/pulls/{number}/reviews", + gate: GateClass::ReadGate, + handler: "pulls::list_reviews", + body: None, + needs: &["priv_pr"], + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/pulls/{number}/comments", + gate: GateClass::ReadGate, + handler: "pulls::list_comments", + body: None, + needs: &["priv_pr"], + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + // get_cert: id-keyed by a real ref-certificate issued by an owner push to + // the private repo. No cert-content marker is seeded: get_cert read-gates + // the repo on "/" and 404s a non-reader BEFORE fetching the cert, so cert + // fields cannot reach the deny body — the repo-scoped tokens (private_repo_id, + // private_secret) plus the status check carry the no-leak assertion here. + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/certs/{id}", + gate: GateClass::ReadGate, + handler: "certs::get_cert", + body: None, + needs: &["cert_id"], + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::CertId, + anon_read: AnonRead::Deny404, + }, + // get_bounty: NOT repo-scoped in its path, read-gates on the bounty's repo. + // Seeded against the PRIVATE repo (marker in its title), id-keyed. + Row { + method: "GET", + path: "/api/v1/bounties/{id}", + gate: GateClass::ReadGate, + handler: "bounties::get_bounty", + body: None, + needs: &["priv_bounty_id"], + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::PrivBountyId, + anon_read: AnonRead::Deny404, + }, + // get_tree (path-scoped): genuinely ADDITIONAL to get_tree_root (Q1). Root + // gates on "/"; get_tree gates on the REQUESTED subtree (N3, + // authorize_repo_read(&gate_path)), a distinct path-scoped Deny surface. The + // {*path} fills to the private content path, so a non-reader is denied the + // subtree listing and the owner gets a non-empty 2xx. + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/tree/{*path}", + gate: GateClass::ReadGate, + handler: "repos::get_tree", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + // #113: /replicas and /branches/protected gained an `authorize_repo_read` + // gate on "/" — a private repo's replica URLs / protected-branch list are + // now hidden (404) from a non-reader. Both take `Option` auth (anon and a + // signed non-reader alike get the existence-hiding 404), gate on the whole + // repo, and return an empty-or-seeded 2xx list to the owner, so `ReaderReads` + // (owner re-read) is the twin and no per-row sub-entity seeding is needed. + // Driven as real ReadGate rows rather than left in PUBLIC_REPO_GETS, which + // is why the completeness fence flagged them after the #113 merge. + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/replicas", + gate: GateClass::ReadGate, + handler: "replicas::list_replicas", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/branches/protected", + gate: GateClass::ReadGate, + handler: "protect::list_protected_branches", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + // #113: list_webhooks is DOUBLY gated — authorize_repo_read (404) THEN + // require_repo_owner (403) — and it rejects a headerless caller with 401 + // BEFORE the lookup (callback URLs are owner-secret config with no anon + // form). This is the READ layer: a signed non-reader of the private repo + // gets the existence-hiding 404, while an anon caller gets 401 (AnonRead:: + // Deny401). The OWNER layer (403 for a signed reader-non-owner) is the + // sibling OwnerGate row above, run against the public repo where the read + // gate passes and the owner gate is what fires. Two rows share this path + // (one ReadGate, one OwnerGate); the consistency dedup keys on + // (method, path, gate) to admit exactly this dual-gate case. + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/hooks", + gate: GateClass::ReadGate, + handler: "webhooks::list_webhooks", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny401, + }, + // ── Caller-self / actor (403), #195 (F3). The task/register handlers carry + // genuine caller-self 403 gates a bypass would expose (impersonate a + // delegator/assignee, or forge a trust row for a DID you don't control). + // Before this change they were marker-only (INV-21 vacuous); now each is + // driven hostile-then-authorized. The field-binding shape (a body DID + // field must equal the caller) is `SignerSelfGate`; the actor-vs-stored + // shape (caller must equal the task's stored assignee) is + // `MultiPrincipalGate` with the Assignee arm. Register is DRIVEN, not + // excused: the harness runs icaptcha inert (spawn_node never inits the + // verifier), so a stranger-signed POST /api/register reaches the 403. + Row { + method: "POST", + path: "/api/v1/tasks", + gate: GateClass::SignerSelfGate, + handler: "tasks::create_task", + // delegator_did must equal the signer (tasks.rs:101). Stranger names the + // owner -> mismatch -> 403; owner names the owner -> match -> 201. + body: Some(r#"{"kind":"review","capability":"read","delegator_did":"{owner}"}"#), + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "POST", + path: "/api/v1/tasks/{id}/claim", + gate: GateClass::SignerSelfGate, + handler: "tasks::claim_task", + // assignee_did must equal the signer (tasks.rs:174); the gate fires + // before the DB claim, so the stranger 403 leaves the pending task for + // the owner control to claim. + body: Some(r#"{"assignee_did":"{owner}"}"#), + needs: &["claimable_task_id"], + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::ClaimableTaskId, + anon_read: AnonRead::Deny404, + }, + Row { + method: "POST", + path: "/api/register", + gate: GateClass::SignerSelfGate, + handler: "register::register", + // did must equal the signer (register.rs:55). The did must parse + // (register.rs:50) so the 403 is reached, so `{owner}` is a real DID. + // `message` is an unknown field serde ignores. + body: Some(r#"{"did":"{owner}","capabilities":[],"message":"probe"}"#), + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "POST", + path: "/api/v1/tasks/{id}/complete", + gate: GateClass::MultiPrincipalGate, + handler: "tasks::complete_task", + // Caller must equal the task's STORED assignee (tasks.rs:220). Driven + // against a claimed task: stranger -> 403, the assignee arm -> 200. + body: Some(r#"{"result":"ok"}"#), + needs: &["completable_task_id"], + reach: Reach::None, + principals: &[Principal::Assignee], + id_source: IdSource::CompletableTaskId, + anon_read: AnonRead::Deny404, + }, + Row { + method: "POST", + path: "/api/v1/tasks/{id}/fail", + gate: GateClass::MultiPrincipalGate, + handler: "tasks::fail_task", + // Caller must equal the task's STORED assignee (tasks.rs:273). Its own + // claimed task so the assignee twin (claimed -> failed) does not race + // complete_task's. + body: Some(r#"{"reason":"x"}"#), + needs: &["failable_task_id"], + reach: Reach::None, + principals: &[Principal::Assignee], + id_source: IdSource::FailableTaskId, + anon_read: AnonRead::Deny404, + }, + // The read-gate handlers NOT driven here (deferred GET reads, read-gating + // mutations, git smart-HTTP reads, the content-addressed read, and the + // global list-filters) are no longer tracked by free-text prose: they are + // ENFORCED in deny_harness.rs's `READ_GATE_NOT_DRIVEN` allowlist, which the + // `every_read_gate_handler_is_driven_or_explicitly_allowlisted` guard checks + // against a live scan of every read-gate marker in src/api. That is the + // single source of truth — adding/removing a driven read row here without + // updating the allowlist trips the guard. + ] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn registry_internal_consistency() { + let rows = deny_bearing_routes(); + + // No duplicate method+path+gate. The key is (method, path, gate), not + // (method, path): one endpoint can carry two distinct gate LAYERS that each + // need their own probe substrate — list_webhooks (GET /hooks) read-gates + // (404 on the private repo) THEN owner-gates (403 on the public repo), so it + // is one ReadGate row and one OwnerGate row on the same path. An accidental + // copy-paste of a row keeps the same gate, so this still catches the real + // double-registration bug; it only admits the deliberate dual-gate case. + let mut seen = std::collections::HashSet::new(); + for r in rows { + assert!( + seen.insert((r.method, r.path, r.gate)), + "duplicate deny-bearing row: {} {} {:?}", + r.method, + r.path, + r.gate + ); + } + + for r in rows { + // Every row records its handler fn (review spot-check anchor). + assert!( + !r.handler.is_empty(), + "row {} {} has no handler", + r.method, + r.path + ); + + match r.gate { + // Every read-gate row must carry a positive twin, or a 404 from a + // merely-absent entity is indistinguishable from the gate's 404. + GateClass::ReadGate => assert_ne!( + r.reach, + Reach::None, + "read-gate row {} {} needs a Reach positive twin", + r.method, + r.path + ), + // Owner-gate/signature rows use the class's own twin (owner re-sign) + // or none, never a read-path twin. + _ => assert_eq!( + r.reach, + Reach::None, + "non-read-gate row {} {} must not carry a read twin", + r.method, + r.path + ), + } + + // `principals` is meaningful ONLY on MultiPrincipalGate rows: it must + // be non-empty there (a gate with no declared arm drives nothing) and + // empty everywhere else (a stray arm set on a non-multi row is a bug). + match r.gate { + GateClass::MultiPrincipalGate => assert!( + !r.principals.is_empty(), + "multi-principal row {} {} declares no authorizing arms", + r.method, + r.path + ), + _ => assert!( + r.principals.is_empty(), + "non-multi-principal row {} {} must not declare arms", + r.method, + r.path + ), + } + } + } + + /// STRUCTURAL enforcement (#195, F1): every `MultiPrincipalGate` row's + /// generator output must carry exactly one Not403 twin PER declared arm plus + /// the single stranger hostile, so a row that registered an arm but never + /// emits its twin (the vacuous-guard failure) fails HERE rather than in a + /// runtime sweep nobody re-runs. Invokes `probes_for` directly — the same + /// generator the real sweep drives — so the two cannot drift. + #[test] + fn multi_principal_rows_emit_one_twin_per_arm() { + use crate::support::probe::{probes_for, tests_support::fx, Expect, Signer}; + + let fixture = fx(); + let mut checked = 0usize; + for r in deny_bearing_routes() { + if r.gate != GateClass::MultiPrincipalGate { + continue; + } + checked += 1; + let ps = probes_for(r, &fixture); + + let hostile = ps + .iter() + .filter(|p| p.signer == Signer::Stranger && matches!(p.expect, Expect::Deny(403))) + .count(); + assert_eq!( + hostile, 1, + "multi-principal row {} {} must drive exactly one stranger-403 hostile, drove {hostile}", + r.method, r.path + ); + + let twins = ps + .iter() + .filter(|p| matches!(p.expect, Expect::Not403)) + .count(); + assert_eq!( + twins, + r.principals.len(), + "multi-principal row {} {} declares {} arms but emits {twins} Not403 twins", + r.method, + r.path, + r.principals.len(), + ); + + // Each declared arm maps to a distinct signer, and every twin's signer + // is one of the declared arms (no phantom / wrong-arm twin). + for arm in r.principals { + let want = crate::support::probe::signer_for_principal(*arm); + let present = ps + .iter() + .any(|p| p.signer == want && matches!(p.expect, Expect::Not403)); + assert!( + present, + "multi-principal row {} {} declares arm {:?} but emits no Not403 twin for it", + r.method, r.path, arm + ); + } + + // The twins' signers must be DISTINCT and number the declared arms. The + // per-arm any() check above is satisfied by a single twin when two arms + // map to the same Signer (a signer_for_principal collision), so without + // this a wrong mapping would pass with one arm silently untested. + let mut twin_signers: Vec = ps + .iter() + .filter(|p| matches!(p.expect, Expect::Not403)) + .map(|p| p.signer) + .collect(); + twin_signers.sort_by_key(|s| format!("{s:?}")); + twin_signers.dedup(); + assert_eq!( + twin_signers.len(), + r.principals.len(), + "multi-principal row {} {} emits twins with {} DISTINCT signers but declares {} arms \ + (Principal->Signer collision?)", + r.method, + r.path, + twin_signers.len(), + r.principals.len(), + ); + } + assert!( + checked >= 8, + "expected at least the eight known multi-principal rows (close_pr, \ + close_issue, dispute/submit/approve/cancel bounty, complete_task, \ + fail_task), checked {checked}" + ); + } + + /// STRUCTURAL enforcement for `SignerSelfGate` rows (#195, F3): each caller-self + /// field-binding row's generator output must be exactly one Stranger/Deny(403) + /// hostile plus one Owner/Not403 control, and both bodies must bind the self + /// field to the owner DID (so the hostile is a real signer-vs-body mismatch and + /// the control a real match). Mirrors `multi_principal_rows_emit_one_twin_per_arm` + /// so a SignerSelfGate row that stopped emitting its twin (the vacuous-guard + /// failure) fails HERE, not only in a DB sweep nobody re-runs. Invokes + /// `probes_for` directly, the same generator the real sweep drives. + #[test] + fn signer_self_rows_emit_one_hostile_and_one_control() { + use crate::support::probe::{probes_for, tests_support::fx, Expect, Signer}; + + let fixture = fx(); + let mut checked = 0usize; + for r in deny_bearing_routes() { + if r.gate != GateClass::SignerSelfGate { + continue; + } + checked += 1; + let ps = probes_for(r, &fixture); + assert_eq!( + ps.len(), + 2, + "signer-self row {} {} must emit exactly two probes, emitted {}", + r.method, + r.path, + ps.len() + ); + + let hostile = ps + .iter() + .filter(|p| p.signer == Signer::Stranger && matches!(p.expect, Expect::Deny(403))) + .count(); + assert_eq!( + hostile, 1, + "signer-self row {} {} must drive exactly one stranger-403 hostile, drove {hostile}", + r.method, r.path + ); + + let control = ps + .iter() + .filter(|p| p.signer == Signer::Owner && matches!(p.expect, Expect::Not403)) + .count(); + assert_eq!( + control, 1, + "signer-self row {} {} must drive exactly one owner Not403 control, drove {control}", + r.method, r.path + ); + + // The bound self field (delegator_did / assignee_did / did) must resolve + // to the owner DID in BOTH bodies: the hostile is a stranger signing a + // body that names the owner (mismatch), the control an owner signing the + // same (match). A row whose body did not run through `fill()` would carry + // a literal `{owner}` and fail here. + for p in &ps { + let body = String::from_utf8(p.body.clone()).expect("probe body is UTF-8 JSON"); + assert!( + body.contains(&fixture.owner_did), + "signer-self row {} {} body must bind the self field to the owner DID, got {body}", + r.method, + r.path + ); + } + } + assert!( + checked >= 3, + "expected at least the three SignerSelfGate rows (create_task, claim_task, \ + register), checked {checked}" + ); + } +} diff --git a/crates/gitlawb-node/tests/support/signing.rs b/crates/gitlawb-node/tests/support/signing.rs new file mode 100644 index 00000000..3e0d543c --- /dev/null +++ b/crates/gitlawb-node/tests/support/signing.rs @@ -0,0 +1,32 @@ +//! Real RFC-9421 request signing for the deny harness (U2). Wraps +//! `gitlawb_core::http_sig::sign_request` (the same entry point the +//! git-remote-gitlawb helper uses in production) so a test can present a valid +//! signature for any identity, reaching the authenticated-but-wrong-owner path +//! that an injected-DID shortcut cannot. + +use gitlawb_core::http_sig::sign_request; +use gitlawb_core::identity::Keypair; + +/// Build a reqwest request carrying a valid RFC-9421 signature for `keypair`. +/// +/// `path` is both appended to `base_url` to form the request URL and signed as +/// the `@path` component, so the value the server reconstructs and the value +/// that was signed are guaranteed identical. `body` is signed via its +/// content-digest and sent as the request body. +pub fn signed_request( + client: &reqwest::Client, + method: reqwest::Method, + base_url: &str, + path: &str, + body: Vec, + keypair: &Keypair, +) -> reqwest::RequestBuilder { + let sig = sign_request(keypair, method.as_str(), path, &body); + let url = format!("{base_url}{path}"); + client + .request(method, url) + .header("content-digest", sig.content_digest) + .header("signature-input", sig.signature_input) + .header("signature", sig.signature) + .body(body) +}