From f511eab722307b5b9ad52a04588e93f29c016efb Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 15:54:55 -0500 Subject: [PATCH 01/41] refactor(node): split gitlawb-node into lib+bin so integration tests can spawn a real node Move the module tree and boot logic from main.rs into a new lib.rs crate root exposing the boot surface (build_router, AppState, Config, migrations) as pub; main.rs becomes a thin #[tokio::main] shim over run(). No behavior change: both targets build and the full node suite (488 tests) stays green. Prerequisite for the real-node deny harness (U1). --- crates/gitlawb-node/Cargo.toml | 4 + crates/gitlawb-node/src/lib.rs | 1093 +++++++++++++++++++++++++++++++ crates/gitlawb-node/src/main.rs | 1087 +----------------------------- 3 files changed, 1101 insertions(+), 1083 deletions(-) create mode 100644 crates/gitlawb-node/src/lib.rs diff --git a/crates/gitlawb-node/Cargo.toml b/crates/gitlawb-node/Cargo.toml index 9e42c084..1f631635 100644 --- a/crates/gitlawb-node/Cargo.toml +++ b/crates/gitlawb-node/Cargo.toml @@ -6,6 +6,10 @@ edition.workspace = true rust-version.workspace = true license.workspace = true +[lib] +name = "gitlawb_node" +path = "src/lib.rs" + [[bin]] name = "gitlawb-node" path = "src/main.rs" diff --git a/crates/gitlawb-node/src/lib.rs b/crates/gitlawb-node/src/lib.rs new file mode 100644 index 00000000..fb0a8c55 --- /dev/null +++ b/crates/gitlawb-node/src/lib.rs @@ -0,0 +1,1093 @@ +// Crate builds as both a library (`gitlawb_node`) and the `gitlawb-node` binary. +// The library exposes the boot surface (build_router, AppState, Config, +// migrations) so the out-of-crate deny-harness integration crate can spawn a +// real node; `src/main.rs` is a thin shim over `run()`. Modules the harness +// reaches are `pub`; the rest stay crate-private. +pub mod api; +mod arweave; +pub mod auth; +mod bootstrap; +mod cert; +pub mod config; +pub mod db; +mod encrypted_pin; +pub mod error; +pub mod git; +mod graphql; +mod icaptcha; +mod ipfs_pin; +mod metrics; +mod operator; +mod p2p; +mod pinata; +mod rate_limit; +pub mod server; +pub mod state; +mod sync; +#[cfg(test)] +mod test_support; +mod visibility; +mod webhooks; + +use anyhow::{anyhow, Context, Result}; +use axum::extract::State; +use axum::http::StatusCode; +use axum::response::IntoResponse; +use axum::routing::get; +use axum::{Json, Router}; +use clap::Parser; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use tokio::net::TcpListener; +use tokio::sync::watch; +use tracing::{info, warn}; + +use gitlawb_core::http_sig::sign_request; +use gitlawb_core::identity::Keypair; + +pub use config::Config; +use db::Db; +pub use state::AppState; + +#[derive(Clone)] +struct DegradedState { + node_did: String, + db_startup: Arc, +} + +/// Two independent counters with no cross-field invariant — atomics, not a +/// lock, so the retry loop and the degraded handlers never contend. +#[derive(Default)] +struct DbStartupStatus { + attempts: AtomicU64, + next_retry_secs: AtomicU64, +} + +/// Boot and run the node to completion. The `gitlawb-node` binary is a thin +/// `#[tokio::main]` shim over this; keeping the body here (not in `main.rs`) +/// lets the library own the full module tree so integration tests can link it. +pub async fn run() -> Result<()> { + tracing_subscriber::fmt() + .with_env_filter( + tracing_subscriber::EnvFilter::from_default_env() + .add_directive("gitlawb_node=debug".parse().unwrap()) + .add_directive("tower_http=info".parse().unwrap()), + ) + .init(); + + let mut config = Config::parse(); + + // Merge the embedded seed list of public network nodes into the runtime + // bootstrap peers. Operators can opt out via GITLAWB_BOOTSTRAP_DISABLE_SEEDS. + bootstrap::merge_seeds(&mut config); + + if !config.public_read { + warn!( + "GITLAWB_PUBLIC_READ=false is reserved; per-repository private-read enforcement is not wired in alpha" + ); + } + + // Load or generate the node's identity keypair + let keypair = load_or_create_keypair(&config)?; + let node_did = keypair.did(); + + // One-time metrics init. Must run before any handler that calls into + // `metrics::record_*` so the registry exists when the first event fires. + // Safe to call even when GITLAWB_METRICS_ADDR is unset — those helpers + // are simply no-ops until something reads from the registry. + metrics::init(env!("CARGO_PKG_VERSION"), &node_did.to_string()); + + info!("╔══════════════════════════════════════════╗"); + info!( + "║ gitlawb node v{} ║", + env!("CARGO_PKG_VERSION") + ); + info!("╚══════════════════════════════════════════╝"); + // Process-wide shutdown signal. One sender lives in AppState (cloned + // into every handler); main() keeps a clone and flips it on SIGINT + // or SIGTERM. Tasks that hold a watch::Receiver get notified at + // their next await point. + let (shutdown_tx, _shutdown_rx_for_main) = watch::channel(false); + spawn_shutdown_signal(shutdown_tx.clone()); + + info!(did = %node_did, "node identity"); + info!(addr = %config.bind_addr(), "binding HTTP listener"); + + // Bind HTTP once, before dependency initialization, and keep this socket + // for the life of the process. The degraded server accepts on a dup of the + // same socket, so the degraded→full handoff never closes the port: while + // the full server initializes, connections queue in the shared backlog + // instead of being refused. + let listener = TcpListener::bind(config.bind_addr()) + .await + .with_context(|| format!("failed to bind to {}", config.bind_addr()))?; + let full_std_listener = listener.into_std()?; + let degraded_listener = TcpListener::from_std( + full_std_listener + .try_clone() + .context("failed to clone HTTP listener for degraded server")?, + )?; + + // Metrics must stay observable during a database outage — the degraded + // window is exactly when dashboards need data — so this listener starts + // before the DB connects. + let metrics_handle = if !config.metrics_addr.is_empty() { + match spawn_metrics_server(&config.metrics_addr, shutdown_tx.subscribe()).await { + Ok(handle) => { + info!(addr = %config.metrics_addr, "metrics endpoint listening"); + Some(handle) + } + Err(e) => { + warn!(err = %e, addr = %config.metrics_addr, "failed to start metrics endpoint — continuing without"); + None + } + } + } else { + info!("metrics endpoint disabled (GITLAWB_METRICS_ADDR not set)"); + None + }; + + let db_startup = Arc::new(DbStartupStatus::default()); + let (db_ready_tx, db_ready_rx) = watch::channel(false); + let mut degraded_handle = tokio::spawn(run_degraded_server( + degraded_listener, + node_did.to_string(), + Arc::clone(&db_startup), + db_ready_rx, + shutdown_tx.subscribe(), + )); + + // Connect to PostgreSQL database. A transient outage or bad secret should + // not crash-loop the process and hammer the database provider; permanent + // misconfiguration surfaces through error-level logs and the /ready check. + let db = tokio::select! { + db = connect_db_with_retry(&config, Arc::clone(&db_startup), shutdown_tx.subscribe()) => { + match db { + Some(db) => db, + None => { + // Shutdown requested while waiting for the database. The + // degraded server only serves one-shot 503s — abort it + // rather than drain, so a slow client can't stall exit. + degraded_handle.abort(); + return Ok(()); + } + } + } + degraded = &mut degraded_handle => { + if *shutdown_tx.borrow() { + return Ok(()); + } + return match degraded { + Ok(Ok(())) => Err(anyhow!("degraded HTTP server stopped before database became ready")), + Ok(Err(err)) => Err(err.context("degraded HTTP server failed")), + Err(err) => Err(anyhow!("degraded HTTP server task failed: {err}")), + }; + } + }; + + // Flip the degraded server into graceful shutdown, but do NOT await the + // drain: one slow in-flight request must not delay the full server, and + // the shared socket means there is no port gap to cover. The drain + // finishes (and logs) in the background. + db_ready_tx.send(true).ok(); + tokio::spawn(async move { + match degraded_handle.await { + Ok(Ok(())) => {} + Ok(Err(err)) => warn!(err = %err, "degraded HTTP server exited with error"), + Err(err) => warn!(err = %err, "degraded HTTP server task failed"), + } + }); + info!(addr = %config.bind_addr(), "database ready; starting full HTTP server"); + + // Prune peer rows that point back at this node (stale self-loop entries) + if let Some(public_url) = config.public_url.as_deref() { + match db.prune_self_peers(public_url).await { + Ok(0) => {} + Ok(n) => info!(removed = n, public_url, "pruned self-loop peer rows"), + Err(e) => warn!(err = %e, "prune_self_peers failed (non-fatal)"), + } + } + + // Prune peer rows with non-public hosts (loopback/private/internal) that + // were injected via the unauthenticated announce route — they poison the + // sync-notify fan-out (SSRF + crowding out real peers). + match db.prune_non_public_peers().await { + Ok(0) => {} + Ok(n) => info!(removed = n, "pruned non-public (poisoned) peer rows"), + Err(e) => warn!(err = %e, "prune_non_public_peers failed (non-fatal)"), + } + + // Ensure repos directory exists + std::fs::create_dir_all(&config.repos_dir).context("failed to create repos directory")?; + + // Start libp2p swarm (if p2p_port > 0) + let p2p_handle = if config.p2p_port > 0 { + let bootstrap_addrs = config + .p2p_bootstrap + .iter() + .filter_map(|s| s.parse().ok()) + .collect(); + let shutdown_rx = shutdown_tx.subscribe(); + match p2p::start( + &node_did.to_string(), + config.p2p_port, + bootstrap_addrs, + Arc::clone(&db), + config.auto_sync, + shutdown_rx, + ) + .await + { + Ok(handle) => { + info!(port = config.p2p_port, peer_id = %handle.local_peer_id, "libp2p swarm started"); + Some(Arc::new(handle)) + } + Err(e) => { + tracing::warn!(err = %e, "failed to start libp2p swarm — continuing without p2p"); + None + } + } + } else { + info!("p2p disabled (p2p_port = 0)"); + None + }; + + // Shared no-redirect HTTP client. See build_http_client for the SSRF rationale. + let http_client = Arc::new(build_http_client()?); + + let (ref_update_tx, _) = tokio::sync::broadcast::channel::(256); + let (task_event_tx, _) = tokio::sync::broadcast::channel::(256); + + let graphql_schema = Arc::new(graphql::build_schema( + Arc::clone(&db), + ref_update_tx.clone(), + task_event_tx.clone(), + )); + + let machine_id = std::env::var("FLY_MACHINE_ID").ok(); + if let Some(ref mid) = machine_id { + info!(" fly machine: {mid}"); + } + + // Initialize Tigris S3 client if bucket is configured + let tigris = if !config.tigris_bucket.is_empty() { + match git::tigris::TigrisClient::new(&config.tigris_bucket).await { + Ok(client) => { + info!(bucket = %config.tigris_bucket, "tigris storage enabled"); + Some(client) + } + Err(e) => { + tracing::warn!(err = %e, "failed to initialize Tigris client — using local-only storage"); + None + } + } + } else { + info!("tigris storage disabled (no bucket configured)"); + None + }; + + let repo_store = + git::repo_store::RepoStore::new(config.repos_dir.clone(), tigris, db.pool().clone()); + + // Per-DID limiter for the creation endpoints. Keyed on the authenticated + // DID (attacker-varied), so bound its key set to cap memory. + let rate_limiter = + rate_limit::RateLimiter::new_bounded(10, std::time::Duration::from_secs(3600), 200_000); + + // Per-client-IP flood brake for the creation endpoints. The per-DID limiter + // above is bypassed by a DID farm (one throwaway did:key per repo), which is + // exactly how the recurring spam-repo floods get past both it and the + // iCaptcha gate. Keyed on the resolved client IP so a single-source flood is + // capped regardless of how many identities it mints. Sized well above any + // legitimate per-IP creation rate; GITLAWB_CREATE_RATE_LIMIT overrides, 0 + // disables. Bounded key set — the key is a client-influenced IP. + let create_limit = std::env::var("GITLAWB_CREATE_RATE_LIMIT") + .ok() + .and_then(|v| v.trim().parse::().ok()) + .unwrap_or(120); + let create_ip_rate_limiter = rate_limit::RateLimiter::new_bounded( + create_limit, + std::time::Duration::from_secs(3600), + 200_000, + ); + if create_limit == 0 { + tracing::warn!("GITLAWB_CREATE_RATE_LIMIT=0 — per-IP creation rate limiting disabled"); + } + + // Push-path flood brake: max git-receive-pack requests per client IP per + // hour (counts both the info/refs advertisement and the push POST). Sized + // for heavy agent automation while still stopping flood traffic (the June + // 2026 attack pushed several times per second per IP). GITLAWB_PUSH_RATE_LIMIT + // overrides; 0 disables. Bounded key set — the key is a client-influenced IP. + let push_limit = std::env::var("GITLAWB_PUSH_RATE_LIMIT") + .ok() + .and_then(|v| v.trim().parse::().ok()) + .unwrap_or(600); + let push_rate_limiter = rate_limit::RateLimiter::new_bounded( + push_limit, + std::time::Duration::from_secs(3600), + 200_000, + ); + if push_limit == 0 { + tracing::warn!("GITLAWB_PUSH_RATE_LIMIT=0 — per-IP push rate limiting disabled"); + } + + // Which forwarded header the edge is trusted to set. Default None (trust + // nothing, key on the socket peer). Fly nodes set GITLAWB_TRUSTED_PROXY=fly; + // a node behind Caddy/NGINX sets it to x-forwarded-for. + let push_limiter_trust = rate_limit::TrustedProxy::from_env_value( + &std::env::var("GITLAWB_TRUSTED_PROXY").unwrap_or_default(), + ); + tracing::info!(trust = ?push_limiter_trust, push_limit, "push rate limiter configured"); + + // Peer-sync flood brakes, keyed on the resolved client IP (per-DID is useless + // here — a did:key farm self-registers). Two buckets so an unsigned notify + // flood can't drain the signed trigger caller's quota (#82). Bounded key sets + // (the key is a client-influenced IP); 0 disables each. + let sync_trigger_rate_limiter = rate_limit::RateLimiter::new_bounded( + config.sync_trigger_rate_limit, + std::time::Duration::from_secs(3600), + 200_000, + ); + let peer_write_rate_limiter = rate_limit::RateLimiter::new_bounded( + config.peer_write_rate_limit, + std::time::Duration::from_secs(3600), + 200_000, + ); + if config.sync_trigger_rate_limit == 0 { + tracing::warn!( + "GITLAWB_SYNC_TRIGGER_RATE_LIMIT=0 — /sync/trigger IP rate limiting disabled" + ); + } + if config.peer_write_rate_limit == 0 { + tracing::warn!("GITLAWB_PEER_WRITE_RATE_LIMIT=0 — peer-write IP rate limiting disabled"); + } + + // Initialize the iCaptcha proof gate (inert unless ICAPTCHA_MODE is set). + icaptcha::init().await; + + let state = AppState { + config: Arc::new(config.clone()), + db, + node_did: node_did.clone(), + node_keypair: Arc::new(keypair), + p2p: p2p_handle, + http_client, + ref_update_tx, + task_event_tx, + graphql_schema, + machine_id, + repo_store, + rate_limiter, + create_ip_rate_limiter, + push_rate_limiter, + push_limiter_trust, + sync_trigger_rate_limiter, + peer_write_rate_limiter, + shutdown_tx: shutdown_tx.clone(), + }; + + // Periodic peer-count poll for the metrics gauge. If p2p is disabled + // we still set the gauge to 0 so dashboards don't show "no data". + { + let p2p_for_metrics = state.p2p.clone(); + let mut shutdown_rx = state.subscribe_shutdown(); + tokio::spawn(async move { + let mut interval = tokio::time::interval(std::time::Duration::from_secs(15)); + loop { + tokio::select! { + _ = interval.tick() => { + let count = match &p2p_for_metrics { + Some(h) => h.status().await.map(|s| s.connected_peers).unwrap_or(0), + None => 0, + }; + metrics::set_peers_connected(count as i64); + } + _ = shutdown_rx.changed() => { + if *shutdown_rx.borrow() { + return; + } + } + } + } + }); + } + + // Periodic cleanup of expired rate limit entries + consumed-proof ledger + { + let rl = state.rate_limiter.clone(); + let create_ip_rl = state.create_ip_rate_limiter.clone(); + let push_rl = state.push_rate_limiter.clone(); + let sync_trigger_rl = state.sync_trigger_rate_limiter.clone(); + let peer_write_rl = state.peer_write_rate_limiter.clone(); + let db = state.db.clone(); + let mut shutdown_rx = state.subscribe_shutdown(); + tokio::spawn(async move { + loop { + tokio::select! { + _ = tokio::time::sleep(std::time::Duration::from_secs(300)) => { + rl.cleanup().await; + create_ip_rl.cleanup().await; + push_rl.cleanup().await; + sync_trigger_rl.cleanup().await; + peer_write_rl.cleanup().await; + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs() as i64) + .unwrap_or(0); + if let Err(e) = db.sweep_expired_proofs(now).await { + tracing::warn!(err = %e, "failed to sweep expired iCaptcha proofs"); + } + } + _ = shutdown_rx.changed() => { + if *shutdown_rx.borrow() { + break; + } + } + } + } + }); + } + + let router = server::build_router(state.clone()); + // Re-register the socket bound at startup — same fd, so there was never a + // moment with the port closed between the degraded and full servers. + let listener = TcpListener::from_std(full_std_listener) + .context("failed to re-register HTTP listener with the runtime")?; + + info!("✓ node started — did:{}", node_did); + info!(" repos dir: {}", config.repos_dir.display()); + info!( + " database: PostgreSQL ({})", + &config.database_url.split('@').next_back().unwrap_or("?") + ); + + // Publish our DID record to the Kademlia DHT shortly after startup + if let Some(p2p) = &state.p2p { + let did_record = p2p::DidRecord { + did: node_did.to_string(), + http_url: config.public_url.clone().unwrap_or_default(), + peer_id: p2p.local_peer_id.to_string(), + p2p_port: config.p2p_port, + timestamp: chrono::Utc::now().to_rfc3339(), + }; + let p2p_clone = Arc::clone(p2p); + let mut shutdown_rx = state.subscribe_shutdown(); + tokio::spawn(async move { + // Small delay so Kademlia can find peers first + tokio::select! { + _ = tokio::time::sleep(std::time::Duration::from_secs(5)) => {} + _ = shutdown_rx.changed() => return, + } + p2p_clone.put_did(did_record).await; + info!("DID record published to Kademlia DHT"); + }); + } + + // Spawn background gossip: announce to bootstrap peers, then ping known peers periodically + { + let gossip_state = state.clone(); + let bootstrap_peers = config.bootstrap_peers.clone(); + let shutdown_rx = state.subscribe_shutdown(); + tokio::spawn(async move { + gossip_task(gossip_state, bootstrap_peers, shutdown_rx).await; + }); + } + + // Start multi-node sync worker if auto_sync is enabled + if config.auto_sync { + sync::start( + Arc::clone(&state.db), + Arc::clone(&state.config), + Arc::clone(&state.node_keypair), + state.subscribe_shutdown(), + ); + info!("auto-sync worker started"); + } + + // On-chain operator setup: verify stake + spawn heartbeat loop + if !state.config.contract_node_staking.is_empty() + && !state.config.operator_private_key.is_empty() + { + match build_operator_client(&state.config, &state.node_did.to_string()) { + Ok(client) => match operator::startup_check(&client).await { + Ok(_) => { + let arc_client = Arc::new(client); + arc_client.spawn_heartbeat_loop(state.subscribe_shutdown()); + } + Err(e) => { + if state.config.operator_strict_mode { + return Err(e.context("strict-mode operator check failed")); + } + tracing::warn!(err = %e, "operator startup check failed — continuing without heartbeat loop"); + } + }, + Err(e) => { + if state.config.operator_strict_mode { + return Err(e.context("strict-mode: failed to build operator client")); + } + tracing::warn!(err = %e, "operator client could not be built — continuing without PoS"); + } + } + } else { + info!("on-chain PoS disabled (GITLAWB_CONTRACT_NODE_STAKING or GITLAWB_OPERATOR_PRIVATE_KEY unset)"); + } + + // axum's `with_graceful_shutdown` waits for in-flight requests to + // complete (up to the configured grace) once the future resolves. + let shutdown_signal_for_axum = state.subscribe_shutdown(); + let grace = std::time::Duration::from_secs(config.shutdown_grace_secs); + info!(grace_secs = config.shutdown_grace_secs, "axum server ready"); + + // `into_make_service_with_connect_info` exposes the socket peer address as + // `ConnectInfo` so the push limiter can key on the real client + // when no trusted proxy header applies (see `rate_limit::client_key`). + let serve_result = axum::serve( + listener, + router.into_make_service_with_connect_info::(), + ) + .with_graceful_shutdown(async move { + let mut rx = shutdown_signal_for_axum; + // Wait until the watcher flips to true, then return so axum + // can begin draining. + while !*rx.borrow_and_update() { + if rx.changed().await.is_err() { + // Sender dropped — treat as shutdown. + break; + } + } + }) + .await; + + // Server has stopped accepting new connections and drained in-flight + // requests. Tear the rest of the system down. + info!("HTTP server stopped, beginning process shutdown"); + if let Some(h) = metrics_handle { + h.abort(); + } + let _ = grace; // recorded for operators in the log above; not enforced + serve_result?; + info!("clean exit"); + Ok(()) +} + +fn spawn_shutdown_signal(tx: watch::Sender) { + tokio::spawn(async move { + #[cfg(unix)] + { + use tokio::signal::unix::{signal as unix_signal, SignalKind}; + let mut sigterm = + unix_signal(SignalKind::terminate()).expect("install SIGTERM handler"); + let mut sigint = unix_signal(SignalKind::interrupt()).expect("install SIGINT handler"); + tokio::select! { + _ = sigterm.recv() => info!("SIGTERM received, shutting down"), + _ = sigint.recv() => info!("SIGINT received, shutting down"), + } + } + #[cfg(not(unix))] + { + use tokio::signal; + let _ = signal::ctrl_c().await; + info!("Ctrl-C received, shutting down"); + } + tx.send(true).ok(); + }); +} + +async fn connect_db_with_retry( + config: &Config, + db_startup: Arc, + mut shutdown_rx: watch::Receiver, +) -> Option> { + let initial_retry_secs = config.db_retry_initial_secs; + let max_retry_secs = config.db_retry_max_secs.max(initial_retry_secs); + let acquire_timeout = std::time::Duration::from_secs(config.db_acquire_timeout_secs); + let attempt_timeout = std::time::Duration::from_secs(config.db_connect_timeout_secs); + let mut attempts = 0_u64; + + loop { + if *shutdown_rx.borrow() { + return None; + } + + attempts = attempts.saturating_add(1); + db_startup.attempts.store(attempts, Ordering::Relaxed); + + // Bound the whole attempt, not just the pool connect: migrations + // block on a cross-instance advisory lock, and an unbounded wait + // there would wedge this loop — no retries, no logs, no recovery. + // Timing out and retrying is safe; migrations are idempotent. + let attempt = match tokio::time::timeout( + attempt_timeout, + Db::connect( + &config.database_url, + config.db_max_connections, + acquire_timeout, + ), + ) + .await + { + Ok(result) => result, + Err(_) => Err(anyhow!( + "connect + migrate attempt exceeded {}s (GITLAWB_DB_CONNECT_TIMEOUT_SECS); \ + is another instance holding the migration lock?", + attempt_timeout.as_secs() + )), + }; + + match attempt { + Ok(db) => { + info!(attempts, "database connection established"); + return Some(Arc::new(db)); + } + Err(err) => { + // A bad DATABASE_URL or rejected credentials won't heal on + // their own. Still retry (exiting would crash-loop and hammer + // the provider — and take liveness down with it), but log at + // error level and skip straight to the maximum backoff; the + // /ready health check is what surfaces this to deploys. + let permanent = is_likely_permanent_db_error(&err); + let retry_secs = if permanent { + max_retry_secs + } else { + database_retry_delay_secs(initial_retry_secs, max_retry_secs, attempts) + }; + db_startup + .next_retry_secs + .store(retry_secs, Ordering::Relaxed); + if permanent { + tracing::error!( + attempts, + retry_secs, + err = %err, + "database rejected our configuration (bad DATABASE_URL or credentials?) — retrying, but operator action is likely required" + ); + } else { + warn!( + attempts, + retry_secs, + err = %err, + "database unavailable during startup; retrying" + ); + } + + tokio::select! { + _ = tokio::time::sleep(std::time::Duration::from_secs(retry_secs)) => {} + changed = shutdown_rx.changed() => { + if changed.is_err() || *shutdown_rx.borrow() { + return None; + } + } + } + } + } + } +} + +/// Errors that indicate misconfiguration rather than a transient outage: a +/// malformed DATABASE_URL, or a server that answered and rejected us — +/// Postgres error class 28xxx (invalid authorization) or 3D000 (database +/// does not exist). Best-effort: an error that anyhow can't downcast back to +/// sqlx just counts as transient. +fn is_likely_permanent_db_error(err: &anyhow::Error) -> bool { + match err.downcast_ref::() { + Some(sqlx::Error::Configuration(_)) => true, + Some(sqlx::Error::Database(db)) => db + .code() + .map(|c| c.starts_with("28") || c.starts_with("3D")) + .unwrap_or(false), + _ => false, + } +} + +fn database_retry_delay_secs(initial_secs: u64, max_secs: u64, attempts: u64) -> u64 { + // The exponent bound only keeps the u32 cast safe — max_secs is the real + // (operator-configurable) cap, and saturating math handles overflow. + let exponent = attempts.saturating_sub(1).min(63) as u32; + initial_secs + .saturating_mul(2_u64.saturating_pow(exponent)) + .min(max_secs) +} + +async fn run_degraded_server( + listener: TcpListener, + node_did: String, + db_startup: Arc, + mut db_ready_rx: watch::Receiver, + mut shutdown_rx: watch::Receiver, +) -> Result<()> { + let addr = listener.local_addr().ok(); + let router = build_degraded_router(node_did, db_startup); + info!(?addr, "degraded HTTP server ready"); + + axum::serve(listener, router) + .with_graceful_shutdown(async move { + // wait_for resolves on predicate-true or sender-drop; either way + // this phase is over. + tokio::select! { + _ = db_ready_rx.wait_for(|ready| *ready) => {} + _ = shutdown_rx.wait_for(|stop| *stop) => {} + } + }) + .await?; + + Ok(()) +} + +fn build_degraded_router(node_did: String, db_startup: Arc) -> Router { + let state = DegradedState { + node_did, + db_startup, + }; + // Everything answers 503 with the same body — including /health and + // /ready, so peer pings (which treat any 2xx /health as alive) and + // uptime monitors correctly see a node that cannot serve traffic. + // `/` additionally carries the node identity for probing peers. + Router::new() + .route("/", get(degraded_node_info)) + .fallback(degraded_unavailable) + .with_state(state) +} + +/// One source of truth for the degraded 503 body, sharing the error +/// vocabulary with error.rs so clients see the same code/message for +/// "database unavailable" regardless of which phase produced it. +fn degraded_body(db_startup: &DbStartupStatus) -> serde_json::Value { + serde_json::json!({ + "status": "degraded", + "database": "initializing", + "error": error::DB_UNAVAILABLE_CODE, + "message": error::DB_UNAVAILABLE_MESSAGE, + "db_attempts": db_startup.attempts.load(Ordering::Relaxed), + "db_next_retry_secs": db_startup.next_retry_secs.load(Ordering::Relaxed), + }) +} + +async fn degraded_node_info(State(state): State) -> impl IntoResponse { + let mut body = degraded_body(&state.db_startup); + if let Some(obj) = body.as_object_mut() { + obj.insert("name".into(), "gitlawb-node".into()); + obj.insert("version".into(), env!("CARGO_PKG_VERSION").into()); + obj.insert("did".into(), state.node_did.clone().into()); + } + (StatusCode::SERVICE_UNAVAILABLE, Json(body)) +} + +async fn degraded_unavailable(State(state): State) -> impl IntoResponse { + ( + StatusCode::SERVICE_UNAVAILABLE, + Json(degraded_body(&state.db_startup)), + ) +} + +/// Spawn a small axum router that exposes only `GET /metrics` on its own +/// listener. Returns the JoinHandle so `main()` can abort it on shutdown. +/// This is deliberately separate from the main router so the metrics port +/// can be firewalled differently from the API port — bind to localhost +/// or a private interface only. +async fn spawn_metrics_server( + addr: &str, + mut shutdown_rx: watch::Receiver, +) -> Result> { + use axum::{response::IntoResponse, routing::get, Router}; + + async fn metrics_handler() -> impl IntoResponse { + match metrics::encode() { + Ok(body) => ( + axum::http::StatusCode::OK, + [( + axum::http::header::CONTENT_TYPE, + "text/plain; version=0.0.4; charset=utf-8", + )], + body, + ), + Err(e) => ( + axum::http::StatusCode::INTERNAL_SERVER_ERROR, + [( + axum::http::header::CONTENT_TYPE, + "text/plain; charset=utf-8", + )], + format!("metrics encode error: {e}"), + ), + } + } + + let listener = TcpListener::bind(addr) + .await + .with_context(|| format!("failed to bind metrics listener to {addr}"))?; + let app = Router::new().route("/metrics", get(metrics_handler)); + + let handle = tokio::spawn(async move { + if let Err(e) = axum::serve(listener, app) + .with_graceful_shutdown(async move { + while !*shutdown_rx.borrow_and_update() { + if shutdown_rx.changed().await.is_err() { + break; + } + } + }) + .await + { + warn!(err = %e, "metrics server exited with error"); + } + }); + Ok(handle) +} + +fn build_operator_client( + config: &config::Config, + node_did: &str, +) -> Result { + use alloy::primitives::Address; + use std::str::FromStr; + + let contract_address = Address::from_str(&config.contract_node_staking) + .with_context(|| format!("invalid contract address: {}", config.contract_node_staking))?; + + let cfg = operator::OperatorConfig { + rpc_url: config.chain_rpc_url.clone(), + private_key: config.operator_private_key.clone(), + contract_address, + node_did: node_did.to_string(), + heartbeat_interval: std::time::Duration::from_secs(config.heartbeat_interval_hours * 3600), + strict_mode: config.operator_strict_mode, + }; + Ok(operator::OperatorClient::new(cfg)) +} + +/// Announce to bootstrap peers on startup, then periodically ping all known peers. +async fn gossip_task( + state: AppState, + bootstrap_peers: Vec, + mut shutdown_rx: tokio::sync::watch::Receiver, +) { + // If shutdown arrives during the initial delay, exit before announcing. + tokio::select! { + _ = tokio::time::sleep(std::time::Duration::from_secs(2)) => {} + _ = shutdown_rx.changed() => { + if *shutdown_rx.borrow() { + info!("gossip: shutdown during startup delay, exiting"); + return; + } + } + } + + // Reuse the shared no-redirect client for every gossip outbound call (the + // bootstrap announce POST and the periodic peer /health ping). Peer URLs are + // attacker-influenceable, so a 3xx to a private address must not be followed. + // Do NOT fall back to reqwest::Client::new(): its default follows redirects + // and would reintroduce the SSRF closed here (#93). + let client = state.http_client.clone(); + let my_did = state.node_did.to_string(); + let my_url = state.config.public_url.clone().unwrap_or_default(); + + // Announce ourselves to each bootstrap peer + for peer_url in &bootstrap_peers { + // Cooperative shutdown between peers — a slow peer shouldn't + // block the node exiting. + if *shutdown_rx.borrow() { + info!("gossip: shutdown signalled during peer announce, exiting"); + return; + } + let path = "/api/v1/peers/announce"; + let announce_url = format!("{}{}", peer_url.trim_end_matches('/'), path); + let body = serde_json::json!({ + "did": my_did.clone(), + "http_url": my_url.clone(), + }); + let body_bytes = match serde_json::to_vec(&body) { + Ok(bytes) => bytes, + Err(e) => { + tracing::warn!(err = %e, "failed to serialize peer announce body"); + continue; + } + }; + let signed = sign_request(state.node_keypair.as_ref(), "POST", path, &body_bytes); + // Per-request timeout inside the loop; do not let one hung peer + // block others. The request itself is a normal tokio future so + // it's cancel-safe on shutdown. + match tokio::time::timeout( + std::time::Duration::from_secs(5), + client + .post(&announce_url) + .header("Content-Type", "application/json") + .header("Content-Digest", signed.content_digest) + .header("Signature-Input", signed.signature_input) + .header("Signature", signed.signature) + .body(body_bytes) + .send(), + ) + .await + { + Ok(Ok(resp)) => { + if resp.status().is_success() { + if let Ok(json) = resp.json::().await { + // Add them back to our peer list + if let (Some(their_did), Some(their_url)) = ( + json.get("node_did").and_then(|v| v.as_str()), + json.get("node_url").and_then(|v| v.as_str()), + ) { + if !their_url.is_empty() { + let _ = state.db.upsert_peer(their_did, their_url).await; + tracing::info!(did = %their_did, url = %their_url, "bootstrap peer added"); + } + } + } + } + } + Ok(Err(e)) => { + tracing::warn!(url = %announce_url, err = %e, "failed to announce to bootstrap peer") + } + Err(_) => tracing::warn!(url = %announce_url, "bootstrap peer announce timed out (5s)"), + } + } + + // Periodic ping every 5 minutes — exit on shutdown. + let mut interval = tokio::time::interval(std::time::Duration::from_secs(300)); + loop { + tokio::select! { + _ = interval.tick() => { + let peers = match state.db.list_peers().await { + Ok(p) => p, + Err(_) => continue, + }; + for peer in peers { + let ok = ping_peer_health(&client, &peer.http_url).await; + let _ = state.db.mark_peer_ping(&peer.did, ok).await; + } + } + _ = shutdown_rx.changed() => { + if *shutdown_rx.borrow() { + info!("gossip task: shutdown signal received, exiting"); + return; + } + } + } + } +} + +/// Build the shared node HTTP client used for every outbound fan-out (sync +/// trigger, profile/repo fetches, gossip announce + peer pings). +/// +/// No redirects: peer URLs are attacker-influenceable, so a `3xx` to a private +/// address must not be followed (SSRF guard, #78/#93). Do NOT replace with +/// `reqwest::Client::new()` — its default follows redirects. Kept as a named +/// builder so tests bind the redirect guarantee to the real client the node +/// runs, not a hand-rolled equivalent. +fn build_http_client() -> reqwest::Result { + reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(10)) + .redirect(reqwest::redirect::Policy::none()) + .build() +} + +/// Ping a peer's `/health` endpoint and report whether it answered 2xx. +/// +/// Takes the client by reference so callers supply the shared, no-redirect +/// `state.http_client`. Peer URLs are attacker-influenceable, so a `3xx` to a +/// private address must not be followed. Do NOT call this with a bare +/// `reqwest::Client::new()`: its default follows redirects and would +/// reintroduce the SSRF this guards against (#93). +async fn ping_peer_health(client: &reqwest::Client, http_url: &str) -> bool { + let url = format!("{}/health", http_url.trim_end_matches('/')); + client + .get(&url) + .send() + .await + .map(|r| r.status().is_success()) + .unwrap_or(false) +} + +fn load_or_create_keypair(config: &Config) -> Result { + let key_path = config.resolved_key_path(); + + if key_path.exists() { + let pem = std::fs::read_to_string(&key_path) + .with_context(|| format!("failed to read key from {}", key_path.display()))?; + let kp = Keypair::from_pem(&pem).map_err(|e| anyhow::anyhow!("invalid PEM key: {e}"))?; + info!(path = %key_path.display(), "loaded existing identity"); + Ok(kp) + } else { + let kp = Keypair::generate(); + let pem = kp + .to_pem() + .map_err(|e| anyhow::anyhow!("failed to serialize key: {e}"))?; + + if let Some(parent) = key_path.parent() { + std::fs::create_dir_all(parent)?; + } + + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + std::fs::write(&key_path, pem.as_bytes())?; + std::fs::set_permissions(&key_path, std::fs::Permissions::from_mode(0o600))?; + } + #[cfg(not(unix))] + std::fs::write(&key_path, pem.as_bytes())?; + + info!(path = %key_path.display(), did = %kp.did(), "generated new node identity"); + Ok(kp) + } +} + +#[cfg(test)] +mod gossip_ssrf_tests { + use super::ping_peer_health; + + // Build the client exactly as production does (super::build_http_client) so + // these tests bind the redirect guarantee to the real shared client the + // node runs. A regression that makes build_http_client follow redirects + // fails ping_peer_health_does_not_follow_redirect. + fn production_http_client() -> reqwest::Client { + super::build_http_client().expect("failed to build production http client") + } + + // A peer answering `/health` with a 302 toward an internal address must not + // be followed: the redirect target must never be requested (#93). + #[tokio::test] + async fn ping_peer_health_does_not_follow_redirect() { + let mut server = mockito::Server::new_async().await; + let internal = server + .mock("GET", "/internal-metadata") + .with_status(200) + .expect(0) + .create_async() + .await; + let _health = server + .mock("GET", "/health") + .with_status(302) + .with_header("location", &format!("{}/internal-metadata", server.url())) + .create_async() + .await; + + let ok = ping_peer_health(&production_http_client(), &server.url()).await; + + assert!(!ok, "a 302 must not count as a healthy peer"); + // expect(0) is enforced only at assert time; this fails if the redirect + // was followed to the internal target. + internal.assert_async().await; + } + + #[tokio::test] + async fn ping_peer_health_reports_success_on_200() { + let mut server = mockito::Server::new_async().await; + let _health = server + .mock("GET", "/health") + .with_status(200) + .create_async() + .await; + + let ok = ping_peer_health(&production_http_client(), &server.url()).await; + + assert!(ok, "a 200 /health must count as a healthy peer"); + } + + // A transport error (nothing listening) must map to unhealthy, never a + // spurious healthy — the .unwrap_or(false) arm. + #[tokio::test] + async fn ping_peer_health_reports_unhealthy_on_connection_error() { + let ok = ping_peer_health(&production_http_client(), "http://127.0.0.1:1").await; + assert!(!ok, "a connection error must count as an unhealthy peer"); + } +} diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index aa0483db..efd80ec4 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -1,1086 +1,7 @@ -mod api; -mod arweave; -mod auth; -mod bootstrap; -mod cert; -mod config; -mod db; -mod encrypted_pin; -mod error; -mod git; -mod graphql; -mod icaptcha; -mod ipfs_pin; -mod metrics; -mod operator; -mod p2p; -mod pinata; -mod rate_limit; -mod server; -mod state; -mod sync; -#[cfg(test)] -mod test_support; -mod visibility; -mod webhooks; - -use anyhow::{anyhow, Context, Result}; -use axum::extract::State; -use axum::http::StatusCode; -use axum::response::IntoResponse; -use axum::routing::get; -use axum::{Json, Router}; -use clap::Parser; -use std::sync::atomic::{AtomicU64, Ordering}; -use std::sync::Arc; -use tokio::net::TcpListener; -use tokio::sync::watch; -use tracing::{info, warn}; - -use gitlawb_core::http_sig::sign_request; -use gitlawb_core::identity::Keypair; - -use config::Config; -use db::Db; -use state::AppState; - -#[derive(Clone)] -struct DegradedState { - node_did: String, - db_startup: Arc, -} - -/// Two independent counters with no cross-field invariant — atomics, not a -/// lock, so the retry loop and the degraded handlers never contend. -#[derive(Default)] -struct DbStartupStatus { - attempts: AtomicU64, - next_retry_secs: AtomicU64, -} +//! Thin binary entry point. All boot logic lives in the library crate +//! (`src/lib.rs`) so out-of-crate integration tests can link the same code. #[tokio::main] -async fn main() -> Result<()> { - tracing_subscriber::fmt() - .with_env_filter( - tracing_subscriber::EnvFilter::from_default_env() - .add_directive("gitlawb_node=debug".parse().unwrap()) - .add_directive("tower_http=info".parse().unwrap()), - ) - .init(); - - let mut config = Config::parse(); - - // Merge the embedded seed list of public network nodes into the runtime - // bootstrap peers. Operators can opt out via GITLAWB_BOOTSTRAP_DISABLE_SEEDS. - bootstrap::merge_seeds(&mut config); - - if !config.public_read { - warn!( - "GITLAWB_PUBLIC_READ=false is reserved; per-repository private-read enforcement is not wired in alpha" - ); - } - - // Load or generate the node's identity keypair - let keypair = load_or_create_keypair(&config)?; - let node_did = keypair.did(); - - // One-time metrics init. Must run before any handler that calls into - // `metrics::record_*` so the registry exists when the first event fires. - // Safe to call even when GITLAWB_METRICS_ADDR is unset — those helpers - // are simply no-ops until something reads from the registry. - metrics::init(env!("CARGO_PKG_VERSION"), &node_did.to_string()); - - info!("╔══════════════════════════════════════════╗"); - info!( - "║ gitlawb node v{} ║", - env!("CARGO_PKG_VERSION") - ); - info!("╚══════════════════════════════════════════╝"); - // Process-wide shutdown signal. One sender lives in AppState (cloned - // into every handler); main() keeps a clone and flips it on SIGINT - // or SIGTERM. Tasks that hold a watch::Receiver get notified at - // their next await point. - let (shutdown_tx, _shutdown_rx_for_main) = watch::channel(false); - spawn_shutdown_signal(shutdown_tx.clone()); - - info!(did = %node_did, "node identity"); - info!(addr = %config.bind_addr(), "binding HTTP listener"); - - // Bind HTTP once, before dependency initialization, and keep this socket - // for the life of the process. The degraded server accepts on a dup of the - // same socket, so the degraded→full handoff never closes the port: while - // the full server initializes, connections queue in the shared backlog - // instead of being refused. - let listener = TcpListener::bind(config.bind_addr()) - .await - .with_context(|| format!("failed to bind to {}", config.bind_addr()))?; - let full_std_listener = listener.into_std()?; - let degraded_listener = TcpListener::from_std( - full_std_listener - .try_clone() - .context("failed to clone HTTP listener for degraded server")?, - )?; - - // Metrics must stay observable during a database outage — the degraded - // window is exactly when dashboards need data — so this listener starts - // before the DB connects. - let metrics_handle = if !config.metrics_addr.is_empty() { - match spawn_metrics_server(&config.metrics_addr, shutdown_tx.subscribe()).await { - Ok(handle) => { - info!(addr = %config.metrics_addr, "metrics endpoint listening"); - Some(handle) - } - Err(e) => { - warn!(err = %e, addr = %config.metrics_addr, "failed to start metrics endpoint — continuing without"); - None - } - } - } else { - info!("metrics endpoint disabled (GITLAWB_METRICS_ADDR not set)"); - None - }; - - let db_startup = Arc::new(DbStartupStatus::default()); - let (db_ready_tx, db_ready_rx) = watch::channel(false); - let mut degraded_handle = tokio::spawn(run_degraded_server( - degraded_listener, - node_did.to_string(), - Arc::clone(&db_startup), - db_ready_rx, - shutdown_tx.subscribe(), - )); - - // Connect to PostgreSQL database. A transient outage or bad secret should - // not crash-loop the process and hammer the database provider; permanent - // misconfiguration surfaces through error-level logs and the /ready check. - let db = tokio::select! { - db = connect_db_with_retry(&config, Arc::clone(&db_startup), shutdown_tx.subscribe()) => { - match db { - Some(db) => db, - None => { - // Shutdown requested while waiting for the database. The - // degraded server only serves one-shot 503s — abort it - // rather than drain, so a slow client can't stall exit. - degraded_handle.abort(); - return Ok(()); - } - } - } - degraded = &mut degraded_handle => { - if *shutdown_tx.borrow() { - return Ok(()); - } - return match degraded { - Ok(Ok(())) => Err(anyhow!("degraded HTTP server stopped before database became ready")), - Ok(Err(err)) => Err(err.context("degraded HTTP server failed")), - Err(err) => Err(anyhow!("degraded HTTP server task failed: {err}")), - }; - } - }; - - // Flip the degraded server into graceful shutdown, but do NOT await the - // drain: one slow in-flight request must not delay the full server, and - // the shared socket means there is no port gap to cover. The drain - // finishes (and logs) in the background. - db_ready_tx.send(true).ok(); - tokio::spawn(async move { - match degraded_handle.await { - Ok(Ok(())) => {} - Ok(Err(err)) => warn!(err = %err, "degraded HTTP server exited with error"), - Err(err) => warn!(err = %err, "degraded HTTP server task failed"), - } - }); - info!(addr = %config.bind_addr(), "database ready; starting full HTTP server"); - - // Prune peer rows that point back at this node (stale self-loop entries) - if let Some(public_url) = config.public_url.as_deref() { - match db.prune_self_peers(public_url).await { - Ok(0) => {} - Ok(n) => info!(removed = n, public_url, "pruned self-loop peer rows"), - Err(e) => warn!(err = %e, "prune_self_peers failed (non-fatal)"), - } - } - - // Prune peer rows with non-public hosts (loopback/private/internal) that - // were injected via the unauthenticated announce route — they poison the - // sync-notify fan-out (SSRF + crowding out real peers). - match db.prune_non_public_peers().await { - Ok(0) => {} - Ok(n) => info!(removed = n, "pruned non-public (poisoned) peer rows"), - Err(e) => warn!(err = %e, "prune_non_public_peers failed (non-fatal)"), - } - - // Ensure repos directory exists - std::fs::create_dir_all(&config.repos_dir).context("failed to create repos directory")?; - - // Start libp2p swarm (if p2p_port > 0) - let p2p_handle = if config.p2p_port > 0 { - let bootstrap_addrs = config - .p2p_bootstrap - .iter() - .filter_map(|s| s.parse().ok()) - .collect(); - let shutdown_rx = shutdown_tx.subscribe(); - match p2p::start( - &node_did.to_string(), - config.p2p_port, - bootstrap_addrs, - Arc::clone(&db), - config.auto_sync, - shutdown_rx, - ) - .await - { - Ok(handle) => { - info!(port = config.p2p_port, peer_id = %handle.local_peer_id, "libp2p swarm started"); - Some(Arc::new(handle)) - } - Err(e) => { - tracing::warn!(err = %e, "failed to start libp2p swarm — continuing without p2p"); - None - } - } - } else { - info!("p2p disabled (p2p_port = 0)"); - None - }; - - // Shared no-redirect HTTP client. See build_http_client for the SSRF rationale. - let http_client = Arc::new(build_http_client()?); - - let (ref_update_tx, _) = tokio::sync::broadcast::channel::(256); - let (task_event_tx, _) = tokio::sync::broadcast::channel::(256); - - let graphql_schema = Arc::new(graphql::build_schema( - Arc::clone(&db), - ref_update_tx.clone(), - task_event_tx.clone(), - )); - - let machine_id = std::env::var("FLY_MACHINE_ID").ok(); - if let Some(ref mid) = machine_id { - info!(" fly machine: {mid}"); - } - - // Initialize Tigris S3 client if bucket is configured - let tigris = if !config.tigris_bucket.is_empty() { - match git::tigris::TigrisClient::new(&config.tigris_bucket).await { - Ok(client) => { - info!(bucket = %config.tigris_bucket, "tigris storage enabled"); - Some(client) - } - Err(e) => { - tracing::warn!(err = %e, "failed to initialize Tigris client — using local-only storage"); - None - } - } - } else { - info!("tigris storage disabled (no bucket configured)"); - None - }; - - let repo_store = - git::repo_store::RepoStore::new(config.repos_dir.clone(), tigris, db.pool().clone()); - - // Per-DID limiter for the creation endpoints. Keyed on the authenticated - // DID (attacker-varied), so bound its key set to cap memory. - let rate_limiter = - rate_limit::RateLimiter::new_bounded(10, std::time::Duration::from_secs(3600), 200_000); - - // Per-client-IP flood brake for the creation endpoints. The per-DID limiter - // above is bypassed by a DID farm (one throwaway did:key per repo), which is - // exactly how the recurring spam-repo floods get past both it and the - // iCaptcha gate. Keyed on the resolved client IP so a single-source flood is - // capped regardless of how many identities it mints. Sized well above any - // legitimate per-IP creation rate; GITLAWB_CREATE_RATE_LIMIT overrides, 0 - // disables. Bounded key set — the key is a client-influenced IP. - let create_limit = std::env::var("GITLAWB_CREATE_RATE_LIMIT") - .ok() - .and_then(|v| v.trim().parse::().ok()) - .unwrap_or(120); - let create_ip_rate_limiter = rate_limit::RateLimiter::new_bounded( - create_limit, - std::time::Duration::from_secs(3600), - 200_000, - ); - if create_limit == 0 { - tracing::warn!("GITLAWB_CREATE_RATE_LIMIT=0 — per-IP creation rate limiting disabled"); - } - - // Push-path flood brake: max git-receive-pack requests per client IP per - // hour (counts both the info/refs advertisement and the push POST). Sized - // for heavy agent automation while still stopping flood traffic (the June - // 2026 attack pushed several times per second per IP). GITLAWB_PUSH_RATE_LIMIT - // overrides; 0 disables. Bounded key set — the key is a client-influenced IP. - let push_limit = std::env::var("GITLAWB_PUSH_RATE_LIMIT") - .ok() - .and_then(|v| v.trim().parse::().ok()) - .unwrap_or(600); - let push_rate_limiter = rate_limit::RateLimiter::new_bounded( - push_limit, - std::time::Duration::from_secs(3600), - 200_000, - ); - if push_limit == 0 { - tracing::warn!("GITLAWB_PUSH_RATE_LIMIT=0 — per-IP push rate limiting disabled"); - } - - // Which forwarded header the edge is trusted to set. Default None (trust - // nothing, key on the socket peer). Fly nodes set GITLAWB_TRUSTED_PROXY=fly; - // a node behind Caddy/NGINX sets it to x-forwarded-for. - let push_limiter_trust = rate_limit::TrustedProxy::from_env_value( - &std::env::var("GITLAWB_TRUSTED_PROXY").unwrap_or_default(), - ); - tracing::info!(trust = ?push_limiter_trust, push_limit, "push rate limiter configured"); - - // Peer-sync flood brakes, keyed on the resolved client IP (per-DID is useless - // here — a did:key farm self-registers). Two buckets so an unsigned notify - // flood can't drain the signed trigger caller's quota (#82). Bounded key sets - // (the key is a client-influenced IP); 0 disables each. - let sync_trigger_rate_limiter = rate_limit::RateLimiter::new_bounded( - config.sync_trigger_rate_limit, - std::time::Duration::from_secs(3600), - 200_000, - ); - let peer_write_rate_limiter = rate_limit::RateLimiter::new_bounded( - config.peer_write_rate_limit, - std::time::Duration::from_secs(3600), - 200_000, - ); - if config.sync_trigger_rate_limit == 0 { - tracing::warn!( - "GITLAWB_SYNC_TRIGGER_RATE_LIMIT=0 — /sync/trigger IP rate limiting disabled" - ); - } - if config.peer_write_rate_limit == 0 { - tracing::warn!("GITLAWB_PEER_WRITE_RATE_LIMIT=0 — peer-write IP rate limiting disabled"); - } - - // Initialize the iCaptcha proof gate (inert unless ICAPTCHA_MODE is set). - icaptcha::init().await; - - let state = AppState { - config: Arc::new(config.clone()), - db, - node_did: node_did.clone(), - node_keypair: Arc::new(keypair), - p2p: p2p_handle, - http_client, - ref_update_tx, - task_event_tx, - graphql_schema, - machine_id, - repo_store, - rate_limiter, - create_ip_rate_limiter, - push_rate_limiter, - push_limiter_trust, - sync_trigger_rate_limiter, - peer_write_rate_limiter, - shutdown_tx: shutdown_tx.clone(), - }; - - // Periodic peer-count poll for the metrics gauge. If p2p is disabled - // we still set the gauge to 0 so dashboards don't show "no data". - { - let p2p_for_metrics = state.p2p.clone(); - let mut shutdown_rx = state.subscribe_shutdown(); - tokio::spawn(async move { - let mut interval = tokio::time::interval(std::time::Duration::from_secs(15)); - loop { - tokio::select! { - _ = interval.tick() => { - let count = match &p2p_for_metrics { - Some(h) => h.status().await.map(|s| s.connected_peers).unwrap_or(0), - None => 0, - }; - metrics::set_peers_connected(count as i64); - } - _ = shutdown_rx.changed() => { - if *shutdown_rx.borrow() { - return; - } - } - } - } - }); - } - - // Periodic cleanup of expired rate limit entries + consumed-proof ledger - { - let rl = state.rate_limiter.clone(); - let create_ip_rl = state.create_ip_rate_limiter.clone(); - let push_rl = state.push_rate_limiter.clone(); - let sync_trigger_rl = state.sync_trigger_rate_limiter.clone(); - let peer_write_rl = state.peer_write_rate_limiter.clone(); - let db = state.db.clone(); - let mut shutdown_rx = state.subscribe_shutdown(); - tokio::spawn(async move { - loop { - tokio::select! { - _ = tokio::time::sleep(std::time::Duration::from_secs(300)) => { - rl.cleanup().await; - create_ip_rl.cleanup().await; - push_rl.cleanup().await; - sync_trigger_rl.cleanup().await; - peer_write_rl.cleanup().await; - let now = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .map(|d| d.as_secs() as i64) - .unwrap_or(0); - if let Err(e) = db.sweep_expired_proofs(now).await { - tracing::warn!(err = %e, "failed to sweep expired iCaptcha proofs"); - } - } - _ = shutdown_rx.changed() => { - if *shutdown_rx.borrow() { - break; - } - } - } - } - }); - } - - let router = server::build_router(state.clone()); - // Re-register the socket bound at startup — same fd, so there was never a - // moment with the port closed between the degraded and full servers. - let listener = TcpListener::from_std(full_std_listener) - .context("failed to re-register HTTP listener with the runtime")?; - - info!("✓ node started — did:{}", node_did); - info!(" repos dir: {}", config.repos_dir.display()); - info!( - " database: PostgreSQL ({})", - &config.database_url.split('@').next_back().unwrap_or("?") - ); - - // Publish our DID record to the Kademlia DHT shortly after startup - if let Some(p2p) = &state.p2p { - let did_record = p2p::DidRecord { - did: node_did.to_string(), - http_url: config.public_url.clone().unwrap_or_default(), - peer_id: p2p.local_peer_id.to_string(), - p2p_port: config.p2p_port, - timestamp: chrono::Utc::now().to_rfc3339(), - }; - let p2p_clone = Arc::clone(p2p); - let mut shutdown_rx = state.subscribe_shutdown(); - tokio::spawn(async move { - // Small delay so Kademlia can find peers first - tokio::select! { - _ = tokio::time::sleep(std::time::Duration::from_secs(5)) => {} - _ = shutdown_rx.changed() => return, - } - p2p_clone.put_did(did_record).await; - info!("DID record published to Kademlia DHT"); - }); - } - - // Spawn background gossip: announce to bootstrap peers, then ping known peers periodically - { - let gossip_state = state.clone(); - let bootstrap_peers = config.bootstrap_peers.clone(); - let shutdown_rx = state.subscribe_shutdown(); - tokio::spawn(async move { - gossip_task(gossip_state, bootstrap_peers, shutdown_rx).await; - }); - } - - // Start multi-node sync worker if auto_sync is enabled - if config.auto_sync { - sync::start( - Arc::clone(&state.db), - Arc::clone(&state.config), - Arc::clone(&state.node_keypair), - state.subscribe_shutdown(), - ); - info!("auto-sync worker started"); - } - - // On-chain operator setup: verify stake + spawn heartbeat loop - if !state.config.contract_node_staking.is_empty() - && !state.config.operator_private_key.is_empty() - { - match build_operator_client(&state.config, &state.node_did.to_string()) { - Ok(client) => match operator::startup_check(&client).await { - Ok(_) => { - let arc_client = Arc::new(client); - arc_client.spawn_heartbeat_loop(state.subscribe_shutdown()); - } - Err(e) => { - if state.config.operator_strict_mode { - return Err(e.context("strict-mode operator check failed")); - } - tracing::warn!(err = %e, "operator startup check failed — continuing without heartbeat loop"); - } - }, - Err(e) => { - if state.config.operator_strict_mode { - return Err(e.context("strict-mode: failed to build operator client")); - } - tracing::warn!(err = %e, "operator client could not be built — continuing without PoS"); - } - } - } else { - info!("on-chain PoS disabled (GITLAWB_CONTRACT_NODE_STAKING or GITLAWB_OPERATOR_PRIVATE_KEY unset)"); - } - - // axum's `with_graceful_shutdown` waits for in-flight requests to - // complete (up to the configured grace) once the future resolves. - let shutdown_signal_for_axum = state.subscribe_shutdown(); - let grace = std::time::Duration::from_secs(config.shutdown_grace_secs); - info!(grace_secs = config.shutdown_grace_secs, "axum server ready"); - - // `into_make_service_with_connect_info` exposes the socket peer address as - // `ConnectInfo` so the push limiter can key on the real client - // when no trusted proxy header applies (see `rate_limit::client_key`). - let serve_result = axum::serve( - listener, - router.into_make_service_with_connect_info::(), - ) - .with_graceful_shutdown(async move { - let mut rx = shutdown_signal_for_axum; - // Wait until the watcher flips to true, then return so axum - // can begin draining. - while !*rx.borrow_and_update() { - if rx.changed().await.is_err() { - // Sender dropped — treat as shutdown. - break; - } - } - }) - .await; - - // Server has stopped accepting new connections and drained in-flight - // requests. Tear the rest of the system down. - info!("HTTP server stopped, beginning process shutdown"); - if let Some(h) = metrics_handle { - h.abort(); - } - let _ = grace; // recorded for operators in the log above; not enforced - serve_result?; - info!("clean exit"); - Ok(()) -} - -fn spawn_shutdown_signal(tx: watch::Sender) { - tokio::spawn(async move { - #[cfg(unix)] - { - use tokio::signal::unix::{signal as unix_signal, SignalKind}; - let mut sigterm = - unix_signal(SignalKind::terminate()).expect("install SIGTERM handler"); - let mut sigint = unix_signal(SignalKind::interrupt()).expect("install SIGINT handler"); - tokio::select! { - _ = sigterm.recv() => info!("SIGTERM received, shutting down"), - _ = sigint.recv() => info!("SIGINT received, shutting down"), - } - } - #[cfg(not(unix))] - { - use tokio::signal; - let _ = signal::ctrl_c().await; - info!("Ctrl-C received, shutting down"); - } - tx.send(true).ok(); - }); -} - -async fn connect_db_with_retry( - config: &Config, - db_startup: Arc, - mut shutdown_rx: watch::Receiver, -) -> Option> { - let initial_retry_secs = config.db_retry_initial_secs; - let max_retry_secs = config.db_retry_max_secs.max(initial_retry_secs); - let acquire_timeout = std::time::Duration::from_secs(config.db_acquire_timeout_secs); - let attempt_timeout = std::time::Duration::from_secs(config.db_connect_timeout_secs); - let mut attempts = 0_u64; - - loop { - if *shutdown_rx.borrow() { - return None; - } - - attempts = attempts.saturating_add(1); - db_startup.attempts.store(attempts, Ordering::Relaxed); - - // Bound the whole attempt, not just the pool connect: migrations - // block on a cross-instance advisory lock, and an unbounded wait - // there would wedge this loop — no retries, no logs, no recovery. - // Timing out and retrying is safe; migrations are idempotent. - let attempt = match tokio::time::timeout( - attempt_timeout, - Db::connect( - &config.database_url, - config.db_max_connections, - acquire_timeout, - ), - ) - .await - { - Ok(result) => result, - Err(_) => Err(anyhow!( - "connect + migrate attempt exceeded {}s (GITLAWB_DB_CONNECT_TIMEOUT_SECS); \ - is another instance holding the migration lock?", - attempt_timeout.as_secs() - )), - }; - - match attempt { - Ok(db) => { - info!(attempts, "database connection established"); - return Some(Arc::new(db)); - } - Err(err) => { - // A bad DATABASE_URL or rejected credentials won't heal on - // their own. Still retry (exiting would crash-loop and hammer - // the provider — and take liveness down with it), but log at - // error level and skip straight to the maximum backoff; the - // /ready health check is what surfaces this to deploys. - let permanent = is_likely_permanent_db_error(&err); - let retry_secs = if permanent { - max_retry_secs - } else { - database_retry_delay_secs(initial_retry_secs, max_retry_secs, attempts) - }; - db_startup - .next_retry_secs - .store(retry_secs, Ordering::Relaxed); - if permanent { - tracing::error!( - attempts, - retry_secs, - err = %err, - "database rejected our configuration (bad DATABASE_URL or credentials?) — retrying, but operator action is likely required" - ); - } else { - warn!( - attempts, - retry_secs, - err = %err, - "database unavailable during startup; retrying" - ); - } - - tokio::select! { - _ = tokio::time::sleep(std::time::Duration::from_secs(retry_secs)) => {} - changed = shutdown_rx.changed() => { - if changed.is_err() || *shutdown_rx.borrow() { - return None; - } - } - } - } - } - } -} - -/// Errors that indicate misconfiguration rather than a transient outage: a -/// malformed DATABASE_URL, or a server that answered and rejected us — -/// Postgres error class 28xxx (invalid authorization) or 3D000 (database -/// does not exist). Best-effort: an error that anyhow can't downcast back to -/// sqlx just counts as transient. -fn is_likely_permanent_db_error(err: &anyhow::Error) -> bool { - match err.downcast_ref::() { - Some(sqlx::Error::Configuration(_)) => true, - Some(sqlx::Error::Database(db)) => db - .code() - .map(|c| c.starts_with("28") || c.starts_with("3D")) - .unwrap_or(false), - _ => false, - } -} - -fn database_retry_delay_secs(initial_secs: u64, max_secs: u64, attempts: u64) -> u64 { - // The exponent bound only keeps the u32 cast safe — max_secs is the real - // (operator-configurable) cap, and saturating math handles overflow. - let exponent = attempts.saturating_sub(1).min(63) as u32; - initial_secs - .saturating_mul(2_u64.saturating_pow(exponent)) - .min(max_secs) -} - -async fn run_degraded_server( - listener: TcpListener, - node_did: String, - db_startup: Arc, - mut db_ready_rx: watch::Receiver, - mut shutdown_rx: watch::Receiver, -) -> Result<()> { - let addr = listener.local_addr().ok(); - let router = build_degraded_router(node_did, db_startup); - info!(?addr, "degraded HTTP server ready"); - - axum::serve(listener, router) - .with_graceful_shutdown(async move { - // wait_for resolves on predicate-true or sender-drop; either way - // this phase is over. - tokio::select! { - _ = db_ready_rx.wait_for(|ready| *ready) => {} - _ = shutdown_rx.wait_for(|stop| *stop) => {} - } - }) - .await?; - - Ok(()) -} - -fn build_degraded_router(node_did: String, db_startup: Arc) -> Router { - let state = DegradedState { - node_did, - db_startup, - }; - // Everything answers 503 with the same body — including /health and - // /ready, so peer pings (which treat any 2xx /health as alive) and - // uptime monitors correctly see a node that cannot serve traffic. - // `/` additionally carries the node identity for probing peers. - Router::new() - .route("/", get(degraded_node_info)) - .fallback(degraded_unavailable) - .with_state(state) -} - -/// One source of truth for the degraded 503 body, sharing the error -/// vocabulary with error.rs so clients see the same code/message for -/// "database unavailable" regardless of which phase produced it. -fn degraded_body(db_startup: &DbStartupStatus) -> serde_json::Value { - serde_json::json!({ - "status": "degraded", - "database": "initializing", - "error": error::DB_UNAVAILABLE_CODE, - "message": error::DB_UNAVAILABLE_MESSAGE, - "db_attempts": db_startup.attempts.load(Ordering::Relaxed), - "db_next_retry_secs": db_startup.next_retry_secs.load(Ordering::Relaxed), - }) -} - -async fn degraded_node_info(State(state): State) -> impl IntoResponse { - let mut body = degraded_body(&state.db_startup); - if let Some(obj) = body.as_object_mut() { - obj.insert("name".into(), "gitlawb-node".into()); - obj.insert("version".into(), env!("CARGO_PKG_VERSION").into()); - obj.insert("did".into(), state.node_did.clone().into()); - } - (StatusCode::SERVICE_UNAVAILABLE, Json(body)) -} - -async fn degraded_unavailable(State(state): State) -> impl IntoResponse { - ( - StatusCode::SERVICE_UNAVAILABLE, - Json(degraded_body(&state.db_startup)), - ) -} - -/// Spawn a small axum router that exposes only `GET /metrics` on its own -/// listener. Returns the JoinHandle so `main()` can abort it on shutdown. -/// This is deliberately separate from the main router so the metrics port -/// can be firewalled differently from the API port — bind to localhost -/// or a private interface only. -async fn spawn_metrics_server( - addr: &str, - mut shutdown_rx: watch::Receiver, -) -> Result> { - use axum::{response::IntoResponse, routing::get, Router}; - - async fn metrics_handler() -> impl IntoResponse { - match metrics::encode() { - Ok(body) => ( - axum::http::StatusCode::OK, - [( - axum::http::header::CONTENT_TYPE, - "text/plain; version=0.0.4; charset=utf-8", - )], - body, - ), - Err(e) => ( - axum::http::StatusCode::INTERNAL_SERVER_ERROR, - [( - axum::http::header::CONTENT_TYPE, - "text/plain; charset=utf-8", - )], - format!("metrics encode error: {e}"), - ), - } - } - - let listener = TcpListener::bind(addr) - .await - .with_context(|| format!("failed to bind metrics listener to {addr}"))?; - let app = Router::new().route("/metrics", get(metrics_handler)); - - let handle = tokio::spawn(async move { - if let Err(e) = axum::serve(listener, app) - .with_graceful_shutdown(async move { - while !*shutdown_rx.borrow_and_update() { - if shutdown_rx.changed().await.is_err() { - break; - } - } - }) - .await - { - warn!(err = %e, "metrics server exited with error"); - } - }); - Ok(handle) -} - -fn build_operator_client( - config: &config::Config, - node_did: &str, -) -> Result { - use alloy::primitives::Address; - use std::str::FromStr; - - let contract_address = Address::from_str(&config.contract_node_staking) - .with_context(|| format!("invalid contract address: {}", config.contract_node_staking))?; - - let cfg = operator::OperatorConfig { - rpc_url: config.chain_rpc_url.clone(), - private_key: config.operator_private_key.clone(), - contract_address, - node_did: node_did.to_string(), - heartbeat_interval: std::time::Duration::from_secs(config.heartbeat_interval_hours * 3600), - strict_mode: config.operator_strict_mode, - }; - Ok(operator::OperatorClient::new(cfg)) -} - -/// Announce to bootstrap peers on startup, then periodically ping all known peers. -async fn gossip_task( - state: AppState, - bootstrap_peers: Vec, - mut shutdown_rx: tokio::sync::watch::Receiver, -) { - // If shutdown arrives during the initial delay, exit before announcing. - tokio::select! { - _ = tokio::time::sleep(std::time::Duration::from_secs(2)) => {} - _ = shutdown_rx.changed() => { - if *shutdown_rx.borrow() { - info!("gossip: shutdown during startup delay, exiting"); - return; - } - } - } - - // Reuse the shared no-redirect client for every gossip outbound call (the - // bootstrap announce POST and the periodic peer /health ping). Peer URLs are - // attacker-influenceable, so a 3xx to a private address must not be followed. - // Do NOT fall back to reqwest::Client::new(): its default follows redirects - // and would reintroduce the SSRF closed here (#93). - let client = state.http_client.clone(); - let my_did = state.node_did.to_string(); - let my_url = state.config.public_url.clone().unwrap_or_default(); - - // Announce ourselves to each bootstrap peer - for peer_url in &bootstrap_peers { - // Cooperative shutdown between peers — a slow peer shouldn't - // block the node exiting. - if *shutdown_rx.borrow() { - info!("gossip: shutdown signalled during peer announce, exiting"); - return; - } - let path = "/api/v1/peers/announce"; - let announce_url = format!("{}{}", peer_url.trim_end_matches('/'), path); - let body = serde_json::json!({ - "did": my_did.clone(), - "http_url": my_url.clone(), - }); - let body_bytes = match serde_json::to_vec(&body) { - Ok(bytes) => bytes, - Err(e) => { - tracing::warn!(err = %e, "failed to serialize peer announce body"); - continue; - } - }; - let signed = sign_request(state.node_keypair.as_ref(), "POST", path, &body_bytes); - // Per-request timeout inside the loop; do not let one hung peer - // block others. The request itself is a normal tokio future so - // it's cancel-safe on shutdown. - match tokio::time::timeout( - std::time::Duration::from_secs(5), - client - .post(&announce_url) - .header("Content-Type", "application/json") - .header("Content-Digest", signed.content_digest) - .header("Signature-Input", signed.signature_input) - .header("Signature", signed.signature) - .body(body_bytes) - .send(), - ) - .await - { - Ok(Ok(resp)) => { - if resp.status().is_success() { - if let Ok(json) = resp.json::().await { - // Add them back to our peer list - if let (Some(their_did), Some(their_url)) = ( - json.get("node_did").and_then(|v| v.as_str()), - json.get("node_url").and_then(|v| v.as_str()), - ) { - if !their_url.is_empty() { - let _ = state.db.upsert_peer(their_did, their_url).await; - tracing::info!(did = %their_did, url = %their_url, "bootstrap peer added"); - } - } - } - } - } - Ok(Err(e)) => { - tracing::warn!(url = %announce_url, err = %e, "failed to announce to bootstrap peer") - } - Err(_) => tracing::warn!(url = %announce_url, "bootstrap peer announce timed out (5s)"), - } - } - - // Periodic ping every 5 minutes — exit on shutdown. - let mut interval = tokio::time::interval(std::time::Duration::from_secs(300)); - loop { - tokio::select! { - _ = interval.tick() => { - let peers = match state.db.list_peers().await { - Ok(p) => p, - Err(_) => continue, - }; - for peer in peers { - let ok = ping_peer_health(&client, &peer.http_url).await; - let _ = state.db.mark_peer_ping(&peer.did, ok).await; - } - } - _ = shutdown_rx.changed() => { - if *shutdown_rx.borrow() { - info!("gossip task: shutdown signal received, exiting"); - return; - } - } - } - } -} - -/// Build the shared node HTTP client used for every outbound fan-out (sync -/// trigger, profile/repo fetches, gossip announce + peer pings). -/// -/// No redirects: peer URLs are attacker-influenceable, so a `3xx` to a private -/// address must not be followed (SSRF guard, #78/#93). Do NOT replace with -/// `reqwest::Client::new()` — its default follows redirects. Kept as a named -/// builder so tests bind the redirect guarantee to the real client the node -/// runs, not a hand-rolled equivalent. -fn build_http_client() -> reqwest::Result { - reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(10)) - .redirect(reqwest::redirect::Policy::none()) - .build() -} - -/// Ping a peer's `/health` endpoint and report whether it answered 2xx. -/// -/// Takes the client by reference so callers supply the shared, no-redirect -/// `state.http_client`. Peer URLs are attacker-influenceable, so a `3xx` to a -/// private address must not be followed. Do NOT call this with a bare -/// `reqwest::Client::new()`: its default follows redirects and would -/// reintroduce the SSRF this guards against (#93). -async fn ping_peer_health(client: &reqwest::Client, http_url: &str) -> bool { - let url = format!("{}/health", http_url.trim_end_matches('/')); - client - .get(&url) - .send() - .await - .map(|r| r.status().is_success()) - .unwrap_or(false) -} - -fn load_or_create_keypair(config: &Config) -> Result { - let key_path = config.resolved_key_path(); - - if key_path.exists() { - let pem = std::fs::read_to_string(&key_path) - .with_context(|| format!("failed to read key from {}", key_path.display()))?; - let kp = Keypair::from_pem(&pem).map_err(|e| anyhow::anyhow!("invalid PEM key: {e}"))?; - info!(path = %key_path.display(), "loaded existing identity"); - Ok(kp) - } else { - let kp = Keypair::generate(); - let pem = kp - .to_pem() - .map_err(|e| anyhow::anyhow!("failed to serialize key: {e}"))?; - - if let Some(parent) = key_path.parent() { - std::fs::create_dir_all(parent)?; - } - - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - std::fs::write(&key_path, pem.as_bytes())?; - std::fs::set_permissions(&key_path, std::fs::Permissions::from_mode(0o600))?; - } - #[cfg(not(unix))] - std::fs::write(&key_path, pem.as_bytes())?; - - info!(path = %key_path.display(), did = %kp.did(), "generated new node identity"); - Ok(kp) - } -} - -#[cfg(test)] -mod gossip_ssrf_tests { - use super::ping_peer_health; - - // Build the client exactly as production does (super::build_http_client) so - // these tests bind the redirect guarantee to the real shared client the - // node runs. A regression that makes build_http_client follow redirects - // fails ping_peer_health_does_not_follow_redirect. - fn production_http_client() -> reqwest::Client { - super::build_http_client().expect("failed to build production http client") - } - - // A peer answering `/health` with a 302 toward an internal address must not - // be followed: the redirect target must never be requested (#93). - #[tokio::test] - async fn ping_peer_health_does_not_follow_redirect() { - let mut server = mockito::Server::new_async().await; - let internal = server - .mock("GET", "/internal-metadata") - .with_status(200) - .expect(0) - .create_async() - .await; - let _health = server - .mock("GET", "/health") - .with_status(302) - .with_header("location", &format!("{}/internal-metadata", server.url())) - .create_async() - .await; - - let ok = ping_peer_health(&production_http_client(), &server.url()).await; - - assert!(!ok, "a 302 must not count as a healthy peer"); - // expect(0) is enforced only at assert time; this fails if the redirect - // was followed to the internal target. - internal.assert_async().await; - } - - #[tokio::test] - async fn ping_peer_health_reports_success_on_200() { - let mut server = mockito::Server::new_async().await; - let _health = server - .mock("GET", "/health") - .with_status(200) - .create_async() - .await; - - let ok = ping_peer_health(&production_http_client(), &server.url()).await; - - assert!(ok, "a 200 /health must count as a healthy peer"); - } - - // A transport error (nothing listening) must map to unhealthy, never a - // spurious healthy — the .unwrap_or(false) arm. - #[tokio::test] - async fn ping_peer_health_reports_unhealthy_on_connection_error() { - let ok = ping_peer_health(&production_http_client(), "http://127.0.0.1:1").await; - assert!(!ok, "a connection error must count as an unhealthy peer"); - } +async fn main() -> anyhow::Result<()> { + gitlawb_node::run().await } From f240af4d966c19844d315368678ae4e1bb686fd9 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 16:09:44 -0500 Subject: [PATCH 02/41] test(node): real-node deny harness rig + INV-8 unsigned-push case (U2-U4, U5a) Add a feature-gated (test-harness) spawn surface (src/test_harness.rs) that boots a real node on 127.0.0.1:0 over an ephemeral #[sqlx::test] pool through the production axum::serve stack with connect-info, and an integration crate (tests/deny_harness.rs) that drives deny paths with a real reqwest client: - U2 signing client: wraps gitlawb_core::http_sig::sign_request for reqwest; self-checks that a valid signature clears require_signature and a tampered body is rejected (400 content_digest_mismatch). - U3 spawn_node: real socket, p2p disabled, per-test DB, shutdown-on-drop. - U4 assert_denied: 4xx AND body-no-leak AND not-empty-200 (INV-8); pure core unit-tested for clean-403 / empty-200 / leaking-403 / wrong-status. - U5a INV-8: unsigned git-receive-pack is denied 401 with no leak. Widens the three cfg(test) test builders (Db/RepoStore::for_testing, run_migrations) to also compile under the feature. No production behavior change: prod build (no feature) excludes test_harness; node suite stays green (488) and the 7 integration tests pass. --- crates/gitlawb-node/Cargo.toml | 19 +++ crates/gitlawb-node/src/db/mod.rs | 4 +- crates/gitlawb-node/src/git/repo_store.rs | 2 +- crates/gitlawb-node/src/lib.rs | 3 + crates/gitlawb-node/src/test_harness.rs | 142 +++++++++++++++++++ crates/gitlawb-node/tests/deny_harness.rs | 101 +++++++++++++ crates/gitlawb-node/tests/support/assert.rs | 95 +++++++++++++ crates/gitlawb-node/tests/support/mod.rs | 6 + crates/gitlawb-node/tests/support/signing.rs | 32 +++++ 9 files changed, 401 insertions(+), 3 deletions(-) create mode 100644 crates/gitlawb-node/src/test_harness.rs create mode 100644 crates/gitlawb-node/tests/deny_harness.rs create mode 100644 crates/gitlawb-node/tests/support/assert.rs create mode 100644 crates/gitlawb-node/tests/support/mod.rs create mode 100644 crates/gitlawb-node/tests/support/signing.rs diff --git a/crates/gitlawb-node/Cargo.toml b/crates/gitlawb-node/Cargo.toml index 1f631635..c7618a07 100644 --- a/crates/gitlawb-node/Cargo.toml +++ b/crates/gitlawb-node/Cargo.toml @@ -14,6 +14,20 @@ path = "src/lib.rs" name = "gitlawb-node" path = "src/main.rs" +# Exposes the `test_harness` spawn surface (src/test_harness.rs) to the +# real-node deny-harness integration crate. Off by default so the production +# binary never compiles test-only boot code. Enable with +# `cargo test -p gitlawb-node --features test-harness`. +[features] +test-harness = [] + +# Only builds when `test-harness` is enabled; otherwise skipped (not an error), +# so `cargo test --workspace` without the feature stays green. +[[test]] +name = "deny_harness" +path = "tests/deny_harness.rs" +required-features = ["test-harness"] + [dependencies] gitlawb-core = { path = "../gitlawb-core" } ed25519-dalek = { workspace = true } @@ -81,3 +95,8 @@ libp2p-dns = { version = "0.44.0", features = ["tokio"] } [dev-dependencies] mockito = "1" tempfile = "3" +# Used by the deny-harness integration crate (tests/deny_harness.rs): the +# #[sqlx::test] macro for an ephemeral per-test pool, and reqwest as the real +# HTTP client that drives deny paths over the socket. +sqlx = { version = "0.8", features = ["postgres", "runtime-tokio-rustls", "macros", "migrate"] } +reqwest = { workspace = true } diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 5324a4b9..364d81bd 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -252,7 +252,7 @@ impl Db { &self.pool } - #[cfg(test)] + #[cfg(any(test, feature = "test-harness"))] pub fn for_testing(pool: PgPool) -> Self { Self { pool } } @@ -261,7 +261,7 @@ impl Db { /// provisions an empty per-test database, so DB-backed tests must run this /// before seeding. Reuses the production `migrate()` path (the advisory lock /// is harmless on an isolated test DB and migrations are idempotent). - #[cfg(test)] + #[cfg(any(test, feature = "test-harness"))] pub(crate) async fn run_migrations(&self) -> Result<()> { self.migrate().await } diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index a5c367e9..73d42fa1 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -33,7 +33,7 @@ pub struct RepoStore { } impl RepoStore { - #[cfg(test)] + #[cfg(any(test, feature = "test-harness"))] pub fn for_testing(repos_dir: PathBuf, pool: PgPool) -> Self { Self { repos_dir, diff --git a/crates/gitlawb-node/src/lib.rs b/crates/gitlawb-node/src/lib.rs index fb0a8c55..f173b053 100644 --- a/crates/gitlawb-node/src/lib.rs +++ b/crates/gitlawb-node/src/lib.rs @@ -26,6 +26,9 @@ pub mod state; mod sync; #[cfg(test)] mod test_support; +#[cfg(feature = "test-harness")] +#[doc(hidden)] +pub mod test_harness; mod visibility; mod webhooks; diff --git a/crates/gitlawb-node/src/test_harness.rs b/crates/gitlawb-node/src/test_harness.rs new file mode 100644 index 00000000..10228613 --- /dev/null +++ b/crates/gitlawb-node/src/test_harness.rs @@ -0,0 +1,142 @@ +//! Test-only spawn surface for the real-node deny harness (see +//! `tests/deny_harness.rs`). Feature-gated behind `test-harness` so it never +//! compiles into the production binary. It exposes a single boot constructor so +//! the out-of-crate integration test can bring up a real node over a bound TCP +//! socket without the integration crate needing access to `graphql`, +//! `rate_limit`, `repo_store`, or the other internals `AppState` is built from. +//! +//! The node is built the same way `test_support::build_state` builds it (p2p +//! disabled, real migrated pool), but bound on `127.0.0.1:0` and served through +//! the real `axum::serve` stack with connect-info, so the middleware order, +//! body limits, and per-IP rate limiters all run exactly as in production. + +use std::net::SocketAddr; +use std::path::PathBuf; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Arc; +use std::time::Duration; + +use clap::Parser; +use sqlx::PgPool; +use tokio::net::TcpListener; +use tokio::sync::watch; + +use gitlawb_core::identity::Keypair; + +use crate::config::Config; +use crate::rate_limit::{RateLimiter, TrustedProxy}; +use crate::state::AppState; + +/// A running node bound to an ephemeral port. Dropping it signals graceful +/// shutdown and removes the temporary repository directory. +pub struct TestNode { + /// Base URL of the running node, e.g. `http://127.0.0.1:54321`. + pub base_url: String, + /// The node's own DID (for building requests that reference it). + pub node_did: String, + shutdown_tx: watch::Sender, + repos_dir: PathBuf, +} + +impl Drop for TestNode { + fn drop(&mut self) { + // Flip the shared shutdown signal so the serve task exits, then remove + // the temp repos dir. Both are best-effort: a test that already failed + // should not panic again in teardown. + let _ = self.shutdown_tx.send(true); + let _ = std::fs::remove_dir_all(&self.repos_dir); + } +} + +/// Allocate a process-unique temp directory for a spawned node's repositories +/// without pulling in the `tempfile` dev-dependency (which is unavailable to +/// the library crate under `--features test-harness`). +fn unique_repos_dir() -> PathBuf { + static COUNTER: AtomicU64 = AtomicU64::new(0); + let n = COUNTER.fetch_add(1, Ordering::Relaxed); + std::env::temp_dir().join(format!( + "gitlawb-deny-harness-{}-{}", + std::process::id(), + n + )) +} + +/// Build an [`AppState`] over the given migrated pool, mirroring +/// `test_support::build_state` but with a real on-disk repository directory so +/// git smart-HTTP routes can serve. P2P is always disabled. +fn build_state(db: Arc, pool: PgPool, repos_dir: PathBuf) -> AppState { + let keypair = Keypair::generate(); + let node_did = keypair.did(); + let (ref_tx, _) = tokio::sync::broadcast::channel(1); + let (task_tx, _) = tokio::sync::broadcast::channel(1); + let schema = Arc::new(crate::graphql::build_schema( + db.clone(), + ref_tx.clone(), + task_tx.clone(), + )); + AppState { + config: Arc::new(Config::parse_from(["gitlawb-node"])), + db, + node_did, + node_keypair: Arc::new(keypair), + p2p: None, + http_client: Arc::new(reqwest::Client::new()), + ref_update_tx: ref_tx, + task_event_tx: task_tx, + graphql_schema: schema, + machine_id: None, + repo_store: crate::git::repo_store::RepoStore::for_testing(repos_dir, pool), + rate_limiter: RateLimiter::new(100, Duration::from_secs(60)), + create_ip_rate_limiter: RateLimiter::new(1000, Duration::from_secs(3600)), + push_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + push_limiter_trust: TrustedProxy::None, + sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), + peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + shutdown_tx: watch::channel(false).0, + } +} + +/// Spawn a real node bound to `127.0.0.1:0` over the given (already-created, +/// empty) test pool. Runs the schema migrations, builds the router, and serves +/// it on a background task through the production `axum::serve` stack with +/// connect-info so per-IP layers key on the real peer. Returns once the socket +/// is bound and accepting connections. +pub async fn spawn_node(pool: PgPool) -> TestNode { + let db = Arc::new(crate::db::Db::for_testing(pool.clone())); + db.run_migrations() + .await + .expect("test schema migrations should apply"); + + let repos_dir = unique_repos_dir(); + std::fs::create_dir_all(&repos_dir).expect("create temp repos dir"); + + let state = build_state(db, pool, repos_dir.clone()); + let node_did = state.node_did.to_string(); + let shutdown_tx = state.shutdown_tx.clone(); + let mut shutdown_rx = shutdown_tx.subscribe(); + + let router = crate::server::build_router(state); + + let listener = TcpListener::bind(("127.0.0.1", 0)) + .await + .expect("bind ephemeral port"); + let addr = listener.local_addr().expect("read bound addr"); + + tokio::spawn(async move { + let _ = axum::serve( + listener, + router.into_make_service_with_connect_info::(), + ) + .with_graceful_shutdown(async move { + let _ = shutdown_rx.changed().await; + }) + .await; + }); + + TestNode { + base_url: format!("http://{addr}"), + node_did, + shutdown_tx, + repos_dir, + } +} diff --git a/crates/gitlawb-node/tests/deny_harness.rs b/crates/gitlawb-node/tests/deny_harness.rs new file mode 100644 index 00000000..ad4edeb9 --- /dev/null +++ b/crates/gitlawb-node/tests/deny_harness.rs @@ -0,0 +1,101 @@ +//! Real-node deny harness: boots a real gitlawb-node over a bound TCP socket +//! and drives trust-boundary DENY paths through a real reqwest client, +//! asserting both the refusal status and that no withheld data leaks +//! (INV-1/INV-2/INV-8). Requires `--features test-harness`. +//! +//! Each `#[sqlx::test]` gets an ephemeral per-test database; `spawn_node` runs +//! the schema migrations and serves the real router on `127.0.0.1:0`. + +mod support; + +use support::assert::assert_denied; +use support::signing::signed_request; + +use gitlawb_core::identity::Keypair; +use gitlawb_node::test_harness::spawn_node; + +// ── U2: the signing client produces signatures require_signature accepts ───── + +/// A validly signed receive-pack request clears `require_signature` (it does not +/// get a 401): it proceeds past the signature layer and is denied later for a +/// different reason (the repo does not exist). This proves the signing client +/// is producing signatures the real verifier accepts over the socket. +#[sqlx::test] +async fn signed_receive_pack_clears_signature_layer(pool: sqlx::PgPool) { + let node = spawn_node(pool).await; + let client = reqwest::Client::new(); + let kp = Keypair::generate(); + + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + "/alice/repo/git-receive-pack", + b"0000".to_vec(), + &kp, + ) + .send() + .await + .expect("request sends"); + + assert_ne!( + resp.status().as_u16(), + 401, + "a valid signature must clear require_signature; got 401" + ); +} + +/// Tampering the body after signing invalidates the content-digest, so the +/// server rejects with 400 `content_digest_mismatch` (distinct from the 401 a +/// missing/invalid signature gets). Proves the signature actually covers the +/// body: the digest is signed and the server re-checks it against the bytes it +/// received. +#[sqlx::test] +async fn tampered_body_after_signing_is_rejected(pool: sqlx::PgPool) { + let node = spawn_node(pool).await; + let client = reqwest::Client::new(); + let kp = Keypair::generate(); + + // Sign one body, then replace it before sending. + let mut req = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + "/alice/repo/git-receive-pack", + b"0000".to_vec(), + &kp, + ) + .build() + .expect("build request"); + *req.body_mut() = Some(reqwest::Body::from(b"tampered".to_vec())); + + let resp = client.execute(req).await.expect("request sends"); + assert_eq!( + resp.status().as_u16(), + 400, + "a body that no longer matches its content-digest must be rejected" + ); +} + +// ── U5(a): INV-8 — an unsigned push is denied and leaks nothing ────────────── + +/// An unauthenticated git-receive-pack (no signature headers) is rejected with +/// 401 before any handler runs, and the denial body carries no repo internals. +#[sqlx::test] +async fn unsigned_receive_pack_is_denied(pool: sqlx::PgPool) { + let node = spawn_node(pool).await; + let client = reqwest::Client::new(); + + let url = format!("{}/alice/repo/git-receive-pack", node.base_url); + let resp = client + .post(&url) + .header("content-type", "application/x-git-receive-pack-request") + .body(b"0000".to_vec()) + .send() + .await + .expect("request sends"); + + // No repo was seeded, so there are no OIDs to leak; the assertion still + // enforces the 4xx-and-not-empty-200 INV-8 shape. + assert_denied(resp, 401, &[]).await; +} diff --git a/crates/gitlawb-node/tests/support/assert.rs b/crates/gitlawb-node/tests/support/assert.rs new file mode 100644 index 00000000..50dca250 --- /dev/null +++ b/crates/gitlawb-node/tests/support/assert.rs @@ -0,0 +1,95 @@ +//! INV-8 deny assertion. A denial must be an actual refusal (4xx) whose body +//! carries none of the data it is withholding, and must never be a 2xx (an +//! empty-200 rendered as success is the denial-as-success bug this guards). +//! +//! The check is split into a pure `check_denied` (unit-tested below with the +//! self-check scenarios) and an async `assert_denied` wrapper that reads a real +//! `reqwest::Response`. + +/// Pure deny check. Returns `Err(reason)` when the response is not a clean +/// denial. `withheld` are exact tokens (OIDs, paths, private slugs) that must +/// not appear anywhere in the body; register short-OID prefixes and encoded +/// path forms as separate tokens at the call site (a raw substring scan cannot +/// normalize them). +pub fn check_denied( + status: u16, + body: &str, + expected: u16, + withheld: &[&str], +) -> Result<(), String> { + // The harness only ever expects denials; a non-4xx expectation is a + // programming error in the test, not a node behavior under test. + if !(400..500).contains(&expected) { + return Err(format!( + "test bug: assert_denied expected status {expected} is not a 4xx" + )); + } + // INV-8: a denial rendered as success is the headline bug. Catch it + // explicitly so the failure message names it, rather than only failing the + // status-equality check below. + if (200..300).contains(&status) { + return Err(format!( + "denial rendered as success: got {status}, expected denial {expected}. \ + body={body:?}" + )); + } + if status != expected { + return Err(format!( + "expected denial status {expected}, got {status}. body={body:?}" + )); + } + for token in withheld { + if !token.is_empty() && body.contains(token) { + return Err(format!( + "withheld token {token:?} leaked in denial body: {body:?}" + )); + } + } + Ok(()) +} + +/// Drive a real response through the deny check and panic on failure. Reads the +/// full body once. +pub async fn assert_denied(resp: reqwest::Response, expected: u16, withheld: &[&str]) { + let status = resp.status().as_u16(); + let body = resp.text().await.unwrap_or_default(); + if let Err(reason) = check_denied(status, &body, expected, withheld) { + panic!("{reason}"); + } +} + +#[cfg(test)] +mod tests { + use super::check_denied; + + #[test] + fn clean_403_with_no_leak_passes() { + let r = check_denied(403, r#"{"error":"only the repo owner can do this"}"#, 403, &["a1b2c3"]); + assert!(r.is_ok(), "{r:?}"); + } + + #[test] + fn empty_200_is_flagged_as_denial_rendered_as_success() { + let r = check_denied(200, "", 403, &[]); + assert!(r.is_err()); + assert!(r.unwrap_err().contains("rendered as success")); + } + + #[test] + fn leaking_403_fails_and_names_the_token() { + let withheld = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"; + let body = format!("not found near object {withheld}"); + let r = check_denied(403, &body, 403, &[withheld]); + assert!(r.is_err()); + assert!(r.unwrap_err().contains(withheld)); + } + + #[test] + fn wrong_status_fails() { + // A 404 when the owner gate's 403 was expected must not pass: it means + // an earlier layer denied, so the gate under test was never exercised. + let r = check_denied(404, "not found", 403, &[]); + assert!(r.is_err()); + assert!(r.unwrap_err().contains("expected denial status 403")); + } +} diff --git a/crates/gitlawb-node/tests/support/mod.rs b/crates/gitlawb-node/tests/support/mod.rs new file mode 100644 index 00000000..75828f5f --- /dev/null +++ b/crates/gitlawb-node/tests/support/mod.rs @@ -0,0 +1,6 @@ +//! Shared support for the real-node deny harness: the deny assertion (U4), the +//! RFC-9421 signing client (U2), and (via `gitlawb_node::test_harness`) the +//! node spawner (U3). + +pub mod assert; +pub mod signing; diff --git a/crates/gitlawb-node/tests/support/signing.rs b/crates/gitlawb-node/tests/support/signing.rs new file mode 100644 index 00000000..3e0d543c --- /dev/null +++ b/crates/gitlawb-node/tests/support/signing.rs @@ -0,0 +1,32 @@ +//! Real RFC-9421 request signing for the deny harness (U2). Wraps +//! `gitlawb_core::http_sig::sign_request` (the same entry point the +//! git-remote-gitlawb helper uses in production) so a test can present a valid +//! signature for any identity, reaching the authenticated-but-wrong-owner path +//! that an injected-DID shortcut cannot. + +use gitlawb_core::http_sig::sign_request; +use gitlawb_core::identity::Keypair; + +/// Build a reqwest request carrying a valid RFC-9421 signature for `keypair`. +/// +/// `path` is both appended to `base_url` to form the request URL and signed as +/// the `@path` component, so the value the server reconstructs and the value +/// that was signed are guaranteed identical. `body` is signed via its +/// content-digest and sent as the request body. +pub fn signed_request( + client: &reqwest::Client, + method: reqwest::Method, + base_url: &str, + path: &str, + body: Vec, + keypair: &Keypair, +) -> reqwest::RequestBuilder { + let sig = sign_request(keypair, method.as_str(), path, &body); + let url = format!("{base_url}{path}"); + client + .request(method, url) + .header("content-digest", sig.content_digest) + .header("signature-input", sig.signature_input) + .header("signature", sig.signature) + .body(body) +} From 6ca940bd7fd99cc4b51ee7ab58b6676004bd40f2 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 16:26:19 -0500 Subject: [PATCH 03/41] test(node): INV-1 owner-gate deny case over the real stack (U6) A validly signed non-owner PUT /visibility is rejected 403 by require_owner (no x-ucan, so require_ucan_chain passes through to the gate); the owner's signed PUT reaches the handler (reachability proof, guards against a 404/415 masquerading as a pass). Adds seed_repo/withhold_path seeding helpers to the test harness. Mutation-verified load-bearing: with require_owner forced Ok the non-owner PUT returns 201 and the INV-8 assertion flips the test RED. --- crates/gitlawb-node/src/test_harness.rs | 52 +++++++++++++++++++- crates/gitlawb-node/tests/deny_harness.rs | 58 +++++++++++++++++++++++ 2 files changed, 109 insertions(+), 1 deletion(-) diff --git a/crates/gitlawb-node/src/test_harness.rs b/crates/gitlawb-node/src/test_harness.rs index 10228613..99d439ce 100644 --- a/crates/gitlawb-node/src/test_harness.rs +++ b/crates/gitlawb-node/src/test_harness.rs @@ -16,14 +16,17 @@ use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Arc; use std::time::Duration; +use chrono::Utc; use clap::Parser; use sqlx::PgPool; use tokio::net::TcpListener; use tokio::sync::watch; +use uuid::Uuid; use gitlawb_core::identity::Keypair; use crate::config::Config; +use crate::db::{Db, RepoRecord, VisibilityMode}; use crate::rate_limit::{RateLimiter, TrustedProxy}; use crate::state::AppState; @@ -36,6 +39,10 @@ pub struct TestNode { pub node_did: String, shutdown_tx: watch::Sender, repos_dir: PathBuf, + /// Seeding handle over the same pool the node serves from. Kept private so + /// the integration crate seeds through the methods below rather than + /// naming `Db`/`RepoRecord` directly. + db: Arc, } impl Drop for TestNode { @@ -110,7 +117,7 @@ pub async fn spawn_node(pool: PgPool) -> TestNode { let repos_dir = unique_repos_dir(); std::fs::create_dir_all(&repos_dir).expect("create temp repos dir"); - let state = build_state(db, pool, repos_dir.clone()); + let state = build_state(db.clone(), pool, repos_dir.clone()); let node_did = state.node_did.to_string(); let shutdown_tx = state.shutdown_tx.clone(); let mut shutdown_rx = shutdown_tx.subscribe(); @@ -138,5 +145,48 @@ pub async fn spawn_node(pool: PgPool) -> TestNode { node_did, shutdown_tx, repos_dir, + db, + } +} + +impl TestNode { + /// Insert a repo owned by `owner_did` and return its repo id. Mirrors + /// `test_support::seed_repo` (the `disk_path` field is unused by the + /// integration path — `RepoStore` computes the on-disk path from + /// `repos_dir`/owner/name, see [`Self::seed_bare_repo`]). + pub async fn seed_repo(&self, owner_did: &str, name: &str, is_public: bool) -> String { + let now = Utc::now(); + let record = RepoRecord { + id: Uuid::new_v4().to_string(), + name: name.to_string(), + owner_did: owner_did.to_string(), + description: None, + is_public, + default_branch: "main".to_string(), + created_at: now, + updated_at: now, + disk_path: format!("/tmp/{name}"), + forked_from: None, + machine_id: None, + }; + self.db.create_repo(&record).await.expect("seed repo"); + record.id + } + + /// Add a path-scoped visibility rule restricting `path_glob` to + /// `reader_dids` (empty = only the owner). Mode B keeps object SHAs intact + /// and withholds blob content, which is what the read/replication gates + /// enforce. + pub async fn withhold_path( + &self, + repo_id: &str, + path_glob: &str, + reader_dids: &[String], + created_by: &str, + ) { + self.db + .set_visibility_rule(repo_id, path_glob, VisibilityMode::B, reader_dids, created_by) + .await + .expect("set visibility rule"); } } diff --git a/crates/gitlawb-node/tests/deny_harness.rs b/crates/gitlawb-node/tests/deny_harness.rs index ad4edeb9..fe47e37f 100644 --- a/crates/gitlawb-node/tests/deny_harness.rs +++ b/crates/gitlawb-node/tests/deny_harness.rs @@ -99,3 +99,61 @@ async fn unsigned_receive_pack_is_denied(pool: sqlx::PgPool) { // enforces the 4xx-and-not-empty-200 INV-8 shape. assert_denied(resp, 401, &[]).await; } + +// ── U6: INV-1 — a validly signed NON-owner mutation is owner-gated (403) ────── + +/// The case the in-crate `oneshot` suite cannot reach over the full stack: a +/// fully valid RFC-9421 signature from a non-owner DID hits a mutation and is +/// rejected by `require_owner` with 403, not merely "not authenticated". The +/// non-owner sends no `x-ucan` header, so `require_ucan_chain` passes through +/// and the request reaches the owner gate. +#[sqlx::test] +async fn wrong_owner_visibility_put_is_forbidden(pool: sqlx::PgPool) { + let node = spawn_node(pool).await; + let client = reqwest::Client::new(); + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let stranger = Keypair::generate(); + + node.seed_repo(&owner_did, "harness-repo", true).await; + + let path = format!("/api/v1/repos/{owner_did}/harness-repo/visibility"); + let body = br#"{"path_glob":"/","reader_dids":[]}"#.to_vec(); + + // Non-owner: valid signature, wrong identity -> 403 from the owner gate. + let resp = signed_request( + &client, + reqwest::Method::PUT, + &node.base_url, + &path, + body.clone(), + &stranger, + ) + .header("content-type", "application/json") + .send() + .await + .expect("request sends"); + assert_denied(resp, 403, &[]).await; + + // Reachability proof: the same request signed by the OWNER is not 403 (it + // reaches the handler). Without this, a 403 produced by an earlier layer or + // a mis-seeded repo would masquerade as a passing INV-1 case. + let resp = signed_request( + &client, + reqwest::Method::PUT, + &node.base_url, + &path, + body, + &owner, + ) + .header("content-type", "application/json") + .send() + .await + .expect("request sends"); + assert!( + resp.status().is_success(), + "owner's signed visibility PUT must reach the handler, got {}", + resp.status() + ); +} From 54ceb0dc01a2c396835c24d206b776fadc47b29f Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 16:35:29 -0500 Subject: [PATCH 04/41] test(node): INV-2 read + content-addressed withhold deny cases (U5b, U7) Adds seed_bare_repo (shells git to build a real bare repo at the served path, sha1 or sha256 object format) and two INV-2 deny cases over the real stack: - U7: a public repo with a /secret/** withhold rule denies an anonymous blob read of the withheld path (404) with no content/OID leak, while the sibling public path is served (path-scoped, not blanket). - U5b: the same withhold denies an anonymous /ipfs/{cid} read of the withheld blob's content-addressed id (404, no leak), while the public blob's CID is served. Completes U5 (INV-8) alongside U5a. Both mutation-verified load-bearing: forcing visibility_check to allow leaks the secret at 200 and the INV-8 assertion flips each test RED. --- crates/gitlawb-node/Cargo.toml | 1 + crates/gitlawb-node/src/test_harness.rs | 70 +++++++++++++ crates/gitlawb-node/tests/deny_harness.rs | 121 ++++++++++++++++++++++ 3 files changed, 192 insertions(+) diff --git a/crates/gitlawb-node/Cargo.toml b/crates/gitlawb-node/Cargo.toml index c7618a07..1dc0fb7c 100644 --- a/crates/gitlawb-node/Cargo.toml +++ b/crates/gitlawb-node/Cargo.toml @@ -100,3 +100,4 @@ tempfile = "3" # HTTP client that drives deny paths over the socket. sqlx = { version = "0.8", features = ["postgres", "runtime-tokio-rustls", "macros", "migrate"] } reqwest = { workspace = true } +hex = "0.4" diff --git a/crates/gitlawb-node/src/test_harness.rs b/crates/gitlawb-node/src/test_harness.rs index 99d439ce..550279cf 100644 --- a/crates/gitlawb-node/src/test_harness.rs +++ b/crates/gitlawb-node/src/test_harness.rs @@ -189,4 +189,74 @@ impl TestNode { .await .expect("set visibility rule"); } + + /// Create a real bare git repository on disk at the exact path + /// `RepoStore::acquire` reads (`//.git`), with + /// one commit on `main` containing `files` (`(path, contents)`). Returns the + /// blob OID of each path so callers can assert a withheld OID never appears + /// in served bytes (U8). Shells out to `git`, mirroring + /// `test_support`'s served-content seam. + /// `object_format` is `"sha1"` for the git smart-HTTP surfaces and + /// `"sha256"` for the content-addressed `/ipfs/{cid}` surface (whose CIDs + /// are the sha2-256 object ids). + pub fn seed_bare_repo( + &self, + owner_did: &str, + name: &str, + files: &[(&str, &str)], + object_format: &str, + ) -> std::collections::HashMap { + let run = |args: &[&str], cwd: &std::path::Path| { + let out = std::process::Command::new("git") + .args(args) + .current_dir(cwd) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + + // Build a source work tree, then bare-clone it into the served path. + let src = self.repos_dir.join(format!("src-{name}")); + std::fs::create_dir_all(&src).expect("create src dir"); + let fmt_arg = format!("--object-format={object_format}"); + run(&["init", "-q", "-b", "main", &fmt_arg], &src); + run(&["config", "user.email", "t@t"], &src); + run(&["config", "user.name", "t"], &src); + for (path, contents) in files { + let full = src.join(path); + if let Some(parent) = full.parent() { + std::fs::create_dir_all(parent).expect("create file parent dir"); + } + std::fs::write(&full, contents).expect("write seed file"); + run(&["add", path], &src); + } + run(&["commit", "-q", "-m", "seed"], &src); + + let mut oids = std::collections::HashMap::new(); + for (path, _) in files { + let oid = run(&["rev-parse", &format!("HEAD:{path}")], &src); + oids.insert((*path).to_string(), oid); + } + + let slug = owner_did.replace([':', '/'], "_"); + let bare = self.repos_dir.join(&slug).join(format!("{name}.git")); + std::fs::create_dir_all(bare.parent().unwrap()).expect("create bare parent"); + run( + &[ + "clone", + "--bare", + "-q", + src.to_str().unwrap(), + bare.to_str().unwrap(), + ], + &self.repos_dir, + ); + + oids + } } diff --git a/crates/gitlawb-node/tests/deny_harness.rs b/crates/gitlawb-node/tests/deny_harness.rs index fe47e37f..9acf0077 100644 --- a/crates/gitlawb-node/tests/deny_harness.rs +++ b/crates/gitlawb-node/tests/deny_harness.rs @@ -11,9 +11,18 @@ mod support; use support::assert::assert_denied; use support::signing::signed_request; +use gitlawb_core::cid::Cid; use gitlawb_core::identity::Keypair; use gitlawb_node::test_harness::spawn_node; +/// Build the `/ipfs/{cid}` CID for a 64-hex sha2-256 git object id, matching the +/// node's own `Cid::from_sha256_bytes` (the value `get_by_cid` decodes back). +fn cid_for_oid(oid_hex: &str) -> String { + let bytes = hex::decode(oid_hex).expect("hex oid"); + let arr: [u8; 32] = bytes.as_slice().try_into().expect("32-byte sha256 oid"); + Cid::from_sha256_bytes(&arr).to_string() +} + // ── U2: the signing client produces signatures require_signature accepts ───── /// A validly signed receive-pack request clears `require_signature` (it does not @@ -100,6 +109,60 @@ async fn unsigned_receive_pack_is_denied(pool: sqlx::PgPool) { assert_denied(resp, 401, &[]).await; } +// ── U5(b): INV-8/INV-2 — anonymous /ipfs/{cid} of a withheld blob is denied ── + +/// A public repo with a `/secret/**` withhold rule (readers = one allowed DID). +/// An anonymous content-addressed read of the withheld blob's CID is denied +/// (404) and leaks neither the secret bytes nor its OID; the sibling public +/// blob's CID is served anonymously, proving the withhold is blob-scoped. +#[sqlx::test] +async fn anon_ipfs_read_of_withheld_blob_is_denied(pool: sqlx::PgPool) { + let node = spawn_node(pool).await; + let client = reqwest::Client::new(); + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let reader = Keypair::generate(); + + let repo_id = node.seed_repo(&owner_did, "u5b-repo", true).await; + // sha256 object format: the /ipfs CID is the sha2-256 object id. + let oids = node.seed_bare_repo( + &owner_did, + "u5b-repo", + &[ + ("public/a.txt", "public bytes U5b"), + ("secret/b.txt", "TOPSECRET-U5b"), + ], + "sha256", + ); + node.withhold_path(&repo_id, "/secret/**", &[reader.did().to_string()], &owner_did) + .await; + + let secret_oid = oids["secret/b.txt"].clone(); + let secret_cid = cid_for_oid(&secret_oid); + let public_cid = cid_for_oid(&oids["public/a.txt"]); + + // Anonymous read of the withheld blob's CID: denied, no leak of content or OID. + let resp = client + .get(format!("{}/ipfs/{secret_cid}", node.base_url)) + .send() + .await + .expect("request sends"); + assert_denied(resp, 404, &["TOPSECRET-U5b", &secret_oid, &secret_oid[..12]]).await; + + // The sibling public blob's CID is served to anon (withhold is blob-scoped). + let resp = client + .get(format!("{}/ipfs/{public_cid}", node.base_url)) + .send() + .await + .expect("request sends"); + assert_eq!(resp.status().as_u16(), 200, "public blob CID must be served to anon"); + assert!( + resp.text().await.unwrap().contains("public bytes U5b"), + "the public blob content is returned" + ); +} + // ── U6: INV-1 — a validly signed NON-owner mutation is owner-gated (403) ────── /// The case the in-crate `oneshot` suite cannot reach over the full stack: a @@ -157,3 +220,61 @@ async fn wrong_owner_visibility_put_is_forbidden(pool: sqlx::PgPool) { resp.status() ); } + +// ── U7: INV-2 — a read over a withheld path is denied and leaks nothing ─────── + +/// A public repo with a path-scoped withhold rule on `/secret/**`. An anonymous +/// blob read of the withheld path is denied (404 RepoNotFound) and the body +/// carries neither the secret content nor its blob OID; a read of a sibling +/// public path succeeds, proving the gate is path-scoped, not a blanket refusal. +#[sqlx::test] +async fn withheld_path_blob_read_is_denied(pool: sqlx::PgPool) { + let node = spawn_node(pool).await; + let client = reqwest::Client::new(); + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + + let repo_id = node.seed_repo(&owner_did, "u7-repo", true).await; + let oids = node.seed_bare_repo( + &owner_did, + "u7-repo", + &[ + ("public.txt", "hello public"), + ("secret/data.txt", "TOPSECRET-CONTENT-U7"), + ], + "sha1", + ); + node.withhold_path(&repo_id, "/secret/**", &[], &owner_did) + .await; + + let secret_oid = oids["secret/data.txt"].clone(); + let short_oid = &secret_oid[..12]; + + // Withheld path: denied, and neither the content nor the OID (full or short) + // may appear in the denial body. + let resp = client + .get(format!( + "{}/api/v1/repos/{owner_did}/u7-repo/blob/secret/data.txt", + node.base_url + )) + .send() + .await + .expect("request sends"); + assert_denied(resp, 404, &["TOPSECRET-CONTENT-U7", &secret_oid, short_oid]).await; + + // Sibling public path: served, proving the withhold is path-scoped. + let resp = client + .get(format!( + "{}/api/v1/repos/{owner_did}/u7-repo/blob/public.txt", + node.base_url + )) + .send() + .await + .expect("request sends"); + assert_eq!(resp.status().as_u16(), 200, "public sibling path must be served"); + assert!( + resp.text().await.unwrap().contains("hello public"), + "the public blob content is returned" + ); +} From c3b25fb9910844a62674a3ea2936c550eeb8184f Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 17:01:03 -0500 Subject: [PATCH 05/41] test(node): INV-2 replication-path deny case over git-upload-pack (U8) Drives the git-upload-pack POST directly (v0 stateless-RPC: want HEAD, flush, done) via a bounded reqwest client rather than a vanilla `git clone` (which negotiates protocol v2 and deadlocks against the node's v0 server, and would otherwise wedge the suite). The served pack is indexed with git index-pack and its objects listed with verify-pack -v: a packfile-aware assertion, since a raw byte scan cannot see an OID inside the zlib-compressed stream. A public repo with a /secret/** withhold rule must serve a pack that omits the withheld blob's object while keeping the sibling public blob. Mutation-verified load-bearing: forcing visibility_check to allow puts the withheld blob back in the pack and flips the test RED. Completes the harness (8 units, 11 integration tests). Prod build (no feature) and the 488-test node suite stay green. --- crates/gitlawb-node/src/test_harness.rs | 2 + crates/gitlawb-node/tests/deny_harness.rs | 98 +++++++++++++++++++++++ 2 files changed, 100 insertions(+) diff --git a/crates/gitlawb-node/src/test_harness.rs b/crates/gitlawb-node/src/test_harness.rs index 550279cf..f1db50fa 100644 --- a/crates/gitlawb-node/src/test_harness.rs +++ b/crates/gitlawb-node/src/test_harness.rs @@ -242,6 +242,8 @@ impl TestNode { let oid = run(&["rev-parse", &format!("HEAD:{path}")], &src); oids.insert((*path).to_string(), oid); } + // The HEAD commit oid, used as the `want` when driving upload-pack. + oids.insert("HEAD".to_string(), run(&["rev-parse", "HEAD"], &src)); let slug = owner_did.replace([':', '/'], "_"); let bare = self.repos_dir.join(&slug).join(format!("{name}.git")); diff --git a/crates/gitlawb-node/tests/deny_harness.rs b/crates/gitlawb-node/tests/deny_harness.rs index 9acf0077..3652a793 100644 --- a/crates/gitlawb-node/tests/deny_harness.rs +++ b/crates/gitlawb-node/tests/deny_harness.rs @@ -8,6 +8,8 @@ mod support; +use std::process::Command; + use support::assert::assert_denied; use support::signing::signed_request; @@ -278,3 +280,99 @@ async fn withheld_path_blob_read_is_denied(pool: sqlx::PgPool) { "the public blob content is returned" ); } + +// ── U8: INV-2 — an anonymous clone/fetch excludes withheld subtree blobs ────── + +/// The replication/clone surface, the one `oneshot` cannot serve: a public repo +/// with a `/secret/**` withhold rule. A real anonymous `git clone` over +/// git-upload-pack must yield a packfile that omits the withheld blob's object +/// while keeping the sibling public blob. The assertion is packfile-aware: git +/// parsed the served pack, and `cat-file -e` checks object presence in it. +#[sqlx::test] +async fn anon_clone_excludes_withheld_subtree_blobs(pool: sqlx::PgPool) { + let node = spawn_node(pool).await; + + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + + let repo_id = node.seed_repo(&owner_did, "u8-repo", true).await; + let oids = node.seed_bare_repo( + &owner_did, + "u8-repo", + &[ + ("public/a.txt", "public bytes U8"), + ("secret/b.txt", "TOPSECRET-U8"), + ], + "sha1", + ); + node.withhold_path(&repo_id, "/secret/**", &[], &owner_did) + .await; + + let secret_oid = oids["secret/b.txt"].clone(); + let public_oid = oids["public/a.txt"].clone(); + let head = oids["HEAD"].clone(); + + // Drive git-upload-pack directly (v0 stateless-RPC): a want for HEAD, a flush, + // then done. No side-band capability, so the response is a raw packfile after + // the NAK pkt-line. Vanilla `git clone` negotiates protocol v2 and deadlocks + // against the node's v0 server; the POST is the real replication surface and, + // via a bounded reqwest timeout, cannot wedge the suite. + let pkt = |s: &str| format!("{:04x}{}", s.len() + 4, s).into_bytes(); + let mut req_body: Vec = Vec::new(); + req_body.extend(pkt(&format!("want {head}\n"))); + req_body.extend_from_slice(b"0000"); // flush after wants + req_body.extend(pkt("done\n")); + + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(30)) + .build() + .unwrap(); + let resp = client + .post(format!("{}/{owner_did}/u8-repo/git-upload-pack", node.base_url)) + .header("content-type", "application/x-git-upload-pack-request") + .body(req_body) + .send() + .await + .expect("upload-pack POST sends"); + assert_eq!(resp.status().as_u16(), 200, "anon upload-pack POST must serve a pack"); + let bytes = resp.bytes().await.expect("read pack response"); + + // The packfile starts at the "PACK" magic (after the NAK pkt-line). + let pack_start = bytes + .windows(4) + .position(|w| w == b"PACK") + .expect("response must contain a packfile"); + let pack = &bytes[pack_start..]; + + // Index the served pack and list its objects (packfile-aware: a raw byte scan + // could not see an OID inside the zlib-compressed stream). + let dir = std::env::temp_dir().join(format!("gl-u8-pack-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + let pack_path = dir.join("deny.pack"); + std::fs::write(&pack_path, pack).unwrap(); + let idx = Command::new("git") + .args(["index-pack", pack_path.to_str().unwrap()]) + .output() + .expect("git index-pack runs"); + assert!( + idx.status.success(), + "the served pack must index cleanly: {}", + String::from_utf8_lossy(&idx.stderr) + ); + let verify = Command::new("git") + .args(["verify-pack", "-v", pack_path.with_extension("idx").to_str().unwrap()]) + .output() + .expect("git verify-pack runs"); + let listing = String::from_utf8_lossy(&verify.stdout).to_string(); + let _ = std::fs::remove_dir_all(&dir); + + assert!( + !listing.contains(&secret_oid), + "withheld blob {secret_oid} must be absent from the served pack; listing:\n{listing}" + ); + assert!( + listing.contains(&public_oid), + "public blob {public_oid} must be present (withhold is subtree-scoped); listing:\n{listing}" + ); +} From 216982a4c44e2340f9923ede4a04811e9b9da37a Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 17:17:36 -0500 Subject: [PATCH 06/41] ci(node): run the real-node deny harness on every PR cargo test --workspace skips the harness because it lives behind the test-harness feature (kept off the production binary). Add an explicit step that runs it with the feature and the same Postgres service, so the INV-1/ INV-2/INV-8 trust-boundary regression cases execute on every PR instead of only when run by hand. --- .github/workflows/pr-checks.yml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/.github/workflows/pr-checks.yml b/.github/workflows/pr-checks.yml index d6d9497a..7e8811cf 100644 --- a/.github/workflows/pr-checks.yml +++ b/.github/workflows/pr-checks.yml @@ -92,6 +92,16 @@ jobs: DATABASE_URL: postgres://postgres:postgres@localhost:5432/gitlawb_test run: cargo test --workspace + # The real-node deny harness is behind the `test-harness` feature (so its + # spawn surface never compiles into the production binary), which means + # `--workspace` above skips it. Run it explicitly so the trust-boundary + # regression cases (INV-1/INV-2/INV-8) execute on every PR. Uses the same + # Postgres service; `#[sqlx::test]` provisions an isolated DB per test. + - name: cargo test (real-node deny harness) + env: + DATABASE_URL: postgres://postgres:postgres@localhost:5432/gitlawb_test + run: cargo test -p gitlawb-node --features test-harness --test deny_harness + build-release: name: build --release runs-on: ubuntu-latest From 746b76fba49e341ec6bc8607b45e6e3973dd5327 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 17:33:39 -0500 Subject: [PATCH 07/41] test(node): cover the deny-assertion guard branches (U4) Add unit tests for the two remaining check_denied branches: a non-4xx expected status is rejected as a test bug, and an empty withheld token is skipped rather than matching every body. Closes the last unexecuted branches in the deny assertion. --- crates/gitlawb-node/tests/support/assert.rs | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/crates/gitlawb-node/tests/support/assert.rs b/crates/gitlawb-node/tests/support/assert.rs index 50dca250..fb64597b 100644 --- a/crates/gitlawb-node/tests/support/assert.rs +++ b/crates/gitlawb-node/tests/support/assert.rs @@ -92,4 +92,21 @@ mod tests { assert!(r.is_err()); assert!(r.unwrap_err().contains("expected denial status 403")); } + + #[test] + fn non_4xx_expected_is_a_test_bug() { + // Guards the harness itself: expecting a non-denial status is a test + // authoring error, not a node behavior, and must be rejected. + let r = check_denied(403, "denied", 200, &[]); + assert!(r.is_err()); + assert!(r.unwrap_err().contains("not a 4xx")); + } + + #[test] + fn empty_withheld_token_never_false_positives() { + // An empty token must be skipped, never treated as "contained in every + // body" — otherwise every denial would spuriously fail as a leak. + let r = check_denied(403, "any body", 403, &[""]); + assert!(r.is_ok(), "{r:?}"); + } } From 2cfbcbafe535f56e22be60b0e4bc1d6449ec5260 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 17:44:48 -0500 Subject: [PATCH 08/41] test(node): full-stack owner-gate deny cases for 5 high-value mutations Fan-out of U6 to the security-sensitive owner-gated mutations that had only the source-level authz-table guard and no runtime deny test: protect_branch, unprotect_branch, create_webhook, delete_webhook, remove_visibility. Each rejects a validly-signed non-owner with 403 and lets the owner reach the handler (not 403). Mutation-verified load-bearing on their shared root gate: did_matches forced true opens all five (non-owner protect_branch returns 201) and the test flips RED. Replica register/unregister were intentionally excluded: they are signer-self (you register your own node), not owner-gated, so there is no owner-deny to assert. --- crates/gitlawb-node/tests/deny_harness.rs | 66 +++++++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/crates/gitlawb-node/tests/deny_harness.rs b/crates/gitlawb-node/tests/deny_harness.rs index 3652a793..40865757 100644 --- a/crates/gitlawb-node/tests/deny_harness.rs +++ b/crates/gitlawb-node/tests/deny_harness.rs @@ -376,3 +376,69 @@ async fn anon_clone_excludes_withheld_subtree_blobs(pool: sqlx::PgPool) { "public blob {public_oid} must be present (withhold is subtree-scoped); listing:\n{listing}" ); } + +// ── Additional INV-1 owner-gates over the real stack (fan-out of U6) ────────── + +/// The remaining high-blast-radius owner-gated mutations that previously had +/// only the source-level authz-table guard and no runtime deny test: branch +/// protection, webhook create/delete, and visibility removal. Each rejects a +/// validly-signed non-owner with 403, and the owner reaches the handler (not +/// 403), proving the 403 is the owner gate. All share `did_matches` as the root +/// gate (see the mutation check in the harness commit). +#[sqlx::test] +async fn additional_owner_gates_reject_non_owner(pool: sqlx::PgPool) { + let node = spawn_node(pool).await; + let client = reqwest::Client::new(); + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let stranger = Keypair::generate(); + + node.seed_repo(&owner_did, "gated-repo", true).await; + let base = format!("/api/v1/repos/{owner_did}/gated-repo"); + + // (method, path, body, needs-json-content-type) + let cases: Vec<(reqwest::Method, String, Vec, bool)> = vec![ + (reqwest::Method::POST, format!("{base}/branches/main/protect"), Vec::new(), false), + (reqwest::Method::DELETE, format!("{base}/branches/main/protect"), Vec::new(), false), + ( + reqwest::Method::POST, + format!("{base}/hooks"), + br#"{"url":"https://example.com/hook","events":["*"]}"#.to_vec(), + true, + ), + (reqwest::Method::DELETE, format!("{base}/hooks/deadbeef"), Vec::new(), false), + ( + reqwest::Method::DELETE, + format!("{base}/visibility"), + br#"{"path_glob":"/"}"#.to_vec(), + true, + ), + ]; + + let send = |m: reqwest::Method, path: String, body: Vec, json: bool, kp: &Keypair| { + let mut rb = signed_request(&client, m, &node.base_url, &path, body, kp); + if json { + rb = rb.header("content-type", "application/json"); + } + rb.send() + }; + + for (method, path, body, json) in cases { + // Non-owner: valid signature, wrong identity -> 403 owner gate, no leak. + let resp = send(method.clone(), path.clone(), body.clone(), json, &stranger) + .await + .unwrap_or_else(|e| panic!("{method} {path} sends: {e}")); + assert_denied(resp, 403, &[]).await; + + // Owner reaches the handler: NOT 403 (proves the 403 above was the gate, + // not an earlier layer or a 415/404 masquerade). + let resp = send(method.clone(), path.clone(), body, json, &owner) + .await + .unwrap_or_else(|e| panic!("{method} {path} owner sends: {e}")); + assert_ne!( + resp.status().as_u16(), + 403, + "owner must reach {method} {path} (got 403)" + ); + } +} From 532627f4b9eed66ddad74673a56c70b187e8e9da Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 17:48:48 -0500 Subject: [PATCH 09/41] style(node): cargo fmt the deny-harness sources --- crates/gitlawb-node/src/lib.rs | 4 +- crates/gitlawb-node/src/test_harness.rs | 14 +++-- crates/gitlawb-node/tests/deny_harness.rs | 66 +++++++++++++++++---- crates/gitlawb-node/tests/support/assert.rs | 7 ++- 4 files changed, 71 insertions(+), 20 deletions(-) diff --git a/crates/gitlawb-node/src/lib.rs b/crates/gitlawb-node/src/lib.rs index f173b053..f2a6cb36 100644 --- a/crates/gitlawb-node/src/lib.rs +++ b/crates/gitlawb-node/src/lib.rs @@ -24,11 +24,11 @@ mod rate_limit; pub mod server; pub mod state; mod sync; -#[cfg(test)] -mod test_support; #[cfg(feature = "test-harness")] #[doc(hidden)] pub mod test_harness; +#[cfg(test)] +mod test_support; mod visibility; mod webhooks; diff --git a/crates/gitlawb-node/src/test_harness.rs b/crates/gitlawb-node/src/test_harness.rs index f1db50fa..7dfe6c81 100644 --- a/crates/gitlawb-node/src/test_harness.rs +++ b/crates/gitlawb-node/src/test_harness.rs @@ -61,11 +61,7 @@ impl Drop for TestNode { fn unique_repos_dir() -> PathBuf { static COUNTER: AtomicU64 = AtomicU64::new(0); let n = COUNTER.fetch_add(1, Ordering::Relaxed); - std::env::temp_dir().join(format!( - "gitlawb-deny-harness-{}-{}", - std::process::id(), - n - )) + std::env::temp_dir().join(format!("gitlawb-deny-harness-{}-{}", std::process::id(), n)) } /// Build an [`AppState`] over the given migrated pool, mirroring @@ -185,7 +181,13 @@ impl TestNode { created_by: &str, ) { self.db - .set_visibility_rule(repo_id, path_glob, VisibilityMode::B, reader_dids, created_by) + .set_visibility_rule( + repo_id, + path_glob, + VisibilityMode::B, + reader_dids, + created_by, + ) .await .expect("set visibility rule"); } diff --git a/crates/gitlawb-node/tests/deny_harness.rs b/crates/gitlawb-node/tests/deny_harness.rs index 40865757..491e1ae8 100644 --- a/crates/gitlawb-node/tests/deny_harness.rs +++ b/crates/gitlawb-node/tests/deny_harness.rs @@ -137,8 +137,13 @@ async fn anon_ipfs_read_of_withheld_blob_is_denied(pool: sqlx::PgPool) { ], "sha256", ); - node.withhold_path(&repo_id, "/secret/**", &[reader.did().to_string()], &owner_did) - .await; + node.withhold_path( + &repo_id, + "/secret/**", + &[reader.did().to_string()], + &owner_did, + ) + .await; let secret_oid = oids["secret/b.txt"].clone(); let secret_cid = cid_for_oid(&secret_oid); @@ -150,7 +155,12 @@ async fn anon_ipfs_read_of_withheld_blob_is_denied(pool: sqlx::PgPool) { .send() .await .expect("request sends"); - assert_denied(resp, 404, &["TOPSECRET-U5b", &secret_oid, &secret_oid[..12]]).await; + assert_denied( + resp, + 404, + &["TOPSECRET-U5b", &secret_oid, &secret_oid[..12]], + ) + .await; // The sibling public blob's CID is served to anon (withhold is blob-scoped). let resp = client @@ -158,7 +168,11 @@ async fn anon_ipfs_read_of_withheld_blob_is_denied(pool: sqlx::PgPool) { .send() .await .expect("request sends"); - assert_eq!(resp.status().as_u16(), 200, "public blob CID must be served to anon"); + assert_eq!( + resp.status().as_u16(), + 200, + "public blob CID must be served to anon" + ); assert!( resp.text().await.unwrap().contains("public bytes U5b"), "the public blob content is returned" @@ -274,7 +288,11 @@ async fn withheld_path_blob_read_is_denied(pool: sqlx::PgPool) { .send() .await .expect("request sends"); - assert_eq!(resp.status().as_u16(), 200, "public sibling path must be served"); + assert_eq!( + resp.status().as_u16(), + 200, + "public sibling path must be served" + ); assert!( resp.text().await.unwrap().contains("hello public"), "the public blob content is returned" @@ -328,13 +346,20 @@ async fn anon_clone_excludes_withheld_subtree_blobs(pool: sqlx::PgPool) { .build() .unwrap(); let resp = client - .post(format!("{}/{owner_did}/u8-repo/git-upload-pack", node.base_url)) + .post(format!( + "{}/{owner_did}/u8-repo/git-upload-pack", + node.base_url + )) .header("content-type", "application/x-git-upload-pack-request") .body(req_body) .send() .await .expect("upload-pack POST sends"); - assert_eq!(resp.status().as_u16(), 200, "anon upload-pack POST must serve a pack"); + assert_eq!( + resp.status().as_u16(), + 200, + "anon upload-pack POST must serve a pack" + ); let bytes = resp.bytes().await.expect("read pack response"); // The packfile starts at the "PACK" magic (after the NAK pkt-line). @@ -361,7 +386,11 @@ async fn anon_clone_excludes_withheld_subtree_blobs(pool: sqlx::PgPool) { String::from_utf8_lossy(&idx.stderr) ); let verify = Command::new("git") - .args(["verify-pack", "-v", pack_path.with_extension("idx").to_str().unwrap()]) + .args([ + "verify-pack", + "-v", + pack_path.with_extension("idx").to_str().unwrap(), + ]) .output() .expect("git verify-pack runs"); let listing = String::from_utf8_lossy(&verify.stdout).to_string(); @@ -398,15 +427,30 @@ async fn additional_owner_gates_reject_non_owner(pool: sqlx::PgPool) { // (method, path, body, needs-json-content-type) let cases: Vec<(reqwest::Method, String, Vec, bool)> = vec![ - (reqwest::Method::POST, format!("{base}/branches/main/protect"), Vec::new(), false), - (reqwest::Method::DELETE, format!("{base}/branches/main/protect"), Vec::new(), false), + ( + reqwest::Method::POST, + format!("{base}/branches/main/protect"), + Vec::new(), + false, + ), + ( + reqwest::Method::DELETE, + format!("{base}/branches/main/protect"), + Vec::new(), + false, + ), ( reqwest::Method::POST, format!("{base}/hooks"), br#"{"url":"https://example.com/hook","events":["*"]}"#.to_vec(), true, ), - (reqwest::Method::DELETE, format!("{base}/hooks/deadbeef"), Vec::new(), false), + ( + reqwest::Method::DELETE, + format!("{base}/hooks/deadbeef"), + Vec::new(), + false, + ), ( reqwest::Method::DELETE, format!("{base}/visibility"), diff --git a/crates/gitlawb-node/tests/support/assert.rs b/crates/gitlawb-node/tests/support/assert.rs index fb64597b..ccdfdc85 100644 --- a/crates/gitlawb-node/tests/support/assert.rs +++ b/crates/gitlawb-node/tests/support/assert.rs @@ -64,7 +64,12 @@ mod tests { #[test] fn clean_403_with_no_leak_passes() { - let r = check_denied(403, r#"{"error":"only the repo owner can do this"}"#, 403, &["a1b2c3"]); + let r = check_denied( + 403, + r#"{"error":"only the repo owner can do this"}"#, + 403, + &["a1b2c3"], + ); assert!(r.is_ok(), "{r:?}"); } From 30672cf679ba2ddd1bce04c39db100d0dd8642b3 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 12 Jul 2026 22:27:07 -0500 Subject: [PATCH 10/41] Address PR review feedback (#194) - Create the node identity key atomically with create_new + mode 0600, closing the umask-derived 0644 disclosure window and the exists()->write overwrite race; on AlreadyExists load the winner's key (bounded retry so a loser can't read a half-written PEM) and tighten looser perms on load. - Enforce the configured shutdown grace: bound the axum drain by grace measured from the signal (extracted as drive_serve_with_grace), abandoning in-flight requests once it expires instead of waiting indefinitely. Removes the discarded grace value. - Route the deny-harness reqwest clients through a shared bounded_client (30s timeout) so a wedged node path fails fast instead of hanging to the CI limit. Tests: 0600-on-create, load-tighten, concurrent-start convergence (create race), and grace-race abandon / normal-drain / signal-gated-clock. All RED-then-GREEN by execution. --- crates/gitlawb-node/src/lib.rs | 364 ++++++++++++++++++++-- crates/gitlawb-node/tests/deny_harness.rs | 28 +- 2 files changed, 354 insertions(+), 38 deletions(-) diff --git a/crates/gitlawb-node/src/lib.rs b/crates/gitlawb-node/src/lib.rs index f2a6cb36..88dff5e1 100644 --- a/crates/gitlawb-node/src/lib.rs +++ b/crates/gitlawb-node/src/lib.rs @@ -536,16 +536,22 @@ pub async fn run() -> Result<()> { info!("on-chain PoS disabled (GITLAWB_CONTRACT_NODE_STAKING or GITLAWB_OPERATOR_PRIVATE_KEY unset)"); } - // axum's `with_graceful_shutdown` waits for in-flight requests to - // complete (up to the configured grace) once the future resolves. + // axum's `with_graceful_shutdown` begins draining in-flight requests once + // the shutdown watch flips. That drain is otherwise unbounded, so we bound + // it by `grace`: the closure fires `armed_tx` the instant it observes the + // signal, and `drive_serve_with_grace` abandons the drain if it has not + // finished `grace` after that moment. The clock starts at the signal, not at + // server start, so total uptime is never bounded. let shutdown_signal_for_axum = state.subscribe_shutdown(); let grace = std::time::Duration::from_secs(config.shutdown_grace_secs); info!(grace_secs = config.shutdown_grace_secs, "axum server ready"); + let (armed_tx, armed_rx) = tokio::sync::oneshot::channel::<()>(); + // `into_make_service_with_connect_info` exposes the socket peer address as // `ConnectInfo` so the push limiter can key on the real client // when no trusted proxy header applies (see `rate_limit::client_key`). - let serve_result = axum::serve( + let serve = axum::serve( listener, router.into_make_service_with_connect_info::(), ) @@ -559,21 +565,58 @@ pub async fn run() -> Result<()> { break; } } - }) - .await; + // Start the grace clock at the signal, right before axum drains. + let _ = armed_tx.send(()); + }); + + // Race the drain against the grace deadline (see `drive_serve_with_grace`): + // the clock starts when the closure above arms `armed_rx`, i.e. at the + // signal, and the drain is abandoned if it outlasts `grace`. + let (serve_result, grace_expired) = drive_serve_with_grace(serve, armed_rx, grace).await; + + if grace_expired { + warn!( + grace_secs = config.shutdown_grace_secs, + "shutdown grace expired; abandoning in-flight requests" + ); + } // Server has stopped accepting new connections and drained in-flight - // requests. Tear the rest of the system down. + // requests (or the grace deadline fired). Tear the rest of the system down. info!("HTTP server stopped, beginning process shutdown"); if let Some(h) = metrics_handle { h.abort(); } - let _ = grace; // recorded for operators in the log above; not enforced serve_result?; info!("clean exit"); Ok(()) } +/// Drive the HTTP `serve` future to completion, bounding the post-signal drain +/// by `grace`. `armed` resolves the instant the shutdown signal fires (the serve +/// future's graceful-shutdown closure sends on it), so the grace clock starts at +/// the signal, not at server start — total uptime is never bounded. Returns the +/// serve result plus whether the deadline fired (which abandons in-flight +/// requests so teardown can proceed). +async fn drive_serve_with_grace( + serve: F, + armed: tokio::sync::oneshot::Receiver<()>, + grace: std::time::Duration, +) -> (std::io::Result<()>, bool) +where + F: std::future::IntoFuture>, +{ + let serve = serve.into_future(); + tokio::select! { + result = serve => (result, false), + _ = async { + // Park until the signal arms the clock, then bound the drain. + let _ = armed.await; + tokio::time::sleep(grace).await; + } => (Ok(()), true), + } +} + fn spawn_shutdown_signal(tx: watch::Sender) { tokio::spawn(async move { #[cfg(unix)] @@ -1004,34 +1047,117 @@ async fn ping_peer_health(client: &reqwest::Client, http_url: &str) -> bool { fn load_or_create_keypair(config: &Config) -> Result { let key_path = config.resolved_key_path(); - if key_path.exists() { - let pem = std::fs::read_to_string(&key_path) - .with_context(|| format!("failed to read key from {}", key_path.display()))?; + // Load an already-provisioned key. On Unix, defensively tighten looser + // permissions to 0600 (do NOT reject a loose key — that would break + // existing deployments; just narrow them). + fn load_existing(path: &std::path::Path) -> Result { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + if let Ok(meta) = std::fs::metadata(path) { + if meta.permissions().mode() & 0o777 != 0o600 { + let _ = std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)); + } + } + } + let pem = std::fs::read_to_string(path) + .with_context(|| format!("failed to read key from {}", path.display()))?; let kp = Keypair::from_pem(&pem).map_err(|e| anyhow::anyhow!("invalid PEM key: {e}"))?; - info!(path = %key_path.display(), "loaded existing identity"); + info!(path = %path.display(), "loaded existing identity"); Ok(kp) - } else { - let kp = Keypair::generate(); - let pem = kp - .to_pem() - .map_err(|e| anyhow::anyhow!("failed to serialize key: {e}"))?; + } - if let Some(parent) = key_path.parent() { - std::fs::create_dir_all(parent)?; + // Load a key another concurrent start may still be writing. `create_new` + // makes the inode appear before its PEM is flushed, so a race loser — or a + // fast-path `exists()` that caught the winner mid-write — can momentarily + // read an empty/partial file. Retry briefly until it parses rather than + // crashing the boot. Bounded (~100ms); an already-provisioned key parses on + // the first attempt with no sleep. + fn load_racing(path: &std::path::Path) -> Result { + let mut last_err = None; + for _ in 0..50 { + match load_existing(path) { + Ok(kp) => return Ok(kp), + Err(e) => { + last_err = Some(e); + std::thread::sleep(std::time::Duration::from_millis(2)); + } + } } + Err(last_err.unwrap_or_else(|| { + anyhow::anyhow!("key at {} unreadable after create race", path.display()) + })) + } - #[cfg(unix)] + // Fast path for the common already-provisioned case (still race-safe: a + // concurrently-writing winner is handled by the retry inside `load_racing`). + if key_path.exists() { + return load_racing(&key_path); + } + + let kp = Keypair::generate(); + let pem = kp + .to_pem() + .map_err(|e| anyhow::anyhow!("failed to serialize key: {e}"))?; + + if let Some(parent) = key_path.parent() { + std::fs::create_dir_all(parent)?; + } + + // Create atomically. `create_new` fails if the file already exists, which + // closes the exists()->write TOCTOU: a lost race deterministically loads + // the winner's key instead of overwriting it. + #[cfg(unix)] + { + use std::io::Write; + use std::os::unix::fs::OpenOptionsExt; + match std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .mode(0o600) + .open(&key_path) { - use std::os::unix::fs::PermissionsExt; - std::fs::write(&key_path, pem.as_bytes())?; - std::fs::set_permissions(&key_path, std::fs::Permissions::from_mode(0o600))?; + Ok(mut f) => { + // The file is 0600 from creation, so the PEM is never + // world-readable, even momentarily. + f.write_all(pem.as_bytes()) + .with_context(|| format!("failed to write key to {}", key_path.display()))?; + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { + // A concurrent start won the race and wrote the identity; load + // theirs rather than propagating the error or overwriting. + return load_racing(&key_path); + } + Err(e) => { + return Err(e) + .with_context(|| format!("failed to create key at {}", key_path.display())); + } + } + } + #[cfg(not(unix))] + { + use std::io::Write; + match std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&key_path) + { + Ok(mut f) => { + f.write_all(pem.as_bytes()) + .with_context(|| format!("failed to write key to {}", key_path.display()))?; + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { + return load_racing(&key_path); + } + Err(e) => { + return Err(e) + .with_context(|| format!("failed to create key at {}", key_path.display())); + } } - #[cfg(not(unix))] - std::fs::write(&key_path, pem.as_bytes())?; - - info!(path = %key_path.display(), did = %kp.did(), "generated new node identity"); - Ok(kp) } + + info!(path = %key_path.display(), did = %kp.did(), "generated new node identity"); + Ok(kp) } #[cfg(test)] @@ -1094,3 +1220,187 @@ mod gossip_ssrf_tests { assert!(!ok, "a connection error must count as an unhealthy peer"); } } + +#[cfg(all(test, unix))] +mod identity_key_tests { + use super::{load_or_create_keypair, Config}; + use clap::Parser; + use std::os::unix::fs::PermissionsExt; + + // A freshly created identity key must be 0600 immediately, with no + // world-readable disclosure window (the atomic create_new(...).mode(0o600) + // guarantee). This is the RED-then-GREEN anchor for the perms fix: the old + // fs::write + set_permissions sequence left a 0644 window. + #[test] + fn created_key_is_mode_0600() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let config = Config::parse_from([ + "gitlawb-node", + "--key-path", + key_path.to_str().expect("utf8 path"), + ]); + + let _kp = load_or_create_keypair(&config).expect("create keypair"); + + let mode = std::fs::metadata(&key_path) + .expect("key file exists") + .permissions() + .mode(); + assert_eq!( + mode & 0o777, + 0o600, + "identity key must be 0600, got {:o}", + mode & 0o777 + ); + } + + // A lost create race (file already present) must load the existing key + // rather than overwrite it, and must not error on AlreadyExists. Loading an + // existing loose-permission key tightens it to 0600 defensively. + #[test] + fn existing_key_is_loaded_and_tightened() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let config = Config::parse_from([ + "gitlawb-node", + "--key-path", + key_path.to_str().expect("utf8 path"), + ]); + + let first = load_or_create_keypair(&config).expect("create keypair"); + // Loosen perms to simulate a legacy/loose on-disk key. + std::fs::set_permissions(&key_path, std::fs::Permissions::from_mode(0o644)) + .expect("loosen perms"); + + let second = load_or_create_keypair(&config).expect("load keypair"); + assert_eq!( + first.did(), + second.did(), + "reloading must return the same identity, not a new one" + ); + + let mode = std::fs::metadata(&key_path) + .expect("key file exists") + .permissions() + .mode(); + assert_eq!( + mode & 0o777, + 0o600, + "load path must tighten loose perms to 0600, got {:o}", + mode & 0o777 + ); + } + + // The create-race arm the single-threaded tests skip (they enter via the + // `exists()` fast path). N concurrent starts on ONE fresh path must converge + // on a single identity: the `create_new` losers hit `AlreadyExists` and load + // the winner's key rather than overwriting it. RED: revert the write to a + // plain `fs::write` (no `create_new`) and each thread returns its own + // freshly generated key, so the DIDs diverge. + #[test] + fn concurrent_starts_converge_on_one_identity() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let config = std::sync::Arc::new(Config::parse_from([ + "gitlawb-node", + "--key-path", + key_path.to_str().expect("utf8 path"), + ])); + + let n = 8; + let barrier = std::sync::Arc::new(std::sync::Barrier::new(n)); + let handles: Vec<_> = (0..n) + .map(|_| { + let c = config.clone(); + let b = barrier.clone(); + std::thread::spawn(move || { + // Release all threads at once to maximize the race. + b.wait(); + format!("{}", load_or_create_keypair(&c).expect("keypair").did()) + }) + }) + .collect(); + let dids: Vec = handles + .into_iter() + .map(|h| h.join().expect("thread joins")) + .collect(); + + let first = &dids[0]; + assert!( + dids.iter().all(|d| d == first), + "all concurrent starts must converge on one identity, got: {dids:?}" + ); + } +} + +#[cfg(test)] +mod shutdown_grace_tests { + use super::drive_serve_with_grace; + use std::time::{Duration, Instant}; + + // The must-not case: the drain (serve future) never completes but the signal + // has fired. The grace deadline must fire, report `grace_expired = true`, and + // return promptly so teardown proceeds — not hang forever. RED: a helper that + // just awaits `serve` (ignoring grace) hangs this test past its bound. + #[tokio::test] + async fn hung_drain_is_abandoned_after_grace() { + let (armed_tx, armed_rx) = tokio::sync::oneshot::channel::<()>(); + let _ = armed_tx.send(()); // signal already fired + let serve = std::future::pending::>(); // never drains + let start = Instant::now(); + + let (result, grace_expired) = + drive_serve_with_grace(serve, armed_rx, Duration::from_millis(50)).await; + + assert!(grace_expired, "a drain outlasting grace must be abandoned"); + assert!( + result.is_ok(), + "abandon path returns Ok so teardown proceeds" + ); + assert!( + start.elapsed() < Duration::from_secs(5), + "must not wait indefinitely for the hung drain" + ); + } + + // Normal drain: the serve future completes before grace, so the real serve + // result is propagated and `grace_expired` is false. + #[tokio::test] + async fn completed_drain_keeps_result_and_does_not_expire() { + let (_armed_tx, armed_rx) = tokio::sync::oneshot::channel::<()>(); + let serve = std::future::ready(Ok::<(), std::io::Error>(())); + + let (result, grace_expired) = + drive_serve_with_grace(serve, armed_rx, Duration::from_secs(3600)).await; + + assert!( + !grace_expired, + "a completed drain must not report grace expiry" + ); + assert!(result.is_ok()); + } + + // The grace clock starts at the SIGNAL, not at call time. With the signal + // unsent, a finite drain longer than `grace` still completes normally — total + // uptime is never bounded by grace. RED: drop `armed.await` from the grace + // branch and the deadline fires at 10ms, expiring before the 80ms drain. + #[tokio::test] + async fn grace_clock_starts_at_signal_not_call() { + let (_armed_tx, armed_rx) = tokio::sync::oneshot::channel::<()>(); + // Signal never fires; drain finishes after > grace. + let serve = async { + tokio::time::sleep(Duration::from_millis(80)).await; + Ok::<(), std::io::Error>(()) + }; + + let (result, grace_expired) = + drive_serve_with_grace(serve, armed_rx, Duration::from_millis(10)).await; + + assert!( + !grace_expired, + "grace must not fire while the shutdown signal is unsent" + ); + assert!(result.is_ok()); + } +} diff --git a/crates/gitlawb-node/tests/deny_harness.rs b/crates/gitlawb-node/tests/deny_harness.rs index 491e1ae8..42655232 100644 --- a/crates/gitlawb-node/tests/deny_harness.rs +++ b/crates/gitlawb-node/tests/deny_harness.rs @@ -25,6 +25,15 @@ fn cid_for_oid(oid_hex: &str) -> String { Cid::from_sha256_bytes(&arr).to_string() } +/// A reqwest client with a bounded timeout so a wedged node route fails the test +/// fast rather than hanging until the CI job timeout. +fn bounded_client() -> reqwest::Client { + reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(30)) + .build() + .expect("client builds") +} + // ── U2: the signing client produces signatures require_signature accepts ───── /// A validly signed receive-pack request clears `require_signature` (it does not @@ -34,7 +43,7 @@ fn cid_for_oid(oid_hex: &str) -> String { #[sqlx::test] async fn signed_receive_pack_clears_signature_layer(pool: sqlx::PgPool) { let node = spawn_node(pool).await; - let client = reqwest::Client::new(); + let client = bounded_client(); let kp = Keypair::generate(); let resp = signed_request( @@ -64,7 +73,7 @@ async fn signed_receive_pack_clears_signature_layer(pool: sqlx::PgPool) { #[sqlx::test] async fn tampered_body_after_signing_is_rejected(pool: sqlx::PgPool) { let node = spawn_node(pool).await; - let client = reqwest::Client::new(); + let client = bounded_client(); let kp = Keypair::generate(); // Sign one body, then replace it before sending. @@ -95,7 +104,7 @@ async fn tampered_body_after_signing_is_rejected(pool: sqlx::PgPool) { #[sqlx::test] async fn unsigned_receive_pack_is_denied(pool: sqlx::PgPool) { let node = spawn_node(pool).await; - let client = reqwest::Client::new(); + let client = bounded_client(); let url = format!("{}/alice/repo/git-receive-pack", node.base_url); let resp = client @@ -120,7 +129,7 @@ async fn unsigned_receive_pack_is_denied(pool: sqlx::PgPool) { #[sqlx::test] async fn anon_ipfs_read_of_withheld_blob_is_denied(pool: sqlx::PgPool) { let node = spawn_node(pool).await; - let client = reqwest::Client::new(); + let client = bounded_client(); let owner = Keypair::generate(); let owner_did = owner.did().to_string(); @@ -189,7 +198,7 @@ async fn anon_ipfs_read_of_withheld_blob_is_denied(pool: sqlx::PgPool) { #[sqlx::test] async fn wrong_owner_visibility_put_is_forbidden(pool: sqlx::PgPool) { let node = spawn_node(pool).await; - let client = reqwest::Client::new(); + let client = bounded_client(); let owner = Keypair::generate(); let owner_did = owner.did().to_string(); @@ -246,7 +255,7 @@ async fn wrong_owner_visibility_put_is_forbidden(pool: sqlx::PgPool) { #[sqlx::test] async fn withheld_path_blob_read_is_denied(pool: sqlx::PgPool) { let node = spawn_node(pool).await; - let client = reqwest::Client::new(); + let client = bounded_client(); let owner = Keypair::generate(); let owner_did = owner.did().to_string(); @@ -341,10 +350,7 @@ async fn anon_clone_excludes_withheld_subtree_blobs(pool: sqlx::PgPool) { req_body.extend_from_slice(b"0000"); // flush after wants req_body.extend(pkt("done\n")); - let client = reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(30)) - .build() - .unwrap(); + let client = bounded_client(); let resp = client .post(format!( "{}/{owner_did}/u8-repo/git-upload-pack", @@ -417,7 +423,7 @@ async fn anon_clone_excludes_withheld_subtree_blobs(pool: sqlx::PgPool) { #[sqlx::test] async fn additional_owner_gates_reject_non_owner(pool: sqlx::PgPool) { let node = spawn_node(pool).await; - let client = reqwest::Client::new(); + let client = bounded_client(); let owner = Keypair::generate(); let owner_did = owner.did().to_string(); let stranger = Keypair::generate(); From fb8685d95c440fc07a08bb2b8391f0c1acfd85fe Mon Sep 17 00:00:00 2001 From: t Date: Mon, 13 Jul 2026 20:25:12 -0500 Subject: [PATCH 11/41] fix(node): remove a failed first-write key file; verify key mode after tightening (#194) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit F1 (P1): create_new(true) closed the permission window, but a write_all failure after the file was created left an empty/partial PEM behind. Every later start then took the key_path.exists() branch, re-parsed that corrupt file in load_racing, and exited 'invalid PEM key' instead of regenerating — a transient ENOSPC/EIO on first boot permanently wedged the node. Extract write_key_or_cleanup, which removes the just-created file on write failure, and wire it into both the unix and non-unix create branches. F2 (P2): the load path tightened a loose existing key to 0600 but discarded the set_permissions result. A chmod that failed (read-only mount, ownership/ACL mismatch) left a world/group-readable private key in use while logging a normal 'loaded existing identity'. Surface the tighten failure (propagate it) and add ensure_key_mode_0600, which fails closed if the key is not 0600 after the attempt. RED->GREEN: failed_write_removes_the_partial_key_file (a failed write leaves no file; RED without the remove_file). loose_key_mode_is_rejected_not_used (a 0644 key is rejected; RED without the mode check). Existing created_key_is_mode_0600, existing_key_is_loaded_and_tightened, and concurrent_starts_converge_on_one_identity stay green. Full node lib+bin suite 497 passed, fmt + clippy clean. --- crates/gitlawb-node/src/lib.rs | 121 +++++++++++++++++++++++++++++++-- 1 file changed, 115 insertions(+), 6 deletions(-) diff --git a/crates/gitlawb-node/src/lib.rs b/crates/gitlawb-node/src/lib.rs index 88dff5e1..ecc38ecf 100644 --- a/crates/gitlawb-node/src/lib.rs +++ b/crates/gitlawb-node/src/lib.rs @@ -1044,6 +1044,43 @@ async fn ping_peer_health(client: &reqwest::Client, http_url: &str) -> bool { .unwrap_or(false) } +/// Persist a just-created identity key, removing the file if the write failed +/// (#194, F1). `create_new` makes the inode appear before the PEM is flushed, so a +/// failed `write_all` (ENOSPC/EIO/quota) leaves an empty or partial PEM behind; +/// every later start would then take the `exists()` branch and re-parse that +/// corrupt file forever, exiting `invalid PEM key` instead of regenerating. Remove +/// it on failure so the next start starts clean. +fn write_key_or_cleanup(path: &std::path::Path, write_result: std::io::Result<()>) -> Result<()> { + write_result.map_err(|e| { + // Best-effort: if removal also fails there is nothing more to do, and the + // original write error is the one worth surfacing. + let _ = std::fs::remove_file(path); + anyhow::Error::new(e).context(format!("failed to write key to {}", path.display())) + }) +} + +/// Verify an identity key file is not world/group-readable (#194, F2). A `chmod` +/// that failed or silently no-op'd (read-only mount, ACL mismatch) must not leave +/// a readable private key in use, so this is checked after any tightening attempt +/// and fails closed rather than logging a normal "loaded identity" path. +#[cfg(unix)] +fn ensure_key_mode_0600(path: &std::path::Path) -> Result<()> { + use std::os::unix::fs::PermissionsExt; + let mode = std::fs::metadata(path) + .with_context(|| format!("stat identity key {}", path.display()))? + .permissions() + .mode() + & 0o777; + if mode != 0o600 { + anyhow::bail!( + "identity key {} has mode {mode:o}, expected 0600 — refusing to use a \ + world/group-readable private key", + path.display() + ); + } + Ok(()) +} + fn load_or_create_keypair(config: &Config) -> Result { let key_path = config.resolved_key_path(); @@ -1056,9 +1093,17 @@ fn load_or_create_keypair(config: &Config) -> Result { use std::os::unix::fs::PermissionsExt; if let Ok(meta) = std::fs::metadata(path) { if meta.permissions().mode() & 0o777 != 0o600 { - let _ = std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)); + // Do NOT silently ignore a failed tighten (#194, F2): surface it + // so a key we cannot secure is not read and used exposed. + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) + .with_context(|| { + format!("could not tighten identity key {} to 0600", path.display()) + })?; } } + // Verify the key is actually 0600 before using it — a chmod that + // succeeded-but-no-op'd (some mounts) still leaves it exposed. + ensure_key_mode_0600(path)?; } let pem = std::fs::read_to_string(path) .with_context(|| format!("failed to read key from {}", path.display()))?; @@ -1119,9 +1164,9 @@ fn load_or_create_keypair(config: &Config) -> Result { { Ok(mut f) => { // The file is 0600 from creation, so the PEM is never - // world-readable, even momentarily. - f.write_all(pem.as_bytes()) - .with_context(|| format!("failed to write key to {}", key_path.display()))?; + // world-readable, even momentarily. On a failed write, remove the + // empty/partial file so a later start regenerates (#194, F1). + write_key_or_cleanup(&key_path, f.write_all(pem.as_bytes()))?; } Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { // A concurrent start won the race and wrote the identity; load @@ -1143,8 +1188,9 @@ fn load_or_create_keypair(config: &Config) -> Result { .open(&key_path) { Ok(mut f) => { - f.write_all(pem.as_bytes()) - .with_context(|| format!("failed to write key to {}", key_path.display()))?; + // On a failed write, remove the empty/partial file so a later + // start regenerates instead of re-parsing it forever (#194, F1). + write_key_or_cleanup(&key_path, f.write_all(pem.as_bytes()))?; } Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { return load_racing(&key_path); @@ -1332,6 +1378,69 @@ mod identity_key_tests { "all concurrent starts must converge on one identity, got: {dids:?}" ); } + + /// #194 (F1): a failed write of a just-created key file removes it, so a later + /// start regenerates instead of re-parsing an empty/partial PEM forever and + /// wedging on `invalid PEM key`. RED without the `remove_file` in + /// `write_key_or_cleanup`: the partial file survives the error. + #[test] + fn failed_write_removes_the_partial_key_file() { + use super::write_key_or_cleanup; + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + std::fs::write(&key_path, b"partial-pem").expect("seed partial file"); + assert!(key_path.exists()); + + let err = std::io::Error::other("simulated ENOSPC"); + let r = write_key_or_cleanup(&key_path, Err(err)); + assert!(r.is_err(), "the write error must propagate"); + assert!( + !key_path.exists(), + "a failed first write must not leave a partial key file to wedge later starts" + ); + } + + /// A successful write leaves the file in place (the common case). + #[test] + fn successful_write_keeps_the_key_file() { + use super::write_key_or_cleanup; + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + std::fs::write(&key_path, b"good-pem").expect("seed file"); + assert!(write_key_or_cleanup(&key_path, Ok(())).is_ok()); + assert!( + key_path.exists(), + "a successful write leaves the file in place" + ); + } + + /// #194 (F2): a loose (world/group-readable) key that cannot be tightened is + /// rejected fail-closed rather than read and used exposed; a 0600 key is + /// accepted. RED without `ensure_key_mode_0600`: a 0644 key is used silently. + #[test] + fn loose_key_mode_is_rejected_not_used() { + use super::ensure_key_mode_0600; + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + std::fs::write(&key_path, b"key").expect("seed file"); + + std::fs::set_permissions(&key_path, std::fs::Permissions::from_mode(0o644)) + .expect("loosen perms"); + let err = ensure_key_mode_0600(&key_path) + .expect_err("a world/group-readable key must be rejected") + .to_string(); + assert!( + err.contains("644"), + "the failure must name the exposed mode: {err}" + ); + + std::fs::set_permissions(&key_path, std::fs::Permissions::from_mode(0o600)) + .expect("tighten perms"); + assert!( + ensure_key_mode_0600(&key_path).is_ok(), + "a 0600 key must be accepted" + ); + } } #[cfg(test)] From e3eac372be108f86813dc1da86dfa99a000e181b Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 14 Jul 2026 23:18:51 -0500 Subject: [PATCH 12/41] fix(node): publish the identity key atomically and wait a real deadline The create path did create_new(final)+write_all, so the final path appeared as an empty inode before the PEM was flushed, and a losing/fast-path start only retried ~100ms (50x2ms). On a slow or stalled filesystem the winner's write can exceed that window, so every other start failed boot with 'invalid PEM key' (#194). Publish atomically instead: write the full PEM to a sibling temp, then hard_link it into place. hard_link is atomic and fails if the target exists, so the final path only ever appears COMPLETE (no partial-read window), a lost race never clobbers the winner, and a crashed writer leaves only a temp rather than a partial final that would wedge later starts. load_racing_key now polls on a wall-clock KEY_RACE_DEADLINE (5s) instead of a fixed 100ms count. Hoisted load_existing_key/load_racing_key to module level for testability. Adversarial RED->GREEN: a 250ms-slow winner is waited out (RED at the old 100ms); a reader watching the final never sees a partial file (RED with create_new+write); a losing publish does not clobber the winner; 500 tests pass. --- crates/gitlawb-node/src/lib.rs | 348 +++++++++++++++++++++++---------- 1 file changed, 243 insertions(+), 105 deletions(-) diff --git a/crates/gitlawb-node/src/lib.rs b/crates/gitlawb-node/src/lib.rs index ecc38ecf..a9415f9a 100644 --- a/crates/gitlawb-node/src/lib.rs +++ b/crates/gitlawb-node/src/lib.rs @@ -1081,63 +1081,133 @@ fn ensure_key_mode_0600(path: &std::path::Path) -> Result<()> { Ok(()) } -fn load_or_create_keypair(config: &Config) -> Result { - let key_path = config.resolved_key_path(); +/// Outcome of an atomic key publish. +enum KeyPublish { + /// We created the key file; it now holds the complete PEM. + Won, + /// Another concurrent start already created it; ours was discarded. + Lost, +} - // Load an already-provisioned key. On Unix, defensively tighten looser - // permissions to 0600 (do NOT reject a loose key — that would break - // existing deployments; just narrow them). - fn load_existing(path: &std::path::Path) -> Result { - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - if let Ok(meta) = std::fs::metadata(path) { - if meta.permissions().mode() & 0o777 != 0o600 { - // Do NOT silently ignore a failed tighten (#194, F2): surface it - // so a key we cannot secure is not read and used exposed. - std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) - .with_context(|| { - format!("could not tighten identity key {} to 0600", path.display()) - })?; - } +/// Wall-clock budget for reading a key another start may still be publishing. +/// The atomic publish (`publish_key_atomically`) means a *present* key file is +/// always complete, so this normally succeeds on the first read; the deadline +/// only covers cross-host cache lag. It is a wall-clock budget, NOT a fixed +/// retry count, so a slow/stalled filesystem cannot starve a losing start after +/// an arbitrarily short (~100ms) window (#194). +const KEY_RACE_DEADLINE: std::time::Duration = std::time::Duration::from_secs(5); + +/// Load an already-provisioned identity key. On Unix, defensively tighten looser +/// permissions to 0600 (do NOT reject a loose key — that would break existing +/// deployments; just narrow them), then verify the mode is actually 0600. +fn load_existing_key(path: &std::path::Path) -> Result { + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + if let Ok(meta) = std::fs::metadata(path) { + if meta.permissions().mode() & 0o777 != 0o600 { + // Do NOT silently ignore a failed tighten (#194, F2): surface it + // so a key we cannot secure is not read and used exposed. + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) + .with_context(|| { + format!("could not tighten identity key {} to 0600", path.display()) + })?; } - // Verify the key is actually 0600 before using it — a chmod that - // succeeded-but-no-op'd (some mounts) still leaves it exposed. - ensure_key_mode_0600(path)?; } - let pem = std::fs::read_to_string(path) - .with_context(|| format!("failed to read key from {}", path.display()))?; - let kp = Keypair::from_pem(&pem).map_err(|e| anyhow::anyhow!("invalid PEM key: {e}"))?; - info!(path = %path.display(), "loaded existing identity"); - Ok(kp) + // Verify the key is actually 0600 before using it — a chmod that + // succeeded-but-no-op'd (some mounts) still leaves it exposed. + ensure_key_mode_0600(path)?; } + let pem = std::fs::read_to_string(path) + .with_context(|| format!("failed to read key from {}", path.display()))?; + let kp = Keypair::from_pem(&pem).map_err(|e| anyhow::anyhow!("invalid PEM key: {e}"))?; + info!(path = %path.display(), "loaded existing identity"); + Ok(kp) +} - // Load a key another concurrent start may still be writing. `create_new` - // makes the inode appear before its PEM is flushed, so a race loser — or a - // fast-path `exists()` that caught the winner mid-write — can momentarily - // read an empty/partial file. Retry briefly until it parses rather than - // crashing the boot. Bounded (~100ms); an already-provisioned key parses on - // the first attempt with no sleep. - fn load_racing(path: &std::path::Path) -> Result { - let mut last_err = None; - for _ in 0..50 { - match load_existing(path) { - Ok(kp) => return Ok(kp), - Err(e) => { - last_err = Some(e); - std::thread::sleep(std::time::Duration::from_millis(2)); - } - } +/// Load a key another concurrent start may still be publishing, polling until it +/// parses or `KEY_RACE_DEADLINE` elapses. An already-provisioned key parses on +/// the first attempt with no sleep. +fn load_racing_key(path: &std::path::Path) -> Result { + let deadline = std::time::Instant::now() + KEY_RACE_DEADLINE; + let mut last_err; + loop { + match load_existing_key(path) { + Ok(kp) => return Ok(kp), + Err(e) => last_err = e, } - Err(last_err.unwrap_or_else(|| { - anyhow::anyhow!("key at {} unreadable after create race", path.display()) - })) + if std::time::Instant::now() >= deadline { + return Err(last_err); + } + std::thread::sleep(std::time::Duration::from_millis(2)); + } +} + +/// Publish `pem` to `final_path` atomically: write the full bytes to a sibling +/// temp file, then `hard_link` the temp into place. `hard_link` is atomic and +/// fails if `final_path` already exists, which gives three guarantees at once: +/// (a) the final path only ever appears with COMPLETE content — a concurrent +/// reader never observes an empty/half-written key, unlike a +/// `create_new`+`write_all` that exposes an empty inode before the PEM is +/// flushed (#194); +/// (b) a lost race never clobbers the winner — exactly one publisher links; +/// (c) a crashed publisher leaves only the temp, never a partial final that +/// would wedge every later start on `invalid PEM key`. +/// `before_link` is a no-op in production; tests use it to widen the +/// post-write / pre-link window deterministically. +fn publish_key_atomically( + final_path: &std::path::Path, + pem: &[u8], + before_link: &dyn Fn(), +) -> Result { + use std::io::Write; + let dir = final_path + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or_else(|| std::path::Path::new(".")); + let stem = final_path + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or("identity.pem"); + // Unique per call (process-global counter) so concurrent publishers never + // collide on the temp name; `create_new` below is the backstop. + static SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let seq = SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let tmp = dir.join(format!(".{stem}.tmp.{}.{seq}", std::process::id())); + + let mut opts = std::fs::OpenOptions::new(); + opts.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + opts.mode(0o600); } + let mut f = opts + .open(&tmp) + .with_context(|| format!("create temp key {}", tmp.display()))?; + // On a failed write, remove the temp so nothing partial is ever linked (#194, F1). + write_key_or_cleanup(&tmp, f.write_all(pem))?; + drop(f); + + before_link(); + + let linked = std::fs::hard_link(&tmp, final_path); + let _ = std::fs::remove_file(&tmp); + match linked { + Ok(()) => Ok(KeyPublish::Won), + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => Ok(KeyPublish::Lost), + Err(e) => Err(e) + .with_context(|| format!("link identity key into place at {}", final_path.display())), + } +} + +fn load_or_create_keypair(config: &Config) -> Result { + let key_path = config.resolved_key_path(); // Fast path for the common already-provisioned case (still race-safe: a - // concurrently-writing winner is handled by the retry inside `load_racing`). + // concurrently-publishing winner is handled by the retry in load_racing_key). if key_path.exists() { - return load_racing(&key_path); + return load_racing_key(&key_path); } let kp = Keypair::generate(); @@ -1149,61 +1219,15 @@ fn load_or_create_keypair(config: &Config) -> Result { std::fs::create_dir_all(parent)?; } - // Create atomically. `create_new` fails if the file already exists, which - // closes the exists()->write TOCTOU: a lost race deterministically loads - // the winner's key instead of overwriting it. - #[cfg(unix)] - { - use std::io::Write; - use std::os::unix::fs::OpenOptionsExt; - match std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .mode(0o600) - .open(&key_path) - { - Ok(mut f) => { - // The file is 0600 from creation, so the PEM is never - // world-readable, even momentarily. On a failed write, remove the - // empty/partial file so a later start regenerates (#194, F1). - write_key_or_cleanup(&key_path, f.write_all(pem.as_bytes()))?; - } - Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { - // A concurrent start won the race and wrote the identity; load - // theirs rather than propagating the error or overwriting. - return load_racing(&key_path); - } - Err(e) => { - return Err(e) - .with_context(|| format!("failed to create key at {}", key_path.display())); - } - } - } - #[cfg(not(unix))] - { - use std::io::Write; - match std::fs::OpenOptions::new() - .write(true) - .create_new(true) - .open(&key_path) - { - Ok(mut f) => { - // On a failed write, remove the empty/partial file so a later - // start regenerates instead of re-parsing it forever (#194, F1). - write_key_or_cleanup(&key_path, f.write_all(pem.as_bytes()))?; - } - Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { - return load_racing(&key_path); - } - Err(e) => { - return Err(e) - .with_context(|| format!("failed to create key at {}", key_path.display())); - } + // Publish atomically: the final path only ever appears complete, and a lost + // race loads the winner's key rather than overwriting it. + match publish_key_atomically(&key_path, pem.as_bytes(), &|| {})? { + KeyPublish::Won => { + info!(path = %key_path.display(), did = %kp.did(), "generated new node identity"); + Ok(kp) } + KeyPublish::Lost => load_racing_key(&key_path), } - - info!(path = %key_path.display(), did = %kp.did(), "generated new node identity"); - Ok(kp) } #[cfg(test)] @@ -1269,10 +1293,124 @@ mod gossip_ssrf_tests { #[cfg(all(test, unix))] mod identity_key_tests { - use super::{load_or_create_keypair, Config}; + use super::{ + load_or_create_keypair, load_racing_key, publish_key_atomically, Config, KeyPublish, + Keypair, + }; use clap::Parser; use std::os::unix::fs::PermissionsExt; + // #194 (P2): the racing loader must wait out a SLOW winner on a meaningful + // wall-clock deadline, not give up after an arbitrary ~100ms window. Simulate + // a winner that only publishes the key after 250ms (past the old 100ms budget) + // and assert the loader waits it out instead of failing with `invalid PEM key`. + // RED with the old `for _ in 0..50` (2ms) fixed-count loop. + #[test] + fn load_racing_key_waits_out_a_slow_winner() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let pem = Keypair::generate().to_pem().expect("pem"); + + let writer_path = key_path.clone(); + let writer = std::thread::spawn(move || { + std::thread::sleep(std::time::Duration::from_millis(250)); + publish_key_atomically(&writer_path, pem.as_bytes(), &|| {}).expect("publish"); + }); + + let started = std::time::Instant::now(); + let kp = load_racing_key(&key_path) + .expect("racing load must wait out the slow winner, not fail"); + let waited = started.elapsed(); + writer.join().expect("writer joins"); + + assert!( + waited >= std::time::Duration::from_millis(200), + "loader should have waited for the ~250ms-slow winner, only waited {waited:?}" + ); + assert!( + !format!("{}", kp.did()).is_empty(), + "loaded the published key" + ); + } + + // #194 (P2): the atomic publish preserves the single-winner no-clobber + // guarantee — a second publish of a DIFFERENT key must lose and leave the + // winner's key untouched — and it cleans up its temp files. + #[test] + fn publish_wins_then_second_publish_loses_without_clobbering() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let pem_a = Keypair::generate().to_pem().expect("pem a"); + let pem_b = Keypair::generate().to_pem().expect("pem b"); + assert_ne!(pem_a.as_str(), pem_b.as_str(), "fixtures must differ"); + + let out = publish_key_atomically(&key_path, pem_a.as_bytes(), &|| {}).expect("publish a"); + assert!(matches!(out, KeyPublish::Won), "first publish wins"); + assert_eq!( + std::fs::read_to_string(&key_path).unwrap().as_str(), + pem_a.as_str(), + "final holds the full PEM" + ); + + let out = publish_key_atomically(&key_path, pem_b.as_bytes(), &|| {}).expect("publish b"); + assert!(matches!(out, KeyPublish::Lost), "second publish loses"); + assert_eq!( + std::fs::read_to_string(&key_path).unwrap().as_str(), + pem_a.as_str(), + "the winner's key must NOT be clobbered by a losing publish" + ); + + let leftovers: Vec<_> = std::fs::read_dir(dir.path()) + .unwrap() + .filter_map(|e| e.ok()) + .filter(|e| e.file_name().to_string_lossy().contains(".tmp.")) + .collect(); + assert!( + leftovers.is_empty(), + "temp files must be cleaned up, found {leftovers:?}" + ); + } + + // #194 (P2, the core of jatmn's option b): while a winner is between writing + // its temp and linking it into place, a reader watching the FINAL path must + // see it absent or COMPLETE — never empty/partial. RED with a + // create_new(final)+write approach, which exposes an empty final for the whole + // write window. + #[test] + fn publish_never_exposes_a_partial_final() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = std::sync::Arc::new(dir.path().join("identity.pem")); + let pem = Keypair::generate().to_pem().expect("pem"); + + let wp = key_path.clone(); + let writer = std::thread::spawn(move || { + // Hold the post-write / pre-link window open for 200ms. + publish_key_atomically(&wp, pem.as_bytes(), &|| { + std::thread::sleep(std::time::Duration::from_millis(200)); + }) + .expect("publish"); + }); + + let deadline = std::time::Instant::now() + std::time::Duration::from_millis(400); + let mut saw_complete = false; + while std::time::Instant::now() < deadline { + if key_path.exists() { + let body = std::fs::read_to_string(&*key_path).unwrap_or_default(); + assert!( + Keypair::from_pem(&body).is_ok(), + "final key observed in a partial/empty state: {body:?}" + ); + saw_complete = true; + } + std::thread::sleep(std::time::Duration::from_millis(1)); + } + writer.join().expect("writer joins"); + assert!( + saw_complete, + "reader should have observed the completed key within the window" + ); + } + // A freshly created identity key must be 0600 immediately, with no // world-readable disclosure window (the atomic create_new(...).mode(0o600) // guarantee). This is the RED-then-GREEN anchor for the perms fix: the old @@ -1340,10 +1478,10 @@ mod identity_key_tests { // The create-race arm the single-threaded tests skip (they enter via the // `exists()` fast path). N concurrent starts on ONE fresh path must converge - // on a single identity: the `create_new` losers hit `AlreadyExists` and load - // the winner's key rather than overwriting it. RED: revert the write to a - // plain `fs::write` (no `create_new`) and each thread returns its own - // freshly generated key, so the DIDs diverge. + // on a single identity: the atomic publish links exactly one winner and the + // losers hit `AlreadyExists` and load the winner's key rather than overwriting + // it. RED: replace the atomic publish with a plain `fs::write` and each thread + // returns its own freshly generated key, so the DIDs diverge. #[test] fn concurrent_starts_converge_on_one_identity() { let dir = tempfile::tempdir().expect("tempdir"); From 688b506f13a62b789b76f8ca08beabfd30536f5c Mon Sep 17 00:00:00 2001 From: t Date: Wed, 15 Jul 2026 19:06:05 -0500 Subject: [PATCH 13/41] fix(node): recover key publish from stale temps, make it durable; join TestNode teardown (#194) - pick the temp key name with a bounded per-call retry instead of a process-global counter: a crashed start's leftover temp no longer wedges a PID-stable restart, and cross-container publishers stop colliding - fsync the temp PEM before it can be linked and the key directory after publication, so a power crash cannot leave a durable-but-truncated identity.pem that permanently blocks startup - TestNode::shutdown() joins the serve task and closes the pool before sqlx's DROP DATABASE cleanup runs; every deny-harness test now ends with it instead of relying on drop-only teardown --- crates/gitlawb-node/src/lib.rs | 157 +++++++++++++++++++--- crates/gitlawb-node/src/test_harness.rs | 57 +++++++- crates/gitlawb-node/tests/deny_harness.rs | 62 ++++++++- 3 files changed, 248 insertions(+), 28 deletions(-) diff --git a/crates/gitlawb-node/src/lib.rs b/crates/gitlawb-node/src/lib.rs index a9415f9a..0e8b4f81 100644 --- a/crates/gitlawb-node/src/lib.rs +++ b/crates/gitlawb-node/src/lib.rs @@ -1097,6 +1097,14 @@ enum KeyPublish { /// an arbitrarily short (~100ms) window (#194). const KEY_RACE_DEADLINE: std::time::Duration = std::time::Duration::from_secs(5); +/// Bounded number of temp-file names a publish will try before giving up. +/// Attempt 0 is the deterministic `.{stem}.tmp.{pid}.0`, so a stale temp left +/// by a crashed prior start with the same PID (a restarted container is +/// commonly PID 1 again) is skipped rather than wedging the restart (#194). +/// 64 names is far more than any plausible pile-up of stale temps plus live +/// concurrent publishers. +const KEY_TEMP_ATTEMPTS: u32 = 64; + /// Load an already-provisioned identity key. On Unix, defensively tighten looser /// permissions to 0600 (do NOT reject a loose key — that would break existing /// deployments; just narrow them), then verify the mode is actually 0600. @@ -1152,7 +1160,10 @@ fn load_racing_key(path: &std::path::Path) -> Result { /// flushed (#194); /// (b) a lost race never clobbers the winner — exactly one publisher links; /// (c) a crashed publisher leaves only the temp, never a partial final that -/// would wedge every later start on `invalid PEM key`. +/// would wedge every later start on `invalid PEM key`; +/// (d) durability: the PEM bytes are fsynced before the name appears and the +/// namespace change is fsynced (on Unix) before success is reported, so a +/// power crash cannot leave a durable-but-truncated final key (#194). /// `before_link` is a no-op in production; tests use it to widen the /// post-write / pre-link window deterministically. fn publish_key_atomically( @@ -1169,24 +1180,50 @@ fn publish_key_atomically( .file_name() .and_then(|s| s.to_str()) .unwrap_or("identity.pem"); - // Unique per call (process-global counter) so concurrent publishers never - // collide on the temp name; `create_new` below is the backstop. - static SEQ: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); - let seq = SEQ.fetch_add(1, std::sync::atomic::Ordering::Relaxed); - let tmp = dir.join(format!(".{stem}.tmp.{}.{seq}", std::process::id())); - - let mut opts = std::fs::OpenOptions::new(); - opts.write(true).create_new(true); - #[cfg(unix)] - { - use std::os::unix::fs::OpenOptionsExt; - opts.mode(0o600); + + // Pick the temp name with a bounded per-call retry, not a process-global + // counter: a counter is only unique within one process (two containers + // sharing a volume can both run as PID 1), and a crash between the temp + // write and its cleanup leaves the old name behind, so a single + // deterministic name would fail `create_new` on every restart until an + // operator deleted it (#194). `AlreadyExists` means either such a stale + // temp or a live concurrent publisher's temp; the two are indistinguishable, + // so never delete a temp this call did not create (unlinking a live one + // between its write and its hard_link would fail that healthy start), just + // skip to the next name. Consequence: a crashed start leaks at most one + // 0600 temp file, which later starts skip. + let mut opened = None; + for attempt in 0..KEY_TEMP_ATTEMPTS { + let tmp = dir.join(format!(".{stem}.tmp.{}.{attempt}", std::process::id())); + let mut opts = std::fs::OpenOptions::new(); + opts.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + opts.mode(0o600); + } + match opts.open(&tmp) { + Ok(f) => { + opened = Some((tmp, f)); + break; + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(e) => return Err(e).with_context(|| format!("create temp key {}", tmp.display())), + } } - let mut f = opts - .open(&tmp) - .with_context(|| format!("create temp key {}", tmp.display()))?; - // On a failed write, remove the temp so nothing partial is ever linked (#194, F1). - write_key_or_cleanup(&tmp, f.write_all(pem))?; + let Some((tmp, mut f)) = opened else { + anyhow::bail!( + "all {KEY_TEMP_ATTEMPTS} temp key names .{stem}.tmp.{}.* in {} are taken; \ + remove the stale temp files and restart", + std::process::id(), + dir.display() + ); + }; + // On a failed write OR fsync, remove the temp so nothing partial is ever + // linked (#194). The fsync makes the PEM bytes durable BEFORE the name can + // appear: without it a power crash after the link could leave a durable + // final name over truncated bytes, wedging every later start. + write_key_or_cleanup(&tmp, f.write_all(pem).and_then(|()| f.sync_all()))?; drop(f); before_link(); @@ -1194,7 +1231,22 @@ fn publish_key_atomically( let linked = std::fs::hard_link(&tmp, final_path); let _ = std::fs::remove_file(&tmp); match linked { - Ok(()) => Ok(KeyPublish::Won), + Ok(()) => { + // Fsync the parent directory so the link (and the temp unlink) are + // durable before success is reported (#194). Skipped on non-Unix, + // where opening a directory fails. On fsync failure do NOT remove + // the final file: it is complete and data-synced, a concurrent + // loser may already have loaded it, and a later start loads it via + // the exists() path. + #[cfg(unix)] + { + std::fs::File::open(dir) + .with_context(|| format!("open key directory {} to fsync", dir.display()))? + .sync_all() + .with_context(|| format!("fsync key directory {}", dir.display()))?; + } + Ok(KeyPublish::Won) + } Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => Ok(KeyPublish::Lost), Err(e) => Err(e) .with_context(|| format!("link identity key into place at {}", final_path.display())), @@ -1295,7 +1347,7 @@ mod gossip_ssrf_tests { mod identity_key_tests { use super::{ load_or_create_keypair, load_racing_key, publish_key_atomically, Config, KeyPublish, - Keypair, + Keypair, KEY_TEMP_ATTEMPTS, }; use clap::Parser; use std::os::unix::fs::PermissionsExt; @@ -1411,6 +1463,71 @@ mod identity_key_tests { ); } + // #194: a temp file left by a crash between the temp write and its cleanup + // must not wedge the restart. A restarted container commonly runs as PID 1 + // again, so the first name tried collides with the stale one; the publish + // must skip it and still win with the complete PEM. RED with a single + // deterministic temp name: `create_new` fails AlreadyExists and the node + // never starts until an operator deletes the temp. + #[test] + fn publish_recovers_from_a_stale_crashed_temp() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let pem = Keypair::generate().to_pem().expect("pem"); + + // The crashed prior start's leftover: the deterministic attempt-0 name, + // 0600, holding a partial write. + let stale = dir + .path() + .join(format!(".identity.pem.tmp.{}.0", std::process::id())); + std::fs::write(&stale, b"partial-").expect("seed stale temp"); + std::fs::set_permissions(&stale, std::fs::Permissions::from_mode(0o600)) + .expect("stale temp perms"); + + let out = publish_key_atomically(&key_path, pem.as_bytes(), &|| {}) + .expect("publish must recover from a stale temp, not wedge the start"); + assert!(matches!(out, KeyPublish::Won), "publish wins"); + assert_eq!( + std::fs::read_to_string(&key_path).unwrap().as_str(), + pem.as_str(), + "final holds the complete PEM, not the stale partial" + ); + } + + // #194: temp-name probing is bounded. With every candidate name taken the + // publish must return a clear error naming the directory and prefix (not + // hang or spin) and must not create the final path. Together with the + // stale-temp test this runs the exhaustion branch both ways. + #[test] + fn publish_errors_when_all_temp_names_are_taken() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let pem = Keypair::generate().to_pem().expect("pem"); + + for attempt in 0..KEY_TEMP_ATTEMPTS { + let tmp = dir.path().join(format!( + ".identity.pem.tmp.{}.{attempt}", + std::process::id() + )); + std::fs::write(&tmp, b"taken").expect("seed taken temp"); + } + + let err = match publish_key_atomically(&key_path, pem.as_bytes(), &|| {}) { + Err(e) => e, + Ok(_) => panic!("publish must fail when every temp name is taken"), + }; + let msg = format!("{err:#}"); + assert!( + msg.contains(&format!(".identity.pem.tmp.{}", std::process::id())) + && msg.contains(&dir.path().display().to_string()), + "the error must name the temp prefix and directory: {msg}" + ); + assert!( + !key_path.exists(), + "an exhausted publish must not create or modify the final key" + ); + } + // A freshly created identity key must be 0600 immediately, with no // world-readable disclosure window (the atomic create_new(...).mode(0o600) // guarantee). This is the RED-then-GREEN anchor for the perms fix: the old diff --git a/crates/gitlawb-node/src/test_harness.rs b/crates/gitlawb-node/src/test_harness.rs index 7dfe6c81..550551ff 100644 --- a/crates/gitlawb-node/src/test_harness.rs +++ b/crates/gitlawb-node/src/test_harness.rs @@ -30,8 +30,12 @@ use crate::db::{Db, RepoRecord, VisibilityMode}; use crate::rate_limit::{RateLimiter, TrustedProxy}; use crate::state::AppState; -/// A running node bound to an ephemeral port. Dropping it signals graceful -/// shutdown and removes the temporary repository directory. +/// A running node bound to an ephemeral port. End every test with +/// [`TestNode::shutdown`]: it joins the serve task and closes the pool, so +/// `#[sqlx::test]`'s `DROP DATABASE` cleanup (which runs the moment the test +/// future returns) never races the server's still-open connections. Dropping +/// without it only signals shutdown and removes the temp repository directory, +/// a best-effort fallback for tests that panic before reaching teardown. pub struct TestNode { /// Base URL of the running node, e.g. `http://127.0.0.1:54321`. pub base_url: String, @@ -43,13 +47,23 @@ pub struct TestNode { /// the integration crate seeds through the methods below rather than /// naming `Db`/`RepoRecord` directly. db: Arc, + /// The detached `axum::serve` task, joined by [`Self::shutdown`]. `None` + /// once taken, so the `Drop` fallback never overlaps a completed teardown. + server: Option>, + /// A clone of the served pool, kept so [`Self::shutdown`] can close it + /// (sqlx pool clones share one inner, so closing this closes the clones + /// inside `AppState`/`Db` too). + pool: PgPool, } impl Drop for TestNode { fn drop(&mut self) { - // Flip the shared shutdown signal so the serve task exits, then remove - // the temp repos dir. Both are best-effort: a test that already failed - // should not panic again in teardown. + // Best-effort fallback for tests that panic before reaching + // [`TestNode::shutdown`]: flip the shared shutdown signal so the serve + // task exits, then remove the temp repos dir. Drop cannot await, so it + // cannot join the server or close the pool; a test that already failed + // should not panic again in teardown. Both actions are idempotent, so + // running after `shutdown()` is harmless. let _ = self.shutdown_tx.send(true); let _ = std::fs::remove_dir_all(&self.repos_dir); } @@ -113,7 +127,7 @@ pub async fn spawn_node(pool: PgPool) -> TestNode { let repos_dir = unique_repos_dir(); std::fs::create_dir_all(&repos_dir).expect("create temp repos dir"); - let state = build_state(db.clone(), pool, repos_dir.clone()); + let state = build_state(db.clone(), pool.clone(), repos_dir.clone()); let node_did = state.node_did.to_string(); let shutdown_tx = state.shutdown_tx.clone(); let mut shutdown_rx = shutdown_tx.subscribe(); @@ -125,7 +139,7 @@ pub async fn spawn_node(pool: PgPool) -> TestNode { .expect("bind ephemeral port"); let addr = listener.local_addr().expect("read bound addr"); - tokio::spawn(async move { + let server = tokio::spawn(async move { let _ = axum::serve( listener, router.into_make_service_with_connect_info::(), @@ -142,10 +156,36 @@ pub async fn spawn_node(pool: PgPool) -> TestNode { shutdown_tx, repos_dir, db, + server: Some(server), + pool, } } impl TestNode { + /// Graceful async teardown: signal shutdown, join the serve task, close + /// the pool, remove the temp repos dir. Call this at the end of every + /// test. `#[sqlx::test]` issues `DROP DATABASE` as soon as the test + /// future returns, so the server (which owns pool clones inside + /// `AppState`/`Db`) must be gone and the pool closed before then, or the + /// cleanup fails and leaks per-test databases. A serve task that does not + /// exit within 10s panics loudly: a wedged graceful shutdown must fail + /// the test, not hang CI. + /// + /// When `self` drops on return, `Drop` re-sends the signal and re-removes + /// the dir; both are idempotent, and the serve handle was already taken, + /// so the overlap is harmless. + pub async fn shutdown(mut self) { + let _ = self.shutdown_tx.send(true); + if let Some(handle) = self.server.take() { + tokio::time::timeout(Duration::from_secs(10), handle) + .await + .expect("serve task must exit within 10s of the shutdown signal (wedged graceful shutdown)") + .expect("serve task must not panic"); + } + self.pool.close().await; + let _ = std::fs::remove_dir_all(&self.repos_dir); + } + /// Insert a repo owned by `owner_did` and return its repo id. Mirrors /// `test_support::seed_repo` (the `disk_path` field is unused by the /// integration path — `RepoStore` computes the on-disk path from @@ -209,6 +249,9 @@ impl TestNode { object_format: &str, ) -> std::collections::HashMap { let run = |args: &[&str], cwd: &std::path::Path| { + // allow-unbounded-git: test-harness-only fixture seeding, feature-gated + // out of the production binary; runs local git against a tempdir inside + // a test's own lifetime, never on a request path holding a permit. let out = std::process::Command::new("git") .args(args) .current_dir(cwd) diff --git a/crates/gitlawb-node/tests/deny_harness.rs b/crates/gitlawb-node/tests/deny_harness.rs index 42655232..ee92a76e 100644 --- a/crates/gitlawb-node/tests/deny_harness.rs +++ b/crates/gitlawb-node/tests/deny_harness.rs @@ -4,7 +4,9 @@ //! (INV-1/INV-2/INV-8). Requires `--features test-harness`. //! //! Each `#[sqlx::test]` gets an ephemeral per-test database; `spawn_node` runs -//! the schema migrations and serves the real router on `127.0.0.1:0`. +//! the schema migrations and serves the real router on `127.0.0.1:0`. Every +//! test ends with `node.shutdown().await` so the serve task is joined and the +//! pool closed before sqlx's `DROP DATABASE` cleanup runs. mod support; @@ -63,6 +65,8 @@ async fn signed_receive_pack_clears_signature_layer(pool: sqlx::PgPool) { 401, "a valid signature must clear require_signature; got 401" ); + + node.shutdown().await; } /// Tampering the body after signing invalidates the content-digest, so the @@ -95,6 +99,8 @@ async fn tampered_body_after_signing_is_rejected(pool: sqlx::PgPool) { 400, "a body that no longer matches its content-digest must be rejected" ); + + node.shutdown().await; } // ── U5(a): INV-8 — an unsigned push is denied and leaks nothing ────────────── @@ -118,6 +124,8 @@ async fn unsigned_receive_pack_is_denied(pool: sqlx::PgPool) { // No repo was seeded, so there are no OIDs to leak; the assertion still // enforces the 4xx-and-not-empty-200 INV-8 shape. assert_denied(resp, 401, &[]).await; + + node.shutdown().await; } // ── U5(b): INV-8/INV-2 — anonymous /ipfs/{cid} of a withheld blob is denied ── @@ -186,6 +194,8 @@ async fn anon_ipfs_read_of_withheld_blob_is_denied(pool: sqlx::PgPool) { resp.text().await.unwrap().contains("public bytes U5b"), "the public blob content is returned" ); + + node.shutdown().await; } // ── U6: INV-1 — a validly signed NON-owner mutation is owner-gated (403) ────── @@ -244,6 +254,8 @@ async fn wrong_owner_visibility_put_is_forbidden(pool: sqlx::PgPool) { "owner's signed visibility PUT must reach the handler, got {}", resp.status() ); + + node.shutdown().await; } // ── U7: INV-2 — a read over a withheld path is denied and leaks nothing ─────── @@ -306,6 +318,8 @@ async fn withheld_path_blob_read_is_denied(pool: sqlx::PgPool) { resp.text().await.unwrap().contains("hello public"), "the public blob content is returned" ); + + node.shutdown().await; } // ── U8: INV-2 — an anonymous clone/fetch excludes withheld subtree blobs ────── @@ -410,6 +424,8 @@ async fn anon_clone_excludes_withheld_subtree_blobs(pool: sqlx::PgPool) { listing.contains(&public_oid), "public blob {public_oid} must be present (withhold is subtree-scoped); listing:\n{listing}" ); + + node.shutdown().await; } // ── Additional INV-1 owner-gates over the real stack (fan-out of U6) ────────── @@ -491,4 +507,48 @@ async fn additional_owner_gates_reject_non_owner(pool: sqlx::PgPool) { "owner must reach {method} {path} (got 403)" ); } + + node.shutdown().await; +} + +// ── Teardown: shutdown() joins the serve task and closes the pool ───────────── + +/// `shutdown()` must join the detached serve task and close the pool before +/// returning. RED condition: with drop-only teardown the pool clones held by +/// the still-running server are open when the test future returns, so +/// `#[sqlx::test]`'s `DROP DATABASE` cleanup fails (sqlx prints and ignores +/// it), leaking per-test databases and flaking under parallel execution. The +/// `observer` clone shares the pool's inner, so `is_closed()` here is exactly +/// the state sqlx cleanup will see. +#[sqlx::test] +async fn shutdown_joins_server_and_closes_pool(pool: sqlx::PgPool) { + let observer = pool.clone(); + + let node = spawn_node(pool).await; + let base_url = node.base_url.clone(); + let client = bounded_client(); + + // One real request so at least one server connection (and its keep-alive) + // has existed; graceful shutdown must close it, not wait on it. + client + .get(format!("{base_url}/health")) + .send() + .await + .expect("probe request sends"); + + node.shutdown().await; + + assert!( + observer.is_closed(), + "shutdown() must close the pool before the test returns, or sqlx's \ + DROP DATABASE cleanup races the server's open connections" + ); + // The serve task was joined, not merely signalled: a fresh request to the + // old address must fail to connect. + let after = client.get(format!("{base_url}/health")).send().await; + assert!( + after.is_err(), + "the server must be gone after shutdown(); got {:?} instead", + after.map(|r| r.status()) + ); } From de21c646b80188e2cdedbe83da949ffe6a669255 Mon Sep 17 00:00:00 2001 From: t Date: Wed, 15 Jul 2026 19:44:21 -0500 Subject: [PATCH 14/41] test(node): let the partial-final reader wait out the fsynced publish (#194) The added temp-file and directory fsyncs can push the hard_link past the reader's old 400ms window under parallel-suite disk load, flaking publish_never_exposes_a_partial_final (observed ~1 in 5 local runs). The reader now polls a 5s deadline and stops at the first complete observation; the never-see-partial assertion still runs before the break, so the exposure mutation still fails the test. Also moves the allow-unbounded-git marker in seed_bare_repo directly above the spawn line, where the pre-push gate looks for it. --- crates/gitlawb-node/src/lib.rs | 13 +++++++++++-- crates/gitlawb-node/src/test_harness.rs | 6 +++--- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/crates/gitlawb-node/src/lib.rs b/crates/gitlawb-node/src/lib.rs index 0e8b4f81..496de798 100644 --- a/crates/gitlawb-node/src/lib.rs +++ b/crates/gitlawb-node/src/lib.rs @@ -1443,7 +1443,15 @@ mod identity_key_tests { .expect("publish"); }); - let deadline = std::time::Instant::now() + std::time::Duration::from_millis(400); + // Poll until the final path appears (bounded by a generous deadline: + // the publish now fsyncs the temp and the directory, so under parallel + // suite load the link can land well after the writer's 200ms hold). + // Every observation of an existing final must parse; the first + // complete observation ends the test, since a linked-complete final + // cannot become partial afterwards. The partial assertion still runs + // BEFORE the break, so a publish that exposes an empty/partial final + // during the write window is caught on the first poll that sees it. + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(5); let mut saw_complete = false; while std::time::Instant::now() < deadline { if key_path.exists() { @@ -1453,13 +1461,14 @@ mod identity_key_tests { "final key observed in a partial/empty state: {body:?}" ); saw_complete = true; + break; } std::thread::sleep(std::time::Duration::from_millis(1)); } writer.join().expect("writer joins"); assert!( saw_complete, - "reader should have observed the completed key within the window" + "reader should have observed the completed key within the deadline" ); } diff --git a/crates/gitlawb-node/src/test_harness.rs b/crates/gitlawb-node/src/test_harness.rs index 550551ff..ad38f92c 100644 --- a/crates/gitlawb-node/src/test_harness.rs +++ b/crates/gitlawb-node/src/test_harness.rs @@ -249,9 +249,9 @@ impl TestNode { object_format: &str, ) -> std::collections::HashMap { let run = |args: &[&str], cwd: &std::path::Path| { - // allow-unbounded-git: test-harness-only fixture seeding, feature-gated - // out of the production binary; runs local git against a tempdir inside - // a test's own lifetime, never on a request path holding a permit. + // Test-harness-only fixture seeding, feature-gated out of the production + // binary; runs local git against a tempdir inside a test's own lifetime. + // allow-unbounded-git: test-only seeding, never on a request path holding a permit let out = std::process::Command::new("git") .args(args) .current_dir(cwd) From 8df95619dbc8f1b4522dd6b28ee51424abe08865 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 19 Jul 2026 15:59:42 -0500 Subject: [PATCH 15/41] fix(node): fall back to a direct create_new publish when hard_link fails (#194) On a non-AlreadyExists hard_link error, publish the identity key by creating the final path directly with create_new(0600): no-clobber is preserved via the exclusive create (AlreadyExists routes to the lost-race load), write failures remove the partial final, and fsync failures keep the complete final on disk since a concurrent loser may already have loaded it (the Won-arm policy). The fallback chains the original link error and the rustdoc now scopes the complete-only-visibility guarantee to the hard-link path. Link step is injectable via compile-time fault hooks; six new tests drive both fallback arms, the lost race, the concurrent convergence, and both failure phases. --- crates/gitlawb-node/src/lib.rs | 381 +++++++++++++++++++++++++++++++-- 1 file changed, 363 insertions(+), 18 deletions(-) diff --git a/crates/gitlawb-node/src/lib.rs b/crates/gitlawb-node/src/lib.rs index 496de798..bc4aad5b 100644 --- a/crates/gitlawb-node/src/lib.rs +++ b/crates/gitlawb-node/src/lib.rs @@ -1151,9 +1151,37 @@ fn load_racing_key(path: &std::path::Path) -> Result { } } +/// Compile-time fault-injection seam for `publish_key_atomically`, extending +/// the `before_link` closure precedent: every hook is a no-op `Ok` in +/// production (`PublishFaults::NONE`); tests swap in a hook returning `Err` to +/// force that step to fail deterministically. Plain `fn` pointers, not a +/// runtime/env knob: production code only ever passes `NONE`. +#[derive(Clone, Copy)] +struct PublishFaults { + /// Runs before the `hard_link`; an `Err` is taken as the link result and + /// the real link is never attempted. + link: fn() -> std::io::Result<()>, + /// Runs before the fallback's `write_all`; an `Err` is taken as the write + /// result. + fallback_write: fn() -> std::io::Result<()>, + /// Runs before the fallback's file `sync_all`; an `Err` is taken as the + /// fsync result. + fallback_fsync: fn() -> std::io::Result<()>, +} + +impl PublishFaults { + /// The production value: no fault injected at any step. + const NONE: Self = Self { + link: || Ok(()), + fallback_write: || Ok(()), + fallback_fsync: || Ok(()), + }; +} + /// Publish `pem` to `final_path` atomically: write the full bytes to a sibling /// temp file, then `hard_link` the temp into place. `hard_link` is atomic and -/// fails if `final_path` already exists, which gives three guarantees at once: +/// fails if `final_path` already exists, which on the hard-link path gives +/// these guarantees at once: /// (a) the final path only ever appears with COMPLETE content — a concurrent /// reader never observes an empty/half-written key, unlike a /// `create_new`+`write_all` that exposes an empty inode before the PEM is @@ -1164,12 +1192,25 @@ fn load_racing_key(path: &std::path::Path) -> Result { /// (d) durability: the PEM bytes are fsynced before the name appears and the /// namespace change is fsynced (on Unix) before success is reported, so a /// power crash cannot leave a durable-but-truncated final key (#194). +/// +/// If the link fails for any reason other than `AlreadyExists` (a filesystem +/// without hard-link support, or a transient link error), the publish falls +/// back to `create_new(0o600)` directly on the final path +/// (`publish_key_fallback`). The fallback keeps (b): `create_new` still admits +/// exactly one winner and a lost race loads the existing key. It weakens (a) +/// and (c): the final name is visible while the PEM is being written, a window +/// concurrent readers ride out via `load_racing_key`'s wall-clock deadline, +/// and a crash mid-write leaves a partial final that fails loudly +/// (`invalid PEM key`) at the next start rather than being cleaned up. +/// /// `before_link` is a no-op in production; tests use it to widen the -/// post-write / pre-link window deterministically. +/// post-write / pre-link window deterministically. `faults` is the +/// compile-time fault-injection seam (`PublishFaults::NONE` in production). fn publish_key_atomically( final_path: &std::path::Path, pem: &[u8], before_link: &dyn Fn(), + faults: PublishFaults, ) -> Result { use std::io::Write; let dir = final_path @@ -1228,7 +1269,7 @@ fn publish_key_atomically( before_link(); - let linked = std::fs::hard_link(&tmp, final_path); + let linked = (faults.link)().and_then(|()| std::fs::hard_link(&tmp, final_path)); let _ = std::fs::remove_file(&tmp); match linked { Ok(()) => { @@ -1248,9 +1289,86 @@ fn publish_key_atomically( Ok(KeyPublish::Won) } Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => Ok(KeyPublish::Lost), - Err(e) => Err(e) - .with_context(|| format!("link identity key into place at {}", final_path.display())), + Err(e) => publish_key_fallback(final_path, pem, e, faults), + } +} + +/// Fallback publish for a key directory whose filesystem cannot hard-link +/// (Unsupported/EPERM on some network and overlay mounts, or a transient link +/// error): create the FINAL path directly with `create_new(true).mode(0o600)` +/// (the INV-23 creation pattern) and write the PEM into it. `AlreadyExists` +/// still routes to the lost-race path, so no-clobber holds. Split-phase error +/// handling, mirroring the Won arm's fsync policy in `publish_key_atomically`: +/// a failed `write_all` removes the final (partial content cannot parse and +/// would wedge every later start), but a failed file- or dir-fsync does NOT +/// (the content is complete, and a concurrent loser may already have loaded +/// it). Every error context chains `link_err` so a two-step failure is +/// diagnosable. +fn publish_key_fallback( + final_path: &std::path::Path, + pem: &[u8], + link_err: std::io::Error, + faults: PublishFaults, +) -> Result { + use std::io::Write; + warn!( + path = %final_path.display(), + err = %link_err, + "hard_link failed; publishing identity key via direct create_new fallback" + ); + let mut opts = std::fs::OpenOptions::new(); + opts.write(true).create_new(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt; + opts.mode(0o600); + } + let mut f = match opts.open(final_path) { + Ok(f) => f, + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => return Ok(KeyPublish::Lost), + Err(e) => { + return Err(e).with_context(|| { + format!( + "create identity key {} in hard_link fallback (link failed: {link_err})", + final_path.display() + ) + }) + } + }; + write_key_or_cleanup( + final_path, + (faults.fallback_write)().and_then(|()| f.write_all(pem)), + ) + .with_context(|| { + format!( + "hard_link fallback publish to {} (link failed: {link_err})", + final_path.display() + ) + })?; + (faults.fallback_fsync)() + .and_then(|()| f.sync_all()) + .with_context(|| { + format!( + "fsync identity key {} in hard_link fallback (link failed: {link_err})", + final_path.display() + ) + })?; + drop(f); + // Fsync the parent directory so the new name is durable before success is + // reported, as on the hard-link path. Same policy on failure: the final is + // complete and data-synced, so it is NOT removed. + #[cfg(unix)] + { + let dir = final_path + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or_else(|| std::path::Path::new(".")); + std::fs::File::open(dir) + .with_context(|| format!("open key directory {} to fsync", dir.display()))? + .sync_all() + .with_context(|| format!("fsync key directory {}", dir.display()))?; } + Ok(KeyPublish::Won) } fn load_or_create_keypair(config: &Config) -> Result { @@ -1273,7 +1391,7 @@ fn load_or_create_keypair(config: &Config) -> Result { // Publish atomically: the final path only ever appears complete, and a lost // race loads the winner's key rather than overwriting it. - match publish_key_atomically(&key_path, pem.as_bytes(), &|| {})? { + match publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, PublishFaults::NONE)? { KeyPublish::Won => { info!(path = %key_path.display(), did = %kp.did(), "generated new node identity"); Ok(kp) @@ -1347,7 +1465,7 @@ mod gossip_ssrf_tests { mod identity_key_tests { use super::{ load_or_create_keypair, load_racing_key, publish_key_atomically, Config, KeyPublish, - Keypair, KEY_TEMP_ATTEMPTS, + Keypair, PublishFaults, KEY_TEMP_ATTEMPTS, }; use clap::Parser; use std::os::unix::fs::PermissionsExt; @@ -1366,7 +1484,8 @@ mod identity_key_tests { let writer_path = key_path.clone(); let writer = std::thread::spawn(move || { std::thread::sleep(std::time::Duration::from_millis(250)); - publish_key_atomically(&writer_path, pem.as_bytes(), &|| {}).expect("publish"); + publish_key_atomically(&writer_path, pem.as_bytes(), &|| {}, PublishFaults::NONE) + .expect("publish"); }); let started = std::time::Instant::now(); @@ -1396,7 +1515,8 @@ mod identity_key_tests { let pem_b = Keypair::generate().to_pem().expect("pem b"); assert_ne!(pem_a.as_str(), pem_b.as_str(), "fixtures must differ"); - let out = publish_key_atomically(&key_path, pem_a.as_bytes(), &|| {}).expect("publish a"); + let out = publish_key_atomically(&key_path, pem_a.as_bytes(), &|| {}, PublishFaults::NONE) + .expect("publish a"); assert!(matches!(out, KeyPublish::Won), "first publish wins"); assert_eq!( std::fs::read_to_string(&key_path).unwrap().as_str(), @@ -1404,7 +1524,8 @@ mod identity_key_tests { "final holds the full PEM" ); - let out = publish_key_atomically(&key_path, pem_b.as_bytes(), &|| {}).expect("publish b"); + let out = publish_key_atomically(&key_path, pem_b.as_bytes(), &|| {}, PublishFaults::NONE) + .expect("publish b"); assert!(matches!(out, KeyPublish::Lost), "second publish loses"); assert_eq!( std::fs::read_to_string(&key_path).unwrap().as_str(), @@ -1437,9 +1558,14 @@ mod identity_key_tests { let wp = key_path.clone(); let writer = std::thread::spawn(move || { // Hold the post-write / pre-link window open for 200ms. - publish_key_atomically(&wp, pem.as_bytes(), &|| { - std::thread::sleep(std::time::Duration::from_millis(200)); - }) + publish_key_atomically( + &wp, + pem.as_bytes(), + &|| { + std::thread::sleep(std::time::Duration::from_millis(200)); + }, + PublishFaults::NONE, + ) .expect("publish"); }); @@ -1493,7 +1619,7 @@ mod identity_key_tests { std::fs::set_permissions(&stale, std::fs::Permissions::from_mode(0o600)) .expect("stale temp perms"); - let out = publish_key_atomically(&key_path, pem.as_bytes(), &|| {}) + let out = publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, PublishFaults::NONE) .expect("publish must recover from a stale temp, not wedge the start"); assert!(matches!(out, KeyPublish::Won), "publish wins"); assert_eq!( @@ -1521,10 +1647,11 @@ mod identity_key_tests { std::fs::write(&tmp, b"taken").expect("seed taken temp"); } - let err = match publish_key_atomically(&key_path, pem.as_bytes(), &|| {}) { - Err(e) => e, - Ok(_) => panic!("publish must fail when every temp name is taken"), - }; + let err = + match publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, PublishFaults::NONE) { + Err(e) => e, + Ok(_) => panic!("publish must fail when every temp name is taken"), + }; let msg = format!("{err:#}"); assert!( msg.contains(&format!(".identity.pem.tmp.{}", std::process::id())) @@ -1705,6 +1832,224 @@ mod identity_key_tests { "a 0600 key must be accepted" ); } + + /// Shared body for the link-failure fallback tests: with the link step + /// forced to fail with `kind`, the publish must fall back to a direct + /// `create_new(0o600)` on the final path and win; the key must be 0600 and + /// round-trip through the normal loader. RED without the fallback: the + /// link error propagates and the start fails. + fn assert_link_failure_falls_back(faults: PublishFaults) { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let kp = Keypair::generate(); + let pem = kp.to_pem().expect("pem"); + + let out = publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, faults) + .expect("a failed hard_link must fall back to a direct publish, not fail the start"); + assert!(matches!(out, KeyPublish::Won), "fallback publish wins"); + + let mode = std::fs::metadata(&key_path) + .expect("key file exists") + .permissions() + .mode(); + assert_eq!( + mode & 0o777, + 0o600, + "fallback-published key must be 0600, got {:o}", + mode & 0o777 + ); + let loaded = super::load_existing_key(&key_path) + .expect("fallback-published key loads via the normal loader"); + assert_eq!( + format!("{}", loaded.did()), + format!("{}", kp.did()), + "fallback publish must round-trip the same identity" + ); + } + + // A key directory whose filesystem rejects hard links with EPERM (some + // network/overlay mounts) must not brick a new node. + #[test] + fn permission_denied_link_falls_back_to_direct_create() { + assert_link_failure_falls_back(PublishFaults { + link: || Err(std::io::ErrorKind::PermissionDenied.into()), + ..PublishFaults::NONE + }); + } + + // A filesystem without hard-link support at all (Unsupported). + #[test] + fn unsupported_link_falls_back_to_direct_create() { + assert_link_failure_falls_back(PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + ..PublishFaults::NONE + }); + } + + // Link failure with the final ALREADY present must route to the lost-race + // path: the fallback's create_new hits AlreadyExists, the existing key is + // loaded, and it is never clobbered. + #[test] + fn link_failure_with_existing_final_loses_without_clobbering() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let winner = Keypair::generate(); + let pem_winner = winner.to_pem().expect("pem winner"); + let out = publish_key_atomically( + &key_path, + pem_winner.as_bytes(), + &|| {}, + PublishFaults::NONE, + ) + .expect("winner publishes"); + assert!(matches!(out, KeyPublish::Won), "winner publish wins"); + + let pem_loser = Keypair::generate().to_pem().expect("pem loser"); + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::PermissionDenied.into()), + ..PublishFaults::NONE + }; + let out = publish_key_atomically(&key_path, pem_loser.as_bytes(), &|| {}, faults) + .expect("an existing final must read as a lost race, not an error"); + assert!( + matches!(out, KeyPublish::Lost), + "fallback loses to the existing key" + ); + assert_eq!( + std::fs::read_to_string(&key_path).unwrap().as_str(), + pem_winner.as_str(), + "the existing key must NOT be clobbered by the fallback" + ); + let loaded = load_racing_key(&key_path).expect("lost race loads the winner"); + assert_eq!( + format!("{}", loaded.did()), + format!("{}", winner.did()), + "the lost race must return the winner's identity" + ); + } + + // Two concurrent publishers, both with the link step failing: the + // fallback's create_new must admit exactly one winner and the loser must + // converge on the winner's key (no-clobber survives the fallback path). + #[test] + fn concurrent_fallback_publishers_converge_on_one_winner() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = std::sync::Arc::new(dir.path().join("identity.pem")); + let barrier = std::sync::Arc::new(std::sync::Barrier::new(2)); + let handles: Vec<_> = (0..2) + .map(|_| { + let kp = key_path.clone(); + let b = barrier.clone(); + std::thread::spawn(move || { + let me = Keypair::generate(); + let pem = me.to_pem().expect("pem"); + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + ..PublishFaults::NONE + }; + // Release both threads at once to maximize the race. + b.wait(); + match publish_key_atomically(&kp, pem.as_bytes(), &|| {}, faults) + .expect("fallback publish") + { + KeyPublish::Won => (true, format!("{}", me.did())), + KeyPublish::Lost => ( + false, + format!( + "{}", + load_racing_key(&kp).expect("loser loads winner").did() + ), + ), + } + }) + }) + .collect(); + let results: Vec<(bool, String)> = handles + .into_iter() + .map(|h| h.join().expect("thread joins")) + .collect(); + + let winners = results.iter().filter(|(won, _)| *won).count(); + assert_eq!( + winners, 1, + "exactly one fallback create_new must win, got {results:?}" + ); + assert_eq!( + results[0].1, results[1].1, + "the loser must converge on the winner's identity: {results:?}" + ); + } + + // A failed fallback WRITE removes the final: partial content cannot parse + // and would wedge every later start on `invalid PEM key`. The error must + // surface the write failure AND the original link error, so a two-step + // failure is diagnosable. + #[test] + fn failed_fallback_write_removes_the_partial_final() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let pem = Keypair::generate().to_pem().expect("pem"); + + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + fallback_write: || Err(std::io::Error::other("injected write failure")), + ..PublishFaults::NONE + }; + let err = match publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, faults) { + Err(e) => e, + Ok(_) => panic!("a failed fallback write must error the start"), + }; + let msg = format!("{err:#}"); + assert!( + msg.contains("injected write failure"), + "the error must surface the write failure: {msg}" + ); + let link_display = std::io::Error::from(std::io::ErrorKind::Unsupported).to_string(); + assert!( + msg.contains(&link_display), + "the error must chain the original link failure ({link_display}): {msg}" + ); + assert!( + !key_path.exists(), + "a failed fallback write must remove the partial final" + ); + } + + // A failed fallback FSYNC after a complete write errors the start but + // leaves the final in place: the content is complete, and a concurrent + // loser may already have loaded it (mirrors the Won arm's fsync policy). + #[test] + fn failed_fallback_fsync_keeps_the_complete_final() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let pem = Keypair::generate().to_pem().expect("pem"); + + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + fallback_fsync: || Err(std::io::Error::other("injected fsync failure")), + ..PublishFaults::NONE + }; + let err = match publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, faults) { + Err(e) => e, + Ok(_) => panic!("a failed fallback fsync must error the start"), + }; + let msg = format!("{err:#}"); + assert!( + msg.contains("injected fsync failure"), + "the error must surface the fsync failure: {msg}" + ); + let body = std::fs::read_to_string(&key_path) + .expect("an fsync failure after a complete write must NOT remove the final"); + assert_eq!( + body.as_str(), + pem.as_str(), + "the surviving final must hold the complete PEM" + ); + assert!( + Keypair::from_pem(&body).is_ok(), + "the surviving final must be loadable at the next start" + ); + } } #[cfg(test)] From 2b37daf7aabfc5e56fc3e0d3dbda24ae70ce258d Mon Sep 17 00:00:00 2001 From: t Date: Sun, 19 Jul 2026 15:59:42 -0500 Subject: [PATCH 16/41] test(node): abort the server task in TestNode::drop (#194) A test that returns without calling shutdown() previously relied on the watch signal alone to unwind the server before sqlx's DROP DATABASE. Drop now also aborts any remaining serve handle, so teardown holds even when the graceful chain is broken; signal and abort are independently sufficient (verified by a four-quadrant neuter matrix in the new drop_without_shutdown_unblocks_database test, which observes leaked sessions via a standalone connection with sqlx pool reaping disabled). --- crates/gitlawb-node/src/test_harness.rs | 28 ++++++-- crates/gitlawb-node/tests/deny_harness.rs | 88 ++++++++++++++++++++++- 2 files changed, 108 insertions(+), 8 deletions(-) diff --git a/crates/gitlawb-node/src/test_harness.rs b/crates/gitlawb-node/src/test_harness.rs index ad38f92c..08514db6 100644 --- a/crates/gitlawb-node/src/test_harness.rs +++ b/crates/gitlawb-node/src/test_harness.rs @@ -34,8 +34,9 @@ use crate::state::AppState; /// [`TestNode::shutdown`]: it joins the serve task and closes the pool, so /// `#[sqlx::test]`'s `DROP DATABASE` cleanup (which runs the moment the test /// future returns) never races the server's still-open connections. Dropping -/// without it only signals shutdown and removes the temp repository directory, -/// a best-effort fallback for tests that panic before reaching teardown. +/// without it signals shutdown, aborts any remaining serve task, and removes +/// the temp repository directory, a fallback for tests that return or panic +/// before reaching teardown. pub struct TestNode { /// Base URL of the running node, e.g. `http://127.0.0.1:54321`. pub base_url: String, @@ -58,13 +59,26 @@ pub struct TestNode { impl Drop for TestNode { fn drop(&mut self) { - // Best-effort fallback for tests that panic before reaching + // Fallback for tests that return or panic before reaching // [`TestNode::shutdown`]: flip the shared shutdown signal so the serve - // task exits, then remove the temp repos dir. Drop cannot await, so it - // cannot join the server or close the pool; a test that already failed - // should not panic again in teardown. Both actions are idempotent, so - // running after `shutdown()` is harmless. + // task exits gracefully, abort whatever is left of it, and remove the + // temp repos dir. Drop cannot await, so it cannot join the server or + // close the pool; a test that already failed should not panic again in + // teardown. All three actions are idempotent or no-ops after + // `shutdown()` (which takes the handle), so running afterwards is + // harmless. + // + // The abort matters when the graceful chain does not run to completion + // before sqlx's `DROP DATABASE` cleanup: `JoinHandle::abort()` is a + // synchronous cancellation request callable from non-async Drop, and + // the aborted task's future (with the `PgPool` clones inside its + // router/state) is dropped on a subsequent scheduler tick, which sqlx + // cleanup's own awaits provide before the drop statement runs. The + // race is closed by those scheduling points, not by the abort alone. let _ = self.shutdown_tx.send(true); + if let Some(handle) = self.server.take() { + handle.abort(); + } let _ = std::fs::remove_dir_all(&self.repos_dir); } } diff --git a/crates/gitlawb-node/tests/deny_harness.rs b/crates/gitlawb-node/tests/deny_harness.rs index ee92a76e..db18790d 100644 --- a/crates/gitlawb-node/tests/deny_harness.rs +++ b/crates/gitlawb-node/tests/deny_harness.rs @@ -6,7 +6,8 @@ //! Each `#[sqlx::test]` gets an ephemeral per-test database; `spawn_node` runs //! the schema migrations and serves the real router on `127.0.0.1:0`. Every //! test ends with `node.shutdown().await` so the serve task is joined and the -//! pool closed before sqlx's `DROP DATABASE` cleanup runs. +//! pool closed before sqlx's `DROP DATABASE` cleanup runs (except the Drop +//! regression test at the bottom, which deliberately skips it). mod support; @@ -552,3 +553,88 @@ async fn shutdown_joins_server_and_closes_pool(pool: sqlx::PgPool) { after.map(|r| r.status()) ); } + +// ── Teardown: Drop without shutdown() must still release the database ───────── + +/// A test that returns WITHOUT calling `shutdown()` (early return, forgotten +/// teardown) must not leave the detached serve task holding pool clones open +/// against sqlx's cleanup: `#[sqlx::test]` issues a plain `DROP DATABASE` (no +/// FORCE) the moment the test future returns, a still-connected session makes +/// the drop fail, and sqlx only prints the error, silently leaking the +/// per-test database. `TestNode`'s `Drop` tears the server down two redundant +/// ways: the graceful watch signal, and an abort of the remaining serve +/// handle (which drops the task's future, and the pool clones inside its +/// router/state, on a subsequent scheduler tick without depending on the +/// graceful chain of watcher/connection tasks completing). This test fails if +/// BOTH are broken (verified by neutering them in turn: with signal and abort +/// both removed the probe sees the leaked sessions for the full bound; either +/// mechanism alone drains them by the first probe), so it fences the Drop +/// teardown contract as a whole rather than either mechanism individually. +/// +/// Two guards against a vacuous pass: the node pool is built with ambient +/// reaping disabled (`idle_timeout`/`max_lifetime` `None`; sqlx's default test +/// pool options reap idle connections after ~1s, which would release the +/// database by itself and mask a broken teardown), and the probe runs over a +/// standalone connection rather than a clone of the node pool, so observing +/// does not itself keep the pool alive. +#[sqlx::test] +async fn drop_without_shutdown_unblocks_database( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, +) { + use sqlx::{ConnectOptions as _, Connection as _}; + + let pool = pool_opts + .idle_timeout(None) + .max_lifetime(None) + .connect_with(connect_opts.clone()) + .await + .expect("node pool connects"); + + // Inner scope so the node (and the client's keep-alive connection) drop + // exactly as they would when a test body returns, with no shutdown() call. + { + let node = spawn_node(pool).await; + let client = bounded_client(); + let resp = client + .get(format!("{}/health", node.base_url)) + .send() + .await + .expect("probe request sends"); + assert!( + resp.status().is_success(), + "the server must be live before the drop" + ); + } + + // Standalone observer: count OTHER sessions connected to the per-test + // database. Zero means the only session left is the probe itself, i.e. the + // database is droppable the instant the probe disconnects, which is + // exactly the state sqlx's cleanup needs. Bounded poll: the abort takes + // effect on a scheduler tick, not synchronously in Drop. + let mut probe = connect_opts + .connect() + .await + .expect("standalone probe connects"); + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(5); + loop { + let others: i64 = sqlx::query_scalar( + "SELECT count(*) FROM pg_stat_activity \ + WHERE datname = current_database() AND pid <> pg_backend_pid()", + ) + .fetch_one(&mut probe) + .await + .expect("session count query runs"); + if others == 0 { + break; + } + assert!( + tokio::time::Instant::now() < deadline, + "dropped-without-shutdown node still holds {others} session(s) \ + against the per-test database after 5s; sqlx's DROP DATABASE \ + cleanup would fail and leak the database" + ); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + probe.close().await.expect("probe connection closes"); +} From bea3158dfb06b4b786f78bb1d53fcd0ee74074d4 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 19 Jul 2026 15:59:54 -0500 Subject: [PATCH 17/41] chore: sync Cargo.lock with the 0.5.1 workspace manifests --- Cargo.lock | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d12daa43..24beb9ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3300,7 +3300,7 @@ dependencies = [ [[package]] name = "git-remote-gitlawb" -version = "0.5.0" +version = "0.5.1" dependencies = [ "anyhow", "gitlawb-core", @@ -3312,7 +3312,7 @@ dependencies = [ [[package]] name = "gitlawb-attest" -version = "0.5.0" +version = "0.5.1" dependencies = [ "base64", "ed25519-dalek", @@ -3329,7 +3329,7 @@ dependencies = [ [[package]] name = "gitlawb-core" -version = "0.5.0" +version = "0.5.1" dependencies = [ "anyhow", "base64", @@ -3356,7 +3356,7 @@ dependencies = [ [[package]] name = "gitlawb-node" -version = "0.5.0" +version = "0.5.1" dependencies = [ "alloy", "anyhow", @@ -3412,7 +3412,7 @@ dependencies = [ [[package]] name = "gl" -version = "0.5.0" +version = "0.5.1" dependencies = [ "alloy", "anyhow", @@ -3824,6 +3824,7 @@ dependencies = [ "reqwest", "serde", "serde_json", + "sha2", "thiserror 2.0.18", "tracing", ] From 89a8d186187c1a88612e58ff476cde202f8d9d4a Mon Sep 17 00:00:00 2001 From: t Date: Sun, 19 Jul 2026 16:03:06 -0500 Subject: [PATCH 18/41] test(node): surface axum::serve's result through TestNode::shutdown (#194) The spawned serve task previously discarded its io::Result; shutdown() now joins and checks it. On axum 0.8.8 the Err arm is unreachable (graceful shutdown hard-codes Ok and accept errors retry forever), so this is forward-compat insurance documented as such inline; the Ok path is exercised by every harness test. --- crates/gitlawb-node/src/test_harness.rs | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/crates/gitlawb-node/src/test_harness.rs b/crates/gitlawb-node/src/test_harness.rs index 08514db6..0abdb8ab 100644 --- a/crates/gitlawb-node/src/test_harness.rs +++ b/crates/gitlawb-node/src/test_harness.rs @@ -50,7 +50,9 @@ pub struct TestNode { db: Arc, /// The detached `axum::serve` task, joined by [`Self::shutdown`]. `None` /// once taken, so the `Drop` fallback never overlaps a completed teardown. - server: Option>, + /// Carries `serve`'s own `io::Result<()>` so `shutdown()` can observe it + /// rather than discard it (see the coverage note there). + server: Option>>, /// A clone of the served pool, kept so [`Self::shutdown`] can close it /// (sqlx pool clones share one inner, so closing this closes the clones /// inside `AppState`/`Db` too). @@ -154,14 +156,14 @@ pub async fn spawn_node(pool: PgPool) -> TestNode { let addr = listener.local_addr().expect("read bound addr"); let server = tokio::spawn(async move { - let _ = axum::serve( + axum::serve( listener, router.into_make_service_with_connect_info::(), ) .with_graceful_shutdown(async move { let _ = shutdown_rx.changed().await; }) - .await; + .await }); TestNode { @@ -191,10 +193,20 @@ impl TestNode { pub async fn shutdown(mut self) { let _ = self.shutdown_tx.send(true); if let Some(handle) = self.server.take() { - tokio::time::timeout(Duration::from_secs(10), handle) + let result = tokio::time::timeout(Duration::from_secs(10), handle) .await .expect("serve task must exit within 10s of the shutdown signal (wedged graceful shutdown)") .expect("serve task must not panic"); + // Honest coverage contract: on axum 0.8.8, + // `WithGracefulShutdown::into_future` always resolves `Ok(())` + // (the shutdown signal is treated as success and `accept()` + // errors are retried forever rather than returned), so this + // `Err` arm is unreachable today - every existing harness test + // exercises only the `Ok` path through this line. It is + // forward-compat insurance, added at reviewer request, for a + // future axum where `serve` can actually fail: that should + // surface here as a clear panic, not be silently discarded. + result.expect("axum::serve exited with an error"); } self.pool.close().await; let _ = std::fs::remove_dir_all(&self.repos_dir); From 48ce8afb8928b9bfc7340bf0b08713fc6a19df72 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 19 Jul 2026 16:18:45 -0500 Subject: [PATCH 19/41] fix(node): verify key file mode before writing PEM bytes at both creation sites (#194) Both creation sites (the temp on the hard-link path, the final on the fallback path) now run the loader's tighten-then-verify pair right after create_new and before any private bytes are written: set_permissions repairs a umask-narrowed mode (0277 landed 0400 and previously shipped that way), and on mode-ignoring mounts where the tighten silently no-ops the verify fails closed with the empty file removed so retries are not wedged. A verified temp implies a verified hard-linked final (shared inode). Both call sites are mutation-vetted; umask cases run in child processes because umask is process-global. --- crates/gitlawb-node/src/lib.rs | 266 +++++++++++++++++++++++++++++++++ 1 file changed, 266 insertions(+) diff --git a/crates/gitlawb-node/src/lib.rs b/crates/gitlawb-node/src/lib.rs index bc4aad5b..bcc0a675 100644 --- a/crates/gitlawb-node/src/lib.rs +++ b/crates/gitlawb-node/src/lib.rs @@ -1081,6 +1081,36 @@ fn ensure_key_mode_0600(path: &std::path::Path) -> Result<()> { Ok(()) } +/// Secure a just-created, still-EMPTY key file before any PEM byte is written +/// (#194, U2): tighten to 0600, then verify with `ensure_key_mode_0600`. The +/// pair matters: `create_new`'s requested 0600 is narrowed by the process +/// umask (0277 lands 0400), so the tighten repairs that first; the verify +/// alone would falsely fail closed. On a mode-ignoring mount (vfat: chmod +/// silently no-ops returning Ok) the tighten changes nothing and the verify +/// fails CLOSED, so the private key never hits a group/other-readable file. +/// On failure the empty file is removed so a retry is not wedged on +/// `AlreadyExists`; removal is safe precisely because nothing has been +/// written yet (the split-phase rule protects only files holding PEM bytes). +/// `verify_fault` is the fault-injection hook (`Ok` in production). +#[cfg(unix)] +fn tighten_and_verify_created( + path: &std::path::Path, + verify_fault: fn() -> std::io::Result<()>, +) -> Result<()> { + use std::os::unix::fs::PermissionsExt; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) + .map_err(anyhow::Error::new) + .and_then(|()| verify_fault().map_err(anyhow::Error::new)) + .and_then(|()| ensure_key_mode_0600(path)) + .map_err(|e| { + let _ = std::fs::remove_file(path); + e.context(format!( + "could not secure just-created identity key {} to 0600", + path.display() + )) + }) +} + /// Outcome of an atomic key publish. enum KeyPublish { /// We created the key file; it now holds the complete PEM. @@ -1167,6 +1197,15 @@ struct PublishFaults { /// Runs before the fallback's file `sync_all`; an `Err` is taken as the /// fsync result. fallback_fsync: fn() -> std::io::Result<()>, + /// Runs between the tighten and the 0600 verification of the just-created + /// TEMP file; an `Err` is taken as the verification result. Simulates a + /// mode-ignoring mount (vfat: chmod no-ops returning Ok), which cannot be + /// reproduced on ext4. + #[cfg(unix)] + temp_mode_verify: fn() -> std::io::Result<()>, + /// Same, for the FINAL file created on the hard-link fallback path. + #[cfg(unix)] + fallback_mode_verify: fn() -> std::io::Result<()>, } impl PublishFaults { @@ -1175,6 +1214,10 @@ impl PublishFaults { link: || Ok(()), fallback_write: || Ok(()), fallback_fsync: || Ok(()), + #[cfg(unix)] + temp_mode_verify: || Ok(()), + #[cfg(unix)] + fallback_mode_verify: || Ok(()), }; } @@ -1260,6 +1303,12 @@ fn publish_key_atomically( dir.display() ); }; + // Secure the empty temp BEFORE the PEM bytes hit it: on a mode-ignoring + // mount this fails closed with nothing sensitive on disk, and the + // hard_link later carries the verified 0600 inode to the final name + // (#194, U2). + #[cfg(unix)] + tighten_and_verify_created(&tmp, faults.temp_mode_verify)?; // On a failed write OR fsync, remove the temp so nothing partial is ever // linked (#194). The fsync makes the PEM bytes durable BEFORE the name can // appear: without it a power crash after the link could leave a durable @@ -1335,6 +1384,15 @@ fn publish_key_fallback( }) } }; + // Secure the empty final BEFORE the PEM bytes hit it (#194, U2). The + // helper's fail-closed removal is what keeps a retry from wedging on + // AlreadyExists, and is safe here for the same reason as everywhere: the + // file is still empty, so the split-phase keep-the-final rule below does + // not apply yet. + #[cfg(unix)] + tighten_and_verify_created(final_path, faults.fallback_mode_verify).with_context(|| { + format!("secure identity key in hard_link fallback (link failed: {link_err})") + })?; write_key_or_cleanup( final_path, (faults.fallback_write)().and_then(|()| f.write_all(pem)), @@ -2050,6 +2108,214 @@ mod identity_key_tests { "the surviving final must be loadable at the next start" ); } + + /// Env var carrying `mask:site` for `umask_worker_publish_lands_0600`. + const UMASK_WORKER_ENV: &str = "GITLAWB_TEST_UMASK_CASE"; + + /// Spawns this test binary again, running only the ignored umask worker + /// with `mask:site` in the environment. The umask is process-global, so + /// setting it in the shared test process narrows every concurrent test's + /// file and tempdir creation modes (observed: a parallel test's fresh + /// tempdir landing dr-x------ and its file create failing EACCES). A + /// child process that runs nothing but the one case confines the mask. + fn run_umask_case(mask: &str, site: &str) { + let exe = std::env::current_exe().expect("test binary path"); + let out = std::process::Command::new(exe) + .args([ + "identity_key_tests::umask_worker_publish_lands_0600", + "--exact", + "--ignored", + ]) + .env(UMASK_WORKER_ENV, format!("{mask}:{site}")) + .output() + .expect("spawn umask worker"); + let stdout = String::from_utf8_lossy(&out.stdout); + // status alone is not enough: a drifted worker name would match zero + // tests and still exit 0, a vacuous green. + assert!( + out.status.success() && stdout.contains("1 passed"), + "umask {mask} {site} worker did not pass:\nstdout:\n{stdout}\nstderr:\n{}", + String::from_utf8_lossy(&out.stderr) + ); + } + + /// In-child body of the umask tests: never runs in the shared suite + /// process (ignored without the spawning tests' env var). Publishes under + /// the given umask and asserts the key lands 0600 and round-trips. The + /// mode assertion runs BEFORE the loader call, which would otherwise + /// tighten a loose key itself and mask a publish-site regression. + #[test] + #[ignore = "umask worker: spawned in a child process by the umask_* tests"] + fn umask_worker_publish_lands_0600() { + let Ok(spec) = std::env::var(UMASK_WORKER_ENV) else { + // A manual `--ignored` sweep without the env var: nothing to do. + return; + }; + let (mask_str, site) = spec.split_once(':').expect("spec is mask:site"); + let mask = libc::mode_t::from_str_radix(mask_str, 8).expect("octal mask"); + let faults = match site { + "hardlink" => PublishFaults::NONE, + "fallback" => PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + ..PublishFaults::NONE + }, + other => panic!("unknown umask worker site {other:?}"), + }; + + // The tempdir is created BEFORE the mask narrows, as a deploy's key + // directory predates the process umask; 0277 would land the dir + // itself 0400. No restore: this process exists only for this case. + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let kp = Keypair::generate(); + let pem = kp.to_pem().expect("pem"); + unsafe { libc::umask(mask) }; + + let out = publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, faults) + .expect("publish must succeed: the tighten repairs the umask-narrowed create mode"); + assert!(matches!(out, KeyPublish::Won), "publish wins"); + + let mode = std::fs::metadata(&key_path) + .expect("key file exists") + .permissions() + .mode(); + assert_eq!( + mode & 0o777, + 0o600, + "under umask {mask:o} the published key must land 0600, got {:o}", + mode & 0o777 + ); + let loaded = super::load_existing_key(&key_path).expect("loader accepts the key"); + assert_eq!( + format!("{}", loaded.did()), + format!("{}", kp.did()), + "publish under umask {mask:o} must round-trip the same identity" + ); + } + + /// #194 (U2): `create_new`'s requested 0600 is narrowed by the process + /// umask, so 0277 lands the file 0400; the tighten half of the + /// tighten-then-verify pair must repair it to 0600 (the verify alone + /// would falsely fail closed). Hard-link site. Pre-state note: before + /// U2 this publish already SUCCEEDED but left the key 0400 on disk, so + /// the mode assertion is what was RED. + #[test] + fn umask_0277_hardlink_publish_is_repaired_to_0600() { + run_umask_case("277", "hardlink"); + } + + /// Same repair, at the fallback creation site (link forced to fail). + #[test] + fn umask_0277_fallback_publish_is_repaired_to_0600() { + run_umask_case("277", "fallback"); + } + + /// #194 (U2): permissive umasks stay healthy: 0600 is already 0600 + /// under 0077 and 0000, the tighten is a no-op, and the verify must not + /// disturb a clean publish. (`created_key_is_mode_0600` pins the suite's + /// ambient umask; these pin the boundary values explicitly.) + #[test] + fn permissive_umasks_0077_and_0000_still_publish_0600() { + run_umask_case("077", "hardlink"); + run_umask_case("000", "hardlink"); + } + + /// #194 (U2): on a mode-ignoring mount (vfat: chmod silently no-ops) the + /// 0600 verification of the just-created TEMP file must fail the publish + /// CLOSED, BEFORE any PEM byte is written. Afterwards the key directory + /// must be empty: the final never appeared, and the empty temp was + /// removed so a retry is not wedged. The error must name the temp file + /// and surface the verify failure. + #[test] + fn forced_temp_mode_verify_failure_fails_closed_before_any_pem_write() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let pem = Keypair::generate().to_pem().expect("pem"); + + let faults = PublishFaults { + temp_mode_verify: || Err(std::io::Error::other("injected mode-ignoring mount")), + ..PublishFaults::NONE + }; + let err = match publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, faults) { + Err(e) => e, + Ok(_) => panic!("an unverifiable temp mode must fail the publish closed"), + }; + let msg = format!("{err:#}"); + assert!( + msg.contains("injected mode-ignoring mount") && msg.contains(".identity.pem.tmp."), + "the error must surface the verify failure and name the temp file: {msg}" + ); + assert!(!key_path.exists(), "the final key must never appear"); + let leftovers: Vec<_> = std::fs::read_dir(dir.path()) + .unwrap() + .filter_map(|e| e.ok().map(|e| e.path())) + .collect(); + assert!( + leftovers.is_empty(), + "no PEM byte may hit disk and the empty temp must be removed: {leftovers:?}" + ); + } + + /// #194 (U2): the same fail-closed check at the FALLBACK creation site + /// (link fault + verify fault). The just-created EMPTY final must be + /// removed, no PEM byte may have hit it, the error must chain the + /// original link failure, and a subsequent retry on a healthy mount must + /// publish cleanly rather than wedge on AlreadyExists. + #[test] + fn forced_fallback_mode_verify_failure_removes_the_empty_final() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let kp = Keypair::generate(); + let pem = kp.to_pem().expect("pem"); + + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + fallback_mode_verify: || Err(std::io::Error::other("injected mode-ignoring mount")), + ..PublishFaults::NONE + }; + let err = match publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, faults) { + Err(e) => e, + Ok(_) => panic!("an unverifiable final mode must fail the publish closed"), + }; + let msg = format!("{err:#}"); + assert!( + msg.contains("injected mode-ignoring mount") && msg.contains("identity.pem"), + "the error must surface the verify failure and name the final: {msg}" + ); + let link_display = std::io::Error::from(std::io::ErrorKind::Unsupported).to_string(); + assert!( + msg.contains(&link_display), + "the error must chain the original link failure ({link_display}): {msg}" + ); + assert!( + !key_path.exists(), + "the empty final must be removed so a retry is not wedged" + ); + let leftovers: Vec<_> = std::fs::read_dir(dir.path()) + .unwrap() + .filter_map(|e| e.ok().map(|e| e.path())) + .collect(); + assert!( + leftovers.is_empty(), + "nothing may remain on disk after the fail-closed publish: {leftovers:?}" + ); + + // The removal is what un-wedges the retry: after a remount (or on a + // start whose chmod works) the same path must publish cleanly. + let retry_faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + ..PublishFaults::NONE + }; + let out = publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, retry_faults) + .expect("a retry after the fail-closed publish must not be wedged"); + assert!(matches!(out, KeyPublish::Won), "retry wins cleanly"); + let loaded = super::load_existing_key(&key_path).expect("retried key loads"); + assert_eq!( + format!("{}", loaded.did()), + format!("{}", kp.did()), + "the retry must publish the intended identity" + ); + } } #[cfg(test)] From ce21a84cfa9419d86bf6583d03e3ebf45bc8ef7b Mon Sep 17 00:00:00 2001 From: t Date: Sun, 19 Jul 2026 16:25:54 -0500 Subject: [PATCH 20/41] docs(node): state the real shutdown-grace contract and lock it with a real-socket test (#194) GITLAWB_SHUTDOWN_GRACE_SECS promised a 503 to in-flight clients at expiry; drive_serve_with_grace (correctly) abandons them instead, and a retroactive 503 on an in-flight response is not constructible under hyper. The config prose now states the abandon contract, and a new test drives a real listener with a slow handler through the driver at a 250ms grace, asserting the client never sees a 503 status line and the driver returns bounded with grace_expired set. This commit changes only a doc comment and a test, so it is typed docs() rather than fix() to avoid a spurious release-please patch bump. --- crates/gitlawb-node/src/config.rs | 6 +- crates/gitlawb-node/src/lib.rs | 95 +++++++++++++++++++++++++++++++ 2 files changed, 99 insertions(+), 2 deletions(-) diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index fc2247d9..2a8f8cc1 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -162,8 +162,10 @@ pub struct Config { pub metrics_addr: String, /// Maximum time to wait for in-flight requests to drain on shutdown, in - /// seconds. After this elapses, the server returns 503 to anything still - /// in flight and exits. Default: 30s. + /// seconds. On the shutdown signal the server stops accepting new + /// connections and gives in-flight requests this long to complete; at + /// expiry it abandons whatever remains (clients observe a closed + /// connection, not an error response) and exits. Default: 30s. #[arg(long, env = "GITLAWB_SHUTDOWN_GRACE_SECS", default_value_t = 30)] pub shutdown_grace_secs: u64, diff --git a/crates/gitlawb-node/src/lib.rs b/crates/gitlawb-node/src/lib.rs index bcc0a675..7eee3bfa 100644 --- a/crates/gitlawb-node/src/lib.rs +++ b/crates/gitlawb-node/src/lib.rs @@ -2387,4 +2387,99 @@ mod shutdown_grace_tests { ); assert!(result.is_ok()); } + + // Real-socket contract: at grace expiry the server abandons a request still + // in flight; the client never receives a 503 (or any late error response), + // it just observes the connection going away or no response at all. Wires a + // real `axum::serve` future through `drive_serve_with_grace` exactly as the + // production path does. RED: asserting the client reads back an + // "HTTP/1.1 503" status line fails (no response bytes ever arrive). + #[tokio::test] + async fn real_socket_in_flight_request_abandoned_without_503() { + use axum::routing::get; + use std::sync::Arc; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + + // The slow handler signals once the request is in flight, then outlasts + // the grace by a wide margin. + let entered = Arc::new(tokio::sync::Notify::new()); + let entered_tx = entered.clone(); + let app = axum::Router::new().route( + "/slow", + get(move || { + let entered_tx = entered_tx.clone(); + async move { + entered_tx.notify_one(); + tokio::time::sleep(Duration::from_secs(10)).await; + "done" + } + }), + ); + + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + + // Mirror the production wiring: axum drains on a watch flip, and the + // graceful-shutdown closure arms the grace clock at the signal. + let (shutdown_tx, mut shutdown_rx) = tokio::sync::watch::channel(false); + let (armed_tx, armed_rx) = tokio::sync::oneshot::channel::<()>(); + let serve = axum::serve(listener, app).with_graceful_shutdown(async move { + while !*shutdown_rx.borrow_and_update() { + if shutdown_rx.changed().await.is_err() { + break; + } + } + let _ = armed_tx.send(()); + }); + + let driver = tokio::spawn(drive_serve_with_grace( + serve, + armed_rx, + Duration::from_millis(250), + )); + + // Put a request in flight on the slow route and wait until the handler + // is actually running, so the drain has something to abandon. + let mut client = tokio::net::TcpStream::connect(addr).await.unwrap(); + client + .write_all(b"GET /slow HTTP/1.1\r\nhost: test\r\nconnection: close\r\n\r\n") + .await + .unwrap(); + tokio::time::timeout(Duration::from_secs(5), entered.notified()) + .await + .expect("slow handler must start before the shutdown signal fires"); + + // Fire the shutdown signal; the in-flight request now outlasts grace. + let signal_at = Instant::now(); + shutdown_tx.send(true).unwrap(); + + let (result, grace_expired) = tokio::time::timeout(Duration::from_secs(5), driver) + .await + .expect("driver must return soon after grace expiry, not hang on the drain") + .expect("driver task must not panic"); + assert!( + grace_expired, + "an in-flight request outlasting grace must be abandoned" + ); + assert!( + result.is_ok(), + "abandon path returns Ok so teardown proceeds" + ); + assert!( + signal_at.elapsed() < Duration::from_secs(5), + "must return within a bounded window past the grace" + ); + + // The client must never see a 503 status line: the request is abandoned + // at the transport, not answered. Read whatever arrives before a bounded + // window; a clean close (EOF) and no bytes at all both satisfy the + // contract. + let mut buf = Vec::new(); + let _ = tokio::time::timeout(Duration::from_secs(2), client.read_to_end(&mut buf)).await; + assert!( + !buf.starts_with(b"HTTP/1.1 503"), + "abandoned in-flight request must not be answered with a 503; got: {:?}", + String::from_utf8_lossy(&buf) + ); + } } From 09bfe36e45117a8a0b116546d86ec85fad8a6425 Mon Sep 17 00:00:00 2001 From: t Date: Sun, 19 Jul 2026 16:52:59 -0500 Subject: [PATCH 21/41] test(node): close review gaps on the key-publish fallback and Drop abort (#194) Round-five review follow-ups, no production behavior change beyond a pure fsync_parent_dir extraction: - Recondition the KEY_RACE_DEADLINE and publish_key_atomically rustdocs: a present key is complete-on-first-read only on the hard-link path; on the fallback the final name is visible mid-write (the deadline covers it) and the fallback also weakens durability (d), failing loud at next start, never a silently used key. - Cover the fallback create_new open-error arm via a new fault hook, asserting the returned error chains the original link error. - Make the Drop abort load-bearing (INV-21): a test-only flag suppresses the graceful signal so handle.abort() carries teardown alone; removing the abort turns the new test RED while the existing one stays green. - Observe the crash-residue contract: empty/garbage finals fail loudly. - Extract fsync_parent_dir, shared by the Won and fallback arms. - Split the permissive-umask test so a 0000 regression can't hide behind 0077. --- crates/gitlawb-node/src/lib.rs | 182 ++++++++++++++++++---- crates/gitlawb-node/src/test_harness.rs | 23 ++- crates/gitlawb-node/tests/deny_harness.rs | 72 +++++++++ 3 files changed, 244 insertions(+), 33 deletions(-) diff --git a/crates/gitlawb-node/src/lib.rs b/crates/gitlawb-node/src/lib.rs index 7eee3bfa..585252d8 100644 --- a/crates/gitlawb-node/src/lib.rs +++ b/crates/gitlawb-node/src/lib.rs @@ -1111,6 +1111,23 @@ fn tighten_and_verify_created( }) } +/// Fsync the parent directory of `final_path` so the namespace change (a link +/// or a create into it) is durable before success is reported (#194). Unix-only: +/// opening a directory as a file is not portable. An empty or missing parent +/// resolves to `"."`. +#[cfg(unix)] +fn fsync_parent_dir(final_path: &std::path::Path) -> anyhow::Result<()> { + let dir = final_path + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or_else(|| std::path::Path::new(".")); + std::fs::File::open(dir) + .with_context(|| format!("open key directory {} to fsync", dir.display()))? + .sync_all() + .with_context(|| format!("fsync key directory {}", dir.display()))?; + Ok(()) +} + /// Outcome of an atomic key publish. enum KeyPublish { /// We created the key file; it now holds the complete PEM. @@ -1120,11 +1137,15 @@ enum KeyPublish { } /// Wall-clock budget for reading a key another start may still be publishing. -/// The atomic publish (`publish_key_atomically`) means a *present* key file is -/// always complete, so this normally succeeds on the first read; the deadline -/// only covers cross-host cache lag. It is a wall-clock budget, NOT a fixed -/// retry count, so a slow/stalled filesystem cannot starve a losing start after -/// an arbitrarily short (~100ms) window (#194). +/// On the atomic hard-link path (`publish_key_atomically`) a *present* key file +/// is always complete, so a read there succeeds on the first try. On the +/// hard-link fallback (`publish_key_fallback`) the final NAME becomes visible +/// while the PEM is still being written, so a losing/concurrent reader can +/// observe an empty/partial final; this deadline is the window it rides out +/// until the winner's write completes. Cross-host cache lag is a secondary +/// reason a present key may not read on the first attempt. It is a wall-clock +/// budget, NOT a fixed retry count, so a slow/stalled filesystem cannot starve a +/// losing start after an arbitrarily short (~100ms) window (#194). const KEY_RACE_DEADLINE: std::time::Duration = std::time::Duration::from_secs(5); /// Bounded number of temp-file names a publish will try before giving up. @@ -1191,6 +1212,10 @@ struct PublishFaults { /// Runs before the `hard_link`; an `Err` is taken as the link result and /// the real link is never attempted. link: fn() -> std::io::Result<()>, + /// Runs before the fallback's `create_new` open of the FINAL path; an `Err` + /// is taken as the open result (a non-`AlreadyExists` open error), so the + /// publish surfaces it chained with `link_err`. + fallback_create: fn() -> std::io::Result<()>, /// Runs before the fallback's `write_all`; an `Err` is taken as the write /// result. fallback_write: fn() -> std::io::Result<()>, @@ -1212,6 +1237,7 @@ impl PublishFaults { /// The production value: no fault injected at any step. const NONE: Self = Self { link: || Ok(()), + fallback_create: || Ok(()), fallback_write: || Ok(()), fallback_fsync: || Ok(()), #[cfg(unix)] @@ -1240,11 +1266,15 @@ impl PublishFaults { /// without hard-link support, or a transient link error), the publish falls /// back to `create_new(0o600)` directly on the final path /// (`publish_key_fallback`). The fallback keeps (b): `create_new` still admits -/// exactly one winner and a lost race loads the existing key. It weakens (a) -/// and (c): the final name is visible while the PEM is being written, a window -/// concurrent readers ride out via `load_racing_key`'s wall-clock deadline, -/// and a crash mid-write leaves a partial final that fails loudly -/// (`invalid PEM key`) at the next start rather than being cleaned up. +/// exactly one winner and a lost race loads the existing key. It weakens (a), +/// (c), and (d): the final name is visible while the PEM is being written, a +/// window concurrent readers ride out via `load_racing_key`'s wall-clock +/// deadline; a crash mid-write leaves a partial final that fails loudly +/// (`invalid PEM key`) at the next start rather than being cleaned up; and the +/// final name can become durable before the PEM bytes are fsynced, so a power +/// loss can additionally leave a durable empty/truncated final. In every case +/// the failure mode is the same LOUD invalid-PEM (or mode) error at the next +/// start, never a silently used key. /// /// `before_link` is a no-op in production; tests use it to widen the /// post-write / pre-link window deterministically. `faults` is the @@ -1329,12 +1359,7 @@ fn publish_key_atomically( // loser may already have loaded it, and a later start loads it via // the exists() path. #[cfg(unix)] - { - std::fs::File::open(dir) - .with_context(|| format!("open key directory {} to fsync", dir.display()))? - .sync_all() - .with_context(|| format!("fsync key directory {}", dir.display()))?; - } + fsync_parent_dir(final_path)?; Ok(KeyPublish::Won) } Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => Ok(KeyPublish::Lost), @@ -1372,7 +1397,7 @@ fn publish_key_fallback( use std::os::unix::fs::OpenOptionsExt; opts.mode(0o600); } - let mut f = match opts.open(final_path) { + let mut f = match (faults.fallback_create)().and_then(|()| opts.open(final_path)) { Ok(f) => f, Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => return Ok(KeyPublish::Lost), Err(e) => { @@ -1416,16 +1441,7 @@ fn publish_key_fallback( // reported, as on the hard-link path. Same policy on failure: the final is // complete and data-synced, so it is NOT removed. #[cfg(unix)] - { - let dir = final_path - .parent() - .filter(|p| !p.as_os_str().is_empty()) - .unwrap_or_else(|| std::path::Path::new(".")); - std::fs::File::open(dir) - .with_context(|| format!("open key directory {} to fsync", dir.display()))? - .sync_all() - .with_context(|| format!("fsync key directory {}", dir.display()))?; - } + fsync_parent_dir(final_path)?; Ok(KeyPublish::Won) } @@ -2073,6 +2089,42 @@ mod identity_key_tests { ); } + // A non-AlreadyExists error from the fallback's create_new open of the + // final path must surface, chained with the original link error, so a + // two-step failure (link failed, then the direct create also failed) is + // diagnosable. RED without the gate wired: the injected open error never + // reaches the real open, so the publish succeeds instead of erroring. + #[test] + fn failed_fallback_create_surfaces_the_open_error_and_chains_link_err() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let pem = Keypair::generate().to_pem().expect("pem"); + + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + fallback_create: || Err(std::io::Error::other("injected fallback create failure")), + ..PublishFaults::NONE + }; + let err = match publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, faults) { + Err(e) => e, + Ok(_) => panic!("a failed fallback create must error the start"), + }; + let msg = format!("{err:#}"); + assert!( + msg.contains("injected fallback create failure"), + "the error must surface the fallback create failure: {msg}" + ); + let link_display = std::io::Error::from(std::io::ErrorKind::Unsupported).to_string(); + assert!( + msg.contains(&link_display), + "the error must chain the original link failure ({link_display}): {msg}" + ); + assert!( + !key_path.exists(), + "a failed fallback create must not leave the final key" + ); + } + // A failed fallback FSYNC after a complete write errors the start but // leaves the final in place: the content is complete, and a concurrent // loser may already have loaded it (mirrors the Won arm's fsync policy). @@ -2109,6 +2161,64 @@ mod identity_key_tests { ); } + // #194: the fallback path can leave a durable empty/truncated final if a + // crash lands between the name becoming durable and the PEM fsync (the + // reconditioned KEY_RACE_DEADLINE / publish_key_atomically doc). This + // observes the promised behavior at the next start: such a residue must + // fail LOUDLY (an invalid-PEM parse error, or a mode rejection), never + // parse as Ok and never be replaced by a freshly generated key. This is a + // characterization/observation test of behavior that already exists, not a + // RED-first guard: it passes on current code. + #[test] + fn fallback_crash_residue_fails_loudly_not_silently() { + use super::{ensure_key_mode_0600, load_existing_key}; + let dir = tempfile::tempdir().expect("tempdir"); + + // (1) an empty final at 0600 (durable name, no PEM bytes yet). + let empty = dir.path().join("empty.pem"); + std::fs::write(&empty, b"").expect("seed empty final"); + std::fs::set_permissions(&empty, std::fs::Permissions::from_mode(0o600)) + .expect("0600 empty"); + let err = match load_existing_key(&empty) { + Err(e) => e.to_string(), + Ok(_) => panic!("an empty final must fail loudly, not parse as Ok"), + }; + assert!( + err.contains("invalid PEM"), + "an empty final must surface a parse error, got: {err}" + ); + + // (2) a truncated/garbage non-PEM final at 0600. + let garbage = dir.path().join("garbage.pem"); + std::fs::write(&garbage, b"-----BEGIN nonsense truncated").expect("seed garbage final"); + std::fs::set_permissions(&garbage, std::fs::Permissions::from_mode(0o600)) + .expect("0600 garbage"); + let err = match load_existing_key(&garbage) { + Err(e) => e.to_string(), + Ok(_) => panic!("a truncated final must fail loudly, not parse as Ok"), + }; + assert!( + err.contains("invalid PEM"), + "a truncated final must surface a parse error, got: {err}" + ); + + // (3) mode-rejection path: an empty final at 0400. The loader tightens a + // loose mode where the mount allows it, so the reachable rejection is + // `ensure_key_mode_0600` failing closed on a mode it cannot narrow (a + // mode-ignoring mount); assert that branch directly on a 0400 file. + let loose = dir.path().join("loose.pem"); + std::fs::write(&loose, b"").expect("seed loose final"); + std::fs::set_permissions(&loose, std::fs::Permissions::from_mode(0o400)) + .expect("0400 loose"); + let err = ensure_key_mode_0600(&loose) + .expect_err("a non-0600 final must be rejected, not used exposed") + .to_string(); + assert!( + err.contains("400"), + "the mode rejection must name the exposed mode, got: {err}" + ); + } + /// Env var carrying `mask:site` for `umask_worker_publish_lands_0600`. const UMASK_WORKER_ENV: &str = "GITLAWB_TEST_UMASK_CASE"; @@ -2210,13 +2320,21 @@ mod identity_key_tests { run_umask_case("277", "fallback"); } - /// #194 (U2): permissive umasks stay healthy: 0600 is already 0600 - /// under 0077 and 0000, the tighten is a no-op, and the verify must not - /// disturb a clean publish. (`created_key_is_mode_0600` pins the suite's - /// ambient umask; these pin the boundary values explicitly.) + /// #194 (U2): a permissive umask stays healthy: 0600 is already 0600 under + /// 0077, the tighten is a no-op, and the verify must not disturb a clean + /// publish. (`created_key_is_mode_0600` pins the suite's ambient umask; + /// these pin the boundary values explicitly.) Split per-umask so a + /// 0000-specific regression is not masked by a 0077 failure. #[test] - fn permissive_umasks_0077_and_0000_still_publish_0600() { + fn permissive_umask_0077_still_publishes_0600() { run_umask_case("077", "hardlink"); + } + + /// #194 (U2): the fully permissive umask 0000 stays healthy too — 0600 is + /// already 0600, the tighten is a no-op, and the verify must not disturb a + /// clean publish. + #[test] + fn permissive_umask_0000_still_publishes_0600() { run_umask_case("000", "hardlink"); } diff --git a/crates/gitlawb-node/src/test_harness.rs b/crates/gitlawb-node/src/test_harness.rs index 0abdb8ab..654285a7 100644 --- a/crates/gitlawb-node/src/test_harness.rs +++ b/crates/gitlawb-node/src/test_harness.rs @@ -57,6 +57,12 @@ pub struct TestNode { /// (sqlx pool clones share one inner, so closing this closes the clones /// inside `AppState`/`Db` too). pool: PgPool, + /// Test-only: when set, `Drop` skips the graceful watch signal so ONLY + /// `handle.abort()` tears the serve task down. Defaults false, so every + /// other test keeps both redundant teardown mechanisms. Set via + /// [`Self::force_abort_only_teardown`] to fence that the abort alone + /// releases the database when the graceful chain is broken. + suppress_graceful_shutdown_on_drop: bool, } impl Drop for TestNode { @@ -77,7 +83,13 @@ impl Drop for TestNode { // router/state) is dropped on a subsequent scheduler tick, which sqlx // cleanup's own awaits provide before the drop statement runs. The // race is closed by those scheduling points, not by the abort alone. - let _ = self.shutdown_tx.send(true); + // + // `suppress_graceful_shutdown_on_drop` (test-only) forces the abort to + // carry teardown by itself, so a test can fence that the abort line is + // load-bearing; it is false everywhere else. + if !self.suppress_graceful_shutdown_on_drop { + let _ = self.shutdown_tx.send(true); + } if let Some(handle) = self.server.take() { handle.abort(); } @@ -174,10 +186,19 @@ pub async fn spawn_node(pool: PgPool) -> TestNode { db, server: Some(server), pool, + suppress_graceful_shutdown_on_drop: false, } } impl TestNode { + /// Test-only: make `Drop` tear the serve task down through `handle.abort()` + /// alone by suppressing the graceful watch signal. Used to prove the abort + /// is load-bearing (that the database is released even when the graceful + /// chain is broken); no production or normal-teardown path calls this. + pub fn force_abort_only_teardown(&mut self) { + self.suppress_graceful_shutdown_on_drop = true; + } + /// Graceful async teardown: signal shutdown, join the serve task, close /// the pool, remove the temp repos dir. Call this at the end of every /// test. `#[sqlx::test]` issues `DROP DATABASE` as soon as the test diff --git a/crates/gitlawb-node/tests/deny_harness.rs b/crates/gitlawb-node/tests/deny_harness.rs index db18790d..6fdf9efe 100644 --- a/crates/gitlawb-node/tests/deny_harness.rs +++ b/crates/gitlawb-node/tests/deny_harness.rs @@ -638,3 +638,75 @@ async fn drop_without_shutdown_unblocks_database( } probe.close().await.expect("probe connection closes"); } + +/// Isolates the `handle.abort()` half of `TestNode`'s `Drop` teardown. The +/// existing `drop_without_shutdown_unblocks_database` fences both redundant +/// mechanisms together and stays GREEN if only the abort is removed (the +/// graceful watch signal alone drains the serve task). This test suppresses the +/// graceful signal (`force_abort_only_teardown`) so ONLY the abort runs, making +/// the abort line load-bearing: with the abort removed from `Drop` neither +/// mechanism runs, the serve task keeps its pool clones open, and the probe sees +/// the leaked sessions for the full bound (RED); with the abort present the +/// aborted task's future is dropped on a scheduler tick and the sessions drain +/// (GREEN). Same vacuous-pass guards as the sibling test: node pool built with +/// reaping disabled, and a standalone observer connection. +#[sqlx::test] +async fn drop_with_broken_graceful_chain_still_unblocks_via_abort( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, +) { + use sqlx::{ConnectOptions as _, Connection as _}; + + let pool = pool_opts + .idle_timeout(None) + .max_lifetime(None) + .connect_with(connect_opts.clone()) + .await + .expect("node pool connects"); + + // Inner scope so the node drops as a test body's would, with no shutdown(). + { + let mut node = spawn_node(pool).await; + // Break the graceful chain: Drop will not send the watch signal, so + // only handle.abort() can tear the serve task down. + node.force_abort_only_teardown(); + let client = bounded_client(); + let resp = client + .get(format!("{}/health", node.base_url)) + .send() + .await + .expect("probe request sends"); + assert!( + resp.status().is_success(), + "the server must be live before the drop" + ); + } + + // Standalone observer: with only the abort running, the serve task's future + // (and its pool clones) must still be dropped, leaving zero other sessions. + let mut probe = connect_opts + .connect() + .await + .expect("standalone probe connects"); + let deadline = tokio::time::Instant::now() + std::time::Duration::from_secs(5); + loop { + let others: i64 = sqlx::query_scalar( + "SELECT count(*) FROM pg_stat_activity \ + WHERE datname = current_database() AND pid <> pg_backend_pid()", + ) + .fetch_one(&mut probe) + .await + .expect("session count query runs"); + if others == 0 { + break; + } + assert!( + tokio::time::Instant::now() < deadline, + "abort-only teardown still holds {others} session(s) against the \ + per-test database after 5s; handle.abort() is not releasing the \ + serve task's pool clones and sqlx's DROP DATABASE cleanup would fail" + ); + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + probe.close().await.expect("probe connection closes"); +} From 33ea63bc29add4c6a904428efc57515e53bfd865 Mon Sep 17 00:00:00 2001 From: t Date: Mon, 20 Jul 2026 15:37:44 -0500 Subject: [PATCH 22/41] fix(node): recover identity-key fallback fsync failure and abort wedged serve task F1: the hard-link fallback wrote the PEM directly to the final path then fsync'd it separately; an fsync failure left a durable name over non-durable content, so a later crash wedged startup (load_or_create_keypair parses the existing truncated file instead of regenerating). Fold the file fsync into write_key_or_cleanup so the final is removed on fsync failure, matching the hard-link temp policy; the next start regenerates. Parent-dir fsync still keeps-on-failure. F2: TestNode::shutdown moved the serve JoinHandle into tokio::time::timeout and, on elapse, the panicking .expect dropped it, detaching the still-running task and leaking the test DB across DROP DATABASE. Retain the handle (&mut), abort and reap it on timeout before panicking. Also: honor a task that finishes in the timeout race as graceful (not a false wedge), propagate a real serve-task panic, and close the pool on every path (including the wedged arm) before returning or panicking so the leak is closed on the path that needs it most. --- crates/gitlawb-node/src/lib.rs | 68 ++++++----- crates/gitlawb-node/src/test_harness.rs | 146 +++++++++++++++++++++--- 2 files changed, 169 insertions(+), 45 deletions(-) diff --git a/crates/gitlawb-node/src/lib.rs b/crates/gitlawb-node/src/lib.rs index 585252d8..7d9395bc 100644 --- a/crates/gitlawb-node/src/lib.rs +++ b/crates/gitlawb-node/src/lib.rs @@ -1371,13 +1371,14 @@ fn publish_key_atomically( /// (Unsupported/EPERM on some network and overlay mounts, or a transient link /// error): create the FINAL path directly with `create_new(true).mode(0o600)` /// (the INV-23 creation pattern) and write the PEM into it. `AlreadyExists` -/// still routes to the lost-race path, so no-clobber holds. Split-phase error -/// handling, mirroring the Won arm's fsync policy in `publish_key_atomically`: -/// a failed `write_all` removes the final (partial content cannot parse and -/// would wedge every later start), but a failed file- or dir-fsync does NOT -/// (the content is complete, and a concurrent loser may already have loaded -/// it). Every error context chains `link_err` so a two-step failure is -/// diagnosable. +/// still routes to the lost-race path, so no-clobber holds. Error handling: a +/// failed `write_all` OR file `sync_all` removes the final (#194 F1) — the +/// `create_new` already made the final NAME durable, so a name left over +/// non-durable content lets a later crash leave a truncated final that wedges +/// every later start; removal lets the next start regenerate. Only a failed +/// parent-DIR fsync keeps the final (the content is complete and data-synced, +/// and a lost directory entry just disappears -> clean regen, no wedge). Every +/// error context chains `link_err` so a two-step failure is diagnosable. fn publish_key_fallback( final_path: &std::path::Path, pem: &[u8], @@ -1418,6 +1419,8 @@ fn publish_key_fallback( tighten_and_verify_created(final_path, faults.fallback_mode_verify).with_context(|| { format!("secure identity key in hard_link fallback (link failed: {link_err})") })?; + // A failed write removes the final (partial content cannot parse and would + // wedge every later start). write_key_or_cleanup( final_path, (faults.fallback_write)().and_then(|()| f.write_all(pem)), @@ -1428,14 +1431,25 @@ fn publish_key_fallback( final_path.display() ) })?; - (faults.fallback_fsync)() - .and_then(|()| f.sync_all()) - .with_context(|| { - format!( - "fsync identity key {} in hard_link fallback (link failed: {link_err})", - final_path.display() - ) - })?; + // Fsync the file, and REMOVE the final on failure too (#194 F1). Unlike the + // hard-link path (which fsyncs a TEMP, so the final NAME only ever appears + // over a durable inode), the fallback's `create_new` already made the final + // name durable, so a bytes-accepted-but-not-durable final would let a later + // crash leave a truncated file that `load_or_create_keypair` parses forever + // via the existing-file path instead of regenerating -> permanent startup + // wedge. Removing it mirrors the hard-link temp policy and lets the next + // start regenerate. A DISTINCT context is kept so an operator debugging + // ENOSPC/EIO can tell "durability failed" (bytes accepted, not synced) from + // the write-rejected case above. Only the parent-DIR fsync below keeps the + // final on failure (a lost directory entry just disappears -> clean regen). + if let Err(e) = (faults.fallback_fsync)().and_then(|()| f.sync_all()) { + let _ = std::fs::remove_file(final_path); + return Err(anyhow::Error::new(e).context(format!( + "fsync identity key {} in hard_link fallback (durability failed, \ + removed; link failed: {link_err})", + final_path.display() + ))); + } drop(f); // Fsync the parent directory so the new name is durable before success is // reported, as on the hard-link path. Same policy on failure: the final is @@ -2125,11 +2139,15 @@ mod identity_key_tests { ); } - // A failed fallback FSYNC after a complete write errors the start but - // leaves the final in place: the content is complete, and a concurrent - // loser may already have loaded it (mirrors the Won arm's fsync policy). + // A failed fallback FILE fsync must REMOVE the final: create_new has already + // made the final NAME durable, so a name left over non-durable content lets a + // later crash leave a truncated final that load_or_create_keypair parses + // forever (existing-file path) instead of regenerating -> permanent startup + // wedge (#194 F1, jatmn). The fallback now mirrors the hard-link temp policy + // (write_key_or_cleanup removes on write OR fsync failure); removal makes the + // next start regenerate cleanly. Only the parent-DIR fsync keeps on failure. #[test] - fn failed_fallback_fsync_keeps_the_complete_final() { + fn failed_fallback_fsync_removes_the_unsynced_final() { let dir = tempfile::tempdir().expect("tempdir"); let key_path = dir.path().join("identity.pem"); let pem = Keypair::generate().to_pem().expect("pem"); @@ -2148,16 +2166,10 @@ mod identity_key_tests { msg.contains("injected fsync failure"), "the error must surface the fsync failure: {msg}" ); - let body = std::fs::read_to_string(&key_path) - .expect("an fsync failure after a complete write must NOT remove the final"); - assert_eq!( - body.as_str(), - pem.as_str(), - "the surviving final must hold the complete PEM" - ); assert!( - Keypair::from_pem(&body).is_ok(), - "the surviving final must be loadable at the next start" + !key_path.exists(), + "an fsync failure on the fallback must remove the un-synced final so \ + the next start regenerates instead of wedging on a truncated file" ); } diff --git a/crates/gitlawb-node/src/test_harness.rs b/crates/gitlawb-node/src/test_harness.rs index 654285a7..86a19700 100644 --- a/crates/gitlawb-node/src/test_harness.rs +++ b/crates/gitlawb-node/src/test_harness.rs @@ -97,6 +97,47 @@ impl Drop for TestNode { } } +/// Outcome of joining the serve task during teardown, split so the wedged-task +/// path is observable to a test (which injects a short timeout) without waiting +/// the production teardown window. +enum ServeTeardown { + /// The task returned (carrying `serve`'s `io::Result<()>`) — either within + /// the timeout, or in the race between the timeout firing and the abort + /// landing. A task that actually finished is never a wedge. + Graceful(std::io::Result<()>), + /// The task did not finish within the timeout and was aborted (cancelled) + /// and reaped. + TimedOutAborted, +} + +/// Join `handle` within `timeout`; on expiry `abort()` it and await the aborted +/// handle so the serve task (and the `PgPool` clones inside its router/state) is +/// reaped BEFORE the caller returns and `#[sqlx::test]` runs `DROP DATABASE`. +/// Takes `&mut` (a `JoinHandle` is `Unpin`) so ownership is retained across the +/// timeout and the handle can be aborted on expiry — the previous code moved the +/// handle into `timeout` and, on elapse, dropped it, which DETACHES the still +/// running task rather than aborting it, leaking the test database (#194 F2). +async fn join_or_abort( + handle: &mut tokio::task::JoinHandle>, + timeout: Duration, +) -> ServeTeardown { + match tokio::time::timeout(timeout, &mut *handle).await { + Ok(join_result) => ServeTeardown::Graceful(join_result.expect("serve task must not panic")), + Err(_elapsed) => { + handle.abort(); + match handle.await { + // The task finished in the race between the timeout firing and + // the abort landing — honor its result, not a false wedge. + Ok(result) => ServeTeardown::Graceful(result), + Err(e) if e.is_cancelled() => ServeTeardown::TimedOutAborted, + // A genuine panic in the serve task surfaces here, mirroring the + // in-time arm's `.expect("serve task must not panic")`. + Err(e) => std::panic::resume_unwind(e.into_panic()), + } + } + } +} + /// Allocate a process-unique temp directory for a spawned node's repositories /// without pulling in the `tempfile` dev-dependency (which is unavailable to /// the library crate under `--features test-harness`). @@ -211,26 +252,49 @@ impl TestNode { /// When `self` drops on return, `Drop` re-sends the signal and re-removes /// the dir; both are idempotent, and the serve handle was already taken, /// so the overlap is harmless. - pub async fn shutdown(mut self) { + pub async fn shutdown(self) { + self.shutdown_with_timeout(Duration::from_secs(10)).await; + } + + /// Body of [`Self::shutdown`] with an injectable teardown timeout so a test + /// can drive the wedged-shutdown path without waiting the production 10s. + /// Production always calls this with 10s. + async fn shutdown_with_timeout(mut self, timeout: Duration) { let _ = self.shutdown_tx.send(true); - if let Some(handle) = self.server.take() { - let result = tokio::time::timeout(Duration::from_secs(10), handle) - .await - .expect("serve task must exit within 10s of the shutdown signal (wedged graceful shutdown)") - .expect("serve task must not panic"); - // Honest coverage contract: on axum 0.8.8, - // `WithGracefulShutdown::into_future` always resolves `Ok(())` - // (the shutdown signal is treated as success and `accept()` - // errors are retried forever rather than returned), so this - // `Err` arm is unreachable today - every existing harness test - // exercises only the `Ok` path through this line. It is - // forward-compat insurance, added at reviewer request, for a - // future axum where `serve` can actually fail: that should - // surface here as a clear panic, not be silently discarded. - result.expect("axum::serve exited with an error"); - } + let outcome = match self.server.take() { + Some(mut handle) => Some(join_or_abort(&mut handle, timeout).await), + None => None, + }; + // Release the pool and temp dir on EVERY path before returning OR + // panicking. A wedged graceful shutdown is exactly when in-flight + // per-connection tasks still hold `PgPool` clones; aborting the serve + // future alone does not reliably drop them, so the explicit + // `pool.close()` (which closes the shared inner) is what frees them + // before `#[sqlx::test]` runs `DROP DATABASE`. Panicking before this + // (the previous behavior) skipped the close on the one path that needs + // it most (#194 F2). self.pool.close().await; let _ = std::fs::remove_dir_all(&self.repos_dir); + match outcome { + // Honest coverage contract: on axum 0.8.8, + // `WithGracefulShutdown::into_future` always resolves `Ok(())` (the + // shutdown signal is treated as success and `accept()` errors are + // retried forever rather than returned), so this `Err` arm is + // unreachable today - every existing harness test exercises only the + // `Ok` path here. Forward-compat insurance for a future axum where + // `serve` can fail: that should surface as a clear panic, not be + // silently discarded. + Some(ServeTeardown::Graceful(result)) => { + result.expect("axum::serve exited with an error"); + } + Some(ServeTeardown::TimedOutAborted) => { + panic!( + "serve task did not exit within {timeout:?} of the shutdown \ + signal (wedged graceful shutdown)" + ); + } + None => {} + } } /// Insert a repo owned by `owner_did` and return its repo id. Mirrors @@ -354,3 +418,51 @@ impl TestNode { oids } } + +#[cfg(test)] +mod tests { + use super::{join_or_abort, ServeTeardown}; + use std::time::Duration; + + /// A wedged serve task (never returns within the window) must be ABORTED and + /// reaped on timeout, not detached. The previous teardown moved the + /// `JoinHandle` into `timeout` and dropped it on elapse, which detaches the + /// still-running task and leaks the test DB across `DROP DATABASE` (#194 F2). + /// Load-bearing: neutralize the abort in `join_or_abort` (return without + /// abort+await) and this flips RED — `handle.is_finished()` stays false + /// because the task was left running. + #[tokio::test] + async fn join_or_abort_aborts_a_wedged_task_on_timeout() { + let mut handle: tokio::task::JoinHandle> = tokio::spawn(async { + loop { + tokio::time::sleep(Duration::from_secs(3600)).await; + } + }); + assert!( + matches!( + join_or_abort(&mut handle, Duration::from_millis(50)).await, + ServeTeardown::TimedOutAborted + ), + "a task that never returns must be reported timed-out, not graceful" + ); + assert!( + handle.is_finished(), + "the wedged serve task must be aborted and reaped, not detached" + ); + } + + /// A task that returns within the window is reported graceful, carrying its + /// `io::Result` — the normal teardown path. + #[tokio::test] + async fn join_or_abort_reports_graceful_when_task_exits_in_time() { + let mut handle: tokio::task::JoinHandle> = + tokio::spawn(async { Ok(()) }); + assert!( + matches!( + join_or_abort(&mut handle, Duration::from_secs(5)).await, + ServeTeardown::Graceful(Ok(())) + ), + "a prompt clean exit must be reported graceful" + ); + } +} From 88616b2aac231313a6b3917da37c953b191d6df2 Mon Sep 17 00:00:00 2001 From: beardthelion <56458543+beardthelion@users.noreply.github.com> Date: Tue, 21 Jul 2026 19:30:38 -0500 Subject: [PATCH 23/41] test(node): invariant deny-prober (driven registry + completeness cross-check) (#195) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * test(node): U1 deny-bearing route registry (owner-gate + signature tranches) Foundation for the invariant deny-prober (plan 002): a declarative registry of ONLY the routes that carry a runtime deny (owner-gate 403 / read-gate 404 / signature 401), each classified by reading its handler and recording the fn name. Owner-gate (12 rows: merge/close_pr, close_issue, webhooks, labels, protect, visibility set/remove/list incl the GET-is-403 correction) and signature-required (git-receive-pack) tranches are verified; read-gate tranche stubbed with a TODO (per-handler 404-vs-filter verification pending). A consistency self-test asserts no dup method+path, every row names its handler, and every read-gate row carries a positive-twin Reach. Decoupled from the #194 primitives (U1 uses none), so classification is not hostage to harness churn. * test(node): U2 fixture state matrix + per-gate-class probe generators Two-repo fixture (public owner-gate substrate + private read-gate substrate) plus probes_for(): each deny-bearing row expands to a hostile probe asserting the exact deny status and a positive twin (owner-reachability for owner-gate, authorized read for read-gate) so a deny for the wrong reason cannot false-pass. Three self-checks cover the owner-gate, read-gate, and signature classes. * test(node): U1 read-gate tranche (14 repo-scoped 404-deny reads) Each row's handler was read to confirm it gates on "/" via authorize_repo_read / visibility_check and returns RepoNotFound (404) to a non-reader of a private repo, with the owner re-read as the ReaderReads positive twin. Sub-entity gets (get_issue/get_pr/get_cert/get_bounty and the comment/review lists) are deferred to the fixture expansion that seeds them; global list-filter surfaces and the KNOWN_UNGATED reads are excluded with reasons. Runtime behavior is asserted by U3. * test(node): U3 drive the deny-bearing registry over a real node Boots a real node, seeds the two-repo fixture (plus a PR and an issue so the author-or-owner close gates are reached, not 404'd on an absent entity), then walks every deny-bearing route: the hostile probe must return the exact deny status and leak nothing, and the positive twin must reach the handler (owner: not 403; read: 2xx). A terminal invariant asserts one hostile probe per row was driven and the count equals the registry, so a row that produced no probe fails loud rather than passing vacuously. Adds seed_pr/seed_issue to TestNode. Mutation-verified load-bearing: breaking did_matches drove the merge_pr owner-gate hostile to 500 (RED, named); neutering authorize_repo_read's deny drove the get_repo read-gate hostile to 200 (RED, named); both reverted green. * test(node): U4 completeness cross-check keeps the registry from drifting A pure source scrape of server.rs (multiline .route parser, floor 90) that the runtime sweep cannot give: no deny-bearing row points at a route that no longer mounts (anti-stale), and no handler carrying an unambiguous owner-gate marker (require_repo_owner / require_owner) escapes the registry (orphan). The api dir is read at test time so a new module is covered. Complements authz_guard (which proves handlers are gated) by proving the deny-bearing ones are actually driven, and reaches the non-API git mounts authz_guard never sees. Mutation-verified: a bogus row path tripped anti-stale (named the row); removing the merge_pr row tripped the orphan guard (named merge_pr); both reverted green. * test(node): close vacuous-green gaps in the deny prober's own assertions Vetting the harness against itself surfaced branches that ran green but were never exercised adversarially: - Ok2xx twin dropped its unused token Option; the arm now asserts the authorized read returns a NON-EMPTY 2xx, so an empty-200 denial-rendered-as-success is caught (executed by every read twin). - U4 orphan guard gains a floor on the owner-marker scan (>=6), so a marker-scan regression to zero fails loud instead of passing by checking nothing. - Added probe unit tests that execute the previously-uncovered probes_for arms: the SiblingPublic path-scoped twin, and the Reach::None fail-loud panic. All 22 harness tests green. Every send_probe/probes_for branch is now executed; the Deny(401), Not403, and Ok2xx-status arms were additionally mutation-verified RED (signature-expect flip; did_matches->false; authorize_repo_read->deny) and both U4 floors RED via threshold flips, all reverted. * test(node): close 3 completeness gaps in the deny prober's own guards (#195) jatmn's review flagged three ways the prober's self-guards stay green when the protection they check is removed: - F1: the U4 completeness check enforced owner-gate orphans but not read-gate ones, so dropping authorize_repo_read on get_issue/get_cert/ get_bounty/etc. left the sweep green. Add a symmetric read-gate orphan guard that scans src/api for authorize_repo_read/visibility_check handlers and requires each to be a driven ReadGate row or an enumerated READ_GATE_NOT_DRIVEN entry with a reason, replacing the free-text prose. A staleness check keeps the allowlist from carrying dead exemptions. - F2: the U3 registry sweep called assert_denied with an empty withheld list, proving status only. Seed the private fixture with a distinctive secret and its blob OID, carry them on the read-gate hostile probes, and pass them through so a denial body echoing withheld data fails. - F3: the owner-gate orphan scan matched only require_owner/require_repo_owner and missed the inline did_matches(caller, &record.owner_did) idiom that protect_branch/unprotect_branch use. Fold in a has_owner_did_matches discriminator that catches the owner form without false-matching the signer-self (register_replica) or author (close_pr) did_matches forms, pinned by unit tests on the exact bodies. * test(node): make the deny-prober guards load-bearing (#195, F1-F4) Resolves jatmn's four INV-21 completeness findings — guards that were green but not load-bearing: F1: every ReadGate hostile probe was anonymous, so the signed-non-reader branch of each private read was untested — a regression treating any valid signature as authorized would leak while the sweep stayed green. Add a Signer::Stranger 404/ no-leak probe per read row, with a dedicated readgate_stranger counter so a dropped probe fails loudly. RED: authorize any signed caller -> the stranger probe leaks (200) while the anon probe stays green. F2: get_star_status, get_icaptcha_proof, and replicate_encrypted_blobs were source-only exemptions (a marker scan, never driven). Drive them as real ReadGate rows and remove the allowlist entries. RED: a runtime bypass that keeps the authorize_repo_read marker leaks -> the driven probe RED, the source scan green. F3: git_receive_pack's protected-branch owner 403 was never driven (its registry row is the unsigned-401 path). Add signed_stranger_protected_branch_push_is_forbidden (a signed non-owner push to a protected branch -> 403, owner control -> not-403), plus a seed_protected_branch harness helper. RED: invert the gate -> the probe RED. F4: the read-gate no-leak assertion omitted the private repo id (seed discarded it). Preserve it on the Fixture and withhold it; a synthetic 404 body carrying the UUID must fail check_denied. RED: a UUID-bearing body passed before the token was added. Each guard proven load-bearing by reverting the exact production auth-line (reverts not shipped). deny_harness suite 30 passed, fmt + clippy clean. * test(node): drive every arm of the multi-principal close/dispute gates close_pr, close_issue (owner-or-author) and dispute_bounty (creator-or-claimant) were each driven with a single owner/creator twin, so reverting the untested arm stayed green. Model gates by their authorizing-principal set (MultiPrincipalGate + Principal), seed distinct author/creator/claimant identities, and emit one Not403 twin per arm plus the stranger 403. registry_internal_consistency now invokes probes_for and asserts a twin per declared arm, so a future multi-arm handler cannot register with an untested arm. Each arm proven load-bearing: reverting is_owner/is_author on close_pr/close_issue and is_creator/is_claimant on dispute_bounty turns that arm's twin RED. * test(node): drive the deferred private reads with per-read leak assertions The deferred reads (get_issue, get_pr, get_pr_diff, list_issue_comments, list_reviews, list_comments, get_cert, get_bounty, path-scoped get_tree) sat in READ_GATE_NOT_DRIVEN, checked only for an authorize_repo_read marker, so an ignored result or early return stayed green. Seed each sub-entity in the private repo (issues/PRs via the real create handlers, a cert via a signed owner push, a private-repo bounty), thread id-keyed paths through IdSource, and drive each as a ReadGate row: anon/signed-stranger 404, owner 2xx. Each seeded entity carries a distinctive marker added to the read-gate withheld set, so a 404 that leaks the entity's own content fails (not just a status check). get_pr_diff exercises both its repo-root gate and its per-path visibility_check loop. get_encrypted_blob stays deferred (owner twin needs live IPFS; reclassified structurally next). Load-bearing: an ignored-result bypass on each read turns its hostile probe RED; a 404 leaking a seeded marker fails the body assertion. * test(node): close the prose-excuse hatch with a mounted-route completeness gate READ_GATE_NOT_DRIVEN allowed a free-text reason to park a read-gated handler out of the runtime sweep. Replace the reasons with a closed NotDrivenReason enum (ReadGatingMutation, GitSmartHttpRead, ContentAddressedRead, GlobalListFilter, ExternalDependencyUnavailable); get_encrypted_blob moves to the last. A drivable 404-deny GET no longer type-checks a reason and is caught by a driven-not- relabeled assertion. Add a completeness gate that derives the required read-gate set from the mounted route table (scrape_mounts, INV-21b), not only the source-marker scan: every mounted repo-scoped GET must be a driven ReadGate row, a deny-bearing GET, structurally excused, or a declared public GET — else it fails. Catches a read that gates via a helper the marker scan does not recognize. Load-bearing: re-parking a driven read trips the relabel assert; removing a driven ReadGate row trips the mounts cross-check. * test(review): harden multi-principal twin check against signer collision Add a distinct-signer assertion to multi_principal_rows_emit_one_twin_per_arm: the per-arm any() check is satisfied by a single twin when two Principals map to the same Signer, so a signer_for_principal collision would leave one arm silently untested. Assert the emitted twins carry principals.len() distinct signers (proven RED by colliding Author->Owner). Also correct the get_cert / priv_markers comments, which claimed a withheld cert-content marker that is not seeded — no cert marker is needed because get_cert 404s at the repo-root gate before the cert is fetched, so cert fields cannot reach the deny body. * style(test): apply rustfmt to the deny-prober additions * test(node): drive the bounty mutation gates and pin fixture object format (#195) - submit/approve/cancel_bounty become MultiPrincipalGate registry rows with status-appropriate bounty fixtures (claimed/submitted/open): their status check fires before the auth gate, so each row seeds its own bounty through the real API and the arm twins consume the status - pin --object-format=sha1 in both fixture worktrees so a sha256 default object format cannot break the OID equality with the served sha1 repos * test(node): drive the caller-self task/register gates in the deny-prober (#195) Resolve the three review findings on head 07abcc6. Bound the two new real-node probes (protected-branch push, PR-diff) on a timeout-bearing client, so a wedged git subprocess fails the suite rather than hanging it until CI kills the job. Withhold the pushed secret blob's OID (full and [..12] short form) in the PR-diff denial. The old branch was dead code: the OID was never in the fixture seed map, so build_branch_push_body now returns the pushed blob OIDs. Drive every caller-self runtime deny the registry claimed but only source-marker checked. create_task, claim_task, and register are a new SignerSelfGate class (a body DID field bound to the authenticated signer); complete_task and fail_task are a MultiPrincipalGate assignee arm (the caller must equal the task's stored assignee). Each runs hostile-then-authorized against a real node. A completeness guard, derived from a scan of src/api, now requires every did_matches(caller, X) handler to be a driven registry row, and the module doc is reconciled to the REST surface (the parallel GraphQL mutation gates are fenced separately, #219). Every gate is mutation-verified load-bearing: reverting a production did_matches line drives its hostile probe RED, and the completeness guard goes RED when a driven row is dropped or the classifier is starved. No production code changed. --------- Co-authored-by: t --- crates/gitlawb-node/src/test_harness.rs | 47 +- crates/gitlawb-node/tests/deny_harness.rs | 1413 +++++++++++++++++++ crates/gitlawb-node/tests/support/mod.rs | 18 + crates/gitlawb-node/tests/support/probe.rs | 1406 ++++++++++++++++++ crates/gitlawb-node/tests/support/routes.rs | 1059 ++++++++++++++ 5 files changed, 3942 insertions(+), 1 deletion(-) create mode 100644 crates/gitlawb-node/tests/support/probe.rs create mode 100644 crates/gitlawb-node/tests/support/routes.rs diff --git a/crates/gitlawb-node/src/test_harness.rs b/crates/gitlawb-node/src/test_harness.rs index 86a19700..6c80e171 100644 --- a/crates/gitlawb-node/src/test_harness.rs +++ b/crates/gitlawb-node/src/test_harness.rs @@ -26,7 +26,7 @@ use uuid::Uuid; use gitlawb_core::identity::Keypair; use crate::config::Config; -use crate::db::{Db, RepoRecord, VisibilityMode}; +use crate::db::{Db, PullRequest, RepoRecord, VisibilityMode}; use crate::rate_limit::{RateLimiter, TrustedProxy}; use crate::state::AppState; @@ -320,6 +320,51 @@ impl TestNode { record.id } + /// Mark `branch` as protected on `repo_id`, so a non-owner push to it is + /// rejected by the branch-protection gate (used by the deny-prober's + /// protected-push probe — #195, F3). + pub async fn seed_protected_branch(&self, repo_id: &str, branch: &str, created_by: &str) { + self.db + .protect_branch(repo_id, branch, created_by) + .await + .expect("seed protected branch"); + } + + /// Seed an open PR `number` in `repo_id`, authored by `author_did`. The + /// author-or-owner close gate (`close_pr`) loads the PR *before* it runs, so a + /// deny-prober row for `.../pulls/{number}/close` needs the PR to exist or a + /// stranger gets a 404 (absent entity) instead of the 403 the gate emits. + pub async fn seed_pr(&self, repo_id: &str, number: i64, author_did: &str) { + let now = Utc::now().to_rfc3339(); + let pr = PullRequest { + id: Uuid::new_v4().to_string(), + repo_id: repo_id.to_string(), + number, + title: "seed pr".to_string(), + body: None, + author_did: author_did.to_string(), + source_branch: "feature".to_string(), + target_branch: "main".to_string(), + status: "open".to_string(), + merged_by_did: None, + merged_at: None, + created_at: now.clone(), + updated_at: now, + }; + self.db.create_pr(&pr).await.expect("seed pr"); + } + + /// Seed issue `issue_id` (a git-ref JSON blob authored by `author_did`) in the + /// repo's bare on-disk store. Like [`Self::seed_pr`], the `close_issue` + /// owner-or-author gate loads the issue before it runs, so the deny-prober row + /// for `.../issues/{id}/close` needs the issue present to reach the 403. + pub fn seed_issue(&self, owner_did: &str, name: &str, issue_id: &str, author_did: &str) { + let slug = owner_did.replace([':', '/'], "_"); + let bare = self.repos_dir.join(&slug).join(format!("{name}.git")); + let json = format!(r#"{{"author":"{author_did}"}}"#); + crate::git::issues::create_issue(&bare, issue_id, &json).expect("seed issue"); + } + /// Add a path-scoped visibility rule restricting `path_glob` to /// `reader_dids` (empty = only the owner). Mode B keeps object SHAs intact /// and withholds blob content, which is what the read/replication gates diff --git a/crates/gitlawb-node/tests/deny_harness.rs b/crates/gitlawb-node/tests/deny_harness.rs index 6fdf9efe..88e60336 100644 --- a/crates/gitlawb-node/tests/deny_harness.rs +++ b/crates/gitlawb-node/tests/deny_harness.rs @@ -14,6 +14,8 @@ mod support; use std::process::Command; use support::assert::assert_denied; +use support::probe::{probes_for, Expect, Fixture, Probe, Signer}; +use support::routes::{deny_bearing_routes, GateClass}; use support::signing::signed_request; use gitlawb_core::cid::Cid; @@ -129,6 +131,81 @@ async fn unsigned_receive_pack_is_denied(pool: sqlx::PgPool) { node.shutdown().await; } +/// Build a minimal git-receive-pack request body: one ref-update pkt-line for +/// `refs/heads/` (dummy 40-hex old/new SHAs — the branch-protection gate +/// only reads the ref NAME) plus a flush. Enough to reach the ref-update parse +/// and the protection gate, not a real pack. +fn receive_pack_update_body(branch: &str) -> Vec { + let old = "0".repeat(40); + let new = "1".repeat(40); + let line = format!("{old} {new} refs/heads/{branch}\0report-status\n"); + let pkt = format!("{:04x}{line}", line.len() + 4); + let mut body = pkt.into_bytes(); + body.extend_from_slice(b"0000"); + body +} + +/// #195 (F3): a signed NON-OWNER pushing to a PROTECTED branch is forbidden (403) +/// by the branch-protection gate (`repos.rs`), and the owner pushing to the same +/// branch is NOT blocked (control). git_receive_pack's registry row only drives the +/// unsigned-401 signature path, so without this probe inverting the 403 leaves the +/// sweep AND the completeness scan green. Drives the 403 so the gate can't rot. +#[sqlx::test] +async fn signed_stranger_protected_branch_push_is_forbidden(pool: sqlx::PgPool) { + let node = spawn_node(pool).await; + // #195 (F1): a bounded timeout so a wedged git-receive-pack fails the suite + // rather than hanging it until CI kills the job. + let client = support::bounded_client(); + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let stranger = Keypair::generate(); + + let repo_id = node.seed_repo(&owner_did, "protrepo", true).await; + node.seed_protected_branch(&repo_id, "main", &owner_did) + .await; + + let path = format!("/{owner_did}/protrepo/git-receive-pack"); + let body = receive_pack_update_body("main"); + + // Signed non-owner -> 403 branch protection; the denial leaks no repo internals. + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &path, + body.clone(), + &stranger, + ) + .send() + .await + .expect("request sends"); + assert_eq!( + resp.status().as_u16(), + 403, + "a signed non-owner push to a protected branch must be forbidden (403)" + ); + assert_denied(resp, 403, &[repo_id.as_str()]).await; + + // Owner control: the owner is NOT blocked by branch protection (it may fail + // later on the dummy pack, but must not be the 403 the stranger got). + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &path, + body, + &owner, + ) + .send() + .await + .expect("request sends"); + assert_ne!( + resp.status().as_u16(), + 403, + "the owner must not be blocked by their own branch protection (control)" + ); +} + // ── U5(b): INV-8/INV-2 — anonymous /ipfs/{cid} of a withheld blob is denied ── /// A public repo with a `/secret/**` withhold rule (readers = one allowed DID). @@ -323,6 +400,206 @@ async fn withheld_path_blob_read_is_denied(pool: sqlx::PgPool) { node.shutdown().await; } +// ── U3: get_pr_diff SECOND (per-path) gate — a diff touching a withheld path ── + +/// Build a git v0 receive-pack body creating `refs/heads/` at a fresh +/// commit that adds `files` on top of the existing `main`, force-updating main to +/// a deterministic base so `` shares history with main. Returns the body +/// bytes plus a map of each pushed file's blob OID (full sha1), so the caller can +/// withhold the OID of a pushed secret path (#195, F2). Shells out to the local +/// `git`. +fn build_branch_push_body( + server_main_tip: &str, + branch: &str, + files: &[(&str, &str)], +) -> (Vec, std::collections::HashMap) { + use std::io::Write; + use std::process::{Command, Stdio}; + + let work = std::env::temp_dir().join(format!( + "gl-u3-prdiff-{}-{}", + std::process::id(), + server_main_tip + )); + let _ = std::fs::remove_dir_all(&work); + std::fs::create_dir_all(&work).unwrap(); + let run = |args: &[&str]| -> String { + let out = Command::new("git") + .args(args) + .current_dir(&work) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + // Pin sha1 to match the served fixture repo; init.defaultObjectFormat=sha256 breaks the ref update. + run(&["init", "-q", "-b", "main", "--object-format=sha1"]); + run(&["config", "user.email", "t@t"]); + run(&["config", "user.name", "t"]); + std::fs::write(work.join("base.txt"), "prdiff base").unwrap(); + run(&["add", "base.txt"]); + run(&["commit", "-q", "-m", "base"]); + let new_main = run(&["rev-parse", "HEAD"]); + run(&["checkout", "-q", "-b", branch]); + for (p, c) in files { + let full = work.join(p); + if let Some(parent) = full.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(&full, c).unwrap(); + run(&["add", p]); + } + run(&["commit", "-q", "-m", "feature"]); + let feature_tip = run(&["rev-parse", "HEAD"]); + + // #195 (F2): capture each pushed file's blob OID (full sha1). seed_bare_repo + // never seeds these pushed paths, so this is the only place their OID is known; + // the PR-diff denial withholds the secret path's OID (full + [..12] short form). + let mut pushed_oids = std::collections::HashMap::new(); + for (p, _) in files { + pushed_oids.insert((*p).to_string(), run(&["rev-parse", &format!("HEAD:{p}")])); + } + + let child = Command::new("git") + .args(["pack-objects", "--stdout", "--revs"]) + .current_dir(&work) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + child + .stdin + .as_ref() + .unwrap() + .write_all(format!("{new_main}\n{feature_tip}\n").as_bytes()) + .unwrap(); + let pack = child.wait_with_output().unwrap().stdout; + + let zero = "0".repeat(server_main_tip.len()); + let l1 = format!("{server_main_tip} {new_main} refs/heads/main\0report-status\n"); + let l2 = format!("{zero} {feature_tip} refs/heads/{branch}\n"); + let mut body: Vec = Vec::new(); + body.extend(format!("{:04x}{l1}", l1.len() + 4).into_bytes()); + body.extend(format!("{:04x}{l2}", l2.len() + 4).into_bytes()); + body.extend_from_slice(b"0000"); + body.extend_from_slice(&pack); + let _ = std::fs::remove_dir_all(&work); + (body, pushed_oids) +} + +/// get_pr_diff has a SECOND, path-scoped gate after the repo-root read: it +/// iterates the diff's touched paths and 404s if any is withheld (pulls.rs +/// visibility_check loop). On a PUBLIC repo with a `/secret/**` withhold rule and a +/// PR whose diff touches `secret/x.txt`, an anon caller PASSES the public repo-root +/// gate but must be DENIED by the per-path gate (404), leaking neither the withheld +/// content nor its OID; the owner sees the full diff. This drives the per-path gate +/// the private-repo registry row cannot reach (there, the root gate fires first for +/// every non-owner), so reverting the visibility_check Deny-return turns this RED. +#[sqlx::test] +async fn get_pr_diff_withheld_path_is_denied(pool: sqlx::PgPool) { + let node = spawn_node(pool).await; + // #195 (F1): a bounded timeout so a wedged git subprocess push fails the suite + // rather than hanging it until CI kills the job. + let client = support::bounded_client(); + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + + let repo_id = node.seed_repo(&owner_did, "prdiff-repo", true).await; + let oids = node.seed_bare_repo( + &owner_did, + "prdiff-repo", + &[("public/a.txt", "public seed")], + "sha1", + ); + node.withhold_path(&repo_id, "/secret/**", &[], &owner_did) + .await; + + // Push a `feature` branch whose diff vs main touches the withheld secret path. + let (body, pushed_oids) = build_branch_push_body( + &oids["HEAD"], + "feature", + &[("secret/x.txt", "TOPSECRET-PRDIFF-PATH")], + ); + let push = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &format!("/{owner_did}/prdiff-repo/git-receive-pack"), + body, + &owner, + ) + .header("content-type", "application/x-git-receive-pack-request") + .send() + .await + .expect("push sends"); + assert_eq!(push.status().as_u16(), 200, "owner push must land"); + + // Open a PR main <- feature (as owner, a reader of the public repo). + let create = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &format!("/api/v1/repos/{owner_did}/prdiff-repo/pulls"), + br#"{"title":"prdiff","source_branch":"feature","target_branch":"main"}"#.to_vec(), + &owner, + ) + .header("content-type", "application/json") + .send() + .await + .expect("create PR sends"); + assert_eq!(create.status().as_u16(), 201, "PR create must return 201"); + let pr: serde_json::Value = create.json().await.unwrap(); + let number = pr["number"].as_i64().expect("PR number"); + + // Anon PASSES the public repo-root gate, then the per-path gate DENIES the diff + // (it touches the withheld secret path): 404, leaking no content or OID. + // #195 (F2): withhold the pushed secret blob's OID in both forms (the full + // sha1 and `oid[..12]`), matching the private_blob_oid_short convention + // (probe.rs:200), NOT `git rev-parse --short` (variable-length). + let secret_oid_full = pushed_oids["secret/x.txt"].clone(); + let secret_oid_short = secret_oid_full[..12].to_string(); + let resp = client + .get(format!( + "{}/api/v1/repos/{owner_did}/prdiff-repo/pulls/{number}/diff", + node.base_url + )) + .send() + .await + .expect("anon diff read sends"); + let withheld = vec![ + "TOPSECRET-PRDIFF-PATH", + secret_oid_full.as_str(), + secret_oid_short.as_str(), + ]; + assert_denied(resp, 404, &withheld).await; + + // Owner sees the full diff (the per-path gate admits the reader). + let resp = signed_request( + &client, + reqwest::Method::GET, + &node.base_url, + &format!("/api/v1/repos/{owner_did}/prdiff-repo/pulls/{number}/diff"), + Vec::new(), + &owner, + ) + .send() + .await + .expect("owner diff read sends"); + assert_eq!( + resp.status().as_u16(), + 200, + "owner must see the full PR diff (per-path gate admits the reader)" + ); + assert!( + resp.text().await.unwrap().contains("TOPSECRET-PRDIFF-PATH"), + "the owner's diff returns the touched file content" + ); +} + // ── U8: INV-2 — an anonymous clone/fetch excludes withheld subtree blobs ────── /// The replication/clone surface, the one `oneshot` cannot serve: a public repo @@ -429,6 +706,208 @@ async fn anon_clone_excludes_withheld_subtree_blobs(pool: sqlx::PgPool) { node.shutdown().await; } +// ── U3: drive the whole deny-bearing route registry over the real stack ─────── + +/// Send one [`Probe`] against the running node, signing as the probe's identity. +/// Anon requests are unsigned; Owner/Stranger requests carry a real RFC-9421 +/// signature for the fixture's owner / stranger keypair. +async fn send_probe( + client: &reqwest::Client, + base_url: &str, + fixture: &Fixture, + probe: &Probe, +) -> reqwest::Response { + let rb = match probe.signer { + Signer::Anon => client + .request(probe.method.clone(), format!("{base_url}{}", probe.path)) + .body(probe.body.clone()), + Signer::Owner => signed_request( + client, + probe.method.clone(), + base_url, + &probe.path, + probe.body.clone(), + &fixture.owner, + ), + Signer::Stranger => signed_request( + client, + probe.method.clone(), + base_url, + &probe.path, + probe.body.clone(), + &fixture.stranger, + ), + // #195 (F1): each multi-principal arm signs with its own distinct fixture + // identity, so reverting one arm in the handler turns only that arm's twin + // RED. + Signer::Author => signed_request( + client, + probe.method.clone(), + base_url, + &probe.path, + probe.body.clone(), + &fixture.author, + ), + Signer::Creator => signed_request( + client, + probe.method.clone(), + base_url, + &probe.path, + probe.body.clone(), + &fixture.creator, + ), + Signer::Claimant => signed_request( + client, + probe.method.clone(), + base_url, + &probe.path, + probe.body.clone(), + &fixture.claimant, + ), + // #195 (F3): the assignee actor arm of complete_task / fail_task signs with + // the fixture's assignee identity (the one that claimed the seeded task), so + // reverting the assignee gate turns that arm's Not403 twin RED. + Signer::Assignee => signed_request( + client, + probe.method.clone(), + base_url, + &probe.path, + probe.body.clone(), + &fixture.assignee, + ), + }; + let rb = if probe.json { + rb.header("content-type", "application/json") + } else { + rb + }; + rb.send().await.unwrap_or_else(|e| { + panic!( + "probe failed to send: {} [{} {}]: {e}", + probe.label, probe.method, probe.path + ) + }) +} + +/// Walk every deny-bearing route (U1 registry), expand each into its hostile +/// probe plus positive twin (U2), and drive them against a real node: the +/// hostile request must return the exact deny status and leak nothing, and the +/// twin must reach the handler (owner-gate: not 403; read-gate: 2xx). This is +/// the runtime discharge of INV-1/INV-2/INV-8 across the whole registry, not one +/// hand-written case at a time. +/// +/// Terminal anti-vacuous-green invariant: exactly one hostile probe per row is +/// driven and the count equals the registry size, so a row that silently +/// produced no probe fails here instead of the sweep passing by testing nothing. +#[sqlx::test] +async fn deny_bearing_registry_denies_hostile_and_admits_authorized(pool: sqlx::PgPool) { + let node = spawn_node(pool).await; + // A bounded timeout so a wedged route fails the suite rather than hanging it. + let client = reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(30)) + .build() + .unwrap(); + let fixture = Fixture::seed(&node).await; + + let rows = deny_bearing_routes(); + assert!( + !rows.is_empty(), + "deny-bearing route registry is empty — nothing to sweep" + ); + + // #195 (F1): every read-gate row is now probed by a signed non-reader too, so a + // dropped stranger probe must fail loudly rather than being absorbed into the + // total. Count the read-gate rows and the stranger hostiles separately. + let readgate_rows = rows + .iter() + .filter(|r| matches!(r.gate, GateClass::ReadGate)) + .count(); + + let mut hostile_driven = 0usize; + let mut readgate_stranger_driven = 0usize; + let mut twins_driven = 0usize; + + for row in rows { + let probes = probes_for(row, &fixture); + assert!( + !probes.is_empty(), + "row {} {} produced no probes", + row.method, + row.path + ); + for probe in &probes { + let ctx = format!("{} [{} {}]", probe.label, probe.method, probe.path); + let resp = send_probe(&client, &node.base_url, &fixture, probe).await; + let status = resp.status(); + match &probe.expect { + Expect::Deny(code) => { + hostile_driven += 1; + // A signed-stranger 404 is the read-gate signed-non-reader probe + // (owner-gate strangers are 403, signature hostiles are anon 401). + if probe.signer == Signer::Stranger && *code == 404 { + readgate_stranger_driven += 1; + } + assert_eq!( + status.as_u16(), + *code, + "hostile probe must deny with {code}, got {status}: {ctx}" + ); + // INV-8 shape guard: rechecks the status and that the body is + // neither an empty-200-as-success nor carrying withheld data. The + // probe's `withheld` tokens (the private repo's secret bytes and + // blob OID on read-gate probes) are passed through, so a 404 that + // spilled the private content fails here instead of being counted + // as a clean denial. + let tokens: Vec<&str> = probe.withheld.iter().map(String::as_str).collect(); + assert_denied(resp, *code, &tokens).await; + } + Expect::Not403 => { + twins_driven += 1; + assert_ne!( + status.as_u16(), + 403, + "owner-reachability twin must reach the handler (not 403): {ctx}" + ); + } + Expect::Ok2xx => { + twins_driven += 1; + assert!( + status.is_success(), + "read-reachability twin must be 2xx, got {status}: {ctx}" + ); + // A 2xx with an empty body is a denial rendered as success: + // the authorized read must actually return the resource. + let body = resp.bytes().await.unwrap_or_default(); + assert!( + !body.is_empty(), + "read-reachability twin returned an empty 2xx body (denial-as-success?): {ctx}" + ); + } + } + } + } + + // One hostile per row (a signed stranger for the 403 gates, including the + // #195 N2 status-gated bounty rows; anon for the signature and read-gate + // rows), plus the EXTRA signed-stranger 404 each read-gate row drives. + assert_eq!( + hostile_driven, + rows.len() + readgate_rows, + "expected one hostile per row ({} rows) plus one signed-stranger hostile per read-gate row ({readgate_rows}), drove {hostile_driven}", + rows.len() + ); + // #195 (F1): every read-gate row must be probed by a signed non-reader — a + // dropped stranger probe fails HERE rather than silently reducing coverage. + assert_eq!( + readgate_stranger_driven, readgate_rows, + "every read-gate row must drive a signed-non-reader 404 probe ({readgate_rows} rows), drove {readgate_stranger_driven}" + ); + assert!( + twins_driven >= 1, + "no positive twins were driven — the reachability proof is missing" + ); +} + // ── Additional INV-1 owner-gates over the real stack (fan-out of U6) ────────── /// The remaining high-blast-radius owner-gated mutations that previously had @@ -710,3 +1189,937 @@ async fn drop_with_broken_graceful_chain_still_unblocks_via_abort( } probe.close().await.expect("probe connection closes"); } + +// ── U4: completeness cross-check (no DB) — the registry cannot silently drift ── +// +// The runtime sweep (U3) only proves the routes it is HANDED. This guard, a pure +// source scrape, keeps that hand-off honest against `server.rs`: no registry row +// points at a route that no longer mounts (stale row), and no owner-gated mount +// escapes the registry (orphan). It complements — does not duplicate — the +// in-crate `authz_guard` egress guard (`src/api/mod.rs`), which proves every +// repo-scoped API handler is *gated*; this proves the deny-bearing ones are +// *driven*, and it reaches the non-API git mounts `authz_guard` never sees. +mod completeness { + use std::collections::HashSet; + + use super::deny_bearing_routes; + use crate::support::routes::{GateClass, Principal}; + + /// The CLOSED set of structural reasons a read-gate handler may sit out of the + /// runtime 404 sweep (#195, U4/R4). This replaces the free-text `reason` + /// strings the old allowlist carried: a "deferred / needs seeding" prose excuse + /// no longer type-checks. Every `READ_GATE_NOT_DRIVEN` entry must name one of + /// these variants, so a reviewer sees a STRUCTURAL class (why the read is not a + /// drivable 404-deny GET), never a soft deferral that hides an undriven private + /// read. A drivable GET (the nine U3 reads) cannot be parked here because none + /// of these classes fits it — and the driven-not-relabeled assertion below + /// makes that a hard failure rather than a judgment call. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + enum NotDrivenReason { + /// A create/write path that read-gates the caller before mutating, not a + /// 404-deny GET. Its owner/author behaviour is covered by the mutation + /// authz guard in `src/api/mod.rs`, not this GET-read sweep. + ReadGatingMutation, + /// A git smart-HTTP read (`info/refs`, `git-upload-pack`): its + /// withheld-subtree 404/exclusion is driven by the U7/U8 real-clone cases, + /// not the API GET sweep. + GitSmartHttpRead, + /// A content-addressed read (`/ipfs/{cid}`): its 404 deny + no-leak is + /// driven by the U5b/U8 anon_ipfs/clone cases in this file. + ContentAddressedRead, + /// A global list-FILTER: returns 200 with unreadable rows removed, never a + /// 404, so it is not a deny-bearing read at all. + GlobalListFilter, + /// The owner-2xx twin needs a live external dependency the harness lacks: + /// `get_encrypted_blob` reads through `ipfs_pin::cat` with no harness stub, + /// so it cannot be driven as a 2xx twin here. An honest exclusion, not a + /// deferral (#195, KTD-5). + ExternalDependencyUnavailable, + } + + /// Read-gate handlers NOT driven as a `ReadGate` row, each carrying a CLOSED + /// structural reason ([`NotDrivenReason`]) — never free-text. This is the + /// enforced form of the prose that used to live in routes.rs; a handler leaving + /// the sweep must be moved here under a structural class, never with a soft + /// "deferred / needs seeding" excuse (which no longer type-checks). + /// + /// Single source of truth: both the marker-scan guard + /// (`every_read_gate_handler_is_driven_or_explicitly_allowlisted`) and the + /// route-table cross-check (`mounted_repo_gets_are_driven_excused_or_declared_public`) + /// read this, so the two cannot drift. + /// + /// get_issue / get_pr / get_pr_diff / list_issue_comments / list_reviews / + /// list_comments / get_cert / get_bounty / get_tree are DRIVEN as real ReadGate + /// rows in deny_bearing_routes() (#195, U3); the driven-not-relabeled assertion + /// bars any of them from being quietly re-parked here. + const READ_GATE_NOT_DRIVEN: &[(&str, NotDrivenReason)] = &[ + // Owner-2xx twin needs a live IPFS backend (`ipfs_pin::cat`) the harness has + // no stub for — an honest external-dependency exclusion (#195, KTD-5). + ( + "get_encrypted_blob", + NotDrivenReason::ExternalDependencyUnavailable, + ), + // Read-gates but is a mutation, not a 404-deny GET. + ("create_review", NotDrivenReason::ReadGatingMutation), + ("create_comment", NotDrivenReason::ReadGatingMutation), + ("create_pr", NotDrivenReason::ReadGatingMutation), + ("create_issue", NotDrivenReason::ReadGatingMutation), + ("create_issue_comment", NotDrivenReason::ReadGatingMutation), + ("create_bounty", NotDrivenReason::ReadGatingMutation), + ("claim_bounty", NotDrivenReason::ReadGatingMutation), + ("fork_repo", NotDrivenReason::ReadGatingMutation), + ("star_repo", NotDrivenReason::ReadGatingMutation), + ("unstar_repo", NotDrivenReason::ReadGatingMutation), + // Content-addressed read: driven by the U5b/U8 anon_ipfs/clone cases. + ("get_by_cid", NotDrivenReason::ContentAddressedRead), + // Git smart-HTTP reads: driven by the U7/U8 real-clone cases. + ("git_info_refs", NotDrivenReason::GitSmartHttpRead), + ("git_upload_pack", NotDrivenReason::GitSmartHttpRead), + // Global list-FILTER: 200 with unreadable rows removed, never a 404. + ("list_all_bounties", NotDrivenReason::GlobalListFilter), + ]; + + /// Every `.route("", ()…)` mount in `src`, as + /// (METHOD, path). Multiline-aware: most mounts put the path on the line after + /// `.route(`, so a per-line scan false-greens. Walks balanced parens from each + /// `.route(` so chained and multi-method (`put().delete().get()`) mounts are + /// all captured. + fn scrape_mounts(src: &str) -> Vec<(String, String)> { + let bytes = src.as_bytes(); + let mut out = Vec::new(); + let mut i = 0; + while let Some(rel) = src[i..].find(".route(") { + let open = i + rel + ".route(".len(); + let mut depth = 1i32; + let mut j = open; + while j < src.len() && depth > 0 { + match bytes[j] { + b'(' => depth += 1, + b')' => depth -= 1, + _ => {} + } + j += 1; + } + let call = &src[open..j.saturating_sub(1)]; + i = j; + // The path is the first string literal in the call. + let Some(qs) = call.find('"') else { continue }; + let Some(qe) = call[qs + 1..].find('"') else { + continue; + }; + let path = &call[qs + 1..qs + 1 + qe]; + for m in ["get", "post", "put", "delete", "patch"] { + let needle = format!("{m}("); + let mut k = 0; + while let Some(mrel) = call[k..].find(&needle) { + let at = k + mrel; + // Reject a match that is the tail of a longer ident (e.g. the + // `get(` inside `budget(`): the char before must be a boundary. + let boundary = call[..at] + .chars() + .last() + .map(|c| !(c.is_alphanumeric() || c == '_')) + .unwrap_or(true); + if boundary { + out.push((m.to_uppercase(), path.to_string())); + } + k = at + needle.len(); + } + } + } + out + } + + /// Names of `fn`s in `src` whose body contains any `marker`. The body is the + /// slice from a fn's `(` to the next top-level fn declaration — the same + /// boundary set `authz_guard::fn_body` uses — so a marker can't leak across + /// into the next handler. + fn handlers_with_marker(src: &str, markers: &[&str]) -> HashSet { + let decls = [ + "\npub async fn ", + "\npub(crate) async fn ", + "\nasync fn ", + "\npub(crate) fn ", + "\npub fn ", + "\nfn ", + ]; + // Ordered start offsets of every fn declaration. + let mut starts: Vec = Vec::new(); + for d in decls { + let mut k = 0; + while let Some(r) = src[k..].find(d) { + starts.push(k + r + 1); // +1: skip the leading '\n' + k = k + r + d.len(); + } + } + starts.sort_unstable(); + let mut out = HashSet::new(); + for (idx, &s) in starts.iter().enumerate() { + let end = starts.get(idx + 1).copied().unwrap_or(src.len()); + let seg = &src[s..end]; + // The name is the ident after this decl's `fn ` (handles `pub(crate)`, + // whose own `(` would otherwise be mistaken for the arg list). + let Some(fnpos) = seg.find("fn ") else { + continue; + }; + let name: String = seg[fnpos + 3..] + .chars() + .take_while(|c| c.is_alphanumeric() || *c == '_') + .collect(); + if name.is_empty() { + continue; + } + if markers.iter().any(|m| seg.contains(m)) { + out.insert(name); + } + } + out + } + + /// Like [`handlers_with_marker`] but selects handlers whose body satisfies a + /// PREDICATE rather than a substring — used for the inline owner-gate + /// did_matches idiom (F3), whose "first arg is the caller, second is + /// owner_did" shape a plain substring cannot express. Same fn-segmentation as + /// `handlers_with_marker` so a match cannot leak across into the next handler. + fn handlers_matching(src: &str, pred: fn(&str) -> bool) -> HashSet { + let decls = [ + "\npub async fn ", + "\npub(crate) async fn ", + "\nasync fn ", + "\npub(crate) fn ", + "\npub fn ", + "\nfn ", + ]; + let mut starts: Vec = Vec::new(); + for d in decls { + let mut k = 0; + while let Some(r) = src[k..].find(d) { + starts.push(k + r + 1); + k = k + r + d.len(); + } + } + starts.sort_unstable(); + let mut out = HashSet::new(); + for (idx, &s) in starts.iter().enumerate() { + let end = starts.get(idx + 1).copied().unwrap_or(src.len()); + let seg = &src[s..end]; + let Some(fnpos) = seg.find("fn ") else { + continue; + }; + let name: String = seg[fnpos + 3..] + .chars() + .take_while(|c| c.is_alphanumeric() || *c == '_') + .collect(); + if name.is_empty() { + continue; + } + if pred(seg) { + out.insert(name); + } + } + out + } + + fn short_handler(h: &str) -> &str { + h.rsplit("::").next().unwrap_or(h) + } + + /// Like [`handlers_with_marker`] but ignores markers that appear only inside a + /// full-line `//` comment. The owner-gate markers never show up in prose, but + /// the read-gate markers (`authorize_repo_read(` / `visibility_check(`) do: + /// several docstrings and a test comment name them (repos.rs's + /// `owner_push_rejection` / `dedupe_canonical_repos` regions, the fork + /// docstring), and a raw `contains` would misattribute those to the nearest + /// preceding fn and force a phantom non-handler onto the read-gate allowlist. + /// Mirrors `authz_guard::fn_body`'s comment-stripping so the scan sees code, + /// not prose. + fn handlers_with_code_marker(src: &str, markers: &[&str]) -> HashSet { + let stripped: String = src + .lines() + .map(|l| { + if l.trim_start().starts_with("//") { + "" + } else { + l + } + }) + .collect::>() + .join("\n"); + handlers_with_marker(&stripped, markers) + } + + /// True when `body` contains an INLINE owner-gate `did_matches` call: one + /// whose FIRST arg is the caller (`caller` / `&caller` / `&auth.0`) AND whose + /// SECOND arg is `&record.owner_did`. That two-arg shape is the discriminator + /// (F3): `require_owner`/`require_repo_owner` are the named markers, but + /// protect_branch / unprotect_branch owner-gate with this raw idiom instead, so + /// a plain marker misses them and a future owner-only handler using the same + /// pattern could slip past the registry while U4 stayed green. + /// + /// It must NOT fire on the signer-self / author forms that share the helper: + /// register_replica's `did_matches(replica_did, &record.owner_did)` (first arg + /// is the replica, not the caller — signer-self by design), close_pr's + /// `did_matches(&auth.0, &pr.author_did)`, and the bounty/task self forms + /// (`&bounty.creator_did`, delegator/assignee dids). Whitespace and newlines + /// inside the arg list are tolerated so a reformatted call still matches. + fn has_owner_did_matches(body: &str) -> bool { + let bytes = body.as_bytes(); + let mut i = 0; + while let Some(rel) = body[i..].find("did_matches(") { + let open = i + rel + "did_matches(".len(); + // Walk to the matching close paren, splitting the top-level args on the + // first depth-0 comma (there are exactly two args). + let mut depth = 1i32; + let mut comma: Option = None; + let mut j = open; + while j < body.len() && depth > 0 { + match bytes[j] { + b'(' => depth += 1, + b')' => depth -= 1, + b',' if depth == 1 && comma.is_none() => comma = Some(j), + _ => {} + } + j += 1; + } + i = j; + let close = j.saturating_sub(1); + let Some(comma) = comma else { continue }; + let arg1 = body[open..comma].split_whitespace().collect::(); + let arg2 = body[comma + 1..close] + .split_whitespace() + .collect::(); + let caller_first = arg1 == "caller" || arg1 == "&caller" || arg1 == "&auth.0"; + let owner_second = arg2 == "&record.owner_did"; + if caller_first && owner_second { + return true; + } + } + false + } + + /// The CLOSED set of caller-self REST gates deliberately NOT driven as a + /// registry row. INTENTIONALLY EMPTY (#195, F3/R4): every caller-self + /// `did_matches(caller, X)` gate in `src/api` is driven by a real-node + /// hostile+control probe (a `SignerSelfGate` field-binding row or a + /// `MultiPrincipalGate` assignee-arm row), so no caller-self REST gate is + /// un-drivable. A new classifier-marked handler that is not a driven row must + /// fail `signer_self_gates_are_all_driven`; it is not parked here. + const SIGNER_SELF_NOT_DRIVEN: &[&str] = &[]; + + /// True when `body` contains a CALLER-SELF `did_matches` call: one whose FIRST + /// arg is the caller (`caller` / `&caller` / `&auth.0`) AND whose SECOND arg is + /// NOT `&record.owner_did`. Sibling to [`has_owner_did_matches`] (the owner form + /// is guarded there); this is the field-binding / actor idiom the task and + /// register handlers use: create_task (`&body.delegator_did`), claim_task + /// (`&body.assignee_did`), complete_task / fail_task (the stored `assignee_did`, + /// spelled across lines), register (`agent_did.as_str()`). The arg1 set matches + /// its owner sibling's (NOT just `&auth.0`), so a `let caller = &auth.0; + /// did_matches(caller, X)` gate cannot escape. The paren-walk is multi-line-robust + /// because complete_task / fail_task spell the call across lines. + fn has_signer_self_did_matches(body: &str) -> bool { + let bytes = body.as_bytes(); + let mut i = 0; + while let Some(rel) = body[i..].find("did_matches(") { + let open = i + rel + "did_matches(".len(); + let mut depth = 1i32; + let mut comma: Option = None; + let mut j = open; + while j < body.len() && depth > 0 { + match bytes[j] { + b'(' => depth += 1, + b')' => depth -= 1, + b',' if depth == 1 && comma.is_none() => comma = Some(j), + _ => {} + } + j += 1; + } + i = j; + let close = j.saturating_sub(1); + let Some(comma) = comma else { continue }; + let arg1 = body[open..comma].split_whitespace().collect::(); + let arg2 = body[comma + 1..close] + .split_whitespace() + .collect::(); + let caller_first = arg1 == "caller" || arg1 == "&caller" || arg1 == "&auth.0"; + let owner_second = arg2 == "&record.owner_did"; + if caller_first && !owner_second { + return true; + } + } + false + } + + #[test] + fn deny_registry_is_not_stale_and_owner_gates_are_all_driven() { + let server = include_str!("../src/server.rs"); + let mounts: HashSet<(String, String)> = scrape_mounts(server).into_iter().collect(); + + // Scraper-integrity floor: if the parser silently stopped finding mounts, + // the anti-stale check below would pass vacuously. The tree has ~95 mounts. + assert!( + mounts.len() >= 90, + "mount scrape found only {} routes — the parser likely broke (floor 90)", + mounts.len() + ); + + // ANTI-STALE: every deny-bearing row must still point at a live mount, so a + // renamed/removed route can't leave a row that the sweep drives against a + // 404 and calls a passing deny. + for row in deny_bearing_routes() { + assert!( + mounts.contains(&(row.method.to_string(), row.path.to_string())), + "stale registry row: {} {} is not mounted in server.rs (renamed or removed?)", + row.method, + row.path + ); + } + + // ORPHAN GUARD: every handler that owner-gates must be a registry owner-gate + // row, so a newly added owner-gated mutation cannot escape the runtime + // sweep. The api dir is read at test time (like authz_guard's completeness + // scan) so a brand-new module is covered too. Two owner-gate shapes are + // recognized: + // (1) the named markers `require_repo_owner(` / `require_owner(`, and + // (2) the INLINE `did_matches(caller, &record.owner_did)` idiom (F3) that + // protect_branch / unprotect_branch use instead of a named helper. + // Only the SECOND-arg-is-owner_did form of did_matches is treated as an + // owner gate; the signer-self / author forms (register_replica's + // did_matches(replica_did, …), close_pr's did_matches(&auth.0, &author_did)) + // are deliberately NOT matched, so they don't false-orphan. + // A handler satisfies the owner marker if it is a plain OwnerGate row OR a + // MultiPrincipalGate row that declares the Owner arm (#195, F1: close_pr / + // close_issue owner-gate via require_repo_owner but are owner-OR-author, so + // they register as MultiPrincipalGate — their owner arm is still driven). + let registry_owner_handlers: HashSet<&str> = deny_bearing_routes() + .iter() + .filter(|r| { + r.gate == GateClass::OwnerGate + || (r.gate == GateClass::MultiPrincipalGate + && r.principals.contains(&Principal::Owner)) + }) + .map(|r| short_handler(r.handler)) + .collect(); + + let api_dir = concat!(env!("CARGO_MANIFEST_DIR"), "/src/api"); + let mut owner_marked: HashSet = HashSet::new(); + for entry in std::fs::read_dir(api_dir).expect("read api dir") { + let path = entry.expect("dir entry").path(); + if path.extension().is_some_and(|e| e == "rs") { + let src = std::fs::read_to_string(&path).expect("read api file"); + owner_marked.extend(handlers_with_marker( + &src, + &["require_repo_owner(", "require_owner("], + )); + // Fold in the inline did_matches-owner idiom (F3). + owner_marked.extend(handlers_matching(&src, has_owner_did_matches)); + } + } + // The gate helpers' own definitions carry the owner-gate idiom (the named + // marker string, and visibility::require_owner's body is itself a + // `did_matches(caller, &record.owner_did)` check); they are helpers, not + // mounted handlers, so drop them. + owner_marked.remove("require_owner"); + owner_marked.remove("require_repo_owner"); + // git_receive_pack also carries the inline did_matches-owner idiom, but for + // a SECONDARY gate: its branch-protection push check (repos.rs), not a + // whole-handler owner gate. Its 403 is DRIVEN by the dedicated + // `signed_stranger_protected_branch_push_is_forbidden` test (#195, F3) — a + // signed non-owner push to a protected branch, with an owner control — so + // inverting the gate goes RED there. It is therefore not an owner-gate + // orphan (the OwnerGate registry rows can't carry the protected-branch + // substrate); drop it so the owner-orphan assert does not misflag a route + // whose owner gate is already driven. + owner_marked.remove("git_receive_pack"); + // Non-vacuous floor: if the marker scan silently found nothing (a parser + // regression), the orphan loop below would pass by checking zero handlers. + // The tree has 10 owner-marker handlers today; 6 trips only on a real break. + assert!( + owner_marked.len() >= 6, + "owner-gate marker scan found only {} handlers — the scan likely broke", + owner_marked.len() + ); + + for h in &owner_marked { + assert!( + registry_owner_handlers.contains(h.as_str()), + "handler `{h}` carries an owner-gate marker but is not an owner-gate \ + row in deny_bearing_routes() — add it so the runtime sweep drives its 403" + ); + } + } + + /// CALLER-SELF ORPHAN GUARD (#195, F3/R4, INV-21b). Sibling to the owner-gate + /// and read-gate orphan guards above: every handler carrying a caller-self + /// `did_matches(caller, X)` gate (X != `&record.owner_did`) must be a DRIVEN + /// registry row (ANY gate class: a `SignerSelfGate` field-binding row or a + /// `MultiPrincipalGate` assignee-arm row) OR listed in the (empty) + /// `SIGNER_SELF_NOT_DRIVEN`. Derived from a SOURCE scan of the `src/api` `.rs` + /// files at test time (KTD-2): a caller-self gate spelled with the inline + /// `did_matches(caller|&caller|&auth.0, X)` idiom in an existing OR new api file + /// is caught and forced to be a driven row. It does NOT catch two shapes (the + /// same limits the owner/read sibling guards have): a check hidden behind a + /// helper (the owner side tracks `require_owner` / `require_repo_owner` as markers + /// for exactly this reason -- add a marker if a caller-self helper is introduced), + /// and a gate in a new SUBDIRECTORY module (the scan is flat, not recursive). The + /// parallel GraphQL mutation gates (`src/graphql/mutation.rs`) are out of this + /// scan's scope, fenced separately (#219). + #[test] + fn signer_self_gates_are_all_driven() { + // A caller-self gate may be driven as a SignerSelfGate row (body-field bind) + // OR a MultiPrincipalGate row (the complete/fail assignee actor gate), so the + // required set is EVERY registry handler regardless of class. + let registry_handlers: HashSet<&str> = deny_bearing_routes() + .iter() + .map(|r| short_handler(r.handler)) + .collect(); + + let api_dir = concat!(env!("CARGO_MANIFEST_DIR"), "/src/api"); + let mut signer_self_marked: HashSet = HashSet::new(); + for entry in std::fs::read_dir(api_dir).expect("read api dir") { + let path = entry.expect("dir entry").path(); + if path.extension().is_some_and(|e| e == "rs") { + let src = std::fs::read_to_string(&path).expect("read api file"); + signer_self_marked.extend(handlers_matching(&src, has_signer_self_did_matches)); + } + } + // Drop the gate helpers' own definitions (mirror of the owner scan dropping + // require_owner / require_repo_owner): none of these is a mounted handler. + signer_self_marked.remove("did_matches"); + signer_self_marked.remove("require_owner"); + signer_self_marked.remove("require_repo_owner"); + + // Non-vacuous floor PINNED to the exact caller-self handler count (close_pr, + // close_issue, dispute/submit/approve/cancel_bounty, create/claim/complete/ + // fail_task, register = 11). Pinned, not loose, so a PARTIAL classifier + // regression (dropping even one) trips here rather than being absorbed by the + // runtime sweep; bump it when the caller-self surface legitimately changes. + assert!( + signer_self_marked.len() >= 11, + "caller-self marker scan found only {} handlers (expected 11); the scan broke or the caller-self surface changed without bumping this floor", + signer_self_marked.len() + ); + + for h in &signer_self_marked { + assert!( + registry_handlers.contains(h.as_str()) + || SIGNER_SELF_NOT_DRIVEN.contains(&h.as_str()), + "handler `{h}` carries a caller-self did_matches gate but is neither a \ + driven registry row nor in SIGNER_SELF_NOT_DRIVEN; drive its 403" + ); + } + + // Staleness: every SIGNER_SELF_NOT_DRIVEN entry must still be a real + // caller-self handler. Vacuous while the list is empty, but it keeps a future + // exemption honest (the mirror of the owner/read staleness checks). + for name in SIGNER_SELF_NOT_DRIVEN { + assert!( + signer_self_marked.contains(*name), + "SIGNER_SELF_NOT_DRIVEN lists `{name}`, which no longer carries a \ + caller-self did_matches gate; update the list" + ); + } + } + + /// READ-GATE ORPHAN GUARD (F1). The owner-gate orphan guard above stops an + /// owner-gated mount from escaping the sweep; this is its read-gate twin. Every + /// handler carrying a read-gate marker (`authorize_repo_read(` / + /// `visibility_check(` — the two markers repos.rs/issues.rs/pulls.rs/ + /// bounties.rs/certs.rs/labels.rs/changelog.rs/events.rs/encrypted.rs/stars.rs/ + /// visibility.rs all use, verified there is no third) must be EITHER a + /// `ReadGate` row in `deny_bearing_routes()` (driven for its 404) OR an explicit + /// entry in `READ_GATE_NOT_DRIVEN` with a reason. That enumerated allowlist + /// REPLACES the old free-text "DEFERRED / EXCLUDED" prose in routes.rs: it is + /// now enforced code, so removing or bypassing the read gate on any of these + /// endpoints (or adding a brand-new read-gated handler) trips this instead of + /// leaving the real-node sweep silently green. The api dir is read at test time, + /// like the owner scan, so a new module is covered too. + #[test] + fn every_read_gate_handler_is_driven_or_explicitly_allowlisted() { + // The allowlist is the module-level `READ_GATE_NOT_DRIVEN` (single source of + // truth, shared with the route-table cross-check). Its entries carry a closed + // `NotDrivenReason`, so a soft "deferred" excuse cannot be added. + + let driven_read_handlers: HashSet<&str> = deny_bearing_routes() + .iter() + .filter(|r| r.gate == GateClass::ReadGate) + .map(|r| short_handler(r.handler)) + .collect(); + let allowlisted: HashSet<&str> = READ_GATE_NOT_DRIVEN.iter().map(|(h, _)| *h).collect(); + + let api_dir = concat!(env!("CARGO_MANIFEST_DIR"), "/src/api"); + let mut read_marked: HashSet = HashSet::new(); + for entry in std::fs::read_dir(api_dir).expect("read api dir") { + let path = entry.expect("dir entry").path(); + // Skip api/mod.rs: it holds the gate HELPERS themselves + // (`authorize_repo_read`, `visibility_check`, `require_repo_owner`, …), + // not mounted route handlers — every mounted read handler lives in a + // sibling module (repos.rs, issues.rs, …). Scanning mod.rs would attach + // the marker to helper definitions and force phantom allowlist entries. + if path.file_name().is_some_and(|f| f == "mod.rs") { + continue; + } + if path.extension().is_some_and(|e| e == "rs") { + let src = std::fs::read_to_string(&path).expect("read api file"); + // Comment-stripped scan: the read-gate markers appear in several + // docstrings, and a raw scan would allowlist phantom non-handlers. + read_marked.extend(handlers_with_code_marker( + &src, + &["authorize_repo_read(", "visibility_check("], + )); + } + } + // The read-gate helper `visibility_check` is re-declared/aliased in + // visibility.rs's own use-path; drop the helper name if the scan attaches + // the marker to it (it is a helper, not a mounted handler) — the mirror of + // the owner scan dropping require_owner / require_repo_owner. + read_marked.remove("authorize_repo_read"); + read_marked.remove("visibility_check"); + + // Non-vacuous floor: the tree has 41 read-gate-marked handlers today. If the + // marker scan silently found far fewer, the orphan loop below would pass by + // checking almost nothing; 30 trips only on a real parser break. + assert!( + read_marked.len() >= 30, + "read-gate marker scan found only {} handlers — the scan likely broke (floor 30)", + read_marked.len() + ); + + for h in &read_marked { + let name = h.as_str(); + assert!( + driven_read_handlers.contains(name) || allowlisted.contains(name), + "handler `{name}` carries a read-gate marker but is neither a ReadGate \ + row in deny_bearing_routes() nor in READ_GATE_NOT_DRIVEN — drive its \ + 404 or add it to the allowlist with a reason" + ); + } + + // Staleness: every allowlist entry must still be a real read-gate handler, + // so a rename/removal can't leave a dead exemption that masks a future gap. + for (name, _reason) in READ_GATE_NOT_DRIVEN { + assert!( + read_marked.contains(*name), + "READ_GATE_NOT_DRIVEN lists `{name}`, which no longer carries a \ + read-gate marker (renamed, removed, or gate dropped?) — update the list" + ); + } + + // DRIVEN-NOT-RELABELED (#195, U4/R5). The nine reads U3 drove must each be a + // driven `ReadGate` row AND absent from `READ_GATE_NOT_DRIVEN`, so none can + // be quietly re-parked under a mislabeled structural reason. A structural + // class (mutation / git / content-addressed / list-filter / external-dep) + // does not fit any of these nine, but a reviewer could still mistype one in; + // this makes that a hard failure rather than a judgment call. Together with + // the closed `NotDrivenReason` enum, a drivable private read cannot leave the + // sweep: either the class does not compile, or this assertion fires. + const U3_DRIVEN_READS: &[&str] = &[ + "get_issue", + "get_pr", + "get_pr_diff", + "list_issue_comments", + "list_reviews", + "list_comments", + "get_cert", + "get_bounty", + "get_tree", + ]; + for name in U3_DRIVEN_READS { + assert!( + driven_read_handlers.contains(name), + "U3 drove `{name}` as a private read, but it is no longer a ReadGate \ + row in deny_bearing_routes() — a driven read was dropped or renamed" + ); + assert!( + !allowlisted.contains(name), + "`{name}` is a U3-driven private read but was re-parked in \ + READ_GATE_NOT_DRIVEN — a drivable 404-deny GET may not be excused \ + under a structural reason; drive it, do not relabel it" + ); + } + } + + /// The repo-scoped GET mounts the [`scrape_mounts`] route-table walk does NOT + /// treat as read-gated — genuinely public repo listings that fetch the repo and + /// return metadata WITHOUT an `authorize_repo_read` / `visibility_check` call + /// (verified in their handlers). They carry no read gate, so the cross-check + /// below must not demand they be driven or structurally excused. Keyed by path + /// (the unit `scrape_mounts` yields) so a new gate added to one of these later + /// still surfaces via the marker scan; this list only says "no gate today". + const PUBLIC_REPO_GETS: &[&str] = &[ + "/api/v1/repos/{owner}/{repo}/hooks", // list_webhooks + "/api/v1/repos/{owner}/{repo}/branches/protected", // list_protected_branches + "/api/v1/repos/{owner}/{repo}/replicas", // list_replicas + ]; + + /// True for a mounted path that is scoped to a single repo — the API form + /// `/api/v1/repos/{owner}/{repo}/…` and the bare git form `/{owner}/{repo}/…`. + /// These are the paths whose GET, if read-gated, must be driven or excused; a + /// collection/global path (`/api/v1/repos`, `/api/v1/bounties`, `/api/v1/agents` + /// …) is out of scope for the repo-read cross-check. + fn is_repo_scoped_path(path: &str) -> bool { + if let Some(rest) = path.strip_prefix("/api/v1/repos/") { + // `.../{owner}/{repo}/…` — at least owner + repo + a trailing segment, + // and NOT the `/api/v1/repos/federated` or bare `/api/v1/repos` forms + // (those have no `{owner}` placeholder). + return rest.starts_with("{owner}/{repo}/"); + } + // Bare git form: `/{owner}/{repo}/info/refs` etc. It is repo-scoped when the + // first two segments are the owner/repo placeholders. + let segs: Vec<&str> = path.trim_start_matches('/').split('/').collect(); + segs.len() >= 3 && segs[0] == "{owner}" && segs[1] == "{repo}" + } + + /// Extract the handler name from a single `.route("path", …get(handler)…)` mount + /// body: the last `::`-tail ident inside the FIRST routing-method call for the + /// requested HTTP method. Returns `None` if the method is not mounted on this + /// route. Reuses the same balanced-paren mount slice `scrape_mounts` walks. + fn handler_for_mount(call: &str, method: &str) -> Option { + let needle = format!("{}(", method.to_lowercase()); + let mut k = 0; + while let Some(rel) = call[k..].find(&needle) { + let at = k + rel; + let boundary = call[..at] + .chars() + .last() + .map(|c| !(c.is_alphanumeric() || c == '_')) + .unwrap_or(true); + if boundary { + // The handler is the first path-expression after `get(`; take up to + // the closing `)` and keep the final `::`-segment ident. + let inner_start = at + needle.len(); + let inner = &call[inner_start..]; + let end = inner.find([')', ',']).unwrap_or(inner.len()); + let expr = inner[..end].trim(); + let name: String = expr + .rsplit("::") + .next() + .unwrap_or(expr) + .chars() + .take_while(|c| c.is_alphanumeric() || *c == '_') + .collect(); + if !name.is_empty() { + return Some(name); + } + } + k = at + needle.len(); + } + None + } + + /// Like [`scrape_mounts`] but also captures the handler name, so a mounted + /// (METHOD, path) can be resolved to its handler for the read-gate cross-check. + /// Same balanced-paren walk as `scrape_mounts`. + fn scrape_mounts_with_handler(src: &str) -> Vec<(String, String, String)> { + let bytes = src.as_bytes(); + let mut out = Vec::new(); + let mut i = 0; + while let Some(rel) = src[i..].find(".route(") { + let open = i + rel + ".route(".len(); + let mut depth = 1i32; + let mut j = open; + while j < src.len() && depth > 0 { + match bytes[j] { + b'(' => depth += 1, + b')' => depth -= 1, + _ => {} + } + j += 1; + } + let call = &src[open..j.saturating_sub(1)]; + i = j; + let Some(qs) = call.find('"') else { continue }; + let Some(qe) = call[qs + 1..].find('"') else { + continue; + }; + let path = &call[qs + 1..qs + 1 + qe]; + for m in ["get", "post", "put", "delete", "patch"] { + let needle = format!("{m}("); + let mut k = 0; + while let Some(mrel) = call[k..].find(&needle) { + let at = k + mrel; + let boundary = call[..at] + .chars() + .last() + .map(|c| !(c.is_alphanumeric() || c == '_')) + .unwrap_or(true); + if boundary { + if let Some(h) = handler_for_mount(call, m) { + out.push((m.to_uppercase(), path.to_string(), h)); + } + } + k = at + needle.len(); + } + } + } + out + } + + /// SCRAPE-MOUNTS CROSS-CHECK (#195, U4/R5, INV-21b). The read-gate orphan guard + /// above derives its required set from a SOURCE-MARKER scan of `src/api`; this + /// derives it instead from the MOUNTED ROUTE TABLE (`server.rs`). Every mounted + /// repo-scoped GET must be EITHER a driven `ReadGate` row, OR a deny-bearing GET + /// of another class already in the registry (`list_visibility` is an owner-gated + /// GET), OR structurally excused in `READ_GATE_NOT_DRIVEN`, OR a declared public + /// repo-GET (`PUBLIC_REPO_GETS`). A mounted repo-GET that is none of these fails + /// HERE — which catches a read that gates via a helper the marker scan does not + /// recognize: it would still be mounted, so it must be classified rather than + /// slipping past silently. + #[test] + fn mounted_repo_gets_are_driven_excused_or_declared_public() { + let server = include_str!("../src/server.rs"); + let mounts = scrape_mounts_with_handler(server); + + // Integrity floor: the handler-aware walk must find roughly as many mounts + // as the plain scrape; if it collapsed, the cross-check would pass vacuously. + assert!( + mounts.len() >= 90, + "handler-aware mount scrape found only {} routes — the parser likely broke (floor 90)", + mounts.len() + ); + + // Every path that is a driven ReadGate row, and every deny-bearing GET path + // of any class (so the owner-gated `list_visibility` GET is accounted for). + let driven_read_paths: HashSet<&str> = deny_bearing_routes() + .iter() + .filter(|r| r.gate == GateClass::ReadGate) + .map(|r| r.path) + .collect(); + let deny_bearing_get_paths: HashSet<&str> = deny_bearing_routes() + .iter() + .filter(|r| r.method == "GET") + .map(|r| r.path) + .collect(); + let excused_handlers: HashSet<&str> = + READ_GATE_NOT_DRIVEN.iter().map(|(h, _)| *h).collect(); + let public_paths: HashSet<&str> = PUBLIC_REPO_GETS.iter().copied().collect(); + + let mut repo_gets_seen = 0usize; + for (method, path, handler) in &mounts { + if method != "GET" || !is_repo_scoped_path(path) { + continue; + } + repo_gets_seen += 1; + let classified = driven_read_paths.contains(path.as_str()) + || deny_bearing_get_paths.contains(path.as_str()) + || excused_handlers.contains(handler.as_str()) + || public_paths.contains(path.as_str()); + assert!( + classified, + "mounted repo-scoped GET `{path}` (handler `{handler}`) is neither a \ + driven ReadGate row, a deny-bearing GET row, structurally excused in \ + READ_GATE_NOT_DRIVEN, nor a declared public repo-GET — classify it \ + (it may gate via a helper the source-marker scan does not recognize)" + ); + } + + // Non-vacuous floor: the tree mounts well over a dozen repo-scoped GETs; if + // the filter found almost none, the loop above proved nothing. + assert!( + repo_gets_seen >= 15, + "found only {repo_gets_seen} mounted repo-scoped GETs — the path filter \ + or the scrape likely broke (floor 15)" + ); + } + + // ── F3 unit tests: the inline owner-gate did_matches discriminator ─────────── + // + // These pin `has_owner_did_matches` against the EXACT real bodies (positive and + // negative), so the two-arg discriminator can't silently regress into matching + // the signer-self / author forms that share the helper. + + #[test] + fn has_owner_did_matches_catches_the_protect_branch_owner_idiom() { + // protect.rs:28 form: first arg the caller, second `&record.owner_did`. + let body = "let caller = &auth.0;\n\ + if !crate::api::did_matches(caller, &record.owner_did) {\n\ + return Err(AppError::Forbidden(\"only the owner\".into()));\n\ + }"; + assert!( + has_owner_did_matches(body), + "the protect_branch owner idiom must be recognized" + ); + } + + #[test] + fn has_owner_did_matches_catches_the_auth0_owner_form() { + // The `&auth.0` first-arg spelling (repos.rs's branch-protection check), + // tolerating a newline before the second arg. + let body = + "if x\n && !crate::api::did_matches(&auth.0,\n &record.owner_did)\n {"; + assert!( + has_owner_did_matches(body), + "the &auth.0 / owner_did form must be recognized across a newline" + ); + } + + #[test] + fn has_owner_did_matches_ignores_register_replica_signer_self() { + // replicas.rs:52: FIRST arg is `replica_did`, NOT the caller — signer-self + // by design. The second arg is owner_did, so a naive "second arg is + // owner_did" check would wrongly flag it. It must NOT match. + let body = "if crate::api::did_matches(replica_did, &record.owner_did) {\n\ + // the signer is registering itself as a replica\n\ + }"; + assert!( + !has_owner_did_matches(body), + "register_replica's signer-self did_matches must NOT be treated as an owner gate" + ); + } + + #[test] + fn has_owner_did_matches_ignores_close_pr_author_form() { + // pulls.rs:277: caller-first but SECOND arg is `&pr.author_did`, not + // owner_did — the owner-or-author close gate. Must NOT match as owner-only. + let body = "let is_author = crate::api::did_matches(&auth.0, &pr.author_did);"; + assert!( + !has_owner_did_matches(body), + "close_pr's author did_matches must NOT be treated as an owner gate" + ); + } + + #[test] + fn has_owner_did_matches_ignores_bounty_and_task_self_forms() { + // bounties.rs / tasks.rs: caller-first, but the second arg is a + // creator/delegator/assignee did, never owner_did. + for body in [ + "if !crate::api::did_matches(&auth.0, &bounty.creator_did) {", + "if !crate::api::did_matches(&auth.0, &body.delegator_did) {", + "if !crate::api::did_matches(&auth.0, &body.assignee_did) {", + ] { + assert!( + !has_owner_did_matches(body), + "self/author did_matches form must NOT be treated as an owner gate: {body}" + ); + } + } + + // ── F3 unit tests: the caller-self did_matches discriminator ───────────────── + + #[test] + fn has_signer_self_did_matches_fires_on_caller_self_and_ignores_owner_form() { + // Fires: caller-spelled first arg (bound via `let caller = &auth.0;`), a + // non-owner second arg, the field-binding form the task handlers use. Proves + // the arg1 set is honored (a `&auth.0`-only classifier would miss this). + let self_form = "let caller = &auth.0;\n\ + if !crate::api::did_matches(caller, &x.foo_did) {\n\ + return Err(forbidden(\"...\"));\n\ + }"; + assert!( + has_signer_self_did_matches(self_form), + "the caller-self field-binding did_matches must be recognized" + ); + // The `&auth.0` spelling with a non-owner second arg fires too. + let auth0_form = "if !crate::api::did_matches(&auth.0, &body.assignee_did) {"; + assert!( + has_signer_self_did_matches(auth0_form), + "the &auth.0 caller-self form must be recognized" + ); + // Does NOT fire on the owner form (that is has_owner_did_matches' job), so a + // caller-self classifier that matched it would double-count. + let owner_form = "if !crate::api::did_matches(caller, &record.owner_did) {"; + assert!( + !has_signer_self_did_matches(owner_form), + "the owner form must NOT be treated as a caller-self gate (no double-count)" + ); + } +} diff --git a/crates/gitlawb-node/tests/support/mod.rs b/crates/gitlawb-node/tests/support/mod.rs index 75828f5f..8b5bf093 100644 --- a/crates/gitlawb-node/tests/support/mod.rs +++ b/crates/gitlawb-node/tests/support/mod.rs @@ -3,4 +3,22 @@ //! node spawner (U3). pub mod assert; +// U2 fixture state matrix + per-gate-class probe generators; driven by U3. +#[allow(dead_code)] +pub mod probe; +// U1 deny-bearing route registry; fields consumed by U2/U3 as they land. +#[allow(dead_code)] +pub mod routes; pub mod signing; + +/// A reqwest client with a bounded request timeout, so a wedged git subprocess or +/// route fails the suite rather than hanging it until CI kills the job (#195, F1). +/// The two real-node probes that drive git subprocesses use this; the inline +/// twins at the upload-pack and registry-sweep sites use the same pattern. +#[allow(dead_code)] +pub fn bounded_client() -> reqwest::Client { + reqwest::Client::builder() + .timeout(std::time::Duration::from_secs(30)) + .build() + .unwrap() +} diff --git a/crates/gitlawb-node/tests/support/probe.rs b/crates/gitlawb-node/tests/support/probe.rs new file mode 100644 index 00000000..fcd2f740 --- /dev/null +++ b/crates/gitlawb-node/tests/support/probe.rs @@ -0,0 +1,1406 @@ +//! U2: fixture state matrix + per-gate-class probe generators. +//! +//! The fixture is a STATE MATRIX, not one repo: owner-gate rows need a repo the +//! signed stranger can REACH (so the request hits the owner gate, not an +//! existence-hiding 404), and read-gate rows need a repo the caller CANNOT see. +//! A single repo cannot be both, so we seed a public repo (owner-gate substrate) +//! and a private repo with content (read-gate substrate). +//! +//! Each deny-bearing row expands to probes: the hostile request (asserting the +//! exact deny) plus a positive twin (owner-reachability for owner-gate, +//! reader/sibling-public read for read-gate) so a 403/404 for the wrong reason +//! cannot false-pass. + +use gitlawb_core::identity::Keypair; +use gitlawb_node::test_harness::TestNode; + +use super::routes::{GateClass, IdSource, Principal, Reach, Row}; + +/// A seeded two-repo state matrix plus the owner and stranger identities. +pub struct Fixture { + pub owner: Keypair, + pub stranger: Keypair, + /// The PR/issue author — distinct from owner and stranger — so the + /// owner-OR-author close gates (#195, F1) drive their author arm with an + /// identity that is NOT the owner (the original bug seeded author == owner). + pub author: Keypair, + /// The bounty creator, distinct from every other identity. + pub creator: Keypair, + /// The bounty claimant, distinct from every other identity. + pub claimant: Keypair, + /// The task assignee, distinct from every other identity (#195, F3). The + /// complete/fail actor gates drive their Assignee arm as this identity, so it + /// must differ from owner and stranger or the arm cannot false-pass on an + /// owner fallback. + pub assignee: Keypair, + pub owner_did: String, + pub author_did: String, + pub creator_did: String, + pub claimant_did: String, + pub assignee_did: String, + /// The id (UUID) of the seeded disputable bounty, filled into the + /// `dispute_bounty` row's `{id}` placeholder (IdSource::BountyId). + pub bounty_id: String, + /// #195 (N2): one bounty per status-gated bounty mutation row. submit/approve/ + /// cancel check the bounty's STATUS before their 403 auth gate, so the stranger + /// hostile only exercises the gate against a bounty in the gate-reaching + /// status, and each row's authorized twin CONSUMES that status; the rows + /// cannot share a bounty. This one stays "claimed" (create + claim) for the + /// submit_bounty row (IdSource::ClaimedBountyId). + pub claimed_bounty_id: String, + /// Driven to "submitted" (create + claim + submit) for the approve_bounty row + /// (IdSource::SubmittedBountyId). + pub submitted_bounty_id: String, + /// Left "open" (create only) for the cancel_bounty row (IdSource::OpenBountyId). + pub open_bounty_id: String, + /// #195 (F3): a task seeded "pending" (delegated by the owner) for the + /// claim_task caller-self row (IdSource::ClaimableTaskId). + pub claimable_task_id: String, + /// A task seeded "claimed" by the assignee for the complete_task actor row + /// (IdSource::CompletableTaskId). + pub completable_task_id: String, + /// A second assignee-claimed task for the fail_task actor row + /// (IdSource::FailableTaskId). + pub failable_task_id: String, + /// The id (UUID) of the seeded issue (authored by `author`), filled into the + /// `close_issue` row's `{id}` placeholder (IdSource::IssueId). + pub issue_id: String, + /// The id (UUID) of an issue seeded in the PRIVATE repo (marker in its title), + /// filled into the get_issue / list_issue_comments read-gate rows + /// (IdSource::PrivIssueId). Distinct from `issue_id` (the public close issue). + pub priv_issue_id: String, + /// The id (UUID) of a bounty seeded against the PRIVATE repo (marker in its + /// title), filled into the get_bounty read-gate row (IdSource::PrivBountyId). + pub priv_bounty_id: String, + /// The id (UUID) of a ref-certificate issued by a real owner push to the + /// private repo, filled into the get_cert read-gate row (IdSource::CertId). + pub cert_id: String, + /// Per-read secret markers seeded into the private sub-entities (issue title, + /// PR title, PR diff, bounty title). Each read's own marker is added to + /// that read's withheld set so a 404 leaking THAT read's private content fails + /// (#195, R3/KTD-4). A union is used so every seeded marker is withheld from + /// every read-gate hostile — a per-read withheld token, never a status-only + /// 404 check. + pub priv_markers: Vec, + /// Public repo: owner-gate rows run against this so the stranger reaches the + /// owner gate rather than a hidden-repo 404. + pub public_repo: String, + /// The public repo's id, needed to seed the PR the close-gate rows require. + pub public_repo_id: String, + /// Private repo (is_public=false) with seeded content: read-gate rows run + /// against this so an anon caller gets the existence-hiding 404 and the owner + /// twin gets 2xx. + pub private_repo: String, + /// The private repo's internal id (UUID). A read-gate denial body must never + /// echo it: a 404 that serializes the private record's internal id has leaked + /// repository metadata just as much as the blob content (#195, F4). + pub private_repo_id: String, + /// A seeded blob path present in both repos, for `{*path}` reads. + pub content_path: String, + /// The DISTINCTIVE secret bytes seeded into the private repo's blob. A + /// read-gate denial body must never echo this (INV-8: the sweep claims the + /// denial "leaks nothing", so it must actually check for the private content). + pub private_secret: String, + /// The private blob's full sha1 object id, and its short prefix. Neither may + /// appear in a read-gate denial body — a 404 that names the withheld object's + /// OID has leaked its existence. + pub private_blob_oid: String, + pub private_blob_oid_short: String, +} + +impl Fixture { + pub async fn seed(node: &TestNode) -> Fixture { + let owner = Keypair::generate(); + let owner_did = owner.did().to_string(); + let stranger = Keypair::generate(); + // #195 (F1): distinct identities for each authorizing arm of the + // multi-principal gates. All four must differ from owner and stranger and + // from each other, or an arm silently collapses onto another and reverting + // it stays invisible (the original author == owner bug). + let author = Keypair::generate(); + let author_did = author.did().to_string(); + let creator = Keypair::generate(); + let creator_did = creator.did().to_string(); + let claimant = Keypair::generate(); + let claimant_did = claimant.did().to_string(); + // #195 (F3): a distinct assignee identity (!= owner, != stranger) for the + // complete/fail actor gates, so their Assignee arm cannot false-pass on an + // owner fallback. + let assignee = Keypair::generate(); + let assignee_did = assignee.did().to_string(); + let content_path = "public/a.txt".to_string(); + + let pub_repo = "prober-pub"; + let public_repo_id = node.seed_repo(&owner_did, pub_repo, true).await; + node.seed_bare_repo( + &owner_did, + pub_repo, + &[("public/a.txt", "pub content")], + "sha1", + ); + // The author-or-owner close gates load the PR/issue before they run, so + // the close rows need the entity to exist. Seed both authored by `author` + // (NOT the owner) so the author arm is granted by a non-owner identity and + // reverting either arm is observable. The PR seeds via the DB (author_did + // is a real column). The issue is seeded over HTTP as `author` (below): + // close_issue deserializes the WHOLE git-JSON IssueRecord before reading + // its author, so a bare `{"author":..}` DB-seed fails to deserialize and + // the author falls back to None — the author arm would then 403. + node.seed_pr(&public_repo_id, 1, &author_did).await; + let issue_id = seed_authored_issue(node, &owner_did, pub_repo, &author).await; + + // Seed a DISPUTABLE bounty (status "claimed", creator + claimant set) so the + // dispute_bounty creator/claimant arms reach their auth check. dispute_bounty + // gates on creator/claimant, NOT the repo, so the bounty is created on the + // public repo (both signers can read it). Seeded over HTTP through the real + // handlers — create (creator) then claim (claimant) — because the DB seam is + // private to TestNode and a test-only seed method would be a src/ change. + let bounty_id = seed_bounty_at_stage( + node, + &owner_did, + pub_repo, + &creator, + &claimant, + "prober dispute bounty", + BountyStage::Claimed, + ) + .await; + + // #195 (N2): submit/approve/cancel check the bounty's STATUS before their + // 403 auth gate, so each row gets its OWN bounty in the gate-reaching + // status (claimed / submitted / open); the authorized twins consume that + // status, so the rows cannot share a bounty with each other or with the + // disputable one above. + let claimed_bounty_id = seed_bounty_at_stage( + node, + &owner_did, + pub_repo, + &creator, + &claimant, + "prober submit bounty", + BountyStage::Claimed, + ) + .await; + let submitted_bounty_id = seed_bounty_at_stage( + node, + &owner_did, + pub_repo, + &creator, + &claimant, + "prober approve bounty", + BountyStage::Submitted, + ) + .await; + let open_bounty_id = seed_bounty_at_stage( + node, + &owner_did, + pub_repo, + &creator, + &claimant, + "prober cancel bounty", + BountyStage::Open, + ) + .await; + + let priv_repo = "prober-priv"; + // Seed the private blob with an UNMISTAKABLE marker (not "priv content"), + // so a read-gate denial body that echoes the content is caught as a leak + // rather than blending into generic prose. The OID is captured so the OID + // (full + short) is withheld too — a 404 naming the object's id has leaked + // its existence just as much as its bytes. + let private_secret = "TOPSECRET-PRIVREAD-U3".to_string(); + let private_repo_id = node.seed_repo(&owner_did, priv_repo, false).await; + let priv_oids = node.seed_bare_repo( + &owner_did, + priv_repo, + &[("public/a.txt", &private_secret)], + "sha1", + ); + let private_blob_oid = priv_oids["public/a.txt"].clone(); + let private_blob_oid_short = private_blob_oid[..12].to_string(); + // The private repo's current `main` commit tip, needed as the `old` oid of + // the ref-update that force-points main at the deterministic push below. + let private_main_tip = priv_oids["HEAD"].clone(); + + // #195 (F2/U3): seed the private sub-entities the deferred reads return, each + // carrying its OWN distinctive marker so the per-read leak assertion is + // load-bearing. All are seeded as the OWNER (the only reader of the private + // repo), so anon/stranger get the existence-hiding 404 and the owner twin + // gets a 2xx that actually returns the entity. + // + // A real owner push to the private repo advances a `feature` branch with a + // marker file (so get_pr_diff's branch_diff_names(main, feature) is + // NON-EMPTY) AND issues a ref-certificate (get_cert). Do it first so the PR's + // source branch exists before create_pr and the cert id is captured. + let pr_diff_marker = "TOPSECRET-PRDIFF-U3".to_string(); + let cert_id = seed_private_push_and_cert( + node, + &owner, + &owner_did, + priv_repo, + &private_main_tip, + &private_secret, + &private_blob_oid, + &pr_diff_marker, + ) + .await; + + let issue_marker = "TOPSECRET-ISSUE-U3".to_string(); + let priv_issue_id = + seed_private_issue(node, &owner, &owner_did, priv_repo, &issue_marker).await; + + let pr_marker = "TOPSECRET-PRTITLE-U3".to_string(); + seed_private_pr(node, &owner, &owner_did, priv_repo, &pr_marker).await; + + let bounty_marker = "TOPSECRET-BOUNTY-U3".to_string(); + let priv_bounty_id = + seed_private_bounty(node, &owner, &owner_did, priv_repo, &bounty_marker).await; + + let priv_markers = vec![issue_marker, pr_marker, pr_diff_marker, bounty_marker]; + + // #195 (F3): seed the caller-self task entities through the real task write + // routes (behind add_auth_layers only, no icaptcha). The claimable task + // stays "pending" so its claim control (the owner) can claim it; the + // completable and failable tasks are claimed by the assignee, so + // complete_task / fail_task reach their assignee actor gate against a + // "claimed" task. + let claimable_task_id = seed_pending_task(node, &owner).await; + let completable_task_id = seed_claimed_task(node, &owner, &assignee).await; + let failable_task_id = seed_claimed_task(node, &owner, &assignee).await; + + Fixture { + owner, + stranger, + author, + creator, + claimant, + assignee, + owner_did, + author_did, + creator_did, + claimant_did, + assignee_did, + bounty_id, + claimed_bounty_id, + submitted_bounty_id, + open_bounty_id, + claimable_task_id, + completable_task_id, + failable_task_id, + issue_id, + priv_issue_id, + priv_bounty_id, + cert_id, + priv_markers, + public_repo: pub_repo.to_string(), + public_repo_id, + private_repo: priv_repo.to_string(), + private_repo_id, + content_path, + private_secret, + private_blob_oid, + private_blob_oid_short, + } + } +} + +/// Seed an issue authored by `author` through the real `create_issue` handler +/// and return its minted UUID id. Written as a full git-JSON `IssueRecord` (the +/// handler serializes the whole record), so `close_issue`'s deserialize-then-read +/// author path finds the author — which a bare-field DB seed cannot satisfy. +async fn seed_authored_issue( + node: &TestNode, + owner_did: &str, + repo: &str, + author: &Keypair, +) -> String { + use super::signing::signed_request; + + let client = reqwest::Client::new(); + let path = format!("/api/v1/repos/{owner_did}/{repo}/issues"); + let body = br#"{"title":"prober close issue"}"#.to_vec(); + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &path, + body, + author, + ) + .header("content-type", "application/json") + .send() + .await + .expect("create issue sends"); + assert_eq!( + resp.status().as_u16(), + 201, + "seeding: issue create must return 201 (author can read the public repo)" + ); + let created: serde_json::Value = resp.json().await.expect("issue create returns JSON"); + created["id"] + .as_str() + .expect("created issue carries an id") + .to_string() +} + +/// Create a "pending" task delegated by `delegator` through the real `create_task` +/// handler and return its minted id. `create_task` binds `delegator_did` to the +/// signer, so the body names the signer's DID. Task write routes sit behind +/// `add_auth_layers` only (no icaptcha), so a signed create reaches the handler. +async fn seed_pending_task(node: &TestNode, delegator: &Keypair) -> String { + use super::signing::signed_request; + + let client = reqwest::Client::new(); + let delegator_did = delegator.did().to_string(); + let body = + format!(r#"{{"kind":"review","capability":"read","delegator_did":"{delegator_did}"}}"#) + .into_bytes(); + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + "/api/v1/tasks", + body, + delegator, + ) + .header("content-type", "application/json") + .send() + .await + .expect("create task sends"); + assert_eq!( + resp.status().as_u16(), + 201, + "seeding: task create must return 201 (delegator_did binds to the signer)" + ); + let created: serde_json::Value = resp.json().await.expect("task create returns JSON"); + created["id"] + .as_str() + .expect("created task carries an id") + .to_string() +} + +/// Create a "pending" task (delegated by `owner`), then claim it as `assignee` +/// (`claim_task` binds `assignee_did` to the signer), leaving it "claimed" so +/// `complete_task` / `fail_task` reach their assignee actor gate. Returns the id. +async fn seed_claimed_task(node: &TestNode, owner: &Keypair, assignee: &Keypair) -> String { + use super::signing::signed_request; + + let id = seed_pending_task(node, owner).await; + + let client = reqwest::Client::new(); + let assignee_did = assignee.did().to_string(); + let claim_path = format!("/api/v1/tasks/{id}/claim"); + let body = format!(r#"{{"assignee_did":"{assignee_did}"}}"#).into_bytes(); + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &claim_path, + body, + assignee, + ) + .header("content-type", "application/json") + .send() + .await + .expect("claim task sends"); + assert_eq!( + resp.status().as_u16(), + 200, + "seeding: task claim must return 200 (assignee_did binds to the signer)" + ); + id +} + +/// How far along its lifecycle a seeded bounty is driven, expressed as the real +/// API transitions that get it there. Each status-gated bounty row needs the +/// status its handler's pre-auth check demands (#195, N2): cancel needs "open", +/// submit (and dispute) need "claimed", approve needs "submitted". +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum BountyStage { + /// `create_bounty` only: status "open". + Open, + /// create + claim: status "claimed" (creator + claimant set). + Claimed, + /// create + claim + submit: status "submitted". + Submitted, +} + +/// Seed a bounty on the (public) repo, drive it to `stage`, and return its id. +/// +/// Driven through the real HTTP handlers rather than the DB: `create_bounty` +/// (creator) mints a UUID id we read back from the response, `claim_bounty` +/// (claimant) sets the claimant, `submit_bounty` (claimant) records a PR id. +/// All read-gate the (public) repo, which the creator/claimant can read, and the +/// claim/submit transitions pass their own auth gates as the claimant. A Claimed +/// bounty's deadline has not been exceeded, so a creator/claimant dispute returns +/// 400 (not 403) after passing auth, which is still a Not403 twin, while a +/// stranger is denied 403 at the auth check first. +async fn seed_bounty_at_stage( + node: &TestNode, + owner_did: &str, + repo: &str, + creator: &Keypair, + claimant: &Keypair, + title: &str, + stage: BountyStage, +) -> String { + use super::signing::signed_request; + + let client = reqwest::Client::new(); + + // create_bounty (creator) -> 201 with the minted BountyRecord (carries the id). + let create_path = format!("/api/v1/repos/{owner_did}/{repo}/bounties"); + let body = format!(r#"{{"title":"{title}","amount":1}}"#).into_bytes(); + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &create_path, + body, + creator, + ) + .header("content-type", "application/json") + .send() + .await + .expect("create bounty sends"); + assert_eq!( + resp.status().as_u16(), + 201, + "seeding: bounty create must return 201 (creator can read the public repo)" + ); + let created: serde_json::Value = resp.json().await.expect("bounty create returns JSON"); + let bounty_id = created["id"] + .as_str() + .expect("created bounty carries an id") + .to_string(); + let creator_did = creator.did().to_string(); + assert_eq!( + created["creator_did"].as_str(), + Some(creator_did.as_str()), + "seeded bounty creator must be the fixture creator" + ); + if stage == BountyStage::Open { + return bounty_id; + } + + // claim_bounty (claimant) -> 200, status becomes "claimed", claimant recorded. + let claim_path = format!("/api/v1/bounties/{bounty_id}/claim"); + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &claim_path, + br#"{}"#.to_vec(), + claimant, + ) + .header("content-type", "application/json") + .send() + .await + .expect("claim bounty sends"); + assert_eq!( + resp.status().as_u16(), + 200, + "seeding: bounty claim must return 200 (claimant can read the public repo)" + ); + if stage == BountyStage::Claimed { + return bounty_id; + } + + // submit_bounty (claimant) -> 200, status becomes "submitted". pr_id is a + // free-text column; the seed value only needs to clear serde. + let submit_path = format!("/api/v1/bounties/{bounty_id}/submit"); + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &submit_path, + br#"{"pr_id":"1"}"#.to_vec(), + claimant, + ) + .header("content-type", "application/json") + .send() + .await + .expect("submit bounty sends"); + assert_eq!( + resp.status().as_u16(), + 200, + "seeding: bounty submit must return 200 (the claimant passes the submit gate)" + ); + + bounty_id +} + +/// Seed an issue in the PRIVATE repo (as the owner, the only reader) whose title +/// carries `marker`, and return its id. get_issue returns the whole IssueRecord, +/// so a 404 that echoes the title has leaked private content; the marker is added +/// to the withheld set. +async fn seed_private_issue( + node: &TestNode, + owner: &Keypair, + owner_did: &str, + repo: &str, + marker: &str, +) -> String { + use super::signing::signed_request; + + let client = reqwest::Client::new(); + let path = format!("/api/v1/repos/{owner_did}/{repo}/issues"); + let body = format!(r#"{{"title":"{marker}"}}"#).into_bytes(); + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &path, + body, + owner, + ) + .header("content-type", "application/json") + .send() + .await + .expect("create private issue sends"); + assert_eq!( + resp.status().as_u16(), + 201, + "seeding: private issue create must return 201 (owner reads its own private repo)" + ); + let created: serde_json::Value = resp.json().await.expect("issue create returns JSON"); + created["id"] + .as_str() + .expect("created private issue carries an id") + .to_string() +} + +/// Seed a PR in the PRIVATE repo (as the owner) with `marker` in its title and +/// `feature` as its source branch (which the prior push created). get_pr returns +/// the whole PullRequest and get_pr_diff a real non-empty diff, so both the title +/// marker and the diff marker are withheld. Lands at PR number 1 (first PR). +async fn seed_private_pr( + node: &TestNode, + owner: &Keypair, + owner_did: &str, + repo: &str, + marker: &str, +) { + use super::signing::signed_request; + + let client = reqwest::Client::new(); + let path = format!("/api/v1/repos/{owner_did}/{repo}/pulls"); + let body = + format!(r#"{{"title":"{marker}","source_branch":"feature","target_branch":"main"}}"#) + .into_bytes(); + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &path, + body, + owner, + ) + .header("content-type", "application/json") + .send() + .await + .expect("create private PR sends"); + assert_eq!( + resp.status().as_u16(), + 201, + "seeding: private PR create must return 201 (owner reads its own private repo)" + ); + let created: serde_json::Value = resp.json().await.expect("PR create returns JSON"); + assert_eq!( + created["number"].as_i64(), + Some(1), + "seeded private PR must be number 1 (the read-gate rows fill {{number}} = 1)" + ); +} + +/// Seed a bounty against the PRIVATE repo (as the owner) with `marker` in its +/// title, and return its id. get_bounty read-gates the bounty's own repo, so +/// anon/stranger get a 404 while the owner gets the bounty JSON (title marker +/// withheld from the 404). +async fn seed_private_bounty( + node: &TestNode, + owner: &Keypair, + owner_did: &str, + repo: &str, + marker: &str, +) -> String { + use super::signing::signed_request; + + let client = reqwest::Client::new(); + let path = format!("/api/v1/repos/{owner_did}/{repo}/bounties"); + let body = format!(r#"{{"title":"{marker}","amount":1}}"#).into_bytes(); + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &path, + body, + owner, + ) + .header("content-type", "application/json") + .send() + .await + .expect("create private bounty sends"); + assert_eq!( + resp.status().as_u16(), + 201, + "seeding: private bounty create must return 201 (owner reads its own private repo)" + ); + let created: serde_json::Value = resp.json().await.expect("bounty create returns JSON"); + created["id"] + .as_str() + .expect("created private bounty carries an id") + .to_string() +} + +/// Point the PRIVATE repo's `main` at a deterministic commit and add a `feature` +/// branch (a marker file on top of it) via a real owner git-receive-pack push, +/// which also makes the node issue a ref-certificate. Returns the issued cert's +/// id, read back from the owner's list_certs. +/// +/// seed_bare_repo's `main` tip is non-deterministic (its commit dates are ambient), +/// so we cannot build a `feature` that descends from it without the parent objects. +/// Instead we push a full (non-thin) history: a fresh `main` commit and `feature` +/// on top, force-updating the served `main` (the ref-update carries the server's +/// current tip as its `old` oid; receive.denyNonFastForwards is off by default, so +/// the non-fast-forward update applies). After the push, `feature` descends from +/// the new `main`, so get_pr_diff's `git diff main...feature` has a merge base and +/// returns the marker file. The push is signed as the owner (require_signature) and, +/// on a non-protected branch of an owner-owned repo, lands cleanly and issues a cert. +#[allow(clippy::too_many_arguments)] +async fn seed_private_push_and_cert( + node: &TestNode, + owner: &Keypair, + owner_did: &str, + repo: &str, + server_main_tip: &str, + blob_contents: &str, + expected_blob_oid: &str, + marker_file_contents: &str, +) -> String { + use super::signing::signed_request; + use std::process::Command; + + let work = std::env::temp_dir().join(format!( + "gl-u3-certpush-{}-{}", + std::process::id(), + server_main_tip + )); + let _ = std::fs::remove_dir_all(&work); + std::fs::create_dir_all(&work).expect("create cert-push workdir"); + let run = |args: &[&str], cwd: &std::path::Path| -> String { + let out = Command::new("git") + .args(args) + .current_dir(cwd) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + + // Pin sha1 to match the served fixture repo; init.defaultObjectFormat=sha256 skews the OIDs. + run(&["init", "-q", "-b", "main", "--object-format=sha1"], &work); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + // Keep `public/a.txt` (the seeded blob) on the new main so the get_blob / + // get_tree owner twins still resolve it — the blob OID depends only on content, + // so preserving the bytes preserves `private_blob_oid`. + std::fs::create_dir_all(work.join("public")).expect("mk public dir"); + std::fs::write(work.join("public/a.txt"), blob_contents).expect("seed blob file"); + run(&["add", "public/a.txt"], &work); + run(&["commit", "-q", "-m", "u3 main"], &work); + let new_main = run(&["rev-parse", "HEAD"], &work); + let new_blob = run(&["rev-parse", "HEAD:public/a.txt"], &work); + assert_eq!( + new_blob, expected_blob_oid, + "the pushed public/a.txt blob OID must equal the fixture's private_blob_oid" + ); + + // feature = new main + a marker file. + run(&["checkout", "-q", "-b", "feature"], &work); + std::fs::write(work.join("feature.txt"), marker_file_contents).expect("seed marker file"); + run(&["add", "feature.txt"], &work); + run(&["commit", "-q", "-m", "feature commit"], &work); + let feature_tip = run(&["rev-parse", "HEAD"], &work); + + // Full (non-thin) pack of both branches' objects — the server keeps its old + // main objects but needs all of the new ones since feature does not descend + // from the server's current main. + let pack = { + let out = Command::new("git") + .args(["pack-objects", "--stdout", "--revs"]) + .current_dir(&work) + .stdin(std::process::Stdio::piped()) + .stdout(std::process::Stdio::piped()) + .spawn() + .expect("spawn pack-objects"); + use std::io::Write; + out.stdin + .as_ref() + .expect("pack-objects stdin") + .write_all(format!("{new_main}\n{feature_tip}\n").as_bytes()) + .expect("write revs"); + let done = out.wait_with_output().expect("pack-objects completes"); + assert!(done.status.success(), "pack-objects must succeed"); + done.stdout + }; + + // git v0 receive-pack body: two ref-update pkt-lines — force-update main + // (old = the server's current tip) and create feature — the first carrying the + // capabilities after a NUL, a flush, then the packfile. + let zero = "0".repeat(server_main_tip.len()); + let l1 = format!("{server_main_tip} {new_main} refs/heads/main\0report-status\n"); + let l2 = format!("{zero} {feature_tip} refs/heads/feature\n"); + let mut body: Vec = Vec::new(); + body.extend(format!("{:04x}{l1}", l1.len() + 4).into_bytes()); + body.extend(format!("{:04x}{l2}", l2.len() + 4).into_bytes()); + body.extend_from_slice(b"0000"); + body.extend_from_slice(&pack); + + let client = reqwest::Client::new(); + let push_path = format!("/{owner_did}/{repo}/git-receive-pack"); + let resp = signed_request( + &client, + reqwest::Method::POST, + &node.base_url, + &push_path, + body, + owner, + ) + .header("content-type", "application/x-git-receive-pack-request") + .send() + .await + .expect("receive-pack push sends"); + assert_eq!( + resp.status().as_u16(), + 200, + "seeding: owner push to a non-protected feature branch must return 200 so a cert issues" + ); + + let _ = std::fs::remove_dir_all(&work); + + // Read the issued cert id back from the owner's list_certs on the private repo. + let certs_path = format!("/api/v1/repos/{owner_did}/{repo}/certs"); + let resp = signed_request( + &client, + reqwest::Method::GET, + &node.base_url, + &certs_path, + Vec::new(), + owner, + ) + .send() + .await + .expect("list_certs sends"); + assert_eq!( + resp.status().as_u16(), + 200, + "seeding: owner list_certs must return 200" + ); + let listed: serde_json::Value = resp.json().await.expect("list_certs returns JSON"); + listed["certificates"][0]["id"] + .as_str() + .expect("a ref-certificate must have been issued by the push") + .to_string() +} + +/// Who signs a probe request. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Signer { + Anon, + Owner, + Stranger, + /// The PR/issue author arm of a multi-principal close gate. + Author, + /// The bounty creator arm of dispute_bounty. + Creator, + /// The bounty claimant arm of dispute_bounty. + Claimant, + /// The task assignee arm of complete_task / fail_task (#195, F3). + Assignee, +} + +/// The signer identity that grants a given multi-principal arm. The runtime sweep +/// resolves each to its fixture keypair; the structural consistency test uses it +/// to confirm every declared arm has a twin signed by ITS identity. +pub fn signer_for_principal(p: Principal) -> Signer { + match p { + Principal::Owner => Signer::Owner, + Principal::Author => Signer::Author, + Principal::Creator => Signer::Creator, + Principal::Claimant => Signer::Claimant, + Principal::Assignee => Signer::Assignee, + } +} + +/// What a probe expects. +#[derive(Debug, Clone)] +pub enum Expect { + /// Hostile request: the exact deny status, body leaking none of `withheld`. + Deny(u16), + /// Owner-gate twin: the owner reaches the handler, so NOT 403. + Not403, + /// Read-gate twin: an authorized read returns a non-empty 2xx (an empty 2xx + /// would be a denial rendered as success). + Ok2xx, +} + +/// A single request to drive against the node. +#[derive(Debug, Clone)] +pub struct Probe { + pub label: String, + pub method: reqwest::Method, + /// Path used both for the URL and for RFC-9421 `@path` signing. + pub path: String, + pub body: Vec, + pub signer: Signer, + pub json: bool, + pub expect: Expect, + /// Private tokens (secret content, blob OID) that the DENY body must not echo. + /// Non-empty only for read-gate hostile probes, where a private repo/blob is + /// actually seeded; the owner-gate (403) and signature (401) hostile probes + /// fire on NO_ENTITY repos before any entity lookup, so there is genuinely + /// nothing seeded to leak and an empty list is the honest assertion there. + pub withheld: Vec, +} + +/// Substitute path-template placeholders from the fixture. `repo` is the public +/// repo for owner-gate rows and the private repo for read-gate rows. +/// +/// `id_source` selects what `{id}` resolves to: `Fixed` uses the static seed id +/// `"1"` (entities seeded at #1), while a per-row source such as `BountyId` +/// injects a UUID minted at seed time. This is the general id-threading hook U3 +/// reuses for the other id-keyed reads (a cert id, etc.): add an `IdSource` +/// variant + fixture field and map it here. +fn fill(path: &str, fixture: &Fixture, repo: &str, id_source: IdSource) -> String { + let id = match id_source { + IdSource::Fixed => "1", + IdSource::BountyId => fixture.bounty_id.as_str(), + IdSource::IssueId => fixture.issue_id.as_str(), + IdSource::PrivIssueId => fixture.priv_issue_id.as_str(), + IdSource::PrivBountyId => fixture.priv_bounty_id.as_str(), + IdSource::CertId => fixture.cert_id.as_str(), + // #195 (N2): the status-gated bounty mutations, each keyed to the bounty + // seeded in its gate-reaching status. + IdSource::ClaimedBountyId => fixture.claimed_bounty_id.as_str(), + IdSource::SubmittedBountyId => fixture.submitted_bounty_id.as_str(), + IdSource::OpenBountyId => fixture.open_bounty_id.as_str(), + // #195 (F3): the caller-self task rows, each keyed to a seeded task in the + // state its gate needs (pending for claim, claimed for complete/fail). + IdSource::ClaimableTaskId => fixture.claimable_task_id.as_str(), + IdSource::CompletableTaskId => fixture.completable_task_id.as_str(), + IdSource::FailableTaskId => fixture.failable_task_id.as_str(), + }; + path.replace("{owner}", &fixture.owner_did) + .replace("{repo}", repo) + .replace("{number}", "1") + .replace("{id}", id) + .replace("{label}", "bug") + .replace("{branch}", "main") + .replace("{*path}", &fixture.content_path) + .replace("{path}", &fixture.content_path) +} + +/// Expand one deny-bearing row into its hostile probe plus positive twin. +pub fn probes_for(row: &Row, fixture: &Fixture) -> Vec { + let method: reqwest::Method = row.method.parse().expect("valid method"); + let body = row.body.map(|b| b.as_bytes().to_vec()).unwrap_or_default(); + let json = row.body.is_some(); + + match row.gate { + GateClass::OwnerGate => { + let path = fill(row.path, fixture, &fixture.public_repo, row.id_source); + vec![ + Probe { + label: format!("{} owner-gate hostile", row.handler), + method: method.clone(), + path: path.clone(), + body: body.clone(), + signer: Signer::Stranger, + json, + expect: Expect::Deny(403), + // No entity is seeded on the public owner-gate substrate before + // the 403 fires, so there is nothing private to leak here. + withheld: Vec::new(), + }, + Probe { + label: format!("{} owner-reachability twin", row.handler), + method, + path, + body, + signer: Signer::Owner, + json, + expect: Expect::Not403, + withheld: Vec::new(), + }, + ] + } + GateClass::MultiPrincipalGate => { + // #195 (F1): a single stranger-403 hostile plus one Not403 twin PER + // declared arm. Each arm is signed by ITS distinct fixture identity, so + // reverting any single arm in the handler turns that arm's twin RED + // while the others (and the stranger 403) stay green. + let path = fill(row.path, fixture, &fixture.public_repo, row.id_source); + let mut v = vec![Probe { + label: format!("{} multi-principal hostile (stranger)", row.handler), + method: method.clone(), + path: path.clone(), + body: body.clone(), + signer: Signer::Stranger, + json, + expect: Expect::Deny(403), + // The gate fires before returning any entity; nothing private is + // seeded to leak on this substrate. + withheld: Vec::new(), + }]; + for arm in row.principals { + let signer = signer_for_principal(*arm); + v.push(Probe { + label: format!("{} arm twin ({:?})", row.handler, arm), + method: method.clone(), + path: path.clone(), + body: body.clone(), + signer, + json, + expect: Expect::Not403, + withheld: Vec::new(), + }); + } + v + } + GateClass::ReadGate => { + let path = fill(row.path, fixture, &fixture.private_repo, row.id_source); + // INV-8: the private repo's blob and its OID are actually seeded here, so + // the 404 body must echo neither the secret content nor the blob OID + // (full or short) nor the private repo's internal id (#195, F4). An empty + // withheld list would let a 404 that spilled the private content pass as + // a clean denial. + // #195 (R3/KTD-4): the fixed blob-scoped withheld set PLUS every seeded + // sub-entity marker. The new reads (get_issue/get_pr/get_pr_diff/get_cert + // /get_bounty and the list rows off them) return their OWN private + // content, whose markers are NOT in the blob set — a 404 that leaks an + // issue title or PR diff must fail, so the per-read markers are withheld + // from every read-gate hostile. A status-only 404 check would be vacuous. + let mut withheld = vec![ + fixture.private_secret.clone(), + fixture.private_blob_oid.clone(), + fixture.private_blob_oid_short.clone(), + fixture.private_repo_id.clone(), + ]; + withheld.extend(fixture.priv_markers.iter().cloned()); + let mut v = vec![ + Probe { + label: format!("{} read-gate hostile (anon)", row.handler), + method: method.clone(), + path: path.clone(), + body: body.clone(), + signer: Signer::Anon, + json, + expect: Expect::Deny(404), + withheld: withheld.clone(), + }, + // #195 (F1): a signed NON-READER must also get the existence-hiding + // 404 — a signature is not access (INV-12). Without this probe, a + // regression that treats any valid signature as authorized would leak + // the private repo while the anon probe stayed green. + Probe { + label: format!("{} read-gate hostile (signed non-reader)", row.handler), + method: method.clone(), + path: path.clone(), + body: body.clone(), + signer: Signer::Stranger, + json, + expect: Expect::Deny(404), + withheld: withheld.clone(), + }, + ]; + // Positive twin: prove the 404 is the gate, not an absent entity. + let twin = match row.reach { + Reach::ReaderReads => Probe { + label: format!("{} read-reachability twin (owner)", row.handler), + method, + path, + body, + signer: Signer::Owner, + json, + expect: Expect::Ok2xx, + // The twin is a GRANTED read; it is expected to return the + // content, so nothing is withheld from it. + withheld: Vec::new(), + }, + Reach::SiblingPublic(sibling) => Probe { + label: format!("{} read-reachability twin (sibling public)", row.handler), + method, + path: fill(sibling, fixture, &fixture.private_repo, row.id_source), + body, + signer: Signer::Anon, + json, + expect: Expect::Ok2xx, + withheld: Vec::new(), + }, + Reach::None => panic!( + "read-gate row {} {} has no Reach twin (U1 consistency should have caught this)", + row.method, row.path + ), + }; + v.push(twin); + v + } + GateClass::SignatureRequired => { + let path = fill(row.path, fixture, &fixture.public_repo, row.id_source); + vec![Probe { + label: format!("{} signature-required hostile", row.handler), + method, + path, + body, + signer: Signer::Anon, + json, + expect: Expect::Deny(401), + // The 401 fires at the signature layer before any repo/entity is + // looked up (NO_ENTITY substrate), so nothing private is in scope. + withheld: Vec::new(), + }] + } + GateClass::SignerSelfGate => { + // #195 (F3, KTD-1): a body-field caller-self gate. The shared body runs + // through `fill()` so `{owner}` resolves to the real owner DID; hostile + // and control differ ONLY in signer. Hostile: a Stranger signs but the + // body names the owner -> `did_matches(stranger, owner) == false` -> 403. + // Control: the Owner signs and the body names the owner -> match -> + // Not403. (The repo arg to `fill` is inert here: task/register paths + // carry no `{repo}` placeholder, but the signature requires it.) + let path = fill(row.path, fixture, &fixture.public_repo, row.id_source); + let filled_body = row + .body + .map(|b| fill(b, fixture, &fixture.public_repo, row.id_source).into_bytes()) + .unwrap_or_default(); + vec![ + Probe { + label: format!("{} signer-self hostile (stranger)", row.handler), + method: method.clone(), + path: path.clone(), + body: filled_body.clone(), + signer: Signer::Stranger, + json, + expect: Expect::Deny(403), + // Each gate returns a fixed forbidden() message before serializing + // any entity, and nothing private is seeded on this substrate, so + // an empty withheld list is the honest assertion (KTD-4). + withheld: Vec::new(), + }, + Probe { + label: format!("{} signer-self control (owner)", row.handler), + method, + path, + body: filled_body, + signer: Signer::Owner, + json, + expect: Expect::Not403, + withheld: Vec::new(), + }, + ] + } + } +} + +/// A DB-less `Fixture` for the pure-generator unit tests (here and the structural +/// consistency test in `routes.rs`). No node is seeded, so the identities are +/// generated keypairs with placeholder repo/secret metadata; only the arm DIDs +/// and the injected `{id}` matter to `probes_for`. Shared so both test modules +/// build the same shape. +#[cfg(test)] +pub mod tests_support { + use super::*; + + pub fn fx() -> Fixture { + let author = Keypair::generate(); + let creator = Keypair::generate(); + let claimant = Keypair::generate(); + let assignee = Keypair::generate(); + Fixture { + author_did: author.did().to_string(), + creator_did: creator.did().to_string(), + claimant_did: claimant.did().to_string(), + assignee_did: assignee.did().to_string(), + author, + creator, + claimant, + assignee, + bounty_id: "bounty-uuid-1234".to_string(), + claimed_bounty_id: "claimed-bounty-uuid-1234".to_string(), + submitted_bounty_id: "submitted-bounty-uuid-1234".to_string(), + open_bounty_id: "open-bounty-uuid-1234".to_string(), + claimable_task_id: "claimable-task-uuid-1234".to_string(), + completable_task_id: "completable-task-uuid-1234".to_string(), + failable_task_id: "failable-task-uuid-1234".to_string(), + issue_id: "issue-uuid-1234".to_string(), + priv_issue_id: "priv-issue-uuid-1234".to_string(), + priv_bounty_id: "priv-bounty-uuid-1234".to_string(), + cert_id: "cert-uuid-1234".to_string(), + priv_markers: vec![ + "TOPSECRET-ISSUE-U3".to_string(), + "TOPSECRET-PRTITLE-U3".to_string(), + "TOPSECRET-PRDIFF-U3".to_string(), + "TOPSECRET-BOUNTY-U3".to_string(), + ], + owner: Keypair::generate(), + stranger: Keypair::generate(), + owner_did: "did:key:zOWNER".to_string(), + public_repo: "prober-pub".to_string(), + public_repo_id: "pub-id".to_string(), + private_repo: "prober-priv".to_string(), + private_repo_id: "priv-repo-uuid-0BADF00D".to_string(), + content_path: "public/a.txt".to_string(), + private_secret: "TOPSECRET-PRIVREAD-U3".to_string(), + private_blob_oid: "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef".to_string(), + private_blob_oid_short: "deadbeefdead".to_string(), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::support::routes::{GateClass, Reach, Row}; + + use super::tests_support::fx; + + #[test] + fn owner_gate_row_yields_stranger_403_and_owner_twin() { + let row = Row { + method: "PUT", + path: "/api/v1/repos/{owner}/{repo}/visibility", + gate: GateClass::OwnerGate, + handler: "visibility::set_visibility", + body: Some(r#"{"path_glob":"/"}"#), + needs: &[], + reach: Reach::None, + principals: &[], + id_source: crate::support::routes::IdSource::Fixed, + }; + let ps = probes_for(&row, &fx()); + assert_eq!(ps.len(), 2); + assert_eq!(ps[0].signer, Signer::Stranger); + assert!(ps[0].json); + assert!(matches!(ps[0].expect, Expect::Deny(403))); + assert!( + ps[0].path.contains("prober-pub"), + "owner-gate uses the public repo" + ); + assert_eq!(ps[1].signer, Signer::Owner); + assert!(matches!(ps[1].expect, Expect::Not403)); + // Owner-gate hostile fires on a NO_ENTITY public repo before any lookup, so + // there is nothing private seeded to leak: an empty withheld list is honest. + assert!( + ps[0].withheld.is_empty(), + "owner-gate hostile probe seeds no private entity, so withholds nothing" + ); + } + + #[test] + fn read_gate_row_yields_anon_and_stranger_404_and_positive_twin() { + let row = Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/blob/{*path}", + gate: GateClass::ReadGate, + handler: "repos::get_blob", + body: None, + needs: &[], + reach: Reach::ReaderReads, + principals: &[], + id_source: crate::support::routes::IdSource::Fixed, + }; + let f = fx(); + let ps = probes_for(&row, &f); + // anon hostile, signed-stranger hostile (#195 F1), owner twin. + assert_eq!(ps.len(), 3); + assert_eq!(ps[0].signer, Signer::Anon); + assert!(matches!(ps[0].expect, Expect::Deny(404))); + // #195 (F1): the signed non-reader hostile must also expect a 404 no-leak. + assert_eq!(ps[1].signer, Signer::Stranger); + assert!(matches!(ps[1].expect, Expect::Deny(404))); + assert!( + !ps[1].withheld.is_empty(), + "the signed-stranger hostile must carry the same withheld tokens" + ); + assert!( + ps[0].path.contains("prober-priv"), + "read-gate uses the private repo" + ); + assert!(ps[0].path.contains("public/a.txt"), "{{*path}} substituted"); + // F2 belt-and-suspenders: the read-gate hostile probe MUST carry the private + // tokens, so a future refactor that drops them (reverting F2 to a vacuous + // empty-withheld 404 check) fails loudly here. + assert!( + !ps[0].withheld.is_empty(), + "read-gate hostile probe must carry withheld private tokens" + ); + assert!( + ps[0].withheld.contains(&f.private_secret), + "the seeded secret content must be a withheld token" + ); + assert!( + ps[0].withheld.contains(&f.private_blob_oid), + "the private blob OID must be a withheld token" + ); + assert!( + ps[0].withheld.contains(&f.private_blob_oid_short), + "the short blob OID prefix must be a withheld token" + ); + assert_eq!(ps[2].signer, Signer::Owner); + assert!(matches!(ps[2].expect, Expect::Ok2xx)); + // The granted twin returns the content, so it withholds nothing. + assert!( + ps[2].withheld.is_empty(), + "the granted read twin must not carry withheld tokens" + ); + } + + #[test] + fn signature_row_yields_unsigned_401_no_twin() { + let row = Row { + method: "POST", + path: "/{owner}/{repo}/git-receive-pack", + gate: GateClass::SignatureRequired, + handler: "repos::git_receive_pack", + body: None, + needs: &[], + reach: Reach::None, + principals: &[], + id_source: crate::support::routes::IdSource::Fixed, + }; + let ps = probes_for(&row, &fx()); + assert_eq!(ps.len(), 1); + assert_eq!(ps[0].signer, Signer::Anon); + assert!(matches!(ps[0].expect, Expect::Deny(401))); + // The 401 fires before any entity lookup, so nothing private is in scope. + assert!( + ps[0].withheld.is_empty(), + "signature-required hostile probe withholds nothing (fires pre-lookup)" + ); + } + + #[test] + fn read_gate_sibling_public_twin_reads_the_sibling_path_anon() { + let row = Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/blob/secret/x.txt", + gate: GateClass::ReadGate, + handler: "repos::get_blob", + body: None, + needs: &[], + reach: Reach::SiblingPublic("/api/v1/repos/{owner}/{repo}/blob/{*path}"), + principals: &[], + id_source: crate::support::routes::IdSource::Fixed, + }; + let ps = probes_for(&row, &fx()); + // anon hostile, signed-stranger hostile (#195 F1), sibling-public twin. + assert_eq!(ps.len(), 3); + // Hostile: anon on the withheld path -> 404. + assert_eq!(ps[0].signer, Signer::Anon); + assert!(matches!(ps[0].expect, Expect::Deny(404))); + assert!(ps[0].path.ends_with("/blob/secret/x.txt")); + // Signed non-reader on the withheld path -> 404 too. + assert_eq!(ps[1].signer, Signer::Stranger); + assert!(matches!(ps[1].expect, Expect::Deny(404))); + assert!(ps[1].path.ends_with("/blob/secret/x.txt")); + // Twin: anon on the sibling PUBLIC path -> non-empty 2xx (proves the 404 + // is path-scoped withholding, not a blanket refusal). + assert_eq!(ps[2].signer, Signer::Anon); + assert!(matches!(ps[2].expect, Expect::Ok2xx)); + assert!( + ps[2].path.contains("public/a.txt"), + "sibling template's {{*path}} is filled from the fixture content path" + ); + } + + // F2 RED proof at the check_denied level (no DB): feed a synthetic denial body + // that DOES contain the private secret through the SAME withheld tokens the + // read-gate probe carries. If the tokens are truly threaded and non-empty, the + // deny check must reject it as a leak. This is the fail case a real node's + // clean 404 avoids; it proves the sweep would actually catch a leaking 404. + #[test] + fn read_gate_withheld_tokens_reject_a_leaking_denial_body() { + use crate::support::assert::check_denied; + + let row = Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}", + gate: GateClass::ReadGate, + handler: "repos::get_repo", + body: None, + needs: &[], + reach: Reach::ReaderReads, + principals: &[], + id_source: crate::support::routes::IdSource::Fixed, + }; + let f = fx(); + let ps = probes_for(&row, &f); + let tokens: Vec<&str> = ps[0].withheld.iter().map(String::as_str).collect(); + assert!( + !tokens.is_empty(), + "precondition: read-gate probe carries tokens" + ); + + // A hostile 404 that spilled the secret content: must be rejected as a leak. + let leaking = format!( + r#"{{"error":"blob {} at {}"}}"#, + f.private_secret, f.private_blob_oid + ); + let r = check_denied(404, &leaking, 404, &tokens); + assert!( + r.is_err(), + "a 404 body echoing the secret must be flagged: {r:?}" + ); + assert!( + r.unwrap_err().contains(&f.private_secret), + "the failure must name the leaked secret token" + ); + + // #195 (F4): a 404 that serialized the private repo's internal id (UUID) is + // a repository-metadata leak and must be flagged too — proving the UUID is a + // load-bearing withheld token, not just the blob content. + let leaking_meta = format!(r#"{{"error":"repo {} not found"}}"#, f.private_repo_id); + let rm = check_denied(404, &leaking_meta, 404, &tokens); + assert!( + rm.is_err(), + "a 404 body echoing the private repo id must be flagged: {rm:?}" + ); + assert!( + rm.unwrap_err().contains(&f.private_repo_id), + "the failure must name the leaked private repo id token" + ); + + // A clean 404 (no private tokens) with the same tokens passes: the tokens do + // not spuriously trip on an honest denial. + let clean = check_denied(404, r#"{"error":"repository not found"}"#, 404, &tokens); + assert!( + clean.is_ok(), + "a clean 404 must pass the withheld check: {clean:?}" + ); + } + + #[test] + #[should_panic(expected = "has no Reach twin")] + fn read_gate_row_without_a_reach_twin_panics() { + // A read-gate row with Reach::None is a registry bug the U1 consistency + // test rejects; if one slips through, the probe generator must fail loud + // rather than silently emit a hostile probe with no positive twin. + let row = Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/issues", + gate: GateClass::ReadGate, + handler: "issues::list_issues", + body: None, + needs: &[], + reach: Reach::None, + principals: &[], + id_source: crate::support::routes::IdSource::Fixed, + }; + let _ = probes_for(&row, &fx()); + } +} diff --git a/crates/gitlawb-node/tests/support/routes.rs b/crates/gitlawb-node/tests/support/routes.rs new file mode 100644 index 00000000..b29fb161 --- /dev/null +++ b/crates/gitlawb-node/tests/support/routes.rs @@ -0,0 +1,1059 @@ +//! U1: the deny-bearing route set for the invariant deny-prober. +//! +//! Every route that carries a runtime deny to assert lives here: owner-gate +//! (403), multi-principal (403), read-gate (404), signature-required (401), and +//! caller-self / actor (403): `SignerSelfGate` for the request-body field +//! bindings (create_task delegator, claim_task assignee, register did) and +//! `MultiPrincipalGate` with the assignee arm for the complete/fail actor gates. +//! The completeness cross-check in `deny_harness.rs` derives the caller-self set +//! from `src/api` and requires each `did_matches(caller, X)` handler to be a +//! driven row, so a new one cannot silently escape the sweep. +//! +//! Scope: this doc claims the REST/api surface only. The parallel GraphQL +//! mutation caller-self gates (`src/graphql/mutation.rs`) are out of this +//! harness and fenced separately (#219). +//! +//! Each row is classified by READING its handler, never inferred from the route +//! name, and records the handler fn name so a reviewer can spot-check. Guessing +//! is how a false test enters the sweep (the register_replica lesson: it looks +//! like an owner-gate, it is signer-self; and list_visibility is a 403 +//! owner-gate despite being a GET). +//! +//! Rows are declarative (gate class + entities + reachability), not prebuilt +//! requests, so Flavor B (the cross-surface differential prober) can reuse them +//! (R9). + +/// The gate class of a deny-bearing route and the exact status its deny emits. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum GateClass { + /// `require_repo_owner` / `require_owner` / inline `did_matches(caller, owner)` + /// -> `AppError::Forbidden` == 403. Probed with a validly-signed non-owner. + OwnerGate, + /// A 403 gate with ONE OR MORE NAMED authorizing principals (owner-OR-author, + /// creator-OR-claimant, claimant-only, …). Probed with a validly-signed + /// stranger (403) plus one Not403 twin per declared arm (`Row.principals`), + /// so reverting ANY single arm turns that arm's twin RED (#195, F1). + /// `OwnerGate` is the owner-arm special case; a single-arm gate whose + /// principal is NOT the repo owner (submit_bounty's claimant-only check, + /// approve/cancel's creator-only check) belongs here, because the point is + /// that the arms are DECLARED and each one gets its own twin (#195, N2). + MultiPrincipalGate, + /// `authorize_repo_read` / `visibility_check` -> `RepoNotFound` == 404 + /// (existence-hiding). Probed with a non-reader against a private/withheld + /// target. A 404 alone is ambiguous (a missing entity also 404s), so every + /// read-gate row carries a `Reach` positive twin. + ReadGate, + /// `require_signature` layer -> 401 on the git write routes. Probed unsigned. + SignatureRequired, + /// A caller-self 403 gate that binds a DID field in the request BODY to the + /// authenticated signer (`did_matches(caller, &body._did)`): create_task + /// (delegator_did), claim_task (assignee_did), register (did). Probed with a + /// validly-signed stranger whose body names the OWNER (mismatch -> 403) plus an + /// owner control whose body names the owner (match -> Not403). Distinct from + /// `OwnerGate`: the bound identity is a body field, not the stored repo owner, + /// and the completeness classifier keys on exactly that (#195, F3, KTD-1). The + /// actor-vs-stored caller-self gates (complete_task/fail_task) are the + /// `MultiPrincipalGate` assignee-arm shape instead, not this class. + SignerSelfGate, +} + +impl GateClass { + pub fn expected_status(self) -> u16 { + match self { + GateClass::OwnerGate | GateClass::MultiPrincipalGate | GateClass::SignerSelfGate => 403, + GateClass::ReadGate => 404, + GateClass::SignatureRequired => 401, + } + } +} + +/// An authorizing principal (arm) of a multi-principal 403 gate. Each maps to a +/// distinct fixture identity in `probe.rs` so reverting one arm cannot silently +/// re-collapse onto another (the original bug seeded `author == owner`). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Principal { + /// The repo owner (`require_repo_owner` / `did_matches(caller, owner)`). + Owner, + /// The PR/issue author (`did_matches(caller, author_did)`). + Author, + /// The bounty creator (`did_matches(caller, creator_did)`). + Creator, + /// The bounty claimant (`did_matches(caller, claimant_did)`). + Claimant, + /// The task assignee (`did_matches(caller, stored assignee_did)`): the actor + /// arm of complete_task / fail_task (#195, F3). Unlike the field-binding + /// `SignerSelfGate`, the identity is the task's STORED assignee, so the arm is + /// driven against a seeded claimed task, not a body field. + Assignee, +} + +/// How a read-gate row's positive twin proves the 404 is the gate and not a +/// merely-absent entity/repo. Owner-gate and signature rows use `None` +/// (the owner-gate twin is the same request re-signed as the owner, handled by +/// the probe generator from the class, not a path). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Reach { + /// Not a read-gate row; no read-twin. + None, + /// The authorized reader/owner issues the same request and gets 2xx. + ReaderReads, + /// A sibling PUBLIC path on the same repo returns 2xx-with-content, proving + /// the 404 is path-scoped withholding, not a blanket/absent 404. Mirrors the + /// existing U7/U8/anon_ipfs cases in `tests/deny_harness.rs`. + SiblingPublic(&'static str), +} + +/// How a row's `{id}` placeholder is filled. Most id-keyed paths use a fixed +/// seed id (`"1"`), but some entities are keyed by a UUID minted at seed time +/// (a bounty, a cert), so the fixture must inject the captured id per row. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum IdSource { + /// `{id}` -> the fixed seed id `"1"` (static-template entities seeded at #1). + Fixed, + /// `{id}` -> the fixture's seeded disputable bounty id (UUID). + BountyId, + /// `{id}` -> the fixture's seeded issue id (UUID). The issue is stored as a + /// full git-JSON `IssueRecord` (authored by the fixture author), which a + /// bare `{"author":..}` seed cannot satisfy — close_issue deserializes the + /// whole record before reading its author, so the author arm needs the + /// complete record. + IssueId, + /// `{id}` -> the fixture's seeded PRIVATE-repo issue id (UUID). Distinct from + /// `IssueId` (the public-repo close-gate issue): the get_issue / + /// list_issue_comments read-gate rows run against the PRIVATE repo, so their + /// `{id}` must be an issue that actually exists there (#195, F2/U3). + PrivIssueId, + /// `{id}` -> the fixture's seeded PRIVATE-repo bounty id (UUID). get_bounty is + /// NOT repo-scoped in its path (`/api/v1/bounties/{id}`) but read-gates on the + /// bounty's own repo; the bounty is seeded against the private repo so anon / + /// stranger get the existence-hiding 404 and the owner gets 2xx (#195, U3). + PrivBountyId, + /// `{id}` -> the fixture's seeded ref-certificate id (UUID), issued by a real + /// owner push to the private repo. get_cert read-gates the private repo on "/" + /// then fetches the cert by id, so the owner-2xx twin needs a real cert whose + /// repo_id is the private repo's (#195, U3). + CertId, + /// `{id}` -> the fixture's bounty seeded in status "claimed" (#195, N2). + /// submit_bounty checks status BEFORE auth (bounties.rs:294), so the stranger + /// only reaches the claimant 403 gate on a bounty that is already "claimed"; + /// the claimant twin then consumes that status (claimed -> submitted), so the + /// row needs its own bounty. + ClaimedBountyId, + /// `{id}` -> the fixture's bounty driven to status "submitted" (#195, N2). + /// approve_bounty's status check fires before its creator gate + /// (bounties.rs:334), so only a "submitted" bounty reaches the 403; the + /// creator twin consumes it (submitted -> completed). + SubmittedBountyId, + /// `{id}` -> the fixture's bounty left in status "open" (#195, N2). + /// cancel_bounty's status check fires before its creator gate + /// (bounties.rs:381), so only an "open" bounty reaches the 403; the creator + /// twin consumes it (open -> cancelled). + OpenBountyId, + /// `{id}` -> the fixture's task seeded "pending" (#195, F3). The claim_task + /// caller-self gate fires before the DB claim, so the stranger 403 does not + /// consume it; the owner control then claims it (pending -> claimed). + ClaimableTaskId, + /// `{id}` -> the fixture's task seeded "claimed" by the assignee (#195, F3). + /// complete_task's assignee actor gate reaches it; the assignee twin then + /// finishes it (claimed -> completed). + CompletableTaskId, + /// `{id}` -> a second assignee-claimed task (#195, F3), so fail_task's twin + /// (claimed -> failed) does not race complete_task for one entity. + FailableTaskId, +} + +/// One deny-bearing route. `path` is a template with `{owner}`/`{repo}`/`{id}` +/// /`{number}`/`{*path}` placeholders the probe generator (U2) fills from the +/// fixture. `needs` names the seeded sub-entities the path requires (empty for +/// owner-gate rows: the gate fires before the entity lookup). +#[derive(Debug, Clone, Copy)] +pub struct Row { + pub method: &'static str, + pub path: &'static str, + pub gate: GateClass, + /// Handler fn name, recorded for review spot-check (per-row verification). + pub handler: &'static str, + /// Sample request body (JSON) or `None`. JSON bodies drive the + /// `Content-Type: application/json` the probe attaches to clear the extractor. + pub body: Option<&'static str>, + /// Seeded sub-entities the path template consumes. + pub needs: &'static [&'static str], + /// Positive-twin strategy for read-gate rows. + pub reach: Reach, + /// For `MultiPrincipalGate` rows, the authorizing arms this gate must drive + /// (one Not403 twin each). Empty for every other class — only meaningful for + /// `MultiPrincipalGate`, and the consistency test enforces that pairing. + pub principals: &'static [Principal], + /// How the `{id}` placeholder is filled for this row (see [`IdSource`]). + pub id_source: IdSource, +} + +const NO_ENTITY: &[&str] = &[]; +const NO_PRINCIPAL: &[Principal] = &[]; + +/// The deny-bearing route set. Owner-gate and signature-required tranches are +/// fully verified against their handlers this session; the read-gate tranche is +/// populated as each handler's deny path (404-deny vs list-filter) is verified. +pub fn deny_bearing_routes() -> &'static [Row] { + &[ + // ── Owner-gate (403) — verified: each calls require_repo_owner / + // require_owner / inline did_matches against the repo owner. ────────── + Row { + method: "POST", + path: "/api/v1/repos/{owner}/{repo}/pulls/{number}/merge", + gate: GateClass::OwnerGate, + handler: "pulls::merge_pr", + body: None, + needs: &["pr_number"], + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + }, + // #195 (F1): close_pr / close_issue are owner-OR-author gates (pulls.rs:276, + // issues.rs:255). A plain OwnerGate tests only the owner arm, so reverting + // the author arm is invisible. MultiPrincipalGate drives BOTH arms. + Row { + method: "POST", + path: "/api/v1/repos/{owner}/{repo}/pulls/{number}/close", + gate: GateClass::MultiPrincipalGate, + handler: "pulls::close_pr", + body: None, + needs: &["pr_number"], + reach: Reach::None, + principals: &[Principal::Owner, Principal::Author], + id_source: IdSource::Fixed, + }, + Row { + method: "POST", + path: "/api/v1/repos/{owner}/{repo}/issues/{id}/close", + gate: GateClass::MultiPrincipalGate, + handler: "issues::close_issue", + body: None, + needs: &["issue_id"], + reach: Reach::None, + principals: &[Principal::Owner, Principal::Author], + id_source: IdSource::IssueId, + }, + // #195 (F1): dispute_bounty is creator-OR-claimant (bounties.rs:425). Not + // repo-scoped: it gates on the bounty's creator/claimant, so the row is + // id-keyed by the seeded disputable bounty's UUID (IdSource::BountyId). + Row { + method: "POST", + path: "/api/v1/bounties/{id}/dispute", + gate: GateClass::MultiPrincipalGate, + handler: "bounties::dispute_bounty", + body: None, + needs: &["bounty_id"], + reach: Reach::None, + principals: &[Principal::Creator, Principal::Claimant], + id_source: IdSource::BountyId, + }, + // #195 (N2): submit/approve/cancel each 403 a non-claimant/non-creator, but + // their STATUS check fires before the auth check, so each row gets its own + // bounty seeded in the gate-reaching status (claimed / submitted / open). + // The sweep drives probes in emission order (hostile first, probes_for), + // so the stranger 403 fires while the bounty still holds that status; the + // arm twin then consumes it (submit: claimed -> submitted, approve: + // submitted -> completed, cancel: open -> cancelled), which is why the + // rows cannot share a bounty with each other or with dispute_bounty. + Row { + method: "POST", + path: "/api/v1/bounties/{id}/submit", + gate: GateClass::MultiPrincipalGate, + handler: "bounties::submit_bounty", + // pr_id is a free-text column (no FK); any string clears serde and + // the UPDATE. The gate fires before the value is used. + body: Some(r#"{"pr_id":"1"}"#), + needs: &["claimed_bounty_id"], + reach: Reach::None, + principals: &[Principal::Claimant], + id_source: IdSource::ClaimedBountyId, + }, + Row { + method: "POST", + path: "/api/v1/bounties/{id}/approve", + gate: GateClass::MultiPrincipalGate, + handler: "bounties::approve_bounty", + // ApproveBountyRequest's only field (tx_hash) is Option, so an empty + // JSON object clears the Json extractor; a missing body would 4xx at + // the extractor, before the gate under test. + body: Some(r#"{}"#), + needs: &["submitted_bounty_id"], + reach: Reach::None, + principals: &[Principal::Creator], + id_source: IdSource::SubmittedBountyId, + }, + Row { + method: "POST", + path: "/api/v1/bounties/{id}/cancel", + gate: GateClass::MultiPrincipalGate, + handler: "bounties::cancel_bounty", + body: None, + needs: &["open_bounty_id"], + reach: Reach::None, + principals: &[Principal::Creator], + id_source: IdSource::OpenBountyId, + }, + Row { + method: "POST", + path: "/api/v1/repos/{owner}/{repo}/hooks", + gate: GateClass::OwnerGate, + handler: "webhooks::create_webhook", + body: Some(r#"{"url":"https://e.example/h","events":["*"]}"#), + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + }, + Row { + method: "DELETE", + path: "/api/v1/repos/{owner}/{repo}/hooks/{id}", + gate: GateClass::OwnerGate, + handler: "webhooks::delete_webhook", + body: None, + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + }, + Row { + method: "POST", + path: "/api/v1/repos/{owner}/{repo}/labels", + gate: GateClass::OwnerGate, + handler: "labels::add_label", + body: Some(r#"{"label":"bug"}"#), + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + }, + Row { + method: "DELETE", + path: "/api/v1/repos/{owner}/{repo}/labels/{label}", + gate: GateClass::OwnerGate, + handler: "labels::remove_label", + body: None, + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + }, + Row { + method: "POST", + path: "/api/v1/repos/{owner}/{repo}/branches/{branch}/protect", + gate: GateClass::OwnerGate, + handler: "protect::protect_branch", + body: None, + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + }, + Row { + method: "DELETE", + path: "/api/v1/repos/{owner}/{repo}/branches/{branch}/protect", + gate: GateClass::OwnerGate, + handler: "protect::unprotect_branch", + body: None, + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + }, + Row { + method: "PUT", + path: "/api/v1/repos/{owner}/{repo}/visibility", + gate: GateClass::OwnerGate, + handler: "visibility::set_visibility", + body: Some(r#"{"path_glob":"/","reader_dids":[]}"#), + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + }, + Row { + method: "DELETE", + path: "/api/v1/repos/{owner}/{repo}/visibility", + gate: GateClass::OwnerGate, + handler: "visibility::remove_visibility", + body: Some(r#"{"path_glob":"/"}"#), + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + }, + // list_visibility is a 403 owner-gate despite being a GET (calls + // require_owner); the /visibility mount chains put+delete+get, all gated. + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/visibility", + gate: GateClass::OwnerGate, + handler: "visibility::list_visibility", + body: None, + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + }, + // ── Signature-required (401) — verified: git write route wrapped by the + // require_signature layer (add_auth_layers in server.rs). ───────────── + Row { + method: "POST", + path: "/{owner}/{repo}/git-receive-pack", + gate: GateClass::SignatureRequired, + handler: "repos::git_receive_pack", + body: None, + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + }, + // ── Read-gate (404) — verified: each handler calls + // authorize_repo_read / visibility_check on "/" which returns + // AppError::RepoNotFound (404) when a non-reader hits a private repo + // (api/mod.rs:44-62). The Reach twin is the owner issuing the same + // request against the private repo and getting 2xx, proving the 404 is + // the gate and not a merely-absent entity/repo. Every row here gates on + // the whole repo ("/"), so ReaderReads (owner re-read) is the twin; + // the fully-private fixture repo needs no per-row sub-entity seeding + // because these reads either return the seeded repo/blob or an empty + // 2xx list. + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}", + gate: GateClass::ReadGate, + handler: "repos::get_repo", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + }, + // #195 (F2): repo-root reads that gate on "/" — driven as real ReadGate rows + // rather than source-only exemptions, so a runtime bypass that keeps the + // authorize_repo_read/visibility_check marker but leaks is caught. + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/star", + gate: GateClass::ReadGate, + handler: "stars::get_star_status", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/icaptcha-proof", + gate: GateClass::ReadGate, + handler: "repos::get_icaptcha_proof", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/encrypted-blobs/replicate", + gate: GateClass::ReadGate, + handler: "encrypted::replicate_encrypted_blobs", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/commits", + gate: GateClass::ReadGate, + handler: "repos::list_commits", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/tree", + gate: GateClass::ReadGate, + handler: "repos::get_tree_root", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + }, + // Path-scoped read: get_blob gates on "/{path}" (repos.rs:390). The + // fully-private fixture repo denies any path to anon (404); the owner + // reads the seeded blob at content_path and gets 2xx bytes. (Path-scoped + // subtree WITHHOLDING on a *public* repo — the SiblingPublic twin — is + // already covered by the U7/U8 cases in deny_harness.rs.) + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/blob/{*path}", + gate: GateClass::ReadGate, + handler: "repos::get_blob", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/refs", + gate: GateClass::ReadGate, + handler: "repos::list_refs", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/issues", + gate: GateClass::ReadGate, + handler: "issues::list_issues", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/labels", + gate: GateClass::ReadGate, + handler: "labels::list_labels", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/certs", + gate: GateClass::ReadGate, + handler: "certs::list_certs", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/events", + gate: GateClass::ReadGate, + handler: "events::list_repo_events", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/pulls", + gate: GateClass::ReadGate, + handler: "pulls::list_prs", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/changelog", + gate: GateClass::ReadGate, + handler: "changelog::get_changelog", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/bounties", + gate: GateClass::ReadGate, + handler: "bounties::list_repo_bounties", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/withheld-paths", + gate: GateClass::ReadGate, + handler: "visibility::withheld_paths", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/encrypted-blobs", + gate: GateClass::ReadGate, + handler: "encrypted::list_encrypted_blobs", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + }, + // #195 (F2, U3): sub-entity reads that gate on "/" (the private repo) but + // require the entity to EXIST at the request path for the owner-2xx twin. + // Each carries its own distinctive marker (seeded in probe.rs) added to the + // per-read withheld set, so a 404 that echoes THAT read's private content + // (issue title, PR title, PR diff, bounty title) fails — a status- + // only 404 check would be vacuous. These were source-only exemptions in + // READ_GATE_NOT_DRIVEN before; they are now driven as real ReadGate rows. + // get_issue: id-keyed by the seeded PRIVATE issue (marker in its title). + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/issues/{id}", + gate: GateClass::ReadGate, + handler: "issues::get_issue", + body: None, + needs: &["priv_issue_id"], + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::PrivIssueId, + }, + // list_issue_comments: parent issue must exist (private); child list may be + // empty — an empty `{"comments":[]}` is a non-empty 2xx body. + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/issues/{id}/comments", + gate: GateClass::ReadGate, + handler: "issues::list_issue_comments", + body: None, + needs: &["priv_issue_id"], + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::PrivIssueId, + }, + // get_pr: the seeded PRIVATE PR #1 (marker in its title). + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/pulls/{number}", + gate: GateClass::ReadGate, + handler: "pulls::get_pr", + body: None, + needs: &["priv_pr"], + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + }, + // get_pr_diff: the PRIVATE PR #1 has a real `feature` source branch with a + // marker file, so branch_diff_names(main, feature) is NON-EMPTY and the + // owner twin returns a real diff (a bail!/500 if the source ref were + // missing). The per-path visibility_check Deny-return is additionally + // driven by the hand-written get_pr_diff_withheld_path_is_denied test. + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/pulls/{number}/diff", + gate: GateClass::ReadGate, + handler: "pulls::get_pr_diff", + body: None, + needs: &["priv_pr"], + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + }, + // list_reviews / list_comments: parent PR #1 must exist (private); child + // lists may be empty. + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/pulls/{number}/reviews", + gate: GateClass::ReadGate, + handler: "pulls::list_reviews", + body: None, + needs: &["priv_pr"], + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/pulls/{number}/comments", + gate: GateClass::ReadGate, + handler: "pulls::list_comments", + body: None, + needs: &["priv_pr"], + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + }, + // get_cert: id-keyed by a real ref-certificate issued by an owner push to + // the private repo. No cert-content marker is seeded: get_cert read-gates + // the repo on "/" and 404s a non-reader BEFORE fetching the cert, so cert + // fields cannot reach the deny body — the repo-scoped tokens (private_repo_id, + // private_secret) plus the status check carry the no-leak assertion here. + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/certs/{id}", + gate: GateClass::ReadGate, + handler: "certs::get_cert", + body: None, + needs: &["cert_id"], + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::CertId, + }, + // get_bounty: NOT repo-scoped in its path, read-gates on the bounty's repo. + // Seeded against the PRIVATE repo (marker in its title), id-keyed. + Row { + method: "GET", + path: "/api/v1/bounties/{id}", + gate: GateClass::ReadGate, + handler: "bounties::get_bounty", + body: None, + needs: &["priv_bounty_id"], + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::PrivBountyId, + }, + // get_tree (path-scoped): genuinely ADDITIONAL to get_tree_root (Q1). Root + // gates on "/"; get_tree gates on the REQUESTED subtree (N3, + // authorize_repo_read(&gate_path)), a distinct path-scoped Deny surface. The + // {*path} fills to the private content path, so a non-reader is denied the + // subtree listing and the owner gets a non-empty 2xx. + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/tree/{*path}", + gate: GateClass::ReadGate, + handler: "repos::get_tree", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + }, + // ── Caller-self / actor (403), #195 (F3). The task/register handlers carry + // genuine caller-self 403 gates a bypass would expose (impersonate a + // delegator/assignee, or forge a trust row for a DID you don't control). + // Before this change they were marker-only (INV-21 vacuous); now each is + // driven hostile-then-authorized. The field-binding shape (a body DID + // field must equal the caller) is `SignerSelfGate`; the actor-vs-stored + // shape (caller must equal the task's stored assignee) is + // `MultiPrincipalGate` with the Assignee arm. Register is DRIVEN, not + // excused: the harness runs icaptcha inert (spawn_node never inits the + // verifier), so a stranger-signed POST /api/register reaches the 403. + Row { + method: "POST", + path: "/api/v1/tasks", + gate: GateClass::SignerSelfGate, + handler: "tasks::create_task", + // delegator_did must equal the signer (tasks.rs:101). Stranger names the + // owner -> mismatch -> 403; owner names the owner -> match -> 201. + body: Some(r#"{"kind":"review","capability":"read","delegator_did":"{owner}"}"#), + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + }, + Row { + method: "POST", + path: "/api/v1/tasks/{id}/claim", + gate: GateClass::SignerSelfGate, + handler: "tasks::claim_task", + // assignee_did must equal the signer (tasks.rs:174); the gate fires + // before the DB claim, so the stranger 403 leaves the pending task for + // the owner control to claim. + body: Some(r#"{"assignee_did":"{owner}"}"#), + needs: &["claimable_task_id"], + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::ClaimableTaskId, + }, + Row { + method: "POST", + path: "/api/register", + gate: GateClass::SignerSelfGate, + handler: "register::register", + // did must equal the signer (register.rs:55). The did must parse + // (register.rs:50) so the 403 is reached, so `{owner}` is a real DID. + // `message` is an unknown field serde ignores. + body: Some(r#"{"did":"{owner}","capabilities":[],"message":"probe"}"#), + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + }, + Row { + method: "POST", + path: "/api/v1/tasks/{id}/complete", + gate: GateClass::MultiPrincipalGate, + handler: "tasks::complete_task", + // Caller must equal the task's STORED assignee (tasks.rs:220). Driven + // against a claimed task: stranger -> 403, the assignee arm -> 200. + body: Some(r#"{"result":"ok"}"#), + needs: &["completable_task_id"], + reach: Reach::None, + principals: &[Principal::Assignee], + id_source: IdSource::CompletableTaskId, + }, + Row { + method: "POST", + path: "/api/v1/tasks/{id}/fail", + gate: GateClass::MultiPrincipalGate, + handler: "tasks::fail_task", + // Caller must equal the task's STORED assignee (tasks.rs:273). Its own + // claimed task so the assignee twin (claimed -> failed) does not race + // complete_task's. + body: Some(r#"{"reason":"x"}"#), + needs: &["failable_task_id"], + reach: Reach::None, + principals: &[Principal::Assignee], + id_source: IdSource::FailableTaskId, + }, + // The read-gate handlers NOT driven here (deferred GET reads, read-gating + // mutations, git smart-HTTP reads, the content-addressed read, and the + // global list-filters) are no longer tracked by free-text prose: they are + // ENFORCED in deny_harness.rs's `READ_GATE_NOT_DRIVEN` allowlist, which the + // `every_read_gate_handler_is_driven_or_explicitly_allowlisted` guard checks + // against a live scan of every read-gate marker in src/api. That is the + // single source of truth — adding/removing a driven read row here without + // updating the allowlist trips the guard. + ] +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn registry_internal_consistency() { + let rows = deny_bearing_routes(); + + // No duplicate method+path. + let mut seen = std::collections::HashSet::new(); + for r in rows { + assert!( + seen.insert((r.method, r.path)), + "duplicate deny-bearing row: {} {}", + r.method, + r.path + ); + } + + for r in rows { + // Every row records its handler fn (review spot-check anchor). + assert!( + !r.handler.is_empty(), + "row {} {} has no handler", + r.method, + r.path + ); + + match r.gate { + // Every read-gate row must carry a positive twin, or a 404 from a + // merely-absent entity is indistinguishable from the gate's 404. + GateClass::ReadGate => assert_ne!( + r.reach, + Reach::None, + "read-gate row {} {} needs a Reach positive twin", + r.method, + r.path + ), + // Owner-gate/signature rows use the class's own twin (owner re-sign) + // or none, never a read-path twin. + _ => assert_eq!( + r.reach, + Reach::None, + "non-read-gate row {} {} must not carry a read twin", + r.method, + r.path + ), + } + + // `principals` is meaningful ONLY on MultiPrincipalGate rows: it must + // be non-empty there (a gate with no declared arm drives nothing) and + // empty everywhere else (a stray arm set on a non-multi row is a bug). + match r.gate { + GateClass::MultiPrincipalGate => assert!( + !r.principals.is_empty(), + "multi-principal row {} {} declares no authorizing arms", + r.method, + r.path + ), + _ => assert!( + r.principals.is_empty(), + "non-multi-principal row {} {} must not declare arms", + r.method, + r.path + ), + } + } + } + + /// STRUCTURAL enforcement (#195, F1): every `MultiPrincipalGate` row's + /// generator output must carry exactly one Not403 twin PER declared arm plus + /// the single stranger hostile, so a row that registered an arm but never + /// emits its twin (the vacuous-guard failure) fails HERE rather than in a + /// runtime sweep nobody re-runs. Invokes `probes_for` directly — the same + /// generator the real sweep drives — so the two cannot drift. + #[test] + fn multi_principal_rows_emit_one_twin_per_arm() { + use crate::support::probe::{probes_for, tests_support::fx, Expect, Signer}; + + let fixture = fx(); + let mut checked = 0usize; + for r in deny_bearing_routes() { + if r.gate != GateClass::MultiPrincipalGate { + continue; + } + checked += 1; + let ps = probes_for(r, &fixture); + + let hostile = ps + .iter() + .filter(|p| p.signer == Signer::Stranger && matches!(p.expect, Expect::Deny(403))) + .count(); + assert_eq!( + hostile, 1, + "multi-principal row {} {} must drive exactly one stranger-403 hostile, drove {hostile}", + r.method, r.path + ); + + let twins = ps + .iter() + .filter(|p| matches!(p.expect, Expect::Not403)) + .count(); + assert_eq!( + twins, + r.principals.len(), + "multi-principal row {} {} declares {} arms but emits {twins} Not403 twins", + r.method, + r.path, + r.principals.len(), + ); + + // Each declared arm maps to a distinct signer, and every twin's signer + // is one of the declared arms (no phantom / wrong-arm twin). + for arm in r.principals { + let want = crate::support::probe::signer_for_principal(*arm); + let present = ps + .iter() + .any(|p| p.signer == want && matches!(p.expect, Expect::Not403)); + assert!( + present, + "multi-principal row {} {} declares arm {:?} but emits no Not403 twin for it", + r.method, r.path, arm + ); + } + + // The twins' signers must be DISTINCT and number the declared arms. The + // per-arm any() check above is satisfied by a single twin when two arms + // map to the same Signer (a signer_for_principal collision), so without + // this a wrong mapping would pass with one arm silently untested. + let mut twin_signers: Vec = ps + .iter() + .filter(|p| matches!(p.expect, Expect::Not403)) + .map(|p| p.signer) + .collect(); + twin_signers.sort_by_key(|s| format!("{s:?}")); + twin_signers.dedup(); + assert_eq!( + twin_signers.len(), + r.principals.len(), + "multi-principal row {} {} emits twins with {} DISTINCT signers but declares {} arms \ + (Principal->Signer collision?)", + r.method, + r.path, + twin_signers.len(), + r.principals.len(), + ); + } + assert!( + checked >= 8, + "expected at least the eight known multi-principal rows (close_pr, \ + close_issue, dispute/submit/approve/cancel bounty, complete_task, \ + fail_task), checked {checked}" + ); + } + + /// STRUCTURAL enforcement for `SignerSelfGate` rows (#195, F3): each caller-self + /// field-binding row's generator output must be exactly one Stranger/Deny(403) + /// hostile plus one Owner/Not403 control, and both bodies must bind the self + /// field to the owner DID (so the hostile is a real signer-vs-body mismatch and + /// the control a real match). Mirrors `multi_principal_rows_emit_one_twin_per_arm` + /// so a SignerSelfGate row that stopped emitting its twin (the vacuous-guard + /// failure) fails HERE, not only in a DB sweep nobody re-runs. Invokes + /// `probes_for` directly, the same generator the real sweep drives. + #[test] + fn signer_self_rows_emit_one_hostile_and_one_control() { + use crate::support::probe::{probes_for, tests_support::fx, Expect, Signer}; + + let fixture = fx(); + let mut checked = 0usize; + for r in deny_bearing_routes() { + if r.gate != GateClass::SignerSelfGate { + continue; + } + checked += 1; + let ps = probes_for(r, &fixture); + assert_eq!( + ps.len(), + 2, + "signer-self row {} {} must emit exactly two probes, emitted {}", + r.method, + r.path, + ps.len() + ); + + let hostile = ps + .iter() + .filter(|p| p.signer == Signer::Stranger && matches!(p.expect, Expect::Deny(403))) + .count(); + assert_eq!( + hostile, 1, + "signer-self row {} {} must drive exactly one stranger-403 hostile, drove {hostile}", + r.method, r.path + ); + + let control = ps + .iter() + .filter(|p| p.signer == Signer::Owner && matches!(p.expect, Expect::Not403)) + .count(); + assert_eq!( + control, 1, + "signer-self row {} {} must drive exactly one owner Not403 control, drove {control}", + r.method, r.path + ); + + // The bound self field (delegator_did / assignee_did / did) must resolve + // to the owner DID in BOTH bodies: the hostile is a stranger signing a + // body that names the owner (mismatch), the control an owner signing the + // same (match). A row whose body did not run through `fill()` would carry + // a literal `{owner}` and fail here. + for p in &ps { + let body = String::from_utf8(p.body.clone()).expect("probe body is UTF-8 JSON"); + assert!( + body.contains(&fixture.owner_did), + "signer-self row {} {} body must bind the self field to the owner DID, got {body}", + r.method, + r.path + ); + } + } + assert!( + checked >= 3, + "expected at least the three SignerSelfGate rows (create_task, claim_task, \ + register), checked {checked}" + ); + } +} From 1720c38b01c97a5cc00b3319a8f583d6a7539ab2 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 21 Jul 2026 13:41:39 -0500 Subject: [PATCH 24/41] fix(node): bracket the fallback identity-key publish with a durable marker The hard-link fallback exposes the final key name before the PEM is complete, so a crash in that window used to leave a partial final that wedged every later start. Create a bounded-name .publishing marker and fsync its name durable before create_new(final), dispose of it on every exit with the final's removal ordered first, and remove it only once the final is complete and durable. The marker dir fsync is warn-and-continue so dir-fsync-hostile mounts degrade to reduced power-loss protection instead of never provisioning. --- crates/gitlawb-node/src/lib.rs | 427 ++++++++++++++++++++++++++++++++- 1 file changed, 415 insertions(+), 12 deletions(-) diff --git a/crates/gitlawb-node/src/lib.rs b/crates/gitlawb-node/src/lib.rs index 7d9395bc..3ee143df 100644 --- a/crates/gitlawb-node/src/lib.rs +++ b/crates/gitlawb-node/src/lib.rs @@ -1153,7 +1153,8 @@ const KEY_RACE_DEADLINE: std::time::Duration = std::time::Duration::from_secs(5) /// by a crashed prior start with the same PID (a restarted container is /// commonly PID 1 again) is skipped rather than wedging the restart (#194). /// 64 names is far more than any plausible pile-up of stale temps plus live -/// concurrent publishers. +/// concurrent publishers. Also bounds the fallback publish-marker names, +/// which follow the same stale-name discipline (#194, U1). const KEY_TEMP_ATTEMPTS: u32 = 64; /// Load an already-provisioned identity key. On Unix, defensively tighten looser @@ -1216,6 +1217,16 @@ struct PublishFaults { /// is taken as the open result (a non-`AlreadyExists` open error), so the /// publish surfaces it chained with `link_err`. fallback_create: fn() -> std::io::Result<()>, + /// Runs before the fallback's publish-marker `create_new` (each name + /// attempt); an `Err` is taken as the create result (a non-`AlreadyExists` + /// open error), so the publish surfaces it chained with `link_err` (#194, + /// U1). + marker_create: fn() -> std::io::Result<()>, + /// Runs before the fsync of the marker's parent directory; an `Err` is + /// taken as that fsync's result. This fsync is warn-and-continue, so an + /// injected `Err` must NOT fail the publish (#194, U1). + #[cfg(unix)] + marker_fsync: fn() -> std::io::Result<()>, /// Runs before the fallback's `write_all`; an `Err` is taken as the write /// result. fallback_write: fn() -> std::io::Result<()>, @@ -1238,6 +1249,9 @@ impl PublishFaults { const NONE: Self = Self { link: || Ok(()), fallback_create: || Ok(()), + marker_create: || Ok(()), + #[cfg(unix)] + marker_fsync: || Ok(()), fallback_write: || Ok(()), fallback_fsync: || Ok(()), #[cfg(unix)] @@ -1379,6 +1393,26 @@ fn publish_key_atomically( /// parent-DIR fsync keeps the final (the content is complete and data-synced, /// and a lost directory entry just disappears -> clean regen, no wedge). Every /// error context chains `link_err` so a two-step failure is diagnosable. +/// +/// The whole non-atomic window is bracketed by a durable publish marker +/// (`.{stem}.publishing.{pid}.{attempt}`, empty, same dir as the final): the +/// marker is created and its NAME fsynced BEFORE the final can exist, and it +/// is removed only once the final is complete-and-durable (success) or +/// disposed of (error / lost race). A crash inside the window therefore +/// leaves marker+final together, so a later boot can tell "partial final from +/// a crashed fallback publish" from a key an operator corrupted (#194, U1). +/// Two constraints: +/// - Disposal ORDER is pinned: wherever the error policy removes the final, +/// that removal is ISSUED BEFORE the marker's. Cleanup is deliberately +/// NOT a Drop guard: a drop guard can run ahead of the final's disposal +/// on an early exit, and a crash between the reordered unlinks would +/// recreate exactly the marker-less partial final this bracket exists to +/// prevent. +/// - The marker dir fsync is warn-and-continue, not a hard gate: on +/// dir-fsync-hostile mounts a hard failure would turn "wedge once, then +/// recover" into "never provisions". Without that durability the marker +/// still survives SIGKILL-class crashes (the fs state persists); only the +/// power-loss protection narrows. fn publish_key_fallback( final_path: &std::path::Path, pem: &[u8], @@ -1391,6 +1425,79 @@ fn publish_key_fallback( err = %link_err, "hard_link failed; publishing identity key via direct create_new fallback" ); + let dir = final_path + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or_else(|| std::path::Path::new(".")); + let stem = final_path + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or("identity.pem"); + + // Create the publish marker BEFORE the final name can exist, probing + // bounded per-call names exactly like the temp-name loop and for the same + // reasons: a stale marker left by a crashed same-PID start must not wedge + // the restart, and a stale marker is indistinguishable from a live + // concurrent publisher's, so never delete a name this call did not + // create; just skip to the next one (#194, U1). + let mut marker = None; + for attempt in 0..KEY_TEMP_ATTEMPTS { + let candidate = dir.join(format!( + ".{stem}.publishing.{}.{attempt}", + std::process::id() + )); + match (faults.marker_create)().and_then(|()| { + std::fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&candidate) + }) { + Ok(f) => { + drop(f); + marker = Some(candidate); + break; + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => continue, + Err(e) => { + return Err(e).with_context(|| { + format!( + "create publish marker {} in hard_link fallback (link failed: {link_err})", + candidate.display() + ) + }) + } + } + } + let Some(marker) = marker else { + anyhow::bail!( + "all {KEY_TEMP_ATTEMPTS} publish marker names .{stem}.publishing.{}.* in {} are \ + taken; remove the stale marker files and restart (link failed: {link_err})", + std::process::id(), + dir.display() + ); + }; + // Marker disposal is explicit at every exit, never a Drop guard; see the + // order constraint in the doc comment. + let dispose_marker = || { + let _ = std::fs::remove_file(&marker); + }; + // Make the marker NAME durable before the final can exist, so no crash + // can leave a partial final without its marker. Warn-and-continue on + // failure per the doc comment's degradation constraint. + #[cfg(unix)] + { + if let Err(e) = (faults.marker_fsync)() + .map_err(anyhow::Error::new) + .and_then(|()| fsync_parent_dir(final_path)) + { + warn!( + marker = %marker.display(), + err = %e, + "publish marker dir fsync failed; continuing with reduced power-loss protection" + ); + } + } + let mut opts = std::fs::OpenOptions::new(); opts.write(true).create_new(true); #[cfg(unix)] @@ -1400,28 +1507,45 @@ fn publish_key_fallback( } let mut f = match (faults.fallback_create)().and_then(|()| opts.open(final_path)) { Ok(f) => f, - Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => return Ok(KeyPublish::Lost), + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { + // Lost race: the existing final is the winner's complete key, not + // ours to bracket; drop our marker and defer to it. + dispose_marker(); + return Ok(KeyPublish::Lost); + } Err(e) => { + // The final was never created here, so there is no final removal + // to order ahead of the marker's. + dispose_marker(); return Err(e).with_context(|| { format!( "create identity key {} in hard_link fallback (link failed: {link_err})", final_path.display() ) - }) + }); } }; // Secure the empty final BEFORE the PEM bytes hit it (#194, U2). The // helper's fail-closed removal is what keeps a retry from wedging on // AlreadyExists, and is safe here for the same reason as everywhere: the // file is still empty, so the split-phase keep-the-final rule below does - // not apply yet. + // not apply yet. The helper removes the final before returning Err, so + // the pinned final-before-marker disposal order holds. #[cfg(unix)] - tighten_and_verify_created(final_path, faults.fallback_mode_verify).with_context(|| { - format!("secure identity key in hard_link fallback (link failed: {link_err})") - })?; + { + if let Err(e) = tighten_and_verify_created(final_path, faults.fallback_mode_verify) + .with_context(|| { + format!("secure identity key in hard_link fallback (link failed: {link_err})") + }) + { + dispose_marker(); + return Err(e); + } + } // A failed write removes the final (partial content cannot parse and would - // wedge every later start). - write_key_or_cleanup( + // wedge every later start). The helper removes the final before returning + // Err, so the pinned final-before-marker disposal order holds. + if let Err(e) = write_key_or_cleanup( final_path, (faults.fallback_write)().and_then(|()| f.write_all(pem)), ) @@ -1430,7 +1554,10 @@ fn publish_key_fallback( "hard_link fallback publish to {} (link failed: {link_err})", final_path.display() ) - })?; + }) { + dispose_marker(); + return Err(e); + } // Fsync the file, and REMOVE the final on failure too (#194 F1). Unlike the // hard-link path (which fsyncs a TEMP, so the final NAME only ever appears // over a durable inode), the fallback's `create_new` already made the final @@ -1442,8 +1569,10 @@ fn publish_key_fallback( // ENOSPC/EIO can tell "durability failed" (bytes accepted, not synced) from // the write-rejected case above. Only the parent-DIR fsync below keeps the // final on failure (a lost directory entry just disappears -> clean regen). + // Final removal is issued first, marker disposal second (pinned order). if let Err(e) = (faults.fallback_fsync)().and_then(|()| f.sync_all()) { let _ = std::fs::remove_file(final_path); + dispose_marker(); return Err(anyhow::Error::new(e).context(format!( "fsync identity key {} in hard_link fallback (durability failed, \ removed; link failed: {link_err})", @@ -1453,9 +1582,24 @@ fn publish_key_fallback( drop(f); // Fsync the parent directory so the new name is durable before success is // reported, as on the hard-link path. Same policy on failure: the final is - // complete and data-synced, so it is NOT removed. + // complete and data-synced, so it is NOT removed; the marker is still + // disposed of best-effort. #[cfg(unix)] - fsync_parent_dir(final_path)?; + { + if let Err(e) = fsync_parent_dir(final_path) { + dispose_marker(); + return Err(e); + } + } + // Success: the final is complete and durable, so the bracket closes. + // Remove the marker, then best-effort fsync the removal so it tends to be + // durable too. (A later unit turns this removal into a commit check; U1 + // keeps it best-effort.) + dispose_marker(); + #[cfg(unix)] + { + let _ = fsync_parent_dir(final_path); + } Ok(KeyPublish::Won) } @@ -2446,6 +2590,265 @@ mod identity_key_tests { "the retry must publish the intended identity" ); } + + /// Lists leftover `.{stem}.publishing.*` marker files in `dir` (#194, U1). + fn publishing_markers(dir: &std::path::Path) -> Vec { + std::fs::read_dir(dir) + .unwrap() + .filter_map(|e| e.ok().map(|e| e.path())) + .filter(|p| { + p.file_name() + .map(|n| n.to_string_lossy().contains(".publishing.")) + .unwrap_or(false) + }) + .collect() + } + + /// #194 (U1): a successful fallback publish (link forced to fail) must win, + /// round-trip the key through the normal loader, and leave NO publish + /// marker behind: the marker exists only for the non-atomic window. + #[test] + fn fallback_publish_succeeds_and_leaves_no_marker() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let kp = Keypair::generate(); + let pem = kp.to_pem().expect("pem"); + + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + ..PublishFaults::NONE + }; + let out = publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, faults) + .expect("fallback publish"); + assert!(matches!(out, KeyPublish::Won), "fallback publish wins"); + let loaded = super::load_existing_key(&key_path).expect("published key loads"); + assert_eq!( + format!("{}", loaded.did()), + format!("{}", kp.did()), + "fallback publish must round-trip the same identity" + ); + let markers = publishing_markers(dir.path()); + assert!( + markers.is_empty(), + "a successful fallback publish must remove its marker, found {markers:?}" + ); + } + + /// #194 (U1): a fallback that loses the create race (final already present) + /// must still behave as today (existing key loaded, never clobbered) and + /// must remove its own marker on the lost-race exit. + #[test] + fn lost_fallback_race_leaves_no_marker() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let winner = Keypair::generate(); + let pem_winner = winner.to_pem().expect("pem winner"); + let out = publish_key_atomically( + &key_path, + pem_winner.as_bytes(), + &|| {}, + PublishFaults::NONE, + ) + .expect("winner publishes"); + assert!(matches!(out, KeyPublish::Won), "winner publish wins"); + + let pem_loser = Keypair::generate().to_pem().expect("pem loser"); + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::PermissionDenied.into()), + ..PublishFaults::NONE + }; + let out = publish_key_atomically(&key_path, pem_loser.as_bytes(), &|| {}, faults) + .expect("an existing final must read as a lost race, not an error"); + assert!(matches!(out, KeyPublish::Lost), "fallback loses"); + let loaded = load_racing_key(&key_path).expect("lost race loads the winner"); + assert_eq!( + format!("{}", loaded.did()), + format!("{}", winner.did()), + "the lost race must return the winner's identity" + ); + let markers = publishing_markers(dir.path()); + assert!( + markers.is_empty(), + "a lost fallback race must remove its own marker, found {markers:?}" + ); + } + + /// Shared body for the fallback error-arm marker tests (#194, U1): with the + /// link fault plus one injected fallback fault, the error must surface as + /// today, the final must be absent (today's removal policy for every one of + /// these arms), and no publish marker may remain. + fn assert_fallback_error_leaves_no_marker_or_final(faults: PublishFaults, injected: &str) { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let pem = Keypair::generate().to_pem().expect("pem"); + + let err = match publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, faults) { + Err(e) => e, + Ok(_) => panic!("the injected fallback fault must error the publish"), + }; + let msg = format!("{err:#}"); + assert!( + msg.contains(injected), + "the error must surface the injected fault: {msg}" + ); + assert!( + !key_path.exists(), + "the final must be absent per the existing removal policy" + ); + let markers = publishing_markers(dir.path()); + assert!( + markers.is_empty(), + "an errored fallback must dispose its marker, found {markers:?}" + ); + } + + #[test] + fn failed_fallback_create_leaves_no_marker() { + assert_fallback_error_leaves_no_marker_or_final( + PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + fallback_create: || Err(std::io::Error::other("injected fallback create failure")), + ..PublishFaults::NONE + }, + "injected fallback create failure", + ); + } + + #[test] + fn failed_fallback_write_leaves_no_marker() { + assert_fallback_error_leaves_no_marker_or_final( + PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + fallback_write: || Err(std::io::Error::other("injected write failure")), + ..PublishFaults::NONE + }, + "injected write failure", + ); + } + + #[test] + fn failed_fallback_fsync_leaves_no_marker() { + assert_fallback_error_leaves_no_marker_or_final( + PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + fallback_fsync: || Err(std::io::Error::other("injected fsync failure")), + ..PublishFaults::NONE + }, + "injected fsync failure", + ); + } + + #[test] + fn failed_fallback_mode_verify_leaves_no_marker() { + assert_fallback_error_leaves_no_marker_or_final( + PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + fallback_mode_verify: || Err(std::io::Error::other("injected mode-ignoring mount")), + ..PublishFaults::NONE + }, + "injected mode-ignoring mount", + ); + } + + /// #194 (U1): marker-name probing is bounded like the temp names. With + /// every candidate marker name taken the fallback must fail loudly, naming + /// the stale prefix and directory, and must never create the final. + #[test] + fn marker_name_exhaustion_errors_loudly_and_never_creates_the_final() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let pem = Keypair::generate().to_pem().expect("pem"); + + for attempt in 0..KEY_TEMP_ATTEMPTS { + let stale = dir.path().join(format!( + ".identity.pem.publishing.{}.{attempt}", + std::process::id() + )); + std::fs::write(&stale, b"").expect("seed stale marker"); + } + + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + ..PublishFaults::NONE + }; + let err = match publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, faults) { + Err(e) => e, + Ok(_) => panic!("the fallback must fail when every marker name is taken"), + }; + let msg = format!("{err:#}"); + assert!( + msg.contains(&format!(".identity.pem.publishing.{}", std::process::id())) + && msg.contains(&dir.path().display().to_string()), + "the error must name the marker prefix and directory: {msg}" + ); + assert!( + !key_path.exists(), + "an exhausted marker probe must not create the final key" + ); + } + + /// #194 (U1): a failed marker DIR fsync must NOT fail the publish + /// (warn-and-continue): on dir-fsync-hostile mounts a hard gate would turn + /// "wedge once then recover" into "never provisions". The publish must + /// still win, the key must load, and the marker must still be removed. + #[test] + fn marker_fsync_failure_does_not_fail_the_publish() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let kp = Keypair::generate(); + let pem = kp.to_pem().expect("pem"); + + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + marker_fsync: || Err(std::io::Error::other("injected marker dir fsync failure")), + ..PublishFaults::NONE + }; + let out = publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, faults) + .expect("a marker dir fsync failure must not fail the publish"); + assert!(matches!(out, KeyPublish::Won), "publish still wins"); + let loaded = super::load_existing_key(&key_path).expect("published key loads"); + assert_eq!( + format!("{}", loaded.did()), + format!("{}", kp.did()), + "publish must round-trip the same identity" + ); + let markers = publishing_markers(dir.path()); + assert!( + markers.is_empty(), + "the marker must still be removed, found {markers:?}" + ); + } + + /// #194 (U1): the primary hard-link path must never touch the marker + /// machinery. The `marker_create` hook is poisoned, so if the primary path + /// ever created a marker the publish would error; it must win untouched + /// and leave no `.publishing.` file. + #[test] + fn primary_hardlink_publish_never_creates_a_marker() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let kp = Keypair::generate(); + let pem = kp.to_pem().expect("pem"); + + let faults = PublishFaults { + marker_create: || Err(std::io::Error::other("marker created on the primary path")), + ..PublishFaults::NONE + }; + let out = publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, faults) + .expect("the primary path must never reach the marker machinery"); + assert!(matches!(out, KeyPublish::Won), "primary publish wins"); + let loaded = super::load_existing_key(&key_path).expect("published key loads"); + assert_eq!( + format!("{}", loaded.did()), + format!("{}", kp.did()), + "primary publish must round-trip the same identity" + ); + let markers = publishing_markers(dir.path()); + assert!( + markers.is_empty(), + "the primary path must never create a marker, found {markers:?}" + ); + } } #[cfg(test)] From 3229022e5eed12507a47cb09846678a0a448c03b Mon Sep 17 00:00:00 2001 From: t Date: Tue, 21 Jul 2026 14:12:03 -0500 Subject: [PATCH 25/41] fix(node): recover a crash-interrupted fallback key publish at boot A marker plus a content-failed final (empty or unparseable PEM) is now provably an interrupted first write: boot claims one marker by atomic rename to a .recovering file, quarantines the final preserving its bytes, sweeps the remaining markers, and regenerates, at most once per start. Every other failure keeps today's loud fail bit-for-bit: a bad final without a marker, any non-content load failure (transient EIO, mode, stat), and a second-iteration failure never touch the key. The error class is carried as a typed root error so recovery cannot key on message text. --- crates/gitlawb-node/src/lib.rs | 739 +++++++++++++++++++++++++++++++-- 1 file changed, 715 insertions(+), 24 deletions(-) diff --git a/crates/gitlawb-node/src/lib.rs b/crates/gitlawb-node/src/lib.rs index 3ee143df..94f9f537 100644 --- a/crates/gitlawb-node/src/lib.rs +++ b/crates/gitlawb-node/src/lib.rs @@ -1157,6 +1157,29 @@ const KEY_RACE_DEADLINE: std::time::Duration = std::time::Duration::from_secs(5) /// which follow the same stale-name discipline (#194, U1). const KEY_TEMP_ATTEMPTS: u32 = 64; +/// A key-load failure caused by the file's CONTENT (an empty or unparseable +/// PEM) rather than IO or permissions (#194, U2). Used as the anyhow error's +/// root so the boot loop can classify a failure as crash-recoverable via +/// `is_key_content_error` WITHOUT changing the message text operators and +/// tests rely on ("invalid PEM key"). Every other load failure (stat, +/// tighten, mode verification, read IO) stays a plain anyhow error and is +/// never recovered from. +#[derive(Debug)] +struct KeyContentError(String); + +impl std::fmt::Display for KeyContentError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +impl std::error::Error for KeyContentError {} + +/// True when `err` is a content-class load failure (see `KeyContentError`). +fn is_key_content_error(err: &anyhow::Error) -> bool { + err.downcast_ref::().is_some() +} + /// Load an already-provisioned identity key. On Unix, defensively tighten looser /// permissions to 0600 (do NOT reject a loose key — that would break existing /// deployments; just narrow them), then verify the mode is actually 0600. @@ -1180,24 +1203,101 @@ fn load_existing_key(path: &std::path::Path) -> Result { } let pem = std::fs::read_to_string(path) .with_context(|| format!("failed to read key from {}", path.display()))?; - let kp = Keypair::from_pem(&pem).map_err(|e| anyhow::anyhow!("invalid PEM key: {e}"))?; + // Content-class failure (#194, U2): an empty file reads Ok("") and is + // rejected here too, so this one arm classifies both the zero-length and + // the unparseable case. The message text is unchanged. + let kp = Keypair::from_pem(&pem) + .map_err(|e| anyhow::Error::new(KeyContentError(format!("invalid PEM key: {e}"))))?; info!(path = %path.display(), "loaded existing identity"); Ok(kp) } +/// The publish markers (`.{stem}.publishing.*`) beside `final_path`, sorted: +/// U1's crash signature for a fallback publish interrupted mid-window (#194, +/// U2). An unreadable directory reads as no markers, so recovery stays off +/// and the load error surfaces unchanged (the loud-fail default); entries +/// with non-UTF-8 names cannot be our markers and are skipped. +fn list_publish_markers(final_path: &std::path::Path) -> Vec { + let dir = final_path + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or_else(|| std::path::Path::new(".")); + let stem = final_path + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or("identity.pem"); + let prefix = format!(".{stem}.publishing."); + let Ok(entries) = std::fs::read_dir(dir) else { + return Vec::new(); + }; + let mut markers: Vec<_> = entries + .filter_map(|e| e.ok()) + .filter(|e| { + e.file_name() + .to_str() + .is_some_and(|n| n.starts_with(&prefix)) + }) + .map(|e| e.path()) + .collect(); + markers.sort(); + markers +} + /// Load a key another concurrent start may still be publishing, polling until it /// parses or `KEY_RACE_DEADLINE` elapses. An already-provisioned key parses on /// the first attempt with no sleep. fn load_racing_key(path: &std::path::Path) -> Result { + match boot_load_key(path, false, || Ok(())) { + BootLoad::Loaded(kp) => Ok(*kp), + BootLoad::CrashSignature(e) | BootLoad::Failed(e) => Err(e), + } +} + +/// Outcome of `boot_load_key` (#194, U2). The tri-state exists so the boot +/// loop can tell "content failure WITH the crash signature observed" (a +/// recovery candidate) from every other failure; folding both into one error +/// would let a concurrent recoverer's marker sweep turn a recoverable boot +/// into a loud failure. +enum BootLoad { + /// Boxed to keep the variants' sizes comparable (clippy: + /// large_enum_variant); the enum lives only across a boot-time match. + Loaded(Box), + /// Content-class failure observed while a `.publishing.` marker was + /// present; returned immediately, without riding the deadline. + CrashSignature(anyhow::Error), + /// Any other failure, after the full deadline. + Failed(anyhow::Error), +} + +/// `load_racing_key` for the boot loop (#194, U2). With `crash_exit` set, a +/// content-class failure (`KeyContentError`) observed while a `.publishing.` +/// marker is present returns `CrashSignature` IMMEDIATELY instead of riding +/// the deadline: marker plus unreadable content is the crash signature +/// recovery exists for, and the caller's atomic claim rename arbitrates the +/// (rare) race against a live fallback publisher still inside its window. +/// Every other failure keeps the full wall-clock deadline. `load_fault` is +/// the injection hook (a no-op `Ok` in production); an injected `Err` is +/// taken as that attempt's result and is NOT content-class. +fn boot_load_key( + path: &std::path::Path, + crash_exit: bool, + load_fault: fn() -> std::io::Result<()>, +) -> BootLoad { let deadline = std::time::Instant::now() + KEY_RACE_DEADLINE; let mut last_err; loop { - match load_existing_key(path) { - Ok(kp) => return Ok(kp), + match load_fault() + .map_err(anyhow::Error::new) + .and_then(|()| load_existing_key(path)) + { + Ok(kp) => return BootLoad::Loaded(Box::new(kp)), Err(e) => last_err = e, } + if crash_exit && is_key_content_error(&last_err) && !list_publish_markers(path).is_empty() { + return BootLoad::CrashSignature(last_err); + } if std::time::Instant::now() >= deadline { - return Err(last_err); + return BootLoad::Failed(last_err); } std::thread::sleep(std::time::Duration::from_millis(2)); } @@ -1603,33 +1703,270 @@ fn publish_key_fallback( Ok(KeyPublish::Won) } +/// Compile-time test seam for the load-side crash recovery in +/// `load_or_create_keypair_with` (#194, U2), following the `PublishFaults` +/// precedent: production passes `RecoverySeam::NONE` (every hook a no-op). +#[derive(Clone, Copy)] +struct RecoverySeam<'a> { + /// Runs before every load attempt of the boot loader; an `Err` is taken as + /// that attempt's result. The injected failure is NOT content-class, so it + /// deterministically exercises the transient-failure (never-recover) policy. + load_fault: fn() -> std::io::Result<()>, + /// Runs at the top of each generate+publish pass, so a test can mutate + /// on-disk state between a completed recovery and the retry publish. + before_publish: &'a dyn Fn(), +} + +impl RecoverySeam<'_> { + /// The production value: no fault, no hook. + const NONE: RecoverySeam<'static> = RecoverySeam { + load_fault: || Ok(()), + before_publish: &|| {}, + }; +} + fn load_or_create_keypair(config: &Config) -> Result { - let key_path = config.resolved_key_path(); + load_or_create_keypair_with( + &config.resolved_key_path(), + &|| {}, + PublishFaults::NONE, + RecoverySeam::NONE, + ) +} + +/// Outcome of `recover_crashed_publish` (#194, U2). +enum Recovery { + /// A marker was claimed and the corrupt final quarantined (or removed); + /// the caller regenerates. Holds the claim file's path, removed only + /// after the follow-up publish resolves. + Claimed(std::path::PathBuf), + /// The marker vanished before it could be claimed (the publisher — or a + /// competing recoverer — resolved the window concurrently): the single + /// follow-up load's result is final, success or error, no recovery. + /// Boxed to keep the variants' sizes comparable (clippy: + /// large_enum_variant); the enum lives only across a boot-time match. + Reloaded(Box>), +} + +/// Claim-and-quarantine for a boot whose load returned `CrashSignature`: a +/// content-class failure beside a `.publishing.` marker — U1's signature of +/// a fallback publish crashed inside its non-atomic window (#194, U2). +/// Atomically claims ONE marker (rename to `.{stem}.recovering.{pid}`, so N +/// racing recoverers admit exactly one), quarantines the corrupt final by +/// rename (preserving bytes and mode for post-mortem; falls back to removal +/// only if every quarantine name fails), consumes any remaining markers +/// best-effort, and hands control back to regenerate. The caller observed a +/// marker, so finding none here (or losing the claim rename) means the +/// window was resolved concurrently: one immediate re-load decides, with no +/// recovery. +fn recover_crashed_publish( + final_path: &std::path::Path, + load_err: anyhow::Error, +) -> Result { + let dir = final_path + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or_else(|| std::path::Path::new(".")); + let stem = final_path + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or("identity.pem"); + let markers = list_publish_markers(final_path); + let Some(first_marker) = markers.first() else { + return Ok(Recovery::Reloaded(Box::new(load_racing_key(final_path)))); + }; - // Fast path for the common already-provisioned case (still race-safe: a - // concurrently-publishing winner is handled by the retry in load_racing_key). - if key_path.exists() { - return load_racing_key(&key_path); + // CLAIM one marker atomically. rename admits exactly one winner: a + // competing recoverer (or the publisher finishing and removing its + // marker) leaves NotFound, which means the window resolved concurrently; + // one immediate re-load decides, with no recovery. + let claim = dir.join(format!(".{stem}.recovering.{}", std::process::id())); + match std::fs::rename(first_marker, &claim) { + Ok(()) => {} + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + return Ok(Recovery::Reloaded(Box::new(load_racing_key(final_path)))); + } + Err(e) => { + return Err(load_err.context(format!( + "could not claim publish marker {} for crash recovery: {e}", + first_marker.display() + ))); + } } - let kp = Keypair::generate(); - let pem = kp - .to_pem() - .map_err(|e| anyhow::anyhow!("failed to serialize key: {e}"))?; + // QUARANTINE the final under a bounded name probe (the KEY_TEMP_ATTEMPTS + // discipline: a stale quarantine from a crashed same-PID start must not + // wedge this one). The warn precedes the move and carries stable + // operator-greppable fields; the destination it names is the first free + // candidate, which the rename below tries first. + let dest_for = |n: u32| dir.join(format!(".{stem}.quarantined.{}.{n}", std::process::id())); + let start = (0..KEY_TEMP_ATTEMPTS) + .find(|&n| !dest_for(n).exists()) + .unwrap_or(0); + warn!( + path = %final_path.display(), + markers = ?markers, + claim = %claim.display(), + quarantine = %dest_for(start).display(), + "crash-interrupted identity key publish: quarantining the unreadable key and \ + regenerating the identity" + ); + let mut quarantined = false; + for n in start..KEY_TEMP_ATTEMPTS { + let dest = dest_for(n); + if dest.exists() { + continue; + } + match std::fs::rename(final_path, &dest) { + Ok(()) => { + quarantined = true; + break; + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + // The final vanished concurrently; nothing left to quarantine. + quarantined = true; + break; + } + Err(_) => continue, + } + } + if !quarantined { + // Every rename failed: removal still unblocks the regeneration + // (mirroring the publish paths' removal policy); only if even that + // fails is the boot left to its loud error. + if let Err(e) = std::fs::remove_file(final_path) { + if e.kind() != std::io::ErrorKind::NotFound { + return Err(anyhow::Error::new(e).context(format!( + "could not quarantine or remove corrupt identity key {} during crash \ + recovery", + final_path.display() + ))); + } + } + } - if let Some(parent) = key_path.parent() { - std::fs::create_dir_all(parent)?; + // Consume any remaining markers (a multi-crash pile-up) best-effort, and + // best-effort fsync the directory so the transition tends to be durable. + for marker in list_publish_markers(final_path) { + let _ = std::fs::remove_file(&marker); + } + #[cfg(unix)] + { + let _ = fsync_parent_dir(final_path); } + Ok(Recovery::Claimed(claim)) +} + +/// Body of `load_or_create_keypair` with the test seams threaded in (#194, +/// U2): `before_link` and `faults` pass through to `publish_key_atomically` +/// (production: no-op / `NONE`), `seam` is the load-side recovery seam +/// (production: `RecoverySeam::NONE`). +/// +/// Load-or-generate with AT MOST ONE crash recovery per start: a load +/// failure recovers — claim a marker, quarantine the final, regenerate — +/// only when it is content-class (`KeyContentError`), a `.publishing.` +/// marker exists (U1's crash signature), and no recovery has run yet this +/// start. Every other failure surfaces unchanged. The Lost arm participates +/// in the same policy; a recovery fired there loops back for one retry +/// publish, and any failure on that second pass surfaces unchanged. +fn load_or_create_keypair_with( + key_path: &std::path::Path, + before_link: &dyn Fn(), + faults: PublishFaults, + seam: RecoverySeam<'_>, +) -> Result { + // The claim file of a fired recovery. Removed (best-effort) only at exits + // AFTER the follow-up publish resolves (Won or Lost), so a stalled winner + // our claim demoted can still observe it while waiting; U3 relies on + // this. The pre-publish error exits inside recover_crashed_publish + // deliberately leave it in place. + let mut claim: Option = None; + let remove_claim = |claim: &Option| { + if let Some(c) = claim { + let _ = std::fs::remove_file(c); + } + }; + let mut recovery_allowed = true; + + // At most two passes: the second exists only so a Lost-arm recovery can + // retry the generate+publish once. + for _pass in 0..2 { + // Fast path for the common already-provisioned case (still race-safe: a + // concurrently-publishing winner is handled by the retry in + // boot_load_key). + if key_path.exists() { + match boot_load_key(key_path, recovery_allowed, seam.load_fault) { + BootLoad::Loaded(kp) => { + remove_claim(&claim); + return Ok(*kp); + } + // Only emitted while recovery is still allowed (crash_exit is + // gated on it). + BootLoad::CrashSignature(e) => match recover_crashed_publish(key_path, e)? { + Recovery::Claimed(c) => { + claim = Some(c); + recovery_allowed = false; + // Fall through to regenerate and publish. + } + Recovery::Reloaded(result) => return *result, + }, + BootLoad::Failed(e) => { + remove_claim(&claim); + return Err(e); + } + } + } + + (seam.before_publish)(); + let kp = Keypair::generate(); + let pem = kp + .to_pem() + .map_err(|e| anyhow::anyhow!("failed to serialize key: {e}"))?; + + if let Some(parent) = key_path.parent() { + std::fs::create_dir_all(parent)?; + } - // Publish atomically: the final path only ever appears complete, and a lost - // race loads the winner's key rather than overwriting it. - match publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, PublishFaults::NONE)? { - KeyPublish::Won => { - info!(path = %key_path.display(), did = %kp.did(), "generated new node identity"); - Ok(kp) + // Publish atomically: the final path only ever appears complete, and a + // lost race loads the winner's key rather than overwriting it. + match publish_key_atomically(key_path, pem.as_bytes(), before_link, faults)? { + KeyPublish::Won => { + remove_claim(&claim); + info!(path = %key_path.display(), did = %kp.did(), "generated new node identity"); + return Ok(kp); + } + KeyPublish::Lost => { + match boot_load_key(key_path, recovery_allowed, seam.load_fault) { + BootLoad::Loaded(kp) => { + remove_claim(&claim); + return Ok(*kp); + } + // Only emitted while recovery is still allowed (crash_exit + // is gated on it). + BootLoad::CrashSignature(e) => match recover_crashed_publish(key_path, e) { + Ok(Recovery::Claimed(c)) => { + claim = Some(c); + recovery_allowed = false; + continue; // Second pass: regenerate and publish. + } + Ok(Recovery::Reloaded(result)) => { + remove_claim(&claim); + return *result; + } + Err(err) => { + remove_claim(&claim); + return Err(err); + } + }, + BootLoad::Failed(e) => { + remove_claim(&claim); + return Err(e); + } + } + } } - KeyPublish::Lost => load_racing_key(&key_path), } + unreachable!("the boot loop returns within two passes"); } #[cfg(test)] @@ -1696,8 +2033,9 @@ mod gossip_ssrf_tests { #[cfg(all(test, unix))] mod identity_key_tests { use super::{ - load_or_create_keypair, load_racing_key, publish_key_atomically, Config, KeyPublish, - Keypair, PublishFaults, KEY_TEMP_ATTEMPTS, + load_or_create_keypair, load_or_create_keypair_with, load_racing_key, + publish_key_atomically, Config, KeyPublish, Keypair, PublishFaults, RecoverySeam, + KEY_TEMP_ATTEMPTS, }; use clap::Parser; use std::os::unix::fs::PermissionsExt; @@ -2849,6 +3187,359 @@ mod identity_key_tests { "the primary path must never create a marker, found {markers:?}" ); } + + /// Seed the on-disk signature of a fallback publish crashed mid-write + /// (#194, U2): a 0600 final holding `content` (empty or truncated) beside + /// a `.publishing.` marker carrying a foreign (crashed) process's pid. + /// Returns the final's path. + fn seed_crash_state(dir: &std::path::Path, content: &[u8]) -> std::path::PathBuf { + let key_path = dir.join("identity.pem"); + std::fs::write(&key_path, content).expect("seed crashed final"); + std::fs::set_permissions(&key_path, std::fs::Permissions::from_mode(0o600)) + .expect("0600 crashed final"); + std::fs::write(dir.join(".identity.pem.publishing.99999.0"), b"") + .expect("seed publish marker"); + key_path + } + + /// File names in `dir` containing `needle` (marker/claim/quarantine sweeps). + fn names_containing(dir: &std::path::Path, needle: &str) -> Vec { + let mut names: Vec = std::fs::read_dir(dir) + .expect("read_dir") + .filter_map(|e| e.ok()) + .map(|e| e.file_name().to_string_lossy().into_owned()) + .filter(|n| n.contains(needle)) + .collect(); + names.sort(); + names + } + + fn key_config(key_path: &std::path::Path) -> Config { + Config::parse_from([ + "gitlawb-node", + "--key-path", + key_path.to_str().expect("utf8 path"), + ]) + } + + // #194 (U2): a `.publishing.` marker beside an EMPTY final is the crash + // signature of an interrupted fallback publish; boot must claim the + // marker, quarantine the empty final, and regenerate instead of wedging + // on `invalid PEM key` forever. RED before U2: load_or_create_keypair + // errors with `invalid PEM key`. + #[test] + fn recovery_regenerates_after_crash_leaves_empty_final() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + + let kp = load_or_create_keypair(&key_config(&key_path)) + .expect("a crash-marked empty final must recover, not wedge the start"); + + let mode = std::fs::metadata(&key_path) + .expect("regenerated final exists") + .permissions() + .mode(); + assert_eq!(mode & 0o777, 0o600, "regenerated final must be 0600"); + let on_disk = super::load_existing_key(&key_path).expect("regenerated final parses"); + assert_eq!( + format!("{}", on_disk.did()), + format!("{}", kp.did()), + "the returned identity must match the regenerated on-disk key" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "recovery must consume the publish marker(s)" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "the recovery claim must be removed after the publish resolves" + ); + let quarantined = names_containing(dir.path(), ".quarantined."); + assert_eq!( + quarantined.len(), + 1, + "exactly one quarantine file must exist: {quarantined:?}" + ); + let bytes = std::fs::read(dir.path().join(&quarantined[0])).expect("read quarantine"); + assert!( + bytes.is_empty(), + "the quarantine must preserve the crashed final's (empty) content" + ); + } + + // #194 (U2): same recovery for a final holding a truncated PEM, and the + // quarantine (a rename) must preserve the truncated bytes exactly for + // post-mortem inspection. + #[test] + fn recovery_quarantines_truncated_final_preserving_bytes() { + let dir = tempfile::tempdir().expect("tempdir"); + let pem = Keypair::generate().to_pem().expect("pem"); + let truncated = &pem.as_bytes()[..pem.len() / 2]; + let key_path = seed_crash_state(dir.path(), truncated); + + let kp = load_or_create_keypair(&key_config(&key_path)) + .expect("a crash-marked truncated final must recover, not wedge the start"); + + let on_disk = super::load_existing_key(&key_path).expect("regenerated final parses"); + assert_eq!( + format!("{}", on_disk.did()), + format!("{}", kp.did()), + "the returned identity must match the regenerated on-disk key" + ); + let quarantined = names_containing(dir.path(), ".quarantined."); + assert_eq!( + quarantined.len(), + 1, + "exactly one quarantine file must exist: {quarantined:?}" + ); + let bytes = std::fs::read(dir.path().join(&quarantined[0])).expect("read quarantine"); + assert_eq!( + bytes, truncated, + "the quarantine must hold exactly the truncated input bytes" + ); + } + + // MUST-NOT (#194, U2): an unparseable final WITHOUT a marker is operator + // corruption, not a crash signature; it must keep today's loud fail + // bit-for-bit: same `invalid PEM key` message, final left on disk with + // identical bytes, no quarantine, no regeneration. Rides the full + // KEY_RACE_DEADLINE (~5s), as today. Pre-implementation this must PASS; + // after GREEN it is the mutation guard for the marker-presence check. + #[test] + fn bad_final_without_marker_still_fails_loudly() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let garbage: &[u8] = b"-----BEGIN nonsense truncated"; + std::fs::write(&key_path, garbage).expect("seed garbage final"); + std::fs::set_permissions(&key_path, std::fs::Permissions::from_mode(0o600)) + .expect("0600 garbage final"); + + let err = match load_or_create_keypair(&key_config(&key_path)) { + Err(e) => e, + Ok(_) => panic!("a marker-less corrupt final must fail loudly, never regenerate"), + }; + assert!( + err.to_string().contains("invalid PEM key"), + "the load failure must surface unchanged: {err:#}" + ); + assert_eq!( + std::fs::read(&key_path).expect("final still on disk"), + garbage, + "the corrupt final must be left untouched" + ); + assert!( + names_containing(dir.path(), ".quarantined.").is_empty(), + "no quarantine may be created without a marker" + ); + } + + // MUST-NOT (#194, U2): a VALID final beside a stale marker loads + // unchanged: same identity, same bytes, no quarantine. (U4 adds marker + // hygiene; U2 may leave the stale marker in place.) + #[test] + fn valid_final_with_stale_marker_loads_unchanged() { + let dir = tempfile::tempdir().expect("tempdir"); + let existing = Keypair::generate(); + let pem = existing.to_pem().expect("pem"); + let key_path = seed_crash_state(dir.path(), pem.as_bytes()); + + let kp = load_or_create_keypair(&key_config(&key_path)) + .expect("a valid final with a stale marker must load"); + + assert_eq!( + format!("{}", kp.did()), + format!("{}", existing.did()), + "the pre-existing identity must be preserved, not regenerated" + ); + assert_eq!( + std::fs::read(&key_path).expect("final still on disk"), + pem.as_bytes(), + "the valid final's bytes must be untouched" + ); + assert!( + names_containing(dir.path(), ".quarantined.").is_empty(), + "a valid final must never be quarantined" + ); + } + + // MUST-NOT (#194, U2): a TRANSIENT (non-content) load failure must never + // recover, even with a marker present: loud error, no quarantine, final + // and marker untouched. The seam's load fault injects the transient class + // deterministically. Rides the full KEY_RACE_DEADLINE (~5s). RED if + // recovery keys on any-error instead of content-class. + #[test] + fn transient_error_with_marker_never_recovers() { + let dir = tempfile::tempdir().expect("tempdir"); + let pem = Keypair::generate().to_pem().expect("pem"); + let key_path = seed_crash_state(dir.path(), pem.as_bytes()); + let seam = RecoverySeam { + load_fault: || Err(std::io::Error::other("injected transient load failure")), + ..RecoverySeam::NONE + }; + + let err = match load_or_create_keypair_with(&key_path, &|| {}, PublishFaults::NONE, seam) { + Err(e) => e, + Ok(_) => panic!("a transient-class load failure must stay loud, never recover"), + }; + assert!( + format!("{err:#}").contains("injected transient load failure"), + "the transient failure must surface: {err:#}" + ); + assert_eq!( + std::fs::read(&key_path).expect("final still on disk"), + pem.as_bytes(), + "the final must be untouched by a transient failure" + ); + assert_eq!( + names_containing(dir.path(), ".publishing."), + vec![".identity.pem.publishing.99999.0".to_string()], + "the marker must be untouched" + ); + assert!( + names_containing(dir.path(), ".quarantined.").is_empty(), + "a transient failure must never quarantine" + ); + } + + // #194 (U2): recovery runs AT MOST ONCE per start. After the first + // recovery completes, the seam re-plants a crash state (marker + empty + // final) before the retry publish; the second load failure must surface + // as an error, not recover again. Exactly one quarantine, from the first + // pass. Rides the full KEY_RACE_DEADLINE (~5s) on the second load. + #[test] + fn recovery_runs_at_most_once_per_start() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + let replanted = std::sync::atomic::AtomicBool::new(false); + let replant_path = key_path.clone(); + let replant_marker = dir.path().join(".identity.pem.publishing.88888.0"); + let before_publish = move || { + if !replanted.swap(true, std::sync::atomic::Ordering::SeqCst) { + std::fs::write(&replant_marker, b"").expect("re-plant marker"); + std::fs::write(&replant_path, b"").expect("re-plant empty final"); + std::fs::set_permissions(&replant_path, std::fs::Permissions::from_mode(0o600)) + .expect("0600 re-planted final"); + } + }; + let seam = RecoverySeam { + before_publish: &before_publish, + ..RecoverySeam::NONE + }; + + let err = match load_or_create_keypair_with(&key_path, &|| {}, PublishFaults::NONE, seam) { + Err(e) => e, + Ok(_) => panic!("a second crash state in the same start must error, not recover again"), + }; + assert!( + err.to_string().contains("invalid PEM key"), + "the second load failure must surface unchanged: {err:#}" + ); + let quarantined = names_containing(dir.path(), ".quarantined."); + assert_eq!( + quarantined.len(), + 1, + "exactly one quarantine, from the first recovery: {quarantined:?}" + ); + } + + // #194 (U2): two concurrent boots against the same crash state must admit + // exactly ONE recoverer (the atomic claim rename arbitrates); both end on + // the SAME identity, one quarantine, no markers or claims left behind. + // Modeled on concurrent_starts_converge_on_one_identity. + #[test] + fn claim_race_admits_exactly_one_recoverer() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + let config = std::sync::Arc::new(key_config(&key_path)); + + let barrier = std::sync::Arc::new(std::sync::Barrier::new(2)); + let handles: Vec<_> = (0..2) + .map(|_| { + let c = config.clone(); + let b = barrier.clone(); + std::thread::spawn(move || { + b.wait(); + format!( + "{}", + load_or_create_keypair(&c).expect("recovering start").did() + ) + }) + }) + .collect(); + let dids: Vec = handles + .into_iter() + .map(|h| h.join().expect("thread joins")) + .collect(); + + assert_eq!( + dids[0], dids[1], + "both concurrent recovering starts must converge on one identity" + ); + let quarantined = names_containing(dir.path(), ".quarantined."); + assert_eq!( + quarantined.len(), + 1, + "exactly one recoverer must quarantine: {quarantined:?}" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "no publish markers may remain" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "no recovery claims may remain" + ); + } + + // #194 (U2): the Lost arm participates in recovery. The publisher's link + // is forced to fail (fallback path) and the crash state (foreign marker + + // empty final) appears between the temp write and the link, so the + // publish loses to the crashed final; the Lost-arm load must then take + // the recovery path and regenerate. + #[test] + fn lost_race_load_failure_takes_recovery_path() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let planted = std::sync::atomic::AtomicBool::new(false); + let plant_path = key_path.clone(); + let plant_marker = dir.path().join(".identity.pem.publishing.77777.0"); + let before_link = move || { + if !planted.swap(true, std::sync::atomic::Ordering::SeqCst) { + std::fs::write(&plant_marker, b"").expect("plant marker"); + std::fs::write(&plant_path, b"").expect("plant empty final"); + std::fs::set_permissions(&plant_path, std::fs::Permissions::from_mode(0o600)) + .expect("0600 planted final"); + } + }; + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + ..PublishFaults::NONE + }; + + let kp = load_or_create_keypair_with(&key_path, &before_link, faults, RecoverySeam::NONE) + .expect("a lost race onto a crash-state final must recover, not fail the start"); + + let on_disk = super::load_existing_key(&key_path).expect("regenerated final parses"); + assert_eq!( + format!("{}", on_disk.did()), + format!("{}", kp.did()), + "the returned identity must match the regenerated on-disk key" + ); + let quarantined = names_containing(dir.path(), ".quarantined."); + assert_eq!( + quarantined.len(), + 1, + "exactly one quarantine file must exist: {quarantined:?}" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "no publish markers may remain" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "no recovery claims may remain" + ); + } } #[cfg(test)] From e43f4d2920509763a32e68725e5e6d30f7dfe55d Mon Sep 17 00:00:00 2001 From: t Date: Tue, 21 Jul 2026 14:26:01 -0500 Subject: [PATCH 26/41] fix(node): make the fallback marker removal a commit check for stalled winners A fallback winner stalled past the race deadline can have its round claimed by a recovering peer: the peer renames the marker, quarantines the in-progress final, and republishes. The winner's own-marker removal now decides the round: removal succeeding linearizes it ahead of any recovery, while any failure demotes it to Lost, after waiting out live or stale .recovering claims so the follow-up load cannot observe the pre-quarantine key. Demotion is always safe: absent interference the Lost path re-loads the winner's own key. --- crates/gitlawb-node/src/lib.rs | 470 +++++++++++++++++++++++++++++---- 1 file changed, 414 insertions(+), 56 deletions(-) diff --git a/crates/gitlawb-node/src/lib.rs b/crates/gitlawb-node/src/lib.rs index 94f9f537..486200b6 100644 --- a/crates/gitlawb-node/src/lib.rs +++ b/crates/gitlawb-node/src/lib.rs @@ -1303,6 +1303,45 @@ fn boot_load_key( } } +/// Wait until no `.{stem}.recovering.*` claim remains beside `final_path`, +/// polling on `load_racing_key`'s cadence (~2ms), bounded by +/// `KEY_RACE_DEADLINE` (#194, U3). Why the wait exists: a fallback winner +/// that failed its commit check was demoted by a recovering peer, and that +/// peer is mid-round — it quarantines the (old) final, republishes a fresh +/// key, and only then clears its claim. A demoted winner that re-loaded +/// before the claim cleared could observe the pre-quarantine key and diverge +/// from what the peer ends up publishing; waiting for claim clearance makes +/// the follow-up load observe the settled state. Deadline expiry is NOT an +/// error: a claim left by a recovery that itself crashed never clears, and +/// the caller's load path copes with whatever state remains (loudly, if the +/// key is unreadable). An unreadable directory reads as no claims, matching +/// `list_publish_markers`' policy. +fn wait_for_recovery_claims(final_path: &std::path::Path) { + let dir = final_path + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or_else(|| std::path::Path::new(".")); + let stem = final_path + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or("identity.pem"); + let prefix = format!(".{stem}.recovering."); + let deadline = std::time::Instant::now() + KEY_RACE_DEADLINE; + loop { + let claimed = std::fs::read_dir(dir).is_ok_and(|entries| { + entries.filter_map(|e| e.ok()).any(|e| { + e.file_name() + .to_str() + .is_some_and(|n| n.starts_with(&prefix)) + }) + }); + if !claimed || std::time::Instant::now() >= deadline { + return; + } + std::thread::sleep(std::time::Duration::from_millis(2)); + } +} + /// Compile-time fault-injection seam for `publish_key_atomically`, extending /// the `before_link` closure precedent: every hook is a no-op `Ok` in /// production (`PublishFaults::NONE`); tests swap in a hook returning `Err` to @@ -1390,13 +1429,28 @@ impl PublishFaults { /// the failure mode is the same LOUD invalid-PEM (or mode) error at the next /// start, never a silently used key. /// +/// Against the boot-time crash RECOVERY race (#194, U3) the fallback's marker +/// protocol restores (b)-style convergence: removing the publish marker is +/// the winner's commit check, and recovery claims a round by renaming that +/// same marker, so exactly one side takes the round. A winner that loses its +/// marker demotes itself to Lost, waits out any `.recovering.` claim +/// (bounded by `KEY_RACE_DEADLINE`), and re-loads the settled key. Honest +/// residuals: a winner stalled past `KEY_RACE_DEADLINE` that loads in the +/// sub-ms window before a recoverer's claim exists, and a crash-interrupted +/// recovery whose stale claim just expires the bounded wait — both degrade +/// to a loud error or a re-load, never silent two-sided divergence. +/// /// `before_link` is a no-op in production; tests use it to widen the -/// post-write / pre-link window deterministically. `faults` is the -/// compile-time fault-injection seam (`PublishFaults::NONE` in production). +/// post-write / pre-link window deterministically. `before_commit` runs +/// between the fallback final's dir fsync and the marker-removal commit +/// check — a no-op in production; tests use it to interleave a concurrent +/// recovery into that window (#194, U3). `faults` is the compile-time +/// fault-injection seam (`PublishFaults::NONE` in production). fn publish_key_atomically( final_path: &std::path::Path, pem: &[u8], before_link: &dyn Fn(), + before_commit: &dyn Fn(), faults: PublishFaults, ) -> Result { use std::io::Write; @@ -1477,7 +1531,7 @@ fn publish_key_atomically( Ok(KeyPublish::Won) } Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => Ok(KeyPublish::Lost), - Err(e) => publish_key_fallback(final_path, pem, e, faults), + Err(e) => publish_key_fallback(final_path, pem, e, before_commit, faults), } } @@ -1498,7 +1552,10 @@ fn publish_key_atomically( /// (`.{stem}.publishing.{pid}.{attempt}`, empty, same dir as the final): the /// marker is created and its NAME fsynced BEFORE the final can exist, and it /// is removed only once the final is complete-and-durable (success) or -/// disposed of (error / lost race). A crash inside the window therefore +/// disposed of (error / lost race). On success the removal doubles as the +/// COMMIT CHECK (#194, U3): a removal that fails means a recovering peer +/// claimed this round, and the publish demotes itself to Lost (see the +/// success arm). A crash inside the window therefore /// leaves marker+final together, so a later boot can tell "partial final from /// a crashed fallback publish" from a key an operator corrupted (#194, U1). /// Two constraints: @@ -1517,6 +1574,7 @@ fn publish_key_fallback( final_path: &std::path::Path, pem: &[u8], link_err: std::io::Error, + before_commit: &dyn Fn(), faults: PublishFaults, ) -> Result { use std::io::Write; @@ -1691,16 +1749,37 @@ fn publish_key_fallback( return Err(e); } } - // Success: the final is complete and durable, so the bracket closes. - // Remove the marker, then best-effort fsync the removal so it tends to be - // durable too. (A later unit turns this removal into a commit check; U1 - // keeps it best-effort.) - dispose_marker(); - #[cfg(unix)] - { - let _ = fsync_parent_dir(final_path); + // Success: the final is complete and durable. Removing our OWN marker is + // the COMMIT CHECK (#194, U3): recovery claims a round by renaming this + // exact marker, so a successful removal proves no recovery can ever claim + // it and linearizes this winner ahead of any recoverer. Any failure + // (NotFound after a peer's claim rename, or any other error) means + // ownership of the round cannot be proven: a recovering peer may have + // quarantined our final and republished, so returning Won could report + // key A while disk holds key B. Demote to Lost instead; demotion is + // always safe, because absent real interference the caller's Lost arm + // re-loads our own just-published key (same identity). + before_commit(); + match std::fs::remove_file(&marker) { + Ok(()) => { + // Best-effort fsync so the marker's removal tends to be durable. + #[cfg(unix)] + { + let _ = fsync_parent_dir(final_path); + } + Ok(KeyPublish::Won) + } + Err(e) => { + warn!( + marker = %marker.display(), + err = %e, + "fallback publish lost its marker before commit; a recovering peer owns \ + this round, demoting the publish to lost" + ); + wait_for_recovery_claims(final_path); + Ok(KeyPublish::Lost) + } } - Ok(KeyPublish::Won) } /// Compile-time test seam for the load-side crash recovery in @@ -1729,6 +1808,7 @@ fn load_or_create_keypair(config: &Config) -> Result { load_or_create_keypair_with( &config.resolved_key_path(), &|| {}, + &|| {}, PublishFaults::NONE, RecoverySeam::NONE, ) @@ -1858,9 +1938,9 @@ fn recover_crashed_publish( } /// Body of `load_or_create_keypair` with the test seams threaded in (#194, -/// U2): `before_link` and `faults` pass through to `publish_key_atomically` -/// (production: no-op / `NONE`), `seam` is the load-side recovery seam -/// (production: `RecoverySeam::NONE`). +/// U2): `before_link`, `before_commit`, and `faults` pass through to +/// `publish_key_atomically` (production: no-op / `NONE`), `seam` is the +/// load-side recovery seam (production: `RecoverySeam::NONE`). /// /// Load-or-generate with AT MOST ONE crash recovery per start: a load /// failure recovers — claim a marker, quarantine the final, regenerate — @@ -1872,6 +1952,7 @@ fn recover_crashed_publish( fn load_or_create_keypair_with( key_path: &std::path::Path, before_link: &dyn Fn(), + before_commit: &dyn Fn(), faults: PublishFaults, seam: RecoverySeam<'_>, ) -> Result { @@ -1929,7 +2010,8 @@ fn load_or_create_keypair_with( // Publish atomically: the final path only ever appears complete, and a // lost race loads the winner's key rather than overwriting it. - match publish_key_atomically(key_path, pem.as_bytes(), before_link, faults)? { + match publish_key_atomically(key_path, pem.as_bytes(), before_link, before_commit, faults)? + { KeyPublish::Won => { remove_claim(&claim); info!(path = %key_path.display(), did = %kp.did(), "generated new node identity"); @@ -2054,8 +2136,14 @@ mod identity_key_tests { let writer_path = key_path.clone(); let writer = std::thread::spawn(move || { std::thread::sleep(std::time::Duration::from_millis(250)); - publish_key_atomically(&writer_path, pem.as_bytes(), &|| {}, PublishFaults::NONE) - .expect("publish"); + publish_key_atomically( + &writer_path, + pem.as_bytes(), + &|| {}, + &|| {}, + PublishFaults::NONE, + ) + .expect("publish"); }); let started = std::time::Instant::now(); @@ -2085,8 +2173,14 @@ mod identity_key_tests { let pem_b = Keypair::generate().to_pem().expect("pem b"); assert_ne!(pem_a.as_str(), pem_b.as_str(), "fixtures must differ"); - let out = publish_key_atomically(&key_path, pem_a.as_bytes(), &|| {}, PublishFaults::NONE) - .expect("publish a"); + let out = publish_key_atomically( + &key_path, + pem_a.as_bytes(), + &|| {}, + &|| {}, + PublishFaults::NONE, + ) + .expect("publish a"); assert!(matches!(out, KeyPublish::Won), "first publish wins"); assert_eq!( std::fs::read_to_string(&key_path).unwrap().as_str(), @@ -2094,8 +2188,14 @@ mod identity_key_tests { "final holds the full PEM" ); - let out = publish_key_atomically(&key_path, pem_b.as_bytes(), &|| {}, PublishFaults::NONE) - .expect("publish b"); + let out = publish_key_atomically( + &key_path, + pem_b.as_bytes(), + &|| {}, + &|| {}, + PublishFaults::NONE, + ) + .expect("publish b"); assert!(matches!(out, KeyPublish::Lost), "second publish loses"); assert_eq!( std::fs::read_to_string(&key_path).unwrap().as_str(), @@ -2134,6 +2234,7 @@ mod identity_key_tests { &|| { std::thread::sleep(std::time::Duration::from_millis(200)); }, + &|| {}, PublishFaults::NONE, ) .expect("publish"); @@ -2189,8 +2290,14 @@ mod identity_key_tests { std::fs::set_permissions(&stale, std::fs::Permissions::from_mode(0o600)) .expect("stale temp perms"); - let out = publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, PublishFaults::NONE) - .expect("publish must recover from a stale temp, not wedge the start"); + let out = publish_key_atomically( + &key_path, + pem.as_bytes(), + &|| {}, + &|| {}, + PublishFaults::NONE, + ) + .expect("publish must recover from a stale temp, not wedge the start"); assert!(matches!(out, KeyPublish::Won), "publish wins"); assert_eq!( std::fs::read_to_string(&key_path).unwrap().as_str(), @@ -2217,11 +2324,16 @@ mod identity_key_tests { std::fs::write(&tmp, b"taken").expect("seed taken temp"); } - let err = - match publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, PublishFaults::NONE) { - Err(e) => e, - Ok(_) => panic!("publish must fail when every temp name is taken"), - }; + let err = match publish_key_atomically( + &key_path, + pem.as_bytes(), + &|| {}, + &|| {}, + PublishFaults::NONE, + ) { + Err(e) => e, + Ok(_) => panic!("publish must fail when every temp name is taken"), + }; let msg = format!("{err:#}"); assert!( msg.contains(&format!(".identity.pem.tmp.{}", std::process::id())) @@ -2414,7 +2526,7 @@ mod identity_key_tests { let kp = Keypair::generate(); let pem = kp.to_pem().expect("pem"); - let out = publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, faults) + let out = publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, &|| {}, faults) .expect("a failed hard_link must fall back to a direct publish, not fail the start"); assert!(matches!(out, KeyPublish::Won), "fallback publish wins"); @@ -2469,6 +2581,7 @@ mod identity_key_tests { &key_path, pem_winner.as_bytes(), &|| {}, + &|| {}, PublishFaults::NONE, ) .expect("winner publishes"); @@ -2479,7 +2592,7 @@ mod identity_key_tests { link: || Err(std::io::ErrorKind::PermissionDenied.into()), ..PublishFaults::NONE }; - let out = publish_key_atomically(&key_path, pem_loser.as_bytes(), &|| {}, faults) + let out = publish_key_atomically(&key_path, pem_loser.as_bytes(), &|| {}, &|| {}, faults) .expect("an existing final must read as a lost race, not an error"); assert!( matches!(out, KeyPublish::Lost), @@ -2519,7 +2632,7 @@ mod identity_key_tests { }; // Release both threads at once to maximize the race. b.wait(); - match publish_key_atomically(&kp, pem.as_bytes(), &|| {}, faults) + match publish_key_atomically(&kp, pem.as_bytes(), &|| {}, &|| {}, faults) .expect("fallback publish") { KeyPublish::Won => (true, format!("{}", me.did())), @@ -2565,7 +2678,7 @@ mod identity_key_tests { fallback_write: || Err(std::io::Error::other("injected write failure")), ..PublishFaults::NONE }; - let err = match publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, faults) { + let err = match publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, &|| {}, faults) { Err(e) => e, Ok(_) => panic!("a failed fallback write must error the start"), }; @@ -2601,7 +2714,7 @@ mod identity_key_tests { fallback_create: || Err(std::io::Error::other("injected fallback create failure")), ..PublishFaults::NONE }; - let err = match publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, faults) { + let err = match publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, &|| {}, faults) { Err(e) => e, Ok(_) => panic!("a failed fallback create must error the start"), }; @@ -2639,7 +2752,7 @@ mod identity_key_tests { fallback_fsync: || Err(std::io::Error::other("injected fsync failure")), ..PublishFaults::NONE }; - let err = match publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, faults) { + let err = match publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, &|| {}, faults) { Err(e) => e, Ok(_) => panic!("a failed fallback fsync must error the start"), }; @@ -2775,7 +2888,7 @@ mod identity_key_tests { let pem = kp.to_pem().expect("pem"); unsafe { libc::umask(mask) }; - let out = publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, faults) + let out = publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, &|| {}, faults) .expect("publish must succeed: the tighten repairs the umask-narrowed create mode"); assert!(matches!(out, KeyPublish::Won), "publish wins"); @@ -2848,7 +2961,7 @@ mod identity_key_tests { temp_mode_verify: || Err(std::io::Error::other("injected mode-ignoring mount")), ..PublishFaults::NONE }; - let err = match publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, faults) { + let err = match publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, &|| {}, faults) { Err(e) => e, Ok(_) => panic!("an unverifiable temp mode must fail the publish closed"), }; @@ -2885,7 +2998,7 @@ mod identity_key_tests { fallback_mode_verify: || Err(std::io::Error::other("injected mode-ignoring mount")), ..PublishFaults::NONE }; - let err = match publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, faults) { + let err = match publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, &|| {}, faults) { Err(e) => e, Ok(_) => panic!("an unverifiable final mode must fail the publish closed"), }; @@ -2918,7 +3031,7 @@ mod identity_key_tests { link: || Err(std::io::ErrorKind::Unsupported.into()), ..PublishFaults::NONE }; - let out = publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, retry_faults) + let out = publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, &|| {}, retry_faults) .expect("a retry after the fail-closed publish must not be wedged"); assert!(matches!(out, KeyPublish::Won), "retry wins cleanly"); let loaded = super::load_existing_key(&key_path).expect("retried key loads"); @@ -2956,7 +3069,7 @@ mod identity_key_tests { link: || Err(std::io::ErrorKind::Unsupported.into()), ..PublishFaults::NONE }; - let out = publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, faults) + let out = publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, &|| {}, faults) .expect("fallback publish"); assert!(matches!(out, KeyPublish::Won), "fallback publish wins"); let loaded = super::load_existing_key(&key_path).expect("published key loads"); @@ -2985,6 +3098,7 @@ mod identity_key_tests { &key_path, pem_winner.as_bytes(), &|| {}, + &|| {}, PublishFaults::NONE, ) .expect("winner publishes"); @@ -2995,7 +3109,7 @@ mod identity_key_tests { link: || Err(std::io::ErrorKind::PermissionDenied.into()), ..PublishFaults::NONE }; - let out = publish_key_atomically(&key_path, pem_loser.as_bytes(), &|| {}, faults) + let out = publish_key_atomically(&key_path, pem_loser.as_bytes(), &|| {}, &|| {}, faults) .expect("an existing final must read as a lost race, not an error"); assert!(matches!(out, KeyPublish::Lost), "fallback loses"); let loaded = load_racing_key(&key_path).expect("lost race loads the winner"); @@ -3020,7 +3134,7 @@ mod identity_key_tests { let key_path = dir.path().join("identity.pem"); let pem = Keypair::generate().to_pem().expect("pem"); - let err = match publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, faults) { + let err = match publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, &|| {}, faults) { Err(e) => e, Ok(_) => panic!("the injected fallback fault must error the publish"), }; @@ -3109,7 +3223,7 @@ mod identity_key_tests { link: || Err(std::io::ErrorKind::Unsupported.into()), ..PublishFaults::NONE }; - let err = match publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, faults) { + let err = match publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, &|| {}, faults) { Err(e) => e, Ok(_) => panic!("the fallback must fail when every marker name is taken"), }; @@ -3141,7 +3255,7 @@ mod identity_key_tests { marker_fsync: || Err(std::io::Error::other("injected marker dir fsync failure")), ..PublishFaults::NONE }; - let out = publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, faults) + let out = publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, &|| {}, faults) .expect("a marker dir fsync failure must not fail the publish"); assert!(matches!(out, KeyPublish::Won), "publish still wins"); let loaded = super::load_existing_key(&key_path).expect("published key loads"); @@ -3172,7 +3286,7 @@ mod identity_key_tests { marker_create: || Err(std::io::Error::other("marker created on the primary path")), ..PublishFaults::NONE }; - let out = publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, faults) + let out = publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, &|| {}, faults) .expect("the primary path must never reach the marker machinery"); assert!(matches!(out, KeyPublish::Won), "primary publish wins"); let loaded = super::load_existing_key(&key_path).expect("published key loads"); @@ -3377,10 +3491,12 @@ mod identity_key_tests { ..RecoverySeam::NONE }; - let err = match load_or_create_keypair_with(&key_path, &|| {}, PublishFaults::NONE, seam) { - Err(e) => e, - Ok(_) => panic!("a transient-class load failure must stay loud, never recover"), - }; + let err = + match load_or_create_keypair_with(&key_path, &|| {}, &|| {}, PublishFaults::NONE, seam) + { + Err(e) => e, + Ok(_) => panic!("a transient-class load failure must stay loud, never recover"), + }; assert!( format!("{err:#}").contains("injected transient load failure"), "the transient failure must surface: {err:#}" @@ -3426,10 +3542,14 @@ mod identity_key_tests { ..RecoverySeam::NONE }; - let err = match load_or_create_keypair_with(&key_path, &|| {}, PublishFaults::NONE, seam) { - Err(e) => e, - Ok(_) => panic!("a second crash state in the same start must error, not recover again"), - }; + let err = + match load_or_create_keypair_with(&key_path, &|| {}, &|| {}, PublishFaults::NONE, seam) + { + Err(e) => e, + Ok(_) => { + panic!("a second crash state in the same start must error, not recover again") + } + }; assert!( err.to_string().contains("invalid PEM key"), "the second load failure must surface unchanged: {err:#}" @@ -3516,8 +3636,14 @@ mod identity_key_tests { ..PublishFaults::NONE }; - let kp = load_or_create_keypair_with(&key_path, &before_link, faults, RecoverySeam::NONE) - .expect("a lost race onto a crash-state final must recover, not fail the start"); + let kp = load_or_create_keypair_with( + &key_path, + &before_link, + &|| {}, + faults, + RecoverySeam::NONE, + ) + .expect("a lost race onto a crash-state final must recover, not fail the start"); let on_disk = super::load_existing_key(&key_path).expect("regenerated final parses"); assert_eq!( @@ -3540,6 +3666,238 @@ mod identity_key_tests { "no recovery claims may remain" ); } + + // #194 (U3): THE divergence race. A fallback winner whose final is already + // durable but which stalls before removing its marker can have its round + // stolen by a recovering peer: the peer claims the winner's marker, + // quarantines the winner's final, and republishes key B. The winner's + // OWN-MARKER removal is the commit check; losing it must demote the winner + // to Lost so it converges on B, never returning Won with key A while disk + // holds B. RED before U3: the publish returns Won with the quarantined key + // A while the final holds B (two-sided divergence). + #[test] + fn stalled_winner_demotes_when_recovery_claims_its_round() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let kp_b = Keypair::generate(); + let pem_b = kp_b.to_pem().expect("pem b"); + + let fired = std::sync::atomic::AtomicBool::new(false); + let steal_dir = dir.path().to_path_buf(); + let steal_path = key_path.clone(); + let before_commit = move || { + if fired.swap(true, std::sync::atomic::Ordering::SeqCst) { + return; + } + // A full concurrent recovery inside A's stall window (post + // final-fsync, pre commit check), step for step what + // recover_crashed_publish does: claim A's marker by rename, + // quarantine A's final, republish key B, clear the claim. + let markers = names_containing(&steal_dir, ".publishing."); + assert_eq!( + markers.len(), + 1, + "A's window must be bracketed: {markers:?}" + ); + let claim = steal_dir.join(".identity.pem.recovering.55555"); + std::fs::rename(steal_dir.join(&markers[0]), &claim).expect("claim A's marker"); + std::fs::rename( + &steal_path, + steal_dir.join(".identity.pem.quarantined.55555.0"), + ) + .expect("quarantine A's final"); + let out = publish_key_atomically( + &steal_path, + pem_b.as_bytes(), + &|| {}, + &|| {}, + PublishFaults::NONE, + ) + .expect("peer republishes key B"); + assert!(matches!(out, KeyPublish::Won), "peer's republish wins"); + std::fs::remove_file(&claim).expect("clear the claim"); + }; + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + ..PublishFaults::NONE + }; + + let kp = load_or_create_keypair_with( + &key_path, + &|| {}, + &before_commit, + faults, + RecoverySeam::NONE, + ) + .expect("a demoted winner must converge, not fail the start"); + + let on_disk = super::load_existing_key(&key_path).expect("final parses"); + assert_eq!( + format!("{}", kp.did()), + format!("{}", kp_b.did()), + "the demoted winner must converge on the recovering peer's key B" + ); + assert_eq!( + format!("{}", on_disk.did()), + format!("{}", kp_b.did()), + "the final must hold key B" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "no publish markers may remain" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "no recovery claims may remain" + ); + let quarantined = names_containing(dir.path(), ".quarantined."); + assert_eq!( + quarantined.len(), + 1, + "exactly one live identity: A's old final stays quarantined: {quarantined:?}" + ); + } + + // #194 (U3): with no interference the commit check is invisible. The + // winner removes its own marker, returns Won, and behaves exactly as U1's + // success test: the key round-trips and no marker is left. Also pins that + // the pre-commit seam runs on the fallback success path. + #[test] + fn winner_commits_first_without_interference() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let kp = Keypair::generate(); + let pem = kp.to_pem().expect("pem"); + + let fired = std::sync::atomic::AtomicBool::new(false); + let before_commit = || { + fired.store(true, std::sync::atomic::Ordering::SeqCst); + }; + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + ..PublishFaults::NONE + }; + let out = publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, &before_commit, faults) + .expect("uncontended fallback publish"); + assert!( + matches!(out, KeyPublish::Won), + "the uncontended winner commits and wins" + ); + assert!( + fired.load(std::sync::atomic::Ordering::SeqCst), + "the pre-commit seam must run on the fallback success path" + ); + let loaded = super::load_existing_key(&key_path).expect("published key loads"); + assert_eq!( + format!("{}", loaded.did()), + format!("{}", kp.did()), + "the committed publish must round-trip the same identity" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "no marker may remain after commit" + ); + } + + // #194 (U3): a recovery attempt arriving AFTER the winner committed finds + // no marker to claim and must abort into a plain re-load: no quarantine, + // the winner's identity returned. Honest scope: recover_crashed_publish + // is invoked directly with a synthetic load error, the way the boot loop + // would after observing a (by now resolved) crash signature; the final on + // disk is GOOD here, so the vanished-marker arm's re-load succeeds. + #[test] + fn late_recovery_aborts_after_winner_commit() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let kp = Keypair::generate(); + let pem = kp.to_pem().expect("pem"); + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + ..PublishFaults::NONE + }; + let out = publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, &|| {}, faults) + .expect("winner publishes and commits"); + assert!(matches!(out, KeyPublish::Won), "winner commits"); + + let recovery = super::recover_crashed_publish( + &key_path, + anyhow::anyhow!("synthetic post-commit crash signature"), + ) + .expect("a late recovery must not error"); + let loaded = match recovery { + super::Recovery::Reloaded(result) => { + result.expect("the vanished-marker arm re-loads the winner's key") + } + super::Recovery::Claimed(claim) => { + panic!("no marker exists to claim after commit, got claim {claim:?}") + } + }; + assert_eq!( + format!("{}", loaded.did()), + format!("{}", kp.did()), + "late recovery must resolve to the committed winner's identity" + ); + assert!( + names_containing(dir.path(), ".quarantined.").is_empty(), + "a committed final must never be quarantined" + ); + } + + // #194 (U3): a demoted winner facing a STALE `.recovering.` claim (a + // recovery that crashed before clearing it) must wait the claim out on + // the bounded KEY_RACE_DEADLINE and then return Lost, not error. The + // marker is stolen WITHOUT a real recovery (final left in place), so the + // follow-up load resolves the winner's OWN key: the demotion-is-safe + // property. RED before U3: the publish returns Won immediately and the + // elapsed floor fails. + #[test] + fn demoted_winner_waits_out_a_stale_claim() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + + let fired = std::sync::atomic::AtomicBool::new(false); + let steal_dir = dir.path().to_path_buf(); + let before_commit = move || { + if fired.swap(true, std::sync::atomic::Ordering::SeqCst) { + return; + } + let markers = names_containing(&steal_dir, ".publishing."); + assert_eq!( + markers.len(), + 1, + "A's window must be bracketed: {markers:?}" + ); + std::fs::remove_file(steal_dir.join(&markers[0])).expect("steal A's marker"); + std::fs::write(steal_dir.join(".identity.pem.recovering.44444"), b"") + .expect("plant stale claim"); + }; + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + ..PublishFaults::NONE + }; + + let started = std::time::Instant::now(); + let kp = load_or_create_keypair_with( + &key_path, + &|| {}, + &before_commit, + faults, + RecoverySeam::NONE, + ) + .expect("a demoted winner behind a stale claim must still resolve a key"); + let waited = started.elapsed(); + + assert!( + waited >= std::time::Duration::from_secs(4), + "the demotion must wait out the stale claim on the deadline, waited {waited:?}" + ); + let on_disk = super::load_existing_key(&key_path).expect("final parses"); + assert_eq!( + format!("{}", kp.did()), + format!("{}", on_disk.did()), + "with the final untouched the demoted winner must resolve its own key" + ); + } } #[cfg(test)] From 64aa15eecc406ab813b734c2c0436accf1bad9b7 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 21 Jul 2026 14:38:15 -0500 Subject: [PATCH 27/41] fix(node): sweep stale key-publish markers on every healthy boot A marker orphaned between its durability fsync and create_new(final), or a claim left by a crashed recovery, must not linger to misclassify a much-later corruption of a good key as an interrupted first write. Both healthy-boot arms (successful load, and a Won publish on the generate path) now sweep stale .publishing and .recovering entries, but only after the sweeper itself has made the final durable: a loadable final is not necessarily synced, and removing a live winner's marker before its sync_all would recreate the marker-less partial-final wedge on power loss. Any durability failure skips the sweep and leaves the markers in place. --- crates/gitlawb-node/src/lib.rs | 225 +++++++++++++++++++++++++++++++-- 1 file changed, 217 insertions(+), 8 deletions(-) diff --git a/crates/gitlawb-node/src/lib.rs b/crates/gitlawb-node/src/lib.rs index 486200b6..4feadbbd 100644 --- a/crates/gitlawb-node/src/lib.rs +++ b/crates/gitlawb-node/src/lib.rs @@ -1342,6 +1342,76 @@ fn wait_for_recovery_claims(final_path: &std::path::Path) { } } +/// Best-effort sweep of stale `.{stem}.publishing.*` markers and +/// `.{stem}.recovering.*` claims beside a healthy final, run on both +/// healthy-boot arms of `load_or_create_keypair_with` (#194, U4). Without it +/// a marker orphaned by a crash between its durability fsync and the final's +/// `create_new`, or a claim left by a crashed recovery, outlives every +/// healthy boot and later misclassifies a real corruption of a good key as +/// an interrupted first write (quarantine-and-regenerate instead of the loud +/// error a corrupted good key deserves). +/// +/// DURABILITY PRECONDITION (critical, the reason this is not a bare unlink +/// loop): a loadable final is not necessarily durable — a racing fallback +/// winner's `write_all` is visible via the page cache before its `sync_all`, +/// so sweeping on loadability alone could remove a marker ahead of the +/// content it vouches for and let a power loss recreate exactly the +/// marker-less partial final the bracket exists to prevent. The sweeper +/// therefore makes the content durable ITSELF before touching any marker: +/// open the final read-only, `sync_all` it, then `fsync_parent_dir` (Unix). +/// If either step fails the sweep is skipped entirely — every marker stays +/// in place, at most a warn. Only after both succeed is every matching entry +/// removed (each best-effort), followed by a best-effort dir fsync. On +/// non-Unix the dir-fsync halves are skipped like every other dir fsync here +/// (reasoned, not run: no non-Unix target is exercised by these tests). +/// +/// Sweeping a FOREIGN marker is safe once durability holds: the swept +/// publisher's own disposal tolerates NotFound on every arm; a live +/// publisher still before its final `create_new` hits AlreadyExists and +/// routes to Lost regardless of its marker; and the content the marker +/// vouched for is durable by the sweeper's own hand. `sweep_sync` is the +/// `RecoverySeam` test hook, run before the final's `sync_all` with an `Err` +/// taken as that sync's result; production passes a no-op `Ok`. +fn sweep_stale_markers(final_path: &std::path::Path, sweep_sync: fn() -> std::io::Result<()>) { + let durable = std::fs::File::open(final_path) + .and_then(|f| sweep_sync().and_then(|()| f.sync_all())) + .map_err(anyhow::Error::new); + #[cfg(unix)] + let durable = durable.and_then(|()| fsync_parent_dir(final_path)); + if let Err(e) = durable { + warn!( + path = %final_path.display(), + err = %e, + "skipping stale publish-marker sweep: could not make the final key durable" + ); + return; + } + let dir = final_path + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or_else(|| std::path::Path::new(".")); + let stem = final_path + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or("identity.pem"); + let publishing = format!(".{stem}.publishing."); + let recovering = format!(".{stem}.recovering."); + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + for entry in entries.filter_map(|e| e.ok()) { + let name = entry.file_name(); + let Some(name) = name.to_str() else { continue }; + if name.starts_with(&publishing) || name.starts_with(&recovering) { + let _ = std::fs::remove_file(entry.path()); + } + } + #[cfg(unix)] + { + let _ = fsync_parent_dir(final_path); + } +} + /// Compile-time fault-injection seam for `publish_key_atomically`, extending /// the `before_link` closure precedent: every hook is a no-op `Ok` in /// production (`PublishFaults::NONE`); tests swap in a hook returning `Err` to @@ -1419,15 +1489,19 @@ impl PublishFaults { /// without hard-link support, or a transient link error), the publish falls /// back to `create_new(0o600)` directly on the final path /// (`publish_key_fallback`). The fallback keeps (b): `create_new` still admits -/// exactly one winner and a lost race loads the existing key. It weakens (a), -/// (c), and (d): the final name is visible while the PEM is being written, a +/// exactly one winner and a lost race loads the existing key. It weakens (a) +/// and (d): the final name is visible while the PEM is being written, a /// window concurrent readers ride out via `load_racing_key`'s wall-clock -/// deadline; a crash mid-write leaves a partial final that fails loudly -/// (`invalid PEM key`) at the next start rather than being cleaned up; and the -/// final name can become durable before the PEM bytes are fsynced, so a power -/// loss can additionally leave a durable empty/truncated final. In every case -/// the failure mode is the same LOUD invalid-PEM (or mode) error at the next -/// start, never a silently used key. +/// deadline; and the final name can become durable before the PEM bytes are +/// fsynced, so a power loss can additionally leave a durable empty/truncated +/// final. Guarantee (c) now holds on the fallback too, via the marker +/// protocol (#194, U2/U4): a crash inside the non-atomic window leaves +/// marker+final together, and the next boot claims the marker, quarantines +/// the partial final, and regenerates instead of wedging. Honest residual on +/// top of the U3 list below: that recovery runs at the next boot, not at +/// crash time, so the partial final persists until then. In every unrecovered +/// case the failure mode is the same LOUD invalid-PEM (or mode) error at the +/// next start, never a silently used key. /// /// Against the boot-time crash RECOVERY race (#194, U3) the fallback's marker /// protocol restores (b)-style convergence: removing the publish marker is @@ -1794,6 +1868,10 @@ struct RecoverySeam<'a> { /// Runs at the top of each generate+publish pass, so a test can mutate /// on-disk state between a completed recovery and the retry publish. before_publish: &'a dyn Fn(), + /// Runs inside `sweep_stale_markers` before the final's `sync_all`; an + /// `Err` is taken as that sync's result, so a test can fail the sweep's + /// durability gate deterministically (#194, U4). + sweep_sync: fn() -> std::io::Result<()>, } impl RecoverySeam<'_> { @@ -1801,6 +1879,7 @@ impl RecoverySeam<'_> { const NONE: RecoverySeam<'static> = RecoverySeam { load_fault: || Ok(()), before_publish: &|| {}, + sweep_sync: || Ok(()), }; } @@ -1979,6 +2058,7 @@ fn load_or_create_keypair_with( match boot_load_key(key_path, recovery_allowed, seam.load_fault) { BootLoad::Loaded(kp) => { remove_claim(&claim); + sweep_stale_markers(key_path, seam.sweep_sync); return Ok(*kp); } // Only emitted while recovery is still allowed (crash_exit is @@ -2014,6 +2094,7 @@ fn load_or_create_keypair_with( { KeyPublish::Won => { remove_claim(&claim); + sweep_stale_markers(key_path, seam.sweep_sync); info!(path = %key_path.display(), did = %kp.did(), "generated new node identity"); return Ok(kp); } @@ -2021,6 +2102,7 @@ fn load_or_create_keypair_with( match boot_load_key(key_path, recovery_allowed, seam.load_fault) { BootLoad::Loaded(kp) => { remove_claim(&claim); + sweep_stale_markers(key_path, seam.sweep_sync); return Ok(*kp); } // Only emitted while recovery is still allowed (crash_exit @@ -3898,6 +3980,133 @@ mod identity_key_tests { "with the final untouched the demoted winner must resolve its own key" ); } + + // #194 (U4): a HEALTHY load must sweep stale `.publishing.` markers and + // `.recovering.` claims, or they linger to misclassify a much-later real + // corruption of a good key as an interrupted first write. RED before U4: + // all three files survive the load. + #[test] + fn healthy_load_sweeps_stale_markers_and_claims() { + let dir = tempfile::tempdir().expect("tempdir"); + let existing = Keypair::generate(); + let pem = existing.to_pem().expect("pem"); + let key_path = seed_crash_state(dir.path(), pem.as_bytes()); + std::fs::write(dir.path().join(".identity.pem.publishing.88888.1"), b"") + .expect("seed second stale marker"); + std::fs::write(dir.path().join(".identity.pem.recovering.77777"), b"") + .expect("seed stale recovery claim"); + + let kp = load_or_create_keypair(&key_config(&key_path)) + .expect("a valid final with stale markers must load"); + + assert_eq!( + format!("{}", kp.did()), + format!("{}", existing.did()), + "the pre-existing identity must be preserved, not regenerated" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "a healthy load must sweep stale publish markers" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "a healthy load must sweep stale recovery claims" + ); + } + + // #194 (U4): the generate path must sweep too. A crash between the + // marker's durability fsync and the final's `create_new` leaves a durable + // marker with NO final; only the Won arm of a later boot can ever clean + // that state, since no load will ever succeed beside it. RED before U4 + // (the marker survives); RED again under a load-arm-only implementation + // (mutation check: with the Won-arm call site removed this test must + // fail while healthy_load_sweeps_stale_markers_and_claims stays green). + #[test] + fn generate_path_sweeps_stale_markers() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + std::fs::write(dir.path().join(".identity.pem.publishing.66666.0"), b"") + .expect("seed stale marker without a final"); + + let kp = load_or_create_keypair(&key_config(&key_path)) + .expect("a stale marker with no final must not block generation"); + + let on_disk = super::load_existing_key(&key_path).expect("published key loads"); + assert_eq!( + format!("{}", on_disk.did()), + format!("{}", kp.did()), + "the generated identity must be on disk" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "a Won publish on a healthy boot must sweep the stale marker" + ); + } + + // #194 (U4) MUST-NOT: the sweep must never remove a marker ahead of the + // content it vouches for. With the seam failing the sweep's durability + // sync, the load still succeeds but the marker SURVIVES. Vacuously green + // before U4 (no sweep exists, so the marker trivially survives); its + // load-bearing proof is the mutation check: with the durability gate + // removed (sweep unconditionally) this test must fail. + #[test] + fn sweep_durability_gate_leaves_markers_on_sync_failure() { + let dir = tempfile::tempdir().expect("tempdir"); + let existing = Keypair::generate(); + let pem = existing.to_pem().expect("pem"); + let key_path = seed_crash_state(dir.path(), pem.as_bytes()); + let seam = RecoverySeam { + sweep_sync: || Err(std::io::Error::other("injected sweep sync failure")), + ..RecoverySeam::NONE + }; + + let kp = load_or_create_keypair_with(&key_path, &|| {}, &|| {}, PublishFaults::NONE, seam) + .expect("a failed sweep durability sync must not fail the load"); + + assert_eq!( + format!("{}", kp.did()), + format!("{}", existing.did()), + "the identity must still load when the sweep is skipped" + ); + assert_eq!( + names_containing(dir.path(), ".publishing."), + vec![".identity.pem.publishing.99999.0".to_string()], + "the marker must survive when the sweep cannot make the final durable" + ); + } + + // #194 (U4): the sweep's removals are best-effort; unremovable markers + // must never fail an otherwise healthy load. The key directory is made + // read-only (0555): the durability gate still passes (the file sync_all + // and the dir fsync both open read-only) but every remove_file fails + // EACCES. Vacuously green before U4 (no sweep exists); load-bearing once + // the sweep runs, since a sweep that surfaced removal errors would fail + // this load. + #[test] + fn sweep_failure_tolerated() { + let dir = tempfile::tempdir().expect("tempdir"); + let existing = Keypair::generate(); + let pem = existing.to_pem().expect("pem"); + let key_path = seed_crash_state(dir.path(), pem.as_bytes()); + std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o555)) + .expect("read-only key dir"); + + let result = load_or_create_keypair(&key_config(&key_path)); + + std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o755)) + .expect("restore key dir"); + let kp = result.expect("unremovable markers must not fail the load"); + assert_eq!( + format!("{}", kp.did()), + format!("{}", existing.did()), + "the identity must load despite the failed sweep removals" + ); + assert_eq!( + names_containing(dir.path(), ".publishing."), + vec![".identity.pem.publishing.99999.0".to_string()], + "the unremovable marker survives, harmlessly" + ); + } } #[cfg(test)] From e673271170d74371f3916325236433b0c5a1373a Mon Sep 17 00:00:00 2001 From: t Date: Tue, 21 Jul 2026 15:33:45 -0500 Subject: [PATCH 28/41] fix(node): make recovery claims complete, releasable, and themselves recoverable Three review findings against the marker protocol. Recovery now claims every publishing marker and stale recovering claim (per-item bounded claim names), so a live publisher's own-marker commit check can no longer falsely succeed while its round is stolen through a stale marker; a post-claim re-parse aborts to reload when a publisher completed in the window. Claims count toward the crash signature and are claimable by rename, so a recoverer crash between claim and quarantine no longer recreates the permanent wedge. Claims are released on every exit including publish failures, closing the leak that made demoted publishers ride the full race deadline on a dead claim. The crash signature also needs two consecutive observations, giving a live mid-write publisher one poll of grace before a reader can classify its round as a crash. --- crates/gitlawb-node/src/lib.rs | 675 +++++++++++++++++++++++++++------ 1 file changed, 564 insertions(+), 111 deletions(-) diff --git a/crates/gitlawb-node/src/lib.rs b/crates/gitlawb-node/src/lib.rs index 4feadbbd..bce086e5 100644 --- a/crates/gitlawb-node/src/lib.rs +++ b/crates/gitlawb-node/src/lib.rs @@ -1243,6 +1243,41 @@ fn list_publish_markers(final_path: &std::path::Path) -> Vec markers } +/// The publish markers AND recovery claims (`.{stem}.publishing.*` plus +/// `.{stem}.recovering.*`) beside `final_path`, sorted: the full claimable +/// set for crash recovery (#194, F3). Claims count because a recoverer's +/// claim IS a renamed marker: a recoverer that crashes between its claim +/// rename and the quarantine leaves a content-bad final with a claim and +/// ZERO markers, and a signature keyed on markers alone would leave that +/// state unrecoverable forever. Directory-read and non-UTF-8 policy match +/// `list_publish_markers`. +fn list_recovery_claimables(final_path: &std::path::Path) -> Vec { + let dir = final_path + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or_else(|| std::path::Path::new(".")); + let stem = final_path + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or("identity.pem"); + let publishing = format!(".{stem}.publishing."); + let recovering = format!(".{stem}.recovering."); + let Ok(entries) = std::fs::read_dir(dir) else { + return Vec::new(); + }; + let mut claimables: Vec<_> = entries + .filter_map(|e| e.ok()) + .filter(|e| { + e.file_name() + .to_str() + .is_some_and(|n| n.starts_with(&publishing) || n.starts_with(&recovering)) + }) + .map(|e| e.path()) + .collect(); + claimables.sort(); + claimables +} + /// Load a key another concurrent start may still be publishing, polling until it /// parses or `KEY_RACE_DEADLINE` elapses. An already-provisioned key parses on /// the first attempt with no sleep. @@ -1262,8 +1297,9 @@ enum BootLoad { /// Boxed to keep the variants' sizes comparable (clippy: /// large_enum_variant); the enum lives only across a boot-time match. Loaded(Box), - /// Content-class failure observed while a `.publishing.` marker was - /// present; returned immediately, without riding the deadline. + /// Content-class failure observed while a `.publishing.` marker or a + /// `.recovering.` claim was present, on two consecutive polls; returned + /// without riding the rest of the deadline. CrashSignature(anyhow::Error), /// Any other failure, after the full deadline. Failed(anyhow::Error), @@ -1271,13 +1307,19 @@ enum BootLoad { /// `load_racing_key` for the boot loop (#194, U2). With `crash_exit` set, a /// content-class failure (`KeyContentError`) observed while a `.publishing.` -/// marker is present returns `CrashSignature` IMMEDIATELY instead of riding -/// the deadline: marker plus unreadable content is the crash signature -/// recovery exists for, and the caller's atomic claim rename arbitrates the -/// (rare) race against a live fallback publisher still inside its window. -/// Every other failure keeps the full wall-clock deadline. `load_fault` is -/// the injection hook (a no-op `Ok` in production); an injected `Err` is -/// taken as that attempt's result and is NOT content-class. +/// marker or a `.recovering.` claim is present, on TWO consecutive polls, +/// returns `CrashSignature` instead of riding the deadline: claimable plus +/// unreadable content is the crash signature recovery exists for, and the +/// caller's atomic claim renames arbitrate the (rare) race against a live +/// fallback publisher still inside its window. Claims count toward the +/// signature for F3 (see `list_recovery_claimables`). The two-poll grace +/// exists so a reader that catches a live publisher's mid-write window is +/// not instantly classified as a crash: one ~2ms iteration is enough for +/// most writes to complete, and a real crash state persists across every +/// poll. Every other failure keeps the full wall-clock deadline. +/// `load_fault` is the injection hook (a no-op `Ok` in production); an +/// injected `Err` is taken as that attempt's result and is NOT +/// content-class. fn boot_load_key( path: &std::path::Path, crash_exit: bool, @@ -1285,6 +1327,7 @@ fn boot_load_key( ) -> BootLoad { let deadline = std::time::Instant::now() + KEY_RACE_DEADLINE; let mut last_err; + let mut signature_seen = false; loop { match load_fault() .map_err(anyhow::Error::new) @@ -1293,9 +1336,13 @@ fn boot_load_key( Ok(kp) => return BootLoad::Loaded(Box::new(kp)), Err(e) => last_err = e, } - if crash_exit && is_key_content_error(&last_err) && !list_publish_markers(path).is_empty() { + let signature = crash_exit + && is_key_content_error(&last_err) + && !list_recovery_claimables(path).is_empty(); + if signature && signature_seen { return BootLoad::CrashSignature(last_err); } + signature_seen = signature; if std::time::Instant::now() >= deadline { return BootLoad::Failed(last_err); } @@ -1505,14 +1552,17 @@ impl PublishFaults { /// /// Against the boot-time crash RECOVERY race (#194, U3) the fallback's marker /// protocol restores (b)-style convergence: removing the publish marker is -/// the winner's commit check, and recovery claims a round by renaming that -/// same marker, so exactly one side takes the round. A winner that loses its -/// marker demotes itself to Lost, waits out any `.recovering.` claim -/// (bounded by `KEY_RACE_DEADLINE`), and re-loads the settled key. Honest -/// residuals: a winner stalled past `KEY_RACE_DEADLINE` that loads in the -/// sub-ms window before a recoverer's claim exists, and a crash-interrupted -/// recovery whose stale claim just expires the bounded wait — both degrade -/// to a loud error or a re-load, never silent two-sided divergence. +/// the winner's commit check, and recovery claims a round by renaming EVERY +/// marker and stale claim beside the final (a live winner's own marker is +/// necessarily among them), so exactly one side takes the round. A winner +/// that loses its marker demotes itself to Lost, waits out any +/// `.recovering.` claim (bounded by `KEY_RACE_DEADLINE`), and re-loads the +/// settled key; a recovery that finds the final completed after claiming +/// re-loads it instead of quarantining. Honest residuals: a winner stalled +/// past `KEY_RACE_DEADLINE` that loads in the sub-ms window before a +/// recoverer's claim exists, and a crash-interrupted recovery whose stale +/// claim just expires the bounded wait: both degrade to a loud error or a +/// re-load, never silent two-sided divergence. /// /// `before_link` is a no-op in production; tests use it to widen the /// post-write / pre-link window deterministically. `before_commit` runs @@ -1824,9 +1874,10 @@ fn publish_key_fallback( } } // Success: the final is complete and durable. Removing our OWN marker is - // the COMMIT CHECK (#194, U3): recovery claims a round by renaming this - // exact marker, so a successful removal proves no recovery can ever claim - // it and linearizes this winner ahead of any recoverer. Any failure + // the COMMIT CHECK (#194, U3): recovery claims a round by renaming EVERY + // marker beside the final, so our marker is necessarily among any + // claimed round and a successful removal proves no recovery can ever + // claim it, linearizing this winner ahead of any recoverer. Any failure // (NotFound after a peer's claim rename, or any other error) means // ownership of the round cannot be proven: a recovering peer may have // quarantined our final and republished, so returning Won could report @@ -1872,6 +1923,10 @@ struct RecoverySeam<'a> { /// `Err` is taken as that sync's result, so a test can fail the sweep's /// durability gate deterministically (#194, U4). sweep_sync: fn() -> std::io::Result<()>, + /// Runs inside `recover_crashed_publish` between the claim renames and + /// the post-claim re-parse of the final, so a test can interleave a + /// concurrent publisher's resolution into exactly that window (#194, F2). + before_reparse: &'a dyn Fn(), } impl RecoverySeam<'_> { @@ -1880,6 +1935,7 @@ impl RecoverySeam<'_> { load_fault: || Ok(()), before_publish: &|| {}, sweep_sync: || Ok(()), + before_reparse: &|| {}, }; } @@ -1895,10 +1951,10 @@ fn load_or_create_keypair(config: &Config) -> Result { /// Outcome of `recover_crashed_publish` (#194, U2). enum Recovery { - /// A marker was claimed and the corrupt final quarantined (or removed); - /// the caller regenerates. Holds the claim file's path, removed only - /// after the follow-up publish resolves. - Claimed(std::path::PathBuf), + /// The round was claimed and the corrupt final quarantined (or removed); + /// the caller regenerates. Holds every claim file this recovery created, + /// removed once the follow-up publish resolves (or on any error exit). + Claimed(Vec), /// The marker vanished before it could be claimed (the publisher — or a /// competing recoverer — resolved the window concurrently): the single /// follow-up load's result is final, success or error, no recovery. @@ -1907,20 +1963,55 @@ enum Recovery { Reloaded(Box>), } +/// Best-effort removal of the claim files a recovery created, plus a +/// best-effort dir fsync. Claims vouch for NO on-disk content (unlike +/// publish markers, whose disposal order against the final is pinned): they +/// only gate a demoted publisher's bounded wait and recovery arbitration. +/// Release therefore needs no ordering against the final, the markers, or +/// anything else: any exit may drop the claims at any point once their owner +/// stops recovering, and the worst a lost claim can cause is a waiter +/// re-loading early, which the load path copes with (#194, F1/F3). +fn release_recovery_claims(final_path: &std::path::Path, claims: &[std::path::PathBuf]) { + for claim in claims { + let _ = std::fs::remove_file(claim); + } + #[cfg(unix)] + { + let _ = fsync_parent_dir(final_path); + } +} + /// Claim-and-quarantine for a boot whose load returned `CrashSignature`: a -/// content-class failure beside a `.publishing.` marker — U1's signature of -/// a fallback publish crashed inside its non-atomic window (#194, U2). -/// Atomically claims ONE marker (rename to `.{stem}.recovering.{pid}`, so N -/// racing recoverers admit exactly one), quarantines the corrupt final by -/// rename (preserving bytes and mode for post-mortem; falls back to removal -/// only if every quarantine name fails), consumes any remaining markers -/// best-effort, and hands control back to regenerate. The caller observed a -/// marker, so finding none here (or losing the claim rename) means the -/// window was resolved concurrently: one immediate re-load decides, with no -/// recovery. +/// content-class failure beside a `.publishing.` marker or `.recovering.` +/// claim, the signature of a fallback publish (or a recovery) crashed +/// inside its window (#194, U2/F2/F3). Claims the WHOLE round: every marker +/// and every pre-existing claim is renamed to a fresh claim of our own +/// (`.{stem}.recovering.{pid}.{n}`, bounded fresh-name probing per item, the +/// KEY_TEMP_ATTEMPTS discipline; a NotFound on an individual rename means +/// that item resolved concurrently and is skipped). Claiming everything is +/// what makes the publisher's commit check sound: a live fallback publisher +/// still inside its window necessarily has its own marker among the +/// claimables, so its marker removal fails and it demotes (F2); claiming +/// only one marker would let a stale marker's claim steal a live round. +/// +/// If NO rename lands, the round resolved concurrently: one immediate +/// re-load decides, with no recovery. If at least one lands, this recovery +/// owns the round, and it RE-PARSES the final once before touching it: a +/// publisher that completed inside the claim window has already resolved the +/// round, so a parseable final is returned as `Reloaded` and the claims are +/// released, never quarantining a good key. Only if the final still +/// content-fails does the recovery quarantine it by rename (preserving bytes +/// and mode for post-mortem; falling back to removal only if every +/// quarantine name fails), consume any remaining markers best-effort, and +/// hand control back to regenerate. Every error exit after claiming releases +/// the claims first (see `release_recovery_claims`; a leaked claim would +/// stall later demoted publishers on the full deadline, F1). `before_reparse` +/// is the test seam for the claim-to-re-parse window (a no-op in +/// production). fn recover_crashed_publish( final_path: &std::path::Path, load_err: anyhow::Error, + before_reparse: &dyn Fn(), ) -> Result { let dir = final_path .parent() @@ -1930,28 +2021,59 @@ fn recover_crashed_publish( .file_name() .and_then(|s| s.to_str()) .unwrap_or("identity.pem"); - let markers = list_publish_markers(final_path); - let Some(first_marker) = markers.first() else { + let claimables = list_recovery_claimables(final_path); + if claimables.is_empty() { return Ok(Recovery::Reloaded(Box::new(load_racing_key(final_path)))); - }; + } - // CLAIM one marker atomically. rename admits exactly one winner: a - // competing recoverer (or the publisher finishing and removing its - // marker) leaves NotFound, which means the window resolved concurrently; - // one immediate re-load decides, with no recovery. - let claim = dir.join(format!(".{stem}.recovering.{}", std::process::id())); - match std::fs::rename(first_marker, &claim) { - Ok(()) => {} - Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - return Ok(Recovery::Reloaded(Box::new(load_racing_key(final_path)))); - } - Err(e) => { - return Err(load_err.context(format!( - "could not claim publish marker {} for crash recovery: {e}", - first_marker.display() - ))); + // CLAIM every marker and foreign claim by rename. Each rename admits + // exactly one winner; NotFound means that item resolved concurrently (a + // publisher committed, or a competing recoverer claimed it) and is + // skipped. Names are never reused or clobbered: the monotonic counter + // plus the exists() probe keeps every destination fresh, bounded per + // item like the temp/marker/quarantine probes. + let mut claims: Vec = Vec::new(); + let mut next_name = 0u32; + for item in &claimables { + for _ in 0..KEY_TEMP_ATTEMPTS { + let dest = dir.join(format!( + ".{stem}.recovering.{}.{next_name}", + std::process::id() + )); + next_name = next_name.wrapping_add(1); + if dest.exists() { + continue; + } + match std::fs::rename(item, &dest) { + Ok(()) => { + claims.push(dest); + break; + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => break, + // Any other rename failure: probe the next fresh name, up to + // the per-item bound; an unclaimable item is left for the + // best-effort marker consumption below (or a later sweep). + Err(_) => continue, + } } } + if claims.is_empty() { + // Every item vanished before it could be claimed: the round resolved + // concurrently; one immediate re-load decides, with no recovery. + return Ok(Recovery::Reloaded(Box::new(load_racing_key(final_path)))); + } + + // POST-CLAIM RE-PARSE (#194, F2): between the crash-signature + // observation and the renames above, a live publisher may have completed + // its final. Now that the round is claimed no publisher can commit + // anymore, so one parse decides: a readable final means the round is + // already resolved, so release the claims and defer to it, never + // quarantining a good key. + (before_reparse)(); + if let Ok(kp) = load_existing_key(final_path) { + release_recovery_claims(final_path, &claims); + return Ok(Recovery::Reloaded(Box::new(Ok(kp)))); + } // QUARANTINE the final under a bounded name probe (the KEY_TEMP_ATTEMPTS // discipline: a stale quarantine from a crashed same-PID start must not @@ -1964,8 +2086,8 @@ fn recover_crashed_publish( .unwrap_or(0); warn!( path = %final_path.display(), - markers = ?markers, - claim = %claim.display(), + claimables = ?claimables, + claims = ?claims, quarantine = %dest_for(start).display(), "crash-interrupted identity key publish: quarantining the unreadable key and \ regenerating the identity" @@ -1992,20 +2114,24 @@ fn recover_crashed_publish( if !quarantined { // Every rename failed: removal still unblocks the regeneration // (mirroring the publish paths' removal policy); only if even that - // fails is the boot left to its loud error. + // fails is the boot left to its loud error, with the claims + // released first, so the dead round cannot stall later publishers + // (#194, F1). if let Err(e) = std::fs::remove_file(final_path) { if e.kind() != std::io::ErrorKind::NotFound { - return Err(anyhow::Error::new(e).context(format!( + release_recovery_claims(final_path, &claims); + return Err(load_err.context(format!( "could not quarantine or remove corrupt identity key {} during crash \ - recovery", + recovery: {e}", final_path.display() ))); } } } - // Consume any remaining markers (a multi-crash pile-up) best-effort, and - // best-effort fsync the directory so the transition tends to be durable. + // Consume any remaining markers (a multi-crash pile-up, or an item the + // claim probe could not rename) best-effort, and best-effort fsync the + // directory so the transition tends to be durable. for marker in list_publish_markers(final_path) { let _ = std::fs::remove_file(&marker); } @@ -2013,7 +2139,7 @@ fn recover_crashed_publish( { let _ = fsync_parent_dir(final_path); } - Ok(Recovery::Claimed(claim)) + Ok(Recovery::Claimed(claims)) } /// Body of `load_or_create_keypair` with the test seams threaded in (#194, @@ -2022,12 +2148,13 @@ fn recover_crashed_publish( /// load-side recovery seam (production: `RecoverySeam::NONE`). /// /// Load-or-generate with AT MOST ONE crash recovery per start: a load -/// failure recovers — claim a marker, quarantine the final, regenerate — +/// failure recovers (claim the round, quarantine the final, regenerate) /// only when it is content-class (`KeyContentError`), a `.publishing.` -/// marker exists (U1's crash signature), and no recovery has run yet this -/// start. Every other failure surfaces unchanged. The Lost arm participates -/// in the same policy; a recovery fired there loops back for one retry -/// publish, and any failure on that second pass surfaces unchanged. +/// marker or `.recovering.` claim exists (the crash signature, observed on +/// two consecutive polls), and no recovery has run yet this start. Every +/// other failure surfaces unchanged. The Lost arm participates in the same +/// policy; a recovery fired there loops back for one retry publish, and any +/// failure on that second pass surfaces unchanged. fn load_or_create_keypair_with( key_path: &std::path::Path, before_link: &dyn Fn(), @@ -2035,17 +2162,14 @@ fn load_or_create_keypair_with( faults: PublishFaults, seam: RecoverySeam<'_>, ) -> Result { - // The claim file of a fired recovery. Removed (best-effort) only at exits - // AFTER the follow-up publish resolves (Won or Lost), so a stalled winner - // our claim demoted can still observe it while waiting; U3 relies on - // this. The pre-publish error exits inside recover_crashed_publish - // deliberately leave it in place. - let mut claim: Option = None; - let remove_claim = |claim: &Option| { - if let Some(c) = claim { - let _ = std::fs::remove_file(c); - } - }; + // The claim files of a fired recovery. Held while the follow-up + // generate+publish runs, so a stalled winner our claims demoted can still + // observe them while waiting (U3 relies on this), and released on EVERY + // exit after that, success, Lost, and error alike (#194, F1): a leaked claim + // stalls every later demoted publisher on the full KEY_RACE_DEADLINE. + // Release order against the final or the markers does not matter; see + // release_recovery_claims. + let mut claims: Vec = Vec::new(); let mut recovery_allowed = true; // At most two passes: the second exists only so a Lost-arm recovery can @@ -2057,22 +2181,24 @@ fn load_or_create_keypair_with( if key_path.exists() { match boot_load_key(key_path, recovery_allowed, seam.load_fault) { BootLoad::Loaded(kp) => { - remove_claim(&claim); + release_recovery_claims(key_path, &claims); sweep_stale_markers(key_path, seam.sweep_sync); return Ok(*kp); } // Only emitted while recovery is still allowed (crash_exit is // gated on it). - BootLoad::CrashSignature(e) => match recover_crashed_publish(key_path, e)? { - Recovery::Claimed(c) => { - claim = Some(c); - recovery_allowed = false; - // Fall through to regenerate and publish. + BootLoad::CrashSignature(e) => { + match recover_crashed_publish(key_path, e, seam.before_reparse)? { + Recovery::Claimed(c) => { + claims = c; + recovery_allowed = false; + // Fall through to regenerate and publish. + } + Recovery::Reloaded(result) => return *result, } - Recovery::Reloaded(result) => return *result, - }, + } BootLoad::Failed(e) => { - remove_claim(&claim); + release_recovery_claims(key_path, &claims); return Err(e); } } @@ -2080,20 +2206,42 @@ fn load_or_create_keypair_with( (seam.before_publish)(); let kp = Keypair::generate(); - let pem = kp - .to_pem() - .map_err(|e| anyhow::anyhow!("failed to serialize key: {e}"))?; + // Every error exit below this point may hold a fired recovery's + // claims; release them before propagating (#194, F1). The bare `?` + // this replaces leaked the claim files on a failed retry publish. + let pem = match kp.to_pem() { + Ok(pem) => pem, + Err(e) => { + release_recovery_claims(key_path, &claims); + return Err(anyhow::anyhow!("failed to serialize key: {e}")); + } + }; if let Some(parent) = key_path.parent() { - std::fs::create_dir_all(parent)?; + if let Err(e) = std::fs::create_dir_all(parent) { + release_recovery_claims(key_path, &claims); + return Err(e.into()); + } } // Publish atomically: the final path only ever appears complete, and a // lost race loads the winner's key rather than overwriting it. - match publish_key_atomically(key_path, pem.as_bytes(), before_link, before_commit, faults)? - { + let published = match publish_key_atomically( + key_path, + pem.as_bytes(), + before_link, + before_commit, + faults, + ) { + Ok(published) => published, + Err(e) => { + release_recovery_claims(key_path, &claims); + return Err(e); + } + }; + match published { KeyPublish::Won => { - remove_claim(&claim); + release_recovery_claims(key_path, &claims); sweep_stale_markers(key_path, seam.sweep_sync); info!(path = %key_path.display(), did = %kp.did(), "generated new node identity"); return Ok(kp); @@ -2101,29 +2249,31 @@ fn load_or_create_keypair_with( KeyPublish::Lost => { match boot_load_key(key_path, recovery_allowed, seam.load_fault) { BootLoad::Loaded(kp) => { - remove_claim(&claim); + release_recovery_claims(key_path, &claims); sweep_stale_markers(key_path, seam.sweep_sync); return Ok(*kp); } // Only emitted while recovery is still allowed (crash_exit // is gated on it). - BootLoad::CrashSignature(e) => match recover_crashed_publish(key_path, e) { - Ok(Recovery::Claimed(c)) => { - claim = Some(c); - recovery_allowed = false; - continue; // Second pass: regenerate and publish. - } - Ok(Recovery::Reloaded(result)) => { - remove_claim(&claim); - return *result; - } - Err(err) => { - remove_claim(&claim); - return Err(err); + BootLoad::CrashSignature(e) => { + match recover_crashed_publish(key_path, e, seam.before_reparse) { + Ok(Recovery::Claimed(c)) => { + claims = c; + recovery_allowed = false; + continue; // Second pass: regenerate and publish. + } + Ok(Recovery::Reloaded(result)) => { + release_recovery_claims(key_path, &claims); + return *result; + } + Err(err) => { + release_recovery_claims(key_path, &claims); + return Err(err); + } } - }, + } BootLoad::Failed(e) => { - remove_claim(&claim); + release_recovery_claims(key_path, &claims); return Err(e); } } @@ -3904,14 +4054,15 @@ mod identity_key_tests { let recovery = super::recover_crashed_publish( &key_path, anyhow::anyhow!("synthetic post-commit crash signature"), + &|| {}, ) .expect("a late recovery must not error"); let loaded = match recovery { super::Recovery::Reloaded(result) => { result.expect("the vanished-marker arm re-loads the winner's key") } - super::Recovery::Claimed(claim) => { - panic!("no marker exists to claim after commit, got claim {claim:?}") + super::Recovery::Claimed(claims) => { + panic!("no marker exists to claim after commit, got claims {claims:?}") } }; assert_eq!( @@ -4107,6 +4258,308 @@ mod identity_key_tests { "the unremovable marker survives, harmlessly" ); } + + // #194 (F1): a claimed recovery whose retry publish FAILS must release + // its `.recovering.` claim before surfacing the error. A leaked claim + // makes every later demoted fallback publisher ride the full + // KEY_RACE_DEADLINE in wait_for_recovery_claims until some healthy boot + // sweeps it. RED before F1: the bare `?` on the publish call returns + // with the claim still on disk. + #[test] + fn claim_released_when_post_recovery_publish_fails() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + fallback_create: || Err(std::io::Error::other("injected fallback create failure")), + ..PublishFaults::NONE + }; + + let err = match load_or_create_keypair_with( + &key_path, + &|| {}, + &|| {}, + faults, + RecoverySeam::NONE, + ) { + Err(e) => e, + Ok(_) => panic!("the forced publish failure must surface, not resolve a key"), + }; + assert!( + format!("{err:#}").contains("injected fallback create failure"), + "the publish failure must propagate: {err:#}" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "a failed retry publish must release the recovery claim, not leak it" + ); + } + + // #194 (F2): a stale marker must not let recovery steal a LIVE fallback + // publisher's round. Deterministic construction: publisher A parks at + // its commit check (final complete and durable, own marker M present, + // stale marker S beside it); the real recovery runs, and its + // before-reparse seam (inside the window between the claim step and the + // post-claim re-parse) unparks A and waits for A to finish, landing A's + // commit exactly in the claim window. Under F2 recovery claims EVERY + // claimable, so A's removal of M fails and A demotes; the re-parse then + // sees A's completed key and recovery aborts to Reloaded. Everyone + // converges on one on-disk identity. RED before F2: recovery claims only + // the lexicographically-first stale S, A's commit on M succeeds, and A + // returns Won with key A while recovery quarantines A's final and + // republishes key B: the two-sided divergence R4 forbids. + #[test] + fn stale_marker_cannot_steal_a_live_publishers_round() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + // Sorts before any real-pid marker (pids never start with '0'). + std::fs::write(dir.path().join(".identity.pem.publishing.00000.0"), b"") + .expect("seed stale marker"); + + let (parked_tx, parked_rx) = std::sync::mpsc::channel::<()>(); + let (unpark_tx, unpark_rx) = std::sync::mpsc::channel::<()>(); + let (done_tx, done_rx) = std::sync::mpsc::channel::(); + let p_path = key_path.clone(); + let publisher = std::thread::spawn(move || { + let before_commit = move || { + parked_tx.send(()).expect("signal parked"); + unpark_rx + .recv() + .expect("wait inside the recovery's claim window"); + }; + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + ..PublishFaults::NONE + }; + let kp = load_or_create_keypair_with( + &p_path, + &|| {}, + &before_commit, + faults, + RecoverySeam::NONE, + ) + .expect("the publisher must resolve a key, demoted or not"); + done_tx + .send(format!("{}", kp.did())) + .expect("report the publisher's identity"); + }); + parked_rx + .recv() + .expect("publisher reaches its commit check"); + + // The concurrent recovery, via the REAL recovery path. Its + // before-reparse hook interleaves the publisher's commit into the + // post-claim window and records the identity the publisher resolved. + let publisher_did: std::sync::Mutex> = std::sync::Mutex::new(None); + let before_reparse = || { + unpark_tx.send(()).expect("unpark the publisher"); + let did = done_rx + .recv() + .expect("publisher finishes inside the window"); + *publisher_did.lock().expect("publisher_did lock") = Some(did); + }; + let recovery = super::recover_crashed_publish( + &key_path, + anyhow::anyhow!("synthetic crash signature"), + &before_reparse, + ) + .expect("recovery must not error"); + let recovered_did = match recovery { + super::Recovery::Reloaded(result) => format!( + "{}", + result + .expect("the post-claim re-parse resolves the completed key") + .did() + ), + super::Recovery::Claimed(recovery_claims) => { + // The boot loop's follow-through: regenerate, publish, release. + let kp_b = Keypair::generate(); + let pem_b = kp_b.to_pem().expect("pem b"); + let out = publish_key_atomically( + &key_path, + pem_b.as_bytes(), + &|| {}, + &|| {}, + PublishFaults::NONE, + ) + .expect("recovery's republish"); + assert!(matches!(out, KeyPublish::Won), "recovery's republish wins"); + for c in &recovery_claims { + let _ = std::fs::remove_file(c); + } + format!("{}", kp_b.did()) + } + }; + publisher.join().expect("publisher joins"); + + let p_did = publisher_did + .lock() + .expect("publisher_did lock") + .take() + .expect("the publisher reported its identity"); + let disk_did = format!( + "{}", + super::load_existing_key(&key_path) + .expect("final parses") + .did() + ); + assert_eq!( + p_did, disk_did, + "the publisher must never resolve an identity that diverges from disk" + ); + assert_eq!( + recovered_did, disk_did, + "recovery must converge on the same on-disk identity" + ); + assert!( + names_containing(dir.path(), ".quarantined.").is_empty(), + "a completed publisher's final must not be quarantined" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "no publish markers may remain" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "no recovery claims may remain" + ); + } + + // #194 (F3): a recoverer crashed between its claim rename and the + // quarantine leaves a content-bad final beside a stale `.recovering.` + // claim and ZERO `.publishing.` markers: the claim IS the renamed + // marker. The claim must count toward the crash signature and be + // claimable itself, or no later boot can ever recover: the permanent + // wedge this protocol exists to eliminate. RED before F3: the signature + // counts only `.publishing.` names, so the boot rides the deadline and + // fails loudly forever. + #[test] + fn claim_crash_state_is_recoverable() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + std::fs::write(&key_path, b"").expect("seed crashed final"); + std::fs::set_permissions(&key_path, std::fs::Permissions::from_mode(0o600)) + .expect("0600 crashed final"); + std::fs::write(dir.path().join(".identity.pem.recovering.99999.0"), b"") + .expect("seed stale recovery claim"); + + let kp = load_or_create_keypair(&key_config(&key_path)) + .expect("a claim-crash state must recover, not wedge every later boot"); + + let on_disk = super::load_existing_key(&key_path).expect("regenerated final parses"); + assert_eq!( + format!("{}", on_disk.did()), + format!("{}", kp.did()), + "the returned identity must match the regenerated on-disk key" + ); + let quarantined = names_containing(dir.path(), ".quarantined."); + assert_eq!( + quarantined.len(), + 1, + "exactly one quarantine file must exist: {quarantined:?}" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "the stale claim must be consumed and the fresh claim released" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "no publish markers may remain" + ); + } + + // #194 (grace): the crash signature requires TWO consecutive + // observations. A reader that catches a live fallback publisher's + // mid-write window sees content-bad-plus-marker ONCE; if the state heals + // before the next poll, no CrashSignature may fire, since a single + // observation classifying a crash would let a crash-free boot hijack the + // publisher's round and churn a quarantine. The load-fault hook heals + // the state on the second attempt; the boot must LOAD the healed key. + // RED before the grace: the first observation returns CrashSignature + // immediately. + #[test] + fn mid_write_reader_does_not_hijack_before_grace() { + use std::sync::atomic::{AtomicU32, Ordering}; + use std::sync::OnceLock; + static CALLS: AtomicU32 = AtomicU32::new(0); + static STATE: OnceLock<(std::path::PathBuf, std::path::PathBuf, String)> = OnceLock::new(); + fn heal_on_second_attempt() -> std::io::Result<()> { + if CALLS.fetch_add(1, Ordering::SeqCst) == 1 { + let (marker, final_path, pem) = STATE.get().expect("state seeded"); + std::fs::remove_file(marker)?; + std::fs::write(final_path, pem.as_bytes())?; + } + Ok(()) + } + + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + let marker = dir.path().join(".identity.pem.publishing.99999.0"); + let pem = Keypair::generate().to_pem().expect("pem"); + let expected = format!("{}", Keypair::from_pem(&pem).expect("fixture parses").did()); + STATE + .set((marker, key_path.clone(), pem.to_string())) + .expect("state seeded once"); + + match super::boot_load_key(&key_path, true, heal_on_second_attempt) { + super::BootLoad::Loaded(kp) => { + assert_eq!( + format!("{}", kp.did()), + expected, + "the healed key must be the one loaded" + ); + } + super::BootLoad::CrashSignature(e) => { + panic!("a single mid-write observation must not classify a crash: {e:#}") + } + super::BootLoad::Failed(e) => panic!("the healed key must load: {e:#}"), + } + } + + // #194 (F2, window-1): a publisher that completes between recovery's + // claim step and its re-parse must win the round retroactively. The + // post-claim re-parse sees the completed key, so recovery releases its + // claims and aborts to Reloaded: no quarantine of a good key, no claim + // left behind, the completed identity returned. RED before F2: no + // re-parse exists, so recovery quarantines the just-completed key and + // regenerates over it. + #[test] + fn post_claim_reparse_aborts_to_reload() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + let completed = Keypair::generate(); + let pem = completed.to_pem().expect("pem"); + let heal_path = key_path.clone(); + let before_reparse = move || { + std::fs::write(&heal_path, pem.as_bytes()) + .expect("complete the final inside the claim window"); + }; + let seam = RecoverySeam { + before_reparse: &before_reparse, + ..RecoverySeam::NONE + }; + + let kp = load_or_create_keypair_with(&key_path, &|| {}, &|| {}, PublishFaults::NONE, seam) + .expect("recovery must abort to the completed key"); + + assert_eq!( + format!("{}", kp.did()), + format!("{}", completed.did()), + "the identity completed inside the window must be the one returned" + ); + assert!( + names_containing(dir.path(), ".quarantined.").is_empty(), + "a key completed before the re-parse must never be quarantined" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "the aborting recovery must release its claims" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "the claimed markers must not reappear" + ); + } } #[cfg(test)] From a713d45853dc87e9e64722f9fbec0e18c4fcae78 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 21 Jul 2026 15:46:57 -0500 Subject: [PATCH 29/41] test(node): cover the quarantine-exhaustion arm and pin recovery latency Add the missing coverage on recover_crashed_publish's degraded arms: quarantine-name exhaustion must fall back to removing the bad final and still regenerate, and a blocked removal must surface the chained error loudly, with the leftover claim proven claimable on the next boot. Route all marker-class name construction through publishing_prefix and recovering_prefix as the single source of truth, state the dir-fsync degradation caveat in publish_key_atomically's guarantee prose, and pin the crash-signature short-circuit with a timing assertion so recovery never silently regresses to riding the full race deadline. --- crates/gitlawb-node/src/lib.rs | 184 ++++++++++++++++++++++++++++++--- 1 file changed, 168 insertions(+), 16 deletions(-) diff --git a/crates/gitlawb-node/src/lib.rs b/crates/gitlawb-node/src/lib.rs index bce086e5..1441e296 100644 --- a/crates/gitlawb-node/src/lib.rs +++ b/crates/gitlawb-node/src/lib.rs @@ -1212,6 +1212,20 @@ fn load_existing_key(path: &std::path::Path) -> Result { Ok(kp) } +/// `".{stem}.publishing."`: the single source of truth for the publish +/// marker name class (#194, U1). Every production site that constructs a +/// marker name or prefix goes through here; the test module keeps raw +/// literals on purpose, as a guard against prefix drift. +fn publishing_prefix(stem: &str) -> String { + format!(".{stem}.publishing.") +} + +/// `".{stem}.recovering."`: the single source of truth for the recovery +/// claim name class (#194, F3). Same policy as `publishing_prefix`. +fn recovering_prefix(stem: &str) -> String { + format!(".{stem}.recovering.") +} + /// The publish markers (`.{stem}.publishing.*`) beside `final_path`, sorted: /// U1's crash signature for a fallback publish interrupted mid-window (#194, /// U2). An unreadable directory reads as no markers, so recovery stays off @@ -1226,7 +1240,7 @@ fn list_publish_markers(final_path: &std::path::Path) -> Vec .file_name() .and_then(|s| s.to_str()) .unwrap_or("identity.pem"); - let prefix = format!(".{stem}.publishing."); + let prefix = publishing_prefix(stem); let Ok(entries) = std::fs::read_dir(dir) else { return Vec::new(); }; @@ -1260,8 +1274,8 @@ fn list_recovery_claimables(final_path: &std::path::Path) -> Vec std::io .file_name() .and_then(|s| s.to_str()) .unwrap_or("identity.pem"); - let publishing = format!(".{stem}.publishing."); - let recovering = format!(".{stem}.recovering."); + let publishing = publishing_prefix(stem); + let recovering = recovering_prefix(stem); let Ok(entries) = std::fs::read_dir(dir) else { return; }; @@ -1544,7 +1558,12 @@ impl PublishFaults { /// final. Guarantee (c) now holds on the fallback too, via the marker /// protocol (#194, U2/U4): a crash inside the non-atomic window leaves /// marker+final together, and the next boot claims the marker, quarantines -/// the partial final, and regenerates instead of wedging. Honest residual on +/// the partial final, and regenerates instead of wedging. On +/// dir-fsync-hostile mounts the marker dir fsync is warn-and-continue (see +/// `publish_key_fallback`), so there the recovery guarantee narrows to +/// non-power-loss (SIGKILL-class) crashes: a power loss can drop the +/// undurable marker name and leave the partial final to the loud +/// invalid-PEM error instead. Honest residual on /// top of the U3 list below: that recovery runs at the next boot, not at /// crash time, so the partial final persists until then. In every unrecovered /// case the failure mode is the same LOUD invalid-PEM (or mode) error at the @@ -1722,12 +1741,10 @@ fn publish_key_fallback( // the restart, and a stale marker is indistinguishable from a live // concurrent publisher's, so never delete a name this call did not // create; just skip to the next one (#194, U1). + let marker_prefix = publishing_prefix(stem); let mut marker = None; for attempt in 0..KEY_TEMP_ATTEMPTS { - let candidate = dir.join(format!( - ".{stem}.publishing.{}.{attempt}", - std::process::id() - )); + let candidate = dir.join(format!("{marker_prefix}{}.{attempt}", std::process::id())); match (faults.marker_create)().and_then(|()| { std::fs::OpenOptions::new() .write(true) @@ -1752,7 +1769,7 @@ fn publish_key_fallback( } let Some(marker) = marker else { anyhow::bail!( - "all {KEY_TEMP_ATTEMPTS} publish marker names .{stem}.publishing.{}.* in {} are \ + "all {KEY_TEMP_ATTEMPTS} publish marker names {marker_prefix}{}.* in {} are \ taken; remove the stale marker files and restart (link failed: {link_err})", std::process::id(), dir.display() @@ -2032,14 +2049,12 @@ fn recover_crashed_publish( // skipped. Names are never reused or clobbered: the monotonic counter // plus the exists() probe keeps every destination fresh, bounded per // item like the temp/marker/quarantine probes. + let claim_prefix = recovering_prefix(stem); let mut claims: Vec = Vec::new(); let mut next_name = 0u32; for item in &claimables { for _ in 0..KEY_TEMP_ATTEMPTS { - let dest = dir.join(format!( - ".{stem}.recovering.{}.{next_name}", - std::process::id() - )); + let dest = dir.join(format!("{claim_prefix}{}.{next_name}", std::process::id())); next_name = next_name.wrapping_add(1); if dest.exists() { continue; @@ -3578,8 +3593,18 @@ mod identity_key_tests { let dir = tempfile::tempdir().expect("tempdir"); let key_path = seed_crash_state(dir.path(), b""); + let started = std::time::Instant::now(); let kp = load_or_create_keypair(&key_config(&key_path)) .expect("a crash-marked empty final must recover, not wedge the start"); + let elapsed = started.elapsed(); + // The crash signature short-circuits after two ~2ms polls; 2s is + // generous for CI machines yet far under the 5s KEY_RACE_DEADLINE a + // deadline-riding load would burn before recovery could even start. + assert!( + elapsed < std::time::Duration::from_secs(2), + "recovery must short-circuit on the crash signature, not ride \ + KEY_RACE_DEADLINE: took {elapsed:?}" + ); let mode = std::fs::metadata(&key_path) .expect("regenerated final exists") @@ -3645,6 +3670,133 @@ mod identity_key_tests { ); } + // #194 (U2): quarantine-name probing is bounded like the temp and marker + // names. With EVERY candidate quarantine name taken, recovery must fall + // back to REMOVING the corrupt final (not rename it, not clobber a stale + // quarantine) and still regenerate successfully. Sentinel bytes in each + // stale quarantine prove no rename landed. Load-bearing by mutation + // check: with the remove_file fallback disabled, the corrupt final + // survives, the retry publish loses to it, and the boot fails loudly. + #[test] + fn quarantine_exhaustion_falls_back_to_removal() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + for n in 0..KEY_TEMP_ATTEMPTS { + let stale = dir.path().join(format!( + ".identity.pem.quarantined.{}.{n}", + std::process::id() + )); + std::fs::write(&stale, b"sentinel").expect("seed stale quarantine"); + } + + let kp = load_or_create_keypair(&key_config(&key_path)) + .expect("an exhausted quarantine probe must fall back to removal, not wedge"); + + let on_disk = super::load_existing_key(&key_path).expect("regenerated final parses"); + assert_eq!( + format!("{}", on_disk.did()), + format!("{}", kp.did()), + "the returned identity must match the regenerated on-disk key" + ); + let quarantined = names_containing(dir.path(), ".quarantined."); + assert_eq!( + quarantined.len(), + KEY_TEMP_ATTEMPTS as usize, + "the final must be removed, not quarantined under a fresh name: {quarantined:?}" + ); + for name in &quarantined { + let bytes = std::fs::read(dir.path().join(name)).expect("read stale quarantine"); + assert_eq!( + bytes, b"sentinel", + "no stale quarantine may be clobbered by a rename: {name}" + ); + } + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "recovery must consume the publish marker(s)" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "the recovery claim must be released after the publish resolves" + ); + } + + // #194 (U2/F1): with every quarantine name taken AND removal blocked, + // recovery must surface the loud chained error, never a silent success. + // The dir turns read-only via the before_reparse seam (after the claim + // rename lands; a dir made read-only up front would fail the claim step + // and never reach the quarantine arm at all). The same read-only dir + // necessarily blocks the claim release too, so phase two restores + // permissions and proves the leftover claim is itself recoverable (F3): + // the next boot claims it, removes the final, and regenerates cleanly. + #[test] + fn quarantine_and_removal_both_blocked_errors_loudly() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + for n in 0..KEY_TEMP_ATTEMPTS { + let stale = dir.path().join(format!( + ".identity.pem.quarantined.{}.{n}", + std::process::id() + )); + std::fs::write(&stale, b"sentinel").expect("seed stale quarantine"); + } + let lock_path = dir.path().to_path_buf(); + let lock_dir = move || { + std::fs::set_permissions(&lock_path, std::fs::Permissions::from_mode(0o555)) + .expect("read-only key dir inside the claim window"); + }; + let seam = RecoverySeam { + before_reparse: &lock_dir, + ..RecoverySeam::NONE + }; + + let result = + load_or_create_keypair_with(&key_path, &|| {}, &|| {}, PublishFaults::NONE, seam); + + std::fs::set_permissions(dir.path(), std::fs::Permissions::from_mode(0o755)) + .expect("restore key dir"); + let err = match result { + Err(e) => e, + Ok(_) => panic!("a blocked quarantine AND removal must fail loudly, not resolve"), + }; + let msg = format!("{err:#}"); + assert!( + msg.contains("could not quarantine or remove") + && msg.contains(&key_path.display().to_string()) + && msg.contains("invalid PEM key"), + "the error must chain the recovery failure and the load failure: {msg}" + ); + assert!( + key_path.exists(), + "the corrupt final must survive for post-mortem when nothing can dispose of it" + ); + + // Phase two: the claim left behind (the read-only dir blocked its + // release too) must make the state recoverable, not wedge it. + let kp = load_or_create_keypair(&key_config(&key_path)) + .expect("the leftover claim must be claimable by the next boot, not a wedge"); + let on_disk = super::load_existing_key(&key_path).expect("regenerated final parses"); + assert_eq!( + format!("{}", on_disk.did()), + format!("{}", kp.did()), + "the returned identity must match the regenerated on-disk key" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "the follow-up boot must consume and release every claim" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "no publish markers may remain" + ); + let quarantined = names_containing(dir.path(), ".quarantined."); + assert_eq!( + quarantined.len(), + KEY_TEMP_ATTEMPTS as usize, + "the follow-up boot must also fall back to removal: {quarantined:?}" + ); + } + // MUST-NOT (#194, U2): an unparseable final WITHOUT a marker is operator // corruption, not a crash signature; it must keep today's loud fail // bit-for-bit: same `invalid PEM key` message, final left on disk with From c1d1532ae6f49edbc1ed62c0aadf533c2ff409f9 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 21 Jul 2026 16:47:25 -0500 Subject: [PATCH 30/41] fix(node): protect live recoveries from the sweep and restore hijacked keys Second-model review findings. The healthy-boot sweep now age-gates recovering claims (2x the race deadline; a live claim is always younger), because sweeping a live recovery's claim cancelled a demoted publisher's clearance wait and reopened two-sided divergence; publishing markers keep the unconditional sweep since stealing one only forces a safe demotion. The post-claim re-parse quarantines only on content-class failures (a transient error there releases the claims and stays loud), and a key completed inside the re-parse-to-quarantine window is detected by parsing the quarantined bytes and restored via no-clobber link instead of being discarded. The crash signature must now persist for 250ms of consecutive observations, above realistic mid-write latency on shared volumes, so concurrent starts stop hijacking live publishes. --- crates/gitlawb-node/src/lib.rs | 455 +++++++++++++++++++++++++++++---- 1 file changed, 410 insertions(+), 45 deletions(-) diff --git a/crates/gitlawb-node/src/lib.rs b/crates/gitlawb-node/src/lib.rs index 1441e296..965f58ee 100644 --- a/crates/gitlawb-node/src/lib.rs +++ b/crates/gitlawb-node/src/lib.rs @@ -1148,6 +1148,31 @@ enum KeyPublish { /// losing start after an arbitrarily short (~100ms) window (#194). const KEY_RACE_DEADLINE: std::time::Duration = std::time::Duration::from_secs(5); +/// Minimum age (metadata mtime) before a `.recovering.` claim is sweepable +/// by a healthy boot (#194, G1). A LIVE recoverer's claims exist only within +/// one bounded recovery round (claim, quarantine, republish, release, itself +/// bounded by `KEY_RACE_DEADLINE`), so anything older is orphaned by a +/// crashed recovery; the 2x margin absorbs clock skew between writers on +/// network mounts. The gate matters because live claims are load-bearing: a +/// demoted publisher's `wait_for_recovery_claims` waits on them, and a third +/// start sweeping a live claim would cancel that wait mid-recovery, letting +/// the publisher re-load the pre-quarantine key while the recoverer +/// quarantines and republishes: two-sided divergence. An unreadable mtime is +/// treated as YOUNG (skip removal; fail safe). +const CLAIM_SWEEP_MIN_AGE: std::time::Duration = KEY_RACE_DEADLINE.saturating_mul(2); + +/// How long the crash signature (a content-bad final beside a claimable) +/// must persist across consecutive observations before boot classifies a +/// crash (#194, G3). A single ~2ms poll sits well inside a LIVE fallback +/// publisher's mid-write window once chmod/stat/scheduling latency on shared +/// volumes is real, so a shorter grace lets a concurrent starter classify a +/// live publish as a crash and churn quarantines and identities. 250ms is +/// far above that latency yet far under the <2s recovery budget the tests +/// pin (250ms of persistence plus claim, quarantine, and regenerate stays +/// well under 2s). Any successful load or signature-free observation resets +/// the clock. +const CRASH_SIGNATURE_MIN_PERSIST: std::time::Duration = std::time::Duration::from_millis(250); + /// Bounded number of temp-file names a publish will try before giving up. /// Attempt 0 is the deterministic `.{stem}.tmp.{pid}.0`, so a stale temp left /// by a crashed prior start with the same PID (a restarted container is @@ -1312,8 +1337,9 @@ enum BootLoad { /// large_enum_variant); the enum lives only across a boot-time match. Loaded(Box), /// Content-class failure observed while a `.publishing.` marker or a - /// `.recovering.` claim was present, on two consecutive polls; returned - /// without riding the rest of the deadline. + /// `.recovering.` claim was present, continuously for + /// `CRASH_SIGNATURE_MIN_PERSIST`; returned without riding the rest of + /// the deadline. CrashSignature(anyhow::Error), /// Any other failure, after the full deadline. Failed(anyhow::Error), @@ -1321,19 +1347,22 @@ enum BootLoad { /// `load_racing_key` for the boot loop (#194, U2). With `crash_exit` set, a /// content-class failure (`KeyContentError`) observed while a `.publishing.` -/// marker or a `.recovering.` claim is present, on TWO consecutive polls, -/// returns `CrashSignature` instead of riding the deadline: claimable plus +/// marker or a `.recovering.` claim is present, continuously for +/// `CRASH_SIGNATURE_MIN_PERSIST` of consecutive observations, returns +/// `CrashSignature` instead of riding the deadline: claimable plus /// unreadable content is the crash signature recovery exists for, and the /// caller's atomic claim renames arbitrate the (rare) race against a live /// fallback publisher still inside its window. Claims count toward the -/// signature for F3 (see `list_recovery_claimables`). The two-poll grace -/// exists so a reader that catches a live publisher's mid-write window is -/// not instantly classified as a crash: one ~2ms iteration is enough for -/// most writes to complete, and a real crash state persists across every -/// poll. Every other failure keeps the full wall-clock deadline. -/// `load_fault` is the injection hook (a no-op `Ok` in production); an -/// injected `Err` is taken as that attempt's result and is NOT -/// content-class. +/// signature for F3 (see `list_recovery_claimables`). The persistence window +/// (#194, G3) exists so a reader that catches a live publisher's mid-write +/// window is not classified as a crash: chmod/stat/scheduling latency on +/// shared volumes routinely stretches that window past any small fixed poll +/// count, while a real crash state persists across every poll, so the +/// signature clock starts at its first observation and any successful load +/// or signature-free observation resets it. Every other failure keeps the +/// full wall-clock deadline. `load_fault` is the injection hook (a no-op +/// `Ok` in production); an injected `Err` is taken as that attempt's result +/// and is NOT content-class. fn boot_load_key( path: &std::path::Path, crash_exit: bool, @@ -1341,7 +1370,7 @@ fn boot_load_key( ) -> BootLoad { let deadline = std::time::Instant::now() + KEY_RACE_DEADLINE; let mut last_err; - let mut signature_seen = false; + let mut signature_since: Option = None; loop { match load_fault() .map_err(anyhow::Error::new) @@ -1353,10 +1382,14 @@ fn boot_load_key( let signature = crash_exit && is_key_content_error(&last_err) && !list_recovery_claimables(path).is_empty(); - if signature && signature_seen { - return BootLoad::CrashSignature(last_err); + if signature { + let since = *signature_since.get_or_insert_with(std::time::Instant::now); + if since.elapsed() >= CRASH_SIGNATURE_MIN_PERSIST { + return BootLoad::CrashSignature(last_err); + } + } else { + signature_since = None; } - signature_seen = signature; if std::time::Instant::now() >= deadline { return BootLoad::Failed(last_err); } @@ -1433,6 +1466,17 @@ fn wait_for_recovery_claims(final_path: &std::path::Path) { /// vouched for is durable by the sweeper's own hand. `sweep_sync` is the /// `RecoverySeam` test hook, run before the final's `sync_all` with an `Err` /// taken as that sync's result; production passes a no-op `Ok`. +/// +/// SWEEP ASYMMETRY (#194, G1): `.publishing.` markers are removed +/// unconditionally, but `.recovering.` claims only once older than +/// `CLAIM_SWEEP_MIN_AGE`. Sweeping a LIVE publisher's marker is safe (it +/// only fails that publisher's commit check, forcing a safe +/// demotion-and-reload), while sweeping a LIVE recoverer's claim is not: a +/// demoted publisher's `wait_for_recovery_claims` waits on that claim until +/// the recovery resolves, and removing it early lets the publisher re-load +/// the pre-quarantine key while the recoverer quarantines and republishes: +/// two-sided divergence. An unreadable claim mtime reads as young, so the +/// claim is kept (fail safe; the next boot past the age floor sweeps it). fn sweep_stale_markers(final_path: &std::path::Path, sweep_sync: fn() -> std::io::Result<()>) { let durable = std::fs::File::open(final_path) .and_then(|f| sweep_sync().and_then(|()| f.sync_all())) @@ -1463,8 +1507,20 @@ fn sweep_stale_markers(final_path: &std::path::Path, sweep_sync: fn() -> std::io for entry in entries.filter_map(|e| e.ok()) { let name = entry.file_name(); let Some(name) = name.to_str() else { continue }; - if name.starts_with(&publishing) || name.starts_with(&recovering) { + if name.starts_with(&publishing) { let _ = std::fs::remove_file(entry.path()); + } else if name.starts_with(&recovering) { + // G1 age gate: only an orphaned claim (older than + // CLAIM_SWEEP_MIN_AGE) is sweepable; see the doc comment. + let orphaned = entry + .metadata() + .and_then(|m| m.modified()) + .ok() + .and_then(|mtime| std::time::SystemTime::now().duration_since(mtime).ok()) + .is_some_and(|age| age >= CLAIM_SWEEP_MIN_AGE); + if orphaned { + let _ = std::fs::remove_file(entry.path()); + } } } #[cfg(unix)] @@ -1944,6 +2000,17 @@ struct RecoverySeam<'a> { /// the post-claim re-parse of the final, so a test can interleave a /// concurrent publisher's resolution into exactly that window (#194, F2). before_reparse: &'a dyn Fn(), + /// Runs immediately before the post-claim RE-parse load inside + /// `recover_crashed_publish`; an `Err` is taken as that load's result. + /// The injected failure is NOT content-class, so it deterministically + /// exercises the transient-reparse (release-and-fail-loudly) policy + /// (#194, G2a). Narrower than `load_fault`, which only reaches + /// `boot_load_key`'s attempts, never this re-parse. + reparse_fault: fn() -> std::io::Result<()>, + /// Runs inside `recover_crashed_publish` between the post-claim re-parse + /// and the quarantine rename, so a test can land a publisher's completed + /// write in exactly the re-parse-to-rename window (#194, G2b). + before_quarantine: &'a dyn Fn(), } impl RecoverySeam<'_> { @@ -1953,6 +2020,8 @@ impl RecoverySeam<'_> { before_publish: &|| {}, sweep_sync: || Ok(()), before_reparse: &|| {}, + reparse_fault: || Ok(()), + before_quarantine: &|| {}, }; } @@ -2016,19 +2085,25 @@ fn release_recovery_claims(final_path: &std::path::Path, claims: &[std::path::Pa /// owns the round, and it RE-PARSES the final once before touching it: a /// publisher that completed inside the claim window has already resolved the /// round, so a parseable final is returned as `Reloaded` and the claims are -/// released, never quarantining a good key. Only if the final still -/// content-fails does the recovery quarantine it by rename (preserving bytes -/// and mode for post-mortem; falling back to removal only if every -/// quarantine name fails), consume any remaining markers best-effort, and -/// hand control back to regenerate. Every error exit after claiming releases +/// released, never quarantining a good key. A TRANSIENT (non-content) +/// re-parse failure releases the claims and surfaces the error loudly with +/// nothing destroyed (#194, G2a). Only if the final still content-fails does +/// the recovery quarantine it by rename (preserving bytes and mode for +/// post-mortem; falling back to removal only if every quarantine name +/// fails); a quarantine whose bytes then PARSE is a publisher that completed +/// in the re-parse-to-rename window and is restored to the final name +/// (#194, G2b; see the inline comment). Otherwise recovery consumes any +/// remaining markers best-effort and hands control back to regenerate. +/// Every error exit after claiming releases /// the claims first (see `release_recovery_claims`; a leaked claim would -/// stall later demoted publishers on the full deadline, F1). `before_reparse` -/// is the test seam for the claim-to-re-parse window (a no-op in -/// production). +/// stall later demoted publishers on the full deadline, F1). `seam` carries +/// the test hooks for the claim-to-re-parse and re-parse-to-quarantine +/// windows (`before_reparse`, `reparse_fault`, `before_quarantine`; all +/// no-ops in production). fn recover_crashed_publish( final_path: &std::path::Path, load_err: anyhow::Error, - before_reparse: &dyn Fn(), + seam: RecoverySeam<'_>, ) -> Result { let dir = final_path .parent() @@ -2083,18 +2158,50 @@ fn recover_crashed_publish( // its final. Now that the round is claimed no publisher can commit // anymore, so one parse decides: a readable final means the round is // already resolved, so release the claims and defer to it, never - // quarantining a good key. - (before_reparse)(); - if let Ok(kp) = load_existing_key(final_path) { - release_recovery_claims(final_path, &claims); - return Ok(Recovery::Reloaded(Box::new(Ok(kp)))); + // quarantining a good key. Only a CONTENT-class failure proceeds to the + // quarantine (#194, G2a): a transient (IO/permissions) failure of a + // PRESENT final proves nothing about the content, and quarantining on it + // would destroy a possibly good key, so the claims are released and the + // error surfaces loudly with nothing destroyed. A NotFound is neither: a + // vanished final means a competing recoverer disposed of it + // concurrently, there is nothing a quarantine could destroy, and the + // quarantine loop's own NotFound arm resolves it into a clean + // regenerate, so it falls through like the content class. + (seam.before_reparse)(); + match (seam.reparse_fault)() + .map_err(anyhow::Error::new) + .and_then(|()| load_existing_key(final_path)) + { + Ok(kp) => { + release_recovery_claims(final_path, &claims); + return Ok(Recovery::Reloaded(Box::new(Ok(kp)))); + } + Err(e) + if is_key_content_error(&e) + || e.downcast_ref::() + .is_some_and(|io| io.kind() == std::io::ErrorKind::NotFound) => + { + // Still the crashed content, or vanished concurrently: fall + // through to the quarantine. + } + Err(e) => { + release_recovery_claims(final_path, &claims); + return Err(e.context(format!( + "transient failure re-parsing identity key {} after claiming crash \ + recovery; claims released, nothing quarantined", + final_path.display() + ))); + } } // QUARANTINE the final under a bounded name probe (the KEY_TEMP_ATTEMPTS // discipline: a stale quarantine from a crashed same-PID start must not // wedge this one). The warn precedes the move and carries stable // operator-greppable fields; the destination it names is the first free - // candidate, which the rename below tries first. + // candidate, which the rename below tries first. `before_quarantine` is + // the test seam for the re-parse-to-rename window (#194, G2b; a no-op in + // production). + (seam.before_quarantine)(); let dest_for = |n: u32| dir.join(format!(".{stem}.quarantined.{}.{n}", std::process::id())); let start = (0..KEY_TEMP_ATTEMPTS) .find(|&n| !dest_for(n).exists()) @@ -2108,6 +2215,7 @@ fn recover_crashed_publish( regenerating the identity" ); let mut quarantined = false; + let mut quarantined_at: Option = None; for n in start..KEY_TEMP_ATTEMPTS { let dest = dest_for(n); if dest.exists() { @@ -2116,6 +2224,7 @@ fn recover_crashed_publish( match std::fs::rename(final_path, &dest) { Ok(()) => { quarantined = true; + quarantined_at = Some(dest); break; } Err(e) if e.kind() == std::io::ErrorKind::NotFound => { @@ -2144,6 +2253,49 @@ fn recover_crashed_publish( } } + // POST-QUARANTINE COMPLETED-WRITE CHECK (#194, G2b): the re-parse above + // and the quarantine rename are not atomic, so a publisher that + // completed its write inside the re-parse-to-rename window has its + // VALID key swept into the quarantine. The rename preserved the bytes + // (and the 0600 inode), so parse them: a parseable quarantine means that + // publisher already resolved the round, and its key is restored to the + // final name instead of being replaced by a regeneration, which would + // diverge from the Won the publisher already reported. The restore is a + // no-clobber hard_link plus unlink of the quarantine name, not a rename: + // the final name is expected free (our own rename freed it and the round + // is claimed), but a fresh start that saw no final may have republished + // it; AlreadyExists leaves the quarantine behind as forensics and the + // follow-up load defers to the republished key. On link-hostile mounts + // (where hard_link cannot work) a plain rename restores instead, + // mirroring publish_key_fallback's weakened no-clobber on the same + // mounts; if even that fails the quarantine stays as forensics and the + // follow-up load fails loudly, nothing destroyed. An unparseable + // quarantine is the expected crash state and falls through to + // regenerate. + if let Some(quarantine) = &quarantined_at { + let completed = + std::fs::read_to_string(quarantine).is_ok_and(|pem| Keypair::from_pem(&pem).is_ok()); + if completed { + match std::fs::hard_link(quarantine, final_path) { + Ok(()) => { + let _ = std::fs::remove_file(quarantine); + } + Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { + // Someone republished the final concurrently; keep it. + } + Err(_) => { + let _ = std::fs::rename(quarantine, final_path); + } + } + #[cfg(unix)] + { + let _ = fsync_parent_dir(final_path); + } + release_recovery_claims(final_path, &claims); + return Ok(Recovery::Reloaded(Box::new(load_racing_key(final_path)))); + } + } + // Consume any remaining markers (a multi-crash pile-up, or an item the // claim probe could not rename) best-effort, and best-effort fsync the // directory so the transition tends to be durable. @@ -2165,8 +2317,8 @@ fn recover_crashed_publish( /// Load-or-generate with AT MOST ONE crash recovery per start: a load /// failure recovers (claim the round, quarantine the final, regenerate) /// only when it is content-class (`KeyContentError`), a `.publishing.` -/// marker or `.recovering.` claim exists (the crash signature, observed on -/// two consecutive polls), and no recovery has run yet this start. Every +/// marker or `.recovering.` claim exists (the crash signature, persisting +/// for `CRASH_SIGNATURE_MIN_PERSIST`), and no recovery has run yet this start. Every /// other failure surfaces unchanged. The Lost arm participates in the same /// policy; a recovery fired there loops back for one retry publish, and any /// failure on that second pass surfaces unchanged. @@ -2203,7 +2355,7 @@ fn load_or_create_keypair_with( // Only emitted while recovery is still allowed (crash_exit is // gated on it). BootLoad::CrashSignature(e) => { - match recover_crashed_publish(key_path, e, seam.before_reparse)? { + match recover_crashed_publish(key_path, e, seam)? { Recovery::Claimed(c) => { claims = c; recovery_allowed = false; @@ -2271,7 +2423,7 @@ fn load_or_create_keypair_with( // Only emitted while recovery is still allowed (crash_exit // is gated on it). BootLoad::CrashSignature(e) => { - match recover_crashed_publish(key_path, e, seam.before_reparse) { + match recover_crashed_publish(key_path, e, seam) { Ok(Recovery::Claimed(c)) => { claims = c; recovery_allowed = false; @@ -3563,6 +3715,19 @@ mod identity_key_tests { key_path } + /// Set `path`'s mtime an hour into the past, far beyond + /// `CLAIM_SWEEP_MIN_AGE`, via GNU `touch -d` (std::fs cannot set mtimes; + /// spawning coreutils is test-only). + fn age_beyond_claim_sweep(path: &std::path::Path) { + let status = std::process::Command::new("touch") + .arg("-d") + .arg("1 hour ago") + .arg(path) + .status() + .expect("run touch -d"); + assert!(status.success(), "touch -d must succeed to age {path:?}"); + } + /// File names in `dir` containing `needle` (marker/claim/quarantine sweeps). fn names_containing(dir: &std::path::Path, needle: &str) -> Vec { let mut names: Vec = std::fs::read_dir(dir) @@ -4206,7 +4371,7 @@ mod identity_key_tests { let recovery = super::recover_crashed_publish( &key_path, anyhow::anyhow!("synthetic post-commit crash signature"), - &|| {}, + RecoverySeam::NONE, ) .expect("a late recovery must not error"); let loaded = match recovery { @@ -4286,8 +4451,10 @@ mod identity_key_tests { // #194 (U4): a HEALTHY load must sweep stale `.publishing.` markers and // `.recovering.` claims, or they linger to misclassify a much-later real - // corruption of a good key as an interrupted first write. RED before U4: - // all three files survive the load. + // corruption of a good key as an interrupted first write. The claim is + // AGED past CLAIM_SWEEP_MIN_AGE: only orphaned claims are sweepable + // (#194, G1; healthy_load_spares_young_claims covers the young side). + // RED before U4: all three files survive the load. #[test] fn healthy_load_sweeps_stale_markers_and_claims() { let dir = tempfile::tempdir().expect("tempdir"); @@ -4296,8 +4463,9 @@ mod identity_key_tests { let key_path = seed_crash_state(dir.path(), pem.as_bytes()); std::fs::write(dir.path().join(".identity.pem.publishing.88888.1"), b"") .expect("seed second stale marker"); - std::fs::write(dir.path().join(".identity.pem.recovering.77777"), b"") - .expect("seed stale recovery claim"); + let claim = dir.path().join(".identity.pem.recovering.77777"); + std::fs::write(&claim, b"").expect("seed stale recovery claim"); + age_beyond_claim_sweep(&claim); let kp = load_or_create_keypair(&key_config(&key_path)) .expect("a valid final with stale markers must load"); @@ -4317,6 +4485,51 @@ mod identity_key_tests { ); } + // #194 (G1) MUST-NOT: the healthy-boot sweep must SPARE a young + // `.recovering.` claim. A live recoverer's claims exist only within one + // bounded recovery round, and a demoted publisher's + // wait_for_recovery_claims waits on them; a third start sweeping a live + // claim cancels that wait early, so the publisher re-loads the + // pre-quarantine key while the recoverer proceeds to quarantine and + // republish: two-sided divergence. Only a claim older than + // CLAIM_SWEEP_MIN_AGE (an orphan of a crashed recovery) may go. + // `.publishing.` markers keep the unconditional sweep: sweeping a live + // publisher's marker only fails its commit check, a safe demotion. RED + // before G1: the young claim is swept too. + #[test] + fn healthy_load_spares_young_claims() { + let dir = tempfile::tempdir().expect("tempdir"); + let existing = Keypair::generate(); + let pem = existing.to_pem().expect("pem"); + let key_path = seed_crash_state(dir.path(), pem.as_bytes()); + let young = dir.path().join(".identity.pem.recovering.11111"); + std::fs::write(&young, b"").expect("seed young (possibly live) claim"); + let old = dir.path().join(".identity.pem.recovering.22222"); + std::fs::write(&old, b"").expect("seed orphaned claim"); + age_beyond_claim_sweep(&old); + + let kp = load_or_create_keypair(&key_config(&key_path)) + .expect("a valid final must load regardless of claim ages"); + + assert_eq!( + format!("{}", kp.did()), + format!("{}", existing.did()), + "the pre-existing identity must be preserved" + ); + assert!( + young.exists(), + "a young claim may belong to a LIVE recovery and must survive the sweep" + ); + assert!( + !old.exists(), + "an orphaned claim past CLAIM_SWEEP_MIN_AGE must be swept" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "publish markers keep the unconditional sweep" + ); + } + // #194 (U4): the generate path must sweep too. A crash between the // marker's durability fsync and the final's `create_new` leaves a durable // marker with NO final; only the Won arm of a later boot can ever clean @@ -4513,7 +4726,10 @@ mod identity_key_tests { let recovery = super::recover_crashed_publish( &key_path, anyhow::anyhow!("synthetic crash signature"), - &before_reparse, + RecoverySeam { + before_reparse: &before_reparse, + ..RecoverySeam::NONE + }, ) .expect("recovery must not error"); let recovered_did = match recovery { @@ -4620,15 +4836,17 @@ mod identity_key_tests { ); } - // #194 (grace): the crash signature requires TWO consecutive - // observations. A reader that catches a live fallback publisher's + // #194 (grace): the crash signature must persist beyond a single + // observation. A reader that catches a live fallback publisher's // mid-write window sees content-bad-plus-marker ONCE; if the state heals // before the next poll, no CrashSignature may fire, since a single // observation classifying a crash would let a crash-free boot hijack the // publisher's round and churn a quarantine. The load-fault hook heals // the state on the second attempt; the boot must LOAD the healed key. // RED before the grace: the first observation returns CrashSignature - // immediately. + // immediately. (G3 widened the grace from two polls to a + // CRASH_SIGNATURE_MIN_PERSIST window; signature_requires_sustained_observation + // covers the widened floor.) #[test] fn mid_write_reader_does_not_hijack_before_grace() { use std::sync::atomic::{AtomicU32, Ordering}; @@ -4668,6 +4886,56 @@ mod identity_key_tests { } } + // #194 (G3): the crash signature must persist for + // CRASH_SIGNATURE_MIN_PERSIST of consecutive observations before boot + // classifies a crash. A few-millisecond glimpse of claimable+content-bad + // is exactly what a reader sees inside a LIVE publisher's mid-write + // window once chmod/stat/scheduling latency on shared volumes exceeds + // one ~2ms poll, and classifying on it hijacks a live round into a + // quarantine churn. Construction: the crash state persists for THREE + // observations (~6ms, far under the persistence floor) and then heals; + // boot must LOAD the healed key. RED before G3: the two-poll grace fires + // CrashSignature on the second observation, before the heal. + #[test] + fn signature_requires_sustained_observation() { + use std::sync::atomic::{AtomicU32, Ordering}; + use std::sync::OnceLock; + static CALLS: AtomicU32 = AtomicU32::new(0); + static STATE: OnceLock<(std::path::PathBuf, std::path::PathBuf, String)> = OnceLock::new(); + fn heal_before_persistence() -> std::io::Result<()> { + if CALLS.fetch_add(1, Ordering::SeqCst) == 3 { + let (marker, final_path, pem) = STATE.get().expect("state seeded"); + std::fs::remove_file(marker)?; + std::fs::write(final_path, pem.as_bytes())?; + } + Ok(()) + } + + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + let marker = dir.path().join(".identity.pem.publishing.99999.0"); + let pem = Keypair::generate().to_pem().expect("pem"); + let expected = format!("{}", Keypair::from_pem(&pem).expect("fixture parses").did()); + STATE + .set((marker, key_path.clone(), pem.to_string())) + .expect("state seeded once"); + + match super::boot_load_key(&key_path, true, heal_before_persistence) { + super::BootLoad::Loaded(kp) => { + assert_eq!( + format!("{}", kp.did()), + expected, + "the healed key must be the one loaded" + ); + } + super::BootLoad::CrashSignature(e) => panic!( + "a signature cleared inside the persistence window must not classify \ + a crash: {e:#}" + ), + super::BootLoad::Failed(e) => panic!("the healed key must load: {e:#}"), + } + } + // #194 (F2, window-1): a publisher that completes between recovery's // claim step and its re-parse must win the round retroactively. The // post-claim re-parse sees the completed key, so recovery releases its @@ -4712,6 +4980,103 @@ mod identity_key_tests { "the claimed markers must not reappear" ); } + + // #194 (G2a) MUST-NOT: a TRANSIENT (non-content) failure of the + // post-claim RE-parse proves nothing about the final's content, so + // recovery must release its claims and surface the error loudly, + // destroying nothing: quarantining on it would destroy a possibly good + // key, the protocol's own must-not. The reparse_fault seam injects the + // transient class deterministically (load_fault only reaches + // boot_load_key's attempts, never this re-parse). RED before G2a: any + // re-parse error is treated as still-crashed and the final is + // quarantined. + #[test] + fn transient_reparse_error_releases_claims_and_fails_loudly() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + let seam = RecoverySeam { + reparse_fault: || Err(std::io::Error::other("injected transient reparse failure")), + ..RecoverySeam::NONE + }; + + let err = + match load_or_create_keypair_with(&key_path, &|| {}, &|| {}, PublishFaults::NONE, seam) + { + Err(e) => e, + Ok(_) => panic!("a transient re-parse failure must stay loud, never resolve"), + }; + assert!( + format!("{err:#}").contains("injected transient reparse failure"), + "the transient failure must surface: {err:#}" + ); + assert_eq!( + std::fs::read(&key_path).expect("final still on disk"), + b"", + "the final's bytes must be untouched" + ); + assert!( + names_containing(dir.path(), ".quarantined.").is_empty(), + "a transient re-parse failure must never quarantine" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "the failing recovery must release its claims, not leak them" + ); + } + + // #194 (G2b): a publisher that completes its write AFTER the re-parse + // observation and BEFORE the quarantine rename has its VALID key swept + // into quarantine; the rename preserves the bytes, so recovery must + // parse the quarantined bytes, detect the completed key, restore it to + // the final name, release its claims, and return that identity: no + // regeneration, no divergence from the Won the publisher already + // reported. The before_quarantine seam lands the completed write in + // exactly that window (before_reparse fires too early: the re-parse + // would catch it). RED before G2b: the completed key stays quarantined + // and a fresh identity is regenerated over it. + #[test] + fn completed_key_is_restored_from_quarantine() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + let completed = Keypair::generate(); + let pem = completed.to_pem().expect("pem"); + let heal_path = key_path.clone(); + let before_quarantine = move || { + std::fs::write(&heal_path, pem.as_bytes()) + .expect("complete the final inside the re-parse-to-rename window"); + }; + let seam = RecoverySeam { + before_quarantine: &before_quarantine, + ..RecoverySeam::NONE + }; + + let kp = load_or_create_keypair_with(&key_path, &|| {}, &|| {}, PublishFaults::NONE, seam) + .expect("recovery must restore the completed key, not fail or regenerate"); + + assert_eq!( + format!("{}", kp.did()), + format!("{}", completed.did()), + "the completed identity must be the one returned, never a regeneration" + ); + let on_disk = super::load_existing_key(&key_path).expect("restored final parses"); + assert_eq!( + format!("{}", on_disk.did()), + format!("{}", completed.did()), + "the final must hold the restored key" + ); + assert!( + names_containing(dir.path(), ".quarantined.").is_empty(), + "the quarantine must be renamed back to the final, not left behind" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "the aborting recovery must release its claims" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "the claimed markers must not reappear" + ); + } } #[cfg(test)] From 8d33fbc83aa06b7f0c32ea03b3677814118e7747 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 21 Jul 2026 17:19:11 -0500 Subject: [PATCH 31/41] fix(node): keep no-clobber on the quarantine restore and scope cleanup to owned inodes The restore now republishes the quarantined bytes through the atomic publisher, so every tier refuses an existing final and a concurrent winner survives; a lost restore keeps the quarantine as forensics and loads the winner. The fallback's post-write error arms remove the final only after proving the name still refers to the inode this publish created, so a failing publisher whose round was claimed away can no longer unlink the recoverer's republished key. Pre-write arms keep by-name removal: they complete inside the crash-signature persistence window, before any recovery can have claimed the round. --- crates/gitlawb-node/src/lib.rs | 369 +++++++++++++++++++++++++++++---- 1 file changed, 324 insertions(+), 45 deletions(-) diff --git a/crates/gitlawb-node/src/lib.rs b/crates/gitlawb-node/src/lib.rs index 965f58ee..ba7f2881 100644 --- a/crates/gitlawb-node/src/lib.rs +++ b/crates/gitlawb-node/src/lib.rs @@ -1059,6 +1059,74 @@ fn write_key_or_cleanup(path: &std::path::Path, write_result: std::io::Result<() }) } +/// Remove `final_path` after a failed fallback write or file fsync ONLY if +/// the name still refers to the inode this publish created (#194). A +/// recovery that claimed this publisher's round has quarantined our inode +/// away by rename and may have republished a LIVE key at the name; removing +/// by name there would unlink the recoverer's final and destroy an identity +/// whose owner already returned Won. On Unix the guard compares dev+ino of +/// our open handle against `symlink_metadata` of the name and removes only +/// on a match; a mismatch or NotFound means the name is no longer ours, so +/// the removal is skipped with a warn carrying both identities. Any stat +/// failure also skips: ownership of the name cannot be proven, and the worst +/// a skipped removal costs is the loud invalid-PEM error at the next start, +/// while a wrong removal destroys a live key. Best-effort throughout, like +/// every other cleanup here. On non-Unix the by-name removal stands +/// (reasoned, not run: no non-Unix target is exercised by these tests). +fn remove_final_if_still_ours(final_path: &std::path::Path, file: &std::fs::File) { + #[cfg(unix)] + { + use std::os::unix::fs::MetadataExt; + let ours = match file.metadata() { + Ok(m) => m, + Err(e) => { + warn!( + path = %final_path.display(), + err = %e, + "skipping failed-publish cleanup: could not stat our own key handle" + ); + return; + } + }; + match std::fs::symlink_metadata(final_path) { + Ok(named) if named.dev() == ours.dev() && named.ino() == ours.ino() => { + let _ = std::fs::remove_file(final_path); + } + Ok(named) => { + warn!( + path = %final_path.display(), + our_dev = ours.dev(), + our_ino = ours.ino(), + named_dev = named.dev(), + named_ino = named.ino(), + "skipping failed-publish cleanup: the final name no longer holds our \ + inode (a recovery claimed this round and republished)" + ); + } + Err(e) if e.kind() == std::io::ErrorKind::NotFound => { + warn!( + path = %final_path.display(), + our_dev = ours.dev(), + our_ino = ours.ino(), + "skipping failed-publish cleanup: the final name is already gone" + ); + } + Err(e) => { + warn!( + path = %final_path.display(), + err = %e, + "skipping failed-publish cleanup: could not stat the final name" + ); + } + } + } + #[cfg(not(unix))] + { + let _ = file; + let _ = std::fs::remove_file(final_path); + } +} + /// Verify an identity key file is not world/group-readable (#194, F2). A `chmod` /// that failed or silently no-op'd (read-only mount, ACL mismatch) must not leave /// a readable private key in use, so this is checked after any tightening attempt @@ -1739,7 +1807,10 @@ fn publish_key_atomically( /// error): create the FINAL path directly with `create_new(true).mode(0o600)` /// (the INV-23 creation pattern) and write the PEM into it. `AlreadyExists` /// still routes to the lost-race path, so no-clobber holds. Error handling: a -/// failed `write_all` OR file `sync_all` removes the final (#194 F1) — the +/// failed `write_all` OR file `sync_all` removes the final (#194 F1), guarded +/// by inode identity (`remove_final_if_still_ours`): only the inode this +/// publish created is removed, never a key a claiming recovery republished +/// at the name. Removal is needed at all because the /// `create_new` already made the final NAME durable, so a name left over /// non-durable content lets a later crash leave a truncated final that wedges /// every later start; removal lets the next start regenerate. Only a failed @@ -1885,7 +1956,10 @@ fn publish_key_fallback( // AlreadyExists, and is safe here for the same reason as everywhere: the // file is still empty, so the split-phase keep-the-final rule below does // not apply yet. The helper removes the final before returning Err, so - // the pinned final-before-marker disposal order holds. + // the pinned final-before-marker disposal order holds. By-name removal + // (no inode guard) stays right on this and the other pre-write arms + // (create/mode-verify): they run within CRASH_SIGNATURE_MIN_PERSIST of + // the final's creation, so no recovery can have claimed this round yet. #[cfg(unix)] { if let Err(e) = tighten_and_verify_created(final_path, faults.fallback_mode_verify) @@ -1898,20 +1972,22 @@ fn publish_key_fallback( } } // A failed write removes the final (partial content cannot parse and would - // wedge every later start). The helper removes the final before returning - // Err, so the pinned final-before-marker disposal order holds. - if let Err(e) = write_key_or_cleanup( - final_path, - (faults.fallback_write)().and_then(|()| f.write_all(pem)), - ) - .with_context(|| { - format!( - "hard_link fallback publish to {} (link failed: {link_err})", - final_path.display() - ) - }) { + // wedge every later start), but only if the name still holds OUR inode: + // a recovery that claimed this round may have quarantined our inode away + // and republished a live key at the name, which a by-name removal would + // destroy (see remove_final_if_still_ours). Not write_key_or_cleanup, + // whose by-name removal stays right for the primary path's TEMP file + // (a private name no recovery can race). Final removal is issued first, + // marker disposal second (pinned order). + if let Err(e) = (faults.fallback_write)().and_then(|()| f.write_all(pem)) { + remove_final_if_still_ours(final_path, &f); dispose_marker(); - return Err(e); + return Err(anyhow::Error::new(e) + .context(format!("failed to write key to {}", final_path.display())) + .context(format!( + "hard_link fallback publish to {} (link failed: {link_err})", + final_path.display() + ))); } // Fsync the file, and REMOVE the final on failure too (#194 F1). Unlike the // hard-link path (which fsyncs a TEMP, so the final NAME only ever appears @@ -1924,13 +2000,15 @@ fn publish_key_fallback( // ENOSPC/EIO can tell "durability failed" (bytes accepted, not synced) from // the write-rejected case above. Only the parent-DIR fsync below keeps the // final on failure (a lost directory entry just disappears -> clean regen). - // Final removal is issued first, marker disposal second (pinned order). + // Final removal is issued first, marker disposal second (pinned order), + // and is inode-guarded like the write arm's: the name may hold a + // recoverer's republished key by now. if let Err(e) = (faults.fallback_fsync)().and_then(|()| f.sync_all()) { - let _ = std::fs::remove_file(final_path); + remove_final_if_still_ours(final_path, &f); dispose_marker(); return Err(anyhow::Error::new(e).context(format!( "fsync identity key {} in hard_link fallback (durability failed, \ - removed; link failed: {link_err})", + removed if still ours; link failed: {link_err})", final_path.display() ))); } @@ -2007,6 +2085,15 @@ struct RecoverySeam<'a> { /// (#194, G2a). Narrower than `load_fault`, which only reaches /// `boot_load_key`'s attempts, never this re-parse. reparse_fault: fn() -> std::io::Result<()>, + /// Runs inside `recover_crashed_publish` between the quarantine's + /// completed-parse and the restore of the completed key to the final + /// name, so a test can interleave a concurrent starter's publish into + /// exactly the quarantine-to-restore window (#194, G2b follow-up). + before_restore: &'a dyn Fn(), + /// Faults threaded into the restore's republish of a completed + /// quarantined key, so a test can force the restore onto its link-hostile + /// degradation tier (production: `PublishFaults::NONE`). + restore_faults: PublishFaults, /// Runs inside `recover_crashed_publish` between the post-claim re-parse /// and the quarantine rename, so a test can land a publisher's completed /// write in exactly the re-parse-to-rename window (#194, G2b). @@ -2021,6 +2108,8 @@ impl RecoverySeam<'_> { sweep_sync: || Ok(()), before_reparse: &|| {}, reparse_fault: || Ok(()), + before_restore: &|| {}, + restore_faults: PublishFaults::NONE, before_quarantine: &|| {}, }; } @@ -2261,38 +2350,57 @@ fn recover_crashed_publish( // publisher already resolved the round, and its key is restored to the // final name instead of being replaced by a regeneration, which would // diverge from the Won the publisher already reported. The restore is a - // no-clobber hard_link plus unlink of the quarantine name, not a rename: - // the final name is expected free (our own rename freed it and the round - // is claimed), but a fresh start that saw no final may have republished - // it; AlreadyExists leaves the quarantine behind as forensics and the - // follow-up load defers to the republished key. On link-hostile mounts - // (where hard_link cannot work) a plain rename restores instead, - // mirroring publish_key_fallback's weakened no-clobber on the same - // mounts; if even that fails the quarantine stays as forensics and the - // follow-up load fails loudly, nothing destroyed. An unparseable - // quarantine is the expected crash state and falls through to - // regenerate. + // REPUBLISH of the quarantined PEM through publish_key_atomically, which + // holds no-clobber on EVERY tier: the hard-link tier refuses an existing + // final, and so does the fallback's create_new on link-hostile mounts (a + // plain rename here would clobber a final a fresh start published into + // the freed name on exactly those mounts). Won means the completed key + // is durably back at the final; the quarantine is then removed, because + // its bytes are an exact copy of what now lives at the final, so keeping + // it buys no forensics. The keypair returned is the one already parsed + // from the quarantined bytes, not a re-read of the final, which could + // race a writer that lands after our republish. Lost means someone else + // durably published meanwhile: keep the quarantine as forensics and + // defer to the settled final. A publish error keeps the quarantine, + // releases the claims, and surfaces loudly, nothing destroyed. An + // unparseable quarantine is the expected crash state and falls through + // to regenerate. if let Some(quarantine) = &quarantined_at { - let completed = - std::fs::read_to_string(quarantine).is_ok_and(|pem| Keypair::from_pem(&pem).is_ok()); - if completed { - match std::fs::hard_link(quarantine, final_path) { - Ok(()) => { + let completed = std::fs::read_to_string(quarantine) + .ok() + .and_then(|pem| Keypair::from_pem(&pem).ok().map(|kp| (pem, kp))); + if let Some((pem, kp)) = completed { + (seam.before_restore)(); + match publish_key_atomically( + final_path, + pem.as_bytes(), + &|| {}, + &|| {}, + seam.restore_faults, + ) { + Ok(KeyPublish::Won) => { let _ = std::fs::remove_file(quarantine); + #[cfg(unix)] + { + let _ = fsync_parent_dir(final_path); + } + release_recovery_claims(final_path, &claims); + return Ok(Recovery::Reloaded(Box::new(Ok(kp)))); } - Err(e) if e.kind() == std::io::ErrorKind::AlreadyExists => { - // Someone republished the final concurrently; keep it. + Ok(KeyPublish::Lost) => { + release_recovery_claims(final_path, &claims); + return Ok(Recovery::Reloaded(Box::new(load_racing_key(final_path)))); } - Err(_) => { - let _ = std::fs::rename(quarantine, final_path); + Err(e) => { + release_recovery_claims(final_path, &claims); + return Err(e.context(format!( + "could not restore the completed identity key to {} during crash \ + recovery; quarantine kept at {}", + final_path.display(), + quarantine.display() + ))); } } - #[cfg(unix)] - { - let _ = fsync_parent_dir(final_path); - } - release_recovery_claims(final_path, &claims); - return Ok(Recovery::Reloaded(Box::new(load_racing_key(final_path)))); } } @@ -5066,7 +5174,7 @@ mod identity_key_tests { ); assert!( names_containing(dir.path(), ".quarantined.").is_empty(), - "the quarantine must be renamed back to the final, not left behind" + "the quarantine must be removed once its key is republished at the final" ); assert!( names_containing(dir.path(), ".recovering.").is_empty(), @@ -5077,6 +5185,177 @@ mod identity_key_tests { "the claimed markers must not reappear" ); } + + // R1: the post-quarantine restore must hold no-clobber on EVERY tier. A + // completed key A is swept into quarantine (G2b), a concurrent starter + // publishes a fresh key B at the freed final name inside the + // quarantine-to-restore window (before_restore), and the restore's link + // tier is forced onto its link-hostile degradation (restore_faults.link). + // The restore must defer to B (keep the quarantine as forensics, converge + // on B), never replace B with A: B's owner already holds B in memory, so + // a clobber leaves that node's memory and disk diverged. RED before the + // fix: the old restore ladder degraded to a plain rename(quarantine -> + // final), which clobbered B with A. + #[test] + fn restore_never_clobbers_a_concurrent_winner() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + let completed = Keypair::generate(); + let pem_a = completed.to_pem().expect("pem a"); + let kp_b = Keypair::generate(); + let pem_b = kp_b.to_pem().expect("pem b"); + + let heal_path = key_path.clone(); + let before_quarantine = move || { + std::fs::write(&heal_path, pem_a.as_bytes()) + .expect("complete the final inside the re-parse-to-rename window"); + }; + let steal_path = key_path.clone(); + let pem_b_disk = pem_b.clone(); + let before_restore = move || { + let out = publish_key_atomically( + &steal_path, + pem_b_disk.as_bytes(), + &|| {}, + &|| {}, + PublishFaults::NONE, + ) + .expect("concurrent starter republishes key B at the freed name"); + assert!( + matches!(out, KeyPublish::Won), + "B's publish wins the free name" + ); + }; + let seam = RecoverySeam { + before_quarantine: &before_quarantine, + before_restore: &before_restore, + restore_faults: PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + ..PublishFaults::NONE + }, + ..RecoverySeam::NONE + }; + + let kp = load_or_create_keypair_with(&key_path, &|| {}, &|| {}, PublishFaults::NONE, seam) + .expect("the restore must converge on B, not fail the start"); + + assert_eq!( + format!("{}", kp.did()), + format!("{}", kp_b.did()), + "the recovering boot must converge on the concurrent winner's key B" + ); + let on_disk = super::load_existing_key(&key_path).expect("final parses"); + assert_eq!( + format!("{}", on_disk.did()), + format!("{}", kp_b.did()), + "key B must survive at the final name on every restore tier" + ); + let quarantined = names_containing(dir.path(), ".quarantined."); + assert_eq!( + quarantined.len(), + 1, + "the losing restore must keep A's quarantine as forensics: {quarantined:?}" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "the recovery must release its claims" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "no publish markers may remain" + ); + } + + // R2: a fallback publisher whose round was claimed by a recovery (its + // inode quarantined away, a live key B republished at the name) must NOT + // let its own failure cleanup unlink B: the removal on the write/fsync + // error arms is guarded by inode identity, not name. The interleave runs + // inside the fallback_fsync fault (before_commit never fires on the error + // arms), step for step what a claiming recovery does: claim A's marker, + // quarantine A's final by rename, republish B, clear the claim; then the + // injected Err forces A's fsync-failure arm. RED before the fix: A's + // by-name removal deletes B, destroying the live identity whose owner + // already returned Won. + #[test] + fn failed_publisher_cleanup_spares_a_republished_final() { + thread_local! { + static STEAL: std::cell::RefCell< + Option<(std::path::PathBuf, std::path::PathBuf, String)>, + > = const { std::cell::RefCell::new(None) }; + } + fn steal_round_then_fail_fsync() -> std::io::Result<()> { + STEAL.with(|s| { + if let Some((dir, key_path, pem_b)) = s.borrow_mut().take() { + let markers = names_containing(&dir, ".publishing."); + assert_eq!( + markers.len(), + 1, + "A's window must be bracketed: {markers:?}" + ); + let claim = dir.join(".identity.pem.recovering.55555"); + std::fs::rename(dir.join(&markers[0]), &claim).expect("claim A's marker"); + std::fs::rename(&key_path, dir.join(".identity.pem.quarantined.55555.0")) + .expect("quarantine A's inode away"); + let out = publish_key_atomically( + &key_path, + pem_b.as_bytes(), + &|| {}, + &|| {}, + PublishFaults::NONE, + ) + .expect("recoverer republishes key B"); + assert!(matches!(out, KeyPublish::Won), "recoverer's republish wins"); + std::fs::remove_file(&claim).expect("clear the claim"); + } + }); + Err(std::io::Error::other("injected fsync failure")) + } + + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let kp_b = Keypair::generate(); + let pem_b = kp_b.to_pem().expect("pem b"); + STEAL.with(|s| { + *s.borrow_mut() = Some((dir.path().to_path_buf(), key_path.clone(), (*pem_b).clone())); + }); + let pem_a = Keypair::generate().to_pem().expect("pem a"); + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + fallback_fsync: steal_round_then_fail_fsync, + ..PublishFaults::NONE + }; + + let err = match publish_key_atomically(&key_path, pem_a.as_bytes(), &|| {}, &|| {}, faults) + { + Err(e) => e, + Ok(_) => panic!("A's failed fsync must still error A's publish"), + }; + assert!( + format!("{err:#}").contains("injected fsync failure"), + "the fsync failure must surface: {err:#}" + ); + let on_disk = super::load_existing_key(&key_path) + .expect("the republished final must survive A's cleanup"); + assert_eq!( + format!("{}", on_disk.did()), + format!("{}", kp_b.did()), + "A's error-arm cleanup must spare the recoverer's key B" + ); + let quarantined = names_containing(dir.path(), ".quarantined."); + assert_eq!( + quarantined.len(), + 1, + "A's quarantined inode stays where the recoverer put it: {quarantined:?}" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "no recovery claims may remain" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "no publish markers may remain" + ); + } } #[cfg(test)] From 0ebdd1a16a051aab86efe28035bda6413594666a Mon Sep 17 00:00:00 2001 From: t Date: Tue, 21 Jul 2026 17:39:45 -0500 Subject: [PATCH 32/41] fix(node): hand post-write publish failures to boot recovery, stamp claim freshness A failed write or fsync on the fallback path now removes nothing: the partial final and its marker stay in place as exactly the crash state boot recovery claims, quarantines, and regenerates from. That deletes the stat-then-unlink cleanup whose race could destroy a recoverer's republished key, and can no longer strand a marker-less partial final when an unlink fails. Pre-write failures keep removal and disposal; they complete inside the crash-signature persistence window. Claim renames also stamp the claim's mtime to now, because rename preserves the source mtime and an aged stale marker otherwise births a live claim the healthy-boot sweep would treat as orphaned mid-round. --- crates/gitlawb-node/src/lib.rs | 433 +++++++++++++++++++++------------ 1 file changed, 271 insertions(+), 162 deletions(-) diff --git a/crates/gitlawb-node/src/lib.rs b/crates/gitlawb-node/src/lib.rs index ba7f2881..f122787c 100644 --- a/crates/gitlawb-node/src/lib.rs +++ b/crates/gitlawb-node/src/lib.rs @@ -1059,74 +1059,6 @@ fn write_key_or_cleanup(path: &std::path::Path, write_result: std::io::Result<() }) } -/// Remove `final_path` after a failed fallback write or file fsync ONLY if -/// the name still refers to the inode this publish created (#194). A -/// recovery that claimed this publisher's round has quarantined our inode -/// away by rename and may have republished a LIVE key at the name; removing -/// by name there would unlink the recoverer's final and destroy an identity -/// whose owner already returned Won. On Unix the guard compares dev+ino of -/// our open handle against `symlink_metadata` of the name and removes only -/// on a match; a mismatch or NotFound means the name is no longer ours, so -/// the removal is skipped with a warn carrying both identities. Any stat -/// failure also skips: ownership of the name cannot be proven, and the worst -/// a skipped removal costs is the loud invalid-PEM error at the next start, -/// while a wrong removal destroys a live key. Best-effort throughout, like -/// every other cleanup here. On non-Unix the by-name removal stands -/// (reasoned, not run: no non-Unix target is exercised by these tests). -fn remove_final_if_still_ours(final_path: &std::path::Path, file: &std::fs::File) { - #[cfg(unix)] - { - use std::os::unix::fs::MetadataExt; - let ours = match file.metadata() { - Ok(m) => m, - Err(e) => { - warn!( - path = %final_path.display(), - err = %e, - "skipping failed-publish cleanup: could not stat our own key handle" - ); - return; - } - }; - match std::fs::symlink_metadata(final_path) { - Ok(named) if named.dev() == ours.dev() && named.ino() == ours.ino() => { - let _ = std::fs::remove_file(final_path); - } - Ok(named) => { - warn!( - path = %final_path.display(), - our_dev = ours.dev(), - our_ino = ours.ino(), - named_dev = named.dev(), - named_ino = named.ino(), - "skipping failed-publish cleanup: the final name no longer holds our \ - inode (a recovery claimed this round and republished)" - ); - } - Err(e) if e.kind() == std::io::ErrorKind::NotFound => { - warn!( - path = %final_path.display(), - our_dev = ours.dev(), - our_ino = ours.ino(), - "skipping failed-publish cleanup: the final name is already gone" - ); - } - Err(e) => { - warn!( - path = %final_path.display(), - err = %e, - "skipping failed-publish cleanup: could not stat the final name" - ); - } - } - } - #[cfg(not(unix))] - { - let _ = file; - let _ = std::fs::remove_file(final_path); - } -} - /// Verify an identity key file is not world/group-readable (#194, F2). A `chmod` /// that failed or silently no-op'd (read-only mount, ACL mismatch) must not leave /// a readable private key in use, so this is checked after any tightening attempt @@ -1806,35 +1738,46 @@ fn publish_key_atomically( /// (Unsupported/EPERM on some network and overlay mounts, or a transient link /// error): create the FINAL path directly with `create_new(true).mode(0o600)` /// (the INV-23 creation pattern) and write the PEM into it. `AlreadyExists` -/// still routes to the lost-race path, so no-clobber holds. Error handling: a -/// failed `write_all` OR file `sync_all` removes the final (#194 F1), guarded -/// by inode identity (`remove_final_if_still_ours`): only the inode this -/// publish created is removed, never a key a claiming recovery republished -/// at the name. Removal is needed at all because the -/// `create_new` already made the final NAME durable, so a name left over -/// non-durable content lets a later crash leave a truncated final that wedges -/// every later start; removal lets the next start regenerate. Only a failed -/// parent-DIR fsync keeps the final (the content is complete and data-synced, -/// and a lost directory entry just disappears -> clean regen, no wedge). Every -/// error context chains `link_err` so a two-step failure is diagnosable. +/// still routes to the lost-race path, so no-clobber holds. Error handling +/// splits at the first PEM byte. PRE-write failures (marker create, final +/// create, tighten/mode-verify) remove the just-created EMPTY final and +/// dispose the marker: nothing recoverable exists yet, and those arms +/// complete within `CRASH_SIGNATURE_MIN_PERSIST` of the final's creation, +/// so no recovery can have claimed the round. POST-write failures +/// (`write_all` or the file `sync_all`) remove NOTHING: the partial final +/// and its marker are left in place together and the error surfaces naming +/// the handoff. That pair is exactly the crash state boot-time recovery +/// handles (claim, quarantine, regenerate), while an unlink here is a +/// stat-then-unlink race against a recovery that already claimed this round +/// and republished a live key at the name, and a marker disposed beside a +/// kept partial final recreates the marker-less wedge the bracket below +/// exists to prevent. The complete-but-unsynced `sync_all`-failure state is +/// safe too: a later healthy load fsyncs the final ITSELF before sweeping +/// the marker (`sweep_stale_markers`' durability gate), and a crash before +/// then re-enters recovery via the surviving marker. A failed parent-DIR +/// fsync also keeps the final but DOES dispose the marker (the content is +/// complete and data-synced, and a lost directory entry just disappears -> +/// clean regen, no wedge). Every error context chains `link_err` so a +/// two-step failure is diagnosable. /// /// The whole non-atomic window is bracketed by a durable publish marker /// (`.{stem}.publishing.{pid}.{attempt}`, empty, same dir as the final): the /// marker is created and its NAME fsynced BEFORE the final can exist, and it -/// is removed only once the final is complete-and-durable (success) or -/// disposed of (error / lost race). On success the removal doubles as the -/// COMMIT CHECK (#194, U3): a removal that fails means a recovering peer -/// claimed this round, and the publish demotes itself to Lost (see the -/// success arm). A crash inside the window therefore +/// is removed only once the final is complete-and-durable (success) or the +/// exit leaves nothing needing vouching (pre-write error / lost race / +/// dir-fsync failure); on a post-write error the marker intentionally +/// OUTLIVES the attempt to vouch for the partial final beside it. On success +/// the removal doubles as the COMMIT CHECK (#194, U3): a removal that fails +/// means a recovering peer claimed this round, and the publish demotes +/// itself to Lost (see the success arm). A crash inside the window therefore /// leaves marker+final together, so a later boot can tell "partial final from /// a crashed fallback publish" from a key an operator corrupted (#194, U1). /// Two constraints: -/// - Disposal ORDER is pinned: wherever the error policy removes the final, -/// that removal is ISSUED BEFORE the marker's. Cleanup is deliberately -/// NOT a Drop guard: a drop guard can run ahead of the final's disposal -/// on an early exit, and a crash between the reordered unlinks would -/// recreate exactly the marker-less partial final this bracket exists to -/// prevent. +/// - Disposal ORDER on the pre-write arms is pinned: the empty final's +/// removal is ISSUED BEFORE the marker's. Cleanup is deliberately NOT a +/// Drop guard: a drop guard can run ahead of the final's disposal on an +/// early exit, and it would also fire on the post-write error exits, +/// whose whole point is that the marker survives. /// - The marker dir fsync is warn-and-continue, not a hard gate: on /// dir-fsync-hostile mounts a hard failure would turn "wedge once, then /// recover" into "never provisions". Without that durability the marker @@ -1902,8 +1845,9 @@ fn publish_key_fallback( dir.display() ); }; - // Marker disposal is explicit at every exit, never a Drop guard; see the - // order constraint in the doc comment. + // Marker disposal is explicit at every disposing exit, never a Drop + // guard; see the order constraint in the doc comment (the post-write + // error exits deliberately keep the marker). let dispose_marker = || { let _ = std::fs::remove_file(&marker); }; @@ -1953,13 +1897,14 @@ fn publish_key_fallback( }; // Secure the empty final BEFORE the PEM bytes hit it (#194, U2). The // helper's fail-closed removal is what keeps a retry from wedging on - // AlreadyExists, and is safe here for the same reason as everywhere: the - // file is still empty, so the split-phase keep-the-final rule below does - // not apply yet. The helper removes the final before returning Err, so - // the pinned final-before-marker disposal order holds. By-name removal - // (no inode guard) stays right on this and the other pre-write arms - // (create/mode-verify): they run within CRASH_SIGNATURE_MIN_PERSIST of - // the final's creation, so no recovery can have claimed this round yet. + // AlreadyExists, and is safe here for the same reason as on every + // pre-write arm: the file is still empty, so the post-write + // keep-everything rule below does not apply yet. The helper removes the + // final before returning Err, so the pinned final-before-marker disposal + // order holds. By-name removal stays right on this and the other + // pre-write arms (create/mode-verify): they run within + // CRASH_SIGNATURE_MIN_PERSIST of the final's creation, so no recovery + // can have claimed this round yet. #[cfg(unix)] { if let Err(e) = tighten_and_verify_created(final_path, faults.fallback_mode_verify) @@ -1971,44 +1916,43 @@ fn publish_key_fallback( return Err(e); } } - // A failed write removes the final (partial content cannot parse and would - // wedge every later start), but only if the name still holds OUR inode: - // a recovery that claimed this round may have quarantined our inode away - // and republished a live key at the name, which a by-name removal would - // destroy (see remove_final_if_still_ours). Not write_key_or_cleanup, + // A failed write LEAVES the partial final and its marker in place and + // only surfaces the error: marker+final together is exactly the crash + // state the next boot recovers from (claim, quarantine, regenerate). + // Any removal here would be a stat-then-unlink race against a recovery + // that already claimed this round and republished a live key at the + // name, and disposing the marker while the partial final stays would + // recreate the marker-less wedge the bracket exists to prevent, so this + // exit removes nothing and disposes nothing. Not write_key_or_cleanup, // whose by-name removal stays right for the primary path's TEMP file - // (a private name no recovery can race). Final removal is issued first, - // marker disposal second (pinned order). + // (a private name no recovery can race). if let Err(e) = (faults.fallback_write)().and_then(|()| f.write_all(pem)) { - remove_final_if_still_ours(final_path, &f); - dispose_marker(); return Err(anyhow::Error::new(e) - .context(format!("failed to write key to {}", final_path.display())) + .context(format!( + "failed to write key to {}; the partial final and its publish marker \ + are left in place for boot-time crash recovery", + final_path.display() + )) .context(format!( "hard_link fallback publish to {} (link failed: {link_err})", final_path.display() ))); } - // Fsync the file, and REMOVE the final on failure too (#194 F1). Unlike the - // hard-link path (which fsyncs a TEMP, so the final NAME only ever appears - // over a durable inode), the fallback's `create_new` already made the final - // name durable, so a bytes-accepted-but-not-durable final would let a later - // crash leave a truncated file that `load_or_create_keypair` parses forever - // via the existing-file path instead of regenerating -> permanent startup - // wedge. Removing it mirrors the hard-link temp policy and lets the next - // start regenerate. A DISTINCT context is kept so an operator debugging - // ENOSPC/EIO can tell "durability failed" (bytes accepted, not synced) from - // the write-rejected case above. Only the parent-DIR fsync below keeps the - // final on failure (a lost directory entry just disappears -> clean regen). - // Final removal is issued first, marker disposal second (pinned order), - // and is inode-guarded like the write arm's: the name may hold a - // recoverer's republished key by now. + // A failed file fsync takes the same keep-everything exit as the failed + // write: the bytes were accepted but may not be durable, and marker+ + // final left together is the recoverable crash state (the next boot's + // load either parses the complete content, after which the sweep's + // durability gate fsyncs the final BEFORE removing the marker, or + // content-fails beside the marker and recovers). A DISTINCT context is + // kept so an operator debugging ENOSPC/EIO can tell "durability failed" + // (bytes accepted, not synced) from the write-rejected case above. Only + // the parent-DIR fsync below keeps the final while disposing the marker + // (a lost directory entry just disappears -> clean regen). if let Err(e) = (faults.fallback_fsync)().and_then(|()| f.sync_all()) { - remove_final_if_still_ours(final_path, &f); - dispose_marker(); return Err(anyhow::Error::new(e).context(format!( - "fsync identity key {} in hard_link fallback (durability failed, \ - removed if still ours; link failed: {link_err})", + "fsync identity key {} in hard_link fallback (durability failed; the \ + final and its publish marker are left in place for boot-time crash \ + recovery; link failed: {link_err})", final_path.display() ))); } @@ -2225,6 +2169,27 @@ fn recover_crashed_publish( } match std::fs::rename(item, &dest) { Ok(()) => { + // The rename preserves the SOURCE's mtime, but the G1 + // sweep gate reads the CLAIM file's mtime: a claim made + // from an aged stale marker would be born already past + // CLAIM_SWEEP_MIN_AGE, and a concurrent healthy boot + // could sweep this LIVE claim mid-round, early-waking + // the demoted publisher it gates. The age gate needs + // claim-time freshness, so stamp it now. Best-effort: + // on failure the worst case is today's bounded-wait + // degradation, so warn and continue. + if let Err(e) = std::fs::File::options() + .write(true) + .open(&dest) + .and_then(|f| f.set_modified(std::time::SystemTime::now())) + { + warn!( + claim = %dest.display(), + err = %e, + "could not refresh a recovery claim's mtime; a concurrent \ + healthy boot may sweep the claim early" + ); + } claims.push(dest); break; } @@ -3170,12 +3135,16 @@ mod identity_key_tests { ); } - // A failed fallback WRITE removes the final: partial content cannot parse - // and would wedge every later start on `invalid PEM key`. The error must - // surface the write failure AND the original link error, so a two-step - // failure is diagnosable. + // A failed fallback WRITE leaves the partial final AND its marker in + // place: the pair is exactly the crash state boot-time recovery handles + // (claim, quarantine, regenerate), while an unlink here is a + // stat-then-unlink race against a recovery that already claimed this + // round and republished a live key at the name, and a marker disposed + // beside a kept partial final recreates the marker-less wedge the + // bracket exists to prevent. The error must surface the write failure + // AND the original link error, so a two-step failure is diagnosable. #[test] - fn failed_fallback_write_removes_the_partial_final() { + fn failed_fallback_write_leaves_the_partial_final_for_recovery() { let dir = tempfile::tempdir().expect("tempdir"); let key_path = dir.path().join("identity.pem"); let pem = Keypair::generate().to_pem().expect("pem"); @@ -3200,8 +3169,14 @@ mod identity_key_tests { "the error must chain the original link failure ({link_display}): {msg}" ); assert!( - !key_path.exists(), - "a failed fallback write must remove the partial final" + key_path.exists(), + "a failed fallback write must leave the partial final for boot-time recovery" + ); + assert!( + std::fs::read(&key_path) + .expect("read partial final") + .is_empty(), + "the write fault fires before any byte lands, so the partial final is empty" ); } @@ -3241,15 +3216,15 @@ mod identity_key_tests { ); } - // A failed fallback FILE fsync must REMOVE the final: create_new has already - // made the final NAME durable, so a name left over non-durable content lets a - // later crash leave a truncated final that load_or_create_keypair parses - // forever (existing-file path) instead of regenerating -> permanent startup - // wedge (#194 F1, jatmn). The fallback now mirrors the hard-link temp policy - // (write_key_or_cleanup removes on write OR fsync failure); removal makes the - // next start regenerate cleanly. Only the parent-DIR fsync keeps on failure. + // A failed fallback FILE fsync takes the same keep-everything exit as + // the failed write: the final (complete but possibly non-durable bytes) + // and its marker stay in place for boot-time recovery. The + // complete-but-unsynced state is safe because a later healthy load + // fsyncs the final ITSELF before sweeping the marker + // (sweep_stale_markers' durability gate), and a crash before then + // re-enters recovery via the surviving marker. #[test] - fn failed_fallback_fsync_removes_the_unsynced_final() { + fn failed_fallback_fsync_leaves_the_final_for_recovery() { let dir = tempfile::tempdir().expect("tempdir"); let key_path = dir.path().join("identity.pem"); let pem = Keypair::generate().to_pem().expect("pem"); @@ -3269,9 +3244,13 @@ mod identity_key_tests { "the error must surface the fsync failure: {msg}" ); assert!( - !key_path.exists(), - "an fsync failure on the fallback must remove the un-synced final so \ - the next start regenerates instead of wedging on a truncated file" + key_path.exists(), + "an fsync failure must leave the final in place for boot-time recovery" + ); + assert_eq!( + std::fs::read(&key_path).expect("read final"), + pem.as_bytes(), + "the fsync fault fires after write_all, so the final holds the full PEM" ); } @@ -3632,10 +3611,12 @@ mod identity_key_tests { ); } - /// Shared body for the fallback error-arm marker tests (#194, U1): with the + /// Shared body for the PRE-write fallback error arms (#194, U1): with the /// link fault plus one injected fallback fault, the error must surface as - /// today, the final must be absent (today's removal policy for every one of - /// these arms), and no publish marker may remain. + /// today, the final must be absent (nothing recoverable exists before the + /// first PEM byte, so those arms remove it), and no publish marker may + /// remain. The POST-write arms (write/fsync) have the opposite contract; + /// see `assert_post_write_failure_leaves_marker_and_final`. fn assert_fallback_error_leaves_no_marker_or_final(faults: PublishFaults, injected: &str) { let dir = tempfile::tempdir().expect("tempdir"); let key_path = dir.path().join("identity.pem"); @@ -3673,9 +3654,44 @@ mod identity_key_tests { ); } + /// Shared body for the POST-write fallback error arms (write/fsync): the + /// error must surface the injected fault AND name the boot-time recovery + /// handoff, the final must SURVIVE, and its `.publishing.` marker must + /// SURVIVE beside it: marker+final together is the recoverable crash + /// state, and marker-less partial final is the permanent wedge. + fn assert_post_write_failure_leaves_marker_and_final(faults: PublishFaults, injected: &str) { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let pem = Keypair::generate().to_pem().expect("pem"); + + let err = match publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, &|| {}, faults) { + Err(e) => e, + Ok(_) => panic!("the injected fallback fault must error the publish"), + }; + let msg = format!("{err:#}"); + assert!( + msg.contains(injected), + "the error must surface the injected fault: {msg}" + ); + assert!( + msg.contains("left in place for boot-time crash recovery"), + "the error must name the recovery handoff: {msg}" + ); + assert!( + key_path.exists(), + "the final must survive a post-write failure for boot-time recovery" + ); + let markers = publishing_markers(dir.path()); + assert_eq!( + markers.len(), + 1, + "the marker must survive to vouch for the partial final, found {markers:?}" + ); + } + #[test] - fn failed_fallback_write_leaves_no_marker() { - assert_fallback_error_leaves_no_marker_or_final( + fn failed_fallback_write_leaves_the_marker_and_final() { + assert_post_write_failure_leaves_marker_and_final( PublishFaults { link: || Err(std::io::ErrorKind::Unsupported.into()), fallback_write: || Err(std::io::Error::other("injected write failure")), @@ -3686,8 +3702,8 @@ mod identity_key_tests { } #[test] - fn failed_fallback_fsync_leaves_no_marker() { - assert_fallback_error_leaves_no_marker_or_final( + fn failed_fallback_fsync_leaves_the_marker_and_final() { + assert_post_write_failure_leaves_marker_and_final( PublishFaults { link: || Err(std::io::ErrorKind::Unsupported.into()), fallback_fsync: || Err(std::io::Error::other("injected fsync failure")), @@ -3697,6 +3713,54 @@ mod identity_key_tests { ); } + // The handoff the post-write arms promise must actually work: take the + // exact state a failed fallback write leaves behind (partial final plus + // its marker) and run a fresh boot over it. The boot must classify the + // crash signature, claim the marker, quarantine the partial final, and + // regenerate: no wedge, no leftover marker or claim. + #[test] + fn post_write_failure_state_is_recoverable() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let pem = Keypair::generate().to_pem().expect("pem"); + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + fallback_write: || Err(std::io::Error::other("injected write failure")), + ..PublishFaults::NONE + }; + if publish_key_atomically(&key_path, pem.as_bytes(), &|| {}, &|| {}, faults).is_ok() { + panic!("the injected write fault must error the publish"); + } + assert!( + key_path.exists() && publishing_markers(dir.path()).len() == 1, + "precondition: the failed publish leaves final plus marker" + ); + + let kp = load_or_create_keypair(&key_config(&key_path)) + .expect("the left-behind crash state must recover, not wedge the start"); + + let on_disk = super::load_existing_key(&key_path).expect("regenerated final parses"); + assert_eq!( + format!("{}", on_disk.did()), + format!("{}", kp.did()), + "the returned identity must match the regenerated on-disk key" + ); + let quarantined = names_containing(dir.path(), ".quarantined."); + assert_eq!( + quarantined.len(), + 1, + "the partial final must be quarantined: {quarantined:?}" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "recovery must consume the publish marker" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "the recovery claim must be released" + ); + } + #[test] fn failed_fallback_mode_verify_leaves_no_marker() { assert_fallback_error_leaves_no_marker_or_final( @@ -4638,6 +4702,50 @@ mod identity_key_tests { ); } + // The G1 age gate reads the CLAIM file's mtime, but a claim is made by + // RENAMING a marker, and rename preserves the source's mtime: a claim + // made from an hours-old stale marker would be born already past + // CLAIM_SWEEP_MIN_AGE, so a concurrent healthy boot's sweep could remove + // the LIVE claim mid-round and early-wake the demoted publisher it + // gates. Recovery must therefore stamp each claim's mtime fresh at + // claim time. RED before the fix: the claim inherits the aged mtime. + #[test] + fn claim_freshness_is_stamped_at_claim_time() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + let marker = dir.path().join(".identity.pem.publishing.99999.0"); + age_beyond_claim_sweep(&marker); + + let recovery = super::recover_crashed_publish( + &key_path, + anyhow::anyhow!("synthetic crash signature"), + RecoverySeam::NONE, + ) + .expect("recovery over an aged marker must not error"); + let claims = match recovery { + super::Recovery::Claimed(claims) => claims, + super::Recovery::Reloaded(_) => { + panic!("an aged marker beside an empty final must be claimed") + } + }; + assert!(!claims.is_empty(), "the aged marker must produce a claim"); + for claim in &claims { + let mtime = std::fs::metadata(claim) + .expect("stat claim") + .modified() + .expect("claim mtime"); + let age = std::time::SystemTime::now() + .duration_since(mtime) + .unwrap_or_default(); + assert!( + age < super::CLAIM_SWEEP_MIN_AGE, + "a claim must be born fresh, not inherit the aged marker mtime \ + through the rename: age {age:?} at {claim:?}" + ); + } + super::release_recovery_claims(&key_path, &claims); + } + // #194 (U4): the generate path must sweep too. A crash between the // marker's durability fsync and the final's `create_new` leaves a durable // marker with NO final; only the Won arm of a later boot can ever clean @@ -5266,16 +5374,17 @@ mod identity_key_tests { ); } - // R2: a fallback publisher whose round was claimed by a recovery (its - // inode quarantined away, a live key B republished at the name) must NOT - // let its own failure cleanup unlink B: the removal on the write/fsync - // error arms is guarded by inode identity, not name. The interleave runs - // inside the fallback_fsync fault (before_commit never fires on the error - // arms), step for step what a claiming recovery does: claim A's marker, - // quarantine A's final by rename, republish B, clear the claim; then the - // injected Err forces A's fsync-failure arm. RED before the fix: A's - // by-name removal deletes B, destroying the live identity whose owner - // already returned Won. + // Regression guard: a fallback publisher whose round was claimed by a + // recovery (its inode quarantined away, a live key B republished at the + // name) must NOT let its own failure cleanup unlink B. The post-write + // error arms now remove nothing at all (any unlink there is a + // stat-then-unlink race against the recoverer), which this interleave + // pins. It runs inside the fallback_fsync fault (before_commit never + // fires on the error arms), step for step what a claiming recovery does: + // claim A's marker, quarantine A's final by rename, republish B, clear + // the claim; then the injected Err forces A's fsync-failure arm. RED if + // any by-name removal returns to that arm: A deletes B, destroying the + // live identity whose owner already returned Won. #[test] fn failed_publisher_cleanup_spares_a_republished_final() { thread_local! { From 159f04c03c868edda025c4f39cb95298e6fbfdf7 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 21 Jul 2026 18:02:14 -0500 Subject: [PATCH 33/41] fix(node): key recovery claims by process nonce and harden the freshness stamp Two PID-1 containers on a shared identity volume derive identical claim names, and rename replaces an existing destination, so one recoverer's release could unlink the other's live claim. Claim names now carry a per-process nonce between pid and counter, so the release set can never alias another process's claims. The claim-freshness stamp gains a write fallback (a one-byte append bumps mtime where set_modified is unsupported), and its failure warning names the real hazard. Also lock the vanished-final re-parse arm with a dedicated regression test. --- crates/gitlawb-node/src/lib.rs | 249 +++++++++++++++++++++++++++++++-- 1 file changed, 238 insertions(+), 11 deletions(-) diff --git a/crates/gitlawb-node/src/lib.rs b/crates/gitlawb-node/src/lib.rs index f122787c..901d74a4 100644 --- a/crates/gitlawb-node/src/lib.rs +++ b/crates/gitlawb-node/src/lib.rs @@ -2082,6 +2082,84 @@ enum Recovery { Reloaded(Box>), } +/// The per-process nonce embedded in recovery-claim names +/// (`.{stem}.recovering.{pid}.{nonce}.{n}`). The pid alone is NOT unique +/// across recoverers on a shared volume: two containers each running as +/// PID 1 (the same restart topology `KEY_TEMP_ATTEMPTS`' stale-name +/// discipline already handles) can recover the same directory concurrently +/// and derive identical destination names, and `rename` REPLACES an +/// existing destination, so the second recoverer's claim rename would +/// clobber the first's LIVE claim inode (the claim loop's `exists()` probe +/// is check-then-act, not an arbiter). Both would then track the same +/// path, and the first release would unlink the other's live claim, +/// early-waking the demoted publisher that claim gates. The nonce makes +/// the two name sequences disjoint. Std-only seeding: boot-time nanos +/// XORed with this static's address (ASLR-shifted per process), folded to +/// 32 bits; uniqueness is probabilistic-by-construction, which suffices +/// because a collision needs the same pid AND the same nonce against the +/// same directory at the same time. Threads within one process share the +/// nonce; intra-process claim arbitration remains the per-source rename +/// (production calls this once, at boot). +fn claim_nonce() -> u32 { + static NONCE: std::sync::OnceLock = std::sync::OnceLock::new(); + *NONCE.get_or_init(|| { + let nanos = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos() as u64) + .unwrap_or(0); + let mixed = nanos ^ (&NONCE as *const _ as u64); + (mixed ^ (mixed >> 32)) as u32 + }) +} + +/// Production primary stamp for `refresh_claim_mtime`: set the file's mtime +/// to now via `set_modified`. +fn set_mtime_now(path: &std::path::Path) -> std::io::Result<()> { + std::fs::File::options() + .write(true) + .open(path)? + .set_modified(std::time::SystemTime::now()) +} + +/// Stamp `claim`'s mtime fresh, for the G1 age gate (see the call site in +/// `recover_crashed_publish`). `primary_stamp` is `set_mtime_now` in +/// production (injectable, the file's fn-pointer seam style; some +/// filesystems reject explicit timestamps while honoring write-driven +/// updates). If the primary fails, fall back to appending one byte, which +/// updates the mtime on effectively every filesystem, plus a best-effort +/// data sync. The claim is then no longer empty, which is safe: claims +/// vouch for no content and every consumer matches them by NAME only +/// (`list_recovery_claimables`, `sweep_stale_markers`, +/// `wait_for_recovery_claims`); nothing reads a claim's bytes. Errs only +/// when BOTH rungs fail, carrying both failures in the message. +fn refresh_claim_mtime( + claim: &std::path::Path, + primary_stamp: fn(&std::path::Path) -> std::io::Result<()>, +) -> std::io::Result<()> { + let primary_err = match primary_stamp(claim) { + Ok(()) => return Ok(()), + Err(e) => e, + }; + std::fs::File::options() + .append(true) + .open(claim) + .and_then(|mut f| { + use std::io::Write; + f.write_all(b".")?; + let _ = f.sync_data(); + Ok(()) + }) + .map_err(|fallback_err| { + std::io::Error::new( + fallback_err.kind(), + format!( + "set_modified failed ({primary_err}) and the write fallback \ + failed ({fallback_err})" + ), + ) + }) +} + /// Best-effort removal of the claim files a recovery created, plus a /// best-effort dir fsync. Claims vouch for NO on-disk content (unlike /// publish markers, whose disposal order against the final is pinned): they @@ -2162,7 +2240,16 @@ fn recover_crashed_publish( let mut next_name = 0u32; for item in &claimables { for _ in 0..KEY_TEMP_ATTEMPTS { - let dest = dir.join(format!("{claim_prefix}{}.{next_name}", std::process::id())); + // The name carries a per-process nonce beside the pid: pid alone + // is not unique across recoverers (two PID-1 containers on one + // shared volume), and a colliding destination would let one + // recoverer's rename clobber the other's live claim; see + // `claim_nonce`. + let dest = dir.join(format!( + "{claim_prefix}{}.{:08x}.{next_name}", + std::process::id(), + claim_nonce() + )); next_name = next_name.wrapping_add(1); if dest.exists() { continue; @@ -2175,19 +2262,21 @@ fn recover_crashed_publish( // CLAIM_SWEEP_MIN_AGE, and a concurrent healthy boot // could sweep this LIVE claim mid-round, early-waking // the demoted publisher it gates. The age gate needs - // claim-time freshness, so stamp it now. Best-effort: - // on failure the worst case is today's bounded-wait - // degradation, so warn and continue. - if let Err(e) = std::fs::File::options() - .write(true) - .open(&dest) - .and_then(|f| f.set_modified(std::time::SystemTime::now())) - { + // claim-time freshness, so stamp it now (two rungs; see + // refresh_claim_mtime). Only if both rungs fail is the + // live claim left exposed to the age-gated sweep, so + // warn naming that hazard and continue: the exposure is + // bounded to churn, not divergence, by the post-claim + // re-parse and the restore's no-clobber republish. + if let Err(e) = refresh_claim_mtime(&dest, set_mtime_now) { warn!( claim = %dest.display(), err = %e, - "could not refresh a recovery claim's mtime; a concurrent \ - healthy boot may sweep the claim early" + "could not refresh a recovery claim's mtime by any rung: a \ + concurrent healthy boot's age-gated sweep may remove this \ + LIVE claim mid-round and early-wake the demoted publisher \ + it gates (bounded to churn, not divergence, by the \ + post-claim re-parse and the restore's no-clobber republish)" ); } claims.push(dest); @@ -4746,6 +4835,144 @@ mod identity_key_tests { super::release_recovery_claims(&key_path, &claims); } + // The G1 freshness stamp must survive a failing set_modified: some + // filesystems reject explicit timestamps while honoring write-driven + // mtime updates, and a claim left with its inherited aged mtime for the + // whole round is exposed to a concurrent healthy boot's age-gated sweep. + // The ladder's fallback appends a byte, which bumps the mtime on + // effectively every filesystem (claims are matched by name only, so a + // non-empty claim is safe). RED with the fallback rung absent: the + // claim keeps the aged mtime. + #[test] + fn stamp_fallback_bumps_mtime_by_write() { + let dir = tempfile::tempdir().expect("tempdir"); + let claim = dir.path().join(".identity.pem.recovering.99999.deadbeef.0"); + std::fs::write(&claim, b"").expect("seed claim"); + age_beyond_claim_sweep(&claim); + + super::refresh_claim_mtime(&claim, |_| Err(std::io::ErrorKind::Unsupported.into())) + .expect("the write fallback must stamp the mtime when the primary fails"); + + let mtime = std::fs::metadata(&claim) + .expect("stat claim") + .modified() + .expect("claim mtime"); + let age = std::time::SystemTime::now() + .duration_since(mtime) + .unwrap_or_default(); + assert!( + age < super::CLAIM_SWEEP_MIN_AGE, + "the fallback write must leave the claim fresh, age {age:?}" + ); + } + + // Claim destinations must carry a per-process nonce beyond the pid: + // `.{stem}.recovering.{pid}.{nonce}.{n}`. The pid alone is not unique + // across recoverers on a shared volume (two PID-1 containers), so two + // concurrent recoverers could derive the SAME destination name, and the + // second's claim rename would replace the first's live claim inode (the + // exists() probe is a check-then-act, not an arbiter); the first release + // would then unlink the other's LIVE claim and early-wake the demoted + // publisher it gates. The cross-process collision itself cannot be + // executed from one test process (one pid, one nonce), so this test + // fences the mechanism: every claim name must carry the process nonce + // field between the pid and the counter. RED under pid-only naming. + #[test] + fn same_pid_recoverers_get_unique_claim_names() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + + let recovery = super::recover_crashed_publish( + &key_path, + anyhow::anyhow!("synthetic crash signature"), + RecoverySeam::NONE, + ) + .expect("recovery over a crash state must not error"); + let claims = match recovery { + super::Recovery::Claimed(claims) => claims, + super::Recovery::Reloaded(_) => { + panic!("a marker beside an empty final must be claimed") + } + }; + assert!(!claims.is_empty(), "the marker must produce a claim"); + let pid_prefix = format!(".identity.pem.recovering.{}.", std::process::id()); + for claim in &claims { + let name = claim + .file_name() + .and_then(|n| n.to_str()) + .expect("claim name is utf8"); + let rest = name + .strip_prefix(&pid_prefix) + .unwrap_or_else(|| panic!("claim {name:?} must start with {pid_prefix:?}")); + let fields: Vec<&str> = rest.split('.').collect(); + assert_eq!( + fields.len(), + 2, + "claim {name:?} must end in {{nonce}}.{{counter}}, got {rest:?}" + ); + assert_eq!( + fields[0], + format!("{:08x}", super::claim_nonce()), + "the field between pid and counter must be the process nonce" + ); + assert!( + fields[1].parse::().is_ok(), + "the final field must be the claim counter, got {:?}", + fields[1] + ); + } + super::release_recovery_claims(&key_path, &claims); + } + + // A final that VANISHES between the claim renames and the post-claim + // re-parse (a competing recoverer disposed of it concurrently) must join + // the content class and regenerate, not surface as a loud transient + // error: there is nothing a quarantine could destroy, and the quarantine + // loop's own NotFound arm resolves it cleanly. Locks the NotFound arm of + // the re-parse match (anyhow's downcast_ref:: sees the + // NotFound through the with_context layers). RED with NotFound routed + // into the transient arm (mutation check): the boot fails loudly. + #[test] + fn vanished_final_at_reparse_still_regenerates() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + let vanish_path = key_path.clone(); + let before_reparse = move || { + std::fs::remove_file(&vanish_path).expect("vanish the final inside the claim window"); + }; + + let kp = load_or_create_keypair_with( + &key_path, + &|| {}, + &|| {}, + PublishFaults::NONE, + RecoverySeam { + before_reparse: &before_reparse, + ..RecoverySeam::NONE + }, + ) + .expect("a final vanished at the re-parse must regenerate, not fail the start"); + + let on_disk = super::load_existing_key(&key_path).expect("regenerated final parses"); + assert_eq!( + format!("{}", on_disk.did()), + format!("{}", kp.did()), + "the returned identity must match the regenerated on-disk key" + ); + assert!( + names_containing(dir.path(), ".quarantined.").is_empty(), + "a vanished final leaves nothing to quarantine" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "no publish markers may remain" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "no recovery claims may remain" + ); + } + // #194 (U4): the generate path must sweep too. A crash between the // marker's durability fsync and the final's `create_new` leaves a durable // marker with NO final; only the Won arm of a later boot can ever clean From 8501285a57a7f2fb3881cff45c93167eb975dcb2 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 21 Jul 2026 18:23:49 -0500 Subject: [PATCH 34/41] fix(node): adopt stranded quarantined identities and age-gate claim stealing A completed key stranded in quarantine by a failed or crash-interrupted restore is no longer abandoned: the generate arm first scans quarantines newest-first and republishes the first parseable one through the atomic publisher, preserving identity continuity instead of silently minting a fresh DID; unparseable quarantines stay as forensics. Recovering claims become claimable only past the same age floor the sweep uses, so a second starter can never steal a live recoverer's round mid-flight; a claim-crash state recovers after the bounded age floor passes. The crash-signature persistence doc now states the slow-mount trade-off plainly: a pre-write window exceeding the floor yields churn that converges, never divergence. --- crates/gitlawb-node/src/lib.rs | 281 ++++++++++++++++++++++++++++++++- 1 file changed, 274 insertions(+), 7 deletions(-) diff --git a/crates/gitlawb-node/src/lib.rs b/crates/gitlawb-node/src/lib.rs index 901d74a4..9d1490e9 100644 --- a/crates/gitlawb-node/src/lib.rs +++ b/crates/gitlawb-node/src/lib.rs @@ -1159,6 +1159,12 @@ const KEY_RACE_DEADLINE: std::time::Duration = std::time::Duration::from_secs(5) /// the publisher re-load the pre-quarantine key while the recoverer /// quarantines and republishes: two-sided divergence. An unreadable mtime is /// treated as YOUNG (skip removal; fail safe). +/// +/// The same floor also gates which `.recovering.` claims are CLAIMABLE +/// (`list_recovery_claimables`), so a claim-crash state (a content-bad +/// final whose only claimable is a claim) becomes recoverable only after +/// this floor passes. That delay is bounded (~2x the deadline) and is the +/// price of never stealing a live recovery round. const CLAIM_SWEEP_MIN_AGE: std::time::Duration = KEY_RACE_DEADLINE.saturating_mul(2); /// How long the crash signature (a content-bad final beside a claimable) @@ -1171,6 +1177,14 @@ const CLAIM_SWEEP_MIN_AGE: std::time::Duration = KEY_RACE_DEADLINE.saturating_mu /// pin (250ms of persistence plus claim, quarantine, and regenerate stays /// well under 2s). Any successful load or signature-free observation resets /// the clock. +/// +/// Honest trade-off: on mounts where a live first publish's pre-write window +/// (the fallback's `create_new` through `write_all`) can itself exceed this +/// floor, a concurrent starter can still classify that live publish as +/// crashed. Every such path converges (the demotion commit check, the +/// post-claim re-parse, the no-clobber restore, the quarantine adoption), so +/// the cost is churn, not divergence; raising the floor trades that +/// probability against recovery latency after a real crash. const CRASH_SIGNATURE_MIN_PERSIST: std::time::Duration = std::time::Duration::from_millis(250); /// Bounded number of temp-file names a publish will try before giving up. @@ -1251,6 +1265,59 @@ fn recovering_prefix(stem: &str) -> String { format!(".{stem}.recovering.") } +/// `".{stem}.quarantined."`: the single source of truth for the quarantine +/// name class. Same policy as `publishing_prefix`. +fn quarantined_prefix(stem: &str) -> String { + format!(".{stem}.quarantined.") +} + +/// The newest `.{stem}.quarantined.*` entry beside `final_path` whose bytes +/// parse as a keypair, with those bytes: a COMPLETED key a crashed or failed +/// recovery left stranded, adoptable by the generate arm of +/// `load_or_create_keypair_with` (F1). Candidates are tried newest mtime +/// first (the most recent quarantine is the most recently live identity; +/// unreadable mtimes sort last); unreadable or unparseable entries are +/// skipped, staying on disk as forensics. Directory-read and non-UTF-8 +/// policy match `list_publish_markers`. +fn find_adoptable_quarantine( + final_path: &std::path::Path, +) -> Option<(std::path::PathBuf, String, Keypair)> { + let dir = final_path + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or_else(|| std::path::Path::new(".")); + let stem = final_path + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or("identity.pem"); + let prefix = quarantined_prefix(stem); + let entries = std::fs::read_dir(dir).ok()?; + let mut quarantines: Vec<(std::path::PathBuf, Option)> = entries + .filter_map(|e| e.ok()) + .filter(|e| { + e.file_name() + .to_str() + .is_some_and(|n| n.starts_with(&prefix)) + }) + .map(|e| { + let mtime = e.metadata().and_then(|m| m.modified()).ok(); + (e.path(), mtime) + }) + .collect(); + // Newest first; None (unreadable mtime) orders before every Some under + // Option's Ord, so the descending sort puts it last. + quarantines.sort_by_key(|q| std::cmp::Reverse(q.1)); + for (path, _) in quarantines { + let Ok(pem) = std::fs::read_to_string(&path) else { + continue; + }; + if let Ok(kp) = Keypair::from_pem(&pem) { + return Some((path, pem, kp)); + } + } + None +} + /// The publish markers (`.{stem}.publishing.*`) beside `final_path`, sorted: /// U1's crash signature for a fallback publish interrupted mid-window (#194, /// U2). An unreadable directory reads as no markers, so recovery stays off @@ -1290,6 +1357,17 @@ fn list_publish_markers(final_path: &std::path::Path) -> Vec /// ZERO markers, and a signature keyed on markers alone would leave that /// state unrecoverable forever. Directory-read and non-UTF-8 policy match /// `list_publish_markers`. +/// +/// A `.recovering.` claim counts only once older than `CLAIM_SWEEP_MIN_AGE` +/// (same age gate and rationale as the G1 sweep): a fresh claim may gate a +/// LIVE recovery round, and a second starter that claims it mid-round steals +/// the round from the recoverer that owns it. Because both the crash +/// signature (`boot_load_key`) and the claim loop +/// (`recover_crashed_publish`) read this one list, the gate holds at a +/// single point. An unreadable claim mtime reads as young, so the claim is +/// skipped (fail safe, matching the sweep). `.publishing.` markers stay +/// unconditionally claimable: claiming a live publisher's marker only fails +/// its commit check, a safe demotion. fn list_recovery_claimables(final_path: &std::path::Path) -> Vec { let dir = final_path .parent() @@ -1307,9 +1385,23 @@ fn list_recovery_claimables(final_path: &std::path::Path) -> Vec = entries .filter_map(|e| e.ok()) .filter(|e| { - e.file_name() - .to_str() - .is_some_and(|n| n.starts_with(&publishing) || n.starts_with(&recovering)) + let name = e.file_name(); + let Some(name) = name.to_str() else { + return false; + }; + if name.starts_with(&publishing) { + return true; + } + if !name.starts_with(&recovering) { + return false; + } + // F2 age gate (see the doc comment): only an ORPHANED claim is + // claimable; a fresh one may gate a live round. + e.metadata() + .and_then(|m| m.modified()) + .ok() + .and_then(|mtime| std::time::SystemTime::now().duration_since(mtime).ok()) + .is_some_and(|age| age >= CLAIM_SWEEP_MIN_AGE) }) .map(|e| e.path()) .collect(); @@ -2345,7 +2437,13 @@ fn recover_crashed_publish( // the test seam for the re-parse-to-rename window (#194, G2b; a no-op in // production). (seam.before_quarantine)(); - let dest_for = |n: u32| dir.join(format!(".{stem}.quarantined.{}.{n}", std::process::id())); + let dest_for = |n: u32| { + dir.join(format!( + "{}{}.{n}", + quarantined_prefix(stem), + std::process::id() + )) + }; let start = (0..KEY_TEMP_ATTEMPTS) .find(|&n| !dest_for(n).exists()) .unwrap_or(0); @@ -2534,6 +2632,58 @@ fn load_or_create_keypair_with( } (seam.before_publish)(); + + // ADOPTION SCAN (F1): with no loadable final, a COMPLETED key may be + // stranded in a quarantine: the recoverer's restore republish + // errored, or the recoverer crashed between its quarantine rename + // and the restore. Generating here would mint a fresh DID over a + // still-live identity, so a parseable quarantine is republished and + // adopted instead: adoption preserves identity continuity across a + // crashed or failed restore. An unparseable quarantine is the + // expected crash state and stays on disk as forensics; only when no + // quarantine parses (or none exists) does the boot fall through to + // generate. The republish holds no-clobber on every tier, so Lost + // means a concurrent starter durably published meanwhile: defer to + // it and keep the quarantine, exactly like the G2b restore. A + // republish error surfaces loudly with the quarantine kept: minting + // a fresh identity over an adoptable one is the failure F1 exists to + // prevent. + if let Some((quarantine, pem, kp)) = find_adoptable_quarantine(key_path) { + match publish_key_atomically(key_path, pem.as_bytes(), &|| {}, &|| {}, faults) { + Ok(KeyPublish::Won) => { + // The quarantine's bytes are an exact copy of what now + // durably lives at the final, so keeping it buys no + // forensics (the G2b restore's removal rationale). + let _ = std::fs::remove_file(&quarantine); + #[cfg(unix)] + { + let _ = fsync_parent_dir(key_path); + } + release_recovery_claims(key_path, &claims); + sweep_stale_markers(key_path, seam.sweep_sync); + info!( + path = %key_path.display(), + did = %kp.did(), + "adopted a completed identity stranded in quarantine" + ); + return Ok(kp); + } + Ok(KeyPublish::Lost) => { + let result = load_racing_key(key_path); + release_recovery_claims(key_path, &claims); + return result; + } + Err(e) => { + release_recovery_claims(key_path, &claims); + return Err(e.context(format!( + "could not republish the completed identity key stranded at {}; \ + quarantine kept", + quarantine.display() + ))); + } + } + } + let kp = Keypair::generate(); // Every error exit below this point may hold a fired recovery's // claims; release them before propagating (#194, F1). The bare `?` @@ -4198,7 +4348,13 @@ mod identity_key_tests { ); // Phase two: the claim left behind (the read-only dir blocked its - // release too) must make the state recoverable, not wedge it. + // release too) must make the state recoverable, not wedge it. F2 + // makes a fresh claim invisible to recovery (it could gate a LIVE + // round), so age the leftovers past the floor first, standing in for + // the bounded wait a real next boot would incur. + for name in names_containing(dir.path(), ".recovering.") { + age_beyond_claim_sweep(&dir.path().join(name)); + } let kp = load_or_create_keypair(&key_config(&key_path)) .expect("the leftover claim must be claimable by the next boot, not a wedge"); let on_disk = super::load_existing_key(&key_path).expect("regenerated final parses"); @@ -5251,8 +5407,12 @@ mod identity_key_tests { std::fs::write(&key_path, b"").expect("seed crashed final"); std::fs::set_permissions(&key_path, std::fs::Permissions::from_mode(0o600)) .expect("0600 crashed final"); - std::fs::write(dir.path().join(".identity.pem.recovering.99999.0"), b"") - .expect("seed stale recovery claim"); + let claim = dir.path().join(".identity.pem.recovering.99999.0"); + std::fs::write(&claim, b"").expect("seed stale recovery claim"); + // F2 age gate: only a claim past CLAIM_SWEEP_MIN_AGE is claimable; a + // fresh one may gate a LIVE round (live_claims_are_not_claimable is + // the must-not side). Age the orphan so this stays the positive case. + age_beyond_claim_sweep(&claim); let kp = load_or_create_keypair(&key_config(&key_path)) .expect("a claim-crash state must recover, not wedge every later boot"); @@ -5279,6 +5439,113 @@ mod identity_key_tests { ); } + // F2 MUST-NOT: a FRESH `.recovering.` claim may gate a LIVE recovery + // round (claim, quarantine, republish), and a second starter claiming it + // mid-round steals that round: the recoverer's own re-parse and restore + // then race the thief's quarantine-and-regenerate. A fresh claim must + // therefore never count as claimable: with no `.publishing.` marker + // beside it the claimable set is empty, no crash signature fires, and + // the load fails loudly with the claim untouched. The positive case (an + // ORPHANED claim past CLAIM_SWEEP_MIN_AGE) is + // claim_crash_state_is_recoverable. Rides the full KEY_RACE_DEADLINE + // (~5s), as every loud-fail load does. RED before F2: the fresh claim is + // claimed and recovery proceeds. + #[test] + fn live_claims_are_not_claimable() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + std::fs::write(&key_path, b"").expect("seed crashed final"); + std::fs::set_permissions(&key_path, std::fs::Permissions::from_mode(0o600)) + .expect("0600 crashed final"); + let claim = dir.path().join(".identity.pem.recovering.99999.0"); + std::fs::write(&claim, b"").expect("seed fresh (possibly live) claim"); + + let err = match load_or_create_keypair(&key_config(&key_path)) { + Err(e) => e, + Ok(_) => panic!("a fresh claim must never be stolen; the load must fail loudly"), + }; + + assert!( + err.to_string().contains("invalid PEM key"), + "the load failure must surface unchanged: {err:#}" + ); + assert!( + claim.exists(), + "the possibly-live claim must survive untouched" + ); + assert!( + names_containing(dir.path(), ".quarantined.").is_empty(), + "no quarantine may be created off a live claim" + ); + } + + // F1: a COMPLETED key stranded in a quarantine (the recoverer's restore + // republish errored, or the recoverer crashed between its quarantine + // rename and the restore) must be ADOPTED by the next boot: the final is + // absent, so without adoption the boot mints a fresh DID and the node's + // durable identity silently changes. The generate arm scans for + // parseable quarantines and republishes the stranded bytes instead. RED + // before F1: a fresh identity is minted and the quarantine is stranded + // forever. + #[test] + fn stranded_quarantine_is_adopted_not_replaced() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let stranded = Keypair::generate(); + let pem = stranded.to_pem().expect("pem"); + std::fs::write( + dir.path().join(".identity.pem.quarantined.99999.0"), + pem.as_bytes(), + ) + .expect("seed stranded quarantine"); + + let kp = load_or_create_keypair(&key_config(&key_path)) + .expect("a stranded completed key must be adopted, not fail the boot"); + + assert_eq!( + format!("{}", kp.did()), + format!("{}", stranded.did()), + "the stranded identity must be adopted, never replaced by a fresh DID" + ); + let on_disk = super::load_existing_key(&key_path).expect("republished final parses"); + assert_eq!( + format!("{}", on_disk.did()), + format!("{}", stranded.did()), + "the final must hold the adopted key" + ); + assert!( + names_containing(dir.path(), ".quarantined.").is_empty(), + "the quarantine must be consumed once its bytes are durably republished" + ); + } + + // F1 MUST-NOT: an UNPARSEABLE quarantine is the expected crash state and + // stays as forensics; boot must fall through to a fresh generation, not + // wedge on it or delete it. + #[test] + fn unparseable_quarantine_falls_through_to_generate() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let garbage: &[u8] = b"-----BEGIN nonsense truncated"; + let quarantine = dir.path().join(".identity.pem.quarantined.99999.0"); + std::fs::write(&quarantine, garbage).expect("seed unparseable quarantine"); + + let kp = load_or_create_keypair(&key_config(&key_path)) + .expect("an unparseable quarantine must not block generation"); + + let on_disk = super::load_existing_key(&key_path).expect("generated final parses"); + assert_eq!( + format!("{}", on_disk.did()), + format!("{}", kp.did()), + "the generated identity must be on disk" + ); + assert_eq!( + std::fs::read(&quarantine).expect("quarantine still on disk"), + garbage, + "the unparseable quarantine must survive byte-for-byte as forensics" + ); + } + // #194 (grace): the crash signature must persist beyond a single // observation. A reader that catches a live fallback publisher's // mid-write window sees content-bad-plus-marker ONCE; if the state heals From 0955504889406aa2793e607dcdf80f2ba5244e5f Mon Sep 17 00:00:00 2001 From: t Date: Tue, 21 Jul 2026 18:52:51 -0500 Subject: [PATCH 35/41] fix(node): bound quarantine adoption, expire orphan claims, align the waiter Adoption of a stranded quarantined identity is now bounded by the claim age floor, so a current round's completed key is preserved while a historical forensic quarantine can never resurrect an old DID or defeat the delete-identity-for-fresh-DID operator procedure. A healthy boot also schedules one delayed re-sweep after the age floor passes, so claims spared as possibly-live cannot linger as aged residue that would let later real corruption of a good final masquerade as a crash state and silently regenerate. The demotion waiter now shares the sweep's liveness rule: it waits out claims younger than the age floor rather than a flat race deadline, and skips aged orphans quickly. --- crates/gitlawb-node/Cargo.toml | 3 + crates/gitlawb-node/src/lib.rs | 378 +++++++++++++++++++++++++++++---- 2 files changed, 336 insertions(+), 45 deletions(-) diff --git a/crates/gitlawb-node/Cargo.toml b/crates/gitlawb-node/Cargo.toml index 1dc0fb7c..16746c3c 100644 --- a/crates/gitlawb-node/Cargo.toml +++ b/crates/gitlawb-node/Cargo.toml @@ -95,6 +95,9 @@ libp2p-dns = { version = "0.44.0", features = ["tokio"] } [dev-dependencies] mockito = "1" tempfile = "3" +# test-util enables tokio::test(start_paused): the delayed claim re-sweep +# test drives the real CLAIM_SWEEP_MIN_AGE-plus-margin sleep on paused time. +tokio = { workspace = true, features = ["test-util"] } # Used by the deny-harness integration crate (tests/deny_harness.rs): the # #[sqlx::test] macro for an ephemeral per-test pool, and reqwest as the real # HTTP client that drives deny paths over the socket. diff --git a/crates/gitlawb-node/src/lib.rs b/crates/gitlawb-node/src/lib.rs index 9d1490e9..576dfce0 100644 --- a/crates/gitlawb-node/src/lib.rs +++ b/crates/gitlawb-node/src/lib.rs @@ -92,6 +92,11 @@ pub async fn run() -> Result<()> { // Load or generate the node's identity keypair let keypair = load_or_create_keypair(&config)?; + // The boot sweep spares young `.recovering.` claims (they may gate a + // live round); this one delayed re-sweep clears any spared orphan once + // it has aged past the liveness floor, ~12s into uptime, so aged claim + // residue cannot linger beside a healthy final (F2). Fire-and-forget. + spawn_delayed_claim_resweep(config.resolved_key_path()); let node_did = keypair.did(); // One-time metrics init. Must run before any handler that calls into @@ -1271,14 +1276,22 @@ fn quarantined_prefix(stem: &str) -> String { format!(".{stem}.quarantined.") } -/// The newest `.{stem}.quarantined.*` entry beside `final_path` whose bytes -/// parse as a keypair, with those bytes: a COMPLETED key a crashed or failed -/// recovery left stranded, adoptable by the generate arm of -/// `load_or_create_keypair_with` (F1). Candidates are tried newest mtime -/// first (the most recent quarantine is the most recently live identity; -/// unreadable mtimes sort last); unreadable or unparseable entries are -/// skipped, staying on disk as forensics. Directory-read and non-UTF-8 -/// policy match `list_publish_markers`. +/// The newest `.{stem}.quarantined.*` entry beside `final_path` YOUNGER than +/// `CLAIM_SWEEP_MIN_AGE` whose bytes parse as a keypair, with those bytes: a +/// COMPLETED key the CURRENT round's crashed or failed recovery left +/// stranded, adoptable by the generate arm of `load_or_create_keypair_with` +/// (F1). The age bound is the point: a stranded current round's quarantine +/// is seconds old, so continuity across a just-crashed restore is preserved, +/// while a HISTORICAL parseable quarantine (forensics kept after a Lost +/// restore, or any prior round) is never resurrected; without the bound, a +/// final that goes missing later silently revives an old identity, and the +/// operator procedure "delete identity.pem for a fresh DID" is defeated by +/// whatever quarantine still sits beside it. Once the floor passes, that +/// procedure mints a genuinely fresh identity. An unreadable mtime reads as +/// too old (skip; fail safe against resurrection). Candidates are tried +/// newest mtime first (the most recent quarantine is the most recently live +/// identity); unparseable entries are skipped, staying on disk as forensics. +/// Directory-read and non-UTF-8 policy match `list_publish_markers`. fn find_adoptable_quarantine( final_path: &std::path::Path, ) -> Option<(std::path::PathBuf, String, Keypair)> { @@ -1292,20 +1305,24 @@ fn find_adoptable_quarantine( .unwrap_or("identity.pem"); let prefix = quarantined_prefix(stem); let entries = std::fs::read_dir(dir).ok()?; - let mut quarantines: Vec<(std::path::PathBuf, Option)> = entries + let now = std::time::SystemTime::now(); + let mut quarantines: Vec<(std::path::PathBuf, std::time::SystemTime)> = entries .filter_map(|e| e.ok()) .filter(|e| { e.file_name() .to_str() .is_some_and(|n| n.starts_with(&prefix)) }) - .map(|e| { - let mtime = e.metadata().and_then(|m| m.modified()).ok(); - (e.path(), mtime) + .filter_map(|e| { + // The adoption age bound (see the doc comment): an unreadable + // mtime is skipped outright, and a future mtime (clock skew) + // reads as age zero, matching the sweep's young side. + let mtime = e.metadata().and_then(|m| m.modified()).ok()?; + let age = now.duration_since(mtime).unwrap_or_default(); + (age < CLAIM_SWEEP_MIN_AGE).then_some((e.path(), mtime)) }) .collect(); - // Newest first; None (unreadable mtime) orders before every Some under - // Option's Ord, so the descending sort puts it last. + // Newest first. quarantines.sort_by_key(|q| std::cmp::Reverse(q.1)); for (path, _) in quarantines { let Ok(pem) = std::fs::read_to_string(&path) else { @@ -1489,18 +1506,28 @@ fn boot_load_key( } } -/// Wait until no `.{stem}.recovering.*` claim remains beside `final_path`, -/// polling on `load_racing_key`'s cadence (~2ms), bounded by -/// `KEY_RACE_DEADLINE` (#194, U3). Why the wait exists: a fallback winner -/// that failed its commit check was demoted by a recovering peer, and that -/// peer is mid-round — it quarantines the (old) final, republishes a fresh -/// key, and only then clears its claim. A demoted winner that re-loaded -/// before the claim cleared could observe the pre-quarantine key and diverge -/// from what the peer ends up publishing; waiting for claim clearance makes -/// the follow-up load observe the settled state. Deadline expiry is NOT an -/// error: a claim left by a recovery that itself crashed never clears, and -/// the caller's load path copes with whatever state remains (loudly, if the -/// key is unreadable). An unreadable directory reads as no claims, matching +/// Wait until no `.{stem}.recovering.*` claim YOUNGER than +/// `CLAIM_SWEEP_MIN_AGE` remains beside `final_path`, polling on +/// `load_racing_key`'s cadence (~2ms), bounded by `CLAIM_SWEEP_MIN_AGE` plus +/// a one-second margin (#194, U3; F3). Why the wait exists: a fallback +/// winner that failed its commit check was demoted by a recovering peer, and +/// that peer is mid-round — it quarantines the (old) final, republishes a +/// fresh key, and only then clears its claim. A demoted winner that +/// re-loaded before the claim cleared could observe the pre-quarantine key +/// and diverge from what the peer ends up publishing; waiting for claim +/// clearance makes the follow-up load observe the settled state. +/// +/// The waiter and the sweep share ONE liveness rule (F3): a claim is live +/// only while younger than `CLAIM_SWEEP_MIN_AGE`, the protocol's +/// claim-liveness floor. An AGED claim is orphaned residue by definition and +/// does not hold the waiter, while a young claim holds it up to the floor +/// (the old `KEY_RACE_DEADLINE` bound expired while a slow mount's live +/// round could still be in flight). The margin lets a claim born at wait +/// start age out naturally, so bound expiry means only unreadable-mtime +/// residue remains. Bound expiry is NOT an error: the caller's load path +/// copes with whatever state remains (loudly, if the key is unreadable). An +/// unreadable claim mtime reads as young (hold; fail safe, matching the +/// sweep), and an unreadable directory reads as no claims, matching /// `list_publish_markers`' policy. fn wait_for_recovery_claims(final_path: &std::path::Path) { let dir = final_path @@ -1512,13 +1539,26 @@ fn wait_for_recovery_claims(final_path: &std::path::Path) { .and_then(|s| s.to_str()) .unwrap_or("identity.pem"); let prefix = recovering_prefix(stem); - let deadline = std::time::Instant::now() + KEY_RACE_DEADLINE; + let deadline = + std::time::Instant::now() + CLAIM_SWEEP_MIN_AGE + std::time::Duration::from_secs(1); loop { let claimed = std::fs::read_dir(dir).is_ok_and(|entries| { entries.filter_map(|e| e.ok()).any(|e| { - e.file_name() + if !e + .file_name() .to_str() .is_some_and(|n| n.starts_with(&prefix)) + { + return false; + } + // The shared liveness rule (see the doc comment): only a + // young claim holds the waiter; unreadable mtime = young. + let age = e + .metadata() + .and_then(|m| m.modified()) + .ok() + .and_then(|mtime| std::time::SystemTime::now().duration_since(mtime).ok()); + age.is_none_or(|age| age < CLAIM_SWEEP_MIN_AGE) }) }); if !claimed || std::time::Instant::now() >= deadline { @@ -1621,6 +1661,34 @@ fn sweep_stale_markers(final_path: &std::path::Path, sweep_sync: fn() -> std::io } } +/// Fire-and-forget companion to the boot sweep's young-claim sparing (F2): +/// one delayed re-sweep of `key_path`, `CLAIM_SWEEP_MIN_AGE` plus a +/// two-second margin into uptime. The boot-time sweep must SPARE a young +/// `.recovering.` claim (it may gate a live round), but a spared orphan is +/// never re-examined in-process, and once it ages on disk a LATER content +/// failure of the good final would match the crash signature (content-bad +/// beside an aged claimable) and silently regenerate where marker-less +/// corruption must fail loudly. By the time this fires, any claim the boot +/// sweep spared has aged past the liveness floor, so `sweep_stale_markers` +/// clears it and aged residue cannot coexist with a long-healthy final. The +/// node never waits on this task; failures stay at warn (the sweep itself +/// warns internally when it skips). +fn spawn_delayed_claim_resweep(key_path: std::path::PathBuf) -> tokio::task::JoinHandle<()> { + tokio::spawn(async move { + tokio::time::sleep(CLAIM_SWEEP_MIN_AGE + std::time::Duration::from_secs(2)).await; + let sweep_path = key_path.clone(); + if let Err(e) = + tokio::task::spawn_blocking(move || sweep_stale_markers(&sweep_path, || Ok(()))).await + { + warn!( + path = %key_path.display(), + err = %e, + "delayed identity-claim re-sweep task failed" + ); + } + }) +} + /// Compile-time fault-injection seam for `publish_key_atomically`, extending /// the `before_link` closure precedent: every hook is a no-op `Ok` in /// production (`PublishFaults::NONE`); tests swap in a hook returning `Err` to @@ -2637,12 +2705,13 @@ fn load_or_create_keypair_with( // stranded in a quarantine: the recoverer's restore republish // errored, or the recoverer crashed between its quarantine rename // and the restore. Generating here would mint a fresh DID over a - // still-live identity, so a parseable quarantine is republished and - // adopted instead: adoption preserves identity continuity across a - // crashed or failed restore. An unparseable quarantine is the - // expected crash state and stays on disk as forensics; only when no - // quarantine parses (or none exists) does the boot fall through to - // generate. The republish holds no-clobber on every tier, so Lost + // still-live identity, so a YOUNG parseable quarantine (the current + // round's; see find_adoptable_quarantine's age bound) is republished + // and adopted instead: adoption preserves identity continuity across + // a crashed or failed restore. An unparseable quarantine is the + // expected crash state and stays on disk as forensics, as does an + // AGED parseable one (history, never resurrected); only when no + // quarantine qualifies does the boot fall through to generate. The republish holds no-clobber on every tier, so Lost // means a concurrent starter durably published meanwhile: defer to // it and keep the quarantine, exactly like the G2b restore. A // republish error surfaces loudly with the quarantine kept: minting @@ -4810,13 +4879,16 @@ mod identity_key_tests { ); } - // #194 (U3): a demoted winner facing a STALE `.recovering.` claim (a - // recovery that crashed before clearing it) must wait the claim out on - // the bounded KEY_RACE_DEADLINE and then return Lost, not error. The - // marker is stolen WITHOUT a real recovery (final left in place), so the - // follow-up load resolves the winner's OWN key: the demotion-is-safe - // property. RED before U3: the publish returns Won immediately and the - // elapsed floor fails. + // #194 (U3, reshaped by F3): a demoted winner facing an AGED + // `.recovering.` claim (a recovery that crashed before clearing it) must + // SKIP it quickly and return Lost, not error: the waiter now shares the + // sweep's liveness rule, and a claim past CLAIM_SWEEP_MIN_AGE is + // orphaned residue by definition. The marker is stolen WITHOUT a real + // recovery (final left in place), so the follow-up load resolves the + // winner's OWN key: the demotion-is-safe property. + // waiter_holds_for_live_young_claims covers the young-claim hold side. + // RED before F3: the age-blind waiter rides the full KEY_RACE_DEADLINE + // (~5s) on the aged claim and the elapsed ceiling fails. #[test] fn demoted_winner_waits_out_a_stale_claim() { let dir = tempfile::tempdir().expect("tempdir"); @@ -4835,8 +4907,12 @@ mod identity_key_tests { "A's window must be bracketed: {markers:?}" ); std::fs::remove_file(steal_dir.join(&markers[0])).expect("steal A's marker"); - std::fs::write(steal_dir.join(".identity.pem.recovering.44444"), b"") - .expect("plant stale claim"); + let claim = steal_dir.join(".identity.pem.recovering.44444"); + std::fs::write(&claim, b"").expect("plant stale claim"); + // The planted claim models an orphan of a LONG-crashed recovery, + // so it must be aged; a fresh claim reads as live and holds the + // waiter (waiter_holds_for_live_young_claims). + age_beyond_claim_sweep(&claim); }; let faults = PublishFaults { link: || Err(std::io::ErrorKind::Unsupported.into()), @@ -4855,8 +4931,86 @@ mod identity_key_tests { let waited = started.elapsed(); assert!( - waited >= std::time::Duration::from_secs(4), - "the demotion must wait out the stale claim on the deadline, waited {waited:?}" + waited < std::time::Duration::from_secs(4), + "an aged (orphaned) claim must not hold the demoted winner, waited {waited:?}" + ); + let on_disk = super::load_existing_key(&key_path).expect("final parses"); + assert_eq!( + format!("{}", kp.did()), + format!("{}", on_disk.did()), + "with the final untouched the demoted winner must resolve its own key" + ); + } + + // F3: the waiter's hold must follow the claim-liveness floor + // (CLAIM_SWEEP_MIN_AGE), not KEY_RACE_DEADLINE: a live claim's round is + // allowed the full liveness floor to finish, so a demoted winner that + // gives up at the 5s deadline can re-load the pre-quarantine key while a + // slow-mount recovery is still mid-round. Plant a FRESH claim, hold it + // live for ~7s (past the old deadline), then remove it mid-wait: the + // waiter must still be holding past 5s and return shortly after the + // removal, well under the CLAIM_SWEEP_MIN_AGE-plus-margin bound. RED + // before F3: the waiter expires at KEY_RACE_DEADLINE and the elapsed + // floor fails. + #[test] + fn waiter_holds_for_live_young_claims() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + + let fired = std::sync::atomic::AtomicBool::new(false); + let steal_dir = dir.path().to_path_buf(); + let before_commit = move || { + if fired.swap(true, std::sync::atomic::Ordering::SeqCst) { + return; + } + let markers = names_containing(&steal_dir, ".publishing."); + assert_eq!( + markers.len(), + 1, + "A's window must be bracketed: {markers:?}" + ); + std::fs::remove_file(steal_dir.join(&markers[0])).expect("steal A's marker"); + std::fs::write(steal_dir.join(".identity.pem.recovering.55555"), b"") + .expect("plant fresh (live) claim"); + }; + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + ..PublishFaults::NONE + }; + + // Model the live round resolving mid-wait: once the claim appears, + // keep it live for 7s (past the old 5s deadline, under the ~11s + // bound), then clear it as a finishing recoverer would. + let claim_path = dir.path().join(".identity.pem.recovering.55555"); + let remover_path = claim_path.clone(); + let remover = std::thread::spawn(move || { + while !remover_path.exists() { + std::thread::sleep(std::time::Duration::from_millis(5)); + } + std::thread::sleep(std::time::Duration::from_secs(7)); + std::fs::remove_file(&remover_path).expect("clear the live claim mid-wait"); + }); + + let started = std::time::Instant::now(); + let kp = load_or_create_keypair_with( + &key_path, + &|| {}, + &before_commit, + faults, + RecoverySeam::NONE, + ) + .expect("a demoted winner behind a live claim must still resolve a key"); + let waited = started.elapsed(); + remover.join().expect("remover joins"); + + assert!( + waited > super::KEY_RACE_DEADLINE + std::time::Duration::from_millis(1500), + "a young claim must hold the waiter past the old deadline, waited {waited:?}" + ); + assert!( + waited < std::time::Duration::from_secs(10), + "the waiter must release on the claim's removal, not ride the full \ + bound: waited {waited:?}" ); let on_disk = super::load_existing_key(&key_path).expect("final parses"); assert_eq!( @@ -4947,6 +5101,72 @@ mod identity_key_tests { ); } + // F2 (function level): the boot sweep SPARES a young claim (it may gate + // a live round), so orphaned residue can outlive a healthy boot; once + // the claim ages past CLAIM_SWEEP_MIN_AGE a SECOND sweep over the same + // final must clear it, or the residue later pairs with a real content + // failure of the good final and misclassifies marker-less corruption as + // a crash. GREEN at introduction (the sweep is already age-correct); + // load-bearing by mutation check: inverting the sweep's age gate turns + // it RED. spawn_delayed_claim_resweep is the boot wiring that provides + // the second sweep in-process. + #[test] + fn second_sweep_clears_aged_orphan_claims() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let pem = Keypair::generate().to_pem().expect("pem"); + std::fs::write(&key_path, pem.as_bytes()).expect("seed valid final"); + std::fs::set_permissions(&key_path, std::fs::Permissions::from_mode(0o600)) + .expect("0600 final"); + let claim = dir.path().join(".identity.pem.recovering.33333"); + std::fs::write(&claim, b"").expect("seed young claim"); + + super::sweep_stale_markers(&key_path, || Ok(())); + assert!( + claim.exists(), + "the first sweep must spare the young (possibly live) claim" + ); + + age_beyond_claim_sweep(&claim); + super::sweep_stale_markers(&key_path, || Ok(())); + assert!( + !claim.exists(), + "a second sweep must clear the claim once it ages into an orphan" + ); + } + + // F2 (wiring): the delayed boot re-sweep must actually fire and sweep. + // Paused tokio time drives the real CLAIM_SWEEP_MIN_AGE-plus-margin + // sleep instantly; the claim is pre-aged on disk because paused time + // advances the tokio clock, not SystemTime mtimes. This covers "the + // spawned task runs the sweep after the delay"; + // second_sweep_clears_aged_orphan_claims covers the age semantics. + #[tokio::test(start_paused = true)] + async fn delayed_resweep_clears_aged_orphan_claims() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let pem = Keypair::generate().to_pem().expect("pem"); + std::fs::write(&key_path, pem.as_bytes()).expect("seed valid final"); + std::fs::set_permissions(&key_path, std::fs::Permissions::from_mode(0o600)) + .expect("0600 final"); + let claim = dir.path().join(".identity.pem.recovering.66666"); + std::fs::write(&claim, b"").expect("seed claim"); + age_beyond_claim_sweep(&claim); + + super::spawn_delayed_claim_resweep(key_path.clone()) + .await + .expect("the re-sweep task must complete"); + + assert!( + !claim.exists(), + "the delayed re-sweep must clear the aged orphan claim" + ); + assert!( + super::load_existing_key(&key_path).is_ok(), + "the final must survive the re-sweep intact" + ); + } + // The G1 age gate reads the CLAIM file's mtime, but a claim is made by // RENAMING a marker, and rename preserves the source's mtime: a claim // made from an hours-old stale marker would be born already past @@ -5519,6 +5739,74 @@ mod identity_key_tests { ); } + // F1 MUST-NOT (age bound): adoption exists for the CURRENT round only (a + // recoverer that crashed or failed seconds ago), so a parseable + // quarantine AGED past CLAIM_SWEEP_MIN_AGE is history, not a stranded + // round: kept forensics after a Lost restore, or an old identity the + // operator already moved past. Adopting it would resurrect a dead DID + // whenever the final later goes missing. The aged quarantine must + // survive untouched as forensics while boot generates fresh. RED before + // the age bound: the aged quarantine is adopted. + #[test] + fn aged_quarantine_is_never_adopted() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let stranded = Keypair::generate(); + let pem = stranded.to_pem().expect("pem"); + let quarantine = dir.path().join(".identity.pem.quarantined.99999.0"); + std::fs::write(&quarantine, pem.as_bytes()).expect("seed aged quarantine"); + age_beyond_claim_sweep(&quarantine); + + let kp = load_or_create_keypair(&key_config(&key_path)) + .expect("an aged quarantine must not block a fresh generation"); + + assert_ne!( + format!("{}", kp.did()), + format!("{}", stranded.did()), + "an aged quarantine is history and must never be resurrected" + ); + assert_eq!( + std::fs::read(&quarantine).expect("quarantine still on disk"), + pem.as_bytes(), + "the aged quarantine must survive byte-for-byte as forensics" + ); + } + + // F1 (operator procedure): deleting identity.pem is the documented way + // to mint a fresh DID. With an old parseable quarantine sitting beside + // the final as forensics, the post-delete boot must GENERATE a new + // identity, not adopt the quarantined one; unbounded adoption silently + // defeats the procedure. RED before the age bound: the old quarantine's + // DID comes back. + #[test] + fn operator_fresh_identity_procedure_works() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let first = load_or_create_keypair(&key_config(&key_path)).expect("provision identity"); + let old = Keypair::generate(); + let old_pem = old.to_pem().expect("pem"); + let quarantine = dir.path().join(".identity.pem.quarantined.88888.0"); + std::fs::write(&quarantine, old_pem.as_bytes()).expect("seed forensic quarantine"); + age_beyond_claim_sweep(&quarantine); + + // The operator action: delete the final to get a fresh DID. + std::fs::remove_file(&key_path).expect("operator deletes the final"); + + let kp = load_or_create_keypair(&key_config(&key_path)) + .expect("the post-delete boot must provision a fresh identity"); + + assert_ne!( + format!("{}", kp.did()), + format!("{}", old.did()), + "the old quarantined identity must not be resurrected" + ); + assert_ne!( + format!("{}", kp.did()), + format!("{}", first.did()), + "the operator must get a genuinely fresh DID" + ); + } + // F1 MUST-NOT: an UNPARSEABLE quarantine is the expected crash state and // stays as forensics; boot must fall through to a fresh generation, not // wedge on it or delete it. From facc024fe194dae4f7ab8cc90892e45953b0c178 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 21 Jul 2026 19:26:37 -0500 Subject: [PATCH 36/41] fix(node): sweep every healthy return, supersede outcompeted quarantines, fail closed on unsettled rounds Every arm that returns a loaded, adopted, or reloaded key now funnels through one sweep-then-return closure, so stale markers cannot outlive any healthy boot path. Quarantines that provably lost their round (a Lost restore, or siblings of an adopted quarantine) move to an inert superseded class: bytes preserved as forensics, never adoptable, so the delete-identity-for-fresh-DID operator procedure can no longer resurrect an outcompeted key. The demotion waiter fails closed when its bound expires while a live young claim persists: an unsettled round now surfaces as a loud failed publish instead of resolving a key that disk may be about to contradict. --- crates/gitlawb-node/src/lib.rs | 620 ++++++++++++++++++++++++++++++--- 1 file changed, 570 insertions(+), 50 deletions(-) diff --git a/crates/gitlawb-node/src/lib.rs b/crates/gitlawb-node/src/lib.rs index 576dfce0..e156d1e7 100644 --- a/crates/gitlawb-node/src/lib.rs +++ b/crates/gitlawb-node/src/lib.rs @@ -1271,11 +1271,111 @@ fn recovering_prefix(stem: &str) -> String { } /// `".{stem}.quarantined."`: the single source of truth for the quarantine -/// name class. Same policy as `publishing_prefix`. +/// name class. Same policy as `publishing_prefix`. Class distinction (N2/N4): +/// a `.quarantined.` file is possibly-current residue, the crashed round's +/// own key, adoptable while young; a `.superseded.` file +/// (`superseded_prefix`) is provably outcompeted and forensics only. fn quarantined_prefix(stem: &str) -> String { format!(".{stem}.quarantined.") } +/// `".{stem}.superseded."`: the single source of truth for the SUPERSEDED +/// quarantine class (N2/N4). A quarantine moves into this class when its key +/// provably LOST its round: a restore republish returned Lost (a concurrent +/// winner durably published meanwhile), or a sibling quarantine won the +/// adoption. Superseded files are forensics only: never adoptable +/// (`find_adoptable_quarantine` scans `.quarantined.` alone), never part of +/// the boot crash signature, and never swept; every consumer matches its own +/// prefix class, so this class is inert by construction. +fn superseded_prefix(stem: &str) -> String { + format!(".{stem}.superseded.") +} + +/// Rename `quarantine` into the superseded class +/// (`.{stem}.superseded.{pid}.{n}`), probing bounded fresh names per the +/// KEY_TEMP_ATTEMPTS discipline (skip an existing name, never clobber it). +/// A NotFound source means the quarantine resolved concurrently and there is +/// nothing left to reclassify. Best-effort: if every attempt fails the file +/// stays a plain quarantine, still adoptable while young, and a warn names +/// that hazard. +fn supersede_quarantine(final_path: &std::path::Path, quarantine: &std::path::Path) { + let dir = final_path + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or_else(|| std::path::Path::new(".")); + let stem = final_path + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or("identity.pem"); + let prefix = superseded_prefix(stem); + for n in 0..KEY_TEMP_ATTEMPTS { + let dest = dir.join(format!("{prefix}{}.{n}", std::process::id())); + if dest.exists() { + continue; + } + match std::fs::rename(quarantine, &dest) { + Ok(()) => return, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return, + Err(_) => continue, + } + } + warn!( + quarantine = %quarantine.display(), + "could not rename an outcompeted quarantine into the superseded class; \ + it remains adoptable while young" + ); +} + +/// Rename every YOUNG parseable `.{stem}.quarantined.*` sibling other than +/// `adopted` into the superseded class (N4). The adoption decided this round +/// in favor of the adopted key, so any other young parseable quarantine is +/// provably outcompeted; left adoptable, it would resurrect a different +/// historical DID the next time the final goes missing. Unparseable entries +/// stay plain quarantines (crash forensics, never adoptable anyway), and so +/// do AGED parseable ones (already outside the adoption bound; unreadable +/// mtime reads as aged, matching `find_adoptable_quarantine`). Best-effort +/// throughout. +fn supersede_sibling_quarantines(final_path: &std::path::Path, adopted: &std::path::Path) { + let dir = final_path + .parent() + .filter(|p| !p.as_os_str().is_empty()) + .unwrap_or_else(|| std::path::Path::new(".")); + let stem = final_path + .file_name() + .and_then(|s| s.to_str()) + .unwrap_or("identity.pem"); + let prefix = quarantined_prefix(stem); + let Ok(entries) = std::fs::read_dir(dir) else { + return; + }; + let now = std::time::SystemTime::now(); + for entry in entries.filter_map(|e| e.ok()) { + if !entry + .file_name() + .to_str() + .is_some_and(|n| n.starts_with(&prefix)) + { + continue; + } + let path = entry.path(); + if path == *adopted { + continue; + } + let Ok(mtime) = entry.metadata().and_then(|m| m.modified()) else { + continue; + }; + let age = now.duration_since(mtime).unwrap_or_default(); + if age >= CLAIM_SWEEP_MIN_AGE { + continue; + } + let parseable = + std::fs::read_to_string(&path).is_ok_and(|pem| Keypair::from_pem(&pem).is_ok()); + if parseable { + supersede_quarantine(final_path, &path); + } + } +} + /// The newest `.{stem}.quarantined.*` entry beside `final_path` YOUNGER than /// `CLAIM_SWEEP_MIN_AGE` whose bytes parse as a keypair, with those bytes: a /// COMPLETED key the CURRENT round's crashed or failed recovery left @@ -1523,13 +1623,19 @@ fn boot_load_key( /// does not hold the waiter, while a young claim holds it up to the floor /// (the old `KEY_RACE_DEADLINE` bound expired while a slow mount's live /// round could still be in flight). The margin lets a claim born at wait -/// start age out naturally, so bound expiry means only unreadable-mtime -/// residue remains. Bound expiry is NOT an error: the caller's load path -/// copes with whatever state remains (loudly, if the key is unreadable). An +/// start age out naturally. +/// +/// A wait that CLEARS (claims removed, or only aged orphans remain) returns +/// `Ok`. Bound expiry with a YOUNG claim still present returns an `Err` +/// naming the claim file(s) (N3): the recovery round did not settle within +/// the bound, and a demoted publisher that loaded the final there could +/// return a pre-settlement identity into `run()` while the still-live +/// recoverer settles disk on a different one, the exact divergence the +/// protocol forbids, so the caller must fail closed instead of loading. An /// unreadable claim mtime reads as young (hold; fail safe, matching the /// sweep), and an unreadable directory reads as no claims, matching /// `list_publish_markers`' policy. -fn wait_for_recovery_claims(final_path: &std::path::Path) { +fn wait_for_recovery_claims(final_path: &std::path::Path) -> Result<()> { let dir = final_path .parent() .filter(|p| !p.as_os_str().is_empty()) @@ -1539,30 +1645,46 @@ fn wait_for_recovery_claims(final_path: &std::path::Path) { .and_then(|s| s.to_str()) .unwrap_or("identity.pem"); let prefix = recovering_prefix(stem); - let deadline = - std::time::Instant::now() + CLAIM_SWEEP_MIN_AGE + std::time::Duration::from_secs(1); + let bound = CLAIM_SWEEP_MIN_AGE + std::time::Duration::from_secs(1); + let deadline = std::time::Instant::now() + bound; loop { - let claimed = std::fs::read_dir(dir).is_ok_and(|entries| { - entries.filter_map(|e| e.ok()).any(|e| { - if !e - .file_name() - .to_str() - .is_some_and(|n| n.starts_with(&prefix)) - { - return false; - } - // The shared liveness rule (see the doc comment): only a - // young claim holds the waiter; unreadable mtime = young. - let age = e - .metadata() - .and_then(|m| m.modified()) - .ok() - .and_then(|mtime| std::time::SystemTime::now().duration_since(mtime).ok()); - age.is_none_or(|age| age < CLAIM_SWEEP_MIN_AGE) + let live: Vec = std::fs::read_dir(dir) + .map(|entries| { + entries + .filter_map(|e| e.ok()) + .filter(|e| { + if !e + .file_name() + .to_str() + .is_some_and(|n| n.starts_with(&prefix)) + { + return false; + } + // The shared liveness rule (see the doc comment): only + // a young claim holds the waiter; unreadable mtime = + // young. + let age = e + .metadata() + .and_then(|m| m.modified()) + .ok() + .and_then(|mtime| { + std::time::SystemTime::now().duration_since(mtime).ok() + }); + age.is_none_or(|age| age < CLAIM_SWEEP_MIN_AGE) + }) + .map(|e| e.file_name().to_string_lossy().into_owned()) + .collect() }) - }); - if !claimed || std::time::Instant::now() >= deadline { - return; + .unwrap_or_default(); + if live.is_empty() { + return Ok(()); + } + if std::time::Instant::now() >= deadline { + anyhow::bail!( + "recovery round did not settle within the {bound:?} wait bound: live \ + recovery claim(s) {live:?} still present beside {}", + final_path.display() + ); } std::thread::sleep(std::time::Duration::from_millis(2)); } @@ -2136,9 +2258,14 @@ fn publish_key_fallback( // (NotFound after a peer's claim rename, or any other error) means // ownership of the round cannot be proven: a recovering peer may have // quarantined our final and republished, so returning Won could report - // key A while disk holds key B. Demote to Lost instead; demotion is - // always safe, because absent real interference the caller's Lost arm - // re-loads our own just-published key (same identity). + // key A while disk holds key B. Demote to Lost instead. Demotion is safe + // ONLY for a SETTLED round (N3): the caller's Lost arm re-loads the + // final, and that load observes the recoverer's outcome only once the + // recoverer's claims have cleared, so the demoted publish first waits + // them out. An UNSETTLED round past the wait bound (a young claim still + // live) fails the publish closed with a loud error instead: loading + // there could return a pre-settlement key while the still-live recoverer + // settles disk on a different identity. before_commit(); match std::fs::remove_file(&marker) { Ok(()) => { @@ -2156,7 +2283,15 @@ fn publish_key_fallback( "fallback publish lost its marker before commit; a recovering peer owns \ this round, demoting the publish to lost" ); - wait_for_recovery_claims(final_path); + wait_for_recovery_claims(final_path).with_context(|| { + format!( + "fallback publish of {} was demoted by a recovering peer, and the \ + recovery round did not settle within the wait bound; failing the \ + publish closed instead of loading a possibly pre-settlement key \ + (marker removal failed: {e})", + final_path.display() + ) + })?; Ok(KeyPublish::Lost) } } @@ -2436,7 +2571,9 @@ fn recover_crashed_publish( concurrent healthy boot's age-gated sweep may remove this \ LIVE claim mid-round and early-wake the demoted publisher \ it gates (bounded to churn, not divergence, by the \ - post-claim re-parse and the restore's no-clobber republish)" + post-claim re-parse, the restore's no-clobber republish, \ + and the demoted publisher's waiter failing closed on an \ + unsettled round)" ); } claims.push(dest); @@ -2580,8 +2717,13 @@ fn recover_crashed_publish( // it buys no forensics. The keypair returned is the one already parsed // from the quarantined bytes, not a re-read of the final, which could // race a writer that lands after our republish. Lost means someone else - // durably published meanwhile: keep the quarantine as forensics and - // defer to the settled final. A publish error keeps the quarantine, + // durably published meanwhile: the quarantined key provably LOST its + // round, so it is renamed into the superseded class (bytes preserved as + // forensics, never adoptable; a young parseable quarantine left in the + // adoptable class here would let the operator's delete-final-for-a- + // fresh-DID procedure resurrect the losing key, N2) and the boot defers + // to the settled final. A publish error keeps the quarantine ADOPTABLE + // (the next boot's F1 adoption is the remedy for a failed restore), // releases the claims, and surfaces loudly, nothing destroyed. An // unparseable quarantine is the expected crash state and falls through // to regenerate. @@ -2608,6 +2750,7 @@ fn recover_crashed_publish( return Ok(Recovery::Reloaded(Box::new(Ok(kp)))); } Ok(KeyPublish::Lost) => { + supersede_quarantine(final_path, quarantine); release_recovery_claims(final_path, &claims); return Ok(Recovery::Reloaded(Box::new(load_racing_key(final_path)))); } @@ -2667,6 +2810,19 @@ fn load_or_create_keypair_with( let mut claims: Vec = Vec::new(); let mut recovery_allowed = true; + // N1 invariant: EVERY arm that returns a successfully resolved keypair + // (loaded, reloaded, adopted, or generated) must run the durability-gated + // stale-marker sweep before returning, by funneling through this closure. + // A healthy return that skips the sweep leaves unclaimable markers or + // aged orphan claims beside a good final, and a LATER content failure of + // that final would pair with the leftover residue and misclassify plain + // corruption as a recoverable crash. Route any future healthy return + // through here too, so no arm can forget the sweep. + let finish_healthy = |kp: Keypair| -> Result { + sweep_stale_markers(key_path, seam.sweep_sync); + Ok(kp) + }; + // At most two passes: the second exists only so a Lost-arm recovery can // retry the generate+publish once. for _pass in 0..2 { @@ -2677,8 +2833,7 @@ fn load_or_create_keypair_with( match boot_load_key(key_path, recovery_allowed, seam.load_fault) { BootLoad::Loaded(kp) => { release_recovery_claims(key_path, &claims); - sweep_stale_markers(key_path, seam.sweep_sync); - return Ok(*kp); + return finish_healthy(*kp); } // Only emitted while recovery is still allowed (crash_exit is // gated on it). @@ -2689,7 +2844,7 @@ fn load_or_create_keypair_with( recovery_allowed = false; // Fall through to regenerate and publish. } - Recovery::Reloaded(result) => return *result, + Recovery::Reloaded(result) => return (*result).and_then(&finish_healthy), } } BootLoad::Failed(e) => { @@ -2724,23 +2879,28 @@ fn load_or_create_keypair_with( // durably lives at the final, so keeping it buys no // forensics (the G2b restore's removal rationale). let _ = std::fs::remove_file(&quarantine); + // N4: any OTHER young parseable quarantine sibling lost + // this round to the adopted key; move each into the + // superseded class so a later missing final cannot + // resurrect a different historical DID. Unparseable + // siblings stay plain quarantines as crash forensics. + supersede_sibling_quarantines(key_path, &quarantine); #[cfg(unix)] { let _ = fsync_parent_dir(key_path); } release_recovery_claims(key_path, &claims); - sweep_stale_markers(key_path, seam.sweep_sync); info!( path = %key_path.display(), did = %kp.did(), "adopted a completed identity stranded in quarantine" ); - return Ok(kp); + return finish_healthy(kp); } Ok(KeyPublish::Lost) => { let result = load_racing_key(key_path); release_recovery_claims(key_path, &claims); - return result; + return result.and_then(&finish_healthy); } Err(e) => { release_recovery_claims(key_path, &claims); @@ -2790,16 +2950,14 @@ fn load_or_create_keypair_with( match published { KeyPublish::Won => { release_recovery_claims(key_path, &claims); - sweep_stale_markers(key_path, seam.sweep_sync); info!(path = %key_path.display(), did = %kp.did(), "generated new node identity"); - return Ok(kp); + return finish_healthy(kp); } KeyPublish::Lost => { match boot_load_key(key_path, recovery_allowed, seam.load_fault) { BootLoad::Loaded(kp) => { release_recovery_claims(key_path, &claims); - sweep_stale_markers(key_path, seam.sweep_sync); - return Ok(*kp); + return finish_healthy(*kp); } // Only emitted while recovery is still allowed (crash_exit // is gated on it). @@ -2812,7 +2970,7 @@ fn load_or_create_keypair_with( } Ok(Recovery::Reloaded(result)) => { release_recovery_claims(key_path, &claims); - return *result; + return (*result).and_then(&finish_healthy); } Err(err) => { release_recovery_claims(key_path, &claims); @@ -5020,6 +5178,103 @@ mod identity_key_tests { ); } + // N3: a demoted fallback winner whose wait bound expires while a LIVE + // young claim still persists has NOT observed a settled round: loading + // the final there can return a pre-settlement key into run() while the + // still-live recoverer settles disk on a different identity, the exact + // divergence the protocol forbids. The publish must fail closed (a loud + // Err naming the unsettled round), never resolve Lost and load. A + // refresher thread keeps the claim young past the full wait bound, + // modeling a recovery round still live (or stuck) beyond it. RED before + // N3: the waiter expires silently and the demoted winner returns its + // own key. + #[test] + fn unsettled_round_fails_closed() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + + let fired = std::sync::atomic::AtomicBool::new(false); + let steal_dir = dir.path().to_path_buf(); + let final_bytes = std::sync::Arc::new(std::sync::Mutex::new(Vec::::new())); + let final_at_demotion = final_bytes.clone(); + let observe_path = key_path.clone(); + let before_commit = move || { + if fired.swap(true, std::sync::atomic::Ordering::SeqCst) { + return; + } + let markers = names_containing(&steal_dir, ".publishing."); + assert_eq!( + markers.len(), + 1, + "A's window must be bracketed: {markers:?}" + ); + std::fs::remove_file(steal_dir.join(&markers[0])).expect("steal A's marker"); + std::fs::write(steal_dir.join(".identity.pem.recovering.55555"), b"") + .expect("plant live claim"); + *final_at_demotion.lock().expect("final bytes lock") = + std::fs::read(&observe_path).expect("final is complete at the commit check"); + }; + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + ..PublishFaults::NONE + }; + + // Keep the claim YOUNG for the waiter's whole bound: a recovery + // round that never settles. Without the refresher the claim would + // age past CLAIM_SWEEP_MIN_AGE before the bound expires and the + // wait would clear as aged-orphan residue (the Ok side). + let claim_path = dir.path().join(".identity.pem.recovering.55555"); + let stop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let stop_refresher = stop.clone(); + let refresher_path = claim_path.clone(); + let refresher = std::thread::spawn(move || { + while !refresher_path.exists() { + if stop_refresher.load(std::sync::atomic::Ordering::SeqCst) { + return; + } + std::thread::sleep(std::time::Duration::from_millis(5)); + } + while !stop_refresher.load(std::sync::atomic::Ordering::SeqCst) { + let _ = std::fs::File::options() + .write(true) + .open(&refresher_path) + .and_then(|f| f.set_modified(std::time::SystemTime::now())); + std::thread::sleep(std::time::Duration::from_millis(500)); + } + }); + + let result = load_or_create_keypair_with( + &key_path, + &|| {}, + &before_commit, + faults, + RecoverySeam::NONE, + ); + stop.store(true, std::sync::atomic::Ordering::SeqCst); + refresher.join().expect("refresher joins"); + + let err = match result { + Err(e) => e, + Ok(_) => { + panic!("an unsettled round past the wait bound must fail closed, not resolve a key") + } + }; + let msg = format!("{err:#}"); + assert!( + msg.contains(".identity.pem.recovering.55555"), + "the error must name the unsettled claim file: {msg}" + ); + assert!( + msg.contains("settle"), + "the error must name the unsettled round: {msg}" + ); + assert_eq!( + std::fs::read(&key_path).expect("final still on disk"), + *final_bytes.lock().expect("final bytes lock"), + "the failed-closed publish must leave the final untouched" + ); + } + // #194 (U4): a HEALTHY load must sweep stale `.publishing.` markers and // `.recovering.` claims, or they linger to misclassify a much-later real // corruption of a good key as an interrupted first write. The claim is @@ -5807,6 +6062,162 @@ mod identity_key_tests { ); } + // N2: a quarantine kept by a LOST restore is provably outcompeted (a + // concurrent winner durably published while its key sat quarantined), + // yet it is young and parseable: exactly what adoption prefers. Left in + // the `.quarantined.` class, the operator's + // delete-identity.pem-for-a-fresh-DID procedure resurrects the LOSING + // key instead of minting fresh. The Lost restore must rename its kept + // quarantine into the `.superseded.` class: bytes preserved as + // forensics, never adoptable. RED before N2: the post-delete boot + // adopts and returns the losing key A. + #[test] + fn lost_restore_quarantine_is_never_adopted() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + let completed = Keypair::generate(); + let pem_a = completed.to_pem().expect("pem a"); + let kp_b = Keypair::generate(); + let pem_b = kp_b.to_pem().expect("pem b"); + + // restore_never_clobbers_a_concurrent_winner's construction: A's + // completed key is swept into quarantine, B takes the freed name + // inside the quarantine-to-restore window, and the link-hostile + // restore republish of A loses to B. + let heal_path = key_path.clone(); + let pem_a_disk = pem_a.clone(); + let before_quarantine = move || { + std::fs::write(&heal_path, pem_a_disk.as_bytes()) + .expect("complete the final inside the re-parse-to-rename window"); + }; + let steal_path = key_path.clone(); + let pem_b_disk = pem_b.clone(); + let before_restore = move || { + let out = publish_key_atomically( + &steal_path, + pem_b_disk.as_bytes(), + &|| {}, + &|| {}, + PublishFaults::NONE, + ) + .expect("concurrent starter republishes key B at the freed name"); + assert!( + matches!(out, KeyPublish::Won), + "B's publish wins the free name" + ); + }; + let seam = RecoverySeam { + before_quarantine: &before_quarantine, + before_restore: &before_restore, + restore_faults: PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + ..PublishFaults::NONE + }, + ..RecoverySeam::NONE + }; + let kp = load_or_create_keypair_with(&key_path, &|| {}, &|| {}, PublishFaults::NONE, seam) + .expect("the losing restore must converge on B"); + assert_eq!( + format!("{}", kp.did()), + format!("{}", kp_b.did()), + "the recovering boot must converge on the concurrent winner's key B" + ); + + // The operator action: delete the final for a fresh DID. + std::fs::remove_file(&key_path).expect("operator deletes the final"); + let fresh = load_or_create_keypair(&key_config(&key_path)) + .expect("the post-delete boot must provision an identity"); + assert_ne!( + format!("{}", fresh.did()), + format!("{}", completed.did()), + "the key that LOST its round must never be resurrected by adoption" + ); + assert_ne!( + format!("{}", fresh.did()), + format!("{}", kp_b.did()), + "the operator must get a genuinely fresh DID" + ); + // A's bytes survive as forensics, in the never-adoptable class. + assert!( + names_containing(dir.path(), ".quarantined.").is_empty(), + "no adoptable quarantine may survive a lost restore" + ); + let superseded = names_containing(dir.path(), ".superseded."); + assert_eq!( + superseded.len(), + 1, + "the losing key must be kept once, superseded: {superseded:?}" + ); + assert_eq!( + std::fs::read(dir.path().join(&superseded[0])).expect("superseded readable"), + pem_a.as_bytes(), + "the superseded file must preserve the losing key's bytes as forensics" + ); + } + + // N4: adoption decides its round in favor of ONE quarantine, but a + // crashed pile-up can leave several young parseable quarantines side by + // side. Adopting the newest while leaving the siblings adoptable means + // a later missing final resurrects a DIFFERENT historical DID. On Won, + // adoption must rename every other young parseable sibling into the + // `.superseded.` class (bytes intact, never adoptable). RED before N4: + // the sibling stays `.quarantined.` and the post-delete boot adopts it. + #[test] + fn adoption_supersedes_sibling_quarantines() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let older = Keypair::generate(); + let older_pem = older.to_pem().expect("pem"); + let q_old = dir.path().join(".identity.pem.quarantined.99999.0"); + std::fs::write(&q_old, older_pem.as_bytes()).expect("seed older quarantine"); + // A strictly newer mtime on the second quarantine, so newest-first + // adoption is deterministic. + std::thread::sleep(std::time::Duration::from_millis(20)); + let newer = Keypair::generate(); + let q_new = dir.path().join(".identity.pem.quarantined.99999.1"); + std::fs::write(&q_new, newer.to_pem().expect("pem").as_bytes()) + .expect("seed newer quarantine"); + + let kp = load_or_create_keypair(&key_config(&key_path)) + .expect("adoption must resolve the stranded round"); + assert_eq!( + format!("{}", kp.did()), + format!("{}", newer.did()), + "adoption must prefer the newest quarantine" + ); + assert!( + names_containing(dir.path(), ".quarantined.").is_empty(), + "the adopted quarantine is consumed and the sibling must be superseded" + ); + let superseded = names_containing(dir.path(), ".superseded."); + assert_eq!( + superseded.len(), + 1, + "exactly the outcompeted sibling is superseded: {superseded:?}" + ); + assert_eq!( + std::fs::read(dir.path().join(&superseded[0])).expect("superseded readable"), + older_pem.as_bytes(), + "the superseded sibling must preserve its bytes as forensics" + ); + + // The sibling must not come back as a different historical DID once + // the final goes missing: the post-delete boot mints fresh. + std::fs::remove_file(&key_path).expect("operator deletes the final"); + let fresh = load_or_create_keypair(&key_config(&key_path)) + .expect("the post-delete boot must provision an identity"); + assert_ne!( + format!("{}", fresh.did()), + format!("{}", older.did()), + "the superseded sibling must never be resurrected" + ); + assert_ne!( + format!("{}", fresh.did()), + format!("{}", newer.did()), + "the operator must get a genuinely fresh DID" + ); + } + // F1 MUST-NOT: an UNPARSEABLE quarantine is the expected crash state and // stays as forensics; boot must fall through to a fresh generation, not // wedge on it or delete it. @@ -5979,6 +6390,109 @@ mod identity_key_tests { ); } + // N1: EVERY arm of load_or_create_keypair_with that returns a + // successfully resolved keypair must run the stale-marker sweep first, + // including the post-claim-reparse Reloaded arm and the adoption-Lost + // re-load. Residue that lands after the claim step (or beside a missing + // final) otherwise persists past a healthy return, and a LATER content + // failure of the good final would pair with it and misclassify plain + // corruption as a recoverable crash. RED before N1: both arms return + // without sweeping and the planted marker and aged claim survive. + #[test] + fn reloaded_and_adoption_paths_sweep_before_returning() { + // Scenario 1: the post-claim-reparse Reloaded arm + // (post_claim_reparse_aborts_to_reload's construction), with fresh + // residue planted inside the claim window. + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + let completed = Keypair::generate(); + let pem = completed.to_pem().expect("pem"); + let heal_path = key_path.clone(); + let residue_dir = dir.path().to_path_buf(); + let before_reparse = move || { + std::fs::write(&heal_path, pem.as_bytes()) + .expect("complete the final inside the claim window"); + std::fs::write(residue_dir.join(".identity.pem.publishing.31337.0"), b"") + .expect("plant stale marker inside the claim window"); + let claim = residue_dir.join(".identity.pem.recovering.31337"); + std::fs::write(&claim, b"").expect("plant orphan claim"); + age_beyond_claim_sweep(&claim); + }; + let seam = RecoverySeam { + before_reparse: &before_reparse, + ..RecoverySeam::NONE + }; + let kp = load_or_create_keypair_with(&key_path, &|| {}, &|| {}, PublishFaults::NONE, seam) + .expect("the healed final must load"); + assert_eq!( + format!("{}", kp.did()), + format!("{}", completed.did()), + "the reloaded identity must be the healed key" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "the Reloaded return must sweep stale publish markers" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "the Reloaded return must sweep aged orphan claims" + ); + + // Scenario 2: the adoption-Lost re-load. A young adoptable + // quarantine loses its republish to a final that appears + // concurrently (before_publish), and the follow-up load returns + // the winner; the same residue must not survive that return. + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let stranded = Keypair::generate(); + std::fs::write( + dir.path().join(".identity.pem.quarantined.99999.0"), + stranded.to_pem().expect("pem").as_bytes(), + ) + .expect("seed young quarantine"); + std::fs::write(dir.path().join(".identity.pem.publishing.31337.0"), b"") + .expect("seed stale marker"); + let claim = dir.path().join(".identity.pem.recovering.31337"); + std::fs::write(&claim, b"").expect("seed orphan claim"); + age_beyond_claim_sweep(&claim); + let winner = Keypair::generate(); + let winner_pem = winner.to_pem().expect("pem"); + let race_path = key_path.clone(); + let before_publish = move || { + if race_path.exists() { + return; + } + let out = publish_key_atomically( + &race_path, + winner_pem.as_bytes(), + &|| {}, + &|| {}, + PublishFaults::NONE, + ) + .expect("concurrent winner publishes"); + assert!(matches!(out, KeyPublish::Won), "the winner takes the name"); + }; + let seam = RecoverySeam { + before_publish: &before_publish, + ..RecoverySeam::NONE + }; + let kp = load_or_create_keypair_with(&key_path, &|| {}, &|| {}, PublishFaults::NONE, seam) + .expect("the adoption-Lost re-load must resolve the winner"); + assert_eq!( + format!("{}", kp.did()), + format!("{}", winner.did()), + "the adoption-Lost arm must defer to the concurrent winner" + ); + assert!( + names_containing(dir.path(), ".publishing.").is_empty(), + "the adoption-Lost return must sweep stale publish markers" + ); + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "the adoption-Lost return must sweep aged orphan claims" + ); + } + // #194 (G2a) MUST-NOT: a TRANSIENT (non-content) failure of the // post-claim RE-parse proves nothing about the final's content, so // recovery must release its claims and surface the error loudly, @@ -6081,11 +6595,13 @@ mod identity_key_tests { // publishes a fresh key B at the freed final name inside the // quarantine-to-restore window (before_restore), and the restore's link // tier is forced onto its link-hostile degradation (restore_faults.link). - // The restore must defer to B (keep the quarantine as forensics, converge + // The restore must defer to B (keep A's bytes as forensics, converge // on B), never replace B with A: B's owner already holds B in memory, so // a clobber leaves that node's memory and disk diverged. RED before the // fix: the old restore ladder degraded to a plain rename(quarantine -> - // final), which clobbered B with A. + // final), which clobbered B with A. Since N2, the kept forensics live in + // the SUPERSEDED class (A provably lost the round, so it must never be + // adoptable); lost_restore_quarantine_is_never_adopted covers that side. #[test] fn restore_never_clobbers_a_concurrent_winner() { let dir = tempfile::tempdir().expect("tempdir"); @@ -6140,11 +6656,15 @@ mod identity_key_tests { format!("{}", kp_b.did()), "key B must survive at the final name on every restore tier" ); - let quarantined = names_containing(dir.path(), ".quarantined."); + let superseded = names_containing(dir.path(), ".superseded."); assert_eq!( - quarantined.len(), + superseded.len(), 1, - "the losing restore must keep A's quarantine as forensics: {quarantined:?}" + "the losing restore must keep A's bytes as superseded forensics: {superseded:?}" + ); + assert!( + names_containing(dir.path(), ".quarantined.").is_empty(), + "the losing key must not stay in the adoptable quarantine class" ); assert!( names_containing(dir.path(), ".recovering.").is_empty(), From dd1a93493f41fe7075079a225553a3742c5a47d4 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 21 Jul 2026 19:55:19 -0500 Subject: [PATCH 37/41] fix(node): supersede on every Lost arm, heartbeat live claims, retire old forensics Complete the supersede rule across its mirror arms: a lost adoption and a restore-Won round now both demote every outcompeted adoptable quarantine, so no arm leaves bytes the fresh-DID procedure could resurrect. Live recovery rounds now heartbeat their claim mtimes every quarter of the age floor under an RAII stop-and-join guard, making age-as-orphan sound: an aged claim provably lost its recoverer, and a stalled-but-live round on a slow volume can no longer be mistaken for residue by the waiter, the sweep, or a competing recoverer. Superseded forensic files are retention-swept after 24 hours, bounding growth on long-lived key directories. --- crates/gitlawb-node/src/lib.rs | 574 +++++++++++++++++++++++++++++---- 1 file changed, 505 insertions(+), 69 deletions(-) diff --git a/crates/gitlawb-node/src/lib.rs b/crates/gitlawb-node/src/lib.rs index e156d1e7..5805f51b 100644 --- a/crates/gitlawb-node/src/lib.rs +++ b/crates/gitlawb-node/src/lib.rs @@ -1154,11 +1154,16 @@ enum KeyPublish { const KEY_RACE_DEADLINE: std::time::Duration = std::time::Duration::from_secs(5); /// Minimum age (metadata mtime) before a `.recovering.` claim is sweepable -/// by a healthy boot (#194, G1). A LIVE recoverer's claims exist only within -/// one bounded recovery round (claim, quarantine, republish, release, itself -/// bounded by `KEY_RACE_DEADLINE`), so anything older is orphaned by a -/// crashed recovery; the 2x margin absorbs clock skew between writers on -/// network mounts. The gate matters because live claims are load-bearing: a +/// by a healthy boot (#194, G1). LIVENESS CONTRACT: a live recovery +/// HEARTBEATS every claim it holds (`spawn_claim_heartbeat`), refreshing +/// each mtime every quarter of this floor until release, so a live round's +/// claims NEVER age out, however long the round runs on a slow shared +/// volume. An AGED claim therefore means its recoverer stopped heartbeating +/// (crashed or killed): orphaned residue, claimable and sweepable. A YOUNG +/// claim means live or recently dead, an ambiguity bounded by one floor +/// (the price of never stealing a live round). The 2x margin over +/// `KEY_RACE_DEADLINE` absorbs clock skew between writers on network +/// mounts. The gate matters because live claims are load-bearing: a /// demoted publisher's `wait_for_recovery_claims` waits on them, and a third /// start sweeping a live claim would cancel that wait mid-recovery, letting /// the publisher re-load the pre-quarantine key while the recoverer @@ -1172,6 +1177,16 @@ const KEY_RACE_DEADLINE: std::time::Duration = std::time::Duration::from_secs(5) /// price of never stealing a live recovery round. const CLAIM_SWEEP_MIN_AGE: std::time::Duration = KEY_RACE_DEADLINE.saturating_mul(2); +/// Retention window for `.{stem}.superseded.` forensic files. A superseded +/// quarantine is kept for operator inspection after a lost round, but with +/// no expiry the residue accumulates forever (every lost round adds one +/// file beside the final). 24 hours is a generous post-mortem window while +/// keeping growth bounded: once a superseded file is older than this, the +/// healthy-boot sweep removes it (behind the same durability gate as every +/// other removal in `sweep_stale_markers`). An unreadable mtime reads as +/// young (kept; fail safe). +const SUPERSEDED_RETENTION: std::time::Duration = std::time::Duration::from_secs(24 * 60 * 60); + /// How long the crash signature (a content-bad final beside a claimable) /// must persist across consecutive observations before boot classifies a /// crash (#194, G3). A single ~2ms poll sits well inside a LIVE fallback @@ -1285,8 +1300,9 @@ fn quarantined_prefix(stem: &str) -> String { /// winner durably published meanwhile), or a sibling quarantine won the /// adoption. Superseded files are forensics only: never adoptable /// (`find_adoptable_quarantine` scans `.quarantined.` alone), never part of -/// the boot crash signature, and never swept; every consumer matches its own -/// prefix class, so this class is inert by construction. +/// the boot crash signature, and swept only once older than +/// `SUPERSEDED_RETENTION` (bounded forensics); every consumer matches its +/// own prefix class, so this class is inert by construction. fn superseded_prefix(stem: &str) -> String { format!(".{stem}.superseded.") } @@ -1690,8 +1706,9 @@ fn wait_for_recovery_claims(final_path: &std::path::Path) -> Result<()> { } } -/// Best-effort sweep of stale `.{stem}.publishing.*` markers and -/// `.{stem}.recovering.*` claims beside a healthy final, run on both +/// Best-effort sweep of stale `.{stem}.publishing.*` markers, +/// `.{stem}.recovering.*` claims, and expired `.{stem}.superseded.*` +/// forensics beside a healthy final, run on both /// healthy-boot arms of `load_or_create_keypair_with` (#194, U4). Without it /// a marker orphaned by a crash between its durability fsync and the final's /// `create_new`, or a claim left by a crashed recovery, outlives every @@ -1755,6 +1772,7 @@ fn sweep_stale_markers(final_path: &std::path::Path, sweep_sync: fn() -> std::io .unwrap_or("identity.pem"); let publishing = publishing_prefix(stem); let recovering = recovering_prefix(stem); + let superseded = superseded_prefix(stem); let Ok(entries) = std::fs::read_dir(dir) else { return; }; @@ -1775,6 +1793,20 @@ fn sweep_stale_markers(final_path: &std::path::Path, sweep_sync: fn() -> std::io if orphaned { let _ = std::fs::remove_file(entry.path()); } + } else if name.starts_with(&superseded) { + // Superseded forensics expire after SUPERSEDED_RETENTION (the + // operator's inspection window); without an expiry every lost + // round's residue accumulates forever. Unreadable mtime reads + // as young (kept; fail safe), matching the claim gate above. + let expired = entry + .metadata() + .and_then(|m| m.modified()) + .ok() + .and_then(|mtime| std::time::SystemTime::now().duration_since(mtime).ok()) + .is_some_and(|age| age >= SUPERSEDED_RETENTION); + if expired { + let _ = std::fs::remove_file(entry.path()); + } } } #[cfg(unix)] @@ -2366,9 +2398,10 @@ fn load_or_create_keypair(config: &Config) -> Result { /// Outcome of `recover_crashed_publish` (#194, U2). enum Recovery { /// The round was claimed and the corrupt final quarantined (or removed); - /// the caller regenerates. Holds every claim file this recovery created, - /// removed once the follow-up publish resolves (or on any error exit). - Claimed(Vec), + /// the caller regenerates. Holds every claim file this recovery created + /// plus their heartbeat (`ClaimSet`), released once the follow-up + /// publish resolves (or on any error exit). + Claimed(ClaimSet), /// The marker vanished before it could be claimed (the publisher — or a /// competing recoverer — resolved the window concurrently): the single /// follow-up load's result is final, success or error, no recovery. @@ -2455,18 +2488,132 @@ fn refresh_claim_mtime( }) } +/// Handle to a live recovery's claim-heartbeat thread. Dropping it stops +/// the beat and joins the thread, so every exit path (release, error, or +/// unwind) reclaims it; the join is bounded because the thread checks the +/// stop flag at least every ~250ms (the sleep granule in +/// `spawn_claim_heartbeat`). +#[derive(Debug)] +struct ClaimHeartbeat { + stop: std::sync::Arc, + handle: Option>, +} + +impl Drop for ClaimHeartbeat { + fn drop(&mut self) { + self.stop.store(true, std::sync::atomic::Ordering::SeqCst); + if let Some(handle) = self.handle.take() { + let _ = handle.join(); + } + } +} + +/// Spawn the claim heartbeat: a thread refreshing every claim's mtime every +/// `CLAIM_SWEEP_MIN_AGE / 4` until stopped, sleeping in ~250ms granules so +/// the joining release is prompt. This is what makes the G1 age-as-orphan +/// rule sound (see `CLAIM_SWEEP_MIN_AGE`'s liveness contract): without it a +/// recoverer stalled longer than the floor between claim and settlement +/// reads as an orphan, the demotion waiter returns Ok (aged out), a demoted +/// publisher loads the pre-settlement final, and a sweep removes the +/// still-live claim: divergence returns on slow shared volumes. Best-effort: +/// a failed spawn warns and degrades to the single claim-time stamp (the +/// pre-heartbeat exposure, churn not divergence); a failed refresh warns and +/// retries at the next beat. +fn spawn_claim_heartbeat(files: Vec) -> Option { + let stop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + let thread_stop = std::sync::Arc::clone(&stop); + let spawned = std::thread::Builder::new() + .name("identity-claim-heartbeat".into()) + .spawn(move || { + let beat = CLAIM_SWEEP_MIN_AGE / 4; + let granule = std::time::Duration::from_millis(250); + 'beat: loop { + let mut slept = std::time::Duration::ZERO; + while slept < beat { + if thread_stop.load(std::sync::atomic::Ordering::SeqCst) { + break 'beat; + } + std::thread::sleep(granule); + slept += granule; + } + for claim in &files { + if thread_stop.load(std::sync::atomic::Ordering::SeqCst) { + break 'beat; + } + if let Err(e) = refresh_claim_mtime(claim, set_mtime_now) { + warn!( + claim = %claim.display(), + err = %e, + "claim heartbeat could not refresh a live claim's mtime; if \ + this persists past the liveness floor the claim reads as \ + orphaned (bounded to churn, not divergence)" + ); + } + } + } + }); + match spawned { + Ok(handle) => Some(ClaimHeartbeat { + stop, + handle: Some(handle), + }), + Err(e) => { + warn!( + err = %e, + "could not spawn the claim heartbeat; falling back to the single \ + claim-time stamp: a recovery stalled past the liveness floor may \ + read as orphaned (bounded to churn, not divergence)" + ); + None + } + } +} + +/// The claim files a recovery holds, plus the heartbeat keeping them young +/// (see `spawn_claim_heartbeat`). Carried through `Recovery::Claimed` so the +/// boot loop's follow-up publish keeps the round visibly live until +/// `release_recovery_claims`; dropping the set (any path) stops and joins +/// the heartbeat via `ClaimHeartbeat`'s `Drop`. +#[derive(Debug)] +struct ClaimSet { + files: Vec, + heartbeat: Option, +} + +impl ClaimSet { + /// No claims, no heartbeat: the boot loop's initial state. + fn empty() -> Self { + ClaimSet { + files: Vec::new(), + heartbeat: None, + } + } + + /// Wrap freshly created claim files and start their heartbeat + /// (best-effort; `None` on spawn failure). + fn with_heartbeat(files: Vec) -> Self { + let heartbeat = spawn_claim_heartbeat(files.clone()); + ClaimSet { files, heartbeat } + } +} + /// Best-effort removal of the claim files a recovery created, plus a -/// best-effort dir fsync. Claims vouch for NO on-disk content (unlike +/// best-effort dir fsync. Stops and joins the claim heartbeat FIRST (via +/// `ClaimHeartbeat`'s `Drop`; bounded by its ~250ms flag granule), so no +/// beat can restamp a claim after its removal. Claims vouch for NO on-disk +/// content (unlike /// publish markers, whose disposal order against the final is pinned): they /// only gate a demoted publisher's bounded wait and recovery arbitration. /// Release therefore needs no ordering against the final, the markers, or /// anything else: any exit may drop the claims at any point once their owner /// stops recovering, and the worst a lost claim can cause is a waiter /// re-loading early, which the load path copes with (#194, F1/F3). -fn release_recovery_claims(final_path: &std::path::Path, claims: &[std::path::PathBuf]) { - for claim in claims { +fn release_recovery_claims(final_path: &std::path::Path, claims: &mut ClaimSet) { + drop(claims.heartbeat.take()); + for claim in &claims.files { let _ = std::fs::remove_file(claim); } + claims.files.clear(); #[cfg(unix)] { let _ = fsync_parent_dir(final_path); @@ -2592,6 +2739,11 @@ fn recover_crashed_publish( // concurrently; one immediate re-load decides, with no recovery. return Ok(Recovery::Reloaded(Box::new(load_racing_key(final_path)))); } + // CLAIM HEARTBEAT: from here until release these claims are load-bearing + // (they gate demoted publishers and arbitrate the round), and the G1 age + // rule reads aged-out as orphaned, so keep them visibly live for as long + // as this recovery actually runs; see spawn_claim_heartbeat. + let mut claims = ClaimSet::with_heartbeat(claims); // POST-CLAIM RE-PARSE (#194, F2): between the crash-signature // observation and the renames above, a live publisher may have completed @@ -2613,7 +2765,7 @@ fn recover_crashed_publish( .and_then(|()| load_existing_key(final_path)) { Ok(kp) => { - release_recovery_claims(final_path, &claims); + release_recovery_claims(final_path, &mut claims); return Ok(Recovery::Reloaded(Box::new(Ok(kp)))); } Err(e) @@ -2625,7 +2777,7 @@ fn recover_crashed_publish( // through to the quarantine. } Err(e) => { - release_recovery_claims(final_path, &claims); + release_recovery_claims(final_path, &mut claims); return Err(e.context(format!( "transient failure re-parsing identity key {} after claiming crash \ recovery; claims released, nothing quarantined", @@ -2655,7 +2807,7 @@ fn recover_crashed_publish( warn!( path = %final_path.display(), claimables = ?claimables, - claims = ?claims, + claims = ?claims.files, quarantine = %dest_for(start).display(), "crash-interrupted identity key publish: quarantining the unreadable key and \ regenerating the identity" @@ -2689,7 +2841,7 @@ fn recover_crashed_publish( // (#194, F1). if let Err(e) = std::fs::remove_file(final_path) { if e.kind() != std::io::ErrorKind::NotFound { - release_recovery_claims(final_path, &claims); + release_recovery_claims(final_path, &mut claims); return Err(load_err.context(format!( "could not quarantine or remove corrupt identity key {} during crash \ recovery: {e}", @@ -2716,7 +2868,11 @@ fn recover_crashed_publish( // its bytes are an exact copy of what now lives at the final, so keeping // it buys no forensics. The keypair returned is the one already parsed // from the quarantined bytes, not a re-read of the final, which could - // race a writer that lands after our republish. Lost means someone else + // race a writer that lands after our republish. Won also supersedes + // every OTHER young parseable quarantine sibling (they lost this round + // to the restored key, the N4 rule adoption-Won already applies); left + // adoptable, a later missing final would resurrect a different + // historical DID. Lost means someone else // durably published meanwhile: the quarantined key provably LOST its // round, so it is renamed into the superseded class (bytes preserved as // forensics, never adoptable; a young parseable quarantine left in the @@ -2742,20 +2898,26 @@ fn recover_crashed_publish( ) { Ok(KeyPublish::Won) => { let _ = std::fs::remove_file(quarantine); + // Mirror-arm invariant (N4): the restore decided this + // round in favor of the restored key, so any OTHER + // young parseable quarantine sibling provably lost it; + // supersede each, exactly like adoption-Won, or a later + // missing final resurrects a different historical DID. + supersede_sibling_quarantines(final_path, quarantine); #[cfg(unix)] { let _ = fsync_parent_dir(final_path); } - release_recovery_claims(final_path, &claims); + release_recovery_claims(final_path, &mut claims); return Ok(Recovery::Reloaded(Box::new(Ok(kp)))); } Ok(KeyPublish::Lost) => { supersede_quarantine(final_path, quarantine); - release_recovery_claims(final_path, &claims); + release_recovery_claims(final_path, &mut claims); return Ok(Recovery::Reloaded(Box::new(load_racing_key(final_path)))); } Err(e) => { - release_recovery_claims(final_path, &claims); + release_recovery_claims(final_path, &mut claims); return Err(e.context(format!( "could not restore the completed identity key to {} during crash \ recovery; quarantine kept at {}", @@ -2800,14 +2962,15 @@ fn load_or_create_keypair_with( faults: PublishFaults, seam: RecoverySeam<'_>, ) -> Result { - // The claim files of a fired recovery. Held while the follow-up + // The claim set (files plus heartbeat) of a fired recovery. Held while + // the follow-up // generate+publish runs, so a stalled winner our claims demoted can still // observe them while waiting (U3 relies on this), and released on EVERY // exit after that, success, Lost, and error alike (#194, F1): a leaked claim // stalls every later demoted publisher on the full KEY_RACE_DEADLINE. // Release order against the final or the markers does not matter; see // release_recovery_claims. - let mut claims: Vec = Vec::new(); + let mut claims = ClaimSet::empty(); let mut recovery_allowed = true; // N1 invariant: EVERY arm that returns a successfully resolved keypair @@ -2832,7 +2995,7 @@ fn load_or_create_keypair_with( if key_path.exists() { match boot_load_key(key_path, recovery_allowed, seam.load_fault) { BootLoad::Loaded(kp) => { - release_recovery_claims(key_path, &claims); + release_recovery_claims(key_path, &mut claims); return finish_healthy(*kp); } // Only emitted while recovery is still allowed (crash_exit is @@ -2848,7 +3011,7 @@ fn load_or_create_keypair_with( } } BootLoad::Failed(e) => { - release_recovery_claims(key_path, &claims); + release_recovery_claims(key_path, &mut claims); return Err(e); } } @@ -2867,8 +3030,10 @@ fn load_or_create_keypair_with( // expected crash state and stays on disk as forensics, as does an // AGED parseable one (history, never resurrected); only when no // quarantine qualifies does the boot fall through to generate. The republish holds no-clobber on every tier, so Lost - // means a concurrent starter durably published meanwhile: defer to - // it and keep the quarantine, exactly like the G2b restore. A + // means a concurrent starter durably published meanwhile: the + // quarantine provably LOST its round, so it moves to the superseded + // class and the boot defers to the winner, exactly like the G2b + // restore's Lost arm. A // republish error surfaces loudly with the quarantine kept: minting // a fresh identity over an adoptable one is the failure F1 exists to // prevent. @@ -2889,7 +3054,7 @@ fn load_or_create_keypair_with( { let _ = fsync_parent_dir(key_path); } - release_recovery_claims(key_path, &claims); + release_recovery_claims(key_path, &mut claims); info!( path = %key_path.display(), did = %kp.did(), @@ -2898,12 +3063,20 @@ fn load_or_create_keypair_with( return finish_healthy(kp); } Ok(KeyPublish::Lost) => { + // Mirror-arm invariant (N2): EVERY Lost outcome + // supersedes the losing bytes. This quarantine just + // provably LOST its round to the concurrent winner; + // left adoptable, the operator's delete-final-for-a- + // fresh-DID procedure would resurrect it within the age + // floor, exactly the hazard the restore's Lost arm + // already closes. + supersede_quarantine(key_path, &quarantine); let result = load_racing_key(key_path); - release_recovery_claims(key_path, &claims); + release_recovery_claims(key_path, &mut claims); return result.and_then(&finish_healthy); } Err(e) => { - release_recovery_claims(key_path, &claims); + release_recovery_claims(key_path, &mut claims); return Err(e.context(format!( "could not republish the completed identity key stranded at {}; \ quarantine kept", @@ -2920,14 +3093,14 @@ fn load_or_create_keypair_with( let pem = match kp.to_pem() { Ok(pem) => pem, Err(e) => { - release_recovery_claims(key_path, &claims); + release_recovery_claims(key_path, &mut claims); return Err(anyhow::anyhow!("failed to serialize key: {e}")); } }; if let Some(parent) = key_path.parent() { if let Err(e) = std::fs::create_dir_all(parent) { - release_recovery_claims(key_path, &claims); + release_recovery_claims(key_path, &mut claims); return Err(e.into()); } } @@ -2943,20 +3116,20 @@ fn load_or_create_keypair_with( ) { Ok(published) => published, Err(e) => { - release_recovery_claims(key_path, &claims); + release_recovery_claims(key_path, &mut claims); return Err(e); } }; match published { KeyPublish::Won => { - release_recovery_claims(key_path, &claims); + release_recovery_claims(key_path, &mut claims); info!(path = %key_path.display(), did = %kp.did(), "generated new node identity"); return finish_healthy(kp); } KeyPublish::Lost => { match boot_load_key(key_path, recovery_allowed, seam.load_fault) { BootLoad::Loaded(kp) => { - release_recovery_claims(key_path, &claims); + release_recovery_claims(key_path, &mut claims); return finish_healthy(*kp); } // Only emitted while recovery is still allowed (crash_exit @@ -2969,17 +3142,17 @@ fn load_or_create_keypair_with( continue; // Second pass: regenerate and publish. } Ok(Recovery::Reloaded(result)) => { - release_recovery_claims(key_path, &claims); + release_recovery_claims(key_path, &mut claims); return (*result).and_then(&finish_healthy); } Err(err) => { - release_recovery_claims(key_path, &claims); + release_recovery_claims(key_path, &mut claims); return Err(err); } } } BootLoad::Failed(e) => { - release_recovery_claims(key_path, &claims); + release_recovery_claims(key_path, &mut claims); return Err(e); } } @@ -4353,19 +4526,24 @@ mod identity_key_tests { key_path } - /// Set `path`'s mtime an hour into the past, far beyond - /// `CLAIM_SWEEP_MIN_AGE`, via GNU `touch -d` (std::fs cannot set mtimes; - /// spawning coreutils is test-only). - fn age_beyond_claim_sweep(path: &std::path::Path) { + /// Set `path`'s mtime `spec` into the past via GNU `touch -d` (std::fs + /// cannot set arbitrary mtimes; spawning coreutils is test-only). + fn set_mtime_past(path: &std::path::Path, spec: &str) { let status = std::process::Command::new("touch") .arg("-d") - .arg("1 hour ago") + .arg(spec) .arg(path) .status() .expect("run touch -d"); assert!(status.success(), "touch -d must succeed to age {path:?}"); } + /// Set `path`'s mtime an hour into the past, far beyond + /// `CLAIM_SWEEP_MIN_AGE`. + fn age_beyond_claim_sweep(path: &std::path::Path) { + set_mtime_past(path, "1 hour ago"); + } + /// File names in `dir` containing `needle` (marker/claim/quarantine sweeps). fn names_containing(dir: &std::path::Path, needle: &str) -> Vec { let mut names: Vec = std::fs::read_dir(dir) @@ -5390,6 +5568,38 @@ mod identity_key_tests { ); } + // M4: `.superseded.` forensic files never expired, so every lost round + // accumulated one beside the final forever. The healthy sweep must + // remove a superseded file older than SUPERSEDED_RETENTION (the + // operator's inspection window) while a YOUNG one survives; the sweep's + // durability gate already ran before any removal. RED before M4: the + // aged file survives every healthy boot. + #[test] + fn superseded_residue_is_retention_swept() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let pem = Keypair::generate().to_pem().expect("pem"); + std::fs::write(&key_path, pem.as_bytes()).expect("seed valid final"); + std::fs::set_permissions(&key_path, std::fs::Permissions::from_mode(0o600)) + .expect("0600 final"); + let aged = dir.path().join(".identity.pem.superseded.99999.0"); + std::fs::write(&aged, b"old forensics").expect("seed aged superseded file"); + set_mtime_past(&aged, "2 days ago"); + let young = dir.path().join(".identity.pem.superseded.99999.1"); + std::fs::write(&young, b"fresh forensics").expect("seed young superseded file"); + + load_or_create_keypair(&key_config(&key_path)).expect("healthy load"); + + assert!( + !aged.exists(), + "a superseded file past the retention window must be swept" + ); + assert!( + young.exists(), + "a superseded file inside the retention window must survive" + ); + } + // F2 (wiring): the delayed boot re-sweep must actually fire and sweep. // Paused tokio time drives the real CLAIM_SWEEP_MIN_AGE-plus-margin // sleep instantly; the claim is pre-aged on disk because paused time @@ -5442,14 +5652,17 @@ mod identity_key_tests { RecoverySeam::NONE, ) .expect("recovery over an aged marker must not error"); - let claims = match recovery { + let mut claims = match recovery { super::Recovery::Claimed(claims) => claims, super::Recovery::Reloaded(_) => { panic!("an aged marker beside an empty final must be claimed") } }; - assert!(!claims.is_empty(), "the aged marker must produce a claim"); - for claim in &claims { + assert!( + !claims.files.is_empty(), + "the aged marker must produce a claim" + ); + for claim in &claims.files { let mtime = std::fs::metadata(claim) .expect("stat claim") .modified() @@ -5463,7 +5676,7 @@ mod identity_key_tests { through the rename: age {age:?} at {claim:?}" ); } - super::release_recovery_claims(&key_path, &claims); + super::release_recovery_claims(&key_path, &mut claims); } // The G1 freshness stamp must survive a failing set_modified: some @@ -5497,6 +5710,75 @@ mod identity_key_tests { ); } + // M3: claim liveness is age < CLAIM_SWEEP_MIN_AGE, but a single + // claim-time stamp lets a recoverer stalled longer than the floor + // between claim and settlement read as an orphan: the demotion waiter + // returns Ok (aged out), a demoted publisher loads the pre-settlement + // final, and the delayed re-sweep or a healthy boot sweeps the + // still-live claim: divergence returns on slow shared volumes. A live + // recovery must HEARTBEAT its claims so they never age out while it + // runs. The park holds recovery inside the claim-to-settlement window + // (before_quarantine) just past the floor while the test samples the + // claim's age throughout. RED before M3: the single stamp ages past the + // floor mid-park. The ~12s wall cost is the test's point: the floor is + // real time. + #[test] + fn heartbeat_keeps_live_claims_young() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + + let park_dir = dir.path().to_path_buf(); + let before_quarantine = move || { + let deadline = std::time::Instant::now() + + super::CLAIM_SWEEP_MIN_AGE + + std::time::Duration::from_secs(2); + while std::time::Instant::now() < deadline { + std::thread::sleep(std::time::Duration::from_millis(900)); + let claims = names_containing(&park_dir, ".recovering."); + assert!(!claims.is_empty(), "the parked round's claim must persist"); + for name in &claims { + let mtime = std::fs::metadata(park_dir.join(name)) + .and_then(|m| m.modified()) + .expect("claim mtime"); + let age = std::time::SystemTime::now() + .duration_since(mtime) + .unwrap_or_default(); + assert!( + age < super::CLAIM_SWEEP_MIN_AGE, + "a LIVE recovery's claim must stay young mid-round: age {age:?}" + ); + } + } + }; + let seam = RecoverySeam { + before_quarantine: &before_quarantine, + ..RecoverySeam::NONE + }; + + let started = std::time::Instant::now(); + let kp = load_or_create_keypair_with(&key_path, &|| {}, &|| {}, PublishFaults::NONE, seam) + .expect("the parked recovery must still resolve a key"); + let elapsed = started.elapsed(); + + // The recovery finished: claims released (the on-disk leak + // evidence) and the heartbeat joined (the elapsed ceiling bounds + // release plus join after the ~12s park). + assert!( + names_containing(dir.path(), ".recovering.").is_empty(), + "the finished recovery must release its claims" + ); + assert!( + elapsed < super::CLAIM_SWEEP_MIN_AGE + std::time::Duration::from_secs(8), + "release and heartbeat join must be prompt after the park: took {elapsed:?}" + ); + let on_disk = super::load_existing_key(&key_path).expect("regenerated final parses"); + assert_eq!( + format!("{}", on_disk.did()), + format!("{}", kp.did()), + "the returned identity must match the on-disk key" + ); + } + // Claim destinations must carry a per-process nonce beyond the pid: // `.{stem}.recovering.{pid}.{nonce}.{n}`. The pid alone is not unique // across recoverers on a shared volume (two PID-1 containers), so two @@ -5519,15 +5801,15 @@ mod identity_key_tests { RecoverySeam::NONE, ) .expect("recovery over a crash state must not error"); - let claims = match recovery { + let mut claims = match recovery { super::Recovery::Claimed(claims) => claims, super::Recovery::Reloaded(_) => { panic!("a marker beside an empty final must be claimed") } }; - assert!(!claims.is_empty(), "the marker must produce a claim"); + assert!(!claims.files.is_empty(), "the marker must produce a claim"); let pid_prefix = format!(".identity.pem.recovering.{}.", std::process::id()); - for claim in &claims { + for claim in &claims.files { let name = claim .file_name() .and_then(|n| n.to_str()) @@ -5552,7 +5834,7 @@ mod identity_key_tests { fields[1] ); } - super::release_recovery_claims(&key_path, &claims); + super::release_recovery_claims(&key_path, &mut claims); } // A final that VANISHES between the claim renames and the post-claim @@ -5787,15 +6069,17 @@ mod identity_key_tests { .expect("publisher reaches its commit check"); // The concurrent recovery, via the REAL recovery path. Its - // before-reparse hook interleaves the publisher's commit into the - // post-claim window and records the identity the publisher resolved. - let publisher_did: std::sync::Mutex> = std::sync::Mutex::new(None); + // before-reparse hook unparks A inside the window between the claim + // step and the post-claim re-parse: A's marker is already claimed, + // so A's commit check fails and A demotes, then waits out the LIVE + // claims until this recovery releases them. (The hook must not park + // recovery until A finishes: A's waiter holds while the claims are + // young, and the M3 heartbeat keeps a live round's claims young + // until release, so that cycle would correctly fail A closed; the + // pre-heartbeat version only unblocked because the live claims aged + // out mid-round, the exact divergence window M3 closes.) let before_reparse = || { unpark_tx.send(()).expect("unpark the publisher"); - let did = done_rx - .recv() - .expect("publisher finishes inside the window"); - *publisher_did.lock().expect("publisher_did lock") = Some(did); }; let recovery = super::recover_crashed_publish( &key_path, @@ -5813,7 +6097,7 @@ mod identity_key_tests { .expect("the post-claim re-parse resolves the completed key") .did() ), - super::Recovery::Claimed(recovery_claims) => { + super::Recovery::Claimed(mut recovery_claims) => { // The boot loop's follow-through: regenerate, publish, release. let kp_b = Keypair::generate(); let pem_b = kp_b.to_pem().expect("pem b"); @@ -5826,19 +6110,13 @@ mod identity_key_tests { ) .expect("recovery's republish"); assert!(matches!(out, KeyPublish::Won), "recovery's republish wins"); - for c in &recovery_claims { - let _ = std::fs::remove_file(c); - } + super::release_recovery_claims(&key_path, &mut recovery_claims); format!("{}", kp_b.did()) } }; publisher.join().expect("publisher joins"); - let p_did = publisher_did - .lock() - .expect("publisher_did lock") - .take() - .expect("the publisher reported its identity"); + let p_did = done_rx.recv().expect("the publisher reported its identity"); let disk_did = format!( "{}", super::load_existing_key(&key_path) @@ -6218,6 +6496,93 @@ mod identity_key_tests { ); } + // M1, the N2 mirror on the ADOPTION arm: a quarantine whose adoption + // republish LOSES (a concurrent winner durably published first) is + // provably outcompeted, yet the adoption-Lost arm used to leave it in + // the adoptable class while restore-Lost superseded its kept + // quarantine. Within the age floor, the operator's + // delete-identity.pem-for-a-fresh-DID procedure then resurrects the key + // that just LOST its round. EVERY Lost outcome must supersede the + // losing bytes. RED before M1: A survives as `.quarantined.` and the + // post-delete boot adopts A. + #[test] + fn adoption_lost_supersedes_the_losing_quarantine() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let stranded = Keypair::generate(); + let pem_a = stranded.to_pem().expect("pem a"); + let quarantine = dir.path().join(".identity.pem.quarantined.99999.0"); + std::fs::write(&quarantine, pem_a.as_bytes()).expect("seed young quarantine A"); + + // A concurrent starter durably publishes C at the free final name + // before the adoption scan (before_publish runs at the top of the + // pass), so the adoption republish of A returns Lost. + let winner = Keypair::generate(); + let winner_pem = winner.to_pem().expect("pem c"); + let race_path = key_path.clone(); + let before_publish = move || { + if race_path.exists() { + return; + } + let out = publish_key_atomically( + &race_path, + winner_pem.as_bytes(), + &|| {}, + &|| {}, + PublishFaults::NONE, + ) + .expect("concurrent winner publishes C"); + assert!( + matches!(out, KeyPublish::Won), + "C's publish takes the free name" + ); + }; + let seam = RecoverySeam { + before_publish: &before_publish, + ..RecoverySeam::NONE + }; + let kp = load_or_create_keypair_with(&key_path, &|| {}, &|| {}, PublishFaults::NONE, seam) + .expect("the adoption-Lost arm must defer to the winner"); + assert_eq!( + format!("{}", kp.did()), + format!("{}", winner.did()), + "the boot must converge on the concurrent winner's key C" + ); + + // A's bytes survive as forensics, in the never-adoptable class. + assert!( + names_containing(dir.path(), ".quarantined.").is_empty(), + "no adoptable quarantine may survive a lost adoption" + ); + let superseded = names_containing(dir.path(), ".superseded."); + assert_eq!( + superseded.len(), + 1, + "the losing quarantine must be superseded: {superseded:?}" + ); + assert_eq!( + std::fs::read(dir.path().join(&superseded[0])).expect("superseded readable"), + pem_a.as_bytes(), + "the superseded file must preserve A's bytes as forensics" + ); + + // The operator action: delete the final for a fresh DID; A must not + // come back. + std::fs::remove_file(&key_path).expect("operator deletes the final"); + let fresh = load_or_create_keypair(&key_config(&key_path)) + .expect("the post-delete boot must provision an identity"); + assert_ne!( + format!("{}", fresh.did()), + format!("{}", stranded.did()), + "the key that LOST its adoption round must never be resurrected" + ); + assert_ne!( + format!("{}", fresh.did()), + format!("{}", winner.did()), + "the operator must get a genuinely fresh DID" + ); + } + // F1 MUST-NOT: an UNPARSEABLE quarantine is the expected crash state and // stays as forensics; boot must fall through to a fresh generation, not // wedge on it or delete it. @@ -6590,6 +6955,77 @@ mod identity_key_tests { ); } + // M2, the N4 mirror on the RESTORE arm: the G2b restore decides its + // round in favor of the restored key, but it used to remove only the + // quarantine it consumed. A second young parseable quarantine (an + // earlier crash's residue) stayed adoptable, so a later missing final + // resurrected a DIFFERENT historical DID. On Won the restore must + // supersede every other young parseable sibling, exactly like + // adoption-Won. RED before M2: Q_old stays `.quarantined.` and the + // post-delete boot adopts it. + #[test] + fn restore_won_supersedes_siblings() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = seed_crash_state(dir.path(), b""); + let older = Keypair::generate(); + let older_pem = older.to_pem().expect("pem"); + let q_old = dir.path().join(".identity.pem.quarantined.11111.0"); + std::fs::write(&q_old, older_pem.as_bytes()).expect("seed sibling quarantine"); + + // completed_key_is_restored_from_quarantine's construction: the + // publisher completes inside the re-parse-to-rename window, so its + // key is swept into a fresh quarantine and restored from there. + let completed = Keypair::generate(); + let pem = completed.to_pem().expect("pem"); + let heal_path = key_path.clone(); + let before_quarantine = move || { + std::fs::write(&heal_path, pem.as_bytes()) + .expect("complete the final inside the re-parse-to-rename window"); + }; + let seam = RecoverySeam { + before_quarantine: &before_quarantine, + ..RecoverySeam::NONE + }; + let kp = load_or_create_keypair_with(&key_path, &|| {}, &|| {}, PublishFaults::NONE, seam) + .expect("recovery must restore the completed key"); + assert_eq!( + format!("{}", kp.did()), + format!("{}", completed.did()), + "the restored identity must be the completed key" + ); + + assert!( + names_containing(dir.path(), ".quarantined.").is_empty(), + "the restore-Won round must leave no adoptable quarantine behind" + ); + let superseded = names_containing(dir.path(), ".superseded."); + assert_eq!( + superseded.len(), + 1, + "exactly the outcompeted sibling is superseded: {superseded:?}" + ); + assert_eq!( + std::fs::read(dir.path().join(&superseded[0])).expect("superseded readable"), + older_pem.as_bytes(), + "the superseded sibling must preserve its bytes as forensics" + ); + + // The sibling must not come back once the final goes missing. + std::fs::remove_file(&key_path).expect("operator deletes the final"); + let fresh = load_or_create_keypair(&key_config(&key_path)) + .expect("the post-delete boot must provision an identity"); + assert_ne!( + format!("{}", fresh.did()), + format!("{}", older.did()), + "the superseded sibling must never be resurrected" + ); + assert_ne!( + format!("{}", fresh.did()), + format!("{}", completed.did()), + "the operator must get a genuinely fresh DID" + ); + } + // R1: the post-quarantine restore must hold no-clobber on EVERY tier. A // completed key A is swept into quarantine (G2b), a concurrent starter // publishes a fresh key B at the freed final name inside the From 32903c451f112723b8a6582c5f87b6edd02c6b04 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 21 Jul 2026 20:37:49 -0500 Subject: [PATCH 38/41] fix(node): order supersede before consume, retire quarantine forensics, key the waiter on absence Both Won arms now supersede sibling quarantines before consuming the chosen one, so a crash in the gap cannot leave an outcompeted key adoptable. Plain quarantine forensics get the same 24h retention sweep as superseded files, bounding growth on long-lived key directories. The demotion waiter drops age logic entirely: it returns Ok only when every claim file is gone and fails the publish closed while any claim survives the bound, young or aged, so a heartbeat failure on a hostile volume can no longer convert a live round into orphan residue mid-wait. Doc prose reconciled with the shipped wait contract. --- crates/gitlawb-node/src/lib.rs | 485 ++++++++++++++++++++++++--------- 1 file changed, 353 insertions(+), 132 deletions(-) diff --git a/crates/gitlawb-node/src/lib.rs b/crates/gitlawb-node/src/lib.rs index 5805f51b..bbcfd484 100644 --- a/crates/gitlawb-node/src/lib.rs +++ b/crates/gitlawb-node/src/lib.rs @@ -1159,7 +1159,9 @@ const KEY_RACE_DEADLINE: std::time::Duration = std::time::Duration::from_secs(5) /// each mtime every quarter of this floor until release, so a live round's /// claims NEVER age out, however long the round runs on a slow shared /// volume. An AGED claim therefore means its recoverer stopped heartbeating -/// (crashed or killed): orphaned residue, claimable and sweepable. A YOUNG +/// (crashed or killed): orphaned residue, claimable and sweepable (the +/// demotion waiter, by contrast, keys on claim-file ABSENCE, not age; see +/// `wait_for_recovery_claims`). A YOUNG /// claim means live or recently dead, an ambiguity bounded by one floor /// (the price of never stealing a live round). The 2x margin over /// `KEY_RACE_DEADLINE` absorbs clock skew between writers on network @@ -1177,14 +1179,20 @@ const KEY_RACE_DEADLINE: std::time::Duration = std::time::Duration::from_secs(5) /// price of never stealing a live recovery round. const CLAIM_SWEEP_MIN_AGE: std::time::Duration = KEY_RACE_DEADLINE.saturating_mul(2); -/// Retention window for `.{stem}.superseded.` forensic files. A superseded -/// quarantine is kept for operator inspection after a lost round, but with -/// no expiry the residue accumulates forever (every lost round adds one -/// file beside the final). 24 hours is a generous post-mortem window while -/// keeping growth bounded: once a superseded file is older than this, the -/// healthy-boot sweep removes it (behind the same durability gate as every -/// other removal in `sweep_stale_markers`). An unreadable mtime reads as -/// young (kept; fail safe). +/// Retention window for forensic residue beside a healthy final: both +/// `.{stem}.superseded.` files and plain `.{stem}.quarantined.` files. A +/// superseded quarantine is kept for operator inspection after a lost +/// round, and a plain quarantine lingers as crash forensics (unparseable +/// bytes) or as aged history (a parseable key already past the adoption +/// floor, never resurrected); with no expiry either class accumulates +/// forever (every lost round or crash adds one file beside the final). 24 +/// hours is a generous post-mortem window while keeping growth bounded: +/// once a file in either class is older than this, the healthy-boot sweep +/// removes it (behind the same durability gate as every other removal in +/// `sweep_stale_markers`). The window is orders of magnitude above the +/// adoption floor (`CLAIM_SWEEP_MIN_AGE`), so no quarantine this sweep can +/// touch was still adoptable. An unreadable mtime reads as young (kept; +/// fail safe). const SUPERSEDED_RETENTION: std::time::Duration = std::time::Duration::from_secs(24 * 60 * 60); /// How long the crash signature (a content-bad final beside a claimable) @@ -1622,34 +1630,38 @@ fn boot_load_key( } } -/// Wait until no `.{stem}.recovering.*` claim YOUNGER than -/// `CLAIM_SWEEP_MIN_AGE` remains beside `final_path`, polling on -/// `load_racing_key`'s cadence (~2ms), bounded by `CLAIM_SWEEP_MIN_AGE` plus -/// a one-second margin (#194, U3; F3). Why the wait exists: a fallback -/// winner that failed its commit check was demoted by a recovering peer, and -/// that peer is mid-round — it quarantines the (old) final, republishes a -/// fresh key, and only then clears its claim. A demoted winner that -/// re-loaded before the claim cleared could observe the pre-quarantine key -/// and diverge from what the peer ends up publishing; waiting for claim -/// clearance makes the follow-up load observe the settled state. +/// Wait until NO `.{stem}.recovering.*` claim FILE remains beside +/// `final_path`, polling on `load_racing_key`'s cadence (~2ms), bounded by +/// `CLAIM_SWEEP_MIN_AGE` plus a one-second margin (#194, U3; F3; N3). Why +/// the wait exists: a fallback winner that failed its commit check was +/// demoted by a recovering peer, and that peer is mid-round — it +/// quarantines the (old) final, republishes a fresh key, and only then +/// clears its claim. A demoted winner that re-loaded before the claim +/// cleared could observe the pre-quarantine key and diverge from what the +/// peer ends up publishing; waiting for claim clearance makes the follow-up +/// load observe the settled state. /// -/// The waiter and the sweep share ONE liveness rule (F3): a claim is live -/// only while younger than `CLAIM_SWEEP_MIN_AGE`, the protocol's -/// claim-liveness floor. An AGED claim is orphaned residue by definition and -/// does not hold the waiter, while a young claim holds it up to the floor -/// (the old `KEY_RACE_DEADLINE` bound expired while a slow mount's live -/// round could still be in flight). The margin lets a claim born at wait -/// start age out naturally. +/// The waiter keys purely on ABSENCE, never on age. Age stays the liveness +/// rule for the SWEEP (`sweep_stale_markers`) and CLAIMABILITY +/// (`list_recovery_claimables`), but an aged claim is not proof its +/// recoverer is gone: with the heartbeat spawn failed AND both mtime stamp +/// rungs failing on a hostile volume, a LIVE recoverer's claim can sit +/// mtime-frozen while the round is still settling disk, and a waiter that +/// read it as orphaned residue would let the demoted publisher load a +/// pre-settlement key while that recoverer settles disk differently: +/// two-sided divergence under compounded failure. A file that is GONE is +/// unambiguous: the round settled (release) or a sweep decided the claim +/// was orphaned; either way the follow-up load observes a settled state. /// -/// A wait that CLEARS (claims removed, or only aged orphans remain) returns -/// `Ok`. Bound expiry with a YOUNG claim still present returns an `Err` -/// naming the claim file(s) (N3): the recovery round did not settle within -/// the bound, and a demoted publisher that loaded the final there could -/// return a pre-settlement identity into `run()` while the still-live -/// recoverer settles disk on a different one, the exact divergence the -/// protocol forbids, so the caller must fail closed instead of loading. An -/// unreadable claim mtime reads as young (hold; fail safe, matching the -/// sweep), and an unreadable directory reads as no claims, matching +/// So: `Ok` only when the claim files are gone. Bound expiry with ANY claim +/// file still present (young or aged) returns an `Err` naming the file(s): +/// the round cannot be proven settled, and the caller must fail closed +/// instead of loading. Consequence: an aged ORPHAN claim (its recoverer +/// long dead) now fails a concurrently-demoted publisher loudly until a +/// healthy boot's sweep or the delayed re-sweep +/// (`spawn_delayed_claim_resweep`) clears it — a bounded, +/// operator-visible degradation, priced against the silent divergence the +/// age rule risked. An unreadable directory reads as no claims, matching /// `list_publish_markers`' policy. fn wait_for_recovery_claims(final_path: &std::path::Path) -> Result<()> { let dir = final_path @@ -1664,41 +1676,26 @@ fn wait_for_recovery_claims(final_path: &std::path::Path) -> Result<()> { let bound = CLAIM_SWEEP_MIN_AGE + std::time::Duration::from_secs(1); let deadline = std::time::Instant::now() + bound; loop { - let live: Vec = std::fs::read_dir(dir) + let present: Vec = std::fs::read_dir(dir) .map(|entries| { entries .filter_map(|e| e.ok()) .filter(|e| { - if !e - .file_name() + e.file_name() .to_str() .is_some_and(|n| n.starts_with(&prefix)) - { - return false; - } - // The shared liveness rule (see the doc comment): only - // a young claim holds the waiter; unreadable mtime = - // young. - let age = e - .metadata() - .and_then(|m| m.modified()) - .ok() - .and_then(|mtime| { - std::time::SystemTime::now().duration_since(mtime).ok() - }); - age.is_none_or(|age| age < CLAIM_SWEEP_MIN_AGE) }) .map(|e| e.file_name().to_string_lossy().into_owned()) .collect() }) .unwrap_or_default(); - if live.is_empty() { + if present.is_empty() { return Ok(()); } if std::time::Instant::now() >= deadline { anyhow::bail!( - "recovery round did not settle within the {bound:?} wait bound: live \ - recovery claim(s) {live:?} still present beside {}", + "recovery round did not settle within the {bound:?} wait bound: \ + recovery claim(s) {present:?} still present beside {}", final_path.display() ); } @@ -1707,8 +1704,8 @@ fn wait_for_recovery_claims(final_path: &std::path::Path) -> Result<()> { } /// Best-effort sweep of stale `.{stem}.publishing.*` markers, -/// `.{stem}.recovering.*` claims, and expired `.{stem}.superseded.*` -/// forensics beside a healthy final, run on both +/// `.{stem}.recovering.*` claims, and expired `.{stem}.superseded.*` / +/// `.{stem}.quarantined.*` forensics beside a healthy final, run on both /// healthy-boot arms of `load_or_create_keypair_with` (#194, U4). Without it /// a marker orphaned by a crash between its durability fsync and the final's /// `create_new`, or a claim left by a crashed recovery, outlives every @@ -1773,6 +1770,7 @@ fn sweep_stale_markers(final_path: &std::path::Path, sweep_sync: fn() -> std::io let publishing = publishing_prefix(stem); let recovering = recovering_prefix(stem); let superseded = superseded_prefix(stem); + let quarantined = quarantined_prefix(stem); let Ok(entries) = std::fs::read_dir(dir) else { return; }; @@ -1793,11 +1791,14 @@ fn sweep_stale_markers(final_path: &std::path::Path, sweep_sync: fn() -> std::io if orphaned { let _ = std::fs::remove_file(entry.path()); } - } else if name.starts_with(&superseded) { - // Superseded forensics expire after SUPERSEDED_RETENTION (the + } else if name.starts_with(&superseded) || name.starts_with(&quarantined) { + // Forensic residue expires after SUPERSEDED_RETENTION (the // operator's inspection window); without an expiry every lost - // round's residue accumulates forever. Unreadable mtime reads - // as young (kept; fail safe), matching the claim gate above. + // round's superseded file, and every crash's plain quarantine + // (unparseable forensics, or parseable history already far past + // the adoption floor), accumulates forever. Unreadable mtime + // reads as young (kept; fail safe), matching the claim gate + // above. let expired = entry .metadata() .and_then(|m| m.modified()) @@ -1944,14 +1945,17 @@ impl PublishFaults { /// the winner's commit check, and recovery claims a round by renaming EVERY /// marker and stale claim beside the final (a live winner's own marker is /// necessarily among them), so exactly one side takes the round. A winner -/// that loses its marker demotes itself to Lost, waits out any -/// `.recovering.` claim (bounded by `KEY_RACE_DEADLINE`), and re-loads the -/// settled key; a recovery that finds the final completed after claiming -/// re-loads it instead of quarantining. Honest residuals: a winner stalled -/// past `KEY_RACE_DEADLINE` that loads in the sub-ms window before a -/// recoverer's claim exists, and a crash-interrupted recovery whose stale -/// claim just expires the bounded wait: both degrade to a loud error or a -/// re-load, never silent two-sided divergence. +/// that loses its marker demotes itself, waits for every `.recovering.` +/// claim FILE to clear (bounded by `CLAIM_SWEEP_MIN_AGE` plus a margin; see +/// `wait_for_recovery_claims`), and re-loads the settled key as Lost; if +/// any claim file survives the bound the publish fails closed with a loud +/// error instead of loading. A recovery that finds the final completed +/// after claiming re-loads it instead of quarantining. Honest residuals: a +/// winner stalled past `KEY_RACE_DEADLINE` that loads in the sub-ms window +/// before a recoverer's claim exists degrades to a re-load, and a +/// crash-interrupted recovery whose orphaned claim outlives the bounded +/// wait fails the demoted publish loudly until a sweep clears the claim: +/// loud error or re-load, never silent two-sided divergence. /// /// `before_link` is a no-op in production; tests use it to widen the /// post-write / pre-link window deterministically. `before_commit` runs @@ -2294,10 +2298,10 @@ fn publish_key_fallback( // ONLY for a SETTLED round (N3): the caller's Lost arm re-loads the // final, and that load observes the recoverer's outcome only once the // recoverer's claims have cleared, so the demoted publish first waits - // them out. An UNSETTLED round past the wait bound (a young claim still - // live) fails the publish closed with a loud error instead: loading - // there could return a pre-settlement key while the still-live recoverer - // settles disk on a different identity. + // them out. An UNSETTLED round past the wait bound (any claim file + // still present) fails the publish closed with a loud error instead: + // loading there could return a pre-settlement key while the still-live + // recoverer settles disk on a different identity. before_commit(); match std::fs::remove_file(&marker) { Ok(()) => { @@ -2513,9 +2517,13 @@ impl Drop for ClaimHeartbeat { /// the joining release is prompt. This is what makes the G1 age-as-orphan /// rule sound (see `CLAIM_SWEEP_MIN_AGE`'s liveness contract): without it a /// recoverer stalled longer than the floor between claim and settlement -/// reads as an orphan, the demotion waiter returns Ok (aged out), a demoted -/// publisher loads the pre-settlement final, and a sweep removes the -/// still-live claim: divergence returns on slow shared volumes. Best-effort: +/// reads as an orphan to the sweep and to claimability, so a healthy boot +/// can remove the still-live claim (early-releasing the demoted waiter to +/// a pre-settlement load) or a competing recoverer can steal the round: +/// divergence returns on slow shared volumes. The demotion waiter itself is +/// age-blind (it keys on claim-file absence; see +/// `wait_for_recovery_claims`), so a frozen mtime alone fails a demoted +/// publisher closed rather than releasing it early. Best-effort: /// a failed spawn warns and degrades to the single claim-time stamp (the /// pre-heartbeat exposure, churn not divergence); a failed refresh warns and /// retries at the next beat. @@ -2649,7 +2657,8 @@ fn release_recovery_claims(final_path: &std::path::Path, claims: &mut ClaimSet) /// remaining markers best-effort and hands control back to regenerate. /// Every error exit after claiming releases /// the claims first (see `release_recovery_claims`; a leaked claim would -/// stall later demoted publishers on the full deadline, F1). `seam` carries +/// ride later demoted publishers through the full wait bound and fail +/// their publish closed until a sweep cleared it, F1). `seam` carries /// the test hooks for the claim-to-re-parse and re-parse-to-quarantine /// windows (`before_reparse`, `reparse_fault`, `before_quarantine`; all /// no-ops in production). @@ -2897,13 +2906,17 @@ fn recover_crashed_publish( seam.restore_faults, ) { Ok(KeyPublish::Won) => { - let _ = std::fs::remove_file(quarantine); // Mirror-arm invariant (N4): the restore decided this // round in favor of the restored key, so any OTHER // young parseable quarantine sibling provably lost it; // supersede each, exactly like adoption-Won, or a later // missing final resurrects a different historical DID. + // ORDER: siblings first, THEN consume the chosen + // quarantine, so a crash in the gap leaves the siblings + // already superseded, never adoptable beside a final + // that already holds the winner. supersede_sibling_quarantines(final_path, quarantine); + let _ = std::fs::remove_file(quarantine); #[cfg(unix)] { let _ = fsync_parent_dir(final_path); @@ -2966,8 +2979,9 @@ fn load_or_create_keypair_with( // the follow-up // generate+publish runs, so a stalled winner our claims demoted can still // observe them while waiting (U3 relies on this), and released on EVERY - // exit after that, success, Lost, and error alike (#194, F1): a leaked claim - // stalls every later demoted publisher on the full KEY_RACE_DEADLINE. + // exit after that, success, Lost, and error alike (#194, F1): a leaked + // claim rides every later demoted publisher through the full wait bound + // and fails its publish closed until a sweep clears the orphan. // Release order against the final or the markers does not matter; see // release_recovery_claims. let mut claims = ClaimSet::empty(); @@ -3040,16 +3054,20 @@ fn load_or_create_keypair_with( if let Some((quarantine, pem, kp)) = find_adoptable_quarantine(key_path) { match publish_key_atomically(key_path, pem.as_bytes(), &|| {}, &|| {}, faults) { Ok(KeyPublish::Won) => { - // The quarantine's bytes are an exact copy of what now - // durably lives at the final, so keeping it buys no - // forensics (the G2b restore's removal rationale). - let _ = std::fs::remove_file(&quarantine); // N4: any OTHER young parseable quarantine sibling lost // this round to the adopted key; move each into the // superseded class so a later missing final cannot // resurrect a different historical DID. Unparseable // siblings stay plain quarantines as crash forensics. + // ORDER: siblings first, THEN consume the chosen + // quarantine, so a crash in the gap leaves the siblings + // already superseded, never adoptable beside a final + // that already holds the winner. supersede_sibling_quarantines(key_path, &quarantine); + // The quarantine's bytes are an exact copy of what now + // durably lives at the final, so keeping it buys no + // forensics (the G2b restore's removal rationale). + let _ = std::fs::remove_file(&quarantine); #[cfg(unix)] { let _ = fsync_parent_dir(key_path); @@ -5215,18 +5233,18 @@ mod identity_key_tests { ); } - // #194 (U3, reshaped by F3): a demoted winner facing an AGED - // `.recovering.` claim (a recovery that crashed before clearing it) must - // SKIP it quickly and return Lost, not error: the waiter now shares the - // sweep's liveness rule, and a claim past CLAIM_SWEEP_MIN_AGE is - // orphaned residue by definition. The marker is stolen WITHOUT a real - // recovery (final left in place), so the follow-up load resolves the - // winner's OWN key: the demotion-is-safe property. - // waiter_holds_for_live_young_claims covers the young-claim hold side. - // RED before F3: the age-blind waiter rides the full KEY_RACE_DEADLINE - // (~5s) on the aged claim and the elapsed ceiling fails. + // #194 (U3, reshaped by F3 and the absence-keyed waiter): a demoted + // winner whose round is SETTLED (no `.recovering.` claim file remains, + // the recoverer released before the demoted publish reached its wait) + // must resolve Lost immediately: the waiter keys purely on claim-file + // absence, so an empty directory clears it on the first poll. The + // marker is stolen WITHOUT a real recovery (final left in place), so + // the follow-up load resolves the winner's OWN key: the + // demotion-is-safe property. waiter_holds_for_live_young_claims covers + // the hold side; frozen_claim_fails_the_waiter_closed covers a claim + // that never clears. #[test] - fn demoted_winner_waits_out_a_stale_claim() { + fn demoted_winner_resolves_fast_when_claims_are_gone() { let dir = tempfile::tempdir().expect("tempdir"); let key_path = dir.path().join("identity.pem"); @@ -5242,13 +5260,9 @@ mod identity_key_tests { 1, "A's window must be bracketed: {markers:?}" ); + // The demoting peer already resolved AND released its claims: + // only the stolen marker remains as evidence of the demotion. std::fs::remove_file(steal_dir.join(&markers[0])).expect("steal A's marker"); - let claim = steal_dir.join(".identity.pem.recovering.44444"); - std::fs::write(&claim, b"").expect("plant stale claim"); - // The planted claim models an orphan of a LONG-crashed recovery, - // so it must be aged; a fresh claim reads as live and holds the - // waiter (waiter_holds_for_live_young_claims). - age_beyond_claim_sweep(&claim); }; let faults = PublishFaults { link: || Err(std::io::ErrorKind::Unsupported.into()), @@ -5263,12 +5277,12 @@ mod identity_key_tests { faults, RecoverySeam::NONE, ) - .expect("a demoted winner behind a stale claim must still resolve a key"); + .expect("a demoted winner with no claims left must still resolve a key"); let waited = started.elapsed(); assert!( waited < std::time::Duration::from_secs(4), - "an aged (orphaned) claim must not hold the demoted winner, waited {waited:?}" + "an absent claim must not hold the demoted winner, waited {waited:?}" ); let on_disk = super::load_existing_key(&key_path).expect("final parses"); assert_eq!( @@ -5278,14 +5292,89 @@ mod identity_key_tests { ); } - // F3: the waiter's hold must follow the claim-liveness floor - // (CLAIM_SWEEP_MIN_AGE), not KEY_RACE_DEADLINE: a live claim's round is - // allowed the full liveness floor to finish, so a demoted winner that - // gives up at the 5s deadline can re-load the pre-quarantine key while a - // slow-mount recovery is still mid-round. Plant a FRESH claim, hold it - // live for ~7s (past the old deadline), then remove it mid-wait: the - // waiter must still be holding past 5s and return shortly after the - // removal, well under the CLAIM_SWEEP_MIN_AGE-plus-margin bound. RED + // Q3: a demoted winner facing a claim whose mtime is FROZEN (its + // recoverer's heartbeat never span up and both stamp rungs failed on a + // hostile volume, or it is a long-crashed recovery's orphan) must fail + // the publish CLOSED, never resolve a key: the waiter keys purely on + // claim-file PRESENCE, so a claim that never leaves the directory rides + // the full bound and errors, whatever its age. Before this fix the + // waiter read an aged claim as orphaned residue and returned Ok, and + // the demoted publisher loaded a possibly pre-settlement key while an + // mtime-frozen LIVE recoverer could still settle disk differently. RED + // before the fix: the aged claim clears the wait as Ok and the demoted + // winner resolves its own key. + #[test] + fn frozen_claim_fails_the_waiter_closed() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + + let fired = std::sync::atomic::AtomicBool::new(false); + let steal_dir = dir.path().to_path_buf(); + let final_bytes = std::sync::Arc::new(std::sync::Mutex::new(Vec::::new())); + let final_at_demotion = final_bytes.clone(); + let observe_path = key_path.clone(); + let before_commit = move || { + if fired.swap(true, std::sync::atomic::Ordering::SeqCst) { + return; + } + let markers = names_containing(&steal_dir, ".publishing."); + assert_eq!( + markers.len(), + 1, + "A's window must be bracketed: {markers:?}" + ); + std::fs::remove_file(steal_dir.join(&markers[0])).expect("steal A's marker"); + let claim = steal_dir.join(".identity.pem.recovering.44444"); + std::fs::write(&claim, b"").expect("plant frozen claim"); + // Freeze the claim's mtime far in the past: no heartbeat, no + // stamp, only the file itself, which never leaves the directory. + age_beyond_claim_sweep(&claim); + *final_at_demotion.lock().expect("final bytes lock") = + std::fs::read(&observe_path).expect("final is complete at the commit check"); + }; + let faults = PublishFaults { + link: || Err(std::io::ErrorKind::Unsupported.into()), + ..PublishFaults::NONE + }; + + let result = load_or_create_keypair_with( + &key_path, + &|| {}, + &before_commit, + faults, + RecoverySeam::NONE, + ); + + let err = match result { + Err(e) => e, + Ok(_) => { + panic!("a claim still present at the wait bound must fail closed, not resolve") + } + }; + let msg = format!("{err:#}"); + assert!( + msg.contains(".identity.pem.recovering.44444"), + "the error must name the surviving claim file: {msg}" + ); + assert!( + msg.contains("settle"), + "the error must name the unsettled round: {msg}" + ); + assert_eq!( + std::fs::read(&key_path).expect("final still on disk"), + *final_bytes.lock().expect("final bytes lock"), + "the failed-closed publish must leave the final untouched" + ); + } + + // F3: the waiter's hold must follow the CLAIM_SWEEP_MIN_AGE-plus-margin + // bound, not KEY_RACE_DEADLINE: a live claim's round is allowed the + // full bound to finish, so a demoted winner that gives up at the 5s + // deadline can re-load the pre-quarantine key while a slow-mount + // recovery is still mid-round. Plant a claim, hold it in place for ~7s + // (past the old deadline), then remove it mid-wait: the waiter must + // still be holding past 5s (the file is present) and return shortly + // after the removal (absence is the release), well under the bound. RED // before F3: the waiter expires at KEY_RACE_DEADLINE and the elapsed // floor fails. #[test] @@ -5356,14 +5445,16 @@ mod identity_key_tests { ); } - // N3: a demoted fallback winner whose wait bound expires while a LIVE - // young claim still persists has NOT observed a settled round: loading - // the final there can return a pre-settlement key into run() while the + // N3: a demoted fallback winner whose wait bound expires while a claim + // still persists has NOT observed a settled round: loading the final + // there can return a pre-settlement key into run() while the // still-live recoverer settles disk on a different identity, the exact // divergence the protocol forbids. The publish must fail closed (a loud // Err naming the unsettled round), never resolve Lost and load. A // refresher thread keeps the claim young past the full wait bound, - // modeling a recovery round still live (or stuck) beyond it. RED before + // modeling a recovery round still heartbeating (live but stuck) beyond + // it; frozen_claim_fails_the_waiter_closed covers the mtime-frozen + // side, which the absence-keyed waiter treats identically. RED before // N3: the waiter expires silently and the demoted winner returns its // own key. #[test] @@ -5397,10 +5488,12 @@ mod identity_key_tests { ..PublishFaults::NONE }; - // Keep the claim YOUNG for the waiter's whole bound: a recovery - // round that never settles. Without the refresher the claim would - // age past CLAIM_SWEEP_MIN_AGE before the bound expires and the - // wait would clear as aged-orphan residue (the Ok side). + // Keep the claim YOUNG for the waiter's whole bound, modeling a + // recovery round that heartbeats but never settles. The + // absence-keyed waiter fails closed on presence regardless of age + // (frozen_claim_fails_the_waiter_closed pins the frozen-mtime + // side); the refresher keeps this test honest about the LIVE-round + // shape it models. let claim_path = dir.path().join(".identity.pem.recovering.55555"); let stop = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); let stop_refresher = stop.clone(); @@ -5600,6 +5693,54 @@ mod identity_key_tests { ); } + // Q2: plain `.quarantined.` residue also never expired: an unparseable + // quarantine (crash forensics) and an aged parseable one (past the + // adoption floor, so pure history) sat beside a healthy final forever. + // The healthy sweep must remove a `.quarantined.` entry older than + // SUPERSEDED_RETENTION, while a YOUNG unparseable one (fresh crash + // forensics) and a parseable one past the adoption floor but inside + // retention (recent history, an operator may still want it) survive. + // RED before Q2: the aged unparseable quarantine survives every + // healthy boot. + #[test] + fn old_quarantine_forensics_are_retention_swept() { + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let pem = Keypair::generate().to_pem().expect("pem"); + std::fs::write(&key_path, pem.as_bytes()).expect("seed valid final"); + std::fs::set_permissions(&key_path, std::fs::Permissions::from_mode(0o600)) + .expect("0600 final"); + let aged_garbage = dir.path().join(".identity.pem.quarantined.99999.0"); + std::fs::write(&aged_garbage, b"-----BEGIN nonsense").expect("seed aged forensics"); + set_mtime_past(&aged_garbage, "2 days ago"); + let young_garbage = dir.path().join(".identity.pem.quarantined.99999.1"); + std::fs::write(&young_garbage, b"-----BEGIN nonsense").expect("seed young forensics"); + let recent_parseable = dir.path().join(".identity.pem.quarantined.99999.2"); + std::fs::write( + &recent_parseable, + Keypair::generate().to_pem().expect("pem").as_bytes(), + ) + .expect("seed recent parseable quarantine"); + // Past the adoption floor (CLAIM_SWEEP_MIN_AGE) but well inside the + // 24h retention window. + set_mtime_past(&recent_parseable, "1 hour ago"); + + load_or_create_keypair(&key_config(&key_path)).expect("healthy load"); + + assert!( + !aged_garbage.exists(), + "a quarantine past the retention window must be swept" + ); + assert!( + young_garbage.exists(), + "young crash forensics must survive the healthy sweep" + ); + assert!( + recent_parseable.exists(), + "a quarantine past the adoption floor but inside retention must survive" + ); + } + // F2 (wiring): the delayed boot re-sweep must actually fire and sweep. // Paused tokio time drives the real CLAIM_SWEEP_MIN_AGE-plus-margin // sleep instantly; the claim is pre-aged on disk because paused time @@ -5712,10 +5853,12 @@ mod identity_key_tests { // M3: claim liveness is age < CLAIM_SWEEP_MIN_AGE, but a single // claim-time stamp lets a recoverer stalled longer than the floor - // between claim and settlement read as an orphan: the demotion waiter - // returns Ok (aged out), a demoted publisher loads the pre-settlement - // final, and the delayed re-sweep or a healthy boot sweeps the - // still-live claim: divergence returns on slow shared volumes. A live + // between claim and settlement read as an orphan to the sweep and to + // claimability: the delayed re-sweep or a healthy boot removes the + // still-live claim (early-releasing any demoted publisher's + // absence-keyed wait to a pre-settlement load), or a competing + // recoverer steals the round: divergence returns on slow shared + // volumes. A live // recovery must HEARTBEAT its claims so they never age out while it // runs. The park holds recovery inside the claim-to-settlement window // (before_quarantine) just past the floor while the test samples the @@ -5982,10 +6125,10 @@ mod identity_key_tests { // #194 (F1): a claimed recovery whose retry publish FAILS must release // its `.recovering.` claim before surfacing the error. A leaked claim - // makes every later demoted fallback publisher ride the full - // KEY_RACE_DEADLINE in wait_for_recovery_claims until some healthy boot - // sweeps it. RED before F1: the bare `?` on the publish call returns - // with the claim still on disk. + // makes every later demoted fallback publisher ride the full wait + // bound in wait_for_recovery_claims and fail its publish closed until + // some healthy boot sweeps it. RED before F1: the bare `?` on the + // publish call returns with the claim still on disk. #[test] fn claim_released_when_post_recovery_publish_fails() { let dir = tempfile::tempdir().expect("tempdir"); @@ -6073,11 +6216,11 @@ mod identity_key_tests { // step and the post-claim re-parse: A's marker is already claimed, // so A's commit check fails and A demotes, then waits out the LIVE // claims until this recovery releases them. (The hook must not park - // recovery until A finishes: A's waiter holds while the claims are - // young, and the M3 heartbeat keeps a live round's claims young - // until release, so that cycle would correctly fail A closed; the - // pre-heartbeat version only unblocked because the live claims aged - // out mid-round, the exact divergence window M3 closes.) + // recovery until A finishes: A's waiter holds while the claim + // files are present, until release, so that cycle would correctly + // fail A closed; the pre-heartbeat, age-keyed version only + // unblocked because the live claims aged out mid-round, the exact + // divergence window M3 and the absence-keyed waiter close.) let before_reparse = || { unpark_tx.send(()).expect("unpark the publisher"); }; @@ -6496,6 +6639,84 @@ mod identity_key_tests { ); } + // Q1: both Won arms now supersede the sibling quarantines BEFORE + // consuming the chosen one, so a crash in the gap leaves the + // POST-SUPERSEDE image: final present, siblings already superseded, + // only the chosen quarantine (the winner's own bytes) still adoptable. + // No seam reaches the reordered gap, so this test pins the two crash + // images directly rather than interposing. First the post-fix image: a + // later missing final may re-adopt only the winner's own key, never a + // different historical DID. Then the PRE-fix image (chosen consumed + // first, sibling still plain): a concrete demonstration that the old + // ordering's crash gap left the sibling adoptable, resurrecting a + // different DID after delete-final. The ordering itself is not + // runtime-observable without a production seam; see the fix report. + #[test] + fn crash_between_won_and_sibling_supersede_leaves_no_adoptable() { + // POST-fix crash image: crash fell between the sibling supersede + // and the chosen quarantine's removal. + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + let winner = Keypair::generate(); + let winner_pem = winner.to_pem().expect("pem"); + let sibling = Keypair::generate(); + let sibling_pem = sibling.to_pem().expect("pem"); + std::fs::write(&key_path, winner_pem.as_bytes()).expect("seed final"); + std::fs::set_permissions(&key_path, std::fs::Permissions::from_mode(0o600)) + .expect("0600 final"); + std::fs::write( + dir.path().join(".identity.pem.superseded.99999.0"), + sibling_pem.as_bytes(), + ) + .expect("seed superseded sibling"); + let chosen = dir.path().join(".identity.pem.quarantined.99999.1"); + std::fs::write(&chosen, winner_pem.as_bytes()).expect("seed un-consumed chosen"); + + let kp = load_or_create_keypair(&key_config(&key_path)) + .expect("the post-supersede crash image must boot healthy"); + assert_eq!( + format!("{}", kp.did()), + format!("{}", winner.did()), + "the final's key wins the crash-image boot" + ); + + std::fs::remove_file(&key_path).expect("final goes missing"); + let readopted = load_or_create_keypair(&key_config(&key_path)) + .expect("the post-delete boot must provision an identity"); + assert_ne!( + format!("{}", readopted.did()), + format!("{}", sibling.did()), + "a crash in the supersede-then-consume gap must leave no sibling adoptable" + ); + assert_eq!( + format!("{}", readopted.did()), + format!("{}", winner.did()), + "only the winner's own quarantined bytes may be re-adopted" + ); + + // PRE-fix crash image (the old ordering's gap): chosen already + // consumed, sibling still in the plain adoptable class. This is the + // hazard: the sibling IS adopted once the final goes missing. + let dir = tempfile::tempdir().expect("tempdir"); + let key_path = dir.path().join("identity.pem"); + std::fs::write(&key_path, winner_pem.as_bytes()).expect("seed final"); + std::fs::set_permissions(&key_path, std::fs::Permissions::from_mode(0o600)) + .expect("0600 final"); + std::fs::write( + dir.path().join(".identity.pem.quarantined.99999.0"), + sibling_pem.as_bytes(), + ) + .expect("seed still-plain sibling"); + std::fs::remove_file(&key_path).expect("final goes missing"); + let resurrected = load_or_create_keypair(&key_config(&key_path)) + .expect("the post-delete boot must provision an identity"); + assert_eq!( + format!("{}", resurrected.did()), + format!("{}", sibling.did()), + "the pre-fix image demonstrably resurrects the sibling's DID" + ); + } + // M1, the N2 mirror on the ADOPTION arm: a quarantine whose adoption // republish LOSES (a concurrent winner durably published first) is // provably outcompeted, yet the adoption-Lost arm used to leave it in From 14de621c5007ebb0bed8442b021a8c01b2cc9188 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 21 Jul 2026 21:50:51 -0500 Subject: [PATCH 39/41] test(node): drive the #113 read/owner gates in the deny-harness fence After merging main, the completeness fence flagged three repo-scoped list GETs that #113 gated but the harness still treated as ungated public reads: list_webhooks (authorize_repo_read + require_repo_owner), list_replicas and list_protected_branches (authorize_repo_read). The runtime marker scan caught its own hand-maintained PUBLIC_REPO_GETS list drifting stale. Drive each real deny instead of exempting it: - list_replicas and list_protected_branches take Option auth, so they slot in as standard ReadGate rows (anon and signed-non-reader both 404, owner 2xx, no-leak body). - list_webhooks is doubly gated and 401s a headerless caller before any lookup, so a vanilla ReadGate row's hardcoded anon->404 leg does not fit. Add a per-Row anon_read signal (Deny404 default, Deny401 for auth-required reads) so its anon leg expects 401 while the signed-non-reader still drives the existence-hiding 404. Its owner layer is a sibling OwnerGate row on the public repo (signed stranger -> 403). The two share the /hooks path, so the consistency dedup key moves from (method, path) to (method, path, gate), which still catches an accidental same-gate double-registration. - Empty PUBLIC_REPO_GETS: the three are now driven rows, classified via the driven-path set rather than declared public. The mechanism stays for the next genuinely-public repo-GET. Each new row proven load-bearing: reverting the production gate turns the matching hostile probe red (403->200 owner, 404->403/200 read), and flipping list_webhooks' anon_read to Deny404 reddens its anon probe (got 401). Full deny_harness green (39/39). No production changes. --- crates/gitlawb-node/tests/deny_harness.rs | 14 +- crates/gitlawb-node/tests/support/probe.rs | 20 ++- crates/gitlawb-node/tests/support/routes.rs | 161 +++++++++++++++++++- 3 files changed, 182 insertions(+), 13 deletions(-) diff --git a/crates/gitlawb-node/tests/deny_harness.rs b/crates/gitlawb-node/tests/deny_harness.rs index 88e60336..77bcc6f2 100644 --- a/crates/gitlawb-node/tests/deny_harness.rs +++ b/crates/gitlawb-node/tests/deny_harness.rs @@ -1845,11 +1845,15 @@ mod completeness { /// below must not demand they be driven or structurally excused. Keyed by path /// (the unit `scrape_mounts` yields) so a new gate added to one of these later /// still surfaces via the marker scan; this list only says "no gate today". - const PUBLIC_REPO_GETS: &[&str] = &[ - "/api/v1/repos/{owner}/{repo}/hooks", // list_webhooks - "/api/v1/repos/{owner}/{repo}/branches/protected", // list_protected_branches - "/api/v1/repos/{owner}/{repo}/replicas", // list_replicas - ]; + /// + /// Currently EMPTY: `/hooks`, `/branches/protected`, and `/replicas` were listed + /// here as ungated public GETs, but #113 added an `authorize_repo_read` gate to + /// all three. They are now DRIVEN as real `ReadGate` rows in + /// `deny_bearing_routes()` (list_webhooks also as an OwnerGate row for its owner + /// layer), so they are classified via the driven-path set, not this list. The + /// mechanism is kept (not deleted) so the next genuinely-public repo-GET has a + /// home — the marker scan is the source of truth that keeps it honest. + const PUBLIC_REPO_GETS: &[&str] = &[]; /// True for a mounted path that is scoped to a single repo — the API form /// `/api/v1/repos/{owner}/{repo}/…` and the bare git form `/{owner}/{repo}/…`. diff --git a/crates/gitlawb-node/tests/support/probe.rs b/crates/gitlawb-node/tests/support/probe.rs index fcd2f740..3a587732 100644 --- a/crates/gitlawb-node/tests/support/probe.rs +++ b/crates/gitlawb-node/tests/support/probe.rs @@ -14,7 +14,7 @@ use gitlawb_core::identity::Keypair; use gitlawb_node::test_harness::TestNode; -use super::routes::{GateClass, IdSource, Principal, Reach, Row}; +use super::routes::{AnonRead, GateClass, IdSource, Principal, Reach, Row}; /// A seeded two-repo state matrix plus the owner and stranger identities. pub struct Fixture { @@ -992,15 +992,23 @@ pub fn probes_for(row: &Row, fixture: &Fixture) -> Vec { fixture.private_repo_id.clone(), ]; withheld.extend(fixture.priv_markers.iter().cloned()); + // The anon leg: an `Option`-auth read existence-hides with a 404, but an + // auth-required read (`AnonRead::Deny401`, e.g. list_webhooks) rejects a + // headerless caller 401 BEFORE any lookup. The signed-non-reader 404 + // below is the existence-hiding gate either way; only this leg differs. + let anon_status = match row.anon_read { + AnonRead::Deny404 => 404, + AnonRead::Deny401 => 401, + }; let mut v = vec![ Probe { - label: format!("{} read-gate hostile (anon)", row.handler), + label: format!("{} read-gate hostile (anon -> {anon_status})", row.handler), method: method.clone(), path: path.clone(), body: body.clone(), signer: Signer::Anon, json, - expect: Expect::Deny(404), + expect: Expect::Deny(anon_status), withheld: withheld.clone(), }, // #195 (F1): a signed NON-READER must also get the existence-hiding @@ -1181,6 +1189,7 @@ mod tests { reach: Reach::None, principals: &[], id_source: crate::support::routes::IdSource::Fixed, + anon_read: crate::support::routes::AnonRead::Deny404, }; let ps = probes_for(&row, &fx()); assert_eq!(ps.len(), 2); @@ -1213,6 +1222,7 @@ mod tests { reach: Reach::ReaderReads, principals: &[], id_source: crate::support::routes::IdSource::Fixed, + anon_read: crate::support::routes::AnonRead::Deny404, }; let f = fx(); let ps = probes_for(&row, &f); @@ -1272,6 +1282,7 @@ mod tests { reach: Reach::None, principals: &[], id_source: crate::support::routes::IdSource::Fixed, + anon_read: crate::support::routes::AnonRead::Deny404, }; let ps = probes_for(&row, &fx()); assert_eq!(ps.len(), 1); @@ -1296,6 +1307,7 @@ mod tests { reach: Reach::SiblingPublic("/api/v1/repos/{owner}/{repo}/blob/{*path}"), principals: &[], id_source: crate::support::routes::IdSource::Fixed, + anon_read: crate::support::routes::AnonRead::Deny404, }; let ps = probes_for(&row, &fx()); // anon hostile, signed-stranger hostile (#195 F1), sibling-public twin. @@ -1337,6 +1349,7 @@ mod tests { reach: Reach::ReaderReads, principals: &[], id_source: crate::support::routes::IdSource::Fixed, + anon_read: crate::support::routes::AnonRead::Deny404, }; let f = fx(); let ps = probes_for(&row, &f); @@ -1400,6 +1413,7 @@ mod tests { reach: Reach::None, principals: &[], id_source: crate::support::routes::IdSource::Fixed, + anon_read: crate::support::routes::AnonRead::Deny404, }; let _ = probes_for(&row, &fx()); } diff --git a/crates/gitlawb-node/tests/support/routes.rs b/crates/gitlawb-node/tests/support/routes.rs index b29fb161..c898361b 100644 --- a/crates/gitlawb-node/tests/support/routes.rs +++ b/crates/gitlawb-node/tests/support/routes.rs @@ -24,7 +24,7 @@ //! (R9). /// The gate class of a deny-bearing route and the exact status its deny emits. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum GateClass { /// `require_repo_owner` / `require_owner` / inline `did_matches(caller, owner)` /// -> `AppError::Forbidden` == 403. Probed with a validly-signed non-owner. @@ -186,6 +186,30 @@ pub struct Row { pub principals: &'static [Principal], /// How the `{id}` placeholder is filled for this row (see [`IdSource`]). pub id_source: IdSource, + /// For `ReadGate` rows, the status an ANONYMOUS (headerless) caller gets on + /// the withheld target. Most read GETs take `Option` auth and existence-hide + /// with a 404 (`Deny404`), but a read GET that *requires* auth rejects a + /// headerless caller with 401 BEFORE any lookup (`Deny401`) — e.g. + /// `list_webhooks`, whose callback URLs are owner-secret config with no anon + /// form. The signed-non-reader 404 (existence-hiding) is unchanged either way; + /// only the anon leg differs. Meaningless on non-`ReadGate` rows, which set + /// `Deny404` as the not-applicable sentinel (mirrors `reach: Reach::None`). + /// A new auth-required read added as a `ReadGate` row that forgets `Deny401` + /// fails loudly (its anon probe gets 401, not the expected 404), so the field + /// is self-correcting rather than a silent default. + pub anon_read: AnonRead, +} + +/// The status class an anonymous caller gets from a `ReadGate` row's withheld +/// target — 404 existence-hiding (the default) or a 401 auth-required rejection. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AnonRead { + /// `Option`-auth read: anon gets the existence-hiding 404, same as a signed + /// non-reader. The default for every read GET that has an anonymous form. + Deny404, + /// Auth-required read: a headerless caller is rejected 401 before any lookup + /// (no anonymous form). The signed-non-reader still gets the 404. + Deny401, } const NO_ENTITY: &[&str] = &[]; @@ -208,6 +232,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::None, principals: NO_PRINCIPAL, id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, }, // #195 (F1): close_pr / close_issue are owner-OR-author gates (pulls.rs:276, // issues.rs:255). A plain OwnerGate tests only the owner arm, so reverting @@ -222,6 +247,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::None, principals: &[Principal::Owner, Principal::Author], id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, }, Row { method: "POST", @@ -233,6 +259,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::None, principals: &[Principal::Owner, Principal::Author], id_source: IdSource::IssueId, + anon_read: AnonRead::Deny404, }, // #195 (F1): dispute_bounty is creator-OR-claimant (bounties.rs:425). Not // repo-scoped: it gates on the bounty's creator/claimant, so the row is @@ -247,6 +274,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::None, principals: &[Principal::Creator, Principal::Claimant], id_source: IdSource::BountyId, + anon_read: AnonRead::Deny404, }, // #195 (N2): submit/approve/cancel each 403 a non-claimant/non-creator, but // their STATUS check fires before the auth check, so each row gets its own @@ -268,6 +296,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::None, principals: &[Principal::Claimant], id_source: IdSource::ClaimedBountyId, + anon_read: AnonRead::Deny404, }, Row { method: "POST", @@ -282,6 +311,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::None, principals: &[Principal::Creator], id_source: IdSource::SubmittedBountyId, + anon_read: AnonRead::Deny404, }, Row { method: "POST", @@ -293,6 +323,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::None, principals: &[Principal::Creator], id_source: IdSource::OpenBountyId, + anon_read: AnonRead::Deny404, }, Row { method: "POST", @@ -304,6 +335,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::None, principals: NO_PRINCIPAL, id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, }, Row { method: "DELETE", @@ -315,6 +347,25 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::None, principals: NO_PRINCIPAL, id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + // #113: the OWNER layer of list_webhooks (GET /hooks). On the PUBLIC repo a + // signed stranger passes authorize_repo_read (public) and is then denied 403 + // by require_repo_owner — this row drives that 403. The READ layer (404 on + // the private repo, anon 401) is the sibling ReadGate row; the two share the + // path, admitted by the (method, path, gate) dedup key. A GET owner-gate + // like list_visibility below. + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/hooks", + gate: GateClass::OwnerGate, + handler: "webhooks::list_webhooks", + body: None, + needs: NO_ENTITY, + reach: Reach::None, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, }, Row { method: "POST", @@ -326,6 +377,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::None, principals: NO_PRINCIPAL, id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, }, Row { method: "DELETE", @@ -337,6 +389,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::None, principals: NO_PRINCIPAL, id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, }, Row { method: "POST", @@ -348,6 +401,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::None, principals: NO_PRINCIPAL, id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, }, Row { method: "DELETE", @@ -359,6 +413,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::None, principals: NO_PRINCIPAL, id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, }, Row { method: "PUT", @@ -370,6 +425,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::None, principals: NO_PRINCIPAL, id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, }, Row { method: "DELETE", @@ -381,6 +437,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::None, principals: NO_PRINCIPAL, id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, }, // list_visibility is a 403 owner-gate despite being a GET (calls // require_owner); the /visibility mount chains put+delete+get, all gated. @@ -394,6 +451,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::None, principals: NO_PRINCIPAL, id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, }, // ── Signature-required (401) — verified: git write route wrapped by the // require_signature layer (add_auth_layers in server.rs). ───────────── @@ -407,6 +465,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::None, principals: NO_PRINCIPAL, id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, }, // ── Read-gate (404) — verified: each handler calls // authorize_repo_read / visibility_check on "/" which returns @@ -428,6 +487,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::ReaderReads, principals: NO_PRINCIPAL, id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, }, // #195 (F2): repo-root reads that gate on "/" — driven as real ReadGate rows // rather than source-only exemptions, so a runtime bypass that keeps the @@ -442,6 +502,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::ReaderReads, principals: NO_PRINCIPAL, id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, }, Row { method: "GET", @@ -453,6 +514,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::ReaderReads, principals: NO_PRINCIPAL, id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, }, Row { method: "GET", @@ -464,6 +526,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::ReaderReads, principals: NO_PRINCIPAL, id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, }, Row { method: "GET", @@ -475,6 +538,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::ReaderReads, principals: NO_PRINCIPAL, id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, }, Row { method: "GET", @@ -486,6 +550,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::ReaderReads, principals: NO_PRINCIPAL, id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, }, // Path-scoped read: get_blob gates on "/{path}" (repos.rs:390). The // fully-private fixture repo denies any path to anon (404); the owner @@ -502,6 +567,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::ReaderReads, principals: NO_PRINCIPAL, id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, }, Row { method: "GET", @@ -513,6 +579,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::ReaderReads, principals: NO_PRINCIPAL, id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, }, Row { method: "GET", @@ -524,6 +591,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::ReaderReads, principals: NO_PRINCIPAL, id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, }, Row { method: "GET", @@ -535,6 +603,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::ReaderReads, principals: NO_PRINCIPAL, id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, }, Row { method: "GET", @@ -546,6 +615,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::ReaderReads, principals: NO_PRINCIPAL, id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, }, Row { method: "GET", @@ -557,6 +627,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::ReaderReads, principals: NO_PRINCIPAL, id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, }, Row { method: "GET", @@ -568,6 +639,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::ReaderReads, principals: NO_PRINCIPAL, id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, }, Row { method: "GET", @@ -579,6 +651,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::ReaderReads, principals: NO_PRINCIPAL, id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, }, Row { method: "GET", @@ -590,6 +663,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::ReaderReads, principals: NO_PRINCIPAL, id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, }, Row { method: "GET", @@ -601,6 +675,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::ReaderReads, principals: NO_PRINCIPAL, id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, }, Row { method: "GET", @@ -612,6 +687,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::ReaderReads, principals: NO_PRINCIPAL, id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, }, // #195 (F2, U3): sub-entity reads that gate on "/" (the private repo) but // require the entity to EXIST at the request path for the owner-2xx twin. @@ -631,6 +707,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::ReaderReads, principals: NO_PRINCIPAL, id_source: IdSource::PrivIssueId, + anon_read: AnonRead::Deny404, }, // list_issue_comments: parent issue must exist (private); child list may be // empty — an empty `{"comments":[]}` is a non-empty 2xx body. @@ -644,6 +721,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::ReaderReads, principals: NO_PRINCIPAL, id_source: IdSource::PrivIssueId, + anon_read: AnonRead::Deny404, }, // get_pr: the seeded PRIVATE PR #1 (marker in its title). Row { @@ -656,6 +734,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::ReaderReads, principals: NO_PRINCIPAL, id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, }, // get_pr_diff: the PRIVATE PR #1 has a real `feature` source branch with a // marker file, so branch_diff_names(main, feature) is NON-EMPTY and the @@ -672,6 +751,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::ReaderReads, principals: NO_PRINCIPAL, id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, }, // list_reviews / list_comments: parent PR #1 must exist (private); child // lists may be empty. @@ -685,6 +765,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::ReaderReads, principals: NO_PRINCIPAL, id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, }, Row { method: "GET", @@ -696,6 +777,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::ReaderReads, principals: NO_PRINCIPAL, id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, }, // get_cert: id-keyed by a real ref-certificate issued by an owner push to // the private repo. No cert-content marker is seeded: get_cert read-gates @@ -712,6 +794,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::ReaderReads, principals: NO_PRINCIPAL, id_source: IdSource::CertId, + anon_read: AnonRead::Deny404, }, // get_bounty: NOT repo-scoped in its path, read-gates on the bounty's repo. // Seeded against the PRIVATE repo (marker in its title), id-keyed. @@ -725,6 +808,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::ReaderReads, principals: NO_PRINCIPAL, id_source: IdSource::PrivBountyId, + anon_read: AnonRead::Deny404, }, // get_tree (path-scoped): genuinely ADDITIONAL to get_tree_root (Q1). Root // gates on "/"; get_tree gates on the REQUESTED subtree (N3, @@ -741,6 +825,61 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::ReaderReads, principals: NO_PRINCIPAL, id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + // #113: /replicas and /branches/protected gained an `authorize_repo_read` + // gate on "/" — a private repo's replica URLs / protected-branch list are + // now hidden (404) from a non-reader. Both take `Option` auth (anon and a + // signed non-reader alike get the existence-hiding 404), gate on the whole + // repo, and return an empty-or-seeded 2xx list to the owner, so `ReaderReads` + // (owner re-read) is the twin and no per-row sub-entity seeding is needed. + // Driven as real ReadGate rows rather than left in PUBLIC_REPO_GETS, which + // is why the completeness fence flagged them after the #113 merge. + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/replicas", + gate: GateClass::ReadGate, + handler: "replicas::list_replicas", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/branches/protected", + gate: GateClass::ReadGate, + handler: "protect::list_protected_branches", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, + }, + // #113: list_webhooks is DOUBLY gated — authorize_repo_read (404) THEN + // require_repo_owner (403) — and it rejects a headerless caller with 401 + // BEFORE the lookup (callback URLs are owner-secret config with no anon + // form). This is the READ layer: a signed non-reader of the private repo + // gets the existence-hiding 404, while an anon caller gets 401 (AnonRead:: + // Deny401). The OWNER layer (403 for a signed reader-non-owner) is the + // sibling OwnerGate row above, run against the public repo where the read + // gate passes and the owner gate is what fires. Two rows share this path + // (one ReadGate, one OwnerGate); the consistency dedup keys on + // (method, path, gate) to admit exactly this dual-gate case. + Row { + method: "GET", + path: "/api/v1/repos/{owner}/{repo}/hooks", + gate: GateClass::ReadGate, + handler: "webhooks::list_webhooks", + body: None, + needs: NO_ENTITY, + reach: Reach::ReaderReads, + principals: NO_PRINCIPAL, + id_source: IdSource::Fixed, + anon_read: AnonRead::Deny401, }, // ── Caller-self / actor (403), #195 (F3). The task/register handlers carry // genuine caller-self 403 gates a bypass would expose (impersonate a @@ -764,6 +903,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::None, principals: NO_PRINCIPAL, id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, }, Row { method: "POST", @@ -778,6 +918,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::None, principals: NO_PRINCIPAL, id_source: IdSource::ClaimableTaskId, + anon_read: AnonRead::Deny404, }, Row { method: "POST", @@ -792,6 +933,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::None, principals: NO_PRINCIPAL, id_source: IdSource::Fixed, + anon_read: AnonRead::Deny404, }, Row { method: "POST", @@ -805,6 +947,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::None, principals: &[Principal::Assignee], id_source: IdSource::CompletableTaskId, + anon_read: AnonRead::Deny404, }, Row { method: "POST", @@ -819,6 +962,7 @@ pub fn deny_bearing_routes() -> &'static [Row] { reach: Reach::None, principals: &[Principal::Assignee], id_source: IdSource::FailableTaskId, + anon_read: AnonRead::Deny404, }, // The read-gate handlers NOT driven here (deferred GET reads, read-gating // mutations, git smart-HTTP reads, the content-addressed read, and the @@ -839,14 +983,21 @@ mod tests { fn registry_internal_consistency() { let rows = deny_bearing_routes(); - // No duplicate method+path. + // No duplicate method+path+gate. The key is (method, path, gate), not + // (method, path): one endpoint can carry two distinct gate LAYERS that each + // need their own probe substrate — list_webhooks (GET /hooks) read-gates + // (404 on the private repo) THEN owner-gates (403 on the public repo), so it + // is one ReadGate row and one OwnerGate row on the same path. An accidental + // copy-paste of a row keeps the same gate, so this still catches the real + // double-registration bug; it only admits the deliberate dual-gate case. let mut seen = std::collections::HashSet::new(); for r in rows { assert!( - seen.insert((r.method, r.path)), - "duplicate deny-bearing row: {} {}", + seen.insert((r.method, r.path, r.gate)), + "duplicate deny-bearing row: {} {} {:?}", r.method, - r.path + r.path, + r.gate ); } From 8d81cbe0a04303a70c5babcbda053da816a281f2 Mon Sep 17 00:00:00 2001 From: t Date: Wed, 22 Jul 2026 00:25:10 -0500 Subject: [PATCH 40/41] test(node): harden deny-harness so its guarantees can't pass vacuously Resolve three review findings on the real-node deny harness, all test-only. - assert_denied no longer folds a body-read error into an empty body: an unreadable denial (a mid-body reset/truncation) now fails loud instead of being certified leak-free by a withheld-token scan over "". - The three sqlx::test cases that returned via TestNode's async Drop (signed_stranger_protected_branch_push_is_forbidden, get_pr_diff_withheld_path_is_denied, deny_bearing_registry_denies_hostile_and_admits_authorized) now end with node.shutdown().await, releasing the serve task's pool clones before sqlx's synchronous DROP DATABASE cleanup instead of racing it. - A source-scrape guard, every_spawn_node_test_shuts_the_node_down, makes that contract load-bearing for every spawn_node test, with a closed allowlist for the two Drop-regression tests that deliberately skip shutdown. Removing any one shutdown() turns it RED (verified). - The eight fixture seeders in support/probe.rs use the bounded client, so a hung setup route fails fast rather than running to the CI job timeout. Full deny_harness suite green (40 tests). --- crates/gitlawb-node/tests/deny_harness.rs | 141 ++++++++++++++++++++ crates/gitlawb-node/tests/support/assert.rs | 10 +- crates/gitlawb-node/tests/support/probe.rs | 16 +-- 3 files changed, 158 insertions(+), 9 deletions(-) diff --git a/crates/gitlawb-node/tests/deny_harness.rs b/crates/gitlawb-node/tests/deny_harness.rs index 77bcc6f2..e7a31c03 100644 --- a/crates/gitlawb-node/tests/deny_harness.rs +++ b/crates/gitlawb-node/tests/deny_harness.rs @@ -204,6 +204,8 @@ async fn signed_stranger_protected_branch_push_is_forbidden(pool: sqlx::PgPool) 403, "the owner must not be blocked by their own branch protection (control)" ); + + node.shutdown().await; } // ── U5(b): INV-8/INV-2 — anonymous /ipfs/{cid} of a withheld blob is denied ── @@ -598,6 +600,8 @@ async fn get_pr_diff_withheld_path_is_denied(pool: sqlx::PgPool) { resp.text().await.unwrap().contains("TOPSECRET-PRDIFF-PATH"), "the owner's diff returns the touched file content" ); + + node.shutdown().await; } // ── U8: INV-2 — an anonymous clone/fetch excludes withheld subtree blobs ────── @@ -906,6 +910,8 @@ async fn deny_bearing_registry_denies_hostile_and_admits_authorized(pool: sqlx:: twins_driven >= 1, "no positive twins were driven — the reachability proof is missing" ); + + node.shutdown().await; } // ── Additional INV-1 owner-gates over the real stack (fan-out of U6) ────────── @@ -2126,4 +2132,139 @@ mod completeness { "the owner form must NOT be treated as a caller-self gate (no double-count)" ); } + + // ── Shutdown-completeness guard: every spawn_node test must shutdown() ──────── + + /// `(name, body)` for every real `#[sqlx::test]` in `src`. The attribute is only + /// counted when nothing but whitespace precedes it on its line, so a backtick + /// mention inside a doc comment (the module header names the attribute in prose) + /// is not mistaken for a test. The needle is spelled in two pieces so this + /// scanner does not match its own source when it scrapes this file. The body is + /// brace-matched from the signature's opening `{` to its close, so a marker cannot + /// leak across into the next item. + fn sqlx_test_bodies(src: &str) -> Vec<(String, String)> { + // Split so the literal attribute never appears contiguously in this fn's own + // source (belt-and-suspenders with the whitespace-before-on-line check below). + let attr = concat!("#[sqlx", "::test]"); + let bytes = src.as_bytes(); + let mut out = Vec::new(); + let mut search = 0; + while let Some(rel) = src[search..].find(attr) { + let at = search + rel; + search = at + attr.len(); + // Real attribute only: the text between the line start and the attribute + // must be blank (rejects `//! … #[sqlx::test] …` and any string mention). + let line_start = src[..at].rfind('\n').map(|n| n + 1).unwrap_or(0); + if !src[line_start..at].trim().is_empty() { + continue; + } + let Some(fnrel) = src[search..].find("fn ") else { + break; + }; + let name_start = search + fnrel + "fn ".len(); + let name: String = src[name_start..] + .chars() + .take_while(|c| c.is_alphanumeric() || *c == '_') + .collect(); + if name.is_empty() { + continue; + } + let Some(brel) = src[name_start..].find('{') else { + break; + }; + let open = name_start + brel; + let mut depth = 0i32; + let mut j = open; + while j < src.len() { + match bytes[j] { + b'{' => depth += 1, + b'}' => { + depth -= 1; + if depth == 0 { + j += 1; + break; + } + } + _ => {} + } + j += 1; + } + out.push((name, src[open..j].to_string())); + search = j; + } + out + } + + /// Every `#[sqlx::test]` that spawns a node must end it with + /// `node.shutdown().await`, so the serve task's pool clones are released before + /// `#[sqlx::test]`'s synchronous `DROP DATABASE` cleanup fires. A test that returns + /// via `TestNode`'s async `Drop` instead only *signals* teardown (it drains on a + /// later scheduler tick — the Drop-regression tests poll up to 5s for it), so it + /// races the cleanup and can leak the per-test database / flake under parallelism. + /// The module header states this contract in prose; this makes it load-bearing — + /// removing any one `shutdown()` turns this RED. + /// + /// The two Drop-regression tests are the CLOSED allowlist: they deliberately + /// return WITHOUT `shutdown()` to exercise the Drop teardown, so they must NOT call + /// it. The allowlist is checked both ways — an allowlisted name that no longer + /// exists (rename/removal) or that grows a `shutdown()` call fails here — so it + /// cannot silently rot into covering a test it no longer describes. + #[test] + fn every_spawn_node_test_shuts_the_node_down() { + // The ONLY tests allowed to return without shutdown(): they test Drop itself. + const DROP_REGRESSION_ALLOW: &[&str] = &[ + "drop_without_shutdown_unblocks_database", + "drop_with_broken_graceful_chain_still_unblocks_via_abort", + ]; + + let src = include_str!("deny_harness.rs"); + let tests = sqlx_test_bodies(src); + + // Non-vacuous floor: if the scraper silently found nothing, every loop below + // would pass by checking zero tests. The file has 14 sqlx::test fns today. + assert!( + tests.len() >= 12, + "sqlx::test scrape found only {} tests — the parser likely broke (floor 12)", + tests.len() + ); + + // Each allowlisted name must resolve to a real spawn_node test that (correctly) + // does NOT call shutdown — so the allowlist cannot cover a renamed/deleted test + // or one that later grew a shutdown() call and no longer exercises Drop. + for allowed in DROP_REGRESSION_ALLOW { + let (_, body) = tests + .iter() + .find(|(name, _)| name == allowed) + .unwrap_or_else(|| { + panic!( + "allowlisted Drop-regression test `{allowed}` not found — renamed or \ + removed? update DROP_REGRESSION_ALLOW" + ) + }); + assert!( + body.contains("spawn_node"), + "allowlisted test `{allowed}` no longer spawns a node — it does not belong \ + on the shutdown allowlist" + ); + assert!( + !body.contains("node.shutdown("), + "allowlisted Drop-regression test `{allowed}` now calls shutdown() — it no \ + longer exercises Drop-only teardown; remove it from DROP_REGRESSION_ALLOW" + ); + } + + // Every other spawn_node test must shut its node down. + for (name, body) in &tests { + if !body.contains("spawn_node") || DROP_REGRESSION_ALLOW.contains(&name.as_str()) { + continue; + } + assert!( + body.contains("node.shutdown("), + "test `{name}` spawns a node but never calls node.shutdown().await — it \ + returns via TestNode::Drop, racing sqlx's DROP DATABASE cleanup (add the \ + awaited shutdown, or add it to DROP_REGRESSION_ALLOW if it deliberately \ + tests Drop)" + ); + } + } } diff --git a/crates/gitlawb-node/tests/support/assert.rs b/crates/gitlawb-node/tests/support/assert.rs index ccdfdc85..57333c8e 100644 --- a/crates/gitlawb-node/tests/support/assert.rs +++ b/crates/gitlawb-node/tests/support/assert.rs @@ -52,7 +52,15 @@ pub fn check_denied( /// full body once. pub async fn assert_denied(resp: reqwest::Response, expected: u16, withheld: &[&str]) { let status = resp.status().as_u16(); - let body = resp.text().await.unwrap_or_default(); + // A body-read failure must NOT fold to "" — an empty body passes the withheld + // scan vacuously, so an unreadable denial (a mid-body reset/truncation) would be + // certified leak-free without ever inspecting the bytes. Fail loud instead. A + // legitimately empty body is `Ok("")`, not `Err`, so the honest empty-401 denials + // still pass. + let body = resp + .text() + .await + .expect("read denial body: an unreadable denial cannot be certified leak-free"); if let Err(reason) = check_denied(status, &body, expected, withheld) { panic!("{reason}"); } diff --git a/crates/gitlawb-node/tests/support/probe.rs b/crates/gitlawb-node/tests/support/probe.rs index 3a587732..a449d77c 100644 --- a/crates/gitlawb-node/tests/support/probe.rs +++ b/crates/gitlawb-node/tests/support/probe.rs @@ -316,7 +316,7 @@ async fn seed_authored_issue( ) -> String { use super::signing::signed_request; - let client = reqwest::Client::new(); + let client = super::bounded_client(); let path = format!("/api/v1/repos/{owner_did}/{repo}/issues"); let body = br#"{"title":"prober close issue"}"#.to_vec(); let resp = signed_request( @@ -350,7 +350,7 @@ async fn seed_authored_issue( async fn seed_pending_task(node: &TestNode, delegator: &Keypair) -> String { use super::signing::signed_request; - let client = reqwest::Client::new(); + let client = super::bounded_client(); let delegator_did = delegator.did().to_string(); let body = format!(r#"{{"kind":"review","capability":"read","delegator_did":"{delegator_did}"}}"#) @@ -387,7 +387,7 @@ async fn seed_claimed_task(node: &TestNode, owner: &Keypair, assignee: &Keypair) let id = seed_pending_task(node, owner).await; - let client = reqwest::Client::new(); + let client = super::bounded_client(); let assignee_did = assignee.did().to_string(); let claim_path = format!("/api/v1/tasks/{id}/claim"); let body = format!(r#"{{"assignee_did":"{assignee_did}"}}"#).into_bytes(); @@ -446,7 +446,7 @@ async fn seed_bounty_at_stage( ) -> String { use super::signing::signed_request; - let client = reqwest::Client::new(); + let client = super::bounded_client(); // create_bounty (creator) -> 201 with the minted BountyRecord (carries the id). let create_path = format!("/api/v1/repos/{owner_did}/{repo}/bounties"); @@ -543,7 +543,7 @@ async fn seed_private_issue( ) -> String { use super::signing::signed_request; - let client = reqwest::Client::new(); + let client = super::bounded_client(); let path = format!("/api/v1/repos/{owner_did}/{repo}/issues"); let body = format!(r#"{{"title":"{marker}"}}"#).into_bytes(); let resp = signed_request( @@ -583,7 +583,7 @@ async fn seed_private_pr( ) { use super::signing::signed_request; - let client = reqwest::Client::new(); + let client = super::bounded_client(); let path = format!("/api/v1/repos/{owner_did}/{repo}/pulls"); let body = format!(r#"{{"title":"{marker}","source_branch":"feature","target_branch":"main"}}"#) @@ -626,7 +626,7 @@ async fn seed_private_bounty( ) -> String { use super::signing::signed_request; - let client = reqwest::Client::new(); + let client = super::bounded_client(); let path = format!("/api/v1/repos/{owner_did}/{repo}/bounties"); let body = format!(r#"{{"title":"{marker}","amount":1}}"#).into_bytes(); let resp = signed_request( @@ -761,7 +761,7 @@ async fn seed_private_push_and_cert( body.extend_from_slice(b"0000"); body.extend_from_slice(&pack); - let client = reqwest::Client::new(); + let client = super::bounded_client(); let push_path = format!("/{owner_did}/{repo}/git-receive-pack"); let resp = signed_request( &client, From 996e1b3fb2c0d91723723f9011add76900cdbfb7 Mon Sep 17 00:00:00 2001 From: t Date: Wed, 22 Jul 2026 00:41:00 -0500 Subject: [PATCH 41/41] test(node): close two vacuity holes in the shutdown-completeness guard An adversarial review flagged that the new every_spawn_node_test_shuts_the_ node_down guard could itself pass vacuously - the exact class it exists to prevent (INV-21 on the guard). - A spawn_node test whose only `node.shutdown(` occurrence was a comment (`// node.shutdown().await`) satisfied the raw substring check while still returning via Drop. - An unbalanced `{` inside a string literal in a test body could over-extend the brace-matched body slice and swallow a later test's shutdown, clearing the earlier one (latent: no current test triggers it). Both are closed by sanitizing the source through a new `code_only` pass (blanks comment and string/raw-string content, length-preserving) before the scan sees it, and the offender check is extracted to a pure spawn_node_tests_missing_shutdown so it can be driven with synthetic sources. Three adversarial unit tests pin both holes shut; stubbing code_only to a no-op turns the comment and brace tests RED (verified), so the sanitizer is load-bearing. deny_harness suite green (43 tests); clippy clean. --- crates/gitlawb-node/tests/deny_harness.rs | 250 ++++++++++++++++++++-- 1 file changed, 232 insertions(+), 18 deletions(-) diff --git a/crates/gitlawb-node/tests/deny_harness.rs b/crates/gitlawb-node/tests/deny_harness.rs index e7a31c03..c14203a9 100644 --- a/crates/gitlawb-node/tests/deny_harness.rs +++ b/crates/gitlawb-node/tests/deny_harness.rs @@ -2135,10 +2135,138 @@ mod completeness { // ── Shutdown-completeness guard: every spawn_node test must shutdown() ──────── - /// `(name, body)` for every real `#[sqlx::test]` in `src`. The attribute is only - /// counted when nothing but whitespace precedes it on its line, so a backtick - /// mention inside a doc comment (the module header names the attribute in prose) - /// is not mistaken for a test. The needle is spelled in two pieces so this + /// Blank the CONTENT of comments and double-quoted / raw string literals in `src` + /// (length-preserving, newlines kept) so the brace slicer and the marker scans see + /// CODE only. Without it this guard carries two vacuity holes of exactly the class + /// it exists to close (INV-21 on the guard itself): a `{` inside a format string + /// unbalances the body slice and can swallow a later test's `shutdown()`, and a + /// `// node.shutdown()` comment satisfies the presence check while the test still + /// returns through `Drop`. Handles `//` and nested `/* */` comments, `"..."` (with + /// `\` escapes), and hash-delimited raw strings (`r#"..."#`, `br#"..."#`). Char + /// literals and lifetimes are left as code: they carry neither braces-of-interest + /// nor the shutdown marker, and telling a `'a` lifetime from a `'a'` char would add + /// hazard for no coverage. + fn code_only(src: &str) -> String { + fn blank(out: &mut [u8], from: usize, to: usize) { + for c in &mut out[from..to] { + if *c != b'\n' { + *c = b' '; + } + } + } + let b = src.as_bytes(); + let n = b.len(); + let mut out = b.to_vec(); + let mut i = 0; + while i < n { + // Line comment: `//` … EOL. + if b[i] == b'/' && i + 1 < n && b[i + 1] == b'/' { + let start = i; + while i < n && b[i] != b'\n' { + i += 1; + } + blank(&mut out, start, i); + continue; + } + // Block comment `/* … */`, nesting allowed (Rust nests them). + if b[i] == b'/' && i + 1 < n && b[i + 1] == b'*' { + let start = i; + let mut depth = 1i32; + i += 2; + while i < n && depth > 0 { + if b[i] == b'/' && i + 1 < n && b[i + 1] == b'*' { + depth += 1; + i += 2; + } else if b[i] == b'*' && i + 1 < n && b[i + 1] == b'/' { + depth -= 1; + i += 2; + } else { + i += 1; + } + } + blank(&mut out, start, i.min(n)); + continue; + } + // Raw string `r"…"` / `r#"…"#` (also the b-prefixed byte form, entered at + // the `r`). A raw ident `r#foo` has no `"` after the hashes and falls through. + if b[i] == b'r' && i + 1 < n && (b[i + 1] == b'"' || b[i + 1] == b'#') { + let mut h = i + 1; + let mut hashes = 0usize; + while h < n && b[h] == b'#' { + hashes += 1; + h += 1; + } + if h < n && b[h] == b'"' { + let content = h + 1; + let mut k = content; + let mut close = None; + while k < n { + if b[k] == b'"' { + let mut extra = 0; + while k + 1 + extra < n && extra < hashes && b[k + 1 + extra] == b'#' { + extra += 1; + } + if extra == hashes { + close = Some(k); + break; + } + } + k += 1; + } + blank(&mut out, content, close.unwrap_or(n)); + i = close.map(|c| c + 1 + hashes).unwrap_or(n); + continue; + } + } + // Normal string `"…"`, honoring `\` escapes (so `\"` does not close it). + if b[i] == b'"' { + let content = i + 1; + let mut k = content; + while k < n { + if b[k] == b'\\' { + k += 2; + continue; + } + if b[k] == b'"' { + break; + } + k += 1; + } + blank(&mut out, content, k.min(n)); + i = if k >= n { n } else { k + 1 }; + continue; + } + i += 1; + } + String::from_utf8(out).expect("code_only blanks whole literal spans, so utf-8 survives") + } + + /// The spawn_node tests that fail the shutdown contract: they spawn a node, are not + /// allowlisted, and their (code-only) body carries no `node.shutdown(` call. Pure so + /// the adversarial unit tests below can drive synthetic sources; the `#[test]` guard + /// runs it over this file's real, sanitized source. + fn spawn_node_tests_missing_shutdown( + tests: &[(String, String)], + allow: &[&str], + ) -> Vec { + tests + .iter() + .filter(|(name, body)| { + body.contains("spawn_node") + && !allow.contains(&name.as_str()) + && !body.contains("node.shutdown(") + }) + .map(|(name, _)| name.clone()) + .collect() + } + + /// `(name, body)` for every real `#[sqlx::test]` in `src`. `src` is expected to be + /// [`code_only`]-sanitized so string/comment content cannot unbalance the body + /// slice. The attribute is only counted when nothing but whitespace precedes it on + /// its line (belt-and-suspenders with sanitization, which already blanks any + /// backtick mention inside a doc comment, e.g. the module header naming the + /// attribute in prose, so it is not mistaken for a test). The needle is spelled in + /// two pieces so this /// scanner does not match its own source when it scrapes this file. The body is /// brace-matched from the signature's opening `{` to its close, so a marker cannot /// leak across into the next item. @@ -2217,8 +2345,8 @@ mod completeness { "drop_with_broken_graceful_chain_still_unblocks_via_abort", ]; - let src = include_str!("deny_harness.rs"); - let tests = sqlx_test_bodies(src); + let src = code_only(include_str!("deny_harness.rs")); + let tests = sqlx_test_bodies(&src); // Non-vacuous floor: if the scraper silently found nothing, every loop below // would pass by checking zero tests. The file has 14 sqlx::test fns today. @@ -2254,17 +2382,103 @@ mod completeness { } // Every other spawn_node test must shut its node down. - for (name, body) in &tests { - if !body.contains("spawn_node") || DROP_REGRESSION_ALLOW.contains(&name.as_str()) { - continue; - } - assert!( - body.contains("node.shutdown("), - "test `{name}` spawns a node but never calls node.shutdown().await — it \ - returns via TestNode::Drop, racing sqlx's DROP DATABASE cleanup (add the \ - awaited shutdown, or add it to DROP_REGRESSION_ALLOW if it deliberately \ - tests Drop)" - ); - } + let missing = spawn_node_tests_missing_shutdown(&tests, DROP_REGRESSION_ALLOW); + assert!( + missing.is_empty(), + "these tests spawn a node but never call node.shutdown().await — they return \ + via TestNode::Drop, racing sqlx's DROP DATABASE cleanup (add the awaited \ + shutdown, or add to DROP_REGRESSION_ALLOW if they deliberately test Drop): \ + {missing:?}" + ); + } + + // ── Adversarial unit tests for the guard itself (INV-21 on the guard) ──────── + // These drive synthetic sources so the two vacuity holes an external review + // flagged are proven CLOSED by execution: a commented-out shutdown, and an + // unbalanced brace inside a string that would otherwise swallow a sibling's + // shutdown. Each passes (false green) against a raw substring/brace scan; + // sanitizing via `code_only` first is what makes them fail as they must. + + #[test] + fn shutdown_guard_flags_a_commented_out_shutdown() { + // The ONLY node.shutdown( occurrence is a comment: the node is never torn + // down, so the test must be flagged despite the textual match. + let synth = concat!( + "#[sqlx", + "::test]\n", + "async fn foo(pool: PgPool) {\n", + " let node = spawn_node(pool).await;\n", + " // node.shutdown().await; (deleted, left as a comment)\n", + "}\n" + ); + let tests = sqlx_test_bodies(&code_only(synth)); + assert_eq!( + spawn_node_tests_missing_shutdown(&tests, &[]), + vec!["foo".to_string()], + "a commented-out shutdown must NOT satisfy the guard" + ); + } + + #[test] + fn shutdown_guard_is_not_fooled_by_a_brace_inside_a_string() { + // `a` has no shutdown and an unbalanced `{` inside a string; `b` follows and + // does shut down. A brace-blind slicer would extend `a` into `b`, borrow b's + // shutdown, and clear `a` — sanitization keeps the two bodies separate. + let synth = concat!( + "#[sqlx", + "::test]\n", + "async fn a(pool: PgPool) {\n", + " let node = spawn_node(pool).await;\n", + " let _ = \"oops {\";\n", + "}\n", + "#[sqlx", + "::test]\n", + "async fn b(pool: PgPool) {\n", + " let node = spawn_node(pool).await;\n", + " node.shutdown().await;\n", + "}\n" + ); + let tests = sqlx_test_bodies(&code_only(synth)); + let missing = spawn_node_tests_missing_shutdown(&tests, &[]); + assert!( + missing.contains(&"a".to_string()), + "test `a` (no shutdown; unbalanced brace in a string) must be flagged, not \ + masked by test `b`'s shutdown: got {missing:?}" + ); + assert!( + !missing.contains(&"b".to_string()), + "test `b` shuts down and must not be flagged: {missing:?}" + ); + } + + #[test] + fn shutdown_guard_passes_clean_and_exempts_only_via_allowlist() { + let synth = concat!( + "#[sqlx", + "::test]\n", + "async fn good(pool: PgPool) {\n", + " let node = spawn_node(pool).await;\n", + " node.shutdown().await;\n", + "}\n", + "#[sqlx", + "::test]\n", + "async fn drop_probe(pool: PgPool) {\n", + " let node = spawn_node(pool).await;\n", + " // deliberately no shutdown: exercises Drop\n", + "}\n" + ); + let tests = sqlx_test_bodies(&code_only(synth)); + // Clean test + allowlisted Drop test -> no offenders. + assert!( + spawn_node_tests_missing_shutdown(&tests, &["drop_probe"]).is_empty(), + "a clean test plus an allowlisted Drop test must produce no offenders" + ); + // Without the allowlist the Drop test IS an offender, proving the exemption + // comes from the allowlist, not from an accident of the scan. + assert_eq!( + spawn_node_tests_missing_shutdown(&tests, &[]), + vec!["drop_probe".to_string()], + "un-allowlisted, the no-shutdown test must be flagged" + ); } }