diff --git a/.env.example b/.env.example index bbd9a342..74c9c913 100644 --- a/.env.example +++ b/.env.example @@ -13,9 +13,16 @@ GITLAWB_PUBLIC_URL=https://your-node.example.com # ── Server ──────────────────────────────────────────────────────────────── GITLAWB_HOST=0.0.0.0 GITLAWB_PORT=7545 +# Optional address to bind a Prometheus /metrics exposition endpoint on (e.g. +# 127.0.0.1:9091). Leave empty (default) to disable. Bind to localhost or a +# private interface — the metrics endpoint is unauthenticated. +GITLAWB_METRICS_ADDR= # ── Storage ─────────────────────────────────────────────────────────────── GITLAWB_REPOS_DIR=/data/repos +# Tigris (S3-compatible) bucket for repo storage. Leave empty (default) to +# disable Tigris and use local-only storage. +GITLAWB_TIGRIS_BUCKET= # PostgreSQL connection URL. Required. # When using the bundled docker-compose, this is wired automatically. @@ -26,6 +33,12 @@ DATABASE_URL=postgresql://gitlawb:changeme@localhost:5432/gitlawb # connections open lazily. Size against the DB server's max_connections, # remembering admin tooling opens its own pool. GITLAWB_DB_MAX_CONNECTIONS=20 +# Maximum connections in the dedicated advisory-lock pool (separate from the +# pool above). Each in-flight repo write pins one connection here for its whole +# lifetime, so size it to peak concurrent distinct-repo writers — keeping it +# separate is what stops a push burst from starving request handlers. Keep +# (main pool + lock pool) within the DB server's max_connections. +GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS=32 # Seconds a request waits for a pool connection before failing with 503. GITLAWB_DB_ACQUIRE_TIMEOUT_SECS=5 # Upper bound on each startup connect+migrate attempt, in seconds. Keep it @@ -39,6 +52,9 @@ GITLAWB_DB_RETRY_INITIAL_SECS=5 GITLAWB_DB_RETRY_MAX_SECS=60 # ── IPFS pinning (Pinata) ───────────────────────────────────────────────── +# URL of a local IPFS/Kubo node HTTP API (e.g. http://127.0.0.1:5001). Leave +# empty (default) to disable local IPFS. +GITLAWB_IPFS_API= # Get a JWT at https://app.pinata.cloud/developers/api-keys GITLAWB_PINATA_JWT= GITLAWB_PINATA_UPLOAD_URL=https://uploads.pinata.cloud/v3/files @@ -127,6 +143,20 @@ GITLAWB_PUSH_RATE_LIMIT=600 # the client IP. 0 disables. Default 120. GITLAWB_CREATE_RATE_LIMIT=120 +# ── Write rate limiting (non-creation authenticated writes) ─────────────── +# Max non-creation write requests per client IP per hour: issue/PR comments, +# labels, stars, merges, protect/unprotect, replicas, visibility, tasks, +# bounties, profile, and all GraphQL HTTP requests. The brake wraps the whole +# /graphql route, so queries and the playground GET consume this bucket too, not +# only mutations (GraphQL WebSocket subscriptions are excluded). Its own bucket, +# separate from the creation and push brakes. Uses GITLAWB_TRUSTED_PROXY to +# resolve the client IP. +# NOTE: this is a per-IP aggregate across ALL those write actions, so behind a +# shared NAT/egress IP (or with GITLAWB_TRUSTED_PROXY unset) many users collapse +# onto one bucket — raise this for automation-heavy or multi-user single-IP +# deployments. 0 disables. Default 600. +GITLAWB_WRITE_RATE_LIMIT=600 + # ── Peer-sync rate limiting (per client IP, uses GITLAWB_TRUSTED_PROXY below) ─ # /api/v1/peers/announce and /api/v1/sync/notify accept unsigned requests from # known peers and run at higher frequency, so a generous bucket. Separate from diff --git a/Cargo.lock b/Cargo.lock index 5dd2903d..f2ddd6bc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3300,7 +3300,7 @@ dependencies = [ [[package]] name = "git-remote-gitlawb" -version = "0.5.1" +version = "0.6.0" dependencies = [ "anyhow", "gitlawb-core", @@ -3312,7 +3312,7 @@ dependencies = [ [[package]] name = "gitlawb-attest" -version = "0.5.1" +version = "0.6.0" dependencies = [ "base64", "ed25519-dalek", @@ -3329,7 +3329,7 @@ dependencies = [ [[package]] name = "gitlawb-core" -version = "0.5.1" +version = "0.6.0" dependencies = [ "anyhow", "base64", @@ -3356,13 +3356,14 @@ dependencies = [ [[package]] name = "gitlawb-node" -version = "0.5.1" +version = "0.6.0" dependencies = [ "alloy", "anyhow", "async-compression", "async-graphql", "async-graphql-axum", + "async-trait", "aws-config", "aws-sdk-s3", "axum", @@ -3412,10 +3413,11 @@ dependencies = [ [[package]] name = "gl" -version = "0.5.1" +version = "0.6.0" dependencies = [ "alloy", "anyhow", + "base64", "chrono", "clap", "dirs", diff --git a/crates/gitlawb-node/Cargo.toml b/crates/gitlawb-node/Cargo.toml index 7cc3727f..103e0ca5 100644 --- a/crates/gitlawb-node/Cargo.toml +++ b/crates/gitlawb-node/Cargo.toml @@ -12,6 +12,7 @@ path = "src/main.rs" [dependencies] gitlawb-core = { path = "../gitlawb-core" } +async-trait = "0.1" ed25519-dalek = { workspace = true } base64 = { workspace = true } tokio = { workspace = true } diff --git a/crates/gitlawb-node/src/admin.rs b/crates/gitlawb-node/src/admin.rs new file mode 100644 index 00000000..2e84d774 --- /dev/null +++ b/crates/gitlawb-node/src/admin.rs @@ -0,0 +1,1695 @@ +//! Admin subcommands invoked out-of-band, not part of the running node. +//! +//! `purge-spam` produces a reviewable dry-run list of empty spam-burst repos and, +//! only behind an explicit `--execute` flag, deletes them one at a time. The +//! selection logic is the load-bearing security part: a repo qualifies ONLY if it +//! is owned by the named burst DID AND is verified empty (zero git refs) PER REPO, +//! and a hard exclusion gate (evaluated BEFORE the empty check) keeps +//! content-bearing and intern/mirror-bot DIDs out no matter what. + +use std::path::{Path, PathBuf}; + +use anyhow::{Context, Result}; +use tracing::{info, warn}; + +use crate::db::{Db, RepoRecord}; +use crate::git::store; + +/// The did:key of the spam burst this tool targets. The purge is scoped to +/// exactly this owner; an empty repo owned by anyone else is never a candidate. +pub const SPAM_BURST_TARGET_DID: &str = "did:key:z6Mkopj6mhcMayipekXbTRFMZPM6Bsgy4FQZuN9fannXSLTC"; + +/// DIDs that must never be touched, even when they own an empty repo whose +/// signature otherwise matches the burst. The gate is evaluated BEFORE the empty +/// check so it wins unconditionally: +/// - `z6Mkk4L…` is a content-bearing live user. +/// - `z6MkqRz…` is the intern / mirror-bot DID. +pub const EXCLUDED_DIDS: &[&str] = &[ + "did:key:z6Mkk4LDvfA8VQmdehbJDvxp133sdtXUhR2UkUnMPguX7gnP", + "did:key:z6MkqRzACJ5iCDdkiymAPK3gq18z2iecZHeAuUyW6JnwRfoM", +]; + +/// Outcome tally of a `run_purge_spam` execute pass. Returned so the DB-delete +/// count is never conflated with full success: `disk_failed` records repos whose +/// row was deleted but whose on-disk dir could not be removed (or escaped the +/// repos_dir containment check), so an operator sees the DB/disk drift instead of +/// a clean "N deleted" summary. +#[derive(Debug, Default, Clone, PartialEq, Eq)] +pub struct PurgeSummary { + /// Repo rows actually deleted from the DB. + pub deleted: u64, + /// Candidates skipped because they were no longer empty (pre-filter or the + /// authoritative recheck under the lock). + pub skipped_not_empty: usize, + /// Candidates skipped because a live writer held the per-repo lock. + pub skipped_locked: usize, + /// Candidates skipped because the object store could not be consulted for the + /// authoritative emptiness recheck (fail-closed: never delete on a store + /// error rather than risk deleting a repo with live remote refs). + pub skipped_store_error: usize, + /// Rows deleted whose on-disk dir removal FAILED (or was refused by the + /// containment guard) — DB/disk drift the operator must reconcile. + pub disk_failed: usize, + /// Rows+dirs deleted whose object-store archive removal FAILED — the archive + /// survives and could be re-downloaded into a later same-owner/name repo, so + /// this is tracked separately and never folded into a clean success. + pub archive_failed: usize, + /// Candidates with no local copy, admitted only because an object store is + /// configured (emptiness decided under the lock at execute time). Reported so + /// the dry-run can surface them distinctly from locally-verified candidates. + pub remote_unverified: usize, +} + +/// A repo selected for purge, with the evidence that qualified it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Candidate { + pub id: String, + pub owner_did: String, + pub name: String, + /// Number of git refs found on disk. 0 for a locally-verified empty candidate; + /// also 0 for a remote-unverified one whose emptiness is decided under the lock. + pub ref_count: usize, + /// True when the repo has no local copy and was admitted only because an object + /// store is configured. Its emptiness has NOT been verified — the execute path + /// must refresh from the archive and recheck UNDER the per-repo lock before any + /// delete; the dry-run lists it distinctly and never touches it. + pub remote_unverified: bool, +} + +/// Whether a DID is on the hard exclusion list. Compared under did:key +/// normalization (the same convention the repos table and every ownership check +/// use), so an excluded identity stored in either `did:key:z6…` or bare `z6…` +/// form is protected regardless of the form the exclusion constant is written in. +fn is_excluded(owner_did: &str) -> bool { + let owner_key = crate::db::normalize_owner_key(owner_did); + EXCLUDED_DIDS + .iter() + .any(|d| crate::db::normalize_owner_key(d) == owner_key) +} + +/// Pure candidate selector — the security core, isolated from disk and DB so the +/// exclusion + empty logic is directly testable. +/// +/// `repos` is the raw row set to consider (the caller supplies the target DID's +/// rows). `local_refs_of` returns `Some(n)` when a local bare repo exists (n +/// refs) and `None` when there is no local copy; the CLI wires the real on-disk +/// source, tests inject precomputed states. `store_configured` gates whether a +/// missing-local repo may be admitted. +/// +/// A repo qualifies ONLY if, PER REPO: +/// 1. its owner is NOT on the exclusion list (gate evaluated FIRST), AND +/// 2. its owner is exactly the target burst DID, AND +/// 3. EITHER it is locally verified empty (`Some(0)`), OR it has no local copy +/// (`None`) AND an object store is configured — in which case it is admitted +/// as remote-unverified and its emptiness is decided under the lock later. +/// +/// The exclusion gate is checked before everything so an empty repo owned by an +/// excluded DID is dropped regardless of its ref signature. A missing-local repo +/// with no object store fails closed (skipped). +pub fn select_spam_candidates( + repos: &[RepoRecord], + target_did: &str, + store_configured: bool, + mut local_refs_of: F, +) -> Vec +where + F: FnMut(&RepoRecord) -> Option, +{ + let mut out = Vec::new(); + for repo in repos { + // Hard exclusion gate FIRST — wins over the empty signature. + if is_excluded(&repo.owner_did) { + continue; + } + // Scope to the named burst only, under did:key normalization so a burst + // row stored in either did:key or bare form is matched consistently. + if crate::db::normalize_owner_key(&repo.owner_did) + != crate::db::normalize_owner_key(target_did) + { + continue; + } + // `local_refs_of` is `Some(n)` when a local bare repo exists and `None` + // when it does not. A local empty repo (Some(0)) is a verified candidate; + // a local non-empty repo is skipped; a missing local copy is a candidate + // ONLY when an object store is configured (its emptiness is then decided + // authoritatively under the lock after refresh), else it fails closed. + let remote_unverified = match local_refs_of(repo) { + Some(0) => false, + Some(_) => continue, + None => { + if store_configured { + true + } else { + continue; + } + } + }; + out.push(Candidate { + id: repo.id.clone(), + owner_did: repo.owner_did.clone(), + name: repo.name.clone(), + ref_count: 0, + remote_unverified, + }); + } + out +} + +/// Count the git refs of a repo on disk. Zero refs means the repo is empty. +/// +/// Resolves the repo's bare path from `repos_dir` + owner/name (the same layout +/// `git::store::repo_disk_path` writes) and shells to `git for-each-ref` via +/// `store::list_refs`. A repo whose on-disk path is missing or unreadable is +/// treated as having an unknown, non-empty ref count so it is NOT selected — the +/// tool fails closed and never deletes on a read error. +/// +/// Critically, a `0` count must come from THIS exact bare repo and never from +/// git's upward repository discovery. `git for-each-ref` runs with the repo path +/// as its cwd and no explicit `--git-dir`, so if the path exists but is not +/// itself a git dir, git walks parent directories for a `.git` — and `repos_dir` +/// may live inside the operator's own git checkout. That would read a DIFFERENT +/// repo's refs (possibly `0`) and delete a real repo. We defend by requiring the +/// bare-repo markers (`HEAD` file + `objects/` dir) before trusting any count; +/// anything else fails closed (treated non-empty, skipped). +/// Local ref state for selection: `Some(n)` when a local bare repo exists (n +/// refs), `None` ONLY when the path is verifiably absent (a `NotFound` from +/// `symlink_metadata`). `None` is what lets selection distinguish a truly +/// missing-local repo (a remote-unverified candidate when a store is configured) +/// from a one-ref repo — both of which `ref_count_on_disk` collapses to a +/// non-zero count. Everything else fails closed to `Some(1)` so it is skipped and +/// never admitted as remote-unverified: an unsafe name, an unreadable/unstat-able +/// path, a dangling symlink, or an existing directory that is not a bare git repo +/// (returning `None` for the latter would promote it, and the remote_unverified +/// refresh's `remove_dir_all` would overwrite that non-repository directory). +fn local_refs_on_disk(repos_dir: &Path, owner_did: &str, name: &str) -> Option { + // Fail closed on an unsafe repo name BEFORE building any on-disk path (a + // peer-mirror row can carry a `../` name). Report it as non-empty so it is + // never a candidate — never as missing (which could admit it remote-unverified). + if let Err(e) = crate::git::repo_store::validate_repo_name(name) { + warn!(name = %name, err = %e, + "purge-spam: unsafe repo name — treating as non-empty (skipped)"); + return Some(1); + } + let path = store::repo_disk_path(repos_dir, owner_did, name); + // Only a VERIFIED NotFound (the path truly does not exist) may return None, + // which promotes the repo to remote_unverified — and that refresh path does a + // remove_dir_all on the target. Use `symlink_metadata` (not `metadata`) so a + // dangling symlink counts as PRESENT, not NotFound. Any existing path, or any + // stat error other than NotFound, fails closed to Some(1) so the refresh never + // overwrites a non-repository directory sitting at the target. + match std::fs::symlink_metadata(&path) { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return None, + Err(e) => { + warn!(path = %path.display(), err = %e, + "purge-spam: could not stat path — treating as non-empty (skipped)"); + return Some(1); + } + // A symlink at the target is never a legitimate bare repo we may treat as + // empty-and-purgeable. The marker checks below FOLLOW the link, so a + // symlink to a real empty bare repo outside `repos_dir` would otherwise + // read 0 refs and be admitted — deleting the DB row while `path_within` + // spares the outside dir. Fail closed as present so it is never a candidate. + Ok(m) if m.file_type().is_symlink() => { + warn!(path = %path.display(), + "purge-spam: path is a symlink — treating as non-empty (skipped)"); + return Some(1); + } + Ok(_) => {} + } + if !path.join("HEAD").is_file() || !path.join("objects").is_dir() { + // The path exists but is not a bare repo — fail closed as present so it is + // never promoted to remote_unverified and overwritten by a refresh. + warn!(path = %path.display(), + "purge-spam: path exists but is not a bare git repo — treating as non-empty (skipped)"); + return Some(1); + } + match store::list_refs(&path) { + Ok(refs) => Some(refs.len()), + Err(e) => { + warn!(path = %path.display(), err = %e, + "purge-spam: could not read refs — treating as non-empty (skipped)"); + Some(1) + } + } +} + +/// Ref count keyed on owner+name (returns 1 for a missing/unsafe/unreadable +/// repo — fail closed), used by the execute path to re-verify emptiness right +/// before deleting (using only a [`Candidate`]) and, after `refresh_from_archive` +/// downloads a remote-unverified candidate, to decide its emptiness under the lock. +fn ref_count_on_disk(repos_dir: &Path, owner_did: &str, name: &str) -> usize { + // Fail closed on an unsafe repo name BEFORE building any on-disk path. A + // peer-mirror row (which skips API name validation) can carry a `../` name; + // `repo_disk_path` would join it verbatim and resolve OUTSIDE `repos_dir`, + // pointing this "empty" check — and later the delete — at an unrelated repo. + // Reject it here so such a row is never a candidate (treated non-empty). + if let Err(e) = crate::git::repo_store::validate_repo_name(name) { + warn!(name = %name, err = %e, + "purge-spam: unsafe repo name — treating as non-empty (skipped)"); + return 1; + } + let path = store::repo_disk_path(repos_dir, owner_did, name); + if !path.join("HEAD").is_file() || !path.join("objects").is_dir() { + // Not a bare git repo at the exact expected path. Do NOT trust a ref + // count that git discovery could have read from a parent repository. + warn!(path = %path.display(), + "purge-spam: path is not a bare git repo — treating as non-empty (skipped)"); + return 1; + } + match store::list_refs(&path) { + Ok(refs) => refs.len(), + Err(e) => { + // Fail closed: an unreadable repo is not provably empty, so keep it + // out of the candidate set (report it as one ref so it's excluded). + warn!(path = %path.display(), err = %e, + "purge-spam: could not read refs — treating as non-empty (skipped)"); + 1 + } + } +} + +/// Whether `path` resolves canonically inside `root`. Both are canonicalized so +/// symlinks and `..` segments are fully resolved before the containment test; a +/// path that does not exist (or a root that cannot be canonicalized) fails closed +/// to `false`. Used as the last gate before a destructive `remove_dir_all`. +fn path_within(path: &Path, root: &Path) -> bool { + match (std::fs::canonicalize(path), std::fs::canonicalize(root)) { + (Ok(p), Ok(r)) => p.starts_with(&r), + _ => false, + } +} + +/// Split selected candidates into (delete, skip) by a fresh emptiness re-check, +/// so a repo that gained a ref between selection and deletion (a TOCTOU push) +/// is never deleted. Pure over the `recheck` closure so the skip branch is +/// directly testable; the CLI wires the real on-disk re-check. +fn partition_for_delete( + candidates: &[Candidate], + mut recheck: F, +) -> (Vec<&Candidate>, Vec<&Candidate>) +where + F: FnMut(&Candidate) -> usize, +{ + let mut to_delete = Vec::new(); + let mut to_skip = Vec::new(); + for c in candidates { + if recheck(c) == 0 { + to_delete.push(c); + } else { + to_skip.push(c); + } + } + (to_delete, to_skip) +} + +/// Run the `purge-spam` admin subcommand. +/// +/// Enumerates the target burst DID's repos, verifies each is empty on disk, +/// applies the exclusion gate, prints one dry-run row per candidate with owner + +/// ref-count evidence, and — only when `execute` is true — deletes the DB row of +/// each candidate one at a time. Dry-run (the default) deletes nothing. +pub async fn run_purge_spam( + db: &Db, + repo_store: &crate::git::repo_store::RepoStore, + repos_dir: &Path, + execute: bool, +) -> Result { + let repos_dir: PathBuf = repos_dir.to_path_buf(); + let rows = db + .list_repos_by_owner_did(SPAM_BURST_TARGET_DID) + .await + .context("listing repos for the spam-burst target DID")?; + + // A repo with no local copy is admitted as a remote-unverified candidate + // only when an object store is configured — its emptiness is then decided + // under the lock after refresh_from_archive. Without a store, missing-local + // fails closed (skipped), preserving the wrong-machine safety rule. + let store_configured = repo_store.has_object_store(); + let candidates = + select_spam_candidates(&rows, SPAM_BURST_TARGET_DID, store_configured, |repo| { + local_refs_on_disk(&repos_dir, &repo.owner_did, &repo.name) + }); + let remote_unverified_count = candidates.iter().filter(|c| c.remote_unverified).count(); + + info!( + target = SPAM_BURST_TARGET_DID, + scanned = rows.len(), + candidates = candidates.len(), + execute, + "purge-spam: candidate selection complete" + ); + + if candidates.is_empty() { + println!("purge-spam: no empty spam-burst repos found for {SPAM_BURST_TARGET_DID}"); + return Ok(PurgeSummary::default()); + } + + println!( + "purge-spam: {} candidate(s) for {} ({} mode)", + candidates.len(), + SPAM_BURST_TARGET_DID, + if execute { "EXECUTE" } else { "dry-run" } + ); + for c in &candidates { + let marker = if c.remote_unverified { + " [remote-only, emptiness verified under lock at execute]" + } else { + "" + }; + println!( + " {} owner={} name={} refs={}{marker}", + c.id, c.owner_did, c.name, c.ref_count + ); + } + + if !execute { + println!( + "purge-spam: dry-run — nothing deleted ({remote_unverified_count} remote-only, verified under lock only on --execute). Re-run with --execute to delete the {} candidate(s).", + candidates.len() + ); + return Ok(PurgeSummary { + remote_unverified: remote_unverified_count, + ..PurgeSummary::default() + }); + } + + // Re-verify emptiness immediately before deleting: a push may have landed + // between selection and now (TOCTOU). A remote-unverified candidate has no + // local copy yet, so `ref_count_on_disk` would report it non-empty and drop + // it here — pass it straight through instead; the authoritative emptiness + // check for it happens under the lock in the execute loop after refresh. + let (to_delete, to_skip) = partition_for_delete(&candidates, |c| { + if c.remote_unverified { + 0 + } else { + ref_count_on_disk(&repos_dir, &c.owner_did, &c.name) + } + }); + for c in &to_skip { + warn!(repo = %c.id, "purge-spam: repo no longer empty at delete time — skipped (TOCTOU)"); + } + + // Execute: delete per-repo, never a single blanket "delete all of owner X". + // A per-repo failure warns and continues rather than aborting the batch. + let mut summary = PurgeSummary { + remote_unverified: remote_unverified_count, + skipped_not_empty: to_skip.len(), + ..PurgeSummary::default() + }; + for c in &to_delete { + // Hold the per-repo advisory lock across the FINAL emptiness recheck and + // the delete, so a concurrent receive-pack cannot land a ref in the window + // between recheck and delete (M4). A repo currently locked by a live writer + // is skipped, never force-deleted out from under the push. + let guard = match repo_store.try_lock_repo(&c.owner_did, &c.name).await { + Ok(Some(g)) => g, + Ok(None) => { + warn!(repo = %c.id, "purge-spam: repo is locked by a live writer — skipped"); + summary.skipped_locked += 1; + continue; + } + Err(e) => { + warn!(repo = %c.id, err = %e, "purge-spam: could not acquire repo lock — skipped"); + summary.skipped_locked += 1; + continue; + } + }; + // Refresh the local copy from the authoritative object store (if any) + // before the recheck: on a Tigris deployment the admin node's local disk + // can be stale, so an emptiness check against local alone could delete a + // repo that has live remote refs. Fail closed on any store error — never + // delete on an unverified view. + if let Err(e) = repo_store.refresh_from_archive(&c.owner_did, &c.name).await { + warn!(repo = %c.id, err = %e, + "purge-spam: could not consult the object store — skipped (fail-closed)"); + summary.skipped_store_error += 1; + guard.release().await; + continue; + } + // Authoritative recheck UNDER the lock: a ref that landed before we locked + // (or that the object-store refresh surfaced) makes the repo non-empty, so + // it must not be deleted. + if ref_count_on_disk(&repos_dir, &c.owner_did, &c.name) != 0 { + warn!(repo = %c.id, "purge-spam: repo no longer empty under lock — skipped (TOCTOU)"); + summary.skipped_not_empty += 1; + guard.release().await; + continue; + } + match db.delete_repo_by_id(&c.id).await { + Ok(0) => { + warn!(repo = %c.id, "purge-spam: repo row already gone — nothing to delete"); + } + Ok(n) => { + summary.deleted += n; + // Remove the now-orphaned on-disk bare repo (empty, so cheap) so + // the DB row and disk stay consistent. Belt-and-suspenders: assert + // the resolved path is canonically INSIDE repos_dir before any + // remove_dir_all, so a symlinked slug dir or any residual traversal + // can never delete outside the repo root (the name is already + // validated at selection; this guards the destructive op itself). + // A disk-removal failure is counted separately, NOT folded into the + // deleted total, so the summary never reports a clean success while + // an on-disk dir survives (DB/disk drift the operator must fix). + let path = store::repo_disk_path(&repos_dir, &c.owner_did, &c.name); + if !path_within(&path, &repos_dir) { + warn!(repo = %c.id, path = %path.display(), + "purge-spam: on-disk path escapes repos_dir — refusing to remove"); + summary.disk_failed += 1; + } else if let Err(e) = std::fs::remove_dir_all(&path) { + warn!(repo = %c.id, path = %path.display(), err = %e, + "purge-spam: deleted DB row but could not remove on-disk repo dir"); + summary.disk_failed += 1; + } else { + info!(repo = %c.id, rows = n, "purge-spam: deleted repo row + on-disk dir"); + } + // Delete the object-store archive too (a no-op when no store is + // configured), else it survives and can be downloaded into a + // later repo created with the same owner/name. Counted separately + // so a surviving archive never reads as a clean success. + if let Err(e) = repo_store.delete_archive(&c.owner_did, &c.name).await { + warn!(repo = %c.id, err = %e, + "purge-spam: deleted row + dir but could not delete the object-store archive"); + summary.archive_failed += 1; + } + } + Err(e) => { + warn!(repo = %c.id, err = %e, "purge-spam: failed to delete repo row — continuing"); + } + } + guard.release().await; + } + println!( + "purge-spam: deleted {} repo row(s); skipped {} (no longer empty), {} (locked by a live writer), {} (object store unreachable); {} on-disk removal(s) failed, {} archive removal(s) failed.", + summary.deleted, + summary.skipped_not_empty, + summary.skipped_locked, + summary.skipped_store_error, + summary.disk_failed, + summary.archive_failed + ); + Ok(summary) +} + +#[cfg(test)] +mod tests { + use super::*; + use chrono::Utc; + + const TARGET: &str = SPAM_BURST_TARGET_DID; + const EXCLUDED_CONTENT: &str = EXCLUDED_DIDS[0]; // z6Mkk4L… content-bearing live user + const EXCLUDED_INTERN: &str = EXCLUDED_DIDS[1]; // z6MkqRz… intern/mirror-bot + const UNRELATED: &str = "did:key:z6MkUnrelatedStrangerDidThatIsNotTheBurst"; + + fn repo(id: &str, owner: &str, name: &str) -> RepoRecord { + RepoRecord { + id: id.to_string(), + name: name.to_string(), + owner_did: owner.to_string(), + description: None, + is_public: true, + default_branch: "main".to_string(), + created_at: Utc::now(), + updated_at: Utc::now(), + disk_path: format!("/srv/{id}.git"), + forked_from: None, + machine_id: None, + } + } + + /// Ref counts keyed by repo id; anything absent defaults to 0 (empty). + fn refs_by_id<'a>(map: &'a [(&'a str, usize)]) -> impl Fn(&RepoRecord) -> Option + 'a { + // A local bare repo exists for every test row (Some), with `n` refs. + move |r: &RepoRecord| { + Some( + map.iter() + .find(|(id, _)| *id == r.id) + .map(|(_, n)| *n) + .unwrap_or(0), + ) + } + } + + // Test 1: an empty repo owned by the target DID is a candidate. + #[test] + fn empty_target_repo_is_a_candidate() { + let repos = vec![repo("t-empty", TARGET, "spam1")]; + let got = select_spam_candidates(&repos, TARGET, false, refs_by_id(&[])); + assert_eq!(got.len(), 1, "empty target repo must be selected"); + assert_eq!(got[0].id, "t-empty"); + assert_eq!(got[0].owner_did, TARGET); + assert_eq!( + got[0].ref_count, 0, + "candidate must carry ref-count evidence" + ); + } + + // Test 2: a target-owned repo WITH refs is absent (per-repo empty check, not + // per-DID). + #[test] + fn target_repo_with_refs_is_absent() { + let repos = vec![ + repo("t-empty", TARGET, "spam1"), + repo("t-nonempty", TARGET, "real"), + ]; + let got = select_spam_candidates(&repos, TARGET, false, refs_by_id(&[("t-nonempty", 3)])); + let ids: Vec<&str> = got.iter().map(|c| c.id.as_str()).collect(); + assert!(ids.contains(&"t-empty"), "empty target repo still selected"); + assert!( + !ids.contains(&"t-nonempty"), + "a target repo WITH refs must NOT be selected (per-repo, not per-DID)" + ); + } + + // Test 3 (MUST ASSERT): an EMPTY repo owned by the excluded content DID is + // absent — the exclusion gate wins over the empty signature. + // + // Driven with the excluded DID passed AS the target so the exclusion gate is + // the ONLY barrier: with the gate removed this repo would match owner==target + // and be selected, so the test goes RED. This is what makes the gate + // load-bearing rather than shadowed by the target-scope check. + #[test] + fn empty_excluded_content_repo_is_absent() { + let repos = vec![repo("x-content", EXCLUDED_CONTENT, "anything")]; + let got = select_spam_candidates(&repos, EXCLUDED_CONTENT, false, refs_by_id(&[])); + assert!( + got.is_empty(), + "an empty repo owned by the excluded content DID must be excluded even \ + if that DID were the target, got {got:?}" + ); + // And of course it is also absent when the real burst DID is the target. + let got = select_spam_candidates(&repos, TARGET, false, refs_by_id(&[])); + assert!(got.is_empty()); + } + + // Test 4 (MUST ASSERT): an EMPTY repo owned by the intern DID is absent. + // Same construction as test 3: the intern DID is passed as the target so the + // exclusion gate is the sole reason it is dropped (RED without the gate). + #[test] + fn empty_intern_repo_is_absent() { + let repos = vec![repo("x-intern", EXCLUDED_INTERN, "mirror")]; + let got = select_spam_candidates(&repos, EXCLUDED_INTERN, false, refs_by_id(&[])); + assert!( + got.is_empty(), + "an empty repo owned by the intern/mirror-bot DID must be excluded even \ + if that DID were the target, got {got:?}" + ); + let got = select_spam_candidates(&repos, TARGET, false, refs_by_id(&[])); + assert!(got.is_empty()); + } + + // Test 5: an empty repo owned by an unrelated DID is absent — the tool targets + // the named burst only. + #[test] + fn empty_unrelated_repo_is_absent() { + let repos = vec![repo("u-empty", UNRELATED, "whatever")]; + let got = select_spam_candidates(&repos, TARGET, false, refs_by_id(&[])); + assert!( + got.is_empty(), + "an empty repo owned by a non-target DID must not be selected, got {got:?}" + ); + } + + // The full matrix in one selector pass: only the empty target repo survives. + #[test] + fn full_matrix_selects_only_empty_target() { + let repos = vec![ + repo("t-empty", TARGET, "spam1"), // selected + repo("t-nonempty", TARGET, "real"), // has refs → out + repo("x-content", EXCLUDED_CONTENT, "a"), // excluded, empty → out + repo("x-intern", EXCLUDED_INTERN, "b"), // excluded, empty → out + repo("u-empty", UNRELATED, "c"), // wrong owner → out + ]; + let got = select_spam_candidates(&repos, TARGET, false, refs_by_id(&[("t-nonempty", 2)])); + assert_eq!( + got.iter().map(|c| c.id.as_str()).collect::>(), + vec!["t-empty"], + "only the empty target repo may survive the full matrix" + ); + } + + // An excluded DID that is empty is still out even when that excluded DID is + // itself the target — pins that the exclusion gate runs BEFORE the owner/empty + // checks and is the sole barrier here (RED without the gate). + #[test] + fn exclusion_gate_precedes_empty_check() { + let repos = vec![repo("collision", EXCLUDED_CONTENT, "spam1")]; + let got = select_spam_candidates(&repos, EXCLUDED_CONTENT, false, refs_by_id(&[])); + assert!(got.is_empty(), "exclusion must win even on an empty repo"); + } + + // TOCTOU: a candidate that gained a ref between selection and the pre-delete + // re-check must be skipped, not deleted; the rest of the batch still deletes. + #[test] + fn partition_for_delete_skips_repos_no_longer_empty() { + let cand = |id: &str| Candidate { + id: id.into(), + owner_did: "o".into(), + name: id.into(), + ref_count: 0, + remote_unverified: false, + }; + let cands = vec![cand("still-empty"), cand("now-nonempty")]; + // Re-check reports the second repo as no longer empty. + let (to_delete, to_skip) = + partition_for_delete(&cands, |c| usize::from(c.id == "now-nonempty")); + assert_eq!( + to_delete.iter().map(|c| c.id.as_str()).collect::>(), + ["still-empty"] + ); + assert_eq!( + to_skip.iter().map(|c| c.id.as_str()).collect::>(), + ["now-nonempty"] + ); + } +} + +#[cfg(test)] +mod db_tests { + use super::*; + use crate::db::{Db, RepoRecord}; + use chrono::Utc; + use sqlx::PgPool; + + async fn db(pool: &PgPool) -> Db { + let db = Db::for_testing(pool.clone()); + db.run_migrations().await.unwrap(); + db + } + + /// Lock-only RepoStore over a test pool + repos_dir (no Tigris), for the purge + /// callers that now take the advisory lock (M4). + fn test_store(repos_dir: &Path, pool: &PgPool) -> crate::git::repo_store::RepoStore { + crate::git::repo_store::RepoStore::for_testing(repos_dir.to_path_buf(), pool.clone()) + } + + /// A Postgres pool of a fixed size with a short acquire timeout and no ambient + /// reaping, so a purge exercised at GITLAWB_DB_MAX_CONNECTIONS=1 fails fast + /// (rather than stalling the whole test) if the single connection is pinned. + /// min/idle/lifetime pinned so a held connection is never reclaimed + /// mid-assertion. + async fn sized_no_reap_pool( + connect_opts: &sqlx::postgres::PgConnectOptions, + max_connections: u32, + ) -> PgPool { + sqlx::postgres::PgPoolOptions::new() + .max_connections(max_connections) + .acquire_timeout(std::time::Duration::from_secs(2)) + .min_connections(0) + .idle_timeout(None) + .max_lifetime(None) + .test_before_acquire(false) + .connect_with(connect_opts.clone()) + .await + .unwrap() + } + + fn rec(id: &str, owner: &str, name: &str) -> RepoRecord { + RepoRecord { + id: id.to_string(), + name: name.to_string(), + owner_did: owner.to_string(), + description: None, + is_public: true, + default_branch: "main".to_string(), + created_at: Utc::now(), + updated_at: Utc::now(), + disk_path: format!("/srv/{id}.git"), + forked_from: None, + machine_id: None, + } + } + + async fn count_rows(db: &Db) -> i64 { + sqlx::query_scalar::<_, i64>("SELECT COUNT(*) FROM repos") + .fetch_one(db.pool()) + .await + .unwrap() + } + + // Test 6: a dry-run over the full matrix deletes nothing. The empty check is + // driven off a repos_dir with NO repos on disk, so every row reads as empty + // (list_refs on a missing repo returns Err → treated as non-empty and skipped), + // which is fine here: the assertion is that dry-run mutates no rows regardless. + #[sqlx::test] + async fn dry_run_deletes_nothing(pool: PgPool) { + let db = db(&pool).await; + for r in [ + rec("t-empty", SPAM_BURST_TARGET_DID, "spam1"), + rec("t-nonempty", SPAM_BURST_TARGET_DID, "real"), + rec("x-content", EXCLUDED_DIDS[0], "a"), + rec("x-intern", EXCLUDED_DIDS[1], "b"), + rec("u-empty", "did:key:z6MkUnrelated", "c"), + ] { + db.create_repo(&r).await.unwrap(); + } + let before = count_rows(&db).await; + assert_eq!(before, 5); + + // Materialize a REAL empty bare repo for the empty target so it is a genuine + // purge candidate. Without an on-disk candidate, run_purge_spam hits the + // no-candidate early return and the `if !execute { return }` guard is never + // exercised with candidates present — the L10 gap this test now closes. + let tmp = tempfile::TempDir::new().unwrap(); + let target_dir = store::repo_disk_path(tmp.path(), SPAM_BURST_TARGET_DID, "spam1"); + store::init_bare(&target_dir).unwrap(); + assert_eq!( + store::list_refs(&target_dir).unwrap().len(), + 0, + "precondition: the candidate is a real empty bare repo" + ); + + let store = test_store(tmp.path(), &pool); + let summary = run_purge_spam(&db, &store, tmp.path(), false) + .await + .unwrap(); + + // A candidate existed, but dry-run deletes nothing — DB row and on-disk dir + // both survive. RED if the `if !execute` guard is removed. + assert_eq!(summary.deleted, 0, "dry-run must delete no rows"); + let after = count_rows(&db).await; + assert_eq!(after, before, "dry-run must not delete any repo rows"); + assert!( + db.get_repo(SPAM_BURST_TARGET_DID, "spam1") + .await + .unwrap() + .is_some(), + "the candidate row must survive a dry-run" + ); + assert!( + target_dir.exists(), + "dry-run must not remove the on-disk repo dir" + ); + } + + // The DB accessor lists exactly the target DID's rows (exact owner match), and + // delete_repo_by_id removes exactly one row, so the execute path deletes per + // repo. This exercises the DB wiring end-to-end with a real empty repo on disk. + #[sqlx::test] + async fn execute_deletes_only_the_empty_target_repo_on_disk(pool: PgPool) { + let db = db(&pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + + // One empty target repo (real bare repo, zero refs) and one target repo + // with a ref, plus an excluded-owner empty repo. + let empty = rec("t-empty", SPAM_BURST_TARGET_DID, "spam1"); + let nonempty = rec("t-refs", SPAM_BURST_TARGET_DID, "real"); + let excluded = rec("x-content", EXCLUDED_DIDS[0], "keep"); + for r in [&empty, &nonempty, &excluded] { + db.create_repo(r).await.unwrap(); + } + + // Materialize the two target repos on disk. + let empty_path = store::repo_disk_path(tmp.path(), &empty.owner_did, &empty.name); + store::init_bare(&empty_path).unwrap(); + let refs_path = store::repo_disk_path(tmp.path(), &nonempty.owner_did, &nonempty.name); + store::init_bare(&refs_path).unwrap(); + // Give the non-empty repo an actual ref. + seed_one_ref(&refs_path); + + // Sanity: our on-disk ref reader sees the expected counts. + assert_eq!(store::list_refs(&empty_path).unwrap().len(), 0); + assert!(!store::list_refs(&refs_path).unwrap().is_empty()); + + let repo_store = test_store(tmp.path(), &pool); + run_purge_spam(&db, &repo_store, tmp.path(), true) + .await + .unwrap(); + + // Only the empty target repo row is gone. + assert!(db + .get_repo(SPAM_BURST_TARGET_DID, "spam1") + .await + .unwrap() + .is_none()); + assert!(db + .get_repo(SPAM_BURST_TARGET_DID, "real") + .await + .unwrap() + .is_some()); + assert!(db + .get_repo(EXCLUDED_DIDS[0], "keep") + .await + .unwrap() + .is_some()); + } + + // Execute removes the on-disk bare repo dir too, so the DB row and disk stay + // consistent (no orphaned empty git dir left behind). + #[sqlx::test] + async fn execute_removes_the_on_disk_repo_dir(pool: PgPool) { + let db = db(&pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + let empty = rec("t-empty", SPAM_BURST_TARGET_DID, "spam1"); + db.create_repo(&empty).await.unwrap(); + let path = store::repo_disk_path(tmp.path(), &empty.owner_did, &empty.name); + store::init_bare(&path).unwrap(); + assert!(path.exists(), "precondition: on-disk repo exists"); + + let repo_store = test_store(tmp.path(), &pool); + run_purge_spam(&db, &repo_store, tmp.path(), true) + .await + .unwrap(); + + assert!( + db.get_repo(SPAM_BURST_TARGET_DID, "spam1") + .await + .unwrap() + .is_none(), + "DB row deleted" + ); + assert!(!path.exists(), "on-disk bare repo dir must be removed too"); + } + + // U3 (R3/KTD3): a purge at GITLAWB_DB_MAX_CONNECTIONS=1 (app pool size 1) must + // still delete. U2 split the pools: the advisory-lock guard pins a connection + // from the dedicated lock_pool while delete_repo_by_id runs on the app pool, so + // the single app connection is free for the delete's begin() even while the guard + // holds the repo lock. Load-bearing for the admin wiring: if the purge store were + // (mis)wired to share ONE pool for the guard and the delete, the held guard + // connection would leave begin() nothing to acquire, delete would time out + // (PoolTimedOut), and the row would survive — deleted stays 0. The split store + // built here is what keeps the purge correct at one connection. + #[sqlx::test] + async fn purge_at_one_app_connection_deletes_with_split_pools( + _pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + // Migrate + seed on a normal multi-connection pool: migrate() pins one + // connection for its advisory lock while running migration queries on + // others, so it cannot run on a size-1 pool. + let setup_db = Db::for_testing(sized_no_reap_pool(&connect_opts, 5).await); + setup_db.run_migrations().await.unwrap(); + let empty = rec("t-empty", SPAM_BURST_TARGET_DID, "spam1"); + setup_db.create_repo(&empty).await.unwrap(); + + let tmp = tempfile::TempDir::new().unwrap(); + let path = store::repo_disk_path(tmp.path(), &empty.owner_did, &empty.name); + store::init_bare(&path).unwrap(); + assert_eq!( + store::list_refs(&path).unwrap().len(), + 0, + "precondition: the candidate is a real empty bare repo" + ); + + // The purge runs at ONE app connection (finding E's MAX_CONNECTIONS=1) with a + // SEPARATE lock pool. Split store: the guard draws from lock_pool, the delete + // runs on the app pool via `db` — the production wiring from main.rs's + // purge-spam path. + let app_pool = sized_no_reap_pool(&connect_opts, 1).await; + let lock_pool = sized_no_reap_pool(&connect_opts, 1).await; + let db = Db::for_testing(app_pool); + let store = + crate::git::repo_store::RepoStore::new(tmp.path().to_path_buf(), None, lock_pool); + let summary = run_purge_spam(&db, &store, tmp.path(), true).await.unwrap(); + + assert_eq!( + summary.deleted, 1, + "a split-pool purge at one app connection must delete the candidate row" + ); + assert!( + db.get_repo(SPAM_BURST_TARGET_DID, "spam1") + .await + .unwrap() + .is_none(), + "the target row must be gone after a size-1 purge" + ); + } + + // U4 (M4): a repo whose per-repo advisory lock is held by a live writer must be + // SKIPPED by purge, not deleted out from under the push. Holds the lock via the + // same RepoStore (a separate pooled connection), runs execute, and asserts the + // row + on-disk dir survive; once the writer releases, purge deletes it. + #[sqlx::test] + async fn locked_repo_is_skipped_not_deleted(pool: PgPool) { + let db = db(&pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + let empty = rec("t-empty", SPAM_BURST_TARGET_DID, "spam1"); + db.create_repo(&empty).await.unwrap(); + let path = store::repo_disk_path(tmp.path(), &empty.owner_did, &empty.name); + store::init_bare(&path).unwrap(); + + let repo_store = test_store(tmp.path(), &pool); + // A live writer holds the per-repo advisory lock. + let held = repo_store + .try_lock_repo(&empty.owner_did, &empty.name) + .await + .unwrap() + .expect("lock should be free initially"); + + run_purge_spam(&db, &repo_store, tmp.path(), true) + .await + .unwrap(); + + // Locked → skipped: both the row and the on-disk dir survive. + assert!( + db.get_repo(SPAM_BURST_TARGET_DID, "spam1") + .await + .unwrap() + .is_some(), + "a repo locked by a live writer must NOT be deleted" + ); + assert!(path.exists(), "on-disk dir must survive while locked"); + + // Once the writer releases, the empty repo is deleted (the lock was the + // only thing protecting it — baseline both ways). + held.release().await; + run_purge_spam(&db, &repo_store, tmp.path(), true) + .await + .unwrap(); + assert!( + db.get_repo(SPAM_BURST_TARGET_DID, "spam1") + .await + .unwrap() + .is_none(), + "once unlocked, the empty repo is deleted" + ); + } + + // U5 (M6): a repo whose DB row is deleted but whose on-disk removal FAILS must + // be counted in `disk_failed`, never folded into a clean "deleted" success — + // else the summary reports success while the on-disk dir survives (DB/disk + // drift). Forces the failure by making the parent (slug) dir read-only. + #[sqlx::test] + async fn disk_removal_failure_is_counted_not_reported_as_success(pool: PgPool) { + use std::os::unix::fs::PermissionsExt; + let db = db(&pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + let empty = rec("t-empty", SPAM_BURST_TARGET_DID, "spam1"); + db.create_repo(&empty).await.unwrap(); + let path = store::repo_disk_path(tmp.path(), &empty.owner_did, &empty.name); + store::init_bare(&path).unwrap(); + + // Read-only parent (slug) dir: remove_dir_all cannot unlink the repo dir. + let slug_dir = path.parent().unwrap().to_path_buf(); + let orig = std::fs::metadata(&slug_dir).unwrap().permissions(); + std::fs::set_permissions(&slug_dir, std::fs::Permissions::from_mode(0o555)).unwrap(); + + let repo_store = test_store(tmp.path(), &pool); + let summary = run_purge_spam(&db, &repo_store, tmp.path(), true) + .await + .unwrap(); + + // Restore perms so TempDir cleanup works regardless of assertion outcome. + std::fs::set_permissions(&slug_dir, orig).unwrap(); + + assert_eq!(summary.deleted, 1, "the DB row was deleted"); + assert_eq!( + summary.disk_failed, 1, + "a failed on-disk removal must be counted, not reported as clean success" + ); + assert!( + db.get_repo(SPAM_BURST_TARGET_DID, "spam1") + .await + .unwrap() + .is_none(), + "DB row is gone (delete succeeded)" + ); + assert!( + path.exists(), + "on-disk dir survived the failed removal (drift)" + ); + } + + // U1 (M3): a burst-owned row whose NAME traverses out of repos_dir must never + // cause a delete OUTSIDE repos_dir. Adversarial must-not: a real empty bare repo + // planted as a "victim" beside repos_dir is reachable from repos_dir// via + // a `../../victim` name; because the traversed path IS a real bare repo, the + // marker check passes and the candidate is selected — then remove_dir_all would + // delete the victim. The name validator must reject it so the victim survives. + #[sqlx::test] + async fn traversal_name_cannot_delete_a_repo_outside_repos_dir(pool: PgPool) { + let db = db(&pool).await; + let root = tempfile::TempDir::new().unwrap(); + let repos_dir = root.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + + // The burst DID's own slug dir must exist for the OS to resolve the `..` + // segments (a burst that owns any normal repo already has this dir). + let slug = SPAM_BURST_TARGET_DID.replace([':', '/'], "_"); + std::fs::create_dir_all(repos_dir.join(&slug)).unwrap(); + + // Victim: a real empty bare repo OUTSIDE repos_dir (sibling under root). + let victim = root.path().join("victim.git"); + store::init_bare(&victim).unwrap(); + assert!(victim.join("HEAD").is_file(), "victim precondition"); + + // Sanity: the evil name resolves from repos_dir// onto the victim. + let evil = rec("evil", SPAM_BURST_TARGET_DID, "../../victim"); + let traversed = store::repo_disk_path(&repos_dir, &evil.owner_did, &evil.name); + assert_eq!( + std::fs::canonicalize(&traversed).unwrap(), + std::fs::canonicalize(&victim).unwrap(), + "test setup: the evil name must resolve onto the victim" + ); + + db.create_repo(&evil).await.unwrap(); + let repo_store = test_store(&repos_dir, &pool); + run_purge_spam(&db, &repo_store, &repos_dir, true) + .await + .unwrap(); + + assert!( + victim.join("HEAD").is_file(), + "a repo OUTSIDE repos_dir must never be deleted via a traversal name" + ); + } + + // The DB query normalizes did:key form (OWNER_KEY_CASE_SQL), so a burst repo + // stored in SHORT (bare) form is still found when querying by the full-form + // target DID — the SQL side of the normalization fix. + #[sqlx::test] + async fn list_repos_by_owner_did_matches_short_form(pool: PgPool) { + let db = db(&pool).await; + let short = crate::db::normalize_owner_key(SPAM_BURST_TARGET_DID); + assert_ne!(short, SPAM_BURST_TARGET_DID, "fixture must be short form"); + let repo = rec("short-owned", short, "spam"); + db.create_repo(&repo).await.unwrap(); + + let rows = db + .list_repos_by_owner_did(SPAM_BURST_TARGET_DID) + .await + .unwrap(); + assert!( + rows.iter().any(|r| r.id == "short-owned"), + "a short-form burst row must be found when querying by the full-form target" + ); + } + + /// Create a single commit + ref in a bare repo via a throwaway worktree, so the + /// repo reads as non-empty (≥1 ref). + fn seed_one_ref(bare: &Path) { + use std::process::Command; + let wt = bare.join("_seed"); + let run = |args: &[&str], dir: &Path| { + let ok = Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(); + assert!(ok, "git {args:?} failed"); + }; + run( + &["worktree", "add", "--orphan", "-b", "main", "_seed"], + bare, + ); + std::fs::write(wt.join("f.txt"), b"x").unwrap(); + run(&["config", "user.email", "t@t"], &wt); + run(&["config", "user.name", "t"], &wt); + run(&["add", "."], &wt); + run(&["commit", "-qm", "seed"], &wt); + let _ = Command::new("git") + .args(["worktree", "remove", "--force", "_seed"]) + .current_dir(bare) + .status(); + } + + /// A `0` ref count must come only from a real bare repo at the exact path, + /// never from git discovery walking up to a parent `.git`. Load-bearing: the + /// tempdir root is itself a git repo (zero refs), so a naive `for-each-ref` + /// run from a child non-git dir would discover it and report 0 — the delete-a- + /// real-repo fail-open. The marker check must make that path fail closed. + #[test] + fn nongit_path_fails_closed_even_under_a_git_ancestor() { + let tmp = tempfile::TempDir::new().unwrap(); + // Make the tempdir root a git repo with zero refs (the discovery trap). + std::process::Command::new("git") + .args(["init", "-q"]) + .current_dir(tmp.path()) + .status() + .unwrap(); + + let repo = rec("t-x", SPAM_BURST_TARGET_DID, "spam1"); + let path = store::repo_disk_path(tmp.path(), &repo.owner_did, &repo.name); + + // (a0) A genuinely MISSING path is a verified NotFound — the only case that + // returns None (eligible for a remote-unverified archive refresh). + assert!(!path.exists()); + assert_eq!( + local_refs_on_disk(tmp.path(), &repo.owner_did, &repo.name), + None, + "a verifiably-absent path must return None (eligible for archive refresh)" + ); + + // (a) Path exists as a plain (non-git) directory under the git ancestor. + // Without the marker guard, git discovery would read the ancestor's 0 refs + // and this repo would be deleted. It must fail CLOSED to Some(1) — NOT + // None: returning None would promote it to remote_unverified, and the + // refresh's remove_dir_all would overwrite this existing directory. And it + // must never run list_refs to read the ancestor's 0. + std::fs::create_dir_all(&path).unwrap(); + assert_eq!( + local_refs_on_disk(tmp.path(), &repo.owner_did, &repo.name), + Some(1), + "an existing non-git dir must fail closed to Some(1), never None (which would overwrite it)" + ); + assert_eq!( + ref_count_on_disk(tmp.path(), &repo.owner_did, &repo.name), + 1, + "the under-lock recheck must also fail closed on a non-git dir" + ); + + // (b) A real empty bare repo at the same path reads Some(0) — a candidate. + std::fs::remove_dir_all(&path).unwrap(); + store::init_bare(&path).unwrap(); + assert_eq!( + local_refs_on_disk(tmp.path(), &repo.owner_did, &repo.name), + Some(0) + ); + } + + /// A symlink at a burst repo's expected on-disk path is never a legitimate + /// bare repo we may treat as empty-and-purgeable, even when its target is a + /// real EMPTY bare repo living OUTSIDE repos_dir. `symlink_metadata` only + /// distinguishes NotFound; the marker checks (`HEAD` file, `objects` dir) + /// FOLLOW the link, so the outside repo's 0 refs would be admitted — the DB + /// row is deleted (the `path_within` gate spares the outside dir, but only + /// AFTER delete_repo_by_id). A symlink at the target must fail CLOSED. + #[test] + fn symlink_to_outside_empty_bare_fails_closed() { + let tmp = tempfile::TempDir::new().unwrap(); + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + + // A real empty bare repo OUTSIDE repos_dir (HEAD + objects/ => looks bare). + let outside = tmp.path().join("outside.git"); + store::init_bare(&outside).unwrap(); + assert!(outside.join("HEAD").is_file() && outside.join("objects").is_dir()); + + let repo = rec("t-sym", SPAM_BURST_TARGET_DID, "spam1"); + let target = store::repo_disk_path(&repos_dir, &repo.owner_did, &repo.name); + std::fs::create_dir_all(target.parent().unwrap()).unwrap(); + std::os::unix::fs::symlink(&outside, &target).unwrap(); + + // Following the link the empty bare repo reads 0 refs (admitted as empty). + // A symlink at the target must fail closed to Some(1), never Some(0)/None. + assert_eq!( + local_refs_on_disk(&repos_dir, &repo.owner_did, &repo.name), + Some(1), + "a symlink at the target (even to a real empty bare repo outside repos_dir) must fail closed to Some(1)" + ); + } + + /// The DANGLING-symlink case (target does not exist) is why the stat uses + /// `symlink_metadata`, not `metadata`: `metadata` would surface NotFound and + /// wrongly return None (promoting the repo to remote-unverified, whose + /// refresh does a remove_dir_all). `symlink_metadata` succeeds, so the path + /// is present, and the symlink classification fails it closed to Some(1). + #[test] + fn dangling_symlink_fails_closed() { + let tmp = tempfile::TempDir::new().unwrap(); + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + + let repo = rec("t-dangle", SPAM_BURST_TARGET_DID, "spam1"); + let target = store::repo_disk_path(&repos_dir, &repo.owner_did, &repo.name); + std::fs::create_dir_all(target.parent().unwrap()).unwrap(); + std::os::unix::fs::symlink("/nonexistent/target", &target).unwrap(); + + assert_eq!( + local_refs_on_disk(&repos_dir, &repo.owner_did, &repo.name), + Some(1), + "a dangling symlink is present (not NotFound) and must fail closed to Some(1)" + ); + } + + /// The belt-and-suspenders containment gate (`path_within`) must reject a path + /// that resolves outside repos_dir even when the *name* itself is innocuous — + /// e.g. the owner slug dir is a symlink pointing elsewhere. Layer 1 (name + /// validation) can't see this; only the canonical-containment check catches it. + #[test] + fn path_within_rejects_symlink_escape() { + let root = tempfile::TempDir::new().unwrap(); + let repos_dir = root.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + + // A real dir outside repos_dir, and a symlink INTO repos_dir that targets it. + let outside = root.path().join("outside.git"); + std::fs::create_dir_all(&outside).unwrap(); + let link = repos_dir.join("evil.git"); + std::os::unix::fs::symlink(&outside, &link).unwrap(); + + // The name is innocuous, but the path resolves outside repos_dir. + assert!( + !path_within(&link, &repos_dir), + "a symlink escaping repos_dir must fail the containment gate" + ); + // A genuine path inside repos_dir passes. + let inside = repos_dir.join("real.git"); + std::fs::create_dir_all(&inside).unwrap(); + assert!( + path_within(&inside, &repos_dir), + "an in-root path must pass" + ); + } + + /// The exclusion gate and the target scope must never overlap: if the burst + /// target were ever set to an excluded DID, the gate would fail to protect it. + #[test] + fn target_did_is_never_excluded() { + assert!( + !EXCLUDED_DIDS.contains(&SPAM_BURST_TARGET_DID), + "the purge target must never be an excluded (protected) DID" + ); + } + + /// Exclusion is normalization-consistent: an excluded identity stored in the + /// bare short form (as mirror upserts write it) is still excluded, even though + /// the exclusion constants are full did:key form — and an empty repo it owns + /// is never selected. + #[test] + fn short_form_excluded_did_is_still_protected() { + let short = crate::db::normalize_owner_key(EXCLUDED_DIDS[1]); // bare z6MkqRz… + assert_ne!( + short, EXCLUDED_DIDS[1], + "fixture must actually be short form" + ); + assert!( + is_excluded(short), + "short-form of an excluded DID must be excluded" + ); + + // An empty repo owned by the short-form excluded DID is spared even though + // its ref signature (0) otherwise matches the burst. + let empty_excluded_short = rec("x-short", short, "spam"); + let cands = select_spam_candidates( + &[empty_excluded_short], + SPAM_BURST_TARGET_DID, + false, + |_| Some(0), + ); + assert!( + cands.is_empty(), + "an empty repo owned by a short-form excluded DID must never be a candidate" + ); + } + + // ── Tigris-authoritative purge (P1b) ─────────────────────────────────── + + /// In-test object store. `download` materializes a bare repo with (or + /// without) a ref so the purge tool's authoritative recheck can be driven + /// without a live bucket; error flags exercise the fail-closed and + /// archive-delete-failure paths. + struct FakeStore { + has_archive: bool, + archive_has_refs: bool, + fail_recheck: bool, + fail_delete: bool, + deleted: std::sync::Arc, + } + + #[async_trait::async_trait] + impl crate::git::tigris::ObjectStore for FakeStore { + async fn exists(&self, _owner: &str, _repo: &str) -> Result { + if self.fail_recheck { + anyhow::bail!("fake object store unreachable"); + } + Ok(self.has_archive) + } + async fn upload(&self, _owner: &str, _repo: &str, _path: &Path) -> Result<()> { + Ok(()) + } + async fn download(&self, _owner: &str, _repo: &str, local_path: &Path) -> Result<()> { + if self.fail_recheck { + anyhow::bail!("fake object store unreachable"); + } + // Materialize the authoritative archive on local disk so + // ref_count_on_disk reflects remote state. + let _ = std::fs::remove_dir_all(local_path); + store::init_bare(local_path).unwrap(); + if self.archive_has_refs { + seed_one_ref(local_path); + } + Ok(()) + } + async fn delete(&self, _owner: &str, _repo: &str) -> Result<()> { + self.deleted + .store(true, std::sync::atomic::Ordering::SeqCst); + if self.fail_delete { + anyhow::bail!("fake object store delete failed"); + } + Ok(()) + } + } + + fn store_backed( + repos_dir: &Path, + pool: &PgPool, + fake: FakeStore, + ) -> crate::git::repo_store::RepoStore { + crate::git::repo_store::RepoStore::new( + repos_dir.to_path_buf(), + Some(std::sync::Arc::new(fake)), + pool.clone(), + ) + } + + fn fake( + has_archive: bool, + archive_has_refs: bool, + fail_recheck: bool, + fail_delete: bool, + ) -> (FakeStore, std::sync::Arc) { + let deleted = std::sync::Arc::new(std::sync::atomic::AtomicBool::new(false)); + ( + FakeStore { + has_archive, + archive_has_refs, + fail_recheck, + fail_delete, + deleted: deleted.clone(), + }, + deleted, + ) + } + + // A locally-empty repo whose authoritative archive HAS refs (pushed via + // another machine) must NOT be purged on the stale-local view. Load-bearing: + // RED on the Tigris-blind purge (deletes it), GREEN once the recheck + // consults the object store. + #[sqlx::test] + async fn purge_skips_repo_with_remote_refs_when_local_empty(pool: PgPool) { + let db = db(&pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + let target = rec("t-remote", SPAM_BURST_TARGET_DID, "spam1"); + db.create_repo(&target).await.unwrap(); + let path = store::repo_disk_path(tmp.path(), &target.owner_did, &target.name); + store::init_bare(&path).unwrap(); + assert_eq!( + store::list_refs(&path).unwrap().len(), + 0, + "local starts empty" + ); + + let (f, _deleted) = fake(true, true, false, false); + let repo_store = store_backed(tmp.path(), &pool, f); + let summary = run_purge_spam(&db, &repo_store, tmp.path(), true) + .await + .unwrap(); + + assert!( + db.get_repo(SPAM_BURST_TARGET_DID, "spam1") + .await + .unwrap() + .is_some(), + "a repo with live remote refs must not be purged on a stale-local view" + ); + assert_eq!(summary.deleted, 0); + assert_eq!(summary.skipped_not_empty, 1); + } + + // A genuinely-empty repo (empty archive) is deleted AND its archive removed, + // else the archive can be re-downloaded into a later same-name repo. + // Load-bearing: RED on the Tigris-blind purge (archive survives), GREEN once + // the delete wires the archive removal. + #[sqlx::test] + async fn purge_deletes_archive_on_successful_delete(pool: PgPool) { + let db = db(&pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + let target = rec("t-empty", SPAM_BURST_TARGET_DID, "spam1"); + db.create_repo(&target).await.unwrap(); + let path = store::repo_disk_path(tmp.path(), &target.owner_did, &target.name); + store::init_bare(&path).unwrap(); + + let (f, deleted) = fake(true, false, false, false); + let repo_store = store_backed(tmp.path(), &pool, f); + let summary = run_purge_spam(&db, &repo_store, tmp.path(), true) + .await + .unwrap(); + + assert!( + db.get_repo(SPAM_BURST_TARGET_DID, "spam1") + .await + .unwrap() + .is_none(), + "a genuinely-empty repo is deleted" + ); + assert!( + deleted.load(std::sync::atomic::Ordering::SeqCst), + "the object-store archive must be deleted on a successful purge" + ); + assert_eq!(summary.deleted, 1); + assert_eq!(summary.archive_failed, 0); + } + + // An unreachable object store during the recheck must fail closed (skip), + // never delete on an unverified view. Load-bearing: RED on the Tigris-blind + // purge (deletes), GREEN once the recheck consults (and fails closed on) the + // store. + #[sqlx::test] + async fn purge_fails_closed_when_store_unreachable(pool: PgPool) { + let db = db(&pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + let target = rec("t-empty", SPAM_BURST_TARGET_DID, "spam1"); + db.create_repo(&target).await.unwrap(); + let path = store::repo_disk_path(tmp.path(), &target.owner_did, &target.name); + store::init_bare(&path).unwrap(); + + let (f, _deleted) = fake(true, false, true, false); + let repo_store = store_backed(tmp.path(), &pool, f); + let summary = run_purge_spam(&db, &repo_store, tmp.path(), true) + .await + .unwrap(); + + assert!( + db.get_repo(SPAM_BURST_TARGET_DID, "spam1") + .await + .unwrap() + .is_some(), + "must not delete when the object store is unreachable (fail-closed)" + ); + assert_eq!(summary.deleted, 0); + assert_eq!(summary.skipped_store_error, 1); + } + + // An archive-delete failure after the row+dir are removed is counted + // separately, never folded into a clean success. + #[sqlx::test] + async fn purge_archive_delete_failure_counted_separately(pool: PgPool) { + let db = db(&pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + let target = rec("t-empty", SPAM_BURST_TARGET_DID, "spam1"); + db.create_repo(&target).await.unwrap(); + let path = store::repo_disk_path(tmp.path(), &target.owner_did, &target.name); + store::init_bare(&path).unwrap(); + + let (f, deleted) = fake(true, false, false, true); + let repo_store = store_backed(tmp.path(), &pool, f); + let summary = run_purge_spam(&db, &repo_store, tmp.path(), true) + .await + .unwrap(); + + assert!( + db.get_repo(SPAM_BURST_TARGET_DID, "spam1") + .await + .unwrap() + .is_none(), + "the row+dir are still deleted" + ); + assert!(deleted.load(std::sync::atomic::Ordering::SeqCst)); + assert_eq!(summary.deleted, 1); + assert_eq!( + summary.archive_failed, 1, + "a surviving archive must be counted, not reported as clean success" + ); + } + + // AE3/R4: a repo that exists ONLY as an object-store archive (no local copy) + // with an EMPTY archive is reached, refreshed under the lock, and deleted + // (row + dir + archive). Pre-U4 a missing-local row was never a candidate + // (treated non-empty, skipped) -> RED; admitting it remote-unverified -> GREEN. + #[sqlx::test] + async fn purge_deletes_remote_only_empty_archive(pool: PgPool) { + let db = db(&pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + let target = rec("t-remote-empty", SPAM_BURST_TARGET_DID, "spam1"); + db.create_repo(&target).await.unwrap(); + // NO local repo — it exists only as an archive. + let path = store::repo_disk_path(tmp.path(), &target.owner_did, &target.name); + assert!(!path.exists(), "no local copy — remote-only"); + + let (f, deleted) = fake(true, false, false, false); // archive exists, empty + let repo_store = store_backed(tmp.path(), &pool, f); + let summary = run_purge_spam(&db, &repo_store, tmp.path(), true) + .await + .unwrap(); + + assert!( + db.get_repo(SPAM_BURST_TARGET_DID, "spam1") + .await + .unwrap() + .is_none(), + "a remote-only empty archive must be reached and deleted" + ); + assert!( + deleted.load(std::sync::atomic::Ordering::SeqCst), + "the archive must be deleted too" + ); + assert_eq!(summary.deleted, 1); + assert_eq!( + summary.remote_unverified, 1, + "the candidate was admitted as remote-unverified" + ); + } + + // AE4/R4: a remote-only archive that turns out to HAVE refs is refreshed under + // the lock and then skipped — never deleted on the missing-local view. + #[sqlx::test] + async fn purge_skips_remote_only_archive_with_refs(pool: PgPool) { + let db = db(&pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + let target = rec("t-remote-refs", SPAM_BURST_TARGET_DID, "spam1"); + db.create_repo(&target).await.unwrap(); + let path = store::repo_disk_path(tmp.path(), &target.owner_did, &target.name); + assert!(!path.exists()); + + let (f, _deleted) = fake(true, true, false, false); // archive exists, HAS refs + let repo_store = store_backed(tmp.path(), &pool, f); + let summary = run_purge_spam(&db, &repo_store, tmp.path(), true) + .await + .unwrap(); + + assert!( + db.get_repo(SPAM_BURST_TARGET_DID, "spam1") + .await + .unwrap() + .is_some(), + "a remote-only archive with refs must be refreshed and skipped, not deleted" + ); + assert_eq!(summary.deleted, 0); + assert_eq!(summary.skipped_not_empty, 1); + } + + // U7/R7: an existing NON-repository directory (holding operator data) sits at + // the exact target path. It is not a bare git repo, so it must fail CLOSED — + // never be admitted remote-unverified, because that promotion drives + // refresh_from_archive, whose remove_dir_all + extract would OVERWRITE the + // directory. Load-bearing: on base, local_refs_on_disk returns None for a + // non-bare path, the candidate is promoted, refresh wipes the dir and the row + // is deleted (RED). After the fix, Some(1) keeps it out entirely (GREEN). + #[sqlx::test] + async fn purge_does_not_overwrite_non_repo_dir_at_target(pool: PgPool) { + let db = db(&pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + let target = rec("t-nonrepo", SPAM_BURST_TARGET_DID, "spam1"); + db.create_repo(&target).await.unwrap(); + + // A plain directory with a file — deliberately NOT a bare git repo. + let path = store::repo_disk_path(tmp.path(), &target.owner_did, &target.name); + std::fs::create_dir_all(&path).unwrap(); + let sentinel = path.join("keep.txt"); + std::fs::write(&sentinel, b"operator data").unwrap(); + + let (f, deleted) = fake(true, false, false, false); // archive exists, empty + let repo_store = store_backed(tmp.path(), &pool, f); + let summary = run_purge_spam(&db, &repo_store, tmp.path(), true) + .await + .unwrap(); + + assert!( + sentinel.exists(), + "an existing non-repo dir must not be overwritten by the purge refresh" + ); + assert_eq!( + std::fs::read(&sentinel).unwrap(), + b"operator data", + "the non-repo dir's contents must be intact" + ); + assert!( + db.get_repo(SPAM_BURST_TARGET_DID, "spam1") + .await + .unwrap() + .is_some(), + "a repo whose disk path is a non-bare dir must not be purged" + ); + assert_eq!(summary.deleted, 0); + assert_eq!( + summary.remote_unverified, 0, + "a non-bare existing dir must never be admitted remote-unverified" + ); + assert!( + !deleted.load(std::sync::atomic::Ordering::SeqCst), + "the archive must not be deleted" + ); + } + + // AE5/R5: no local copy AND no archive — the candidate is admitted (a store is + // configured) but the under-lock recheck finds nothing and fails closed. + #[sqlx::test] + async fn purge_skips_remote_unverified_with_no_archive(pool: PgPool) { + let db = db(&pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + let target = rec("t-missing-both", SPAM_BURST_TARGET_DID, "spam1"); + db.create_repo(&target).await.unwrap(); + + let (f, _deleted) = fake(false, false, false, false); // NO archive + let repo_store = store_backed(tmp.path(), &pool, f); + let summary = run_purge_spam(&db, &repo_store, tmp.path(), true) + .await + .unwrap(); + + assert!( + db.get_repo(SPAM_BURST_TARGET_DID, "spam1") + .await + .unwrap() + .is_some(), + "missing local AND no archive must fail closed (not deleted)" + ); + assert_eq!(summary.deleted, 0); + assert_eq!( + summary.skipped_not_empty, 1, + "skipped by the authoritative under-lock recheck" + ); + } + + // R5: with NO object store, a repo with no local copy is not even a candidate + // (the missing-local admission is gated on a configured store). + #[sqlx::test] + async fn purge_storeless_missing_local_is_not_a_candidate(pool: PgPool) { + let db = db(&pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + let target = rec("t-nolocal", SPAM_BURST_TARGET_DID, "spam1"); + db.create_repo(&target).await.unwrap(); + // Storeless RepoStore, no local repo on disk. + let repo_store = test_store(tmp.path(), &pool); + let summary = run_purge_spam(&db, &repo_store, tmp.path(), true) + .await + .unwrap(); + + assert!( + db.get_repo(SPAM_BURST_TARGET_DID, "spam1") + .await + .unwrap() + .is_some(), + "storeless + missing-local must fail closed — never a candidate" + ); + assert_eq!(summary.deleted, 0); + assert_eq!(summary.remote_unverified, 0); + } + + // R7: with no object store configured (single-machine), an empty repo is + // deleted exactly as before and nothing touches an archive. + #[sqlx::test] + async fn purge_tigris_disabled_deletes_empty_unchanged(pool: PgPool) { + let db = db(&pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + let target = rec("t-empty", SPAM_BURST_TARGET_DID, "spam1"); + db.create_repo(&target).await.unwrap(); + let path = store::repo_disk_path(tmp.path(), &target.owner_did, &target.name); + store::init_bare(&path).unwrap(); + + let repo_store = test_store(tmp.path(), &pool); // Tigris = None + let summary = run_purge_spam(&db, &repo_store, tmp.path(), true) + .await + .unwrap(); + + assert!(db + .get_repo(SPAM_BURST_TARGET_DID, "spam1") + .await + .unwrap() + .is_none()); + assert_eq!(summary.deleted, 1); + assert_eq!(summary.skipped_store_error, 0); + assert_eq!(summary.archive_failed, 0); + } +} diff --git a/crates/gitlawb-node/src/api/issues.rs b/crates/gitlawb-node/src/api/issues.rs index 17acf9af..750aaa4b 100644 --- a/crates/gitlawb-node/src/api/issues.rs +++ b/crates/gitlawb-node/src/api/issues.rs @@ -70,11 +70,37 @@ pub async fn create_issue( let create_result = git_issues::create_issue(&disk_path, &issue_id, &json_str); - // Always release the advisory lock — even on error; upload to Tigris only on success. - guard.release(create_result.is_ok()).await; + // Always release the advisory lock, even on error; upload to Tigris only on + // success. A false return means the durable upload failed/timed out: the + // issue ref applied to local disk but never reached object storage, so the + // next acquire_write would revert it from the stale archive. Report 5xx so + // the client retries rather than trusting an issue that never landed durably, + // and skip the trust bump below. (Same P1 data-loss guard as receive-pack.) + // On that failure, roll back the local ref BEFORE the lock releases — an + // orphan ref would make the retry a duplicate, and in an unlock-to-delete + // window a concurrent same-repo write could upload an archive still + // carrying it. Best-effort: a failed delete logs and the 5xx still returns. + let upload_ok = guard + .release_with_failure_cleanup(create_result.is_ok(), |path| { + if let Err(e) = git_issues::delete_issue_ref(path, &issue_id) { + tracing::warn!(issue = %issue_id, err = %e, + "failed to roll back local issue ref after failed durable upload; \ + a retry may duplicate this issue"); + } + }) + .await; create_result.map_err(|e| AppError::Git(e.to_string()))?; + if !upload_ok { + tracing::error!(repo = %record.name, + "durable upload failed after issue create; reporting failure so the client retries"); + return Err(AppError::Git(format!( + "durable storage upload failed for {}", + record.name + ))); + } + // Bump trust score for the issue author — increment current score by 0.05 // (avoids the push_count=0 stuck-at-0.05 bug for agents who only file issues) if let Some(ref author_did) = issue.author { @@ -244,11 +270,14 @@ pub async fn close_issue( .ok() .and_then(|i| i.author), Ok(None) => { - guard.release(false).await; + // release(false): no upload attempted, so the bool is always true + // (nothing to fail on). Ignore it on this pre-write error path. + let _ = guard.release(false).await; return Err(AppError::NotFound(format!("issue {issue_id} not found"))); } Err(e) => { - guard.release(false).await; + // release(false): no upload attempted, nothing to fail on. + let _ = guard.release(false).await; return Err(AppError::Git(e.to_string())); } }; @@ -257,21 +286,57 @@ pub async fn close_issue( .as_deref() .is_some_and(|a| crate::api::did_matches(&auth.0, a)); if !is_owner && !is_author { - guard.release(false).await; + // release(false): no upload attempted, nothing to fail on. + let _ = guard.release(false).await; return Err(AppError::Forbidden( "only the repo owner or the issue author can close this issue".into(), )); } + // Snapshot the issue ref's pre-close OID so a failed durable upload can roll + // the close back on local disk, mirroring create_issue. Best-effort: if the + // snapshot read fails we proceed without a rollback closure (the close still + // applies and a failed upload still surfaces as a 5xx below). + let prior_ref = git_issues::issue_ref_oid(&disk_path, &issue_id) + .ok() + .flatten(); + let close_result = git_issues::close_issue(&disk_path, &issue_id); - // Always release the advisory lock — even on error; upload to Tigris only on success. - guard.release(close_result.is_ok()).await; + // Always release the advisory lock, even on error; upload to Tigris only on + // success. A false return means the durable upload failed: the close applied + // to local disk but never reached object storage, so a later acquire_write + // reverts it. Report 5xx so the client retries. (P1 data-loss guard.) + // On that failure, roll the local close BACK to the captured open ref BEFORE + // the lock releases — otherwise the local fast path in RepoStore::acquire + // serves the "closed" state until a later write refreshes the archive, even + // though the client got a 5xx. The cleanup runs only when an upload was + // attempted and failed, while the advisory lock is still held. + let upload_ok = guard + .release_with_failure_cleanup(close_result.is_ok(), |path| { + if let Some((full_id, oid)) = &prior_ref { + if let Err(e) = git_issues::restore_issue_ref(path, full_id, oid) { + tracing::warn!(issue = %issue_id, err = %e, + "failed to roll back local issue close after failed durable upload; \ + a local-fast-path read may serve a stale closed state"); + } + } + }) + .await; let updated = close_result .map_err(|e| AppError::Git(e.to_string()))? .ok_or_else(|| AppError::RepoNotFound(format!("issue {issue_id} not found")))?; + if !upload_ok { + tracing::error!(repo = %repo, + "durable upload failed after issue close; reporting failure so the client retries"); + return Err(AppError::Git(format!( + "durable storage upload failed for {}", + record.name + ))); + } + let issue: serde_json::Value = serde_json::from_str(&updated) .map_err(|e| AppError::BadRequest(format!("invalid issue data: {e}")))?; @@ -279,3 +344,421 @@ pub async fn close_issue( Ok(Json(issue)) } + +#[cfg(test)] +mod tests { + use super::*; + use axum::extract::{Path as AxPath, State}; + use axum::Extension; + use std::path::Path as StdPath; + use std::time::Duration; + + // Object store whose upload() parks forever, so release()'s bounded timeout + // is the only thing that completes it, modeling a stalled durable PUT. exists() + // is false so acquire_write never downloads over the fresh local bare repo. + struct StallStore; + #[async_trait::async_trait] + impl crate::git::tigris::ObjectStore for StallStore { + async fn exists(&self, _o: &str, _r: &str) -> anyhow::Result { + Ok(false) + } + async fn upload(&self, _o: &str, _r: &str, _p: &StdPath) -> anyhow::Result<()> { + std::future::pending::<()>().await; + Ok(()) + } + async fn download(&self, _o: &str, _r: &str, _p: &StdPath) -> anyhow::Result<()> { + Ok(()) + } + async fn delete(&self, _o: &str, _r: &str) -> anyhow::Result<()> { + Ok(()) + } + } + + // Object store whose upload succeeds immediately. exists() is false so + // acquire_write never performs a reverting refresh over the local dir — + // the retry test below pins that contract explicitly. + struct OkStore; + #[async_trait::async_trait] + impl crate::git::tigris::ObjectStore for OkStore { + async fn exists(&self, _o: &str, _r: &str) -> anyhow::Result { + Ok(false) + } + async fn upload(&self, _o: &str, _r: &str, _p: &StdPath) -> anyhow::Result<()> { + Ok(()) + } + async fn download(&self, _o: &str, _r: &str, _p: &StdPath) -> anyhow::Result<()> { + Ok(()) + } + async fn delete(&self, _o: &str, _r: &str) -> anyhow::Result<()> { + Ok(()) + } + } + + // Shared setup for the failed-upload rollback tests: a state whose repo + // store stalls uploads (200ms release timeout), a public repo row for + // `owner/name`, and an initialized bare repo at the returned path. + async fn stall_state_with_repo( + pool: sqlx::PgPool, + repos_dir: &tempfile::TempDir, + owner: &str, + name: &str, + ) -> (crate::state::AppState, std::path::PathBuf) { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.repo_store = crate::git::repo_store::RepoStore::new( + repos_dir.path().to_path_buf(), + Some(std::sync::Arc::new(StallStore)), + pool.clone(), + ) + .with_release_upload_timeout(Duration::from_millis(200)); + + let owner_slug = owner.replace([':', '/'], "_"); + let bare = repos_dir + .path() + .join(&owner_slug) + .join(format!("{name}.git")); + + state + .db + .upsert_mirror_repo(owner, name, &bare.to_string_lossy(), None, false) + .await + .unwrap(); + + std::fs::create_dir_all(&bare).unwrap(); + let out = std::process::Command::new("git") + .args(["init", "--bare", "-q", &bare.to_string_lossy()]) + .output() + .unwrap(); + assert!(out.status.success(), "git init --bare failed"); + + (state, bare) + } + + async fn create_issue_status( + state: &crate::state::AppState, + owner: &str, + name: &str, + ) -> axum::http::StatusCode { + use axum::response::IntoResponse; + let resp = super::create_issue( + State(state.clone()), + Extension(crate::auth::AuthenticatedDid(owner.to_string())), + AxPath((owner.to_string(), name.to_string())), + axum::Json(CreateIssueRequest { + title: "t".into(), + body: None, + signed_payload: None, + }), + ) + .await; + match resp { + Ok((code, _)) => code, + Err(e) => e.into_response().status(), + } + } + + async fn close_issue_status( + state: &crate::state::AppState, + owner: &str, + name: &str, + issue_id: &str, + ) -> axum::http::StatusCode { + use axum::response::IntoResponse; + let resp = super::close_issue( + State(state.clone()), + Extension(crate::auth::AuthenticatedDid(owner.to_string())), + AxPath((owner.to_string(), name.to_string(), issue_id.to_string())), + ) + .await; + match resp { + Ok(_) => axum::http::StatusCode::OK, + Err(e) => e.into_response().status(), + } + } + + // Seed an OPEN issue directly on local disk, bypassing the (stalling) upload + // path so the close test starts from a durably-consistent open issue. + fn seed_open_issue(bare: &StdPath, owner: &str, issue_id: &str) { + let issue = IssueRecord { + id: issue_id.to_string(), + title: "t".into(), + body: None, + author: Some(owner.to_string()), + created_at: Utc::now().to_rfc3339(), + status: "open".to_string(), + signed_payload: None, + }; + let json = serde_json::to_string(&issue).unwrap(); + git_issues::create_issue(bare, issue_id, &json).unwrap(); + } + + fn issue_status(bare: &StdPath, issue_id: &str) -> String { + let raw = git_issues::get_issue(bare, issue_id).unwrap().unwrap(); + let v: serde_json::Value = serde_json::from_str(&raw).unwrap(); + v["status"].as_str().unwrap().to_string() + } + + // U2 [F2, P1] — the failed-upload rollback on the close path. close_issue + // applies the "closed" ref to LOCAL disk, then release() uploads to durable + // storage. When that upload times out the local close is NOT persisted, so a + // later acquire_write reverts it from the stale archive — but until then, a + // local-fast-path read via RepoStore::acquire serves the stale "closed" + // state even though the client got a 5xx. The fix rolls the close back to the + // captured open ref while the advisory lock is held. RED with the cleanup + // closure reverted to `|_| {}`: the read sees "closed". + #[sqlx::test] + async fn issue_close_failed_upload_rolls_back_to_open(pool: sqlx::PgPool) { + let repos_dir = tempfile::TempDir::new().unwrap(); + let owner = "z6issuecloserollbackowner"; + let name = "issuecloserollback"; + let (state, bare) = stall_state_with_repo(pool, &repos_dir, owner, name).await; + + let issue_id = "aaaa1111-0000-0000-0000-000000000000"; + seed_open_issue(&bare, owner, issue_id); + assert_eq!(issue_status(&bare, issue_id), "open", "seed must be open"); + + // Close via the handler — the durable upload stalls and times out (5xx). + let status = close_issue_status(&state, owner, name, issue_id).await; + assert!( + status.is_server_error(), + "a stalled durable upload on issue-close must 500, got {status}" + ); + + // The local fast path must still see the issue OPEN: the failed upload + // rolled the close back while the lock was held. + assert_eq!( + issue_status(&bare, issue_id), + "open", + "a failed durable upload must roll the local close back to open; a \ + local-fast-path read served the stale closed state" + ); + } + + // Success path: a working store closes the issue (2xx) and the read reflects + // the close. Confirms the rollback wiring does not fire on success. + #[sqlx::test] + async fn issue_close_success_persists_close(pool: sqlx::PgPool) { + let repos_dir = tempfile::TempDir::new().unwrap(); + let owner = "z6issueclosesuccessowner"; + let name = "issueclosesuccess"; + let (mut state, bare) = stall_state_with_repo(pool.clone(), &repos_dir, owner, name).await; + // Swap the stalling store for one whose upload succeeds immediately. + state.repo_store = crate::git::repo_store::RepoStore::new( + repos_dir.path().to_path_buf(), + Some(std::sync::Arc::new(OkStore)), + pool.clone(), + ) + .with_release_upload_timeout(Duration::from_millis(200)); + + let issue_id = "bbbb2222-0000-0000-0000-000000000000"; + seed_open_issue(&bare, owner, issue_id); + + let status = close_issue_status(&state, owner, name, issue_id).await; + assert_eq!(status, StatusCode::OK, "working store must close (200)"); + assert_eq!( + issue_status(&bare, issue_id), + "closed", + "a successful close must persist as closed" + ); + } + + // Pre-write error path: closing a nonexistent issue returns 404 and never + // attempts an upload (release(false)), so the stalling store cannot hang it. + #[sqlx::test] + async fn issue_close_not_found_releases_without_upload(pool: sqlx::PgPool) { + let repos_dir = tempfile::TempDir::new().unwrap(); + let owner = "z6issueclosemissingowner"; + let name = "issueclosemissing"; + let (state, _bare) = stall_state_with_repo(pool, &repos_dir, owner, name).await; + + let status = + close_issue_status(&state, owner, name, "cccc3333-0000-0000-0000-000000000000").await; + assert_eq!( + status, + StatusCode::NOT_FOUND, + "closing a missing issue must 404 without attempting an upload" + ); + } + + fn issue_refs(bare: &StdPath) -> Vec { + let out = std::process::Command::new("git") + .args([ + "for-each-ref", + "--format=%(refname)", + "refs/gitlawb/issues/", + ]) + .current_dir(bare) + .output() + .unwrap(); + assert!(out.status.success(), "git for-each-ref failed"); + String::from_utf8_lossy(&out.stdout) + .lines() + .filter(|l| !l.trim().is_empty()) + .map(|l| l.trim().to_string()) + .collect() + } + + // Rollback half of the #196 round-four issue-create fix. When the durable + // upload fails/times out, the handler 500s (covered below) — but the issue + // ref it wrote to LOCAL disk must also be rolled back, while the advisory + // lock is still held. An orphan ref makes the 500 dishonest: a retry + // re-creates the issue under a new uuid while the orphan is still listed, + // and a concurrent same-repo write can upload an archive carrying it. + // RED pre-fix: the orphan ref survives the failed create. + #[sqlx::test] + async fn issue_create_failed_upload_rolls_back_local_ref(pool: sqlx::PgPool) { + let repos_dir = tempfile::TempDir::new().unwrap(); + let owner = "z6issuerollbackowner"; + let name = "issuerollback"; + let (state, bare) = stall_state_with_repo(pool, &repos_dir, owner, name).await; + + let status = create_issue_status(&state, owner, name).await; + assert!( + status.is_server_error(), + "stalled upload must 500, got {status}" + ); + + let refs = issue_refs(&bare); + assert!( + refs.is_empty(), + "a failed durable upload must roll back the local issue ref while \ + the advisory lock is held; orphan ref(s) survived: {refs:?}" + ); + } + + // Retry half: after the failed-upload 500, a retry with a WORKING store + // must yield exactly one issue. Pre-fix the orphan ref from the failed + // attempt makes the retry a duplicate (two issues in list_issues). + #[sqlx::test] + async fn issue_create_failed_upload_retry_does_not_duplicate(pool: sqlx::PgPool) { + use crate::git::tigris::ObjectStore as _; + + let repos_dir = tempfile::TempDir::new().unwrap(); + let owner = "z6issueretryowner"; + let name = "issueretry"; + let (state, bare) = stall_state_with_repo(pool.clone(), &repos_dir, owner, name).await; + + let status = create_issue_status(&state, owner, name).await; + assert!( + status.is_server_error(), + "stalled upload must 500, got {status}" + ); + + // PIN the double's contract: exists() stays false after the failed + // upload, so the retry's acquire_write performs no reverting refresh + // from a stale archive. Without this pin, a download could revert the + // orphan ref and the assertion below would pass without the rollback + // fix — it must rest on the rollback, not a stale-archive revert. + let owner_slug = owner.replace([':', '/'], "_"); + assert!( + !OkStore.exists(&owner_slug, name).await.unwrap(), + "test double contract broken: exists() must stay false" + ); + + // Retry with a store whose upload succeeds. + let mut retry_state = state.clone(); + retry_state.repo_store = crate::git::repo_store::RepoStore::new( + repos_dir.path().to_path_buf(), + Some(std::sync::Arc::new(OkStore)), + pool.clone(), + ) + .with_release_upload_timeout(Duration::from_millis(200)); + + let status = create_issue_status(&retry_state, owner, name).await; + assert_eq!( + status, + StatusCode::CREATED, + "retry with working store must 201" + ); + + let issues = crate::git::issues::list_issues(&bare).unwrap(); + assert_eq!( + issues.len(), + 1, + "retry after a failed-upload 500 must yield exactly ONE issue; the \ + failed attempt's orphan ref duplicated it (got {} issues)", + issues.len() + ); + } + + // P1-class data-loss regression, issue-create path. create_issue writes the + // issue ref to local disk, then release(true) uploads the repo to durable + // storage. When that upload times out the local write is NOT persisted, so + // the next acquire_write reverts it from the stale archive. The pre-fix + // handler ignored release()'s bool and still returned 201 AND bumped the + // author's trust score, so the client trusted an issue that never landed + // durably. The fix surfaces a failed/timed-out upload as a 5xx and skips the + // success tail (the trust bump). Stall the upload past a tiny release timeout + // and assert (a) the handler returns 5xx and (b) the trust bump did not run. + // RED pre-fix: 201 + trust score bumped to 0.05. + #[sqlx::test] + async fn issue_create_durable_upload_timeout_fails_and_skips_side_effects(pool: sqlx::PgPool) { + use axum::response::IntoResponse; + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tempfile::TempDir::new().unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::new( + repos_dir.path().to_path_buf(), + Some(std::sync::Arc::new(StallStore)), + pool.clone(), + ) + .with_release_upload_timeout(Duration::from_millis(200)); + + let owner = "z6issueownerdurablefail"; + let name = "issuedurablefail"; + let owner_slug = owner.replace([':', '/'], "_"); + let bare = repos_dir + .path() + .join(&owner_slug) + .join(format!("{name}.git")); + + // Public mirror repo so the owner clears authorize_repo_read, and a real + // agent row so the post-write trust bump is an observable durable side + // effect (register_agent seeds trust_score = 0.0). + state + .db + .upsert_mirror_repo(owner, name, &bare.to_string_lossy(), None, false) + .await + .unwrap(); + state.db.register_agent(owner, &[]).await.unwrap(); + + std::fs::create_dir_all(&bare).unwrap(); + let out = std::process::Command::new("git") + .args(["init", "--bare", "-q", &bare.to_string_lossy()]) + .output() + .unwrap(); + assert!(out.status.success(), "git init --bare failed"); + + // acquire_write → git_issues::create_issue (writes the ref) → release + // (upload stalls, times out after 200ms). The client MUST see a failure. + let resp = super::create_issue( + State(state.clone()), + Extension(crate::auth::AuthenticatedDid(owner.to_string())), + AxPath((owner.to_string(), name.to_string())), + axum::Json(CreateIssueRequest { + title: "t".into(), + body: None, + signed_payload: None, + }), + ) + .await; + let status = match resp { + Ok((code, _)) => code, + Err(e) => e.into_response().status(), + }; + assert!( + status.is_server_error(), + "a timed-out durable upload on issue-create must fail (5xx) so the \ + client retries (got {status}), which the client trusts as a landed \ + issue that a later acquire_write would silently revert" + ); + + // The success tail (trust bump) must NOT have run: the issue was never + // durably accepted. register_agent seeded 0.0; a bump would make it 0.05. + let score = state.db.get_trust_score(owner).await.unwrap(); + assert_eq!( + score, 0.0, + "the trust bump must NOT run when the durable upload failed; the \ + issue was not durably accepted (score bumped to {score})" + ); + } +} diff --git a/crates/gitlawb-node/src/api/pulls.rs b/crates/gitlawb-node/src/api/pulls.rs index 26be6109..3be8b1d3 100644 --- a/crates/gitlawb-node/src/api/pulls.rs +++ b/crates/gitlawb-node/src/api/pulls.rs @@ -216,6 +216,14 @@ pub async fn merge_pr( .map_err(|e| AppError::Git(e.to_string()))?; let disk_path = guard.path().to_path_buf(); let merger_did = auth.0; + + // Snapshot the target branch's pre-merge tip so a failed durable upload can + // roll the merge commit back on local disk (mirrors create_issue). Without + // this the local fast path in RepoStore::acquire serves the merged tree until + // a later write refreshes the archive, even though the client got a 5xx. + let target_ref = format!("refs/heads/{}", pr.target_branch); + let prior_target = store::ref_oid(&disk_path, &target_ref).ok().flatten(); + let merge_result = store::merge_branch( &disk_path, &pr.target_branch, @@ -224,11 +232,37 @@ pub async fn merge_pr( &pr.title, ); - // Always release the advisory lock — even on error; upload to Tigris only on success. - guard.release(merge_result.is_ok()).await; + // Always release the advisory lock, even on error; upload to Tigris only on + // success. A false return means the durable upload failed/timed out: the merge + // commit applied to local disk but never reached object storage, so the next + // acquire_write reverts it from the stale archive. Report 5xx so the client + // retries rather than marking the PR merged in the DB while the merged tree + // was silently lost. Fail BEFORE merge_pr / touch_repo / webhooks. (P1 + // data-loss guard, same as receive-pack.) On that failure, roll the local + // merge back to the captured tip BEFORE the lock releases. + let upload_ok = guard + .release_with_failure_cleanup(merge_result.is_ok(), |path| { + if let Some(oid) = &prior_target { + if let Err(e) = store::set_ref(path, &target_ref, oid) { + tracing::warn!(repo = %record.name, err = %e, + "failed to roll back local merge commit after failed durable upload; \ + a local-fast-path read may serve the un-uploaded merged tree"); + } + } + }) + .await; let merge_sha = merge_result.map_err(|e| AppError::Git(e.to_string()))?; + if !upload_ok { + tracing::error!(repo = %record.name, + "durable upload failed after merge; reporting failure so the client retries"); + return Err(AppError::Git(format!( + "durable storage upload failed for {}", + record.name + ))); + } + state.db.merge_pr(&pr.id, &merger_did).await?; let _ = state.db.touch_repo(&record.id).await; @@ -424,3 +458,169 @@ pub async fn list_comments( let comments = state.db.list_pr_comments(&pr.id).await?; Ok(Json(serde_json::json!({ "comments": comments }))) } + +#[cfg(test)] +mod tests { + // super::* brings in the handler plus the axum Extension/Path/State + // extractors and the Utc/Uuid/PullRequest types the seed uses. + use super::*; + use std::path::Path as StdPath; + use std::time::Duration; + + // Object store whose upload() parks forever, so release()'s bounded timeout + // is the only thing that completes it, modeling a stalled durable PUT. + // exists() is false so acquire_write never downloads over the local bare repo. + struct StallStore; + #[async_trait::async_trait] + impl crate::git::tigris::ObjectStore for StallStore { + async fn exists(&self, _o: &str, _r: &str) -> anyhow::Result { + Ok(false) + } + async fn upload(&self, _o: &str, _r: &str, _p: &StdPath) -> anyhow::Result<()> { + std::future::pending::<()>().await; + Ok(()) + } + async fn download(&self, _o: &str, _r: &str, _p: &StdPath) -> anyhow::Result<()> { + Ok(()) + } + async fn delete(&self, _o: &str, _r: &str) -> anyhow::Result<()> { + Ok(()) + } + } + + // The failed-upload rollback on the merge path. merge_pr writes the merge + // commit to LOCAL disk, then release() uploads to durable storage. When + // that upload times out the handler must 5xx (before merge_pr/webhooks) + // AND roll the target branch back to its pre-merge tip while the advisory + // lock is held, so a local-fast-path read never serves the un-uploaded + // merged tree. RED with the cleanup closure at ~pulls.rs:244 reverted to + // `|_| {}`: main stays at the merge commit. + #[sqlx::test] + async fn merge_failed_upload_rolls_back_target_ref(pool: sqlx::PgPool) { + use axum::response::IntoResponse; + use std::process::Command; + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tempfile::TempDir::new().unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::new( + repos_dir.path().to_path_buf(), + Some(std::sync::Arc::new(StallStore)), + pool.clone(), + ) + .with_release_upload_timeout(Duration::from_millis(200)); + + let owner = "z6mergerollbackowner"; + let name = "mergerollback"; + let owner_slug = owner.replace([':', '/'], "_"); + let bare = repos_dir + .path() + .join(&owner_slug) + .join(format!("{name}.git")); + + state + .db + .upsert_mirror_repo(owner, name, &bare.to_string_lossy(), None, false) + .await + .unwrap(); + + fn git(args: &[&str], dir: &StdPath) -> String { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + + // Scratch repo: c1 on main, c2 on feature; the bare clone carries both + // branches, so the merge of feature into main has real work to do. + let work = tempfile::TempDir::new().unwrap(); + git(&["init", "-q", "-b", "main", "."], work.path()); + git(&["config", "user.email", "t@t"], work.path()); + git(&["config", "user.name", "t"], work.path()); + std::fs::write(work.path().join("f.txt"), "one").unwrap(); + git(&["add", "f.txt"], work.path()); + git(&["commit", "-q", "-m", "c1"], work.path()); + git(&["checkout", "-q", "-b", "feature"], work.path()); + std::fs::write(work.path().join("g.txt"), "two").unwrap(); + git(&["add", "g.txt"], work.path()); + git(&["commit", "-q", "-m", "c2"], work.path()); + + std::fs::create_dir_all(bare.parent().unwrap()).unwrap(); + let out = Command::new("git") + .args([ + "clone", + "--bare", + "-q", + &work.path().to_string_lossy(), + &bare.to_string_lossy(), + ]) + .output() + .unwrap(); + assert!( + out.status.success(), + "clone --bare failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + + let pre_merge_tip = store::ref_oid(&bare, "refs/heads/main") + .unwrap() + .expect("main exists before the merge"); + + // Seed the open PR row the handler looks up. + let record = state.db.get_repo(owner, name).await.unwrap().unwrap(); + let now = Utc::now().to_rfc3339(); + state + .db + .create_pr(&PullRequest { + id: Uuid::new_v4().to_string(), + repo_id: record.id.clone(), + number: 1, + title: "t".into(), + body: None, + author_did: owner.to_string(), + source_branch: "feature".into(), + target_branch: "main".into(), + status: "open".to_string(), + merged_by_did: None, + merged_at: None, + created_at: now.clone(), + updated_at: now, + }) + .await + .unwrap(); + + // Merge as the owner: the merge commit applies locally, the durable + // upload stalls and times out, and the handler must report 5xx. + let resp = super::merge_pr( + State(state.clone()), + Extension(AuthenticatedDid(owner.to_string())), + Path((owner.to_string(), name.to_string(), 1)), + ) + .await; + let status = match resp { + Ok(_) => StatusCode::OK, + Err(e) => e.into_response().status(), + }; + assert!( + status.is_server_error(), + "a stalled durable upload on merge must 5xx, got {status}" + ); + + // The target branch must still point at the pre-merge tip: the failed + // upload rolled the merge commit back while the lock was held. + let tip = store::ref_oid(&bare, "refs/heads/main") + .unwrap() + .expect("main still exists after the rollback"); + assert_eq!( + tip, pre_merge_tip, + "a failed durable upload must roll the target branch back to its \ + pre-merge tip; a local-fast-path read served the un-uploaded merge" + ); + } +} diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index b38b177b..49b4a8e8 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -199,6 +199,42 @@ pub async fn create_repo( // Owner is the authenticated agent's DID let owner_did = auth.0; + // Serialize against a concurrent purge of the SAME owner/name. The purge holds + // this per-repo advisory lock across its delete-row + remove-dir, so without + // taking the same lock here a create could slip into the row-deleted-but-dir- + // not-yet-removed window and land a repos row pointing at a directory the purge + // then removes (a dangling row). Held across the existence-check -> init -> + // create_repo sequence so the two operations cannot interleave; released + // explicitly on the success path, and its guard's Drop frees the lock on every + // error path below. Bounded-wait (no object-store I/O), so the uncontended hot + // path returns on the first attempt. (KTD6, R6.) + // + // Lock on the write lock_pool (via `lock_repo_blocking`), NOT the app pool. The + // create's own work (get_repo, proof.consume, db.create_repo) runs on the app + // pool via state.db, so drawing the lock guard from a SEPARATE pool keeps it from + // pinning an app connection that work then needs. At GITLAWB_DB_MAX_CONNECTIONS=1 + // an app-pool lock self-deadlocks (get_repo waits for the one connection the + // guard holds -> PoolTimedOut -> 500). The advisory lock is DB-global, so it + // still serializes against a same-key purge holding it from lock_pool. Accepted + // residual: a sustained 32-writer burst that pins lock_pool can make this acquire + // wait -> a retryable 503, far less bad than breaking create at MAX_CONNECTIONS=1. + let repo_lock = state + .repo_store + .lock_repo_blocking(&owner_did, &req.name) + .await + // Both arms are a transient resource condition, not a create failure: the + // lock pool is pinned (Err from acquiring the lock connection) or the key + // stayed held through the bounded retry (None: a live writer or a purge). + // Return a retryable 503 so the client backs off and retries, as the + // call-site comment promises, rather than a misleading 500 git_error. + .map_err(|e| AppError::Unavailable(e.to_string()))? + .ok_or_else(|| { + AppError::Unavailable(format!( + "could not acquire repo lock for {owner_did}/{} — held by a live writer or purge", + req.name + )) + })?; + // Check it doesn't already exist if state.db.get_repo(&owner_did, &req.name).await?.is_some() { return Err(AppError::RepoExists(req.name)); @@ -235,6 +271,10 @@ pub async fn create_repo( state.db.create_repo(&record).await?; + // Row + on-disk dir now both exist consistently; the race window is closed, so + // release the per-repo lock before the (best-effort) proof recording below. + repo_lock.release().await; + // Persist the proof so it can travel with the repo and a mirroring peer can // re-verify it (enforce-mode origins only; off/shadow yield no proof here). if let Some(p) = verified_proof { @@ -554,11 +594,12 @@ pub async fn git_info_refs( // trusted-proxy policy as the POST middleware (shared buckets). if service == "git-receive-pack" { if let Some(key) = crate::rate_limit::client_key(&headers, peer, state.push_limiter_trust) { - if !state.push_rate_limiter.check(&key).await { + // Use check_retry (like U5's other 429 sites) so the rejection carries a + // window-derived Retry-After, built via the shared 429 response helper + // that the per-IP middleware also uses. + if let Some(retry_after) = state.push_rate_limiter.check_retry(&key).await { tracing::warn!(repo = %name, key = %key, "receive-pack advertisement rate limited"); - return Err(AppError::TooManyRequests( - "push rate limit exceeded — try again later".into(), - )); + return Ok(crate::rate_limit::too_many_requests(retry_after)); } } } @@ -850,6 +891,32 @@ async fn notify_peer_of_refs( } } +/// Roll a push whose durable upload failed back to the pre-push ref snapshot. +/// `None` means the pre-push listing itself failed: rolling back to an empty +/// snapshot would delete EVERY ref in the repo, so the rollback is skipped +/// instead. A genuinely empty repo snapshots as `Some(vec![])` and still rolls +/// back (deleting the refs the failed push created). +fn rollback_push_refs( + path: &std::path::Path, + repo_name: &str, + pre_push_refs: &Option>, +) { + match pre_push_refs { + Some(refs) => { + if let Err(e) = store::restore_refs(path, refs) { + tracing::warn!(repo = %repo_name, err = %e, + "failed to roll back receive-pack refs after failed durable upload; \ + a local-fast-path read may serve un-uploaded refs"); + } + } + None => { + tracing::warn!(repo = %repo_name, + "skipping ref rollback after failed durable upload: pre-push snapshot \ + unavailable; a local-fast-path read may serve un-uploaded refs"); + } + } +} + /// POST /:owner/:repo.git/git-receive-pack (AUTH REQUIRED — enforced by middleware) pub async fn git_receive_pack( State(state): State, @@ -928,454 +995,558 @@ pub async fn git_receive_pack( } tracing::debug!(repo = %name, "acquiring write lock"); - let guard = state - .repo_store - .acquire_write(&record.owner_did, &record.name) - .await - .map_err(|e| { - tracing::error!(repo = %name, err = %e, "acquire_write failed"); - AppError::Git(e.to_string()) - })?; - let disk_path = guard.path().to_path_buf(); - tracing::debug!(repo = %name, path = %disk_path.display(), "running git receive-pack"); let body_len = body.len(); let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); - let receive_result = smart_http::receive_pack(&disk_path, body, git_timeout).await; - // Always release the advisory lock — even on error — to prevent stale locks - // from blocking subsequent pushes. Only upload to Tigris when the push - // succeeded; uploading a half-applied repo would propagate corruption. - guard.release(receive_result.is_ok()).await; + // Detach the WHOLE push — acquire → receive-pack → release AND the success + // tail (metadata + fan-out) — from THIS handler future. The pack `body` is + // already fully buffered, so the spawned task is self-contained: a client + // disconnect drops the handler future but does NOT cancel the task, so a + // fully-received push still applies the pack, releases the lock, AND runs its + // full metadata/fan-out tail server-side. Folding the tail INTO the task (vs + // leaving it in the cancellable request future) is what prevents a split-brain + // on disconnect: committed git with no push record / certs / broadcast. (KTD1, + // R1.) + // + // report-status is handed to the (possibly still-connected) client over a + // oneshot the request future awaits — NOT by awaiting the JoinHandle, which + // would re-couple the client response to the full tail completing (a latency + // regression). The task sends the receive-pack result, then continues the tail + // independently; if the client has disconnected the receiver is dropped and the + // send fails, which the task ignores and runs the tail regardless. + let (report_tx, report_rx) = tokio::sync::oneshot::channel::>(); + tokio::spawn(async move { + let guard = match state + .repo_store + .acquire_write(&record.owner_did, &record.name) + .await + { + Ok(g) => g, + Err(e) => { + tracing::error!(repo = %record.name, err = %e, "acquire_write failed"); + let _ = report_tx.send(Err(AppError::Git(e.to_string()))); + return; + } + }; + let disk_path = guard.path().to_path_buf(); + + // Snapshot the pre-push refs so a failed durable upload can roll the + // applied pack back on local disk (mirrors create_issue). Without this + // the local fast path in RepoStore::acquire serves the pushed refs until + // a later write refreshes the archive, even though the client got a 5xx. + // `.ok()`, not `.unwrap_or_default()`: a listing error must read as + // "snapshot unavailable", never as an empty snapshot, or the rollback + // below would delete every ref in the repo. Fail closed on a snapshot + // failure: without a restore plan, a failed durable upload would leave + // the pushed refs on the local fast path with no way back (the class + // this rollback exists to close), so refuse the push before mutating. + let pre_push_refs = match store::list_refs(&disk_path) { + Ok(refs) => Some(refs), + Err(e) => { + tracing::error!(repo = %record.name, err = %e, "pre-push ref snapshot failed"); + // release(false): no upload attempted, so the bool is always true. + let _ = guard.release(false).await; + let _ = report_tx.send(Err(AppError::Internal(anyhow::anyhow!( + "cannot snapshot pre-push refs; refusing push: {e}" + )))); + return; + } + }; + + tracing::debug!(repo = %record.name, path = %disk_path.display(), "running git receive-pack"); + let result = smart_http::receive_pack(&disk_path, body, git_timeout).await; + // Always release the advisory lock — even on error, and BEFORE the tail — + // to prevent stale locks and to avoid holding the write lock across the + // fan-out. Only upload to Tigris when the push succeeded; uploading a + // half-applied repo would propagate corruption. On a failed durable + // upload, restore the pre-push refs BEFORE the lock releases so a + // local-fast-path read does not serve refs that never landed durably. + let upload_ok = guard + .release_with_failure_cleanup(result.is_ok(), |path| { + rollback_push_refs(path, &record.name, &pre_push_refs) + }) + .await; - let result = receive_result.map_err(|e| { - let app = git_service_app_error(&e); - match &app { - AppError::Timeout(_) => tracing::warn!(repo = %name, "git receive-pack timed out"), - AppError::BadRequest(msg) => { - tracing::warn!(repo = %name, err = %msg, "git receive-pack: bad client request") + // On receive-pack error, deliver the classified error to the client and run + // NO tail. On success, hand report-status to the client now, then continue + // the tail regardless of whether the client is still listening. + let response = match result { + Ok(resp) => resp, + Err(e) => { + let app = git_service_app_error(&e); + match &app { + AppError::Timeout(_) => { + tracing::warn!(repo = %record.name, "git receive-pack timed out") + } + AppError::BadRequest(msg) => { + tracing::warn!(repo = %record.name, err = %msg, "git receive-pack: bad client request") + } + _ => tracing::error!(repo = %record.name, err = %e, "git receive-pack failed"), + } + let _ = report_tx.send(Err(app)); + return; } - _ => tracing::error!(repo = %name, err = %e, "git receive-pack failed"), + }; + + // The refs applied locally, but the durable upload to object storage + // FAILED or timed out. Report the push as failed (5xx) instead of 200 so + // the idempotent client re-pushes and re-uploads: otherwise the next + // acquire_write downloads the STALE pre-push archive over local disk and + // reverts these refs, silently losing the commit the client believes it + // landed. Skip the whole success tail — a push whose durable copy never + // landed must not be recorded/broadcast as accepted. (P1 data-loss fix.) + if !upload_ok { + tracing::error!(repo = %record.name, + "durable upload failed after receive-pack — reporting push failure so the client retries"); + let _ = report_tx.send(Err(AppError::Git(format!( + "durable storage upload failed for {}", + record.name + )))); + return; } - app - })?; - // Update the repo's updated_at timestamp after a successful push - let _ = state.db.touch_repo(&record.id).await; + let _ = report_tx.send(Ok(response)); - // Record the successful push for metrics. The body has already been - // consumed by smart_http::receive_pack so we observe size up front. - crate::metrics::record_push(&record.id); - crate::metrics::observe_pack_size(body_len as f64); + // Update the repo's updated_at timestamp after a successful push + let _ = state.db.touch_repo(&record.id).await; - // Record push event for trust score and issue a signed ref certificate. - // The route is behind `require_signature`, so the verified pusher identity is - // always present; use it directly rather than re-parsing the headers. - let did = auth.0.as_str(); - { - // Use the first new commit hash we parsed, fall back to timestamp - let commit_hash = ref_updates - .first() - .map(|u| u.new_sha.clone()) - .unwrap_or_else(|| Utc::now().timestamp().to_string()); - - let _ = state.db.record_push(did, &record.id, &commit_hash, 0).await; - if let Ok(push_count) = state.db.get_push_count(did).await { - // 0.05 base (from registration) + 0.05 per push, capped at 1.0 - // 1 push → 0.10, 5 pushes → 0.30, 19 pushes → 1.0 - let new_score = (push_count as f64 * 0.05 + 0.05).min(1.0); - let _ = state.db.update_trust_score(did, new_score).await; - } + // Record the successful push for metrics. The body has already been + // consumed by smart_http::receive_pack so we observe size up front. + crate::metrics::record_push(&record.id); + crate::metrics::observe_pack_size(body_len as f64); - // Issue a signed certificate for every ref this push advanced, each - // carrying that ref's real old→new transition. A multi-ref push must - // not collapse to a single cert covering only the first ref. - for update in &ref_updates { - match cert::issue_ref_certificate( - &state, - &record.id, - &update.ref_name, - &update.old_sha, - &update.new_sha, - did, - ) - .await - { - Ok(c) => { - tracing::info!(cert_id = %c.id, repo = %record.name, ref_name = %update.ref_name, pusher = %did, "issued ref certificate") - } - Err(e) => { - tracing::warn!(err = %e, ref_name = %update.ref_name, "failed to issue ref certificate") + // Record push event for trust score and issue a signed ref certificate. + // The route is behind `require_signature`, so the verified pusher identity is + // always present; use it directly rather than re-parsing the headers. + let did = auth.0.as_str(); + { + // Use the first new commit hash we parsed, fall back to timestamp + let commit_hash = ref_updates + .first() + .map(|u| u.new_sha.clone()) + .unwrap_or_else(|| Utc::now().timestamp().to_string()); + + let _ = state.db.record_push(did, &record.id, &commit_hash, 0).await; + if let Ok(push_count) = state.db.get_push_count(did).await { + // 0.05 base (from registration) + 0.05 per push, capped at 1.0 + // 1 push → 0.10, 5 pushes → 0.30, 19 pushes → 1.0 + let new_score = (push_count as f64 * 0.05 + 0.05).min(1.0); + let _ = state.db.update_trust_score(did, new_score).await; + } + + // Issue a signed certificate for every ref this push advanced, each + // carrying that ref's real old→new transition. A multi-ref push must + // not collapse to a single cert covering only the first ref. + for update in &ref_updates { + match cert::issue_ref_certificate( + &state, + &record.id, + &update.ref_name, + &update.old_sha, + &update.new_sha, + did, + ) + .await + { + Ok(c) => { + tracing::info!(cert_id = %c.id, repo = %record.name, ref_name = %update.ref_name, pusher = %did, "issued ref certificate") + } + Err(e) => { + tracing::warn!(err = %e, ref_name = %update.ref_name, "failed to issue ref certificate") + } } } } - } - // Fire push webhooks — one per ref update - if !ref_updates.is_empty() { - let base_url = state - .config - .public_url - .as_deref() - .unwrap_or("http://127.0.0.1:7545") - .trim_end_matches('/'); - let owner_short = crate::db::normalize_owner_key(&record.owner_did); - let clone_url = format!("{}/{}/{}.git", base_url, owner_short, record.name); - - for update in &ref_updates { - let payload = serde_json::json!({ - "ref": update.ref_name, - "before": update.old_sha, - "after": update.new_sha, - "created": update.old_sha == ZERO_SHA, - "forced": false, - "pusher": { - "did": did, - }, - "repository": { - "id": record.id, - "name": record.name, - "owner_did": record.owner_did, - "clone_url": clone_url, - }, - }); - webhooks::fire_event( - state.db.clone(), - state.http_client.clone(), - &record.id, - "push", - payload, - ); + // Fire push webhooks — one per ref update + if !ref_updates.is_empty() { + let base_url = state + .config + .public_url + .as_deref() + .unwrap_or("http://127.0.0.1:7545") + .trim_end_matches('/'); + let owner_short = crate::db::normalize_owner_key(&record.owner_did); + let clone_url = format!("{}/{}/{}.git", base_url, owner_short, record.name); + + for update in &ref_updates { + let payload = serde_json::json!({ + "ref": update.ref_name, + "before": update.old_sha, + "after": update.new_sha, + "created": update.old_sha == ZERO_SHA, + "forced": false, + "pusher": { + "did": did, + }, + "repository": { + "id": record.id, + "name": record.name, + "owner_did": record.owner_did, + "clone_url": clone_url, + }, + }); + webhooks::fire_event( + state.db.clone(), + state.http_client.clone(), + &record.id, + "push", + payload, + ); + } } - } - // Replication enforcement (Phase 2): decide once per push whether the public - // may read this repo at all and, if so, which blob OIDs must not leave the - // node. `withheld == None` means replicate nothing (private / mode A / - // undetermined): skip every pin so even commit and tree objects (which - // withheld_blob_oids never lists) stay local. `announce` gates the - // network-facing announcements. Fail closed: a private or undetermined repo - // never leaks. - let rules_opt = state.db.list_visibility_rules(&record.id).await.ok(); - let (announce, withheld) = replication_withheld_set( - rules_opt.clone(), - &record.owner_did, - record.is_public, - disk_path.clone(), - ) - .await; - - // Resolve the per-push pin candidate set once, off the async worker, then - // filter to what may actually replicate. Delta path: the reachable-only - // `withheld` set suffices (delta objects are reachable). Full-scan path: the - // candidate set can include dangling blobs the withheld set never classified, - // so fail closed — replicate a blob only if it is reachable AND - // visibility-allowed (#99). Only computed when something will actually - // replicate; every degraded path logs rather than failing silently. - let object_list: Vec = if let Some(withheld_set) = withheld.clone() { - let new_tips: Vec = ref_updates - .iter() - .map(|u| u.new_sha.clone()) - .filter(|s| s != ZERO_SHA) - .collect(); - let old_tips: Vec = ref_updates - .iter() - .map(|u| u.old_sha.clone()) - .filter(|s| s != ZERO_SHA) - .collect(); - let pin_set = crate::git::push_delta::resolve_candidates_for_push( + // Replication enforcement (Phase 2): decide once per push whether the public + // may read this repo at all and, if so, which blob OIDs must not leave the + // node. `withheld == None` means replicate nothing (private / mode A / + // undetermined): skip every pin so even commit and tree objects (which + // withheld_blob_oids never lists) stay local. `announce` gates the + // network-facing announcements. Fail closed: a private or undetermined repo + // never leaks. + let rules_opt = state.db.list_visibility_rules(&record.id).await.ok(); + let (announce, withheld) = replication_withheld_set( + rules_opt.clone(), + &record.owner_did, + record.is_public, disk_path.clone(), - new_tips, - old_tips, ) .await; - if pin_set.full_scan { - fail_closed_full_scan_objects( - disk_path.clone(), - rules_opt.clone().unwrap_or_default(), - record.is_public, - record.owner_did.clone(), - pin_set.candidates, - ) - .await - } else { - crate::git::visibility_pack::replicable_objects(pin_set.candidates, &withheld_set) - } - } else { - Vec::new() - }; - // Pin new git objects to the local IPFS node (no-op if ipfs_api is empty). - // Skipped entirely when the public cannot read the repo (withheld == None). - if withheld.is_some() { - let object_list_ipfs = object_list.clone(); - let ipfs_api = state.config.ipfs_api.clone(); - let repo_path_clone = disk_path.clone(); - let db_clone = state.db.clone(); - let rules_for_enc = rules_opt.clone(); - let repo_id = record.id.clone(); - let owner_did = record.owner_did.clone(); - let is_public = record.is_public; - let irys_url = state.config.irys_url.clone(); - let http_client = std::sync::Arc::clone(&state.http_client); - let node_did_str = state.node_did.to_string(); - let node_seed = state.node_keypair.to_seed(); - let repo_name = record.name.clone(); - tokio::spawn(async move { - let pinned = crate::ipfs_pin::pin_new_objects( - &ipfs_api, - &repo_path_clone, - object_list_ipfs, - &db_clone, + // Resolve the per-push pin candidate set once, off the async worker, then + // filter to what may actually replicate. Delta path: the reachable-only + // `withheld` set suffices (delta objects are reachable). Full-scan path: the + // candidate set can include dangling blobs the withheld set never classified, + // so fail closed — replicate a blob only if it is reachable AND + // visibility-allowed (#99). Only computed when something will actually + // replicate; every degraded path logs rather than failing silently. + let object_list: Vec = if let Some(withheld_set) = withheld.clone() { + let new_tips: Vec = ref_updates + .iter() + .map(|u| u.new_sha.clone()) + .filter(|s| s != ZERO_SHA) + .collect(); + let old_tips: Vec = ref_updates + .iter() + .map(|u| u.old_sha.clone()) + .filter(|s| s != ZERO_SHA) + .collect(); + let pin_set = crate::git::push_delta::resolve_candidates_for_push( + disk_path.clone(), + new_tips, + old_tips, ) .await; - if !pinned.is_empty() { - tracing::info!(count = pinned.len(), "pinned git objects to IPFS"); - for (sha, cid) in &pinned { - tracing::info!(sha = %sha, %cid, "pinned"); - } + if pin_set.full_scan { + fail_closed_full_scan_objects( + disk_path.clone(), + rules_opt.clone().unwrap_or_default(), + record.is_public, + record.owner_did.clone(), + pin_set.candidates, + ) + .await + } else { + crate::git::visibility_pack::replicable_objects(pin_set.candidates, &withheld_set) } + } else { + Vec::new() + }; - // Option B1: encrypt-then-pin the withheld blobs so authorized - // readers can recover them when the origin cannot serve them. - // No path-scoped rule can withhold a blob, so withheld_blob_recipients - // would return an empty map after a full per-ref walk; skip it. Mirrors - // the has_path_scoped_rule gate on the other two withheld-walk sites. - if let Some(rules) = rules_for_enc.filter(|r| visibility_pack::has_path_scoped_rule(r)) - { - let p = repo_path_clone.clone(); - let owner = owner_did.clone(); - let recip = tokio::task::spawn_blocking(move || { - crate::git::visibility_pack::withheld_blob_recipients( - &p, &rules, is_public, &owner, - ) - }) + // Pin new git objects to the local IPFS node (no-op if ipfs_api is empty). + // Skipped entirely when the public cannot read the repo (withheld == None). + if withheld.is_some() { + let object_list_ipfs = object_list.clone(); + let ipfs_api = state.config.ipfs_api.clone(); + let repo_path_clone = disk_path.clone(); + let db_clone = state.db.clone(); + let rules_for_enc = rules_opt.clone(); + let repo_id = record.id.clone(); + let owner_did = record.owner_did.clone(); + let is_public = record.is_public; + let irys_url = state.config.irys_url.clone(); + let http_client = std::sync::Arc::clone(&state.http_client); + let node_did_str = state.node_did.to_string(); + let node_seed = state.node_keypair.to_seed(); + let repo_name = record.name.clone(); + tokio::spawn(async move { + let pinned = crate::ipfs_pin::pin_new_objects( + &ipfs_api, + &repo_path_clone, + object_list_ipfs, + &db_clone, + ) .await; - if let Ok(Ok(recipients)) = recip { - let delta = crate::encrypted_pin::encrypt_and_pin( - &ipfs_api, - &repo_path_clone, - &db_clone, - &repo_id, - &node_seed, - &recipients, - ) - .await; + if !pinned.is_empty() { + tracing::info!(count = pinned.len(), "pinned git objects to IPFS"); + for (sha, cid) in &pinned { + tracing::info!(sha = %sha, %cid, "pinned"); + } + } - // Option B3: anchor a per-push manifest of the blobs sealed - // this push to Arweave, so the oid->cid index survives total - // node loss. Best-effort; never fails the push. - if !delta.is_empty() && !irys_url.is_empty() { - let owner_short = crate::db::normalize_owner_key(&owner_did); - let repo_slug = format!("{owner_short}/{repo_name}"); - let ts = chrono::Utc::now().to_rfc3339(); - let manifest = crate::arweave::EncryptedManifest { - repo: &repo_slug, - owner_did: &owner_did, - node_did: &node_did_str, - timestamp: &ts, - blobs: &delta, - }; - match crate::arweave::anchor_encrypted_manifest( - &http_client, - &irys_url, - &manifest, + // Option B1: encrypt-then-pin the withheld blobs so authorized + // readers can recover them when the origin cannot serve them. + // No path-scoped rule can withhold a blob, so withheld_blob_recipients + // would return an empty map after a full per-ref walk; skip it. Mirrors + // the has_path_scoped_rule gate on the other two withheld-walk sites. + if let Some(rules) = + rules_for_enc.filter(|r| visibility_pack::has_path_scoped_rule(r)) + { + let p = repo_path_clone.clone(); + let owner = owner_did.clone(); + let recip = tokio::task::spawn_blocking(move || { + crate::git::visibility_pack::withheld_blob_recipients( + &p, &rules, is_public, &owner, ) - .await - { - Ok(tx) if !tx.is_empty() => tracing::info!( - repo = %repo_slug, - tx_id = %tx, - "anchored encrypted manifest to Arweave" - ), - Ok(_) => {} - Err(e) => tracing::warn!( - repo = %repo_slug, - err = %e, - "encrypted manifest anchor failed" - ), + }) + .await; + if let Ok(Ok(recipients)) = recip { + let delta = crate::encrypted_pin::encrypt_and_pin( + &ipfs_api, + &repo_path_clone, + &db_clone, + &repo_id, + &node_seed, + &recipients, + ) + .await; + + // Option B3: anchor a per-push manifest of the blobs sealed + // this push to Arweave, so the oid->cid index survives total + // node loss. Best-effort; never fails the push. + if !delta.is_empty() && !irys_url.is_empty() { + let owner_short = crate::db::normalize_owner_key(&owner_did); + let repo_slug = format!("{owner_short}/{repo_name}"); + let ts = chrono::Utc::now().to_rfc3339(); + let manifest = crate::arweave::EncryptedManifest { + repo: &repo_slug, + owner_did: &owner_did, + node_did: &node_did_str, + timestamp: &ts, + blobs: &delta, + }; + match crate::arweave::anchor_encrypted_manifest( + &http_client, + &irys_url, + &manifest, + ) + .await + { + Ok(tx) if !tx.is_empty() => tracing::info!( + repo = %repo_slug, + tx_id = %tx, + "anchored encrypted manifest to Arweave" + ), + Ok(_) => {} + Err(e) => tracing::warn!( + repo = %repo_slug, + err = %e, + "encrypted manifest anchor failed" + ), + } } } } - } - }); - } + }); + } - // Pin new git objects to Pinata, then record branch→CID and gossip - { - let pinata_jwt = state.config.pinata_jwt.clone(); - let pinata_upload_url = state.config.pinata_upload_url.clone(); - let repo_path_clone = disk_path.clone(); - let db_clone = state.db.clone(); - let http_client = Arc::clone(&state.http_client); - let node_did_str = state.node_did.to_string(); - let repo_slug = format!( - "{}/{}", - crate::db::normalize_owner_key(&record.owner_did), - record.name - ); - let ref_updates_clone = ref_updates - .iter() - .map(|u| (u.ref_name.clone(), u.old_sha.clone(), u.new_sha.clone())) - .collect::>(); - let p2p_handle = state.p2p.clone(); - let pusher_did_clone = did.to_string(); - let db_for_peers = state.db.clone(); - let ref_update_tx = state.ref_update_tx.clone(); - let irys_url = state.config.irys_url.clone(); - let owner_did_for_arweave = record.owner_did.clone(); - let self_public_url = state.config.public_url.clone(); - let node_keypair = Arc::clone(&state.node_keypair); - let object_list_pinata = object_list; - let do_pinata_replication = withheld.is_some(); - tokio::spawn(async move { - let pinned = if do_pinata_replication { - crate::pinata::pin_new_objects( - &http_client, - &pinata_upload_url, - &pinata_jwt, - &repo_path_clone, - object_list_pinata, - &db_clone, - ) - .await - } else { - Vec::new() - }; + // Pin new git objects to Pinata, then record branch→CID and gossip + { + let pinata_jwt = state.config.pinata_jwt.clone(); + let pinata_upload_url = state.config.pinata_upload_url.clone(); + let repo_path_clone = disk_path.clone(); + let db_clone = state.db.clone(); + let http_client = Arc::clone(&state.http_client); + let node_did_str = state.node_did.to_string(); + let repo_slug = format!( + "{}/{}", + crate::db::normalize_owner_key(&record.owner_did), + record.name + ); + let ref_updates_clone = ref_updates + .iter() + .map(|u| (u.ref_name.clone(), u.old_sha.clone(), u.new_sha.clone())) + .collect::>(); + let p2p_handle = state.p2p.clone(); + let pusher_did_clone = did.to_string(); + let db_for_peers = state.db.clone(); + let ref_update_tx = state.ref_update_tx.clone(); + let irys_url = state.config.irys_url.clone(); + let owner_did_for_arweave = record.owner_did.clone(); + let self_public_url = state.config.public_url.clone(); + let node_keypair = Arc::clone(&state.node_keypair); + let object_list_pinata = object_list; + let do_pinata_replication = withheld.is_some(); + tokio::spawn(async move { + let pinned = if do_pinata_replication { + crate::pinata::pin_new_objects( + &http_client, + &pinata_upload_url, + &pinata_jwt, + &repo_path_clone, + object_list_pinata, + &db_clone, + ) + .await + } else { + Vec::new() + }; - if !pinned.is_empty() { - tracing::info!(count = pinned.len(), "pinned git objects to Pinata"); - } + if !pinned.is_empty() { + tracing::info!(count = pinned.len(), "pinned git objects to Pinata"); + } - // Build sha→cid map from pinned objects - let cid_map: std::collections::HashMap = pinned.into_iter().collect(); + // Build sha→cid map from pinned objects + let cid_map: std::collections::HashMap = + pinned.into_iter().collect(); - // Record branch→CID for each ref update and publish gossip - for (ref_name, old_sha, new_sha) in &ref_updates_clone { - let cid = cid_map.get(new_sha).map(|s| s.as_str()); + // Record branch→CID for each ref update and publish gossip + for (ref_name, old_sha, new_sha) in &ref_updates_clone { + let cid = cid_map.get(new_sha).map(|s| s.as_str()); + + if let Some(cid_str) = cid { + let _ = db_clone + .upsert_branch_cid( + &repo_slug, + ref_name, + new_sha, + cid_str, + &node_did_str, + ) + .await; + } - if let Some(cid_str) = cid { - let _ = db_clone - .upsert_branch_cid(&repo_slug, ref_name, new_sha, cid_str, &node_did_str) - .await; + if announce { + if let Some(p2p) = &p2p_handle { + p2p.publish_ref_update(crate::p2p::RefUpdateEvent { + node_did: node_did_str.clone(), + pusher_did: pusher_did_clone.clone(), + owner_did: Some(record.owner_did.clone()), + repo: repo_slug.clone(), + ref_name: ref_name.clone(), + old_sha: old_sha.clone(), + new_sha: new_sha.clone(), + timestamp: chrono::Utc::now().to_rfc3339(), + cert_id: None, + cid: cid.map(|s| s.to_string()), + }) + .await; + } + } } + // Broadcast ref update to GraphQL subscription listeners — one per ref. + // Gated on `announce`: /graphql/ws is unauthenticated (mounted after + // the optional_signature layer), and the subscription resolver has no + // caller to gate against, so only publicly-readable ref updates may + // reach anonymous subscribers. Mirrors the gossip (above) and Arweave + // (below) sends, which are already `announce`-gated. Without this a + // private-repo push would leak live ref metadata over the socket — + // the subscription analog of #112/#114. + let now_ts = chrono::Utc::now().to_rfc3339(); if announce { - if let Some(p2p) = &p2p_handle { - p2p.publish_ref_update(crate::p2p::RefUpdateEvent { - node_did: node_did_str.clone(), - pusher_did: pusher_did_clone.clone(), + for (ref_name, old_sha, new_sha) in &ref_updates_clone { + let _ = ref_update_tx.send(crate::state::RefUpdateBroadcast { repo: repo_slug.clone(), - owner_did: Some(record.owner_did.clone()), + owner_did: record.owner_did.clone(), ref_name: ref_name.clone(), old_sha: old_sha.clone(), new_sha: new_sha.clone(), - timestamp: chrono::Utc::now().to_rfc3339(), - cert_id: None, - cid: cid.map(|s| s.to_string()), - }) - .await; + pusher_did: pusher_did_clone.clone(), + node_did: node_did_str.clone(), + timestamp: now_ts.clone(), + }); } } - } - - // Broadcast ref update to GraphQL subscription listeners — one per ref. - // Gated on `announce`: /graphql/ws is unauthenticated (mounted after - // the optional_signature layer), and the subscription resolver has no - // caller to gate against, so only publicly-readable ref updates may - // reach anonymous subscribers. Mirrors the gossip (above) and Arweave - // (below) sends, which are already `announce`-gated. Without this a - // private-repo push would leak live ref metadata over the socket — - // the subscription analog of #112/#114. - let now_ts = chrono::Utc::now().to_rfc3339(); - if announce { - for (ref_name, old_sha, new_sha) in &ref_updates_clone { - let _ = ref_update_tx.send(crate::state::RefUpdateBroadcast { - repo: repo_slug.clone(), - ref_name: ref_name.clone(), - old_sha: old_sha.clone(), - new_sha: new_sha.clone(), - pusher_did: pusher_did_clone.clone(), - node_did: node_did_str.clone(), - timestamp: now_ts.clone(), - owner_did: record.owner_did.clone(), - }); - } - } - // Arweave permanent anchoring — fire for each ref update. - // Suppressed for repos the public cannot read (public permanent ledger). - if announce && !irys_url.is_empty() { - for (ref_name, old_sha, new_sha) in &ref_updates_clone { - let cid = cid_map.get(new_sha).cloned(); - let anchor = crate::arweave::RefAnchor { - repo: repo_slug.clone(), - owner_did: owner_did_for_arweave.clone(), - ref_name: ref_name.clone(), - old_sha: old_sha.clone(), - new_sha: new_sha.clone(), - cid: cid.clone(), - timestamp: now_ts.clone(), - node_did: node_did_str.clone(), - }; - match crate::arweave::anchor_ref_update(&http_client, &irys_url, &anchor).await - { - Ok(tx_id) if !tx_id.is_empty() => { - let arweave_url = crate::arweave::arweave_url(&tx_id); - let _ = db_clone - .record_arweave_anchor(&crate::db::RecordAnchorInput { - repo: &repo_slug, - owner_did: &owner_did_for_arweave, - ref_name, - old_sha, - new_sha, - cid: cid.as_deref(), - irys_tx_id: &tx_id, - arweave_url: &arweave_url, - node_did: &node_did_str, - }) - .await; + // Arweave permanent anchoring — fire for each ref update. + // Suppressed for repos the public cannot read (public permanent ledger). + if announce && !irys_url.is_empty() { + for (ref_name, old_sha, new_sha) in &ref_updates_clone { + let cid = cid_map.get(new_sha).cloned(); + let anchor = crate::arweave::RefAnchor { + repo: repo_slug.clone(), + owner_did: owner_did_for_arweave.clone(), + ref_name: ref_name.clone(), + old_sha: old_sha.clone(), + new_sha: new_sha.clone(), + cid: cid.clone(), + timestamp: now_ts.clone(), + node_did: node_did_str.clone(), + }; + match crate::arweave::anchor_ref_update(&http_client, &irys_url, &anchor) + .await + { + Ok(tx_id) if !tx_id.is_empty() => { + let arweave_url = crate::arweave::arweave_url(&tx_id); + let _ = db_clone + .record_arweave_anchor(&crate::db::RecordAnchorInput { + repo: &repo_slug, + owner_did: &owner_did_for_arweave, + ref_name, + old_sha, + new_sha, + cid: cid.as_deref(), + irys_tx_id: &tx_id, + arweave_url: &arweave_url, + node_did: &node_did_str, + }) + .await; + } + Ok(_) => {} + Err(e) => { + tracing::warn!(repo=%repo_slug, err=%e, "Arweave anchor failed") + } } - Ok(_) => {} - Err(e) => tracing::warn!(repo=%repo_slug, err=%e, "Arweave anchor failed"), } } - } - // HTTP peer notification — notify all known peers to pull from us. - // This is the reliable fallback when Gossipsub p2p is not yet connected. - // Suppressed for repos the public cannot read. Runs last so a slow or - // unreachable peer cannot delay the local GraphQL broadcast or Arweave - // anchoring above; this is the lowest-priority best-effort step. - if announce { - if let Ok(peers) = db_for_peers.list_peers().await { - for peer in peers { - if peer.http_url.is_empty() { - continue; - } - let peer_url = peer.http_url.trim_end_matches('/'); - if let Some(self_url) = self_public_url.as_deref() { - if peer_url == self_url.trim_end_matches('/') { + // HTTP peer notification — notify all known peers to pull from us. + // This is the reliable fallback when Gossipsub p2p is not yet connected. + // Suppressed for repos the public cannot read. Runs last so a slow or + // unreachable peer cannot delay the local GraphQL broadcast or Arweave + // anchoring above; this is the lowest-priority best-effort step. + if announce { + if let Ok(peers) = db_for_peers.list_peers().await { + for peer in peers { + if peer.http_url.is_empty() { continue; } + let peer_url = peer.http_url.trim_end_matches('/'); + if let Some(self_url) = self_public_url.as_deref() { + if peer_url == self_url.trim_end_matches('/') { + continue; + } + } + let notify_url = format!("{peer_url}{SYNC_NOTIFY_PATH}"); + notify_peer_of_refs( + &http_client, + node_keypair.as_ref(), + &peer.did, + ¬ify_url, + &repo_slug, + &ref_updates_clone, + &node_did_str, + &pusher_did_clone, + &record.owner_did, + ) + .await; } - let notify_url = format!("{peer_url}{SYNC_NOTIFY_PATH}"); - notify_peer_of_refs( - &http_client, - node_keypair.as_ref(), - &peer.did, - ¬ify_url, - &repo_slug, - &ref_updates_clone, - &node_did_str, - &pusher_did_clone, - &record.owner_did, - ) - .await; } } - } - }); - } + }); + } + }); + + // Await report-status from the detached task WITHOUT blocking on the tail. If + // the client disconnected, this future is already dropped and `report_rx` with + // it — the task then runs the tail regardless. A send-before-report task panic + // drops the sender, surfaced here as an internal error (matching the prior + // JoinError → 500 mapping). + let result = report_rx.await.map_err(|_| { + AppError::Internal(anyhow::anyhow!( + "receive-pack task failed before reporting report-status" + )) + })??; Ok(result) } @@ -1550,6 +1721,38 @@ pub async fn fork_repo( )); } + // Materialize the SOURCE on local disk BEFORE taking the target lock + // (downloads from Tigris on cache miss). On a cold source (archive-only, no + // local dir) this download takes a nested advisory lock on the write + // lock_pool for the source's OWN namespace; doing it under the held target + // lock would need two lock-pool connections at once from a pool sized + // one-per-writer. The source is a read-only, different-namespace concern and + // needs no target-namespace protection, and nothing under the lock below + // depends on it beyond the clone reading source_path. + let source_path = state + .repo_store + .acquire(&source.owner_did, &source.name) + .await + .map_err(|e| AppError::Git(e.to_string()))?; + + // Serialize against a concurrent same-key purge or creation on the FORK + // TARGET namespace, mirroring create_repo's lock above (same rationale, + // same 503 mapping). Held across the conflict check, clone, durable upload, + // and row insert, so none of it can interleave with a purge's delete-row + + // remove-dir window, and so the row is only published after the archive + // durably landed. Released explicitly on both exits below; the guard's + // Drop frees the lock on every intermediate error path. + let repo_lock = state + .repo_store + .lock_repo_blocking(&forker_did, &fork_name) + .await + .map_err(|e| AppError::Unavailable(e.to_string()))? + .ok_or_else(|| { + AppError::Unavailable(format!( + "could not acquire repo lock for {forker_did}/{fork_name}: held by a live writer or purge" + )) + })?; + // Check no name conflict under the forker's ownership let forker_short = crate::db::normalize_owner_key(&forker_did); if state.db.get_repo(forker_short, &fork_name).await?.is_some() { @@ -1561,38 +1764,93 @@ pub async fn fork_repo( // Request is admissible — spend the proof now, immediately before the write. let verified_proof = proof.consume(&state.db).await?; - // Ensure source repo is on local disk (downloads from Tigris on cache miss) - let source_path = state - .repo_store - .acquire(&source.owner_did, &source.name) - .await - .map_err(|e| AppError::Git(e.to_string()))?; - let disk_path = store::repo_disk_path(&state.config.repos_dir, &forker_did, &fork_name); - // Clone the source repo as a mirror - let output = std::process::Command::new("git") - .args([ - "clone", - "--mirror", - source_path.to_str().unwrap_or(""), - disk_path.to_str().unwrap_or(""), - ]) - .output() - .map_err(|e| AppError::Git(format!("git clone --mirror failed: {e}")))?; + // Clone the source repo as a mirror, bounded so a pathological or huge source + // cannot pin the held target lock (and its lock-pool connection) + // indefinitely. Reuses the served-git ceiling (git_service_timeout_secs, + // generous for large clones); after the source reorder above source_path is a + // LOCAL path so a normal clone is fast and never approaches the bound, which + // is a safety ceiling only. tokio::process with kill_on_drop tears the child + // down when the timeout drops the future. + let clone_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let output = match tokio::time::timeout( + clone_timeout, + tokio::process::Command::new("git") + .args([ + "clone", + "--mirror", + source_path.to_str().unwrap_or(""), + disk_path.to_str().unwrap_or(""), + ]) + .kill_on_drop(true) + .output(), + ) + .await + { + Ok(Ok(output)) => output, + // Spawn/IO failure, or the clone exceeded the bound (child killed on + // drop). Clear any partial mirror and fail the fork with the same + // retryable shape the durable-upload arm below returns, before any + // RepoRecord exists. + Ok(Err(e)) => { + if let Err(rm) = std::fs::remove_dir_all(&disk_path) { + tracing::warn!(fork = %fork_name, err = %rm, + "failed to remove fork mirror after a failed clone spawn"); + } + repo_lock.release().await; + return Err(AppError::Git(format!("git clone --mirror failed: {e}"))); + } + Err(_elapsed) => { + if let Err(rm) = std::fs::remove_dir_all(&disk_path) { + tracing::warn!(fork = %fork_name, err = %rm, + "failed to remove fork mirror after a clone timeout"); + } + repo_lock.release().await; + return Err(AppError::Git(format!( + "git clone --mirror timed out after {}s for {fork_name}", + clone_timeout.as_secs() + ))); + } + }; if !output.status.success() { let stderr = String::from_utf8_lossy(&output.stderr); + // git clone --mirror can create the destination dir then exit non-zero + // (authz, corrupt/partial source), leaving a half-created mirror. Clear + // it best-effort and free the lock, matching the timeout/spawn arms, so + // a retry is not blocked by an existing dest and a same-name create sees + // an empty path. + if let Err(rm) = std::fs::remove_dir_all(&disk_path) { + tracing::warn!(fork = %fork_name, err = %rm, + "failed to remove fork mirror after a non-zero clone exit"); + } + repo_lock.release().await; return Err(AppError::Git(format!( "git clone --mirror failed: {stderr}" ))); } - // Upload fork to Tigris - state + // Upload the fork to durable storage under the held target lock, bounded by + // the release-upload timeout. Publish-after-durability: an attempted upload + // that failed or timed out fails the fork (the sibling write paths' 5xx) + // BEFORE any RepoRecord exists, and removes the cloned mirror so no later + // acquire serves a fork whose archive never landed. Store-less nodes take + // the success arm (nothing to upload is not a failure). + if !state .repo_store - .release_after_write(&forker_did, &fork_name) - .await; + .upload_under_guard(&forker_did, &fork_name, &repo_lock) + .await + { + if let Err(e) = std::fs::remove_dir_all(&disk_path) { + tracing::warn!(fork = %fork_name, err = %e, + "failed to remove fork mirror after failed durable upload"); + } + repo_lock.release().await; + return Err(AppError::Git(format!( + "durable storage upload failed for {fork_name}" + ))); + } let now = Utc::now(); let record = crate::db::RepoRecord { @@ -1609,7 +1867,32 @@ pub async fn fork_repo( machine_id: state.machine_id.clone(), }; - state.db.create_repo(&record).await?; + // The mirror is cloned and (with a store configured) the durable archive is + // already uploaded, so a create_repo failure here would orphan BOTH with no + // repos row: a retry blocks on the existing dest, and a later same-key + // create could download the stale archive. Roll both back best-effort, free + // the lock, then fail the fork. delete_archive is a no-op on a store-less + // node, so this is safe regardless of configuration. + if let Err(e) = state.db.create_repo(&record).await { + if let Err(rm) = std::fs::remove_dir_all(&disk_path) { + tracing::warn!(fork = %fork_name, err = %rm, + "failed to remove fork mirror after a failed row insert"); + } + if let Err(del) = state + .repo_store + .delete_archive(&forker_did, &fork_name) + .await + { + tracing::warn!(fork = %fork_name, err = %del, + "failed to delete fork archive after a failed row insert"); + } + repo_lock.release().await; + return Err(e.into()); + } + + // Row, archive, and on-disk dir now all exist consistently; the race window + // is closed, so the target lock can be released before the best-effort tail. + repo_lock.release().await; // Persist the proof so the fork carries it when it propagates to peers. if let Some(p) = verified_proof { @@ -2597,14 +2880,14 @@ mod tests { ); } - /// Repo creation must be throttled by the per-IP creation limiter BEFORE - /// signature verification — otherwise a DID farm (one throwaway did:key per - /// repo, each carrying a valid but machine-solved iCaptcha proof) walks past - /// the per-DID limiter and floods the network, as in the recurring spam-repo - /// incidents. A 429 (not a 401) on an unsigned request from an exhausted IP - /// proves the IP brake runs outermost, ahead of auth. + /// The receive-pack advertisement 429 must carry a window-derived Retry-After, + /// consistent with the other push 429 sites (U5). Before the fix this site + /// returned a bare 429 with no Retry-After header at all — a client had nothing + /// to back off on. A freshly-filled bucket's oldest + /// entry is ~now, so the advertised delay must be close to the whole window + /// (100s), never missing and never the old constant 60. #[sqlx::test] - async fn repo_creation_is_rate_limited_by_ip(pool: sqlx::PgPool) { + async fn receive_pack_advertisement_429_carries_window_derived_retry_after(pool: sqlx::PgPool) { use axum::body::Body; use axum::extract::ConnectInfo; use axum::http::{Method, Request, StatusCode}; @@ -2613,34 +2896,2341 @@ mod tests { use tower::ServiceExt; let mut state = crate::test_support::test_state(pool).await; - // Tiny limit, keyed on the socket peer (no trusted proxy). - state.create_ip_rate_limiter = - crate::rate_limit::RateLimiter::new(1, Duration::from_secs(60)); + // Budget 1, 100s window, keyed on the socket peer (no trusted proxy). + state.push_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(100)); state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6advretry", "adv", "/tmp/advretry", None, false) + .await + .unwrap(); - let peer: SocketAddr = "203.0.113.77:7000".parse().unwrap(); - // Exhaust this peer's single-request budget up front. - assert!( - state - .create_ip_rate_limiter - .check(&peer.ip().to_string()) - .await - ); + let peer: SocketAddr = "203.0.113.56:6000".parse().unwrap(); + // Fill the single-request budget so the handler's own check rejects. + assert!(state.push_rate_limiter.check(&peer.ip().to_string()).await); let router = crate::server::build_router(state); let mut req = Request::builder() - .method(Method::POST) - .uri("/api/v1/repos") - .header("content-type", "application/json") - .body(Body::from(r#"{"name":"flood","is_public":true}"#)) + .method(Method::GET) + .uri("/z6advretry/adv/info/refs?service=git-receive-pack") + .body(Body::empty()) .unwrap(); req.extensions_mut().insert(ConnectInfo(peer)); - let status = router.oneshot(req).await.unwrap().status(); - assert_eq!( - status, - StatusCode::TOO_MANY_REQUESTS, - "repo creation must be IP-throttled before signature verification" + let resp = router.oneshot(req).await.unwrap(); + assert_eq!(resp.status(), StatusCode::TOO_MANY_REQUESTS); + let retry: u64 = resp + .headers() + .get("retry-after") + .expect("advertisement 429 must carry a retry-after header") + .to_str() + .unwrap() + .parse() + .expect("retry-after must be an integer number of seconds"); + assert!( + (95..=100).contains(&retry), + "freshly-filled bucket must advertise ~window (95..=100s), got {retry} \ + (missing header or constant-60 bug otherwise)" + ); + } + + // U2/R3/AE1: a fully-received push must complete server-side even when the + // client disconnects during the apply. The pack is buffered before the + // handler runs, so the acquire→receive→release core is detached from the + // handler future; dropping that future (the disconnect) must NOT cancel the + // push. A sleeping pre-receive hook creates a deterministic mid-apply window; + // dropping the handler during it kills the git group on the inline (pre-fix) + // code but not on the detached code, so the hook completes and the ref lands + // only after the fix. RED pre-fix (marker + ref absent), GREEN after. + #[sqlx::test] + async fn fully_received_push_completes_after_client_disconnect(pool: sqlx::PgPool) { + use axum::extract::{Path as AxPath, State}; + use axum::Extension; + use std::os::unix::fs::PermissionsExt; + use std::process::{Command, Stdio}; + use std::time::Duration; + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tempfile::TempDir::new().unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing( + repos_dir.path().to_path_buf(), + pool.clone(), + ); + + let owner = "z6u2pushowner"; + let name = "u2push"; + let owner_slug = owner.replace([':', '/'], "_"); + let bare = repos_dir + .path() + .join(&owner_slug) + .join(format!("{name}.git")); + + state + .db + .upsert_mirror_repo(owner, name, &bare.to_string_lossy(), None, false) + .await + .unwrap(); + + // Helper: run git, asserting success. + fn git(args: &[&str], dir: &std::path::Path) -> String { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + + // Build a commit in a scratch working repo. + let work = tempfile::TempDir::new().unwrap(); + git(&["init", "-q", "-b", "main", "."], work.path()); + git(&["config", "user.email", "t@t"], work.path()); + git(&["config", "user.name", "t"], work.path()); + std::fs::write(work.path().join("f.txt"), "hi").unwrap(); + git(&["add", "f.txt"], work.path()); + git(&["commit", "-q", "-m", "c"], work.path()); + let oid = git(&["rev-parse", "HEAD"], work.path()); + + // Server bare repo: has the object (from the clone) but no refs/heads/main, + // so the push is a create satisfiable with an empty pack. + std::fs::create_dir_all(bare.parent().unwrap()).unwrap(); + let out = Command::new("git") + .args([ + "clone", + "--bare", + "-q", + &work.path().to_string_lossy(), + &bare.to_string_lossy(), + ]) + .output() + .unwrap(); + assert!( + out.status.success(), + "clone --bare failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + git( + &[ + "-C", + &bare.to_string_lossy(), + "update-ref", + "-d", + "refs/heads/main", + ], + work.path(), + ); + + // Sleeping pre-receive hook — the mid-apply window; writes a marker at the + // end so we can see the git child ran to completion. + let marker = repos_dir.path().join("hook_ran.marker"); + let hook = bare.join("hooks").join("pre-receive"); + std::fs::create_dir_all(hook.parent().unwrap()).unwrap(); + std::fs::write( + &hook, + format!( + "#!/bin/sh\ncat >/dev/null\nsleep 2\necho done > '{}'\nexit 0\n", + marker.display() + ), + ) + .unwrap(); + std::fs::set_permissions(&hook, std::fs::Permissions::from_mode(0o755)).unwrap(); + + // Receive-pack POST body: create refs/heads/main -> oid, plus an empty pack. + let zero = "0".repeat(40); + let cmd = format!("{zero} {oid} refs/heads/main\0report-status\n"); + let mut body = Vec::new(); + let len = cmd.len() + 4; + body.extend_from_slice(format!("{len:04x}").as_bytes()); + body.extend_from_slice(cmd.as_bytes()); + body.extend_from_slice(b"0000"); + let pack = Command::new("git") + .args([ + "-C", + &bare.to_string_lossy(), + "pack-objects", + "--stdout", + "-q", + ]) + .stdin(Stdio::null()) + .output() + .unwrap(); + assert!( + pack.status.success(), + "pack-objects failed: {}", + String::from_utf8_lossy(&pack.stderr) + ); + body.extend_from_slice(&pack.stdout); + let body = Bytes::from(body); + + // Drive the handler, then DROP it mid-hook (the "client disconnect"). + let handler = super::git_receive_pack( + State(state.clone()), + AxPath((owner.to_string(), format!("{name}.git"))), + Extension(crate::auth::AuthenticatedDid(owner.to_string())), + body, + ); + let _ = tokio::time::timeout(Duration::from_millis(800), handler).await; + + // Wait past the hook's sleep so a surviving (detached) push can finish. + tokio::time::sleep(Duration::from_millis(3000)).await; + + let ref_present = bare.join("refs/heads/main").exists() + || std::fs::read_to_string(bare.join("packed-refs")) + .unwrap_or_default() + .contains("refs/heads/main"); + assert!( + marker.exists(), + "pre-receive hook must run to completion despite the client disconnect (detached push)" + ); + assert!( + ref_present, + "refs/heads/main must be created after a fully-received push despite client disconnect" + ); + } + + // U1/R1: a fully-received push must produce its metadata + fan-out TAIL even + // when the client disconnects mid-apply. The pre-fix handler detaches only + // acquire→receive→release; the whole success tail (touch_repo, record_push, + // certs, webhooks, replication, ref broadcast) runs inline in the cancellable + // request future, so a disconnect after receive-pack succeeds commits the git + // refs (in the surviving task) but drops the tail — a split-brain: committed + // git, no push record. Here we drop the handler mid-apply (sleeping hook), let + // the surviving task finish, and assert the push record landed. RED pre-fix + // (push_events row absent), GREEN after (tail folded into the detached task). + #[sqlx::test] + async fn push_metadata_tail_completes_after_client_disconnect(pool: sqlx::PgPool) { + use axum::extract::{Path as AxPath, State}; + use axum::Extension; + use std::os::unix::fs::PermissionsExt; + use std::process::{Command, Stdio}; + use std::time::Duration; + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tempfile::TempDir::new().unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing( + repos_dir.path().to_path_buf(), + pool.clone(), + ); + + let owner = "z6u1tailowner"; + let name = "u1tail"; + let owner_slug = owner.replace([':', '/'], "_"); + let bare = repos_dir + .path() + .join(&owner_slug) + .join(format!("{name}.git")); + + state + .db + .upsert_mirror_repo(owner, name, &bare.to_string_lossy(), None, false) + .await + .unwrap(); + + fn git(args: &[&str], dir: &std::path::Path) -> String { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + + // Build a commit in a scratch working repo. + let work = tempfile::TempDir::new().unwrap(); + git(&["init", "-q", "-b", "main", "."], work.path()); + git(&["config", "user.email", "t@t"], work.path()); + git(&["config", "user.name", "t"], work.path()); + std::fs::write(work.path().join("f.txt"), "hi").unwrap(); + git(&["add", "f.txt"], work.path()); + git(&["commit", "-q", "-m", "c"], work.path()); + let oid = git(&["rev-parse", "HEAD"], work.path()); + + std::fs::create_dir_all(bare.parent().unwrap()).unwrap(); + let out = Command::new("git") + .args([ + "clone", + "--bare", + "-q", + &work.path().to_string_lossy(), + &bare.to_string_lossy(), + ]) + .output() + .unwrap(); + assert!( + out.status.success(), + "clone --bare failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + git( + &[ + "-C", + &bare.to_string_lossy(), + "update-ref", + "-d", + "refs/heads/main", + ], + work.path(), + ); + + // Sleeping pre-receive hook: the deterministic mid-apply window we drop in. + let hook = bare.join("hooks").join("pre-receive"); + std::fs::create_dir_all(hook.parent().unwrap()).unwrap(); + std::fs::write(&hook, "#!/bin/sh\ncat >/dev/null\nsleep 2\nexit 0\n").unwrap(); + std::fs::set_permissions(&hook, std::fs::Permissions::from_mode(0o755)).unwrap(); + + // Receive-pack POST body: create refs/heads/main -> oid, plus a real pack. + let zero = "0".repeat(40); + let cmd = format!("{zero} {oid} refs/heads/main\0report-status\n"); + let mut body = Vec::new(); + let len = cmd.len() + 4; + body.extend_from_slice(format!("{len:04x}").as_bytes()); + body.extend_from_slice(cmd.as_bytes()); + body.extend_from_slice(b"0000"); + let pack = Command::new("git") + .args([ + "-C", + &bare.to_string_lossy(), + "pack-objects", + "--stdout", + "-q", + ]) + .stdin(Stdio::null()) + .output() + .unwrap(); + assert!( + pack.status.success(), + "pack-objects failed: {}", + String::from_utf8_lossy(&pack.stderr) + ); + body.extend_from_slice(&pack.stdout); + let body = Bytes::from(body); + + // Drive the handler, then DROP it mid-hook (the "client disconnect"). + let handler = super::git_receive_pack( + State(state.clone()), + AxPath((owner.to_string(), format!("{name}.git"))), + Extension(crate::auth::AuthenticatedDid(owner.to_string())), + body, + ); + let _ = tokio::time::timeout(Duration::from_millis(800), handler).await; + + // Poll for the tail's push record past the hook's sleep. The pusher DID is + // the repo owner, so the push_events count for `owner` becomes 1 once the + // surviving task runs the tail. + let mut pushes = 0i64; + for _ in 0..40 { + pushes = state.db.get_push_count(owner).await.unwrap(); + if pushes >= 1 { + break; + } + tokio::time::sleep(Duration::from_millis(150)).await; + } + assert_eq!( + pushes, 1, + "the push-metadata tail (record_push) must land after a fully-received \ + push despite the client disconnect — it must run in the detached task, \ + not the cancelled request future" + ); + } + + // U1/R1 no-regression: a CONNECTED push (handler awaited to completion) must + // still get report-status back over the oneshot AND run the full tail. Guards + // the two ways the refactor could regress a connected client: the response + // coupling to tail completion (a latency regression), and the oneshot never + // delivering (client would see a spurious 500). + #[sqlx::test] + async fn connected_push_returns_report_status_and_runs_tail(pool: sqlx::PgPool) { + use axum::extract::{Path as AxPath, State}; + use axum::Extension; + use std::process::{Command, Stdio}; + use std::time::Duration; + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tempfile::TempDir::new().unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing( + repos_dir.path().to_path_buf(), + pool.clone(), + ); + + let owner = "z6u1connowner"; + let name = "u1conn"; + let owner_slug = owner.replace([':', '/'], "_"); + let bare = repos_dir + .path() + .join(&owner_slug) + .join(format!("{name}.git")); + + state + .db + .upsert_mirror_repo(owner, name, &bare.to_string_lossy(), None, false) + .await + .unwrap(); + + fn git(args: &[&str], dir: &std::path::Path) -> String { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + + let work = tempfile::TempDir::new().unwrap(); + git(&["init", "-q", "-b", "main", "."], work.path()); + git(&["config", "user.email", "t@t"], work.path()); + git(&["config", "user.name", "t"], work.path()); + std::fs::write(work.path().join("f.txt"), "hi").unwrap(); + git(&["add", "f.txt"], work.path()); + git(&["commit", "-q", "-m", "c"], work.path()); + let oid = git(&["rev-parse", "HEAD"], work.path()); + + std::fs::create_dir_all(bare.parent().unwrap()).unwrap(); + let out = Command::new("git") + .args([ + "clone", + "--bare", + "-q", + &work.path().to_string_lossy(), + &bare.to_string_lossy(), + ]) + .output() + .unwrap(); + assert!(out.status.success(), "clone --bare failed"); + git( + &[ + "-C", + &bare.to_string_lossy(), + "update-ref", + "-d", + "refs/heads/main", + ], + work.path(), + ); + + // No hook: the push applies immediately, so the handler returns promptly. + let zero = "0".repeat(40); + let cmd = format!("{zero} {oid} refs/heads/main\0report-status\n"); + let mut body = Vec::new(); + let len = cmd.len() + 4; + body.extend_from_slice(format!("{len:04x}").as_bytes()); + body.extend_from_slice(cmd.as_bytes()); + body.extend_from_slice(b"0000"); + let pack = Command::new("git") + .args([ + "-C", + &bare.to_string_lossy(), + "pack-objects", + "--stdout", + "-q", + ]) + .stdin(Stdio::null()) + .output() + .unwrap(); + assert!(pack.status.success(), "pack-objects failed"); + body.extend_from_slice(&pack.stdout); + let body = Bytes::from(body); + + // Await the handler to completion (connected client) and assert it returned + // report-status (200) over the oneshot — not a 500 from a dropped sender. + let resp = super::git_receive_pack( + State(state.clone()), + AxPath((owner.to_string(), format!("{name}.git"))), + Extension(crate::auth::AuthenticatedDid(owner.to_string())), + body, + ) + .await + .expect("connected push must return report-status"); + assert_eq!(resp.status(), axum::http::StatusCode::OK); + + // And the tail runs (detached task): the push record lands. + let mut pushes = 0i64; + for _ in 0..40 { + pushes = state.db.get_push_count(owner).await.unwrap(); + if pushes >= 1 { + break; + } + tokio::time::sleep(Duration::from_millis(150)).await; + } + assert_eq!(pushes, 1, "the tail must run on a connected push too"); + } + + // P1 data-loss regression: when the durable (Tigris) upload in release() + // times out, the refs are applied on local disk but never persisted to + // object storage. The pre-fix handler still reported 200 to the client AND + // ran the success tail, so the client trusted a push that the NEXT + // acquire_write would revert from the stale pre-push archive — silent data + // loss. The fix surfaces a failed/timed-out upload as a FAILED push (5xx) so + // the idempotent client re-pushes, and skips the whole success tail. Here we + // stall the upload past a tiny release timeout, drive a fully-applied push, + // and assert (a) the client gets a non-2xx and (b) the tail did NOT run + // (no push record). RED pre-fix: 200 + push record present. + #[sqlx::test] + async fn durable_upload_timeout_fails_push_and_skips_tail(pool: sqlx::PgPool) { + use axum::extract::{Path as AxPath, State}; + use axum::response::IntoResponse; + use axum::Extension; + use std::path::Path as StdPath; + use std::process::{Command, Stdio}; + use std::time::Duration; + + // Object store whose upload() parks forever: the release() timeout is the + // only thing that unblocks it. `exists()` is false so acquire_write never + // downloads over the fresh local bare repo. + struct StallStore; + #[async_trait::async_trait] + impl crate::git::tigris::ObjectStore for StallStore { + async fn exists(&self, _o: &str, _r: &str) -> anyhow::Result { + Ok(false) + } + async fn upload(&self, _o: &str, _r: &str, _p: &StdPath) -> anyhow::Result<()> { + std::future::pending::<()>().await; + Ok(()) + } + async fn download(&self, _o: &str, _r: &str, _p: &StdPath) -> anyhow::Result<()> { + Ok(()) + } + async fn delete(&self, _o: &str, _r: &str) -> anyhow::Result<()> { + Ok(()) + } + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tempfile::TempDir::new().unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::new( + repos_dir.path().to_path_buf(), + Some(std::sync::Arc::new(StallStore)), + pool.clone(), + ) + .with_release_upload_timeout(Duration::from_millis(200)); + + let owner = "z6durablefailowner"; + let name = "durablefail"; + let owner_slug = owner.replace([':', '/'], "_"); + let bare = repos_dir + .path() + .join(&owner_slug) + .join(format!("{name}.git")); + + state + .db + .upsert_mirror_repo(owner, name, &bare.to_string_lossy(), None, false) + .await + .unwrap(); + + fn git(args: &[&str], dir: &std::path::Path) -> String { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + + let work = tempfile::TempDir::new().unwrap(); + git(&["init", "-q", "-b", "main", "."], work.path()); + git(&["config", "user.email", "t@t"], work.path()); + git(&["config", "user.name", "t"], work.path()); + std::fs::write(work.path().join("f.txt"), "hi").unwrap(); + git(&["add", "f.txt"], work.path()); + git(&["commit", "-q", "-m", "c"], work.path()); + let oid = git(&["rev-parse", "HEAD"], work.path()); + + std::fs::create_dir_all(bare.parent().unwrap()).unwrap(); + let out = Command::new("git") + .args([ + "clone", + "--bare", + "-q", + &work.path().to_string_lossy(), + &bare.to_string_lossy(), + ]) + .output() + .unwrap(); + assert!(out.status.success(), "clone --bare failed"); + git( + &[ + "-C", + &bare.to_string_lossy(), + "update-ref", + "-d", + "refs/heads/main", + ], + work.path(), + ); + + // No hook: the push applies immediately; the stall is entirely in release(). + let zero = "0".repeat(40); + let cmd = format!("{zero} {oid} refs/heads/main\0report-status\n"); + let mut body = Vec::new(); + let len = cmd.len() + 4; + body.extend_from_slice(format!("{len:04x}").as_bytes()); + body.extend_from_slice(cmd.as_bytes()); + body.extend_from_slice(b"0000"); + let pack = Command::new("git") + .args([ + "-C", + &bare.to_string_lossy(), + "pack-objects", + "--stdout", + "-q", + ]) + .stdin(Stdio::null()) + .output() + .unwrap(); + assert!(pack.status.success(), "pack-objects failed"); + body.extend_from_slice(&pack.stdout); + let body = Bytes::from(body); + + // Await the handler: acquire → receive-pack (applies) → release (upload + // stalls, timing out after 200ms). The client MUST see a failure, not 200. + let resp = super::git_receive_pack( + State(state.clone()), + AxPath((owner.to_string(), format!("{name}.git"))), + Extension(crate::auth::AuthenticatedDid(owner.to_string())), + body, + ) + .await; + let status = match resp { + Ok(r) => r.status(), + Err(e) => e.into_response().status(), + }; + assert!( + status.is_server_error(), + "a timed-out durable upload must fail the push (5xx) so the client \ + retries — got {status}, which the client trusts as a landed push \ + that a later acquire_write would silently revert" + ); + + // And the success tail must NOT have run: no push record for a push whose + // durable copy never landed. Give the (detached) tail time to run if it + // were going to, then assert it did not. + tokio::time::sleep(Duration::from_millis(400)).await; + let pushes = state.db.get_push_count(owner).await.unwrap(); + assert_eq!( + pushes, 0, + "the success tail (record_push) must NOT run when the durable upload \ + failed — the push was not durably accepted" + ); + } + + // Rollback completeness on the receive-pack path: a failed durable upload + // must restore the EXACT pre-push snapshot (a ref the push updated is + // rewound AND a ref the push created is deleted) while the advisory lock + // is held, so the local fast path never serves refs that never landed + // durably. RED with the cleanup closure reverted to `|_| {}` (or with + // restore_refs not deleting created refs): main stays at the pushed tip + // and/or the created ref survives. + #[sqlx::test] + async fn push_failed_upload_rolls_back_created_and_updated_refs(pool: sqlx::PgPool) { + use axum::extract::{Path as AxPath, State}; + use axum::response::IntoResponse; + use axum::Extension; + use std::path::Path as StdPath; + use std::process::{Command, Stdio}; + use std::time::Duration; + + struct StallStore; + #[async_trait::async_trait] + impl crate::git::tigris::ObjectStore for StallStore { + async fn exists(&self, _o: &str, _r: &str) -> anyhow::Result { + Ok(false) + } + async fn upload(&self, _o: &str, _r: &str, _p: &StdPath) -> anyhow::Result<()> { + std::future::pending::<()>().await; + Ok(()) + } + async fn download(&self, _o: &str, _r: &str, _p: &StdPath) -> anyhow::Result<()> { + Ok(()) + } + async fn delete(&self, _o: &str, _r: &str) -> anyhow::Result<()> { + Ok(()) + } + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tempfile::TempDir::new().unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::new( + repos_dir.path().to_path_buf(), + Some(std::sync::Arc::new(StallStore)), + pool.clone(), + ) + .with_release_upload_timeout(Duration::from_millis(200)); + + let owner = "z6refrollbackowner"; + let name = "refrollback"; + let owner_slug = owner.replace([':', '/'], "_"); + let bare = repos_dir + .path() + .join(&owner_slug) + .join(format!("{name}.git")); + + state + .db + .upsert_mirror_repo(owner, name, &bare.to_string_lossy(), None, false) + .await + .unwrap(); + + fn git(args: &[&str], dir: &std::path::Path) -> String { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + + // Two commits in a scratch repo: the bare repo starts at c1 and the + // push advances main to c2 AND creates a second ref at c2. + let work = tempfile::TempDir::new().unwrap(); + git(&["init", "-q", "-b", "main", "."], work.path()); + git(&["config", "user.email", "t@t"], work.path()); + git(&["config", "user.name", "t"], work.path()); + std::fs::write(work.path().join("f.txt"), "one").unwrap(); + git(&["add", "f.txt"], work.path()); + git(&["commit", "-q", "-m", "c1"], work.path()); + let old_oid = git(&["rev-parse", "HEAD"], work.path()); + std::fs::write(work.path().join("f.txt"), "two").unwrap(); + git(&["add", "f.txt"], work.path()); + git(&["commit", "-q", "-m", "c2"], work.path()); + let new_oid = git(&["rev-parse", "HEAD"], work.path()); + + // Bare clone (has both commits' objects), then rewind main to c1 so the + // push is a genuine update satisfiable with an empty pack. + std::fs::create_dir_all(bare.parent().unwrap()).unwrap(); + let out = Command::new("git") + .args([ + "clone", + "--bare", + "-q", + &work.path().to_string_lossy(), + &bare.to_string_lossy(), + ]) + .output() + .unwrap(); + assert!(out.status.success(), "clone --bare failed"); + git( + &[ + "-C", + &bare.to_string_lossy(), + "update-ref", + "refs/heads/main", + &old_oid, + ], + work.path(), + ); + + let snapshot = store::list_refs(&bare).unwrap(); + assert_eq!( + snapshot, + vec![("refs/heads/main".to_string(), old_oid.clone())], + "pre-push snapshot must be exactly main at c1" + ); + + // Two commands: update main c1 -> c2, create refs/heads/feature at c2. + let zero = "0".repeat(40); + let mut body = Vec::new(); + let cmd1 = format!("{old_oid} {new_oid} refs/heads/main\0report-status\n"); + body.extend_from_slice(format!("{:04x}", cmd1.len() + 4).as_bytes()); + body.extend_from_slice(cmd1.as_bytes()); + let cmd2 = format!("{zero} {new_oid} refs/heads/feature\n"); + body.extend_from_slice(format!("{:04x}", cmd2.len() + 4).as_bytes()); + body.extend_from_slice(cmd2.as_bytes()); + body.extend_from_slice(b"0000"); + let pack = Command::new("git") + .args([ + "-C", + &bare.to_string_lossy(), + "pack-objects", + "--stdout", + "-q", + ]) + .stdin(Stdio::null()) + .output() + .unwrap(); + assert!(pack.status.success(), "pack-objects failed"); + body.extend_from_slice(&pack.stdout); + let body = Bytes::from(body); + + let resp = super::git_receive_pack( + State(state.clone()), + AxPath((owner.to_string(), format!("{name}.git"))), + Extension(crate::auth::AuthenticatedDid(owner.to_string())), + body, + ) + .await; + let status = match resp { + Ok(r) => r.status(), + Err(e) => e.into_response().status(), + }; + assert!( + status.is_server_error(), + "a timed-out durable upload must fail the push (5xx), got {status}" + ); + + // The refs must equal the pre-push snapshot EXACTLY: the created ref is + // gone and the updated ref is rewound. + let after = store::list_refs(&bare).unwrap(); + assert_eq!( + after, snapshot, + "a failed durable upload must restore the exact pre-push snapshot \ + (created ref deleted, updated ref rewound)" + ); + } + + // Fail-closed fence on the pre-push ref snapshot, driven through the real + // handler: when `store::list_refs` on the acquired disk path fails, the + // handler must refuse the push (Internal, 5xx) BEFORE running receive-pack, + // because without a snapshot a failed durable upload has no restore plan. + // The repo on disk is a real bare repo whose config declares + // repositoryformatversion=999, so every git invocation in it fails + // deterministically at repo-setup time (`for-each-ref` included) while the + // directory itself is untouched valid state. No object store is configured, + // so acquire_write's local fast path uses the existing directory as-is and + // never repairs or re-downloads it. + // + // Load-bearing (RED) checks: remove the fence at the `store::list_refs` + // match in `git_receive_pack` (e.g. fall back to `.ok()` and proceed) and + // the failure comes from receive-pack instead, so the "cannot snapshot + // pre-push refs" assertion on the error text fails. The directory-listing + // comparison pins that nothing mutated the repo before the refusal. + #[sqlx::test] + async fn push_fails_closed_when_ref_snapshot_unavailable(pool: sqlx::PgPool) { + use axum::extract::{Path as AxPath, State}; + use axum::response::IntoResponse; + use std::process::{Command, Stdio}; + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tempfile::TempDir::new().unwrap(); + // No object store: acquire_write takes the local fast path and hands the + // on-disk directory to the handler exactly as seeded below. + state.repo_store = crate::git::repo_store::RepoStore::new( + repos_dir.path().to_path_buf(), + None, + pool.clone(), + ); + + let owner = "z6snapshotfenceowner"; + let name = "snapfence"; + let owner_slug = owner.replace([':', '/'], "_"); + let bare = repos_dir + .path() + .join(&owner_slug) + .join(format!("{name}.git")); + + state + .db + .upsert_mirror_repo(owner, name, &bare.to_string_lossy(), None, false) + .await + .unwrap(); + + fn git(args: &[&str], dir: &std::path::Path) -> String { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + + // A scratch repo provides a real oid and a well-formed (empty) pack so + // the request body is a genuine push, not garbage the parser rejects. + let work = tempfile::TempDir::new().unwrap(); + git(&["init", "-q", "-b", "main", "."], work.path()); + git(&["config", "user.email", "t@t"], work.path()); + git(&["config", "user.name", "t"], work.path()); + std::fs::write(work.path().join("f.txt"), "hi").unwrap(); + git(&["add", "f.txt"], work.path()); + git(&["commit", "-q", "-m", "c"], work.path()); + let oid = git(&["rev-parse", "HEAD"], work.path()); + + // Seed the target: a REAL bare repo, then declare an unsupported + // repository format version. Discovery still finds the repo (HEAD, + // objects/, refs/ all present) but every git command in it dies with + // "expected git repo version <= 1": the deterministic corruption that + // makes `list_refs` fail without any racy filesystem state. + std::fs::create_dir_all(bare.parent().unwrap()).unwrap(); + let out = Command::new("git") + .args(["init", "--bare", "-q", &bare.to_string_lossy()]) + .output() + .unwrap(); + assert!(out.status.success(), "git init --bare failed"); + std::fs::write( + bare.join("config"), + "[core]\n\trepositoryformatversion = 999\n\tbare = true\n", + ) + .unwrap(); + assert!( + store::list_refs(&bare).is_err(), + "precondition: list_refs must fail on the corrupted repo" + ); + + // Recursive sorted listing of the repo dir, to pin "nothing modified". + fn listing(root: &std::path::Path) -> Vec { + fn walk(root: &std::path::Path, dir: &std::path::Path, out: &mut Vec) { + for entry in std::fs::read_dir(dir).unwrap() { + let p = entry.unwrap().path(); + out.push(p.strip_prefix(root).unwrap().to_string_lossy().into_owned()); + if p.is_dir() { + walk(root, &p, out); + } + } + } + let mut out = Vec::new(); + walk(root, root, &mut out); + out.sort(); + out + } + let before = listing(&bare); + + // Minimal push body: create refs/heads/main at the scratch oid plus an + // empty pack (same shape as the rollback test above). + let zero = "0".repeat(40); + let cmd = format!("{zero} {oid} refs/heads/main\0report-status\n"); + let mut body = Vec::new(); + body.extend_from_slice(format!("{:04x}", cmd.len() + 4).as_bytes()); + body.extend_from_slice(cmd.as_bytes()); + body.extend_from_slice(b"0000"); + let pack = Command::new("git") + .args([ + "-C", + &work.path().to_string_lossy(), + "pack-objects", + "--stdout", + "-q", + ]) + .stdin(Stdio::null()) + .output() + .unwrap(); + assert!(pack.status.success(), "pack-objects failed"); + body.extend_from_slice(&pack.stdout); + let body = Bytes::from(body); + + let resp = super::git_receive_pack( + State(state.clone()), + AxPath((owner.to_string(), format!("{name}.git"))), + Extension(crate::auth::AuthenticatedDid(owner.to_string())), + body, + ) + .await; + let err = match resp { + Ok(r) => panic!( + "push must fail closed when the ref snapshot is unavailable, got {}", + r.status() + ), + Err(e) => e, + }; + let msg = err.to_string(); + let status = err.into_response().status(); + assert!( + status.is_server_error(), + "snapshot failure must surface as a 5xx, got {status}" + ); + // This is the fence-specific assert: without the fail-closed branch the + // failure comes from receive-pack ("git error: ...") instead. + assert!( + msg.contains("cannot snapshot pre-push refs"), + "the refusal must come from the snapshot fence, before receive-pack; got: {msg}" + ); + + // Nothing ran against the repo: the directory contents are unchanged + // (no new refs, no objects, no receive-pack side effects). + let after = listing(&bare); + assert_eq!( + after, before, + "the corrupted repo must not be modified by a refused push" + ); + } + + // A bare repo at HEAD with one commit on main, for the rollback-decision + // unit tests below. Returns the tempdirs (kept alive by the caller), the + // bare path, and main's oid. + fn seeded_bare_repo() -> ( + tempfile::TempDir, + tempfile::TempDir, + std::path::PathBuf, + String, + ) { + use std::process::Command; + + fn git(args: &[&str], dir: &std::path::Path) -> String { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + + let work = tempfile::TempDir::new().unwrap(); + git(&["init", "-q", "-b", "main", "."], work.path()); + git(&["config", "user.email", "t@t"], work.path()); + git(&["config", "user.name", "t"], work.path()); + std::fs::write(work.path().join("f.txt"), "hi").unwrap(); + git(&["add", "f.txt"], work.path()); + git(&["commit", "-q", "-m", "c"], work.path()); + let oid = git(&["rev-parse", "HEAD"], work.path()); + + let dir = tempfile::TempDir::new().unwrap(); + let bare = dir.path().join("repo.git"); + let out = Command::new("git") + .args([ + "clone", + "--bare", + "-q", + &work.path().to_string_lossy(), + &bare.to_string_lossy(), + ]) + .output() + .unwrap(); + assert!(out.status.success(), "clone --bare failed"); + (work, dir, bare, oid) + } + + // The None arm of the rollback decision: a failed pre-push listing means + // "snapshot unavailable", and the rollback must be SKIPPED: the repo's + // existing refs survive. RED on the pre-fix shape (unwrap_or_default + an + // unconditional restore_refs), which reads the error as an empty snapshot + // and deletes every ref. + #[test] + fn ref_rollback_skips_when_snapshot_unavailable() { + let (_work, _dir, bare, oid) = seeded_bare_repo(); + + super::rollback_push_refs(&bare, "r", &None); + + assert_eq!( + store::list_refs(&bare).unwrap(), + vec![("refs/heads/main".to_string(), oid)], + "a None snapshot (listing failed) must skip the rollback, not \ + mass-delete the repo's refs" + ); + } + + // The must-keep negative of the fix: a genuinely EMPTY repo snapshots as + // Some(vec![]) and still rolls back, deleting the refs the failed push + // created. Guards against the `.ok()` change accidentally widening the + // skip to empty snapshots. + #[test] + fn ref_rollback_empty_snapshot_still_deletes_created_refs() { + let (_work, _dir, bare, _oid) = seeded_bare_repo(); + + super::rollback_push_refs(&bare, "r", &Some(vec![])); + + assert_eq!( + store::list_refs(&bare).unwrap(), + Vec::<(String, String)>::new(), + "an empty (but present) snapshot must still roll back: the ref the \ + push created has to be deleted" + ); + } + + /// Repo creation must be throttled by the per-IP creation limiter BEFORE + /// signature verification — otherwise a DID farm (one throwaway did:key per + /// repo, each carrying a valid but machine-solved iCaptcha proof) walks past + /// the per-DID limiter and floods the network, as in the recurring spam-repo + /// incidents. A 429 (not a 401) on an unsigned request from an exhausted IP + /// proves the IP brake runs outermost, ahead of auth. + #[sqlx::test] + async fn repo_creation_is_rate_limited_by_ip(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::time::Duration; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + // Tiny limit, keyed on the socket peer (no trusted proxy). + state.create_ip_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(60)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let peer: SocketAddr = "203.0.113.77:7000".parse().unwrap(); + // Exhaust this peer's single-request budget up front. + assert!( + state + .create_ip_rate_limiter + .check(&peer.ip().to_string()) + .await + ); + + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::POST) + .uri("/api/v1/repos") + .header("content-type", "application/json") + .body(Body::from(r#"{"name":"flood","is_public":true}"#)) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + + let status = router.oneshot(req).await.unwrap().status(); + assert_eq!( + status, + StatusCode::TOO_MANY_REQUESTS, + "repo creation must be IP-throttled before signature verification" + ); + } + + // Shared request driver for the per-IP write-brake tests below: build a + // request with the given method/uri/headers/body, attach the socket peer as + // ConnectInfo (what the IP limiter keys on), send it through the router, and + // return the status. Mirrors post_from/post_with in rate_limit.rs. + async fn send_from( + router: &axum::Router, + method: axum::http::Method, + uri: &str, + headers: &[(&str, &str)], + body: axum::body::Body, + peer: std::net::SocketAddr, + ) -> axum::http::StatusCode { + use tower::ServiceExt; + let mut b = axum::http::Request::builder().method(method).uri(uri); + for (k, v) in headers { + b = b.header(*k, *v); + } + let mut req = b.body(body).unwrap(); + req.extensions_mut() + .insert(axum::extract::ConnectInfo(peer)); + router.clone().oneshot(req).await.unwrap().status() + } + + #[sqlx::test] + async fn write_route_is_rate_limited_by_ip(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::http::{Method, StatusCode}; + use std::net::SocketAddr; + use std::time::Duration; + + let mut state = crate::test_support::test_state(pool).await; + // Tiny write bucket, keyed on the socket peer (no trusted proxy). + state.write_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(60)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let peer: SocketAddr = "203.0.113.88:7000".parse().unwrap(); + // Exhaust this peer's single-request write budget up front. + assert!(state.write_rate_limiter.check(&peer.ip().to_string()).await); + + let router = crate::server::build_router(state); + // A write_routes sink (star). The IP brake is outermost, so the 429 + // fires before auth/handler — the path only needs to match. + let status = send_from( + &router, + Method::PUT, + "/api/v1/repos/someowner/somerepo/star", + &[], + Body::empty(), + peer, + ) + .await; + assert_eq!( + status, + StatusCode::TOO_MANY_REQUESTS, + "a write_routes sink must be IP-throttled before signature verification" + ); + } + + // KTD-1: the write bucket is separate from the creation bucket, so a write + // flood must not consume the creation budget (and vice versa). + #[sqlx::test] + async fn write_flood_does_not_drain_creation_budget(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::http::{Method, StatusCode}; + use std::net::SocketAddr; + use std::time::Duration; + + let mut state = crate::test_support::test_state(pool).await; + // Exhaust the write bucket for this peer; leave the creation bucket ample. + state.write_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(60)); + state.create_ip_rate_limiter = + crate::rate_limit::RateLimiter::new(1000, Duration::from_secs(60)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let peer: SocketAddr = "203.0.113.99:7000".parse().unwrap(); + assert!(state.write_rate_limiter.check(&peer.ip().to_string()).await); + + let router = crate::server::build_router(state); + + // Anchor the test: prove the write bucket is genuinely drained at the + // router (a write sink from this peer 429s) so the creation assertion + // below cannot pass vacuously on some unrelated non-429 status. + assert_eq!( + send_from( + &router, + Method::PUT, + "/api/v1/repos/someowner/somerepo/star", + &[], + Body::empty(), + peer, + ) + .await, + StatusCode::TOO_MANY_REQUESTS, + "write bucket must be drained for this peer (test precondition)" + ); + + // Creation from the same peer must NOT be 429 — its bucket is untouched. + // (It fails later on missing signature; the point is it is not throttled.) + let status = send_from( + &router, + Method::POST, + "/api/v1/repos", + &[("content-type", "application/json")], + Body::from(r#"{"name":"legit","is_public":true}"#), + peer, + ) + .await; + assert_ne!( + status, + StatusCode::TOO_MANY_REQUESTS, + "an exhausted write bucket must not throttle repo creation (separate buckets)" + ); + } + + // KTD-5: /graphql POST (the MutationRoot surface) draws from the write bucket. + #[sqlx::test] + async fn graphql_post_is_rate_limited_by_ip(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::http::{Method, StatusCode}; + use std::net::SocketAddr; + use std::time::Duration; + + let mut state = crate::test_support::test_state(pool).await; + state.write_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(60)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let peer: SocketAddr = "203.0.113.111:7000".parse().unwrap(); + assert!(state.write_rate_limiter.check(&peer.ip().to_string()).await); + + let router = crate::server::build_router(state); + let status = send_from( + &router, + Method::POST, + "/graphql", + &[("content-type", "application/json")], + Body::from(r#"{"query":"{ __typename }"}"#), + peer, + ) + .await; + assert_eq!( + status, + StatusCode::TOO_MANY_REQUESTS, + "/graphql must be IP-throttled by the write brake" + ); + } + + // Representative REST write group (issue comment) — same attachment as the + // task/bounty/profile groups. + #[sqlx::test] + async fn issue_comment_is_rate_limited_by_ip(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::http::{Method, StatusCode}; + use std::net::SocketAddr; + use std::time::Duration; + + let mut state = crate::test_support::test_state(pool).await; + state.write_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(60)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let peer: SocketAddr = "203.0.113.122:7000".parse().unwrap(); + assert!(state.write_rate_limiter.check(&peer.ip().to_string()).await); + + let router = crate::server::build_router(state); + let status = send_from( + &router, + Method::POST, + "/api/v1/repos/someowner/somerepo/issues/1/comments", + &[("content-type", "application/json")], + Body::from(r#"{"body":"flood"}"#), + peer, + ) + .await; + assert_eq!( + status, + StatusCode::TOO_MANY_REQUESTS, + "issue-write routes must be IP-throttled by the write brake" + ); + } + + // Adoption floor: an under-limit write must NOT be throttled. Guards against + // an off-by-one that braked the first request (invisible to the 429 tests, + // which all pre-exhaust the bucket). + #[sqlx::test] + async fn under_limit_write_is_not_throttled(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::http::{Method, StatusCode}; + use std::net::SocketAddr; + use std::time::Duration; + + let mut state = crate::test_support::test_state(pool).await; + // Ample budget; bucket NOT exhausted. + state.write_rate_limiter = + crate::rate_limit::RateLimiter::new(100, Duration::from_secs(60)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let peer: SocketAddr = "203.0.113.150:7000".parse().unwrap(); + + let router = crate::server::build_router(state); + assert_ne!( + send_from( + &router, + Method::PUT, + "/api/v1/repos/someowner/somerepo/star", + &[], + Body::empty(), + peer, + ) + .await, + StatusCode::TOO_MANY_REQUESTS, + "an under-limit write must pass the brake, not be 429'd" + ); + } + + // GITLAWB_WRITE_RATE_LIMIT=0 disables the brake end-to-end: no write is 429'd + // however many arrive from one IP. + #[sqlx::test] + async fn write_rate_limit_zero_disables_the_brake(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::http::{Method, StatusCode}; + use std::net::SocketAddr; + use std::time::Duration; + + let mut state = crate::test_support::test_state(pool).await; + state.write_rate_limiter = crate::rate_limit::RateLimiter::new(0, Duration::from_secs(60)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let peer: SocketAddr = "203.0.113.151:7000".parse().unwrap(); + + let router = crate::server::build_router(state); + for _ in 0..5 { + assert_ne!( + send_from( + &router, + Method::PUT, + "/api/v1/repos/someowner/somerepo/star", + &[], + Body::empty(), + peer, + ) + .await, + StatusCode::TOO_MANY_REQUESTS, + "a 0 write limit must disable the brake" + ); + } + } + + // The task/bounty/profile write groups share the write brake (same + // attachment as write_routes); prove each 429s at the route level. + #[sqlx::test] + async fn task_bounty_profile_writes_are_rate_limited(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::http::{Method, StatusCode}; + use std::net::SocketAddr; + use std::time::Duration; + + let mut state = crate::test_support::test_state(pool).await; + state.write_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(60)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let peer: SocketAddr = "203.0.113.152:7000".parse().unwrap(); + assert!(state.write_rate_limiter.check(&peer.ip().to_string()).await); + + let router = crate::server::build_router(state); + for (method, uri) in [ + (Method::POST, "/api/v1/tasks"), + (Method::POST, "/api/v1/repos/o/r/bounties"), + (Method::PUT, "/api/v1/profile"), + ] { + assert_eq!( + send_from( + &router, + method, + uri, + &[("content-type", "application/json")], + Body::from("{}"), + peer, + ) + .await, + StatusCode::TOO_MANY_REQUESTS, + "write group {uri} must be IP-throttled by the write brake" + ); + } + } + + // /graphql/ws (subscriptions) is deliberately mounted AFTER the write brake + // layer, so it must stay unbraked even when the write bucket is exhausted. + #[sqlx::test] + async fn graphql_ws_is_not_braked(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::http::{Method, StatusCode}; + use std::net::SocketAddr; + use std::time::Duration; + + let mut state = crate::test_support::test_state(pool).await; + state.write_rate_limiter = crate::rate_limit::RateLimiter::new(1, Duration::from_secs(60)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let peer: SocketAddr = "203.0.113.153:7000".parse().unwrap(); + assert!(state.write_rate_limiter.check(&peer.ip().to_string()).await); + + let router = crate::server::build_router(state); + // Not a real ws upgrade, so the subscription service rejects it with some + // non-429 status; the point is the write brake never sees it. + assert_ne!( + send_from( + &router, + Method::GET, + "/graphql/ws", + &[], + Body::empty(), + peer + ) + .await, + StatusCode::TOO_MANY_REQUESTS, + "/graphql/ws must not be behind the write brake" + ); + } + + // Adoption floor, per group: with an un-exhausted bucket, a write to EVERY + // braked group passes the brake (reaches auth/handler), not 429. Guards each + // group's grant path, not just the star representative. + #[sqlx::test] + async fn every_write_group_passes_under_limit(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::http::{Method, StatusCode}; + use std::net::SocketAddr; + use std::time::Duration; + + let mut state = crate::test_support::test_state(pool).await; + state.write_rate_limiter = + crate::rate_limit::RateLimiter::new(100, Duration::from_secs(60)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let peer: SocketAddr = "203.0.113.160:7000".parse().unwrap(); + + let router = crate::server::build_router(state); + for (method, uri) in [ + (Method::PUT, "/api/v1/repos/o/r/star"), + (Method::POST, "/graphql"), + (Method::POST, "/api/v1/repos/o/r/issues/1/comments"), + (Method::POST, "/api/v1/tasks"), + (Method::POST, "/api/v1/repos/o/r/bounties"), + (Method::PUT, "/api/v1/profile"), + ] { + assert_ne!( + send_from( + &router, + method, + uri, + &[("content-type", "application/json")], + Body::from(r#"{"query":"{ __typename }"}"#), + peer, + ) + .await, + StatusCode::TOO_MANY_REQUESTS, + "under-limit write to {uri} must pass the brake, not 429" + ); + } + } + + // ── U6/R6: create_repo serializes against a same-key purge ─────────────── + + // create_repo must take the SAME per-owner/name advisory lock the purge holds + // (try_lock_repo / RepoLockGuard), so a create cannot slip into the purge's + // delete-row -> [window] -> remove-dir gap and land a repos row pointing at a + // directory the purge then removes (a dangling row). Deterministic form of the + // race: hold the purge lock, then prove create BLOCKS on it rather than + // proceeding into the window — it must not insert its row while the lock is + // held, and must complete cleanly (row + on-disk dir both present) once the + // lock frees. RED on base (create takes no lock): the row lands while the lock + // is held. GREEN after (create serializes on the same key). + #[sqlx::test] + async fn create_repo_serializes_against_purge_lock( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + use axum::extract::State; + use axum::Extension; + use std::time::Duration; + + // Multi-connection pool: while the purge guard pins one connection for the + // held lock, create's own try_lock_repo attempt must get a DIFFERENT + // connection and observe the advisory lock held (Ok(None) -> it retries), + // not merely block on pool exhaustion. + let pool = pool_opts + .max_connections(5) + .connect_with(connect_opts) + .await + .unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tempfile::TempDir::new().unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing( + repos_dir.path().to_path_buf(), + pool.clone(), + ); + + let owner = "did:key:z6MkCreatePurgeRaceAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "racerepo"; + + // The purge path holds this exact per-repo advisory lock across its + // delete-row + remove-dir. Take it via the same helper the purge uses. + let purge_guard = state + .repo_store + .try_lock_repo(owner, name) + .await + .unwrap() + .expect("lock is free before the create"); + + // Kick off a create for the SAME owner/name while the purge holds the lock. + let state2 = state.clone(); + let owner2 = owner.to_string(); + let handle = tokio::spawn(async move { + super::create_repo( + State(state2), + Extension(crate::auth::AuthenticatedDid(owner2)), + axum::http::HeaderMap::new(), + Json(CreateRepoRequest { + name: name.to_string(), + description: None, + is_public: true, + default_branch: "main".to_string(), + }), + ) + .await + }); + + // While the purge lock is held, create must NOT complete its init+insert. + tokio::time::sleep(Duration::from_millis(600)).await; + assert!( + state.db.get_repo(owner, name).await.unwrap().is_none(), + "create_repo must not insert a repos row while a purge holds the same-key \ + lock (RED on base: create takes no lock and the row lands in the window)" + ); + assert!( + !handle.is_finished(), + "create must be blocked on the purge lock, not have completed" + ); + + // Purge releases; create can now proceed and create cleanly. + purge_guard.release().await; + + let created = tokio::time::timeout(Duration::from_secs(8), handle) + .await + .expect("create should finish once the lock frees") + .expect("create task join") + .expect("create_repo returns Ok once it wins the lock"); + assert_eq!(created.0, StatusCode::CREATED); + + // End state is consistent: the row AND its on-disk dir are both present — + // never a row pointing at a removed directory. + assert!( + state.db.get_repo(owner, name).await.unwrap().is_some(), + "repos row must be present after the create wins the lock" + ); + let dir = repos_dir + .path() + .join(owner.replace([':', '/'], "_")) + .join(format!("{name}.git")); + assert!( + dir.exists(), + "the created repo's on-disk dir must be present — no dangling row" + ); + } + + // U6 no-regression: with no concurrent purge, create_repo still succeeds and + // leaves a consistent row + on-disk dir. Guards against the lock acquisition + // wedging the uncontended hot path. + #[sqlx::test] + async fn create_repo_succeeds_without_lock_contention( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + use axum::extract::State; + use axum::Extension; + + let pool = pool_opts + .max_connections(5) + .connect_with(connect_opts) + .await + .unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tempfile::TempDir::new().unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing( + repos_dir.path().to_path_buf(), + pool.clone(), + ); + + let owner = "did:key:z6MkCreateNoContendBBBBBBBBBBBBBBBBBBBBBBBB"; + let name = "solo"; + + let created = super::create_repo( + State(state.clone()), + Extension(crate::auth::AuthenticatedDid(owner.to_string())), + axum::http::HeaderMap::new(), + Json(CreateRepoRequest { + name: name.to_string(), + description: None, + is_public: true, + default_branch: "main".to_string(), + }), + ) + .await + .expect("uncontended create_repo must succeed"); + assert_eq!(created.0, StatusCode::CREATED); + + assert!( + state.db.get_repo(owner, name).await.unwrap().is_some(), + "row present after an uncontended create" + ); + let dir = repos_dir + .path() + .join(owner.replace([':', '/'], "_")) + .join(format!("{name}.git")); + assert!( + dir.exists(), + "on-disk dir present after an uncontended create" + ); + } + + // ── U3/R2/R4: fork tail, guarded span, publish only after durability ───── + + const FORK_SRC_OWNER: &str = "z6forksrcowner"; + const FORK_SRC_NAME: &str = "forksrc"; + + /// Build a state whose repo_store AND config.repos_dir point at `repos_dir` + /// (fork_repo computes its clone target from config.repos_dir, so the two + /// must agree), seeded with a bare public source repo any caller may fork. + async fn fork_test_state( + repos_dir: &std::path::Path, + repo_store: crate::git::repo_store::RepoStore, + pool: sqlx::PgPool, + ) -> AppState { + let mut state = crate::test_support::test_state(pool).await; + state.repo_store = repo_store; + let mut cfg = (*state.config).clone(); + cfg.repos_dir = repos_dir.to_path_buf(); + state.config = std::sync::Arc::new(cfg); + + let bare = repos_dir + .join(FORK_SRC_OWNER) // slug == owner: no ':' or '/' to replace + .join(format!("{FORK_SRC_NAME}.git")); + std::fs::create_dir_all(bare.parent().unwrap()).unwrap(); + let out = std::process::Command::new("git") + .args(["init", "--bare", "-q", &bare.to_string_lossy()]) + .output() + .unwrap(); + assert!(out.status.success(), "git init --bare failed"); + state + .db + .upsert_mirror_repo( + FORK_SRC_OWNER, + FORK_SRC_NAME, + &bare.to_string_lossy(), + None, + false, + ) + .await + .unwrap(); + state + } + + async fn call_fork( + state: &AppState, + forker: &str, + fork_name: &str, + ) -> Result { + use axum::extract::{Path as AxPath, State}; + use axum::response::IntoResponse; + use axum::Extension; + super::fork_repo( + State(state.clone()), + Extension(crate::auth::AuthenticatedDid(forker.to_string())), + AxPath((FORK_SRC_OWNER.to_string(), FORK_SRC_NAME.to_string())), + axum::http::HeaderMap::new(), + Json(ForkRepoRequest { + name: Some(fork_name.to_string()), + }), + ) + .await + .map(|r| r.into_response()) + } + + // R2: fork must take the SAME per-owner/name advisory lock on its TARGET + // namespace that purge and create hold, so it cannot interleave with a + // purge's delete-row + remove-dir (or another creator) on that key. + // Deterministic form: hold the target lock, prove the fork BLOCKS (no clone + // dir, no repos row) rather than proceeding, then completes cleanly once the + // lock frees. RED on base: fork takes no lock and completes in the window. + #[sqlx::test] + async fn fork_serializes_against_target_namespace_lock( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + use std::time::Duration; + + // Multi-connection pool: the held guard pins one connection, so the + // fork's own lock attempt must get a DIFFERENT one and observe the key + // held (Ok(None) -> retry), not merely block on pool exhaustion. + let pool = pool_opts + .max_connections(5) + .connect_with(connect_opts) + .await + .unwrap(); + let repos_dir = tempfile::TempDir::new().unwrap(); + let repo_store = crate::git::repo_store::RepoStore::for_testing( + repos_dir.path().to_path_buf(), + pool.clone(), + ); + let state = fork_test_state(repos_dir.path(), repo_store, pool).await; + + let forker = "did:key:z6MkForkSerializeAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let fork_name = "serialfork"; + + // A purge (or another creator) holds the fork target's advisory lock. + let held = state + .repo_store + .try_lock_repo(forker, fork_name) + .await + .unwrap() + .expect("target lock free before the fork"); + + let state2 = state.clone(); + let forker2 = forker.to_string(); + let fork_name2 = fork_name.to_string(); + let handle = tokio::spawn(async move { call_fork(&state2, &forker2, &fork_name2).await }); + + // While the target lock is held the fork must make no progress past it. + tokio::time::sleep(Duration::from_millis(500)).await; + let clone_dir = store::repo_disk_path(&state.config.repos_dir, forker, fork_name); + assert!( + state + .db + .get_repo(forker, fork_name) + .await + .unwrap() + .is_none(), + "fork must not insert a repos row while the target-namespace lock is \ + held (RED on base: fork takes no lock and the row lands in the window)" + ); + assert!( + !clone_dir.exists(), + "fork must not clone the mirror while the target-namespace lock is held" + ); + assert!( + !handle.is_finished(), + "fork must be blocked on the target lock, not have completed" + ); + + held.release().await; + let resp = tokio::time::timeout(Duration::from_secs(8), handle) + .await + .expect("fork should finish once the lock frees") + .expect("fork task join") + .expect("fork_repo returns Ok once it wins the lock"); + assert_eq!(resp.status(), StatusCode::CREATED); + assert!( + state + .db + .get_repo(forker, fork_name) + .await + .unwrap() + .is_some(), + "repos row present after the fork wins the lock" + ); + assert!(clone_dir.exists(), "cloned mirror present after the fork"); + } + + // R4: publish-after-durability. When the fork's durable upload stalls, the + // fork must fail (5xx) within the release-upload bound, insert NO repos row, + // and remove the half-created mirror from disk. RED on base: the foreground + // upload's unbounded PUT hangs the handler on a stall (and an erroring + // upload would return 201 + insert the row despite the failed upload). + #[sqlx::test] + async fn fork_durable_upload_failure_fails_before_row_insert(pool: sqlx::PgPool) { + use axum::response::IntoResponse; + use std::path::Path as StdPath; + use std::time::Duration; + + // Object store whose upload() parks forever: the fork-tail upload bound + // is the only thing that unblocks it. `exists()` is false so acquire + // never downloads over the local source repo. + struct StallStore; + #[async_trait::async_trait] + impl crate::git::tigris::ObjectStore for StallStore { + async fn exists(&self, _o: &str, _r: &str) -> anyhow::Result { + Ok(false) + } + async fn upload(&self, _o: &str, _r: &str, _p: &StdPath) -> anyhow::Result<()> { + std::future::pending::<()>().await; + Ok(()) + } + async fn download(&self, _o: &str, _r: &str, _p: &StdPath) -> anyhow::Result<()> { + Ok(()) + } + async fn delete(&self, _o: &str, _r: &str) -> anyhow::Result<()> { + Ok(()) + } + } + + let repos_dir = tempfile::TempDir::new().unwrap(); + let repo_store = crate::git::repo_store::RepoStore::new( + repos_dir.path().to_path_buf(), + Some(std::sync::Arc::new(StallStore)), + pool.clone(), + ) + .with_release_upload_timeout(Duration::from_millis(200)); + let state = fork_test_state(repos_dir.path(), repo_store, pool).await; + + let forker = "did:key:z6MkForkUploadFailAAAAAAAAAAAAAAAAAAAAAAAAA"; + let fork_name = "failfork"; + + let resp = + tokio::time::timeout(Duration::from_secs(5), call_fork(&state, forker, fork_name)) + .await + .expect( + "fork must fail within the upload bound when the durable upload \ + stalls (RED on base: the unbounded foreground PUT hangs the handler)", + ); + let status = match resp { + Ok(r) => r.status(), + Err(e) => e.into_response().status(), + }; + assert!( + status.is_server_error(), + "a failed durable upload must fail the fork (5xx); got {status}, \ + which the client trusts as a fork whose archive never landed" + ); + + // Publish-after-durability: no repos row for a fork with no archive. + assert!( + state + .db + .get_repo(forker, fork_name) + .await + .unwrap() + .is_none(), + "no repos row may exist for a fork whose durable upload failed" + ); + let clone_dir = store::repo_disk_path(&state.config.repos_dir, forker, fork_name); + assert!( + !clone_dir.exists(), + "the cloned mirror must be removed when the durable upload fails" + ); + } + + // KTD-2 trap case (must-not-break negative): a store-less node has nothing + // to upload, so fork succeeds exactly as today: "no object store" is + // nothing-to-do, never a failed upload. GREEN both before and after the fix. + #[sqlx::test] + async fn fork_without_object_store_succeeds(pool: sqlx::PgPool) { + let repos_dir = tempfile::TempDir::new().unwrap(); + let repo_store = crate::git::repo_store::RepoStore::for_testing( + repos_dir.path().to_path_buf(), + pool.clone(), + ); + let state = fork_test_state(repos_dir.path(), repo_store, pool).await; + + let forker = "did:key:z6MkForkNoStoreAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let fork_name = "nostorefork"; + + let resp = call_fork(&state, forker, fork_name) + .await + .expect("a store-less fork must succeed"); + assert_eq!(resp.status(), StatusCode::CREATED); + assert!( + state + .db + .get_repo(forker, fork_name) + .await + .unwrap() + .is_some(), + "repos row present after a store-less fork" + ); + let clone_dir = store::repo_disk_path(&state.config.repos_dir, forker, fork_name); + assert!(clone_dir.exists(), "cloned mirror present after the fork"); + } + + // R2: a target lock held past the fork's bounded wait must surface as the + // same retryable 503 create_repo returns, with no repos row and no clone + // residue. RED on base: fork ignores the lock and completes with a 201. + #[sqlx::test] + async fn fork_lock_contention_returns_503( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + use axum::response::IntoResponse; + + let pool = pool_opts + .max_connections(5) + .connect_with(connect_opts) + .await + .unwrap(); + let repos_dir = tempfile::TempDir::new().unwrap(); + let repo_store = crate::git::repo_store::RepoStore::for_testing( + repos_dir.path().to_path_buf(), + pool.clone(), + ); + let state = fork_test_state(repos_dir.path(), repo_store, pool).await; + + let forker = "did:key:z6MkForkContendedAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let fork_name = "contendedfork"; + + // Held for the whole test: the fork's bounded wait must give up. + let held = state + .repo_store + .try_lock_repo(forker, fork_name) + .await + .unwrap() + .expect("target lock free before the fork"); + + let resp = call_fork(&state, forker, fork_name).await; + let status = match resp { + Ok(r) => r.status(), + Err(e) => e.into_response().status(), + }; + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "a target lock held past the bounded wait must 503 like create_repo \ + (RED on base: fork ignores the lock and returns 201)" + ); + assert!( + state + .db + .get_repo(forker, fork_name) + .await + .unwrap() + .is_none(), + "no repos row after a 503'd fork" + ); + let clone_dir = store::repo_disk_path(&state.config.repos_dir, forker, fork_name); + assert!(!clone_dir.exists(), "no clone residue after a 503'd fork"); + held.release().await; + } + + // Finding 1: forking a COLD source (archive-only, no local dir) must + // materialize the source out from under the target lock, so it works even + // when the write lock_pool is sized 1. On a cold source, acquire()'s + // download takes a nested advisory lock on that SAME lock_pool for the + // SOURCE's namespace; if the target lock (which pins the pool's one + // connection) were held first, that nested acquire would find the pool + // exhausted, PoolTimedOut -> the source publish degrades -> the source never + // lands on disk -> the clone of a missing source fails the fork. RED on the + // pre-reorder code (source acquire AFTER lock_repo_blocking): fork fails + // with a 5xx at a size-1 lock pool. GREEN after: acquire runs before the + // lock, so the two nested acquires never overlap. + #[sqlx::test] + async fn fork_cold_source_materializes_before_target_lock_at_pool_size_one( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + use std::path::Path as StdPath; + + // An object store standing in for a cold source: exists() is always + // true, and download() materializes a valid bare repo at the target path + // (mirroring the real store's extract-then-swap), so acquire() publishes + // the source on disk. upload() succeeds so the fork's own durable upload + // is a no-op success. + struct ColdMaterializeStore; + #[async_trait::async_trait] + impl crate::git::tigris::ObjectStore for ColdMaterializeStore { + async fn exists(&self, _o: &str, _r: &str) -> anyhow::Result { + Ok(true) + } + async fn upload(&self, _o: &str, _r: &str, _p: &StdPath) -> anyhow::Result<()> { + Ok(()) + } + async fn download(&self, _o: &str, _r: &str, p: &StdPath) -> anyhow::Result<()> { + if p.exists() { + std::fs::remove_dir_all(p)?; + } + crate::git::store::init_bare(p)?; + Ok(()) + } + async fn delete(&self, _o: &str, _r: &str) -> anyhow::Result<()> { + Ok(()) + } + } + + // App pool for state.db (get_repo / proof.consume / create_repo), sized + // generously so unrelated app work never contends. The hazard under test + // is on the SEPARATE lock pool below. + let app_pool = pool_opts + .max_connections(5) + .connect_with(connect_opts.clone()) + .await + .unwrap(); + // The write lock_pool sized to ONE connection: a target lock plus an + // overlapping source-namespace lock cannot both be satisfied from it, so + // this size is exactly what surfaces the double-hold hazard. + let lock_pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(1) + .acquire_timeout(std::time::Duration::from_secs(2)) + .min_connections(0) + .connect_with(connect_opts) + .await + .unwrap(); + + let repos_dir = tempfile::TempDir::new().unwrap(); + let repo_store = crate::git::repo_store::RepoStore::new( + repos_dir.path().to_path_buf(), + Some(std::sync::Arc::new(ColdMaterializeStore)), + lock_pool, + ); + + // Build state WITHOUT creating the source bare on disk: the source lives + // only in the object store, so acquire() must download it. (fork_test_state + // would materialize it locally, which is the warm path, not this one.) + let mut state = crate::test_support::test_state(app_pool).await; + state.repo_store = repo_store; + let mut cfg = (*state.config).clone(); + cfg.repos_dir = repos_dir.path().to_path_buf(); + state.config = std::sync::Arc::new(cfg); + state + .db + .upsert_mirror_repo( + FORK_SRC_OWNER, + FORK_SRC_NAME, + "unused-cold-path", + None, + false, + ) + .await + .unwrap(); + // The source must NOT be on local disk, or acquire takes the warm path + // and never exercises the cold download + nested lock. + let src_local = + store::repo_disk_path(&state.config.repos_dir, FORK_SRC_OWNER, FORK_SRC_NAME); + assert!(!src_local.exists(), "source must be cold (absent locally)"); + + let forker = "did:key:z6MkForkColdSrcAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let fork_name = "coldfork"; + + let resp = call_fork(&state, forker, fork_name) + .await + .expect("cold-source fork must succeed at a size-1 lock pool"); + assert_eq!(resp.status(), StatusCode::CREATED); + assert!( + state + .db + .get_repo(forker, fork_name) + .await + .unwrap() + .is_some(), + "repos row present after the cold-source fork" + ); + let clone_dir = store::repo_disk_path(&state.config.repos_dir, forker, fork_name); + assert!( + clone_dir.exists(), + "cloned mirror present after the cold-source fork" + ); + } + + // Finding 1: a `git clone --mirror` that exits non-zero after the dest dir + // exists must clear that dest and free the lock, so a retry of the same fork + // name is not permanently blocked by a leftover half-mirror. Driven by + // pre-seeding the dest so the clone fails "destination already exists" + // (exit 128, the `!output.status.success()` arm, not the timeout/spawn + // arms). RED on base: the arm returns without removing the dest, so the dest + // survives and the second fork's clone fails the same way (5xx forever). + #[sqlx::test] + async fn fork_nonzero_clone_exit_cleans_dest_and_allows_retry(pool: sqlx::PgPool) { + let repos_dir = tempfile::TempDir::new().unwrap(); + let repo_store = crate::git::repo_store::RepoStore::for_testing( + repos_dir.path().to_path_buf(), + pool.clone(), + ); + let state = fork_test_state(repos_dir.path(), repo_store, pool).await; + + let forker = "did:key:z6MkForkCloneNonzeroAAAAAAAAAAAAAAAAAAAAAA"; + let fork_name = "nonzerofork"; + + // Seed the dest as a non-empty dir so `git clone --mirror` refuses it and + // exits 128, standing in for a half-created mirror a prior failed clone + // (authz, corrupt source) would leave behind. + let clone_dir = store::repo_disk_path(&state.config.repos_dir, forker, fork_name); + std::fs::create_dir_all(&clone_dir).unwrap(); + std::fs::write(clone_dir.join("stale.txt"), b"half-created mirror").unwrap(); + + let resp = call_fork(&state, forker, fork_name).await; + let status = match resp { + Ok(r) => r.status(), + Err(e) => { + use axum::response::IntoResponse; + e.into_response().status() + } + }; + assert!( + status.is_server_error(), + "a non-zero clone exit must fail the fork (5xx); got {status}" + ); + assert!( + !clone_dir.exists(), + "the leftover dest must be removed after a non-zero clone exit \ + (RED on base: the !success arm leaves it, blocking every retry)" + ); + + // The dest is clean and the lock freed, so re-forking the same name now + // clones fresh and succeeds. + let resp2 = call_fork(&state, forker, fork_name) + .await + .expect("a retry after the dest is cleaned must succeed"); + assert_eq!(resp2.status(), StatusCode::CREATED); + assert!( + state + .db + .get_repo(forker, fork_name) + .await + .unwrap() + .is_some(), + "repos row present after the successful retry" + ); + assert!(clone_dir.exists(), "cloned mirror present after the retry"); + } + + // Finding 2: when the row insert fails AFTER the durable upload landed, the + // fork must roll back BOTH the on-disk mirror and the just-uploaded archive, + // so no orphaned dest blocks a retry and no stale archive can be downloaded + // into a later same-key repo. The insert failure is injected deterministically + // via the `disk_path UNIQUE` constraint: a decoy row occupying the fork's + // exact disk_path passes the owner+name conflict guard (different owner/name) + // yet collides on insert, which happens only after upload_under_guard. RED on + // base: the `?` short-circuits, leaving the mirror dir and the archive behind. + #[sqlx::test] + async fn fork_row_insert_failure_rolls_back_mirror_and_archive(pool: sqlx::PgPool) { + use axum::response::IntoResponse; + use std::path::Path as StdPath; + use std::sync::{Arc, Mutex}; + + // Store double that uploads OK and records/removes the archive key so the + // test can assert the rollback deleted it. exists() is false so acquire + // never downloads over the local source. + type Archives = Arc>>; + struct TrackingStore { + archives: Archives, + } + #[async_trait::async_trait] + impl crate::git::tigris::ObjectStore for TrackingStore { + async fn exists(&self, _o: &str, _r: &str) -> anyhow::Result { + Ok(false) + } + async fn upload(&self, o: &str, r: &str, _p: &StdPath) -> anyhow::Result<()> { + self.archives + .lock() + .unwrap() + .insert((o.to_string(), r.to_string())); + Ok(()) + } + async fn download(&self, _o: &str, _r: &str, _p: &StdPath) -> anyhow::Result<()> { + Ok(()) + } + async fn delete(&self, o: &str, r: &str) -> anyhow::Result<()> { + self.archives + .lock() + .unwrap() + .remove(&(o.to_string(), r.to_string())); + Ok(()) + } + } + + let archives: Archives = Arc::new(Mutex::new(std::collections::HashSet::new())); + let repos_dir = tempfile::TempDir::new().unwrap(); + let repo_store = crate::git::repo_store::RepoStore::new( + repos_dir.path().to_path_buf(), + Some(Arc::new(TrackingStore { + archives: archives.clone(), + })), + pool.clone(), + ); + let state = fork_test_state(repos_dir.path(), repo_store, pool).await; + + let forker = "did:key:z6MkForkInsertFailAAAAAAAAAAAAAAAAAAAAAAAAA"; + let fork_name = "insertfailfork"; + // The store keys on the slugified owner DID (`:`/`/` -> `_`), the same + // transform local_path applies. Track the FORK's archive specifically: + // the source repo is independently (re)uploaded via the read acquire, so + // the set is not empty even when the fork's archive is correctly rolled + // back. + let fork_archive = (forker.replace([':', '/'], "_"), fork_name.to_string()); + + // Pre-insert a decoy row that occupies the fork's exact disk_path but has + // a different owner/name, so the fork's own insert violates disk_path's + // UNIQUE constraint. get_repo(forker, fork_name) still returns None, so the + // handler proceeds past its conflict guard, clones, and uploads first. + let clone_dir = store::repo_disk_path(&state.config.repos_dir, forker, fork_name); + let now = Utc::now(); + let decoy = crate::db::RepoRecord { + id: Uuid::new_v4().to_string(), + name: "decoyname".to_string(), + owner_did: "did:key:z6MkDecoyOwnerBBBBBBBBBBBBBBBBBBBBBBBBBBBB".to_string(), + description: None, + is_public: true, + default_branch: "main".to_string(), + created_at: now, + updated_at: now, + disk_path: clone_dir.to_string_lossy().to_string(), + forked_from: None, + machine_id: None, + }; + state.db.create_repo(&decoy).await.unwrap(); + + let resp = call_fork(&state, forker, fork_name).await; + let status = match resp { + Ok(r) => r.status(), + Err(e) => e.into_response().status(), + }; + assert!( + status.is_server_error(), + "a row-insert failure must fail the fork (5xx); got {status}" + ); + assert!( + state + .db + .get_repo(forker, fork_name) + .await + .unwrap() + .is_none(), + "no fork row may exist after the insert failure" + ); + assert!( + !clone_dir.exists(), + "the cloned mirror must be removed after the failed insert \ + (RED on base: the `?` short-circuits and leaves it)" + ); + assert!( + !archives.lock().unwrap().contains(&fork_archive), + "the uploaded fork archive must be deleted after the failed insert \ + (RED on base: the archive is orphaned)" + ); + + // The target lock must be free again for a retry. + let probe = state + .repo_store + .try_lock_repo(forker, fork_name) + .await + .unwrap() + .expect("target lock freed after the rolled-back fork"); + probe.release().await; + + // Remove the decoy so the retry's insert can land, then re-fork the same + // name: dest is clean, lock is free, and the archive is re-uploaded. + state.db.delete_repo_by_id(&decoy.id).await.unwrap(); + let resp2 = call_fork(&state, forker, fork_name) + .await + .expect("a retry after the rollback must succeed"); + assert_eq!(resp2.status(), StatusCode::CREATED); + assert!( + state + .db + .get_repo(forker, fork_name) + .await + .unwrap() + .is_some(), + "repos row present after the successful retry" + ); + assert!( + archives.lock().unwrap().contains(&fork_archive), + "the retry re-uploads the fork archive" + ); + } + + // create_repo must NOT take its serialization lock from the APP pool. Doing so + // pins an app-pool connection in the lock guard while the create's own work + // (get_repo / proof.consume / db.create_repo, all on state.db = the app pool) + // still needs an app connection. At GITLAWB_DB_MAX_CONNECTIONS=1 (a supported + // config) the guard holds the only app connection and get_repo self-deadlocks + // (PoolTimedOut -> 500). Locking on the separate write lock_pool instead keeps + // the lock and the work on different pools, so a size-1 app pool never wedges. + // RED on the app-pool-lock code: create times out -> Err. GREEN after reverting + // create to lock on lock_pool. + #[sqlx::test] + async fn create_repo_does_not_self_deadlock_at_one_app_connection( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + use axum::extract::State; + use axum::Extension; + use std::time::Duration; + + // Migrate the per-test DB on a normal pool: `migrate()` pins one connection for + // its cross-process advisory lock while applying statements on a second, so it + // needs >1 connection and cannot itself run on the size-1 pool under test. + let migrate_pool = pool_opts.connect_with(connect_opts.clone()).await.unwrap(); + let mut state = crate::test_support::test_state(migrate_pool).await; + + // Now point state.db at an APP pool of MAX size 1 on the SAME database, the + // minimal supported config (GITLAWB_DB_MAX_CONNECTIONS=1). The schema is + // already applied, so this pool only serves create's work (get_repo -> insert). + // A short acquire timeout makes the RED case (the lock guard pinning this one + // connection while get_repo waits for another) fail fast instead of hanging. + let app_pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(1) + .acquire_timeout(Duration::from_secs(2)) + .connect_with(connect_opts.clone()) + .await + .unwrap(); + state.db = std::sync::Arc::new(crate::db::Db::for_testing(app_pool.clone())); + + // A SEPARATE write lock pool (where create's serialization lock belongs). Its + // being separate is the whole point: the lock guard must not draw from the app + // pool the create work runs on. + let lock_pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(5) + .connect_with(connect_opts.clone()) + .await + .unwrap(); + let repos_dir = tempfile::TempDir::new().unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing( + repos_dir.path().to_path_buf(), + lock_pool.clone(), + ); + + let owner = "did:key:z6MkOneAppConnNoDeadlockDDDDDDDDDDDDDDDDDDDD"; + let name = "solo"; + let created = super::create_repo( + State(state.clone()), + Extension(crate::auth::AuthenticatedDid(owner.to_string())), + axum::http::HeaderMap::new(), + Json(CreateRepoRequest { + name: name.to_string(), + description: None, + is_public: true, + default_branch: "main".to_string(), + }), + ) + .await + .expect("create must not self-deadlock at app-pool size 1 (lock on lock_pool)"); + assert_eq!(created.0, StatusCode::CREATED); + assert!( + state.db.get_repo(owner, name).await.unwrap().is_some(), + "row present after a create with a size-1 app pool" ); } } diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 720fb3ae..a9baa314 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -515,6 +515,7 @@ mod tests { repo_store: crate::git::repo_store::RepoStore::for_testing(PathBuf::from("/tmp"), pool), rate_limiter: RateLimiter::new(100, Duration::from_secs(60)), create_ip_rate_limiter: RateLimiter::new(1000, Duration::from_secs(3600)), + write_rate_limiter: RateLimiter::new(1000, Duration::from_secs(3600)), push_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), push_limiter_trust: crate::rate_limit::TrustedProxy::None, sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index fc2247d9..7dd77876 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -1,18 +1,45 @@ -use clap::Parser; +use clap::{Parser, Subcommand}; use std::path::PathBuf; +/// Optional admin subcommands. When none is given, the binary runs the node +/// daemon as before — the default (no-subcommand) startup path is unchanged. +#[derive(Subcommand, Debug, Clone)] +pub enum Command { + /// Dry-run (default) or, with --execute, delete empty spam-burst repos owned + /// by the known burst DID. Never touches the hard-excluded DIDs, and verifies + /// each repo is empty (zero git refs) per repo before selecting it. + PurgeSpam { + /// Actually delete the candidates. Omit for a dry-run that prints the + /// candidate list and deletes nothing. + #[arg(long, default_value_t = false)] + execute: bool, + }, +} + #[derive(Parser, Debug, Clone)] #[command(name = "gitlawb-node", about = "gitlawb node daemon", version)] pub struct Config { - /// Directory where bare git repositories are stored - #[arg(long, env = "GITLAWB_REPOS_DIR", default_value = "./data/repos")] + /// Admin subcommand to run instead of the node daemon. Absent = run the node. + #[command(subcommand)] + pub command: Option, + + /// Directory where bare git repositories are stored. `global` so it can + /// follow a subcommand (e.g. `gitlawb-node purge-spam --repos-dir …`). + #[arg( + long, + env = "GITLAWB_REPOS_DIR", + default_value = "./data/repos", + global = true + )] pub repos_dir: PathBuf, - /// PostgreSQL connection URL (Supabase or any Postgres instance) + /// PostgreSQL connection URL (Supabase or any Postgres instance). `global` so + /// admin subcommands accept it in either position. #[arg( long, env = "DATABASE_URL", - default_value = "postgresql://localhost/gitlawb" + default_value = "postgresql://localhost/gitlawb", + global = true )] pub database_url: String, @@ -194,6 +221,22 @@ pub struct Config { )] pub db_max_connections: u32, + /// Maximum connections in the dedicated advisory-lock pool, separate from + /// the main pool above. Every in-flight repo write (git-receive-pack, fork, + /// merge) pins one connection here for its whole lifetime — the write, its + /// metadata/fan-out tail, and the bounded post-write archive upload — so + /// size it to the expected peak number of concurrent distinct-repo writers, + /// NOT small. Keeping it separate from GITLAWB_DB_MAX_CONNECTIONS is what + /// stops a push burst from starving normal request handlers. Keep + /// (main pool + lock pool) within the database server's max_connections. + #[arg( + long, + env = "GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS", + default_value_t = 32, + value_parser = clap::value_parser!(u32).range(1..) + )] + pub db_lock_pool_max_connections: u32, + /// Maximum time a request waits for a pool connection before failing with /// 503, in seconds. Bounds queueing when the database is slow or down. #[arg( diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 33f5bd67..51499ab6 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -250,7 +250,11 @@ pub struct Db { } impl Db { - /// Access the underlying Postgres connection pool. + /// Access the underlying Postgres connection pool. Test-only: all production DB + /// access goes through `Db`'s methods, and the write advisory-lock subsystem uses + /// its own pool (`connect_lock_pool`). Tests reach for the raw pool to seed and + /// assert directly. + #[cfg(test)] pub fn pool(&self) -> &PgPool { &self.pool } @@ -295,6 +299,30 @@ impl Db { Ok(db) } + /// Build a standalone Postgres pool for the advisory-lock subsystem, + /// separate from the app pool. No migrations run here (the app pool owns + /// the schema); this pool only serves the session-scoped advisory-lock + /// connections that `RepoStore`'s write guards pin for a write's whole + /// lifetime, so a concurrent-push burst pins connections from here rather + /// than starving the app pool. Sized by GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS. + pub async fn connect_lock_pool( + database_url: &str, + max_connections: u32, + acquire_timeout: Duration, + ) -> Result { + info!( + max_connections, + acquire_timeout_secs = acquire_timeout.as_secs(), + "connecting advisory-lock pool" + ); + PgPoolOptions::new() + .max_connections(max_connections) + .acquire_timeout(acquire_timeout) + .connect(database_url) + .await + .context("connecting advisory-lock pool") + } + /// Cheap liveness probe against the pool, for readiness checks: one /// `SELECT 1` that fails fast when the database is unreachable. pub async fn ping(&self) -> Result<()> { @@ -899,6 +927,49 @@ pub(crate) fn normalize_owner_key(did: &str) -> &str { /// drift apart. If you change `normalize_owner_key`, update this const too. const OWNER_KEY_CASE_SQL: &str = "CASE WHEN owner_did LIKE 'did:key:%' AND position(':' in substr(owner_did, 9)) = 0 THEN substr(owner_did, 9) ELSE owner_did END"; +/// `OWNER_KEY_CASE_SQL` over an arbitrary owner column. The const above hardcodes +/// `owner_did` (the `repos` column the `idx_repos_owner_key_name` index is built +/// on); the bounty tombstone needs the same normalizer over `bounties.repo_owner`, +/// which is not index-backed. Byte-identical to `normalize_owner_key`; update this +/// alongside the const and the fn if the rule changes. +fn owner_key_case_sql(col: &str) -> String { + format!( + "CASE WHEN {col} LIKE 'did:key:%' AND position(':' in substr({col}, 9)) = 0 \ + THEN substr({col}, 9) ELSE {col} END" + ) +} + +/// Take a transaction-scoped Postgres advisory lock keyed on the normalized repo +/// slug (`normalize_owner_key(owner)/name`). `delete_repo_by_id`'s purge cascade and +/// `upsert_mirror_repo`'s ingest both take this lock on the identical slug string, so +/// a peer-sync mirror insert cannot interleave between the purge's point-in-time +/// sibling check and its slug-scoped deletes (which would half-wipe a live mirror's +/// child rows / bounty). `hashtextextended($1, 0)` folds the slug to the `bigint` the +/// lock API needs; both call sites hash the identical string, so their keys agree. +/// XACT-scoped, so it auto-releases on commit/rollback with no unlock bookkeeping +/// (mirrors the session-scoped `pg_advisory_lock` migration guard, but self-releasing). +async fn lock_repo_slug(conn: &mut sqlx::PgConnection, slug: &str) -> Result<()> { + sqlx::query("SELECT pg_advisory_xact_lock(hashtextextended($1, 0))") + .bind(slug) + .execute(conn) + .await?; + Ok(()) +} + +/// Normalize an `owner/name` slug to the exact string `delete_repo_by_id` builds its +/// advisory-lock key from: `normalize_owner_key(owner)/name`, splitting at the first +/// `/` (owners never contain `/`; DID owners use `:`). Writers that receive the slug +/// as one string (`branch_cids.repo`, `sync_queue.repo`, `received_ref_updates.repo`) +/// must lock this normalized form, not the verbatim input, or a full-DID caller would +/// hash a different key than the purge and slip past the serialization. A string with +/// no `/` is returned as-is; the purge never locks such a key, so it is inert. +fn normalize_repo_slug(repo: &str) -> String { + match repo.split_once('/') { + Some((owner, name)) => format!("{}/{}", normalize_owner_key(owner), name), + None => repo.to_string(), + } +} + #[cfg(test)] mod normalize_owner_key_tests { use super::normalize_owner_key; @@ -1001,6 +1072,14 @@ impl Db { ) -> Result<()> { let now = Utc::now().to_rfc3339(); let id = format!("{owner_short}/{name}"); + // Slug the purge cascade keys on. Take the shared advisory lock on it inside a + // tx so a concurrent `delete_repo_by_id` purge of a colliding row cannot + // interleave its sibling-check-then-cascade around this insert and wipe the + // mirror we are ingesting. Normalize the owner the same way the cascade does so + // both call sites hash the identical slug string. + let slug = format!("{}/{}", normalize_owner_key(owner_short), name); + let mut tx = self.pool.begin().await?; + lock_repo_slug(&mut tx, &slug).await?; // `quarantined` is set only on first insert (the admission decision). // A re-sync (ON CONFLICT) preserves the existing flag — admission runs // once, and an operator's later release must not be silently reverted. @@ -1021,8 +1100,9 @@ impl Db { .bind(disk_path) .bind(machine_id) .bind(quarantined) - .execute(&self.pool) + .execute(&mut *tx) .await?; + tx.commit().await?; Ok(()) } @@ -1270,6 +1350,176 @@ impl Db { .await?; Ok(()) } + + /// Every repo row whose owner resolves to `owner_did` under did:key + /// normalization, so `did:key:z6…` and bare `z6…` rows of the same identity + /// both match (mirroring how ownership is resolved everywhere else via + /// `OWNER_KEY_CASE_SQL` / the `idx_repos_owner_key_name` index). No dedup, so a + /// caller enumerating one DID's repos sees every physical row. Backs the + /// `purge-spam` admin tool, whose selection then applies its own per-repo empty + /// check and exclusion gate (both normalization-consistent). Ordered by `id`. + pub async fn list_repos_by_owner_did(&self, owner_did: &str) -> Result> { + let sql = format!( + "SELECT id, name, owner_did, description, is_public, default_branch, + created_at, updated_at, disk_path, forked_from, machine_id + FROM repos WHERE {key} = $1 ORDER BY id", + key = OWNER_KEY_CASE_SQL + ); + let rows = sqlx::query(&sql) + .bind(normalize_owner_key(owner_did)) + .fetch_all(&self.pool) + .await?; + Ok(rows.into_iter().map(row_to_repo).collect()) + } + + /// Delete a single repo row by its primary key `id`. Returns the number of + /// rows removed (0 if no such repo). Operates on one repo at a time by design: + /// the `purge-spam` tool deletes a vetted candidate list per-repo, never a + /// blanket "delete all repos of owner X". + pub async fn delete_repo_by_id(&self, id: &str) -> Result { + let mut tx = self.pool.begin().await?; + + // Resolve the row's identity so the `repo`-slug-keyed children (keyed on + // `normalize_owner_key(owner)/name`, not `repos.id`) can be matched. Absent + // row → nothing to delete; commit the empty tx and report 0. + let row: Option<(String, String)> = + sqlx::query_as("SELECT owner_did, name FROM repos WHERE id = $1") + .bind(id) + .fetch_optional(&mut *tx) + .await?; + let Some((owner_did, name)) = row else { + tx.commit().await?; + return Ok(0); + }; + let slug = format!("{}/{}", normalize_owner_key(&owner_did), name); + + // Serialize this purge against a concurrent `upsert_mirror_repo` ingest that + // resolves to the same slug: both take this xact-scoped advisory lock on the + // identical slug string BEFORE the sibling check below, so a peer-sync insert + // cannot land between that point-in-time check and the slug-scoped cascade + // deletes (which would half-wipe the live mirror). Auto-released on commit. + lock_repo_slug(&mut tx, &slug).await?; + + // Grandchildren first: PR reviews/comments key on pr_id, so delete them + // before the parent PRs vanish. + for table in ["pr_reviews", "pr_comments"] { + sqlx::query(&format!( + "DELETE FROM {table} WHERE pr_id IN (SELECT id FROM pull_requests WHERE repo_id = $1)" + )) + .bind(id) + .execute(&mut *tx) + .await?; + } + + // Direct children keyed on repos.id. + for table in [ + "push_events", + "ref_certificates", + "pull_requests", + "webhooks", + "agent_tasks", + "protected_branches", + "repo_stars", + "repo_replicas", + "repo_labels", + "visibility_rules", + "encrypted_blobs", + "repo_icaptcha_proofs", + ] { + sqlx::query(&format!("DELETE FROM {table} WHERE repo_id = $1")) + .bind(id) + .execute(&mut *tx) + .await?; + } + + // Children keyed on the derived `owner_short/name` slug rather than the id. + // INV-9 same-method slug collision: a canonical `did:key:zX` row and a bare + // `zX` mirror both normalize (via OWNER_KEY_CASE_SQL) to slug `zX/name`, so + // these tables cannot tell whose rows they are. Purging one row must not wipe + // a surviving sibling's records. Gate the slug-scoped deletes on the absence + // of any OTHER repos row resolving to the same slug, using the same + // OWNER_KEY_CASE_SQL the slug was built from so the check is byte-identical to + // the cascade key. The xact advisory lock taken at the top of this tx (and + // matched in `upsert_mirror_repo` on the identical slug) serializes this + // sibling-check-then-cascade against a concurrent peer-sync insert for the same + // slug, so this is no longer a bare point-in-time read a mirror upsert can slip + // past (closing the old KTD4 residual: the concurrent inserter is peer sync, + // not an operator, so the window was never operator-scoped). + let sibling_exists: bool = sqlx::query_scalar(&format!( + "SELECT EXISTS(SELECT 1 FROM repos WHERE id <> $1 AND {key} || '/' || name = $2)", + key = OWNER_KEY_CASE_SQL + )) + .bind(id) + .bind(&slug) + .fetch_one(&mut *tx) + .await?; + if !sibling_exists { + for table in [ + "branch_cids", + "sync_queue", + "received_ref_updates", + "arweave_anchors", + ] { + sqlx::query(&format!("DELETE FROM {table} WHERE repo = $1")) + .bind(&slug) + .execute(&mut *tx) + .await?; + } + } + // NOTE: `bounties` (financial: amount/wallet/tx_hash) and `issue_comments` + // (no issues table to map issue_id → repo) are deliberately NOT cascaded + // here — dropping money records or unmappable rows on a repo delete would + // be wrong. They are left intact by design. + // + // But bounties key on (repo_owner, repo_name), not an immutable repo id, so + // an untouched OPEN bounty would silently re-attach to a same-name repo the + // owner recreates and become claimable again (finding F). Tombstone the + // repo's non-terminal bounties to a dead `purged` status instead of deleting + // them: this preserves the financial record while making them unclaimable + // (claim/submit/approve/cancel/dispute all gate on specific non-`purged` + // source statuses, so `purged` is inert and no transition reopens it). + // Scope by (repo_owner, repo_name) as the surviving purge does; the already + // terminal `completed`/`cancelled` bounties are left as-is. (Residual: a + // tombstoned bounty stays readable/listable under the same owner/name after + // recreation — a view, not a claim/payout re-attach; see the plan's KTD8.) + // + // Gate the tombstone on the SAME sibling check as the cascade: only when no + // other repos row resolves to this slug. If a mirror row survives, the logical + // repo still exists (and still serves via the mirror), so its bounty is still + // valid — tombstoning it would kill a live repo's bounty. Only when the last + // row for the slug is gone can a recreate re-attach an open bounty. + // + // When it does fire, match EVERY owner representation by normalizing BOTH sides. + // `create_bounty` stores `repo_owner` verbatim from the URL Path param, commonly + // the bare SHORT form (`z6Mk...`), but a full-DID URL that resolves to a bare + // mirror row yields the FULL `did:key:z6Mk...` form. The repo row here is likewise + // either form (`delete_repo_by_id` is reachable on bare mirror rows). Comparing the + // SQL-normalized stored `repo_owner` against the normalized target covers all four + // combinations (did:key-row + short-bounty AND bare-row + full-bounty); binding + // only one side left the mirror form `open`, so it re-attached to the recreated + // repo. It's a rare purge, so a seq scan on `bounties` (the normalizer is not + // index-backed) is fine. + if !sibling_exists { + sqlx::query(&format!( + "UPDATE bounties SET status = 'purged' + WHERE {key} = $1 AND repo_name = $2 + AND status IN ('open', 'claimed', 'submitted')", + key = owner_key_case_sql("repo_owner") + )) + .bind(normalize_owner_key(&owner_did)) + .bind(&name) + .execute(&mut *tx) + .await?; + } + + let result = sqlx::query("DELETE FROM repos WHERE id = $1") + .bind(id) + .execute(&mut *tx) + .await?; + + tx.commit().await?; + Ok(result.rows_affected()) + } } // ── Agents / Trust ──────────────────────────────────────────────────────────── @@ -1557,6 +1807,15 @@ impl Db { cid: &str, node_did: &str, ) -> Result<()> { + // `branch_cids` is one of the slug-scoped tables `delete_repo_by_id` cascades + // over. Take the shared advisory lock (same normalized slug the purge hashes) + // inside a tx so this row cannot land between the purge's point-in-time + // sibling check and its cascade deletes (which would strand an orphan row or + // half-wipe a live mirror's state). Single lock, nothing else held, so the + // ordering matches every other slug-lock site and cannot deadlock. + let slug = normalize_repo_slug(repo); + let mut tx = self.pool.begin().await?; + lock_repo_slug(&mut tx, &slug).await?; sqlx::query( "INSERT INTO branch_cids (repo, ref_name, sha, cid, node_did, updated_at) VALUES ($1, $2, $3, $4, $5, $6) @@ -1570,8 +1829,9 @@ impl Db { .bind(cid) .bind(node_did) .bind(Utc::now().to_rfc3339()) - .execute(&self.pool) + .execute(&mut *tx) .await?; + tx.commit().await?; Ok(()) } @@ -1620,6 +1880,13 @@ impl Db { new_sha: &str, cid: Option<&str>, ) -> Result<()> { + // `sync_queue` is slug-scoped and cascaded by `delete_repo_by_id`; take the + // shared advisory lock on the same normalized slug so an enqueue cannot + // interleave with the purge's sibling-check-then-cascade (same shape and + // rationale as `upsert_branch_cid`; single lock, no deadlock exposure). + let slug = normalize_repo_slug(repo); + let mut tx = self.pool.begin().await?; + lock_repo_slug(&mut tx, &slug).await?; sqlx::query( "INSERT INTO sync_queue (id, repo, node_did, ref_name, new_sha, cid, status, enqueued_at) VALUES ($1, $2, $3, $4, $5, $6, 'pending', $7) @@ -1632,8 +1899,9 @@ impl Db { .bind(new_sha) .bind(cid) .bind(Utc::now().to_rfc3339()) - .execute(&self.pool) + .execute(&mut *tx) .await?; + tx.commit().await?; Ok(()) } @@ -2312,6 +2580,14 @@ impl Db { impl Db { pub async fn insert_ref_update(&self, update: &ReceivedRefUpdate) -> Result<()> { + // `received_ref_updates` is slug-scoped and cascaded by `delete_repo_by_id`; + // take the shared advisory lock on the same normalized slug so a peer-fed + // insert cannot land inside the purge's sibling-check-then-cascade window + // (same shape and rationale as `upsert_branch_cid`; single lock, no deadlock + // exposure). + let slug = normalize_repo_slug(&update.repo); + let mut tx = self.pool.begin().await?; + lock_repo_slug(&mut tx, &slug).await?; sqlx::query( "INSERT INTO received_ref_updates (id, node_did, pusher_did, repo, ref_name, old_sha, new_sha, timestamp, @@ -2331,8 +2607,9 @@ impl Db { .bind(&update.received_at) .bind(&update.from_peer) .bind(&update.owner_did) - .execute(&self.pool) + .execute(&mut *tx) .await?; + tx.commit().await?; Ok(()) } @@ -2676,6 +2953,13 @@ impl Db { pub async fn record_arweave_anchor(&self, input: &RecordAnchorInput<'_>) -> Result<()> { let id = Uuid::new_v4().to_string(); let now = Utc::now().to_rfc3339(); + // `arweave_anchors` is slug-scoped and cascaded by `delete_repo_by_id`; + // take the shared advisory lock on the same normalized slug so a post-push + // anchor cannot land inside the purge's sibling-check-then-cascade window + // (same shape as `upsert_branch_cid`; single lock, no deadlock exposure). + let slug = normalize_repo_slug(input.repo); + let mut tx = self.pool.begin().await?; + lock_repo_slug(&mut tx, &slug).await?; sqlx::query( "INSERT INTO arweave_anchors (id, repo, owner_did, ref_name, old_sha, new_sha, cid, irys_tx_id, arweave_url, node_did, anchored_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11)", @@ -2691,8 +2975,9 @@ impl Db { .bind(input.arweave_url) .bind(input.node_did) .bind(&now) - .execute(&self.pool) + .execute(&mut *tx) .await?; + tx.commit().await?; Ok(()) } @@ -3746,7 +4031,7 @@ mod agent_discovery_tests { #[cfg(test)] mod dedup_db_tests { - use super::{Db, RepoRecord}; + use super::{normalize_owner_key, BountyRecord, Db, RecordAnchorInput, RepoRecord}; use chrono::{DateTime, Utc}; use sqlx::PgPool; @@ -3785,6 +4070,709 @@ mod dedup_db_tests { } } + /// U2 (M7): deleting a repo must not orphan its child rows. Seeds one child in + /// a `repo_id`-keyed table (ref_certificates), one in a `repo`-slug-keyed table + /// (branch_cids), and a PR with a grandchild review (`pr_id`-keyed). Before the + /// transactional cascade these all survive `delete_repo_by_id` (RED); after, the + /// row and every child are gone (GREEN). + #[sqlx::test] + async fn delete_repo_by_id_removes_child_rows(pool: PgPool) { + let db = db(pool).await; + let owner = "did:key:z6MkChildOwnerFixtureForCascadeDelete"; + let repo = rec( + "rid-cascade", + owner, + "victim", + "d", + "2026-01-01T00:00:00Z", + "2026-01-01T00:00:00Z", + ); + db.create_repo(&repo).await.unwrap(); + let slug = format!("{}/victim", crate::db::normalize_owner_key(owner)); + + sqlx::query( + "INSERT INTO ref_certificates (id, repo_id, ref_name, old_sha, new_sha, pusher_did, node_did, signature, issued_at) + VALUES ('rc1', 'rid-cascade', 'refs/heads/main', '0', '1', 'p', 'n', 'sig', '2026-01-01T00:00:00Z')", + ).execute(db.pool()).await.unwrap(); + sqlx::query( + "INSERT INTO branch_cids (repo, ref_name, sha, cid, node_did, updated_at) + VALUES ($1, 'refs/heads/main', '1', 'cid', 'n', '2026-01-01T00:00:00Z')", + ) + .bind(&slug) + .execute(db.pool()) + .await + .unwrap(); + sqlx::query( + "INSERT INTO pull_requests (id, repo_id, number, title, author_did, source_branch, target_branch, created_at, updated_at) + VALUES ('pr1', 'rid-cascade', 1, 't', 'a', 'b', 'main', '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z')", + ).execute(db.pool()).await.unwrap(); + sqlx::query( + "INSERT INTO pr_reviews (id, pr_id, reviewer_did, status, created_at) + VALUES ('rev1', 'pr1', 'r', 'approved', '2026-01-01T00:00:00Z')", + ) + .execute(db.pool()) + .await + .unwrap(); + + let removed = db.delete_repo_by_id("rid-cascade").await.unwrap(); + assert_eq!(removed, 1, "parent repo row deleted"); + + async fn count(db: &Db, sql: &str, arg: &str) -> i64 { + sqlx::query_scalar::<_, i64>(sql) + .bind(arg) + .fetch_one(db.pool()) + .await + .unwrap() + } + assert_eq!( + count( + &db, + "SELECT COUNT(*) FROM ref_certificates WHERE repo_id=$1", + "rid-cascade" + ) + .await, + 0, + "repo_id-keyed child (ref_certificates) must be deleted" + ); + assert_eq!( + count(&db, "SELECT COUNT(*) FROM branch_cids WHERE repo=$1", &slug).await, + 0, + "slug-keyed child (branch_cids) must be deleted" + ); + assert_eq!( + count( + &db, + "SELECT COUNT(*) FROM pull_requests WHERE repo_id=$1", + "rid-cascade" + ) + .await, + 0, + "pull_requests must be deleted" + ); + assert_eq!( + count(&db, "SELECT COUNT(*) FROM pr_reviews WHERE pr_id=$1", "pr1").await, + 0, + "PR grandchild (pr_reviews) must be deleted with its parent PR" + ); + } + + /// U4 gate-raise (INV-9 same-method slug collision): a canonical `did:key:zX` + /// row and a bare `zX` mirror of the same name both normalize to slug `zX/name`, + /// which is the key the slug-scoped child tables use. Purging the EMPTY canonical + /// row must NOT run the slug-scoped cascade, because that would wipe the surviving + /// non-empty mirror's branch_cids/arweave_anchors (keyed on the shared slug). The + /// physical `did:key:` row still goes; the sibling's slug-scoped records stay. + /// RED on base: the cascade fires and both survivor rows are deleted. + #[sqlx::test] + async fn delete_repo_keeps_sibling_slug_records_on_collision(pool: PgPool) { + let db = db(pool).await; + // Two rows collapsing to slug `z6MkSiblingCollisionFixture/victim`: an empty + // canonical `did:key:` row (deletion target) and a non-empty bare mirror. + let canonical = rec( + "rid-canonical-empty", + "did:key:z6MkSiblingCollisionFixture", + "victim", + "empty canonical", + "2026-01-01T00:00:00Z", + "2026-01-01T00:00:00Z", + ); + let mirror = rec( + "z6MkSiblingCollisionFixture/victim", + "z6MkSiblingCollisionFixture", + "victim", + "non-empty mirror", + "2026-02-01T00:00:00Z", + "2026-02-01T00:00:00Z", + ); + db.create_repo(&canonical).await.unwrap(); + db.create_repo(&mirror).await.unwrap(); + + // Both rows resolve to this slug; the child rows belong to the mirror. + let slug = format!( + "{}/victim", + crate::db::normalize_owner_key("did:key:z6MkSiblingCollisionFixture") + ); + sqlx::query( + "INSERT INTO branch_cids (repo, ref_name, sha, cid, node_did, updated_at) + VALUES ($1, 'refs/heads/main', '1', 'cid', 'n', '2026-02-01T00:00:00Z')", + ) + .bind(&slug) + .execute(db.pool()) + .await + .unwrap(); + sqlx::query( + "INSERT INTO arweave_anchors (id, repo, owner_did, ref_name, old_sha, new_sha, cid, irys_tx_id, arweave_url, node_did, anchored_at) + VALUES ('anch1', $1, 'z6MkSiblingCollisionFixture', 'refs/heads/main', '0', '1', 'cid', 'irys', 'https://ar/1', 'n', '2026-02-01T00:00:00Z')", + ) + .bind(&slug) + .execute(db.pool()) + .await + .unwrap(); + + // An OPEN bounty on the shared owner/name — it belongs to the logical repo the + // surviving mirror still serves, so purging the empty sibling must NOT tombstone it. + db.create_bounty(&BountyRecord { + id: "bnt-collision".to_string(), + repo_owner: "z6MkSiblingCollisionFixture".to_string(), + repo_name: "victim".to_string(), + issue_id: None, + title: "sibling bounty".to_string(), + amount: 5_000, + creator_did: "did:key:z6MkCreator".to_string(), + claimant_did: None, + claimant_wallet: None, + pr_id: None, + status: "open".to_string(), + created_at: "2026-02-01T00:00:00Z".to_string(), + claimed_at: None, + submitted_at: None, + completed_at: None, + deadline_secs: 604_800, + tx_hash: None, + }) + .await + .unwrap(); + + // Purge the empty canonical row. + let removed = db.delete_repo_by_id("rid-canonical-empty").await.unwrap(); + assert_eq!(removed, 1, "the empty canonical row is deleted"); + + async fn count(db: &Db, sql: &str, arg: &str) -> i64 { + sqlx::query_scalar::<_, i64>(sql) + .bind(arg) + .fetch_one(db.pool()) + .await + .unwrap() + } + // The surviving mirror keeps its slug-scoped records (the collision guard + // skips the slug-scoped cascade because a sibling row shares the slug). + assert!( + count(&db, "SELECT COUNT(*) FROM branch_cids WHERE repo=$1", &slug).await > 0, + "sibling's branch_cids must survive the collision purge" + ); + assert!( + count( + &db, + "SELECT COUNT(*) FROM arweave_anchors WHERE repo=$1", + &slug + ) + .await + > 0, + "sibling's arweave_anchors must survive the collision purge" + ); + assert_eq!( + count( + &db, + "SELECT COUNT(*) FROM repos WHERE id=$1", + "z6MkSiblingCollisionFixture/victim" + ) + .await, + 1, + "the surviving mirror repos row is untouched" + ); + // The bounty is NOT tombstoned: the mirror still serves the logical repo, so + // the tombstone is gated on no sibling surviving. + assert_eq!( + db.get_bounty("bnt-collision") + .await + .unwrap() + .unwrap() + .status, + "open", + "a bounty on a repo whose mirror survives the purge must stay open, not be tombstoned" + ); + } + + /// U1 (F1): the purge cascade must be serialized with mirror ingestion on the + /// shared slug. `delete_repo_by_id` runs a point-in-time sibling check and then, + /// only if no sibling was seen, wipes the slug-scoped child rows / tombstones the + /// bounty. A concurrent `upsert_mirror_repo` that inserts a colliding mirror row + /// AFTER that check but before/around the cascade would leave a half-wipe: the + /// mirror's `repos` row survives while its live child rows / bounty are gone. + /// + /// Deterministic race, with BOTH sides going through the real code: hold a session + /// advisory lock on the same `hashtextextended` slug key both sides hash, spawn the + /// REAL `upsert_mirror_repo` (parks on its own `lock_repo_slug`, first in the wait + /// queue), assert the mirror row has NOT landed while parked, then spawn the purge + /// (parks second) and release. Advisory-lock waiters are granted FIFO, so the + /// ingest commits the mirror row first and the purge's sibling check then SEES it + /// and skips the cascade: mirror row, child rows, and bounty all survive (GREEN). + /// Remove `lock_repo_slug` from `upsert_mirror_repo` and the ingest no longer + /// parks: the mirror row appears while the session lock is still held and the + /// mid-park absence assert fails (RED). Remove it from `delete_repo_by_id` and the + /// purge no longer blocks: it runs its sibling check before the parked ingest has + /// committed, cascades, and half-wipes the child rows / bounty (RED). + #[sqlx::test] + async fn purge_serializes_with_mirror_ingest_via_advisory_lock(pool: PgPool) { + let db = db(pool).await; + // Purge target: an empty canonical `did:key:` row. + let owner = "did:key:z6MkPurgeRaceMirrorFixtureForLock"; + let target = rec( + "rid-purge-race-target", + owner, + "victim", + "empty canonical", + "2026-01-01T00:00:00Z", + "2026-01-01T00:00:00Z", + ); + db.create_repo(&target).await.unwrap(); + + // The mirror's slug-scoped live state, keyed on the shared slug. The mirror's + // `repos` row is deliberately NOT present yet — ingestion lands it mid-race. + let slug = format!("{}/victim", crate::db::normalize_owner_key(owner)); + sqlx::query( + "INSERT INTO branch_cids (repo, ref_name, sha, cid, node_did, updated_at) + VALUES ($1, 'refs/heads/main', '1', 'cid', 'n', '2026-02-01T00:00:00Z')", + ) + .bind(&slug) + .execute(db.pool()) + .await + .unwrap(); + db.create_bounty(&BountyRecord { + id: "bnt-purge-race".to_string(), + repo_owner: "z6MkPurgeRaceMirrorFixtureForLock".to_string(), + repo_name: "victim".to_string(), + issue_id: None, + title: "mirror bounty".to_string(), + amount: 5_000, + creator_did: "did:key:z6MkCreator".to_string(), + claimant_did: None, + claimant_wallet: None, + pr_id: None, + status: "open".to_string(), + created_at: "2026-02-01T00:00:00Z".to_string(), + claimed_at: None, + submitted_at: None, + completed_at: None, + deadline_secs: 604_800, + tx_hash: None, + }) + .await + .unwrap(); + + // Hold a session-scoped advisory lock on the exact slug key the cascade takes. + // A session lock and the cascade's xact lock share Postgres' advisory lock + // space, so the purge's `pg_advisory_xact_lock` blocks on this until we unlock. + let mut lock_conn = db.pool().acquire().await.unwrap(); + sqlx::query("SELECT pg_advisory_lock(hashtextextended($1, 0))") + .bind(&slug) + .execute(&mut *lock_conn) + .await + .unwrap(); + + // Spawn the REAL mirror ingest; its own `lock_repo_slug` inside + // `upsert_mirror_repo` parks it on the held session lock, first in the wait + // queue. Bare owner short form, so it normalizes to the identical slug the + // purge locks. + let pool_for_ingest = db.pool().clone(); + let ingest = tokio::spawn(async move { + Db::for_testing(pool_for_ingest) + .upsert_mirror_repo( + "z6MkPurgeRaceMirrorFixtureForLock", + "victim", + "/srv/mirror", + None, + false, + ) + .await + }); + + // Give the ingest time to reach the lock wait, then assert it is actually + // parked: the mirror row must NOT exist while the session lock is held. This + // is the load-bearing check for the ingest side; without `lock_repo_slug` in + // `upsert_mirror_repo` the insert lands during this sleep (RED). + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + let landed_early: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM repos WHERE id=$1") + .bind(&slug) + .fetch_one(db.pool()) + .await + .unwrap(); + assert_eq!( + landed_early, 0, + "the real upsert_mirror_repo must block on the slug advisory lock, not insert past it" + ); + + // Spawn the purge; it parks on the same lock, second in the queue. Without + // `lock_repo_slug` in `delete_repo_by_id` it runs immediately instead: its + // sibling check sees no mirror row (the ingest is still parked), so it + // cascades the child rows and tombstones the bounty (RED below). + let pool_for_purge = db.pool().clone(); + let purge = tokio::spawn(async move { + Db::for_testing(pool_for_purge) + .delete_repo_by_id("rid-purge-race-target") + .await + }); + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + + // Release the lock. Advisory-lock waiters are woken FIFO: the ingest (queued + // first) commits the mirror row, then the purge acquires the lock and its + // sibling check observes the committed row. + sqlx::query("SELECT pg_advisory_unlock(hashtextextended($1, 0))") + .bind(&slug) + .execute(&mut *lock_conn) + .await + .unwrap(); + + ingest.await.unwrap().unwrap(); + let removed = purge.await.unwrap().unwrap(); + assert_eq!(removed, 1, "the empty canonical target row is purged"); + + async fn count(db: &Db, sql: &str, arg: &str) -> i64 { + sqlx::query_scalar::<_, i64>(sql) + .bind(arg) + .fetch_one(db.pool()) + .await + .unwrap() + } + // No half-wipe: the mirror row AND its slug-scoped state are all present. + assert_eq!( + count(&db, "SELECT COUNT(*) FROM repos WHERE id=$1", &slug).await, + 1, + "the mirror repos row survives the purge" + ); + assert!( + count(&db, "SELECT COUNT(*) FROM branch_cids WHERE repo=$1", &slug).await > 0, + "mirror branch_cids must survive: the purge serialized behind the ingest and saw the sibling" + ); + assert_eq!( + db.get_bounty("bnt-purge-race") + .await + .unwrap() + .unwrap() + .status, + "open", + "the mirror's bounty must stay open, not be tombstoned by the racing purge" + ); + } + + /// Sibling of the mirror-ingest race above, for the arweave-anchor writer. + /// `record_arweave_anchor` inserts a slug-scoped child row (`arweave_anchors`) + /// that `delete_repo_by_id`'s cascade wipes, so it must serialize on the same + /// `lock_repo_slug` key or a post-push anchor can land inside the purge's + /// sibling-check-then-cascade window. Deterministic choreography, same as the + /// mirror test: hold a session advisory lock on the exact `hashtextextended` + /// slug key, spawn the REAL `record_arweave_anchor` (its own `lock_repo_slug` + /// parks it), assert mid-park that the anchor row has NOT landed (this is the + /// load-bearing check: remove `lock_repo_slug` from `record_arweave_anchor` + /// and the insert lands during the sleep, RED), then release and assert the + /// row lands. The writer is handed the full-DID `owner/name` form so the test + /// also pins the `normalize_repo_slug` folding to the purge's key. + #[sqlx::test] + async fn purge_serializes_with_arweave_anchor_via_advisory_lock(pool: PgPool) { + let db = db(pool).await; + let owner = "did:key:z6MkAnchorRaceFixtureForSlugLock"; + let target = rec( + "rid-anchor-race-target", + owner, + "victim", + "canonical", + "2026-01-01T00:00:00Z", + "2026-01-01T00:00:00Z", + ); + db.create_repo(&target).await.unwrap(); + let slug = format!("{}/victim", crate::db::normalize_owner_key(owner)); + + // Hold a session-scoped advisory lock on the exact slug key the anchor + // writer's xact lock hashes; they share Postgres' advisory lock space. + let mut lock_conn = db.pool().acquire().await.unwrap(); + sqlx::query("SELECT pg_advisory_lock(hashtextextended($1, 0))") + .bind(&slug) + .execute(&mut *lock_conn) + .await + .unwrap(); + + // Spawn the REAL anchor writer with the full-DID slug form; its + // `normalize_repo_slug` folds it to the identical key the session lock + // holds, so `lock_repo_slug` parks it here. + let pool_for_anchor = db.pool().clone(); + let repo_arg = format!("{owner}/victim"); + let repo_arg_for_task = repo_arg.clone(); + let anchor = tokio::spawn(async move { + Db::for_testing(pool_for_anchor) + .record_arweave_anchor(&RecordAnchorInput { + repo: &repo_arg_for_task, + owner_did: "did:key:z6MkAnchorRaceFixtureForSlugLock", + ref_name: "refs/heads/main", + old_sha: "0", + new_sha: "1", + cid: Some("cid"), + irys_tx_id: "irys-tx-race", + arweave_url: "https://ar/anchor-race", + node_did: "did:key:z6MkNodeFixture", + }) + .await + }); + + // Give the writer time to reach the lock wait, then assert it is actually + // parked: no anchor row while the session lock is held. Without + // `lock_repo_slug` in `record_arweave_anchor` the insert commits during + // this sleep and the assert fails (RED). + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + let landed_early: i64 = + sqlx::query_scalar("SELECT COUNT(*) FROM arweave_anchors WHERE repo=$1") + .bind(&repo_arg) + .fetch_one(db.pool()) + .await + .unwrap(); + assert_eq!( + landed_early, 0, + "record_arweave_anchor must block on the slug advisory lock, not insert past it" + ); + + // Release; the parked writer acquires the lock and commits. + sqlx::query("SELECT pg_advisory_unlock(hashtextextended($1, 0))") + .bind(&slug) + .execute(&mut *lock_conn) + .await + .unwrap(); + anchor.await.unwrap().unwrap(); + + let landed: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM arweave_anchors WHERE repo=$1") + .bind(&repo_arg) + .fetch_one(db.pool()) + .await + .unwrap(); + assert_eq!(landed, 1, "the anchor lands once the lock is released"); + } + + /// U8 (F): bounties key on (repo_owner, repo_name), not an immutable repo id, + /// and `delete_repo_by_id` deliberately does not delete them (financial record). + /// Without tombstoning, a purged repo's OPEN bounty stays claimable and silently + /// re-attaches to a same-name repo recreated by the (spam) owner. The purge must + /// transition the affected non-terminal bounties to the terminal `purged` status + /// (preserving amount/wallet/tx_hash) so no claim/dispute path can re-expose them. + /// A bounty on an unrelated surviving repo must be left untouched. + /// RED on base: the bounty stays `open` and `claim_bounty` succeeds after recreate. + #[sqlx::test] + async fn delete_repo_by_id_tombstones_bounties_against_recreation(pool: PgPool) { + let db = db(pool).await; + let owner = "did:key:z6MkBountyPurgeFixtureForTombstone"; + let victim = rec( + "rid-bounty-victim", + owner, + "victim", + "spam repo", + "2026-01-01T00:00:00Z", + "2026-01-01T00:00:00Z", + ); + db.create_repo(&victim).await.unwrap(); + + // An unrelated surviving repo with its own open bounty. + let survivor_owner = "did:key:z6MkBountySurvivorFixtureUntouched"; + let survivor = rec( + "rid-bounty-survivor", + survivor_owner, + "keeper", + "legit repo", + "2026-01-01T00:00:00Z", + "2026-01-01T00:00:00Z", + ); + db.create_repo(&survivor).await.unwrap(); + + let bounty = |id: &str, r_owner: &str, r_name: &str, status: &str| BountyRecord { + id: id.to_string(), + repo_owner: r_owner.to_string(), + repo_name: r_name.to_string(), + issue_id: None, + title: "fix the bug".to_string(), + amount: 5_000, + creator_did: "did:key:z6MkCreator".to_string(), + claimant_did: None, + claimant_wallet: None, + pr_id: None, + status: status.to_string(), + created_at: "2026-01-01T00:00:00Z".to_string(), + claimed_at: None, + submitted_at: None, + completed_at: None, + deadline_secs: 604_800, + tx_hash: None, + }; + db.create_bounty(&bounty("bnt-victim", owner, "victim", "open")) + .await + .unwrap(); + // The common `create_bounty` path stores `repo_owner` verbatim from the URL + // Path param, which is the bare SHORT owner form (`z6Mk...`), not the full + // `did:key:z6Mk...`. The tombstone must match this form too or a short-form + // bounty stays `open` and re-attaches to the recreated repo. + let short_owner = normalize_owner_key(owner); + db.create_bounty(&bounty("bnt-victim-short", short_owner, "victim", "open")) + .await + .unwrap(); + // A mid-flight `claimed` bounty must also be tombstoned: otherwise a later + // dispute (which resets `claimed`/`submitted` back to `open`) would re-expose + // it on the recreated repo. + db.create_bounty(&bounty("bnt-victim-claimed", owner, "victim", "claimed")) + .await + .unwrap(); + // A mid-flight `submitted` bounty is in the tombstone set too. + db.create_bounty(&bounty( + "bnt-victim-submitted", + owner, + "victim", + "submitted", + )) + .await + .unwrap(); + // An already-terminal `completed` bounty is outside the tombstone set. + db.create_bounty(&bounty( + "bnt-victim-completed", + owner, + "victim", + "completed", + )) + .await + .unwrap(); + db.create_bounty(&bounty("bnt-survivor", survivor_owner, "keeper", "open")) + .await + .unwrap(); + + // Purge the spam repo, then let the owner recreate the same owner/name. + let removed = db.delete_repo_by_id("rid-bounty-victim").await.unwrap(); + assert_eq!(removed, 1, "the purged repo row is deleted"); + let recreated = rec( + "rid-bounty-recreated", + owner, + "victim", + "recreated repo", + "2026-03-01T00:00:00Z", + "2026-03-01T00:00:00Z", + ); + db.create_repo(&recreated).await.unwrap(); + + // The purged repo's bounty is tombstoned, so a claim is a no-op: the + // `UPDATE ... WHERE status='open'` matches nothing and it stays `purged`. + db.claim_bounty( + "bnt-victim", + "did:key:z6MkAttacker", + Some("0xattacker"), + "2026-03-02T00:00:00Z", + ) + .await + .unwrap(); + let victim_bounty = db.get_bounty("bnt-victim").await.unwrap().unwrap(); + assert_eq!( + victim_bounty.status, "purged", + "the purged repo's bounty must be tombstoned, not re-claimable" + ); + assert!( + victim_bounty.claimant_did.is_none(), + "the tombstoned bounty must not accept a new claimant" + ); + // The financial record is preserved (tombstoned, not deleted). + assert_eq!(victim_bounty.amount, 5_000, "amount preserved on tombstone"); + assert_eq!( + victim_bounty.creator_did, "did:key:z6MkCreator", + "creator_did preserved on tombstone" + ); + + // The short-owner-form bounty (the common create_bounty path) is tombstoned + // too, so it cannot re-attach to the recreated repo and become claimable. + db.claim_bounty( + "bnt-victim-short", + "did:key:z6MkAttacker", + Some("0xattacker"), + "2026-03-02T00:00:00Z", + ) + .await + .unwrap(); + let short_bounty = db.get_bounty("bnt-victim-short").await.unwrap().unwrap(); + assert_eq!( + short_bounty.status, "purged", + "a short-owner-form bounty on the purged repo must also be tombstoned" + ); + assert!( + short_bounty.claimant_did.is_none(), + "the tombstoned short-form bounty must not accept a new claimant" + ); + + // A mid-flight claimed bounty on the same repo is tombstoned too (so a + // dispute cannot reopen it against the recreated repo). + let claimed_bounty = db.get_bounty("bnt-victim-claimed").await.unwrap().unwrap(); + assert_eq!( + claimed_bounty.status, "purged", + "a claimed bounty on the purged repo must also be tombstoned" + ); + // Dispute gates on status IN ('claimed','submitted'); a tombstoned bounty is + // `purged`, so dispute is inert and cannot reopen it back to `open`. + db.dispute_bounty("bnt-victim-claimed").await.unwrap(); + let disputed_bounty = db.get_bounty("bnt-victim-claimed").await.unwrap().unwrap(); + assert_eq!( + disputed_bounty.status, "purged", + "dispute must not reopen a tombstoned bounty" + ); + + // A mid-flight submitted bounty on the same repo is tombstoned too. + let submitted_bounty = db + .get_bounty("bnt-victim-submitted") + .await + .unwrap() + .unwrap(); + assert_eq!( + submitted_bounty.status, "purged", + "a submitted bounty on the purged repo must also be tombstoned" + ); + + // An already-terminal completed bounty is left as-is (not in the tombstone set). + let completed_bounty = db + .get_bounty("bnt-victim-completed") + .await + .unwrap() + .unwrap(); + assert_eq!( + completed_bounty.status, "completed", + "a terminal completed bounty must be left untouched" + ); + + // The unrelated survivor's bounty is untouched and still claimable. + let survivor_bounty = db.get_bounty("bnt-survivor").await.unwrap().unwrap(); + assert_eq!( + survivor_bounty.status, "open", + "an unrelated repo's bounty must stay open" + ); + + // Symmetric owner-form case (the mirror of `bnt-victim-short`). Here the repo + // row is the BARE short-owner mirror (`owner_did = z6X`), while the bounty was + // stored in the FULL `did:key:z6X` form, the shape a full-DID URL that + // resolves to the bare mirror produces. The tombstone must normalize BOTH the + // stored `repo_owner` and the target, or this full-form bounty stays `open` + // when the bare row is purged and re-attaches to a recreated repo, the same + // re-attach hole in the opposite direction. `delete_repo_by_id` is reachable on + // bare mirror rows (`list_repos_by_owner_did` matches under normalization). + // RED before the symmetric fix (only the target side was normalized): stays + // `open`. GREEN after: `purged`. + let bare_owner = "z6MkBareOwnerFixtureForSymmetricTombstone"; + let bare = rec( + "rid-bounty-bare-victim", + bare_owner, + "victim", + "bare spam repo", + "2026-01-01T00:00:00Z", + "2026-01-01T00:00:00Z", + ); + db.create_repo(&bare).await.unwrap(); + db.create_bounty(&bounty( + "bnt-bare-full", + &format!("did:key:{bare_owner}"), + "victim", + "open", + )) + .await + .unwrap(); + let removed_bare = db + .delete_repo_by_id("rid-bounty-bare-victim") + .await + .unwrap(); + assert_eq!(removed_bare, 1, "the bare-owner repo row is deleted"); + let bare_full_bounty = db.get_bounty("bnt-bare-full").await.unwrap().unwrap(); + assert_eq!( + bare_full_bounty.status, "purged", + "a FULL-form bounty on a BARE-owner repo row must be tombstoned too \ + (symmetric normalization); RED before: only the target side was \ + normalized, so the full-form bounty stayed open" + ); + } + /// The canonical `did:key:` row and the short-owner mirror row of one logical /// repo collapse to a single deduped entry: the canonical row wins and inherits /// the group's most recent `updated_at`. diff --git a/crates/gitlawb-node/src/error.rs b/crates/gitlawb-node/src/error.rs index bdd5aec9..5c93c2f5 100644 --- a/crates/gitlawb-node/src/error.rs +++ b/crates/gitlawb-node/src/error.rs @@ -35,9 +35,6 @@ pub enum AppError { #[error("invalid request: {0}")] BadRequest(String), - #[error("too many requests: {0}")] - TooManyRequests(String), - #[error("incomplete: {0}")] Incomplete(String), @@ -47,6 +44,9 @@ pub enum AppError { #[error("git service timed out: {0}")] Timeout(String), + #[error("service unavailable: {0}")] + Unavailable(String), + #[error("database error: {0}")] Db(#[from] sqlx::Error), @@ -132,9 +132,6 @@ impl IntoResponse for AppError { // IcaptchaProofRequired is handled above (it carries extra headers/fields). AppError::IcaptchaProofRequired { .. } => unreachable!("handled before this match"), AppError::BadRequest(msg) => (StatusCode::BAD_REQUEST, "bad_request", msg.clone()), - AppError::TooManyRequests(msg) => { - (StatusCode::TOO_MANY_REQUESTS, "rate_limited", msg.clone()) - } AppError::Incomplete(msg) => { (StatusCode::UNPROCESSABLE_ENTITY, "incomplete", msg.clone()) } @@ -142,6 +139,14 @@ impl IntoResponse for AppError { // 504, distinct from the 500 git_error and from the read-gate's 404 / // the auth 401, so the client can tell a deadline from a failure. AppError::Timeout(msg) => (StatusCode::GATEWAY_TIMEOUT, "git_timeout", msg.clone()), + // 503, retryable: a transient resource contention (e.g. the repo lock + // pool is pinned by a write burst or a purge holds the key), distinct + // from the 500 git_error so the client knows to back off and retry. + AppError::Unavailable(msg) => ( + StatusCode::SERVICE_UNAVAILABLE, + "service_unavailable", + msg.clone(), + ), AppError::Db(e) if db_unavailable(e) => ( StatusCode::SERVICE_UNAVAILABLE, DB_UNAVAILABLE_CODE, @@ -182,4 +187,20 @@ mod tests { StatusCode::INTERNAL_SERVER_ERROR ); } + + #[test] + fn unavailable_maps_to_503_distinct_from_git_500() { + // The create-repo lock-acquire failure returns Unavailable so a client sees + // a retryable 503, not a terminal 500 git_error. + assert_eq!( + AppError::Unavailable("lock pool pinned".into()) + .into_response() + .status(), + StatusCode::SERVICE_UNAVAILABLE + ); + assert_eq!( + AppError::Git("x".into()).into_response().status(), + StatusCode::INTERNAL_SERVER_ERROR + ); + } } diff --git a/crates/gitlawb-node/src/git/issues.rs b/crates/gitlawb-node/src/git/issues.rs index 73098302..9e1277ff 100644 --- a/crates/gitlawb-node/src/git/issues.rs +++ b/crates/gitlawb-node/src/git/issues.rs @@ -61,6 +61,51 @@ pub fn create_issue(repo_path: &Path, issue_id: &str, json: &str) -> Result<()> Ok(()) } +/// Delete an issue ref. Used to roll back a locally written issue whose +/// durable upload failed, so a retry does not duplicate it. The dangling +/// blob object is harmless. +pub fn delete_issue_ref(repo_path: &Path, issue_id: &str) -> Result<()> { + let ref_name = format!("refs/gitlawb/issues/{issue_id}"); + let output = Command::new("git") + .args(["update-ref", "-d", &ref_name]) + .current_dir(repo_path) + .output() + .context("failed to run git update-ref -d")?; + + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + anyhow::bail!("git update-ref -d failed: {stderr}"); + } + + Ok(()) +} + +/// Read the object id an issue ref currently points to (resolving an 8-char +/// prefix), returning `(full_id, oid)`. Returns None if no such issue exists. +/// Used to snapshot an issue ref's pre-mutation state so a failed durable +/// upload can restore it, mirroring `create_issue`'s rollback. +pub fn issue_ref_oid(repo_path: &Path, issue_id: &str) -> Result> { + let full_id = match resolve_issue_id(repo_path, issue_id)? { + Some(id) => id, + None => return Ok(None), + }; + let ref_name = format!("refs/gitlawb/issues/{full_id}"); + let oid = match crate::git::store::ref_oid(repo_path, &ref_name)? { + Some(oid) => oid, + None => return Ok(None), + }; + Ok(Some((full_id, oid))) +} + +/// Reset an issue ref back to a captured object id — rolls back a local +/// mutation (e.g. a close) whose durable upload failed, so a local-fast-path +/// read does not serve the un-uploaded state. `full_id` must be the resolved +/// UUID (from [`issue_ref_oid`]). The now-dangling object is harmless. +pub fn restore_issue_ref(repo_path: &Path, full_id: &str, oid: &str) -> Result<()> { + let ref_name = format!("refs/gitlawb/issues/{full_id}"); + crate::git::store::set_ref(repo_path, &ref_name, oid) +} + /// List all issue refs and return their JSON content. pub fn list_issues(repo_path: &Path) -> Result> { // List all refs under refs/gitlawb/issues/ diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index a5c367e9..8b9c540f 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -3,12 +3,13 @@ //! Every handler that needs access to a git repo on disk goes through `RepoStore`: //! //! - `acquire()` — ensures the repo is on local disk (downloads from Tigris on cache miss). -//! - `release_after_write()` — uploads the updated repo to Tigris after a write operation. +//! - `upload_under_guard()`: uploads the updated repo to Tigris after a write, under a +//! per-repo advisory lock the caller already holds (the fork tail's durability step). //! - `init()` — creates a new bare repo locally and uploads to Tigris. //! //! When Tigris is disabled (bucket empty), this is a simple passthrough to local disk. -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::path::{Path, PathBuf}; use std::sync::Arc; @@ -18,40 +19,84 @@ use tokio::sync::Mutex; use tracing::{debug, info, warn}; use super::store; -use super::tigris::TigrisClient; +use super::tigris::ObjectStore; -/// Centralized repo storage: local disk cache + optional Tigris backend. +mod download; +use download::DownloadOutcome; + +/// Default bound on `release()`'s post-write archive upload (INV-22). Generous +/// for a large-repo PUT while still guaranteeing the guard's pinned advisory- +/// lock connection returns to the pool within a fixed window even if the upload +/// stalls indefinitely. +const DEFAULT_RELEASE_UPLOAD_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(300); + +/// Centralized repo storage: local disk cache + optional object-store backend. #[derive(Clone)] pub struct RepoStore { repos_dir: PathBuf, - tigris: Option, - /// Shared Postgres pool for advisory locks. - pool: PgPool, - /// Tracks repos already confirmed to exist in Tigris — avoids redundant - /// HEAD checks and background uploads for repos we've already migrated. + object_store: Option>, + /// Dedicated Postgres pool for advisory-lock guard connections, separate + /// from the app pool that serves normal request handlers. Each write guard + /// pins one connection from here for its whole lifetime, so a concurrent- + /// push burst pins connections from this pool rather than starving the app + /// pool. Sized by GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS to peak writers. + lock_pool: PgPool, + /// Upper bound on the post-write archive upload in `release()`, so a stalled + /// upload cannot pin the guard's lock connection indefinitely. + release_upload_timeout: std::time::Duration, + /// Tracks repos already confirmed to exist in the object store — avoids + /// redundant HEAD checks and background uploads for repos we've migrated. migrated: Arc>>, + /// Per-repo async download coordination (KTD-3): concurrent readers of the + /// same repo serialize here so only one runs the network download while + /// the rest await it and serve what the winner published. Entries are + /// created only after a caller has confirmed the archive exists (reached + /// the download branch) and are removed when the holder finishes (on every + /// exit, including handler cancellation, via `DownloadEntryGuard`), so + /// permissionless requests for arbitrary names cannot grow the map. The + /// OUTER map is a std::sync::Mutex (only ever held briefly for a get/insert/ + /// remove, never across an await), so the RAII entry guard's synchronous + /// `Drop` can prune it. The INNER per-repo `Arc>` stays a + /// tokio::sync::Mutex — it is the async serialization primitive readers await. + download_locks: Arc>>>>, } impl RepoStore { #[cfg(test)] - pub fn for_testing(repos_dir: PathBuf, pool: PgPool) -> Self { + pub fn for_testing(repos_dir: PathBuf, lock_pool: PgPool) -> Self { Self { repos_dir, - tigris: None, - pool, + object_store: None, + lock_pool, + release_upload_timeout: DEFAULT_RELEASE_UPLOAD_TIMEOUT, migrated: Arc::new(tokio::sync::Mutex::new(std::collections::HashSet::new())), + download_locks: Arc::new(std::sync::Mutex::new(HashMap::new())), } } - pub fn new(repos_dir: PathBuf, tigris: Option, pool: PgPool) -> Self { + pub fn new( + repos_dir: PathBuf, + object_store: Option>, + lock_pool: PgPool, + ) -> Self { Self { repos_dir, - tigris, - pool, + object_store, + lock_pool, + release_upload_timeout: DEFAULT_RELEASE_UPLOAD_TIMEOUT, migrated: Arc::new(Mutex::new(HashSet::new())), + download_locks: Arc::new(std::sync::Mutex::new(HashMap::new())), } } + /// Shrink the `release()` upload timeout — test-only, so a stall test can + /// assert the lock connection is freed within a short bound. + #[cfg(test)] + pub fn with_release_upload_timeout(mut self, timeout: std::time::Duration) -> Self { + self.release_upload_timeout = timeout; + self + } + /// Ensure a repo is available on local disk, downloading from Tigris if needed. /// If the repo exists locally but not yet in Tigris, a background upload is /// spawned to lazily migrate it (on-demand migration for pre-Tigris repos). @@ -63,14 +108,15 @@ impl RepoStore { if local_path.exists() { // Lazy migration: if Tigris is enabled and we haven't confirmed this // repo is in Tigris yet, check and upload in the background. - if let Some(ref tigris) = self.tigris { + if let Some(ref tigris) = self.object_store { let key = format!("{owner_slug}/{repo_name}"); let already_migrated = self.migrated.lock().await.contains(&key); if !already_migrated { + let store = self.clone(); let tigris = tigris.clone(); + let did = owner_did.to_string(); let slug = owner_slug.clone(); let name = repo_name.to_string(); - let path = local_path.clone(); let migrated = Arc::clone(&self.migrated); tokio::spawn(async move { // Check if already in Tigris before uploading @@ -80,8 +126,10 @@ impl RepoStore { } Ok(false) => { info!(repo = %name, "migrating local repo to tigris"); - if let Err(e) = tigris.upload(&slug, &name, &path).await { - warn!(repo = %name, err = %e, "lazy migration to tigris failed"); + // Upload under the per-repo lock so it can't race + // a purge. If it skipped (lock contended or dir + // gone), do NOT mark migrated — retry next acquire. + if !store.upload_locked(&did, &name, false).await { return; } info!(repo = %name, "lazy migration to tigris complete"); @@ -99,19 +147,36 @@ impl RepoStore { } // Try downloading from Tigris - if let Some(ref tigris) = self.tigris { + if let Some(ref tigris) = self.object_store { if tigris.exists(&owner_slug, repo_name).await.unwrap_or(false) { debug!(repo = %repo_name, "cache miss — downloading from tigris"); - tigris - .download(&owner_slug, repo_name, &local_path) - .await - .context("downloading repo from tigris")?; - // Mark as migrated since we just downloaded it - self.migrated - .lock() + match self + .download_published( + owner_did, + repo_name, + &owner_slug, + &local_path, + tigris, + false, + ) .await - .insert(format!("{owner_slug}/{repo_name}")); - return Ok(local_path); + .context("downloading repo from tigris")? + { + DownloadOutcome::Published => { + // Mark as migrated since we just downloaded it + self.migrated + .lock() + .await + .insert(format!("{owner_slug}/{repo_name}")); + return Ok(local_path); + } + DownloadOutcome::Skipped => { + // Degraded: the archive vanished under the lock (purged + // mid-download) or a writer/purge holds the advisory + // lock. Publish nothing and fall through to the + // missing-path outcome below. + } + } } } @@ -127,23 +192,43 @@ impl RepoStore { pub async fn acquire_fresh(&self, owner_did: &str, repo_name: &str) -> Result { let (owner_slug, local_path) = self.local_path(owner_did, repo_name)?; - if let Some(ref tigris) = self.tigris { + if let Some(ref tigris) = self.object_store { if tigris.exists(&owner_slug, repo_name).await.unwrap_or(false) { debug!(repo = %repo_name, "acquire_fresh: downloading latest from tigris"); - if let Err(e) = tigris.download(&owner_slug, repo_name, &local_path).await { - // The Tigris archive is present (HEAD ok) but unreadable — a - // corrupt/partial upload, or a transient GET failure. If we have a - // valid local copy, proceed with it rather than blocking the write; - // the post-write upload re-syncs (self-heals) Tigris. Only hard-fail - // when there is no local copy to fall back to. - if local_path.exists() { - warn!(repo = %repo_name, err = %e, - "acquire_fresh: tigris download failed — falling back to local copy"); + match self + .download_published( + owner_did, + repo_name, + &owner_slug, + &local_path, + tigris, + true, + ) + .await + { + // Published covers both the refreshed copy and a concurrent + // reader's just-published one. Skipped is the degraded + // outcome: the archive vanished under the lock (purged) or + // a writer/purge holds the advisory lock — serve the local + // copy when one still exists (the same fallback as the + // download-error arm), else the missing path. + Ok(DownloadOutcome::Published) | Ok(DownloadOutcome::Skipped) => { return Ok(local_path); } - return Err(e).context("downloading repo from tigris (fresh)"); + Err(e) => { + // The Tigris archive is present (HEAD ok) but unreadable — a + // corrupt/partial upload, or a transient GET failure. If we have a + // valid local copy, proceed with it rather than blocking the write; + // the post-write upload re-syncs (self-heals) Tigris. Only hard-fail + // when there is no local copy to fall back to. + if local_path.exists() { + warn!(repo = %repo_name, err = %e, + "acquire_fresh: tigris download failed — falling back to local copy"); + return Ok(local_path); + } + return Err(e).context("downloading repo from tigris (fresh)"); + } } - return Ok(local_path); } } @@ -157,93 +242,385 @@ impl RepoStore { let (owner_slug, local_path) = self.local_path(owner_did, repo_name)?; let lock_key = advisory_lock_key(&owner_slug, repo_name); - // Acquire Postgres advisory lock with retry using pg_try_advisory_lock - // to avoid blocking indefinitely on stale locks from crashed connections. - let mut acquired = false; - for attempt in 0..60 { - let row: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") - .bind(lock_key) - .fetch_one(&self.pool) - .await - .context("trying advisory lock")?; - if row.0 { - acquired = true; - break; + // Acquire the advisory lock on a DEDICATED connection, held for the + // guard's whole lifetime so the session-scoped lock lives on ONE + // connection and `release` unlocks it on that SAME connection, and so a + // concurrent `try_lock_repo`'s `pool.acquire()` cannot be handed the + // lock-owning connection and reentrantly re-grab it. Mirrors + // `RepoLockGuard`. Use pg_try_advisory_lock with retry to avoid blocking + // indefinitely on a stale lock from a crashed connection. + // + // Between FAILED attempts the connection is returned to the pool (the + // `drop` below) so a writer spinning on a contended repo does NOT hold a + // lock-pool connection through the retry backoff — holding one per + // spinner would starve the lock pool under a same-repo write burst. Only + // the WINNING attempt keeps its connection (the lock lives on it). + let conn = { + let mut acquired = None; + for attempt in 0..60 { + let mut c = self + .lock_pool + .acquire() + .await + .context("acquiring write-lock connection")?; + let row: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(lock_key) + .fetch_one(&mut *c) + .await + .context("trying advisory lock")?; + if row.0 { + acquired = Some(c); + break; + } + drop(c); + if attempt < 59 { + tokio::time::sleep(std::time::Duration::from_secs(1)).await; + } } - if attempt < 59 { - tokio::time::sleep(std::time::Duration::from_secs(1)).await; + match acquired { + Some(c) => c, + None => anyhow::bail!( + "could not acquire advisory lock after 60s — possible stale lock for {owner_slug}/{repo_name}" + ), } - } - if !acquired { - anyhow::bail!("could not acquire advisory lock after 60s — possible stale lock for {owner_slug}/{repo_name}"); - } + }; + + // Wrap the winning connection in the guard IMMEDIATELY, before any + // further await. The download below can be cancelled (the caller's + // future dropped mid-op) or fail; once the connection lives inside the + // guard, its `Drop` frees the advisory lock on every such exit. Before + // this ordering the lock orphaned on a bare connection returned to the + // pool holding it. (KTD-2.) + let guard = RepoWriteGuard { + owner_slug, + repo_name: repo_name.to_string(), + local_path, + lock_key, + conn: Some(conn), + object_store: self.object_store.clone(), + release_upload_timeout: self.release_upload_timeout, + }; - // Always download the latest from Tigris before writing. + // Always download the latest from the object store before writing. // Local disk may be stale if another machine pushed since our last access. - if let Some(ref tigris) = self.tigris { - if tigris.exists(&owner_slug, repo_name).await.unwrap_or(false) { - debug!(repo = %repo_name, "write acquire: downloading latest from tigris"); - if let Err(e) = tigris.download(&owner_slug, repo_name, &local_path).await { + // + // Both the exists() re-check and the download() are bounded by + // release_upload_timeout (INV-22): they run while the write advisory lock + // is held on a pinned lock-pool connection, so a stalled (never-erroring) + // GET would pin the lock and its connection indefinitely, and the local- + // copy fallback below (which only fires on Err) would never trigger on a + // hang. On timeout, exists() collapses to "not present" (skip the + // download, matching download_and_publish's bounded exists re-check), and + // download() takes the SAME arm as a download error so a hang bails + // instead of hanging. The guard was wrapped before this await (KTD-2), so + // every bail here still frees the lock via the guard's Drop. + if let Some(ref store) = self.object_store { + let archive_present = matches!( + tokio::time::timeout( + self.release_upload_timeout, + store.exists(&guard.owner_slug, &guard.repo_name), + ) + .await, + Ok(Ok(true)) + ); + if archive_present { + debug!(repo = %guard.repo_name, "write acquire: downloading latest from object store"); + let download_result = match tokio::time::timeout( + self.release_upload_timeout, + store.download(&guard.owner_slug, &guard.repo_name, &guard.local_path), + ) + .await + { + Ok(res) => res, + Err(_) => Err(anyhow::anyhow!( + "object-store download timed out after {}s under the write lock", + self.release_upload_timeout.as_secs() + )), + }; + if let Err(e) = download_result { // Same self-healing fallback as acquire_fresh: a corrupt/unreadable - // Tigris archive must not block a write when a valid local copy - // exists — release(success) will re-upload a good archive. - if local_path.exists() { - warn!(repo = %repo_name, err = %e, - "write acquire: tigris download failed — falling back to local copy"); + // archive OR a stalled-then-timed-out GET must not block a write + // when a valid local copy exists — release(success) will re-upload + // a good archive. + if guard.local_path.exists() { + warn!(repo = %guard.repo_name, err = %e, + "write acquire: object-store download failed — falling back to local copy"); } else { - return Err(e).context("downloading repo from tigris for write"); + // Dropping the guard frees the advisory lock (its `Drop` + // closes the connection). No Tigris upload on a bail, + // unlike release(true). + return Err(e).context("downloading repo from object store for write"); } } } } - Ok(RepoWriteGuard { - owner_slug, - repo_name: repo_name.to_string(), - local_path, + Ok(guard) + } + + /// Try to acquire ONLY the per-repo advisory lock, non-blocking, with no + /// Tigris I/O — the lightweight counterpart to [`acquire_write`](Self::acquire_write) + /// for out-of-band admin ops (purge-spam) that must mutually exclude a live + /// push during a destructive delete but never download/re-upload the repo. + /// + /// Returns `Some(guard)` if the lock was free, `None` if another writer holds + /// it (so the caller can skip rather than block). The guard holds a dedicated + /// pooled connection for its whole lifetime, so the lock lives on that one + /// connection and `release()` unlocks it on the SAME connection — a plain + /// pool query could unlock on a different connection and silently fail. + pub async fn try_lock_repo( + &self, + owner_did: &str, + repo_name: &str, + ) -> Result> { + self.try_lock_repo_on(&self.lock_pool, owner_did, repo_name) + .await + } + + /// [`try_lock_repo`](Self::try_lock_repo) against a caller-chosen pool. The + /// Postgres advisory lock is DATABASE-global, so the lock still mutually + /// excludes holders of the same key from any other pool — only the connection + /// the guard pins comes from `pool`. Backs the `lock_pool`-based delegators. + async fn try_lock_repo_on( + &self, + pool: &PgPool, + owner_did: &str, + repo_name: &str, + ) -> Result> { + let (owner_slug, _local) = self.local_path(owner_did, repo_name)?; + let lock_key = advisory_lock_key(&owner_slug, repo_name); + let mut conn = pool.acquire().await.context("acquiring lock connection")?; + let row: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(lock_key) + .fetch_one(&mut *conn) + .await + .context("try advisory lock")?; + if !row.0 { + return Ok(None); + } + Ok(Some(RepoLockGuard { + conn: Some(conn), lock_key, - pool: self.pool.clone(), - tigris: self.tigris.clone(), - }) + repo_name: repo_name.to_string(), + })) } /// Initialize a new bare repo on local disk and upload to Tigris. pub async fn init(&self, owner_did: &str, repo_name: &str) -> Result { - let (owner_slug, local_path) = self.local_path(owner_did, repo_name)?; + let (_owner_slug, local_path) = self.local_path(owner_did, repo_name)?; store::init_bare(&local_path).context("initializing bare repo")?; - // Upload to Tigris in background - if let Some(ref tigris) = self.tigris { - let tigris = tigris.clone(); - let owner_slug = owner_slug.clone(); - let repo_name = repo_name.to_string(); - let path = local_path.clone(); + // Upload to the object store in the background, UNDER the per-repo lock + // so the PUT can't race a purge. Waits (bounded) for the lock: + // create_repo calls init while holding this repo's lock, so the upload + // waits out the creator's own insert-and-release rather than dying on it + // (a skipped init upload has no retry until the first write). + if self.object_store.is_some() { + let store = self.clone(); + let did = owner_did.to_string(); + let name = repo_name.to_string(); tokio::spawn(async move { - if let Err(e) = tigris.upload(&owner_slug, &repo_name, &path).await { - warn!(repo = %repo_name, err = %e, "failed to upload new repo to tigris"); - } + store.upload_locked(&did, &name, true).await; }); } Ok(local_path) } - /// Upload a repo to Tigris after a write operation (push, merge, fork, etc.). - /// Call this after any operation that modifies the git repo on disk. - pub async fn release_after_write(&self, owner_did: &str, repo_name: &str) { - if let Some(ref tigris) = self.tigris { - let (owner_slug, local_path) = match self.local_path(owner_did, repo_name) { - Ok(p) => p, + /// Upload a repo to the object store under a per-repo advisory lock the + /// CALLER already holds. The fork tail uses this: it takes the target's + /// lock before its conflict check and holds it through clone, upload, and + /// row insert, so the PUT cannot race a concurrent purge (which deletes the + /// repo under the same lock) and the repos row is only published after the + /// archive durably landed. The PUT is bounded by `release_upload_timeout`, + /// exactly as `RepoWriteGuard::release` bounds its upload: an unbounded PUT + /// under a held namespace lock is the INV-22 hazard. A timeout counts as an + /// attempted-and-failed upload. + /// + /// Return semantics follow [`RepoWriteGuard::release`], NOT `upload_locked`: + /// `false` only when an ATTEMPTED upload failed or timed out. No configured + /// object store means nothing to upload, so store-less nodes see success and + /// their writes proceed unchanged. The local dir gone under the lock (a + /// purge won the key before the caller took it) is likewise a skip, not a + /// failure: nothing exists to upload and a deleted archive must stay gone. + /// A path-validation failure returns `false` (fail closed: never report + /// durable success for a path we refused to touch). + #[must_use = "a false return means the durable upload failed; the write must be reported as failed, not 2xx"] + pub async fn upload_under_guard( + &self, + owner_did: &str, + repo_name: &str, + _guard: &RepoLockGuard, + ) -> bool { + let Some(ref store) = self.object_store else { + // No durable backend configured: nothing to upload, nothing failed. + return true; + }; + // Dir-gone-under-lock is SUCCESS here (`true`): the fork tail + // (api/repos.rs) treats a `false` return as a hard failure and deletes + // the freshly-cloned mirror, so a purge that removed the dir before this + // caller took the lock is nothing-to-upload, not a failed durable write. + self.upload_dir_locked(store, owner_did, repo_name, true) + .await + } + + /// Shared under-lock upload body for [`upload_under_guard`](Self::upload_under_guard) + /// and [`upload_locked`](Self::upload_locked). Both resolve+validate the + /// local path, re-check the on-disk dir still exists UNDER the caller's + /// advisory lock (a purge removes the dir under the same lock, so a caller + /// that took the key post-purge must NOT recreate the archive), then upload + /// bounded by `release_upload_timeout`. An unbounded PUT under a held + /// namespace lock is the INV-22 hazard, so the bound guarantees the guard's + /// pinned lock connection returns to the pool within a fixed window even if + /// the PUT stalls. A path-validation failure, an upload error, and a timeout + /// all return `false` (fail closed: never report durable success for a write + /// that did not land). + /// + /// `dir_gone_is_success` threads the ONE place the two callers diverge, and + /// both values are load-bearing. `upload_under_guard` passes `true`: its + /// fork-tail caller reads `false` as a hard failure and deletes the mirror, + /// so a dir purged out from under it must read as nothing-to-upload success. + /// `upload_locked` passes `false`: its lazy-migration caller must then NOT + /// mark the repo migrated, so the next `acquire` retries the migration. + async fn upload_dir_locked( + &self, + store: &Arc, + owner_did: &str, + repo_name: &str, + dir_gone_is_success: bool, + ) -> bool { + let (owner_slug, local_path) = match self.local_path(owner_did, repo_name) { + Ok(p) => p, + Err(e) => { + warn!(repo = %repo_name, err = %e, "rejected unsafe path before object-store upload"); + return false; + } + }; + // Re-check under the lock: a purge removes the on-disk dir under this + // same lock, so a caller that took the key post-purge finds the dir gone + // and must NOT recreate the archive. + if !local_path.exists() { + warn!(repo = %repo_name, "object-store upload skipped: local repo dir gone under lock (purged?)"); + return dir_gone_is_success; + } + match tokio::time::timeout( + self.release_upload_timeout, + store.upload(&owner_slug, repo_name, &local_path), + ) + .await + { + Ok(Ok(())) => true, + Ok(Err(e)) => { + warn!(repo = %repo_name, err = %e, "failed to upload repo to object store"); + false + } + Err(_) => { + warn!(repo = %repo_name, timeout_secs = self.release_upload_timeout.as_secs(), + "object-store upload timed out under the held repo lock"); + false + } + } + } + + /// Whether an object store is configured. Used by purge-spam to decide + /// whether a repo with no local copy can be a remote-unverified candidate + /// (its emptiness verified under the lock after a refresh) versus fail-closed. + pub fn has_object_store(&self) -> bool { + self.object_store.is_some() + } + + /// Upload the local repo to the object store while holding the per-repo + /// advisory lock, then release. The lock serializes the PUT against a + /// concurrent `purge-spam` that deletes the repo (row + dir + archive) under + /// the same lock, so an in-flight upload cannot resurrect a just-deleted + /// archive. Re-checks the local dir exists UNDER the lock — a purge removes + /// the dir under its lock, so a post-purge uploader finds it gone and skips. + /// Returns true iff a PUT was performed. A no-op when no store is configured. + /// + /// `wait`: init's background upload waits (bounded) for the lock, since the + /// creator holds this key across create_repo's insert-and-release and a + /// skipped init upload has no later retry (the next re-upload is the first + /// write's release, which may never come). The background lazy-migration + /// upload passes `false` and skips on contention; that skip genuinely + /// self-heals, since the next `acquire` retries the migration. The fork + /// tail does NOT come through here: it already holds the target's lock and + /// uploads via `upload_under_guard` (a second acquire would self-contend). + async fn upload_locked(&self, owner_did: &str, repo_name: &str, wait: bool) -> bool { + let Some(ref store) = self.object_store else { + return false; + }; + let guard = if wait { + match self.lock_repo_blocking(owner_did, repo_name).await { + Ok(Some(g)) => g, + Ok(None) => { + warn!(repo = %repo_name, "object-store upload skipped — repo lock still held after retry"); + return false; + } + Err(e) => { + warn!(repo = %repo_name, err = %e, "object-store upload skipped — could not acquire repo lock"); + return false; + } + } + } else { + match self.try_lock_repo(owner_did, repo_name).await { + Ok(Some(g)) => g, + Ok(None) => { + debug!(repo = %repo_name, "object-store upload skipped — repo locked by a live writer"); + return false; + } Err(e) => { - warn!(repo = %repo_name, err = %e, "rejected unsafe path in release_after_write"); - return; + warn!(repo = %repo_name, err = %e, "object-store upload skipped — could not acquire repo lock"); + return false; } - }; - if let Err(e) = tigris.upload(&owner_slug, repo_name, &local_path).await { - warn!(repo = %repo_name, err = %e, "failed to upload repo to tigris after write"); + } + }; + // Shared under-lock body: re-check the dir exists and PUT bounded by + // `release_upload_timeout` (INV-22 — an untimed PUT here would pin this + // guard's lock connection forever). Dir-gone returns `false` so the + // lazy-migration caller does NOT mark the repo migrated and retries. + let uploaded = self + .upload_dir_locked(store, owner_did, repo_name, false) + .await; + guard.release().await; + uploaded + } + + /// Bounded-wait lock-only acquire, without any object-store I/O. Retries + /// `try_lock_repo` with short backoff. Used by `create_repo` and `fork_repo` + /// to serialize their existence-check -> write -> row-insert spans against a + /// concurrent same-key purge (which holds this same lock across delete-row + + /// remove-dir), and by init's background upload to wait out the creator's + /// own insert-and-release. A key held past the bounded wait surfaces as + /// `Ok(None)`, which the create/fork handlers map to a retryable 503. + pub async fn lock_repo_blocking( + &self, + owner_did: &str, + repo_name: &str, + ) -> Result> { + self.lock_repo_blocking_on(&self.lock_pool, owner_did, repo_name) + .await + } + + /// [`lock_repo_blocking`](Self::lock_repo_blocking) against a caller-chosen + /// pool. The advisory lock is DB-global, so the guard mutually excludes the same + /// key across pools; only the pinned connection comes from `pool`. Backs the + /// `lock_pool`-based [`lock_repo_blocking`](Self::lock_repo_blocking) delegator. + pub async fn lock_repo_blocking_on( + &self, + pool: &PgPool, + owner_did: &str, + repo_name: &str, + ) -> Result> { + for attempt in 0..30 { + if let Some(g) = self.try_lock_repo_on(pool, owner_did, repo_name).await? { + return Ok(Some(g)); + } + if attempt < 29 { + tokio::time::sleep(std::time::Duration::from_millis(200)).await; } } + Ok(None) } /// Compute the local disk path and owner slug for a repo. @@ -291,6 +668,34 @@ impl RepoStore { Ok((owner_slug, local_path)) } + + /// Refresh the local copy from the authoritative object-store archive so an + /// emptiness recheck reflects remote state (used by purge-spam on a Tigris + /// deployment, where the admin node's local disk can be stale). Downloads + /// the archive over the local path when it exists; a no-op when no object + /// store is configured (single-machine). Errors propagate so the caller can + /// fail closed rather than delete on a stale-local view. + pub async fn refresh_from_archive(&self, owner_did: &str, repo_name: &str) -> Result<()> { + let Some(ref store) = self.object_store else { + return Ok(()); + }; + let (owner_slug, local_path) = self.local_path(owner_did, repo_name)?; + if store.exists(&owner_slug, repo_name).await? { + store.download(&owner_slug, repo_name, &local_path).await?; + } + Ok(()) + } + + /// Delete the object-store archive for a repo. A no-op when no object store + /// is configured. Used by purge-spam so a deleted repo's archive cannot be + /// downloaded into a later repo created with the same owner/name. + pub async fn delete_archive(&self, owner_did: &str, repo_name: &str) -> Result<()> { + let Some(ref store) = self.object_store else { + return Ok(()); + }; + let (owner_slug, _local_path) = self.local_path(owner_did, repo_name)?; + store.delete(&owner_slug, repo_name).await + } } /// Strict allowlist validator for `owner_did` and `repo_name`. @@ -324,7 +729,7 @@ fn validate_owner_did(owner_did: &str) -> Result<()> { Ok(()) } -fn validate_repo_name(repo_name: &str) -> Result<()> { +pub(crate) fn validate_repo_name(repo_name: &str) -> Result<()> { if repo_name.is_empty() { anyhow::bail!("repo_name is empty"); } @@ -354,8 +759,16 @@ pub struct RepoWriteGuard { repo_name: String, pub local_path: PathBuf, lock_key: i64, - pool: PgPool, - tigris: Option, + /// Dedicated pooled connection the advisory lock lives on. Held for the + /// guard's lifetime so `release` unlocks on the SAME session that locked, + /// and so a concurrent `try_lock_repo` cannot be handed this connection and + /// reentrantly re-grab the lock. `Option` so `release` can take it (return + /// it to the pool) while `Drop` closes it when release was never reached. + conn: Option>, + object_store: Option>, + /// Upper bound on the `release()` archive upload — copied from the store so + /// a stalled upload cannot pin this connection past the bound. + release_upload_timeout: std::time::Duration, } impl RepoWriteGuard { @@ -369,26 +782,175 @@ impl RepoWriteGuard { /// half-applied or otherwise inconsistent repo would propagate corruption to /// Tigris (and to every node that later downloads it). The lock is always /// released regardless, to avoid stale locks blocking future writes. - pub async fn release(self, success: bool) { - // Upload to Tigris only on success. + #[must_use = "a false return means the durable upload failed; the write must be reported as failed, not 2xx"] + pub async fn release(self, success: bool) -> bool { + self.release_with_failure_cleanup(success, |_| {}).await + } + + /// Like [`release`](Self::release), but when an ATTEMPTED upload fails or + /// times out, runs `cleanup` on the local repo path BEFORE the advisory + /// lock is freed. Lets a caller roll back a local write that never reached + /// durable storage without an unlock-to-cleanup window in which a + /// concurrent same-repo write could upload an archive still carrying it. + /// `cleanup` does NOT run when `success == false` (the write itself + /// failed, no upload attempted) or when the upload succeeded. + #[must_use = "a false return means the durable upload failed; the write must be reported as failed, not 2xx"] + pub async fn release_with_failure_cleanup( + mut self, + success: bool, + cleanup: impl FnOnce(&Path), + ) -> bool { + // Whether the durable copy is safe. Stays `true` when there is nothing to + // upload (no object store, or `success == false` where no upload is + // attempted) — those are not upload failures. Only an upload that was + // ATTEMPTED and then errored or timed out flips it `false`, which the + // caller surfaces as a FAILED push so the client re-pushes rather than + // trusting a commit that never reached durable storage. (P1 data-loss.) + let mut upload_ok = true; + // Whether the failed upload TIMED OUT (as opposed to erroring). On the + // error arm the PUT definitively failed; on the timeout arm the dropped + // upload future may have already transmitted the full body, so the store + // can still commit the POST-state archive; that arm alone needs the + // compensating upload after cleanup below. + let mut upload_timed_out = false; + + // Upload to Tigris only on success. Bound the upload so a stalled PUT + // cannot pin the guard's advisory-lock connection indefinitely — on + // timeout we log and fall through to the unlock, returning the + // connection to the pool within the bound. (INV-22.) The lock is freed on + // EVERY arm below regardless of `upload_ok`; only the return value tells + // the caller whether the durable write landed. if success { - if let Some(ref tigris) = self.tigris { - if let Err(e) = tigris - .upload(&self.owner_slug, &self.repo_name, &self.local_path) - .await + if let Some(ref tigris) = self.object_store { + match tokio::time::timeout( + self.release_upload_timeout, + tigris.upload(&self.owner_slug, &self.repo_name, &self.local_path), + ) + .await { - warn!(repo = %self.repo_name, err = %e, "failed to upload repo to tigris after write"); + Ok(Ok(())) => {} + Ok(Err(e)) => { + upload_ok = false; + warn!(repo = %self.repo_name, err = %e, "failed to upload repo to tigris after write"); + } + Err(_) => { + upload_ok = false; + upload_timed_out = true; + warn!(repo = %self.repo_name, timeout_secs = self.release_upload_timeout.as_secs(), + "tigris upload timed out after write — releasing the advisory lock without a completed upload"); + } } } } else { warn!(repo = %self.repo_name, "write failed — skipping tigris upload to avoid propagating an inconsistent repo"); } - // Release advisory lock - let _ = sqlx::query("SELECT pg_advisory_unlock($1)") - .bind(self.lock_key) - .execute(&self.pool) - .await; + // Failed-upload cleanup runs HERE, while the advisory lock is still + // held: after the unlock below, a concurrent same-repo write could + // upload an archive still carrying the state the caller is rolling + // back, which a later download would resurrect. + if !upload_ok { + cleanup(&self.local_path); + + // TIMEOUT arm only: the dropped upload future may have fully + // transmitted the PUT body with only the response in flight, so the + // store can commit the POST-state archive even though `cleanup` + // just rolled the local refs back, and a later cold download would + // resurrect the rolled-back write. Attempt ONE bounded compensating + // upload of the now-rolled-back tree: it starts strictly after the + // rollback, so if it lands it commits after the orphan PUT's + // near-immediate commit and the store converges to the rolled-back + // state either way. This at most doubles the bounded lock hold on + // this rare arm; the advisory lock must still be held during the + // compensation, and it is, since this runs before the unlock below. + // `upload_ok` stays false regardless: the client was already told + // the write failed; the compensation only converges durable state. + // The error arm needs none of this: its PUT definitively failed. + if upload_timed_out { + if let Some(ref tigris) = self.object_store { + match tokio::time::timeout( + self.release_upload_timeout, + tigris.upload(&self.owner_slug, &self.repo_name, &self.local_path), + ) + .await + { + Ok(Ok(())) => {} + Ok(Err(e)) => { + warn!(repo = %self.repo_name, err = %e, + "compensating upload after timed-out release upload failed: store may hold a write that was reported failed until the next successful upload"); + } + Err(_) => { + warn!(repo = %self.repo_name, timeout_secs = self.release_upload_timeout.as_secs(), + "compensating upload after timed-out release upload timed out: store may hold a write that was reported failed until the next successful upload"); + } + } + } + } + } + + // Unlock on the SAME connection that took the lock, then TAKE the + // connection so it returns to the pool on drop and the guard's `Drop` + // sees `None` (no close-on-drop). If this future is cancelled before the + // take completes, the connection stays in `self.conn` and `Drop` frees + // the lock by closing it instead. + if let Some(mut conn) = self.conn.take() { + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(self.lock_key) + .execute(&mut *conn) + .await; + } + + upload_ok + } +} + +impl Drop for RepoWriteGuard { + fn drop(&mut self) { + // If `release()` was never reached (an early `?`, a panic, or the + // caller's future cancelled on client disconnect), the connection still + // holds the session advisory lock. Close it so Postgres frees the lock + // at session end, rather than returning a lock-holding connection to the + // pool where it would block every future write to this repo. (R1.) + if let Some(mut conn) = self.conn.take() { + warn!(repo = %self.repo_name, + "RepoWriteGuard dropped without release() — closing its connection to free the advisory lock"); + conn.close_on_drop(); + } + } +} + +/// Lock-only guard from [`RepoStore::try_lock_repo`]. Holds the Postgres advisory +/// lock (and the dedicated connection it lives on) until `release()`. No Tigris +/// I/O — unlike [`RepoWriteGuard`], releasing does NOT upload anything. +pub struct RepoLockGuard { + conn: Option>, + lock_key: i64, + repo_name: String, +} + +impl RepoLockGuard { + /// Release the advisory lock on the same connection that took it, then return + /// the connection to the pool. + pub async fn release(mut self) { + if let Some(mut conn) = self.conn.take() { + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(self.lock_key) + .execute(&mut *conn) + .await; + } + } +} + +impl Drop for RepoLockGuard { + fn drop(&mut self) { + // Same guarantee as RepoWriteGuard: a lock guard dropped without + // release() closes its connection so Postgres frees the advisory lock, + // rather than returning a lock-holding connection to the pool. (R1.) + if let Some(mut conn) = self.conn.take() { + warn!(repo = %self.repo_name, + "RepoLockGuard dropped without release() — closing its connection to free the advisory lock"); + conn.close_on_drop(); + } } } @@ -562,4 +1124,2102 @@ mod tests { ); } } + + // ── advisory-lock mutual exclusion (P1a) ──────────────────────────────── + + // A live `acquire_write` guard must exclude a concurrent `try_lock_repo` + // (the purge path) on the same repo. This is load-bearing for the purge + // tool's M4 mutual exclusion. + // + // The pool MUST be single-connection. `try_lock_repo`'s own return is the + // only observable that exposes the bug (it acquires through the pool), and + // it is only deterministic when the writer's connection is the ONLY one, so + // `try_lock_repo`'s `pool.acquire()` is forced onto it: + // * Pre-fix, `acquire_write` locks via `.fetch_one(&self.pool)` and returns + // the lock-owning connection to the pool. `try_lock_repo` re-acquires + // that same connection and `pg_try_advisory_lock` re-locks it + // reentrantly -> Ok(Some) (RED). (On a multi-connection pool the reentrant + // grab is non-deterministic: `acquire` may hand back a different idle + // connection, so the bug hides — verified by execution.) + // * Post-fix, `acquire_write` pins the connection, so `try_lock_repo`'s + // `pool.acquire()` cannot get one and times out -> Err. Either way the + // fix guarantees try_lock does NOT reentrantly grab the held lock. + #[sqlx::test] + async fn acquire_write_guard_blocks_try_lock( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + let pool = pool_opts + .max_connections(1) + .acquire_timeout(std::time::Duration::from_secs(3)) + .connect_with(connect_opts) + .await + .unwrap(); + let tmp = tempfile::TempDir::new().unwrap(); + let store = RepoStore::for_testing(tmp.path().to_path_buf(), pool); + let owner = "did:key:z6MkWriterLockOwnerAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "locktest"; + + // A live writer holds the per-repo advisory lock. + let guard = store.acquire_write(owner, name).await.unwrap(); + + // The purge path must NOT reentrantly acquire the same lock while held. + // Pre-fix: Ok(Some) (reentrant grab). Post-fix: Err (connection pinned). + // The invariant is "not Ok(Some)". + let contended = store.try_lock_repo(owner, name).await; + assert!( + !matches!(contended, Ok(Some(_))), + "try_lock_repo must not reentrantly acquire a lock a live acquire_write \ + guard holds (got Ok(Some) — the reentrant-grab bug)" + ); + + // After the writer releases (on its pinned connection), the lock is free + // and the single connection is back in the pool. No object store here, so + // release always returns true, so the value is irrelevant to this test. + let _ = guard.release(true).await; + let after = store.try_lock_repo(owner, name).await.unwrap(); + assert!( + after.is_some(), + "lock must be free once the writer releases it" + ); + after.unwrap().release().await; + } + + // The exclusion is real ACROSS sessions, not just the reentrant + // same-connection case: on a multi-connection pool the writer pins its + // connection, so `try_lock_repo`'s `pool.acquire()` gets a DIFFERENT + // connection whose `pg_try_advisory_lock` correctly observes the lock held + // -> Ok(None). This is the actual production topology (writer and purge on + // different sessions), and it also catches a pin-but-don't-actually-lock + // regression that the single-connection test (which passes via a + // pool-exhaustion Err) cannot distinguish from a real held lock. + #[sqlx::test] + async fn acquire_write_guard_excludes_try_lock_across_connections(pool: PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let store = RepoStore::for_testing(tmp.path().to_path_buf(), pool); + let owner = "did:key:z6MkCrossConnExclusionAAAAAAAAAAAAAAAAAAAAA"; + let name = "xconn"; + + let guard = store.acquire_write(owner, name).await.unwrap(); + + // A second, distinct pooled connection must see the lock held. + let contended = store.try_lock_repo(owner, name).await.unwrap(); + assert!( + contended.is_none(), + "a live acquire_write guard must exclude try_lock_repo on a different \ + connection (Ok(None)); Ok(Some) would mean the lock is not held" + ); + + // No object store, so release always returns true, so value irrelevant here. + let _ = guard.release(true).await; + let after = store.try_lock_repo(owner, name).await.unwrap(); + assert!(after.is_some(), "lock is free after release"); + after.unwrap().release().await; + } + + // ── Drop-safety of the advisory-lock guards (U1: R1, R2; AE2) ─────────── + + // A minimal ObjectStore double: `exists()` is configurable, `download()` can + // park on a gate until notified (to hold `acquire_write` inside its + // post-lock await for the cancellation test), and `upload()` records its + // calls. Serves U1's reorder test and U3's serialization tests. + struct GatedStore { + // `exists` is dynamic so a delete() flips it false and an upload() flips + // it true — lets U3's resurrect test assert a deleted archive stays gone. + exists: std::sync::Arc, + download_gate: Option>, + // When set, `upload()` parks on this gate before doing anything — models a + // stalled PUT so the release-upload timeout can be observed (U2). + upload_gate: Option>, + uploads: std::sync::Arc>>, + deletes: std::sync::Arc>>, + // Records every download() call (pushed BEFORE the gate park, so a test + // can poll "a download is in flight" while the gate holds it). U4's + // concurrent-cold-read test counts these. + downloads: std::sync::Arc>>, + } + + impl GatedStore { + fn new(exists: bool) -> Self { + Self { + exists: std::sync::Arc::new(std::sync::atomic::AtomicBool::new(exists)), + download_gate: None, + upload_gate: None, + uploads: std::sync::Arc::new(std::sync::Mutex::new(Vec::new())), + deletes: std::sync::Arc::new(std::sync::Mutex::new(Vec::new())), + downloads: std::sync::Arc::new(std::sync::Mutex::new(Vec::new())), + } + } + } + + #[async_trait::async_trait] + impl crate::git::tigris::ObjectStore for GatedStore { + async fn exists(&self, _o: &str, _r: &str) -> anyhow::Result { + Ok(self.exists.load(std::sync::atomic::Ordering::SeqCst)) + } + async fn upload(&self, o: &str, r: &str, _p: &std::path::Path) -> anyhow::Result<()> { + if let Some(gate) = &self.upload_gate { + // Park indefinitely — the caller's timeout must be what unblocks. + gate.notified().await; + } + self.uploads + .lock() + .unwrap() + .push((o.to_string(), r.to_string())); + self.exists.store(true, std::sync::atomic::Ordering::SeqCst); + Ok(()) + } + async fn download(&self, o: &str, r: &str, p: &std::path::Path) -> anyhow::Result<()> { + self.downloads + .lock() + .unwrap() + .push((o.to_string(), r.to_string())); + if let Some(gate) = &self.download_gate { + gate.notified().await; + } + // Materialize a valid bare repo at the target path so a published + // download is observable on disk (mirrors the real store's + // extract-then-swap, which replaces any existing copy). + if p.exists() { + std::fs::remove_dir_all(p)?; + } + crate::git::store::init_bare(p)?; + Ok(()) + } + async fn delete(&self, o: &str, r: &str) -> anyhow::Result<()> { + self.deletes + .lock() + .unwrap() + .push((o.to_string(), r.to_string())); + self.exists + .store(false, std::sync::atomic::Ordering::SeqCst); + Ok(()) + } + } + + // Non-perturbing probe: true iff advisory lock `key` is free (grabs it and + // immediately releases it on the observer's own separate session). + async fn advisory_lock_is_free(conn: &mut sqlx::PgConnection, key: i64) -> bool { + let (got,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *conn) + .await + .unwrap(); + if got { + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *conn) + .await; + } + got + } + + async fn poll_until_free(conn: &mut sqlx::PgConnection, key: i64) -> bool { + for _ in 0..50 { + if advisory_lock_is_free(conn, key).await { + return true; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + false + } + + async fn poll_until_held(conn: &mut sqlx::PgConnection, key: i64) -> bool { + for _ in 0..50 { + if !advisory_lock_is_free(conn, key).await { + return true; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + false + } + + fn lock_key_for(owner: &str, name: &str) -> i64 { + advisory_lock_key(&owner.replace([':', '/'], "_"), name) + } + + // Build a pool whose idle/lifetime reaping is disabled, so a leaked + // (dropped-without-release) connection is NOT reclaimed by ambient sqlx + // maintenance during the poll window. Without this the default sqlx-test + // pool reaps the connection ~1.8s after drop, freeing the lock on its own + // and masking the leak — the test must observe ONLY the fix's own unlock. + async fn no_reap_pool( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: &sqlx::postgres::PgConnectOptions, + ) -> PgPool { + pool_opts + .max_connections(5) + .min_connections(0) + .idle_timeout(None) + .max_lifetime(None) + .test_before_acquire(false) + .connect_with(connect_opts.clone()) + .await + .unwrap() + } + + // R1/AE2: a RepoWriteGuard dropped WITHOUT release() must free its advisory + // lock — its connection is closed rather than returned to the pool still + // holding the session lock. Pre-fix (no Drop impl) the lock leaks -> RED. + #[sqlx::test] + async fn write_guard_dropped_without_release_frees_lock( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + use sqlx::ConnectOptions; + let pool = no_reap_pool(pool_opts, &connect_opts).await; + let tmp = tempfile::TempDir::new().unwrap(); + let store = RepoStore::new(tmp.path().to_path_buf(), None, pool); + let owner = "did:key:z6MkDropFreesLockAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "droptest"; + let key = lock_key_for(owner, name); + let mut observer = connect_opts.connect().await.unwrap(); + + let guard = store.acquire_write(owner, name).await.unwrap(); + assert!( + !advisory_lock_is_free(&mut observer, key).await, + "lock must be held while the guard is alive" + ); + + drop(guard); // NO release() + + assert!( + poll_until_free(&mut observer, key).await, + "advisory lock must be freed after a RepoWriteGuard is dropped without release()" + ); + } + + // R1: same guarantee for the purge lock-only guard. + #[sqlx::test] + async fn repo_lock_guard_dropped_without_release_frees_lock( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + use sqlx::ConnectOptions; + let pool = no_reap_pool(pool_opts, &connect_opts).await; + let tmp = tempfile::TempDir::new().unwrap(); + let store = RepoStore::new(tmp.path().to_path_buf(), None, pool); + let owner = "did:key:z6MkLockGuardDropFreesAAAAAAAAAAAAAAAAAAAAAA"; + let name = "lockdrop"; + let key = lock_key_for(owner, name); + let mut observer = connect_opts.connect().await.unwrap(); + + let guard = store.try_lock_repo(owner, name).await.unwrap().unwrap(); + assert!( + !advisory_lock_is_free(&mut observer, key).await, + "lock must be held while the lock guard is alive" + ); + + drop(guard); // NO release() + + assert!( + poll_until_free(&mut observer, key).await, + "advisory lock must be freed after a RepoLockGuard is dropped without release()" + ); + } + + // R2/KTD-2: cancelling acquire_write DURING its post-lock freshness download + // must free the lock. Pre-reorder the lock is won onto a bare connection + // before the guard exists, so cancellation leaks it -> RED; post-reorder the + // guard wraps the connection first and its Drop frees the lock. + #[sqlx::test] + async fn acquire_write_cancelled_during_download_frees_lock( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + use sqlx::ConnectOptions; + let pool = no_reap_pool(pool_opts, &connect_opts).await; + let tmp = tempfile::TempDir::new().unwrap(); + let mut ts = GatedStore::new(true); + ts.download_gate = Some(std::sync::Arc::new(tokio::sync::Notify::new())); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ); + let owner = "did:key:z6MkCancelDownloadFreesAAAAAAAAAAAAAAAAAAAA"; + let name = "canceldl"; + let key = lock_key_for(owner, name); + let mut observer = connect_opts.connect().await.unwrap(); + + { + let fut = store.acquire_write(owner, name); + tokio::pin!(fut); + tokio::select! { + _ = &mut fut => panic!("acquire_write should be parked in download, not complete"), + _ = tokio::time::sleep(std::time::Duration::from_millis(400)) => {} + } + // `fut` is dropped here — cancels acquire_write mid-download. + } + + assert!( + poll_until_free(&mut observer, key).await, + "advisory lock must be freed when acquire_write is cancelled mid-download" + ); + } + + // R2 negative: the normal release() path must NOT close the connection — it + // returns to the pool, so writes don't churn the pool. On a single-connection + // pool a second acquire_write can only succeed if the first returned its + // connection (unlocked, not closed). + #[sqlx::test] + async fn release_returns_connection_to_pool( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + let pool = pool_opts + .max_connections(1) + .acquire_timeout(std::time::Duration::from_secs(3)) + .connect_with(connect_opts) + .await + .unwrap(); + let tmp = tempfile::TempDir::new().unwrap(); + let store = RepoStore::new(tmp.path().to_path_buf(), None, pool); + let owner = "did:key:z6MkReleasePoolsConnAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "releasepool"; + + for _ in 0..3 { + let guard = store.acquire_write(owner, name).await.unwrap(); + // No object store, so release always returns true, so value irrelevant. + let _ = guard.release(true).await; + } + } + + // OQ1: settle whether a guard abandoned inside a detached task when the + // tokio runtime tears down panics in Drop. `PoolConnection::drop` spawns a + // task (both the close_on_drop path and the normal return path), and + // `rt::spawn` panics via `missing_rt` if no runtime handle is current. U2 + // makes receive-pack a detached task that graceful shutdown does not drain, + // so at process exit such a task can be dropped mid-flight holding a guard. + // This drops a real runtime with a parked guard-holding task and asserts the + // process survives (a panic-in-Drop during unwind would abort the binary). + #[test] + fn guard_drop_during_runtime_shutdown_does_not_panic() { + let url = match std::env::var("DATABASE_URL") { + Ok(u) => u, + Err(_) => return, // no DB configured — skip (mirrors sqlx::test gating) + }; + let rt = tokio::runtime::Builder::new_multi_thread() + .worker_threads(2) + .enable_all() + .build() + .unwrap(); + let ready = std::sync::Arc::new(tokio::sync::Notify::new()); + let ready2 = ready.clone(); + rt.block_on(async move { + let pool = sqlx::postgres::PgPoolOptions::new() + .max_connections(5) + .connect(&url) + .await + .unwrap(); + let tmp = tempfile::TempDir::new().unwrap(); + let store = RepoStore::new(tmp.path().to_path_buf(), None, pool); + tokio::spawn(async move { + let _guard = store + .acquire_write("did:key:z6MkOQ1ShutdownAAAAAAAAAAAAAAAAAAAAAAAAA", "oq1") + .await + .unwrap(); + ready2.notify_one(); + // Park forever holding the guard. + std::future::pending::<()>().await; + }); + ready.notified().await; + }); + // Drop the runtime while the detached task is parked holding the guard: + // the task is cancelled, dropping the guard during runtime teardown. Its + // Drop must not panic. Reaching the end of the test proves it didn't. + drop(rt); + } + + // ── U3: archive uploads serialize under the per-repo lock (R7, R8, R9) ─── + + fn repo_dir_of(repos_dir: &std::path::Path, owner: &str, name: &str) -> std::path::PathBuf { + repos_dir + .join(owner.replace([':', '/'], "_")) + .join(format!("{name}.git")) + } + + // R1/R7: init's background upload must WAIT (bounded) for the creator's own + // lock rather than dying on it. create_repo holds the per-repo lock across + // existence-check -> init -> row insert, so the spawned upload always finds + // it held at first. Create-shaped flow: hold the lock, init under it, + // release shortly after; no PUT while held, exactly one PUT after release. + // Pre-fix (wait=false) the task try-locks once, skips, and the PUT never + // arrives -> RED on the post-release assert. The "still empty at 500ms" + // assert alone would pass vacuously during the new wait window; the + // post-release upload assert is what makes this test load-bearing. + #[sqlx::test] + async fn init_upload_waits_out_creators_lock_then_uploads( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + let pool = pool_opts + .max_connections(5) + .connect_with(connect_opts) + .await + .unwrap(); + let tmp = tempfile::TempDir::new().unwrap(); + let ts = GatedStore::new(false); + let uploads = ts.uploads.clone(); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ); + let owner = "did:key:z6MkInitWaitAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "initwait"; + let slug = owner.replace([':', '/'], "_"); + + let held = store.try_lock_repo(owner, name).await.unwrap().unwrap(); + store.init(owner, name).await.unwrap(); + // While the creator still holds the lock the upload must not PUT. + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + assert!( + uploads.lock().unwrap().is_empty(), + "init's upload must not PUT while the creator still holds the lock" + ); + held.release().await; + + // The waiting task retries every 200ms; the PUT must land soon after. + let mut landed = false; + for _ in 0..50 { + if uploads.lock().unwrap().len() == 1 { + landed = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + assert!( + landed, + "init's upload must PUT after the creator releases the lock" + ); + let ups = uploads.lock().unwrap(); + assert_eq!( + ups.len(), + 1, + "init's upload must PUT exactly once after the lock frees" + ); + assert_eq!(ups[0], (slug, name.to_string())); + } + + // R1/R7 negative: the bounded wait must not become an unbounded one. A lock + // held past the full 30 x 200ms window yields NO PUT: the task logs a skip + // and exits. The post-release grace assert is what proves boundedness; an + // unbounded waiter would still be parked at release time and would PUT once + // the lock frees. (Reworked from init_upload_skips_while_repo_locked, whose + // skip-immediately premise inverts under wait=true.) + #[sqlx::test] + async fn init_upload_gives_up_after_bounded_lock_wait( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + let pool = pool_opts + .max_connections(5) + .connect_with(connect_opts) + .await + .unwrap(); + let tmp = tempfile::TempDir::new().unwrap(); + let ts = GatedStore::new(false); + let uploads = ts.uploads.clone(); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ); + let owner = "did:key:z6MkInitSkipAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "initskip"; + + let held = store.try_lock_repo(owner, name).await.unwrap().unwrap(); + store.init(owner, name).await.unwrap(); + // Hold past the entire bounded window (29 sleeps x 200ms, ~5.8s). + tokio::time::sleep(std::time::Duration::from_millis(8_000)).await; + assert!( + uploads.lock().unwrap().is_empty(), + "init's upload must give up, not PUT, when the lock outlives the bounded wait" + ); + held.release().await; + // A still-parked (unbounded) waiter would PUT within ~200ms of release. + tokio::time::sleep(std::time::Duration::from_millis(1_500)).await; + assert!( + uploads.lock().unwrap().is_empty(), + "no late PUT after release; the upload task must have exited at the bound" + ); + } + + // R8/AE6: the fork tail's lock acquire WAITS for a held target lock rather + // than skipping, then the under-guard upload PUTs once after it frees. This + // runs the exact lock_repo_blocking -> upload_under_guard sequence + // fork_repo runs (migrated from release_after_write when it was retired; + // assertions unchanged). + #[sqlx::test] + async fn under_guard_upload_waits_then_uploads_after_lock_frees( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + let pool = pool_opts + .max_connections(5) + .connect_with(connect_opts) + .await + .unwrap(); + let tmp = tempfile::TempDir::new().unwrap(); + let ts = GatedStore::new(false); + let uploads = ts.uploads.clone(); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ); + let owner = "did:key:z6MkForkWaitAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "forkwait"; + let slug = owner.replace([':', '/'], "_"); + std::fs::create_dir_all(repo_dir_of(tmp.path(), owner, name)).unwrap(); + + let held = store.try_lock_repo(owner, name).await.unwrap().unwrap(); + let store2 = store.clone(); + let owner2 = owner.to_string(); + let name2 = name.to_string(); + let h = tokio::spawn(async move { + let g = store2 + .lock_repo_blocking(&owner2, &name2) + .await + .unwrap() + .expect("target lock frees within the bounded wait"); + let ok = store2.upload_under_guard(&owner2, &name2, &g).await; + g.release().await; + ok + }); + tokio::time::sleep(std::time::Duration::from_millis(500)).await; + assert!( + uploads.lock().unwrap().is_empty(), + "fork upload must wait (not skip) while the repo lock is held" + ); + + held.release().await; + assert!( + h.await.unwrap(), + "the under-guard upload reports success once the lock frees" + ); + let ups = uploads.lock().unwrap(); + assert_eq!( + ups.len(), + 1, + "fork upload must PUT exactly once after the lock frees" + ); + assert_eq!(ups[0], (slug, name.to_string())); + } + + // R9/AE6: after a purge deletes the archive AND removes the on-disk dir under + // the lock, a late upload (that lost the race) must NOT resurrect the archive + // — it finds the dir gone under the lock and skips. Pre-fix (no dir recheck) + // it re-PUTs and revives the archive -> RED. + #[sqlx::test] + async fn upload_does_not_resurrect_a_purged_archive(pool: sqlx::PgPool) { + use std::sync::atomic::Ordering::SeqCst; + let tmp = tempfile::TempDir::new().unwrap(); + let ts = GatedStore::new(false); + let uploads = ts.uploads.clone(); + let deletes = ts.deletes.clone(); + let exists = ts.exists.clone(); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ); + let owner = "did:key:z6MkResurrectAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "resurrect"; + let dir = repo_dir_of(tmp.path(), owner, name); + std::fs::create_dir_all(&dir).unwrap(); + + let g = store.try_lock_repo(owner, name).await.unwrap().unwrap(); + assert!(store.upload_under_guard(owner, name, &g).await); + g.release().await; + assert!(exists.load(SeqCst), "archive exists after the first upload"); + assert_eq!(uploads.lock().unwrap().len(), 1); + + // Simulate a purge: delete the archive and remove the on-disk dir. + store.delete_archive(owner, name).await.unwrap(); + std::fs::remove_dir_all(&dir).unwrap(); + assert_eq!(deletes.lock().unwrap().len(), 1, "archive delete recorded"); + assert!(!exists.load(SeqCst), "archive is deleted by the purge"); + + // A late upload attempt must find the dir gone under the lock and skip, + // not resurrect. The skip is nothing-to-do (true), never a PUT. + let g = store.try_lock_repo(owner, name).await.unwrap().unwrap(); + assert!(store.upload_under_guard(owner, name, &g).await); + g.release().await; + assert!( + !exists.load(SeqCst), + "a purged archive must stay deleted — the upload found no dir and skipped" + ); + assert_eq!( + uploads.lock().unwrap().len(), + 1, + "no second PUT after the repo dir was purged" + ); + } + + // Negative: an uncontended fork upload PUTs exactly once (the lock is free, + // so the fork-tail sequence acquires it first try and uploads under it). + // Migrated from release_after_write when it was retired; assertion unchanged. + #[sqlx::test] + async fn uncontended_under_guard_upload_uploads_once(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let ts = GatedStore::new(false); + let uploads = ts.uploads.clone(); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ); + let owner = "did:key:z6MkUncontendedAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "uncontended"; + std::fs::create_dir_all(repo_dir_of(tmp.path(), owner, name)).unwrap(); + + let g = store + .lock_repo_blocking(owner, name) + .await + .unwrap() + .expect("uncontended lock acquires first try"); + assert!( + store.upload_under_guard(owner, name, &g).await, + "an uncontended under-guard upload reports success" + ); + g.release().await; + assert_eq!( + uploads.lock().unwrap().len(), + 1, + "an uncontended fork upload must PUT exactly once" + ); + } + + // ── U2: dedicated lock pool + bounded release upload (R2, KTD2) ────────── + + // A no-reap pool sized to `max_connections` with a short acquire timeout, so + // a held guard's connection is not reclaimed mid-assertion (see + // raw-advisory-lock-guard-must-free-on-drop-and-ambient-pool-reap-masks-the-leak.md) + // and an exhausted pool fails fast instead of stalling the whole test. + async fn sized_no_reap_pool( + connect_opts: &sqlx::postgres::PgConnectOptions, + max_connections: u32, + ) -> PgPool { + sqlx::postgres::PgPoolOptions::new() + .max_connections(max_connections) + .acquire_timeout(std::time::Duration::from_secs(2)) + .min_connections(0) + .idle_timeout(None) + .max_lifetime(None) + .test_before_acquire(false) + .connect_with(connect_opts.clone()) + .await + .unwrap() + } + + // R2/KTD2: advisory-lock guards pin connections from the DEDICATED lock pool, + // so N concurrent distinct-repo write guards do NOT consume the app pool — an + // unrelated app-pool handler query still gets a connection. On the pre-split + // base (the store shared the app pool, RepoStore::new(_, _, db.pool())) those + // N guards pinned every app-pool connection and the unrelated query 503'd on + // acquire; the `shared_pool_starves_*` control below pins that RED behavior. + #[sqlx::test] + async fn write_guards_on_lock_pool_do_not_starve_the_app_pool( + _pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + const N: u32 = 3; + let lock_pool = sized_no_reap_pool(&connect_opts, N).await; + let app_pool = sized_no_reap_pool(&connect_opts, N).await; + let tmp = tempfile::TempDir::new().unwrap(); + let store = RepoStore::new(tmp.path().to_path_buf(), None, lock_pool.clone()); + + // Hold N distinct-repo write guards -> N lock-pool connections pinned. + let mut guards = Vec::new(); + for i in 0..N { + let owner = format!("did:key:z6MkStarveApp{i}AAAAAAAAAAAAAAAAAAAAAAAA"); + guards.push(store.acquire_write(&owner, "starve").await.unwrap()); + } + + // The lock pool is now exhausted: a further acquire on it times out — + // proving the guards really pinned all N of its connections (so the app- + // pool assertion below is load-bearing, not vacuously green on idle). + assert!( + lock_pool.acquire().await.is_err(), + "the lock pool must be exhausted by N held write guards" + ); + + // ...but the app pool is a SEPARATE pool, so an unrelated handler query + // still acquires a connection. This is the query that starved pre-split. + assert!( + app_pool.acquire().await.is_ok(), + "the app pool must stay available while N write guards are held on the lock pool" + ); + + for g in guards { + // No object store, so release always returns true, so value irrelevant. + let _ = g.release(true).await; + } + } + + // The pre-split hazard, pinned as a characterization test: when the store + // SHARES its guard pool with the app pool (the pre-U2 wiring), N held write + // guards pin every connection and an unrelated query on that SAME pool + // starves (acquire times out). This is the RED the dedicated lock pool + // closes; keeping it green documents WHY the pools must be split. + #[sqlx::test] + async fn shared_pool_starves_the_app_pool_under_held_write_guards( + _pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + const N: u32 = 3; + let shared = sized_no_reap_pool(&connect_opts, N).await; + let tmp = tempfile::TempDir::new().unwrap(); + // Pre-split topology: the store's guard pool IS the app pool. + let store = RepoStore::new(tmp.path().to_path_buf(), None, shared.clone()); + + let mut guards = Vec::new(); + for i in 0..N { + let owner = format!("did:key:z6MkSharedStarve{i}AAAAAAAAAAAAAAAAAAAAAA"); + guards.push(store.acquire_write(&owner, "starve").await.unwrap()); + } + + // An unrelated query on the SAME pool now starves. + assert!( + shared.acquire().await.is_err(), + "a shared pool must starve under N held write guards — the hazard the split closes" + ); + + for g in guards { + // No object store, so release always returns true, so value irrelevant. + let _ = g.release(true).await; + } + } + + // R2: the lock pool is sized to peak writers, so N = pool-size concurrent + // distinct-repo writers all acquire and proceed (the starvation is not + // merely relocated from the app pool onto pushers). + #[sqlx::test] + async fn lock_pool_admits_peak_writers_concurrently( + _pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + const N: u32 = 4; + let lock_pool = sized_no_reap_pool(&connect_opts, N).await; + let tmp = tempfile::TempDir::new().unwrap(); + let store = RepoStore::new(tmp.path().to_path_buf(), None, lock_pool); + + let mut guards = Vec::new(); + for i in 0..N { + let owner = format!("did:key:z6MkPeakWriter{i}AAAAAAAAAAAAAAAAAAAAAAAA"); + guards.push( + store + .acquire_write(&owner, "peak") + .await + .expect("every writer up to the lock-pool size must acquire and proceed"), + ); + } + assert_eq!(guards.len(), N as usize); + for g in guards { + // No object store, so release always returns true, so value irrelevant. + let _ = g.release(true).await; + } + } + + // R2/INV-22: a stalled release() upload must not pin the guard's advisory- + // lock connection indefinitely. With a short upload bound and an upload that + // never completes, release() still returns and frees the lock within the + // bound. Pre-fix (untimed upload) release() would hang on the stalled PUT + // and never reach the unlock. + #[sqlx::test] + async fn release_upload_stall_is_bounded( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + use sqlx::ConnectOptions; + let pool = no_reap_pool(pool_opts, &connect_opts).await; + let tmp = tempfile::TempDir::new().unwrap(); + let mut ts = GatedStore::new(true); + // The upload parks forever; the release timeout is what must unblock it. + ts.upload_gate = Some(std::sync::Arc::new(tokio::sync::Notify::new())); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ) + .with_release_upload_timeout(std::time::Duration::from_millis(300)); + let owner = "did:key:z6MkReleaseStallBoundAAAAAAAAAAAAAAAAAAAAAA"; + let name = "stall"; + let key = lock_key_for(owner, name); + let mut observer = connect_opts.connect().await.unwrap(); + + let guard = store.acquire_write(owner, name).await.unwrap(); + assert!( + !advisory_lock_is_free(&mut observer, key).await, + "the lock must be held while the guard is alive" + ); + + // release(success) -> the upload stalls, but the bound caps the hold. The + // return here is false (the upload timed out), but this test asserts only + // the timing bound, so the value is intentionally ignored. + let start = std::time::Instant::now(); + let _ = guard.release(true).await; + assert!( + start.elapsed() < std::time::Duration::from_secs(5), + "release() must return within the upload bound, not hang on the stalled PUT" + ); + assert!( + poll_until_free(&mut observer, key).await, + "the advisory lock must be freed after the bounded (timed-out) upload" + ); + } + + // ObjectStore double for the timed-out-release compensation tests: `upload()` + // records its call count and, at entry, whether the caller's rollback marker + // file exists in the repo dir (so a test can prove WHICH state each upload + // observed). When `err_first` is set the first call fails immediately (the + // Err arm); otherwise the first call parks forever past the release timeout + // (the timeout arm) and every later call succeeds instantly. + struct CompensationProbeStore { + err_first: bool, + calls: std::sync::Arc, + marker_at_upload: std::sync::Arc>>, + } + + #[async_trait::async_trait] + impl crate::git::tigris::ObjectStore for CompensationProbeStore { + async fn exists(&self, _o: &str, _r: &str) -> anyhow::Result { + Ok(false) + } + async fn upload(&self, _o: &str, _r: &str, p: &std::path::Path) -> anyhow::Result<()> { + let n = self.calls.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + self.marker_at_upload + .lock() + .unwrap() + .push(p.join("rollback-marker").exists()); + if n == 0 { + if self.err_first { + anyhow::bail!("simulated PUT failure"); + } + // Park forever; the release timeout is what must unblock it. + std::future::pending::<()>().await; + } + Ok(()) + } + async fn download(&self, _o: &str, _r: &str, _p: &std::path::Path) -> anyhow::Result<()> { + unreachable!("exists() is always false, so no download should run"); + } + async fn delete(&self, _o: &str, _r: &str) -> anyhow::Result<()> { + Ok(()) + } + } + + // A release upload that TIMES OUT may have fully transmitted its PUT body + // (only the response in flight), so the store can commit the POST-state + // archive while cleanup rolls the local tree back, and a later cold download + // would resurrect the rolled-back write. release_with_failure_cleanup must + // therefore attempt ONE bounded compensating upload of the rolled-back tree + // after cleanup, on the timeout arm only. Pre-fix (no compensation) upload + // runs exactly once -> the call-count == 2 assert is RED. + #[sqlx::test] + async fn timed_out_release_upload_compensates_with_rolled_back_tree(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let calls = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let marker_at_upload = std::sync::Arc::new(std::sync::Mutex::new(Vec::new())); + let ts = CompensationProbeStore { + err_first: false, + calls: calls.clone(), + marker_at_upload: marker_at_upload.clone(), + }; + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ) + .with_release_upload_timeout(std::time::Duration::from_millis(300)); + let owner = "did:key:z6MkTimeoutCompensateAAAAAAAAAAAAAAAAAAAAAA"; + let name = "compensate"; + + // A real write under the guard, so the first upload has a tree to PUT. + let guard = store.acquire_write(owner, name).await.unwrap(); + crate::git::store::init_bare(guard.path()).unwrap(); + + let cleanup_ran = std::sync::atomic::AtomicBool::new(false); + let ok = guard + .release_with_failure_cleanup(true, |p| { + // Models the rollback: mutate a marker the double snapshots at + // upload entry, so the compensating upload's observed state is + // provably POST-cleanup. + std::fs::write(p.join("rollback-marker"), b"rolled back").unwrap(); + cleanup_ran.store(true, std::sync::atomic::Ordering::SeqCst); + }) + .await; + + assert!(!ok, "a timed-out upload must still be reported as failed"); + assert!( + cleanup_ran.load(std::sync::atomic::Ordering::SeqCst), + "cleanup must run on the timeout arm" + ); + assert_eq!( + calls.load(std::sync::atomic::Ordering::SeqCst), + 2, + "the timeout arm must attempt exactly one compensating upload after cleanup" + ); + assert_eq!( + *marker_at_upload.lock().unwrap(), + vec![false, true], + "the compensating upload must observe the POST-cleanup (rolled-back) tree" + ); + } + + // The Err arm must NOT compensate: an erroring PUT definitively failed, so + // there is no orphan archive to converge over. Exactly one upload call. + #[sqlx::test] + async fn errored_release_upload_does_not_compensate(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let calls = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let ts = CompensationProbeStore { + err_first: true, + calls: calls.clone(), + marker_at_upload: std::sync::Arc::new(std::sync::Mutex::new(Vec::new())), + }; + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ) + .with_release_upload_timeout(std::time::Duration::from_millis(300)); + let owner = "did:key:z6MkErrNoCompensateAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "nocompensate"; + + let guard = store.acquire_write(owner, name).await.unwrap(); + crate::git::store::init_bare(guard.path()).unwrap(); + + let cleanup_ran = std::sync::atomic::AtomicBool::new(false); + let ok = guard + .release_with_failure_cleanup(true, |_| { + cleanup_ran.store(true, std::sync::atomic::Ordering::SeqCst) + }) + .await; + + assert!(!ok, "an erroring upload must be reported as failed"); + assert!( + cleanup_ran.load(std::sync::atomic::Ordering::SeqCst), + "cleanup must run on the error arm" + ); + assert_eq!( + calls.load(std::sync::atomic::Ordering::SeqCst), + 1, + "the error arm must not attempt a compensating upload" + ); + } + + // FINDING 3 / R2 / INV-22: acquire_write's under-lock freshness DOWNLOAD must + // be bounded, not just release()'s upload. acquire_write holds the write + // advisory lock on a pinned lock-pool connection, then does store.exists() + + // store.download() before returning the guard. A stalled (never-erroring) GET + // would pin that lock and its connection forever, and the local-copy fallback + // (which only fires on Err) never triggers on a hang. With a short bound and a + // download that never completes, acquire_write returns within the bound and + // the lock is freed. Pre-fix (untimed download) acquire_write hangs on the + // stalled GET and never returns -> the outer 5s timeout fires -> RED. + #[sqlx::test] + async fn acquire_write_download_stall_is_bounded( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + use sqlx::ConnectOptions; + let pool = no_reap_pool(pool_opts, &connect_opts).await; + let tmp = tempfile::TempDir::new().unwrap(); + let mut ts = GatedStore::new(true); + // The download parks forever; the acquire bound is what must unblock it. + ts.download_gate = Some(std::sync::Arc::new(tokio::sync::Notify::new())); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ) + .with_release_upload_timeout(std::time::Duration::from_millis(300)); + let owner = "did:key:z6MkAcquireWriteStallAAAAAAAAAAAAAAAAAAAAAA"; + let name = "acqwritestall"; + let key = lock_key_for(owner, name); + let mut observer = connect_opts.connect().await.unwrap(); + + // No local copy exists, so the timed-out download has nothing to fall back + // to and bails; the guard drops and its Drop frees the advisory lock. The + // call must RETURN within the bound rather than hang on the stalled GET. + let start = std::time::Instant::now(); + let res = tokio::time::timeout( + std::time::Duration::from_secs(5), + store.acquire_write(owner, name), + ) + .await + .expect("acquire_write must return within the download bound, not hang on the stalled GET"); + assert!( + res.is_err(), + "with no local copy to fall back to, a timed-out download must bail" + ); + assert!( + start.elapsed() < std::time::Duration::from_secs(5), + "acquire_write must return within the download bound" + ); + assert!( + poll_until_free(&mut observer, key).await, + "the advisory lock must be freed after the bounded (timed-out) download" + ); + } + + // R2/INV-22: the OTHER under-lock upload path — upload_locked's PUT (init's + // background upload, wait=true) — must be bounded exactly like release()'s. + // With a stalled PUT, upload_locked acquires the advisory lock, parks on the + // PUT, and the timeout is the only thing that frees the lock. Pre-fix + // (untimed store.upload) the lock is pinned forever -> poll_until_free RED. + #[sqlx::test] + async fn upload_locked_put_stall_is_bounded( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + use sqlx::ConnectOptions; + let pool = no_reap_pool(pool_opts, &connect_opts).await; + let tmp = tempfile::TempDir::new().unwrap(); + let mut ts = GatedStore::new(false); + // The PUT parks forever; the upload bound is what must unblock it. + ts.upload_gate = Some(std::sync::Arc::new(tokio::sync::Notify::new())); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ) + .with_release_upload_timeout(std::time::Duration::from_millis(300)); + let owner = "did:key:z6MkUploadLockedStallAAAAAAAAAAAAAAAAAAAAAA"; + let name = "uploadlockedstall"; + let key = lock_key_for(owner, name); + let mut observer = connect_opts.connect().await.unwrap(); + + // init spawns upload_locked(wait=true); it inits the bare dir, acquires + // the lock, then parks on the stalled PUT. + store.init(owner, name).await.unwrap(); + + // The spawned upload must first take the advisory lock (then stall on the + // PUT) — proving the lock IS held, so the free-within-bound assert below + // is load-bearing and not vacuously green on a never-acquired lock. + assert!( + poll_until_held(&mut observer, key).await, + "the spawned init upload must acquire the advisory lock before its PUT" + ); + + // The PUT is parked forever: only the bound can free the lock. Pre-fix + // (untimed PUT) it never frees within the 5s poll window -> RED. + assert!( + poll_until_free(&mut observer, key).await, + "upload_locked's PUT must be bounded — the advisory lock must free within the timeout" + ); + } + + // ── U4: read-path downloads participate in the purge lock (R5, R7) ────── + + async fn poll_until_true(mut cond: impl FnMut() -> bool) -> bool { + for _ in 0..50 { + if cond() { + return true; + } + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + } + false + } + + // R5: an in-flight cache-miss download must not resurrect a purged repo. + // The reader parks mid-download (holding NO advisory lock, so the purge + // proceeds), the purge runs to completion (archive deleted), and the + // resumed reader's under-lock exists() re-check must discard rather than + // publish. Pre-fix the resumed download publishes straight onto the local + // path and the repo dir returns -> RED. + #[sqlx::test] + async fn download_does_not_resurrect_a_purged_repo(pool: sqlx::PgPool) { + use std::sync::atomic::Ordering::SeqCst; + let tmp = tempfile::TempDir::new().unwrap(); + let mut ts = GatedStore::new(true); + let gate = std::sync::Arc::new(tokio::sync::Notify::new()); + ts.download_gate = Some(gate.clone()); + let downloads = ts.downloads.clone(); + let exists = ts.exists.clone(); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ); + let owner = "did:key:z6MkDlResurrectAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "dlresurrect"; + let dir = repo_dir_of(tmp.path(), owner, name); + + // Cache miss: the reader enters the download path and parks on the gate. + let store2 = store.clone(); + let (o2, n2) = (owner.to_string(), name.to_string()); + let h = tokio::spawn(async move { store2.acquire(&o2, &n2).await }); + assert!( + poll_until_true(|| downloads.lock().unwrap().len() == 1).await, + "the reader's download must be in flight" + ); + + // Purge runs to completion while the download is parked (no local dir + // exists on this cache-miss path; the archive delete is the purge). + store.delete_archive(owner, name).await.unwrap(); + assert!(!exists.load(SeqCst), "archive deleted by the purge"); + + // Resume the download; the reader must NOT publish the purged repo. + gate.notify_one(); + let res = h.await.unwrap().unwrap(); + assert_eq!(res, dir); + assert!( + !dir.exists(), + "a purged repo must not be resurrected by an in-flight read download" + ); + } + + // R5: same interleaving through acquire_fresh's refresh-over-an-existing- + // copy arm — a local dir is present when the refresh starts, and the purge + // removes BOTH the dir and the archive mid-download. Pre-fix the resumed + // refresh publishes onto the purged path and the dir returns -> RED. + #[sqlx::test] + async fn fresh_refresh_does_not_resurrect_a_purged_repo(pool: sqlx::PgPool) { + use std::sync::atomic::Ordering::SeqCst; + let tmp = tempfile::TempDir::new().unwrap(); + let mut ts = GatedStore::new(true); + let gate = std::sync::Arc::new(tokio::sync::Notify::new()); + ts.download_gate = Some(gate.clone()); + let downloads = ts.downloads.clone(); + let exists = ts.exists.clone(); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ); + let owner = "did:key:z6MkFreshResurrectAAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "freshresurrect"; + let dir = repo_dir_of(tmp.path(), owner, name); + std::fs::create_dir_all(&dir).unwrap(); + + // The refresh always downloads, even over an existing local copy. + let store2 = store.clone(); + let (o2, n2) = (owner.to_string(), name.to_string()); + let h = tokio::spawn(async move { store2.acquire_fresh(&o2, &n2).await }); + assert!( + poll_until_true(|| downloads.lock().unwrap().len() == 1).await, + "the refresh download must be in flight" + ); + + // Purge runs to completion mid-download: local dir AND archive removed. + std::fs::remove_dir_all(&dir).unwrap(); + store.delete_archive(owner, name).await.unwrap(); + assert!(!exists.load(SeqCst), "archive deleted by the purge"); + + gate.notify_one(); + let res = h.await.unwrap().unwrap(); + assert_eq!(res, dir); + assert!( + !dir.exists(), + "a purged repo must not be resurrected by an in-flight refresh download" + ); + } + + // R5: the degraded outcome of the under-lock skip, pinned in full. The + // purge completes before the reader reaches the advisory lock; the + // reader's under-lock exists() re-check sees false and publishes nothing: + // Ok(missing path), no dir, no republished archive, exactly one download. + // Its RED form is the same interleaving as the two resurrection tests + // above; this test pins the post-fix outcome shape. + #[sqlx::test] + async fn post_purge_download_skips_under_lock(pool: sqlx::PgPool) { + use std::sync::atomic::Ordering::SeqCst; + let tmp = tempfile::TempDir::new().unwrap(); + let mut ts = GatedStore::new(true); + let gate = std::sync::Arc::new(tokio::sync::Notify::new()); + ts.download_gate = Some(gate.clone()); + let downloads = ts.downloads.clone(); + let uploads = ts.uploads.clone(); + let exists = ts.exists.clone(); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ); + let owner = "did:key:z6MkPostPurgeSkipAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "postpurgeskip"; + let dir = repo_dir_of(tmp.path(), owner, name); + + let store2 = store.clone(); + let (o2, n2) = (owner.to_string(), name.to_string()); + let h = tokio::spawn(async move { store2.acquire(&o2, &n2).await }); + assert!( + poll_until_true(|| downloads.lock().unwrap().len() == 1).await, + "the reader's download must be in flight" + ); + store.delete_archive(owner, name).await.unwrap(); + gate.notify_one(); + + let res = h.await.unwrap().unwrap(); + assert_eq!( + res, dir, + "degraded read returns the missing path, not an error" + ); + assert!(!dir.exists(), "the skipped download must publish nothing"); + assert!( + !exists.load(SeqCst), + "the purged archive must stay deleted after the skip" + ); + assert!( + uploads.lock().unwrap().is_empty(), + "the read path must never upload" + ); + assert_eq!( + downloads.lock().unwrap().len(), + 1, + "exactly one download attempt was made" + ); + } + + // R7: two simultaneous cache-miss reads of the same repo coalesce on the + // per-repo download mutex: exactly one download occurs, both callers end + // with the published dir, neither errors. Pre-fix each caller runs its own + // download -> the count hits 2 -> RED. + #[sqlx::test] + async fn concurrent_cold_reads_download_once(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut ts = GatedStore::new(true); + let gate = std::sync::Arc::new(tokio::sync::Notify::new()); + ts.download_gate = Some(gate.clone()); + let downloads = ts.downloads.clone(); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ); + let owner = "did:key:z6MkColdCoalesceAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "coldcoalesce"; + let dir = repo_dir_of(tmp.path(), owner, name); + + let store1 = store.clone(); + let (o1, n1) = (owner.to_string(), name.to_string()); + let h1 = tokio::spawn(async move { store1.acquire(&o1, &n1).await }); + assert!( + poll_until_true(|| downloads.lock().unwrap().len() == 1).await, + "the first reader's download must be in flight" + ); + + let store2 = store.clone(); + let (o2, n2) = (owner.to_string(), name.to_string()); + let h2 = tokio::spawn(async move { store2.acquire(&o2, &n2).await }); + // Give the second reader time to reach the coordination point. + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + assert_eq!( + downloads.lock().unwrap().len(), + 1, + "the second cold read must await the in-flight download, not start its own" + ); + + // Release the parked download. The second notify covers the pre-fix + // topology where both readers park on the gate; post-fix it leaves an + // unconsumed permit, which is harmless. + gate.notify_one(); + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + gate.notify_one(); + + let p1 = h1.await.unwrap().unwrap(); + let p2 = h2.await.unwrap().unwrap(); + assert_eq!(p1, dir); + assert_eq!(p2, dir); + assert!(dir.exists(), "both callers end with the published dir"); + assert_eq!( + downloads.lock().unwrap().len(), + 1, + "exactly one download for two concurrent cold reads" + ); + } + + // R7/INV-15: the long network phase must not pin a lock-pool connection — + // while a download is parked mid-flight, that repo's advisory lock is + // observably FREE (out-of-pool observer). This is the distinct-repo-burst + // pool guard: N cold reads across N repos must not drain the writer-sized + // lock pool. GREEN by construction pre-fix too (the old download path held + // no lock either); it fences the new design against regression. + #[sqlx::test] + async fn no_advisory_lock_held_while_download_parked( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + use sqlx::ConnectOptions; + let pool = no_reap_pool(pool_opts, &connect_opts).await; + let tmp = tempfile::TempDir::new().unwrap(); + let mut ts = GatedStore::new(true); + let gate = std::sync::Arc::new(tokio::sync::Notify::new()); + ts.download_gate = Some(gate.clone()); + let downloads = ts.downloads.clone(); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ); + let owner = "did:key:z6MkNoLockParkedAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "nolockparked"; + let dir = repo_dir_of(tmp.path(), owner, name); + let key = lock_key_for(owner, name); + let mut observer = connect_opts.connect().await.unwrap(); + + let store2 = store.clone(); + let (o2, n2) = (owner.to_string(), name.to_string()); + let h = tokio::spawn(async move { store2.acquire(&o2, &n2).await }); + assert!( + poll_until_true(|| downloads.lock().unwrap().len() == 1).await, + "the reader's download must be in flight" + ); + + assert!( + advisory_lock_is_free(&mut observer, key).await, + "the repo's advisory lock must be free while the download is parked mid-flight" + ); + + gate.notify_one(); + let res = h.await.unwrap().unwrap(); + assert_eq!(res, dir); + assert!(dir.exists(), "the resumed download publishes normally"); + } + + // R7: acquire's local-dir hit path stays lock-free — deterministic form. + // An out-of-pool observer HOLDS the repo's advisory lock for the whole + // call window; a dir-present acquire must succeed promptly regardless (a + // lock-touching cache hit would park until the hold ends). GREEN by + // construction pre-fix (the hit path never locked); it fences the new + // design's promise that the hit path stays byte-identical and lock-free. + #[sqlx::test] + async fn cache_hit_never_touches_the_lock( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + use sqlx::ConnectOptions; + let pool = no_reap_pool(pool_opts, &connect_opts).await; + let tmp = tempfile::TempDir::new().unwrap(); + // exists=true so the lazy-migration spawn sees "already in tigris" and + // neither uploads nor waits on anything. + let ts = GatedStore::new(true); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ); + let owner = "did:key:z6MkCacheHitNoLockAAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "cachehitnolock"; + let dir = repo_dir_of(tmp.path(), owner, name); + std::fs::create_dir_all(&dir).unwrap(); + let key = lock_key_for(owner, name); + + // The observer takes and HOLDS the repo's advisory lock. + let mut observer = connect_opts.connect().await.unwrap(); + let (got,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut observer) + .await + .unwrap(); + assert!(got, "observer must win the free lock"); + + let res = tokio::time::timeout( + std::time::Duration::from_secs(2), + store.acquire(owner, name), + ) + .await + .expect("a dir-present acquire must not wait on the held advisory lock") + .unwrap(); + assert_eq!(res, dir); + + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut observer) + .await; + } + + // ── Finding 2: stale-swap over a fresher writer copy ──────────────────── + + // A cold `acquire` reader downloads to a temp holding NO advisory lock; + // during that window a writer can lock, publish a fresher on-disk copy, and + // release. When the reader finally takes the lock, a presence-only check + // would swap its now-stale temp over the writer's fresher dir. The guard + // detects the writer's dir (absent at entry, present under the lock) and + // serves it instead. Pre-fix the reader swaps and the writer's copy is lost + // -> RED (the sentinel disappears). + #[sqlx::test] + async fn cold_read_defers_to_writer_published_during_download(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut ts = GatedStore::new(true); + let gate = std::sync::Arc::new(tokio::sync::Notify::new()); + ts.download_gate = Some(gate.clone()); + let downloads = ts.downloads.clone(); + let uploads = ts.uploads.clone(); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ); + let owner = "did:key:z6MkColdDeferAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "colddefer"; + let dir = repo_dir_of(tmp.path(), owner, name); + + // Cache miss: the reader enters the download path and parks (no local dir). + let store2 = store.clone(); + let (o2, n2) = (owner.to_string(), name.to_string()); + let h = tokio::spawn(async move { store2.acquire(&o2, &n2).await }); + assert!( + poll_until_true(|| downloads.lock().unwrap().len() == 1).await, + "the reader's download must be in flight" + ); + + // A writer publishes a fresher copy onto the local path during the + // window (simulated directly: the writer would hold the lock only while + // it works, which is over before the reader takes the lock). + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("writer_sentinel"), b"fresher").unwrap(); + + gate.notify_one(); + let res = h.await.unwrap().unwrap(); + assert_eq!(res, dir); + assert!( + dir.join("writer_sentinel").exists(), + "the reader must serve the writer's fresher copy, not swap its stale temp over it" + ); + assert!( + uploads.lock().unwrap().is_empty(), + "the read path must never upload" + ); + assert_eq!(downloads.lock().unwrap().len(), 1, "exactly one download"); + } + + // acquire_fresh refreshes over an EXISTING local copy, so a presence check + // cannot detect a concurrent writer (the dir is always present). The guard + // compares the local dir's mtime captured at entry against its value under + // the lock; a writer that replaced the dir during the window changed the + // mtime, so the reader serves the writer's copy rather than swapping its + // stale refresh over it. Pre-fix the reader swaps -> RED (version B lost). + #[sqlx::test] + async fn fresh_refresh_defers_to_writer_published_during_download(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut ts = GatedStore::new(true); + let gate = std::sync::Arc::new(tokio::sync::Notify::new()); + ts.download_gate = Some(gate.clone()); + let downloads = ts.downloads.clone(); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ); + let owner = "did:key:z6MkFreshDeferAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "freshdefer"; + let dir = repo_dir_of(tmp.path(), owner, name); + + // Version A already on disk when the refresh starts. + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("version_a"), b"A").unwrap(); + let mtime_before = std::fs::metadata(&dir).unwrap().modified().unwrap(); + + let store2 = store.clone(); + let (o2, n2) = (owner.to_string(), name.to_string()); + let h = tokio::spawn(async move { store2.acquire_fresh(&o2, &n2).await }); + assert!( + poll_until_true(|| downloads.lock().unwrap().len() == 1).await, + "the refresh download must be in flight" + ); + + // Cross a filesystem mtime tick, then a writer replaces the dir with + // version B. The long sleep guarantees B's mtime differs from A's even + // on a coarse-granularity filesystem, so the comparison is deterministic. + tokio::time::sleep(std::time::Duration::from_millis(1_100)).await; + std::fs::remove_dir_all(&dir).unwrap(); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("version_b"), b"B").unwrap(); + let mtime_after = std::fs::metadata(&dir).unwrap().modified().unwrap(); + assert!( + mtime_after != mtime_before, + "precondition: the writer's replacement must change the dir mtime for the guard to fire" + ); + + gate.notify_one(); + let res = h.await.unwrap().unwrap(); + assert_eq!(res, dir); + assert!( + dir.join("version_b").exists(), + "the refresh must serve the writer's fresher copy (B), not swap version A's stale temp over it" + ); + assert!( + !dir.join("version_a").exists(), + "version A must be gone; the writer replaced it, and the reader must not resurrect it" + ); + assert_eq!(downloads.lock().unwrap().len(), 1, "exactly one download"); + } + + // ── Finding 4: the try_lock_repo Ok(None) contention degrade ──────────── + + fn has_leftover_temp(parent: &std::path::Path) -> bool { + std::fs::read_dir(parent) + .map(|rd| { + rd.flatten() + .any(|e| e.file_name().to_string_lossy().contains(".tmp-download.")) + }) + .unwrap_or(false) + } + + // KTD-3 degrade: while a live writer/purge holds the repo's advisory lock, a + // read-path downloader that reaches try_lock AFTER its fetch must discard its + // temp and degrade. This exercises the Ok(None) arm the six U4 tests never + // reach (they simulate purge via delete_archive and never hold the lock + // while a downloader reaches try_lock). An out-of-pool observer holds the + // lock; GatedStore.download_gate parks the reader past the fetch so it + // reaches try_lock while the observer still holds it. + // + // acquire variant: degrades to the missing path, publishes nothing. + #[sqlx::test] + async fn acquire_download_degrades_when_writer_holds_lock( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + use sqlx::ConnectOptions; + let pool = no_reap_pool(pool_opts, &connect_opts).await; + let tmp = tempfile::TempDir::new().unwrap(); + let mut ts = GatedStore::new(true); + let gate = std::sync::Arc::new(tokio::sync::Notify::new()); + ts.download_gate = Some(gate.clone()); + let downloads = ts.downloads.clone(); + let uploads = ts.uploads.clone(); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ); + let owner = "did:key:z6MkAcquireDegradeAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "acqdegrade"; + let dir = repo_dir_of(tmp.path(), owner, name); + let key = lock_key_for(owner, name); + + // An out-of-pool observer holds the repo's advisory lock for the window. + let mut observer = connect_opts.connect().await.unwrap(); + let (got,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut observer) + .await + .unwrap(); + assert!(got, "observer must win the free lock"); + + // Cache miss: reader downloads to temp (no lock held), parks on the gate. + let store2 = store.clone(); + let (o2, n2) = (owner.to_string(), name.to_string()); + let h = tokio::spawn(async move { store2.acquire(&o2, &n2).await }); + assert!( + poll_until_true(|| downloads.lock().unwrap().len() == 1).await, + "the reader's download must be in flight" + ); + + // Release the gate: the reader reaches try_lock while the observer holds + // the lock -> Ok(None) -> discard temp, degrade to the missing path. + gate.notify_one(); + let res = h.await.unwrap().unwrap(); + assert_eq!( + res, dir, + "degraded acquire returns the missing path, not an error" + ); + assert!(!dir.exists(), "the contended download must publish nothing"); + assert!( + uploads.lock().unwrap().is_empty(), + "the read path must never upload" + ); + assert_eq!( + downloads.lock().unwrap().len(), + 1, + "exactly one download attempt" + ); + assert!( + !has_leftover_temp(dir.parent().unwrap()), + "the discarded download temp must be cleaned up" + ); + + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut observer) + .await; + } + + // acquire_fresh variant: degrades to serving the existing local copy. + #[sqlx::test] + async fn fresh_download_degrades_to_local_copy_when_writer_holds_lock( + pool_opts: sqlx::postgres::PgPoolOptions, + connect_opts: sqlx::postgres::PgConnectOptions, + ) { + use sqlx::ConnectOptions; + let pool = no_reap_pool(pool_opts, &connect_opts).await; + let tmp = tempfile::TempDir::new().unwrap(); + let mut ts = GatedStore::new(true); + let gate = std::sync::Arc::new(tokio::sync::Notify::new()); + ts.download_gate = Some(gate.clone()); + let downloads = ts.downloads.clone(); + let uploads = ts.uploads.clone(); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ); + let owner = "did:key:z6MkFreshDegradeAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "freshdegrade"; + let dir = repo_dir_of(tmp.path(), owner, name); + let key = lock_key_for(owner, name); + + // An existing local copy is present when the refresh starts. + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write(dir.join("local_copy"), b"keep").unwrap(); + + let mut observer = connect_opts.connect().await.unwrap(); + let (got,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut observer) + .await + .unwrap(); + assert!(got, "observer must win the free lock"); + + let store2 = store.clone(); + let (o2, n2) = (owner.to_string(), name.to_string()); + let h = tokio::spawn(async move { store2.acquire_fresh(&o2, &n2).await }); + assert!( + poll_until_true(|| downloads.lock().unwrap().len() == 1).await, + "the refresh download must be in flight" + ); + + gate.notify_one(); + let res = h.await.unwrap().unwrap(); + assert_eq!(res, dir, "degraded acquire_fresh returns the local path"); + assert!( + dir.join("local_copy").exists(), + "the contended refresh must serve the existing local copy untouched" + ); + assert!( + uploads.lock().unwrap().is_empty(), + "the read path must never upload" + ); + assert_eq!( + downloads.lock().unwrap().len(), + 1, + "exactly one download attempt" + ); + assert!( + !has_leftover_temp(dir.parent().unwrap()), + "the discarded download temp must be cleaned up" + ); + + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut observer) + .await; + } + + // ── Finding 4: the download-coordination map entry is freed on cancellation ─ + + // A cold-read holder parks mid-download, then its handler future is dropped + // (axum cancels it on client disconnect). The per-repo map entry must be + // removed once the in-flight download settles. The original pre-fix code + // removed it only at explicit return points, which a cancelled future + // skips, so a permissionless caller could fan cold GETs across many public + // repos and disconnect to grow the map. The RAII DownloadEntryGuard's Drop + // closes that gap; the guard now travels with the spawned download task, + // so the entry deliberately STAYS registered while the download is in + // flight (that is what keeps cancel-then-retry from duplicating the + // download) and is pruned at settle. This reads the private download_locks + // map directly (the test module is a child of RepoStore's module). + // Original bug: after abort the entry leaked forever -> never freed -> RED. + #[sqlx::test] + async fn cancelled_cold_read_frees_the_map_entry(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut ts = GatedStore::new(true); + let gate = std::sync::Arc::new(tokio::sync::Notify::new()); + ts.download_gate = Some(gate.clone()); + let downloads = ts.downloads.clone(); + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ); + let owner = "did:key:z6MkCancelMapEntryAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "cancelmapentry"; + let map_key = format!("{}/{name}", owner.replace([':', '/'], "_")); + + // Cache miss: the holder enters download_published, registers the map + // entry, and parks mid-download on the gate. + let store2 = store.clone(); + let (o2, n2) = (owner.to_string(), name.to_string()); + let h = tokio::spawn(async move { store2.acquire(&o2, &n2).await }); + assert!( + poll_until_true(|| downloads.lock().unwrap().len() == 1).await, + "the reader's download must be in flight" + ); + // Precondition: the map entry is registered while the holder runs. + assert!( + store.download_locks.lock().unwrap().contains_key(&map_key), + "the coordination entry must be registered while the holder downloads" + ); + + // Cancel the handler future mid-download (the client-disconnect hazard). + h.abort(); + assert!( + h.await.unwrap_err().is_cancelled(), + "the holder future must be cancelled, not settled" + ); + + // The spawned download task owns the coordination, so the entry stays + // registered while the download is still in flight; a reader arriving + // now queues on it instead of starting a duplicate download. + assert!( + store.download_locks.lock().unwrap().contains_key(&map_key), + "the entry must stay registered while the cancelled holder's download is in flight" + ); + + // Let the parked download settle; the task's returned guards then drop + // in the runtime and prune the entry. + gate.notify_one(); + let freed = poll_until_true(|| store.download_locks.lock().unwrap().is_empty()).await; + assert!( + freed, + "the map entry must be freed once the cancelled holder's download settles" + ); + } + + // ── F3: a timed-out cold read must not resurrect its abandoned temp dir ── + + // An ObjectStore double that reproduces the detached-spawn_blocking + // resurrection. Its download() runs the extraction in spawn_blocking (as the + // real TigrisClient::download does) and awaits the join handle. A tokio + // timeout that fires while the blocking task runs does NOT cancel it: + // dropping the download future detaches the task, which then creates ("renames + // into place") the target path LATE — after the caller's temp-dir cleanup has + // already run. `extracted` is bumped once the blocking task has created the + // dir, so a test can wait for the extraction to actually land. + struct DetachExtractStore { + extract_delay: std::time::Duration, + downloads: std::sync::Arc, + extracted: std::sync::Arc, + } + + #[async_trait::async_trait] + impl crate::git::tigris::ObjectStore for DetachExtractStore { + async fn exists(&self, _o: &str, _r: &str) -> anyhow::Result { + Ok(true) + } + async fn upload(&self, _o: &str, _r: &str, _p: &std::path::Path) -> anyhow::Result<()> { + Ok(()) + } + async fn download(&self, _o: &str, _r: &str, p: &std::path::Path) -> anyhow::Result<()> { + self.downloads + .fetch_add(1, std::sync::atomic::Ordering::SeqCst); + let target = p.to_path_buf(); + let delay = self.extract_delay; + let extracted = self.extracted.clone(); + // Mirror TigrisClient::download: the extraction runs in spawn_blocking + // and we await the join. A timeout that drops THIS future detaches the + // blocking task, which finishes its "rename into place" after cleanup. + tokio::task::spawn_blocking(move || { + std::thread::sleep(delay); + std::fs::create_dir_all(&target).unwrap(); + // A marker so the resurrected dir looks like a real extraction. + std::fs::write(target.join("HEAD"), b"ref: refs/heads/main\n").unwrap(); + extracted.fetch_add(1, std::sync::atomic::Ordering::SeqCst); + }) + .await + .unwrap(); + Ok(()) + } + async fn delete(&self, _o: &str, _r: &str) -> anyhow::Result<()> { + Ok(()) + } + } + + // F3: a cold read whose download times out must not let the detached + // spawn_blocking extraction resurrect the abandoned temp dir after cleanup. + // The store's extraction lands well past the download timeout; the timeout + // fires, acquire bails, and the extraction then creates ("renames into place") + // temp.path(). The fix ties cleanup to the extraction's completion: a spawned + // task owns the download and the temp guard, so the temp dir is removed only + // once the download settles, and no `.tmp-download.*` dir survives. Pre-fix the timeout arm drops the + // download future, the detached extraction recreates temp.path() AFTER the RAII + // guard's Drop, and it survives forever — the second poll never sees a clean + // parent -> RED. + #[sqlx::test] + async fn timed_out_cold_read_does_not_resurrect_temp_dir(pool: sqlx::PgPool) { + use std::sync::atomic::Ordering::SeqCst; + let tmp = tempfile::TempDir::new().unwrap(); + let extracted = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let downloads = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let ts = DetachExtractStore { + // The extraction lands well after the 150ms download timeout below. + extract_delay: std::time::Duration::from_millis(600), + downloads: downloads.clone(), + extracted: extracted.clone(), + }; + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ) + .with_release_upload_timeout(std::time::Duration::from_millis(150)); + let owner = "did:key:z6MkF3ResurrectTempDirAAAAAAAAAAAAAAAAAAAA"; + let name = "f3resurrect"; + let dir = repo_dir_of(tmp.path(), owner, name); + let parent = dir.parent().unwrap().to_path_buf(); + + // Cold cache miss: acquire enters the download path, the download times out + // at 150ms, and the read path bails (no local copy to serve). + let res = store.acquire(owner, name).await; + assert!( + res.is_err(), + "a timed-out cold read with no local copy must bail" + ); + assert_eq!(downloads.load(SeqCst), 1, "exactly one download attempt"); + + // Wait for the detached extraction to actually finish its "rename into + // place" (the resurrection moment). + assert!( + poll_until_true(|| extracted.load(SeqCst) >= 1).await, + "the detached extraction must run to completion" + ); + + // Once the extraction has landed, no temp dir may survive under the parent. + // The fix's janitor removes temp.path() after the extraction settles; + // pre-fix the resurrected dir survives and this poll never sees it cleaned. + assert!( + poll_until_true(|| !has_leftover_temp(&parent)).await, + "a timed-out cold read must not leave a resurrected .tmp-download.* dir" + ); + } + + // Cancellation twin of the resurrection test above: the handler future is + // DROPPED mid-download (axum cancels it on client disconnect) instead of + // timing out. Pre-fix, dropping the handler dropped the download future + // while its spawn_blocking extraction kept running; TempDownloadDir::Drop + // removed the (not yet created) temp dir, and the extraction's later + // "rename into place" resurrected it as an orphan that only a later + // same-repo download would sweep -> the cleanup poll never succeeds -> RED. + // Post-fix the spawned task owns the download and the temp guard, keeps + // running across the caller's cancellation, and removes the temp dir once + // the download settles -> GREEN. + #[sqlx::test] + async fn cancelled_cold_read_does_not_resurrect_temp_dir(pool: sqlx::PgPool) { + use std::sync::atomic::Ordering::SeqCst; + let tmp = tempfile::TempDir::new().unwrap(); + let extracted = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let downloads = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let ts = DetachExtractStore { + // Long enough that the abort below lands while the extraction is + // still running, so the cancellation drops the download mid-flight. + extract_delay: std::time::Duration::from_millis(600), + downloads: downloads.clone(), + extracted: extracted.clone(), + }; + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ); + let owner = "did:key:z6MkCancelResurrectTempDirAAAAAAAAAAAAAAAA"; + let name = "cancelresurrect"; + let dir = repo_dir_of(tmp.path(), owner, name); + let parent = dir.parent().unwrap().to_path_buf(); + + // Cold cache miss: the handler enters the download path and its store + // download (the slow extraction) is in flight. + let store2 = store.clone(); + let (o2, n2) = (owner.to_string(), name.to_string()); + let h = tokio::spawn(async move { store2.acquire(&o2, &n2).await }); + assert!( + poll_until_true(|| downloads.load(SeqCst) >= 1).await, + "the reader's download must be in flight" + ); + + // Drop the handler future mid-download (the client-disconnect hazard). + h.abort(); + + // Wait for the underlying extraction to settle (the resurrection moment + // pre-fix; the cleanup moment post-fix). + assert!( + poll_until_true(|| extracted.load(SeqCst) >= 1).await, + "the detached extraction must run to completion" + ); + + // Once the download has settled, no temp dir may survive under the + // parent. Pre-fix the resurrected dir survives and this poll never + // sees a clean parent -> RED. + assert!( + poll_until_true(|| !has_leftover_temp(&parent)).await, + "a cancelled cold read must not leave a resurrected .tmp-download.* dir" + ); + } + + // Timeout waves must coalesce on the ONE in-flight download. Pre-fix, a + // holder whose download timed out released the per-repo mutex and pruned + // the map entry while its spawned download task was still running; the + // next reader of the same repo then became a fresh holder and started a + // second full store.download, so N timeout waves meant N concurrent + // downloads of the same object. Post-fix the spawned download task itself + // owns `held` and the entry guard until it settles, so the second reader + // queues on the same entry, its bounded wait elapses, and it reports the + // same timeout error WITHOUT invoking store.download again. RED recipe: in + // download.rs, revert the ownership move (make the spawned task return + // only `temp` and keep `held`/`entry_guard` in the caller frame, the + // pre-fix unwind): the holder's timeout then frees the lock and entry + // while the download still runs, the second reader acquires them and + // starts download #2, and the downloads == 1 assert fails. + #[sqlx::test] + async fn timed_out_cold_reads_coalesce_on_the_inflight_download(pool: sqlx::PgPool) { + use std::sync::atomic::Ordering::SeqCst; + let tmp = tempfile::TempDir::new().unwrap(); + let extracted = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let downloads = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let ts = DetachExtractStore { + // The download settles well after BOTH callers' 150ms timeouts. + extract_delay: std::time::Duration::from_millis(600), + downloads: downloads.clone(), + extracted: extracted.clone(), + }; + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ) + .with_release_upload_timeout(std::time::Duration::from_millis(150)); + let owner = "did:key:z6MkTimeoutCoalesceAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "timeoutcoalesce"; + let dir = repo_dir_of(tmp.path(), owner, name); + let parent = dir.parent().unwrap().to_path_buf(); + + // First cold reader: its download stalls and its 150ms timeout fires. + let store1 = store.clone(); + let (o1, n1) = (owner.to_string(), name.to_string()); + let h1 = tokio::spawn(async move { store1.acquire(&o1, &n1).await }); + assert!( + poll_until_true(|| downloads.load(SeqCst) == 1).await, + "the first reader's download must be in flight" + ); + // Start the second reader measurably after the first, so that pre-fix + // (RED) the lock the first holder frees at ~150ms is available well + // before the second reader's own 150ms wait elapses. + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + let store2 = store.clone(); + let (o2, n2) = (owner.to_string(), name.to_string()); + let h2 = tokio::spawn(async move { store2.acquire(&o2, &n2).await }); + + let res1 = h1.await.unwrap(); + let res2 = h2.await.unwrap(); + assert!( + res1.is_err(), + "the stalled holder must report its timeout error" + ); + let err2 = res2.expect_err("the coalesced reader must also time out, not succeed"); + assert!( + format!("{err2:#}").contains("read download timed out"), + "the coalesced reader reports the same timeout error, got: {err2:#}" + ); + assert_eq!( + downloads.load(SeqCst), + 1, + "both timed-out cold reads must share the ONE in-flight download" + ); + + // Once the in-flight download settles, the task-owned coordination must + // be released (map entry pruned) and the temp dir must be cleaned. + assert!( + poll_until_true(|| extracted.load(SeqCst) >= 1).await, + "the detached extraction must run to completion" + ); + assert!( + poll_until_true(|| store.download_locks.lock().unwrap().is_empty()).await, + "the map entry must be pruned once the download settles" + ); + assert!( + poll_until_true(|| !has_leftover_temp(&parent)).await, + "the settled download's temp dir must be cleaned up" + ); + } + + // Cancellation twin of the coalescing test above: the first cold reader is + // ABORTED mid-download (axum drops the handler on client disconnect) + // instead of timing out. Pre-fix (guards in the caller frame), the abort + // dropped `held` and the entry guard immediately while the spawned + // download task kept running, so a retry arriving right after the + // disconnect became a fresh holder and started a second full + // store.download. Post-fix the task owns the coordination, so the retry + // queues on the in-flight download, its bounded wait elapses, and it + // reports the timeout error WITHOUT a second download; the entry is + // pruned once the task settles. RED recipe: same revert as above (task + // returns only `temp`, guards stay caller-side): the abort then frees + // lock and entry at once, the retry starts download #2, and the + // downloads == 1 assert fails. + #[sqlx::test] + async fn cancelled_cold_read_coalesces_next_reader_on_the_inflight_download( + pool: sqlx::PgPool, + ) { + use std::sync::atomic::Ordering::SeqCst; + let tmp = tempfile::TempDir::new().unwrap(); + let extracted = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let downloads = std::sync::Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let ts = DetachExtractStore { + // The download settles well after the retry's 150ms bounded wait. + extract_delay: std::time::Duration::from_millis(600), + downloads: downloads.clone(), + extracted: extracted.clone(), + }; + let store = RepoStore::new( + tmp.path().to_path_buf(), + Some(std::sync::Arc::new(ts)), + pool, + ) + .with_release_upload_timeout(std::time::Duration::from_millis(150)); + let owner = "did:key:z6MkCancelCoalesceAAAAAAAAAAAAAAAAAAAAAAAAA"; + let name = "cancelcoalesce"; + let dir = repo_dir_of(tmp.path(), owner, name); + let parent = dir.parent().unwrap().to_path_buf(); + + // First cold reader: download in flight, then the handler is dropped. + let store1 = store.clone(); + let (o1, n1) = (owner.to_string(), name.to_string()); + let h1 = tokio::spawn(async move { store1.acquire(&o1, &n1).await }); + assert!( + poll_until_true(|| downloads.load(SeqCst) == 1).await, + "the first reader's download must be in flight" + ); + h1.abort(); + assert!( + h1.await.unwrap_err().is_cancelled(), + "the first reader must be cancelled, not settled" + ); + + // Retry after the disconnect, well before the download settles at + // ~600ms: it must queue on the in-flight download, not start its own. + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + let store2 = store.clone(); + let (o2, n2) = (owner.to_string(), name.to_string()); + let res2 = tokio::spawn(async move { store2.acquire(&o2, &n2).await }) + .await + .unwrap(); + let err2 = res2.expect_err("the retry must hit its bounded wait, not succeed"); + assert!( + format!("{err2:#}").contains("read download timed out"), + "the retry reports the bounded-wait timeout error, got: {err2:#}" + ); + assert_eq!( + downloads.load(SeqCst), + 1, + "a retry after a cancelled cold read must share the ONE in-flight download" + ); + + // At settle, the task-owned coordination unwinds: entry pruned, temp + // dir cleaned. + assert!( + poll_until_true(|| extracted.load(SeqCst) >= 1).await, + "the detached extraction must run to completion" + ); + assert!( + poll_until_true(|| store.download_locks.lock().unwrap().is_empty()).await, + "the map entry must be pruned once the cancelled holder's download settles" + ); + assert!( + poll_until_true(|| !has_leftover_temp(&parent)).await, + "the settled download's temp dir must be cleaned up" + ); + } } diff --git a/crates/gitlawb-node/src/git/repo_store/download.rs b/crates/gitlawb-node/src/git/repo_store/download.rs new file mode 100644 index 00000000..8543b11a --- /dev/null +++ b/crates/gitlawb-node/src/git/repo_store/download.rs @@ -0,0 +1,431 @@ +//! Read-path download coordination (KTD-3) for [`RepoStore`]. +//! +//! The read path's cache-miss handling lives here: the per-repo async mutex +//! that serializes concurrent readers of the same repo, the holder's fetch + +//! extract + under-lock publish, and the RAII temp-dir cleanup. `acquire` and +//! `acquire_fresh` stay in the parent module and call into `download_published`. +//! These methods reach the parent's private fields (`download_locks`) and +//! private methods (`try_lock_repo`) directly, because a child module can see +//! its parent's private items. + +use super::*; + +/// Outcome of a coordinated read-path download (KTD-3). +pub(super) enum DownloadOutcome { + /// The local dir is present: this reader downloaded and published it under + /// the advisory lock, or a concurrent reader did and this one served it. + Published, + /// The download was discarded without publishing: the advisory lock was + /// contended (a live writer or purge holds it), the lock pool errored, or + /// the archive vanished under the lock (purged mid-download). The caller + /// degrades to its missing-path or serve-local-copy outcome. + Skipped, +} + +/// RAII cleanup for a read-path download temp dir (finding 1). Its `Drop` +/// removes the dir (best effort), so a handler future cancelled mid-download +/// (axum drops it on client disconnect, the same hazard `RepoWriteGuard` / +/// `RepoLockGuard` Drop impls exist for) cannot leak a fully-populated repo +/// dir. `disarm` forgets the dir once the publish rename has consumed it. +struct TempDownloadDir { + path: PathBuf, + armed: bool, +} + +impl TempDownloadDir { + fn new(path: PathBuf) -> Self { + Self { path, armed: true } + } + + fn path(&self) -> &Path { + &self.path + } + + /// Disarm: the temp has been consumed (renamed into place), so `Drop` must + /// not try to remove it. + fn disarm(mut self) { + self.armed = false; + } +} + +impl Drop for TempDownloadDir { + fn drop(&mut self) { + if self.armed { + let _ = std::fs::remove_dir_all(&self.path); + } + } +} + +/// RAII removal of a download-coordination map entry (finding 4). Its `Drop` +/// removes the entry from `download_locks` whenever the map still holds THIS +/// Arc (ptr_eq-guarded so a newer entry inserted after an earlier removal is +/// never clobbered). Removal is guaranteed on every path (a normal return or +/// `?` drops it in the holder; once a download is in flight it travels with +/// the spawned task and drops at settle, covering holder timeout and a +/// handler future cancelled mid cold-read, which axum triggers on client +/// disconnect), which is what keeps the map from growing per arbitrary +/// requested name: the pre-fix explicit removals ran only at the labelled +/// return points, which a cancelled future skipped, leaking the entry. The +/// outer map is a std::sync::Mutex so +/// this Drop can prune it synchronously (Drop cannot be async); it is only ever +/// held briefly for a get/insert/remove, never across an await. +struct DownloadEntryGuard { + locks: Arc>>>>, + key: String, + entry: Arc>, +} + +impl Drop for DownloadEntryGuard { + fn drop(&mut self) { + // A poisoned map is still safe to prune — recover the guard and remove. + let mut map = self.locks.lock().unwrap_or_else(|p| p.into_inner()); + if map + .get(&self.key) + .is_some_and(|cur| Arc::ptr_eq(cur, &self.entry)) + { + map.remove(&self.key); + } + } +} + +impl RepoStore { + /// Coordinated read-path download (KTD-3). Serializes concurrent readers + /// of the same repo on a per-repo async mutex: the first one in becomes + /// the holder and runs [`download_and_publish`](Self::download_and_publish); + /// a contended reader awaits the holder, re-checks the local dir on wake, + /// and serves what the holder published instead of re-downloading. Only + /// callers that already confirmed the archive exists reach this, and the + /// map entry is removed on completion, so the map cannot grow per + /// arbitrary requested name. + pub(super) async fn download_published( + &self, + owner_did: &str, + repo_name: &str, + owner_slug: &str, + local_path: &Path, + store: &Arc, + local_existed_at_entry: bool, + ) -> Result { + let map_key = format!("{owner_slug}/{repo_name}"); + let entry = { + let mut map = self.download_locks.lock().unwrap(); + Arc::clone( + map.entry(map_key.clone()) + .or_insert_with(|| Arc::new(Mutex::new(()))), + ) + }; + let held = match Arc::clone(&entry).try_lock_owned() { + Ok(g) => g, + Err(_) => { + // Bounded coalescing wait: once a holder's download is in + // flight, the spawned task owns this lock (and the map entry) + // until it settles, whatever happens to the holder (timeout or + // cancellation), so an unbounded lock().await here would park + // this reader for as long as a stall lasts. On elapse, report + // the same timeout error the holder would. Do NOT touch the + // map entry on this path: pruning it while the download is in + // flight would let the next arrival start a duplicate download. + let waited = tokio::time::timeout( + self.release_upload_timeout, + Arc::clone(&entry).lock_owned(), + ) + .await; + let g = match waited { + Ok(g) => g, + Err(_) => { + if local_path.exists() { + // A concurrent reader published while we waited. + return Ok(DownloadOutcome::Published); + } + return Err(anyhow::anyhow!("read download timed out")); + } + }; + if local_path.exists() { + // A concurrent reader published while we waited — serve it. + // The holder's own guard prunes the map entry on its exit. + return Ok(DownloadOutcome::Published); + } + // The holder degraded without publishing; this reader takes over. + // The prior holder removed the map entry on its way out, so + // re-register THIS entry before downloading, so readers arriving + // now coalesce onto it instead of starting a duplicate download + // (finding 3). Narrows the duplicate-download window; publishes + // stay serialized by the advisory lock regardless. + { + let mut map = self.download_locks.lock().unwrap(); + map.entry(map_key.clone()) + .or_insert_with(|| Arc::clone(&entry)); + } + g + } + }; + // RAII: remove the map entry when this holder's work is done (finding + // 4). Created only once THIS reader holds the per-repo lock, so a + // parked or timed-out waiter never prunes the entry out from under a + // live holder; its Drop is ptr_eq-guarded, so it never clobbers a + // newer entry. Ownership moves into download_and_publish and from + // there into the spawned download task itself, so on EVERY settle path + // (success, download error, holder timeout, holder cancellation) the + // entry stays registered and the lock stays held until the in-flight + // download settles: waiters queue on the ONE download instead of each + // timeout or cancel wave starting a fresh full download of the same + // object. + let entry_guard = DownloadEntryGuard { + locks: Arc::clone(&self.download_locks), + key: map_key, + entry: Arc::clone(&entry), + }; + self.download_and_publish( + owner_did, + repo_name, + owner_slug, + local_path, + store, + local_existed_at_entry, + entry_guard, + held, + ) + .await + } + + /// The holder's half of the coordinated download: fetch and extract the + /// archive into a temp sibling with NO advisory lock held (the network + /// phase must never pin a lock-pool connection — a cold-read burst across + /// distinct repos must not drain the writer-sized pool), then take the + /// per-repo advisory lock only around the publish: re-check the archive + /// still exists under the lock (a purge deletes it under this same lock, + /// so a post-purge downloader discards rather than resurrecting), swap + /// the temp copy into place, release. Advisory contention or a lock-pool + /// error discards the temp copy and degrades — no blocking lock or pool + /// wait exists anywhere on the read path. + /// + /// `local_existed_at_entry` selects the stale-swap guard (finding 2): a cold + /// `acquire` passes `false`, an `acquire_fresh` refresh passes `true`. Under + /// the lock, before the swap, we detect a writer that published a fresher + /// copy during our unlocked download window and serve theirs instead of + /// clobbering it with our now-stale temp. + /// + /// `entry_guard` and `held` are this holder's per-repo coordination (the + /// download_locks map entry and the locked per-repo mutex). Both move into + /// the spawned download task, which returns them through its JoinHandle on + /// success; on any other settle path (download error, task panic, holder + /// timeout, holder cancellation) they drop when the task settles, so + /// waiters keep queueing on the SAME in-flight download until it settles + /// rather than each starting a duplicate one. + #[allow(clippy::too_many_arguments)] + async fn download_and_publish( + &self, + owner_did: &str, + repo_name: &str, + owner_slug: &str, + local_path: &Path, + store: &Arc, + local_existed_at_entry: bool, + entry_guard: DownloadEntryGuard, + held: tokio::sync::OwnedMutexGuard<()>, + ) -> Result { + let parent = local_path.parent().context("repo path has no parent")?; + std::fs::create_dir_all(parent).context("creating repo parent dir")?; + let file_name = local_path + .file_name() + .context("repo path has no file name")? + .to_string_lossy(); + + // Best-effort sweep of leftover temp siblings from a prior download whose + // cleanup never ran (a process crash or kill mid-download, where no Drop + // and no owning task survive to remove the temp). Serialized per + // repo by the download mutex, so no concurrent same-repo download owns a + // live temp here; scoped to THIS repo's prefix, so other repos are + // untouched. + let tmp_prefix = format!(".{file_name}.tmp-download."); + if let Ok(entries) = std::fs::read_dir(parent) { + for entry in entries.flatten() { + if entry.file_name().to_string_lossy().starts_with(&tmp_prefix) { + let _ = std::fs::remove_dir_all(entry.path()); + } + } + } + + // Stale-swap signal (finding 2): capture the local dir's mtime BEFORE the + // unlocked download. Under the lock, a changed mtime (acquire_fresh, + // refreshing over an existing copy) or the dir's mere appearance (cold + // acquire) means a writer published a fresher copy during our download + // window, so we serve theirs rather than swap our stale temp over it. + let entry_mtime = std::fs::metadata(local_path) + .and_then(|m| m.modified()) + .ok(); + + // Unique per-download temp target (same parent as local_path so the + // publish rename stays on one filesystem); mirrors the extract temp + // naming in tigris.rs. Wrapped in an RAII guard so a handler future + // cancelled mid-download (axum drops it on client disconnect) cannot + // leak the populated temp dir (the same hazard the guard Drop impls + // below exist for, finding 1). + let temp = TempDownloadDir::new(parent.join(format!( + ".{file_name}.tmp-download.{}", + uuid::Uuid::new_v4() + ))); + + // Ownership of the download AND the temp-dir guard moves into a spawned + // task, so timeout and handler cancellation share ONE cleanup path + // (finding F3 and its cancellation twin). `store.download` runs its tar + // extraction in `spawn_blocking`, and neither a tokio timeout nor a + // dropped handler future cancels a spawn_blocking task: pre-fix, + // dropping the download future (a timed-out await, or axum dropping the + // whole handler on client disconnect) detached the extraction, whose + // later rename resurrected `temp.path()` AFTER the guard's Drop had + // removed it, leaving an orphan only a later same-repo download sweeps. + // The spawned task is never dropped mid-flight, so the download future + // always runs to settle. On success it returns the still-armed guard + // through the JoinHandle: the caller consumes it and publishes below. + // If the caller timed out or was cancelled, the JoinHandle is gone, so + // the runtime drops the returned guard when the task settles and its + // Drop removes the temp dir; cleanup is tied to the extraction's + // completion, never racing it (this subsumes the previous detached + // janitor). A download error drops the guard inside the task with the + // same at-settle timing. + let store_dl = Arc::clone(store); + let (dl_owner, dl_repo, dl_target) = ( + owner_slug.to_string(), + repo_name.to_string(), + temp.path().to_path_buf(), + ); + // The per-repo coordination (`held`, `entry_guard`) moves into the task + // alongside the temp guard: whatever happens to THIS caller (timeout + // below, or a handler future cancelled by axum on client disconnect), + // the lock stays held and the map entry stays registered until the + // in-flight download settles, so waiters and new arrivals queue on the + // ONE download instead of starting duplicates. On success the tuple + // returns through the JoinHandle: the caller publishes under `held` as + // before. If the caller timed out or was cancelled, the runtime drops + // the returned tuple when the task settles, in field order: `temp` + // (dir removed), then `held` (lock released), then `entry_guard` (map + // entry pruned), the same unlock-before-prune order as a normal + // return. + let dl_task = tokio::spawn(async move { + // The guards travel as one `(held, entry_guard)` tuple so that + // wherever it drops (here on error, in the runtime at settle after + // a caller timeout or cancellation, or in the caller after a + // normal publish), its field order gives unlock-before-prune. + match store_dl.download(&dl_owner, &dl_repo, &dl_target).await { + Ok(()) => Ok((temp, (held, entry_guard))), + // The download settled, so cleanup is safe. Explicit drops pin + // the order (async-block captures have no guaranteed one): + // temp removed, lock released, map entry pruned last. + Err(e) => { + drop(temp); + drop(held); + drop(entry_guard); + Err(e) + } + } + }); + // Bound the wait, not the work: a stalled GET would park this reader + // indefinitely. On timeout the caller gets the same error as before, + // but the coordination does NOT unwind: it lives in the task (see + // above) and is released only at settle. Waiters therefore stay queued + // on the SAME in-flight download (each bounded by its own timeout in + // download_published) instead of each timeout wave starting a fresh + // full download of the same object; the task keeps running and cleans + // up its temp when the download settles. + // `_coordination` is `(held, entry_guard)`, kept alive to the end of + // this function exactly as the separate guards were before: the + // publish below still runs with the per-repo lock held, and the map + // entry is pruned only on exit (unlock first, prune second, per the + // tuple's field order). + let (temp, _coordination) = + match tokio::time::timeout(self.release_upload_timeout, dl_task).await { + Ok(Ok(Ok(returned))) => returned, + Ok(Ok(Err(e))) => return Err(e), + // The task panicked; its unwind dropped the guards, removing + // the dir, releasing the lock, and pruning the map entry. + Ok(Err(join_err)) => { + return Err(anyhow::Error::from(join_err).context("read download task failed")) + } + Err(_) => { + warn!(repo = %repo_name, timeout_secs = self.release_upload_timeout.as_secs(), + "read download timed out — discarding"); + // Dropping the JoinHandle detaches the task; the runtime + // frees temp dir, lock, and map entry at settle (see the + // spawn comment above). + return Err(anyhow::anyhow!("read download timed out")); + } + }; + + let guard = match self.try_lock_repo(owner_did, repo_name).await { + Ok(Some(g)) => g, + Ok(None) => { + debug!(repo = %repo_name, + "read download discarded — repo lock held by a live writer or purge"); + return Ok(DownloadOutcome::Skipped); + } + Err(e) => { + warn!(repo = %repo_name, err = %e, + "read download discarded — could not acquire repo lock"); + return Ok(DownloadOutcome::Skipped); + } + }; + // Re-check under the lock: the archive gone here means a purge won the + // key mid-download — discard, never publish (the read-side twin of the + // upload paths' under-lock dir re-check). Bounded so a stalled HEAD + // cannot pin the advisory lock + its lock-pool connection. Both a + // timeout and an exists() error collapse to "not present" and take the + // discard arm: fail closed rather than publish unconfirmed state. + let archive_present = matches!( + tokio::time::timeout( + self.release_upload_timeout, + store.exists(owner_slug, repo_name), + ) + .await, + Ok(Ok(true)) + ); + if !archive_present { + warn!(repo = %repo_name, + "read download discarded — archive gone under lock (purged?)"); + guard.release().await; + return Ok(DownloadOutcome::Skipped); + } + + // Stale-swap guard (finding 2): a writer that locked, published a fresher + // copy, and released during our unlocked download window must not have + // its work clobbered by our now-stale temp. The only concurrent mutator + // of local_path is a writer or a purge (readers of the same repo + // serialize on the download mutex), so the signals below unambiguously + // mean "a writer published here." + let writer_published = match (local_existed_at_entry, entry_mtime) { + // acquire_fresh over an existing copy: presence is always true, so it + // cannot signal a writer; a CHANGED mtime can (the writer replaced + // the dir via rename, minting a fresh mtime). + (true, Some(entry_m)) => std::fs::metadata(local_path) + .and_then(|m| m.modified()) + .map(|now| now != entry_m) + .unwrap_or(false), + // Cold acquire, or acquire_fresh whose dir was absent at entry: the + // dir being present now means a writer published it during the window. + _ => local_path.exists(), + }; + if writer_published { + warn!(repo = %repo_name, + "read download discarded: a writer published a fresher copy during the download window"); + guard.release().await; + // temp guard's Drop removes the stale temp; serve the fresher local copy. + return Ok(DownloadOutcome::Published); + } + + let swapped = (|| -> std::io::Result<()> { + if local_path.exists() { + std::fs::remove_dir_all(local_path)?; + } + std::fs::rename(temp.path(), local_path) + })(); + guard.release().await; + match swapped { + Ok(()) => { + temp.disarm(); // the rename consumed the temp; nothing to clean up + Ok(DownloadOutcome::Published) + } + // temp guard's Drop removes the temp if the rename left it behind. + Err(e) => Err(e).context("publishing downloaded repo into place"), + } + } +} diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index 229ee695..09904aea 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -63,6 +63,105 @@ pub fn list_refs(repo_path: &Path) -> Result> { Ok(refs) } +/// Read the object id a single ref currently points to. Returns None if the +/// ref does not exist. Used to snapshot a ref's pre-write state so a failed +/// durable upload can roll the local write back. +pub fn ref_oid(repo_path: &Path, ref_name: &str) -> Result> { + let output = Command::new("git") + .args(["rev-parse", "--verify", ref_name]) + .current_dir(repo_path) + .output() + .context("failed to run git rev-parse")?; + if !output.status.success() { + return Ok(None); + } + let oid = String::from_utf8_lossy(&output.stdout).trim().to_string(); + if oid.is_empty() { + return Ok(None); + } + Ok(Some(oid)) +} + +/// Force a single ref back to a captured object id — rolls back a local write +/// (e.g. a merge commit) whose durable upload failed, so a local-fast-path +/// read does not serve the un-uploaded state. The now-dangling objects are +/// harmless. +pub fn set_ref(repo_path: &Path, ref_name: &str, oid: &str) -> Result<()> { + let output = Command::new("git") + .args(["update-ref", ref_name, oid]) + .current_dir(repo_path) + .output() + .context("failed to run git update-ref")?; + if !output.status.success() { + let stderr = String::from_utf8_lossy(&output.stderr); + bail!("git update-ref failed: {stderr}"); + } + Ok(()) +} + +/// Restore a repo's refs to a previously captured snapshot: reset every +/// snapshot ref to its captured object id, and delete any ref that exists now +/// but was absent from the snapshot (created by the intervening write). Used +/// to roll back a receive-pack whose durable upload failed, so the next +/// local-fast-path read does not serve refs that never reached object storage. +/// Best-effort per ref: an individual failure is collected and reported, but +/// the remaining refs are still attempted. +pub fn restore_refs(repo_path: &Path, snapshot: &[(String, String)]) -> Result<()> { + use std::collections::HashSet; + + let mut errors: Vec = Vec::new(); + + // Reset every snapshot ref to its captured oid (recreates refs the write + // deleted, and rewinds refs the write advanced). + for (ref_name, oid) in snapshot { + if let Err(e) = set_ref(repo_path, ref_name, oid) { + errors.push(format!("{ref_name}: {e}")); + } + } + + // Delete refs that exist now but were not in the snapshot (created by the + // write being rolled back). A failure to LIST the current refs is collected + // like any per-ref failure, never returned early: the snapshot-reset half + // above already ran, and bailing here would discard its collected errors. + let snapshot_names: HashSet<&str> = snapshot.iter().map(|(r, _)| r.as_str()).collect(); + match list_refs(repo_path) { + Ok(current) => { + for (ref_name, _) in ¤t { + if !snapshot_names.contains(ref_name.as_str()) { + match Command::new("git") + .args(["update-ref", "-d", ref_name]) + .current_dir(repo_path) + .output() + { + Ok(output) if output.status.success() => {} + Ok(output) => { + let stderr = String::from_utf8_lossy(&output.stderr); + errors.push(format!("{ref_name} (delete): {stderr}")); + } + Err(e) => { + errors.push(format!( + "{ref_name} (delete): failed to run git update-ref -d: {e}" + )); + } + } + } + } + } + Err(e) => { + errors.push(format!("(extra-ref sweep) listing current refs: {e}")); + } + } + + if !errors.is_empty() { + bail!( + "failed to restore {} ref(s): {}", + errors.len(), + errors.join("; ") + ); + } + Ok(()) +} + /// Read the current HEAD commit hash of a repository. /// Returns None if the repo is empty (no commits yet). pub fn head_commit(repo_path: &Path) -> Result> { @@ -455,10 +554,176 @@ pub fn repo_disk_path(repos_dir: &Path, owner_did: &str, repo_name: &str) -> Pat #[cfg(test)] mod tests { - use super::branch_diff_names; + use super::{branch_diff_names, list_refs, restore_refs}; use std::path::Path; use std::process::Command; + fn run_git(dir: &Path, args: &[&str]) -> String { + let out = Command::new("git") + .args(args) + .current_dir(dir) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + + /// Bare repo with main at c2 and keeper at c2, whose "pre-push" state was + /// main and keeper both at c1. Returns (workdir, baredir, bare_path, c1, c2). + fn advanced_bare_repo() -> ( + tempfile::TempDir, + tempfile::TempDir, + std::path::PathBuf, + String, + String, + ) { + let work = tempfile::TempDir::new().unwrap(); + run_git(work.path(), &["init", "-q", "-b", "main", "."]); + run_git(work.path(), &["config", "user.email", "t@t"]); + run_git(work.path(), &["config", "user.name", "t"]); + std::fs::write(work.path().join("f.txt"), "one").unwrap(); + run_git(work.path(), &["add", "f.txt"]); + run_git(work.path(), &["commit", "-qm", "c1"]); + let c1 = run_git(work.path(), &["rev-parse", "HEAD"]); + std::fs::write(work.path().join("f.txt"), "two").unwrap(); + run_git(work.path(), &["add", "f.txt"]); + run_git(work.path(), &["commit", "-qm", "c2"]); + let c2 = run_git(work.path(), &["rev-parse", "HEAD"]); + + let dir = tempfile::TempDir::new().unwrap(); + let bare = dir.path().join("repo.git"); + let out = Command::new("git") + .args([ + "clone", + "--bare", + "-q", + &work.path().to_string_lossy(), + &bare.to_string_lossy(), + ]) + .output() + .unwrap(); + assert!(out.status.success(), "clone --bare failed"); + // "The push" advanced main to c2 (already there from the clone), moved + // keeper c1 -> c2, and created feature at c2. + run_git(&bare, &["update-ref", "refs/heads/keeper", &c2]); + run_git(&bare, &["update-ref", "refs/heads/feature", &c2]); + (work, dir, bare, c1, c2) + } + + fn sorted_refs(bare: &Path) -> Vec<(String, String)> { + let mut refs = list_refs(bare).unwrap(); + refs.sort(); + refs + } + + /// A single unrestorable snapshot entry (invalid oid, `set_ref` fails) must + /// not abort the restore: every OTHER snapshot ref is still reset and the + /// push-created extra ref is still deleted, and the fn returns Err carrying + /// the failure. RED with the reset loop reverted to `set_ref(..)?`: the + /// broken entry is first in the snapshot, so an early return leaves keeper + /// at c2 and feature alive. + #[test] + fn restore_refs_partial_failure_still_resets_rest_and_deletes_extras() { + let (_work, _dir, bare, c1, c2) = advanced_bare_repo(); + + let snapshot = vec![ + ( + "refs/heads/broken".to_string(), + "not-a-valid-oid".to_string(), + ), + ("refs/heads/main".to_string(), c1.clone()), + ("refs/heads/keeper".to_string(), c1.clone()), + ]; + let err = restore_refs(&bare, &snapshot).unwrap_err(); + let msg = format!("{err:#}"); + assert!( + msg.contains("refs/heads/broken"), + "the aggregated error names the failed ref: {msg}" + ); + + let refs = sorted_refs(&bare); + assert_eq!( + refs, + vec![ + ("refs/heads/keeper".to_string(), c1.clone()), + ("refs/heads/main".to_string(), c1.clone()), + ], + "keeper and main are reset to c1 and feature is deleted despite the \ + broken entry (c2 was {c2})" + ); + } + + /// Happy path unchanged by the hardening: a fully valid snapshot restores + /// exactly (refs rewound, created ref deleted) and returns Ok. + #[test] + fn restore_refs_happy_path_restores_snapshot_exactly() { + let (_work, _dir, bare, c1, _c2) = advanced_bare_repo(); + + let snapshot = vec![ + ("refs/heads/main".to_string(), c1.clone()), + ("refs/heads/keeper".to_string(), c1.clone()), + ]; + restore_refs(&bare, &snapshot).unwrap(); + + let refs = sorted_refs(&bare); + assert_eq!( + refs, + vec![ + ("refs/heads/keeper".to_string(), c1.clone()), + ("refs/heads/main".to_string(), c1), + ], + "restore must reproduce the snapshot exactly" + ); + } + + /// When the internal `list_refs` (extra-ref sweep) fails, the snapshot-reset + /// half must still have run and its collected failures must surface in the + /// aggregated error, not be discarded by an early `?` return. The repo is a + /// real bare repo declaring repositoryformatversion=999, so both `set_ref` + /// and `list_refs` fail deterministically. RED with the sweep reverted to + /// `let current = list_refs(repo_path)?;`: the raw for-each-ref error + /// propagates without the "failed to restore" aggregate or the ref name. + #[test] + fn restore_refs_aggregates_when_internal_list_refs_fails() { + let dir = tempfile::TempDir::new().unwrap(); + let bare = dir.path().join("repo.git"); + let out = Command::new("git") + .args(["init", "--bare", "-q", &bare.to_string_lossy()]) + .output() + .unwrap(); + assert!(out.status.success(), "git init --bare failed"); + std::fs::write( + bare.join("config"), + "[core]\n\trepositoryformatversion = 999\n\tbare = true\n", + ) + .unwrap(); + assert!( + list_refs(&bare).is_err(), + "precondition: list_refs must fail on the corrupted repo" + ); + + let snapshot = vec![("refs/heads/main".to_string(), "a".repeat(40))]; + let err = restore_refs(&bare, &snapshot).unwrap_err(); + let msg = format!("{err:#}"); + assert!( + msg.contains("failed to restore"), + "must return the aggregated error, not the raw list_refs error: {msg}" + ); + assert!( + msg.contains("refs/heads/main"), + "the set_ref failure from the reset half must survive the sweep \ + failure: {msg}" + ); + assert!( + msg.contains("listing current refs"), + "the sweep failure itself is also collected: {msg}" + ); + } + #[test] fn branch_diff_names_lists_changed_paths() { let td = tempfile::TempDir::new().unwrap(); diff --git a/crates/gitlawb-node/src/git/tigris.rs b/crates/gitlawb-node/src/git/tigris.rs index ad26ddc5..9b447696 100644 --- a/crates/gitlawb-node/src/git/tigris.rs +++ b/crates/gitlawb-node/src/git/tigris.rs @@ -11,6 +11,21 @@ use anyhow::{Context, Result}; use aws_sdk_s3::Client as S3Client; use tracing::{debug, info}; +/// Object storage for git bare-repo archives. The seam that lets `RepoStore` +/// (and tests) depend on storage behavior without a live bucket. `TigrisClient` +/// is the production implementation. +#[async_trait::async_trait] +pub trait ObjectStore: Send + Sync { + /// Whether an archive exists for this repo. + async fn exists(&self, owner_slug: &str, repo_name: &str) -> Result; + /// Upload a local bare repo directory as an archive. + async fn upload(&self, owner_slug: &str, repo_name: &str, local_path: &Path) -> Result<()>; + /// Download and extract a repo archive to local disk. + async fn download(&self, owner_slug: &str, repo_name: &str, local_path: &Path) -> Result<()>; + /// Delete a repo archive. + async fn delete(&self, owner_slug: &str, repo_name: &str) -> Result<()>; +} + /// Wrapper around the S3 client with the configured bucket. #[derive(Clone)] pub struct TigrisClient { @@ -35,9 +50,12 @@ impl TigrisClient { fn repo_key(owner_slug: &str, repo_name: &str) -> String { format!("repos/v1/{owner_slug}/{repo_name}.tar.zst") } +} +#[async_trait::async_trait] +impl ObjectStore for TigrisClient { /// Check if a repo archive exists in Tigris. - pub async fn exists(&self, owner_slug: &str, repo_name: &str) -> Result { + async fn exists(&self, owner_slug: &str, repo_name: &str) -> Result { let key = Self::repo_key(owner_slug, repo_name); match self .s3 @@ -59,7 +77,7 @@ impl TigrisClient { } /// Upload a local bare repo directory to Tigris as a tar.zst archive. - pub async fn upload(&self, owner_slug: &str, repo_name: &str, local_path: &Path) -> Result<()> { + async fn upload(&self, owner_slug: &str, repo_name: &str, local_path: &Path) -> Result<()> { let key = Self::repo_key(owner_slug, repo_name); debug!(key = %key, path = %local_path.display(), "uploading repo to tigris"); @@ -89,12 +107,7 @@ impl TigrisClient { } /// Download a repo archive from Tigris and extract to local disk. - pub async fn download( - &self, - owner_slug: &str, - repo_name: &str, - local_path: &Path, - ) -> Result<()> { + async fn download(&self, owner_slug: &str, repo_name: &str, local_path: &Path) -> Result<()> { let key = Self::repo_key(owner_slug, repo_name); debug!(key = %key, path = %local_path.display(), "downloading repo from tigris"); @@ -128,8 +141,7 @@ impl TigrisClient { } /// Delete a repo archive from Tigris. - #[allow(dead_code)] - pub async fn delete(&self, owner_slug: &str, repo_name: &str) -> Result<()> { + async fn delete(&self, owner_slug: &str, repo_name: &str) -> Result<()> { let key = Self::repo_key(owner_slug, repo_name); self.s3 .delete_object() diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index aa0483db..c27ad3d0 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -1,3 +1,4 @@ +mod admin; mod api; mod arweave; mod auth; @@ -70,6 +71,12 @@ async fn main() -> Result<()> { let mut config = Config::parse(); + // Admin subcommands run out-of-band and exit, never starting the node. The + // no-subcommand path below is the unchanged daemon startup. + if let Some(command) = config.command.clone() { + return run_admin_command(command, &config).await; + } + // Merge the embedded seed list of public network nodes into the runtime // bootstrap peers. Operators can opt out via GITLAWB_BOOTSTRAP_DISABLE_SEEDS. bootstrap::merge_seeds(&mut config); @@ -279,8 +286,22 @@ async fn main() -> Result<()> { None }; + let object_store = + tigris.map(|t| std::sync::Arc::new(t) as std::sync::Arc); + // Advisory-lock guards get their own pool, separate from the app pool + // (db.pool()) that serves normal handlers, so a concurrent-push burst pins + // lock connections here instead of starving request handlers. Sized to peak + // writers via GITLAWB_DB_LOCK_POOL_MAX_CONNECTIONS. + let lock_pool_acquire_timeout = std::time::Duration::from_secs(config.db_acquire_timeout_secs); + let lock_pool = Db::connect_lock_pool( + &config.database_url, + config.db_lock_pool_max_connections, + lock_pool_acquire_timeout, + ) + .await + .context("connecting advisory-lock pool")?; let repo_store = - git::repo_store::RepoStore::new(config.repos_dir.clone(), tigris, db.pool().clone()); + git::repo_store::RepoStore::new(config.repos_dir.clone(), object_store, lock_pool); // Per-DID limiter for the creation endpoints. Keyed on the authenticated // DID (attacker-varied), so bound its key set to cap memory. @@ -294,36 +315,24 @@ async fn main() -> Result<()> { // capped regardless of how many identities it mints. Sized well above any // legitimate per-IP creation rate; GITLAWB_CREATE_RATE_LIMIT overrides, 0 // disables. Bounded key set — the key is a client-influenced IP. - let create_limit = std::env::var("GITLAWB_CREATE_RATE_LIMIT") - .ok() - .and_then(|v| v.trim().parse::().ok()) - .unwrap_or(120); - let create_ip_rate_limiter = rate_limit::RateLimiter::new_bounded( - create_limit, - std::time::Duration::from_secs(3600), - 200_000, - ); - if create_limit == 0 { - tracing::warn!("GITLAWB_CREATE_RATE_LIMIT=0 — per-IP creation rate limiting disabled"); - } + let create_ip_rate_limiter = build_ip_limiter("GITLAWB_CREATE_RATE_LIMIT", 120, "creation"); + + // Per-IP brake for the authenticated non-creation write surface (issue/PR + // comments, labels, stars, merges, protect, replicas, visibility, tasks, + // bounties, profile, GraphQL mutations). Its own bucket, separate from the + // creation brake, so a write flood cannot drain the creation budget and vice + // versa (same rationale as the sync_trigger / peer_write split). Sized above + // any legitimate per-IP write rate — real agent automation makes many small + // writes per hour. GITLAWB_WRITE_RATE_LIMIT overrides; 0 disables. Bounded + // key set — the key is a client-influenced IP. + let write_rate_limiter = build_ip_limiter("GITLAWB_WRITE_RATE_LIMIT", 600, "write"); // Push-path flood brake: max git-receive-pack requests per client IP per // hour (counts both the info/refs advertisement and the push POST). Sized // for heavy agent automation while still stopping flood traffic (the June // 2026 attack pushed several times per second per IP). GITLAWB_PUSH_RATE_LIMIT // overrides; 0 disables. Bounded key set — the key is a client-influenced IP. - let push_limit = std::env::var("GITLAWB_PUSH_RATE_LIMIT") - .ok() - .and_then(|v| v.trim().parse::().ok()) - .unwrap_or(600); - let push_rate_limiter = rate_limit::RateLimiter::new_bounded( - push_limit, - std::time::Duration::from_secs(3600), - 200_000, - ); - if push_limit == 0 { - tracing::warn!("GITLAWB_PUSH_RATE_LIMIT=0 — per-IP push rate limiting disabled"); - } + let push_rate_limiter = build_ip_limiter("GITLAWB_PUSH_RATE_LIMIT", 600, "push"); // Which forwarded header the edge is trusted to set. Default None (trust // nothing, key on the socket peer). Fly nodes set GITLAWB_TRUSTED_PROXY=fly; @@ -331,7 +340,7 @@ async fn main() -> Result<()> { let push_limiter_trust = rate_limit::TrustedProxy::from_env_value( &std::env::var("GITLAWB_TRUSTED_PROXY").unwrap_or_default(), ); - tracing::info!(trust = ?push_limiter_trust, push_limit, "push rate limiter configured"); + tracing::info!(trust = ?push_limiter_trust, "push rate limiter trust configured"); // Peer-sync flood brakes, keyed on the resolved client IP (per-DID is useless // here — a did:key farm self-registers). Two buckets so an unsigned notify @@ -373,6 +382,7 @@ async fn main() -> Result<()> { repo_store, rate_limiter, create_ip_rate_limiter, + write_rate_limiter, push_rate_limiter, push_limiter_trust, sync_trigger_rate_limiter, @@ -408,22 +418,17 @@ async fn main() -> Result<()> { // Periodic cleanup of expired rate limit entries + consumed-proof ledger { - let rl = state.rate_limiter.clone(); - let create_ip_rl = state.create_ip_rate_limiter.clone(); - let push_rl = state.push_rate_limiter.clone(); - let sync_trigger_rl = state.sync_trigger_rate_limiter.clone(); - let peer_write_rl = state.peer_write_rate_limiter.clone(); + // Clone the whole state so the sweep uses AppState::cleanup_rate_limiters + // (the single source of truth that reaps EVERY limiter, including + // write_rate_limiter) rather than a hand-maintained list that can omit one. + let state = state.clone(); let db = state.db.clone(); let mut shutdown_rx = state.subscribe_shutdown(); tokio::spawn(async move { loop { tokio::select! { _ = tokio::time::sleep(std::time::Duration::from_secs(300)) => { - rl.cleanup().await; - create_ip_rl.cleanup().await; - push_rl.cleanup().await; - sync_trigger_rl.cleanup().await; - peer_write_rl.cleanup().await; + state.cleanup_rate_limiters().await; let now = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .map(|d| d.as_secs() as i64) @@ -564,6 +569,110 @@ async fn main() -> Result<()> { Ok(()) } +/// Pure resolver for a usize rate-limit env value. Returns `(resolved, invalid)`: +/// an absent or empty value resolves to `default` and is NOT invalid; a non-empty +/// value that fails to parse resolves to `default` and IS invalid (so the caller +/// warns rather than silently disabling the intended, usually stricter, brake). +fn resolve_rate_limit(raw: Option<&str>, default: usize) -> (usize, bool) { + match raw.map(str::trim) { + None | Some("") => (default, false), + Some(t) => match t.parse::() { + Ok(n) => (n, false), + Err(_) => (default, true), + }, + } +} + +/// Read a rate-limit env var, warning on a present-but-unparseable value. A typo +/// like `GITLAWB_WRITE_RATE_LIMIT=5O` must not silently revert to the default and +/// leave the operator believing a stricter cap is in force. +fn rate_limit_from_env(var: &str, default: usize) -> usize { + let raw = std::env::var(var).ok(); + let (value, invalid) = resolve_rate_limit(raw.as_deref(), default); + if invalid { + tracing::warn!( + var, + value = raw.as_deref().unwrap_or(""), + default, + "invalid rate-limit value — using default; the intended brake is NOT applied" + ); + } + value +} + +/// Build a bounded per-client-IP rate limiter from an env var: resolve the limit +/// (honoring the present-but-unparseable warning in `rate_limit_from_env`), build +/// the limiter with the shared 1-hour window and 200k-key cap, and warn when the +/// limit is 0 (disabled). Collapses the identical create/write/push setup. +fn build_ip_limiter(var: &str, default: usize, label: &str) -> rate_limit::RateLimiter { + let limit = rate_limit_from_env(var, default); + let limiter = + rate_limit::RateLimiter::new_bounded(limit, std::time::Duration::from_secs(3600), 200_000); + if limit == 0 { + tracing::warn!("{var}=0 — per-IP {label} rate limiting disabled"); + } else { + tracing::info!(%var, limit, "per-IP {label} rate limiter configured"); + } + limiter +} + +/// Dispatch an admin subcommand. Connects the database directly (no server, no +/// p2p, no metrics) and runs the requested tool to completion, then exits. +async fn run_admin_command(command: config::Command, config: &Config) -> Result<()> { + match command { + config::Command::PurgeSpam { execute } => { + let acquire_timeout = std::time::Duration::from_secs(config.db_acquire_timeout_secs); + let db = Db::connect( + &config.database_url, + config.db_max_connections, + acquire_timeout, + ) + .await + .context("connecting to database for purge-spam")?; + // Build the object store so purge is authoritative against remote + // state: the emptiness recheck consults the archive (not just + // possibly-stale local disk) and a successful purge deletes the + // archive too. When no bucket is configured this stays None and purge + // is local-only, exactly as before. + let object_store: Option> = if config + .tigris_bucket + .is_empty() + { + None + } else { + match git::tigris::TigrisClient::new(&config.tigris_bucket).await { + Ok(client) => Some(std::sync::Arc::new(client)), + Err(e) => { + // Fail closed: without the configured store we cannot + // do the authoritative recheck, and a local-only purge + // on a Tigris deployment is the stale-view delete this + // guards against. Refuse to run rather than fall back. + anyhow::bail!( + "purge-spam: GITLAWB_TIGRIS_BUCKET is set but the object-store client failed to initialize ({e}); refusing to run a local-only purge on a Tigris deployment" + ); + } + } + }; + // Give the purge-spam store a dedicated advisory-lock pool, separate + // from `db`'s app pool. The guard then holds a lock-pool connection + // while `delete_repo_by_id` runs on the app pool, so a purge at + // GITLAWB_DB_MAX_CONNECTIONS=1 does not self-deadlock (KTD3). + let lock_pool = Db::connect_lock_pool( + &config.database_url, + config.db_lock_pool_max_connections, + acquire_timeout, + ) + .await + .context("connecting advisory-lock pool for purge-spam")?; + let repo_store = + git::repo_store::RepoStore::new(config.repos_dir.clone(), object_store, lock_pool); + admin::run_purge_spam(&db, &repo_store, &config.repos_dir, execute) + .await + .map(|_| ()) + } + } +} + fn spawn_shutdown_signal(tx: watch::Sender) { tokio::spawn(async move { #[cfg(unix)] @@ -1024,6 +1133,40 @@ fn load_or_create_keypair(config: &Config) -> Result { } } +#[cfg(test)] +mod rate_limit_env_tests { + use super::resolve_rate_limit; + + #[test] + fn absent_uses_default_without_warning() { + assert_eq!(resolve_rate_limit(None, 600), (600, false)); + } + + #[test] + fn empty_or_whitespace_uses_default_without_warning() { + assert_eq!(resolve_rate_limit(Some(""), 600), (600, false)); + assert_eq!(resolve_rate_limit(Some(" "), 600), (600, false)); + } + + #[test] + fn valid_value_parses() { + assert_eq!(resolve_rate_limit(Some("50"), 600), (50, false)); + assert_eq!(resolve_rate_limit(Some(" 50 "), 600), (50, false)); + // 0 is a VALID value (disables the brake); the ==0 warning is handled + // separately at the call site. It must NOT be flagged invalid here. + assert_eq!(resolve_rate_limit(Some("0"), 600), (0, false)); + } + + #[test] + fn present_but_unparseable_defaults_and_flags_invalid() { + // The L8 case: a fat-fingered stricter cap ("5O", letter O) must default + // AND be flagged so the caller warns, not silently disable the brake. + assert_eq!(resolve_rate_limit(Some("5O"), 600), (600, true)); + assert_eq!(resolve_rate_limit(Some("abc"), 600), (600, true)); + assert_eq!(resolve_rate_limit(Some("-1"), 600), (600, true)); + } +} + #[cfg(test)] mod gossip_ssrf_tests { use super::ping_peer_health; diff --git a/crates/gitlawb-node/src/rate_limit.rs b/crates/gitlawb-node/src/rate_limit.rs index d40e2691..88967b26 100644 --- a/crates/gitlawb-node/src/rate_limit.rs +++ b/crates/gitlawb-node/src/rate_limit.rs @@ -70,10 +70,23 @@ impl RateLimiter { self.window.min(Duration::from_secs(1)) } + /// Returns `true` if the request is allowed. Thin wrapper over + /// [`check_retry`](Self::check_retry) for callers that only need the + /// allow/deny decision, not the rejection's Retry-After delay. Test-only now: + /// production 429 sites use `check_retry` so they can advertise Retry-After. + #[cfg(test)] pub(crate) async fn check(&self, key: &str) -> bool { + self.check_retry(key).await.is_none() + } + + /// Record a request against `key`. Returns `None` when it is allowed, or + /// `Some(retry_after)` when it is rejected — the delay a client should wait + /// before retrying, so a 429 can advertise a value consistent with the + /// actual window instead of a constant. + pub(crate) async fn check_retry(&self, key: &str) -> Option { // max_requests == 0 means the limiter is disabled, not "block all". if self.max_requests == 0 { - return true; + return None; } let now = Instant::now(); let mut state = self.state.lock().await; @@ -85,10 +98,25 @@ impl RateLimiter { .timestamps .retain(|t| now.duration_since(*t) < self.window); if window.timestamps.len() >= self.max_requests { - return false; + // Insertion order is preserved, so after the retain the first + // element is the oldest live request. It ages out (freeing a + // slot) one window after it landed, so that remaining time is + // the delay to advertise. Clamp to >= 1 so a 429 never tells a + // client to retry immediately. + let retry = window + .timestamps + .first() + .map(|oldest| { + self.window + .as_secs() + .saturating_sub(now.duration_since(*oldest).as_secs()) + }) + .unwrap_or(0) + .max(1); + return Some(Duration::from_secs(retry)); } window.timestamps.push(now); - return true; + return None; } // New key. Enforce the cap BEFORE inserting so a flood of distinct keys @@ -109,7 +137,11 @@ impl RateLimiter { } drop(last_sweep); if state.len() >= self.max_keys { - return false; + // The key is untracked (never inserted), so there is no oldest + // entry to derive a delay from. Advertise the full window as a + // conservative bound: capacity frees as other keys expire, at + // the latest one window from now. + return Some(self.window.max(Duration::from_secs(1))); } } state.insert( @@ -118,7 +150,7 @@ impl RateLimiter { timestamps: vec![now], }, ); - true + None } pub async fn cleanup(&self) { @@ -130,6 +162,41 @@ impl RateLimiter { !w.timestamps.is_empty() }); } + + /// Test-only: insert a fully-expired entry (no live timestamps) that the next + /// `cleanup()` must reclaim. Models a key whose window has aged out, without + /// depending on wall-clock sleeps or a short window. Used cross-module by the + /// AppState cleanup guard test (H2). + #[cfg(test)] + pub(crate) async fn insert_reclaimable_for_test(&self, key: &str) { + self.state + .lock() + .await + .insert(key.to_string(), Window { timestamps: vec![] }); + } + + /// Test-only: number of distinct keys currently tracked. + #[cfg(test)] + pub(crate) async fn tracked_keys(&self) -> usize { + self.state.lock().await.len() + } + + /// Test-only: seed a key's bucket with entries of the given ages (how long + /// ago each request landed), so a test can drive the Retry-After computation + /// off a controlled oldest-entry age without wall-clock sleeps. Ages that + /// would underflow the monotonic clock collapse to `now`. + #[cfg(test)] + pub(crate) async fn seed_aged_entry_for_test(&self, key: &str, ages: &[Duration]) { + let now = Instant::now(); + let timestamps = ages + .iter() + .map(|a| now.checked_sub(*a).unwrap_or(now)) + .collect(); + self.state + .lock() + .await + .insert(key.to_string(), Window { timestamps }); + } } pub async fn rate_limit_by_did(request: Request, next: Next) -> Response { @@ -141,13 +208,8 @@ pub async fn rate_limit_by_did(request: Request, next: Next) -> Response { .map(|a| a.0.clone()); if let (Some(limiter), Some(did)) = (limiter, did) { - if !limiter.check(&did).await { - return ( - StatusCode::TOO_MANY_REQUESTS, - [("retry-after", "60")], - "rate limit exceeded — try again later", - ) - .into_response(); + if let Some(retry_after) = limiter.check_retry(&did).await { + return too_many_requests(retry_after); } } @@ -250,13 +312,13 @@ impl axum::extract::FromRequestParts for PeerAddr { } } -/// The shared 429 response for the per-IP flood brakes. Route-agnostic: this -/// middleware now serves the push path AND the peer-sync routes, so the message -/// stays generic (the offending path is recorded in the warn log below). -pub fn too_many_requests() -> Response { +/// The shared 429 response for the rate-limit middlewares (per-DID and per-IP +/// alike). Route-agnostic: the message stays generic (the offending path, when +/// logged, is recorded at the call site). +pub fn too_many_requests(retry_after: Duration) -> Response { ( StatusCode::TOO_MANY_REQUESTS, - [("retry-after", "60")], + [("retry-after", retry_after.as_secs().to_string())], "rate limit exceeded — try again later", ) .into_response() @@ -274,9 +336,9 @@ pub async fn rate_limit_by_ip(request: Request, next: Next) -> Response { if let Some(limiter) = limiter { if let Some(key) = client_key(request.headers(), peer, limiter.trust) { - if !limiter.limiter.check(&key).await { + if let Some(retry_after) = limiter.limiter.check_retry(&key).await { tracing::warn!(key = %key, path = %request.uri().path(), "per-IP rate limit exceeded"); - return too_many_requests(); + return too_many_requests(retry_after); } } } @@ -605,4 +667,196 @@ mod tests { "an exhausted IP must be braked with 429 before auth runs, not leak to 401" ); } + + // ── Adversarial TrustedProxy verification through the middleware (U5) ── + // These complement the client_key unit tests above by driving hostile + // headers through rate_limit_by_ip on a real router: the property under test + // is that the *bucket key* cannot be rotated by a spoofer. + + async fn post_with( + router: &axum::Router, + peer: SocketAddr, + hdrs: &[(&str, &str)], + ) -> StatusCode { + use tower::ServiceExt; + let mut b = axum::http::Request::builder() + .method(axum::http::Method::POST) + .uri("/api/v1/repos"); + for (k, v) in hdrs { + b = b.header(*k, *v); + } + let mut req = b.body(axum::body::Body::empty()).unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + router.clone().oneshot(req).await.unwrap().status() + } + + #[tokio::test] + async fn xff_spoofed_leftmost_hop_cannot_rotate_bucket() { + // XForwardedFor mode, budget 1. A spoofer varies the client-controlled + // leftmost hop every request but the trusted proxy always appends the + // same rightmost hop. All requests must key to the rightmost hop, so the + // second is braked despite a fresh leftmost value. + let router = ip_limited_over_auth_router(IpRateLimiter { + limiter: RateLimiter::new(1, Duration::from_secs(60)), + trust: TrustedProxy::XForwardedFor, + }); + let p1: SocketAddr = "10.0.0.1:1".parse().unwrap(); + let p2: SocketAddr = "10.0.0.2:1".parse().unwrap(); + // First request passes the brake, reaches (failing) auth. + assert_eq!( + post_with(&router, p1, &[("x-forwarded-for", "9.9.9.1, 203.0.113.5")]).await, + StatusCode::UNAUTHORIZED + ); + // Different leftmost + different socket peer, SAME rightmost trusted hop: + // braked, because the key is the rightmost hop, not the spoofed value. + assert_eq!( + post_with(&router, p2, &[("x-forwarded-for", "9.9.9.2, 203.0.113.5")]).await, + StatusCode::TOO_MANY_REQUESTS, + "a spoofer varying the leftmost XFF hop must not rotate its bucket key" + ); + } + + #[tokio::test] + async fn xff_distinct_real_clients_get_distinct_buckets() { + // Distinct trusted (rightmost) hops are distinct clients: neither throttles + // the other, so a busy legitimate client cannot starve a different one. + let router = ip_limited_over_auth_router(IpRateLimiter { + limiter: RateLimiter::new(1, Duration::from_secs(60)), + trust: TrustedProxy::XForwardedFor, + }); + let peer: SocketAddr = "10.0.0.9:1".parse().unwrap(); + assert_eq!( + post_with( + &router, + peer, + &[("x-forwarded-for", "1.1.1.1, 203.0.113.5")] + ) + .await, + StatusCode::UNAUTHORIZED + ); + assert_eq!( + post_with( + &router, + peer, + &[("x-forwarded-for", "1.1.1.1, 203.0.113.6")] + ) + .await, + StatusCode::UNAUTHORIZED, + "a different trusted client hop must get its own bucket" + ); + } + + #[tokio::test] + async fn absent_trusted_header_collapses_to_socket_peer() { + // The documented fallback: in a trusted-proxy mode, a request with no + // forwarded header keys on the socket peer. Behind a real edge every + // request shares the proxy's socket, so they collapse onto one bucket — + // asserted here by name so any change to this behavior is deliberate. + let router = ip_limited_over_auth_router(IpRateLimiter { + limiter: RateLimiter::new(1, Duration::from_secs(60)), + trust: TrustedProxy::XForwardedFor, + }); + let proxy: SocketAddr = "172.16.0.1:1".parse().unwrap(); + assert_eq!( + post_with(&router, proxy, &[]).await, + StatusCode::UNAUTHORIZED + ); + assert_eq!( + post_with(&router, proxy, &[]).await, + StatusCode::TOO_MANY_REQUESTS, + "with no trusted header, requests fall back to the socket peer key (collapse)" + ); + } + + // ── Retry-After derived from the window, not a constant 60 (U5) ─────── + // A 429 must advertise the delay until the oldest live request ages out of + // the limiter's window, so the value tracks the real window instead of the + // hardcoded 60 that contradicted every limiter's 3600s window. + + async fn resp_from(router: &axum::Router, peer: SocketAddr) -> Response { + use tower::ServiceExt; + let mut req = axum::http::Request::builder() + .method(axum::http::Method::POST) + .uri("/o/r/git-receive-pack") + .body(axum::body::Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + router.clone().oneshot(req).await.unwrap() + } + + fn retry_after_secs(resp: &Response) -> u64 { + resp.headers() + .get("retry-after") + .expect("429 must carry a retry-after header") + .to_str() + .unwrap() + .parse() + .expect("retry-after must be an integer number of seconds") + } + + #[tokio::test] + async fn retry_after_reflects_full_window_for_freshly_filled_bucket() { + // A just-filled bucket's oldest entry is ~now, so the advertised delay + // must be close to the whole window (100s), NOT the old constant 60. + let router = ip_limited_router(IpRateLimiter { + limiter: RateLimiter::new(1, Duration::from_secs(100)), + trust: TrustedProxy::None, + }); + let peer: SocketAddr = "203.0.113.50:6000".parse().unwrap(); + assert_eq!(resp_from(&router, peer).await.status(), StatusCode::OK); + let resp = resp_from(&router, peer).await; + assert_eq!(resp.status(), StatusCode::TOO_MANY_REQUESTS); + let retry = retry_after_secs(&resp); + assert!( + (95..=100).contains(&retry), + "freshly-filled bucket must advertise ~window (95..=100s), got {retry} (constant-60 bug if 60)" + ); + } + + #[tokio::test] + async fn retry_after_shrinks_as_oldest_entry_ages() { + // Seed the sole slot with an entry already 95s into a 100s window: the + // next request is rejected and must advertise only the ~5s remaining, + // not 60. + let limiter = RateLimiter::new(1, Duration::from_secs(100)); + let peer: SocketAddr = "203.0.113.51:6000".parse().unwrap(); + limiter + .seed_aged_entry_for_test(&peer.ip().to_string(), &[Duration::from_secs(95)]) + .await; + let router = ip_limited_router(IpRateLimiter { + limiter, + trust: TrustedProxy::None, + }); + let resp = resp_from(&router, peer).await; + assert_eq!(resp.status(), StatusCode::TOO_MANY_REQUESTS); + let retry = retry_after_secs(&resp); + assert!( + (1..=10).contains(&retry), + "an aged bucket must advertise the small remaining delay (1..=10s), got {retry}" + ); + } + + #[tokio::test] + async fn retry_after_clamps_to_at_least_one_second() { + // Oldest entry is 99s into a 100s window: the raw remainder rounds toward + // 1s and must never be advertised as 0 (a 0/blank retry lets a rejected + // client hammer immediately). + let limiter = RateLimiter::new(1, Duration::from_secs(100)); + let peer: SocketAddr = "203.0.113.52:6000".parse().unwrap(); + limiter + .seed_aged_entry_for_test(&peer.ip().to_string(), &[Duration::from_secs(99)]) + .await; + let router = ip_limited_router(IpRateLimiter { + limiter, + trust: TrustedProxy::None, + }); + let resp = resp_from(&router, peer).await; + assert_eq!(resp.status(), StatusCode::TOO_MANY_REQUESTS); + let retry = retry_after_secs(&resp); + assert!(retry >= 1, "retry-after must clamp to >= 1, got {retry}"); + assert!( + retry <= 2, + "with 99s of a 100s window elapsed, the remaining delay is ~1s, got {retry}" + ); + } } diff --git a/crates/gitlawb-node/src/server.rs b/crates/gitlawb-node/src/server.rs index f4c0d3e3..1456b700 100644 --- a/crates/gitlawb-node/src/server.rs +++ b/crates/gitlawb-node/src/server.rs @@ -55,18 +55,50 @@ fn add_auth_layers(router: Router, state: AppState) -> Router Self; +} + +impl WriteBraked for Router { + fn write_braked(self, limiter: &rate_limit::IpRateLimiter) -> Self { + self.layer(middleware::from_fn(rate_limit::rate_limit_by_ip)) + .layer(axum::Extension(limiter.clone())) + } +} + pub fn build_router(state: AppState) -> Router { // ── GraphQL routes ───────────────────────────────────────────────────── let schema = state.graphql_schema.as_ref().clone(); + + // Per-IP write brake, shared across every authenticated non-creation write + // group (see `AppState::write_rate_limiter`). Built once here so each group + // clones the same bucket; attached outermost on each group so a flood is + // rejected before auth runs. Separate bucket from the creation brake. + let write_ip_limiter = rate_limit::IpRateLimiter { + limiter: state.write_rate_limiter.clone(), + trust: state.push_limiter_trust, + }; + let graphql_routes = Router::new() .route("/graphql", get(graphql_playground).post(graphql_handler)) // Attach the verified DID to /graphql when a signature is present. The // layer covers only routes added before it, so /graphql/ws (added after, - // read-only subscriptions) stays open. + // read-only subscriptions) stays open. The per-IP write brake covers the + // same /graphql (KTD-5: an HTTP-layer brake cannot tell a query POST from + // a mutation POST, so it counts every /graphql request — acceptable, both + // cost work); /graphql/ws subscriptions stay unbraked. .layer(middleware::from_fn(auth::optional_signature)) + .write_braked(&write_ip_limiter) .route_service("/graphql/ws", GraphQLSubscription::new(schema)); - // ── Task routes (write — require HTTP Signature) ─────────────────────── + // ── Task routes (write — require HTTP Signature + per-IP write brake) ── let task_write_routes = add_auth_layers( Router::new() .route("/api/v1/tasks", post(tasks::create_task)) @@ -74,7 +106,8 @@ pub fn build_router(state: AppState) -> Router { .route("/api/v1/tasks/{id}/complete", post(tasks::complete_task)) .route("/api/v1/tasks/{id}/fail", post(tasks::fail_task)), state.clone(), - ); + ) + .write_braked(&write_ip_limiter); // ── Task routes (read — open) ────────────────────────────────────────── let task_read_routes = Router::new() @@ -111,7 +144,13 @@ pub fn build_router(state: AppState) -> Router { .layer(middleware::from_fn(rate_limit::rate_limit_by_ip)) .layer(axum::Extension(create_ip_limiter)); - // ── Write routes — require HTTP Signature (no rate limit) ───────────── + // ── Write routes — require HTTP Signature + per-IP write brake ──────── + // Same per-IP flood defense as the creation routes, on its own `write` + // bucket (see `AppState::write_rate_limiter`). Per-DID is deliberately not + // paired (a DID farm never trips it; busy legitimate agents would false- + // positive). The bundled visibility GET is a read that also draws from the + // write bucket — harmless, since the bucket is sized far above any legit + // per-IP read rate. let write_routes = add_auth_layers( Router::new() .route( @@ -181,7 +220,8 @@ pub fn build_router(state: AppState) -> Router { axum::routing::delete(agents::deregister_agent), ), state.clone(), - ); + ) + .write_braked(&write_ip_limiter); // Body limit is raised to GITLAWB_MAX_PACK_BYTES (default 2 GB) for git // routes only — all other API routes keep axum's default 2 MB cap. @@ -247,7 +287,8 @@ pub fn build_router(state: AppState) -> Router { post(bounties::dispute_bounty), ), state.clone(), - ); + ) + .write_braked(&write_ip_limiter); // ── Bounty routes (read — open) ────────────────────────────────────── let bounty_read_routes = Router::new() @@ -268,9 +309,10 @@ pub fn build_router(state: AppState) -> Router { let profile_write_routes = add_auth_layers( Router::new().route("/api/v1/profile", axum::routing::put(profiles::set_profile)), state.clone(), - ); + ) + .write_braked(&write_ip_limiter); - // ── Issue routes (write — require HTTP Signature, no rate limit) ───── + // ── Issue routes (write — require HTTP Signature + per-IP write brake) ─ let issue_write_routes = add_auth_layers( Router::new() .route( @@ -282,7 +324,8 @@ pub fn build_router(state: AppState) -> Router { post(issues::create_issue_comment), ), state.clone(), - ); + ) + .write_braked(&write_ip_limiter); // ── Peer discovery routes ───────────────────────────────────────────── // Peer writes accept signatures when present and can require them after a diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index d3e53f3a..c18b7147 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -62,6 +62,16 @@ pub struct AppState { /// resolved client IP is what actually stops a single-source flood (same /// rationale as `push_rate_limiter`). Keyed by `push_limiter_trust`. pub create_ip_rate_limiter: RateLimiter, + /// Per-client-IP rate limiter for the authenticated write surface that is + /// not repo/agent creation: issue/PR comments, labels, stars, merges, + /// protect/unprotect, replicas, visibility, tasks, bounties, profile, and + /// the GraphQL `MutationRoot`. Separate bucket from `create_ip_rate_limiter` + /// (its own budget, per the `sync_trigger`/`peer_write` precedent) so a + /// write flood cannot drain the creation budget and vice versa. Per-DID is + /// deliberately NOT paired here: a DID farm never trips it, and busy + /// legitimate agents would false-positive. `GITLAWB_WRITE_RATE_LIMIT` + /// overrides the default, 0 disables. Keyed by `push_limiter_trust`. + pub write_rate_limiter: RateLimiter, /// Per-client-IP rate limiter for git-receive-pack. Per-DID limits cannot /// brake a push flood from a DID farm (one throwaway DID per repo), so the /// push path throttles on the resolved client IP instead. @@ -118,4 +128,48 @@ impl AppState { pub fn is_shutting_down(&self) -> bool { *self.shutdown_tx.borrow() } + + /// Reclaim expired entries from EVERY rate limiter. Single source of truth for + /// the periodic cleanup loop (main.rs), co-located with the limiter fields so a + /// newly-added limiter is cleaned here rather than silently omitted from a + /// hand-maintained list in main() (H2: `write_rate_limiter` was the omitted + /// one). Every limiter field above MUST be swept here; the guard test + /// `cleanup_rate_limiters_reaps_the_write_limiter` fails if it is not. + pub(crate) async fn cleanup_rate_limiters(&self) { + self.rate_limiter.cleanup().await; + self.create_ip_rate_limiter.cleanup().await; + self.write_rate_limiter.cleanup().await; + self.push_rate_limiter.cleanup().await; + self.sync_trigger_rate_limiter.cleanup().await; + self.peer_write_rate_limiter.cleanup().await; + } +} + +#[cfg(test)] +mod tests { + /// H2 guard: the write-surface limiter must be reaped by the shared cleanup + /// routine. Seeds a reclaimable entry into `write_rate_limiter` and asserts + /// `cleanup_rate_limiters` removes it. Goes RED if `write_rate_limiter` is + /// dropped from `cleanup_rate_limiters` — the exact H2 omission, now guarded. + #[tokio::test] + async fn cleanup_rate_limiters_reaps_the_write_limiter() { + let state = crate::test_support::test_state_lazy(); + state + .write_rate_limiter + .insert_reclaimable_for_test("some-ip") + .await; + assert_eq!( + state.write_rate_limiter.tracked_keys().await, + 1, + "precondition: the reclaimable entry is tracked" + ); + + state.cleanup_rate_limiters().await; + + assert_eq!( + state.write_rate_limiter.tracked_keys().await, + 0, + "write_rate_limiter must be reaped by cleanup_rate_limiters (H2)" + ); + } } diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index be6fb7b8..135f3145 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -77,6 +77,7 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { repo_store: crate::git::repo_store::RepoStore::for_testing(PathBuf::from("/tmp"), pool), rate_limiter: RateLimiter::new(100, Duration::from_secs(60)), create_ip_rate_limiter: RateLimiter::new(1000, Duration::from_secs(3600)), + write_rate_limiter: RateLimiter::new(1000, Duration::from_secs(3600)), push_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), push_limiter_trust: crate::rate_limit::TrustedProxy::None, sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)),