diff --git a/.env.example b/.env.example index bbd9a342..9913b41b 100644 --- a/.env.example +++ b/.env.example @@ -24,8 +24,11 @@ DATABASE_URL=postgresql://gitlawb:changeme@localhost:5432/gitlawb # ── Database pool & startup resilience ──────────────────────────────────── # Maximum connections in the PostgreSQL pool. A cap, not a floor — # connections open lazily. Size against the DB server's max_connections, -# remembering admin tooling opens its own pool. -GITLAWB_DB_MAX_CONNECTIONS=20 +# remembering admin tooling opens its own pool. Each concurrent write pins one +# connection for its whole duration (the connection-affine advisory lock), so the +# node REJECTS at boot any value below GITLAWB_MAX_CONCURRENT_GIT_PUSHES + 8 +# headroom — keep this comfortably above that (default 48 for pushes 32). +GITLAWB_DB_MAX_CONNECTIONS=48 # Seconds a request waits for a pool connection before failing with 503. GITLAWB_DB_ACQUIRE_TIMEOUT_SECS=5 # Upper bound on each startup connect+migrate attempt, in seconds. Keep it @@ -109,15 +112,96 @@ GITLAWB_MAX_PACK_BYTES=2147483648 # Max seconds a served git upload-pack / receive-pack (clone / push) may run # before it is aborted with a 504. Bounds a hung git that would otherwise pin a -# worker and, on push, the repo write lock. Does NOT cover the info/refs -# advertisement or the withheld-blob path, which remain unbounded. Default 600. +# worker and, on push, the repo write lock. Also bounds both info/refs +# advertisements, the withheld-blob pack build, and the push-side candidate +# discovery (rev-list / cat-file), all reaped via process-group teardown (#174). +# Default 600. GITLAWB_GIT_SERVICE_TIMEOUT_SECS=600 +# Max seconds the storage-ACQUISITION phase of a served git op may run before the +# request is shed with a 503, separate from the git-run timeout above. A +# concurrency permit is taken before this phase and GITLAWB_GIT_SERVICE_TIMEOUT_SECS +# only starts once git spawns, so without this a stalled backend (a hung Tigris +# HEAD/GET, or a hung pg advisory-lock iteration on push) pins the permit and drains +# the pool until every later request 503s. On expiry the permit is released +# (fail-closed). Kept separate because acquisition and git execution are distinct +# cost centers. Must be positive; set very large to effectively disable. Default 30. +GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS=30 + +# Max concurrent git READ ops (upload-pack + the upload-pack info/refs +# advertisement) served at once, a global pool separate from the push pool below. +# The anon receive-pack info/refs advertisement has its OWN pool (see below), not +# this one. Over-cap sheds a clean 503 + Retry-After. Anonymous reads draw from +# here, so pair it with GITLAWB_MAX_CONCURRENT_READS_PER_CALLER (below) so one +# caller cannot monopolize the pool. Default 128. +GITLAWB_MAX_CONCURRENT_GIT_OPS=128 + +# Max concurrent git-receive-pack (push) POST operations, in a pool separate from +# the read pool (GITLAWB_MAX_CONCURRENT_GIT_OPS) so anonymous reads cannot shed an +# authenticated push at admission. The anon receive-pack info/refs advertisement +# runs in a SEPARATE pool of the same size (disjoint from this one), so an +# advertisement flood cannot shed a push either. Over-cap sheds a 503 + +# Retry-After. Default 32. +GITLAWB_MAX_CONCURRENT_GIT_PUSHES=32 + +# Max concurrent post-push pin loops (IPFS + Pinata pin_new_objects) across all +# repos. Each loop holds a full per-push object-id list while pinning, so this +# bounds that MB-scale memory even though the per-repo encrypt-task set already +# caps the task COUNT. A loop DEFERS (waits) when the pool is full, never drops a +# pin. Default 8. +GITLAWB_MAX_CONCURRENT_PIN_TASKS=8 + +# Max concurrent read ops (upload-pack + the upload-pack info/refs advertisement) +# a single caller may hold, so one caller cannot monopolize the read pool. Keyed +# on the resolved SOURCE IP, never the DID: a signature does not move a caller off +# this cap. The source-IP key is only as granular as GITLAWB_TRUSTED_PROXY below: +# left unset, a node behind an edge/NAT keys all callers on the edge IP and this +# collapses to one global cap. Set GITLAWB_TRUSTED_PROXY for per-client keying; a +# high-fanout caller (CI behind one NAT) then needs the operator to raise this. +# Default 16. +GITLAWB_MAX_CONCURRENT_READS_PER_CALLER=16 + # ── Push rate limiting (git-receive-pack flood brake) ───────────────────── # Max receive-pack requests (info/refs advertisement + push POST) per client # IP per hour. 0 disables. Default 600. GITLAWB_PUSH_RATE_LIMIT=600 +# ── /ipfs/{cid} visibility-walk admission (#174) ────────────────────────── +# GET /ipfs/{cid} runs a per-repo full-history git walk in a blocking thread to +# decide whether the caller may read a path-scoped blob. It is publicly reachable, +# so it is bounded to keep a permissionless caller from fanning out unbounded +# concurrent walks and exhausting blocking-pool threads + PIDs. +# Max concurrent /ipfs walks across all callers (a pool of its own, disjoint from +# the served-git pools). Over-cap sheds a 503. Default 32. +GITLAWB_MAX_CONCURRENT_IPFS_WALKS=32 +# Max concurrent /ipfs walks a single SOURCE IP may hold (keyed like the git +# per-caller caps via GITLAWB_TRUSTED_PROXY; reject-before-insert bounded map). +# Default 4. +GITLAWB_IPFS_WALK_PER_SOURCE=4 +# Max EXPENSIVE path-scope visibility walks per single /ipfs request (only a +# blob in a path-scoped repo costs a full-history walk). Over-cap repos are +# skipped without a verdict and the scan continues; if the object is then found +# nowhere the request sheds a retryable 503 instead of a false 404. Default 64. +GITLAWB_IPFS_MAX_REPOS_WALKED=64 +# Ceiling on repos one /ipfs request may VISIT past the visibility gate. Each +# visit costs a repo acquire — on a Tigris cache miss a full archive download, +# so this is also the worst-case object-store fetch count per request — plus a +# cat-file probe. On exhaustion the scan stops and sheds a retryable 503. +# Default 1024. +GITLAWB_IPFS_MAX_REPO_VISITS=1024 +# Absolute wall-clock budget for one admitted /ipfs request's acquire+walk +# lifetime (all stages of the whole scan). Per-stage clamps bound the acquire +# and walk stages to the remaining budget, and no stage starts once it is +# exhausted (the scan then sheds a retryable 503). The object-type probe and +# content-read cat-file subprocesses are budget-checked before starting but +# have NO duration bound of their own: a hung git probe (corrupt pack, stuck +# filesystem) holds the request's walk slot for the full duration of the hang. +# Default 600. +GITLAWB_IPFS_REQUEST_BUDGET_SECS=600 +# Max /ipfs/{cid} requests per client IP per hour (route flood brake, distinct +# from the concurrency caps above). 0 disables. Default 600. +GITLAWB_IPFS_RATE_LIMIT=600 + # ── Creation rate limiting (repo/agent/issue/PR flood brake) ────────────── # Max creation requests (POST /api/v1/repos, /api/register, fork, issues, # pulls) per client IP per hour, in addition to the per-DID limit. The per-DID diff --git a/README.md b/README.md index 57ce2885..cc32bcf7 100644 --- a/README.md +++ b/README.md @@ -343,7 +343,14 @@ Important node settings: | `GITLAWB_REQUIRE_SIGNED_PEER_WRITES` | Require signed peer announce/sync writes. | | `GITLAWB_AUTO_SYNC` | Enable automatic sync from known peers. | | `GITLAWB_MAX_PACK_BYTES` | Max git pack body size for smart-HTTP routes. | -| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack/receive-pack may run before it is aborted (504). Default 600. Does not bound `info/refs` or the withheld-blob path. | +| `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` | Max seconds a served git upload-pack, receive-pack, or `info/refs` advertisement may run before it is aborted (504). Default 600. Also bounds the withheld-blob classification walk (on both the upload-pack serve and receive-pack replication paths) and the push-side pin-candidate discovery (`rev-list` / `cat-file`), each reaped via process-group teardown at the deadline. | +| `GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS` | Max seconds the storage-acquisition phase (Tigris HEAD/GET, push advisory-lock) of a served git op may run before the request is shed with a 503, separate from the git-run timeout. The concurrency permit is released on expiry so a stalled backend cannot pin the pool. Default 30. | +| `GITLAWB_MAX_CONCURRENT_IPFS_WALKS` | Max concurrent `GET /ipfs/{cid}` visibility walks across all callers (own pool, disjoint from the served-git pools); over-cap sheds 503. Default 32. | +| `GITLAWB_IPFS_WALK_PER_SOURCE` | Max concurrent `/ipfs` walks a single source IP may hold. Default 4. | +| `GITLAWB_IPFS_MAX_REPOS_WALKED` | Max expensive path-scope visibility walks per `/ipfs/{cid}` request; over-cap repos are skipped and the scan continues, shedding a retryable 503 (not a false 404) if the object is then found nowhere. Default 64. | +| `GITLAWB_IPFS_MAX_REPO_VISITS` | Ceiling on repos one `/ipfs/{cid}` request may visit (acquire + probe) past the visibility gate — also the worst-case per-request Tigris fetch count. On exhaustion the scan stops with a retryable 503. Default 1024. | +| `GITLAWB_IPFS_REQUEST_BUDGET_SECS` | Absolute wall-clock budget for one admitted `/ipfs/{cid}` request's acquire+walk lifetime. Per-stage clamps bound the acquire and walk stages to the remaining budget, and no stage starts once it is exhausted; the scan then stops with a retryable 503. The object-type probe and content-read `cat-file` subprocesses are budget-checked before starting but have no duration bound of their own, so a hung git probe (corrupt pack, stuck filesystem) holds the request's walk slot for the full duration of the hang. Default 600. | +| `GITLAWB_IPFS_RATE_LIMIT` | Max `/ipfs/{cid}` requests per client IP per hour (route flood brake). 0 disables. Default 600. | | `GITLAWB_TIGRIS_BUCKET` | Optional S3/Tigris shared repo storage bucket. | | `GITLAWB_PINATA_JWT` | Optional Pinata/IPFS warm-storage pinning. | | `GITLAWB_IRYS_URL` | Optional Irys/Arweave permanent anchoring. | diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index f3de7570..5e7ae05a 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -27,7 +27,7 @@ use std::str::FromStr; use crate::auth::AuthenticatedDid; use crate::error::{AppError, Result}; use crate::git::store; -use crate::git::visibility_pack::{allowed_blob_set_for_caller, has_path_scoped_rule}; +use crate::git::visibility_pack::{allowed_blob_set_for_caller_bounded, has_path_scoped_rule}; use crate::state::AppState; use crate::visibility::{visibility_check, Decision}; @@ -51,6 +51,30 @@ use crate::visibility::{visibility_check, Decision}; /// fail-closed 404'd under path-scoped rules (#126). Denial and genuine /// not-found both fall through to an opaque 404. /// +/// Scan completeness (F2): the 404 above is returned ONLY when every candidate +/// repo reached a VERDICT — visibility deny, probe-says-absent, walk-gate deny, +/// or served. A candidate skipped WITHOUT a verdict (acquire failure/timeout, +/// probe error, walk failure/panic, content-read error, or truncation by +/// `ipfs_max_repos_walked` / `ipfs_max_repo_visits` / +/// `ipfs_request_budget_secs`) taints the scan, and a +/// tainted scan that found nothing sheds a retryable 503 + Retry-After naming +/// the truncation sources — existing content is never misreported absent +/// because of unrelated repos or transient faults. +/// +/// Deterministic fault (F5/U4): a candidate repo that is persistently broken (a +/// corrupt repo, a bad `.git/config`) also yields no absence verdict, but a retry +/// cannot fix it, so a scan that found nothing sheds a TERMINAL, non-retryable 500 +/// (opaque body) rather than the retryable 503 — checked first so a deterministic +/// fault is never downgraded, and gated on nothing-served so a healthy repo that +/// carries the object still serves. +/// +/// Request budget (F3): one absolute clock (`ipfs_request_budget_secs`) spans +/// the whole admitted request. No stage (acquire, probe, walk, content read) +/// starts once it is exhausted, and the acquire wait and walk deadline are +/// clamped to the remainder. The probe and content-read subprocesses have no +/// duration bound of their own past their pre-start budget check, so a hung +/// git probe holds the request's walk slot for the full duration of the hang. +/// /// Scope: this closes the direct unauthenticated scan, including the dangling /// case. A stale-public mirror row still serves withheld content (tracked /// separately, #124). @@ -58,6 +82,11 @@ pub async fn get_by_cid( Path(cid_str): Path, State(state): State, auth: Option>, + // Per-source keying for the walk concurrency sub-cap. Infallible extractors + // (mirror the git handlers in `repos.rs`): `PeerAddr` yields `None` under + // `oneshot` with no `ConnectInfo`, and the header map falls back per `client_key`. + crate::rate_limit::PeerAddr(peer): crate::rate_limit::PeerAddr, + req_headers: HeaderMap, ) -> Result { // 1. Decode the CID and extract the SHA-256 digest let cid = CidGeneric::<64>::from_str(&cid_str) @@ -76,22 +105,108 @@ pub async fn get_by_cid( let caller = auth.as_ref().map(|e| e.0 .0.as_str()); let caller_owned = caller.map(|c| c.to_string()); - // 2. Search all repos for an object with this SHA-256 - let repos = state - .db - .list_all_repos() - .await - .map_err(AppError::Internal)?; + // One absolute budget bounds this request's whole acquire+walk lifetime (F3), + // captured before admission so the clock covers everything the walk permit + // holds. Each stage below (acquire, probe, walk, read) starts only while + // budget remains, and the acquire wait + walk deadline run clamped to the + // remainder, so an admitted request cannot hold its scarce walk slot for + // hours by drawing a fresh per-stage timeout every iteration. The budget + // NEVER aborts a running spawn_blocking walk: the clamped git deadline + // inside the walk is what ends it (a tokio timeout around the walk future + // would free the walk permit while the blocking thread still runs, the + // exact hole the held permit closes). + let request_deadline = std::time::Instant::now() + + std::time::Duration::from_secs(state.config.ipfs_request_budget_secs); + + // Bounded walk admission (#174 P1-3), taken before any DB/git work so a flood sheds + // cheaply. The per-repo `spawn_blocking` walk below is a full-history git walk with + // no served-git admission of its own; a permissionless caller could otherwise fan + // out concurrent walks past every git pool, exhausting the blocking pool + PIDs. + // Acquire the global permit (and, for a resolvable source, the per-source + // sub-permit) ONCE here and hold BOTH for the whole request — across every + // `spawn_blocking` walk in the loop below — so the slot reflects real blocking-thread + // occupancy (a tokio walk-timeout cannot free it while the blocking work still runs) + // and one request cannot open more than its share of concurrent walks. On + // unavailability shed a clean 503. The per-source key is the resolved source IP + // (`client_key`), never the DID (`/ipfs` admits any `did:key` unthrottled, so a DID + // key would be free to mint around); a `None` key (no trusted header, no peer) is + // bounded by the global pool only, never the per-source sub-cap. + let _ipfs_walk_permit = state + .git_ipfs_walk_semaphore + .clone() + .try_acquire_owned() + .map_err(|_| { + tracing::warn!("/ipfs walk concurrency cap reached; shedding request with 503"); + AppError::Overloaded("ipfs service at capacity, retry shortly".into()) + })?; + let source_key = crate::rate_limit::client_key(&req_headers, peer, state.push_limiter_trust); + let _ipfs_caller_permit = match &source_key { + Some(ip) => Some(state.git_ipfs_walk_per_caller.try_acquire(ip).ok_or_else(|| { + tracing::warn!(key = %ip, "/ipfs per-source walk cap reached; shedding request with 503"); + AppError::Overloaded("ipfs service at capacity for this source, retry shortly".into()) + })?), + None => None, + }; + + // 2. Search all repos for an object with this SHA-256. + // + // F6/KTD-5: both initial metadata queries run while the scarce walk permits + // acquired above are ALREADY held (RAII, for the whole request) and BEFORE the + // per-repo loop's first budget gate. With no pool statement_timeout, a query + // blocked in Postgres would otherwise pin those walk slots for the entire stall + // — past the request budget — capacity-503'ing later requests. Clamp BOTH to the + // remaining request budget. On timeout return the same retryable budget 503 the + // later stages shed (the "budget" source); returning here drops the RAII permits, + // freeing the slot. list_visibility_rules_for_repos is the access-control query, + // so its timeout returns BEFORE the loop — the scan can NEVER run with an empty + // rule map and serve an unfiltered listing that exposes private repos (FAIL CLOSED). + let repos = match tokio::time::timeout( + request_deadline.saturating_duration_since(std::time::Instant::now()), + state.db.list_all_repos(), + ) + .await + { + Ok(Ok(repos)) => repos, + Ok(Err(e)) => return Err(AppError::Internal(e)), + Err(_elapsed) => { + tracing::warn!( + budget_secs = state.config.ipfs_request_budget_secs, + "/ipfs list_all_repos exceeded the request budget \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); shedding a retryable 503 and freeing the walk permit" + ); + return Err(AppError::Overloaded(format!( + "ipfs scan incomplete (budget) for CID {cid_str}; retry shortly" + ))); + } + }; // Fetch every repo's visibility rules in one query rather than one per row // (the gate runs each row against its OWN rules — KTD2a). A row absent from // the map has no rules. let repo_ids: Vec = repos.iter().map(|r| r.id.clone()).collect(); - let rules_by_repo = state - .db - .list_visibility_rules_for_repos(&repo_ids) - .await - .map_err(AppError::Internal)?; + let rules_by_repo = match tokio::time::timeout( + request_deadline.saturating_duration_since(std::time::Instant::now()), + state.db.list_visibility_rules_for_repos(&repo_ids), + ) + .await + { + Ok(Ok(rules)) => rules, + Ok(Err(e)) => return Err(AppError::Internal(e)), + // FAIL CLOSED (security-critical): a timeout on the access-control query must + // DENY. Returning here — before the loop — means the scan can never fall + // through and apply an empty rule map, which would serve an unfiltered listing + // exposing private repos. It sheds the same retryable budget 503 as above. + Err(_elapsed) => { + tracing::warn!( + budget_secs = state.config.ipfs_request_budget_secs, + "/ipfs list_visibility_rules_for_repos exceeded the request budget \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); denying (fail closed) and freeing the walk permit" + ); + return Err(AppError::Overloaded(format!( + "ipfs scan incomplete (budget) for CID {cid_str}; retry shortly" + ))); + } + }; // Request-scoped memo of the per-repo allowed-blob set (KTD1, #126). The // caller is constant for one request, so `repo.id` alone is a safe, @@ -104,8 +219,73 @@ pub async fn get_by_cid( // deny entry (#126). let mut allowed_memo: HashMap> = HashMap::new(); + // Verdict-or-taint bookkeeping (F2): a candidate repo the loop cannot bring to a + // VERDICT (visibility deny / probe-says-absent / walk-gate deny / served) marks + // the scan truncated with its source. A truncated scan that finds nothing must + // NOT report 404 — the object may sit in a repo we skipped — so the terminal arm + // sheds a retryable 503 naming the sources, keyed so the operator can tell which + // knob (or backend) to look at. + let mut truncated_by: Vec<&'static str> = Vec::new(); + fn taint(truncated_by: &mut Vec<&'static str>, source: &'static str) { + if !truncated_by.contains(&source) { + truncated_by.push(source); + } + } + // A DETERMINISTIC probe fault (a corrupt repo / bad `.git/config`; #174 F5/U4) is + // separate from a transient taint: a retry cannot fix it, so a scan that found + // nothing must NOT shed the retryable 503 (which would invite a conformant client + // to retry-storm a fresh `git cat-file` per attempt against the broken repo). It + // sheds a terminal, non-retryable 500 instead — but only if nothing served, so one + // corrupt repo never masks a healthy repo that carries the object. + let mut deterministic_fault = false; + + // Budget gate shared by the four per-stage checks (F3): the remaining + // request budget, or — once exhausted — None, after logging the stage and + // knob and tainting "budget"; the call site only breaks (the scan STOPS, + // leaving this and every later candidate unproven, never a false 404). A + // stage is never started with zero remaining; the probe and read + // subprocesses carry no internal duration clamp, so this pre-start check is + // their entire bound (a hung one holds the request's walk slot for the + // duration of the hang). The acquire and walk stages clamp their deadlines + // to the returned remainder. + fn budget_gate( + truncated_by: &mut Vec<&'static str>, + deadline: std::time::Instant, + budget_secs: u64, + repo_name: &str, + stage: &'static str, + ) -> Option { + let left = deadline.saturating_duration_since(std::time::Instant::now()); + if left.is_zero() { + tracing::warn!( + repo = %repo_name, + stage, + budget_secs, + "/ipfs request budget exhausted before the stage \ + (GITLAWB_IPFS_REQUEST_BUDGET_SECS); stopping the scan without a verdict" + ); + taint(truncated_by, "budget"); + return None; + } + Some(left) + } + + // Cap on EXPENSIVE walks only (F2): counts the repos that actually require the + // full-history `allowed_blob_set_for_caller_bounded` walk (a path-scoped blob), + // checked immediately before the spawn_blocking below. Cheap probe-only visits + // are bounded by `repos_visited` — counting them here starved later-ordered + // repos out of a plain 200 on nodes with more readable repos than the cap. + let mut repos_walked: usize = 0; + // Ceiling on VISITS (F2): every repo past the visibility gate costs an acquire + // (worst case a full Tigris archive download on a cache miss) plus a cat-file + // probe, so one request can trigger at most `ipfs_max_repo_visits` object-store + // fetches. On exhaustion the scan STOPS — there is no cheaper way to continue. + let mut repos_visited: usize = 0; + for repo in &repos { - // Repo-level read gate against THIS row's own rules (KTD2a). + // Repo-level read gate against THIS row's own rules (KTD2a). Deny is a + // VERDICT: this repo would never serve the caller, so skipping it cannot + // hide content from them. let rules: &[crate::db::VisibilityRule] = rules_by_repo .get(&repo.id) .map(Vec::as_slice) @@ -114,19 +294,116 @@ pub async fn get_by_cid( continue; } - let repo_path = match state.repo_store.acquire(&repo.owner_did, &repo.name).await { - Ok(p) => p, - Err(_) => continue, + // Budget gate for the acquire stage (F3), checked ahead of the visit + // bookkeeping so an unstarted acquire is not counted as a visit. + let Some(budget_left) = budget_gate( + &mut truncated_by, + request_deadline, + state.config.ipfs_request_budget_secs, + &repo.name, + "repo acquire", + ) else { + break; + }; + + // Visit ceiling (F2): bound the acquire+probe cost class. Stopping here + // leaves the remaining candidates unproven, so the scan is truncated. + if repos_visited >= state.config.ipfs_max_repo_visits { + tracing::warn!( + ceiling = state.config.ipfs_max_repo_visits, + "/ipfs request hit the per-request repo-visit ceiling \ + (GITLAWB_IPFS_MAX_REPO_VISITS); stopping the scan without a verdict" + ); + taint(&mut truncated_by, "visit-ceiling"); + break; + } + repos_visited += 1; + + // Bound the per-repo acquire under `git_acquire_timeout_secs`: this loop shares + // the P1-2 stall vector (a hung Tigris HEAD/GET on one repo would otherwise + // block the whole /ipfs request). On expiry keep the fail-closed skip — never + // serve an un-acquired repo; a public copy (if any) still gets its turn — but + // the repo got no verdict, so the skip taints the scan. Clamped to the + // remaining request budget (F3) so per-repo acquires cannot each draw a + // fresh full timeout past it. + let acquire_deadline = std::cmp::min( + std::time::Duration::from_secs(state.config.git_acquire_timeout_secs), + budget_left, + ); + let repo_path = match tokio::time::timeout( + acquire_deadline, + state.repo_store.acquire(&repo.owner_did, &repo.name), + ) + .await + { + Ok(Ok(p)) => p, + Ok(Err(e)) => { + tracing::warn!(repo = %repo.name, err = %e, "repo acquire failed during /ipfs scan; skipping repo without a verdict"); + taint(&mut truncated_by, "acquire"); + continue; + } + Err(_elapsed) => { + tracing::warn!(repo = %repo.name, "repo acquire timed out during /ipfs scan; skipping repo without a verdict"); + taint(&mut truncated_by, "acquire"); + continue; + } + }; + + // Budget gate for the probe stage (F3): a probe is never STARTED with zero + // remaining, and a started probe now runs its git child under a deadline + // clamped to the remainder (below), so it can never complete past the budget. + let Some(probe_budget) = budget_gate( + &mut truncated_by, + request_deadline, + state.config.ipfs_request_budget_secs, + &repo.name, + "object-type probe", + ) else { + break; }; // Check whether the object exists in this repo before any expensive // reachability walk. This prevents random-CID spray from triggering - // full-history git walks on repos that don't carry the object. - let obj_type = match store::object_type(&repo_path, &sha256_hex) { - Ok(Some(t)) => t, - Ok(None) => continue, - Err(e) => { - tracing::warn!(repo = %repo.name, err = %e, "error checking git object type"); + // full-history git walks on repos that don't carry the object. Absent + // (`Ok(None)`) is a VERDICT; a probe that could not run is not. The + // `git cat-file -t` shells out, so run it OFF the async worker under the + // reaped bounded runner (#174 F3) — a hung/corrupt object store cannot pin a + // runtime worker or the held IPFS permits past the deadline. + let probe_deadline = std::time::Instant::now() + + std::cmp::min( + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + probe_budget, + ); + // The probe shells to the real `git` (as `object_type` historically did), + // independent of `state.git_bin` (which tests point at a fake walk git). + let probe_path = repo_path.clone(); + let probe_sha = sha256_hex.clone(); + let obj_type = match tokio::task::spawn_blocking(move || { + store::object_type_bounded("git", &probe_path, &probe_sha, probe_deadline) + }) + .await + { + Ok(Ok(Some(t))) => t, + Ok(Ok(None)) => continue, + // Transient probe fault (unreadable/mid-repack store): unproven, retryable. + Ok(Err(store::ProbeError::Transient(e))) => { + tracing::warn!(repo = %repo.name, err = %e, "object-type probe hit a transient store fault during /ipfs scan; skipping repo without a verdict"); + taint(&mut truncated_by, "probe"); + continue; + } + // Deterministic probe fault (corrupt repo / bad config): a retry cannot fix + // it, so it does NOT taint (which would shed a retryable 503). It records a + // terminal condition that the terminal arm renders as a non-retryable 500 + // only if nothing served. The raw git detail stays in the log; the client + // body is opaque. + Ok(Err(store::ProbeError::Deterministic(e))) => { + tracing::warn!(repo = %repo.name, err = %e, "object-type probe hit a deterministic fault (corrupt repo/config) during /ipfs scan; skipping repo without a verdict"); + deterministic_fault = true; + continue; + } + Err(join_err) => { + tracing::warn!(repo = %repo.name, err = %join_err, "object-type probe task panicked during /ipfs scan; skipping repo without a verdict"); + taint(&mut truncated_by, "probe"); continue; } }; @@ -137,15 +414,52 @@ pub async fn get_by_cid( let path_scoped = has_path_scoped_rule(rules); if path_scoped && obj_type == "blob" { if !allowed_memo.contains_key(&repo.id) { + // Budget gate for the walk stage (F3): a walk is never STARTED + // with zero remaining (probed-present is not a serve), and a + // started walk runs its git children under a deadline clamped + // to the remainder (the min below), so a walk can never + // complete past the budget. + let Some(budget_left) = budget_gate( + &mut truncated_by, + request_deadline, + state.config.ipfs_request_budget_secs, + &repo.name, + "visibility walk", + ) else { + break; + }; + // Walk cap (F2), checked at the one site that actually spends a walk: + // on exhaustion skip THIS repo without a verdict and KEEP scanning — + // later candidates may still reach a cheap probe-only verdict (a plain + // public copy serves its 200 with no walk at all). + if repos_walked >= state.config.ipfs_max_repos_walked { + tracing::warn!( + cap = state.config.ipfs_max_repos_walked, + repo = %repo.name, + "/ipfs request hit the per-request walk cap \ + (GITLAWB_IPFS_MAX_REPOS_WALKED); skipping repo without a verdict" + ); + taint(&mut truncated_by, "walk-cap"); + continue; + } + repos_walked += 1; let rp = repo_path.clone(); let r = rules.to_vec(); let is_public = repo.is_public; let owner = repo.owner_did.clone(); let caller_for_walk = caller_owned.clone(); - // Full-history walk shells out to git — keep it off the async runtime. + let git_bin = state.git_bin.clone(); + let walk_timeout = std::cmp::min( + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + budget_left, + ); + // Full-history walk shells out to git — keep it off the async runtime, + // bounded and reaped like the served-git ops (#174). let walk = tokio::task::spawn_blocking(move || { - allowed_blob_set_for_caller( + allowed_blob_set_for_caller_bounded( &rp, + &git_bin, + walk_timeout, &r, is_public, &owner, @@ -155,20 +469,25 @@ pub async fn get_by_cid( .await; // Fail closed on EITHER a task panic (JoinError) or a walk error: // we cannot prove the caller may read here, so skip this repo and - // let a public copy (if any) serve. Never serve on an unproven gate. + // let a public copy (if any) serve. Never serve on an unproven gate + // — and never report absent on one either (no verdict, taint). let set = match walk { Ok(Ok(set)) => set, Ok(Err(e)) => { - tracing::warn!(repo = %repo.name, err = %e, "allowed-blob walk failed; skipping repo"); + tracing::warn!(repo = %repo.name, err = %e, "allowed-blob walk failed during /ipfs scan; skipping repo without a verdict"); + taint(&mut truncated_by, "walk-failure"); continue; } Err(e) => { - tracing::warn!(repo = %repo.name, err = %e, "allowed-blob walk task panicked; skipping repo"); + tracing::warn!(repo = %repo.name, err = %e, "allowed-blob walk task panicked during /ipfs scan; skipping repo without a verdict"); + taint(&mut truncated_by, "walk-failure"); continue; } }; allowed_memo.insert(repo.id.clone(), set); } + // Not in the caller's reachable allowed-set: a VERDICT (deny), the walk + // proved this repo would never serve the blob to this caller. let in_allowed = allowed_memo .get(&repo.id) .is_some_and(|set| set.contains(&sha256_hex)); @@ -177,11 +496,55 @@ pub async fn get_by_cid( } } - // Now that we've passed the gate, read the content. - let content = match store::read_object_content(&repo_path, &sha256_hex, &obj_type) { - Ok(c) => c, - Err(e) => { - tracing::warn!(repo = %repo.name, err = %e, "error reading git object content"); + // Budget gate for the content-read stage (F3): the read subprocess is + // unclamped, so it never starts past the budget. Tainting instead of + // serving keeps the terminal arm honest and the stop unconditional; the + // retryable 503 tells the caller to come back rather than letting an + // over-budget request keep spending. + let Some(read_budget) = budget_gate( + &mut truncated_by, + request_deadline, + state.config.ipfs_request_budget_secs, + &repo.name, + "content read", + ) else { + break; + }; + + // Now that we've passed the gate, read the content. A failed read after a + // passed gate is not an absence verdict — the probe just said the object + // exists here — so the skip taints the scan. Like the probe, the read shells + // out to `git cat-file `, so run it OFF the async worker under the reaped + // bounded runner clamped to the remaining budget (#174 F3). + let read_deadline = std::time::Instant::now() + + std::cmp::min( + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + read_budget, + ); + // Real `git`, as the read historically used, independent of `state.git_bin`. + let read_path = repo_path.clone(); + let read_sha = sha256_hex.clone(); + let read_type = obj_type.clone(); + let content = match tokio::task::spawn_blocking(move || { + store::read_object_content_bounded( + "git", + &read_path, + &read_sha, + &read_type, + read_deadline, + ) + }) + .await + { + Ok(Ok(c)) => c, + Ok(Err(e)) => { + tracing::warn!(repo = %repo.name, err = %e, "object content read failed during /ipfs scan; skipping repo without a verdict"); + taint(&mut truncated_by, "read"); + continue; + } + Err(join_err) => { + tracing::warn!(repo = %repo.name, err = %join_err, "object content read task panicked during /ipfs scan; skipping repo without a verdict"); + taint(&mut truncated_by, "read"); continue; } }; @@ -205,7 +568,42 @@ pub async fn get_by_cid( return Ok((StatusCode::OK, headers, content).into_response()); } - // Not found in any repo + // Deterministic fault (F5/U4): a candidate repo is persistently broken (corrupt + // repo / bad `.git/config`), so the object is not proven absent AND a retry cannot + // change that. Shed a TERMINAL, non-retryable 500 rather than the retryable 503 + // below — a 503 would let a conformant client retry-storm a fresh `git cat-file` + // per attempt against the broken repo. The body is opaque (a generic message via + // `AppError::Git` -> 500, no Retry-After): the raw git stderr — which leaks + // filesystem paths / config — was logged at the probe, and never reaches the client. + // + // Gate on `truncated_by.is_empty()`: the terminal 500 fires only when a + // deterministic fault is the SOLE reason nothing served. When a TRANSIENT taint + // co-occurs (a DIFFERENT repo was skipped by budget / acquire / walk-cap / probe / + // visit-ceiling), the requested object may live in that transiently-skipped repo, + // so fall through to the retryable 503 below — a retry can re-probe it and serve + // the content. Reporting a terminal 500 there would wrongly tell a conformant + // client not to retry, leaving reachable content unreachable until the unrelated + // broken repo is repaired. + if deterministic_fault && truncated_by.is_empty() { + return Err(AppError::Git( + "ipfs object probe could not complete: a candidate repository is corrupt".into(), + )); + } + + // Truncated scan (F2): at least one candidate repo yielded no verdict, so the + // object is not proven absent. A 404 here would misreport existing content, so + // shed retryable instead — Overloaded is the single 503 + Retry-After site in + // error.rs, and the message names the truncation sources so the operator can + // map the shed to the right knob or backend. + if !truncated_by.is_empty() { + return Err(AppError::Overloaded(format!( + "ipfs scan incomplete ({}) for CID {cid_str}; retry shortly", + truncated_by.join("+") + ))); + } + + // Complete scan: every candidate reached a verdict and none served, so the + // object is definitively absent (or denied) for this caller. Err(AppError::RepoNotFound(format!( "no git object found for CID {cid_str}" ))) @@ -228,3 +626,2149 @@ pub async fn list_pins(State(state): State) -> Result Router { + Router::new() + .route( + "/ipfs/{cid}", + axum::routing::get(crate::api::ipfs::get_by_cid), + ) + .layer(axum::middleware::from_fn(crate::auth::optional_signature)) + .with_state(state) + } + + /// A syntactically valid CIDv1(raw, sha2-256) string the handler decodes past its + /// CID/hash-code validation, so the request reaches the walk admission (not a 400). + fn valid_cid() -> String { + gitlawb_core::cid::Cid::from_git_object_bytes(b"blob 5\0hello") + .as_str() + .to_string() + } + + fn get_cid(cid: &str, peer: Option) -> Request { + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + if let Some(p) = peer { + req.extensions_mut().insert(ConnectInfo(p)); + } + req + } + + /// Run real git, asserting success. Shared by the F2 scan-verdict tests. + fn run_git(args: &[&str], cwd: &std::path::Path) { + let out = std::process::Command::new("git") + .args(args) + .current_dir(cwd) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + } + + /// Seed a repo row plus a REAL sha256 bare repo at its acquired path holding one + /// committed blob (`src/secret.txt` = `content`). Returns `(repo_id, blob_oid)`. + /// Same recipe as `get_by_cid_walk_permit_held_through_blocking_walk`: the CID + /// digest IS the sha256 object id under `--object-format=sha256`, so the real + /// `cat-file` probe finds the blob. + async fn seed_repo_with_blob( + state: &crate::state::AppState, + tmp: &std::path::Path, + owner: &str, + name: &str, + content: &[u8], + ) -> (String, String) { + state + .db + .upsert_mirror_repo(owner, name, &format!("/unused-{name}"), None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + let _ = std::fs::remove_dir_all(&bare); + std::fs::create_dir_all(&bare).unwrap(); + let work = tmp.join(format!("work-{owner}-{name}")); + std::fs::create_dir_all(work.join("src")).unwrap(); + std::fs::write(work.join("src/secret.txt"), content).unwrap(); + run_git( + &["init", "-q", "--object-format=sha256", "-b", "main"], + &work, + ); + run_git(&["config", "user.email", "t@t"], &work); + run_git(&["config", "user.name", "t"], &work); + run_git(&["add", "src/secret.txt"], &work); + run_git(&["commit", "-q", "-m", "seed"], &work); + run_git( + &[ + "clone", + "--bare", + "-q", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + tmp, + ); + let out = std::process::Command::new("git") + .args(["rev-parse", "HEAD:src/secret.txt"]) + .current_dir(&work) + .output() + .expect("git rev-parse runs"); + assert!(out.status.success(), "rev-parse failed"); + let oid = String::from_utf8_lossy(&out.stdout).trim().to_string(); + (rec.id, oid) + } + + /// CIDv1(raw, sha2-256) for a sha256 object id, as the handler resolves it. + fn cid_for_oid(oid: &str) -> String { + let oid_bytes = gitlawb_core::cid::sha256_hex_to_bytes(oid).unwrap(); + gitlawb_core::cid::Cid::from_sha256_bytes(&oid_bytes) + .as_str() + .to_string() + } + + /// A local endpoint whose TCP accept succeeds instantly but that never writes + /// an HTTP response, so a Tigris HEAD against it stalls deterministically + /// until the caller's timeout. (A non-routable address hangs only if the + /// network blackholes the SYN — a fast RST would end the stall early.) The + /// accepted sockets are parked in the spawned task, which dies with the + /// test's runtime, so the peer never sees a close mid-test. + async fn silent_tigris_endpoint() -> String { + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let endpoint = format!("http://{}", listener.local_addr().unwrap()); + tokio::spawn(async move { + let mut held = Vec::new(); + while let Ok((sock, _)) = listener.accept().await { + held.push(sock); + } + }); + endpoint + } + + /// Fake git for the WALK only (`state.git_bin`): empty refs, `rev-parse` + /// resolves, and each `rev-list` appends one line to `log` and prints nothing — + /// every walked repo yields an EMPTY allowed-set (path-gate deny verdict) and + /// the log's line count == the number of expensive walks run. The probe and the + /// content read shell to the real `git`, so seeded objects must genuinely exist. + #[cfg(unix)] + fn walk_logging_fake_git(dir: &std::path::Path, log: &std::path::Path) -> String { + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + for-each-ref) : ;;\n\ + rev-parse) echo deadbeef ;;\n\ + rev-list) echo walk >> \"{}\" ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + log.display() + ); + let git_path = dir.join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + git_path.to_str().unwrap().to_string() + } + + /// F2 buried-row repro: with more readable repos than `ipfs_max_repos_walked`, + /// existing PUBLIC content past the cap must still serve. The cap counts + /// EXPENSIVE walks only — this request has no path-scoped rules anywhere, so it + /// runs ZERO walks (the fake-git walk log stays empty) and the cap can never cut + /// the scan: the blob buried in the OLDER-updated repo (iterated last under + /// `list_all_repos`' updated_at DESC) serves its 200. Before F2 the cap counted + /// visibility-passing VISITS and broke the loop into the opaque 404 — existing + /// content misreported absent because of unrelated repos. MUTATION (RED): count + /// visits against the cap again (re-add the check+increment at the visibility + /// gate) and the buried row 503s instead of serving. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_buried_public_row_past_walk_cap_still_serves(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let walk_log = tmp.path().join("walks.log"); + state.git_bin = walk_logging_fake_git(tmp.path(), &walk_log); + // Tighter than the repo count: the old visit-counting cap cut the scan here. + let mut cfg = (*state.config).clone(); + cfg.ipfs_max_repos_walked = 1; + state.config = Arc::new(cfg); + + // Seed the blob-carrying repo FIRST so its updated_at is OLDER: the empty + // repo is iterated first and the blob row sits past the old visit budget. + let (_, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6f2buried", + "buried", + b"buried row proof\n", + ) + .await; + seed_repo_with_blob( + &state, + tmp.path(), + "z6f2buried", + "fresh", + b"unrelated content\n", + ) + .await; + + let peer: SocketAddr = "203.0.113.60:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&cid_for_oid(&oid), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::OK, + "a public blob in a repo past the walk cap must still serve — the cap \ + counts expensive walks and this scan needs none" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + assert_eq!(&body[..], b"buried row proof\n"); + let walks = std::fs::read_to_string(&walk_log) + .map(|s| s.lines().count()) + .unwrap_or(0); + assert_eq!( + walks, 0, + "a request with no path-scoped rules anywhere must run zero expensive walks" + ); + } + + /// F2 walk-cap skip-and-continue: exhausting `ipfs_max_repos_walked` skips the + /// walk-NEEDING repo without a verdict but keeps the scan alive. Three public + /// repos carry the same blob, newest first: the first (path-scoped) consumes the + /// cap-of-1 walk and denies (empty allowed-set — a verdict); the second + /// (path-scoped) needs a walk the cap forbids and is skipped WITHOUT one (taint); + /// the third is plain public and serves the 200 from a cheap probe — found beats + /// taint, and exactly one expensive walk ran. Before F2 the cap broke the loop at + /// the second repo and the request 404'd despite the public copy. MUTATION (RED): + /// turn the walk-cap skip back into a `break` and the public copy never serves. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_walk_cap_skip_continues_to_later_public_copy(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let walk_log = tmp.path().join("walks.log"); + state.git_bin = walk_logging_fake_git(tmp.path(), &walk_log); + let mut cfg = (*state.config).clone(); + cfg.ipfs_max_repos_walked = 1; + state.config = Arc::new(cfg); + + // Insert order = oldest first, so iteration (updated_at DESC) is reversed: + // gatedwalk, then gatedskip, then pubcopy. Identical content -> one CID. + let content = b"skip and continue proof\n"; + let (_, oid) = + seed_repo_with_blob(&state, tmp.path(), "z6f2skip", "pubcopy", content).await; + let (skip_id, _) = + seed_repo_with_blob(&state, tmp.path(), "z6f2skip", "gatedskip", content).await; + let (walk_id, _) = + seed_repo_with_blob(&state, tmp.path(), "z6f2skip", "gatedwalk", content).await; + for id in [&walk_id, &skip_id] { + state + .db + .set_visibility_rule( + id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3IpfsReaderCCCCCCCCCCCCCCCCCCCCCCCC".to_string()], + "z6f2skip", + ) + .await + .unwrap(); + } + + let peer: SocketAddr = "203.0.113.61:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&cid_for_oid(&oid), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::OK, + "the walk-cap skip must continue the scan so the plain public copy serves" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + assert_eq!(&body[..], content.as_slice()); + let walks = std::fs::read_to_string(&walk_log) + .map(|s| s.lines().count()) + .unwrap_or(0); + assert_eq!( + walks, 1, + "cap honored exactly: the first path-scoped repo walks, the second is cut" + ); + } + + /// F2 visit ceiling: `ipfs_max_repo_visits` bounds the acquire+probe cost class + /// (each visit can be a full Tigris archive fetch on a cache miss). Unlike the + /// walk cap there is no cheap way to keep scanning, so exhaustion STOPS the scan + /// — and the stop is a truncation, not an absence: with ceiling 1 the newer + /// empty repo consumes the only visit and the blob-carrying older repo is never + /// probed, so the request sheds a retryable 503 + Retry-After, never a false + /// 404. MUTATION (RED): drop the ceiling check and the blob serves (200); drop + /// only the taint on the break and the 503 decays to a 404. + #[sqlx::test] + async fn get_by_cid_visit_ceiling_stops_scan_with_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let mut cfg = (*state.config).clone(); + cfg.ipfs_max_repo_visits = 1; + state.config = Arc::new(cfg); + + // Blob repo first (older, iterated second); empty repo second (newer, + // consumes the single visit). + let (_, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6f2visit", + "buried", + b"visit ceiling proof\n", + ) + .await; + seed_repo_with_blob(&state, tmp.path(), "z6f2visit", "fresh", b"unrelated\n").await; + + let peer: SocketAddr = "203.0.113.62:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&cid_for_oid(&oid), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a visit-ceiling truncation must shed a retryable 503, not report absent" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + } + + /// F2 negative arm: a COMPLETE scan that finds nothing keeps its definitive 404 + /// — the truncation 503 must never fire when every candidate reached a verdict. + /// Two public repos both probe clean (the requested CID is nowhere), no rules, + /// no cap or ceiling hit: 404 with no Retry-After. MUTATION (RED): taint the + /// scan unconditionally and this decays into a 503. + #[sqlx::test] + async fn get_by_cid_complete_scan_keeps_definitive_404(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + seed_repo_with_blob(&state, tmp.path(), "z6f2clean", "one", b"content one\n").await; + seed_repo_with_blob(&state, tmp.path(), "z6f2clean", "two", b"content two\n").await; + + // valid_cid() is the "hello" blob — present in neither repo. + let peer: SocketAddr = "203.0.113.63:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::NOT_FOUND, + "a complete clean scan is a definitive absence — 404, never the 503 shed" + ); + assert!( + resp.headers().get("retry-after").is_none(), + "a definitive 404 must not advertise a retry" + ); + } + + /// F2 acquire taint: a repo row with NO local copy over a Tigris backend that + /// stalls (a silent local endpoint — accepted, never answered) hits the 1s + /// acquire timeout at the read-acquire site. The skip carries no verdict, so the + /// scan is truncated: retryable 503 + Retry-After, never the old silent-skip 404. + /// MUTATION (RED): drop the taint on the acquire-timeout arm and this decays to + /// a 404. + #[sqlx::test] + async fn get_by_cid_acquire_timeout_taints_scan_to_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + // Endpoint-pinned test client (no AWS_* env reads — env is racy under a + // parallel test run); the silent local endpoint stalls the HEAD + // deterministically. + let endpoint = silent_tigris_endpoint().await; + let tigris = + crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint) + .await; + state.repo_store = crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let mut cfg = (*state.config).clone(); + cfg.git_acquire_timeout_secs = 1; + state.config = Arc::new(cfg); + + // Row exists in the DB but has no local copy, so the read acquire must + // consult Tigris (local-miss path) and stall until the timeout. + state + .db + .upsert_mirror_repo("z6f2acq", "ghost", "/unused-ghost", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.64:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an acquire timeout leaves the repo unproven — the scan must shed 503, not 404" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + } + + /// F2 found-beats-taint on the acquire arm: an acquire timeout taints the + /// scan but must NOT stop it — the loop `continue`s, and a later repo that + /// genuinely carries the object still serves. The NEWER row (visited first + /// under `list_all_repos`' updated_at DESC) is a Tigris-backed ghost whose + /// acquire stalls against the silent endpoint and times out at 1s; the + /// OLDER row is a plain public repo carrying the blob, reached next and + /// served from a cheap probe — found beats taint: 200 with the blob bytes, + /// never the truncation 503. MUTATION (RED): turn the acquire-timeout arm's + /// `continue` into a `break` and the public copy never serves (503). + #[sqlx::test] + async fn get_by_cid_acquire_taint_does_not_block_later_public_copy(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Seed the blob repo through a LOCAL-ONLY store first, so seeding never + // consults the (deliberately unreachable) Tigris endpoint. + state.repo_store = + crate::git::repo_store::RepoStore::for_testing(repos_dir.clone(), pool.clone()); + let content = b"acquire taint continue proof\n"; + let (_, oid) = + seed_repo_with_blob(&state, tmp.path(), "z6f2acqcont", "pubcopy", content).await; + // Swap in a Tigris-backed store over the SAME repos_dir (the seeded bare + // repo stays a fast local hit) and add a NEWER ghost row with no local + // copy: its acquire consults the silent local endpoint and stalls to the + // 1s timeout (endpoint-pinned test client, no AWS_* env reads). + let endpoint = silent_tigris_endpoint().await; + let tigris = + crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint) + .await; + state.repo_store = crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), pool); + state + .db + .upsert_mirror_repo("z6f2acqcont", "ghost", "/unused-ghost", None, false) + .await + .unwrap(); + let mut cfg = (*state.config).clone(); + cfg.git_acquire_timeout_secs = 1; + state.config = Arc::new(cfg); + + // Ordering precondition: the ghost must be iterated FIRST (updated_at + // DESC — it was upserted after the blob repo), otherwise the pubcopy + // would serve before the taint ever fires and the continue-vs-break + // distinction would go untested. + let order: Vec = state + .db + .list_all_repos() + .await + .unwrap() + .into_iter() + .map(|r| r.name) + .collect(); + let ghost_pos = order.iter().position(|n| n == "ghost").unwrap(); + let pub_pos = order.iter().position(|n| n == "pubcopy").unwrap(); + assert!( + ghost_pos < pub_pos, + "precondition: the stalling ghost must be iterated before the blob repo; got {order:?}" + ); + + let peer: SocketAddr = "203.0.113.73:5000".parse().unwrap(); + let started = std::time::Instant::now(); + let resp = ipfs_router(state) + .oneshot(get_cid(&cid_for_oid(&oid), Some(peer))) + .await + .unwrap(); + // The taint arm demonstrably FIRED on this run: the response can only + // arrive after the ghost's stalled acquire burned its full 1s timeout + // (a cheap skip or a deny verdict would answer near-instantly). + assert!( + started.elapsed() >= std::time::Duration::from_millis(900), + "the ghost's acquire must stall to its timeout before the scan continues; \ + got {:?}", + started.elapsed() + ); + assert_eq!( + resp.status(), + StatusCode::OK, + "an acquire taint must not stop the scan — the later public copy serves" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + assert_eq!( + &body[..], + content.as_slice(), + "the served body must be the blob content from the later public copy" + ); + } + + /// F2 probe taint: a repo row whose local dir does not exist (no Tigris) — + /// `RepoStore::acquire` returns the path anyway (local passthrough), and the + /// `cat-file -t` probe cannot even spawn (missing working dir), so + /// `object_type` is Err. That is not an absence verdict, so the scan is + /// truncated: 503, never 404. A second, real repo probes clean (absent verdict) + /// — the one bad row is what taints. NOTE: the probe shells to the real `git` + /// (not `state.git_bin`), and a clean missing/invalid-object nonzero exit is + /// still `Ok(None)` (an absent verdict) — this arm needs a probe that could + /// not RUN, hence the missing-dir spawn failure here; the corrupt-repo test + /// below drives the stderr-discriminated Err. MUTATION (RED): drop the + /// taint on the probe-error arm and this decays to a 404. + #[sqlx::test] + async fn get_by_cid_probe_error_taints_scan_to_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // Older row: a real repo that probes clean. Newer row: no dir on disk. + seed_repo_with_blob(&state, tmp.path(), "z6f2probe", "real", b"probe clean\n").await; + state + .db + .upsert_mirror_repo("z6f2probe", "ghost", "/unused-ghost", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.65:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a failed probe leaves the repo unproven — the scan must shed 503, not 404" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + } + + /// F2 probe taint, corrupt-repo arm: a repo whose git dir EXISTS but is broken + /// (objects/ removed, HEAD garbage) makes the real `cat-file -t` die with the + /// repo-level `fatal: not a git repository` — a probe that could not examine + /// the object store, not an absence verdict, so `object_type` must map it to + /// Err and the scan must shed the probe-tainted 503, never the silent-absence + /// 404. A second, real repo probes clean (absent verdict) — the corrupt row is + /// what taints. MUTATION (RED): map every nonzero cat-file exit back to + /// `Ok(None)` in `object_type` (drop the stderr discrimination) and this + /// decays to a 404. + #[sqlx::test] + async fn get_by_cid_corrupt_repo_dir_probe_error_taints_scan_to_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // Older row: a real repo that probes clean. Newer row: a bare repo whose + // git dir exists on disk but is corrupt at the repo level. + seed_repo_with_blob(&state, tmp.path(), "z6f2corrupt", "real", b"probe clean\n").await; + state + .db + .upsert_mirror_repo("z6f2corrupt", "broken", "/unused-broken", None, false) + .await + .unwrap(); + let rec = state + .db + .get_repo("z6f2corrupt", "broken") + .await + .unwrap() + .unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + std::fs::create_dir_all(&bare).unwrap(); + run_git(&["init", "-q", "--bare", "--object-format=sha256"], &bare); + std::fs::remove_dir_all(bare.join("objects")).unwrap(); + std::fs::write(bare.join("HEAD"), b"junk\n").unwrap(); + + let peer: SocketAddr = "203.0.113.68:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a repo-level cat-file fatal leaves the repo unproven — the scan must \ + shed 503, not report the object absent" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("probe"), + "the shed must name the probe taint; got: {body}" + ); + } + + /// #174 F5/U4 (RED-before/GREEN-after): a candidate repo with a corrupt + /// `.git/config` makes `git cat-file` die with `fatal: bad config line N` while + /// `objects/` stays readable. That is a DETERMINISTIC fault, not an absence, and a + /// retry cannot fix it — so the scan must shed a TERMINAL, non-retryable 500, never + /// the old false 404 (`Ok(None)` fell through) and never the retryable 503 (which + /// would invite a conformant client to retry-storm a fresh `cat-file` per attempt). + /// A second, healthy repo probes clean (absent verdict); the corrupt row is what + /// forces the 500. The body must be OPAQUE — no raw git stderr, no filesystem path. + /// MUTATION (RED): route the deterministic fault back to `Ok(None)` in + /// `object_type_bounded` and this decays to a 404; classify it Transient and it + /// decays to a retryable 503. + #[sqlx::test] + async fn get_by_cid_bad_config_repo_is_terminal_500_not_404_or_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // A healthy repo that probes clean (would give a definitive 404 on its own) plus + // a repo whose bare git dir has a corrupt config (objects/ intact). + seed_repo_with_blob(&state, tmp.path(), "z6f5clean", "real", b"probe clean\n").await; + state + .db + .upsert_mirror_repo("z6f5badcfg", "broken", "/unused-badcfg", None, false) + .await + .unwrap(); + let rec = state + .db + .get_repo("z6f5badcfg", "broken") + .await + .unwrap() + .unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + std::fs::create_dir_all(&bare).unwrap(); + run_git(&["init", "-q", "--bare", "--object-format=sha256"], &bare); + // Corrupt the config; leave objects/ readable (the readable-store + git-fails + // combination is exactly what makes this deterministic, not transient). + { + use std::io::Write; + let mut cfg = std::fs::OpenOptions::new() + .append(true) + .open(bare.join("config")) + .unwrap(); + cfg.write_all(b"\n[broken section\nnot a valid = = = line\n") + .unwrap(); + } + + let peer: SocketAddr = "203.0.113.69:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::INTERNAL_SERVER_ERROR, + "a bad-config (deterministic) repo fault must shed a terminal 500, never a \ + 404 (false absence) or a retryable 503" + ); + assert!( + resp.headers().get("retry-after").is_none(), + "a terminal 500 must NOT advertise a retry (that is the whole point vs 503)" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + !body.contains("bad config") + && !body.contains(tmp.path().to_str().unwrap()) + && !body.contains(".git") + && !body.contains("fatal"), + "the 500 body must be opaque — no raw git stderr / config text / filesystem \ + path; got: {body}" + ); + } + + /// #174 F5 co-occurrence (RED-before/GREEN-after): a deterministic fault on ONE + /// repo and a TRANSIENT taint on a DIFFERENT repo occur in the same scan, and the + /// requested CID is served by neither. The transiently-skipped repo could hold the + /// object, so a retry can surface it — the correct shed is the RETRYABLE 503, not + /// the terminal 500. Two broken repos drive it, both local so the outcome is + /// deterministic: a bad-`config` repo whose `objects/` stays readable is a + /// DETERMINISTIC probe fault (`deterministic_fault = true`), while a repo whose + /// `objects/` dir is removed is a TRANSIENT probe fault (taints "probe"). A third + /// healthy repo probes clean (absent verdict) so nothing serves. Before the fix the + /// terminal `if deterministic_fault` arm fired first and shed 500 unconditionally, + /// hiding the transiently-skipped repo behind a non-retryable status. MUTATION + /// (RED): drop the `&& truncated_by.is_empty()` gate and this shes 500 again. + #[sqlx::test] + async fn get_by_cid_deterministic_fault_with_cooccurring_transient_taint_is_503_not_500( + pool: sqlx::PgPool, + ) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // Healthy repo that probes clean (definitive absence on its own). + seed_repo_with_blob(&state, tmp.path(), "z6f5coclean", "real", b"probe clean\n").await; + + // Bad-config repo: objects/ readable, config corrupt -> DETERMINISTIC fault. + state + .db + .upsert_mirror_repo("z6f5cobadcfg", "broken", "/unused-badcfg", None, false) + .await + .unwrap(); + let rec = state + .db + .get_repo("z6f5cobadcfg", "broken") + .await + .unwrap() + .unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + std::fs::create_dir_all(&bare).unwrap(); + run_git(&["init", "-q", "--bare", "--object-format=sha256"], &bare); + { + use std::io::Write; + let mut cfg = std::fs::OpenOptions::new() + .append(true) + .open(bare.join("config")) + .unwrap(); + cfg.write_all(b"\n[broken section\nnot a valid = = = line\n") + .unwrap(); + } + + // Corrupt-dir repo: objects/ removed -> TRANSIENT probe fault (taints "probe"), + // a DIFFERENT repo than the deterministic one above. + state + .db + .upsert_mirror_repo("z6f5cocorrupt", "broken", "/unused-corrupt", None, false) + .await + .unwrap(); + let rec2 = state + .db + .get_repo("z6f5cocorrupt", "broken") + .await + .unwrap() + .unwrap(); + let bare2 = state + .repo_store + .acquire(&rec2.owner_did, &rec2.name) + .await + .unwrap(); + std::fs::create_dir_all(&bare2).unwrap(); + run_git(&["init", "-q", "--bare", "--object-format=sha256"], &bare2); + std::fs::remove_dir_all(bare2.join("objects")).unwrap(); + std::fs::write(bare2.join("HEAD"), b"junk\n").unwrap(); + + let peer: SocketAddr = "203.0.113.71:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a deterministic fault co-occurring with a transient taint on a DIFFERENT \ + repo must shed the retryable 503 (a retry can surface the object in the \ + transiently-skipped repo), never the terminal 500" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the co-occurrence 503 must carry Retry-After" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("probe"), + "the shed must name the transient probe taint; got: {body}" + ); + } + + /// F2 read taint: the gate passes (the probe reads the truncated loose object's + /// intact "blob 64" header) but the content read fails (`cat-file blob` dies on + /// the deflate stream cut mid-content) — the probe just said the object EXISTS + /// here, so the failed read is no absence verdict: 503, never 404. The loose + /// object is hand-rolled: zlib header + one stored deflate block declaring 72 + /// bytes ("blob 64\0" + 64), truncated after the header NUL + 4 content bytes, + /// no adler trailer. MUTATION (RED): drop the taint on the read-error arm and + /// this decays to a 404. + #[sqlx::test] + async fn get_by_cid_read_error_taints_scan_to_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + state + .db + .upsert_mirror_repo("z6f2read", "corrupt", "/unused-corrupt", None, false) + .await + .unwrap(); + let rec = state + .db + .get_repo("z6f2read", "corrupt") + .await + .unwrap() + .unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + std::fs::create_dir_all(&bare).unwrap(); + run_git(&["init", "-q", "--bare", "--object-format=sha256"], &bare); + // Hand-rolled truncated loose object (dangling is fine: no path-scoped rules, + // so the "/" gate is the whole story and the read follows the probe). + let oid = "6bf5122f344554c53bde2ebb8cd2b7e3d1600ad631c385a5d7cce23c7785459c"; + let mut corrupt: Vec = vec![0x78, 0x01, 0x01, 0x48, 0x00, 0xb7, 0xff]; + corrupt.extend_from_slice(b"blob 64\0AAAA"); + let obj_dir = bare.join("objects").join(&oid[..2]); + std::fs::create_dir_all(&obj_dir).unwrap(); + std::fs::write(obj_dir.join(&oid[2..]), &corrupt).unwrap(); + // Preconditions: the probe classifies it as a blob, the full read fails — + // otherwise the test would pass vacuously via some other arm. + assert_eq!( + crate::git::store::object_type(&bare, oid) + .unwrap() + .as_deref(), + Some("blob"), + "the truncated loose object's header must still probe as a blob" + ); + assert!( + crate::git::store::read_object_content(&bare, oid, "blob").is_err(), + "the truncated loose object's content read must fail" + ); + + let peer: SocketAddr = "203.0.113.66:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&cid_for_oid(oid), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a failed read after a passed gate leaves the repo unproven — 503, not 404" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + } + + /// F2 denied-is-a-verdict: repos that DENY the caller at the visibility gate + /// are settled, not skipped — an all-denied scan is COMPLETE: 404, zero visits. + /// The private rows deliberately have no local dirs: if the deny didn't + /// short-circuit before the visit, the missing-dir probe would taint the scan + /// into a 503, which the 404 assertion rules out — so the 404 also proves zero + /// acquires, probes, or walks ran for denied rows. + #[sqlx::test] + async fn get_by_cid_all_denied_is_complete_scan_404(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + for name in ["priv-a", "priv-b"] { + let now = chrono::Utc::now(); + state + .db + .create_repo(&crate::db::RepoRecord { + id: uuid::Uuid::new_v4().to_string(), + name: name.to_string(), + owner_did: "did:key:z6MkF2DenyOwnerAAAAAAAAAAAAAAAAAAAAAAAAA".to_string(), + description: None, + is_public: false, + default_branch: "main".to_string(), + created_at: now, + updated_at: now, + disk_path: format!("/nonexistent/{name}"), + forked_from: None, + machine_id: None, + }) + .await + .unwrap(); + } + + let peer: SocketAddr = "203.0.113.67:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::NOT_FOUND, + "an anonymous caller denied by every repo gets a complete-scan 404 — a deny \ + is a verdict and must not visit, taint, or 503" + ); + } + + /// F3 budget expiry mid-loop: one absolute request budget + /// (`ipfs_request_budget_secs`) bounds the whole admitted scan; per-repo + /// stages may not each draw a fresh timeout past it. Budget 1s, per-iteration + /// acquire timeout 2s; the NEWER row is a Tigris-backed ghost (no local copy, + /// silent local endpoint) whose acquire stalls, the OLDER row is a plain + /// public repo carrying the blob. The ghost's acquire runs clamped to the ~1s + /// remainder and times out; at the next repo the budget gate sees zero + /// remaining, taints "budget", and STOPS the scan, so the blob repo is never + /// visited (a visit would probe the healthy public copy and serve 200, which + /// the 503 assertion rules out) and the shed names the budget. Without the + /// budget the acquire would time out at its own 2s, the scan would continue, + /// and the buried blob would serve 200 (the recorded RED). MUTATION (RED): + /// remove the `request_deadline` capture (or make the remaining budget + /// infinite) and this serves 200 again. + #[sqlx::test] + async fn get_by_cid_request_budget_expiry_stops_scan_with_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Seed the blob repo through a LOCAL-ONLY store first, so seeding never + // consults the (deliberately unreachable) Tigris endpoint. + state.repo_store = + crate::git::repo_store::RepoStore::for_testing(repos_dir.clone(), pool.clone()); + let (_, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6f3budget", + "buried", + b"budget expiry proof\n", + ) + .await; + // Swap in a Tigris-backed store over the SAME repos_dir (the seeded bare + // repo stays a fast local hit) and add a NEWER ghost row with no local + // copy: its acquire consults the silent local endpoint and stalls past + // the budget (endpoint-pinned test client, no AWS_* env reads). + let endpoint = silent_tigris_endpoint().await; + let tigris = + crate::git::tigris::TigrisClient::for_testing_with_endpoint("test-bucket", &endpoint) + .await; + state.repo_store = crate::git::repo_store::RepoStore::new(repos_dir, Some(tigris), pool); + state + .db + .upsert_mirror_repo("z6f3budget", "ghost", "/unused-ghost", None, false) + .await + .unwrap(); + let mut cfg = (*state.config).clone(); + cfg.ipfs_request_budget_secs = 1; + cfg.git_acquire_timeout_secs = 2; + state.config = Arc::new(cfg); + + let peer: SocketAddr = "203.0.113.70:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&cid_for_oid(&oid), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted request budget must stop the scan with a retryable 503; \ + scanning on into the later public blob repo would have served 200" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the budget-truncation 503 must carry Retry-After" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("budget"), + "the truncation body must name the budget taint so the operator can \ + map the shed to GITLAWB_IPFS_REQUEST_BUDGET_SECS; got: {body}" + ); + } + + /// F3 clamped walk at expiry: a walk that starts with little budget left runs + /// its git children under `min(git_service_timeout_secs, remaining)`, so the + /// clamp (not any tokio-level abort) is what ends it and a walk can never + /// complete past the budget. Budget 2s, service timeout at its 600s default, + /// fake walk git that sleeps 8s: the walk STARTS (pid file), the walk permit + /// stays held while the blocking walk runs (`available_permits == 0`), the + /// clamped deadline SIGTERM/SIGKILLs the child group at ~2s remaining (the + /// response lands after the ~1s watchdog grace, far before the 8s sleep, and + /// the recorded pid is already dead: a tokio abort would have left it + /// running), the log shows the walk started but never completed, and the + /// request sheds the terminal budget-truncated 503 without ever reaching the + /// OLDER public copy of the same blob (which would have served 200). After + /// the response the permit is free: the spawn_blocking closure genuinely + /// returned. MUTATION (RED): drop the `min` clamp on `walk_timeout` and the + /// walk runs its full 8s sleep (elapsed and log-completion assertions fail). + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_budget_clamps_walk_deadline_and_holds_permit(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let walk_log = tmp.path().join("walks.log"); + let revlist_pid = tmp.path().join("revlist.pid"); + // Fake git for the WALK only: `rev-list` records its pid and a start + // marker, sleeps far past the budget, then records a done marker. Under + // the clamped walk deadline the whole process group is torn down mid + // sleep, so "done" never appears. The 8s sleep also bounds a RED run. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + for-each-ref) : ;;\n\ + rev-parse) echo deadbeef ;;\n\ + rev-list) echo $$ > \"{pid}\"; echo start >> \"{log}\"; sleep 8; echo done >> \"{log}\" ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + pid = revlist_pid.display(), + log = walk_log.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Global walk pool of 1 so the held permit is observable; per-source cap + // permissive so only the global pool matters. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + let mut cfg = (*state.config).clone(); + // The budget is the ONLY thing that can end this walk early: the service + // timeout stays at its generous 600s default. + cfg.ipfs_request_budget_secs = 2; + state.config = Arc::new(cfg); + + // Older row: a plain public copy of the same blob, which must never be + // reached. Newer row: path-scoped, so its blob costs the clamped walk. + let content = b"budget walk clamp proof\n"; + let (_, oid) = + seed_repo_with_blob(&state, tmp.path(), "z6f3clamp", "pubcopy", content).await; + let (walk_id, _) = + seed_repo_with_blob(&state, tmp.path(), "z6f3clamp", "gated", content).await; + state + .db + .set_visibility_rule( + &walk_id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3IpfsReaderDDDDDDDDDDDDDDDDDDDDDDDD".to_string()], + "z6f3clamp", + ) + .await + .unwrap(); + + let sem = state.git_ipfs_walk_semaphore.clone(); + let router = ipfs_router(state); + let started = std::time::Instant::now(); + let peer: SocketAddr = "203.0.113.71:5000".parse().unwrap(); + let mut fut = Box::pin(router.oneshot(get_cid(&cid_for_oid(&oid), Some(peer)))); + + // Drive until the fake git's rev-list records its pid: the walk is now in + // the blocking pool and the request future is `.await`ing its join. Stop + // polling the instant the future completes (re-polling would panic). + let mut walk_pid: Option = None; + let mut early = None; + for _ in 0..500 { + let done = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&revlist_pid) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + walk_pid = Some(p); + break; + } + if let Ok(resp) = done { + early = Some(resp.map(|r| r.status())); + break; + } + } + let pid = walk_pid.unwrap_or_else(|| { + panic!( + "the budget-clamped walk must have STARTED (nonzero remaining); early: {early:?}" + ) + }); + // Reap the sleeping child on drop so a RED run leaks no orphan. + struct ReapOnDrop(i32); + impl Drop for ReapOnDrop { + fn drop(&mut self) { + unsafe { + libc::kill(self.0, libc::SIGKILL); + } + } + } + let _cleanup = ReapOnDrop(pid); + + // While the blocking walk runs the permit is HELD: the budget never frees + // a slot whose blocking thread is still burning. + assert_eq!( + sem.available_permits(), + 0, + "the walk permit must stay held while the budget-clamped walk runs" + ); + + let resp = tokio::time::timeout(std::time::Duration::from_secs(20), &mut fut) + .await + .expect("the clamped walk deadline must end the request; it never hung") + .unwrap(); + let elapsed = started.elapsed(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a budget-clamped walk that could not finish leaves no verdict: 503, not 404/200" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("budget"), + "the terminal shed must name the budget taint; got: {body}" + ); + // Deadline-killed at ~remaining, not run to completion: the response + // lands at ~budget + the watchdog's kill/reap slack, well before the 8s + // sleep could have finished. + assert!( + elapsed < std::time::Duration::from_secs(7), + "the clamped git deadline must end the walk at ~remaining; got {elapsed:?}" + ); + // The child group is already dead AT response time: the clamp killed it. + // (A tokio-level abort of the walk future would have answered while the + // blocking thread and its child still ran.) + assert_eq!( + unsafe { libc::kill(pid, 0) }, + -1, + "the walk's git child must be reaped by the clamped deadline before the response" + ); + let log = std::fs::read_to_string(&walk_log).unwrap_or_default(); + assert!( + log.contains("start"), + "the walk must have started (the budget gate passed with remaining > 0)" + ); + assert!( + !log.contains("done"), + "the walk must never complete past the budget; the clamp kills it mid-run" + ); + // The spawn_blocking closure returned and the handler finished: the + // permit is free again (held through the blocking run, no longer). + assert_eq!( + sem.available_permits(), + 1, + "the walk permit must free once the blocking walk genuinely returns" + ); + } + + /// #174 F3 hung-probe reap (RED-before/GREEN-after): the `git cat-file -t` + /// probe runs OFF the async worker under the reaped bounded runner, so a hung or + /// corrupt object store cannot pin a runtime worker or the held IPFS permits. + /// `objects/info/alternates` is a FIFO with no writer, so real `git cat-file -t` + /// blocks at odb setup forever. With the probe bounded to + /// `min(git_service_timeout, remaining budget)` (~1s here), the watchdog tears the + /// git process group down at the deadline and the probe returns Err — a taint, not + /// a verdict — so the scan sheds a retryable 503 naming the probe, no walk ever + /// starts, and the whole request returns in bounded time. + /// + /// Load-bearing: with the probe on the bare async worker (pre-fix) this FIFO blocks + /// the handler forever (no feeder frees it) and the request hangs — the wrapping + /// timeout fires (RED). With the reaped bounded probe it returns 503 promptly. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_hung_probe_is_reaped_and_sheds_503(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + let walk_log = tmp.path().join("walks.log"); + state.git_bin = walk_logging_fake_git(tmp.path(), &walk_log); + let mut cfg = (*state.config).clone(); + cfg.ipfs_request_budget_secs = 1; + state.config = Arc::new(cfg); + + let (repo_id, oid) = seed_repo_with_blob( + &state, + tmp.path(), + "z6f3probe", + "gated", + b"probe-then-expire proof\n", + ) + .await; + state + .db + .set_visibility_rule( + &repo_id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3IpfsReaderEEEEEEEEEEEEEEEEEEEEEEEE".to_string()], + "z6f3probe", + ) + .await + .unwrap(); + + // Hang the REAL-git probe indefinitely: `objects/info/alternates` as a FIFO + // with no writer blocks `git cat-file -t` at odb setup forever. There is no + // feeder — the reaped bounded runner must tear the git process group down at + // the deadline; a bare unbounded probe would block the handler here. + let rec = state + .db + .get_repo("z6f3probe", "gated") + .await + .unwrap() + .unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + let fifo = bare.join("objects").join("info").join("alternates"); + let c_path = std::ffi::CString::new(fifo.to_str().unwrap()).unwrap(); + assert_eq!( + unsafe { libc::mkfifo(c_path.as_ptr(), 0o644) }, + 0, + "mkfifo(objects/info/alternates) must succeed" + ); + let peer: SocketAddr = "203.0.113.72:5000".parse().unwrap(); + // The request must return in bounded time: the reaped probe sheds a 503; a + // bare unbounded probe would block on the FIFO forever (no feeder frees it). + let resp = tokio::time::timeout( + std::time::Duration::from_secs(15), + ipfs_router(state).oneshot(get_cid(&cid_for_oid(&oid), Some(peer))), + ) + .await + .expect("the hung probe must be reaped, not block the handler") + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "probed-present with the budget gone must shed the truncation 503: \ + never the walked 404, never a serve" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("probe"), + "the shed must name the reaped-probe taint; got: {body}" + ); + let walks = std::fs::read_to_string(&walk_log) + .map(|s| s.lines().count()) + .unwrap_or(0); + assert_eq!( + walks, 0, + "no walk may START once the budget is exhausted, even for a probed-present object" + ); + } + + /// Shed at capacity: an exhausted `git_ipfs_walk_semaphore` sheds a `/ipfs/{cid}` + /// request with 503 BEFORE any DB/git walk (the acquire is the first thing after CID + /// validation), so a lazy DB-free state suffices — exactly like the served-git shed + /// tests. MUTATION (RED): delete the `git_ipfs_walk_semaphore` acquire in + /// `get_by_cid` and the request no longer sheds here (it falls through to the DB / + /// walk and returns something other than 503). + #[tokio::test] + async fn get_by_cid_sheds_with_503_when_walk_pool_exhausted() { + let mut state = crate::test_support::test_state_lazy(); + // Global /ipfs walk pool exhausted; per-source cap permissive so only the global + // pool can shed. Route rate limit is applied as a layer in production, not here. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(0)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let peer: SocketAddr = "203.0.113.9:5000".parse().unwrap(); + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted /ipfs walk pool must shed the request with 503" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the 503 shed must carry Retry-After" + ); + } + + /// Per-source sub-cap, the `Some(ip)` arm: with per-source = 1 and the source pinned + /// at its single slot, a request from THAT source sheds 503 (global pool has room), + /// while a request from a DIFFERENT source is NOT shed by the cap (it proceeds past + /// admission). Pinning proves the `PeerAddr`/`HeaderMap` extractors resolved the key + /// — an inert `None` key would never shed on the per-source cap. MUTATION (RED): + /// delete the `git_ipfs_walk_per_caller` acquire and the capped source no longer + /// sheds. + #[tokio::test] + async fn get_by_cid_per_source_cap_sheds_same_source_admits_other() { + let mut state = crate::test_support::test_state_lazy(); + // Global pool has room; the per-source cap is 1. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(8)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let capped: SocketAddr = "203.0.113.20:5000".parse().unwrap(); + let other: SocketAddr = "203.0.113.21:5000".parse().unwrap(); + + // Pin the capped source at its single walk slot. + let _slot = state + .git_ipfs_walk_per_caller + .try_acquire(&capped.ip().to_string()) + .expect("first walk slot for the capped source IP"); + + let cid = valid_cid(); + // The capped source sheds on the per-source cap even with global capacity free. + let resp = ipfs_router(state.clone()) + .oneshot(get_cid(&cid, Some(capped))) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a source at its per-source /ipfs walk cap must shed 503 with global capacity free" + ); + + // A DIFFERENT source is NOT shed by the per-source cap: it clears admission and + // proceeds (then errors on the lazy DB, which is not a 503). + let resp = ipfs_router(state) + .oneshot(get_cid(&cid, Some(other))) + .await + .unwrap(); + assert_ne!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a different source must not be shed by the per-source cap" + ); + } + + /// The `None`-key arm: a request with no resolvable source key (no trusted-proxy + /// header, no `ConnectInfo`) is bounded by the GLOBAL pool only, never the per-source + /// sub-cap. With the global pool exhausted it still sheds 503 (the counterpart to the + /// `Some(ip)` arm above, so neither arm is vacuous). + #[tokio::test] + async fn get_by_cid_none_key_arm_sheds_on_global_pool() { + let mut state = crate::test_support::test_state_lazy(); + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(0)); + // Per-source cap permissive so only the global pool can shed. + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + // No ConnectInfo + no trusted header -> client_key resolves None. + let resp = ipfs_router(state) + .oneshot(get_cid(&valid_cid(), None)) + .await + .unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a None-key request must still shed 503 on the exhausted GLOBAL /ipfs walk pool" + ); + } + + /// Map self-bound (INV-15): the `/ipfs` per-source map is a `PerCallerConcurrency` + /// built via `with_default_max_keys`, so a distinct-source-key flood cannot grow it + /// past the cap and a rejected key never allocates (reject-before-insert). Mirrors + /// `per_caller_concurrency_map_is_self_bounding_and_reject_before_insert` for the + /// pool U3 adds. + #[tokio::test] + async fn ipfs_walk_per_caller_map_is_self_bounding_and_reject_before_insert() { + let lim = crate::rate_limit::PerCallerConcurrency::new(4, 3); + // Acquire+drop a flood of distinct keys — the map self-empties (a key is removed + // the instant its in-flight count hits zero). + for i in 0..50 { + let _p = lim.try_acquire(&format!("src{i}")); + } + assert_eq!( + lim.tracked_keys(), + 0, + "an acquire+drop flood of distinct sources leaves the /ipfs map empty" + ); + // Reject-before-insert: hold max_keys distinct sources, then a new one sheds + // without growing the map. + let held: Vec<_> = (0..3) + .map(|i| lim.try_acquire(&format!("h{i}")).unwrap()) + .collect(); + assert_eq!( + lim.tracked_keys(), + 3, + "three distinct sources held concurrently" + ); + assert!( + lim.try_acquire("h3").is_none(), + "a new source key at max_keys is rejected" + ); + assert_eq!( + lim.tracked_keys(), + 3, + "the rejected key did not allocate an entry (reject-before-insert)" + ); + drop(held); + } + + /// Retain-through-blocking (R3, the load-bearing async property): the walk + /// admission is held until the `spawn_blocking` walk actually RETURNS, not when a + /// tokio timeout fires. With the global pool at size 1, drive a request until its + /// walk (a fake git that hangs on `rev-list`) is in flight; the slot must stay held + /// (`available_permits() == 0`) and a replacement from a DIFFERENT source must shed + /// 503 for as long as the blocking walk runs — even though the request future is + /// only `.await`ing the blocking join. When the blocking walk ends the permit frees + /// and a replacement is admitted. The permit lives INSIDE the handler across the + /// blocking `.await`; move it out (drop before the walk) and the replacement would + /// be admitted while the walk still burns a blocking thread (the bug this guards). + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_walk_permit_held_through_blocking_walk(pool: sqlx::PgPool) { + use std::process::Command; + + let tmp = tempfile::TempDir::new().unwrap(); + let revlist_pid = tmp.path().join("revlist.pid"); + // Fake git for the /ipfs WALK only (object_type/read_object_content use the real + // `git`, so the object must genuinely exist below). Empty refs (so + // assert_all_refs_are_commits returns Ok without the peel), `rev-parse` resolves, + // and `rev-list` records its pid then sleeps ~6s so the walk BLOCKS + // deterministically. The sleep bounds the walk so a broken fix cannot wedge the + // suite. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + for-each-ref) : ;;\n\ + rev-parse) echo deadbeef ;;\n\ + rev-list) echo $$ > \"{}\"; sleep 6 ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + revlist_pid.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + // Isolate the global walk pool at size 1; per-source cap permissive so only the + // held global permit can shed the replacement. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let owner = "z6ipfs1"; + let name = "ip1"; + state + .db + .upsert_mirror_repo(owner, name, "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + // The exact bare path the handler's `acquire` resolves. Build a REAL SHA-256 bare + // repo there with a committed blob under `src/`, so real `git cat-file -t ` classifies it as a blob (the CID digest IS the sha256 object id in + // object-format=sha256) and the handler reaches the path-scoped walk branch. + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + let _ = std::fs::remove_dir_all(&bare); + std::fs::create_dir_all(&bare).unwrap(); + let run = |args: &[&str], cwd: &std::path::Path| { + let out = Command::new("git") + .args(args) + .current_dir(cwd) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + }; + let work = tmp.path().join("work"); + std::fs::create_dir_all(work.join("src")).unwrap(); + std::fs::write(work.join("src/secret.txt"), b"ipfs walk retain proof\n").unwrap(); + run( + &["init", "-q", "--object-format=sha256", "-b", "main"], + &work, + ); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + run(&["add", "src/secret.txt"], &work); + run(&["commit", "-q", "-m", "seed"], &work); + run( + &[ + "clone", + "--bare", + "-q", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + tmp.path(), + ); + // The blob's SHA-256 object id (= the CID's digest); build the CID from it. + let oid = { + let out = Command::new("git") + .args(["rev-parse", "HEAD:src/secret.txt"]) + .current_dir(&work) + .output() + .expect("git rev-parse runs"); + assert!(out.status.success(), "rev-parse failed"); + String::from_utf8_lossy(&out.stdout).trim().to_string() + }; + let oid_bytes = gitlawb_core::cid::sha256_hex_to_bytes(&oid).unwrap(); + let cid = gitlawb_core::cid::Cid::from_sha256_bytes(&oid_bytes) + .as_str() + .to_string(); + // Precondition: real git classifies the object as a blob (so the handler reaches + // the walk branch, not an early `continue`). + assert_eq!( + crate::git::store::object_type(&bare, &oid) + .unwrap() + .as_deref(), + Some("blob"), + "the seeded sha256 blob must exist so the handler reaches the walk" + ); + // A path-scoped rule so has_path_scoped_rule() is true (the walk branch) without + // denying the "/" gate on the public repo. + state + .db + .set_visibility_rule( + &rec.id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3IpfsReaderAAAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + + let sem = state.git_ipfs_walk_semaphore.clone(); + assert_eq!( + sem.available_permits(), + 1, + "one walk slot before the request" + ); + + let router = ipfs_router(state); + let make_req = |peer: SocketAddr| { + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req + }; + + let peer: SocketAddr = "203.0.113.81:5000".parse().unwrap(); + let mut fut = Box::pin(router.clone().oneshot(make_req(peer))); + // Drive until the fake git's rev-list records its pid — the walk is now in the + // blocking pool and the request future is `.await`ing its join, holding the walk + // permit. Stop polling the instant the future completes (re-polling a completed + // oneshot panics). + let mut walk_pid: Option = None; + let mut early = None; + for _ in 0..500 { + let done = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&revlist_pid) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + walk_pid = Some(p); + break; + } + if let Ok(resp) = done { + early = Some(resp.map(|r| r.status())); + break; + } + } + let pid = walk_pid + .unwrap_or_else(|| panic!("the fake git rev-list must have spawned; early: {early:?}")); + // Reap the sleeping child on drop so a RED run leaks no orphan. + struct ReapOnDrop(i32); + impl Drop for ReapOnDrop { + fn drop(&mut self) { + unsafe { + libc::kill(self.0, libc::SIGKILL); + } + } + } + let _cleanup = ReapOnDrop(pid); + + // Load-bearing: while the blocking walk runs, the slot is HELD and a replacement + // from a DIFFERENT source sheds 503 — proving the permit is retained across the + // spawn_blocking join, not freed by a tokio timeout. + assert_eq!( + sem.available_permits(), + 0, + "the walk slot must be held while the spawn_blocking walk runs" + ); + let peer2: SocketAddr = "203.0.113.82:5000".parse().unwrap(); + let resp = router.clone().oneshot(make_req(peer2)).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a replacement must shed 503 while the prior request's blocking walk still runs" + ); + + // Drop the in-flight request; the detached blocking walk keeps running (a + // spawn_blocking cannot be cancelled), but on the fix the permit is a handler + // local, so dropping the future releases it once the blocking join is abandoned. + // Either way, kill the sleeping child so the slot frees promptly and poll for + // recovery — the point already proven above is that the slot stayed held for the + // duration of the blocking work. + drop(fut); + unsafe { + libc::kill(pid, libc::SIGKILL); + } + let mut freed = false; + for _ in 0..400 { + if sem.available_permits() == 1 { + freed = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + assert!( + freed, + "once the blocking walk ends the walk permit must free the global slot" + ); + } + + /// Loop bound (cap N) + F2 truncation verdict: one `/ipfs/{cid}` request against a + /// CID present in many path-scoped repos must not serialize an unbounded number of + /// full-history walks — and cutting a candidate WITHOUT a verdict must not report + /// the object absent. With `ipfs_max_repos_walked = 1` and TWO public, path-scoped + /// repos both carrying the blob, the first candidate is walked (empty allowed-set → + /// a deny VERDICT) and the second is cut by the cap (no verdict), so the fake git's + /// `rev-list` runs exactly once and the request sheds a retryable 503 + Retry-After + /// — never the old false 404 (the blob genuinely sits in the second repo). + /// MUTATION (RED): remove the `repos_walked >= cap` skip and both repos are walked + /// (count 2); drop the truncation taint on the skip and the 503 decays to a 404. + #[cfg(unix)] + #[sqlx::test] + async fn get_by_cid_caps_repos_walked_per_request(pool: sqlx::PgPool) { + use std::process::Command; + + let tmp = tempfile::TempDir::new().unwrap(); + let walk_log = tmp.path().join("walks.log"); + // Fake git for the WALK: empty refs, `rev-parse` resolves, and each `rev-list` + // appends one line to a log (so the number of walks == the line count) and exits + // with EMPTY output (the allowed-set is empty, so every repo path-gates to a + // `continue` and the request 404s after walking). object_type uses the REAL git, + // so the seeded blob below must genuinely exist. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + for-each-ref) : ;;\n\ + rev-parse) echo deadbeef ;;\n\ + rev-list) echo walk >> \"{}\" ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + walk_log.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // The bound under test: walk at most one candidate repo per request. + let mut cfg = (*state.config).clone(); + cfg.ipfs_max_repos_walked = 1; + state.config = Arc::new(cfg); + + // Seed TWO public repos, each with the SAME blob (same content -> same sha256 OID + // -> same CID) under a path-scoped rule, so both are walk candidates for one CID. + let run = |args: &[&str], cwd: &std::path::Path| { + let out = Command::new("git") + .args(args) + .current_dir(cwd) + .output() + .expect("git runs"); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + }; + let mut oid = String::new(); + for (i, name) in ["ipa", "ipb"].iter().enumerate() { + let owner = "z6ipfsN"; + state + .db + .upsert_mirror_repo(owner, name, &format!("/unused-{name}"), None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + let bare = state + .repo_store + .acquire(&rec.owner_did, &rec.name) + .await + .unwrap(); + let _ = std::fs::remove_dir_all(&bare); + std::fs::create_dir_all(&bare).unwrap(); + let work = tmp.path().join(format!("work{i}")); + std::fs::create_dir_all(work.join("src")).unwrap(); + // Identical content in both repos -> identical sha256 blob OID -> one CID. + std::fs::write(work.join("src/secret.txt"), b"loop bound proof\n").unwrap(); + run( + &["init", "-q", "--object-format=sha256", "-b", "main"], + &work, + ); + run(&["config", "user.email", "t@t"], &work); + run(&["config", "user.name", "t"], &work); + run(&["add", "src/secret.txt"], &work); + run(&["commit", "-q", "-m", "seed"], &work); + run( + &[ + "clone", + "--bare", + "-q", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + tmp.path(), + ); + if oid.is_empty() { + let out = Command::new("git") + .args(["rev-parse", "HEAD:src/secret.txt"]) + .current_dir(&work) + .output() + .expect("git rev-parse runs"); + oid = String::from_utf8_lossy(&out.stdout).trim().to_string(); + } + state + .db + .set_visibility_rule( + &rec.id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3IpfsReaderBBBBBBBBBBBBBBBBBBBBBBBB".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + } + let oid_bytes = gitlawb_core::cid::sha256_hex_to_bytes(&oid).unwrap(); + let cid = gitlawb_core::cid::Cid::from_sha256_bytes(&oid_bytes) + .as_str() + .to_string(); + + let peer: SocketAddr = "203.0.113.90:5000".parse().unwrap(); + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let resp = ipfs_router(state).oneshot(req).await.unwrap(); + // The first repo's walk yields the empty allowed-set (deny verdict); the second + // repo NEEDS a walk the cap forbids, so the scan is truncated without a verdict + // on it: retryable 503, never a false 404 for the blob it genuinely carries. + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a walk-cap truncation must shed a retryable 503, not report the object absent" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the truncation 503 must carry Retry-After" + ); + + let walks = std::fs::read_to_string(&walk_log) + .map(|s| s.lines().count()) + .unwrap_or(0); + assert_eq!( + walks, 1, + "with the per-request repo-walk cap at 1, only the first candidate repo is \ + walked (the second is cut by the cap), so exactly one walk runs; got {walks}" + ); + } + + /// Route rate limit is WIRED (not a silent no-op): the production `build_router` + /// attaches an `IpRateLimiter` extension to the `/ipfs/{cid}` route, so a per-IP + /// flood is braked with 429. A bare `rate_limit_by_ip` layer with no extension does + /// nothing, so this proves the extension is attached. Drive it through the real + /// router with a tight limiter (1/hr): the second request from the same IP is 429. + /// MUTATION (RED): drop the `axum::Extension(ipfs_limiter)` layer in `server.rs` and + /// the second request is no longer braked (it reaches the handler, 404, not 429). + #[sqlx::test] + async fn ipfs_route_ip_rate_limit_is_attached(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool).await; + // Tight per-IP /ipfs bucket so the second request from one IP trips 429. + state.ipfs_rate_limiter = + crate::rate_limit::RateLimiter::new(1, std::time::Duration::from_secs(3600)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let router = crate::server::build_router(state); + let cid = valid_cid(); + let make = |peer: SocketAddr| { + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/ipfs/{cid}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req + }; + let peer: SocketAddr = "203.0.113.99:5000".parse().unwrap(); + + // First request from this IP passes the brake and reaches the handler (404 — no + // such object anywhere), debiting the single-slot bucket. + let resp = router.clone().oneshot(make(peer)).await.unwrap(); + assert_ne!( + resp.status(), + StatusCode::TOO_MANY_REQUESTS, + "the first /ipfs request from an IP must pass the rate brake" + ); + // Second request from the SAME IP is braked with 429 — proving the limiter + // extension is attached (a bare no-op layer would let it through to 404). + let resp = router.clone().oneshot(make(peer)).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::TOO_MANY_REQUESTS, + "an exhausted per-IP /ipfs bucket must brake with 429 — the IpRateLimiter \ + extension must be attached to the route" + ); + // A DIFFERENT IP still has its own budget (independent bucket). + let other: SocketAddr = "203.0.113.100:5000".parse().unwrap(); + let resp = router.oneshot(make(other)).await.unwrap(); + assert_ne!( + resp.status(), + StatusCode::TOO_MANY_REQUESTS, + "a different IP must not be braked by another IP's exhausted bucket" + ); + } + + /// F6/KTD-5: the two initial metadata queries (`list_all_repos`, + /// `list_visibility_rules_for_repos`) run AFTER the scarce walk permits are + /// acquired (held RAII for the whole request) but BEFORE the per-repo loop's + /// first budget gate. Pre-fix they were bare awaits with no deadline, so a query + /// blocked in Postgres pinned the walk slot for the whole stall, past the request + /// budget. Here we hold an ACCESS EXCLUSIVE lock on `repos` so `list_all_repos` + /// blocks; with the budget clamp the request sheds a retryable budget 503 within + /// ~budget and FREES the walk permit, and a follow-up (lock released) is served. + /// + /// Load-bearing: pre-fix the bare await blocks on the lock until the 10s wrapping + /// timeout fires (RED — "never returned within budget"). After the fix it returns + /// the 503 at ~1s and the permit is free again. MUTATION (RED): drop the + /// `tokio::time::timeout` around `list_all_repos` and this hangs past the wrap. + #[sqlx::test] + async fn get_by_cid_stalled_metadata_query_frees_walk_permit(pool: sqlx::PgPool) { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Global walk pool of 1 so the held/freed permit is directly observable; + // per-source cap permissive so only the global pool matters. + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + let mut cfg = (*state.config).clone(); + cfg.ipfs_request_budget_secs = 1; + state.config = Arc::new(cfg); + + let sem = state.git_ipfs_walk_semaphore.clone(); + let router = ipfs_router(state); + + // Hold an ACCESS EXCLUSIVE lock on `repos` on a dedicated pooled connection: + // `list_all_repos`' SELECT needs ACCESS SHARE, which conflicts, so it blocks + // at lock acquisition regardless of row count. + let mut lock_conn = pool.acquire().await.unwrap(); + sqlx::raw_sql("BEGIN; LOCK TABLE repos IN ACCESS EXCLUSIVE MODE;") + .execute(&mut *lock_conn) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.80:5000".parse().unwrap(); + let started = std::time::Instant::now(); + let resp = tokio::time::timeout( + std::time::Duration::from_secs(10), + router.clone().oneshot(get_cid(&valid_cid(), Some(peer))), + ) + .await + .expect("the budget clamp must return within budget; a bare await hangs on the lock") + .unwrap(); + let elapsed = started.elapsed(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a metadata query blocked past the request budget must shed a retryable 503" + ); + assert!( + elapsed < std::time::Duration::from_secs(3), + "the clamp must end the request at ~budget (1s); got {elapsed:?} \ + (pre-fix the bare await blocks on the lock for the whole stall)" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("budget"), + "the shed must name the budget taint so it maps to \ + GITLAWB_IPFS_REQUEST_BUDGET_SECS; got: {body}" + ); + // The scarce walk permit was RAII-dropped on the early return, not pinned for + // the stall: the slot is free again the instant the request returns. + assert_eq!( + sem.available_permits(), + 1, + "the walk permit must be freed on the budget-shed path, not held for the stall" + ); + + // Release the lock; a follow-up request is now SERVED (404 — empty DB), never + // capacity-503'd, proving the slot was not left pinned. + sqlx::raw_sql("ROLLBACK") + .execute(&mut *lock_conn) + .await + .unwrap(); + drop(lock_conn); + let resp2 = router + .oneshot(get_cid(&valid_cid(), Some(peer))) + .await + .unwrap(); + assert_eq!( + resp2.status(), + StatusCode::NOT_FOUND, + "with the permit freed and the lock released, a follow-up is served (404), \ + not capacity-503'd" + ); + } + + /// F6/KTD-5 FAIL CLOSED (security-critical): `list_visibility_rules_for_repos` is + /// the access-control query. If its timeout let the handler fall through with an + /// empty rule map, the loop would apply no visibility rules and serve an unfiltered + /// listing — exposing a public repo's path-restricted blob. Here a PUBLIC repo + /// carries the blob under a path-scoped rule that denies anon; `visibility_rules` + /// is locked ACCESS EXCLUSIVE so the rule query blocks. The fix returns the budget + /// 503 BEFORE the loop, so the handler NEVER serves (never 200). + /// + /// Load-bearing: pre-fix the bare await blocks on the lock until the 10s wrap fires + /// (RED). After the fix it sheds the 503 at ~1s. The `assert_ne!(200)` is the + /// fail-closed guard: a naive fix that `unwrap_or_default()`s the rules on timeout + /// and falls through would serve the blob 200 and trip it. + #[sqlx::test] + async fn get_by_cid_visibility_rule_timeout_fails_closed(pool: sqlx::PgPool) { + let tmp = tempfile::TempDir::new().unwrap(); + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + let mut state = crate::test_support::test_state(pool.clone()).await; + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool.clone()); + state.git_ipfs_walk_semaphore = Arc::new(Semaphore::new(1)); + state.git_ipfs_walk_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + + // A PUBLIC repo (upsert_mirror_repo sets is_public=true) carrying the blob, + // with a path-scoped rule restricting src/** to a reader that is NOT the anon + // caller. Rules applied => the blob is denied; rules skipped (fall-through) => + // the public repo serves it (the exposure this guard forbids). + let (repo_id, oid) = + seed_repo_with_blob(&state, tmp.path(), "z6f3failclosed", "gated", b"private\n").await; + state + .db + .set_visibility_rule( + &repo_id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3IpfsReaderDDDDDDDDDDDDDDDDDDDDDDDD".to_string()], + "z6f3failclosed", + ) + .await + .unwrap(); + + let mut cfg = (*state.config).clone(); + cfg.ipfs_request_budget_secs = 1; + state.config = Arc::new(cfg); + let sem = state.git_ipfs_walk_semaphore.clone(); + let router = ipfs_router(state); + + // Lock `visibility_rules` ACCESS EXCLUSIVE: list_all_repos (on `repos`) still + // succeeds, but list_visibility_rules_for_repos blocks on the rule query. + let mut lock_conn = pool.acquire().await.unwrap(); + sqlx::raw_sql("BEGIN; LOCK TABLE visibility_rules IN ACCESS EXCLUSIVE MODE;") + .execute(&mut *lock_conn) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.81:5000".parse().unwrap(); + let started = std::time::Instant::now(); + let resp = tokio::time::timeout( + std::time::Duration::from_secs(10), + router.oneshot(get_cid(&cid_for_oid(&oid), Some(peer))), + ) + .await + .expect("the budget clamp must return within budget; a bare await hangs on the lock") + .unwrap(); + let elapsed = started.elapsed(); + + let status = resp.status(); + // Fail closed: the handler must NEVER emit the listing with no rules applied. + assert_ne!( + status, + StatusCode::OK, + "a visibility-rule query timeout must DENY, never serve the path-restricted \ + blob from the public repo (that would expose private content)" + ); + assert_eq!( + status, + StatusCode::SERVICE_UNAVAILABLE, + "a visibility-rule query blocked past the budget must shed the retryable budget 503" + ); + assert!( + elapsed < std::time::Duration::from_secs(3), + "the clamp must end the request at ~budget (1s); got {elapsed:?}" + ); + let body = axum::body::to_bytes(resp.into_body(), 1 << 20) + .await + .unwrap(); + let body = String::from_utf8_lossy(&body); + assert!( + body.contains("budget"), + "the fail-closed shed must name the budget taint; got: {body}" + ); + assert_eq!( + sem.available_permits(), + 1, + "the walk permit must be freed on the fail-closed budget-shed path" + ); + + sqlx::raw_sql("ROLLBACK") + .execute(&mut *lock_conn) + .await + .unwrap(); + drop(lock_conn); + } +} diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index b9cbc352..f258a4bc 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -39,11 +39,21 @@ const ZERO_SHA: &str = "0000000000000000000000000000000000000000"; /// `withheld` is `None`, so an unvetted push neither replicates blobs nor /// announces. Returning both keeps the gate's announce decision a single /// source rather than recomputing it at each call site. +/// +/// The walk arm runs under a `git_encrypt_semaphore` admission permit (#174 F4): +/// by the time the receive-pack tail calls this, the handler's write permit has +/// already been released (receive_pack's AdmissionGuard drops when the git group +/// is reaped), so without the gate a burst of completed pushes accumulates +/// unbounded concurrent full-history walks. `encrypt_sem` is threaded in so the +/// no-walk fast paths (not announceable; no path-scoped rule) never touch it. async fn replication_withheld_set( + encrypt_sem: std::sync::Arc, rules: Option>, owner_did: &str, is_public: bool, disk_path: std::path::PathBuf, + git_bin: String, + timeout: std::time::Duration, ) -> (bool, Option>) { let announce = match &rules { Some(rules) => crate::visibility::listable_at_root(rules, is_public, owner_did, None), @@ -64,9 +74,17 @@ async fn replication_withheld_set( // that off the async worker thread. Some(rules) => { let owner_did = owner_did.to_string(); + // Scan admission (#174 F4): DEFER, never shed — dropping the walk + // would skip the vetting and fail the push's replication closed for + // no reason. Residuals at `acquire_scan_permit`. + let permit = + crate::state::acquire_scan_permit(encrypt_sem, &disk_path, "withheld walk").await; tokio::task::spawn_blocking(move || { - crate::git::visibility_pack::withheld_blob_oids( - &disk_path, &rules, is_public, &owner_did, None, + // The permit lives inside the blocking closure: a started walk + // always completes holding it. + let _permit = permit; + crate::git::visibility_pack::withheld_blob_oids_bounded( + &disk_path, &git_bin, timeout, &rules, is_public, &owner_did, None, ) }) .await @@ -100,18 +118,46 @@ async fn replication_withheld_set( /// non-blobs plus allowed blobs. Any error in either walk (or a task panic) /// pins nothing this push, mirroring the degraded-path shape of /// `replication_withheld_set`. +/// +/// Always walks (there is no no-git arm), so the whole blocking scan runs under +/// one `git_encrypt_semaphore` admission permit (#174 F4) — see +/// `acquire_scan_permit` for the defer rationale and the honest residuals. +#[allow(clippy::too_many_arguments)] async fn fail_closed_full_scan_objects( + encrypt_sem: std::sync::Arc, disk_path: std::path::PathBuf, rules: Vec, is_public: bool, owner_did: String, candidates: Vec, + git_bin: String, + timeout: std::time::Duration, ) -> Vec { + // Scan admission (#174 F4): DEFER, never shed; the permit moves into the + // closure so a started scan always completes holding it. + let permit = + crate::state::acquire_scan_permit(encrypt_sem, &disk_path, "fail-closed full scan").await; tokio::task::spawn_blocking(move || -> anyhow::Result> { - let allowed = crate::git::visibility_pack::replicable_blob_set( - &disk_path, &rules, is_public, &owner_did, + let _permit = permit; + // One whole-scan deadline shared across both phases (#174 F4). A fresh + // `Instant::now() + timeout` for phase 2 let a large-but-successful phase 1 plus + // a full phase 2 hold the scan permit ~2x the configured budget. Sharing the + // deadline caps total occupancy at ~1x: phase 1 runs against the remaining + // budget, and if it consumes the budget phase 2 gets what is left and fails + // closed (pins nothing) rather than over-holding — the safe direction. The cost + // is honest: a genuinely large repo whose phase 1 nears the budget under-pins + // this push rather than the previous silent ~2x hold; size the budget so both + // phases normally fit. + let deadline = std::time::Instant::now() + timeout; + let allowed = crate::git::visibility_pack::replicable_blob_set_bounded( + &disk_path, + &git_bin, + deadline.saturating_duration_since(std::time::Instant::now()), + &rules, + is_public, + &owner_did, )?; - let all_blobs = crate::git::push_delta::all_blob_oids(&disk_path)?; + let all_blobs = crate::git::push_delta::all_blob_oids(&disk_path, &git_bin, deadline)?; Ok(crate::git::visibility_pack::replicable_objects_fail_closed( candidates, &allowed, &all_blobs, )) @@ -508,6 +554,43 @@ pub async fn git_info_refs( auth: Option>, ) -> Result { let name = smart_http_repo_name(&repo)?; + let service = query + .service + .ok_or_else(|| AppError::BadRequest("missing ?service= parameter".into()))?; + // Reject an unsupported service BEFORE taking a read slot or doing any DB/Tigris + // work (#174 P2-1). git_info_refs otherwise treats everything that is not + // git-receive-pack as a read op, so an unauthenticated `?service=anything` to a + // public repo would consume a read permit and the visibility/Tigris work before + // validate_service rejected it downstream in smart_http. + if service != "git-upload-pack" && service != "git-receive-pack" { + return Err(AppError::BadRequest(format!( + "unsupported git service: {service}" + ))); + } + // #62 cheap pre-DB load shed: if the pool this service draws from is already + // saturated, shed with 503 before any DB/disk work. Best-effort (holds no + // permit); the authoritative hold is `git_permit` below, after the per-source + // cap. Restores the shed-before-DB property the reordered held acquire alone + // would drop, while the reorder still prevents one source from occupying global + // slots during the DB/visibility window. + { + // The receive-pack advertisement peeks its DEDICATED advert pool, not the + // write pool the authenticated POST uses (#174) — matching the held acquire + // below, so the pre-DB shed and the authoritative hold agree on the pool. + let pool = if service == "git-receive-pack" { + &state.git_push_advert_semaphore + } else { + &state.git_read_semaphore + }; + if pool.available_permits() == 0 { + tracing::warn!( + "served-git concurrency cap reached; shedding request with 503 (pre-DB)" + ); + return Err(AppError::Overloaded( + "git service at capacity, retry shortly".into(), + )); + } + } tracing::info!(owner = %owner, repo = %name, "info/refs request"); let record = state .db @@ -521,9 +604,6 @@ pub async fn git_info_refs( return Err(AppError::RepoNotFound(format!("{owner}/{name}"))); } - let service = query - .service - .ok_or_else(|| AppError::BadRequest("missing ?service= parameter".into()))?; tracing::debug!(service = %service, repo = %name, "info/refs service"); // Enforce read visibility on the ref advertisement, for BOTH services. The @@ -563,32 +643,557 @@ pub async fn git_info_refs( } } - // For receive-pack (push), download the latest from Tigris so the client - // sees the same refs that acquire_write() will operate on. - let disk_path = if service == "git-receive-pack" { - state - .repo_store - .acquire_fresh(&record.owner_did, &record.name) - .await + // Per-source concurrency sub-cap (#174), keyed on the resolved source IP and + // acquired AFTER the visibility + push-rate gates (KTD7) so a denied or + // rate-limited request never consumes a slot; held for the whole op. The + // upload-pack advertisement is bounded on the read pool (git_read_per_caller). + // The receive-pack advertisement draws from the write pool, so it is bounded per + // source (git_push_advert_per_caller) instead: without this, an anonymous + // multi-source flood of push-handshake advertisements could hold the write pool's + // slots across acquire_fresh and shed authenticated pushes, since the per-IP push + // rate limiter caps rate, not concurrency (#174 review fix). + let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); + let _caller_permit = if service == "git-receive-pack" { + acquire_read_caller_permit( + &state.git_push_advert_per_caller, + caller_key.as_deref(), + name, + "receive-pack advert", + )? } else { - state - .repo_store - .acquire(&record.owner_did, &record.name) - .await - } - .map_err(|e| { - tracing::error!(repo = %name, service = %service, err = %e, "repo acquire failed"); - AppError::Git(e.to_string()) - })?; + acquire_read_caller_permit( + &state.git_read_per_caller, + caller_key.as_deref(), + name, + "info/refs", + )? + }; - smart_http::info_refs(&disk_path, &service) + // Shed with a 503 before spawning git when the concurrency cap is saturated; + // held for the whole op (incl. the smart_http call), released on return. Taken + // AFTER the per-source cap above so one source cannot occupy global slots it + // would be sub-cap-denied for during the DB/visibility window and starve other + // sources; still before acquire_fresh/git so it bounds the fresh Tigris acquire + // and git exec (INV-10). The receive-pack advertisement is phase one of a push, + // but it is ANON-reachable, so it draws from the dedicated advert pool + // (`git_push_advert_semaphore`), NOT the write pool the authenticated POST uses: + // an advert flood can at worst exhaust the advert pool, never a permit a push + // POST needs at admission (#174 U2). A clone flood on the read pool likewise + // can't touch either. The upload-pack advertisement stays on the read pool with + // its per-caller sub-cap. + let _permit = if service == "git-receive-pack" { + git_permit(&state.git_push_advert_semaphore)? + } else { + git_permit(&state.git_read_semaphore)? + }; + + // For receive-pack (push), download the latest from Tigris so the client + // sees the same refs that acquire_write() will operate on. + // + // Bound the acquire under `git_acquire_timeout_secs`: the concurrency permit is + // already held above, and `git_service_timeout_secs` only starts once git spawns, + // so an un-deadlined acquire (a hung Tigris HEAD/GET here) pins the permit until + // the pool drains (#174 P1-2). On expiry the handler-local `_permit`/`_caller_permit` + // drop on the early return (the AdmissionGuard is not built until after acquire), + // so the shed frees the slot; return a bounded 503. + let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); + let acquire_fut = async { + if service == "git-receive-pack" { + state + .repo_store + .acquire_fresh(&record.owner_did, &record.name) + .await + } else { + state + .repo_store + .acquire(&record.owner_did, &record.name) + .await + } + }; + let disk_path = tokio::time::timeout(acquire_deadline, acquire_fut) .await + .map_err(|_elapsed| { + tracing::warn!(repo = %name, service = %service, "repo acquire timed out; shedding with 503"); + AppError::Overloaded("git service acquisition timed out, retry shortly".into()) + })? .map_err(|e| { - tracing::error!(repo = %name, service = %service, err = %e, "info_refs git failed"); + tracing::error!(repo = %name, service = %service, err = %e, "repo acquire failed"); AppError::Git(e.to_string()) + })?; + + // Move the admission permits into the guard so they release only after the spawned + // git process group is confirmed reaped, on complete/timeout/disconnect — not the + // instant a disconnect drops this future while the detached reaper is still tearing + // the group down (#174 P1-a). The handler keeps no copy: `_permit`/`_caller_permit` + // are moved in, so admission tracks the real process lifetime. + let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); + let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); + smart_http::info_refs("git", &service, &disk_path, git_timeout, Some(admission)) + .await + .map_err(|e| { + let app = git_service_app_error(&e); + match &app { + AppError::Timeout(_) => { + tracing::warn!(repo = %name, service = %service, "info/refs advertisement timed out") + } + _ => { + tracing::error!(repo = %name, service = %service, err = %e, "info_refs git failed") + } + } + app }) } +/// Acquire a permit from the served-git concurrency semaphore, or shed the +/// request with a 503 + Retry-After when every slot is in use. Bind the returned +/// permit to a named local so it is held for the whole git op (it releases on +/// drop); a bare `_` would release it immediately. +fn git_permit( + sem: &std::sync::Arc, +) -> Result { + sem.clone().try_acquire_owned().map_err(|_| { + // Surface the shed so operators can see the cap engaging, mirroring the + // receive-pack rate-limit warn above. A silent 503 makes a saturated or + // misconfigured cap look like a client problem instead of a capacity one. + tracing::warn!("served-git concurrency cap reached; shedding request with 503"); + AppError::Overloaded("git service at capacity, retry shortly".into()) + }) +} + +/// Resolve the per-caller key for the read sub-cap (#174): always the resolved +/// source IP (`client_key`), never the signed DID. Public read routes accept any +/// valid `did:key` via `optional_signature` with no admission step, so keying on +/// the DID would let one host mint disposable DIDs to multiply its per-source +/// budget; the push path already throttles on the resolved source IP for exactly +/// this DID-farm reason (`rate_limit.rs`, `IpRateLimiter`). `None` when no key +/// resolves (no trusted header and no peer): such a request is bounded by the +/// global read pool only, never a 500. The per-source-IP key is only as granular +/// as `trust`; see the `max_concurrent_reads_per_caller` config doc. +fn read_caller_key( + headers: &axum::http::HeaderMap, + peer: Option, + trust: crate::rate_limit::TrustedProxy, +) -> Option { + crate::rate_limit::client_key(headers, peer, trust) +} + +/// Acquire the per-caller read sub-cap permit (#174), or shed with a 503. `key` is +/// `None` when no caller key resolves — that request is bounded by the global read +/// pool only and is never shed here (returns `Ok(None)`). `handler` labels the shed +/// log line. Shared by both read handlers so the two acquire sites cannot drift. +fn acquire_read_caller_permit( + limiter: &crate::rate_limit::PerCallerConcurrency, + key: Option<&str>, + repo: &str, + handler: &str, +) -> Result> { + match key { + Some(k) => match limiter.try_acquire(k) { + Some(p) => Ok(Some(p)), + None => { + tracing::warn!(repo = %repo, caller = %k, handler, "per-caller cap reached; shedding with 503"); + Err(AppError::Overloaded( + "git service at capacity for this caller, retry shortly".into(), + )) + } + }, + None => Ok(None), + } +} + +/// Acquire an encryption-walk admission permit, then run the bounded withheld-blob +/// recipients walk. Blocks (defers) when `git_encrypt_semaphore` is full rather than +/// shedding — the walk is background so added latency is fine, and dropping it would +/// lose the withheld-blob recovery copy (#174 P1-e). Bounds the number of concurrent +/// post-push encryption walks so N fast completed pushes cannot spawn N concurrent +/// full-history git walks. Mirrors the original `spawn_blocking(...).await` return +/// shape so the caller's `Ok(Ok(recipients))` match is unchanged. +async fn withheld_recipients_gated( + encrypt_sem: std::sync::Arc, + repo_path: std::path::PathBuf, + git_bin: String, + timeout: std::time::Duration, + rules: Vec, + is_public: bool, + owner_did: String, +) -> std::result::Result< + anyhow::Result>>, + tokio::task::JoinError, +> { + let _permit = encrypt_sem + .acquire_owned() + .await + .expect("git_encrypt_semaphore is never closed"); + tokio::task::spawn_blocking(move || { + crate::git::visibility_pack::withheld_blob_recipients_bounded( + &repo_path, &git_bin, timeout, &rules, is_public, &owner_did, + ) + }) + .await +} + +/// Everything the detached post-push pin/encrypt task needs, cloned once at spawn +/// and shared by the snapshot iteration and every coalesced-drain iteration +/// (#174 F5). +struct EncryptTaskCtx { + ipfs_api: String, + repo_path: std::path::PathBuf, + db: Arc, + repo_id: String, + owner_did: String, + repo_name: String, + irys_url: String, + http_client: Arc, + node_did: String, + node_keypair: Arc, + git_bin: String, + git_timeout: std::time::Duration, + encrypt_sem: Arc, + pin_sem: Arc, +} + +/// The detached post-push pin/encrypt task (#174 P2-2 + F5): run this push's own +/// pre-resolved snapshot through the pipeline, then loop-drain every push that +/// coalesced against the in-flight key until a finish attempt finds nothing +/// pending (which releases the key in the same critical section). +/// +/// The loop holds NO encrypt-pool permit at the task level: each helper it calls +/// (`replication_withheld_set`, `resolve_candidates_for_push`, +/// `fail_closed_full_scan_objects`, `withheld_recipients_gated`) acquires and +/// releases its own walk permit, so the drain makes progress at pool size 1 — a +/// task-level permit would nest over those same-semaphore acquires and deadlock. +async fn run_encrypt_pin_task( + ctx: EncryptTaskCtx, + guard: crate::state::EncryptInflightGuard, + snapshot_objects: Vec, + snapshot_rules: Option>, + snapshot_is_public: bool, +) { + pin_and_encrypt_objects(&ctx, snapshot_objects, snapshot_rules, snapshot_is_public).await; + let mut guard = guard; + loop { + match guard.finish_or_take_pending() { + crate::state::FinishOutcome::Finished(_) => break, + crate::state::FinishOutcome::Pending(g, pending) => { + guard = g; + if let Some((object_list, rules, is_public)) = + resolve_drain_object_list(&ctx, pending).await + { + pin_and_encrypt_objects(&ctx, object_list, rules, is_public).await; + } + } + } + } +} + +/// Resolve a coalesced-drain iteration's replicable object list. Re-fetches the +/// repo record and visibility rules FRESH — rules tightened between the coalesced +/// push and its drain must be honored, fail closed: a newly-withheld blob is not +/// pinned, and a repo that is no longer announceable (or whose record cannot be +/// re-read) pins nothing at all (`None`). Returns the filtered object list plus +/// the fresh rules/is_public snapshot for the encrypt stage — the same +/// resolution → withheld-filter pipeline the receive-pack tail runs. +async fn resolve_drain_object_list( + ctx: &EncryptTaskCtx, + pending: crate::state::PendingWork, +) -> Option<(Vec, Option>, bool)> { + let record = match ctx.db.get_repo(&ctx.owner_did, &ctx.repo_name).await { + Ok(Some(r)) => r, + Ok(None) => { + tracing::warn!( + repo = %ctx.repo_id, + "coalesced drain: repo record is gone; dropping the pending work" + ); + return None; + } + Err(e) => { + tracing::warn!( + repo = %ctx.repo_id, + err = %e, + "coalesced drain: repo re-fetch failed; pinning nothing (fail closed)" + ); + return None; + } + }; + // record.id, never the spawn-time ctx.repo_id: the record above is re-fetched + // fresh by owner/name, and a delete+re-create between spawn and drain gives + // the row a NEW id — rules read against the stale id come back empty and + // would fail open for the new row. + let rules_opt = ctx.db.list_visibility_rules(&record.id).await.ok(); + let (_announce, withheld) = replication_withheld_set( + ctx.encrypt_sem.clone(), + rules_opt.clone(), + &record.owner_did, + record.is_public, + ctx.repo_path.clone(), + ctx.git_bin.clone(), + ctx.git_timeout, + ) + .await; + let withheld_set = match withheld { + Some(w) => w, + None => { + tracing::info!( + repo = %ctx.repo_id, + "coalesced drain: repo is not announceable under current rules; \ + pinning nothing (fail closed)" + ); + return None; + } + }; + let (new_tips, old_tips, force_full_scan) = match pending { + crate::state::PendingWork::Tips(pairs) => { + let new_tips: Vec = pairs + .iter() + .map(|(_, n)| n.clone()) + .filter(|s| s != ZERO_SHA) + .collect(); + let old_tips: Vec = pairs + .into_iter() + .map(|(o, _)| o) + .filter(|s| s != ZERO_SHA) + .collect(); + (new_tips, old_tips, false) + } + // The overflow marker forces the full scan via the explicit flag. It must + // never be encoded as a plain empty-tips call: empty tips resolve to an + // empty delta and would pin nothing (the F5 silent loss again). + crate::state::PendingWork::FullScan => (Vec::new(), Vec::new(), true), + }; + let pin_set = crate::git::push_delta::resolve_candidates_for_push( + ctx.encrypt_sem.clone(), + ctx.repo_path.clone(), + new_tips, + old_tips, + ctx.git_bin.clone(), + ctx.git_timeout, + force_full_scan, + ) + .await; + let object_list = if pin_set.full_scan { + fail_closed_full_scan_objects( + ctx.encrypt_sem.clone(), + ctx.repo_path.clone(), + rules_opt.clone().unwrap_or_default(), + record.is_public, + record.owner_did.clone(), + pin_set.candidates, + ctx.git_bin.clone(), + ctx.git_timeout, + ) + .await + } else { + crate::git::visibility_pack::replicable_objects(pin_set.candidates, &withheld_set) + }; + Some((object_list, rules_opt, record.is_public)) +} + +/// Re-derive the Pinata replication object set for a push from its ref-update +/// tuples (#174 F2 / KTD-3). +/// +/// The detached Pinata task used to MOVE the push's full pre-resolved object list +/// into its closure and hold it across the `pin_semaphore` await. Under a slow +/// Pinata backend every later push then parked a fresh task each still retaining +/// an MB-scale OID list, so outstanding memory grew O(pushes x object-list) — +/// unbounded. The task now captures only the small `(ref, old, new)` tuples and +/// calls this once a pin slot frees, re-deriving the SAME OID set via +/// `git rev-list` (the delta scan) filtered by the current withheld set. Retained +/// memory is O(ref tuples); the object list is materialized only inside the +/// pin-bounded section, so at most `pin_semaphore` permits' worth exist at once. +/// +/// Coalescing / shedding were rejected because the task's per-ref work is +/// non-idempotent (branch->CID upsert, gossip, GraphQL broadcast, Arweave anchor, +/// peer-notify); dropping a later push's task drops its announcements. Only the +/// retained object list is dropped here, not the task, so every push's effects +/// still fire exactly once. +/// +/// Exactly like `resolve_drain_object_list`: the withheld and candidate sets are +/// recomputed from the rules snapshot captured at post-receive tail start (NOT a +/// fresh read at pin-worker time), so a rule tightened up to that point is honored +/// and the filter always fails closed (a newly-withheld blob is not pinned; a +/// no-longer-announceable repo pins nothing). A tightening AFTER tail-start — before +/// a slow re-derivation runs — is not reflected, matching the old retained-list +/// behavior; the reconciliation sweep is the backstop. Every git child runs +/// through the same INV-22 bounded, process-group-reaped helpers the sibling +/// post-receive scans use (`replication_withheld_set`, +/// `resolve_candidates_for_push`, `fail_closed_full_scan_objects`). +#[allow(clippy::too_many_arguments)] +async fn pinata_object_list_for_refs( + encrypt_sem: Arc, + disk_path: std::path::PathBuf, + ref_updates: &[(String, String, String)], + rules_opt: Option>, + is_public: bool, + owner_did: String, + git_bin: String, + timeout: std::time::Duration, +) -> Vec { + let (_announce, withheld) = replication_withheld_set( + encrypt_sem.clone(), + rules_opt.clone(), + &owner_did, + is_public, + disk_path.clone(), + git_bin.clone(), + timeout, + ) + .await; + // Not announceable, or the withheld walk failed: replicate nothing (fail + // closed) — mirrors the receive-pack tail's `withheld == None` handling. + let withheld_set = match withheld { + Some(w) => w, + None => return Vec::new(), + }; + let new_tips: Vec = ref_updates + .iter() + .map(|(_, _, new)| new.clone()) + .filter(|s| s != ZERO_SHA) + .collect(); + let old_tips: Vec = ref_updates + .iter() + .map(|(_, old, _)| old.clone()) + .filter(|s| s != ZERO_SHA) + .collect(); + let pin_set = crate::git::push_delta::resolve_candidates_for_push( + encrypt_sem.clone(), + disk_path.clone(), + new_tips, + old_tips, + git_bin.clone(), + timeout, + false, + ) + .await; + if pin_set.full_scan { + fail_closed_full_scan_objects( + encrypt_sem, + disk_path, + rules_opt.unwrap_or_default(), + is_public, + owner_did, + pin_set.candidates, + git_bin, + timeout, + ) + .await + } else { + crate::git::visibility_pack::replicable_objects(pin_set.candidates, &withheld_set) + } +} + +/// The pin/encrypt pipeline shared by the snapshot iteration and the +/// coalesced-drain iterations: local IPFS pin, then (path-scoped rules only) the +/// admission-gated recipients walk → encrypt-then-pin → Arweave manifest anchor. +/// Pin `object_list` to the local IPFS node under the global pin-admission permit +/// (#174 F6). `EncryptInflight` bounds the pin-task COUNT to one per repo, but each +/// pin loop holds a full per-push object-id list while walking it; this permit bounds +/// how many such MB-scale lists are held at once across all repos. DEFERS (waits) when +/// the pool is full — never drops, since a dropped pin loses the replication copy. +async fn pin_new_objects_gated( + pin_sem: &Arc, + ipfs_api: &str, + repo_path: &std::path::Path, + object_list: Vec, + db: &Arc, +) -> Vec<(String, String)> { + let _permit = pin_sem + .clone() + .acquire_owned() + .await + .expect("pin_semaphore is never closed"); + crate::ipfs_pin::pin_new_objects(ipfs_api, repo_path, object_list, db).await +} + +async fn pin_and_encrypt_objects( + ctx: &EncryptTaskCtx, + object_list: Vec, + rules: Option>, + is_public: bool, +) { + let pinned = pin_new_objects_gated( + &ctx.pin_sem, + &ctx.ipfs_api, + &ctx.repo_path, + object_list, + &ctx.db, + ) + .await; + if !pinned.is_empty() { + tracing::info!(count = pinned.len(), "pinned git objects to IPFS"); + for (sha, cid) in &pinned { + tracing::info!(sha = %sha, %cid, "pinned"); + } + } + + // Option B1: encrypt-then-pin the withheld blobs so authorized readers can + // recover them when the origin cannot serve them. No path-scoped rule can + // withhold a blob, so withheld_blob_recipients would return an empty map + // after a full per-ref walk; skip it. Mirrors the has_path_scoped_rule gate + // on the other two withheld-walk sites. + if let Some(rules) = rules.filter(|r| visibility_pack::has_path_scoped_rule(r)) { + // Bound the number of concurrent post-push encryption walks (#174 P1-e): + // acquire an admission permit before the full-history walk, deferring + // when the pool is full rather than shedding the recovery pin. + let recip = withheld_recipients_gated( + ctx.encrypt_sem.clone(), + ctx.repo_path.clone(), + ctx.git_bin.clone(), + ctx.git_timeout, + rules, + is_public, + ctx.owner_did.clone(), + ) + .await; + if let Ok(Ok(recipients)) = recip { + let node_seed = ctx.node_keypair.to_seed(); + let delta = crate::encrypted_pin::encrypt_and_pin( + &ctx.ipfs_api, + &ctx.repo_path, + &ctx.db, + &ctx.repo_id, + &node_seed, + &recipients, + ) + .await; + + // Option B3: anchor a per-push manifest of the blobs sealed this + // push to Arweave, so the oid->cid index survives total node loss. + // Best-effort; never fails the push. + if !delta.is_empty() && !ctx.irys_url.is_empty() { + let owner_short = crate::db::normalize_owner_key(&ctx.owner_did); + let repo_slug = format!("{owner_short}/{}", ctx.repo_name); + let ts = chrono::Utc::now().to_rfc3339(); + let manifest = crate::arweave::EncryptedManifest { + repo: &repo_slug, + owner_did: &ctx.owner_did, + node_did: &ctx.node_did, + timestamp: &ts, + blobs: &delta, + }; + match crate::arweave::anchor_encrypted_manifest( + &ctx.http_client, + &ctx.irys_url, + &manifest, + ) + .await + { + Ok(tx) if !tx.is_empty() => tracing::info!( + repo = %repo_slug, + tx_id = %tx, + "anchored encrypted manifest to Arweave" + ), + Ok(_) => {} + Err(e) => tracing::warn!( + repo = %repo_slug, + err = %e, + "encrypted manifest anchor failed" + ), + } + } + } + } +} + /// Map an error from a `smart_http` git service call to the right `AppError`: /// [`smart_http::GitServiceTimeout`] to 504, a malformed client request to 400, /// anything else to a 500 git error. Pure (no logging) so it is unit-testable; @@ -614,8 +1219,19 @@ pub async fn git_upload_pack( State(state): State, Path((owner, repo)): Path<(String, String)>, auth: Option>, + crate::rate_limit::PeerAddr(peer): crate::rate_limit::PeerAddr, + headers: axum::http::HeaderMap, body: Bytes, ) -> Result { + // #62 cheap pre-DB load shed (see git_info_refs): shed before DB when the read + // pool is saturated; the authoritative hold is `git_permit` below, after the + // per-source cap. + if state.git_read_semaphore.available_permits() == 0 { + tracing::warn!("served-git concurrency cap reached; shedding request with 503 (pre-DB)"); + return Err(AppError::Overloaded( + "git service at capacity, retry shortly".into(), + )); + } let name = smart_http_repo_name(&repo)?; let record = state .db @@ -636,11 +1252,41 @@ pub async fn git_upload_pack( return Err(AppError::RepoNotFound(format!("{owner}/{name}"))); } - let disk_path = state - .repo_store - .acquire(&record.owner_did, &record.name) - .await - .map_err(|e| AppError::Git(e.to_string()))?; + // Per-caller read sub-cap (#174): after the visibility gate (KTD7) so a + // visibility-denied caller never consumes a scarce read slot. Keyed on the + // resolved source IP (never the signed DID, #174 U1); no resolvable key -> + // global read pool only. + let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); + let _caller_permit = acquire_read_caller_permit( + &state.git_read_per_caller, + caller_key.as_deref(), + name, + "upload-pack", + )?; + + // Shed with a 503 before spawning git when the concurrency cap is saturated; + // held for the whole op (incl. the smart_http call), released on return. Taken + // AFTER the per-source cap above so one source cannot occupy global slots it + // would be sub-cap-denied for during the DB/visibility window and starve other + // sources; still before acquire/git so it bounds the Tigris acquire and git + // exec (INV-10). + let _permit = git_permit(&state.git_read_semaphore)?; + + // Bound the acquire under `git_acquire_timeout_secs` so a hung Tigris HEAD/GET + // cannot pin the read permit indefinitely (#174 P1-2). The permit is a handler + // local here (moved into the AdmissionGuard only below, once git is spawned), so + // the early return on timeout drops it and frees the slot; shed a bounded 503. + let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); + let disk_path = tokio::time::timeout( + acquire_deadline, + state.repo_store.acquire(&record.owner_did, &record.name), + ) + .await + .map_err(|_elapsed| { + tracing::warn!(repo = %name, "repo acquire timed out; shedding with 503"); + AppError::Overloaded("git service acquisition timed out, retry shortly".into()) + })? + .map_err(|e| AppError::Git(e.to_string()))?; let body_len = body.len(); // No path-scoped rule can withhold an individual blob, and the whole-repo @@ -648,35 +1294,72 @@ pub async fn git_upload_pack( // withheld walk and serve the pack directly. let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); let resp = if !visibility_pack::has_path_scoped_rule(&rules) { - smart_http::upload_pack(&disk_path, body, git_timeout).await + // Plain (non-path-scoped) serve: move both admission permits into the guard so + // they release only after the spawned git group is reaped, on + // complete/timeout/disconnect — not the instant a disconnect drops this future + // (#174 P1-a). The handler keeps no copy. + let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); + smart_http::upload_pack(&state.git_bin, &disk_path, body, git_timeout, Some(admission)).await } else { - // withheld_blob_oids walks every ref with blocking `git ls-tree`; keep - // that off the async worker thread. - let withheld = { + // withheld_blob_oids walks every ref with blocking `git ls-tree`; keep that + // off the async worker thread. Move BOTH admission permits INTO the blocking + // task so they are held for the walk's real duration: spawn_blocking cannot be + // cancelled, so on a client disconnect the handler future drops but the walk + // keeps running — and now so do its permits, released only when the walk + // finishes rather than the instant the future drops (#174 P1-b). On success the + // task hands the permits back so the serve phase below keeps them; on a + // dropped future the returned tuple (with the permits) is discarded only when + // the blocking task completes, so admission tracks the real git work. + let (withheld, _permit, _caller_permit) = { let path = disk_path.clone(); let rules = rules.clone(); let owner_did = record.owner_did.clone(); let caller_owned = caller.map(str::to_string); let is_public = record.is_public; + let git_bin = state.git_bin.clone(); tokio::task::spawn_blocking(move || { - visibility_pack::withheld_blob_oids( + let withheld = visibility_pack::withheld_blob_oids_bounded( &path, + &git_bin, + git_timeout, &rules, is_public, &owner_did, caller_owned.as_deref(), - ) + ); + (withheld, _permit, _caller_permit) }) .await .map_err(|e| AppError::Git(e.to_string()))? - .map_err(|e| AppError::Git(e.to_string()))? }; - + // A walk that hit its deadline carries GitServiceTimeout; map it to 504 like + // the smart_http paths, not a generic 500 (#174 U3). + let withheld = withheld.map_err(|e| git_service_app_error(&e))?; + + // Move the permits returned by the walk into the guard, ONE construction + // site for both serve arms below, so admission tracks the served git group's + // reap (complete/timeout/disconnect) whether the pack is plain or filtered. + // The handler keeps no copy (F1: handler-local permits would drop the + // instant a disconnect drops this future, mid-reap). + let admission = smart_http::AdmissionGuard::new(_permit, _caller_permit); if withheld.is_empty() { - smart_http::upload_pack(&disk_path, body, git_timeout).await + // No blobs to withhold: serve the plain pack (the walk already held the + // permits per be0cdd6; the guard hands them to the serve). + smart_http::upload_pack(&state.git_bin, &disk_path, body, git_timeout, Some(admission)).await } else { tracing::info!(repo = %name, caller = ?caller, withheld = withheld.len(), "serving filtered pack"); - smart_http::upload_pack_excluding(&disk_path, body, &withheld).await + // The guard threads through both filtered-pack stages (rev-list, then + // pack-objects), so a disconnect mid-stage keeps the permits held until + // that stage's process group is reaped (F1). + smart_http::upload_pack_excluding( + &state.git_bin, + &disk_path, + body, + &withheld, + git_timeout, + Some(admission), + ) + .await } } .map_err(|e| { @@ -851,9 +1534,21 @@ pub async fn git_receive_pack( State(state): State, Path((owner, repo)): Path<(String, String)>, Extension(auth): Extension, + crate::rate_limit::PeerAddr(peer): crate::rate_limit::PeerAddr, + headers: axum::http::HeaderMap, body: Bytes, ) -> Result { let name = smart_http_repo_name(&repo)?; + // Fast-path shed BEFORE the DB lookup when the write pool is saturated, so a push + // flood on a full pool does not hit Postgres per request. Best-effort (racy) and + // NON-holding: the authoritative, held permit is taken after the per-repo lease below + // (so a lease-blocked waiter pins no write slot — #174 F3 review). This peek only + // restores the cheap pre-DB shed the permit reorder would otherwise have lost. + if state.git_write_semaphore.available_permits() == 0 { + return Err(AppError::Overloaded( + "git service at capacity, retry shortly".into(), + )); + } tracing::info!(owner = %owner, repo = %name, "receive-pack request"); let record = state .db @@ -923,25 +1618,116 @@ pub async fn git_receive_pack( } } + // Per-repo in-process write lease (#174 U2/F3): SUPPLEMENTS the cluster-wide pg + // advisory lock. Acquire it BEFORE acquire_write (one consistent order everywhere, + // so the two serializers can never invert into a self-hang) so a second SAME-NODE + // push to this repo blocks here rather than racing a disconnected first push's git + // group while its detached reaper is still tearing it down over the shared local + // objects/ dir. Taking it before the pg lock also means a blocked second writer pins + // no pooled pg connection while it waits. The lease rides the write-path + // AdmissionGuard into the reaper (clone (a)) and spans the clean-path Tigris upload + // in guard.release (clone (b)); it frees only when the LAST clone drops. steal_after + // is sized above ONE legitimate hold (a full receive-pack under git_service_timeout + + // the ~4s reaper cap + the Tigris upload). It is NOT a guarantee that only a leaked + // lease is reclaimed: a waiter's timeout starts at acquire(), not at the head of the + // FIFO queue, so a same-repo backlog whose CUMULATIVE wait exceeds steal_after can + // steal while an earlier waiter is still writing. Correctness does not rest on the + // bound — on the non-disconnect path the retained pg advisory lock still serializes + // the stealer at acquire_write (a spurious 503, not a race); the only corruption-capable + // overlap is the ~4s disconnect/reap window, which the reaper-carried clone (a) covers. + let lease_steal_after = + std::time::Duration::from_secs(state.config.git_service_timeout_secs * 2 + 60); + let lease = state + .repo_write_leases + .acquire(&record.id, lease_steal_after) + .await; + + // Admission permits are taken HERE, AFTER the per-repo lease and BEFORE acquire_write. + // Ordering is the fix (#174 P2 DoS): the lease is a block-and-wait serializer, so a + // second same-repo push can park on `acquire` above for up to steal_after. Taking the + // scarce write permits only once we own the lease means a lease-blocked waiter pins NO + // write-pool slot while it waits. Otherwise a few hostile sources could stack same-repo + // pushes, hold every global slot on zero-byte lease-waiters, and shed 503 on every push + // to every OTHER repo node-wide. Still before acquire_write, so the git op stays + // admission-gated (INV-10) and a saturated pool sheds 503 before spawning git. + // + // Per-source sub-cap first (#174 P1-d): one source IP cannot occupy the whole write + // pool via many slow pushes. Owner enforcement defaults off, so any valid did:key is + // accepted (auth != authz) and the push rate limiter bounds arrival RATE, not in-flight + // concurrency. Keyed on the resolved source IP, NEVER the signed DID (a DID farm defeats + // a DID key); no resolvable key -> global write pool only. Then the global write permit: + // pushes draw from the dedicated WRITE pool, separate from reads, and it is held for the + // whole op (moved into the AdmissionGuard below). + let caller_key = read_caller_key(&headers, peer, state.push_limiter_trust); + let _caller_permit = acquire_read_caller_permit( + &state.git_write_per_caller, + caller_key.as_deref(), + name, + "receive-pack", + )?; + let _permit = git_permit(&state.git_write_semaphore)?; + tracing::debug!(repo = %name, "acquiring write lock"); - let guard = state - .repo_store - .acquire_write(&record.owner_did, &record.name) - .await - .map_err(|e| { - tracing::error!(repo = %name, err = %e, "acquire_write failed"); - AppError::Git(e.to_string()) - })?; + // Bound the write acquire under `git_acquire_timeout_secs`. acquire_write's + // advisory-lock loop already caps at ~60s, but its per-iteration + // `pg_try_advisory_lock().fetch_one(&pool)` can block indefinitely on a hung / + // exhausted Postgres pool (so the 60-count never advances) — and the write permit + // is held the whole time, draining the pool (#174 P1-2). The outer + // `tokio::time::timeout` cancels a mid-sleep/mid-`fetch_one` future, so it bounds + // both the loop and a hung iteration without any repo_store.rs change (KTD3). The + // permit is a handler local here (moved into the AdmissionGuard only after this), + // so the early return on timeout drops it and frees the slot; shed a bounded 503. + let acquire_deadline = std::time::Duration::from_secs(state.config.git_acquire_timeout_secs); + let guard = tokio::time::timeout( + acquire_deadline, + state + .repo_store + .acquire_write(&record.owner_did, &record.name), + ) + .await + .map_err(|_elapsed| { + tracing::warn!(repo = %name, "acquire_write timed out; shedding with 503"); + AppError::Overloaded("git service acquisition timed out, retry shortly".into()) + })? + .map_err(|e| { + tracing::error!(repo = %name, err = %e, "acquire_write failed"); + AppError::Git(e.to_string()) + })?; let disk_path = guard.path().to_path_buf(); tracing::debug!(repo = %name, path = %disk_path.display(), "running git receive-pack"); let body_len = body.len(); let git_timeout = std::time::Duration::from_secs(state.config.git_service_timeout_secs); - let receive_result = smart_http::receive_pack(&disk_path, body, git_timeout).await; + // Move both admission permits into the guard so they release only after the spawned + // receive-pack process group is reaped, on complete/timeout/disconnect — not the + // instant a disconnect drops this future while the detached reaper runs (#174 P1-a). + // The handler keeps no copy. This is independent of the write-lock `guard.release` + // below: admission tracks the git process lifetime, the write lock tracks the repo. + // Clone (a) of the write lease rides this AdmissionGuard: on a client disconnect the + // guard moves into KillGroupOnDrop's detached reaper, so the lease frees only after + // the receive-pack group is reaped — NOT at the disconnect instant (which is exactly + // when RepoWriteGuard::Drop frees the pg lock). Tying the lease to RepoWriteGuard + // instead would drop it at disconnect and reopen the F3 race. + let admission = + smart_http::AdmissionGuard::new(_permit, _caller_permit).with_lease(lease.clone()); + let receive_result = smart_http::receive_pack( + &state.git_bin, + &disk_path, + body, + git_timeout, + Some(admission), + ) + .await; // Always release the advisory lock — even on error — to prevent stale locks // from blocking subsequent pushes. Only upload to Tigris when the push // succeeded; uploading a half-applied repo would propagate corruption. guard.release(receive_result.is_ok()).await; + // Clean path: clone (a) already dropped inside run_git_service when the receive-pack + // group was reaped; clone (b) held here spanned the success-only Tigris upload that + // ran inside release() above. Drop it now so a second same-repo push proceeds the + // moment this write is durable, rather than at end of the (longer) handler tail. On + // the disconnect path this line is never reached — clone (a) rides the reaper (F3). + drop(lease); let result = receive_result.map_err(|e| { let app = git_service_app_error(&e); @@ -1044,331 +1830,376 @@ pub async fn git_receive_pack( } } - // Replication enforcement (Phase 2): decide once per push whether the public - // may read this repo at all and, if so, which blob OIDs must not leave the - // node. `withheld == None` means replicate nothing (private / mode A / - // undetermined): skip every pin so even commit and tree objects (which - // withheld_blob_oids never lists) stay local. `announce` gates the - // network-facing announcements. Fail closed: a private or undetermined repo - // never leaks. - let rules_opt = state.db.list_visibility_rules(&record.id).await.ok(); - let (announce, withheld) = replication_withheld_set( - rules_opt.clone(), - &record.owner_did, - record.is_public, - disk_path.clone(), - ) - .await; - - // Resolve the per-push pin candidate set once, off the async worker, then - // filter to what may actually replicate. Delta path: the reachable-only - // `withheld` set suffices (delta objects are reachable). Full-scan path: the - // candidate set can include dangling blobs the withheld set never classified, - // so fail closed — replicate a blob only if it is reachable AND - // visibility-allowed (#99). Only computed when something will actually - // replicate; every degraded path logs rather than failing silently. - let object_list: Vec = if let Some(withheld_set) = withheld.clone() { - let new_tips: Vec = ref_updates - .iter() - .map(|u| u.new_sha.clone()) - .filter(|s| s != ZERO_SHA) - .collect(); - let old_tips: Vec = ref_updates - .iter() - .map(|u| u.old_sha.clone()) - .filter(|s| s != ZERO_SHA) - .collect(); - let pin_set = crate::git::push_delta::resolve_candidates_for_push( + // #174 F2: move the whole post-receive replication tail into an independently owned + // task. It parks on `git_encrypt_semaphore` (withheld / candidate / full-scan + // resolution), so leaving it in the request future means a client/proxy disconnect + // while parked silently drops this push's pins, recovery copy, and announcements — + // the residual `state.rs` documented, because the durable `try_begin` gate sat after + // the park. The request future now returns the git response immediately and the tail + // runs detached; a disconnect can no longer drop it. Each push owns its own tail, + // including its own always-spawned announce, so per-push announcements are never + // coalesced away (the announce spawn stays out of the per-repo encrypt coalescing). + let did = did.to_string(); + tokio::spawn(async move { + // Replication enforcement (Phase 2): decide once per push whether the public + // may read this repo at all and, if so, which blob OIDs must not leave the + // node. `withheld == None` means replicate nothing (private / mode A / + // undetermined): skip every pin so even commit and tree objects (which + // withheld_blob_oids never lists) stay local. `announce` gates the + // network-facing announcements. Fail closed: a private or undetermined repo + // never leaks. + let rules_opt = state.db.list_visibility_rules(&record.id).await.ok(); + let (announce, withheld) = replication_withheld_set( + state.git_encrypt_semaphore.clone(), + rules_opt.clone(), + &record.owner_did, + record.is_public, disk_path.clone(), - new_tips, - old_tips, + state.git_bin.clone(), + std::time::Duration::from_secs(state.config.git_service_timeout_secs), ) .await; - if pin_set.full_scan { - fail_closed_full_scan_objects( + + // Resolve the per-push pin candidate set once, off the async worker, then + // filter to what may actually replicate. Delta path: the reachable-only + // `withheld` set suffices (delta objects are reachable). Full-scan path: the + // candidate set can include dangling blobs the withheld set never classified, + // so fail closed — replicate a blob only if it is reachable AND + // visibility-allowed (#99). Only computed when something will actually + // replicate; every degraded path logs rather than failing silently. + let object_list: Vec = if let Some(withheld_set) = withheld.clone() { + let new_tips: Vec = ref_updates + .iter() + .map(|u| u.new_sha.clone()) + .filter(|s| s != ZERO_SHA) + .collect(); + let old_tips: Vec = ref_updates + .iter() + .map(|u| u.old_sha.clone()) + .filter(|s| s != ZERO_SHA) + .collect(); + let pin_set = crate::git::push_delta::resolve_candidates_for_push( + state.git_encrypt_semaphore.clone(), disk_path.clone(), - rules_opt.clone().unwrap_or_default(), - record.is_public, - record.owner_did.clone(), - pin_set.candidates, + new_tips, + old_tips, + state.git_bin.clone(), + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + false, ) - .await + .await; + if pin_set.full_scan { + fail_closed_full_scan_objects( + state.git_encrypt_semaphore.clone(), + disk_path.clone(), + rules_opt.clone().unwrap_or_default(), + record.is_public, + record.owner_did.clone(), + pin_set.candidates, + state.git_bin.clone(), + std::time::Duration::from_secs(state.config.git_service_timeout_secs), + ) + .await + } else { + crate::git::visibility_pack::replicable_objects(pin_set.candidates, &withheld_set) + } } else { - crate::git::visibility_pack::replicable_objects(pin_set.candidates, &withheld_set) - } - } else { - Vec::new() - }; + Vec::new() + }; - // Pin new git objects to the local IPFS node (no-op if ipfs_api is empty). - // Skipped entirely when the public cannot read the repo (withheld == None). - if withheld.is_some() { - let object_list_ipfs = object_list.clone(); - let ipfs_api = state.config.ipfs_api.clone(); - let repo_path_clone = disk_path.clone(); - let db_clone = state.db.clone(); - let rules_for_enc = rules_opt.clone(); - let repo_id = record.id.clone(); - let owner_did = record.owner_did.clone(); - let is_public = record.is_public; - let irys_url = state.config.irys_url.clone(); - let http_client = std::sync::Arc::clone(&state.http_client); - let node_did_str = state.node_did.to_string(); - let node_seed = state.node_keypair.to_seed(); - let repo_name = record.name.clone(); - tokio::spawn(async move { - let pinned = crate::ipfs_pin::pin_new_objects( - &ipfs_api, - &repo_path_clone, - object_list_ipfs, - &db_clone, - ) - .await; - if !pinned.is_empty() { - tracing::info!(count = pinned.len(), "pinned git objects to IPFS"); - for (sha, cid) in &pinned { - tracing::info!(sha = %sha, %cid, "pinned"); + // Pin new git objects to the local IPFS node (no-op if ipfs_api is empty). + // Skipped entirely when the public cannot read the repo (withheld == None). + // + // Coalesce-and-requeue per repo (#174 P2-2 + F5): the spawned task's walks park + // on `git_encrypt_semaphore` (which DEFERS when the pool is full rather than + // dropping the recovery copy). To bound the OUTSTANDING task set, at most one + // task per repo is in flight; a push arriving while one is in flight does NOT + // spawn a duplicate — and is NOT dropped either. The in-flight task pins only + // its own pre-spawn object-list snapshot, so this push's (old, new) tip pairs + // are merged into the in-flight key's pending slot in the same critical section + // as the presence check, and the task loop-drains them (fresh rules, fail + // closed) before releasing the key. Without the requeue a coalesced push's pins + // and recovery copies would be silently absent until an unrelated later push + // (the F5 loss). The guard still releases the key on panic (Drop on unwind), so + // a crashed walk never permanently locks the repo out. + if withheld.is_some() { + let tip_pairs: Vec<(String, String)> = ref_updates + .iter() + .map(|u| (u.old_sha.clone(), u.new_sha.clone())) + .collect(); + match state.encrypt_inflight.try_begin(&record.id, tip_pairs) { + crate::state::BeginOutcome::Coalesced => { + tracing::debug!( + repo = %record.id, + "post-push encryption task already in flight for this repo; coalesced \ + — this push's tip pairs are queued for that task's drain" + ); + } + crate::state::BeginOutcome::Admitted(inflight_guard) => { + let ctx = EncryptTaskCtx { + ipfs_api: state.config.ipfs_api.clone(), + repo_path: disk_path.clone(), + db: state.db.clone(), + repo_id: record.id.clone(), + owner_did: record.owner_did.clone(), + repo_name: record.name.clone(), + irys_url: state.config.irys_url.clone(), + http_client: std::sync::Arc::clone(&state.http_client), + node_did: state.node_did.to_string(), + node_keypair: std::sync::Arc::clone(&state.node_keypair), + git_bin: state.git_bin.clone(), + git_timeout: std::time::Duration::from_secs( + state.config.git_service_timeout_secs, + ), + encrypt_sem: state.git_encrypt_semaphore.clone(), + pin_sem: state.pin_semaphore.clone(), + }; + tokio::spawn(run_encrypt_pin_task( + ctx, + inflight_guard, + object_list.clone(), + rules_opt.clone(), + record.is_public, + )); } } + } - // Option B1: encrypt-then-pin the withheld blobs so authorized - // readers can recover them when the origin cannot serve them. - // No path-scoped rule can withhold a blob, so withheld_blob_recipients - // would return an empty map after a full per-ref walk; skip it. Mirrors - // the has_path_scoped_rule gate on the other two withheld-walk sites. - if let Some(rules) = rules_for_enc.filter(|r| visibility_pack::has_path_scoped_rule(r)) - { - let p = repo_path_clone.clone(); - let owner = owner_did.clone(); - let recip = tokio::task::spawn_blocking(move || { - crate::git::visibility_pack::withheld_blob_recipients( - &p, &rules, is_public, &owner, + // Pin new git objects to Pinata, then record branch→CID and gossip. + // + // #174 P2-2 scope note: this SECOND detached spawn is deliberately NOT brought + // under the per-repo encryption coalescing above, because unlike the idempotent + // recovery-copy walk it does PER-PUSH, PER-REF work — branch→CID upserts, gossip + // publish, GraphQL subscription broadcast, Arweave anchoring, and peer notify, each + // keyed to THIS push's ref_updates. Coalescing (or shedding) it against an in-flight + // task for the same repo would DROP a later push's ref-update announcements (a + // correctness regression), not merely delay a duplicate. So the task stays one per + // push and every push's effects fire exactly once. + // + // #174 F2 / KTD-3: {bounded memory, no dropped effects, no handler latency} are + // jointly unsatisfiable by coalesce/shed/block, so instead of retaining the full + // object list we bound the thing that actually accumulates. The task captures only + // the small ref tuples and RE-DERIVES the object set inside the worker once a pin + // slot frees (see `pinata_object_list_for_refs`); the MB-scale OID list is never + // held by a parked task. + { + let pinata_jwt = state.config.pinata_jwt.clone(); + let pinata_upload_url = state.config.pinata_upload_url.clone(); + let repo_path_clone = disk_path.clone(); + let db_clone = state.db.clone(); + let http_client = Arc::clone(&state.http_client); + let node_did_str = state.node_did.to_string(); + let repo_slug = format!( + "{}/{}", + crate::db::normalize_owner_key(&record.owner_did), + record.name + ); + let ref_updates_clone = ref_updates + .iter() + .map(|u| (u.ref_name.clone(), u.old_sha.clone(), u.new_sha.clone())) + .collect::>(); + let p2p_handle = state.p2p.clone(); + let pusher_did_clone = did.to_string(); + let db_for_peers = state.db.clone(); + let ref_update_tx = state.ref_update_tx.clone(); + let irys_url = state.config.irys_url.clone(); + let owner_did_for_arweave = record.owner_did.clone(); + let self_public_url = state.config.public_url.clone(); + let node_keypair = Arc::clone(&state.node_keypair); + let do_pinata_replication = withheld.is_some(); + // #174 F2 / KTD-3: capture only the small inputs the re-derivation needs; the + // MB-scale object list is NOT moved in. `pinata_object_list_for_refs` recomputes + // it from these once a pin slot frees. rules/owner/is_public drive the fresh + // fail-closed withheld filter; encrypt_sem + git_bin + timeout keep the re-derive + // git children under the same INV-22 bounded, group-reaped scan admission. + let pinata_rules_opt = rules_opt.clone(); + let pinata_owner_did = record.owner_did.clone(); + let pinata_is_public = record.is_public; + let pinata_git_bin = state.git_bin.clone(); + let pinata_git_timeout = + std::time::Duration::from_secs(state.config.git_service_timeout_secs); + let pinata_encrypt_sem = state.git_encrypt_semaphore.clone(); + // Same global pin-admission bound as the IPFS loop (#174 F6): the Pinata pin + // loop holds a re-derived object-id list while pinning it, so it shares the cap. + // It DEFERS on a full pool rather than dropping the pin. + let pin_sem_pinata = state.pin_semaphore.clone(); + tokio::spawn(async move { + let pinned = if do_pinata_replication { + let _pin_permit = pin_sem_pinata + .acquire_owned() + .await + .expect("pin_semaphore is never closed"); + // Re-derive the object set now that a pin slot is free (#174 F2 / + // KTD-3). A parked task retained only `ref_updates_clone` (O(ref + // tuples)), never this list, so a slow Pinata backend cannot grow + // outstanding memory O(pushes x object-list). Fresh + fail-closed; + // each git child is INV-22 bounded and process-group reaped. + let object_list = pinata_object_list_for_refs( + pinata_encrypt_sem, + repo_path_clone.clone(), + &ref_updates_clone, + pinata_rules_opt, + pinata_is_public, + pinata_owner_did, + pinata_git_bin, + pinata_git_timeout, ) - }) - .await; - if let Ok(Ok(recipients)) = recip { - let delta = crate::encrypted_pin::encrypt_and_pin( - &ipfs_api, + .await; + crate::pinata::pin_new_objects( + &http_client, + &pinata_upload_url, + &pinata_jwt, &repo_path_clone, + object_list, &db_clone, - &repo_id, - &node_seed, - &recipients, ) - .await; + .await + } else { + Vec::new() + }; - // Option B3: anchor a per-push manifest of the blobs sealed - // this push to Arweave, so the oid->cid index survives total - // node loss. Best-effort; never fails the push. - if !delta.is_empty() && !irys_url.is_empty() { - let owner_short = crate::db::normalize_owner_key(&owner_did); - let repo_slug = format!("{owner_short}/{repo_name}"); - let ts = chrono::Utc::now().to_rfc3339(); - let manifest = crate::arweave::EncryptedManifest { - repo: &repo_slug, - owner_did: &owner_did, - node_did: &node_did_str, - timestamp: &ts, - blobs: &delta, - }; - match crate::arweave::anchor_encrypted_manifest( - &http_client, - &irys_url, - &manifest, - ) - .await - { - Ok(tx) if !tx.is_empty() => tracing::info!( - repo = %repo_slug, - tx_id = %tx, - "anchored encrypted manifest to Arweave" - ), - Ok(_) => {} - Err(e) => tracing::warn!( - repo = %repo_slug, - err = %e, - "encrypted manifest anchor failed" - ), - } - } + if !pinned.is_empty() { + tracing::info!(count = pinned.len(), "pinned git objects to Pinata"); } - } - }); - } - - // Pin new git objects to Pinata, then record branch→CID and gossip - { - let pinata_jwt = state.config.pinata_jwt.clone(); - let pinata_upload_url = state.config.pinata_upload_url.clone(); - let repo_path_clone = disk_path.clone(); - let db_clone = state.db.clone(); - let http_client = Arc::clone(&state.http_client); - let node_did_str = state.node_did.to_string(); - let repo_slug = format!( - "{}/{}", - crate::db::normalize_owner_key(&record.owner_did), - record.name - ); - let ref_updates_clone = ref_updates - .iter() - .map(|u| (u.ref_name.clone(), u.old_sha.clone(), u.new_sha.clone())) - .collect::>(); - let p2p_handle = state.p2p.clone(); - let pusher_did_clone = did.to_string(); - let db_for_peers = state.db.clone(); - let ref_update_tx = state.ref_update_tx.clone(); - let irys_url = state.config.irys_url.clone(); - let owner_did_for_arweave = record.owner_did.clone(); - let self_public_url = state.config.public_url.clone(); - let node_keypair = Arc::clone(&state.node_keypair); - let object_list_pinata = object_list; - let do_pinata_replication = withheld.is_some(); - tokio::spawn(async move { - let pinned = if do_pinata_replication { - crate::pinata::pin_new_objects( - &http_client, - &pinata_upload_url, - &pinata_jwt, - &repo_path_clone, - object_list_pinata, - &db_clone, - ) - .await - } else { - Vec::new() - }; - - if !pinned.is_empty() { - tracing::info!(count = pinned.len(), "pinned git objects to Pinata"); - } - // Build sha→cid map from pinned objects - let cid_map: std::collections::HashMap = pinned.into_iter().collect(); + // Build sha→cid map from pinned objects + let cid_map: std::collections::HashMap = + pinned.into_iter().collect(); - // Record branch→CID for each ref update and publish gossip - for (ref_name, old_sha, new_sha) in &ref_updates_clone { - let cid = cid_map.get(new_sha).map(|s| s.as_str()); + // Record branch→CID for each ref update and publish gossip + for (ref_name, old_sha, new_sha) in &ref_updates_clone { + let cid = cid_map.get(new_sha).map(|s| s.as_str()); + + if let Some(cid_str) = cid { + let _ = db_clone + .upsert_branch_cid( + &repo_slug, + ref_name, + new_sha, + cid_str, + &node_did_str, + ) + .await; + } - if let Some(cid_str) = cid { - let _ = db_clone - .upsert_branch_cid(&repo_slug, ref_name, new_sha, cid_str, &node_did_str) - .await; + if announce { + if let Some(p2p) = &p2p_handle { + p2p.publish_ref_update(crate::p2p::RefUpdateEvent { + node_did: node_did_str.clone(), + pusher_did: pusher_did_clone.clone(), + repo: repo_slug.clone(), + ref_name: ref_name.clone(), + old_sha: old_sha.clone(), + new_sha: new_sha.clone(), + timestamp: chrono::Utc::now().to_rfc3339(), + cert_id: None, + cid: cid.map(|s| s.to_string()), + }) + .await; + } + } } + // Broadcast ref update to GraphQL subscription listeners — one per ref. + // Gated on `announce`: /graphql/ws is unauthenticated (mounted after + // the optional_signature layer), and the subscription resolver has no + // caller to gate against, so only publicly-readable ref updates may + // reach anonymous subscribers. Mirrors the gossip (above) and Arweave + // (below) sends, which are already `announce`-gated. Without this a + // private-repo push would leak live ref metadata over the socket — + // the subscription analog of #112/#114. + let now_ts = chrono::Utc::now().to_rfc3339(); if announce { - if let Some(p2p) = &p2p_handle { - p2p.publish_ref_update(crate::p2p::RefUpdateEvent { - node_did: node_did_str.clone(), - pusher_did: pusher_did_clone.clone(), + for (ref_name, old_sha, new_sha) in &ref_updates_clone { + let _ = ref_update_tx.send(crate::state::RefUpdateBroadcast { repo: repo_slug.clone(), ref_name: ref_name.clone(), old_sha: old_sha.clone(), new_sha: new_sha.clone(), - timestamp: chrono::Utc::now().to_rfc3339(), - cert_id: None, - cid: cid.map(|s| s.to_string()), - }) - .await; - } - } - } - - // Broadcast ref update to GraphQL subscription listeners — one per ref. - // Gated on `announce`: /graphql/ws is unauthenticated (mounted after - // the optional_signature layer), and the subscription resolver has no - // caller to gate against, so only publicly-readable ref updates may - // reach anonymous subscribers. Mirrors the gossip (above) and Arweave - // (below) sends, which are already `announce`-gated. Without this a - // private-repo push would leak live ref metadata over the socket — - // the subscription analog of #112/#114. - let now_ts = chrono::Utc::now().to_rfc3339(); - if announce { - for (ref_name, old_sha, new_sha) in &ref_updates_clone { - let _ = ref_update_tx.send(crate::state::RefUpdateBroadcast { - repo: repo_slug.clone(), - ref_name: ref_name.clone(), - old_sha: old_sha.clone(), - new_sha: new_sha.clone(), - pusher_did: pusher_did_clone.clone(), - node_did: node_did_str.clone(), - timestamp: now_ts.clone(), - }); + pusher_did: pusher_did_clone.clone(), + node_did: node_did_str.clone(), + timestamp: now_ts.clone(), + }); + } } - } - // Arweave permanent anchoring — fire for each ref update. - // Suppressed for repos the public cannot read (public permanent ledger). - if announce && !irys_url.is_empty() { - for (ref_name, old_sha, new_sha) in &ref_updates_clone { - let cid = cid_map.get(new_sha).cloned(); - let anchor = crate::arweave::RefAnchor { - repo: repo_slug.clone(), - owner_did: owner_did_for_arweave.clone(), - ref_name: ref_name.clone(), - old_sha: old_sha.clone(), - new_sha: new_sha.clone(), - cid: cid.clone(), - timestamp: now_ts.clone(), - node_did: node_did_str.clone(), - }; - match crate::arweave::anchor_ref_update(&http_client, &irys_url, &anchor).await - { - Ok(tx_id) if !tx_id.is_empty() => { - let arweave_url = crate::arweave::arweave_url(&tx_id); - let _ = db_clone - .record_arweave_anchor(&crate::db::RecordAnchorInput { - repo: &repo_slug, - owner_did: &owner_did_for_arweave, - ref_name, - old_sha, - new_sha, - cid: cid.as_deref(), - irys_tx_id: &tx_id, - arweave_url: &arweave_url, - node_did: &node_did_str, - }) - .await; + // Arweave permanent anchoring — fire for each ref update. + // Suppressed for repos the public cannot read (public permanent ledger). + if announce && !irys_url.is_empty() { + for (ref_name, old_sha, new_sha) in &ref_updates_clone { + let cid = cid_map.get(new_sha).cloned(); + let anchor = crate::arweave::RefAnchor { + repo: repo_slug.clone(), + owner_did: owner_did_for_arweave.clone(), + ref_name: ref_name.clone(), + old_sha: old_sha.clone(), + new_sha: new_sha.clone(), + cid: cid.clone(), + timestamp: now_ts.clone(), + node_did: node_did_str.clone(), + }; + match crate::arweave::anchor_ref_update(&http_client, &irys_url, &anchor) + .await + { + Ok(tx_id) if !tx_id.is_empty() => { + let arweave_url = crate::arweave::arweave_url(&tx_id); + let _ = db_clone + .record_arweave_anchor(&crate::db::RecordAnchorInput { + repo: &repo_slug, + owner_did: &owner_did_for_arweave, + ref_name, + old_sha, + new_sha, + cid: cid.as_deref(), + irys_tx_id: &tx_id, + arweave_url: &arweave_url, + node_did: &node_did_str, + }) + .await; + } + Ok(_) => {} + Err(e) => { + tracing::warn!(repo=%repo_slug, err=%e, "Arweave anchor failed") + } } - Ok(_) => {} - Err(e) => tracing::warn!(repo=%repo_slug, err=%e, "Arweave anchor failed"), } } - } - // HTTP peer notification — notify all known peers to pull from us. - // This is the reliable fallback when Gossipsub p2p is not yet connected. - // Suppressed for repos the public cannot read. Runs last so a slow or - // unreachable peer cannot delay the local GraphQL broadcast or Arweave - // anchoring above; this is the lowest-priority best-effort step. - if announce { - if let Ok(peers) = db_for_peers.list_peers().await { - for peer in peers { - if peer.http_url.is_empty() { - continue; - } - let peer_url = peer.http_url.trim_end_matches('/'); - if let Some(self_url) = self_public_url.as_deref() { - if peer_url == self_url.trim_end_matches('/') { + // HTTP peer notification — notify all known peers to pull from us. + // This is the reliable fallback when Gossipsub p2p is not yet connected. + // Suppressed for repos the public cannot read. Runs last so a slow or + // unreachable peer cannot delay the local GraphQL broadcast or Arweave + // anchoring above; this is the lowest-priority best-effort step. + if announce { + if let Ok(peers) = db_for_peers.list_peers().await { + for peer in peers { + if peer.http_url.is_empty() { continue; } + let peer_url = peer.http_url.trim_end_matches('/'); + if let Some(self_url) = self_public_url.as_deref() { + if peer_url == self_url.trim_end_matches('/') { + continue; + } + } + let notify_url = format!("{peer_url}{SYNC_NOTIFY_PATH}"); + notify_peer_of_refs( + &http_client, + node_keypair.as_ref(), + &peer.did, + ¬ify_url, + &repo_slug, + &ref_updates_clone, + &node_did_str, + &pusher_did_clone, + ) + .await; } - let notify_url = format!("{peer_url}{SYNC_NOTIFY_PATH}"); - notify_peer_of_refs( - &http_client, - node_keypair.as_ref(), - &peer.did, - ¬ify_url, - &repo_slug, - &ref_updates_clone, - &node_did_str, - &pusher_did_clone, - ) - .await; } } - } - }); - } + }); + } + }); Ok(result) } @@ -1862,6 +2693,17 @@ mod tests { assert!(matches!(git_service_app_error(&other), AppError::Git(_))); } + #[test] + fn git_permit_sheds_at_capacity_and_releases() { + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + let p1 = git_permit(&sem).expect("first acquire succeeds"); + // At capacity the next request is shed with Overloaded (-> 503), not queued. + assert!(matches!(git_permit(&sem), Err(AppError::Overloaded(_)))); + // Releasing the permit frees the slot for the next request. + drop(p1); + assert!(git_permit(&sem).is_ok()); + } + fn repo_owned_by(owner_did: &str) -> crate::db::RepoRecord { let now = chrono::Utc::now(); crate::db::RepoRecord { @@ -1892,16 +2734,42 @@ mod tests { let dummy = std::path::PathBuf::from("/nonexistent"); // Private: no rules at all. - let (announce, _) = replication_withheld_set(None, OWNER_DID, false, dummy.clone()).await; + let (announce, _) = replication_withheld_set( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), + None, + OWNER_DID, + false, + dummy.clone(), + "git".into(), + std::time::Duration::from_secs(600), + ) + .await; assert!(!announce, "private repo (no rules) must not announce"); // Private: empty rule set, is_public=false → still not listable at root. - let (announce, _) = - replication_withheld_set(Some(vec![]), OWNER_DID, false, dummy.clone()).await; + let (announce, _) = replication_withheld_set( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), + Some(vec![]), + OWNER_DID, + false, + dummy.clone(), + "git".into(), + std::time::Duration::from_secs(600), + ) + .await; assert!(!announce, "private repo (empty rules) must not announce"); // Public: empty rule set, is_public=true → listable at root, announces. - let (announce, _) = replication_withheld_set(Some(vec![]), OWNER_DID, true, dummy).await; + let (announce, _) = replication_withheld_set( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), + Some(vec![]), + OWNER_DID, + true, + dummy, + "git".into(), + std::time::Duration::from_secs(600), + ) + .await; assert!(announce, "public repo must announce"); } @@ -1993,6 +2861,151 @@ mod tests { } } + #[cfg(unix)] + fn write_fake_git(dir: &std::path::Path, body: &str) -> String { + use std::os::unix::fs::PermissionsExt; + let p = dir.join("fakegit"); + std::fs::write(&p, body).unwrap(); + let mut perm = std::fs::metadata(&p).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&p, perm).unwrap(); + p.to_str().unwrap().to_string() + } + + /// #174 (write-pool twin, vetted by execution not reasoning): the receive-pack + /// post-push replication walk is bounded. Drive `replication_withheld_set` with an + /// injected fake git that hangs on `rev-list` and a short budget: it must RETURN + /// within the budget (so `git_receive_pack` releases the write permit it holds + /// across this await, rather than pinning it for the hang) AND fail closed + /// (announce suppressed) because the walk could not be vetted. Proves this path + /// funnels through the bounded `blob_paths`, on the write-permit-holding side. + #[cfg(unix)] + #[tokio::test] + async fn replication_walk_is_bounded_and_fails_closed_on_a_hung_git() { + use std::time::Duration; + let tmp = tempfile::TempDir::new().unwrap(); + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) sleep 30 ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + // Public root (announceable) + a path-scoped rule, so the walk actually runs + // rather than taking the has_path_scoped_rule short-circuit. + let rules = Some(vec![vis_rule("/secret/**", &[])]); + + let result = tokio::time::timeout( + Duration::from_secs(10), + replication_withheld_set( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), + rules, + OWNER_DID, + true, + tmp.path().to_path_buf(), + git_bin, + Duration::from_millis(200), + ), + ) + .await + .expect( + "replication_withheld_set must return within the budget; a hung walk must \ + not pin the write permit git_receive_pack holds across it", + ); + assert_eq!( + result, + (false, None), + "a walk that could not be vetted must suppress the announce (fail closed)" + ); + } + + /// #174 (serve-path 504, vetted by execution): a hung withheld-blob walk on the + /// upload-pack POST maps to 504, not a generic 500. Real repo dir on disk (so + /// acquire's fast path returns it) + a path-scoped rule (so the walk runs) + + /// an injected fake git that hangs on rev-list. The handler must return 504, + /// proving git_upload_pack routes the walk's GitServiceTimeout through + /// git_service_app_error end to end. + #[cfg(unix)] + #[sqlx::test] + async fn upload_pack_hung_withheld_walk_returns_504(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let tmp = tempfile::TempDir::new().unwrap(); + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) sleep 30 ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n"; + let fake = write_fake_git(tmp.path(), body); + + let mut state = crate::test_support::test_state(pool).await; + state.git_bin = fake; + let mut cfg = (*state.config).clone(); + cfg.git_service_timeout_secs = 1; + state.config = std::sync::Arc::new(cfg); + state + .db + .upsert_mirror_repo("z6srv504", "sv", "/tmp/z6srv504-sv", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6srv504", "sv").await.unwrap().unwrap(); + // Path-scoped rule so has_path_scoped_rule() is true and the walk runs; the + // public root still lets an anonymous caller past the "/" gate. + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &[], + OWNER_DID, + ) + .await + .unwrap(); + // acquire()'s fast path returns the local path when it exists on disk. + let disk = std::path::Path::new("/tmp/z6srv504/sv.git"); + std::fs::create_dir_all(disk).unwrap(); + + let peer: SocketAddr = "203.0.113.91:7000".parse().unwrap(); + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::POST) + .uri("/z6srv504/sv/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let status = router.oneshot(req).await.unwrap().status(); + let _ = std::fs::remove_dir_all("/tmp/z6srv504"); + assert_eq!( + status, + StatusCode::GATEWAY_TIMEOUT, + "a hung withheld-blob walk must surface as 504, not a generic 500" + ); + } + + /// #174 (F2 sizing edge, vetted by execution): the receive-pack advertisement + /// per-source cap is derived in main.rs as `(max_concurrent_git_pushes / 8).max(1)`, + /// so it is never 0 even at the minimum write-pool size (1). A 0 cap would make + /// PerCallerConcurrency shed EVERY receive-pack advertisement and break all pushes. + #[test] + fn advert_per_caller_cap_sizing_is_never_zero() { + let cap = |pushes: usize| (pushes / 8).max(1); + for pushes in [1usize, 4, 8, 32, 256] { + assert!( + cap(pushes) >= 1, + "advert cap must be >= 1 for pushes={pushes}" + ); + } + assert_eq!(cap(1), 1, "minimum write pool must derive cap 1, not 0"); + assert_eq!( + cap(32), + 4, + "default write pool 32 derives cap 4 (~8 source IPs to fill)" + ); + // A cap of 1 admits one and sheds the second from the same source. + let lim = crate::rate_limit::PerCallerConcurrency::new(cap(1), 100); + let _held = lim.try_acquire("src").expect("first advert admitted"); + assert!( + lim.try_acquire("src").is_none(), + "second advert from the same source is shed" + ); + } + #[test] fn fork_owner_full_did_with_path_rule_allowed() { // Owner reads everything (implicit reader), so nothing is withheld. @@ -2579,50 +3592,3495 @@ mod tests { ); } - /// Repo creation must be throttled by the per-IP creation limiter BEFORE - /// signature verification — otherwise a DID farm (one throwaway did:key per - /// repo, each carrying a valid but machine-solved iCaptcha proof) walks past - /// the per-DID limiter and floods the network, as in the recurring spam-repo - /// incidents. A 429 (not a 401) on an unsigned request from an exhausted IP - /// proves the IP brake runs outermost, ahead of auth. + /// #174 P2-1: an unsupported `?service=` must be rejected with 400 BEFORE taking a + /// read slot or doing DB/Tigris work. Isolate it: exhaust the read pool so a read + /// op WOULD shed 503 at the pre-DB check — a garbage service must still return 400 + /// (validation runs first), proving `?service=anything` cannot consume the read + /// pool. Removing the validation makes this 503 (RED). #[sqlx::test] - async fn repo_creation_is_rate_limited_by_ip(pool: sqlx::PgPool) { + async fn info_refs_rejects_unsupported_service_before_the_read_slot(pool: sqlx::PgPool) { use axum::body::Body; use axum::extract::ConnectInfo; use axum::http::{Method, Request, StatusCode}; use std::net::SocketAddr; - use std::time::Duration; + use std::sync::Arc; + use tokio::sync::Semaphore; use tower::ServiceExt; let mut state = crate::test_support::test_state(pool).await; - // Tiny limit, keyed on the socket peer (no trusted proxy). - state.create_ip_rate_limiter = - crate::rate_limit::RateLimiter::new(1, Duration::from_secs(60)); - state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6svcowner", "svc", "/tmp/svc", None, false) + .await + .unwrap(); + // Exhaust the read pool: a read op would shed 503 at the pre-DB check. + state.git_read_semaphore = Arc::new(Semaphore::new(0)); - let peer: SocketAddr = "203.0.113.77:7000".parse().unwrap(); - // Exhaust this peer's single-request budget up front. - assert!( + let router = crate::server::build_router(state); + let peer: SocketAddr = "203.0.113.90:7000".parse().unwrap(); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6svcowner/svc/info/refs?service=git-explode") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + + let status = router.oneshot(req).await.unwrap().status(); + assert_eq!( + status, + StatusCode::BAD_REQUEST, + "an unsupported ?service= must be 400 before the read-pool shed, not 503" + ); + } + + /// #174 (jatmn P1): the anon-reachable receive-pack advertisement + /// (`GET info/refs?service=git-receive-pack`) draws from a DEDICATED advert pool + /// (`git_push_advert_semaphore`), NOT the write pool the authenticated POST uses. + /// Proven at the handler by saturating each pool to zero and checking who shares + /// it (INV-10, across the auth boundary). The load-bearing pair: + /// * advert pool at 0 -> the advert SHEDS 503 (it is bound to that pool); + /// * write pool at 0 -> the advert SURVIVES (it can NOT consume a permit the + /// authenticated POST needs — the reservation jatmn asked for). + /// Revert the branch to `git_write_semaphore` and BOTH flip: the advert-pool-0 + /// case stops shedding and the write-pool-0 case starts shedding (the exact + /// anon-sheds-authed-push starvation). + #[sqlx::test] + async fn receive_pack_advertisement_draws_from_dedicated_advert_pool(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + // Build a fresh state with the three pools sized independently, then drive one + // info/refs advertisement for `service` and return its handler status. + async fn advert_status( + pool: &sqlx::PgPool, + read_permits: usize, + write_permits: usize, + advert_permits: usize, + service: &str, + ) -> StatusCode { + let mut state = crate::test_support::test_state(pool.clone()).await; + state.git_read_semaphore = Arc::new(Semaphore::new(read_permits)); + state.git_write_semaphore = Arc::new(Semaphore::new(write_permits)); + state.git_push_advert_semaphore = Arc::new(Semaphore::new(advert_permits)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; state - .create_ip_rate_limiter - .check(&peer.ip().to_string()) + .db + .upsert_mirror_repo("z6wpadv", "wp", "/tmp/wp-nonexistent", None, false) .await + .unwrap(); + let peer: SocketAddr = "203.0.113.61:6000".parse().unwrap(); + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::GET) + .uri(format!("/z6wpadv/wp/info/refs?service={service}")) + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + router.oneshot(req).await.unwrap().status() + } + + // Advert pool saturated (read + write free): the receive-pack advert SHEDS, + // proving it is bound to the dedicated advert pool. + assert_eq!( + advert_status(&pool, 8, 8, 0, "git-receive-pack").await, + StatusCode::SERVICE_UNAVAILABLE, + "receive-pack advertisement draws from the dedicated advert pool: a saturated advert pool sheds it 503" + ); + // WRITE pool saturated (advert + read free): the advert SURVIVES. This is the + // reservation — an advert flood can never occupy a permit the authenticated + // push POST relies on at admission. + assert_ne!( + advert_status(&pool, 8, 0, 8, "git-receive-pack").await, + StatusCode::SERVICE_UNAVAILABLE, + "receive-pack advertisement must NOT draw from the write pool: a saturated write pool must not shed it" + ); + // Read pool saturated (advert + write free): the advert SURVIVES (never on the read pool). + assert_ne!( + advert_status(&pool, 0, 8, 8, "git-receive-pack").await, + StatusCode::SERVICE_UNAVAILABLE, + "receive-pack advertisement must not draw from the read pool" ); + // Read pool saturated: the upload-pack advertisement still SHEDS (unchanged). + assert_eq!( + advert_status(&pool, 0, 8, 8, "git-upload-pack").await, + StatusCode::SERVICE_UNAVAILABLE, + "upload-pack advertisement stays on the read pool: a saturated read pool sheds it 503" + ); + // Write + advert pools saturated, read free: the upload-pack advertisement is + // UNAFFECTED, proving reads never touch either write-side pool. + assert_ne!( + advert_status(&pool, 8, 0, 0, "git-upload-pack").await, + StatusCode::SERVICE_UNAVAILABLE, + "upload-pack advertisement never touches the write or advert pool" + ); + } + + /// #174 U2: the receive-pack advertisement is a write-path op, so it must not be + /// shed by the READ per-caller sub-cap even when the caller's source IP has + /// exhausted its read budget (e.g. concurrent clones from the same host). Fill + /// the IP's read per-caller slot, then the receive-pack advertisement from that + /// same IP must still get through. Restore the unconditional read-cap acquire on + /// the receive-pack branch and this goes 503. + #[sqlx::test] + async fn receive_pack_advertisement_ignores_read_per_caller_cap(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6wpc", "wp", "/tmp/wp-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.71:6000".parse().unwrap(); + // Exhaust the source IP's single READ per-caller slot, as concurrent clones + // from the same host would. + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("fill the IP's read per-caller slot"); let router = crate::server::build_router(state); let mut req = Request::builder() - .method(Method::POST) - .uri("/api/v1/repos") - .header("content-type", "application/json") - .body(Body::from(r#"{"name":"flood","is_public":true}"#)) + .method(Method::GET) + .uri("/z6wpc/wp/info/refs?service=git-receive-pack") + .body(Body::empty()) .unwrap(); req.extensions_mut().insert(ConnectInfo(peer)); + assert_ne!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "receive-pack advertisement must not be shed by the read per-caller cap: it is a write-path op" + ); + } - let status = router.oneshot(req).await.unwrap().status(); + /// #174 (review fix): the anon-reachable receive-pack advertisement draws from + /// the write pool, so it is bounded per source by `git_push_advert_per_caller` to + /// stop one source from monopolizing the write pool and shedding authenticated + /// pushes. Fill one source IP's advert slot; its next receive-pack advertisement + /// sheds 503, while a different source and the upload-pack advertisement are + /// unaffected. Remove the advert-cap acquisition and the same-source assertion + /// goes green-not-503. + #[sqlx::test] + async fn receive_pack_advertisement_capped_per_source(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_push_advert_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6advcap", "ac", "/tmp/ac-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.81:6000".parse().unwrap(); + // Fill this source IP's single receive-pack-advertisement slot. + let _slot = state + .git_push_advert_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first advert slot for this source IP"); + + // Same source: the receive-pack advertisement sheds 503 (advert cap full). + let router = crate::server::build_router(state.clone()); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6advcap/ac/info/refs?service=git-receive-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); assert_eq!( - status, - StatusCode::TOO_MANY_REQUESTS, - "repo creation must be IP-throttled before signature verification" + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a source at its receive-pack advertisement cap must shed 503, so it cannot monopolize the write pool" + ); + + // A DIFFERENT source keeps its own advert budget -> not shed. + let other: SocketAddr = "203.0.113.82:6000".parse().unwrap(); + let router2 = crate::server::build_router(state.clone()); + let mut req2 = Request::builder() + .method(Method::GET) + .uri("/z6advcap/ac/info/refs?service=git-receive-pack") + .body(Body::empty()) + .unwrap(); + req2.extensions_mut().insert(ConnectInfo(other)); + assert_ne!( + router2.oneshot(req2).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a different source must keep its own receive-pack advertisement budget" ); + + // The upload-pack advertisement is NOT bounded by the receive-pack advert cap. + let router3 = crate::server::build_router(state); + let mut req3 = Request::builder() + .method(Method::GET) + .uri("/z6advcap/ac/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req3.extensions_mut().insert(ConnectInfo(peer)); + assert_ne!( + router3.oneshot(req3).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "the upload-pack advertisement must not be shed by the receive-pack advert cap" + ); + } + + /// #174 SC2 (info_refs probe): the per-caller read sub-cap sheds a caller that + /// is already at its concurrency budget on the upload-pack advertisement, while + /// a DIFFERENT caller still enters. Remove the sub-cap from `git_info_refs` and + /// the same-caller assertion goes green-not-503 — this is the info_refs half of + /// the two-handler mutation probe. + #[sqlx::test] + async fn info_refs_per_caller_cap_sheds_one_caller_not_others(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6pcadv", "pc", "/tmp/pc-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.31:5000".parse().unwrap(); + // Fill this caller's single read slot (a clone shares the Arc-backed map). + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first slot for this caller"); + + // Same caller (IP) at its cap -> shed 503 before the git/Tigris work. + let router = crate::server::build_router(state.clone()); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6pcadv/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + assert_eq!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a caller already at its per-caller read cap must shed the advertisement with 503" + ); + + // A DIFFERENT caller (IP) has its own budget -> not shed by the per-caller cap. + let other: SocketAddr = "203.0.113.32:5000".parse().unwrap(); + let router2 = crate::server::build_router(state.clone()); + let mut req2 = Request::builder() + .method(Method::GET) + .uri("/z6pcadv/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req2.extensions_mut().insert(ConnectInfo(other)); + assert_ne!( + router2.oneshot(req2).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a different caller must not be shed by another caller's saturated budget" + ); + } + + /// #174 SC2 (upload_pack probe): the same per-caller shed on the POST + /// upload-pack path. Remove the sub-cap from `git_upload_pack` and this goes + /// green-not-503 — the upload_pack half of the two-handler mutation probe. + #[sqlx::test] + async fn upload_pack_per_caller_cap_sheds_one_caller_not_others(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6pcupl", "pc", "/tmp/pc-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.41:5000".parse().unwrap(); + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first slot for this caller"); + + let router = crate::server::build_router(state.clone()); + let mut req = Request::builder() + .method(Method::POST) + .uri("/z6pcupl/pc/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + assert_eq!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a caller already at its per-caller read cap must shed upload-pack with 503" + ); + + let other: SocketAddr = "203.0.113.42:5000".parse().unwrap(); + let router2 = crate::server::build_router(state.clone()); + let mut req2 = Request::builder() + .method(Method::POST) + .uri("/z6pcupl/pc/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + req2.extensions_mut().insert(ConnectInfo(other)); + assert_ne!( + router2.oneshot(req2).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a different caller must not be shed by another caller's saturated budget" + ); + } + + /// #174 (review fix): the per-source caller cap is an independent brake that + /// sheds a capped source even when the global pool has free capacity — the + /// sub-cap is not a mere pre-filter for pool exhaustion. Proven by leaving the + /// global read pool with capacity (so the pre-DB early shed passes) AND + /// pre-holding the source's upload-pack read sub-cap: the request reaches the + /// caller cap and sheds there, so its 503 body reads "for this caller". Remove + /// the `acquire_read_caller_permit` call and the capped source falls through to + /// the git op instead of shedding with "for this caller" — this is the + /// caller-cap acquire probe for the info/refs upload-pack branch. + #[sqlx::test] + async fn info_refs_upload_pack_per_source_cap_sheds_with_global_capacity(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + // Global read pool has free capacity (early shed passes); source pre-held at + // its per-caller cap so it sheds on the caller cap, not the global pool. + state.git_read_semaphore = Arc::new(Semaphore::new(4)); + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6ordir", "oi", "/tmp/oi-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.91:5000".parse().unwrap(); + // Pin this source at its single upload-pack read slot. + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first read slot for this source IP"); + + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6ordir/oi/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a source at its read sub-cap must shed 503 even with global pool capacity" + ); + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .expect("body bytes"); + let body = String::from_utf8_lossy(&bytes); + assert!( + body.contains("for this caller"), + "the per-source cap is an independent brake: with global capacity free, the capped source must still shed with the caller-cap body, got {body}" + ); + } + + /// #174 (review fix): same independent-brake guarantee for the receive-pack + /// advertisement branch of info/refs — its per-source cap + /// (`git_push_advert_per_caller`) sheds a capped source even when the global + /// write pool has capacity. Leave the global write pool with capacity (so the + /// pre-DB early shed passes) and pre-hold the source's advert slot: the request + /// reaches the caller cap, so the 503 body reads "for this caller". Remove the + /// caller-cap acquire and the capped source falls through instead of shedding + /// with "for this caller". The push rate limiter is left permissive so the + /// request reaches the caller cap. + #[sqlx::test] + async fn info_refs_receive_pack_per_source_cap_sheds_with_global_capacity(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + // Global write pool has free capacity (early shed passes); source pre-held at + // its advert sub-cap so it sheds on the caller cap, not the global pool. + state.git_write_semaphore = Arc::new(Semaphore::new(4)); + state.git_push_advert_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + // Permissive push rate limiter so the advertisement passes the rate gate and + // reaches the per-source concurrency cap. + state.push_rate_limiter = crate::rate_limit::RateLimiter::new(100, Duration::from_secs(60)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6ordrp", "or", "/tmp/or-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.92:5000".parse().unwrap(); + // Pin this source at its single receive-pack advertisement slot. + let _slot = state + .git_push_advert_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first advert slot for this source IP"); + + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6ordrp/or/info/refs?service=git-receive-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a source at its advert sub-cap must shed 503 even with global write pool capacity" + ); + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .expect("body bytes"); + let body = String::from_utf8_lossy(&bytes); + assert!( + body.contains("for this caller"), + "the per-source advert cap is an independent brake: with global write capacity free, the capped source must still shed with the caller-cap body, got {body}" + ); + } + + /// #174 (review fix): same independent-brake guarantee for the POST upload-pack + /// handler — its per-source read cap sheds a capped source even when the global + /// read pool has capacity. Leave the global read pool with capacity (so the + /// pre-DB early shed passes) and pre-hold the source's read slot: the request + /// reaches the caller cap, so the 503 body reads "for this caller". Remove the + /// caller-cap acquire and the capped source falls through instead of shedding + /// with "for this caller". + #[sqlx::test] + async fn upload_pack_per_source_cap_sheds_with_global_capacity(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + // Global read pool has free capacity (early shed passes); source pre-held at + // its per-caller cap so it sheds on the caller cap, not the global pool. + state.git_read_semaphore = Arc::new(Semaphore::new(4)); + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6ordup", "ou", "/tmp/ou-nonexistent", None, false) + .await + .unwrap(); + + let peer: SocketAddr = "203.0.113.93:5000".parse().unwrap(); + // Pin this source at its single read slot. + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first read slot for this source IP"); + + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::POST) + .uri("/z6ordup/ou/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a source at its read sub-cap must shed 503 even with global pool capacity" + ); + let bytes = axum::body::to_bytes(resp.into_body(), usize::MAX) + .await + .expect("body bytes"); + let body = String::from_utf8_lossy(&bytes); + assert!( + body.contains("for this caller"), + "the per-source cap is an independent brake: with global capacity free, the capped source must still shed with the caller-cap body, got {body}" + ); + } + + /// #174 U3 (P1-b, RED-before/GREEN-after): a client disconnect during the + /// path-scoped withheld-blob walk must NOT release the read admission while the + /// uncancellable `spawn_blocking` walk is still running. The handler takes the + /// global read permit, enters the walk (a fake git hangs on rev-list), then the + /// request future is dropped mid-walk. With both permits moved into the blocking + /// task the global slot stays occupied until the walk finishes; on the pre-fix code + /// the handler-local permits drop on future-drop and the slot frees instantly (RED), + /// letting disconnect-spam exceed the cap while real git work keeps running. + #[sqlx::test] + async fn upload_pack_permit_held_through_walk_after_disconnect(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let tmp = tempfile::TempDir::new().unwrap(); + let revlist_pid = tmp.path().join("revlist.pid"); + // Fake git: resolve refs fast, hang on rev-list (recording its pid first). The + // ~6s sleep bounds the walk so a broken fix cannot wedge the suite. + let body = format!( + "#!/bin/sh\ncase \"$1\" in\n rev-list) echo $$ > \"{}\" ; sleep 6 ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n", + revlist_pid.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + // Root the repo store at this test's TempDir so the bare repo is isolated per + // run (the default for_testing store uses a fixed /tmp path that would collide + // across runs). + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.git_read_semaphore = Arc::new(Semaphore::new(1)); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let owner = "z6up3rd"; + let name = "up3"; + state + .db + .upsert_mirror_repo(owner, name, "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + // Real bare repo at the path acquire() computes, so the handler reaches the walk. + state + .repo_store + .init(&rec.owner_did, &rec.name) + .await + .unwrap(); + // A path-scoped rule so has_path_scoped_rule() is true (the walk path) without + // denying the "/" gate for the public repo. + state + .db + .set_visibility_rule( + &rec.id, + "src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkU3ReaderAAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + + let sem = state.git_read_semaphore.clone(); + assert_eq!( + sem.available_permits(), + 1, + "one read slot before the request" + ); + + let router = crate::server::build_router(state); + let peer: SocketAddr = "203.0.113.77:5000".parse().unwrap(); + let mut req = Request::builder() + .method(Method::POST) + .uri(format!("/{owner}/{name}/git-upload-pack")) + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + + let mut fut = Box::pin(router.oneshot(req)); + // Drive until the walk's rev-list starts (its pidfile appears) — i.e. the + // request is inside the spawn_blocking walk, holding the global read permit. + let mut in_walk = false; + for _ in 0..500 { + let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if revlist_pid.exists() { + in_walk = true; + break; + } + } + assert!( + in_walk, + "the walk's rev-list must start (request reached the spawn_blocking walk)" + ); + assert_eq!( + sem.available_permits(), + 0, + "the read slot is held while the walk runs" + ); + + // Client disconnect: drop the request future mid-walk. + drop(fut); + + // Load-bearing: the slot must STAY held while the uncancellable walk runs. On + // the pre-fix code the handler-local permits drop here and the slot frees at + // once (RED). + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + assert_eq!( + sem.available_permits(), + 0, + "on disconnect the read admission must be held until the spawn_blocking walk \ + finishes, not released the instant the future drops (P1-b)" + ); + + // Cleanup: let the walk finish so the slot releases and no blocking task leaks. + for _ in 0..400 { + if sem.available_permits() == 1 { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + if let Some(p) = std::fs::read_to_string(&revlist_pid) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + unsafe { + libc::kill(p, libc::SIGKILL); + } + } + } + + /// #174 U1 (P1-a, plain-spawn residual, RED-before/GREEN-after): on the PLAIN + /// (non-path-scoped) upload-pack path a client disconnect must NOT release the + /// global read admission while the detached process-group reaper is still tearing + /// down a git group that ignores SIGTERM. The `be0cdd6` fix moved permits into the + /// path-scoped `spawn_blocking` walk; this closes the residual plain path, where the + /// permits were handler-locals that dropped the instant the future was dropped. + /// + /// Isolate the GLOBAL pool: read pool = 1, per-source cap + rate limiter permissive, + /// so the only thing that can shed a replacement is the leaked global permit. Drive + /// the handler until git spawns, disconnect, then assert the global slot stays held + /// (`available_permits() == 0`) AND a replacement sheds 503 while the group is alive; + /// after the reaper SIGKILLs+reaps the group the slot frees and a replacement is no + /// longer shed by the global cap. On the pre-fix code the handler-local permit drops + /// on future-drop and the slot frees at once (RED). + #[cfg(unix)] + #[sqlx::test] + async fn upload_pack_plain_permit_held_through_group_reap_after_disconnect(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let tmp = tempfile::TempDir::new().unwrap(); + let descfile = tmp.path().join("desc.pid"); + // Fake git for the plain upload-pack path (invoked as `git upload-pack + // --stateless-rpc `). It forks a descendant that TRAPS SIGTERM, records its + // pid, and loops ~20s, then `wait`s — so on disconnect the group leader dies on + // the reaper's SIGTERM but the descendant survives until the reaper escalates to + // SIGKILL, keeping the group alive (ESRCH not reached) across the observation + // window. Bounded so a broken fix leaks no permanent orphan. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + upload-pack)\n\ + sh -c 'trap \"\" TERM; echo $$ > \"{}\"; i=0; while [ $i -lt 20 ]; do sleep 1; i=$((i+1)); done' &\n\ + wait ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + descfile.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + // Isolate the global read pool: size 1; per-source cap + rate limiter permissive + // so only the leaked global permit can shed the replacement. + state.git_read_semaphore = Arc::new(Semaphore::new(1)); + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let owner = "z6up1st"; + let name = "up1"; + state + .db + .upsert_mirror_repo(owner, name, "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + // Real bare repo at the path acquire() computes, so the handler reaches the + // spawn. No path-scoped rule -> the PLAIN serve branch (this test's target). + state + .repo_store + .init(&rec.owner_did, &rec.name) + .await + .unwrap(); + + let sem = state.git_read_semaphore.clone(); + assert_eq!( + sem.available_permits(), + 1, + "one read slot before the request" + ); + + let router = crate::server::build_router(state); + let make_req = |peer: SocketAddr| { + let mut req = Request::builder() + .method(Method::POST) + .uri(format!("/{owner}/{name}/git-upload-pack")) + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req + }; + + let peer: SocketAddr = "203.0.113.71:5000".parse().unwrap(); + let mut fut = Box::pin(router.clone().oneshot(make_req(peer))); + // Drive until git spawns (the descendant records its pid) — the request is + // inside the plain serve, holding the global read permit. Stop polling the + // instant the future completes (re-polling a completed oneshot panics); read the + // descfile first so a spawn that recorded its pid then returned is still caught. + let mut spawned: Option = None; + let mut early = None; + for _ in 0..500 { + let done = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&descfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + spawned = Some(p); + break; + } + if let Ok(resp) = done { + early = Some(resp.map(|r| r.status())); + break; + } + } + let desc = spawned + .unwrap_or_else(|| panic!("the fake git must have spawned; early finish: {early:?}")); + // Kill the descendant regardless of outcome so a RED run leaks no orphan. + struct ReapOnDrop(i32); + impl Drop for ReapOnDrop { + fn drop(&mut self) { + unsafe { + libc::kill(self.0, libc::SIGKILL); + } + } + } + let _cleanup = ReapOnDrop(desc); + assert!( + unsafe { libc::kill(desc, 0) == 0 }, + "descendant should be running before the disconnect" + ); + assert_eq!( + sem.available_permits(), + 0, + "the read slot is held while the git op runs" + ); + + // Client disconnect: drop the request future. The detached reaper now owns the + // AdmissionGuard and will not drop it until the group is ESRCH-confirmed reaped. + drop(fut); + + // Load-bearing: the slot must STAY held while the SIGTERM-ignoring group is still + // alive. On the pre-fix code the handler-local permit drops here and the slot + // frees at once (RED). Check quickly (before the reaper's ~2s SIGKILL escalation). + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + assert!( + unsafe { libc::kill(desc, 0) == 0 }, + "the SIGTERM-ignoring descendant must still be alive during the hold window" + ); + assert_eq!( + sem.available_permits(), + 0, + "on disconnect the read admission must be HELD until the process group is \ + reaped, not released the instant the future drops (P1-a)" + ); + // A replacement request from a DIFFERENT source must shed 503 — the only pool + // that can shed it is the leaked global permit (per-source cap is permissive). + let peer2: SocketAddr = "203.0.113.72:5000".parse().unwrap(); + let resp = router.clone().oneshot(make_req(peer2)).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "while the prior group is still alive the held global permit must shed a \ + replacement with 503" + ); + + // After the reaper SIGKILLs + reaps the group the AdmissionGuard drops and the + // slot frees. Poll for recovery. + let mut freed = false; + for _ in 0..400 { + if sem.available_permits() == 1 { + freed = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + assert!( + freed, + "once the reaper confirms the group gone the admission guard must drop and \ + free the global slot" + ); + // A replacement is now no longer shed by the global cap (it proceeds past + // admission; it then fails downstream on the fake git, which is not a 503). + let peer3: SocketAddr = "203.0.113.73:5000".parse().unwrap(); + let resp = router.oneshot(make_req(peer3)).await.unwrap(); + assert_ne!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "after the group is reaped the freed slot must admit a replacement" + ); + } + + /// F1 (filtered-serve residual of #174 P1-a, RED-before/GREEN-after): on the + /// FILTERED (path-scoped, non-empty withheld) upload-pack branch a client + /// disconnect mid-pack-objects must NOT release the read admission while the + /// detached reaper is still tearing down the git group. Pre-fix the handler held + /// the permits as locals (`_hold`) across `upload_pack_excluding`, so dropping + /// the future released them instantly and disconnect-spam could exceed the read + /// caps during each reap window (RED). The fix threads the AdmissionGuard through + /// both filtered-pack stages so it rides `KillGroupOnDrop` into the reaper. + /// + /// Same isolation as the plain-path test above: read pool = 1, per-source cap + /// permissive, so only the global permit can shed a replacement. The fake git + /// serves the withheld walk (for-each-ref/rev-parse/rev-list/ls-tree) with a blob + /// under the denied `/src/**` subtree so the filtered branch is taken, answers + /// the pack build's rev-list fast, and hangs pack-objects in a SIGTERM-trapping + /// descendant. The descendant hang is first-invocation-only (keyed on the + /// pidfile's existence) so the post-reap replacement request completes fast. + #[cfg(unix)] + #[sqlx::test] + async fn upload_pack_filtered_permit_held_through_group_reap_after_disconnect( + pool: sqlx::PgPool, + ) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let tmp = tempfile::TempDir::new().unwrap(); + let descfile = tmp.path().join("desc.pid"); + let commit = "1111111111111111111111111111111111111111"; + let blob = "2222222222222222222222222222222222222222"; + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + rev-parse) echo {commit} ;;\n\ + rev-list) echo {commit} ;;\n\ + ls-tree) printf '100644 blob {blob}\\tsrc/x.txt' ;;\n\ + pack-objects)\n\ + if [ ! -e \"{desc}\" ]; then\n\ + sh -c 'trap \"\" TERM; echo $$ > \"{desc}\"; i=0; while [ $i -lt 20 ]; do sleep 1; i=$((i+1)); done' &\n\ + wait\n\ + fi ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + desc = descfile.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.path().join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + // Isolate the global read pool: size 1; per-source cap + rate limiter permissive + // so only the leaked global permit can shed the replacement. + state.git_read_semaphore = Arc::new(Semaphore::new(1)); + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.git_bin = git_path.to_str().unwrap().to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let owner = "z6upf1"; + let name = "upf"; + state + .db + .upsert_mirror_repo(owner, name, "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + // Real bare repo at the path acquire() computes, so the handler reaches the + // walk and the filtered serve. + state + .repo_store + .init(&rec.owner_did, &rec.name) + .await + .unwrap(); + // Path-scoped rule denying the anonymous caller under /src, matching the + // fake ls-tree's blob path, so the withheld set is NON-EMPTY and the + // filtered (upload_pack_excluding) branch is taken. + state + .db + .set_visibility_rule( + &rec.id, + "/src/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkUF1ReaderAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + + let sem = state.git_read_semaphore.clone(); + assert_eq!( + sem.available_permits(), + 1, + "one read slot before the request" + ); + + let router = crate::server::build_router(state); + let make_req = |peer: SocketAddr| { + let mut req = Request::builder() + .method(Method::POST) + .uri(format!("/{owner}/{name}/git-upload-pack")) + .body(Body::from(&b"0000"[..])) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req + }; + + let peer: SocketAddr = "203.0.113.81:5000".parse().unwrap(); + let mut fut = Box::pin(router.clone().oneshot(make_req(peer))); + // Drive until the pack-objects descendant records its pid: the request is + // past the walk, inside the filtered serve's stage 2, holding the read permit. + // Stop polling the instant the future completes (re-polling a completed + // oneshot panics); read the descfile first so a spawn that recorded its pid + // then returned is still caught. + let mut spawned: Option = None; + let mut early = None; + for _ in 0..500 { + let done = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&descfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + spawned = Some(p); + break; + } + if let Ok(resp) = done { + early = Some(resp.map(|r| r.status())); + break; + } + } + let desc = spawned.unwrap_or_else(|| { + panic!("the fake pack-objects must have spawned; early finish: {early:?}") + }); + // Kill the descendant regardless of outcome so a RED run leaks no orphan. + struct ReapOnDrop(i32); + impl Drop for ReapOnDrop { + fn drop(&mut self) { + unsafe { + libc::kill(self.0, libc::SIGKILL); + } + } + } + let _cleanup = ReapOnDrop(desc); + assert!( + unsafe { libc::kill(desc, 0) == 0 }, + "descendant should be running before the disconnect" + ); + assert_eq!( + sem.available_permits(), + 0, + "the read slot is held while the filtered serve runs" + ); + + // Client disconnect: drop the request future mid-pack-objects. The detached + // reaper must now own the AdmissionGuard and hold it until ESRCH. + drop(fut); + + // Load-bearing: the slot must STAY held while the SIGTERM-ignoring group is + // still alive. On the pre-fix code the handler-local `_hold` drops here and + // the slot frees at once (RED). Check quickly (before the reaper's ~2s + // SIGKILL escalation). + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + assert!( + unsafe { libc::kill(desc, 0) == 0 }, + "the SIGTERM-ignoring descendant must still be alive during the hold window" + ); + assert_eq!( + sem.available_permits(), + 0, + "on disconnect the read admission must be HELD until the filtered serve's \ + process group is reaped, not released the instant the future drops (F1)" + ); + // A replacement request from a DIFFERENT source must shed 503: the only pool + // that can shed it is the held global permit (per-source cap is permissive). + let peer2: SocketAddr = "203.0.113.82:5000".parse().unwrap(); + let resp = router.clone().oneshot(make_req(peer2)).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "while the prior group is still alive the held global permit must shed a \ + replacement with 503" + ); + + // After the reaper SIGKILLs + reaps the group the AdmissionGuard drops and + // the slot frees. Poll for recovery. + let mut freed = false; + for _ in 0..400 { + if sem.available_permits() == 1 { + freed = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + assert!( + freed, + "once the reaper confirms the group gone the admission guard must drop and \ + free the global slot" + ); + // A replacement is now admitted and completes: the fake pack-objects takes + // its fast path (the descfile exists), so the filtered serve returns instead + // of hanging. + let peer3: SocketAddr = "203.0.113.83:5000".parse().unwrap(); + let resp = router.oneshot(make_req(peer3)).await.unwrap(); + assert_ne!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "after the group is reaped the freed slot must admit a replacement" + ); + } + + /// #174 U1 (P1-a): the `None`-key arm — a request with no resolvable source key + /// (no trusted-proxy header, no peer) is bounded by the GLOBAL read pool only, never + /// a per-source cap. With the global read pool exhausted such a request still sheds + /// 503, proving the plain path admits/sheds on the global pool for the `None` arm + /// (the counterpart to the `Some(ip)` arm above). Complements the resolver-arm rule: + /// neither arm is vacuous. + #[tokio::test] + async fn upload_pack_plain_none_key_arm_sheds_on_global_pool() { + use axum::body::Body; + use axum::http::{Method, Request, StatusCode}; + use axum::Router; + use std::sync::Arc; + use tokio::sync::Semaphore; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state_lazy(); + // Global read pool exhausted; per-source cap permissive so only the global pool + // can shed. No ConnectInfo + no trusted header -> read_caller_key resolves None. + state.git_read_semaphore = Arc::new(Semaphore::new(0)); + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let router = Router::new() + .route( + "/{owner}/{repo}/git-upload-pack", + axum::routing::post(crate::api::repos::git_upload_pack), + ) + .with_state(state); + // No ConnectInfo extension and no XFF header: the caller key is None. + let req = Request::builder() + .method(Method::POST) + .uri("/alice/repo.git/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + let resp = router.oneshot(req).await.unwrap(); + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "a None-key request must still shed 503 on the exhausted GLOBAL read pool" + ); + } + + /// #174 U4 (P1-d, RED-before/GREEN-after): the authenticated receive-pack POST + /// carries a per-source WRITE sub-cap so one source IP cannot monopolize the write + /// pool with many slow pushes (owner enforcement defaults off, so disposable DIDs + /// are free). Global write pool has capacity; the source is pre-held at its single + /// write slot. A push from THAT source sheds (Overloaded/503) — which also proves + /// the PeerAddr+HeaderMap extractors resolve a key (without them the key is None and + /// the cap is inert, never shedding). A push from a DIFFERENT source is NOT shed by + /// the cap. Called directly so the test needs no signed request; the handler is + /// where the cap lives. Remove the `git_write_per_caller` acquire and the capped + /// source no longer sheds (RED). + #[sqlx::test] + async fn receive_pack_per_source_write_cap_sheds_capped_source_not_others(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + + let mut state = crate::test_support::test_state(pool).await; + // Global write pool has capacity; the per-source cap is 1. + state.git_write_semaphore = Arc::new(Semaphore::new(4)); + state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6rp4wr", "rp4", "/tmp/rp4-nonexistent", None, false) + .await + .unwrap(); + + let did = "did:key:z6MkReceivePackWriteCapProofDidAAAAAAAAAA"; + let capped: SocketAddr = "203.0.113.44:5000".parse().unwrap(); + let other: SocketAddr = "203.0.113.45:5000".parse().unwrap(); + + // Pin the capped source at its single write slot. + let _slot = state + .git_write_per_caller + .try_acquire(&capped.ip().to_string()) + .expect("first write slot for the capped source IP"); + + // A push from the capped source must shed on the per-source write cap even with + // global write capacity free. The shed also proves the source-IP key resolved + // via the extractors (an inert None key would fall through to Ok(None)). + let capped_result = git_receive_pack( + State(state.clone()), + Path(("z6rp4wr".to_string(), "rp4".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(capped)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await; + assert!( + matches!(capped_result, Err(AppError::Overloaded(_))), + "a source at its per-source write cap must shed (Overloaded/503) with global \ + pool capacity free; got {capped_result:?}" + ); + + // A push from a DIFFERENT source must NOT be shed by the per-source cap — it + // proceeds past admission (and fails later on the nonexistent repo, which is not + // an Overloaded error). + let other_result = git_receive_pack( + State(state.clone()), + Path(("z6rp4wr".to_string(), "rp4".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(other)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await; + assert!( + !matches!(other_result, Err(AppError::Overloaded(_))), + "a different source must not be shed by the per-source write cap while the \ + capped source holds its slot; got {other_result:?}" + ); + } + + /// #174 U2 (P1-2, RED-before/GREEN-after): the storage-acquisition phase is bounded + /// by `git_acquire_timeout_secs`, so a stalled backend releases the admission permit + /// and sheds a 503 instead of pinning the pool. The permit is taken BEFORE + /// `acquire_write`, whose advisory-lock loop can spin ~60s (and whose per-iteration + /// `pg_try_advisory_lock` can block indefinitely on a hung pool), so without the + /// `tokio::time::timeout` wrapper the permit is held far past the deadline. + /// + /// Real stall (no `RepoStore` trait to fake): hold the SAME session-level advisory + /// lock `acquire_write` derives (`advisory_lock_key(owner_slug, repo_name)`, where + /// `owner_slug = owner_did.replace([':','/'], "_")`) on a second pooled connection, + /// so the handler's `pg_try_advisory_lock` returns false every iteration and the loop + /// must retry against the deadline. `git_acquire_timeout_secs = 2`; the request must + /// return 503 (Overloaded) at ~2s (NOT ~59s), and the write permit must be released + /// (`available_permits()` recovers to full once the shed returns). Covers R2. + /// + /// Load-bearing / mutation: remove the `tokio::time::timeout` wrapper on + /// `acquire_write` and the loop runs to ~59s with the permit held the whole time — + /// the `< DEADLINE_CEILING` timing assertion goes RED (observed ~59s) and the permit + /// stays pinned past the deadline. Restore to return GREEN. + #[sqlx::test] + async fn receive_pack_acquire_deadline_sheds_and_releases_permit(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + + // Reproduce acquire_write's session-level advisory-lock key exactly so the + // second-connection lock collides with the handler's pg_try_advisory_lock + // (repo_store.rs: advisory_lock_key over owner_slug then repo_name). + fn advisory_lock_key(owner_slug: &str, repo_name: &str) -> i64 { + use std::hash::{Hash, Hasher}; + let mut hasher = std::collections::hash_map::DefaultHasher::new(); + owner_slug.hash(&mut hasher); + repo_name.hash(&mut hasher); + hasher.finish() as i64 + } + + let owner = "z6acqdead"; + let name = "acq1"; + // owner_slug as local_path() computes it from the record's owner_did. The + // mirror row stores the short owner as owner_did, so slug == owner (no ':'/'/'). + let owner_slug = owner.replace([':', '/'], "_"); + let lock_key = advisory_lock_key(&owner_slug, name); + + let mut state = crate::test_support::test_state(pool.clone()).await; + // Isolate the write pool at size 1 so available_permits() cleanly reports + // held (0) vs released (1). Per-source cap + trust permissive so only the + // write pool / acquire path can gate. + state.git_write_semaphore = Arc::new(Semaphore::new(1)); + state.git_write_per_caller = crate::rate_limit::PerCallerConcurrency::new(1000, 1000); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + // Short acquire deadline: the fix must shed here, well before acquire_write's + // ~59s advisory-lock loop would bail on its own. + const ACQUIRE_TIMEOUT_SECS: u64 = 2; + let mut cfg = (*state.config).clone(); + cfg.git_acquire_timeout_secs = ACQUIRE_TIMEOUT_SECS; + // Keep the git-service timeout large so the deadline under test is the acquire + // one, not git execution (which is never reached on the stalled path anyway). + cfg.git_service_timeout_secs = 600; + state.config = std::sync::Arc::new(cfg); + + state + .db + .upsert_mirror_repo(owner, name, "/tmp/z6acqdead-acq1", None, false) + .await + .unwrap(); + + // Hold the advisory lock on a dedicated pooled connection (a distinct session), + // so the handler's pg_try_advisory_lock($lock_key) returns false every iteration + // and acquire_write's real loop must retry against the deadline. Released when + // this connection drops at end of test. + let mut lock_conn = pool + .acquire() + .await + .expect("second connection for the lock"); + sqlx::query("SELECT pg_advisory_lock($1)") + .bind(lock_key) + .execute(&mut *lock_conn) + .await + .expect("hold the advisory lock on the second connection"); + + let did = "did:key:z6MkAcquireDeadlineProofDidAAAAAAAAAAAAAAAA"; + let peer: SocketAddr = "203.0.113.61:5000".parse().unwrap(); + + let sem = state.git_write_semaphore.clone(); + assert_eq!( + sem.available_permits(), + 1, + "one write slot before the request" + ); + + // Drive the authenticated push in the background so we can observe the permit is + // held while acquire_write stalls, then that it is released on the shed. + let state_for_task = state.clone(); + let start = std::time::Instant::now(); + let handle = tokio::spawn(async move { + git_receive_pack( + State(state_for_task), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(peer)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await + }); + + // The handler takes the write permit BEFORE acquire_write, so once it is stalled + // in the advisory-lock loop the pool reports 0 available. Wait for that to prove + // the permit is genuinely held during the stall (and the request really reached + // acquire_write, not an earlier reject). + let mut held = false; + for _ in 0..200 { + if sem.available_permits() == 0 { + held = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert!( + held, + "the write permit must be held while acquire_write stalls on the advisory lock" + ); + + // The bounded acquire deadline must shed with 503 (Overloaded), NOT wait out the + // ~59s advisory-lock loop. Ceiling is comfortably above the 2s deadline + task + // scheduling but far below 59s, so a RED run (no wrapper -> ~59s) fails here. + const DEADLINE_CEILING: std::time::Duration = std::time::Duration::from_secs(20); + let result = tokio::time::timeout( + DEADLINE_CEILING + std::time::Duration::from_secs(10), + handle, + ) + .await + .expect("the handler must return within the ceiling — a hang means the acquire deadline is missing (RED)") + .expect("the receive-pack task must not panic"); + let elapsed = start.elapsed(); + + assert!( + matches!(result, Err(AppError::Overloaded(_))), + "a stalled acquire_write must shed with Overloaded/503 at the acquire deadline; \ + got {result:?}" + ); + assert!( + elapsed < DEADLINE_CEILING, + "the shed must land at ~{ACQUIRE_TIMEOUT_SECS}s (the acquire deadline), not ~59s \ + (the advisory-lock loop). Observed {elapsed:?}; without the timeout wrapper this \ + is ~59s (RED)" + ); + + // Permit release on expiry: the Overloaded return drops the handler-local permit, + // so the isolated write pool must recover to full. A leaked permit here means the + // pool drains under a stalled backend (the #174 P1-2 bug). + let mut freed = false; + for _ in 0..200 { + if sem.available_permits() == 1 { + freed = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert!( + freed, + "on the acquire-deadline shed the write permit must be released; the pool did \ + not recover to full (permit leaked)" + ); + + // Follow-up admits once the contended lock is released: release the second-conn + // lock, then a fresh push proceeds PAST admission (it fails later on the + // nonexistent on-disk repo, which is NOT an Overloaded/503). Proves the freed + // slot is usable, not merely counted. + sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(lock_key) + .execute(&mut *lock_conn) + .await + .expect("release the advisory lock"); + let followup = git_receive_pack( + State(state.clone()), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some("203.0.113.62:5000".parse().unwrap())), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await; + assert!( + !matches!(followup, Err(AppError::Overloaded(_))), + "once the lock frees, a follow-up push must admit past the (recovered) write \ + pool and acquire; got {followup:?}" + ); + } + + /// #174 U5 (P1-e, RED-before/GREEN-after): the post-push encryption walk acquires a + /// `git_encrypt_semaphore` permit before running, so completed pushes cannot spawn + /// unbounded concurrent full-history walks. With the pool exhausted the gated walk + /// must DEFER (block on admission) and NOT run its rev-list; on the pre-fix code + /// (no acquire) the walk runs regardless of the pool (RED). It defers rather than + /// sheds — releasing the permit lets the SAME walk run and pin (durability stays + /// fail-closed). Exercises the gating seam directly; the detached push task calls + /// this exact helper. + #[tokio::test] + async fn encrypt_walk_defers_when_pool_exhausted() { + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let marker = tmp.path().join("revlist.ran"); + // Fake git records when rev-list runs (the walk's first git call). + let body = format!( + "#!/bin/sh\ncase \"$1\" in\n rev-list) echo ran > \"{}\" ;;\n *) : ;;\nesac\nexit 0\n", + marker.display() + ); + let git_path = tmp.path().join("fakegit"); + std::fs::write(&git_path, &body).unwrap(); + { + use std::os::unix::fs::PermissionsExt; + let mut perm = std::fs::metadata(&git_path).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&git_path, perm).unwrap(); + } + let git_bin = git_path.to_str().unwrap().to_string(); + let owner = "did:key:z6MkEncWalkOwnerAAAAAAAAAAAAAAAAAAAAAAAA".to_string(); + + // Exhaust the pool: hold its only permit so a gated walk must defer. + let sem = Arc::new(Semaphore::new(1)); + let held = sem.clone().acquire_owned().await.unwrap(); + + // Blocked: the gated walk must NOT complete or run rev-list while exhausted. + let blocked = tokio::time::timeout( + Duration::from_millis(500), + withheld_recipients_gated( + sem.clone(), + tmp.path().to_path_buf(), + git_bin.clone(), + Duration::from_secs(5), + Vec::new(), + true, + owner.clone(), + ), + ) + .await; + assert!( + blocked.is_err(), + "the encryption walk must defer (block on admission) when the pool is exhausted" + ); + assert!( + !marker.exists(), + "the walk's rev-list must not run while its admission permit is unavailable (P1-e)" + ); + + // Release admission: the SAME walk now runs (defer, not shed) — rev-list fires. + drop(held); + let ran = withheld_recipients_gated( + sem, + tmp.path().to_path_buf(), + git_bin, + Duration::from_secs(5), + Vec::new(), + true, + owner, + ) + .await; + assert!( + ran.is_ok(), + "with a permit the walk runs and joins: {ran:?}" + ); + assert!( + marker.exists(), + "once admission is available the deferred walk runs its rev-list" + ); + } + + /// F4 defer proof 1: `replication_withheld_set`'s WALK arm acquires a + /// `git_encrypt_semaphore` permit before its spawn_blocking git walk, deferring + /// (never shedding) when the pool is exhausted — while its no-walk fast paths + /// (no path-scoped rule; not announceable) complete WITHOUT touching the pool. + /// On ungated code the walk runs regardless of a zero-permit pool (RED). + #[cfg(unix)] + #[tokio::test] + async fn replication_walk_defers_when_scan_pool_exhausted() { + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let marker = tmp.path().join("git.ran"); + // Fake git records ANY invocation (the walk's first call is rev-parse), then + // behaves well enough for a successful empty walk: HEAD probe succeeds, + // rev-list lists no commits. + let body = format!( + "#!/bin/sh\necho ran >> \"{}\"\ncase \"$1\" in\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n", + marker.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + let scoped_rules = || Some(vec![vis_rule("/secret/**", &[])]); + + // Zero-permit pool: every gated walk must park forever. + let sem: Arc = Arc::new(Semaphore::new(0)); + + // Fast path A (negative arm): announceable, NO path-scoped rule -> zero git + // work, must complete immediately without acquiring from the empty pool. + let fast = tokio::time::timeout( + Duration::from_millis(500), + replication_withheld_set( + sem.clone(), + Some(vec![]), + OWNER_DID, + true, + tmp.path().to_path_buf(), + git_bin.clone(), + Duration::from_secs(5), + ), + ) + .await + .expect("the no-path-scoped-rule fast path must not park on the scan pool"); + assert_eq!(fast, (true, Some(std::collections::HashSet::new()))); + + // Fast path B (negative arm): not announceable (no rules) -> zero git work, + // must complete immediately without acquiring. + let fast = tokio::time::timeout( + Duration::from_millis(500), + replication_withheld_set( + sem.clone(), + None, + OWNER_DID, + false, + tmp.path().to_path_buf(), + git_bin.clone(), + Duration::from_secs(5), + ), + ) + .await + .expect("the not-announceable fast path must not park on the scan pool"); + assert_eq!(fast, (false, None)); + assert!(!marker.exists(), "the fast paths must spawn no git at all"); + + // Walk arm with the pool exhausted: must DEFER (park), spawning no git. + let blocked = tokio::time::timeout( + Duration::from_millis(500), + replication_withheld_set( + sem.clone(), + scoped_rules(), + OWNER_DID, + true, + tmp.path().to_path_buf(), + git_bin.clone(), + Duration::from_secs(5), + ), + ) + .await; + assert!( + blocked.is_err(), + "the withheld walk must defer (park on admission) when the pool is exhausted" + ); + assert!( + !marker.exists(), + "the withheld walk's git must not spawn while its admission permit is unavailable (F4)" + ); + + // Release admission: the SAME walk now runs (defer, not shed) and succeeds. + sem.add_permits(1); + let ran = replication_withheld_set( + sem, + scoped_rules(), + OWNER_DID, + true, + tmp.path().to_path_buf(), + git_bin, + Duration::from_secs(5), + ) + .await; + assert!( + marker.exists(), + "once admission is available the deferred withheld walk runs its git" + ); + assert_eq!( + ran, + (true, Some(std::collections::HashSet::new())), + "the released walk completes and vets the (empty) withheld set" + ); + } + + /// F4 defer proof 3: `fail_closed_full_scan_objects` ALWAYS walks, so its + /// spawn_blocking is always admission-gated: with the pool exhausted it defers + /// and spawns no git; with a permit the same call runs. Ungated it runs + /// regardless (RED). + #[cfg(unix)] + #[tokio::test] + async fn full_scan_pin_walk_defers_when_scan_pool_exhausted() { + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let marker = tmp.path().join("git.ran"); + let body = format!( + "#!/bin/sh\necho ran >> \"{}\"\ncase \"$1\" in\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n", + marker.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + let candidates = vec!["3333333333333333333333333333333333333333".to_string()]; + + let sem: Arc = Arc::new(Semaphore::new(0)); + let blocked = tokio::time::timeout( + Duration::from_millis(500), + fail_closed_full_scan_objects( + sem.clone(), + tmp.path().to_path_buf(), + vec![vis_rule("/secret/**", &[])], + true, + OWNER_DID.to_string(), + candidates.clone(), + git_bin.clone(), + Duration::from_secs(5), + ), + ) + .await; + assert!( + blocked.is_err(), + "the fail-closed full scan must defer (park on admission) when the pool is exhausted" + ); + assert!( + !marker.exists(), + "the full scan's git must not spawn while its admission permit is unavailable (F4)" + ); + + // Release admission: the SAME scan now runs (defer, not shed). + sem.add_permits(1); + let _objs = fail_closed_full_scan_objects( + sem, + tmp.path().to_path_buf(), + vec![vis_rule("/secret/**", &[])], + true, + OWNER_DID.to_string(), + candidates, + git_bin, + Duration::from_secs(5), + ) + .await; + assert!( + marker.exists(), + "once admission is available the deferred full scan runs its git" + ); + } + + /// #174 F4 (RED-before/GREEN-after): the two full-scan phases share ONE whole-scan + /// deadline. Phase 1 (`replicable_blob_set_bounded`) succeeds but consumes almost + /// the whole budget; phase 2 (`all_blob_oids`) then gets only the remainder. With a + /// shared deadline phase 2 is reaped and the scan fails closed (pins nothing) — the + /// safe direction. With a FRESH `Instant::now() + timeout` for phase 2 (pre-fix) it + /// gets a full second budget, completes with an empty blob set, and the non-blob + /// candidate is kept — so the result is NON-empty (RED) and the permit is held ~2x. + #[cfg(unix)] + #[tokio::test] + async fn full_scan_shares_one_deadline_across_both_phases() { + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + // Phase 1 (ls-tree) sleeps 1.5s and succeeds (empty tree); phase 2 + // (cat-file --batch-all-objects) sleeps 1.5s. With a 2s whole-scan budget the + // shared deadline leaves phase 2 only ~0.5s, so it is reaped; a fresh 2s budget + // would let it finish. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-parse) echo deadbeef ;;\n rev-list) echo aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa ;;\n ls-tree) sleep 1.5 ;;\n cat-file) case \"$*\" in *--batch-all-objects*) sleep 1.5 ;; *) : ;; esac ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + // A candidate that is NOT a blob (never appears in all_blob_oids): kept by + // replicable_objects_fail_closed only if phase 2 actually ran to completion. + let candidates = vec!["cccccccccccccccccccccccccccccccccccccccc".to_string()]; + + let sem: Arc = Arc::new(Semaphore::new(1)); + let objs = fail_closed_full_scan_objects( + sem, + tmp.path().to_path_buf(), + vec![vis_rule("/secret/**", &[])], + true, + OWNER_DID.to_string(), + candidates, + git_bin, + Duration::from_secs(2), + ) + .await; + assert!( + objs.is_empty(), + "a large-but-successful phase 1 must leave phase 2 only the SHARED whole-scan \ + remainder, so it reaps and the scan fails closed; got {objs:?} (a fresh phase-2 \ + budget completed the scan and kept the candidate — the ~2x-budget bug)" + ); + } + + /// #174 F6 (RED-before/GREEN-after): a post-push pin loop holds this push's full + /// object-id list while walking it, so concurrent pin loops across many repos must + /// be bounded by a global permit, not just the per-repo task count. `pin_new_objects_gated` + /// DEFERS (waits) when the pin pool is exhausted rather than running unbounded. + /// + /// Load-bearing: without the permit acquire the pin loop runs immediately even with + /// the pool held (RED — the deferral assertion fails). With it, it parks. + #[sqlx::test] + async fn pin_new_objects_gated_defers_when_pin_pool_exhausted(pool: sqlx::PgPool) { + use std::sync::Arc; + use tokio::sync::Semaphore; + + let state = crate::test_support::test_state(pool).await; + let db = state.db.clone(); + let tmp = tempfile::TempDir::new().unwrap(); + let pin_sem = Arc::new(Semaphore::new(1)); + // Hold the only pin permit. + let held = pin_sem.clone().acquire_owned().await.unwrap(); + + // Empty ipfs_api makes the pin itself a no-op, but the loop must still DEFER on + // the exhausted pin pool rather than run. + let blocked = tokio::time::timeout( + std::time::Duration::from_millis(500), + pin_new_objects_gated(&pin_sem, "", tmp.path(), vec![], &db), + ) + .await; + assert!( + blocked.is_err(), + "a pin loop must defer while the pin pool is exhausted (#174 F6)" + ); + + // Release admission: the SAME call now completes. + drop(held); + let out = tokio::time::timeout( + std::time::Duration::from_secs(5), + pin_new_objects_gated(&pin_sem, "", tmp.path(), vec![], &db), + ) + .await + .expect("the pin loop completes once admission frees"); + assert!(out.is_empty(), "an empty ipfs_api pins nothing"); + } + + /// Shared fixture for the F4 handler-layer tests: a state whose repo_store and + /// git_bin point at the given tempdir/fake-git, plus a seeded on-disk repo, + /// optionally with a path-scoped rule (so the post-receive walks actually run). + #[cfg(unix)] + async fn f4_state_with_repo( + pool: sqlx::PgPool, + tmp: &std::path::Path, + git_bin: &str, + owner: &str, + name: &str, + path_scoped: bool, + ) -> AppState { + let mut state = crate::test_support::test_state(pool.clone()).await; + let repos_dir = tmp.join("repos"); + std::fs::create_dir_all(&repos_dir).unwrap(); + state.repo_store = crate::git::repo_store::RepoStore::for_testing(repos_dir, pool); + state.git_bin = git_bin.to_string(); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo(owner, name, &format!("/unused-{owner}-{name}"), None, false) + .await + .unwrap(); + let rec = state.db.get_repo(owner, name).await.unwrap().unwrap(); + state + .repo_store + .init(&rec.owner_did, &rec.name) + .await + .unwrap(); + if path_scoped { + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkF4ReaderAAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + } + state + } + + /// A pkt-line receive-pack body carrying one branch-create ref update, so the + /// handler's post-receive tail resolves a non-empty new-tip set (the delta + /// scan's git stages run). + fn ref_update_body(new_sha: &str) -> axum::body::Bytes { + let line = format!("{ZERO_SHA} {new_sha} refs/heads/main"); + axum::body::Bytes::from(format!("{:04x}{}0000", line.len() + 4, line)) + } + + /// F4 scenario 2 — push-burst bound at the handler layer: with a scan pool of + /// ONE, two concurrent pushes to two path-scoped repos never have more than one + /// scan's git alive at a time (an atomic mkdir lock in the fake git detects any + /// overlap), and BOTH pushes still succeed 200 — defer, not shed. Two distinct + /// repos on purpose: the per-repo advisory write lock must not be what + /// serializes the scans. + #[cfg(unix)] + #[sqlx::test] + async fn receive_pack_burst_scans_serialized_and_both_pushes_succeed(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let lockdir = tmp.path().join("scan.lock"); + let ranfile = tmp.path().join("scan.ran"); + let overlap = tmp.path().join("scan.overlap"); + // receive-pack succeeds instantly; every candidate-scan git op (cat-file / + // rev-list / ls-tree) holds an atomic mkdir lock for 150ms — a second scan + // process alive at the same instant records an overlap. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + receive-pack) cat > /dev/null 2>/dev/null ;;\n\ + rev-parse) echo deadbeef ;;\n\ + cat-file|rev-list|ls-tree)\n\ + if mkdir \"{lock}\" 2>/dev/null; then\n\ + echo 1 >> \"{ran}\"\n\ + sleep 0.15\n\ + rmdir \"{lock}\"\n\ + else\n\ + echo 1 >> \"{over}\"\n\ + fi\n\ + if [ \"$1\" = cat-file ]; then echo commit; fi ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + lock = lockdir.display(), + ran = ranfile.display(), + over = overlap.display(), + ); + let git_bin = write_fake_git(tmp.path(), &body); + + let mut state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f4burst1", "b1", true).await; + // Second path-scoped repo on the same state/store. + state + .db + .upsert_mirror_repo("z6f4burst2", "b2", "/unused-z6f4burst2-b2", None, false) + .await + .unwrap(); + let rec2 = state + .db + .get_repo("z6f4burst2", "b2") + .await + .unwrap() + .unwrap(); + state + .repo_store + .init(&rec2.owner_did, &rec2.name) + .await + .unwrap(); + state + .db + .set_visibility_rule( + &rec2.id, + "/secret/**", + crate::db::VisibilityMode::B, + &["did:key:z6MkF4ReaderAAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string()], + &rec2.owner_did, + ) + .await + .unwrap(); + // Scan pool of ONE: at most one post-receive walk may run at a time. + state.git_encrypt_semaphore = Arc::new(Semaphore::new(1)); + + let did = "did:key:z6MkF4BurstPusherAAAAAAAAAAAAAAAAAAAAAAAA"; + let new_sha = "1111111111111111111111111111111111111111"; + let push = |owner: &'static str, name: &'static str, peer: &'static str| { + let state = state.clone(); + tokio::spawn(async move { + git_receive_pack( + State(state), + Path((owner.to_string(), name.to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(peer.parse::().unwrap())), + axum::http::HeaderMap::new(), + ref_update_body(new_sha), + ) + .await + }) + }; + + let (a, b) = ( + push("z6f4burst1", "b1", "203.0.113.71:5000"), + push("z6f4burst2", "b2", "203.0.113.72:5000"), + ); + let a = tokio::time::timeout(std::time::Duration::from_secs(60), a) + .await + .expect("push A must complete — a scan gate must defer, never wedge") + .expect("push A task must not panic"); + let b = tokio::time::timeout(std::time::Duration::from_secs(60), b) + .await + .expect("push B must complete — a scan gate must defer, never wedge") + .expect("push B task must not panic"); + let a = a.expect("push A must succeed"); + let b = b.expect("push B must succeed"); + assert_eq!(a.status(), 200, "push A lands 200 despite scan contention"); + assert_eq!(b.status(), 200, "push B lands 200 despite scan contention"); + + // Wait for both pushes' detached scan tails to drain through the pool of 1 + // before reading the detector files. The WHOLE tail (withheld walk included) now + // runs detached (#174 F2), so poll until every expected scan has run rather than a + // fixed sleep, which is load-sensitive under a parallel test run. + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); + loop { + let ran = std::fs::read_to_string(&ranfile) + .unwrap_or_default() + .lines() + .count(); + if ran >= 6 || std::time::Instant::now() >= deadline { + break; + } + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + } + // Small settle so the last scan's rmdir has landed before the overlap check. + tokio::time::sleep(std::time::Duration::from_millis(100)).await; + assert!( + !overlap.exists(), + "with a scan pool of 1, no two scans' git may ever be alive at once \ + (found overlap records: {:?})", + std::fs::read_to_string(&overlap).unwrap_or_default() + ); + let ran = std::fs::read_to_string(&ranfile).unwrap_or_default(); + assert!( + ran.lines().count() >= 6, + "both pushes' scans must actually have run (withheld walk + delta probe + \ + delta rev-list each); got {} runs", + ran.lines().count() + ); + } + + /// F4 scenario 3 — fast-path non-acquisition at the handler layer: a push to a + /// public repo with NO path-scoped rules does zero post-receive git scanning + /// (the withheld short-circuit; a deletion-free flush-only body resolves no new + /// tips), so it must complete 200 even with the scan pool at ZERO permits. + /// A gate that wrongly captured a no-walk path would park this push forever. + /// Note: `resolve_candidates_for_push` spawns git for ANY non-empty new-tip set + /// (the per-tip cat-file probe), so the genuinely git-free negative arm is the + /// no-ref-update body, not a branch-create push. + #[cfg(unix)] + #[sqlx::test] + async fn receive_pack_no_scan_fast_path_completes_with_zero_scan_permits(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let marker = tmp.path().join("scan.ran"); + let body = format!( + "#!/bin/sh\ncase \"$1\" in\n receive-pack) cat > /dev/null 2>/dev/null ;;\n cat-file|rev-list|ls-tree) echo 1 >> \"{}\" ;;\n *) : ;;\nesac\nexit 0\n", + marker.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + let mut state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f4fast", "f1", false).await; + state.git_encrypt_semaphore = Arc::new(Semaphore::new(0)); + + let peer: SocketAddr = "203.0.113.73:5000".parse().unwrap(); + let resp = tokio::time::timeout( + std::time::Duration::from_secs(30), + git_receive_pack( + State(state), + Path(("z6f4fast".to_string(), "f1".to_string())), + Extension(crate::auth::AuthenticatedDid( + "did:key:z6MkF4FastPusherAAAAAAAAAAAAAAAAAAAAAAAAA".to_string(), + )), + crate::rate_limit::PeerAddr(Some(peer)), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ), + ) + .await + .expect("a no-scan push must not park on the (empty) scan pool") + .expect("the push must succeed"); + assert_eq!(resp.status(), 200); + assert!( + !marker.exists(), + "the no-walk fast paths must spawn no scan git at all" + ); + } + + /// F4 scenario 4 — landed-push-never-fails: a push whose post-receive walk must + /// park (pool held elsewhere) DEFERS and then returns the receive-pack success + /// once admission frees; contention never converts the landed push into a 5xx. + /// #174 F2 (RED-before/GREEN-after): the post-receive replication tail parks on + /// `git_encrypt_semaphore` (withheld/candidate/full-scan resolution). Leaving it in the + /// request future means a client/proxy disconnect while parked silently loses this + /// push's pins, recovery copy, and announcements (state.rs documented this residual). + /// The fix moves the whole tail into an independently owned task, so the handler + /// returns its receive-pack 200 WITHOUT waiting on the scan pool and a disconnect can + /// no longer drop the work. + /// + /// Load-bearing: with the tail inline (pre-fix) the handler parks while the pool is + /// held and does NOT return within the bound (RED — the timeout fires). With the + /// detached tail it returns 200 promptly (GREEN). + #[cfg(unix)] + #[sqlx::test] + async fn receive_pack_landed_push_returns_without_parking_on_scan_pool(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let body = "#!/bin/sh\ncase \"$1\" in\n receive-pack) cat > /dev/null 2>/dev/null ;;\n rev-parse) echo deadbeef ;;\n cat-file) echo commit ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + let mut state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f4park", "p1", true).await; + let sem = Arc::new(Semaphore::new(1)); + state.git_encrypt_semaphore = sem.clone(); + // Hold the pool's only permit: the post-receive scan would park if it ran in the + // request future. + let held = sem.clone().acquire_owned().await.unwrap(); + + let peer: SocketAddr = "203.0.113.74:5000".parse().unwrap(); + // The handler must return its receive-pack 200 WITHOUT waiting on the held scan + // pool — the tail is owned by a detached task. Pre-fix this times out. + let resp = tokio::time::timeout( + std::time::Duration::from_secs(5), + git_receive_pack( + State(state), + Path(("z6f4park".to_string(), "p1".to_string())), + Extension(crate::auth::AuthenticatedDid( + "did:key:z6MkF4ParkPusherAAAAAAAAAAAAAAAAAAAAAAAAA".to_string(), + )), + crate::rate_limit::PeerAddr(Some(peer)), + axum::http::HeaderMap::new(), + ref_update_body("2222222222222222222222222222222222222222"), + ), + ) + .await + .expect("the handler must return without parking on the held scan pool") + .expect("contention must never convert a landed push into an error"); + assert_eq!( + resp.status(), + 200, + "the response is the receive-pack success, returned before the detached tail runs" + ); + + // The detached tail is still owned: release admission and let it drain cleanly. + drop(held); + tokio::time::sleep(std::time::Duration::from_millis(300)).await; + } + + // ---- #174 U4 (P2-2): post-push encryption task set bounded by per-repo coalescing ---- + // + // The residual jatmn found is not the WALK (bounded by `git_encrypt_semaphore`, + // proven by `encrypt_walk_defers_when_pool_exhausted` above) but the OUTER + // `tokio::spawn` + its parked `acquire_owned().await` waiters: N rapid pushes to a + // repo spawn N tasks that each park holding cloned object lists/rules/keys — an + // unbounded outstanding set. U4 bounds it by coalescing per repo: before spawning, + // if a task for the repo is in flight, skip the duplicate. Crucially this DEFERS a + // duplicate walk (the newer push's objects are covered by the pending one) and does + // NOT shed — there is no reconciliation sweep, so a dropped job would permanently + // lose the withheld-blob recovery copy (`2a54c15`'s fail-closed durability stance). + // + // These drive the coalescing seam (`EncryptInflight`) that the detached spawn at + // `repos.rs` consults directly (the try_begin gate on the in-flight set, guarded by + // `withheld.is_some()`). Observing `encrypt_and_pin`'s IPFS effect end-to-end needs a live IPFS node + // (`pin_git_object` hits the API), so the durability property is proven at this + // layer: a coalesced repo's key is released when its task ends, so a later push for + // that repo is processed once — NOT permanently skipped, which is exactly what a + // coalesce->shed mutation would break by dropping the job with no sweep to recover it. + + /// Bounded outstanding set under saturation (R4). Simulate K rapid path-scoped + /// pushes to the SAME repo while the encrypt pool is saturated (every spawned task + /// would park, so none has finished and removed its key): the first `try_begin` + /// admits (spawns), the rest coalesce (skip). The in-flight set holds at 1, not K. + /// + /// MUTATION (RED): removing the coalescing check makes every push spawn — modeled by + /// `simulate_without_coalescing`, which reaches K. If the coalesced count equaled the + /// un-coalesced one the gate would be a no-op; the strict inequality proves it bites. + #[test] + fn u4_outstanding_encrypt_set_is_bounded_to_one_per_repo_under_saturation() { + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkRepoOwnerAAAAAAAAAAAAAAAAAAAAAAAAAAAA/proj"; + const K: usize = 32; + + // Hold every admitted guard so the tasks are "still in flight" (the saturated + // case: all parked on acquire_owned().await, none finished, none removed a key). + let mut admitted = Vec::new(); + let mut coalesced = 0usize; + for _ in 0..K { + match inflight.try_begin(repo, vec![]) { + crate::state::BeginOutcome::Admitted(g) => admitted.push(g), + crate::state::BeginOutcome::Coalesced => coalesced += 1, + } + } + + assert_eq!( + admitted.len(), + 1, + "exactly ONE detached task may spawn per repo while one is in flight — the \ + outstanding set is bounded to 1, not K parked waiters" + ); + assert_eq!( + coalesced, + K - 1, + "the other K-1 rapid pushes to the same repo coalesce (skip spawning)" + ); + assert_eq!( + inflight.len(), + 1, + "the in-flight set holds at most one entry per repo under saturation" + ); + + let no_coalesce = simulate_without_coalescing(K); + assert_eq!( + no_coalesce, K, + "sanity: without the coalescing check all K pushes spawn (the unbounded set \ + the fix prevents) — proves the bound above is not vacuously 1" + ); + assert!( + admitted.len() < no_coalesce, + "coalesced set ({}) must be strictly smaller than the un-coalesced one ({})", + admitted.len(), + no_coalesce + ); + } + + /// Coalescing is PER-REPO: distinct repos are never coalesced against each other, so + /// one repo in flight cannot starve a second repo's recovery copy. + #[test] + fn u4_distinct_repos_each_admit_one_encrypt_task() { + use crate::state::BeginOutcome; + let inflight = crate::state::EncryptInflight::new(); + let a = inflight.try_begin("owner/repo-a", vec![]); + let b = inflight.try_begin("owner/repo-b", vec![]); + let c = inflight.try_begin("owner/repo-c", vec![]); + assert!( + matches!(&a, BeginOutcome::Admitted(_)) + && matches!(&b, BeginOutcome::Admitted(_)) + && matches!(&c, BeginOutcome::Admitted(_)), + "three distinct repos each admit their own encryption task" + ); + assert_eq!(inflight.len(), 3, "one in-flight entry per distinct repo"); + } + + /// NO LOST RECOVERY COPY — the security guard (R4/R6). Coalescing must DELAY a + /// duplicate walk, never permanently drop a repo's recovery copy. Observable + /// property: once an in-flight task ENDS (its guard drops — completion, error, or + /// panic-unwind) the repo key is released, so the NEXT push for that repo is admitted + /// and processed again. A coalesce->shed mutation would drop the job AND never + /// re-admit — with no reconciliation sweep the copy is lost forever. Here re-admission + /// survives normal completion AND a panic, so no permanent skip / no leaked key. + #[test] + fn u4_coalesced_repo_is_reprocessed_after_task_ends_not_permanently_skipped() { + use crate::state::{BeginOutcome, FinishOutcome}; + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkDurableRepoBBBBBBBBBBBBBBBBBBBBBBBBB/repo"; + + // Push #1 admits and "spawns". A concurrent push #2 (task #1 still in flight) + // coalesces — no duplicate spawn; its (empty) tip set is recorded, not lost. + let guard1 = admit(&inflight, repo); + assert!( + matches!(inflight.try_begin(repo, vec![]), BeginOutcome::Coalesced), + "while task #1 is in flight, push #2 to the same repo coalesces" + ); + + // Task #1 finishes normally: nothing pending (push #2 carried no tips), so + // the empty-pending check removes the key in its critical section. + assert!( + matches!(guard1.finish_or_take_pending(), FinishOutcome::Finished(_)), + "no pending tips — the task exits and releases the key" + ); + assert_eq!( + inflight.len(), + 0, + "when the in-flight task ends its repo key is released — the set does not leak" + ); + + // A LATER push for the SAME repo is admitted again (processed, not skipped + // forever). This is what coalesce->shed breaks: shed drops the job and no sweep + // re-derives the missing copy, so the recovery copy is permanently lost. + let guard2 = admit(&inflight, repo); + // An errored task (guard dropped without finishing) still releases the key. + drop(guard2); + assert_eq!(inflight.len(), 0); + + // Durability across PANIC: a task that panics mid-walk must still release its + // key (the still-armed guard's Drop runs on unwind), so one crashed walk never + // permanently locks a repo out of future recovery copies. Coalesce real tips + // first: the panic loses them (logged), and the loss must not corrupt the set. + let panicked = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let _g = admit(&inflight, repo); + assert!(matches!( + inflight.try_begin(repo, vec![("old1".to_string(), "new1".to_string())]), + BeginOutcome::Coalesced + )); + assert_eq!(inflight.len(), 1); + panic!("simulate the detached encryption task panicking mid-walk"); + })); + assert!(panicked.is_err(), "the simulated task panicked"); + assert_eq!( + inflight.len(), + 0, + "a panicked encryption task still releases its repo key (Drop on unwind) — no \ + permanent leak that would block every future recovery copy for the repo" + ); + // The next push is re-admitted, and the panicked task's pending tips did NOT + // survive into it (they are lost-and-logged, recovered only by a later push). + let guard3 = admit(&inflight, repo); + assert!( + matches!(guard3.finish_or_take_pending(), FinishOutcome::Finished(_)), + "the pre-panic pending tips must not leak into the re-admitted task" + ); + } + + /// Degenerate state: the first push on a cold/empty in-flight set always admits + /// (never a false coalesce on an empty set). + #[test] + fn u4_first_push_on_a_cold_set_always_admits() { + let inflight = crate::state::EncryptInflight::new(); + assert!(inflight.is_empty(), "cold set is empty"); + assert!( + matches!( + inflight.try_begin("owner/first", vec![]), + crate::state::BeginOutcome::Admitted(_) + ), + "the first push on a cold in-flight set must admit (never falsely coalesce)" + ); + } + + /// Unwrap an Admitted outcome (panic on Coalesced) — the u4/u5 suites' shorthand. + fn admit( + inflight: &crate::state::EncryptInflight, + repo: &str, + ) -> crate::state::EncryptInflightGuard { + match inflight.try_begin(repo, vec![]) { + crate::state::BeginOutcome::Admitted(g) => g, + crate::state::BeginOutcome::Coalesced => panic!("expected {repo} to admit"), + } + } + + /// Model of the pre-fix / mutated code: no coalescing check, so every push spawns. + /// Returns the count of tasks spawned (== the size of the unbounded outstanding set + /// the fix prevents), used as the RED comparison in the bound test above. + fn simulate_without_coalescing(pushes: usize) -> usize { + (0..pushes).count() + } + + // ---- #174 U5 (F5): a push that loses try_begin is REQUEUED, never dropped ---- + // + // F5: the in-flight task pins only its own pre-spawn object-list snapshot, so a + // push B arriving while task A is in flight used to be SKIPPED outright (the old + // None arm) — B's pins and recovery copies were silently absent until an + // unrelated later push re-walked the repo. U5 records B's (old, new) tip pairs + // into the in-flight key's pending slot in the SAME critical section as the + // presence check, and A's task loop-drains them before releasing the key. + + /// The F5 lost-update repro. Task A is in flight past its snapshot; push B + /// coalesces carrying its tip pair; when A finishes its snapshot iteration the + /// tracker must hand A exactly B's recorded work with the key retained — and only + /// an empty pending check may remove the key. On pre-U5 code this is RED: the + /// coalesce arm records B's work nowhere and there is no drain surface at all. + #[test] + fn u5_coalesced_push_work_is_drained_by_the_inflight_task() { + use crate::state::{BeginOutcome, FinishOutcome, PendingWork}; + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkF5LostUpdateCCCCCCCCCCCCCCCCCCCCCCCC/repo"; + + // Push A admits; its spawned task is "in flight past its snapshot". + let guard_a = match inflight.try_begin(repo, vec![]) { + BeginOutcome::Admitted(g) => g, + BeginOutcome::Coalesced => panic!("first push must admit"), + }; + + // Push B lands while A is in flight: coalesced, tip pair recorded. + let b_pair = ( + "b0ldb0ldb0ldb0ldb0ldb0ldb0ldb0ldb0ldb0ld".to_string(), + "bnewbnewbnewbnewbnewbnewbnewbnewbnewbnew".to_string(), + ); + match inflight.try_begin(repo, vec![b_pair.clone()]) { + BeginOutcome::Coalesced => {} + BeginOutcome::Admitted(_) => panic!("push B must coalesce while A is in flight"), + } + + // A finishes its snapshot iteration: it must be handed B's work (drained), + // not exit — an exit here is exactly the F5 silent loss. + match guard_a.finish_or_take_pending() { + FinishOutcome::Pending(guard_a, work) => { + assert_eq!( + work, + PendingWork::Tips(vec![b_pair]), + "A drains exactly B's recorded tip pair" + ); + assert_eq!(inflight.len(), 1, "the key is retained while A iterates"); + // Nothing further pending: A now exits and releases the key. + match guard_a.finish_or_take_pending() { + FinishOutcome::Finished(_) => {} + FinishOutcome::Pending(..) => panic!("no second batch was recorded"), + } + assert_eq!( + inflight.len(), + 0, + "an empty pending check at task end releases the key" + ); + } + FinishOutcome::Finished(_) => panic!( + "F5: B's coalesced work vanished — the in-flight task exited without draining it" + ), + } + } + + /// Drain-vs-admit race, both orderings driven deterministically through the lock + /// API (the check+merge and check+remove are each ONE critical section, so a push + /// can only land on one side of A's final pending check — never inside it): + /// before it, the push is merged and A drains it; after it, the key is gone and + /// the push is admitted as a fresh task. Neither ordering loses the work. A + /// check-then-record split (merge moved outside try_begin's critical section) + /// turns ordering 1 RED: the work recorded after A's check is never drained. + #[test] + fn u5_drain_vs_admit_race_loses_no_work_in_either_ordering() { + use crate::state::{BeginOutcome, FinishOutcome, PendingWork}; + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkF5RaceOrderDDDDDDDDDDDDDDDDDDDDDDDDD/repo"; + let pair = ("cold".to_string(), "cnew".to_string()); + + // Ordering 1: push C lands BEFORE A's final pending check → merged in + // try_begin's critical section → A must drain it (key retained). + let guard_a = admit(&inflight, repo); + assert!(matches!( + inflight.try_begin(repo, vec![pair.clone()]), + BeginOutcome::Coalesced + )); + match guard_a.finish_or_take_pending() { + FinishOutcome::Pending(g, work) => { + assert_eq!(work, PendingWork::Tips(vec![pair.clone()])); + assert!(matches!( + g.finish_or_take_pending(), + FinishOutcome::Finished(_) + )); + } + FinishOutcome::Finished(_) => { + panic!("a push merged before the final check must be drained, not lost") + } + } + assert!(inflight.is_empty()); + + // Ordering 2: push C lands AFTER A's final pending check removed the key → + // it must be ADMITTED as a fresh task (its own snapshot covers its work). + let guard_a = admit(&inflight, repo); + assert!(matches!( + guard_a.finish_or_take_pending(), + FinishOutcome::Finished(_) + )); + match inflight.try_begin(repo, vec![pair]) { + BeginOutcome::Admitted(g) => drop(g), + BeginOutcome::Coalesced => panic!( + "a push landing after the key was removed must admit a new task — a \ + coalesce here records work no task will ever drain" + ), + } + } + + /// Exit-vs-successor (the double-remove hazard). A's normal exit removes the key + /// and disarms the guard in ONE critical section, and the disarmed guard is + /// handed back — so its eventual Drop lands in the real remove→drop window. A + /// successor task B admitted inside that window must keep ITS key when A's guard + /// finally drops. With the disarm reverted (Drop removing unconditionally) this + /// is RED: dropping A's guard deletes B's key and the third push falsely admits + /// a second task for the repo. + #[test] + fn u5_disarmed_guard_drop_never_removes_a_successor_key() { + use crate::state::{BeginOutcome, FinishOutcome}; + let inflight = crate::state::EncryptInflight::new(); + let repo = "did:key:z6MkF5DisarmEEEEEEEEEEEEEEEEEEEEEEEEEEEE/repo"; + + // A admits and exits normally; HOLD the disarmed guard to keep the window open. + let guard_a = admit(&inflight, repo); + let disarmed = match guard_a.finish_or_take_pending() { + FinishOutcome::Finished(g) => g, + FinishOutcome::Pending(..) => panic!("nothing was pending"), + }; + assert!(inflight.is_empty(), "A's exit released the key"); + + // Successor B is admitted inside the remove→drop window. + let guard_b = admit(&inflight, repo); + assert_eq!(inflight.len(), 1); + + // A's disarmed guard now drops. B's key must SURVIVE: a third push still + // coalesces against B's in-flight task. + drop(disarmed); + assert_eq!( + inflight.len(), + 1, + "dropping A's disarmed guard must not remove successor B's key" + ); + assert!( + matches!(inflight.try_begin(repo, vec![]), BeginOutcome::Coalesced), + "B's task is still the (only) in-flight task — at-most-one-per-repo holds" + ); + drop(guard_b); + assert!(inflight.is_empty()); + } + + /// Pending overflow: past the 1024-pair bound the slot degrades to the FullScan + /// marker (bounded memory under a hostile push burst); at exactly the bound it + /// stays a Tips batch. The marker is an explicit variant, never an empty tip + /// list — an empty-tips encoding would drain to an empty delta and pin nothing. + #[test] + fn u5_pending_overflow_degrades_to_full_scan_marker() { + use crate::state::{BeginOutcome, FinishOutcome, PendingWork}; + let inflight = crate::state::EncryptInflight::new(); + let pair = |i: usize| (format!("old{i}"), format!("new{i}")); + + // At the bound: exactly 1024 pairs stay a Tips batch. + let repo_at = "owner/at-bound"; + let g = admit(&inflight, repo_at); + assert!(matches!( + inflight.try_begin(repo_at, (0..1024).map(pair).collect()), + BeginOutcome::Coalesced + )); + match g.finish_or_take_pending() { + FinishOutcome::Pending(g, PendingWork::Tips(v)) => { + assert_eq!(v.len(), 1024, "at the bound the pairs are kept verbatim"); + assert!(matches!( + g.finish_or_take_pending(), + FinishOutcome::Finished(_) + )); + } + other => panic!( + "expected a Tips batch at the bound, got {:?}", + match other { + FinishOutcome::Pending(_, w) => Some(w), + FinishOutcome::Finished(_) => None, + } + ), + } + + // Past the bound: the accumulated slot degrades to FullScan and later + // merges are absorbed (still one bounded marker, not a growing list). + let repo_over = "owner/over-bound"; + let g = admit(&inflight, repo_over); + assert!(matches!( + inflight.try_begin(repo_over, (0..1024).map(pair).collect()), + BeginOutcome::Coalesced + )); + assert!(matches!( + inflight.try_begin(repo_over, vec![pair(9999)]), + BeginOutcome::Coalesced + )); + assert!(matches!( + inflight.try_begin(repo_over, vec![pair(10000)]), + BeginOutcome::Coalesced + )); + match g.finish_or_take_pending() { + FinishOutcome::Pending(g, work) => { + assert_eq!( + work, + PendingWork::FullScan, + "overflow degrades to the explicit FullScan marker" + ); + assert!(matches!( + g.finish_or_take_pending(), + FinishOutcome::Finished(_) + )); + } + FinishOutcome::Finished(_) => panic!("the overflowed pending work vanished"), + } + } + + // ---- u5 drain-pipeline fixtures: a real git repo + a DB repo row ---- + + fn u5_git(dir: &std::path::Path, args: &[&str]) -> String { + let out = std::process::Command::new("git") + .args(args) + .current_dir(dir) + .output() + .unwrap(); + assert!( + out.status.success(), + "git {args:?} failed: {}", + String::from_utf8_lossy(&out.stderr) + ); + String::from_utf8_lossy(&out.stdout).trim().to_string() + } + + fn u5_init_repo(dir: &std::path::Path) { + u5_git(dir, &["init", "-q", "-b", "main"]); + u5_git(dir, &["config", "user.email", "t@t"]); + u5_git(dir, &["config", "user.name", "t"]); + } + + /// Commit `name` (parent dirs created) with `body`; returns the commit sha. + fn u5_commit_file(dir: &std::path::Path, name: &str, body: &str) -> String { + let path = dir.join(name); + if let Some(parent) = path.parent() { + std::fs::create_dir_all(parent).unwrap(); + } + std::fs::write(&path, body).unwrap(); + u5_git(dir, &["add", name]); + u5_git(dir, &["commit", "-qm", &format!("add {name}")]); + u5_git(dir, &["rev-parse", "HEAD"]) + } + + /// A drain-task context over the test state and an on-disk repo. Empty + /// `ipfs_api`/`irys_url` keep the pin/anchor stages inert (no network). + fn u5_ctx( + state: &AppState, + rec: &crate::db::RepoRecord, + repo_path: std::path::PathBuf, + git_bin: &str, + sem: std::sync::Arc, + ) -> EncryptTaskCtx { + EncryptTaskCtx { + ipfs_api: String::new(), + repo_path, + db: state.db.clone(), + repo_id: rec.id.clone(), + owner_did: rec.owner_did.clone(), + repo_name: rec.name.clone(), + irys_url: String::new(), + http_client: std::sync::Arc::clone(&state.http_client), + node_did: state.node_did.to_string(), + node_keypair: std::sync::Arc::clone(&state.node_keypair), + git_bin: git_bin.to_string(), + git_timeout: std::time::Duration::from_secs(600), + encrypt_sem: sem, + pin_sem: std::sync::Arc::new(tokio::sync::Semaphore::new(64)), + } + } + + /// The drain resolves a coalesced push's tip pair to exactly that push's + /// introduced objects (delta semantics — the F5 observable: push B's pins are + /// recorded by the drain, and pre-existing objects are not re-listed). + #[sqlx::test] + async fn u5_drain_resolves_coalesced_tips_to_their_objects(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + u5_init_repo(tmp.path()); + let c1 = u5_commit_file(tmp.path(), "a.txt", "one\n"); + let c2 = u5_commit_file(tmp.path(), "b.txt", "two\n"); + state + .db + .upsert_mirror_repo("z6u5delta", "d", "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6u5delta", "d").await.unwrap().unwrap(); + let ctx = u5_ctx( + &state, + &rec, + tmp.path().to_path_buf(), + "git", + std::sync::Arc::new(tokio::sync::Semaphore::new(4)), + ); + + // Push B advanced main c1 -> c2 and lost try_begin; its pair was coalesced. + let (list, _rules, is_public) = resolve_drain_object_list( + &ctx, + crate::state::PendingWork::Tips(vec![(c1.clone(), c2.clone())]), + ) + .await + .expect("a public repo drains to a pin list"); + assert!(is_public, "mirror rows are public"); + let got: std::collections::HashSet = list.into_iter().collect(); + let new_blob = u5_git(tmp.path(), &["rev-parse", "HEAD:b.txt"]); + let old_blob = u5_git(tmp.path(), &["rev-parse", &format!("{c1}:a.txt")]); + assert!( + got.contains(&c2) && got.contains(&new_blob), + "B's commit and blob are in the drained pin list (the F5 fix)" + ); + assert!( + !got.contains(&c1) && !got.contains(&old_blob), + "pre-existing objects are not re-listed (delta, not full scan)" + ); + } + + /// The FullScan marker drains through the FLAGGED full-scan path to a NON-EMPTY + /// candidate set. RED arm of the encoding: were the marker a plain empty-tips + /// call, the deletion-only fast path would return an empty delta and the drain + /// would pin nothing (the F5 silent loss resurfacing). + #[sqlx::test] + async fn u5_drain_full_scan_marker_yields_nonempty_candidates(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + u5_init_repo(tmp.path()); + let c1 = u5_commit_file(tmp.path(), "a.txt", "one\n"); + state + .db + .upsert_mirror_repo("z6u5full", "f", "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6u5full", "f").await.unwrap().unwrap(); + let ctx = u5_ctx( + &state, + &rec, + tmp.path().to_path_buf(), + "git", + std::sync::Arc::new(tokio::sync::Semaphore::new(4)), + ); + + let (list, _rules, _pub) = + resolve_drain_object_list(&ctx, crate::state::PendingWork::FullScan) + .await + .expect("a public repo drains to a pin list"); + let got: std::collections::HashSet = list.into_iter().collect(); + assert!( + !got.is_empty(), + "the FullScan drain must enumerate the repo — an empty list means the \ + marker collapsed into the empty-tips fast path" + ); + let blob = u5_git(tmp.path(), &["rev-parse", "HEAD:a.txt"]); + assert!( + got.contains(&c1) && got.contains(&blob), + "the full-scan drain covers the repo's commit and blob" + ); + } + + /// Rules tightened between the coalesced push and its drain are honored, fail + /// closed: the drain re-fetches rules/is_public fresh, so (1) a newly-withheld + /// blob is NOT pinned, and (2) a repo whose root became unreadable to the + /// anonymous public drains to nothing at all. + #[sqlx::test] + async fn u5_drain_honors_rules_tightened_after_the_push(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + u5_init_repo(tmp.path()); + u5_commit_file(tmp.path(), "pub.txt", "public\n"); + let c2 = u5_commit_file(tmp.path(), "secret/hidden.txt", "sealed\n"); + state + .db + .upsert_mirror_repo("z6u5tight", "t", "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6u5tight", "t").await.unwrap().unwrap(); + let ctx = u5_ctx( + &state, + &rec, + tmp.path().to_path_buf(), + "git", + std::sync::Arc::new(tokio::sync::Semaphore::new(4)), + ); + let pending = || crate::state::PendingWork::Tips(vec![(ZERO_SHA.to_string(), c2.clone())]); + + // At push time the repo had no rules. TIGHTEN before the drain: /secret/** + // becomes reader-gated. The drain must re-fetch and withhold the new blob. + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &[READER_DID.to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + let (list, _rules, _pub) = resolve_drain_object_list(&ctx, pending()) + .await + .expect("still announceable at root"); + let got: std::collections::HashSet = list.into_iter().collect(); + let pub_blob = u5_git(tmp.path(), &["rev-parse", "HEAD:pub.txt"]); + let secret_blob = u5_git(tmp.path(), &["rev-parse", "HEAD:secret/hidden.txt"]); + assert!( + got.contains(&pub_blob), + "the still-public blob is pinned by the drain" + ); + assert!( + !got.contains(&secret_blob), + "a blob withheld by a rule added AFTER the push must NOT be pinned by \ + the drain (fresh rules, fail closed)" + ); + + // Tighten further: root becomes reader-gated → not announceable to the + // anonymous public → the drain pins nothing at all. + state + .db + .set_visibility_rule( + &rec.id, + "/", + crate::db::VisibilityMode::A, + &[READER_DID.to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + assert!( + resolve_drain_object_list(&ctx, pending()).await.is_none(), + "a repo no longer announceable under current rules drains to nothing \ + (fail closed)" + ); + } + + /// #174 F2 / KTD-3 re-derivation equivalence: the object set the Pinata worker + /// re-derives from ONLY the ref tuples (`pinata_object_list_for_refs`, run once a + /// pin slot frees) must equal exactly what the old retained `object_list` would + /// have pinned — the inline-resolved delta, filtered by the withheld set. If the + /// two differ, the memory fix changed what gets pinned; they must not. + #[tokio::test] + async fn f2_pinata_rederivation_equals_retained_object_list() { + let tmp = tempfile::TempDir::new().unwrap(); + u5_init_repo(tmp.path()); + let c1 = u5_commit_file(tmp.path(), "a.txt", "one\n"); + let c2 = u5_commit_file(tmp.path(), "b.txt", "two\n"); + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(4)); + let timeout = std::time::Duration::from_secs(600); + + // What the OLD retained-list task WOULD have pinned: the inline pipeline the + // receive-pack tail ran before moving `object_list` into the closure — the + // delta for main c1 -> c2, filtered by the (empty) withheld set. + let candidates = crate::git::push_delta::resolve_candidates_for_push( + sem.clone(), + tmp.path().to_path_buf(), + vec![c2.clone()], + vec![c1.clone()], + "git".to_string(), + timeout, + false, + ) + .await; + assert!( + !candidates.full_scan, + "the c1 -> c2 push is a delta, not a full scan" + ); + let retained: std::collections::HashSet = + crate::git::visibility_pack::replicable_objects( + candidates.candidates, + &std::collections::HashSet::new(), + ) + .into_iter() + .collect(); + + // What the worker re-derives from only the (ref, old, new) tuples. Empty rules + // + is_public => announceable, withheld = {} (the common Pinata case). + let ref_updates = vec![("refs/heads/main".to_string(), c1.clone(), c2.clone())]; + let rederived: std::collections::HashSet = pinata_object_list_for_refs( + sem.clone(), + tmp.path().to_path_buf(), + &ref_updates, + Some(Vec::new()), + true, + "z6MkPinataOwnerAAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string(), + "git".to_string(), + timeout, + ) + .await + .into_iter() + .collect(); + + let new_blob = u5_git(tmp.path(), &["rev-parse", "HEAD:b.txt"]); + assert!( + retained.contains(&c2) && retained.contains(&new_blob), + "the push introduced the new commit and blob" + ); + assert_eq!( + rederived, retained, + "the worker's git rev-list re-derivation must yield exactly the object set \ + the retained list would have pinned — the memory fix must not change what pins" + ); + } + + /// #174 F2 / KTD-3 reaped + deadline-bounded: the worker's re-derivation git children + /// run through the same INV-22 bounded, process-group-reaped helpers the sibling scans + /// use. On a git that hangs on both `rev-list` and `--batch-all-objects`, + /// `pinata_object_list_for_refs` must RETURN within the watchdog budget (the group is + /// SIGKILLed + reaped at the deadline), not block. A bare `Command::output()` here + /// would hang past the ceiling (RED). + #[cfg(unix)] + #[tokio::test] + async fn f2_pinata_rederivation_is_deadline_bounded_and_reaped() { + use std::os::unix::fs::PermissionsExt; + + let dir = tempfile::TempDir::new().unwrap(); + // Empty rules => replication_withheld_set short-circuits (no git). The tip + // peel (`cat-file -t`) reports a commit so the delta stage proceeds; rev-list + // and the full-scan `cat-file --batch-all-objects` both hang (bounded 30s so a + // broken test cannot leak a permanent orphan). + let fake = dir.path().join("fakegit"); + std::fs::write( + &fake, + "#!/bin/sh\ncase \"$1\" in\n \ + cat-file) case \"$*\" in *--batch-all-objects*) i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done ;; *) echo commit ;; esac ;;\n \ + rev-list) i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done ;;\n \ + *) : ;;\nesac\nexit 0\n", + ) + .unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + let git_bin = fake.to_str().unwrap().to_string(); + + let ref_updates = vec![( + "refs/heads/main".to_string(), + ZERO_SHA.to_string(), + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef".to_string(), + )]; + let got = tokio::time::timeout( + std::time::Duration::from_secs(10), + pinata_object_list_for_refs( + std::sync::Arc::new(tokio::sync::Semaphore::new(4)), + dir.path().to_path_buf(), + &ref_updates, + Some(Vec::new()), + true, + "z6MkPinataOwnerAAAAAAAAAAAAAAAAAAAAAAAAAAAA".to_string(), + git_bin, + std::time::Duration::from_millis(400), + ), + ) + .await + .expect( + "pinata_object_list_for_refs must return within the watchdog budget — a hang \ + means the re-derivation git is not deadline-bounded / group-reaped (RED)", + ); + assert!( + got.is_empty(), + "a hung git yields nothing this push (the reconciliation sweep backstops)" + ); + } + + /// Hot-repo drain at encrypt-pool size 1: the task loop holds NO task-level + /// permit, so per-iteration helper acquires (withheld walk, candidate scan, + /// recipients walk) each get the pool's single permit in turn and BOTH the + /// snapshot iteration and the coalesced-drain iteration complete. RED if the + /// loop takes a task-level permit: the first helper acquire nests over the + /// same exhausted semaphore and the task parks forever. + #[sqlx::test] + async fn u5_hot_repo_drain_completes_at_pool_size_one(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + u5_init_repo(tmp.path()); + u5_commit_file(tmp.path(), "pub.txt", "public\n"); + let c2 = u5_commit_file(tmp.path(), "secret/hidden.txt", "sealed\n"); + state + .db + .upsert_mirror_repo("z6u5hot", "h", "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6u5hot", "h").await.unwrap().unwrap(); + // A path-scoped rule so every gated walk actually runs (withheld walk on + // the drain, recipients walk on both iterations). READER_DID carries no + // resolvable key, so the encrypt stage plans no seal and stays offline. + state + .db + .set_visibility_rule( + &rec.id, + "/secret/**", + crate::db::VisibilityMode::B, + &[READER_DID.to_string()], + &rec.owner_did, + ) + .await + .unwrap(); + let rules = state.db.list_visibility_rules(&rec.id).await.unwrap(); + + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + let ctx = u5_ctx(&state, &rec, tmp.path().to_path_buf(), "git", sem); + + let inflight = crate::state::EncryptInflight::new(); + let guard = admit(&inflight, &rec.id); + assert!(matches!( + inflight.try_begin(&rec.id, vec![(ZERO_SHA.to_string(), c2)]), + crate::state::BeginOutcome::Coalesced + )); + + tokio::time::timeout( + std::time::Duration::from_secs(60), + run_encrypt_pin_task(ctx, guard, Vec::new(), Some(rules), true), + ) + .await + .expect( + "the drain must complete at pool size 1 — a task-level permit would \ + deadlock the helper-internal acquires", + ); + assert!( + inflight.is_empty(), + "the drained task released its repo key on exit" + ); + } + + /// The task LOOP is load-bearing: work coalesced during the snapshot iteration + /// is drained (its candidate scan runs git) before the task exits. RED under + /// the drain-loop revert (task drops its guard after the snapshot without + /// checking pending): the fake git never runs and the marker is absent. + #[cfg(unix)] + #[sqlx::test] + async fn u5_task_drains_coalesced_work_before_exiting(pool: sqlx::PgPool) { + let state = crate::test_support::test_state(pool).await; + let tmp = tempfile::TempDir::new().unwrap(); + let marker = tmp.path().join("git.ran"); + // The drain's candidate scan probes the tip type then walks: report a + // commit tip and an empty rev-list, recording every invocation. + let body = format!( + "#!/bin/sh\necho ran >> \"{}\"\ncase \"$1\" in\n cat-file) echo commit ;;\n *) : ;;\nesac\nexit 0\n", + marker.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + state + .db + .upsert_mirror_repo("z6u5loop", "l", "/unused", None, false) + .await + .unwrap(); + let rec = state.db.get_repo("z6u5loop", "l").await.unwrap().unwrap(); + let ctx = u5_ctx( + &state, + &rec, + tmp.path().to_path_buf(), + &git_bin, + std::sync::Arc::new(tokio::sync::Semaphore::new(4)), + ); + + let inflight = crate::state::EncryptInflight::new(); + let guard = admit(&inflight, &rec.id); + // Push B coalesces mid-flight with a real (created-ref) tip pair. + assert!(matches!( + inflight.try_begin( + &rec.id, + vec![( + ZERO_SHA.to_string(), + "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef".to_string(), + )] + ), + crate::state::BeginOutcome::Coalesced + )); + + // Snapshot: empty object list, no rules — the snapshot iteration itself + // spawns no git, so any git invocation below belongs to the DRAIN. + run_encrypt_pin_task(ctx, guard, Vec::new(), None, true).await; + assert!( + marker.exists(), + "the task must drain B's coalesced tips (candidate scan runs git) \ + before exiting — an absent marker is the F5 skip" + ); + assert!( + inflight.is_empty(), + "the empty pending check at task end released the key" + ); + } + + /// #174 SC2 (per-source key, U1): the per-caller read sub-cap keys on the + /// resolved source IP, NOT the signed DID, so a disposable-DID farm cannot + /// multiply its budget. Fill the source IP's single read slot, then drive two + /// requests signed under DIFFERENT DIDs from that SAME IP: both must shed 503 + /// (keyed by the saturated IP, not their own free DID slots). A signed request + /// from a DIFFERENT source IP keeps its own budget. Revert `read_caller_key` to + /// prefer the DID and the same-IP assertions go green-not-503 (each fresh DID + /// gets a free slot) -- the farm-defeat mutation probe. + #[sqlx::test] + async fn info_refs_per_caller_cap_keys_on_ip_not_did(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6pcip", "pc", "/tmp/pc-nonexistent", None, false) + .await + .unwrap(); + + let did_a = "did:key:z6MkPerCallerKeyingProofDidAAAAAAAAAAAAAAAA"; + let did_b = "did:key:z6MkPerCallerKeyingProofDidBBBBBBBBBBBBBBBB"; + let peer: SocketAddr = "203.0.113.51:5000".parse().unwrap(); + + // Fill the SOURCE IP's single read slot; both DIDs' own slots stay free. + let _slot = state + .git_read_per_caller + .try_acquire(&peer.ip().to_string()) + .expect("first slot for this source IP"); + + // Signed as DID_A from `peer`: keyed by the saturated source IP -> shed 503. + let router = crate::server::build_router(state.clone()); + let mut req = Request::builder() + .method(Method::GET) + .uri("/z6pcip/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + req.extensions_mut() + .insert(crate::auth::AuthenticatedDid(did_a.to_string())); + assert_eq!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a signed caller must be keyed by its source IP, not its DID: the saturated IP must shed it 503" + ); + + // Same IP, a DIFFERENT DID: still keyed by the same saturated IP -> also shed. + // The farm defeat: minting a fresh DID buys no fresh per-source budget. + let router2 = crate::server::build_router(state.clone()); + let mut req2 = Request::builder() + .method(Method::GET) + .uri("/z6pcip/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req2.extensions_mut().insert(ConnectInfo(peer)); + req2.extensions_mut() + .insert(crate::auth::AuthenticatedDid(did_b.to_string())); + assert_eq!( + router2.oneshot(req2).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a second DID from the same source IP must also shed 503: a DID farm cannot multiply the per-source budget" + ); + + // A signed caller from a DIFFERENT source IP keeps its own budget -> not shed. + let other: SocketAddr = "203.0.113.52:5000".parse().unwrap(); + let router3 = crate::server::build_router(state.clone()); + let mut req3 = Request::builder() + .method(Method::GET) + .uri("/z6pcip/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + req3.extensions_mut().insert(ConnectInfo(other)); + req3.extensions_mut() + .insert(crate::auth::AuthenticatedDid(did_a.to_string())); + assert_ne!( + router3.oneshot(req3).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a signed caller from a different source IP must keep its own per-source budget" + ); + } + + /// #174 SC2 (None-key): a request with no resolvable caller key (no ConnectInfo, + /// no trusted header) must NOT be shed by the per-caller cap even when another + /// caller's budget is full — it is bounded by the global read pool only. A None + /// key never keys into the map, so it never 503s from the per-caller sub-cap. + #[sqlx::test] + async fn info_refs_none_key_bypasses_per_caller_cap(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::http::{Method, Request, StatusCode}; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + state.git_read_per_caller = crate::rate_limit::PerCallerConcurrency::new(1, 100); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + state + .db + .upsert_mirror_repo("z6pcnone", "pc", "/tmp/pc-nonexistent", None, false) + .await + .unwrap(); + // Saturate an unrelated caller's budget; the None-key request must be + // unaffected because it never keys into the per-caller map. + let _slot = state + .git_read_per_caller + .try_acquire("203.0.113.99") + .expect("hold an unrelated caller's slot"); + + // No ConnectInfo inserted -> PeerAddr is None -> no per-caller key. + let router = crate::server::build_router(state.clone()); + let req = Request::builder() + .method(Method::GET) + .uri("/z6pcnone/pc/info/refs?service=git-upload-pack") + .body(Body::empty()) + .unwrap(); + assert_ne!( + router.oneshot(req).await.unwrap().status(), + StatusCode::SERVICE_UNAVAILABLE, + "a request with no resolvable caller key must not be shed by the per-caller cap" + ); + } + + /// Repo creation must be throttled by the per-IP creation limiter BEFORE + /// signature verification — otherwise a DID farm (one throwaway did:key per + /// repo, each carrying a valid but machine-solved iCaptcha proof) walks past + /// the per-DID limiter and floods the network, as in the recurring spam-repo + /// incidents. A 429 (not a 401) on an unsigned request from an exhausted IP + /// proves the IP brake runs outermost, ahead of auth. + #[sqlx::test] + async fn repo_creation_is_rate_limited_by_ip(pool: sqlx::PgPool) { + use axum::body::Body; + use axum::extract::ConnectInfo; + use axum::http::{Method, Request, StatusCode}; + use std::net::SocketAddr; + use std::time::Duration; + use tower::ServiceExt; + + let mut state = crate::test_support::test_state(pool).await; + // Tiny limit, keyed on the socket peer (no trusted proxy). + state.create_ip_rate_limiter = + crate::rate_limit::RateLimiter::new(1, Duration::from_secs(60)); + state.push_limiter_trust = crate::rate_limit::TrustedProxy::None; + + let peer: SocketAddr = "203.0.113.77:7000".parse().unwrap(); + // Exhaust this peer's single-request budget up front. + assert!( + state + .create_ip_rate_limiter + .check(&peer.ip().to_string()) + .await + ); + + let router = crate::server::build_router(state); + let mut req = Request::builder() + .method(Method::POST) + .uri("/api/v1/repos") + .header("content-type", "application/json") + .body(Body::from(r#"{"name":"flood","is_public":true}"#)) + .unwrap(); + req.extensions_mut().insert(ConnectInfo(peer)); + + let status = router.oneshot(req).await.unwrap().status(); + assert_eq!( + status, + StatusCode::TOO_MANY_REQUESTS, + "repo creation must be IP-throttled before signature verification" + ); + } + + // ── #174 U2 / F3: second same-repo push serialized until a disconnected first ── + // push's git process GROUP is reaped (RepoWriteLease riding the disconnect reaper). + + /// `kill(pid, 0)` liveness probe (same-uid here, so EPERM never applies). + #[cfg(unix)] + fn f3_alive(pid: i32) -> bool { + unsafe { libc::kill(pid, 0) == 0 } + } + + /// F3 (P1, RED-before/GREEN-after): on a client disconnect DURING receive-pack the + /// disconnected push's git group is torn down by KillGroupOnDrop's detached reaper + /// (~4s TERM/grace/KILL/reap), while RepoWriteGuard::Drop releases the pg advisory + /// lock at the disconnect INSTANT. Without the in-process write lease, a second + /// same-node push then acquires the freed pg lock and mutates the shared local repo + /// WHILE the first group is still writing — a torn snapshot. The lease is held by the + /// write-path AdmissionGuard, which rides that reaper, so the second push must not run + /// its receive-pack (mutate the repo) until the first group is reaped. + /// + /// The fake git labels the pushes by receive-pack arrival order (atomic mkdir): the + /// first (push A) forks a SIGTERM-IGNORING descendant, records its pid, then hangs + /// (so A can be dropped mid-transfer and its group survives the SIGTERM grace); the + /// second (push B, a DIFFERENT source) records that its receive-pack ran — i.e. that + /// B mutated the repo. The load-bearing invariant is strictly ordered, not + /// time-windowed: B's marker must NEVER appear while A's descendant is still alive. + /// + /// Load-bearing: pre-fix (no lease) A's disconnect frees the pg lock, B's + /// acquire_write succeeds within its ~1s retry, and B's receive-pack runs (marker + /// appears) WHILE A's descendant is still alive — RED. With the lease the reaper + /// holds it until the group is ESRCH-gone, so B's marker appears only AFTER — GREEN. + #[cfg(unix)] + #[sqlx::test] + async fn f3_second_push_serialized_until_disconnected_group_reaped(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + + let tmp = tempfile::TempDir::new().unwrap(); + let seq_a = tmp.path().join("seq_a"); // first receive-pack wins this mkdir = push A + let descfile = tmp.path().join("desc.pid"); // A's SIGTERM-ignoring descendant pid + let b_ran = tmp.path().join("b.ran"); // set when B's receive-pack runs (B mutates) + // receive-pack: first invocation (A) forks a TERM-ignoring descendant (bounded + // loop so a RED run leaks no permanent orphan), records its pid, and hangs in + // `wait`; second (B) records that it ran. rev-parse feeds any tail probe. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + receive-pack)\n\ + cat >/dev/null 2>/dev/null\n\ + if mkdir \"{seq}\" 2>/dev/null; then\n\ + sh -c 'trap \"\" TERM; echo $$ > \"{desc}\"; i=0; while [ $i -lt 60 ]; do sleep 0.1; i=$((i+1)); done' &\n\ + wait\n\ + else\n\ + echo 1 > \"{bran}\"\n\ + fi ;;\n\ + rev-parse) echo deadbeef ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + seq = seq_a.display(), + desc = descfile.display(), + bran = b_ran.display(), + ); + let git_bin = write_fake_git(tmp.path(), &body); + // One repo; A and B push to it (same record.id -> same lease key). Non-path-scoped + // + flush-only body -> no post-receive scans to muddy the observation. + let state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f3repo", "r1", false).await; + let did = "did:key:z6MkF3PusherAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA"; + + // Push A: drive its handler future in slices until it reaches receive-pack and the + // fake records its descendant pid (A now holds the lease and is hung). + let mut fut_a = Box::pin(git_receive_pack( + State(state.clone()), + Path(("z6f3repo".to_string(), "r1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some("203.0.113.81:5000".parse::().unwrap())), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + )); + let mut desc: Option = None; + for _ in 0..1000 { + let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut_a).await; + if let Some(p) = std::fs::read_to_string(&descfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + desc = Some(p); + break; + } + } + let desc = desc.expect("push A must reach receive-pack and record its descendant pid"); + assert!( + f3_alive(desc), + "A's descendant must be alive before the disconnect" + ); + + // Push B: a DIFFERENT source, same repo. It blocks on the lease A holds. + let state_b = state.clone(); + let handle_b = tokio::spawn(async move { + git_receive_pack( + State(state_b), + Path(("z6f3repo".to_string(), "r1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some( + "203.0.113.82:5000".parse::().unwrap(), + )), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await + }); + // Give B time to reach and block on the lease acquire. + tokio::time::sleep(std::time::Duration::from_millis(200)).await; + assert!( + !b_ran.exists(), + "B must not have mutated the repo while A legitimately holds the lease" + ); + + // Client disconnect on A: drop its future. RepoWriteGuard::Drop frees the pg lock + // immediately; the write-path AdmissionGuard (carrying the lease's clone (a)) + // rides KillGroupOnDrop's detached reaper, which now tears down A's group. + drop(fut_a); + + // Load-bearing ordering invariant: while A's descendant is still alive (group not + // yet reaped), B must NOT have run its receive-pack. Poll until the descendant is + // gone; every step it is alive, B's marker must be absent. Pre-fix, B's marker + // appears here (RED); with the lease it can only appear after the reap (GREEN). + let mut reaped = false; + for _ in 0..800 { + if !f3_alive(desc) { + reaped = true; + break; + } + assert!( + !b_ran.exists(), + "F3 RED: push B mutated the repo while push A's disconnected git group \ + was still alive (descendant pid {desc}) — the second writer must be \ + serialized until the first group is reaped" + ); + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + // Safety net so a RED run never leaks the orphan. + unsafe { + libc::kill(desc, libc::SIGKILL); + } + assert!( + reaped, + "A's disconnected group must be reaped within the teardown cap" + ); + + // GREEN tail: once the group is reaped the lease frees and B proceeds — its + // receive-pack runs (marker appears) and it returns 200. + let mut b_mutated = false; + for _ in 0..1000 { + if b_ran.exists() { + b_mutated = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + assert!( + b_mutated, + "push B must proceed and mutate the repo once A's group is reaped" + ); + let resp = tokio::time::timeout(std::time::Duration::from_secs(30), handle_b) + .await + .expect("push B must complete once the lease frees") + .expect("push B task must not panic") + .expect("push B must succeed"); + assert_eq!(resp.status(), 200, "push B lands 200 after serialization"); + } + + /// F3 clean-path no-regression: a clean push (no disconnect) releases the lease after + /// the receive-pack group is reaped and the (success-only) Tigris upload in + /// guard.release runs, so the per-repo lease entry is GC'd and a second same-repo + /// push proceeds immediately. A lease that failed to free on the clean path would + /// wedge every subsequent push to the repo. + #[cfg(unix)] + #[sqlx::test] + async fn f3_clean_push_frees_lease_and_second_push_proceeds(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + + let tmp = tempfile::TempDir::new().unwrap(); + // Clean receive-pack: drain stdin, exit 0. No hang, no descendant. + let body = "#!/bin/sh\ncase \"$1\" in\n receive-pack) cat >/dev/null 2>/dev/null ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + let state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f3clean", "c1", false).await; + let did = "did:key:z6MkF3CleanPusherAAAAAAAAAAAAAAAAAAAAAAAA"; + + let push = |st: AppState, peer: &'static str| async move { + tokio::time::timeout( + std::time::Duration::from_secs(30), + git_receive_pack( + State(st), + Path(("z6f3clean".to_string(), "c1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some(peer.parse::().unwrap())), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ), + ) + .await + .expect("a clean push must not wedge on the lease") + .expect("the push must succeed") + }; + + let a = push(state.clone(), "203.0.113.91:5000").await; + assert_eq!(a.status(), 200, "clean push A lands 200"); + // The clean push freed its lease (both clones dropped) -> the entry GC'd. + assert!( + state.repo_write_leases.is_empty(), + "a clean push must free the per-repo lease (Drop-frees-key) so it never wedges" + ); + + let b = push(state.clone(), "203.0.113.92:5000").await; + assert_eq!( + b.status(), + 200, + "a second same-repo push proceeds after a clean first" + ); + assert!( + state.repo_write_leases.is_empty(), + "the lease entry must be freed again after the second clean push" + ); + } + + /// F3 DoS (P2, RED-before/GREEN-after): a second same-repo push that BLOCKS on the + /// per-repo write lease must hold NO global write permit while it waits. The lease + /// is a block-and-wait serializer, so a lease-blocked waiter can sit for up to + /// steal_after (~a full git_service_timeout window). If it grabs a scarce global + /// write-pool slot BEFORE blocking, a handful of hostile sources can stack same-repo + /// pushes, pin every write slot on lease-waiters sending zero bytes, and shed 503 on + /// every push to every OTHER repo node-wide. The fix acquires the lease BEFORE the + /// two write permits, so a blocked waiter pins no slot. + /// + /// Load-bearing invariant: with the write pool sized to 2, push A holds the lease and + /// is in-flight in receive-pack (1 permit held), and same-repo push B is blocked on + /// the lease, `git_write_semaphore.available_permits()` must stay 1 (only A holds). + /// Pre-fix B takes its permit BEFORE blocking on the lease, draining the pool to 0 + /// (RED). With the reorder B blocks before any permit, so the pool stays at 1 (GREEN). + #[cfg(unix)] + #[sqlx::test] + async fn f3_lease_blocked_waiter_holds_no_write_permit(pool: sqlx::PgPool) { + use axum::extract::{Path, State}; + use axum::Extension; + use std::net::SocketAddr; + use std::sync::Arc; + use tokio::sync::Semaphore; + + let tmp = tempfile::TempDir::new().unwrap(); + let seq_a = tmp.path().join("seq_a"); // first receive-pack wins this mkdir = push A + let a_inpack = tmp.path().join("a.inpack"); // set when A reaches receive-pack (holds lease+permit) + let b_ran = tmp.path().join("b.ran"); // set when B's receive-pack runs (B got past the lease) + // receive-pack: first invocation (A) marks that it reached the pack and hangs in a + // bounded loop (so a RED run leaks no permanent orphan); second (B) marks it ran. + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + receive-pack)\n\ + cat >/dev/null 2>/dev/null\n\ + if mkdir \"{seq}\" 2>/dev/null; then\n\ + echo 1 > \"{ainp}\"\n\ + i=0; while [ $i -lt 100 ]; do sleep 0.1; i=$((i+1)); done\n\ + else\n\ + echo 1 > \"{bran}\"\n\ + fi ;;\n\ + rev-parse) echo deadbeef ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + seq = seq_a.display(), + ainp = a_inpack.display(), + bran = b_ran.display(), + ); + let git_bin = write_fake_git(tmp.path(), &body); + let mut state = + f4_state_with_repo(pool.clone(), tmp.path(), &git_bin, "z6f3dos", "d1", false).await; + // Size the write pool to 2 so one in-flight holder (A) leaves exactly one slot + // free, and a pool-holding waiter (B, pre-fix) would drain it to zero. Sizing to + // 1 would 503 B on the pool before it could block on the lease, hiding the bug. + state.git_write_semaphore = Arc::new(Semaphore::new(2)); + let did = "did:key:z6MkF3DosPusherAAAAAAAAAAAAAAAAAAAAAAAAAA"; + + // Push A: drive its handler future in slices until it reaches receive-pack (it now + // holds the lease and one write permit and is hung). available_permits() drops to 1. + let mut fut_a = Box::pin(git_receive_pack( + State(state.clone()), + Path(("z6f3dos".to_string(), "d1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some("203.0.113.71:5000".parse::().unwrap())), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + )); + for _ in 0..1000 { + let _ = tokio::time::timeout(std::time::Duration::from_millis(10), &mut fut_a).await; + if a_inpack.exists() { + break; + } + } + assert!( + a_inpack.exists(), + "push A must reach receive-pack and hold the lease" + ); + assert_eq!( + state.git_write_semaphore.available_permits(), + 1, + "with the pool sized to 2, the single in-flight holder (A) leaves one slot free" + ); + + // Push B: a DIFFERENT source, same repo. It blocks on the lease A holds. + let state_b = state.clone(); + let handle_b = tokio::spawn(async move { + git_receive_pack( + State(state_b), + Path(("z6f3dos".to_string(), "d1".to_string())), + Extension(crate::auth::AuthenticatedDid(did.to_string())), + crate::rate_limit::PeerAddr(Some( + "203.0.113.72:5000".parse::().unwrap(), + )), + axum::http::HeaderMap::new(), + axum::body::Bytes::from_static(b"0000"), + ) + .await + }); + + // Load-bearing check: while B is a lease-blocked waiter the pool must stay at 1 + // (only A holds a permit). Poll the invariant across a full window; pre-fix B + // grabs the last slot within ms and the pool falls to 0 (RED), post-fix it never + // does (GREEN). A stable state, not a one-shot race: B stays blocked on the lease + // (steal_after is far larger than this window) so once it settles the pool holds. + for _ in 0..100 { + assert_eq!( + state.git_write_semaphore.available_permits(), + 1, + "F3 DoS RED: a lease-blocked same-repo waiter took a global write permit \ + while sending zero bytes, draining the pool — a blocked waiter must pin \ + no write-pool slot" + ); + assert!( + !b_ran.exists(), + "B must not have run receive-pack while A holds the lease" + ); + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + // Non-vacuous: B is genuinely parked on the lease, not returned early. + assert!( + !handle_b.is_finished(), + "push B must still be blocked on the lease at this point" + ); + + // Client disconnect on A: drop its future. The write-path AdmissionGuard rides the + // reaper, freeing the lease once A's group is reaped; B then proceeds. + drop(fut_a); + let resp = tokio::time::timeout(std::time::Duration::from_secs(30), handle_b) + .await + .expect("push B must complete once A's group is reaped and the lease frees") + .expect("push B task must not panic") + .expect("push B must succeed"); + assert_eq!(resp.status(), 200, "push B lands 200 after serialization"); + assert!(b_ran.exists(), "push B ran its receive-pack once unblocked"); } } diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 720fb3ae..431ac983 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -520,6 +520,22 @@ mod tests { sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), shutdown_tx: tokio::sync::watch::channel(false).0, + git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + pin_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + encrypt_inflight: crate::state::EncryptInflight::new(), + repo_write_leases: crate::state::RepoWriteLeases::new(), + git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), + git_push_advert_per_caller: + crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), + git_write_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), + git_ipfs_walk_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_ipfs_walk_per_caller: + crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), + ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + git_bin: "git".to_string(), } } diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index fc2247d9..d70be5ea 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -172,9 +172,9 @@ pub struct Config { /// its process group torn down, in seconds. Bounds a git that neither /// finishes nor disconnects. Must be positive; set it very large to /// effectively disable the bound. Default: 600s (10 min), generous for large - /// clones. Does not cover the ref advertisement (`info/refs`) or the - /// withheld-blob fetch path (`upload_pack_excluding`, a blocking - /// `spawn_blocking` a tokio timeout cannot cancel); both remain unbounded. + /// clones. Also bounds the ref advertisement (`info/refs`) and the withheld-blob + /// pack build (`upload_pack_excluding`'s pack-objects stage), which now share the + /// same timeout + process-group teardown (#174). #[arg( long, env = "GITLAWB_GIT_SERVICE_TIMEOUT_SECS", @@ -183,13 +183,38 @@ pub struct Config { )] pub git_service_timeout_secs: u64, + /// Maximum wall-clock time the storage-acquisition phase of a served git + /// operation may run before the request is shed with a 503, in seconds. This + /// bounds `RepoStore::{acquire,acquire_fresh,acquire_write}` — the Tigris + /// HEAD/GET on a read/advert acquire and the advisory-lock retry loop (incl. a + /// per-iteration `pg_try_advisory_lock` that can block on a hung Postgres pool) + /// on a write acquire. A concurrency permit is taken BEFORE this phase, and + /// `git_service_timeout_secs` only starts once git spawns, so without this the + /// acquire phase is unbounded: a stalled backend pins the permit and drains the + /// pool until every later request 503s. On expiry the permit is released and a + /// bounded 503 + Retry-After is returned (fail-closed). Kept separate from + /// `git_service_timeout_secs` because acquisition and git execution are distinct + /// cost centers — one shared budget would let a slow acquire starve git. Must be + /// positive; set it very large to effectively disable the bound. Default: 30s. + #[arg( + long, + env = "GITLAWB_GIT_ACQUIRE_TIMEOUT_SECS", + default_value_t = 30, + value_parser = clap::value_parser!(u64).range(1..) + )] + pub git_acquire_timeout_secs: u64, + /// Maximum connections in the PostgreSQL pool. This is a cap, not a floor /// (connections open lazily). Size against the database server's - /// max_connections, remembering admin tooling opens its own pool. + /// max_connections, remembering admin tooling opens its own pool. Each + /// concurrent write pins one pooled connection for its whole duration (the + /// advisory lock in `repo_store::acquire_write` is connection-affine), so this + /// must exceed `max_concurrent_git_pushes` by `DB_POOL_APP_HEADROOM` or slow + /// pushes starve every other DB path — enforced by `Config::validate`. #[arg( long, env = "GITLAWB_DB_MAX_CONNECTIONS", - default_value_t = 20, + default_value_t = 48, value_parser = clap::value_parser!(u32).range(1..) )] pub db_max_connections: u32, @@ -234,6 +259,214 @@ pub struct Config { value_parser = clap::value_parser!(u64).range(1..) )] pub db_retry_max_secs: u64, + + /// Maximum number of served git operations (upload-pack / receive-pack / + /// info-refs) allowed to run concurrently. Beyond this the node sheds the + /// request with a clean 503 + Retry-After instead of spawning another git + /// subprocess and risking PID/thread exhaustion. Portable backstop: the + /// compose `pids_limit` is not present on Fly, whose connection-concurrency + /// cap is a different axis (500 connections each fan out to git + + /// pack-objects + threads). Size below the process budget with headroom. + /// + /// This is the READ pool (`git_read_semaphore`): upload-pack and the UPLOAD-PACK + /// `info/refs` advertisement only. The authenticated push POST draws from a + /// separate write pool (`max_concurrent_git_pushes`) that anonymous reads can + /// never reach, and each read caller is additionally bounded by + /// `max_concurrent_reads_per_caller`, so an anonymous flood cannot shed the actual + /// push nor monopolize reads (#174). The anon-reachable RECEIVE-PACK `info/refs` + /// advertisement draws from its OWN dedicated pool (sized like the write pool but + /// disjoint), so an advertisement flood can never occupy a permit the + /// authenticated push POST needs at admission (#174). + /// + /// A permit is held for the whole op. Every git subprocess that STREAMS is + /// duration-bounded and reaps its process group on disconnect: upload-pack, + /// receive-pack, and both info/refs advertisements run under + /// `git_service_timeout_secs` with `process_group(0)` teardown, and the + /// withheld-blob (`upload_pack_excluding`) pack-objects stage plus the push-side + /// candidate-discovery children (`rev-list` / `cat-file`) now run under the same + /// bounded runner with process-group teardown, so a stuck git child no longer + /// holds its slot indefinitely (#174 closed the duration/cancellation gaps this + /// comment previously tracked). + /// + /// Default: 128. Must be between 1 and 1_048_576; the ceiling keeps the value + /// well under tokio's `Semaphore` permit limit so an oversized value is a + /// clean CLI error rather than a boot-time panic. + #[arg( + long, + env = "GITLAWB_MAX_CONCURRENT_GIT_OPS", + default_value_t = 128, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub max_concurrent_git_ops: usize, + + /// Maximum number of concurrent `git-receive-pack` (push) operations. The + /// authenticated push POST draws from this dedicated pool, separate from + /// `max_concurrent_git_ops` (reads), so a flood of anonymous reads cannot shed an + /// authenticated push at admission (#174). The anon-reachable receive-pack + /// `info/refs` advertisement runs in a SEPARATE pool of the same size (derived + /// from this knob), disjoint from this one, so an advertisement flood cannot + /// occupy a POST's slot either (#174). Beyond this a push sheds a clean 503 + + /// Retry-After. + /// + /// Default: 32. Must be between 1 and 1_048_576 (the ceiling keeps the value + /// under tokio's `Semaphore` permit limit so an oversized value is a clean CLI + /// error rather than a boot-time panic). + #[arg( + long, + env = "GITLAWB_MAX_CONCURRENT_GIT_PUSHES", + default_value_t = 32, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub max_concurrent_git_pushes: usize, + + /// Max concurrent post-push pin loops (`ipfs_pin` and `pinata` + /// `pin_new_objects`) across all repos. `EncryptInflight` bounds the outstanding + /// pin-task COUNT to one per repo, but each pin loop holds a full per-push + /// object-id list (up to `git_max_pack_bytes` worth of OIDs) while it walks it, + /// so N distinct repos could hold N such lists at once. This caps how many run + /// concurrently, bounding that MB-scale memory regardless of how many repos an + /// authenticated actor pushes to (#174 F6). Beyond it a pin loop DEFERS (waits), + /// never drops — a dropped pin would lose the object's replication copy. + /// + /// Default: 8. Must be between 1 and 1_048_576. + #[arg( + long, + env = "GITLAWB_MAX_CONCURRENT_PIN_TASKS", + default_value_t = 8, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub max_concurrent_pin_tasks: usize, + + /// Maximum concurrent read operations (`upload-pack` and the upload-pack + /// `info/refs` advertisement) a single caller may hold at once, so one caller + /// cannot monopolize the `max_concurrent_git_ops` read pool (#174). Callers are + /// keyed on the RESOLVED SOURCE IP, never the DID — a signature does not move a + /// caller off this cap, so an authenticated client cannot mint DIDs to escape it. + /// IMPORTANT: the source-IP key is only as granular as `GITLAWB_TRUSTED_PROXY`. + /// Left unset (the default), a node behind an edge/NAT keys all callers on the + /// edge IP, so this cap collapses to a single global cap rather than per-client. + /// Set `GITLAWB_TRUSTED_PROXY` to key on the real client; a high-fanout caller (a + /// CI fleet behind one NAT) then needs the operator to raise this. Over-cap for a + /// caller sheds a clean 503 + Retry-After. + /// + /// Default: 16. Must be between 1 and 1_048_576. + #[arg( + long, + env = "GITLAWB_MAX_CONCURRENT_READS_PER_CALLER", + default_value_t = 16, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub max_concurrent_reads_per_caller: usize, + + /// Maximum number of concurrent `GET /ipfs/{cid}` requests that may run their + /// visibility walk at once. The publicly-reachable `/ipfs/{cid}` route runs + /// `allowed_blob_set_for_caller_bounded` in `spawn_blocking` — a full-history + /// git walk (up to `git_service_timeout_secs`) — for each candidate repo. It + /// draws from THIS pool, not any served-git pool: a distinct public cost center + /// on a distinct surface, so sharing a git pool would let anonymous /ipfs + /// traffic shed authenticated git ops (the auth-boundary trap). A permit is + /// held for the whole request (across the repo loop) so it reflects real + /// blocking-thread occupancy, not merely the tokio wait. Beyond this the request + /// sheds a clean 503 + Retry-After. Must be between 1 and 1_048_576; the ceiling + /// keeps the value under tokio's `Semaphore` permit limit so an oversized value + /// is a clean CLI error rather than a boot-time panic. Default: 32. + #[arg( + long, + env = "GITLAWB_MAX_CONCURRENT_IPFS_WALKS", + default_value_t = 32, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub max_concurrent_ipfs_walks: usize, + + /// Maximum concurrent `/ipfs/{cid}` walk requests a single source may hold at + /// once, so one source cannot monopolize `max_concurrent_ipfs_walks` (#174). + /// Callers are keyed on the RESOLVED SOURCE IP (`client_key`/`GITLAWB_TRUSTED_PROXY`), + /// never the DID — `/ipfs` accepts any `did:key` via `optional_signature` with no + /// admission step, so keying on the DID would let one host mint disposable DIDs to + /// multiply its budget. A request with no resolvable key (no trusted header, no + /// peer) is bounded by the global pool only, never this sub-cap. Over-cap sheds a + /// clean 503 + Retry-After. Must be between 1 and 1_048_576. Default: 4. + #[arg( + long, + env = "GITLAWB_IPFS_WALK_PER_SOURCE", + default_value_t = 4, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub ipfs_walk_per_source: usize, + + /// Upper bound on the number of EXPENSIVE visibility walks + /// (`allowed_blob_set_for_caller_bounded`, a full-history git walk in a + /// blocking thread) a single `/ipfs/{cid}` request may run. Only a blob in a + /// path-scoped repo costs a walk, so the cap counts exactly those candidates + /// — cheap probe-only visits are bounded by `ipfs_max_repo_visits` instead + /// (counting them here would starve a plain public copy past the cap out of + /// its 200). On exhaustion the walk-needing repo is skipped WITHOUT a verdict + /// and the scan continues; if the request then finds the object nowhere it + /// sheds a retryable 503 + Retry-After rather than misreport existing content + /// absent with a 404. The handler still short-circuits the moment it serves. + /// Must be between 1 and 1_048_576. Default: 64. + #[arg( + long, + env = "GITLAWB_IPFS_MAX_REPOS_WALKED", + default_value_t = 64, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub ipfs_max_repos_walked: usize, + + /// Ceiling on the number of repos a single `/ipfs/{cid}` request may VISIT — + /// pass the repo-level visibility gate into the acquire + `cat-file` probe. + /// Each visit costs a `RepoStore::acquire` (on a Tigris cache miss that is a + /// full repo-archive download from object storage, so the worst-case + /// object-store fetch count for one request equals this ceiling) plus a git + /// probe subprocess. On exhaustion the scan STOPS — unlike + /// `ipfs_max_repos_walked`, which skips just the walk-needing repo, there is + /// no cheaper way to keep scanning — and the request sheds a retryable 503 + + /// Retry-After rather than a false 404. Must be between 1 and 1_048_576. + /// Default: 1024. + #[arg( + long, + env = "GITLAWB_IPFS_MAX_REPO_VISITS", + default_value_t = 1024, + value_parser = clap::builder::RangedU64ValueParser::::new().range(1..=1_048_576) + )] + pub ipfs_max_repo_visits: usize, + + /// Absolute wall-clock budget for one admitted `GET /ipfs/{cid}` request's + /// acquire+walk lifetime, in seconds. `max_concurrent_ipfs_walks` bounds how + /// MANY requests hold walk slots; this bounds how LONG one admitted request + /// may keep its slot. Without it, each repo iteration draws a fresh + /// `git_acquire_timeout_secs` and each expensive walk a fresh + /// `git_service_timeout_secs`, so one request scanning many repos could hold + /// a scarce walk slot for hours. Every stage (acquire, `cat-file` probe, + /// visibility walk, content read) starts only while budget remains, and the + /// acquire wait and walk deadline are clamped to `min(their own timeout, + /// remaining budget)`; a stage is never started with zero remaining. On + /// exhaustion the scan stops without a verdict and the request sheds a + /// retryable 503 + Retry-After rather than a false 404. The clamps bound + /// only the acquire and walk stages (overshoot there is the walk watchdog's + /// SIGTERM grace + SIGKILL settle); the `object_type` / + /// `read_object_content` probe subprocesses are budget-checked before they + /// start but have NO duration bound of their own, so a hung git probe + /// (corrupt pack, stuck filesystem) holds the request's walk slot for the + /// full duration of the hang. + /// Must be positive. Default: 600s (10 min), matching + /// `git_service_timeout_secs` so a single full-length walk still fits. + #[arg( + long, + env = "GITLAWB_IPFS_REQUEST_BUDGET_SECS", + default_value_t = 600, + value_parser = clap::value_parser!(u64).range(1..) + )] + pub ipfs_request_budget_secs: u64, + + /// Per-client-IP rate limit for `GET /ipfs/{cid}`, in requests per hour. The + /// route is publicly reachable (`optional_signature`) and each request can drive + /// a full-history git walk, so it carries a per-IP flood brake in addition to the + /// concurrency cap above (a rate limit bounds request *rate*, the semaphore + /// bounds concurrent slow holds — different axes). Keyed on the resolved client + /// IP via `GITLAWB_TRUSTED_PROXY`. `0` disables. Default: 600. + #[arg(long, env = "GITLAWB_IPFS_RATE_LIMIT", default_value_t = 600)] + pub ipfs_rate_limit: usize, } impl Config { @@ -250,6 +483,36 @@ impl Config { } PathBuf::from(&self.key_path) } + + /// DB connections reserved for everything other than held write-locks: auth + /// lookups, visibility-rule reads, the post-receive tail's own DB writes, and + /// admin tooling. A write pins one pooled connection for its whole duration, so + /// the pool must clear the concurrent-write cap by at least this margin. + pub const DB_POOL_APP_HEADROOM: u32 = 8; + + /// Cross-field boot validation. Single-field ranges are enforced by clap; this + /// catches combinations that ship a denial-of-service under otherwise-valid + /// values. Call once at startup and fail fast on `Err`. + pub fn validate(&self) -> Result<(), String> { + // A write pins one pooled connection for its whole duration (the + // connection-affine advisory lock in repo_store::acquire_write), and + // concurrent writes are capped at max_concurrent_git_pushes. If the pool + // does not exceed that cap by DB_POOL_APP_HEADROOM, a burst of slow pushes + // drains every connection and every other DB path 503s. (#174 F1) + let floor = (self.max_concurrent_git_pushes as u64) + (Self::DB_POOL_APP_HEADROOM as u64); + if (self.db_max_connections as u64) < floor { + return Err(format!( + "GITLAWB_DB_MAX_CONNECTIONS ({}) must be at least max_concurrent_git_pushes ({}) \ + + {} headroom = {}: each concurrent write pins one pooled connection for its whole \ + duration, so a smaller pool lets a burst of slow pushes starve every other DB path.", + self.db_max_connections, + self.max_concurrent_git_pushes, + Self::DB_POOL_APP_HEADROOM, + floor + )); + } + Ok(()) + } } #[cfg(test)] @@ -272,4 +535,245 @@ mod tests { Config::try_parse_from(["gitlawb-node", "--git-service-timeout-secs", "0"]).is_err() ); } + + #[test] + fn max_concurrent_pin_tasks_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).max_concurrent_pin_tasks, + 8 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-pin-tasks", "2"]) + .max_concurrent_pin_tasks, + 2 + ); + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-pin-tasks", "0"]).is_err() + ); + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-pin-tasks", "1048577"]) + .is_err() + ); + } + + #[test] + fn max_concurrent_git_ops_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).max_concurrent_git_ops, + 128 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-git-ops", "8"]) + .max_concurrent_git_ops, + 8 + ); + // 0 permits would shed every served-git request with a 503; clap must reject it. + assert!(Config::try_parse_from(["gitlawb-node", "--max-concurrent-git-ops", "0"]).is_err()); + // Above the ceiling would panic tokio's Semaphore::new at boot (permits > + // usize::MAX >> 3); clap must reject it as a clean CLI error instead. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-git-ops", "1048577"]) + .is_err() + ); + // The ceiling itself is accepted. + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-git-ops", "1048576"]) + .max_concurrent_git_ops, + 1_048_576 + ); + } + + #[test] + fn max_concurrent_git_pushes_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).max_concurrent_git_pushes, + 32 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-git-pushes", "8"]) + .max_concurrent_git_pushes, + 8 + ); + // 0 permits would shed every push with a 503; clap must reject it. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-git-pushes", "0"]).is_err() + ); + // Above the ceiling would panic tokio's Semaphore::new at boot; clap rejects it. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-git-pushes", "1048577"]) + .is_err() + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-git-pushes", "1048576"]) + .max_concurrent_git_pushes, + 1_048_576 + ); + } + + #[test] + fn max_concurrent_ipfs_walks_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).max_concurrent_ipfs_walks, + 32 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-ipfs-walks", "4"]) + .max_concurrent_ipfs_walks, + 4 + ); + // 0 permits would shed every /ipfs walk with a 503; clap must reject it. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-ipfs-walks", "0"]).is_err() + ); + // Above the ceiling would panic tokio's Semaphore::new at boot; clap rejects it. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-ipfs-walks", "1048577"]) + .is_err() + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-ipfs-walks", "1048576"]) + .max_concurrent_ipfs_walks, + 1_048_576 + ); + } + + #[test] + fn ipfs_walk_per_source_defaults_and_rejects_out_of_range() { + assert_eq!(Config::parse_from(["gitlawb-node"]).ipfs_walk_per_source, 4); + assert_eq!( + Config::parse_from(["gitlawb-node", "--ipfs-walk-per-source", "2"]) + .ipfs_walk_per_source, + 2 + ); + // 0 would shed every /ipfs walk from a keyed source; clap must reject it. + assert!(Config::try_parse_from(["gitlawb-node", "--ipfs-walk-per-source", "0"]).is_err()); + assert!( + Config::try_parse_from(["gitlawb-node", "--ipfs-walk-per-source", "1048577"]).is_err() + ); + } + + #[test] + fn ipfs_max_repos_walked_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).ipfs_max_repos_walked, + 64 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "8"]) + .ipfs_max_repos_walked, + 8 + ); + // 0 would walk no repos (serve nothing); clap must reject it. + assert!(Config::try_parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "0"]).is_err()); + assert!( + Config::try_parse_from(["gitlawb-node", "--ipfs-max-repos-walked", "1048577"]).is_err() + ); + } + + #[test] + fn ipfs_max_repo_visits_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).ipfs_max_repo_visits, + 1024 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--ipfs-max-repo-visits", "8"]) + .ipfs_max_repo_visits, + 8 + ); + // 0 would visit no repos (serve nothing); clap must reject it. + assert!(Config::try_parse_from(["gitlawb-node", "--ipfs-max-repo-visits", "0"]).is_err()); + assert!( + Config::try_parse_from(["gitlawb-node", "--ipfs-max-repo-visits", "1048577"]).is_err() + ); + } + + #[test] + fn ipfs_request_budget_secs_defaults_to_600_and_rejects_zero() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).ipfs_request_budget_secs, + 600 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--ipfs-request-budget-secs", "30"]) + .ipfs_request_budget_secs, + 30 + ); + // 0 would expire every /ipfs request at its first stage (unconditional + // 503); clap must reject it. + assert!( + Config::try_parse_from(["gitlawb-node", "--ipfs-request-budget-secs", "0"]).is_err() + ); + } + + #[test] + fn max_concurrent_reads_per_caller_defaults_and_rejects_out_of_range() { + assert_eq!( + Config::parse_from(["gitlawb-node"]).max_concurrent_reads_per_caller, + 16 + ); + assert_eq!( + Config::parse_from(["gitlawb-node", "--max-concurrent-reads-per-caller", "4"]) + .max_concurrent_reads_per_caller, + 4 + ); + // 0 would shed every read from a keyed caller; clap must reject it. + assert!( + Config::try_parse_from(["gitlawb-node", "--max-concurrent-reads-per-caller", "0"]) + .is_err() + ); + assert!(Config::try_parse_from([ + "gitlawb-node", + "--max-concurrent-reads-per-caller", + "1048577" + ]) + .is_err()); + assert_eq!( + Config::parse_from([ + "gitlawb-node", + "--max-concurrent-reads-per-caller", + "1048576" + ]) + .max_concurrent_reads_per_caller, + 1_048_576 + ); + } + + /// #174 F1: a connection-affine write lock pins a pooled connection per + /// concurrent write, so the pool must clear `max_concurrent_git_pushes` by + /// `DB_POOL_APP_HEADROOM` or a push burst starves every other DB path. + /// `validate()` must reject an under-sized pool at boot. + #[test] + fn db_pool_must_clear_the_git_push_cap() { + // Shipped defaults validate (48 >= 32 + 8). + Config::parse_from(["gitlawb-node"]) + .validate() + .expect("default config must validate"); + + // An under-sized pool relative to the push cap is rejected (20 < 32 + 8). + let under = Config::parse_from([ + "gitlawb-node", + "--db-max-connections", + "20", + "--max-concurrent-git-pushes", + "32", + ]); + assert!( + under.validate().is_err(), + "db_max_connections 20 below max_concurrent_git_pushes 32 + headroom must be rejected" + ); + + // Exactly at the floor validates (40 == 32 + 8). + let at_floor = Config::parse_from([ + "gitlawb-node", + "--db-max-connections", + "40", + "--max-concurrent-git-pushes", + "32", + ]); + assert!( + at_floor.validate().is_ok(), + "db_max_connections at the floor (pushes + headroom) must validate" + ); + } } diff --git a/crates/gitlawb-node/src/error.rs b/crates/gitlawb-node/src/error.rs index bdd5aec9..a07235fa 100644 --- a/crates/gitlawb-node/src/error.rs +++ b/crates/gitlawb-node/src/error.rs @@ -47,6 +47,9 @@ pub enum AppError { #[error("git service timed out: {0}")] Timeout(String), + #[error("server overloaded: {0}")] + Overloaded(String), + #[error("database error: {0}")] Db(#[from] sqlx::Error), @@ -147,6 +150,12 @@ impl IntoResponse for AppError { DB_UNAVAILABLE_CODE, DB_UNAVAILABLE_MESSAGE.into(), ), + // 503 with a Retry-After (attached after this match — the shared tail + // can't carry per-variant headers). This is the single place Overloaded + // becomes a response, so it can never ship a 503 without the retry hint. + AppError::Overloaded(msg) => { + (StatusCode::SERVICE_UNAVAILABLE, "overloaded", msg.clone()) + } AppError::Db(e) => (StatusCode::INTERNAL_SERVER_ERROR, "db_error", e.to_string()), AppError::Internal(e) => ( StatusCode::INTERNAL_SERVER_ERROR, @@ -160,7 +169,17 @@ impl IntoResponse for AppError { "message": message, })); - (status, body).into_response() + let mut resp = (status, body).into_response(); + // Overloaded advertises when to retry. It rides the shared tail above for + // its body/status, so the header is attached here rather than in a bespoke + // early return — keeping the variant handled in exactly one place. + if matches!(self, AppError::Overloaded(_)) { + resp.headers_mut().insert( + axum::http::header::RETRY_AFTER, + axum::http::HeaderValue::from_static("1"), + ); + } + resp } } @@ -182,4 +201,14 @@ mod tests { StatusCode::INTERNAL_SERVER_ERROR ); } + + #[test] + fn overloaded_maps_to_503_with_retry_after() { + let resp = AppError::Overloaded("x".into()).into_response(); + assert_eq!(resp.status(), StatusCode::SERVICE_UNAVAILABLE); + assert_eq!( + resp.headers().get("retry-after").unwrap().to_str().unwrap(), + "1" + ); + } } diff --git a/crates/gitlawb-node/src/git/push_delta.rs b/crates/gitlawb-node/src/git/push_delta.rs index 7ab00816..0b569693 100644 --- a/crates/gitlawb-node/src/git/push_delta.rs +++ b/crates/gitlawb-node/src/git/push_delta.rs @@ -29,9 +29,9 @@ use std::collections::HashSet; use std::path::{Path, PathBuf}; -use std::process::Command; +use std::time::{Duration, Instant}; -use anyhow::{bail, Context, Result}; +use anyhow::Result; /// Env var that forces the push path to always full-scan, bypassing the delta /// optimization (KTD7 kill-switch). Reuses the already-tested fallback branch @@ -64,11 +64,21 @@ pub enum PinCandidates { /// Fail-closed: any condition where the introduced set cannot be safely /// determined returns [`PinCandidates::FullScanRequired`] rather than a partial /// set, so the caller full-scans instead of silently under-pinning. -pub fn resolve_push_delta(repo_path: &Path, new_tips: &[&str], old_tips: &[&str]) -> PinCandidates { - // KTD7 kill-switch: force the (already-tested) full-scan fallback. The env - // read is split out from the pure logic so the resolver stays unit-testable - // without touching process-global state. - resolve_push_delta_inner(repo_path, new_tips, old_tips, force_full_scan()) +pub fn resolve_push_delta( + repo_path: &Path, + new_tips: &[&str], + old_tips: &[&str], + git_bin: &str, + deadline: Instant, +) -> PinCandidates { + resolve_push_delta_inner( + repo_path, + new_tips, + old_tips, + force_full_scan(), + git_bin, + deadline, + ) } /// Whether the force-full-scan kill-switch env var is set. @@ -83,6 +93,8 @@ fn resolve_push_delta_inner( new_tips: &[&str], old_tips: &[&str], force_full_scan: bool, + git_bin: &str, + deadline: Instant, ) -> PinCandidates { if force_full_scan { tracing::debug!("{FORCE_FULL_SCAN_ENV} set — forcing full scan"); @@ -102,7 +114,7 @@ fn resolve_push_delta_inner( // commit). Bare `cat-file -t` returns `tag` for an annotated tag, and // `for-each-ref %(*objecttype)` peels only one level — neither is correct. for tip in new_tips { - match peeled_object_type(repo_path, tip) { + match peeled_object_type(repo_path, tip, git_bin, deadline) { Some(t) if t == "commit" => {} other => { tracing::debug!( @@ -115,7 +127,7 @@ fn resolve_push_delta_inner( } } - match rev_list_delta(repo_path, new_tips, old_tips) { + match rev_list_delta(repo_path, new_tips, old_tips, git_bin, deadline) { Ok(oids) => PinCandidates::Delta(oids), Err(e) => { tracing::debug!(err = %e, "push-delta rev-list failed — forcing full scan"); @@ -126,41 +138,43 @@ fn resolve_push_delta_inner( /// Return the fully-peeled object type of `sha` (e.g. `commit`, `tree`, /// `blob`), or `None` if the object is missing/unpeelable or git errored. -fn peeled_object_type(repo_path: &Path, sha: &str) -> Option { - let output = Command::new("git") - .args(["cat-file", "-t", &format!("{sha}^{{}}")]) - .current_dir(repo_path) - .output() - .ok()?; - if !output.status.success() { - return None; - } - Some(String::from_utf8_lossy(&output.stdout).trim().to_string()) +fn peeled_object_type( + repo_path: &Path, + sha: &str, + git_bin: &str, + deadline: Instant, +) -> Option { + let peel = format!("{sha}^{{}}"); + let out = crate::git::visibility_pack::run_bounded_git( + git_bin, + &["cat-file", "-t", &peel], + repo_path, + b"", + deadline, + ) + .ok()?; + Some(String::from_utf8_lossy(&out).trim().to_string()) } /// Run `git rev-list --objects --no-object-names --not ` and return /// the bare OID set. Decides on `status.success()` *before* parsing stdout, so /// a walk that prints a valid prefix then errors mid-walk is discarded. -fn rev_list_delta(repo_path: &Path, new_tips: &[&str], old_tips: &[&str]) -> Result> { +fn rev_list_delta( + repo_path: &Path, + new_tips: &[&str], + old_tips: &[&str], + git_bin: &str, + deadline: Instant, +) -> Result> { let mut args: Vec<&str> = vec!["rev-list", "--objects", "--no-object-names"]; args.extend_from_slice(new_tips); if !old_tips.is_empty() { args.push("--not"); args.extend_from_slice(old_tips); } - - let output = Command::new("git") - .args(&args) - .current_dir(repo_path) - .output() - .context("failed to run git rev-list for push delta")?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - bail!("git rev-list failed: {stderr}"); - } - - let stdout = String::from_utf8_lossy(&output.stdout); + let out = + crate::git::visibility_pack::run_bounded_git(git_bin, &args, repo_path, b"", deadline)?; + let stdout = String::from_utf8_lossy(&out); Ok(stdout .lines() .map(|l| l.trim().to_string()) @@ -175,23 +189,19 @@ fn rev_list_delta(repo_path: &Path, new_tips: &[&str], old_tips: &[&str]) -> Res /// reconciliation sweep relies on. It returns *all* objects (including /// unreachable/dangling ones), which is what the sweep needs to catch /// stragglers — do not swap it for a reachability walk. -pub fn list_all_objects(repo_path: &Path) -> Result> { - let output = Command::new("git") - .args([ +pub fn list_all_objects(repo_path: &Path, git_bin: &str, deadline: Instant) -> Result> { + let out = crate::git::visibility_pack::run_bounded_git( + git_bin, + &[ "cat-file", "--batch-all-objects", "--batch-check=%(objectname)", - ]) - .current_dir(repo_path) - .output() - .context("failed to run git cat-file")?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - bail!("git cat-file failed: {stderr}"); - } - - let stdout = String::from_utf8_lossy(&output.stdout); + ], + repo_path, + b"", + deadline, + )?; + let stdout = String::from_utf8_lossy(&out); Ok(stdout .lines() .map(|l| l.trim().to_string()) @@ -203,23 +213,23 @@ pub fn list_all_objects(repo_path: &Path) -> Result> { /// `--batch-check='%(objectname) %(objecttype)'`. The pin path's fail-closed /// filter needs to tell blobs (content, withholdable) from commits/trees /// (structural, never withheld) without typing the candidate list itself. -pub fn list_all_objects_with_type(repo_path: &Path) -> Result> { - let output = Command::new("git") - .args([ +pub fn list_all_objects_with_type( + repo_path: &Path, + git_bin: &str, + deadline: Instant, +) -> Result> { + let out = crate::git::visibility_pack::run_bounded_git( + git_bin, + &[ "cat-file", "--batch-all-objects", "--batch-check=%(objectname) %(objecttype)", - ]) - .current_dir(repo_path) - .output() - .context("failed to run git cat-file")?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - bail!("git cat-file failed: {stderr}"); - } - - let stdout = String::from_utf8_lossy(&output.stdout); + ], + repo_path, + b"", + deadline, + )?; + let stdout = String::from_utf8_lossy(&out); Ok(stdout .lines() .filter_map(|l| { @@ -235,8 +245,12 @@ pub fn list_all_objects_with_type(repo_path: &Path) -> Result Result> { - Ok(list_all_objects_with_type(repo_path)? +pub fn all_blob_oids( + repo_path: &Path, + git_bin: &str, + deadline: Instant, +) -> Result> { + Ok(list_all_objects_with_type(repo_path, git_bin, deadline)? .into_iter() .filter(|(_, ty)| ty == "blob") .map(|(oid, _)| oid) @@ -268,22 +282,68 @@ pub struct PinCandidateSet { /// it can never leak because the withheld/fail-closed filter still runs on /// whatever set is returned. `full_scan` rides on the returned set so the caller /// knows when the dangling-inclusive filter is required. +/// +/// `scan_sem` is the post-receive scan admission pool (`git_encrypt_semaphore`, +/// #174 F4): both git-spawning stages — the per-tip `cat-file` probe + delta +/// rev-list, and the full-scan fallback — run under ONE permit held for the +/// whole blocking scan. The deletion-only fast path (no new tips) spawns no git +/// and never parks. +/// +/// `force_full_scan` forces the full-scan fallback regardless of the tips — the +/// coalesced-drain `PendingWork::FullScan` marker (#174 F5). It composes with +/// the env kill-switch (either forces), and it disqualifies the empty-tips fast +/// path: a forced-scan call with no tips must still enumerate the repo, never +/// silently resolve to an empty delta (which would pin nothing and lose the +/// drained pushes' work — the F5 bug shape). pub async fn resolve_candidates_for_push( + scan_sem: std::sync::Arc, repo_path: PathBuf, new_tips: Vec, old_tips: Vec, + git_bin: String, + timeout: Duration, + force_full_scan: bool, ) -> PinCandidateSet { + let force = force_full_scan || self::force_full_scan(); + // No-git fast path: a deletion-only push resolves to an empty delta without + // spawning any git child, so it must not park on the scan pool. A forced + // full scan (caller flag or kill-switch) disqualifies it — the scan spawns git. + if new_tips.is_empty() && !force { + tracing::info!(delta = 0usize, repo = %repo_path.display(), "pin candidate set from push delta"); + return PinCandidateSet { + candidates: Vec::new(), + full_scan: false, + }; + } + // Scan admission (#174 F4): DEFER, never shed — a dropped scan would + // silently under-pin this push. The permit moves into the blocking closure + // so a started scan always completes holding it. Residuals at + // `acquire_scan_permit`. + let permit = + crate::state::acquire_scan_permit(scan_sem, &repo_path, "pin-candidate scan").await; tokio::task::spawn_blocking(move || { + let _permit = permit; + // ONE shared deadline for the whole scan, per jatmn ("the same deadline"). + let deadline = Instant::now() + timeout; let new_refs: Vec<&str> = new_tips.iter().map(String::as_str).collect(); let old_refs: Vec<&str> = old_tips.iter().map(String::as_str).collect(); - match resolve_push_delta(&repo_path, &new_refs, &old_refs) { + // `force` already folds in the env kill-switch, so the forced arm skips the + // delta machinery outright; the unforced arm goes through the normal + // resolver (whose own env read is false here by construction). + let resolved = if force { + tracing::debug!("full scan forced (coalesced-drain marker or kill-switch)"); + PinCandidates::FullScanRequired + } else { + resolve_push_delta(&repo_path, &new_refs, &old_refs, &git_bin, deadline) + }; + match resolved { PinCandidates::Delta(objs) => { tracing::info!(delta = objs.len(), repo = %repo_path.display(), "pin candidate set from push delta"); PinCandidateSet { candidates: objs, full_scan: false } } PinCandidates::FullScanRequired => { tracing::warn!(repo = %repo_path.display(), "pin delta unavailable (non-commit tip, git error, or force-full-scan) — full-scan fallback"); - match list_all_objects(&repo_path) { + match list_all_objects(&repo_path, &git_bin, deadline) { Ok(objs) => PinCandidateSet { candidates: objs, full_scan: true }, Err(e) => { tracing::warn!(repo = %repo_path.display(), err = %e, "full-scan fallback failed; pinning nothing this push (reconciliation sweep backstops)"); @@ -304,8 +364,13 @@ pub async fn resolve_candidates_for_push( mod tests { use super::*; use std::collections::HashSet; + use std::process::Command; use tempfile::TempDir; + fn td() -> std::time::Instant { + std::time::Instant::now() + std::time::Duration::from_secs(600) + } + /// Minimal git helper for building test repos. struct Repo { _td: TempDir, @@ -363,9 +428,10 @@ mod tests { let repo = Repo::new(); let c1 = repo.commit_file("a.txt", "one\n"); let c2 = repo.commit_file("b.txt", "two\n"); - let got: HashSet = delta(resolve_push_delta(&repo.path, &[&c2], &[&c1])) - .into_iter() - .collect(); + let got: HashSet = + delta(resolve_push_delta(&repo.path, &[&c2], &[&c1], "git", td())) + .into_iter() + .collect(); // The new blob b.txt and commit c2 are in the delta; the old blob a.txt // and commit c1 are not. let new_blob = repo.rev("HEAD:b.txt"); @@ -383,7 +449,7 @@ mod tests { // genuinely new objects, never fewer. let repo = Repo::new(); let c1 = repo.commit_file("a.txt", "one\n"); - let got: HashSet = delta(resolve_push_delta(&repo.path, &[&c1], &[])) + let got: HashSet = delta(resolve_push_delta(&repo.path, &[&c1], &[], "git", td())) .into_iter() .collect(); assert!(got.contains(&c1)); @@ -399,9 +465,15 @@ mod tests { // Rewrite history: reset to base, commit a different file. repo.git(&["reset", "-q", "--hard", &base]); let new_tip = repo.commit_file("c.txt", "three\n"); - let got: HashSet = delta(resolve_push_delta(&repo.path, &[&new_tip], &[&old_tip])) - .into_iter() - .collect(); + let got: HashSet = delta(resolve_push_delta( + &repo.path, + &[&new_tip], + &[&old_tip], + "git", + td(), + )) + .into_iter() + .collect(); assert!(got.contains(&new_tip), "new tip in delta"); assert!(got.contains(&repo.rev(&format!("{new_tip}:c.txt")))); // No error; force-push computes new-minus-old cleanly. @@ -413,7 +485,7 @@ mod tests { repo.commit_file("a.txt", "one\n"); // All updates were deletions => new_tips empty after the caller strips zeros. assert_eq!( - resolve_push_delta(&repo.path, &[], &[ZERO]), + resolve_push_delta(&repo.path, &[], &[ZERO], "git", td()), PinCandidates::Delta(Vec::new()) ); } @@ -424,7 +496,7 @@ mod tests { repo.commit_file("a.txt", "one\n"); let blob = repo.rev("HEAD:a.txt"); assert_eq!( - resolve_push_delta(&repo.path, &[&blob], &[]), + resolve_push_delta(&repo.path, &[&blob], &[], "git", td()), PinCandidates::FullScanRequired, "a blob tip must force full scan (rev-list would exit 0 and walk it)" ); @@ -436,7 +508,7 @@ mod tests { repo.commit_file("a.txt", "one\n"); let tree = repo.rev("HEAD^{tree}"); assert_eq!( - resolve_push_delta(&repo.path, &[&tree], &[]), + resolve_push_delta(&repo.path, &[&tree], &[], "git", td()), PinCandidates::FullScanRequired ); } @@ -449,7 +521,7 @@ mod tests { repo.git(&["tag", "-a", "treetag", "-m", "x", &tree]); let tag = repo.rev("treetag"); assert_eq!( - resolve_push_delta(&repo.path, &[&tag], &[]), + resolve_push_delta(&repo.path, &[&tag], &[], "git", td()), PinCandidates::FullScanRequired, "annotated tag peeling to a tree must force full scan" ); @@ -465,7 +537,7 @@ mod tests { repo.git(&["tag", "-a", "t2", "-m", "x", &t1]); let t2 = repo.rev("t2"); assert_eq!( - resolve_push_delta(&repo.path, &[&t2], &[]), + resolve_push_delta(&repo.path, &[&t2], &[], "git", td()), PinCandidates::FullScanRequired ); } @@ -480,7 +552,7 @@ mod tests { let c1 = repo.commit_file("a.txt", "one\n"); repo.git(&["tag", "-a", "rel", "-m", "release", &c1]); let tag = repo.rev("rel"); - let got: HashSet = delta(resolve_push_delta(&repo.path, &[&tag], &[])) + let got: HashSet = delta(resolve_push_delta(&repo.path, &[&tag], &[], "git", td())) .into_iter() .collect(); assert!( @@ -501,7 +573,7 @@ mod tests { let t1 = repo.rev("t1"); repo.git(&["tag", "-a", "t2", "-m", "x", &t1]); let t2 = repo.rev("t2"); - let got: HashSet = delta(resolve_push_delta(&repo.path, &[&t2], &[])) + let got: HashSet = delta(resolve_push_delta(&repo.path, &[&t2], &[], "git", td())) .into_iter() .collect(); assert!( @@ -516,9 +588,16 @@ mod tests { let repo = Repo::new(); let c1 = repo.commit_file("a.txt", "one\n"); let c2 = repo.commit_file("b.txt", "two\n"); - let set = - resolve_candidates_for_push(repo.path.clone(), vec![c2.clone()], vec![c1.clone()]) - .await; + let set = resolve_candidates_for_push( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), + repo.path.clone(), + vec![c2.clone()], + vec![c1.clone()], + "git".to_string(), + std::time::Duration::from_secs(600), + false, + ) + .await; assert!(!set.full_scan, "happy-path delta is not a full scan"); let got: HashSet = set.candidates.into_iter().collect(); let new_blob = repo.rev("HEAD:b.txt"); @@ -536,20 +615,206 @@ mod tests { let repo = Repo::new(); repo.commit_file("a.txt", "one\n"); let blob = repo.rev("HEAD:a.txt"); - let all: HashSet = list_all_objects(&repo.path).unwrap().into_iter().collect(); - let set = resolve_candidates_for_push(repo.path.clone(), vec![blob], vec![]).await; + let all: HashSet = list_all_objects(&repo.path, "git", td()) + .unwrap() + .into_iter() + .collect(); + let set = resolve_candidates_for_push( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), + repo.path.clone(), + vec![blob], + vec![], + "git".to_string(), + std::time::Duration::from_secs(600), + false, + ) + .await; assert!(set.full_scan, "non-commit tip is signalled as a full scan"); let got: HashSet = set.candidates.into_iter().collect(); assert_eq!(got, all, "non-commit tip falls back to full repo scan"); } + /// F4 defer proof 2: `resolve_candidates_for_push`'s git stages (the per-tip + /// cat-file type probe + delta rev-list, and the full-scan fallback) run under a + /// scan-admission permit: with a zero-permit pool the call parks and spawns no + /// git; once a permit is available the SAME call runs (defer, not shed). On + /// ungated code the git runs regardless of the pool (RED). + #[cfg(unix)] + #[tokio::test] + async fn resolve_candidates_defers_when_scan_pool_exhausted() { + use std::os::unix::fs::PermissionsExt; + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + + let dir = tempfile::TempDir::new().unwrap(); + let marker = dir.path().join("git.ran"); + // Fake git records ANY invocation; cat-file reports a commit tip so the + // delta stage proceeds, rev-list yields an empty delta. + let fake = dir.path().join("fakegit"); + std::fs::write( + &fake, + format!( + "#!/bin/sh\necho ran >> \"{}\"\ncase \"$1\" in\n cat-file) echo commit ;;\n *) : ;;\nesac\nexit 0\n", + marker.display() + ), + ) + .unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + let git_bin = fake.to_str().unwrap().to_string(); + let tip = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef".to_string(); + + let sem: Arc = Arc::new(Semaphore::new(0)); + let blocked = tokio::time::timeout( + Duration::from_millis(500), + resolve_candidates_for_push( + sem.clone(), + dir.path().to_path_buf(), + vec![tip.clone()], + vec![], + git_bin.clone(), + Duration::from_secs(5), + false, + ), + ) + .await; + assert!( + blocked.is_err(), + "the pin-candidate scan must defer (park on admission) when the pool is exhausted" + ); + assert!( + !marker.exists(), + "the scan's git must not spawn while its admission permit is unavailable (F4)" + ); + + // Release admission: the SAME scan now runs (defer, not shed). + sem.add_permits(1); + let set = resolve_candidates_for_push( + sem, + dir.path().to_path_buf(), + vec![tip], + vec![], + git_bin, + Duration::from_secs(5), + false, + ) + .await; + assert!( + marker.exists(), + "once admission is available the deferred scan runs its git" + ); + assert!(!set.full_scan, "commit tip + empty rev-list is a delta"); + assert!(set.candidates.is_empty()); + } + + /// F4 fast-path negative arm: a deletion-only push (no new tips, kill-switch + /// off) computes its empty delta without spawning ANY git child, so it must + /// complete without acquiring from the scan pool — even at zero permits. The + /// per-tip cat-file probe means every push with a non-empty new tip DOES spawn + /// git, so this is the only genuinely git-free stage. + #[cfg(unix)] + #[tokio::test] + async fn resolve_candidates_no_git_fast_path_skips_admission() { + use std::os::unix::fs::PermissionsExt; + use std::sync::Arc; + use std::time::Duration; + use tokio::sync::Semaphore; + + let dir = tempfile::TempDir::new().unwrap(); + let marker = dir.path().join("git.ran"); + let fake = dir.path().join("fakegit"); + std::fs::write( + &fake, + format!("#!/bin/sh\necho ran >> \"{}\"\nexit 0\n", marker.display()), + ) + .unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + + let set = tokio::time::timeout( + Duration::from_millis(500), + resolve_candidates_for_push( + Arc::new(Semaphore::new(0)), + dir.path().to_path_buf(), + vec![], + vec!["deadbeefdeadbeefdeadbeefdeadbeefdeadbeef".to_string()], + fake.to_str().unwrap().to_string(), + Duration::from_secs(5), + false, + ), + ) + .await + .expect("the no-new-tips fast path must not park on the scan pool"); + assert_eq!( + set, + PinCandidateSet { + candidates: Vec::new(), + full_scan: false + } + ); + assert!(!marker.exists(), "the fast path must spawn no git at all"); + } + + /// #174 F5: the coalesced-drain FullScan marker is signalled via the explicit + /// `force_full_scan` flag, and a flagged call with NO tips must still enumerate + /// the whole repo (full_scan=true, non-empty candidates). The RED arm of the + /// encoding question: if the marker were encoded as a plain empty-tips call + /// (flag off — the fast path), the drain would resolve to an empty delta and + /// pin nothing, silently losing the coalesced pushes' work. + #[tokio::test] + async fn forced_full_scan_with_no_tips_enumerates_the_repo() { + let repo = Repo::new(); + repo.commit_file("a.txt", "one\n"); + let all: HashSet = list_all_objects(&repo.path, "git", td()) + .unwrap() + .into_iter() + .collect(); + assert!(!all.is_empty(), "fixture repo has objects"); + + let set = resolve_candidates_for_push( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), + repo.path.clone(), + vec![], + vec![], + "git".to_string(), + std::time::Duration::from_secs(600), + true, + ) + .await; + assert!(set.full_scan, "the forced call is signalled as a full scan"); + let got: HashSet = set.candidates.into_iter().collect(); + assert_eq!( + got, all, + "a forced full scan with no tips enumerates the repo — never an empty delta" + ); + + // The discriminator: the SAME empty-tips call without the flag is the + // deletion-only fast path (empty delta, pin nothing). The two must differ, + // or the marker encoding has collapsed into the silent-loss shape. + let unforced = resolve_candidates_for_push( + std::sync::Arc::new(tokio::sync::Semaphore::new(64)), + repo.path.clone(), + vec![], + vec![], + "git".to_string(), + std::time::Duration::from_secs(600), + false, + ) + .await; + assert!(!unforced.full_scan); + assert!(unforced.candidates.is_empty()); + } + #[test] fn missing_oid_tip_forces_full_scan() { let repo = Repo::new(); repo.commit_file("a.txt", "one\n"); let bogus = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"; assert_eq!( - resolve_push_delta(&repo.path, &[bogus], &[]), + resolve_push_delta(&repo.path, &[bogus], &[], "git", td()), PinCandidates::FullScanRequired, "a missing/corrupt tip OID must force full scan" ); @@ -564,7 +829,7 @@ mod tests { let c1 = repo.commit_file("a.txt", "one\n"); let c2 = repo.commit_file("b.txt", "two\n"); let old_tree = repo.rev(&format!("{c1}^{{tree}}")); - let result = resolve_push_delta(&repo.path, &[&c2], &[&old_tree]); + let result = resolve_push_delta(&repo.path, &[&c2], &[&old_tree], "git", td()); match result { PinCandidates::FullScanRequired => {} // safe: caller full-scans PinCandidates::Delta(objs) => { @@ -583,10 +848,15 @@ mod tests { // branch2 from base advances independently repo.git(&["checkout", "-q", "-b", "branch2", &base]); let b2 = repo.commit_file("c.txt", "three\n"); - let got: HashSet = - delta(resolve_push_delta(&repo.path, &[&b1, &b2], &[&base, &base])) - .into_iter() - .collect(); + let got: HashSet = delta(resolve_push_delta( + &repo.path, + &[&b1, &b2], + &[&base, &base], + "git", + td(), + )) + .into_iter() + .collect(); assert!(got.contains(&b1), "branch1 new commit"); assert!(got.contains(&b2), "branch2 new commit"); } @@ -595,7 +865,7 @@ mod tests { fn empty_repo_no_tips_yields_empty_delta() { let repo = Repo::new(); assert_eq!( - resolve_push_delta(&repo.path, &[], &[]), + resolve_push_delta(&repo.path, &[], &[], "git", td()), PinCandidates::Delta(Vec::new()) ); } @@ -609,12 +879,12 @@ mod tests { let repo = Repo::new(); let c1 = repo.commit_file("a.txt", "one\n"); assert_eq!( - resolve_push_delta_inner(&repo.path, &[&c1], &[], true), + resolve_push_delta_inner(&repo.path, &[&c1], &[], true, "git", td()), PinCandidates::FullScanRequired ); // And with the flag off, the same push yields a Delta. assert!(matches!( - resolve_push_delta_inner(&repo.path, &[&c1], &[], false), + resolve_push_delta_inner(&repo.path, &[&c1], &[], false, "git", td()), PinCandidates::Delta(_) )); } @@ -624,7 +894,7 @@ mod tests { let repo = Repo::new(); repo.commit_file("a.txt", "one\n"); repo.commit_file("b.txt", "two\n"); - let all = list_all_objects(&repo.path).unwrap(); + let all = list_all_objects(&repo.path, "git", td()).unwrap(); // 2 commits + 2 trees + 2 blobs = 6 objects. assert_eq!(all.len(), 6, "got: {all:?}"); } @@ -642,7 +912,7 @@ mod tests { std::fs::write(repo.path.join("orphan.bin"), b"dangling\n").unwrap(); let dangling = repo.git(&["hash-object", "-w", "orphan.bin"]); - let blobs = all_blob_oids(&repo.path).unwrap(); + let blobs = all_blob_oids(&repo.path, "git", td()).unwrap(); assert!(blobs.contains(&reachable_blob), "reachable blob present"); assert!( blobs.contains(&dangling), @@ -652,7 +922,7 @@ mod tests { assert!(!blobs.contains(&tree), "tree is not a blob"); // The typed lister tags each object; spot-check the dangling blob's type. - let typed = list_all_objects_with_type(&repo.path).unwrap(); + let typed = list_all_objects_with_type(&repo.path, "git", td()).unwrap(); assert!( typed .iter() @@ -660,4 +930,96 @@ mod tests { "dangling object is typed as a blob" ); } + + #[cfg(unix)] + #[test] + fn list_all_objects_times_out_on_a_hung_git_instead_of_blocking() { + use std::os::unix::fs::PermissionsExt; + use std::time::{Duration, Instant}; + let dir = tempfile::TempDir::new().unwrap(); + // Fake git: hang on cat-file (bounded to 30s so a broken test can't wedge). + let fake = dir.path().join("fakegit"); + std::fs::write( + &fake, + "#!/bin/sh\ncase \"$1\" in\n cat-file) i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done ;;\n *) : ;;\nesac\n", + ) + .unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + let git_bin = fake.to_str().unwrap().to_string(); + + let (tx, rx) = std::sync::mpsc::channel(); + let path = dir.path().to_path_buf(); + std::thread::spawn(move || { + let _ = tx.send(list_all_objects( + &path, + &git_bin, + Instant::now() + Duration::from_millis(150), + )); + }); + let res = rx.recv_timeout(Duration::from_secs(10)).expect( + "list_all_objects must return within the watchdog budget, not block on a hung git", + ); + assert!( + res.is_err(), + "a hung git must make list_all_objects error out, not hang" + ); + } + + /// #174 finding 2: the DELTA-path children — `peeled_object_type` (cat-file), + /// `rev_list_delta` (rev-list) — and `list_all_objects_with_type` (cat-file) are + /// the remaining candidate-discovery execs (the common push path). Each must + /// return within the watchdog budget on a hung git rather than block and pin the + /// write permit. Revert any one to a bare `Command::output()` and its arm blocks + /// past the recv budget (RED). + #[cfg(unix)] + #[test] + fn delta_path_exec_fns_time_out_on_a_hung_git() { + use std::os::unix::fs::PermissionsExt; + use std::time::{Duration, Instant}; + let dir = tempfile::TempDir::new().unwrap(); + // Fake git hangs on BOTH cat-file and rev-list (bounded 30s so a broken + // test can't leak a permanent orphan or wedge the suite). + let fake = dir.path().join("fakegit"); + std::fs::write( + &fake, + "#!/bin/sh\ncase \"$1\" in\n cat-file|rev-list) i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done ;;\n *) : ;;\nesac\n", + ) + .unwrap(); + let mut perm = std::fs::metadata(&fake).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&fake, perm).unwrap(); + let git_bin = fake.to_str().unwrap().to_string(); + let path = dir.path().to_path_buf(); + + // Run `f` on a thread and require it to RETURN (not block) within 10s. + fn returns_within(f: impl FnOnce() -> T + Send + 'static) -> T { + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let _ = tx.send(f()); + }); + rx.recv_timeout(Duration::from_secs(10)).expect( + "a delta-path exec fn must return within the watchdog budget, not block on a hung git", + ) + } + let dl = || Instant::now() + Duration::from_millis(150); + let sha = "deadbeefdeadbeefdeadbeefdeadbeefdeadbeef"; + + let (p, g, d) = (path.clone(), git_bin.clone(), dl()); + assert!( + returns_within(move || peeled_object_type(&p, sha, &g, d)).is_none(), + "peeled_object_type must time out to None on a hung cat-file" + ); + let (p, g, d) = (path.clone(), git_bin.clone(), dl()); + assert!( + returns_within(move || rev_list_delta(&p, &[sha], &[], &g, d)).is_err(), + "rev_list_delta must error out on a hung rev-list" + ); + let (p, g, d) = (path.clone(), git_bin.clone(), dl()); + assert!( + returns_within(move || list_all_objects_with_type(&p, &g, d)).is_err(), + "list_all_objects_with_type must error out on a hung cat-file" + ); + } } diff --git a/crates/gitlawb-node/src/git/repo_store.rs b/crates/gitlawb-node/src/git/repo_store.rs index a5c367e9..bd27a186 100644 --- a/crates/gitlawb-node/src/git/repo_store.rs +++ b/crates/gitlawb-node/src/git/repo_store.rs @@ -13,7 +13,8 @@ use std::path::{Path, PathBuf}; use std::sync::Arc; use anyhow::{Context, Result}; -use sqlx::PgPool; +use sqlx::pool::PoolConnection; +use sqlx::{PgPool, Postgres}; use tokio::sync::Mutex; use tracing::{debug, info, warn}; @@ -157,13 +158,44 @@ impl RepoStore { let (owner_slug, local_path) = self.local_path(owner_did, repo_name)?; let lock_key = advisory_lock_key(&owner_slug, repo_name); - // Acquire Postgres advisory lock with retry using pg_try_advisory_lock - // to avoid blocking indefinitely on stale locks from crashed connections. + // Pin a dedicated pooled connection and build the guard holding it BEFORE + // issuing the lock query. Session-level pg advisory locks are + // connection-affine (they can only be released on the session that took + // them), so the guard must own the locking connection; and building the + // guard first means any cancellation after the lock is taken — a + // `tokio::time::timeout` firing during the Tigris download below — drops a + // guard that CAN release, closing the leak the outer timeout otherwise + // opened (#174 F1). + let conn = self + .pool + .acquire() + .await + .context("acquiring db connection for the write advisory lock")?; + let mut guard = RepoWriteGuard { + owner_slug: owner_slug.clone(), + repo_name: repo_name.to_string(), + local_path: local_path.clone(), + lock_key, + conn: Some(conn), + locked: false, + released: false, + tigris: self.tigris.clone(), + #[cfg(test)] + test_pre_unlock_gate: None, + }; + + // Acquire the advisory lock with retry, through the guard's OWN connection, + // so the matching unlock (in release, or the Drop backstop) runs on the same + // session — pg_advisory_unlock on a different pooled connection is a no-op. let mut acquired = false; for attempt in 0..60 { + let c = guard + .conn + .as_deref_mut() + .expect("write guard holds its connection during acquisition"); let row: (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") .bind(lock_key) - .fetch_one(&self.pool) + .fetch_one(&mut *c) .await .context("trying advisory lock")?; if row.0 { @@ -177,9 +209,11 @@ impl RepoStore { if !acquired { anyhow::bail!("could not acquire advisory lock after 60s — possible stale lock for {owner_slug}/{repo_name}"); } + guard.locked = true; - // Always download the latest from Tigris before writing. - // Local disk may be stale if another machine pushed since our last access. + // Always download the latest from Tigris before writing. Local disk may be + // stale if another machine pushed since our last access. The guard already + // owns the lock + its connection, so a cancellation here drops through Drop. if let Some(ref tigris) = self.tigris { if tigris.exists(&owner_slug, repo_name).await.unwrap_or(false) { debug!(repo = %repo_name, "write acquire: downloading latest from tigris"); @@ -197,14 +231,7 @@ impl RepoStore { } } - Ok(RepoWriteGuard { - owner_slug, - repo_name: repo_name.to_string(), - local_path, - lock_key, - pool: self.pool.clone(), - tigris: self.tigris.clone(), - }) + Ok(guard) } /// Initialize a new bare repo on local disk and upload to Tigris. @@ -354,8 +381,24 @@ pub struct RepoWriteGuard { repo_name: String, pub local_path: PathBuf, lock_key: i64, - pool: PgPool, + /// The pooled connection that took the advisory lock. Session-level pg + /// advisory locks are connection-affine, so the guard pins that connection + /// for its whole lifetime and unlocks on it (in `release`, or the `Drop` + /// backstop). `None` only after the connection has been taken to unlock. + conn: Option>, + /// Set once the advisory lock has actually been taken. A guard dropped + /// before the lock is held (or after `release`) performs no unlock. + locked: bool, + /// Set once `release` has run its unlock, making the `Drop` backstop inert. + released: bool, tigris: Option, + /// Test-only seam: when set, `release` parks on this gate at the exact point + /// it is about to await `pg_advisory_unlock` (connection still owned, not yet + /// released). Dropping the `release` future while it is parked reproduces a + /// mid-unlock cancellation, so a test can assert the `Drop` backstop still + /// frees the session lock. Never set outside tests. + #[cfg(test)] + test_pre_unlock_gate: Option>, } impl RepoWriteGuard { @@ -369,7 +412,7 @@ impl RepoWriteGuard { /// half-applied or otherwise inconsistent repo would propagate corruption to /// Tigris (and to every node that later downloads it). The lock is always /// released regardless, to avoid stale locks blocking future writes. - pub async fn release(self, success: bool) { + pub async fn release(mut self, success: bool) { // Upload to Tigris only on success. if success { if let Some(ref tigris) = self.tigris { @@ -384,11 +427,72 @@ impl RepoWriteGuard { warn!(repo = %self.repo_name, "write failed — skipping tigris upload to avoid propagating an inconsistent repo"); } - // Release advisory lock - let _ = sqlx::query("SELECT pg_advisory_unlock($1)") - .bind(self.lock_key) - .execute(&self.pool) - .await; + // Release the advisory lock on the SAME connection that took it (session + // advisory locks are connection-affine). Unlock through the connection + // while it is STILL owned by `self` — do not `take()` it first. If this + // future is cancelled during the unlock await, `self` is dropped with + // `conn == Some(..)` and `released == false`, so the `Drop` backstop still + // runs the detached unlock. `released` is set only AFTER the await + // resolves, so a cancellation cannot make the backstop inert (#174 F4). + if self.locked { + #[cfg(test)] + let pre_unlock_gate = self.test_pre_unlock_gate.clone(); + if let Some(conn) = self.conn.as_deref_mut() { + // Test-only: park right before the unlock await so a test can drop + // this future mid-unlock (connection owned, not yet released). + #[cfg(test)] + if let Some(gate) = pre_unlock_gate { + gate.notified().await; + } + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(self.lock_key) + .execute(&mut *conn) + .await; + } + } + self.released = true; + } +} + +impl Drop for RepoWriteGuard { + /// Cancellation-safe backstop: if the guard is dropped while still holding the + /// advisory lock (a `tokio::time::timeout` cancelled `acquire_write`, or a + /// handler future was dropped before `release`), unlock on the pinned + /// connection. `Drop` cannot await, so spawn a detached unlock — it runs on the + /// same session (connection-affine). An off-runtime drop falls back to a log; + /// the ~60s stale-lock retry loop in `acquire_write` reclaims it. On runtime + /// SHUTDOWN the spawned unlock task may be dropped before it polls, so the unlock + /// may not run — but shutdown tears down the pool, and closing the connection + /// releases the session-level advisory lock server-side, so this too is bounded. + fn drop(&mut self) { + if self.released || !self.locked { + return; + } + let Some(mut conn) = self.conn.take() else { + return; + }; + let lock_key = self.lock_key; + let repo_name = self.repo_name.clone(); + match tokio::runtime::Handle::try_current() { + Ok(handle) => { + handle.spawn(async move { + if let Err(e) = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(lock_key) + .execute(&mut *conn) + .await + { + warn!(repo = %repo_name, err = %e, "detached advisory-unlock on write-guard drop failed"); + } + }); + } + Err(_) => { + warn!( + repo = %repo_name, + "RepoWriteGuard dropped off a Tokio runtime; advisory lock not released \ + synchronously — the stale-lock retry loop will reclaim it" + ); + } + } } } @@ -562,4 +666,205 @@ mod tests { ); } } + + // ── advisory-lock cancellation-safety (#174 F1, RED-before/GREEN-after) ── + + /// F1 (P1): dropping a `RepoWriteGuard` WITHOUT calling `release()` — the + /// state a `tokio::time::timeout` cancellation leaves `acquire_write` in when + /// it fires during the Tigris await — must still release the session advisory + /// lock. A checker connection is held OUT of the pool first, so `acquire_write` + /// is forced onto a distinct session; the checker (a different session) then + /// probes the lock, so advisory-lock re-entrancy cannot mask a leak. + /// + /// Load-bearing: RED today (no `Drop` releases the lock → held by + /// `acquire_write`'s session → checker's `pg_try_advisory_lock` returns + /// false). GREEN after the connection-affine `Drop` backstop. + #[sqlx::test] + async fn write_guard_drop_without_release_frees_the_lock(pool: sqlx::PgPool) { + let dir = tempfile::TempDir::new().unwrap(); + let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone()); + let owner = "did:key:z6MkDropBackstopProofAAAAAAAAAAAAAAAAAAAAAA"; + let name = "leaktest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + // Distinct session for the probe: hold it out of the pool BEFORE acquiring, + // so acquire_write cannot use it and a re-entrant probe cannot falsely read free. + let mut checker = pool.acquire().await.expect("checker connection"); + + let guard = store.acquire_write(owner, name).await.expect("acquire"); + // The cancellation shape: drop without release(). + drop(guard); + // Let the detached unlock task run. + tokio::time::sleep(std::time::Duration::from_millis(400)).await; + + let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *checker) + .await + .unwrap(); + assert!( + free, + "advisory lock must be released when the guard is dropped without release()" + ); + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } + + /// F1 (latent non-affine release): `release()` must unlock on the SAME + /// session that locked, so the lock is freed regardless of which pooled + /// connection would service a fresh query. Observed from a distinct session. + #[sqlx::test] + async fn write_guard_release_frees_the_lock_from_a_distinct_session(pool: sqlx::PgPool) { + let dir = tempfile::TempDir::new().unwrap(); + let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone()); + let owner = "did:key:z6MkAffineReleaseProofBBBBBBBBBBBBBBBBBBBB"; + let name = "affinetest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + let mut checker = pool.acquire().await.expect("checker connection"); + let guard = store.acquire_write(owner, name).await.expect("acquire"); + guard.release(false).await; + + let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *checker) + .await + .unwrap(); + assert!( + free, + "release() must free the advisory lock via connection-affine unlock" + ); + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } + + // ── cancellation-safe unlock (#174 F4, RED-before/GREEN-after) ────────── + + /// F4 (P1): a cancellation DURING the unlock await must still free the session + /// advisory lock. `release` unlocks through the connection while `self` still + /// owns it, so if the future is dropped mid-unlock, `Drop` sees `conn == Some` + /// + `locked && !released` and runs its detached-unlock backstop. A test-only + /// gate parks `release` at the exact pre-unlock point; dropping the future + /// there reproduces the cancellation. + /// + /// Load-bearing: RED on the original ordering (`self.conn.take()` before the + /// await → at cancellation `self.conn == None` → `Drop` skips → the local + /// connection returns to the pool with the session lock still held → the + /// checker's `pg_try_advisory_lock` returns false). GREEN after the reorder. + #[sqlx::test] + async fn write_guard_release_cancelled_mid_unlock_frees_the_lock(pool: sqlx::PgPool) { + let dir = tempfile::TempDir::new().unwrap(); + let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone()); + let owner = "did:key:z6MkCancelMidUnlockProofCCCCCCCCCCCCCCCCCC"; + let name = "canceltest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + // Distinct session for the probe, held out of the pool before acquiring. + let mut checker = pool.acquire().await.expect("checker connection"); + + let mut guard = store.acquire_write(owner, name).await.expect("acquire"); + // Arm the pre-unlock gate; it is never notified, so `release` parks on it + // with the connection still owned and `released` still false. + let gate = Arc::new(tokio::sync::Notify::new()); + guard.test_pre_unlock_gate = Some(gate); + + // Box the future so we can drop it ourselves (tokio::pin! keeps it alive to + // end of scope, which would defer the guard's Drop past the assertions). + let mut fut = Box::pin(guard.release(false)); + let parked = + tokio::time::timeout(std::time::Duration::from_millis(300), fut.as_mut()).await; + assert!( + parked.is_err(), + "release should park on the pre-unlock gate, not complete" + ); + // Cancel mid-unlock: dropping the boxed future runs RepoWriteGuard::drop. + drop(fut); + + // Let the Drop backstop's detached unlock task run. + tokio::time::sleep(std::time::Duration::from_millis(400)).await; + + let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *checker) + .await + .unwrap(); + assert!( + free, + "advisory lock must be freed when release is cancelled mid-unlock — the \ + connection must stay owned by the guard so Drop's backstop can run" + ); + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } + + /// F4: the ordinary success path still frees the lock, and a second write + /// acquire on the same repo returns promptly (no stale-lock retry loop). + #[sqlx::test] + async fn write_guard_release_true_frees_lock_and_second_acquire_succeeds(pool: sqlx::PgPool) { + let dir = tempfile::TempDir::new().unwrap(); + let store = RepoStore::for_testing(dir.path().to_path_buf(), pool.clone()); + let owner = "did:key:z6MkReleaseTrueProofDDDDDDDDDDDDDDDDDDDDDD"; + let name = "reltruetest"; + + let guard = store + .acquire_write(owner, name) + .await + .expect("first acquire"); + guard.release(true).await; + + let again = tokio::time::timeout( + std::time::Duration::from_secs(2), + store.acquire_write(owner, name), + ) + .await + .expect("second acquire_write must not hit the ~60s stale-lock retry loop") + .expect("second acquire"); + again.release(true).await; + } + + /// F4: releasing a guard that never took the lock (`locked == false`, the state + /// acquire_write leaves after a failed acquisition) must not unlock or panic. + #[sqlx::test] + async fn write_guard_release_when_not_locked_does_not_unlock_or_panic(pool: sqlx::PgPool) { + let dir = tempfile::TempDir::new().unwrap(); + let owner = "did:key:z6MkNotLockedProofEEEEEEEEEEEEEEEEEEEEEE"; + let name = "notlockedtest"; + let slug = owner.replace([':', '/'], "_"); + let key = advisory_lock_key(&slug, name); + + let guard = RepoWriteGuard { + owner_slug: slug, + repo_name: name.to_string(), + local_path: dir.path().to_path_buf(), + lock_key: key, + conn: Some(pool.acquire().await.expect("conn")), + locked: false, + released: false, + tigris: None, + test_pre_unlock_gate: None, + }; + // Must complete without panic and issue no unlock. + guard.release(false).await; + + let mut checker = pool.acquire().await.expect("checker"); + let (free,): (bool,) = sqlx::query_as("SELECT pg_try_advisory_lock($1)") + .bind(key) + .fetch_one(&mut *checker) + .await + .unwrap(); + assert!(free, "release on an unlocked guard must not touch the key"); + let _ = sqlx::query("SELECT pg_advisory_unlock($1)") + .bind(key) + .execute(&mut *checker) + .await; + } } diff --git a/crates/gitlawb-node/src/git/smart_http.rs b/crates/gitlawb-node/src/git/smart_http.rs index eeb35b99..9ac742ec 100644 --- a/crates/gitlawb-node/src/git/smart_http.rs +++ b/crates/gitlawb-node/src/git/smart_http.rs @@ -6,29 +6,98 @@ use bytes::Bytes; use std::collections::HashSet; use std::path::Path; use std::process::Stdio; -use std::time::Duration; +use std::time::{Duration, Instant}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::process::Command; +/// Owns the served-git admission permits (global + per-source) for the lifetime of +/// the work they admitted, so admission is released only when that work is truly +/// done — not the instant the handler future drops on a client disconnect. +/// +/// A move-only wrapper: no methods beyond construction and `Drop`. The handler +/// MOVEs its permits in and keeps no copy (a retained copy would drop early and +/// release admission the moment the future is dropped, defeating the guard). It is +/// threaded into `drive_git_child`, whose [`KillGroupOnDrop`] moves it into the +/// detached reaper on disconnect, so both permits drop only after the process group +/// is confirmed reaped (`kill(-pgid,0)==ESRCH`) rather than while the group is still +/// alive holding PIDs past the concurrency cap (#174 P1-a, plain-spawn residual). +/// +/// The `be0cdd6` path-scoped upload-pack walk already applies this discipline by +/// moving its permits into the `spawn_blocking`; this generalizes it to the plain +/// (non-path-scoped) `info_refs` / `run_git_service` spawn paths and, via the +/// hand-back contract on `drive_git_child`, across both stages of the filtered +/// (`upload_pack_excluding`) pack build (F1). +pub struct AdmissionGuard { + // Boxed so the concrete permit types stay private to the caller (repos.rs) — the + // guard only needs to hold them and drop them, never inspect them. `Send + + // 'static` so the guard can move into the detached reaper task. + _global: Option>, + _caller: Option>, + // Per-repo write lease (#174 U2/F3), `Some` ONLY on the receive-pack write path + // (via [`with_lease`](Self::with_lease)); `None` on every read path and every + // non-receive-pack write path. It rides this guard into `KillGroupOnDrop`'s detached + // reaper, so on a client disconnect the lease frees only after the disconnected + // push's git group is reaped — serializing a second same-node push against the + // still-writing group. A `RepoWriteLease` clone is `Send + 'static` and holds no pg + // connection, so it travels with the reaper cleanly. A stray `Some` on a READ path + // would wrongly serialize upload-pack against pushes, hence the None-everywhere-else + // discipline. + _lease: Option, +} + +impl AdmissionGuard { + /// Take ownership of the global permit and an optional per-caller permit. Both are + /// erased to `Box` — the guard's only job is to hold them until it drops. + /// No lease is attached here; the receive-pack write path adds one via + /// [`with_lease`](Self::with_lease), so every other call site is lease-free. + pub fn new(global: impl Send + 'static, caller: Option) -> Self { + Self { + _global: Some(Box::new(global)), + _caller: caller.map(|c| Box::new(c) as Box), + _lease: None, + } + } + + /// Attach the per-repo write lease (#174 U2/F3). Called ONLY on the receive-pack + /// write path, so the lease rides the disconnect reaper; read paths never call this. + pub fn with_lease(mut self, lease: crate::state::RepoWriteLease) -> Self { + self._lease = Some(lease); + self + } +} + /// Handle `GET /:owner/:repo/info/refs?service=git-upload-pack` /// or `?service=git-receive-pack` /// /// This is the ref advertisement — the first step of a clone or push. -pub async fn info_refs(repo_path: &Path, service: &str) -> Result { +/// +/// `git_bin` is injectable purely so the process-group teardown can be driven by a +/// fake `git` in tests (production passes `"git"`). `timeout` bounds the whole child +/// interaction: previously the advertisement ran a bare `Command::output()` with no +/// deadline and no teardown, so a hung git pinned its concurrency slot indefinitely +/// and a client disconnect orphaned the child (#174). It now shares +/// [`drive_git_child`]'s timeout + `process_group(0)` + [`KillGroupOnDrop`] teardown. +pub async fn info_refs( + git_bin: &str, + service: &str, + repo_path: &Path, + timeout: Duration, + admission: Option, +) -> Result { validate_service(service)?; - let output = Command::new("git") + let mut command = Command::new(git_bin); + command .arg(service_to_command(service)) .arg("--stateless-rpc") .arg("--advertise-refs") - .arg(repo_path) - .output() - .await?; - - if !output.status.success() { - let stderr = String::from_utf8_lossy(&output.stderr); - bail!("git {service} --advertise-refs failed: {stderr}"); - } + .arg(repo_path); + // No request body — advertise-refs does not read stdin. + let (stdout, admission) = + drive_git_child(command, Bytes::new(), timeout, "advertise-refs", admission).await?; + // Single-stage op: the advertisement's group is reaped by now; release admission + // rather than holding it across the pure response framing below. + drop(admission); let content_type = format!("application/x-{service}-advertisement"); @@ -38,7 +107,7 @@ pub async fn info_refs(repo_path: &Path, service: &str) -> Result { let mut body = Vec::new(); body.extend_from_slice(&pkt_service); body.extend_from_slice(flush); - body.extend_from_slice(&output.stdout); + body.extend_from_slice(&stdout); Ok(Response::builder() .status(StatusCode::OK) @@ -53,12 +122,21 @@ pub async fn info_refs(repo_path: &Path, service: &str) -> Result { /// Serves pack data for a clone or fetch. This is stateless — the entire /// negotiation happens in a single request/response. pub async fn upload_pack( + git_bin: &str, repo_path: &Path, request_body: Bytes, timeout: Duration, + admission: Option, ) -> Result { - let output = - run_git_service("git", "git-upload-pack", repo_path, request_body, timeout).await?; + let output = run_git_service( + git_bin, + "git-upload-pack", + repo_path, + request_body, + timeout, + admission, + ) + .await?; Ok(Response::builder() .status(StatusCode::OK) @@ -72,12 +150,21 @@ pub async fn upload_pack( /// Accepts a push. The caller MUST verify HTTP Signature auth before /// calling this function. pub async fn receive_pack( + git_bin: &str, repo_path: &Path, request_body: Bytes, timeout: Duration, + admission: Option, ) -> Result { - let output = - run_git_service("git", "git-receive-pack", repo_path, request_body, timeout).await?; + let output = run_git_service( + git_bin, + "git-receive-pack", + repo_path, + request_body, + timeout, + admission, + ) + .await?; Ok(Response::builder() .status(StatusCode::OK) @@ -100,24 +187,86 @@ pub async fn receive_pack( /// `wait_with_output()` returns, so a request that completed cleanly never signals. #[cfg(unix)] struct KillGroupOnDrop { + // Holds the child + its pgid while armed. The interaction drives the child through + // `child_mut()`; the success/timeout paths call `disarm` once they have reaped it. + // On drop — a client disconnect that drops the whole request future — the guard + // launches a detached reaper that OWNS the child and runs the full + // SIGTERM -> grace -> SIGKILL -> reap sequence (`reap_group_on_timeout`), as strong + // as the timeout path, so a SIGTERM-ignoring group member is SIGKILLed and reaped + // rather than left running until EPIPE to accumulate past the concurrency cap + // (#174 P1-c). Owning the tokio `Child` in the reaper (rather than a raw + // `waitpid(-pgid)`) keeps a single reaper of the leader, so it never races tokio's + // own SIGCHLD-driven orphan reaper. + child: Option, pgid: Option, + // The global + per-source admission permits for this op, if any. repos.rs moves + // its permits in here (directly on the plain spawn paths, via the stage threading + // on the filtered rev-list/pack-objects ones) so admission is released only when + // the group is confirmed reaped, on every exit: complete, timeout, or + // client-disconnect (#174 P1-a, F1). `None` when the op carries no admission + // (e.g. the smart_http test harness). + admission: Option, } #[cfg(unix)] impl KillGroupOnDrop { - fn disarm(&mut self) { + /// The child, while armed. The interaction drives stdin/stdout/`wait` through this. + fn child_mut(&mut self) -> &mut tokio::process::Child { + self.child.as_mut().expect("child present while armed") + } + + /// Disarm on the success/timeout path: the body has already reaped the child (its + /// `wait()` returned, or `reap_group_on_timeout` ran), so drop the handle and clear + /// the pgid, leaving the guard's Drop a no-op. Returns the admission guard: the + /// group is already reaped on both callers of this, so the success path hands it + /// back to drive_git_child's caller (a multi-stage build carries it to the next + /// stage) and the timeout path drops it there and then. + fn disarm(&mut self) -> Option { self.pgid = None; + self.child = None; + self.admission.take() } } #[cfg(unix)] impl Drop for KillGroupOnDrop { fn drop(&mut self) { - if let Some(pgid) = self.pgid { - // SAFETY: kill(2) takes only integer arguments and borrows no Rust - // memory. Signalling a stale group just returns ESRCH, which we ignore. - unsafe { - libc::kill(-pgid, libc::SIGTERM); + // Move any admission guard out first so it travels with the reaper (or drops + // here on the no-child / no-runtime fallback), never before the group is gone. + let admission = self.admission.take(); + let (Some(mut child), Some(pgid)) = (self.child.take(), self.pgid) else { + // Nothing to reap (already disarmed); dropping `admission` here is correct — + // the group is already gone. + return; + }; + // A sync Drop cannot await, so launch a detached reaper that owns the child and + // runs the same teardown as the timeout path (TERM -> grace -> SIGKILL -> reap + // the whole group). Owning the tokio `Child` means this task is the sole reaper + // of the leader, so it cannot race tokio's orphan reaper. Prefer the current + // runtime handle; if dropped outside a runtime, fall back to a best-effort + // synchronous SIGTERM so the group is at least signalled. + match tokio::runtime::Handle::try_current() { + Ok(handle) => { + handle.spawn(async move { + reap_group_on_timeout(&mut child).await; + // Release admission only now: the group is ESRCH-confirmed gone (or + // hit the ~4s D-state hard cap, past which nothing in userspace can + // free the PIDs anyway). Holding the permits until here is what + // stops disconnect-spam from admitting replacements while the prior + // group is still alive (#174 P1-a). `admission` is moved in and + // dropped when this closure ends. + drop(admission); + }); + } + Err(_) => { + // SAFETY: kill(2) takes only integers and borrows no Rust memory; + // ESRCH on an already-gone group is ignored. + unsafe { + libc::kill(-pgid, libc::SIGTERM); + } + // No runtime to await the reap; drop admission best-effort after the + // synchronous signal (the fallback path is not on the hot request path). + drop(admission); } } } @@ -202,12 +351,44 @@ async fn run_git_service( repo_path: &Path, input: Bytes, timeout: Duration, + admission: Option, ) -> Result> { let mut command = Command::new(git_bin); command .arg(service_to_command(service)) .arg("--stateless-rpc") - .arg(repo_path) + .arg(repo_path); + let (out, admission) = drive_git_child(command, input, timeout, service, admission).await?; + // Single-stage op: the child group is already reaped when drive_git_child + // returns, so releasing admission here keeps the permits held exactly for the + // process lifetime, no longer. + drop(admission); + Ok(out) +} + +/// Drive a spawned git child under `timeout` with process-group teardown, returning +/// its stdout together with the admission guard. Shared core for +/// [`run_git_service`], [`info_refs`], and the filtered-pack stages: the caller +/// passes a `Command` with its args set; this adds piped stdio and `process_group(0)`. +/// On the deadline the whole group is torn down and reaped before returning +/// [`GitServiceTimeout`]; on a dropped future (client disconnect) the +/// [`KillGroupOnDrop`] guard fires. `input` is written to the child's stdin (empty +/// for the advertise-refs path, which has no request body); `what` labels errors. +/// +/// Admission contract: on success the guard is handed BACK so a multi-stage caller +/// (the filtered-pack build) can carry one admission across consecutive children +/// instead of releasing it between stages. On every Err return (timeout, spawn +/// failure, interaction error, non-zero exit) the guard is dropped internally, and +/// only after the child is reaped: an Err cannot carry the guard, and by then the +/// group is confirmed gone, so releasing admission there is correct. +async fn drive_git_child( + mut command: Command, + input: Bytes, + timeout: Duration, + what: &str, + admission: Option, +) -> Result<(Vec, Option)> { + command .stdin(Stdio::piped()) .stdout(Stdio::piped()) .stderr(Stdio::piped()); @@ -219,20 +400,33 @@ async fn run_git_service( let mut child = command.spawn()?; - // Arm the group-kill guard for the lifetime of the request. With - // process_group(0) the child is its own group leader, so pgid == its pid. - // This fires on a client disconnect (the whole future is dropped mid-request). + // Own the pipes so the child stays reap-able: wait_with_output would consume it, + // but on a timeout we must actively reap the group first (see + // reap_group_on_timeout), and on a disconnect the guard's detached reaper does. + // Take the pipes before the child moves into the guard below. + let mut stdin = child.stdin.take(); + let mut stdout = child.stdout.take().context("git stdout was not piped")?; + let mut stderr = child.stderr.take().context("git stderr was not piped")?; + + // Arm the group-kill guard for the lifetime of the request. With process_group(0) + // the child is its own group leader, so pgid == its pid. On a client disconnect + // (the whole future is dropped mid-request) the guard's Drop launches a detached + // reaper that OWNS the child and runs the full TERM/grace/SIGKILL/reap — as strong + // as the timeout path below (#174 P1-c). The guard owns the child; the interaction + // drives it through `child_mut()`. + #[cfg(unix)] + let pgid = child.id().map(|id| id as i32); #[cfg(unix)] let mut group_guard = KillGroupOnDrop { - pgid: child.id().map(|id| id as i32), + child: Some(child), + pgid, + admission, }; + // On non-unix there is no process-group teardown; the `admission` parameter + // itself stays live for the child's whole interaction (nothing moves it until + // the match below), so the success path hands it back like the unix path and + // the timeout path drops it after the child is waited on. - // Own the pipes so `child` stays reap-able after a timeout: wait_with_output - // would consume it, but on timeout we must actively reap the group before - // returning (see reap_group_on_timeout). - let mut stdin = child.stdin.take(); - let mut stdout = child.stdout.take().context("git stdout was not piped")?; - let mut stderr = child.stderr.take().context("git stderr was not piped")?; let mut out = Vec::new(); let mut err = Vec::new(); @@ -249,11 +443,15 @@ async fn run_git_service( None => Ok(()), } }; + #[cfg(unix)] + let child_ref = group_guard.child_mut(); + #[cfg(not(unix))] + let child_ref = &mut child; let (write_result, r_out, r_err, status) = tokio::join!( write, stdout.read_to_end(&mut out), stderr.read_to_end(&mut err), - child.wait(), + child_ref.wait(), ); r_out?; r_err?; @@ -261,28 +459,34 @@ async fn run_git_service( }; let timed = tokio::time::timeout(timeout, interact).await; - let (write_result, status) = match timed { + let (write_result, status, admission) = match timed { Ok(result) => { // The join runs all arms to completion, so the child is reaped: disarm // before surfacing any interaction error (a read/wait error), else the - // guard's drop would fire SIGTERM on the reaped, possibly-reused pgid. + // guard's drop would reap an already-reaped child / signal a reused pgid. + // Disarming hands the admission guard back; it rides to the Ok return + // below, and every error return between here and there drops it, always + // AFTER the reap, the earliest provably-free point. #[cfg(unix)] - group_guard.disarm(); - result? + let admission = group_guard.disarm(); + let (write_result, status) = result?; + (write_result, status, admission) } Err(_elapsed) => { // Timeout: tear the whole group down and reap it before returning so a // caller releasing a write lock can't race a still-live git. Then disarm - // the (now redundant) guard so its drop can't hit a reused pgid. + // the (now redundant) guard so its drop can't hit a reused pgid. The + // returned admission guard drops here, AFTER the group is confirmed reaped. #[cfg(unix)] { - reap_group_on_timeout(&mut child).await; - group_guard.disarm(); + reap_group_on_timeout(group_guard.child_mut()).await; + drop(group_guard.disarm()); } #[cfg(not(unix))] { let _ = child.start_kill(); let _ = tokio::time::timeout(Duration::from_secs(2), child.wait()).await; + drop(admission); } return Err(GitServiceTimeout.into()); } @@ -291,15 +495,16 @@ async fn run_git_service( // Surface git's own failure (its stderr, which the handler may classify as a // 400) before any stdin-write error: when git rejects a malformed body it // exits non-zero and closes stdin, so the write's EPIPE would otherwise mask - // the real cause. + // the real cause. `admission` drops on these error returns; the child is + // already reaped by the completed join above. if !status.success() { let stderr = String::from_utf8_lossy(&err); - bail!("{service} failed: {stderr}"); + bail!("{what} failed: {stderr}"); } write_result.context("failed to write to git stdin")?; - Ok(out) + Ok((out, admission)) } fn service_to_command(service: &str) -> &str { @@ -324,62 +529,91 @@ fn pkt_line(data: &str) -> Vec { format!("{len:04x}{data}").into_bytes() } -/// Build a packfile containing every object reachable from all refs EXCEPT the -/// given blob OIDs. Commits and trees are always included, so SHAs stay intact; -/// only the named blobs are dropped. -pub fn build_filtered_pack(repo_path: &Path, withheld: &HashSet) -> Result> { - // All reachable objects as "oid [path]" lines. - let rev = std::process::Command::new("git") +/// Run `rev-list --objects --all` under `timeout` and return the reachable object +/// ids minus the withheld blobs. Runs under [`drive_git_child`] (async, with the +/// `tokio::time::timeout` + `process_group(0)` + [`KillGroupOnDrop`] teardown), so a +/// hung/slow enumeration is duration-bounded and reaped on client disconnect just +/// like the pack-objects stage. A bare blocking `Command::output()` inside a +/// `spawn_blocking` is uncancellable, so a hung rev-list would pin the endpoint's +/// concurrency permit for the whole hang (#174). `git_bin` is injectable for the +/// same fake-git testing reason as `run_git_service`. +async fn rev_list_keep( + git_bin: &str, + repo_path: &Path, + withheld: &HashSet, + timeout: Duration, + admission: Option, +) -> Result<(Vec, Option)> { + let mut command = Command::new(git_bin); + command .args(["rev-list", "--objects", "--all"]) - .current_dir(repo_path) - .output()?; - if !rev.status.success() { - bail!( - "git rev-list failed: {}", - String::from_utf8_lossy(&rev.stderr) - ); - } + .current_dir(repo_path); + // The filtered-serve caller's admission rides through drive_git_child so a + // disconnect mid-enumeration keeps the permits held until the rev-list group is + // reaped; on success the guard comes back for the pack-objects stage (F1). + let (stdout, admission) = + drive_git_child(command, Bytes::new(), timeout, "rev-list", admission).await?; let mut keep = Vec::new(); - for line in String::from_utf8_lossy(&rev.stdout).lines() { + for line in String::from_utf8_lossy(&stdout).lines() { let oid = line.split_whitespace().next().unwrap_or(""); if oid.is_empty() || withheld.contains(oid) { continue; } keep.push(oid.to_string()); } - let mut child = std::process::Command::new("git") + Ok((keep, admission)) +} + +/// Build a packfile containing every object reachable from all refs EXCEPT the +/// given blob OIDs. Commits and trees are always included, so SHAs stay intact; +/// only the named blobs are dropped. +/// +/// Both git stages — the `rev-list` enumeration and the streaming `pack-objects` +/// build — run under [`drive_git_child`] sharing one deadline, so a hung/slow git at +/// either stage is duration-bounded and its process group is reaped on client +/// disconnect. An outer `tokio::time::timeout` around a `spawn_blocking` cannot +/// cancel the blocking thread, so neither stage may live off the async side +/// (#174, KTD5). +/// +/// `admission` carries the filtered-serve permits across BOTH stages: stage 1 hands +/// it back on success, stage 2 takes it, and the caller receives it after stage 2 so +/// release happens only once the last child group is reaped. On a disconnect the +/// guard rides the active stage's detached reaper instead of dropping with the +/// future (F1). A stage that fails or times out (including a stage 2 whose +/// remaining budget has saturated to zero) drops the guard inside drive_git_child, +/// after that stage's reap. +pub async fn build_filtered_pack( + git_bin: &str, + repo_path: &Path, + withheld: &HashSet, + timeout: Duration, + admission: Option, +) -> Result<(Vec, Option)> { + // One deadline spans both git stages so a slow rev-list eats into the pack + // budget rather than granting each stage a fresh `timeout` (2x the permit hold). + let deadline = Instant::now() + timeout; + let (keep, admission) = rev_list_keep( + git_bin, + repo_path, + withheld, + deadline.saturating_duration_since(Instant::now()), + admission, + ) + .await?; + let mut data = keep.join("\n").into_bytes(); + data.push(b'\n'); + let mut command = Command::new(git_bin); + command .args(["pack-objects", "--stdout"]) - .current_dir(repo_path) - .stdin(Stdio::piped()) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()) - .spawn()?; - // Feed the object ids on stdin, but always reap the child afterward even if - // the write fails or stdin is missing, so an error can't drop the Child - // unwaited and leak a zombie (#53). - let write_result: std::io::Result<()> = { - use std::io::Write as _; - match child.stdin.take() { - Some(mut stdin) => { - let mut data = keep.join("\n").into_bytes(); - data.push(b'\n'); - stdin.write_all(&data) - } - None => Err(std::io::Error::new( - std::io::ErrorKind::BrokenPipe, - "git pack-objects stdin unavailable", - )), - } - }; - let out = child.wait_with_output()?; - write_result.context("failed to write object ids to git pack-objects stdin")?; - if !out.status.success() { - bail!( - "git pack-objects failed: {}", - String::from_utf8_lossy(&out.stderr) - ); - } - Ok(out.stdout) + .current_dir(repo_path); + drive_git_child( + command, + Bytes::from(data), + deadline.saturating_duration_since(Instant::now()), + "pack-objects", + admission, + ) + .await } /// Serve a clone/fetch with the withheld blobs removed from the response pack. @@ -406,20 +640,24 @@ pub fn build_filtered_pack(repo_path: &Path, withheld: &HashSet) -> Resu /// instead of a thin delta. Honoring negotiation for smaller fetch packs is an /// optimization follow-up, not a correctness requirement. pub async fn upload_pack_excluding( + git_bin: &str, repo_path: &Path, request_body: Bytes, withheld: &HashSet, + timeout: Duration, + admission: Option, ) -> Result { - // build_filtered_pack shells out to git (rev-list, pack-objects) with - // blocking std::process I/O; run it off the async worker so a large repo's - // pack build does not stall the tokio runtime. - let pack = { - let repo_path = repo_path.to_path_buf(); - let withheld = withheld.clone(); - tokio::task::spawn_blocking(move || build_filtered_pack(&repo_path, &withheld)) - .await - .context("filtered-pack build task panicked")?? - }; + // Both filtered-pack stages run async under drive_git_child (duration-bounded, + // process group reaped on disconnect, #174), and `admission` threads through + // them so the caller's permits release only after the active stage's group is + // reaped, never the instant a disconnect drops this future (F1). `git_bin` is + // injectable for the same fake-git testing reason as `run_git_service` + // (production passes the configured git binary). + let (pack, admission) = + build_filtered_pack(git_bin, repo_path, withheld, timeout, admission).await?; + // Both git stages are done and their groups reaped; release admission before the + // pure in-memory response framing below. + drop(admission); // The client lists its capabilities on the first `want` line. Honor // side-band-64k when offered (every modern smart-HTTP client offers it); @@ -538,7 +776,10 @@ mod tests { let mut withheld = std::collections::HashSet::new(); withheld.insert(secret.clone()); - let pack = build_filtered_pack(&bare, &withheld).unwrap(); + let (pack, _admission) = + build_filtered_pack("git", &bare, &withheld, Duration::from_secs(30), None) + .await + .unwrap(); let ids = pack_object_ids(&pack); assert!(ids.contains(&public), "public blob must be in the pack"); assert!( @@ -600,7 +841,10 @@ mod tests { b"0098want 0000000000000000000000000000000000000000 \ side-band-64k ofs-delta agent=git/2\n00000009done\n", ); - let resp = upload_pack_excluding(&bare, req, &withheld).await.unwrap(); + let resp = + upload_pack_excluding("git", &bare, req, &withheld, Duration::from_secs(30), None) + .await + .unwrap(); let body = to_bytes(resp.into_body(), usize::MAX).await.unwrap(); let ids = pack_object_ids(&extract_pack(&body)); assert!( @@ -650,16 +894,25 @@ mod tests { axum::extract::Query(q): axum::extract::Query>, ) -> Response { let service = q.get("service").cloned().unwrap_or_default(); - info_refs(&st.repo, &service).await.unwrap() + info_refs("git", &service, &st.repo, Duration::from_secs(30), None) + .await + .unwrap() } async fn pack_handler( axum::extract::State(st): axum::extract::State>, body: Bytes, ) -> Response { - upload_pack_excluding(&st.repo, body, &st.withheld) - .await - .unwrap() + upload_pack_excluding( + "git", + &st.repo, + body, + &st.withheld, + Duration::from_secs(30), + None, + ) + .await + .unwrap() } /// Spawn the server for `bare`, withholding `withheld`. Returns the clone URL @@ -920,23 +1173,37 @@ mod tests { #[cfg(unix)] #[tokio::test] async fn kill_group_guard_terminates_child_on_drop() { - let mut child = tokio::process::Command::new("sleep") + let child = tokio::process::Command::new("sleep") .arg("300") .process_group(0) .spawn() .unwrap(); - let pgid = child.id().map(|id| id as i32); + let pid = child.id().unwrap() as i32; { - let _guard = KillGroupOnDrop { pgid }; - } // guard drops here -> SIGTERM to the group + // The guard owns the child; on drop its detached reaper TERM/grace/KILL/ + // reaps the group (a plain sleep dies on the first SIGTERM). + let _guard = KillGroupOnDrop { + child: Some(child), + pgid: Some(pid), + admission: None, + }; + } - use std::os::unix::process::ExitStatusExt; - let status = child.wait().await.unwrap(); - assert_eq!( - status.signal(), - Some(libc::SIGTERM), - "child must be terminated by SIGTERM via its process group" + let mut gone = false; + for _ in 0..300 { + if !alive(pid) { + gone = true; + break; + } + tokio::time::sleep(std::time::Duration::from_millis(10)).await; + } + unsafe { + libc::kill(pid, libc::SIGKILL); + } + assert!( + gone, + "child must be terminated and reaped via its process group on guard drop" ); } @@ -954,9 +1221,10 @@ mod tests { .process_group(0) .spawn() .unwrap(); - let pgid = child.id().map(|id| id as i32); + let pid = child.id().unwrap() as i32; - // Read the backgrounded grandchild's pid from the first stdout line. + // Read the backgrounded grandchild's pid from the first stdout line, before + // the child moves into the guard. let mut stdout = child.stdout.take().unwrap(); let mut buf = Vec::new(); loop { @@ -971,20 +1239,26 @@ mod tests { assert!(alive(grandchild), "grandchild should be running"); { - let _guard = KillGroupOnDrop { pgid }; - } // group SIGTERM reaches sh AND the sleep grandchild - - let _ = child.wait().await; // reap sh + // The guard owns the child; on drop the detached reaper group-kills sh AND + // the sleep grandchild. + let _guard = KillGroupOnDrop { + child: Some(child), + pgid: Some(pid), + admission: None, + }; + } - // The grandchild reparents to init and is reaped; poll until it's gone. let mut gone = false; - for _ in 0..200 { + for _ in 0..300 { if !alive(grandchild) { gone = true; break; } tokio::time::sleep(std::time::Duration::from_millis(10)).await; } + unsafe { + libc::kill(grandchild, libc::SIGKILL); + } assert!( gone, "grandchild must be terminated by the group signal (#53)" @@ -996,27 +1270,32 @@ mod tests { async fn kill_group_guard_disarmed_does_not_kill() { // A request that completed cleanly disarms the guard; dropping it must not // signal anything. - let mut child = tokio::process::Command::new("sleep") + let child = tokio::process::Command::new("sleep") .arg("300") .process_group(0) .spawn() .unwrap(); + let pid = child.id().unwrap() as i32; { let mut guard = KillGroupOnDrop { - pgid: child.id().map(|id| id as i32), + child: Some(child), + pgid: Some(pid), + admission: None, }; guard.disarm(); - } // disarmed -> no kill + } // disarmed -> no reaper, no kill - assert!( - child.try_wait().unwrap().is_none(), - "disarmed guard must not kill the child" - ); + // Give any erroneously-spawned reaper a chance to run, then assert alive. + tokio::time::sleep(std::time::Duration::from_millis(50)).await; + assert!(alive(pid), "disarmed guard must not kill the child"); - // Clean up the still-running child. - let _ = child.kill().await; - let _ = child.wait().await; + // Clean up the still-running child (disarm dropped the handle, so reap by pid). + unsafe { + libc::kill(pid, libc::SIGKILL); + let mut status = 0; + libc::waitpid(pid, &mut status, 0); + } } // ── #62 PR1: end-to-end teardown wiring through run_git_service ───────── @@ -1120,6 +1399,66 @@ mod tests { ); } + /// True when `err` is the transient ETXTBSY exec race a freshly-written fake + /// `git` hits under fork-storm load — a concurrent worker forked while this + /// file's write fd was still open, so `execve` sees it as busy (the same race + /// `fake_git_run_with_pids` retries). Deliberately narrow: only this raw-OS + /// error is retried, so a wrong error *type* (e.g. a missing async bound + /// surfacing as anything other than `GitServiceTimeout`) still fails loudly at + /// the caller's assertion instead of being swallowed. `spawn()?` propagates the + /// `io::Error` with no context, so it survives as the anyhow root. + #[cfg(unix)] + fn is_transient_exec_race(err: &anyhow::Error) -> bool { + // ETXTBSY == 26 on Linux and the BSDs/macOS; std has no stable ErrorKind. + err.downcast_ref::() + .and_then(std::io::Error::raw_os_error) + == Some(26) + } + + /// Drive `build_filtered_pack` under a per-attempt watchdog, retrying ONLY the + /// transient ETXTBSY exec race and returning the terminal error for the caller + /// to classify. Every attempt keeps its own outer `tokio::time::timeout`, so a + /// MISSING async bound — the regression the `build_filtered_pack_times_out_*` + /// tests guard — still trips the watchdog loudly on every attempt: the retry + /// can only absorb a fast exec failure, never a hang (a hang never returns, so + /// it can never reach the retry decision). + #[cfg(unix)] + async fn build_filtered_pack_or_exec_race_retry( + git_bin: &str, + repo_path: &std::path::Path, + withheld: &HashSet, + stage_timeout: Duration, + ) -> anyhow::Error { + for i in 0..FAKE_GIT_RETRY_ATTEMPTS { + let result = tokio::time::timeout( + Duration::from_secs(10), + build_filtered_pack(git_bin, repo_path, withheld, stage_timeout, None), + ) + .await + .expect( + "build_filtered_pack must return within the watchdog — the git stage \ + must be timeout-bounded, not an uncancellable spawn_blocking", + ); + // `match` rather than `expect_err`: the Ok arm carries an + // AdmissionGuard, which has no Debug impl for expect_err to print. + let err = match result { + Ok(_) => panic!("a hung git stage must return an error, not hang"), + Err(e) => e, + }; + if is_transient_exec_race(&err) { + // Fresh-fake-git ETXTBSY: back off (growing) so a bursty fork-pressure + // spike subsides before retrying, per fake_git_run_with_pids. + tokio::time::sleep(Duration::from_millis(FAKE_GIT_BACKOFF_STEP_MS * (i + 1))).await; + continue; + } + return err; + } + panic!( + "fake git kept hitting ETXTBSY after {FAKE_GIT_RETRY_ATTEMPTS} attempts \ + (persistent exec failure, not a transient parallel-runner miss)" + ); + } + // Dropping the request future mid-flight (client disconnect) must SIGTERM the // whole group so git AND its pack-objects grandchild die together. Goes RED // if `process_group(0)` or the guard-arming is removed: without its own @@ -1154,6 +1493,7 @@ mod tests { tmp.path(), Bytes::new(), Duration::from_secs(60), + None, )); // Advance the future a slice at a time until the fake records its @@ -1225,6 +1565,464 @@ mod tests { ); } + /// #174 U2 (P1-c, RED-before/GREEN-after): on a client disconnect the teardown must + /// be as strong as the timeout path — a group member that IGNORES SIGTERM is still + /// SIGKILLed and reaped, not left running to accumulate past the concurrency cap. + /// The old `KillGroupOnDrop::drop` sent a lone SIGTERM with no escalation or reap, + /// so a SIGTERM-ignoring descendant survived the disconnect (RED). The fix launches + /// a detached reaper owning the child that runs the full TERM/grace/SIGKILL/reap. + /// This is distinct from the well-behaved-grandchild test above, whose `sleep` dies + /// on the first SIGTERM. + #[cfg(unix)] + #[tokio::test] + async fn run_git_service_sigkills_a_sigterm_ignoring_child_on_disconnect() { + let tmp = tempfile::TempDir::new().unwrap(); + let descfile = tmp.path().join("desc.pid"); + // Leader (dies on the group SIGTERM) spawns a descendant that traps SIGTERM, + // records its own pid, and loops ~30s (bounded so a RED run leaks no permanent + // orphan; the assertion fires well before then). + let body = format!( + "#!/bin/sh\n\ + sh -c 'trap \"\" TERM; echo $$ > \"{}\"; i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done' &\n\ + wait\n", + descfile.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + + let mut fut = Box::pin(run_git_service( + git_bin.to_str().unwrap(), + "git-upload-pack", + tmp.path(), + Bytes::new(), + Duration::from_secs(60), + None, + )); + // Drive the future a slice at a time until the descendant records its pid. + let mut desc: Option = None; + for _ in 0..500 { + let _ = tokio::time::timeout(Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&descfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + desc = Some(p); + break; + } + } + let desc = desc.expect("the fake leader must have spawned and recorded a descendant"); + let _cleanup = ReapOnPanic(vec![desc]); + assert!(alive(desc), "descendant should be running before the drop"); + + // Client disconnect: drop the request future. The guard's detached reaper must + // escalate SIGTERM -> SIGKILL and reap the SIGTERM-ignoring descendant. + drop(fut); + + let mut gone = false; + for _ in 0..500 { + if !alive(desc) { + gone = true; + break; + } + tokio::time::sleep(Duration::from_millis(10)).await; + } + // Kill regardless so a RED run leaks no orphan. + unsafe { + libc::kill(desc, libc::SIGKILL); + } + assert!( + gone, + "a SIGTERM-ignoring descendant (pid {desc}) must be SIGKILLed and reaped on \ + disconnect, not left running (old KillGroupOnDrop sent SIGTERM only)" + ); + } + + // #174 (SC3): the info/refs advertisement is now duration-bounded — previously + // it ran a bare `Command::output()` with no deadline, so a hung git pinned its + // concurrency slot forever. A hung fake git must abort with GitServiceTimeout + // (which the handler maps to 504). The outer watchdog turns a missing timeout + // into a loud failure instead of hanging the suite. info_refs shares + // drive_git_child, so its disconnect/group-teardown is the same code proven by + // run_git_service_tears_down_group_when_future_dropped above. + #[cfg(unix)] + #[tokio::test] + async fn info_refs_times_out_a_hung_advertisement() { + let tmp = tempfile::TempDir::new().unwrap(); + // A fake git that hangs forever instead of advertising refs. + let git_bin = write_fake_git(tmp.path(), "#!/bin/sh\nsleep 300\n"); + + let result = tokio::time::timeout( + Duration::from_secs(10), + info_refs( + git_bin.to_str().unwrap(), + "git-upload-pack", + tmp.path(), + Duration::from_millis(200), + None, + ), + ) + .await + .expect("info_refs must return within the watchdog — its own timeout must fire"); + let err = result.expect_err("a hung advertisement must return an error, not hang"); + assert!( + err.downcast_ref::().is_some(), + "a hung advertisement must abort with GitServiceTimeout (-> 504), got: {err}" + ); + } + + // #174 (SC3, KTD5): the filtered-pack build's streaming pack-objects stage is + // duration-bounded on the ASYNC side. The old build ran the whole thing in a + // spawn_blocking, so an outer tokio timeout could not cancel the blocking thread + // and a disconnect orphaned the git child. A fake git that returns objects fast + // on rev-list but hangs on pack-objects must now abort with GitServiceTimeout; + // the watchdog turns a missing bound into a loud failure. The stage runs under + // drive_git_child, so its disconnect/group-teardown is the same code proven by + // run_git_service_tears_down_group_when_future_dropped. + #[cfg(unix)] + #[tokio::test] + async fn build_filtered_pack_times_out_a_hung_pack_objects() { + let tmp = tempfile::TempDir::new().unwrap(); + // rev-list returns one oid fast; pack-objects hangs forever. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) echo deadbeefdeadbeefdeadbeefdeadbeefdeadbeef ;;\n pack-objects) sleep 300 ;;\n *) exit 1 ;;\nesac\n"; + let git_bin = write_fake_git(tmp.path(), body); + let withheld = HashSet::new(); + + // Retry only the transient ETXTBSY exec race (see the helper); the + // per-attempt watchdog keeps the missing-bound regression loud. + let err = build_filtered_pack_or_exec_race_retry( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_millis(200), + ) + .await; + assert!( + err.downcast_ref::().is_some(), + "a hung pack-objects must abort with GitServiceTimeout, got: {err}" + ); + } + + // #174 (SC3, KTD5): the filtered-pack build's rev-list ENUMERATION stage is now + // duration-bounded on the ASYNC side too. It previously ran inside a + // spawn_blocking via a bare `Command::output()`, which an outer tokio timeout + // cannot cancel, so a hung/slow rev-list pinned the endpoint's concurrency permit + // for the whole hang. A fake git that hangs on rev-list must now abort with + // GitServiceTimeout; the watchdog turns a missing bound into a loud failure. + // rev-list runs under drive_git_child, so its disconnect/group-teardown is the + // same code proven by run_git_service_tears_down_group_when_future_dropped. + #[cfg(unix)] + #[tokio::test] + async fn build_filtered_pack_times_out_a_hung_rev_list() { + let tmp = tempfile::TempDir::new().unwrap(); + // rev-list hangs forever; pack-objects would return fast if it were reached. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) sleep 300 ;;\n pack-objects) printf '' ;;\n *) exit 1 ;;\nesac\n"; + let git_bin = write_fake_git(tmp.path(), body); + let withheld = HashSet::new(); + + // Retry only the transient ETXTBSY exec race (see the helper); the + // per-attempt watchdog keeps the missing-bound regression loud. + let err = build_filtered_pack_or_exec_race_retry( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_millis(200), + ) + .await; + assert!( + err.downcast_ref::().is_some(), + "a hung rev-list must abort with GitServiceTimeout, got: {err}" + ); + } + + // ── F1: filtered-serve admission threaded through both pack stages ────── + // + // The handler-level disconnect regression lives in api/repos.rs + // (upload_pack_filtered_permit_held_through_group_reap_after_disconnect); these + // exercise the guard contract at the smart_http seam: held through a + // mid-rev-list disconnect, handed back on success, and recovered (post-reap) + // on every error return. + + /// F1: a client disconnect mid-REV-LIST (stage 1 of the filtered build) must + /// keep the threaded admission held until the rev-list group is reaped, and + /// stage 2 (pack-objects) must never spawn, since the dropped future cannot + /// advance to it. + #[cfg(unix)] + #[tokio::test] + async fn build_filtered_pack_holds_admission_through_rev_list_reap_on_disconnect() { + let tmp = tempfile::TempDir::new().unwrap(); + let descfile = tmp.path().join("desc.pid"); + let packfile = tmp.path().join("pack.ran"); + // rev-list: SIGTERM-trapping descendant records its pid and loops (bounded + // so a broken fix leaks no permanent orphan); the leader waits, keeping the + // interaction pending. pack-objects: records that it ran (it must never run). + let body = format!( + "#!/bin/sh\n\ + case \"$1\" in\n\ + rev-list)\n\ + sh -c 'trap \"\" TERM; echo $$ > \"{desc}\"; i=0; while [ $i -lt 20 ]; do sleep 1; i=$((i+1)); done' &\n\ + wait ;;\n\ + pack-objects) : > \"{pack}\" ;;\n\ + *) : ;;\n\ + esac\n\ + exit 0\n", + desc = descfile.display(), + pack = packfile.display() + ); + let git_bin = write_fake_git(tmp.path(), &body); + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + let guard = AdmissionGuard::new(sem.clone().try_acquire_owned().unwrap(), None::<()>); + let withheld = HashSet::new(); + + let mut fut = Box::pin(build_filtered_pack( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_secs(60), + Some(guard), + )); + // Drive until the rev-list descendant records its pid. + let mut desc: Option = None; + for _ in 0..500 { + let _ = tokio::time::timeout(Duration::from_millis(10), &mut fut).await; + if let Some(p) = std::fs::read_to_string(&descfile) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + desc = Some(p); + break; + } + } + let desc = desc.expect("the fake rev-list must have spawned its descendant"); + let _cleanup = ReapOnPanic(vec![desc]); + assert_eq!( + sem.available_permits(), + 0, + "admission held while rev-list runs" + ); + + // Client disconnect mid-enumeration. + drop(fut); + + // Load-bearing: the permit must stay held while the SIGTERM-ignoring group + // member is still alive (check before the reaper's ~2s SIGKILL escalation). + tokio::time::sleep(Duration::from_millis(100)).await; + assert!( + alive(desc), + "the SIGTERM-ignoring descendant must still be alive during the hold window" + ); + assert_eq!( + sem.available_permits(), + 0, + "on disconnect mid-rev-list the admission must be held until the group is \ + reaped, not released the instant the future drops (F1)" + ); + + let mut freed = false; + for _ in 0..400 { + if sem.available_permits() == 1 { + freed = true; + break; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + assert!( + freed, + "the reaper must release admission once the rev-list group is gone" + ); + assert!( + !packfile.exists(), + "pack-objects must never spawn after a disconnect mid-rev-list" + ); + } + + /// F1 success path: a filtered serve that completes hands the guard back + /// through both stages and releases admission when the serve returns. No leak, + /// no double-release. + #[tokio::test] + async fn upload_pack_excluding_releases_admission_after_success() { + let td = TempDir::new().unwrap(); + let (_work, bare, secret_oid, _public_oid) = fixture_with_secret(&td); + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + let guard = AdmissionGuard::new(sem.clone().try_acquire_owned().unwrap(), None::<()>); + let withheld = HashSet::from([secret_oid]); + let resp = upload_pack_excluding( + "git", + &bare, + Bytes::from_static(b"0000"), + &withheld, + Duration::from_secs(30), + Some(guard), + ) + .await + .unwrap(); + assert_eq!(resp.status(), StatusCode::OK); + assert_eq!( + sem.available_permits(), + 1, + "admission must be released once the filtered serve returns" + ); + } + + /// F1 error path: a rev-list that exits non-zero fails the build AND recovers + /// the threaded admission: drive_git_child drops the guard only after the + /// completed join has reaped the child, and the error return must not leak it. + #[cfg(unix)] + #[tokio::test] + async fn build_filtered_pack_releases_admission_after_rev_list_failure() { + let tmp = tempfile::TempDir::new().unwrap(); + let body = + "#!/bin/sh\ncase \"$1\" in\n rev-list) echo boom >&2; exit 2 ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + let withheld = HashSet::new(); + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + + for i in 0..FAKE_GIT_RETRY_ATTEMPTS { + let guard = AdmissionGuard::new(sem.clone().try_acquire_owned().unwrap(), None::<()>); + let result = build_filtered_pack( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_secs(30), + Some(guard), + ) + .await; + let err = match result { + Ok(_) => panic!("a non-zero rev-list exit must fail the build"), + Err(e) => e, + }; + // Load-bearing on EVERY attempt: even a transient spawn-failure return + // must recover the permit (the guard drops before the Err surfaces). + assert_eq!( + sem.available_permits(), + 1, + "the admission permit must be recovered on the error return, not leaked" + ); + if is_transient_exec_race(&err) { + tokio::time::sleep(Duration::from_millis(FAKE_GIT_BACKOFF_STEP_MS * (i + 1))).await; + continue; + } + assert!( + err.to_string().contains("rev-list failed"), + "the failure must surface rev-list's own error, got: {err}" + ); + return; + } + panic!( + "fake git kept hitting ETXTBSY after {FAKE_GIT_RETRY_ATTEMPTS} attempts \ + (persistent exec failure, not a transient parallel-runner miss)" + ); + } + + /// F1: a pack-objects stage that times out (stage 2 of the filtered build, + /// guard threaded through) recovers the admission after the reap: the timeout + /// Err cannot carry the guard, so drive_git_child must drop it post-reap. + #[cfg(unix)] + #[tokio::test] + async fn build_filtered_pack_recovers_admission_after_pack_objects_timeout() { + let tmp = tempfile::TempDir::new().unwrap(); + // rev-list returns one oid fast (guard handed back); pack-objects hangs. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) echo deadbeefdeadbeefdeadbeefdeadbeefdeadbeef ;;\n pack-objects) sleep 300 ;;\n *) exit 1 ;;\nesac\n"; + let git_bin = write_fake_git(tmp.path(), body); + let withheld = HashSet::new(); + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + + for i in 0..FAKE_GIT_RETRY_ATTEMPTS { + let guard = AdmissionGuard::new(sem.clone().try_acquire_owned().unwrap(), None::<()>); + let result = tokio::time::timeout( + Duration::from_secs(10), + build_filtered_pack( + git_bin.to_str().unwrap(), + tmp.path(), + &withheld, + Duration::from_millis(200), + Some(guard), + ), + ) + .await + .expect( + "build_filtered_pack must return within the watchdog; the git stage \ + must be timeout-bounded", + ); + let err = match result { + Ok(_) => panic!("a hung pack-objects must fail the build"), + Err(e) => e, + }; + assert_eq!( + sem.available_permits(), + 1, + "admission must be recovered after the stage-2 timeout reap, not leaked" + ); + if is_transient_exec_race(&err) { + tokio::time::sleep(Duration::from_millis(FAKE_GIT_BACKOFF_STEP_MS * (i + 1))).await; + continue; + } + assert!( + err.downcast_ref::().is_some(), + "a hung pack-objects must abort with GitServiceTimeout, got: {err}" + ); + return; + } + panic!( + "fake git kept hitting ETXTBSY after {FAKE_GIT_RETRY_ATTEMPTS} attempts \ + (persistent exec failure, not a transient parallel-runner miss)" + ); + } + + /// F1 zero-budget contract: build_filtered_pack hands pack-objects + /// `deadline.saturating_duration_since(now)`, which saturates to ZERO once + /// rev-list has consumed the whole budget. A zero-budget stage must still route + /// the guard through the timeout's reap-then-drop rather than leak it: the + /// child spawns, the deadline fires immediately, the group is reaped, and only + /// then does the guard drop. + #[cfg(unix)] + #[tokio::test] + async fn drive_git_child_zero_budget_drops_admission_after_reap() { + let tmp = tempfile::TempDir::new().unwrap(); + let git_bin = write_fake_git(tmp.path(), "#!/bin/sh\nsleep 300\n"); + let sem = std::sync::Arc::new(tokio::sync::Semaphore::new(1)); + + for i in 0..FAKE_GIT_RETRY_ATTEMPTS { + let guard = AdmissionGuard::new(sem.clone().try_acquire_owned().unwrap(), None::<()>); + let mut command = tokio::process::Command::new(git_bin.to_str().unwrap()); + command + .args(["pack-objects", "--stdout"]) + .current_dir(tmp.path()); + let result = tokio::time::timeout( + Duration::from_secs(10), + drive_git_child( + command, + Bytes::new(), + Duration::ZERO, + "pack-objects", + Some(guard), + ), + ) + .await + .expect("a zero-budget stage must abort via its own deadline, not hang"); + let err = match result { + Ok(_) => panic!("a zero-budget stage must return an error"), + Err(e) => e, + }; + assert_eq!( + sem.available_permits(), + 1, + "the guard must drop after the zero-budget timeout reap, not leak" + ); + if is_transient_exec_race(&err) { + tokio::time::sleep(Duration::from_millis(FAKE_GIT_BACKOFF_STEP_MS * (i + 1))).await; + continue; + } + assert!( + err.downcast_ref::().is_some(), + "a zero-budget stage must abort with GitServiceTimeout, got: {err}" + ); + return; + } + panic!( + "fake git kept hitting ETXTBSY after {FAKE_GIT_RETRY_ATTEMPTS} attempts \ + (persistent exec failure, not a transient parallel-runner miss)" + ); + } + // A request that runs to completion must DISARM the guard after reaping, so // no stray group SIGTERM fires. The fake exits non-zero (surfacing as Err) // but leaves a grandchild alive; the grandchild must survive. Goes RED if the @@ -1257,6 +2055,7 @@ mod tests { tmp.path(), Bytes::new(), Duration::from_secs(60), + None, ) }) .await; @@ -1301,6 +2100,7 @@ mod tests { tmp.path(), Bytes::new(), Duration::from_secs(60), + None, ) }) .await; @@ -1351,6 +2151,7 @@ mod tests { tmp.path(), Bytes::new(), git_timeout, + None, ), ) }) @@ -1425,6 +2226,7 @@ mod tests { tmp.path(), Bytes::new(), git_timeout, + None, ), ) }) @@ -1462,6 +2264,7 @@ mod tests { tmp.path(), Bytes::new(), Duration::from_millis(200), + None, ), ) .await; @@ -1502,6 +2305,7 @@ mod tests { tmp.path(), big, Duration::from_secs(60), + None, ) .await; diff --git a/crates/gitlawb-node/src/git/store.rs b/crates/gitlawb-node/src/git/store.rs index 229ee695..a18ab472 100644 --- a/crates/gitlawb-node/src/git/store.rs +++ b/crates/gitlawb-node/src/git/store.rs @@ -271,7 +271,8 @@ pub struct TreeEntry { /// `/ipfs/` is computed from these same content bytes via /// `gitlawb_core::cid::Cid::from_git_object_bytes`. /// -/// Get just the object type. Returns `None` if the object doesn't exist. +/// Get just the object type. Returns `None` if the object doesn't exist; a +/// probe that could not examine the object store is `Err`, never `None`. pub fn object_type(repo_path: &Path, sha256_hex: &str) -> Result> { let type_output = Command::new("git") .args(["cat-file", "-t", sha256_hex]) @@ -280,6 +281,19 @@ pub fn object_type(repo_path: &Path, sha256_hex: &str) -> Result> .context("failed to run git cat-file -t")?; if !type_output.status.success() { + // A nonzero exit is an ABSENCE verdict only when git could examine the + // object store: missing-object and invalid-oid probes die with a single + // clean `fatal:` line. A broken repo dir (`fatal: not a git repository`) + // or a corrupt object (`error: inflate` / `error: unable to unpack` + // lines before the fatal) proves nothing about absence, so it must + // surface as Err — the /ipfs scan taints on Err rather than treating + // the repo as probed-clean. + let stderr = String::from_utf8_lossy(&type_output.stderr); + if stderr.contains("not a git repository") + || stderr.lines().any(|l| l.starts_with("error:")) + { + bail!("git cat-file -t failed: {}", stderr.trim()); + } return Ok(None); } @@ -306,6 +320,227 @@ pub fn read_object_content(repo_path: &Path, sha256_hex: &str, obj_type: &str) - Ok(content_output.stdout) } +/// Why an `/ipfs` existence probe could not return an absence verdict (#174 F5/U4). +/// The caller (`api::ipfs`) maps the variant to an HTTP status, and the split exists +/// only for that mapping: a `Transient` fault is retryable (503), a `Deterministic` +/// fault is terminal (500). +/// +/// The discriminator is object-store readability, NOT any English `git` wording, so a +/// future `git` message change cannot silently reclassify a fault (KTD-4): if the +/// store cannot be read (an unreadable or mid-repack pack, a removed `objects/` dir, +/// a permissions fault) the fault may clear on its own -> `Transient`; if the store IS +/// readable yet `git` still fails (a corrupt repo, a bad `.git/config`) a retry cannot +/// fix it -> `Deterministic`, so a conformant client is told not to retry-storm a +/// fresh `git cat-file` per attempt against a persistently broken repo. +#[derive(Debug)] +pub enum ProbeError { + /// Retryable (-> 503): the object store could not be read right now. + Transient(anyhow::Error), + /// Terminal (-> 500): a persistent, deterministic fault a retry cannot fix. + Deterministic(anyhow::Error), +} + +impl std::fmt::Display for ProbeError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + ProbeError::Transient(e) => write!(f, "transient probe fault: {e}"), + ProbeError::Deterministic(e) => write!(f, "deterministic probe fault: {e}"), + } + } +} + +impl std::error::Error for ProbeError {} + +/// Structured outcome of one `git cat-file --batch-check` existence probe. +enum BatchProbe { + /// Present: git printed ` ` on exit 0. + Present(String), + /// A structured, CLEAN absence: git printed ` missing` on exit 0 with no + /// `error:` diagnostics on stderr. This is the ONLY signal that can become an + /// `Ok(None)` (404) absence verdict — and even then only after the store-readable + /// disambiguation below, since an unreadable pack ALSO prints a clean `missing`. + Missing, + /// git could not honestly examine the object store: a hard exit (bad config, not a + /// git repo) OR an exit-0 `missing` accompanied by `error:` diagnostics (a corrupt + /// loose object still prints `missing` on stdout but complains on stderr). Never an + /// absence verdict. Carries the raw git detail for the server log only. + Fault(anyhow::Error), +} + +/// One bounded, reaped `git cat-file --batch-check` probe (the oid is fed on stdin, so +/// no prose is matched to decide presence — the `missing` token and the exit code are +/// the structured signal). See [`object_type_bounded`] for the surrounding teardown +/// guarantees. +fn batch_check_probe( + git_bin: &str, + repo_path: &Path, + sha256_hex: &str, + deadline: std::time::Instant, +) -> std::result::Result { + let stdin = format!("{sha256_hex}\n"); + let (status, stdout, stderr) = crate::git::visibility_pack::run_bounded_git_raw( + git_bin, + &["cat-file", "--batch-check"], + repo_path, + stdin.as_bytes(), + deadline, + ) + // A spawn/timeout failure of the reaped child is not deterministic — retry it. + .map_err(ProbeError::Transient)?; + + let stderr = String::from_utf8_lossy(&stderr); + // A corrupt object makes `--batch-check` print `missing` on stdout (exit 0) yet + // emit `error:` lines on stderr; those diagnostics disqualify a clean-absence read + // regardless of the exit code, so they are checked before anything else. + let has_error_diag = stderr.lines().any(|l| l.starts_with("error:")); + + if status.success() && !has_error_diag { + let line = String::from_utf8_lossy(&stdout); + let line = line.trim(); + // ` missing` is the structured absence token; ` ` is a + // hit. Anything else on a "success" exit is unexpected and not an absence. + if line + .rsplit(' ') + .next() + .is_some_and(|last| last == "missing") + { + return Ok(BatchProbe::Missing); + } + let mut parts = line.split_whitespace(); + if let (Some(_oid), Some(ty), Some(_size)) = (parts.next(), parts.next(), parts.next()) { + return Ok(BatchProbe::Present(ty.to_string())); + } + return Ok(BatchProbe::Fault(anyhow::anyhow!( + "unexpected git cat-file --batch-check output: {line:?}" + ))); + } + + Ok(BatchProbe::Fault(anyhow::anyhow!( + "git cat-file --batch-check failed (exit {:?}): {}", + status.code(), + stderr.trim() + ))) +} + +/// Bounded, reaped variant of [`object_type`] for the async `/ipfs` serve path +/// (#174 F3/F5): runs `git cat-file --batch-check` off the caller's runtime through the +/// process-group + watchdog reaper, so a hung or corrupt object store cannot pin a +/// runtime worker or an IPFS admission permit past `deadline`. +/// +/// Absence is keyed on `--batch-check`'s STRUCTURED ` missing` token on exit 0, +/// never on any English `fatal:` wording (KTD-4): a genuinely-absent object is the only +/// `Ok(None)` (404) path. A probe that could not honestly examine the store is a +/// [`ProbeError`], split by object-store readability into `Transient` (retryable 503) +/// and `Deterministic` (terminal 500) so the serve path can shed the right status. +pub fn object_type_bounded( + git_bin: &str, + repo_path: &Path, + sha256_hex: &str, + deadline: std::time::Instant, +) -> std::result::Result, ProbeError> { + match batch_check_probe(git_bin, repo_path, sha256_hex, deadline)? { + BatchProbe::Present(ty) => Ok(Some(ty)), + BatchProbe::Fault(detail) => Err(classify_store_fault(repo_path, detail)), + BatchProbe::Missing => { + // A clean `missing` is the absence-vs-unreadable-pack COLLISION (#174 F5): + // a genuinely missing object AND a packed object whose pack/idx is + // unreadable (permissions, or a mid-repack race) both print an identical + // clean `missing`. Disambiguate OUT OF BAND on store readability — an + // unreadable store is not an absence verdict (taint -> retryable 503). + if !object_store_readable(repo_path) { + return Err(ProbeError::Transient(anyhow::anyhow!( + "git cat-file inconclusive: object store not readable at {} (not an absence verdict)", + repo_path.display() + ))); + } + // Store readable: re-probe once. Still `missing` on a confirmed-readable + // store is very likely truly absent (Ok(None)); a mid-repack race that + // resolved returns the type. This narrows, but cannot fully close, the + // concurrent-repack window (the readability check samples a different + // instant than the failing probe). + match batch_check_probe(git_bin, repo_path, sha256_hex, deadline)? { + BatchProbe::Present(ty) => Ok(Some(ty)), + BatchProbe::Fault(detail) => Err(classify_store_fault(repo_path, detail)), + BatchProbe::Missing => Ok(None), + } + } + } +} + +/// Classify a probe fault by object-store readability (#174 F5/U4). An unreadable store +/// may be a transient permissions/mid-repack condition (retryable 503); a readable +/// store on which git still fails is a persistent, deterministic fault — a corrupt repo +/// or a bad `.git/config` — that a retry cannot fix (terminal 500). The `detail` is +/// carried for the server log; the client-facing body is opaque (set by the caller). +fn classify_store_fault(repo_path: &Path, detail: anyhow::Error) -> ProbeError { + if object_store_readable(repo_path) { + ProbeError::Deterministic(detail) + } else { + ProbeError::Transient(detail) + } +} + +/// Best-effort check that a repo's object store is readable, used to disambiguate a +/// genuine missing-object `git cat-file` fatal from an unreadable or racing pack +/// (both emit "could not get object info"). Returns false on any unreadable +/// `objects/` dir or any pack/idx that cannot be opened (EACCES / EIO), so the +/// caller surfaces an error rather than a false absence. Cheap — a couple of readdir +/// plus open probes. It narrows, but does not close, the concurrent-repack TOCTOU: it +/// samples a different instant than the failing cat-file. +fn object_store_readable(repo_path: &Path) -> bool { + let objects = repo_path.join("objects"); + // The objects dir itself must be listable; drain the iterator so a mid-listing + // EACCES/EIO surfaces, not just the initial open. + let Ok(entries) = std::fs::read_dir(&objects) else { + return false; + }; + for entry in entries { + if entry.is_err() { + return false; + } + } + // Every pack file and its index must be openable for read. A loose-only store + // (no pack dir) is fine — the objects readdir above already proved reachability. + if let Ok(pack_entries) = std::fs::read_dir(objects.join("pack")) { + for entry in pack_entries { + let Ok(entry) = entry else { + return false; + }; + let path = entry.path(); + if matches!( + path.extension().and_then(|s| s.to_str()), + Some("pack") | Some("idx") + ) && std::fs::File::open(&path).is_err() + { + return false; + } + } + } + true +} + +/// Bounded, reaped variant of [`read_object_content`] for the async `/ipfs` serve +/// path (#174 F3). Same teardown guarantees as [`object_type_bounded`]. +pub fn read_object_content_bounded( + git_bin: &str, + repo_path: &Path, + sha256_hex: &str, + obj_type: &str, + deadline: std::time::Instant, +) -> Result> { + let (status, stdout, stderr) = crate::git::visibility_pack::run_bounded_git_raw( + git_bin, + &["cat-file", obj_type, sha256_hex], + repo_path, + &[], + deadline, + )?; + if !status.success() { + bail!("git cat-file failed: {}", String::from_utf8_lossy(&stderr)); + } + Ok(stdout) +} + /// Read a git object by its SHA-256 hex object ID. /// /// Returns `(object_type, content_bytes)` where `content_bytes` is the raw @@ -501,4 +736,289 @@ mod tests { "unchanged file must not appear: {names:?}" ); } + + /// #174 F5 (RED-before/GREEN-after): a packed object whose pack/idx is unreadable + /// makes `git cat-file -t` emit "could not get object info" — byte-identical to a + /// genuine miss. `object_type_bounded` must report absence ONLY when the object + /// store is confirmed readable; an unreadable store is Err (-> retryable 503), + /// never Ok(None) (-> a wrong 404 for a present object). + #[cfg(unix)] + #[test] + fn object_type_bounded_unreadable_pack_is_error_not_absence() { + use std::os::unix::fs::PermissionsExt; + let td = tempfile::TempDir::new().unwrap(); + let work = td.path().join("work"); + let bare = td.path().join("bare.git"); + std::fs::create_dir_all(&work).unwrap(); + let g = |args: &[&str], dir: &Path| { + assert!( + Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(), + "git {args:?}" + ); + }; + g(&["init", "-q", "--object-format=sha256", "."], &work); + g(&["config", "user.email", "t@t"], &work); + g(&["config", "user.name", "t"], &work); + std::fs::write(work.join("file.txt"), b"packed f5 content\n").unwrap(); + g(&["add", "file.txt"], &work); + g(&["commit", "-qm", "c1"], &work); + let blob = String::from_utf8( + Command::new("git") + .args(["rev-parse", "HEAD:file.txt"]) + .current_dir(&work) + .output() + .unwrap() + .stdout, + ) + .unwrap() + .trim() + .to_string(); + g( + &[ + "clone", + "-q", + "--bare", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + td.path(), + ); + g(&["gc", "-q"], &bare); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + + // Readable store: the packed blob probes present; a genuine miss is Ok(None). + assert_eq!( + super::object_type_bounded("git", &bare, &blob, deadline) + .unwrap() + .as_deref(), + Some("blob"), + "a packed blob on a readable store must probe present" + ); + assert!( + super::object_type_bounded("git", &bare, &"0".repeat(64), deadline) + .unwrap() + .is_none(), + "a genuinely-absent object on a readable store must be Ok(None)" + ); + + // Make the pack unreadable: cat-file -t now emits the collided fatal for the + // PRESENT blob. It must surface as Err, not a false Ok(None). + let pack_dir = bare.join("objects").join("pack"); + let set_pack_mode = |mode: u32| { + for e in std::fs::read_dir(&pack_dir).unwrap() { + let p = e.unwrap().path(); + if matches!( + p.extension().and_then(|s| s.to_str()), + Some("pack") | Some("idx") + ) { + let mut perms = std::fs::metadata(&p).unwrap().permissions(); + perms.set_mode(mode); + std::fs::set_permissions(&p, perms).unwrap(); + } + } + }; + set_pack_mode(0o000); + // Root bypasses file permissions, so the chmod won't block reads there; only + // assert the error path when the pack is genuinely unreadable to this process. + let a_pack = std::fs::read_dir(&pack_dir) + .unwrap() + .filter_map(|e| e.ok()) + .map(|e| e.path()) + .find(|p| p.extension().and_then(|s| s.to_str()) == Some("pack")); + let genuinely_unreadable = a_pack + .as_ref() + .map(|p| std::fs::File::open(p).is_err()) + .unwrap_or(false); + let res = super::object_type_bounded("git", &bare, &blob, deadline); + set_pack_mode(0o644); // restore so TempDir cleanup succeeds + + if genuinely_unreadable { + assert!( + res.is_err(), + "an unreadable pack must surface as Err (-> retryable 503), not Ok(None) \ + (-> a wrong 404 for a present object); got {res:?}" + ); + } + } + + /// Shared setup: a bare sha256 repo carrying one committed blob. Returns the repo + /// path and the blob's oid. + #[cfg(unix)] + fn bare_repo_with_blob(td: &std::path::Path) -> (std::path::PathBuf, String) { + let work = td.join("work"); + let bare = td.join("bare.git"); + std::fs::create_dir_all(&work).unwrap(); + let g = |args: &[&str], dir: &Path| { + assert!( + Command::new("git") + .args(args) + .current_dir(dir) + .status() + .unwrap() + .success(), + "git {args:?}" + ); + }; + g(&["init", "-q", "--object-format=sha256", "."], &work); + g(&["config", "user.email", "t@t"], &work); + g(&["config", "user.name", "t"], &work); + std::fs::write(work.join("file.txt"), b"f5 u4 content\n").unwrap(); + g(&["add", "file.txt"], &work); + g(&["commit", "-qm", "c1"], &work); + let blob = String::from_utf8( + Command::new("git") + .args(["rev-parse", "HEAD:file.txt"]) + .current_dir(&work) + .output() + .unwrap() + .stdout, + ) + .unwrap() + .trim() + .to_string(); + g( + &[ + "clone", + "-q", + "--bare", + work.to_str().unwrap(), + bare.to_str().unwrap(), + ], + td, + ); + (bare, blob) + } + + /// #174 F5/U4 (RED-before/GREEN-after, the CORE regression guard): a repo with a + /// corrupt `.git/config` makes `git cat-file` die with `fatal: bad config line N` + /// (exit 128, NO `error:` line) while `objects/` stays fully readable. The old + /// `-t` path let that fall through to `Ok(None)` — a false 404 for content that + /// may well exist. `object_type_bounded` must instead classify it as a + /// DETERMINISTIC fault (a retry cannot fix it) so the serve path renders a + /// terminal 500, never a 404 and never a retryable 503. + /// + /// LOAD-BEARING: revert the classification (route `BatchProbe::Fault` on a readable + /// store back to `Ok(None)`, or drop the `has_error_diag`/exit checks so a hard + /// `fatal:` is read as `missing`) and this goes RED — the probe reports the corrupt + /// repo as an absent object. + #[cfg(unix)] + #[test] + fn object_type_bounded_bad_config_is_deterministic_not_absence() { + let td = tempfile::TempDir::new().unwrap(); + let (bare, blob) = bare_repo_with_blob(td.path()); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + + // Baseline on the healthy repo: present blob probes present, genuine miss is + // the clean Ok(None) absence verdict. + assert_eq!( + super::object_type_bounded("git", &bare, &blob, deadline) + .unwrap() + .as_deref(), + Some("blob"), + "a present blob on a healthy readable store must probe present" + ); + assert!( + super::object_type_bounded("git", &bare, &"0".repeat(64), deadline) + .unwrap() + .is_none(), + "a genuinely-absent object on a healthy readable store must be Ok(None) (404)" + ); + + // Corrupt the config; objects/ is untouched (and stays readable). + { + use std::io::Write; + let mut cfg = std::fs::OpenOptions::new() + .append(true) + .open(bare.join("config")) + .unwrap(); + cfg.write_all(b"\n[broken section\nnot a valid = = = line\n") + .unwrap(); + } + assert!( + super::object_store_readable(&bare), + "config corruption must leave objects/ readable (that is the whole point: \ + a readable store + a git failure == deterministic, not transient)" + ); + + // Probing the PRESENT blob under the bad config must be a DETERMINISTIC fault, + // never Ok(None) (the old false 404) and never a Transient (retryable 503). + let res = super::object_type_bounded("git", &bare, &blob, deadline); + assert!( + matches!(res, Err(super::ProbeError::Deterministic(_))), + "a bad-config fatal on a readable store must be a terminal Deterministic \ + fault (-> 500), never Ok(None) (-> false 404) or Transient (-> 503); got {res:?}" + ); + // And a genuinely-absent oid under the bad config is ALSO not an absence verdict. + let res_absent = super::object_type_bounded("git", &bare, &"0".repeat(64), deadline); + assert!( + matches!(res_absent, Err(super::ProbeError::Deterministic(_))), + "even a would-be-absent oid must not read as Ok(None) once the config is \ + corrupt; got {res_absent:?}" + ); + } + + /// #174 F5/U4: a corrupt LOOSE object makes `git cat-file --batch-check` print + /// ` missing` on stdout (exit 0) yet emit `error:` diagnostics on stderr. The + /// clean-`missing` absence path must NOT fire here — the `error:` line disqualifies + /// a clean-absence read — so the probe surfaces a fault, not a false Ok(None) 404. + /// The object store is readable (a corrupt object file still opens), so this is a + /// Deterministic fault. LOAD-BEARING: drop the `has_error_diag` guard and a corrupt + /// object reads as `missing` -> Ok(None) -> false 404 (RED). + #[cfg(unix)] + #[test] + fn object_type_bounded_corrupt_loose_object_is_fault_not_absence() { + use std::os::unix::fs::PermissionsExt; + let td = tempfile::TempDir::new().unwrap(); + let work = td.path().join("loose"); + std::fs::create_dir_all(&work).unwrap(); + let g = |args: &[&str]| { + assert!( + Command::new("git") + .args(args) + .current_dir(&work) + .status() + .unwrap() + .success(), + "git {args:?}" + ); + }; + g(&["init", "-q", "--object-format=sha256", "."]); + g(&["config", "user.email", "t@t"]); + g(&["config", "user.name", "t"]); + std::fs::write(work.join("f.txt"), b"loose object content\n").unwrap(); + g(&["add", "f.txt"]); + g(&["commit", "-qm", "c1"]); + let blob = String::from_utf8( + Command::new("git") + .args(["rev-parse", "HEAD:f.txt"]) + .current_dir(&work) + .output() + .unwrap() + .stdout, + ) + .unwrap() + .trim() + .to_string(); + + // Overwrite the loose object file with non-zlib garbage (it is 0o444 by default). + let obj = work.join(".git/objects").join(&blob[0..2]).join(&blob[2..]); + let mut perms = std::fs::metadata(&obj).unwrap().permissions(); + perms.set_mode(0o644); + std::fs::set_permissions(&obj, perms).unwrap(); + std::fs::write(&obj, b"garbage not a zlib stream").unwrap(); + + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(10); + let res = super::object_type_bounded("git", &work, &blob, deadline); + assert!( + res.is_err(), + "a corrupt loose object (error: on stderr, `missing` on stdout) must be Err, \ + never a false Ok(None) 404; got {res:?}" + ); + } } diff --git a/crates/gitlawb-node/src/git/tigris.rs b/crates/gitlawb-node/src/git/tigris.rs index ad26ddc5..cf7abfd5 100644 --- a/crates/gitlawb-node/src/git/tigris.rs +++ b/crates/gitlawb-node/src/git/tigris.rs @@ -31,6 +31,25 @@ impl TigrisClient { }) } + /// Test-only constructor with an explicit S3 endpoint, region, and static + /// credentials — no env-var reads, so parallel tests cannot race each other's + /// `AWS_*` environment the way the env-based `new` would. Lets a test point + /// the client at a non-routable endpoint to exercise acquire-stall paths. + #[cfg(test)] + pub(crate) async fn for_testing_with_endpoint(bucket: &str, endpoint_url: &str) -> Self { + let creds = aws_sdk_s3::config::Credentials::new("test", "test", None, None, "test"); + let config = aws_config::defaults(aws_config::BehaviorVersion::latest()) + .endpoint_url(endpoint_url) + .region(aws_config::Region::new("auto")) + .credentials_provider(creds) + .load() + .await; + Self { + s3: S3Client::new(&config), + bucket: bucket.to_string(), + } + } + /// S3 key for a given repo: `repos/v1/{owner_slug}/{repo_name}.tar.zst` fn repo_key(owner_slug: &str, repo_name: &str) -> String { format!("repos/v1/{owner_slug}/{repo_name}.tar.zst") diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index cb70e39c..e69c4ceb 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -4,11 +4,321 @@ //! content is held back. use crate::db::VisibilityRule; -use crate::git::store; use crate::visibility::{visibility_check, Decision}; use anyhow::{Context, Result}; use std::collections::{BTreeSet, HashMap, HashSet}; use std::path::Path; +use std::process::Stdio; +use std::sync::mpsc; +use std::time::{Duration, Instant}; + +/// Fixed budget bounding the whole withheld-blob classification walk (#174 U3). +/// The walk is fast for a real repo; this bound exists to reap a hung or +/// pathologically slow git child so it cannot pin a served-git permit (the read +/// permit on the upload-pack serve path, the write permit on the receive-pack +/// post-push replication path) past the deadline. Every caller funnels through +/// `blob_paths`, so bounding here bounds both paths at one seam. Production callers +/// pass the operator-configured `GITLAWB_GIT_SERVICE_TIMEOUT_SECS` instead; this +/// fixed budget only backs the `git_bin`-less test wrappers. +#[cfg(test)] +const WALK_TIMEOUT: Duration = Duration::from_secs(600); + +/// How long the process-group watchdog waits after SIGTERM before escalating to +/// SIGKILL, giving a well-behaved git child time to clean up its `*.lock` files. Only +/// paid on a timeout (already the exceptional path). +#[cfg(unix)] +const WATCHDOG_TERM_GRACE: Duration = Duration::from_secs(1); + +/// Run one git child under a shared `deadline` with process-group teardown, +/// BLOCKING, and return its stdout. The child runs in its own process group; a +/// watchdog thread SIGTERMs (lets git clean up its `*.lock` files), then SIGKILLs, +/// the whole group if the deadline passes before the child is reaped, so a hung or +/// slow git can pin neither a served-git permit nor a blocking thread past the +/// deadline (jatmn's "retain admission until they are reaped"). This is the +/// blocking-side counterpart of `smart_http::drive_git_child`, needed because the +/// walk's callers run it inside `spawn_blocking`, which an async timeout cannot +/// cancel. Returns [`crate::git::smart_http::GitServiceTimeout`] on the deadline so +/// the serve handler maps it to 504. `git_bin` is injectable so a fake `git` can +/// drive the teardown in tests without mutating the process-global PATH; +/// `stdin_bytes` feeds children that read stdin (empty for the arg-only children). +/// Returns true if `pid` (a process-group leader we spawned) has terminated, WITHOUT +/// reaping it. `waitid(..., WNOWAIT)` reports the exit state but leaves the child +/// waitable, so the caller's later `child.wait()` still collects the status and the +/// pid/pgid stays live until then — which is what keeps the watchdog's `kill(-pgid)` +/// teardown from ever racing a recycled pgid. Used to distinguish "the child actually +/// exited" from "the child merely closed stdout" after the drain returns (#174 P1-a). +#[cfg(unix)] +fn child_terminated_without_reaping(pid: i32) -> bool { + // SAFETY: waitid writes only into the zeroed siginfo and borrows no Rust memory; + // WNOWAIT leaves the child unreaped, WNOHANG makes the probe non-blocking. + let mut info: libc::siginfo_t = unsafe { std::mem::zeroed() }; + let rc = unsafe { + libc::waitid( + libc::P_PID, + pid as libc::id_t, + &mut info, + libc::WEXITED | libc::WNOHANG | libc::WNOWAIT, + ) + }; + // rc == 0 with si_pid == 0 means "no state change yet" (still running); a non-zero + // si_pid means the child has entered a waitable, exited state. EINTR/other errors + // (rc != 0) are treated as "not yet terminated" and the caller re-polls. + rc == 0 && unsafe { info.si_pid() } != 0 +} + +#[cfg(unix)] +pub(crate) fn run_bounded_git_raw( + git_bin: &str, + args: &[&str], + repo_path: &Path, + stdin_bytes: &[u8], + deadline: Instant, +) -> Result<(std::process::ExitStatus, Vec, Vec)> { + use std::io::{Read, Write}; + use std::os::unix::process::CommandExt; + use std::sync::mpsc::RecvTimeoutError; + + let label = args.first().copied().unwrap_or("git"); + let mut child = std::process::Command::new(git_bin) + .args(args) + .current_dir(repo_path) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .process_group(0) + .spawn() + .with_context(|| format!("failed to spawn git {label}"))?; + // With process_group(0) the child leads its own group, so pgid == its pid. + let pgid = child.id() as i32; + + // Watchdog: on the deadline, tear the WHOLE process group down — SIGTERM, a grace + // for a well-behaved child to clean up its `*.lock` files, then an UNCONDITIONAL + // SIGKILL of the group. It never stands down on leader-reap alone: a group member + // that ignores SIGTERM while the leader exits cleanly would otherwise escape the + // SIGKILL and keep running past the deadline (finding 3, #174). The main thread + // defers reaping the leader until this thread returns (see below), so the leader's + // pid is still unreaped while every `kill(-pgid)` fires and the pgid cannot have + // been recycled — which is why this no longer needs the old `reaped` short-circuit. + // Kept off the main thread because the main thread's stdout drain is exactly what + // blocks until a hung child is torn down. + let (done_tx, done_rx) = mpsc::channel::<()>(); + let watchdog = std::thread::spawn(move || -> bool { + let wait = deadline.saturating_duration_since(Instant::now()); + match done_rx.recv_timeout(wait) { + Ok(()) | Err(RecvTimeoutError::Disconnected) => false, + Err(RecvTimeoutError::Timeout) => { + // SAFETY: kill(2) takes only integers and borrows no Rust memory; + // ESRCH on an already-gone group is ignored. + unsafe { libc::kill(-pgid, libc::SIGTERM) }; + // Fixed grace: because the main thread defers the leader's reap, a + // fully-exited group still shows a zombie leader here, so polling for + // ESRCH cannot detect early completion — just wait the grace, then + // SIGKILL. On a group of only zombies the SIGKILL is a harmless no-op; + // on a SIGTERM-ignoring member it is what actually kills it. + std::thread::sleep(WATCHDOG_TERM_GRACE); + unsafe { libc::kill(-pgid, libc::SIGKILL) }; + // Brief settle so the SIGKILL is delivered before the main thread + // reaps the leader and frees the pgid. A wedged (D-state) member + // survives even SIGKILL — the documented residual, as in smart_http. + std::thread::sleep(Duration::from_millis(20)); + if unsafe { libc::kill(-pgid, 0) } == 0 { + tracing::warn!( + pgid, + "withheld-walk git survived SIGKILL past the watchdog cap (uninterruptible I/O?)" + ); + } + true + } + } + }); + + // Feed stdin on a writer thread and drain stderr on a reader thread so the main + // thread can drain stdout concurrently; writing all of stdin (or draining one + // pipe) before the others can deadlock once a pipe buffer fills. + let mut stdin = child.stdin.take(); + let input = stdin_bytes.to_vec(); + let writer = std::thread::spawn(move || { + if let Some(mut s) = stdin.take() { + let _ = s.write_all(&input); + } + }); + let mut stderr = child.stderr.take().context("git stderr was not piped")?; + let err_reader = std::thread::spawn(move || { + let mut err = Vec::new(); + let _ = stderr.read_to_end(&mut err); + err + }); + let mut stdout = child.stdout.take().context("git stdout was not piped")?; + let mut out = Vec::new(); + // Blocking drain, unblocked by the child closing stdout on exit. The watchdog's + // SIGTERM/SIGKILL is what makes a hung child exit; a git wedged in uninterruptible + // (D-state) I/O survives even SIGKILL, so this drain and the wait below can block + // until the kernel returns, pinning the walk thread and its permit. That residual + // is unreachable in userspace (no signal reaps a D-state process) and matches the + // async `reap_group_on_timeout`, which likewise only warns and gives up there. + let read_result = stdout.read_to_end(&mut out); + // The drain has returned, but that only means all stdout write ends are closed — + // NOT that the child has exited. A group member, or the leader itself, can close + // stdout and keep running; standing the watchdog down on the drain alone (as the + // old code did) would then let `child.wait()` block forever on that live child, + // past the deadline, pinning the walk thread and its permit (finding P1-a, #174). + // So stand the watchdog down only once the child has ACTUALLY terminated, detected + // WITHOUT reaping (waitid + WNOWAIT) so the leader's pid stays unreaped and its + // pgid un-recycled until the watchdog finishes and we join it below. Past the + // deadline the watchdog owns the teardown, so we stop polling and let it run the + // full SIGTERM -> grace -> SIGKILL; joining it before `child.wait()` keeps every + // `kill(-pgid)` firing while the pid is still unreaped and guarantees a + // stdout-closing-then-hanging member has been SIGKILLed rather than left running. + loop { + if child_terminated_without_reaping(pgid) { + let _ = done_tx.send(()); + break; + } + if Instant::now() >= deadline { + break; + } + std::thread::sleep(Duration::from_millis(2)); + } + let killed = watchdog.join().unwrap_or(false); + let status = child.wait().context("git wait failed")?; + let err = err_reader.join().unwrap_or_default(); + let _ = writer.join(); + read_result.context("failed to read git stdout")?; + // The watchdog runs off a wall clock that can race a child finishing right at the + // deadline. A child that exited on its own (success) is not a timeout even if the + // watchdog fired late; only a child that did not exit successfully is a genuine + // timeout, which keeps a walk completing at its budget from a spurious 504. + if killed && !status.success() { + return Err(crate::git::smart_http::GitServiceTimeout.into()); + } + Ok((status, out, err)) +} + +/// Bounded git returning only stdout, `bail!`ing on any nonzero exit. The thin +/// wrapper the walk callers use. Probes that must distinguish exit classes — +/// `git cat-file` absence vs an object-store access failure — call +/// [`run_bounded_git_raw`] and classify the status/stderr themselves. +#[cfg(unix)] +pub(crate) fn run_bounded_git( + git_bin: &str, + args: &[&str], + repo_path: &Path, + stdin_bytes: &[u8], + deadline: Instant, +) -> Result> { + let label = args.first().copied().unwrap_or("git"); + let (status, out, err) = run_bounded_git_raw(git_bin, args, repo_path, stdin_bytes, deadline)?; + if !status.success() { + anyhow::bail!("git {label} failed: {}", String::from_utf8_lossy(&err)); + } + Ok(out) +} + +/// Non-Unix fallback for [`run_bounded_git`]. Windows and other non-Unix targets +/// have no process-group teardown (`process_group(0)` / `kill(-pgid)` are Unix-only), +/// so this bounds a single child on its own: threads feed stdin and drain stderr +/// while the main thread drains stdout, and a watchdog thread kills the child at the +/// deadline (which closes stdout and unblocks the drain). The child is shared with +/// the watchdog behind a mutex that the main thread does NOT hold while draining, so +/// the watchdog can always acquire it to kill. Best-effort — it reaps only the direct +/// child, not a descendant group — which is why the hardened, group-aware path above +/// is gated to Unix, the only target the served node actually runs on (the Windows +/// release binary is best-effort / `continue-on-error` in CI). Kept in lockstep with +/// the Unix version's signature and result semantics so every caller compiles on all +/// targets (#174). +#[cfg(not(unix))] +pub(crate) fn run_bounded_git_raw( + git_bin: &str, + args: &[&str], + repo_path: &Path, + stdin_bytes: &[u8], + deadline: Instant, +) -> Result<(std::process::ExitStatus, Vec, Vec)> { + use std::io::{Read, Write}; + use std::sync::mpsc::RecvTimeoutError; + + let label = args.first().copied().unwrap_or("git"); + let mut child = std::process::Command::new(git_bin) + .args(args) + .current_dir(repo_path) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .with_context(|| format!("failed to spawn git {label}"))?; + + let mut stdin = child.stdin.take(); + let input = stdin_bytes.to_vec(); + let writer = std::thread::spawn(move || { + if let Some(mut s) = stdin.take() { + let _ = s.write_all(&input); + } + }); + let mut stderr = child.stderr.take().context("git stderr was not piped")?; + let err_reader = std::thread::spawn(move || { + let mut err = Vec::new(); + let _ = stderr.read_to_end(&mut err); + err + }); + let mut stdout = child.stdout.take().context("git stdout was not piped")?; + + // Share the child with the watchdog. The main thread drains stdout WITHOUT + // holding this lock, so the watchdog can always acquire it to kill on timeout; + // killing closes stdout and unblocks the drain below. + let child = std::sync::Arc::new(std::sync::Mutex::new(child)); + let (done_tx, done_rx) = mpsc::channel::<()>(); + let watchdog = { + let child = child.clone(); + std::thread::spawn(move || -> bool { + let wait = deadline.saturating_duration_since(Instant::now()); + match done_rx.recv_timeout(wait) { + Ok(()) | Err(RecvTimeoutError::Disconnected) => false, + Err(RecvTimeoutError::Timeout) => { + if let Ok(mut c) = child.lock() { + let _ = c.kill(); + } + true + } + } + }) + }; + + let mut out = Vec::new(); + let read_result = stdout.read_to_end(&mut out); + // The drain has returned (child exited or was killed), so taking the lock here + // cannot deadlock against the watchdog. + let status = child + .lock() + .expect("git child mutex poisoned") + .wait() + .context("git wait failed")?; + let _ = done_tx.send(()); + let killed = watchdog.join().unwrap_or(false); + let err = err_reader.join().unwrap_or_default(); + let _ = writer.join(); + read_result.context("failed to read git stdout")?; + if killed && !status.success() { + return Err(crate::git::smart_http::GitServiceTimeout.into()); + } + Ok((status, out, err)) +} + +/// Non-Unix thin wrapper matching the Unix [`run_bounded_git`] semantics. +#[cfg(not(unix))] +pub(crate) fn run_bounded_git( + git_bin: &str, + args: &[&str], + repo_path: &Path, + stdin_bytes: &[u8], + deadline: Instant, +) -> Result> { + let label = args.first().copied().unwrap_or("git"); + let (status, out, err) = run_bounded_git_raw(git_bin, args, repo_path, stdin_bytes, deadline)?; + if !status.success() { + anyhow::bail!("git {label} failed: {}", String::from_utf8_lossy(&err)); + } + Ok(out) +} /// Fail closed unless every ref ultimately resolves to a commit (a ref pointing /// directly at a blob or tree, or an annotated tag — even a nested one — of such @@ -21,19 +331,15 @@ use std::path::Path; /// Full peeling is why this is not `for-each-ref %(*objecttype)`, which /// dereferences only one tag level and so misclassifies a tag-of-a-tag-of-a- /// commit as a non-commit. -fn assert_all_refs_are_commits(repo_path: &Path) -> Result<()> { - let refs = std::process::Command::new("git") - .args(["for-each-ref", "--format=%(refname)"]) - .current_dir(repo_path) - .output() - .context("git for-each-ref failed")?; - if !refs.status.success() { - anyhow::bail!( - "git for-each-ref failed: {}", - String::from_utf8_lossy(&refs.stderr) - ); - } - let refs_stdout = String::from_utf8_lossy(&refs.stdout); +fn assert_all_refs_are_commits(repo_path: &Path, git_bin: &str, deadline: Instant) -> Result<()> { + let refs_out = run_bounded_git( + git_bin, + &["for-each-ref", "--format=%(refname)"], + repo_path, + b"", + deadline, + )?; + let refs_stdout = String::from_utf8_lossy(&refs_out); let refnames: Vec<&str> = refs_stdout .lines() .map(str::trim) @@ -43,59 +349,25 @@ fn assert_all_refs_are_commits(repo_path: &Path) -> Result<()> { return Ok(()); } - // Peel every ref in one `git cat-file --batch-check` pass: one - // `^{}` query per line, one output line per input line, in order. - // The stdin write runs on a separate thread so this thread can drain stdout - // concurrently. cat-file echoes the full query on a ` missing` line, - // so output scales with refname length (not a fixed size per ref); writing - // all of stdin before reading any stdout would deadlock both pipes once the - // child's stdout buffer fills. Dropping `stdin` at the end of the closure - // sends EOF. + // Peel every ref in one `git cat-file --batch-check` pass: one `^{}` + // query per line, one output line per input line, in order. cat-file echoes the + // full query on a ` missing` line, so output scales with refname length; + // run_bounded_git drains stdout concurrently with the stdin write, so the pipe + // cannot deadlock, and the whole peel is bounded by the shared walk deadline. let queries = refnames .iter() .map(|r| format!("{r}^{{}}")) .collect::>() .join("\n"); - use std::io::Write; - let mut child = std::process::Command::new("git") - .args(["cat-file", "--batch-check=%(objecttype)"]) - .current_dir(repo_path) - .stdin(std::process::Stdio::piped()) - .stdout(std::process::Stdio::piped()) - .stderr(std::process::Stdio::piped()) - .spawn() - .context("failed to spawn git cat-file")?; - // Feed stdin on a writer thread so this thread can drain stdout via - // wait_with_output concurrently; a None handle (the pipe vanished) becomes a - // broken-pipe write error. wait_with_output reaps the child unconditionally - // before any error is surfaced, so no path drops it unwaited (#53), and the - // writer is joined only after the drain so the join cannot deadlock. - let writer = child - .stdin - .take() - .map(|mut stdin| std::thread::spawn(move || stdin.write_all(queries.as_bytes()))); - let peel_result = child.wait_with_output(); - let write_result = match writer { - Some(handle) => handle - .join() - .map_err(|_| anyhow::anyhow!("git cat-file stdin writer thread panicked"))?, - None => Err(std::io::Error::new( - std::io::ErrorKind::BrokenPipe, - "git cat-file stdin unavailable", - )), - }; - // Surface a write error only if the process didn't already fail with a - // clearer status. - let peel = peel_result.context("git cat-file failed")?; - if !peel.status.success() { - anyhow::bail!( - "git cat-file --batch-check failed: {}", - String::from_utf8_lossy(&peel.stderr) - ); - } - write_result.context("failed to write to git cat-file stdin")?; - - let peel_stdout = String::from_utf8_lossy(&peel.stdout); + let peel_out = run_bounded_git( + git_bin, + &["cat-file", "--batch-check=%(objecttype)"], + repo_path, + queries.as_bytes(), + deadline, + )?; + + let peel_stdout = String::from_utf8_lossy(&peel_out); let types: Vec<&str> = peel_stdout.lines().map(str::trim).collect(); // A short read means at least one ref went unclassified — fail closed. if types.len() != refnames.len() { @@ -147,47 +419,46 @@ fn assert_all_refs_are_commits(repo_path: &Path) -> Result<()> { /// Fails closed: if commit enumeration or any tree walk fails, returns an error so /// the caller aborts the serve/pin rather than producing a partial (under-withheld) /// set. -fn blob_paths(repo_path: &Path) -> Result> { - assert_all_refs_are_commits(repo_path)?; +fn blob_paths(repo_path: &Path, git_bin: &str, timeout: Duration) -> Result> { + // One deadline spans the whole walk (the ref check, the HEAD probe, rev-list, + // and every per-commit ls-tree), so a slow or hung walk is bounded as a unit + // rather than granting each git child a fresh timeout. + let deadline = Instant::now() + timeout; + assert_all_refs_are_commits(repo_path, git_bin, deadline)?; // Enumerate every reachable commit, not just ref tips. `--all` walks all refs; // append HEAD so a detached HEAD (reachable by rev-list/upload-pack but in no // ref) is still classified. When HEAD does not resolve (unborn branch on an - // empty repo) `--all` alone yields nothing, which is correct — no objects exist. - let head = store::head_commit(repo_path).context("resolve HEAD failed")?; + // empty repo) `--all` alone yields nothing, which is correct: no objects exist. + // The HEAD probe is a bounded `git rev-parse --verify HEAD` (a clean exit means + // HEAD resolves), replacing the previously unbounded `store::head_commit` child. + let head_resolves = run_bounded_git( + git_bin, + &["rev-parse", "--verify", "HEAD"], + repo_path, + b"", + deadline, + ) + .is_ok(); let mut rev_args = vec!["rev-list", "--all"]; - if head.is_some() { + if head_resolves { rev_args.push("HEAD"); } - let commits = std::process::Command::new("git") - .args(&rev_args) - .current_dir(repo_path) - .output() - .context("git rev-list --all failed")?; - if !commits.status.success() { - anyhow::bail!( - "git rev-list --all failed: {}", - String::from_utf8_lossy(&commits.stderr) - ); - } - let commits_stdout = String::from_utf8_lossy(&commits.stdout); + let commits_out = run_bounded_git(git_bin, &rev_args, repo_path, b"", deadline)?; + let commits_stdout = String::from_utf8_lossy(&commits_out); let mut out: HashSet<(String, String)> = HashSet::new(); for commit in commits_stdout.lines() { let commit = commit.trim(); if commit.is_empty() { continue; } - let listing = std::process::Command::new("git") - .args(["ls-tree", "-rz", commit]) - .current_dir(repo_path) - .output() - .context("git ls-tree -rz failed")?; - if !listing.status.success() { - anyhow::bail!( - "git ls-tree -rz {commit} failed: {}", - String::from_utf8_lossy(&listing.stderr) - ); - } + let listing_out = run_bounded_git( + git_bin, + &["ls-tree", "-rz", commit], + repo_path, + b"", + deadline, + )?; // `-z` NUL-delimits records and emits paths raw; plain `git ls-tree -r` // C-quotes any path with non-ASCII or special bytes (e.g. café.txt becomes // "secret/caf\303\251.txt"), and that quoted literal would not match a @@ -198,7 +469,7 @@ fn blob_paths(repo_path: &Path) -> Result> { // path (e.g. a non-UTF-8 directory name) with U+FFFD, and the mangled string // would no longer match its deny rule — the same under-withholding class, one // layer down. Fail closed instead so the caller aborts rather than leaks. - let Ok(listing_stdout) = std::str::from_utf8(&listing.stdout) else { + let Ok(listing_stdout) = std::str::from_utf8(&listing_out) else { anyhow::bail!( "git ls-tree -rz {commit} returned a non-UTF-8 path; \ refusing to produce a partial (under-withheld) set" @@ -229,6 +500,7 @@ fn blob_paths(repo_path: &Path) -> Result> { /// /// The whole-repo "/" gate is handled by the caller before this function runs: /// if "/" denies, the caller gets a 404 and never reaches the filtered serve. +#[cfg(test)] pub fn withheld_blob_oids( repo_path: &Path, rules: &[VisibilityRule], @@ -236,7 +508,33 @@ pub fn withheld_blob_oids( owner_did: &str, caller: Option<&str>, ) -> Result> { - let pairs = blob_paths(repo_path)?; + withheld_blob_oids_bounded( + repo_path, + "git", + WALK_TIMEOUT, + rules, + is_public, + owner_did, + caller, + ) +} + +/// [`withheld_blob_oids`] with an injectable `git_bin` and walk `timeout`. Served +/// handlers call this with the operator-configured git binary and +/// `GITLAWB_GIT_SERVICE_TIMEOUT_SECS`, so the whole walk is bounded by the same +/// budget as the other served-git ops and a fake `git` can drive its teardown in +/// tests. The `git_bin`-less wrapper above keeps the fixed [`WALK_TIMEOUT`] for the +/// classification tests that run against real git. +pub fn withheld_blob_oids_bounded( + repo_path: &Path, + git_bin: &str, + timeout: Duration, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> Result> { + let pairs = blob_paths(repo_path, git_bin, timeout)?; Ok(withheld_from_pairs( &pairs, rules, is_public, owner_did, caller, )) @@ -304,6 +602,7 @@ pub fn replicable_objects(all: Vec, withheld: &HashSet) -> Vec Result> { + allowed_blob_set_for_caller_bounded( + repo_path, git_bin, timeout, rules, is_public, owner_did, None, + ) +} + /// Reachable blob OIDs that visibility ALLOWS `caller` at some path. The /// caller-aware generalization of `replicable_blob_set` (which is the anonymous /// `caller = None` case). Used by `GET /ipfs/{cid}` to gate fail-closed against @@ -325,6 +639,7 @@ pub fn replicable_blob_set( /// elsewhere (its content is readable to this caller elsewhere). Trees and /// commits are NOT included here; the caller decides per object type whether /// the allow-set applies (it does not for trees/commits — KTD3). +#[cfg(test)] pub fn allowed_blob_set_for_caller( repo_path: &Path, rules: &[VisibilityRule], @@ -332,7 +647,29 @@ pub fn allowed_blob_set_for_caller( owner_did: &str, caller: Option<&str>, ) -> Result> { - let pairs = blob_paths(repo_path)?; + allowed_blob_set_for_caller_bounded( + repo_path, + "git", + WALK_TIMEOUT, + rules, + is_public, + owner_did, + caller, + ) +} + +/// [`allowed_blob_set_for_caller`] with an injectable `git_bin` and walk `timeout`, +/// for the `GET /ipfs/{cid}` gate. +pub fn allowed_blob_set_for_caller_bounded( + repo_path: &Path, + git_bin: &str, + timeout: Duration, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, + caller: Option<&str>, +) -> Result> { + let pairs = blob_paths(repo_path, git_bin, timeout)?; let mut allowed = HashSet::new(); for (oid, path) in &pairs { if visibility_check(rules, is_public, owner_did, caller, path) == Decision::Allow { @@ -365,14 +702,28 @@ pub fn replicable_objects_fail_closed( /// owner plus any reader DID that `visibility_check` Allows at some path the /// blob appears at. Least-privilege: a reader of one private subtree is not a /// recipient of a blob that only lives in another. +#[cfg(test)] pub fn withheld_blob_recipients( repo_path: &Path, rules: &[VisibilityRule], is_public: bool, owner_did: &str, +) -> Result>> { + withheld_blob_recipients_bounded(repo_path, "git", WALK_TIMEOUT, rules, is_public, owner_did) +} + +/// [`withheld_blob_recipients`] with an injectable `git_bin` and walk `timeout`, for +/// the receive-pack encrypt-then-pin path. +pub fn withheld_blob_recipients_bounded( + repo_path: &Path, + git_bin: &str, + timeout: Duration, + rules: &[VisibilityRule], + is_public: bool, + owner_did: &str, ) -> Result>> { // One history walk feeds both the withheld set and the recipient mapping. - let pairs = blob_paths(repo_path)?; + let pairs = blob_paths(repo_path, git_bin, timeout)?; let withheld = withheld_from_pairs(&pairs, rules, is_public, owner_did, None); if withheld.is_empty() { return Ok(HashMap::new()); @@ -402,6 +753,286 @@ pub fn withheld_blob_recipients( #[cfg(test)] mod tests { use super::*; + + /// Write an executable fake `git` shell script into `dir` and return its path, + /// so a test can drive the walk's process-group teardown without a real git and + /// without mutating the process-global PATH (the crate's only injection seam). + #[cfg(unix)] + fn write_fake_git(dir: &Path, body: &str) -> String { + use std::os::unix::fs::PermissionsExt; + let p = dir.join("fakegit"); + std::fs::write(&p, body).unwrap(); + let mut perm = std::fs::metadata(&p).unwrap().permissions(); + perm.set_mode(0o755); + std::fs::set_permissions(&p, perm).unwrap(); + p.to_str().unwrap().to_string() + } + + /// #174 U3: the withheld-blob walk is bounded at the shared `blob_paths` seam, so + /// a hung git child cannot pin the caller's permit past the deadline. A fake git + /// that hangs on `rev-list` must make `blob_paths` return `GitServiceTimeout` + /// within the watchdog budget (not block for the child's lifetime), and the + /// child's process group must be reaped (its recorded leader PID gone). Every + /// caller (upload-pack serve, receive-pack replication) funnels through + /// `blob_paths`, so this seam-level proof covers both permit pools. Neutralize + /// the watchdog SIGTERM and this hangs past the recv budget (RED). + #[cfg(unix)] + #[test] + fn blob_paths_times_out_and_reaps_a_hung_walk() { + use std::time::Duration; + let tmp = TempDir::new().unwrap(); + // Fast on every stage except rev-list, which records its own (group-leader) + // PID and then hangs. `sleep 30` bounds the worst case if the watchdog is + // ever broken, so a regression cannot wedge the suite for 300s. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) echo $$ > revlist.pid ; sleep 30 ;;\n rev-parse) echo deadbeef ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + + // Run the walk on a thread with a short budget; the recv_timeout succeeding + // is itself proof the walk did not block on the hung child. + let (tx, rx) = mpsc::channel(); + let path = tmp.path().to_path_buf(); + std::thread::spawn(move || { + let _ = tx.send(blob_paths(&path, &git_bin, Duration::from_millis(200))); + }); + let result = rx.recv_timeout(Duration::from_secs(10)).expect( + "blob_paths must return within the watchdog budget, not hang on a stuck git child", + ); + let err = result.expect_err("a hung rev-list must abort the walk with an error"); + assert!( + err.downcast_ref::() + .is_some(), + "a hung walk must abort with GitServiceTimeout (mapped to 504), got: {err}" + ); + + // The recorded process-group leader must be gone: the watchdog reaps the + // whole group before blob_paths returns, so no orphaned git lingers. + let pid: i32 = std::fs::read_to_string(tmp.path().join("revlist.pid")) + .expect("the fake git must have recorded its rev-list PID") + .trim() + .parse() + .expect("recorded PID must parse"); + let mut gone = false; + for _ in 0..200 { + // SAFETY: kill(2) with signal 0 only probes existence; ESRCH (-1) means + // the process is gone. Borrows no Rust memory. + if unsafe { libc::kill(pid, 0) } != 0 { + gone = true; + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + assert!( + gone, + "the hung git child (pid {pid}) must be reaped, not orphaned, after the walk aborts" + ); + } + + /// #174 (F1 status-gate, vetted by execution): a child that exits SUCCESSFULLY is + /// never reported as a timeout even when the watchdog fires, so a walk finishing + /// right at its deadline is not a spurious 504. The fake only exits when signalled + /// and exits 0 on SIGTERM, so with a deadline already elapsed the watchdog always + /// reaches its kill path (killed == true) yet the child's status is success. + /// Drop the `!status.success()` guard and this returns GitServiceTimeout (RED). + #[cfg(unix)] + #[test] + fn run_bounded_git_success_at_the_deadline_is_not_a_timeout() { + use std::time::{Duration, Instant}; + let tmp = TempDir::new().unwrap(); + let body = "#!/bin/sh\ntrap 'exit 0' TERM\nsleep 30 &\nwait\n"; + let git_bin = write_fake_git(tmp.path(), body); + let out = run_bounded_git( + &git_bin, + &["rev-list"], + tmp.path(), + b"", + Instant::now() + Duration::from_millis(100), + ); + assert!( + out.is_ok(), + "a child that exited successfully must not be reported as a timeout even if the watchdog fired: {out:?}" + ); + } + + /// #174 (F3, vetted by execution): a child that IGNORES SIGTERM is still reaped + /// via the watchdog's SIGKILL escalation, so it cannot pin the walk thread or its + /// permit. The fake traps SIGTERM and keeps sleeping; run_bounded_git must still + /// return (via SIGKILL at the grace step) with a timeout error and the group must + /// be gone. (A truly uninterruptible D-state child, which no signal can reap, is + /// the documented residual this teardown, like the async twin, cannot cover.) + #[cfg(unix)] + #[test] + fn run_bounded_git_reaps_a_sigterm_ignoring_child_via_sigkill() { + use std::time::{Duration, Instant}; + let tmp = TempDir::new().unwrap(); + let body = "#!/bin/sh\ntrap '' TERM\necho $$ > pid\nwhile true; do sleep 1; done\n"; + let git_bin = write_fake_git(tmp.path(), body); + let (tx, rx) = std::sync::mpsc::channel(); + let path = tmp.path().to_path_buf(); + std::thread::spawn(move || { + let _ = tx.send(run_bounded_git( + &git_bin, + &["rev-list"], + &path, + b"", + Instant::now() + Duration::from_millis(100), + )); + }); + let out = rx + .recv_timeout(Duration::from_secs(10)) + .expect("run_bounded_git must return via SIGKILL even for a SIGTERM-ignoring child"); + assert!( + out.is_err(), + "a SIGTERM-ignoring child killed by SIGKILL is a timeout, not a success: {out:?}" + ); + let pid: i32 = std::fs::read_to_string(tmp.path().join("pid")) + .unwrap() + .trim() + .parse() + .unwrap(); + let mut gone = false; + for _ in 0..300 { + if unsafe { libc::kill(pid, 0) } != 0 { + gone = true; + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + assert!( + gone, + "the SIGTERM-ignoring child (pid {pid}) must be reaped via SIGKILL, not left running" + ); + } + + /// #174 finding 3 (jatmn/CodeRabbit): a group MEMBER that ignores SIGTERM must + /// still be SIGKILLed even when the group LEADER exits cleanly on SIGTERM. The + /// leader traps SIGTERM to exit 0, but first spawns a descendant (`sh -c`, so its + /// `$$` is its OWN pid — a `( )` subshell's `$$` is the parent's) that ignores + /// SIGTERM and closes its inherited stdout/stderr. When the watchdog SIGTERMs the + /// group, the leader exits, its stdout closes, the main drain unblocks, and the + /// leader is reaped — the exact window a `reaped`-gated watchdog stands down in, + /// before escalating to SIGKILL. The descendant must be dead when run_bounded_git + /// returns; a teardown that stands down on leader-reap leaves it running (RED). + #[cfg(unix)] + #[test] + fn run_bounded_git_sigkills_a_sigterm_ignoring_descendant_after_leader_exits() { + use std::time::{Duration, Instant}; + let tmp = TempDir::new().unwrap(); + // Both loops are bounded (~30s) so a broken teardown cannot leak a permanent + // orphan or wedge the suite; the assertion fires well before then. + let body = "#!/bin/sh\n\ +case \"$1\" in\n\ + rev-list)\n\ + sh -c 'trap \"\" TERM; echo $$ > desc.pid; exec 1>&- 2>&-; i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done' &\n\ + trap 'exit 0' TERM\n\ + i=0; while [ $i -lt 30 ]; do sleep 1; i=$((i+1)); done ;;\n\ + *) : ;;\n\ +esac\n"; + let git_bin = write_fake_git(tmp.path(), body); + let (tx, rx) = std::sync::mpsc::channel(); + let path = tmp.path().to_path_buf(); + std::thread::spawn(move || { + let _ = tx.send(run_bounded_git( + &git_bin, + &["rev-list"], + &path, + b"", + Instant::now() + Duration::from_millis(100), + )); + }); + let _ = rx + .recv_timeout(Duration::from_secs(10)) + .expect("run_bounded_git must return within the watchdog budget"); + + // Wait for the descendant to record its OWN pid, then assert it is gone. + let desc_pid_path = tmp.path().join("desc.pid"); + let mut desc: Option = None; + for _ in 0..200 { + if let Some(p) = std::fs::read_to_string(&desc_pid_path) + .ok() + .and_then(|s| s.trim().parse::().ok()) + { + desc = Some(p); + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + let desc = desc.expect("the fake leader must have spawned and recorded a descendant"); + let mut gone = false; + for _ in 0..300 { + if unsafe { libc::kill(desc, 0) } != 0 { + gone = true; + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + // Kill it regardless so a RED run leaks no orphan. + unsafe { libc::kill(desc, libc::SIGKILL) }; + assert!( + gone, + "a SIGTERM-ignoring descendant (pid {desc}) must be SIGKILLed even after the leader exits cleanly, not orphaned" + ); + } + + /// #174 U1 (P1-a, RED-before/GREEN-after): the group LEADER closes its own + /// stdout/stderr BEFORE the deadline and then keeps running. On the pre-fix code + /// the stdout drain returns EOF early, `done_tx.send` stands the watchdog down + /// before it ever fires (`recv` gets `Ok` -> `false`, no kill), and `child.wait()` + /// then blocks on the still-alive leader — pinning the walk thread and its read/ + /// write permit past the deadline, bypassing GITLAWB_GIT_SERVICE_TIMEOUT_SECS. + /// This is distinct from the descendant case above: there the leader sleeps until + /// the deadline so the watchdog DOES time out; here the drain-EOF races ahead of + /// the deadline. The fix keeps the watchdog armed until the child is actually + /// reaped, so the deadline SIGTERM still fires and the call returns within budget. + /// A pre-fix build blocks on `child.wait()` past the recv budget (RED). + #[cfg(unix)] + #[test] + fn run_bounded_git_reaps_a_leader_that_closes_stdout_then_hangs() { + use std::time::{Duration, Instant}; + let tmp = TempDir::new().unwrap(); + // rev-list records its (leader) pid, closes stdout+stderr so the drain EOFs + // immediately, then sleeps without trapping TERM. `sleep 30` bounds the worst + // case so a RED run cannot wedge the suite; the recv budget fires first. + let body = "#!/bin/sh\ncase \"$1\" in\n rev-list) echo $$ > leader.pid; exec 1>&- 2>&-; sleep 30 ;;\n *) : ;;\nesac\nexit 0\n"; + let git_bin = write_fake_git(tmp.path(), body); + let (tx, rx) = std::sync::mpsc::channel(); + let path = tmp.path().to_path_buf(); + std::thread::spawn(move || { + let _ = tx.send(run_bounded_git( + &git_bin, + &["rev-list"], + &path, + b"", + Instant::now() + Duration::from_millis(100), + )); + }); + let out = rx.recv_timeout(Duration::from_secs(10)).expect( + "run_bounded_git must return within the watchdog budget when the leader closes stdout then hangs, not block on child.wait()", + ); + assert!( + out.is_err(), + "a leader killed at the deadline (no TERM trap) is a timeout, not a success: {out:?}" + ); + let pid: i32 = std::fs::read_to_string(tmp.path().join("leader.pid")) + .unwrap() + .trim() + .parse() + .unwrap(); + let mut gone = false; + for _ in 0..300 { + if unsafe { libc::kill(pid, 0) } != 0 { + gone = true; + break; + } + std::thread::sleep(Duration::from_millis(10)); + } + // Kill it regardless so a RED run leaks no orphan. + unsafe { libc::kill(pid, libc::SIGKILL) }; + assert!( + gone, + "the hung leader (pid {pid}) must be killed and reaped at the deadline, not left running" + ); + } + use crate::db::VisibilityMode; use chrono::Utc; use std::process::Command; @@ -735,7 +1366,12 @@ mod tests { String::from_utf8_lossy(&out.stdout).trim().to_string() }; - let all_blobs = crate::git::push_delta::all_blob_oids(&work).unwrap(); + let all_blobs = crate::git::push_delta::all_blob_oids( + &work, + "git", + std::time::Instant::now() + std::time::Duration::from_secs(600), + ) + .unwrap(); assert!( all_blobs.contains(&dangling_oid), "precondition: the dangling blob is in the all-objects universe" diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index aa0483db..d6c0df46 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -74,6 +74,13 @@ async fn main() -> Result<()> { // bootstrap peers. Operators can opt out via GITLAWB_BOOTSTRAP_DISABLE_SEEDS. bootstrap::merge_seeds(&mut config); + // Fail fast on config combinations that are individually in-range but jointly + // unsafe — notably a DB pool too small for the concurrent-write cap, which + // would let a push burst starve every other DB path (#174 F1). + config + .validate() + .map_err(|e| anyhow::anyhow!("invalid configuration: {e}"))?; + if !config.public_read { warn!( "GITLAWB_PUBLIC_READ=false is reserved; per-repository private-read enforcement is not wired in alpha" @@ -378,7 +385,72 @@ async fn main() -> Result<()> { sync_trigger_rate_limiter, peer_write_rate_limiter, shutdown_tx: shutdown_tx.clone(), + git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(config.max_concurrent_git_ops)), + git_write_semaphore: Arc::new(tokio::sync::Semaphore::new( + config.max_concurrent_git_pushes, + )), + // Anon receive-pack advertisements get their OWN pool, same size as the + // write pool but disjoint, so filling it (which takes many source IPs, each + // capped by git_push_advert_per_caller) never occupies a permit the + // authenticated POST needs (#174). + git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new( + config.max_concurrent_git_pushes, + )), + // Bounds concurrent detached post-push encryption walks, sized from the push + // pool (no separate knob — Q1): completed pushes cannot outnumber active + // encryption walks past this (#174 P1-e). + git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new( + config.max_concurrent_git_pushes, + )), + // Bounds concurrent post-push pin loops (their MB-scale object-id lists) across + // all repos (#174 F6), independent of the per-repo encrypt-task coalescing below. + pin_semaphore: Arc::new(tokio::sync::Semaphore::new(config.max_concurrent_pin_tasks)), + // Coalesces the DETACHED post-push encryption tasks per repo so a rapid pusher + // cannot grow the outstanding parked-waiter set past one task per repo (#174 + // P2-2). No knob: it is a natural cap (one entry per distinct repo), not a + // sized pool. + encrypt_inflight: crate::state::EncryptInflight::new(), + // Per-repo in-process write-lease serializer (#174 U2/F3): supplements the pg + // advisory lock so a disconnected push's still-reaping git group can't be raced + // by a second same-node push. Natural cap (one entry per contended repo, freed + // when unreferenced), no sized knob. + repo_write_leases: crate::state::RepoWriteLeases::new(), + git_read_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( + config.max_concurrent_reads_per_caller, + ), + // Per-source cap on the receive-pack advertisement, sized to an eighth of the + // write pool (min 1): a single source can hold at most this many write-pool + // slots via the anon advertisement, so saturating the pool takes ~8 distinct + // source IPs, each also rate-limited (#174). + git_push_advert_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( + (config.max_concurrent_git_pushes / 8).max(1), + ), + // Per-source cap on the authenticated receive-pack POST, sized like the advert + // cap: one source IP can hold at most this many write-pool slots, so + // monopolizing the pool takes ~8 distinct source IPs, each also rate-limited + // (#174 P1-d). + git_write_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( + (config.max_concurrent_git_pushes / 8).max(1), + ), + // Bounds concurrent /ipfs visibility walks — a distinct public cost center, so + // its own pool + per-source sub-cap + per-IP rate limiter, never a git pool + // (#174 P1-3). The per-source map is bounded (reject-before-insert, INV-15). + git_ipfs_walk_semaphore: Arc::new(tokio::sync::Semaphore::new( + config.max_concurrent_ipfs_walks, + )), + git_ipfs_walk_per_caller: rate_limit::PerCallerConcurrency::with_default_max_keys( + config.ipfs_walk_per_source, + ), + ipfs_rate_limiter: rate_limit::RateLimiter::new_bounded( + config.ipfs_rate_limit, + std::time::Duration::from_secs(3600), + 200_000, + ), + git_bin: "git".to_string(), }; + if config.ipfs_rate_limit == 0 { + tracing::warn!("GITLAWB_IPFS_RATE_LIMIT=0 — per-IP /ipfs rate limiting disabled"); + } // Periodic peer-count poll for the metrics gauge. If p2p is disabled // we still set the gauge to 0 so dashboards don't show "no data". diff --git a/crates/gitlawb-node/src/rate_limit.rs b/crates/gitlawb-node/src/rate_limit.rs index d40e2691..22281691 100644 --- a/crates/gitlawb-node/src/rate_limit.rs +++ b/crates/gitlawb-node/src/rate_limit.rs @@ -132,6 +132,96 @@ impl RateLimiter { } } +/// A bounded per-caller CONCURRENCY limiter — distinct from [`RateLimiter`], which +/// caps request RATE. Each caller key may hold at most `per_caller` in-flight +/// permits at once; beyond that [`try_acquire`](Self::try_acquire) returns `None` +/// and the caller sheds. Used to stop one caller (a single anonymous source-IP or +/// DID) monopolizing the served-git read pool (#174). +/// +/// The key map is self-bounding: a key is removed the instant its in-flight count +/// reaches zero, so it never holds more keys than there are concurrently-active +/// callers (itself bounded by the read semaphore). A `max_keys` reject-before-insert +/// backstop guarantees a key farm can never grow the map even if that invariant +/// weakened — a NEW key at the cap is rejected WITHOUT allocating an entry (INV-15). +/// +/// Uses a `std::sync::Mutex` (not the file's `tokio::sync::Mutex`) because the +/// permit's `Drop` must release synchronously; the critical section holds no await. +#[derive(Clone)] +pub struct PerCallerConcurrency { + state: Arc>>, + per_caller: usize, + max_keys: usize, +} + +/// RAII permit from [`PerCallerConcurrency::try_acquire`]. On drop it decrements +/// the caller's in-flight count and removes the key when it reaches zero. +pub struct PerCallerPermit { + state: Arc>>, + key: String, +} + +impl PerCallerConcurrency { + pub fn new(per_caller: usize, max_keys: usize) -> Self { + Self { + state: Arc::new(std::sync::Mutex::new(HashMap::new())), + per_caller: per_caller.max(1), + max_keys: max_keys.max(1), + } + } + + /// Convenience constructor with the default key bound. + pub fn with_default_max_keys(per_caller: usize) -> Self { + Self::new(per_caller, DEFAULT_MAX_KEYS) + } + + /// `Some(permit)` when the caller is under its cap and the map has room; + /// `None` (shed) otherwise. Reject-before-insert: a new key at `max_keys` is + /// rejected without allocating. + pub fn try_acquire(&self, key: &str) -> Option { + // Recover from a poisoned lock rather than panicking: the critical section + // is pure counter arithmetic and cannot itself panic, so a poisoned mutex + // would only ever come from an unrelated abort, and a slightly-off count + // self-heals as permits drop. A panic here would instead brick the limiter + // for every caller (each subsequent lock re-panics). + let mut map = self.state.lock().unwrap_or_else(|e| e.into_inner()); + match map.get_mut(key) { + Some(count) => { + if *count >= self.per_caller { + return None; + } + *count += 1; + } + None => { + if map.len() >= self.max_keys { + return None; + } + map.insert(key.to_string(), 1); + } + } + Some(PerCallerPermit { + state: self.state.clone(), + key: key.to_string(), + }) + } + + #[cfg(test)] + pub fn tracked_keys(&self) -> usize { + self.state.lock().unwrap().len() + } +} + +impl Drop for PerCallerPermit { + fn drop(&mut self) { + let mut map = self.state.lock().unwrap_or_else(|e| e.into_inner()); + if let Some(count) = map.get_mut(&self.key) { + *count -= 1; + if *count == 0 { + map.remove(&self.key); + } + } + } +} + pub async fn rate_limit_by_did(request: Request, next: Next) -> Response { let limiter = request.extensions().get::().cloned(); @@ -288,6 +378,63 @@ pub async fn rate_limit_by_ip(request: Request, next: Next) -> Response { mod tests { use super::*; + #[test] + fn per_caller_concurrency_caps_one_caller_and_frees_on_drop() { + let lim = PerCallerConcurrency::new(2, 100); + let p1 = lim.try_acquire("did:key:zA").expect("first under cap"); + let p2 = lim.try_acquire("did:key:zA").expect("second under cap"); + assert!( + lim.try_acquire("did:key:zA").is_none(), + "a third in-flight op for the same caller sheds (over the per-caller cap)" + ); + // A DIFFERENT caller is unaffected — the cap is per-caller, not global. + let _other = lim + .try_acquire("did:key:zB") + .expect("a different caller has its own budget"); + drop(p1); + assert!( + lim.try_acquire("did:key:zA").is_some(), + "freeing one in-flight slot lets the same caller back in" + ); + drop(p2); + } + + #[test] + fn per_caller_concurrency_map_is_self_bounding_and_reject_before_insert() { + // Self-bounding: acquire+drop many distinct keys — the map never grows + // because a key is removed the instant its in-flight count hits zero. + let lim = PerCallerConcurrency::new(4, 3); + for i in 0..50 { + let _p = lim.try_acquire(&format!("k{i}")); + } + assert_eq!( + lim.tracked_keys(), + 0, + "keys with zero in-flight ops are removed, so an acquire+drop flood leaves the map empty" + ); + // Reject-before-insert: HOLD max_keys distinct keys, then a new key sheds + // WITHOUT growing the map past the cap (INV-15 — a rejected request never + // allocates an entry). + let held: Vec<_> = (0..3) + .map(|i| lim.try_acquire(&format!("h{i}")).unwrap()) + .collect(); + assert_eq!( + lim.tracked_keys(), + 3, + "three distinct callers held concurrently" + ); + assert!( + lim.try_acquire("h3").is_none(), + "a new key at max_keys is rejected" + ); + assert_eq!( + lim.tracked_keys(), + 3, + "the rejected new key did not allocate an entry (reject-before-insert)" + ); + drop(held); + } + #[tokio::test] async fn allows_within_limit() { let limiter = RateLimiter::new(3, Duration::from_secs(60)); diff --git a/crates/gitlawb-node/src/server.rs b/crates/gitlawb-node/src/server.rs index f4c0d3e3..de61fcbe 100644 --- a/crates/gitlawb-node/src/server.rs +++ b/crates/gitlawb-node/src/server.rs @@ -214,9 +214,20 @@ pub fn build_router(state: AppState) -> Router { // identity and can apply per-repo visibility (#110); anonymous callers stay // anonymous and still read genuinely public content. `/api/v1/ipfs/pins` // stays unsigned — gating the pin index is tracked separately (#121). + // `/ipfs/{cid}` also carries a per-IP flood brake: it is anon-reachable and each + // request can drive a full-history git walk, so the per-IP rate limiter is the + // outermost layer (rejects a flood before the walk-admission work), mirroring the + // push/create routers. The extension MUST be attached or rate_limit_by_ip is a + // silent no-op. `/api/v1/ipfs/pins` (no walk) is merged in unbraked, as before. + let ipfs_limiter = rate_limit::IpRateLimiter { + limiter: state.ipfs_rate_limiter.clone(), + trust: state.push_limiter_trust, + }; let ipfs_routes = Router::new() .route("/ipfs/{cid}", get(ipfs::get_by_cid)) .layer(middleware::from_fn(auth::optional_signature)) + .layer(middleware::from_fn(rate_limit::rate_limit_by_ip)) + .layer(axum::Extension(ipfs_limiter)) .merge(Router::new().route("/api/v1/ipfs/pins", get(ipfs::list_pins))); // ── Arweave permanent anchors ────────────────────────────────────────── diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index 6a84b3c2..213b8b35 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -88,6 +88,131 @@ pub struct AppState { /// * the libp2p swarm task /// * the gossip, sync, operator heartbeat, and rate-limit cleanup loops pub shutdown_tx: tokio::sync::watch::Sender, + /// Bounds concurrent served git READ operations (upload-pack + both info/refs + /// advertisements). A read handler acquires a permit before spawning git and + /// holds it for the op; when none are free the request is shed with a 503. + /// Writes draw from `git_write_semaphore` so a read flood cannot shed an + /// authenticated push at admission (#174). + pub git_read_semaphore: Arc, + /// Bounds concurrent `git-receive-pack` (push) operations, a pool separate + /// from `git_read_semaphore` so an anonymous READ flood can never shed an + /// authenticated push (#174). Sized by `max_concurrent_git_pushes`. Drawn from + /// by the `git-receive-pack` POST (owner-gated) ONLY. The anon-reachable + /// receive-pack `info/refs` advertisement draws from the SEPARATE + /// `git_push_advert_semaphore` below, never this pool, so a multi-source flood + /// of push-handshake advertisements can never occupy a permit an authenticated + /// POST needs at admission (#174). + pub git_write_semaphore: Arc, + /// Bounds concurrent anon-reachable `git-receive-pack` `info/refs` + /// advertisements — a pool SEPARATE from `git_write_semaphore` so adverts (which + /// hold a permit across `acquire_fresh` + `info/refs`) can never consume a slot + /// the authenticated POST relies on. A per-source flood can at worst exhaust this + /// advert pool (each source also capped by `git_push_advert_per_caller` and the + /// per-IP push rate limiter), and the reserved POST pool is untouched (#174). + pub git_push_advert_semaphore: Arc, + /// Bounds concurrent post-receive git scans. Each successful push releases its + /// handler write permit the moment receive-pack's git group is reaped, then runs + /// up to four scans over the repo: the anonymous withheld walk + /// (`replication_withheld_set`), the pin-candidate scan + /// (`resolve_candidates_for_push`), the fail-closed full scan + /// (`fail_closed_full_scan_objects`), and the DETACHED encrypt-then-pin walk + /// (`withheld_blob_recipients_bounded`). Without a cap, N fast pushes spawn N + /// concurrent full-history git walks past `max_concurrent_git_pushes` (which only + /// bounds the in-handler receive-pack phase) — #174 P1-e closed the detached walk, + /// F4 closed the other three. Each scan acquires ONE permit here per walk and + /// DEFERS (blocks) when the pool is full rather than shedding — dropping the work + /// would lose the recovery copy or silently under-pin the push. No-walk fast + /// paths (not announceable, no path-scoped rule, deletion-only push) never touch + /// the pool. A pool of its own, not `git_write_semaphore`: a long background + /// walk must not hold a foreground write slot, and a handler already holding a + /// write permit that needed a second would self-deadlock at pool size 1. + pub git_encrypt_semaphore: Arc, + /// Bounds concurrent post-push pin loops (`ipfs_pin` / `pinata` `pin_new_objects`) + /// across all repos (#174 F6). `encrypt_inflight` caps the pin-task COUNT to one + /// per repo, but each pin loop holds a full per-push object-id list while walking + /// it, so N distinct repos could hold N such MB-scale lists at once. This caps how + /// many run concurrently; a loop DEFERS (waits) when the pool is full, never drops. + pub pin_semaphore: Arc, + /// Bounds the outstanding post-push encryption-task set to at most one PER REPO by + /// coalescing (#174 P2-2). This is NOT a global cap: N distinct repos still admit N + /// tasks; the cross-repo residual (an authenticated actor pushing to many repos + /// leaves many parked tasks) is throttled by auth plus the per-IP/per-DID rate + /// limits, and its real cost — the MB-scale per-push object-id list each pin loop + /// holds — is bounded by `pin_semaphore`, not this. `git_encrypt_semaphore` caps + /// *active* walks; this caps duplicate SPAWNS per repo. Before spawning a per-push + /// encryption task, the receive-pack handler consults this set: if the repo already + /// has a task in flight it coalesces (skips the duplicate spawn) rather than parking + /// a new waiter, and its tip pairs are recorded for that task's drain loop (#174 F5). + /// Coalescing only delays the coalesced push's walk — it never drops the withheld-blob + /// recovery copy, which `2a54c15` deliberately kept fail-closed (there is no + /// reconciliation sweep to re-derive a dropped copy). See [`EncryptInflight`]. + pub encrypt_inflight: EncryptInflight, + /// Per-repo in-process write serializer that SUPPLEMENTS the cluster-wide pg + /// advisory lock on the receive-pack path (#174 U2/F3). On a client disconnect + /// mid-`receive-pack`, `RepoWriteGuard::Drop` releases the pg advisory lock at the + /// disconnect instant, but the disconnected push's git process GROUP is still + /// being torn down by `KillGroupOnDrop`'s detached reaper (~4s TERM/grace/KILL/reap) + /// over the shared LOCAL objects/ dir — so a second SAME-NODE push could acquire + /// the repo and race the still-writing group into a torn snapshot. This lease is + /// held by the write-path `AdmissionGuard`, which rides that reaper, so a second + /// same-repo push blocks until the first group is reaped. It is per-NODE (the + /// corruption is same-node: shared local objects/ + in-process reaper, and the + /// disconnect path uploads nothing to Tigris), so it needs no cross-node counterpart + /// and does NOT replace the pg lock (which stays the genuine cluster-wide serializer). + /// See [`RepoWriteLeases`]. + pub repo_write_leases: RepoWriteLeases, + /// Per-caller concurrency sub-cap on the read pool: each caller (keyed on the + /// resolved source IP, #174 U1) may hold at most `max_concurrent_reads_per_caller` + /// in-flight read ops, so one caller cannot monopolize `git_read_semaphore` + /// (#174). Applied by `git_upload_pack` and the upload-pack `info/refs` + /// advertisement. + pub git_read_per_caller: crate::rate_limit::PerCallerConcurrency, + /// Per-source concurrency sub-cap on the anon-reachable receive-pack `info/refs` + /// advertisement: each source IP may hold at most a small share of the write + /// pool, so a multi-source flood of push-handshake advertisements cannot + /// saturate `git_write_semaphore` and shed authenticated pushes (#174). Sized as + /// a fraction of `max_concurrent_git_pushes`, so filling the write pool takes many + /// distinct source IPs (each also braked by the per-IP push rate limiter). + pub git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency, + /// Per-source concurrency sub-cap on the authenticated `git-receive-pack` POST: + /// each source IP may hold at most a small share of `git_write_semaphore`, so one + /// host minting disposable `did:key` identities cannot open enough slow pushes to + /// monopolize the write pool and 503 every other source's push (#174 P1-d). Keyed + /// on the resolved source IP (never the DID — a DID farm defeats a DID key). Sized + /// like `git_push_advert_per_caller`, a fraction of `max_concurrent_git_pushes`. + pub git_write_per_caller: crate::rate_limit::PerCallerConcurrency, + /// Bounds concurrent `GET /ipfs/{cid}` visibility-walk requests. The public + /// `/ipfs/{cid}` route runs `allowed_blob_set_for_caller_bounded` in + /// `spawn_blocking` (a full-history git walk) with NO served-git admission of its + /// own; without this a permissionless caller fans out concurrent walks past every + /// git pool, exhausting the blocking pool + PIDs (#174 P1-3). A request acquires a + /// permit before the repo loop and holds it for the whole request (across every + /// `spawn_blocking` walk), so the slot reflects real thread occupancy — a tokio + /// walk-timeout cannot free it while the blocking work still runs. A pool of its + /// own (`max_concurrent_ipfs_walks`), NOT a git pool: distinct cost center + public + /// surface, so anonymous /ipfs traffic can never shed an authenticated git op. + pub git_ipfs_walk_semaphore: Arc, + /// Per-source concurrency sub-cap on the `/ipfs/{cid}` walk pool: each source + /// (keyed on the resolved source IP, never the DID — `/ipfs` admits any `did:key` + /// unthrottled, so a DID key would be free to mint around) may hold at most + /// `ipfs_walk_per_source` in-flight walk slots, so one source cannot monopolize + /// `git_ipfs_walk_semaphore` (#174 P1-3). A request with no resolvable key is + /// bounded by the global pool only, never this sub-cap. The key map is bounded + /// (`with_default_max_keys`, reject-before-insert) so a source-key farm cannot grow + /// it (INV-15). + pub git_ipfs_walk_per_caller: crate::rate_limit::PerCallerConcurrency, + /// Per-client-IP rate limiter for `GET /ipfs/{cid}`. The route is publicly + /// reachable and each request can drive a full-history git walk, so it carries a + /// per-IP flood brake in addition to the concurrency cap above — a rate limit + /// bounds request *rate*, the semaphore bounds concurrent slow holds (different + /// axes). Keyed on the resolved client IP via `push_limiter_trust`. Layered on the + /// `/ipfs` route via `rate_limit_by_ip`. + pub ipfs_rate_limiter: RateLimiter, + /// The `git` executable the served-git withheld-blob walk spawns. Production is + /// `"git"` (resolved via PATH); injectable so a fake `git` can drive the walk's + /// process-group teardown in handler tests without mutating the process-global + /// PATH (#174). + pub git_bin: String, } impl AppState { @@ -118,3 +243,517 @@ impl AppState { *self.shutdown_tx.borrow() } } + +/// Bounds the OUTSTANDING post-push encryption-task set by per-repo coalescing +/// (#174 P2-2). Each successful path-scoped push `tokio::spawn`s a DETACHED task that +/// parks on `git_encrypt_semaphore.acquire_owned().await` (which DEFERS when the pool +/// is full rather than shedding — `2a54c15` kept it fail-closed so the withheld-blob +/// recovery copy is never dropped). The semaphore caps *active* walks, but nothing +/// capped how many detached tasks *spawn and park* on that await: N rapid pushes to a +/// repo spawn N parked tasks, each holding cloned object lists/rules/paths/keys — an +/// unbounded outstanding set. +/// +/// This tracks the repo keys with an in-flight encryption task. Before spawning, the +/// handler calls [`try_begin`](Self::try_begin) with the push's (old, new) tip pairs: +/// if no task is in-flight the push is [`Admitted`](BeginOutcome::Admitted) and spawns +/// one; if a task IS in-flight the push [`Coalesces`](BeginOutcome::Coalesced) — no +/// duplicate spawn — and its tip pairs are merged into the in-flight key's pending +/// slot in the SAME critical section as the presence check. The in-flight task pins +/// only its own pre-spawn object-list snapshot, so the merge is what keeps coalescing +/// lossless (#174 F5): the task loop-drains the pending slot via +/// [`EncryptInflightGuard::finish_or_take_pending`] before releasing the key, so a +/// coalesced push's pins and recovery copies are delayed, never dropped (there is no +/// reconciliation sweep, so a *dropped* job would be lost forever). Check-then-record +/// as two lock acquisitions would race the task's final pending check — hence one +/// critical section for both. +/// +/// The returned [`EncryptInflightGuard`] is moved into the detached task. On normal +/// exit the key is removed (and the guard disarmed) inside `finish_or_take_pending`'s +/// empty-pending critical section; the guard's Drop is the PANIC backstop (Drop runs +/// on unwind), so one crashed walk can never permanently lock a repo out of future +/// recovery copies. +#[derive(Clone, Default)] +pub struct EncryptInflight { + // std::sync::Mutex: only ever held for O(1)-ish map ops (insert/remove/merge — + // the merge is an O(pairs) Vec extend bounded by MAX_PENDING_TIP_PAIRS) in a + // sync context, never across an await, so a std Mutex is correct and cheaper + // than a tokio one. Key present == task in flight; the value is the work + // recorded by pushes that coalesced against it. + repos: Arc>>, +} + +/// Cap on the accumulated coalesced tip pairs per repo. Past it the pending slot +/// degrades to [`PendingWork::FullScan`], so a hostile pusher cannot grow the slot +/// without bound while a walk is in flight; the drain then costs one full-repo +/// enumeration instead (the same already-tested fallback the push path uses). +const MAX_PENDING_TIP_PAIRS: usize = 1024; + +/// Work recorded by pushes that coalesced against an in-flight encryption task, +/// drained by that task one batch per loop iteration. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum PendingWork { + /// The coalesced pushes' raw (old_sha, new_sha) ref-update pairs, zeros + /// included — the drain strips the create/delete sentinels exactly like the + /// handler tail does. An EMPTY vec is "nothing pending", never a work item. + Tips(Vec<(String, String)>), + /// The pair bound overflowed: drain with a FORCED full-repo scan. This must be + /// signalled explicitly (the `force_full_scan` flag on + /// `resolve_candidates_for_push`), never encoded as an empty tip list — empty + /// tips resolve to an empty delta and would pin nothing (the F5 loss again). + FullScan, +} + +/// Outcome of [`EncryptInflight::try_begin`]. +pub enum BeginOutcome { + /// No task was in flight: the caller spawns one, moving the guard into it. The + /// push's own tip pairs are NOT recorded — the caller's pre-spawn snapshot + /// covers them; the pending slot starts empty. + Admitted(EncryptInflightGuard), + /// A task is in flight; this push's tip pairs were merged into its pending + /// slot (same critical section as the presence check). The in-flight task's + /// drain loop will process them. + Coalesced, +} + +/// Outcome of [`EncryptInflightGuard::finish_or_take_pending`]. +pub enum FinishOutcome { + /// Coalesced work was pending: it is handed back with the still-armed guard + /// (the repo key is retained) and the task must run another drain iteration. + Pending(EncryptInflightGuard, PendingWork), + /// Nothing was pending: the repo key was removed AND the guard disarmed in one + /// critical section, so dropping the returned guard is inert. The task exits. + /// Remove-then-drop as two steps would double-remove: a successor task admitted + /// between them would have ITS key deleted by the late Drop. The disarmed guard + /// is handed back rather than dropped internally so that remove→drop window is + /// real and the disarm is testable; production just lets it fall out of scope + /// (hence the allow). + Finished(#[allow(dead_code)] EncryptInflightGuard), +} + +impl EncryptInflight { + pub fn new() -> Self { + Self::default() + } + + /// Begin-or-coalesce an encryption task for `repo_id`, in one critical section. + /// `tip_pairs` is this push's raw (old_sha, new_sha) ref-update list; it is + /// merged into the pending slot only on the [`Coalesced`](BeginOutcome::Coalesced) + /// arm (an admitted caller's own snapshot already covers its pairs). + pub fn try_begin(&self, repo_id: &str, tip_pairs: Vec<(String, String)>) -> BeginOutcome { + let mut map = self.repos.lock().expect("encrypt_inflight mutex poisoned"); + match map.entry(repo_id.to_string()) { + std::collections::hash_map::Entry::Vacant(slot) => { + slot.insert(PendingWork::Tips(Vec::new())); + BeginOutcome::Admitted(EncryptInflightGuard { + repos: Arc::clone(&self.repos), + repo_id: repo_id.to_string(), + armed: true, + }) + } + std::collections::hash_map::Entry::Occupied(mut slot) => { + merge_pending(slot.get_mut(), tip_pairs); + BeginOutcome::Coalesced + } + } + } + + /// Number of repos with an in-flight encryption task. Test/metrics observability; + /// the bound under saturation is `len() <= number of distinct repos`, i.e. at most + /// one task per repo. + #[allow(dead_code)] + pub fn len(&self) -> usize { + self.repos + .lock() + .expect("encrypt_inflight mutex poisoned") + .len() + } + + #[allow(dead_code)] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +/// Merge a coalesced push's tip pairs into a repo's pending slot. FullScan absorbs +/// everything; a Tips slot that would exceed [`MAX_PENDING_TIP_PAIRS`] degrades to +/// FullScan rather than growing without bound. +fn merge_pending(slot: &mut PendingWork, pairs: Vec<(String, String)>) { + match slot { + PendingWork::FullScan => {} + PendingWork::Tips(acc) => { + if acc.len().saturating_add(pairs.len()) > MAX_PENDING_TIP_PAIRS { + *slot = PendingWork::FullScan; + } else { + acc.extend(pairs); + } + } + } +} + +/// Guard owned by the detached encryption task for its repo key. Move-only — there +/// is no reason to clone a guard, and cloning would double-remove. Normal exit goes +/// through [`finish_or_take_pending`](Self::finish_or_take_pending); Drop is the +/// panic-path backstop only. +pub struct EncryptInflightGuard { + repos: Arc>>, + repo_id: String, + /// True until the normal-exit path removes the key. A disarmed guard's Drop is + /// a no-op: the key slot may already belong to a successor task admitted after + /// our removal, and removing THAT key would break at-most-one-task-per-repo. + armed: bool, +} + +impl EncryptInflightGuard { + /// The task's end-of-iteration step, one critical section: if coalesced work is + /// pending, take it and hand the still-armed guard back (key retained — iterate); + /// if nothing is pending, remove the key and disarm the guard (task exits; the + /// returned guard's Drop is inert). The atomicity is load-bearing both ways: a + /// push landing before this call is merged and therefore drained here; a push + /// landing after it finds the key gone and is admitted as a fresh task. No + /// interleaving can lose the work or admit two tasks for one repo. + pub fn finish_or_take_pending(mut self) -> FinishOutcome { + let mut map = self.repos.lock().expect("encrypt_inflight mutex poisoned"); + match map.get_mut(&self.repo_id) { + Some(PendingWork::Tips(acc)) if acc.is_empty() => { + map.remove(&self.repo_id); + self.armed = false; + drop(map); + FinishOutcome::Finished(self) + } + Some(slot) => { + let work = std::mem::replace(slot, PendingWork::Tips(Vec::new())); + drop(map); + FinishOutcome::Pending(self, work) + } + None => { + // Unreachable while armed (only this method removes a live key), + // but never panic in the release path: treat as finished. + self.armed = false; + drop(map); + FinishOutcome::Finished(self) + } + } + } +} + +impl Drop for EncryptInflightGuard { + fn drop(&mut self) { + // Normal exit disarmed us inside finish_or_take_pending's critical section; + // an armed drop means the task ended abnormally (panic-unwind, or a future + // code path that returns without finishing). Release the key so the repo is + // not permanently locked out, and log any pending work this loses — there + // is no sweep, so it stays lost until a later push re-walks the repo. + if !self.armed { + return; + } + // A poisoned lock is not expected (the critical sections above are small + // and panic-free); remove best-effort. + if let Ok(mut map) = self.repos.lock() { + match map.remove(&self.repo_id) { + Some(PendingWork::Tips(acc)) if !acc.is_empty() => tracing::warn!( + repo = %self.repo_id, + lost_tip_pairs = acc.len(), + "encryption task ended abnormally with coalesced pushes pending; \ + their pins/recovery copies are lost until a later push" + ), + Some(PendingWork::FullScan) => tracing::warn!( + repo = %self.repo_id, + "encryption task ended abnormally with a pending full-scan drain; \ + it is lost until a later push" + ), + _ => {} + } + } + } +} + +/// Per-repo in-process write-lease serializer (#174 U2/F3). Keyed by the repo's DB +/// id (1:1 with the pg advisory lock's owner/name key), each entry is a one-permit +/// semaphore: the receive-pack handler takes it BEFORE `acquire_write` (see the acquire +/// order note on [`acquire`](Self::acquire)) and a second same-repo push BLOCKS on it — +/// block-and-wait, NOT coalesce. It mirrors [`EncryptInflight`]'s keyed-map + guard + +/// Drop-frees-key STRUCTURE; the semantics differ (block-and-wait, so there is no +/// lossy-coalesce degradation to fall back on). +#[derive(Clone, Default)] +pub struct RepoWriteLeases { + // std::sync::Mutex: held only for O(1) map ops (get-or-create + refcount) in a sync + // context, never across an await — the semaphore wait happens OUTSIDE this lock. + repos: Arc>>, +} + +/// A per-repo lease entry: the one-permit semaphore plus a refcount of the handlers +/// currently referencing it (holding or waiting). While `refs > 0` every acquirer +/// shares the SAME semaphore, so mutual exclusion holds; the entry is removed only when +/// `refs` hits 0 (no one references it), so a fresh entry can never split serialization. +struct LeaseSlot { + sem: Arc, + refs: usize, +} + +impl RepoWriteLeases { + pub fn new() -> Self { + Self::default() + } + + /// Acquire the per-repo write lease, blocking until it is free (a second same-repo + /// writer waits). `steal_after` bounds that wait: past it the acquirer STEALS + /// (proceeds permit-less) rather than block forever. + /// + /// Why a bounded steal: block-and-wait has no degradation of its own (unlike the + /// coalescing [`EncryptInflight`], whose lost key merely delays a best-effort copy), + /// and unlike the pg advisory lock (60s stale reclaim) an in-process waiter has no + /// reclaim — so a leaked/never-run Drop (runtime teardown without unwind, task abort, + /// `mem::forget`) would otherwise wedge the repo permanently. A stealer takes NO + /// permit and touches NO count, so a merely-slow holder that later drops can never + /// leave the semaphore over-counted; the caller must therefore set `steal_after` + /// safely ABOVE any legitimate hold (a full receive-pack under + /// `git_service_timeout_secs` + the ~4s reaper cap + the Tigris upload). + /// + /// Acquire order (one consistent order everywhere, so no inversion self-hang): the + /// lease is taken BEFORE the pg advisory lock (`acquire_write`) and released AFTER + /// it. Nothing anywhere takes the pg lock before this lease, so the two serializers + /// can never deadlock; taking the lease first also means a blocked second writer + /// pins no pooled pg connection while it waits. + pub async fn acquire(&self, repo_id: &str, steal_after: std::time::Duration) -> RepoWriteLease { + // Take the entry refcount BEFORE the await, so the entry cannot be GC'd out from + // under a waiter (a fresh entry for a new acquirer would split serialization). + let sem = { + let mut map = self.repos.lock().expect("repo_write_leases mutex poisoned"); + let slot = map.entry(repo_id.to_string()).or_insert_with(|| LeaseSlot { + sem: Arc::new(tokio::sync::Semaphore::new(1)), + refs: 0, + }); + slot.refs += 1; + Arc::clone(&slot.sem) + }; + // Cancellation-safe refcount: hold a reservation across the (cancellable) wait so + // that if this acquire future is DROPPED mid-wait — a client disconnect while a + // second same-repo push is blocked here — the reservation's Drop still decrements + // the ref it just took, rather than stranding it (which would defeat the + // Drop-frees-key GC). On success the reservation is `forget`-transferred into the + // returned guard, which then owns the single decrement. + let reservation = RefReservation { + repos: Arc::clone(&self.repos), + repo_id: repo_id.to_string(), + }; + let permit = match tokio::time::timeout(steal_after, Arc::clone(&sem).acquire_owned()).await + { + Ok(Ok(p)) => Some(p), + // The semaphore is never closed; treat the (unreachable) closed case as a + // steal so acquire always makes forward progress. + Ok(Err(_closed)) => None, + Err(_elapsed) => { + tracing::warn!( + repo = %repo_id, + steal_after_secs = steal_after.as_secs(), + "repo write-lease wait exceeded the steal bound; presuming a leaked \ + lease and proceeding permit-less (in-process serializer reclaim)" + ); + None + } + }; + // Transfer the ref from the reservation to the guard: forget the reservation (so + // it does NOT decrement) and let the guard own the single decrement on its Drop. + std::mem::forget(reservation); + RepoWriteLease(Arc::new(LeaseGuardInner { + repos: Arc::clone(&self.repos), + repo_id: repo_id.to_string(), + _permit: permit, + })) + } + + /// Number of repos with a live lease entry. Test/metrics observability. + #[allow(dead_code)] + pub fn len(&self) -> usize { + self.repos + .lock() + .expect("repo_write_leases mutex poisoned") + .len() + } + + #[allow(dead_code)] + pub fn is_empty(&self) -> bool { + self.len() == 0 + } +} + +/// Shared-ownership handle to a held repo write lease (#174 U2/F3). `Clone` hands a +/// second holder a handle to the SAME inner guard; the lease (permit + map refcount) +/// frees only when the LAST clone drops. The receive-pack handler makes two: +/// * clone (a) rides the write-path [`AdmissionGuard`] into `KillGroupOnDrop`'s +/// detached reaper, so on a client disconnect it drops only AFTER the git group is +/// reaped (this is the F3 fix — a lease tied to `RepoWriteGuard` would instead drop +/// at the disconnect instant, reopening the race); +/// * clone (b) is held by the handler across `guard.release()`, so on the clean path +/// it spans the success-only Tigris upload that runs inside `release`, AFTER +/// `receive_pack` has already dropped clone (a) inside `run_git_service`. +/// +/// `Send + 'static` with NO pg connection (just an `Arc`), so it can ride the reaper. +#[derive(Clone)] +pub struct RepoWriteLease(#[allow(dead_code)] Arc); + +struct LeaseGuardInner { + repos: Arc>>, + repo_id: String, + // `None` only on the steal path (the bounded wait elapsed). Dropping `None` releases + // no permit, so a stealer never corrupts the semaphore's permit count. + _permit: Option, +} + +impl Drop for LeaseGuardInner { + fn drop(&mut self) { + // Runs exactly ONCE per handler acquisition — when the last `RepoWriteLease` + // clone drops (the Arc strong count hits 0) — so the refcount decrements once, + // however many clones existed. `_permit` drops after this body, releasing the + // semaphore permit so a waiting acquirer proceeds. + release_lease_ref(&self.repos, &self.repo_id); + } +} + +/// Holds the entry refcount across the cancellable wait inside +/// [`RepoWriteLeases::acquire`]. If that acquire future is dropped mid-wait, this Drop +/// decrements the ref it took; on success `acquire` `forget`s it and hands the ref to +/// the returned [`LeaseGuardInner`], so the ref is decremented exactly once either way. +struct RefReservation { + repos: Arc>>, + repo_id: String, +} + +impl Drop for RefReservation { + fn drop(&mut self) { + release_lease_ref(&self.repos, &self.repo_id); + } +} + +/// Decrement a lease entry's refcount and remove it once no handler references it, so +/// the map cannot grow without bound (Drop-frees-key, like `EncryptInflight`). Safe +/// under block-and-wait: while `refs > 0` every acquirer shares the SAME semaphore, and +/// a fresh entry is created only after `refs` hits 0, when no one references the old one. +fn release_lease_ref( + repos: &Arc>>, + repo_id: &str, +) { + if let Ok(mut map) = repos.lock() { + if let Some(slot) = map.get_mut(repo_id) { + slot.refs = slot.refs.saturating_sub(1); + if slot.refs == 0 { + map.remove(repo_id); + } + } + } +} + +/// Admit a post-receive git scan to the shared `git_encrypt_semaphore` pool +/// (#174 F4): DEFER (await), never shed — a dropped scan would lose the push's +/// recovery copy or silently under-pin it. The returned permit must move into +/// the blocking closure so a started scan always completes holding it (a +/// disconnect cannot cancel `spawn_blocking` or leak the permit mid-walk). +/// Accepted residual, stated once for every caller: the park wait is queue-depth +/// multiplied — post-receive tails are no longer admission-bounded once the write +/// permit is released, so N landed pushes can queue N scans and the last waits N +/// scan-durations. A client-timeout disconnect no longer loses the work (#174 F2): +/// the whole post-receive replication tail runs in an independently owned task, so +/// dropping the request future cannot drop this parked scan — the park no longer +/// precedes any durable-record gate in a cancellable future. +pub async fn acquire_scan_permit( + scan_sem: Arc, + repo: &std::path::Path, + stage: &'static str, +) -> tokio::sync::OwnedSemaphorePermit { + let parked = std::time::Instant::now(); + let permit = scan_sem + .acquire_owned() + .await + .expect("git_encrypt_semaphore is never closed"); + tracing::debug!( + repo = %repo.display(), + stage, + queue_wait_ms = parked.elapsed().as_millis() as u64, + "post-receive scan admitted to the scan pool" + ); + permit +} + +#[cfg(test)] +mod repo_write_lease_tests { + use super::RepoWriteLeases; + use std::time::Duration; + + /// #174 U2/F3 lease mechanics: block-and-wait serialization on the same repo, + /// no serialization across distinct repos, Drop-frees-key GC, and the bounded-wait + /// steal reclaim so a leaked (never-run Drop) holder cannot wedge the repo forever. + #[tokio::test] + async fn serializes_same_repo_frees_key_and_steals_on_leak() { + let leases = RepoWriteLeases::new(); + let big = Duration::from_secs(3600); + + // Block-and-wait: a second same-repo acquire waits while the first is held. + let a = leases.acquire("repo1", big).await; + let blocked = + tokio::time::timeout(Duration::from_millis(200), leases.acquire("repo1", big)).await; + assert!( + blocked.is_err(), + "a second same-repo acquire must block while the first lease is held" + ); + // ... and proceeds once the first frees. + drop(a); + let b = tokio::time::timeout(Duration::from_millis(500), leases.acquire("repo1", big)) + .await + .expect("the second acquire must proceed once the first lease frees"); + drop(b); + + // Drop-frees-key: with no holders the entry is removed (bounded map growth). + assert!( + leases.is_empty(), + "the lease entry must be removed once no handler references it" + ); + + // Distinct repos never serialize against each other. + let x = leases.acquire("repoX", big).await; + let _y = tokio::time::timeout(Duration::from_millis(200), leases.acquire("repoY", big)) + .await + .expect("distinct repos must not serialize"); + drop(x); + drop(_y); + + // Steal-on-timeout reclaim: a leaked holder (never-run Drop, simulated by + // mem::forget) must NOT wedge the repo — the bounded wait proceeds permit-less. + let leaked = leases.acquire("repoZ", big).await; + std::mem::forget(leaked); + let stolen = tokio::time::timeout( + Duration::from_secs(2), + leases.acquire("repoZ", Duration::from_millis(150)), + ) + .await + .expect("a leaked lease must be reclaimed by the bounded-wait steal, not hang forever"); + drop(stolen); + } + + /// Cancellation safety: dropping an acquire future while it is BLOCKED waiting for + /// the lease (a client disconnect on a second same-repo push) must not strand the + /// entry refcount — after the holder frees and the waiter is cancelled, the key GCs. + #[tokio::test] + async fn cancelled_waiter_does_not_strand_the_refcount() { + let leases = RepoWriteLeases::new(); + let big = Duration::from_secs(3600); + + let a = leases.acquire("repoC", big).await; + // A waiter blocks, then is cancelled (its acquire future dropped) mid-wait. + let cancelled = + tokio::time::timeout(Duration::from_millis(150), leases.acquire("repoC", big)).await; + assert!( + cancelled.is_err(), + "the waiter must be blocked, then cancelled" + ); + + // Release the holder. If the cancelled waiter had stranded its ref, the entry + // would never GC; assert it does once the holder frees. + drop(a); + // Let any pending Drop bookkeeping settle. + tokio::task::yield_now().await; + assert!( + leases.is_empty(), + "a cancelled waiter must not strand the entry refcount (key must GC)" + ); + } +} diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index 4fe52f64..b88ae463 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -82,6 +82,26 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), shutdown_tx: tokio::sync::watch::channel(false).0, + // Generous — no test drives the handler-level shed (git_permit is unit-tested). + git_read_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_write_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_push_advert_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_encrypt_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + pin_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + encrypt_inflight: crate::state::EncryptInflight::new(), + repo_write_leases: crate::state::RepoWriteLeases::new(), + git_read_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(16), + git_push_advert_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( + 8, + ), + git_write_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys(8), + // Generous — a test that drives the /ipfs walk shed overrides these directly. + git_ipfs_walk_semaphore: Arc::new(tokio::sync::Semaphore::new(64)), + git_ipfs_walk_per_caller: crate::rate_limit::PerCallerConcurrency::with_default_max_keys( + 16, + ), + ipfs_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + git_bin: "git".to_string(), } } @@ -187,6 +207,199 @@ mod tests { ); } + /// PR3 (#62): the served-git concurrency cap sheds at the HTTP layer before the + /// DB. The held `git_permit` acquire now sits after the per-source cap, so the + /// shed-before-DB property is carried by an explicit `available_permits() == 0` + /// early check at the top of the handler (the held permit remains the + /// authoritative bound further down). DB-free: an exhausted semaphore sheds + /// before any DB/disk access, so a lazy state works. Remove the early-shed block + /// from git_info_refs and this goes red (the request falls through to the DB and + /// returns something other than 503). + #[tokio::test] + async fn git_info_refs_sheds_with_503_when_semaphore_exhausted() { + let mut state = test_state_lazy(); + state.git_read_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + + let router = Router::new() + .route( + "/{owner}/{repo}/info/refs", + axum::routing::get(crate::api::repos::git_info_refs), + ) + .with_state(state); + let resp = router + .oneshot(anon_get( + "/alice/repo.git/info/refs?service=git-upload-pack", + )) + .await + .unwrap(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted git semaphore must shed info/refs with 503 before touching the DB" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the 503 shed must carry Retry-After" + ); + } + + /// PR3 (#62) sibling of the info/refs shed test: git-upload-pack carries the same + /// explicit `available_permits() == 0` early-shed check at the top, so an + /// exhausted semaphore must shed it with a 503 before any DB/disk work. + /// Anonymous-reachable, so no auth injection is needed. Remove the early-shed + /// block from git_upload_pack and this goes red. + #[tokio::test] + async fn git_upload_pack_sheds_with_503_when_semaphore_exhausted() { + let mut state = test_state_lazy(); + state.git_read_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + + let router = Router::new() + .route( + "/{owner}/{repo}/git-upload-pack", + axum::routing::post(crate::api::repos::git_upload_pack), + ) + .with_state(state); + let req = Request::builder() + .method(Method::POST) + .uri("/alice/repo.git/git-upload-pack") + .body(Body::from(&b"0000"[..])) + .unwrap(); + let resp = router.oneshot(req).await.unwrap(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted git semaphore must shed git-upload-pack with 503 before touching the DB" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the 503 shed must carry Retry-After" + ); + } + + /// PR3 (#62) receive-pack sibling of the info/refs shed test: the early shed + /// selects the dedicated ADVERT pool for a git-receive-pack advertisement (#174), + /// so an exhausted advert pool sheds the advert with 503 before any DB/disk work + /// — while the write pool (reserved for authenticated POSTs) is left free here. + /// Flip the pool selection back to the write pool, or remove the early-shed + /// block, and this goes red. + #[tokio::test] + async fn git_info_refs_receive_pack_sheds_with_503_when_advert_pool_exhausted() { + let mut state = test_state_lazy(); + state.git_push_advert_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + + let router = Router::new() + .route( + "/{owner}/{repo}/info/refs", + axum::routing::get(crate::api::repos::git_info_refs), + ) + .with_state(state); + let resp = router + .oneshot(anon_get( + "/alice/repo.git/info/refs?service=git-receive-pack", + )) + .await + .unwrap(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted ADVERT pool must shed the receive-pack advertisement with 503 before touching the DB" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the 503 shed must carry Retry-After" + ); + } + + /// PR3 (#62) sibling for the push path: git-receive-pack requires an + /// AuthenticatedDid extension (production: require_signature injects it), so the + /// request carries one via signed_request_as — without it the Extension + /// extractor 500s before the handler body reaches git_permit. The permit is the + /// first statement, so an exhausted semaphore still sheds 503 before any DB + /// work. Remove the permit line from git_receive_pack and this goes red. + #[tokio::test] + async fn git_receive_pack_sheds_with_503_when_semaphore_exhausted() { + let mut state = test_state_lazy(); + state.git_write_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + + let router = Router::new() + .route( + "/{owner}/{repo}/git-receive-pack", + axum::routing::post(crate::api::repos::git_receive_pack), + ) + .with_state(state); + let owner = "did:key:zRECVSHEDOWNERAAAAAAAAAAAAAAAAAAAAAAAAAA"; + let resp = router + .oneshot(signed_request_as( + owner, + Method::POST, + "/alice/repo.git/git-receive-pack", + Body::from(&b"0000"[..]), + )) + .await + .unwrap(); + + assert_eq!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted write pool must shed git-receive-pack with 503 before touching the DB" + ); + assert_eq!( + resp.headers() + .get("retry-after") + .and_then(|h| h.to_str().ok()), + Some("1"), + "the 503 shed must carry Retry-After" + ); + } + + /// #174 (SC1, load-bearing): a saturated READ pool must NOT shed an + /// authenticated push — the write pool is a separate budget. Read pool at zero, + /// write pool with capacity: the push proceeds PAST admission (it then errors on + /// the placeholder DB, but crucially it is not a 503). Route git-receive-pack + /// back to the read pool and this goes red — that is the isolation proof. + #[tokio::test] + async fn git_receive_pack_not_shed_by_exhausted_read_pool() { + let mut state = test_state_lazy(); + // Read pool exhausted as if a flood of anonymous clones held every slot. + state.git_read_semaphore = Arc::new(tokio::sync::Semaphore::new(0)); + // Write pool keeps its default capacity from test_state_lazy. + + let router = Router::new() + .route( + "/{owner}/{repo}/git-receive-pack", + axum::routing::post(crate::api::repos::git_receive_pack), + ) + .with_state(state); + let owner = "did:key:zRECVCROSSBOUNDARYAAAAAAAAAAAAAAAAAAAAA"; + let resp = router + .oneshot(signed_request_as( + owner, + Method::POST, + "/alice/repo.git/git-receive-pack", + Body::from(&b"0000"[..]), + )) + .await + .unwrap(); + + assert_ne!( + resp.status(), + StatusCode::SERVICE_UNAVAILABLE, + "an exhausted READ pool must not shed a push — the write pool is a separate budget (#174)" + ); + } + /// N7: merge_pr is owner-only. A non-owner is rejected by require_repo_owner /// before any git work (so no on-disk repo is needed for the rejection). #[sqlx::test] @@ -2198,7 +2411,10 @@ mod tests { /// the handler skips the whole repo rather than serving. Asserts no leak of the /// withheld blob AND that even the *public* blob in that repo is withheld — the /// latter distinguishes fail-closed-skip from normal per-blob withholding and - /// would serve 200 if the error arm wrongly proceeded. + /// would serve 200 if the error arm wrongly proceeded. The skip carries no + /// VERDICT (F2), so the response is the retryable truncation 503, not a 404 + /// claiming the object is absent — never-serve-unproven and never-404-unproven + /// hold together. #[sqlx::test] async fn ipfs_cid_walk_error_fails_closed(pool: PgPool) { use crate::db::VisibilityMode; @@ -2243,7 +2459,8 @@ mod tests { .await .expect("deny rule"); - // Withheld secret CID under a walk error → 404, no leak. + // Withheld secret CID under a walk error → the repo is skipped without a + // verdict, so the scan is truncated (503), and nothing leaks. let (st, body) = cid_parts( cid_router(&state) .oneshot(cid_anon(&secret_cid)) @@ -2253,17 +2470,17 @@ mod tests { .await; assert_eq!( st, - StatusCode::NOT_FOUND, - "walk error must not serve the withheld blob" + StatusCode::SERVICE_UNAVAILABLE, + "walk error must not serve the withheld blob — the unproven skip sheds 503" ); assert!( !body.contains("TOP SECRET"), - "walk-error 404 must not leak the secret" + "walk-error 503 must not leak the secret" ); - // The PUBLIC blob in the same repo is also 404: the walk error fails closed - // by skipping the whole repo, not by serving. Without the fail-closed arm - // this would serve 200, so this assertion is the load-bearing discriminator. + // The PUBLIC blob in the same repo is also not served: the walk error fails + // closed by skipping the whole repo. Without the fail-closed arm this would + // serve 200, so this assertion is the load-bearing discriminator. let (st, _) = cid_parts( cid_router(&state) .oneshot(cid_anon(&public_cid)) @@ -2273,8 +2490,9 @@ mod tests { .await; assert_eq!( st, - StatusCode::NOT_FOUND, - "walk error fails closed: repo skipped, even the public blob is not served" + StatusCode::SERVICE_UNAVAILABLE, + "walk error fails closed: repo skipped without a verdict, even the public \ + blob is not served and the scan sheds 503" ); } diff --git a/crates/gitlawb-node/tests/inv22_gates.rs b/crates/gitlawb-node/tests/inv22_gates.rs new file mode 100644 index 00000000..0ce54088 --- /dev/null +++ b/crates/gitlawb-node/tests/inv22_gates.rs @@ -0,0 +1,358 @@ +//! #174 U6 — INV-22 completeness guard (rung-raising). +//! +//! INV-22: a permit held per op recovers only if every path that holds it is also +//! duration-bounded and reaps the process group before releasing admission, and every +//! detached git/blocking task carries its own admission. PR #174 fixed five paths +//! (U1-U5) that violated this. Each fix has a per-unit RED/GREEN regression; together +//! those form the five-revert matrix. This guard adds the missing piece: a source-scan +//! tripwire that fails when a NEW site reintroduces the class, or when one of the five +//! gates is removed. +//! +//! It lives in `tests/` (a separate crate) on purpose: a guard that scanned the same +//! file it lives in would match its own identifier literals and pass vacuously. Here +//! the scanned `src/` files never contain this file's literals, so each check is +//! load-bearing — reverting the named gate turns the assertion red. +//! +//! These are deliberately coarse structural checks, not a parser. They cannot prove a +//! gate is *correct* (the per-unit tests do that); they prove a gate is *present and +//! not bypassed*, which is what stops the class from silently regressing. + +use std::path::Path; + +fn src(rel: &str) -> String { + let p = Path::new(env!("CARGO_MANIFEST_DIR")).join("src").join(rel); + std::fs::read_to_string(&p).unwrap_or_else(|e| panic!("read {}: {e}", p.display())) +} + +#[test] +fn inv22_concurrency_gates_present_and_not_bypassed() { + let repos = src("api/repos.rs"); + let smart_http = src("git/smart_http.rs"); + let vis = src("git/visibility_pack.rs"); + + // U1 / P1-a: run_bounded_git stands the watchdog down only after confirming the + // child actually terminated (WNOWAIT), not on the raw stdout-drain EOF — otherwise + // a child that closes stdout then hangs pins the permit past the deadline. The + // probe is defined and called, so >= 2 occurrences; reverting the fix removes both. + assert!( + vis.matches("child_terminated_without_reaping").count() >= 2, + "U1/P1-a gate missing: run_bounded_git must confirm child exit via \ + child_terminated_without_reaping before signalling the watchdog" + ); + + // U2 / P1-c: on client disconnect KillGroupOnDrop must launch a detached reaper + // that runs the full TERM/grace/KILL/reap, not a lone SIGTERM. The reaper is spawned + // via a runtime handle in Drop; `Handle::try_current` is unique to that launch (the + // timeout path already has an async context and never calls it). + assert!( + smart_http.contains("Handle::try_current"), + "U2/P1-c gate missing: KillGroupOnDrop::drop must launch the full \ + TERM/grace/KILL reaper on disconnect (Handle::try_current), not a lone SIGTERM" + ); + + // U4 / P1-d: git_receive_pack must acquire the per-source write sub-cap before the + // global write permit. The acquire reads `state.git_write_per_caller`; comments name + // the field without the `state.` prefix, so this targets the real acquire site. + assert!( + repos.contains("state.git_write_per_caller"), + "U4/P1-d gate missing: git_receive_pack must acquire the per-source write cap \ + (state.git_write_per_caller) before the global write permit" + ); + + // U5 / P1-e: the detached post-push encryption walk must run through the + // admission-gated helper, which is wired to the shared encrypt pool. + assert!( + repos.contains("fn withheld_recipients_gated") + && repos.contains("state.git_encrypt_semaphore"), + "U5/P1-e gate missing: the encryption walk must run through \ + withheld_recipients_gated, which acquires git_encrypt_semaphore" + ); + + // P1-e non-bypass tripwire: the bounded recipients walk is spawn_blocking'd nowhere + // but inside withheld_recipients_gated. A second call site (count > 1) is a new + // detached git walk that skips the admission gate — exactly the class U5 closed. + assert_eq!( + repos.matches("withheld_blob_recipients_bounded").count(), + 1, + "P1-e bypass: the bounded recipients walk must be invoked only inside \ + withheld_recipients_gated; a new call site bypasses the encrypt-walk admission cap" + ); + + // U4 / P2-2: the detached post-push encryption task must be gated by the per-repo + // coalescing set (`encrypt_inflight.try_begin`) so the OUTSTANDING parked-task set is + // bounded to <=1 per repo. Removing the gate lets N rapid pushes spawn N parked + // waiters (the unbounded set U4 closed); the semaphore only caps active walks. + assert!( + repos.contains("state.encrypt_inflight.try_begin"), + "U4/P2-2 gate missing: the detached post-push encryption spawn must consult \ + encrypt_inflight.try_begin to coalesce per repo (bound the outstanding-task set)" + ); + + // F5: coalescing must REQUEUE, not shed. The in-flight task pins only its own + // pre-spawn snapshot, so a coalesced push's tip pairs are recorded on its key and + // the task must loop-drain them (`finish_or_take_pending`) before releasing it. + // Removing the drain reverts to the silent loss: a coalesced push's pins and + // recovery copies are absent until an unrelated later push. Scan only the + // production half of the file — the u5 tests in its `mod tests` also name the + // drain call, and matching them would make this check vacuous. + let repos_production = repos + .split("#[cfg(test)]") + .next() + .expect("split always yields a first chunk"); + assert!( + repos_production.contains("finish_or_take_pending"), + "F5 gate missing: the post-push encryption task must loop-drain coalesced \ + pushes via finish_or_take_pending before releasing its repo key" + ); + + // F4: every post-receive scan helper admits itself to the shared scan pool via + // `crate::state::acquire_scan_permit` BEFORE its spawn_blocking git walk, so a + // push burst cannot accumulate unbounded concurrent scans once the write permit + // is released. Two halves, both load-bearing: the helper body must actually + // acquire the pool (state.rs sits the helper at the end of the file, so the + // definition tail contains no other `acquire_owned` to match vacuously), and + // within each scan helper the first qualified call precedes the first + // `spawn_blocking`. Severing a call site pushes the next occurrence past the + // helper's own `spawn_blocking` (or off the end of the file), turning the + // assertion red; comments name the helper without the `crate::state::` prefix, + // so this targets the real call sites. + let state_rs = src("state.rs"); + let helper_def = state_rs + .find("fn acquire_scan_permit") + .expect("F4 gate missing: state.rs no longer defines acquire_scan_permit"); + assert!( + state_rs[helper_def..].contains("acquire_owned"), + "F4 gate gutted: acquire_scan_permit must acquire the scan pool via acquire_owned" + ); + let push_delta = src("git/push_delta.rs"); + for (file_src, file, helper) in [ + (&repos, "api/repos.rs", "fn replication_withheld_set"), + (&repos, "api/repos.rs", "fn fail_closed_full_scan_objects"), + ( + &push_delta, + "git/push_delta.rs", + "fn resolve_candidates_for_push", + ), + ] { + let start = file_src + .find(helper) + .unwrap_or_else(|| panic!("{file}: `{helper}` not found")); + let tail = &file_src[start..]; + let acquire = tail + .find("crate::state::acquire_scan_permit(") + .unwrap_or_else(|| { + panic!("F4 gate missing: {file} `{helper}` no longer acquires a scan permit") + }); + let spawn = tail.find("spawn_blocking").unwrap_or_else(|| { + panic!("{file}: `{helper}` lost its spawn_blocking walk — update this guard") + }); + assert!( + acquire < spawn, + "F4 gate bypassed: {file} `{helper}` must acquire its git_encrypt_semaphore \ + permit BEFORE dispatching the blocking git scan" + ); + } +} + +/// F4 (repo_store advisory-unlock cancellation safety): `RepoWriteGuard::release` +/// must await `pg_advisory_unlock` while `self` still owns the pooled connection, +/// and must not mark itself `released` until that await resolves. Either shape, +/// reintroduced, re-opens the mid-unlock cancellation leak: taking the connection +/// early leaves `Drop` with `conn == None`, and setting `released = true` early +/// leaves the `Drop` backstop inert — both strand the session lock on cancellation. +/// +/// Scoped to the `release` fn body: the `Drop` impl legitimately takes the +/// connection and unlocks, so a whole-file scan would match it and read as a false +/// pass. Reverting either ordering turns this red (proven load-bearing). +#[test] +fn f4_release_keeps_conn_owned_until_unlock_resolves() { + let repo_store = src("git/repo_store.rs"); + + let rel_start = repo_store + .find("pub async fn release(mut self") + .expect("F4 gate: repo_store.rs no longer defines RepoWriteGuard::release"); + let rel_end = repo_store[rel_start..] + .find("impl Drop for RepoWriteGuard") + .map(|off| rel_start + off) + .expect("F4 gate: release fn / Drop impl markers moved — update this guard"); + let release_body = &repo_store[rel_start..rel_end]; + + let unlock = release_body + .find("pg_advisory_unlock") + .expect("F4 gate: release must still issue pg_advisory_unlock"); + let before_unlock = &release_body[..unlock]; + + // (a) the connection must still be owned by `self` at the unlock await. + assert!( + !before_unlock.contains("self.conn.take()"), + "F4 regression: RepoWriteGuard::release takes self.conn BEFORE awaiting \ + pg_advisory_unlock. A cancellation during the unlock await then strands the \ + session advisory lock (Drop sees conn == None and skips its backstop). \ + Unlock through the still-owned connection instead." + ); + // (b) `released` must not be set before the unlock await — the other + // reintroduction shape a single-reorder check on (a) alone is blind to. + assert!( + !before_unlock.contains("released = true"), + "F4 regression: RepoWriteGuard::release sets `released = true` BEFORE awaiting \ + pg_advisory_unlock. A cancellation during the await then leaves the Drop \ + backstop inert (it early-returns on released). Set released only AFTER the \ + unlock await resolves." + ); +} + +/// F6/KTD-5 (initial IPFS metadata queries deadline-wrapped): `get_by_cid` acquires +/// the scarce walk permits (RAII, held for the whole request) BEFORE its two initial +/// metadata queries, and the per-repo loop's first budget gate runs only later. So +/// both `list_all_repos` and `list_visibility_rules_for_repos` must be clamped to the +/// remaining request budget — otherwise a query blocked in Postgres pins the walk slot +/// for the whole stall, past the budget. This scans the PRODUCTION half of `api/ipfs.rs` +/// (the `mod tests` half names the same calls in its own harness and would make the +/// check vacuous) and asserts each query call is immediately preceded by a +/// `tokio::time::timeout(` wrapper. Removing either wrapper turns this red (proven +/// load-bearing). +#[test] +fn f6_ipfs_metadata_queries_are_deadline_wrapped() { + let ipfs = src("api/ipfs.rs"); + let production = ipfs + .split("#[cfg(test)]") + .next() + .expect("split always yields a first chunk"); + + for call in [".list_all_repos()", ".list_visibility_rules_for_repos("] { + assert_eq!( + production.matches(call).count(), + 1, + "F6 guard stale: `{call}` must appear exactly once in the production half \ + of api/ipfs.rs (the deadline-wrapped handler call) — update this guard" + ); + let idx = production + .find(call) + .unwrap_or_else(|| panic!("F6 gate: api/ipfs.rs no longer calls `{call}`")); + // The wrapper opens a few lines above the call (match tokio::time::timeout( ... + // remaining budget ..., )); a 240-char lookback covers it without + // reaching the previous statement. + let window = &production[idx.saturating_sub(240)..idx]; + assert!( + window.contains("tokio::time::timeout("), + "F6 gate missing: `{call}` must be wrapped in tokio::time::timeout(...) \ + clamped to the remaining request budget. An unwrapped await pins the held \ + walk permit for the whole DB stall, past GITLAWB_IPFS_REQUEST_BUDGET_SECS." + ); + } +} + +/// #174 F2 / KTD-3: the detached post-receive Pinata replication task must enqueue +/// only the push's `(ref, old, new)` tuples and RE-DERIVE its object set inside the +/// worker once a pin slot frees — it must NOT move the full per-push object list into +/// the closure and hold it across the `pin_semaphore` acquire. Retaining the list makes +/// every parked task (under a slow Pinata backend) hold an MB-scale OID list, so +/// outstanding memory grows O(pushes x object-list). Coalescing/shedding the task is +/// forbidden (its per-ref effects are non-idempotent), so the fix bounds the retained +/// data, not the task count. +/// +/// Two load-bearing checks, both against the PRODUCTION half of `api/repos.rs` (the +/// `mod tests` half names the same identifiers in its own harness and would make the +/// scan vacuous): (a) the closure-local `object_list_pinata` binding — the retain form — +/// must be gone; reintroducing `let object_list_pinata = object_list;` turns this red. +/// (b) the re-derivation (`pinata_object_list_for_refs`) must appear AFTER the Pinata +/// pin permit is acquired, so the object list is materialized only inside the pin-bounded +/// section and a parked task holds O(ref tuples). +#[test] +fn f2_pinata_enqueues_refs_not_retained_object_lists() { + let repos = src("api/repos.rs"); + let production = repos + .split("#[cfg(test)]") + .next() + .expect("split always yields a first chunk"); + + // (a) the retained-list form must be gone from production. + assert!( + !production.contains("object_list_pinata"), + "F2/KTD-3 regression: the Pinata task retains a full per-push object list \ + (`object_list_pinata`) across the pin-permit acquire. Enqueue only the ref \ + tuples and re-derive the object set inside the worker (pinata_object_list_for_refs)." + ); + + // (b) the re-derivation runs AFTER the pin permit is acquired. Anchor on the + // Pinata pin-admission clone so the window is the Pinata task, not the sibling + // IPFS/encrypt spawn (which shares `pin_semaphore` but never re-derives). + let anchor = production + .find("let pin_sem_pinata") + .expect("F2 gate stale: the Pinata pin-admission clone (pin_sem_pinata) moved"); + let tail = &production[anchor..]; + let acquire = tail + .find(".acquire_owned()") + .expect("F2 gate: the Pinata task no longer acquires a pin permit (acquire_owned)"); + let rederive = tail.find("pinata_object_list_for_refs(").expect( + "F2 gate missing: the Pinata task must re-derive its object set via \ + pinata_object_list_for_refs; it can no longer move a pre-resolved list into the closure", + ); + assert!( + acquire < rederive, + "F2 gate bypassed: the Pinata object-set re-derivation must run AFTER the pin \ + permit is acquired, so a parked task never holds the MB-scale object list" + ); + + // The re-derivation is driven by the push's ref tuples (the enqueued unit), not a + // retained object list — tie "enqueue ref_updates" to the call explicitly. + assert!( + tail[rederive..].starts_with("pinata_object_list_for_refs(") + && tail[rederive..] + .get(..400) + .map(|w| w.contains("ref_updates_clone")) + .unwrap_or(false), + "F2 gate: pinata_object_list_for_refs must re-derive from the push's ref tuples \ + (ref_updates_clone), the small unit the parked task retains" + ); +} + +/// #174 U2 / F3 (second same-repo writer serialized until a disconnected first push's +/// git group is reaped): `git_receive_pack` must take the per-repo in-process write +/// lease and CARRY it on the write-path `AdmissionGuard` (via `.with_lease`), so the +/// lease rides `KillGroupOnDrop`'s detached reaper and frees only after the group is +/// reaped — supplementing the pg advisory lock, which `RepoWriteGuard::Drop` releases at +/// the disconnect instant. Tying the lease to `RepoWriteGuard` instead (or dropping the +/// `.with_lease` carry) reopens the race; the behavioral RED/GREEN test +/// (`f3_second_push_serialized_until_disconnected_group_reaped`) is the real bar, this +/// is the completeness tripwire. Scanned against the PRODUCTION half of `api/repos.rs` +/// (the `mod tests` half names these identifiers in its own harness). +#[test] +fn f3_second_writer_leased_until_reap() { + let repos = src("api/repos.rs"); + let smart_http = src("git/smart_http.rs"); + let repos_production = repos + .split("#[cfg(test)]") + .next() + .expect("split always yields a first chunk"); + + // The lease is acquired, then carried by the AdmissionGuard (.with_lease), before + // receive_pack runs the write. Severing any of the three turns this red. + let lease_acquire = repos_production.find("repo_write_leases").expect( + "F3 gate missing: git_receive_pack no longer takes the per-repo write lease \ + (state.repo_write_leases)", + ); + let with_lease = repos_production.find(".with_lease(").expect( + "F3 gate missing: the write-path AdmissionGuard no longer carries the lease \ + (.with_lease). The lease must ride the disconnect reaper via the AdmissionGuard, \ + NOT RepoWriteGuard (which drops at the disconnect instant, reopening F3).", + ); + let receive = repos_production + .find("smart_http::receive_pack(") + .expect("F3 gate stale: git_receive_pack no longer calls smart_http::receive_pack"); + assert!( + lease_acquire < with_lease && with_lease < receive, + "F3 gate bypassed: the write lease must be acquired, then carried by the \ + AdmissionGuard (.with_lease), BEFORE receive_pack runs the write" + ); + + // The AdmissionGuard must actually hold the lease (Option) via a + // with_lease setter, so it travels into the detached reaper. Removing the field or + // method turns this red. + assert!( + smart_http.contains("_lease: Option") + && smart_http.contains("pub fn with_lease("), + "F3 gate missing: AdmissionGuard must hold an Option set via \ + with_lease, so the lease rides the guard into KillGroupOnDrop's reaper" + ); +}