diff --git a/.env.example b/.env.example index bbd9a342..05c2de47 100644 --- a/.env.example +++ b/.env.example @@ -99,6 +99,18 @@ GITLAWB_ENFORCE_OWNER_PUSH=false # Example: /ip4/1.2.3.4/udp/7546/quic-v1/p2p/12D3KooW... GITLAWB_P2P_BOOTSTRAP= +# ── IPFS pin listing rate limiting ──────────────────────────────────────── +# Per-DID rate limit — requests per hour per signed DID. The listing performs +# expensive git walks and cat-file probes, so a throwaway DID with a valid +# signature can otherwise exhaust resources. Default 60. +GITLAWB_IPFS_LIST_RATE_LIMIT=60 + +# Global (non-sybil) rate limit — total requests per hour regardless of signed +# DID. Prevents DID-rotation attacks from bypassing the per-DID limiter. +# Charged only after the per-DID check passes so a single DID cannot drain the +# shared bucket with rejected requests. Default 1200. +GITLAWB_IPFS_LIST_GLOBAL_RATE_LIMIT=1200 + # ── Access control ──────────────────────────────────────────────────────── # Reserved for private-read mode. Public/private repo read enforcement is not # wired in the current live release; do not rely on this for private repositories. diff --git a/.release-please-manifest.json b/.release-please-manifest.json index bcd05228..e7ca6139 100644 --- a/.release-please-manifest.json +++ b/.release-please-manifest.json @@ -1,3 +1,3 @@ { - ".": "0.6.0" + ".": "0.7.0" } diff --git a/CHANGELOG.md b/CHANGELOG.md index 99e1796e..c4bf3a32 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,23 @@ # Changelog +## [0.7.0](https://github.com/Gitlawb/node/compare/v0.6.0...v0.7.0) (2026-07-22) + + +### Features + +* **gl:** actually verify certificate signatures in gl cert show ([b0a2ed8](https://github.com/Gitlawb/node/commit/b0a2ed830821b5a77b0dafdcf5ce4b1ad904ec25)) +* **gl:** doctor warns when a shell alias shadows the gl binary ([0765c42](https://github.com/Gitlawb/node/commit/0765c42feff47b29054b9a277881a7cd5bfd9135)) + + +### Bug Fixes + +* **gl:** address review findings — non-fatal DID check, exact-host loopback, detached HEAD, multi-URL remotes, command-aware unalias ([3fa38dc](https://github.com/Gitlawb/node/commit/3fa38dce4e435ef017f607d355266c2892161f8d)) +* **gl:** anchor cert --verify to a trusted issuer; pin canonical payload form ([c681af7](https://github.com/Gitlawb/node/commit/c681af70dba3a76b11795d324f2ca0fecf7e42fd)) +* **gl:** detect gitlawb remotes beyond origin in gl status ([2d88937](https://github.com/Gitlawb/node/commit/2d88937fbabf21013f28debb4d4f93270a4bb8cd)) +* **gl:** doctor treats a reachable local node as configuration, not failure ([94c61f7](https://github.com/Gitlawb/node/commit/94c61f79421fa26ecc1bdf7c255e0c48e66a2048)) +* **gl:** make gl init's push guidance match the repo's actual state ([cb42a48](https://github.com/Gitlawb/node/commit/cb42a4830d0d070165e2212f316d009ec716dbee)) +* warn about the oh-my-zsh gl alias at install time ([e3df51d](https://github.com/Gitlawb/node/commit/e3df51d76c0275fde33954ff0af0aab95fd0cb7d)) + ## [0.6.0](https://github.com/Gitlawb/node/compare/v0.5.1...v0.6.0) (2026-07-22) diff --git a/Cargo.lock b/Cargo.lock index 5dd2903d..48ee2e33 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3300,7 +3300,7 @@ dependencies = [ [[package]] name = "git-remote-gitlawb" -version = "0.5.1" +version = "0.7.0" dependencies = [ "anyhow", "gitlawb-core", @@ -3312,7 +3312,7 @@ dependencies = [ [[package]] name = "gitlawb-attest" -version = "0.5.1" +version = "0.7.0" dependencies = [ "base64", "ed25519-dalek", @@ -3329,7 +3329,7 @@ dependencies = [ [[package]] name = "gitlawb-core" -version = "0.5.1" +version = "0.7.0" dependencies = [ "anyhow", "base64", @@ -3356,7 +3356,7 @@ dependencies = [ [[package]] name = "gitlawb-node" -version = "0.5.1" +version = "0.7.0" dependencies = [ "alloy", "anyhow", @@ -3368,6 +3368,7 @@ dependencies = [ "axum", "base64", "bytes", + "chacha20poly1305", "chrono", "cid", "clap", @@ -3376,6 +3377,7 @@ dependencies = [ "futures", "gitlawb-core", "hex", + "hkdf", "hmac", "http-body-util", "libc", @@ -3390,6 +3392,7 @@ dependencies = [ "mockito", "multiaddr", "prometheus", + "rand 0.8.6", "reqwest", "serde", "serde_json", @@ -3412,10 +3415,11 @@ dependencies = [ [[package]] name = "gl" -version = "0.5.1" +version = "0.7.0" dependencies = [ "alloy", "anyhow", + "base64", "chrono", "clap", "dirs", diff --git a/crates/git-remote-gitlawb/Cargo.toml b/crates/git-remote-gitlawb/Cargo.toml index 3810aa97..ba7d126b 100644 --- a/crates/git-remote-gitlawb/Cargo.toml +++ b/crates/git-remote-gitlawb/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "git-remote-gitlawb" description = "Git remote helper — enables 'git clone gitlawb://did:gitlawb:...'" -version = "0.6.0" # x-release-please-version +version = "0.7.0" # x-release-please-version edition.workspace = true rust-version.workspace = true license.workspace = true diff --git a/crates/gitlawb-attest/Cargo.toml b/crates/gitlawb-attest/Cargo.toml index b01fbce9..ce0e9269 100644 --- a/crates/gitlawb-attest/Cargo.toml +++ b/crates/gitlawb-attest/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "gitlawb-attest" description = "External Attestation v1: pluggable provenance attachments for gitlawb ref-update certs" -version = "0.6.0" # x-release-please-version +version = "0.7.0" # x-release-please-version edition.workspace = true rust-version.workspace = true license.workspace = true diff --git a/crates/gitlawb-core/Cargo.toml b/crates/gitlawb-core/Cargo.toml index 3bfc810a..5358ccfd 100644 --- a/crates/gitlawb-core/Cargo.toml +++ b/crates/gitlawb-core/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "gitlawb-core" description = "Core cryptographic primitives for the gitlawb network: DIDs, CIDs, UCAN, HTTP Signatures, ref-update certificates" -version = "0.6.0" # x-release-please-version +version = "0.7.0" # x-release-please-version edition.workspace = true rust-version.workspace = true license.workspace = true diff --git a/crates/gitlawb-node/Cargo.toml b/crates/gitlawb-node/Cargo.toml index 7cc3727f..dc872f6c 100644 --- a/crates/gitlawb-node/Cargo.toml +++ b/crates/gitlawb-node/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "gitlawb-node" description = "The gitlawb node daemon — git hosting over HTTP with DID auth" -version = "0.6.0" # x-release-please-version +version = "0.7.0" # x-release-please-version edition.workspace = true rust-version.workspace = true license.workspace = true @@ -73,6 +73,9 @@ alloy = { version = "1", default-features = false, features = [ "rpc-types-eth", ] } libp2p-dns = { version = "0.44.0", features = ["tokio"] } +rand = { workspace = true } +hkdf = "0.12" +chacha20poly1305 = "0.10" [dev-dependencies] mockito = "1" diff --git a/crates/gitlawb-node/src/api/arweave.rs b/crates/gitlawb-node/src/api/arweave.rs index 0d728c71..bc9a97b4 100644 --- a/crates/gitlawb-node/src/api/arweave.rs +++ b/crates/gitlawb-node/src/api/arweave.rs @@ -1,12 +1,19 @@ //! GET /api/v1/arweave/anchors — list Arweave ref-update anchors. +//! +//! Requires authentication (RFC 9421 HTTP Signature). When `?repo=...` is +//! specified the caller must also be authorized to read that repo, and the +//! query uses the normalized slug. Without `?repo=` the endpoint returns +//! anchors scoped to repos the caller can read, or an empty list for +//! anonymous callers. use axum::{ extract::{Query, State}, - Json, + Extension, Json, }; use serde::Deserialize; -use crate::error::Result; +use crate::auth::AuthenticatedDid; +use crate::error::{AppError, Result}; use crate::state::AppState; #[derive(Debug, Deserialize)] @@ -20,20 +27,284 @@ fn default_limit() -> i64 { 50 } +/// Compute the set of (repo_slug, owner_did) pairs the caller can read. +/// Used when `?repo=` is absent and the caller is authenticated (P1). +fn readable_repo_pairs( + repos: &[crate::db::RepoRecord], + rules_by_repo: &std::collections::HashMap>, + caller: &str, +) -> (Vec, Vec) { + let mut slugs = Vec::new(); + let mut dids = Vec::new(); + for r in repos { + let rules = rules_by_repo.get(&r.id).map(Vec::as_slice).unwrap_or(&[]); + if crate::visibility::listable_at_root(rules, r.is_public, &r.owner_did, Some(caller)) { + let owner_short = crate::db::normalize_owner_key(&r.owner_did); + slugs.push(format!("{owner_short}/{}", r.name)); + dids.push(r.owner_did.clone()); + } + } + (slugs, dids) +} + /// GET /api/v1/arweave/anchors pub async fn list_anchors( State(state): State, Query(q): Query, + auth: Option>, ) -> Result> { - let limit = q.limit.min(200); + let caller = auth.as_ref().map(|e| e.0 .0.as_str()); + let limit = q.limit.clamp(0, 200); + + // Reject missing authentication before any branch — the ?repo= path also + // needs a caller so that public repos are gated by the same auth contract (P1). + if caller.is_none() { + return Err(AppError::Unauthorized( + "authentication required for anchor listing".into(), + )); + } + + if let Some(slug) = &q.repo { + let Some((owner_key, name)) = slug.split_once('/') else { + return Err(AppError::BadRequest(format!("invalid repo slug: {slug}"))); + }; + let (record, _rules) = + crate::api::authorize_repo_read(&state, owner_key, name, Some(caller.unwrap()), "/") + .await?; + + // Use the normalized slug so full-DID queries match persisted values (P2) + let owner_short = crate::db::normalize_owner_key(&record.owner_did); + let normalized_slug = format!("{owner_short}/{}", record.name); + let anchors = state + .db + .list_arweave_anchors(Some(&normalized_slug), limit) + .await + .map_err(AppError::Internal)?; + + return Ok(Json(serde_json::json!({ + "anchors": anchors, + "count": anchors.len(), + }))); + } + + // Authenticated caller without ?repo=: scope to readable repos (P1). + // Use the deduped, quarantine-filtered view (same as the pin listing). + let repos = state + .db + .list_all_repos_deduped() + .await + .map_err(AppError::Internal)?; + let ids: Vec = repos.iter().map(|r| r.id.clone()).collect(); + let rules_by_repo = state + .db + .list_visibility_rules_for_repos(&ids) + .await + .map_err(AppError::Internal)?; + let (repos, owner_dids) = readable_repo_pairs(&repos, &rules_by_repo, caller.unwrap()); let anchors = state .db - .list_arweave_anchors(q.repo.as_deref(), limit) + .list_arweave_anchors_for_repos(&repos, &owner_dids, limit) .await - .map_err(crate::error::AppError::Internal)?; + .map_err(AppError::Internal)?; Ok(Json(serde_json::json!({ "anchors": anchors, "count": anchors.len(), }))) } + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_support::test_state; + use axum::extract::{Extension, Query, State}; + use sqlx::PgPool; + + fn alice_did() -> String { + "did:key:z6MkwAlice".into() + } + + fn bob_did() -> String { + "did:key:z6MkwBob".into() + } + + fn auth_ext(did: &str) -> Option> { + Some(Extension(AuthenticatedDid(did.to_string()))) + } + + #[sqlx::test] + async fn anonymous_is_401_before_any_db_work(pool: PgPool) { + let state = test_state(pool).await; + let q = Query(ListAnchorsQuery { + repo: None, + limit: 50, + }); + let result = list_anchors(State(state), q, None).await; + assert!( + matches!(result, Err(AppError::Unauthorized(_))), + "expected 401 for anonymous, got {result:?}" + ); + } + + #[sqlx::test] + async fn anonymous_with_repo_is_401(pool: PgPool) { + let state = test_state(pool).await; + let q = Query(ListAnchorsQuery { + repo: Some("z6MkwAlice/public-repo".into()), + limit: 50, + }); + let result = list_anchors(State(state), q, None).await; + assert!( + matches!(result, Err(AppError::Unauthorized(_))), + "expected 401 for anonymous with ?repo=, got {result:?}" + ); + } + + #[sqlx::test] + async fn stranger_repo_on_private_is_denied(pool: PgPool) { + let state = test_state(pool).await; + + // Seed a private repo. + sqlx::query( + "INSERT INTO repos (id, name, owner_did, description, is_public, default_branch, created_at, updated_at, disk_path) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", + ) + .bind("arw-test-private") + .bind("priv-repo") + .bind(alice_did()) + .bind("desc") + .bind(false) + .bind("main") + .bind("2026-07-19T00:00:00Z") + .bind("2026-07-19T00:00:00Z") + .bind("/srv/priv-repo") + .execute(state.db.pool()) + .await + .unwrap(); + + // Seed an anchor for the private repo. + sqlx::query( + "INSERT INTO arweave_anchors (id, repo, owner_did, ref_name, old_sha, new_sha, cid, irys_tx_id, arweave_url, node_did, anchored_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)", + ) + .bind("anchor-1") + .bind("z6MkwAlice/priv-repo") + .bind(alice_did()) + .bind("refs/heads/main") + .bind("0000") + .bind("aaaa") + .bind("QmAnchor") + .bind("irys-tx-1") + .bind("https://arweave.net/tx1") + .bind("did:key:z6MkwNode") + .bind("2026-07-19T00:00:00Z") + .execute(state.db.pool()) + .await + .unwrap(); + + // Bob (stranger) tries ?repo= on private repo — denied. + let q = Query(ListAnchorsQuery { + repo: Some("z6MkwAlice/priv-repo".into()), + limit: 50, + }); + let result = list_anchors(State(state), q, auth_ext(&bob_did())).await; + assert!( + matches!(result, Err(AppError::RepoNotFound(_))), + "stranger should get RepoNotFound for private repo, got {result:?}" + ); + } + + #[sqlx::test] + async fn global_path_scopes_to_readable_repos(pool: PgPool) { + let state = test_state(pool).await; + + // Seed two repos: one public (readable by all) and one private (readable + // only by Alice). Bob should only see the public repo's anchors. + sqlx::query( + "INSERT INTO repos (id, name, owner_did, description, is_public, default_branch, created_at, updated_at, disk_path) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", + ) + .bind("arw-pub") + .bind("pub-repo") + .bind(alice_did()) + .bind("desc") + .bind(true) + .bind("main") + .bind("2026-07-19T00:00:00Z") + .bind("2026-07-19T00:00:00Z") + .bind("/srv/pub") + .execute(state.db.pool()) + .await + .unwrap(); + + sqlx::query( + "INSERT INTO repos (id, name, owner_did, description, is_public, default_branch, created_at, updated_at, disk_path) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", + ) + .bind("arw-priv") + .bind("priv-repo") + .bind(alice_did()) + .bind("desc") + .bind(false) + .bind("main") + .bind("2026-07-19T00:00:00Z") + .bind("2026-07-19T00:00:00Z") + .bind("/srv/priv") + .execute(state.db.pool()) + .await + .unwrap(); + + // Seed anchors for both repos. + sqlx::query( + "INSERT INTO arweave_anchors (id, repo, owner_did, ref_name, old_sha, new_sha, cid, irys_tx_id, arweave_url, node_did, anchored_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)", + ) + .bind("anchor-pub") + .bind("z6MkwAlice/pub-repo") + .bind(alice_did()) + .bind("refs/heads/main") + .bind("0000") + .bind("aaaa") + .bind("QmPub") + .bind("irys-tx-pub") + .bind("https://arweave.net/pub") + .bind("did:key:z6MkwNode") + .bind("2026-07-19T00:00:00Z") + .execute(state.db.pool()) + .await + .unwrap(); + + sqlx::query( + "INSERT INTO arweave_anchors (id, repo, owner_did, ref_name, old_sha, new_sha, cid, irys_tx_id, arweave_url, node_did, anchored_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)", + ) + .bind("anchor-priv") + .bind("z6MkwAlice/priv-repo") + .bind(alice_did()) + .bind("refs/heads/main") + .bind("0000") + .bind("bbbb") + .bind("QmPriv") + .bind("irys-tx-priv") + .bind("https://arweave.net/priv") + .bind("did:key:z6MkwNode") + .bind("2026-07-19T00:00:00Z") + .execute(state.db.pool()) + .await + .unwrap(); + + // Bob (stranger) without ?repo=: only public repo anchors returned. + let q = Query(ListAnchorsQuery { + repo: None, + limit: 200, + }); + let Json(body) = list_anchors(State(state), q, auth_ext(&bob_did())) + .await + .unwrap(); + let anchors = body["anchors"].as_array().unwrap(); + let count = body["count"].as_u64().unwrap(); + assert_eq!(count, 1, "bob should see only 1 anchor (public repo)"); + assert_eq!(anchors.len(), 1); + assert_eq!(anchors[0]["cid"].as_str(), Some("QmPub")); + } +} diff --git a/crates/gitlawb-node/src/api/ipfs.rs b/crates/gitlawb-node/src/api/ipfs.rs index f3de7570..53ded39a 100644 --- a/crates/gitlawb-node/src/api/ipfs.rs +++ b/crates/gitlawb-node/src/api/ipfs.rs @@ -15,14 +15,30 @@ //! see `get_by_cid`). use axum::{ - extract::{Path, State}, + extract::{Path, Query, State}, http::{HeaderMap, HeaderName, HeaderValue, StatusCode}, response::{IntoResponse, Response}, Extension, Json, }; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use base64::Engine; +use chacha20poly1305::{ + aead::{Aead, AeadCore, KeyInit}, + XChaCha20Poly1305, XNonce, +}; use cid::CidGeneric; +use hkdf::Hkdf; +use rand::rngs::OsRng; +use serde::Deserialize; +use sha2::Sha256; use std::collections::{HashMap, HashSet}; +use std::io::Write; +use std::path::PathBuf; +use std::process::{Command, Stdio}; use std::str::FromStr; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::Arc; +use std::time::{SystemTime, UNIX_EPOCH}; use crate::auth::AuthenticatedDid; use crate::error::{AppError, Result}; @@ -144,12 +160,14 @@ pub async fn get_by_cid( let caller_for_walk = caller_owned.clone(); // Full-history walk shells out to git — keep it off the async runtime. let walk = tokio::task::spawn_blocking(move || { + let uncancelled = AtomicBool::new(false); allowed_blob_set_for_caller( &rp, &r, is_public, &owner, caller_for_walk.as_deref(), + &uncancelled, ) }) .await; @@ -211,20 +229,1696 @@ pub async fn get_by_cid( ))) } +/// Query parameters for `GET /api/v1/ipfs/pins`. +#[derive(Debug, Deserialize, Clone)] +pub struct ListPinsQuery { + #[serde(default = "default_limit")] + pub limit: i64, + pub cursor: Option, + pub truncated_cursor: Option, +} + +fn default_limit() -> i64 { + 50 +} + +/// Derive a dedicated 32-byte cursor cipher key from the node's Ed25519 seed +/// using HKDF with a domain-separated info string. This decouples cursor +/// confidentiality from the write-signing identity and avoids feeding the raw +/// seed into an unrelated primitive. +fn derive_cursor_key(seed: &[u8; 32]) -> [u8; 32] { + let hk = Hkdf::::new(None, seed.as_slice()); + let mut okm = [0u8; 32]; + hk.expand(b"gitlawb-ipfs-cursor-v1", &mut okm) + .expect("32 bytes is a valid HKDF output length"); + okm +} + +/// Create an opaque, self-contained truncated cursor token using +/// XChaCha20Poly1305 AEAD. +/// +/// Format: `base64_url_no_pad(nonce_24 || ciphertext)` where `ciphertext` = +/// XChaCha20Poly1305-encrypt(expiry_be_8 || cursor_string) with the 16-byte +/// AEAD tag appended by the encryptor. The caller cannot decode hidden-row +/// metadata without the server's Ed25519 seed. Tokens are durable (survive +/// restart, cross-node routing, retries) and expire after 600 seconds. +fn create_opaque_cursor(seed: &[u8; 32], cursor: &str) -> String { + let cursor_key = derive_cursor_key(seed); + let cipher = XChaCha20Poly1305::new_from_slice(&cursor_key) + .expect("32-byte key is valid for XChaCha20Poly1305"); + + let nonce = XChaCha20Poly1305::generate_nonce(&mut OsRng); + + let expiry = (SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs() + + 600) + .to_be_bytes(); + + let mut plaintext = Vec::with_capacity(8 + cursor.len()); + plaintext.extend_from_slice(&expiry); + plaintext.extend_from_slice(cursor.as_bytes()); + + let ciphertext = cipher + .encrypt(&nonce, plaintext.as_ref()) + .expect("AEAD encrypt should never fail"); + + let mut token = Vec::with_capacity(24 + ciphertext.len()); + token.extend_from_slice(nonce.as_ref()); + token.extend_from_slice(&ciphertext); + + URL_SAFE_NO_PAD.encode(&token) +} + +/// Decode and verify an opaque truncated cursor token. +/// Returns the original cursor string if valid and not expired. +fn decode_opaque_cursor(seed: &[u8; 32], token: &str) -> Option<(String, String)> { + let cursor_key = derive_cursor_key(seed); + let cipher = XChaCha20Poly1305::new_from_slice(&cursor_key) + .expect("32-byte key is valid for XChaCha20Poly1305"); + + let data = URL_SAFE_NO_PAD.decode(token.as_bytes()).ok()?; + if data.len() < 24 + 1 { + return None; + } + + let (nonce_bytes, ciphertext) = data.split_at(24); + let nonce = XNonce::from_slice(nonce_bytes); + + let plaintext = cipher.decrypt(nonce, ciphertext).ok()?; + if plaintext.len() < 8 { + return None; + } + + let expiry = u64::from_be_bytes(plaintext[..8].try_into().ok()?); + let now = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default() + .as_secs(); + if now >= expiry { + return None; + } + + let cursor = std::str::from_utf8(&plaintext[8..]).ok()?; + + let parts: Vec<&str> = cursor.splitn(2, '|').collect(); + if parts.len() == 2 { + Some((parts[0].to_string(), parts[1].to_string())) + } else { + None + } +} + +/// Batch-check git object types for many SHAs in a single repo, using one +/// `git cat-file --batch-check` subprocess instead of N individual `cat-file -t` +/// calls. Returns a map from SHA → `Some("blob"|"commit"|"tree"|"tag")` or +/// `None` (missing/dangling). +/// +/// Must be called from a blocking context (e.g. `tokio::task::spawn_blocking`) +/// since it spawns a child process and reads its output synchronously. +fn batch_object_types( + repo_path: &std::path::Path, + shas: &[String], + cancelled: &AtomicBool, +) -> Result>> { + use anyhow::Context; + + let mut child = Command::new("git") + .args(["cat-file", "--batch-check"]) + .current_dir(repo_path) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .spawn() + .context("failed to spawn git cat-file --batch-check")?; + + { + let stdin = child.stdin.as_mut().context("stdin not captured")?; + for sha in shas { + writeln!(stdin, "{sha}").context("failed to write sha to cat-file stdin")?; + } + } + + // Drop stdin so the child sees EOF on its input pipe. + drop(child.stdin.take()); + + // Poll for completion, checking the cancellation flag between iterations. + let output = loop { + if cancelled.load(Ordering::Relaxed) { + let _ = child.kill(); + let _ = child.wait(); + return Ok(HashMap::new()); + } + match child.try_wait() { + Ok(Some(_status)) => { + break child + .wait_with_output() + .context("git cat-file --batch-check failed")?; + } + Ok(None) => { + std::thread::sleep(std::time::Duration::from_millis(50)); + continue; + } + Err(e) => { + return Err(AppError::Git(format!( + "git cat-file --batch-check wait failed: {e}", + ))); + } + } + }; + let stdout = String::from_utf8_lossy(&output.stdout); + + let mut results = HashMap::with_capacity(shas.len()); + for line in stdout.lines() { + let parts: Vec<&str> = line.splitn(3, ' ').collect(); + if parts.len() < 2 { + continue; + } + let sha = parts[0].to_string(); + match parts[1] { + "missing" => { + results.insert(sha, None); + } + obj_type => { + results.insert(sha, Some(obj_type.to_string())); + } + } + } + Ok(results) +} + /// GET /api/v1/ipfs/pins /// /// Returns all CIDs that have been pinned to the local IPFS node from git /// objects received via push. Each entry includes the git SHA-256 hex, the /// CIDv1 string, and the timestamp when it was pinned. -pub async fn list_pins(State(state): State) -> Result> { - let pins = state +/// +/// Requires authentication: the global pin index would otherwise disclose +/// metadata for every object ever pushed here (#121). +/// +/// The global listing filters each pinned object on current repo visibility +/// to prevent metadata disclosure when repos are made private after push (#136). +/// Only pins from repos the caller can currently read are returned. +pub async fn list_pins( + State(state): State, + Query(query): Query, + auth: Option>, +) -> Result> { + let caller = auth.as_ref().map(|e| e.0 .0.as_str()); + + // Reject anonymous callers: the pin index spans the entire node and would + // expose metadata for every object ever pushed here (#121). + if caller.is_none() { + return Err(AppError::Unauthorized( + "authentication required for pin listing".into(), + )); + } + let caller_str = caller.unwrap(); + let caller_owned = Some(caller_str.to_string()); + + // Per-DID rate limit: the listing performs expensive git walks and cat-file + // probes, so a throwaway DID with a valid signature can exhaust resources (P1). + if !state.ipfs_list_rate_limiter.check(caller_str).await { + return Err(AppError::TooManyRequests( + "rate limit exceeded for IPFS pin listing".into(), + )); + } + + // Build the set of readable repo slugs and owner DIDs from the deduped repo view + // (mirror rows already collapsed, quarantined excluded), then query + // pins bounded in SQL. + let repos = state + .db + .list_all_repos_deduped() + .await + .map_err(AppError::Internal)?; + let repo_ids: Vec = repos.iter().map(|r| r.id.clone()).collect(); + let rules_by_repo = state .db - .list_pinned_cids() + .list_visibility_rules_for_repos(&repo_ids) .await .map_err(AppError::Internal)?; - Ok(Json(serde_json::json!({ + // Build parallel vectors of readable (slug, owner_did) pairs to query in SQL, + // plus a boolean flag per pair indicating whether the repo has *no* path-scoped + // visibility rules. The SQL ROW_NUMBER dedup uses this flag to prefer + // associations from rule-free repos (always visible at root level) over those + // from repos with /secret/**-style rules (P2). + let mut query_repos = Vec::new(); + let mut query_owner_dids = Vec::new(); + let mut query_no_rules = Vec::new(); + + for r in &repos { + let rules = rules_by_repo.get(&r.id).map(Vec::as_slice).unwrap_or(&[]); + if visibility_check(rules, r.is_public, &r.owner_did, caller, "/") == Decision::Deny { + continue; + } + let short = crate::db::normalize_owner_key(&r.owner_did); + let slug = format!("{}/{}", short, r.name); + query_repos.push(slug); + query_owner_dids.push(r.owner_did.clone()); + query_no_rules.push(!has_path_scoped_rule(rules)); + } + + let max_visible = query.limit.clamp(0, 200); + + if max_visible == 0 { + return Ok(Json(serde_json::json!({ + "pins": [], + "count": 0, + }))); + } + + // Global rate limit: keyed on a fixed value so DID rotation cannot + // bypass the enumeration cost guard (P1). Checked after the limit=0 + // fast path so short-lived requests do not drain the shared bucket (P2). + if !state.ipfs_list_global_limiter.check("global").await { + return Err(AppError::TooManyRequests( + "rate limit exceeded for IPFS pin listing".into(), + )); + } + + // Decode the optional keyset cursor from base64. + // Internal format: "pinned_at|sha256_hex" (2-tuple) for normal + // pagination, or just "pinned_at" (1-tuple) for the truncated resume. + let decode_cursor = |s: &str| -> Option<(String, String)> { + let bytes = URL_SAFE_NO_PAD.decode(s.as_bytes()).ok()?; + let decoded = String::from_utf8(bytes).ok()?; + let parts: Vec<&str> = decoded.splitn(2, '|').collect(); + if parts.len() == 2 { + Some((parts[0].to_string(), parts[1].to_string())) + } else { + None + } + }; + let encode_cursor = + |pa: &str, sha: &str| -> String { URL_SAFE_NO_PAD.encode(format!("{pa}|{sha}")) }; + + let initial_cursor = match query.cursor.as_ref() { + Some(c) => match decode_cursor(c) { + Some(cursor) => Some(cursor), + None => { + return Err(AppError::BadRequest( + "invalid cursor: expected base64-encoded pinned_at|sha256_hex".into(), + )) + } + }, + None => None, + }; + + // Truncated resume cursor: XChaCha20Poly1305 AEAD token. Decrypts to the + // same (pinned_at, repo, sha256_hex) cursor on the server side but the + // caller cannot decode hidden-row metadata from the wire format. If the + // token is present but undecodable we return an explicit error so the + // client does not silently restart at page 1. + let truncated_resume = match query.truncated_cursor.as_ref() { + Some(t) => { + let seed = state.node_keypair.to_seed(); + match decode_opaque_cursor(&seed, t) { + Some(c) => Some(c), + None => { + return Err(AppError::BadRequest( + "invalid or expired truncated_cursor".into(), + )) + } + } + } + None => None, + }; + + // Build a lookup of slug -> (repo, rules) once. + let mut repos_by_slug = HashMap::new(); + for r in repos { + let short = crate::db::normalize_owner_key(&r.owner_did); + let slug = format!("{}/{}", short, r.name); + let rules = rules_by_repo.get(&r.id).cloned().unwrap_or_default(); + repos_by_slug.insert(slug, (r, rules)); + } + + // Use keyset pagination to fetch batches and post-filter path-scoped + // hidden pins so the caller still receives up to `max_visible` visible + // entries even when newer pins are hidden under /secret/** rules. + // Keyset cursor avoids duplicate/skip rows when new pins land between + // batches (unlike LIMIT/OFFSET) and removes the cost of deep OFFSET + // re-scanning. + // + // The loop is bounded by MAX_BATCHES to prevent a single request from + // scanning an unbounded number of hidden rows. Path-scoped git walks + // are independently bounded by MAX_WALKS as a secondary safeguard. + // + // next_cursor is derived from the last *accepted* (visible) pin, never + // from the last scanned row, to avoid leaking withheld-blob metadata + // or skipping rows the caller was never shown. + const BATCH_SIZE: i64 = 200; + const MAX_BATCHES: usize = 10; + const MAX_WALKS: usize = 50; + const MAX_PROBES: usize = 200; + // P1: hard deadline for cumulative visibility-walk work so a single + // request with many path-scoped repos cannot hold the global permits + // for minutes on end. + const LISTING_DEADLINE_SECS: u64 = 120; + let listing_deadline = + tokio::time::Instant::now() + std::time::Duration::from_secs(LISTING_DEADLINE_SECS); + let mut batch_count = 0usize; + let mut batch_hit_limit = false; + let mut pins = Vec::new(); + // Within-batch dedup by sha256_hex: SQL ROW_NUMBER guarantees one row + // per SHA across pages, but within a single batch of an all-deferred + // page the same object could still appear via a different association + // (P2). Track seen SHAs so each object is emitted at most once. + let mut seen_shas: HashSet = HashSet::new(); + let mut db_cursor: Option<(String, String)> = truncated_resume.or(initial_cursor); + let mut response_cursor: Option<(String, String)> = None; + let mut allowed_blobs_by_repo: HashMap, PathBuf)> = HashMap::new(); + let mut page_truncated = false; + // Per-repo cache of sha256_hex → is_structural (true for commit/tree/tag). + let mut structural_cache: HashMap> = HashMap::new(); + let mut probe_count = 0usize; + let mut probe_limit = usize::MAX; + + 'fetch: loop { + if batch_count >= MAX_BATCHES { + batch_hit_limit = true; + break; + } + batch_count += 1; + + let batch = if query_repos.is_empty() { + Vec::new() + } else { + state + .db + .list_pinned_cids_for_repos( + &query_repos, + &query_owner_dids, + &query_no_rules, + BATCH_SIZE, + db_cursor + .as_ref() + .map(|(pa, sha)| (pa.as_str(), sha.as_str())), + ) + .await + .map_err(AppError::Internal)? + }; + + if batch.is_empty() { + break; + } + + // Snapshot the cursor used to fetch THIS batch so Phase 3 can retry + // the first row inclusively when it is deferred (P2). + let batch_cursor = db_cursor.clone(); + + // ── Phase 1 — collect structural candidates per repo ────────────── + // Track per-pin outcome: None = structural candidate (needs type check + // before final decision), Some(false) = hidden, Some(true) = visible. + let mut pin_outcome: Vec> = Vec::with_capacity(batch.len()); + let mut structural_candidates: HashMap> = HashMap::new(); + let mut walk_limit_idx = batch.len(); + + for (i, pin) in batch.iter().enumerate() { + if pin.repo.is_empty() { + db_cursor = Some((pin.pinned_at.clone(), pin.sha256_hex.clone())); + pin_outcome.push(None); + continue; + } + let Some((repo, rules)) = repos_by_slug.get(&pin.repo) else { + // Unknown slug — advance cursor past it, no visibility check. + db_cursor = Some((pin.pinned_at.clone(), pin.sha256_hex.clone())); + pin_outcome.push(None); + continue; + }; + + if !has_path_scoped_rule(rules) { + // No path-scoped rules — every pin from this repo is visible. + pin_outcome.push(Some(true)); + continue; + } + + // Path-scoped repo — ensure walk result is cached. + if !allowed_blobs_by_repo.contains_key(&repo.id) { + if allowed_blobs_by_repo.len() >= MAX_WALKS { + // Walk budget exhausted. Stop before this pin and leave + // db_cursor at the last processed pin so the next request + // picks up here and retries the walk. + page_truncated = true; + walk_limit_idx = i; + break; + } + // Respect the total visibility-walk deadline so a single + // request cannot hold the global permits for minutes (P1). + if tokio::time::Instant::now() >= listing_deadline { + page_truncated = true; + if i < walk_limit_idx { + walk_limit_idx = i; + } + continue; + } + + // Acquire a concurrency permit so a flood of requests cannot + // exhaust the blocking-pool worker or leave unbounded git + // children running (P1). + let permit = match state.walk_semaphore.clone().try_acquire_owned() { + Ok(p) => p, + Err(_) => { + // All walk slots are occupied — defer this repo's pins + // to the next request. + page_truncated = true; + if i < walk_limit_idx { + walk_limit_idx = i; + } + continue; + } + }; + + // Wrap acquire() in a timeout so a slow Tigris fetch does not + // hold the walk permit unboundedly (P2). + let acquire_fut = state.repo_store.acquire(&repo.owner_did, &repo.name); + match tokio::time::timeout(std::time::Duration::from_secs(30), acquire_fut).await { + Ok(Ok(rp)) => { + let rp_clone = rp.clone(); + let r_clone = rules.clone(); + let is_public = repo.is_public; + let owner = repo.owner_did.clone(); + let caller_for_walk = caller_owned.clone(); + let cancelled = Arc::new(AtomicBool::new(false)); + let cancelled_clone = Arc::clone(&cancelled); + + // Move the semaphore permit into the blocking task so it + // is released only when the walk truly completes — even + // on timeout the permit stays alive until the git child + // is killed and the worker returns (P1). + let walk_fut = tokio::task::spawn_blocking(move || { + let _hold = permit; + allowed_blob_set_for_caller( + &rp_clone, + &r_clone, + is_public, + &owner, + caller_for_walk.as_deref(), + &cancelled_clone, + ) + }); + match tokio::time::timeout(std::time::Duration::from_secs(60), walk_fut) + .await + { + Ok(Ok(Ok(allowed))) => { + allowed_blobs_by_repo.insert(repo.id.clone(), (allowed, rp)); + } + _ => { + // Walk failed (timeout / error / panic). + cancelled.store(true, Ordering::Relaxed); + page_truncated = true; + if i < walk_limit_idx { + walk_limit_idx = i; + } + allowed_blobs_by_repo + .insert(repo.id.clone(), (HashSet::new(), PathBuf::new())); + } + } + } + Ok(Err(_)) | Err(_) => { + // Repo-store acquisition failed or timed out — same + // strategy (P2). The walk permit is dropped here. + page_truncated = true; + if i < walk_limit_idx { + walk_limit_idx = i; + } + allowed_blobs_by_repo + .insert(repo.id.clone(), (HashSet::new(), PathBuf::new())); + } + }; + } + + let (allowed, repo_path) = allowed_blobs_by_repo.get(&repo.id).unwrap(); + if allowed.contains(&pin.sha256_hex) { + pin_outcome.push(Some(true)); + } else if !repo_path.as_os_str().is_empty() { + // Not in the allowed set — could be a withheld blob or a + // structural object (commit/tree/tag). Mark as structural + // candidate; Phase 2 will probe the type. + pin_outcome.push(None); // deferred — decided after phase 2 + structural_candidates + .entry(repo.id.clone()) + .or_default() + .push((i, pin.sha256_hex.clone())); + } else { + pin_outcome.push(Some(false)); + } + } + + // When Phase 1 never advanced db_cursor (all pins are path-scoped and + // no walk permit was available), keep the batch-fetch cursor so the + // next request retries the same batch. Only advance past the batch + // when walk permits were available but the pins had no repo match, + // because those rows are permanently unprocessable (P2). + let all_deferred = walk_limit_idx == 0 && db_cursor.as_ref() == batch_cursor.as_ref(); + if all_deferred { + // db_cursor already equals batch_cursor — the next fetch uses the + // same position and retries the deferred path-scoped pins. + } else if db_cursor.as_ref() == batch_cursor.as_ref() { + // All pins had empty/unmatched repos — advance past the batch so + // we don't loop forever on the same unprocessable rows (P1). + if let Some(last) = batch.last() { + db_cursor = Some((last.pinned_at.clone(), last.sha256_hex.clone())); + } + } + + // ── Phase 2 — batch-check structural candidates per repo ────────── + 'phase2: for (repo_id, candidates) in &structural_candidates { + // Honor the same listing_deadline and walk_semaphore that Phase 1 + // respects, so probe subprocesses don't run past the total request + // budget or outside the concurrency cap (P2). + if tokio::time::Instant::now() >= listing_deadline { + for &(idx, _) in candidates { + if idx < probe_limit { + probe_limit = idx; + } + } + continue 'phase2; + } + let probe_permit = match state.walk_semaphore.clone().try_acquire_owned() { + Ok(p) => p, + Err(_) => { + for &(idx, _) in candidates { + if idx < probe_limit { + probe_limit = idx; + } + } + continue 'phase2; + } + }; + if probe_count >= MAX_PROBES { + // Probe budget exhausted. Fold EVERY remaining structural + // candidate index (not just the current repo's) into + // probe_limit so none are silently dropped as hidden. + for &(idx, _) in candidates { + if idx < probe_limit { + probe_limit = idx; + } + } + continue 'phase2; + } + let rp = allowed_blobs_by_repo + .get(repo_id) + .map(|(_, p)| p.clone()) + .unwrap_or_default(); + if rp.as_os_str().is_empty() { + continue; + } + // Filter to SHAs not already cached. + let repo_cache = structural_cache.entry(repo_id.clone()).or_default(); + let to_check: Vec = candidates + .iter() + .filter(|(_, sha)| !repo_cache.contains_key(sha)) + .map(|(_, sha)| sha.clone()) + .collect(); + if to_check.is_empty() { + continue; + } + let remaining = MAX_PROBES.saturating_sub(probe_count); + let to_check: Vec = to_check.into_iter().take(remaining).collect(); + probe_count += to_check.len(); + + // Fold any unprobed candidates from THIS repo into probe_limit + // so Phase 3 stops before them (P2). + if to_check.len() < candidates.len() { + for &(idx, _) in candidates.iter().skip(to_check.len()) { + if idx < probe_limit { + probe_limit = idx; + } + } + } + + let rp_for_block = rp.clone(); + let cancelled_probe = Arc::new(AtomicBool::new(false)); + let cancelled_probe_clone = Arc::clone(&cancelled_probe); + let probe_fut = tokio::task::spawn_blocking(move || { + let _probe_hold = probe_permit; + batch_object_types(&rp_for_block, &to_check, &cancelled_probe_clone) + }); + let results = + match tokio::time::timeout(std::time::Duration::from_secs(30), probe_fut).await { + Ok(Ok(Ok(map))) => map, + _ => { + // Probe timeout/error — don't cache empty results + // (they'd be classified as hidden). Fold the current + // repo's candidates into probe_limit so Phase 3 + // defers them to the next request (P2). + cancelled_probe.store(true, Ordering::Relaxed); + for &(idx, _) in candidates { + if idx < probe_limit { + probe_limit = idx; + } + } + HashMap::new() + } + }; + for (sha, obj_type) in results { + repo_cache.insert(sha, obj_type.is_some_and(|t| t != "blob")); + } + } + + // ── Phase 3 — emit visible pins ─────────────────────────────────── + for i in 0..batch.len() { + if i >= walk_limit_idx.min(probe_limit) { + // Past the MAX_WALKS wall or an unprobed structural candidate. + // Remaining pins are handled by the next request; db_cursor + // stays at the last processed pin so no row is skipped. + if i < probe_limit && !page_truncated { + page_truncated = true; + } + // Save cursor so the keyset predicate < resumes at the + // first unprocessed row. When i == 0 there is no processed + // pin — use the cursor that fetched this batch so the SQL + // predicate < re-evaluates the deferred row (P2). When + // batch_cursor is None (page 1), keep the Phase 1 fallback + // value so the response can produce a truncated_cursor (P1). + if i == 0 { + if batch_cursor.is_some() { + db_cursor = batch_cursor; + } + } else if let Some(pin) = i.checked_sub(1).and_then(|prev| batch.get(prev)) { + db_cursor = Some((pin.pinned_at.clone(), pin.sha256_hex.clone())); + } + break; + } + + let pin = batch[i].clone(); + let Some((repo, rules)) = repos_by_slug.get(&pin.repo) else { + // Already advanced past in phase 1 — just maintain cursor. + db_cursor = Some((pin.pinned_at.clone(), pin.sha256_hex.clone())); + continue; + }; + + if !has_path_scoped_rule(rules) { + let pa = pin.pinned_at.clone(); + let sha = pin.sha256_hex.clone(); + + // Dedup by sha256_hex (P2): only skip if already *emitted* — + // do not suppress a visible association because a hidden one + // appeared first in the batch. + if !seen_shas.insert(sha.clone()) { + db_cursor = Some((pa, sha)); + continue; + } + + response_cursor = Some((pa.clone(), sha.clone())); + pins.push(pin); + db_cursor = Some((pa, sha)); + } else { + let pa = pin.pinned_at.clone(); + let sha = pin.sha256_hex.clone(); + + let visible = match pin_outcome[i] { + Some(v) => v, + None => { + // Structural candidate — consult cache. If the + // candidate was never probed (MAX_PROBES exhausted) + // this shouldn't be reached (probe_limit stops Phase 3 + // before unprobed rows), but handle it defensively. + structural_cache + .get(&repo.id) + .and_then(|c| c.get(&pin.sha256_hex)) + .copied() + .unwrap_or(false) + } + }; + if visible { + if !seen_shas.insert(sha.clone()) { + db_cursor = Some((pa, sha)); + continue; + } + response_cursor = Some((pa.clone(), sha.clone())); + pins.push(pin); + } + db_cursor = Some((pa, sha)); + } + + if pins.len() >= max_visible as usize { + break 'fetch; + } + } + } + let page_filled = pins.len() >= max_visible as usize; + if !page_truncated && batch_hit_limit { + page_truncated = true; + } + pins.truncate(max_visible as usize); + + // When page 1 is all-deferred (no walk permit available) neither + // response_cursor nor db_cursor was set. Emit a sentinel opaque cursor + // so the client can retry; on the retry the sentinel decodes to + // ("\x7f", "\x7f") which the keyset WHERE < predicate treats + // as "include every row" — effectively restarting from the beginning (P2). + if page_truncated && response_cursor.is_none() && db_cursor.is_none() { + db_cursor = Some(("\x7f".to_string(), "\x7f".to_string())); + } + + let mut body = serde_json::json!({ "pins": pins, "count": pins.len(), - }))) + }); + + if page_truncated { + body["truncated"] = serde_json::json!(true); + } + if page_filled { + // Page is full — provide a cursor from the last visible row so the + // caller can paginate. + if let Some((ref pa, ref sha)) = response_cursor { + body["next_cursor"] = serde_json::json!(encode_cursor(pa, sha)); + } + } else if page_truncated { + // Scan bound hit before filling the page — opaque cursor fallback + // when there are no visible rows to derive a keyset cursor from. + if let Some((ref pa, ref sha)) = response_cursor { + body["next_cursor"] = serde_json::json!(encode_cursor(pa, sha)); + } else if let Some((ref pa, ref sha)) = db_cursor { + let cursor_str = format!("{pa}|{sha}"); + let seed = state.node_keypair.to_seed(); + let token = create_opaque_cursor(&seed, &cursor_str); + body["truncated_cursor"] = serde_json::json!(token); + } + } + + Ok(Json(body)) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::auth::AuthenticatedDid; + use crate::test_support::test_state; + use axum::extract::{Extension, Query, State}; + use sqlx::PgPool; + + #[sqlx::test] + async fn test_ipfs_cursor_guard(pool: PgPool) { + let app_state = test_state(pool.clone()).await; + + // Seed a path-scoped repo + sqlx::query( + "INSERT INTO repos (id, name, owner_did, description, is_public, default_branch, created_at, updated_at, disk_path) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)" + ) + .bind("repo-ipfs-test") + .bind("ipfstest") + .bind("did:key:z6Mkwowner") + .bind("desc") + .bind(true) + .bind("main") + .bind("2026-07-03T00:00:00Z") + .bind("2026-07-03T00:00:00Z") + .bind("/srv/ipfstest") + .execute(app_state.db.pool()) + .await + .unwrap(); + + sqlx::query( + "INSERT INTO visibility_rules (id, repo_id, path_glob, mode, reader_dids, created_by, created_at) + VALUES ($1, $2, $3, $4, $5, $6, $7)", + ) + .bind("rule-1") + .bind("repo-ipfs-test") + .bind("/secret/**") + .bind("deny") + .bind("") + .bind("did:key:z6Mkwowner") + .bind("2026-07-03T00:00:00Z") + .execute(app_state.db.pool()) + .await + .unwrap(); + + // Seed another repo with NO path-scoped rules for visible pagination. + sqlx::query( + "INSERT INTO repos (id, name, owner_did, description, is_public, default_branch, created_at, updated_at, disk_path) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", + ) + .bind("repo-ipfs-vis") + .bind("ipfsvis") + .bind("did:key:z6Mkwowner") + .bind("desc") + .bind(true) + .bind("main") + .bind("2026-07-03T00:00:00Z") + .bind("2026-07-03T00:00:00Z") + .bind("/srv/ipfsvis") + .execute(app_state.db.pool()) + .await + .unwrap(); + + // Insert 1 visible pin, then a 2005-pin hidden stretch, then 1 visible pin. + // The hidden pins go in `ipfstest` (which has a deny rule and no physical repo so allowed_blobs is empty). + // The visible pins go in `ipfsvis` (which has no rules, so they are always visible). + // Note: three separate execute calls — sqlx prepared statements do not + // support multiple semicolon-delimited statements in a single query(). + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo, owner_did) + VALUES ('vis-1-sha', 'vis-1-cid', '2026-07-03T10:00:00Z', 'z6Mkwowner/ipfsvis', 'did:key:z6Mkwowner')", + ) + .execute(app_state.db.pool()) + .await + .unwrap(); + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo, owner_did) + SELECT 'hid-sha-' || i, 'hid-cid-' || i, '2026-07-03T09:00:00Z', 'z6Mkwowner/ipfstest', 'did:key:z6Mkwowner' + FROM generate_series(1, 2005) as i", + ) + .execute(app_state.db.pool()) + .await + .unwrap(); + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo, owner_did) + VALUES ('vis-2-sha', 'vis-2-cid', '2026-07-03T08:00:00Z', 'z6Mkwowner/ipfsvis', 'did:key:z6Mkwowner')", + ) + .execute(app_state.db.pool()) + .await + .unwrap(); + + // Visible pagination case asserting the cursor equals the last returned row + let auth = Extension(AuthenticatedDid("did:key:z6Mkcaller".to_string())); + let mut q = ListPinsQuery { + limit: 1, + cursor: None, + truncated_cursor: None, + }; + + let res1 = list_pins( + State(app_state.clone()), + Query(q.clone()), + Some(auth.clone()), + ) + .await + .unwrap() + .0; + let pins1 = res1["pins"].as_array().unwrap(); + assert_eq!(pins1.len(), 1); + assert_eq!(pins1[0]["sha256_hex"], "vis-1-sha"); + + let cursor1 = res1["next_cursor"].as_str().unwrap().to_string(); + // Decode to ensure it equals the last returned row + let bytes = URL_SAFE_NO_PAD.decode(cursor1.as_bytes()).unwrap(); + let decoded = String::from_utf8(bytes).unwrap(); + assert!(decoded.contains("vis-1-sha")); + + // Case 2: Follow the cursor. The next 2005 rows are hidden. + // It will hit MAX_BATCHES (2000 rows) and return a truncated_cursor + // whose XChaCha20Poly1305-encrypted payload conceals the hidden SHA. + q.cursor = Some(cursor1); + let res2 = list_pins( + State(app_state.clone()), + Query(q.clone()), + Some(auth.clone()), + ) + .await + .unwrap() + .0; + assert!(res2.get("pins").unwrap().as_array().unwrap().is_empty()); + assert_eq!(res2.get("truncated").unwrap().as_bool(), Some(true)); + assert!(res2.get("next_cursor").is_none()); + let truncated_cursor = res2["truncated_cursor"] + .as_str() + .expect("truncated_cursor should be present") + .to_string(); + + // Case 3: Resume with truncated_cursor. It should skip past the hidden + // batch and reach the older visible pin (vis-2-sha at 08:00:00Z). + q.cursor = None; + q.truncated_cursor = Some(truncated_cursor); + let res3 = list_pins( + State(app_state.clone()), + Query(q.clone()), + Some(auth.clone()), + ) + .await + .unwrap() + .0; + let pins3 = res3["pins"].as_array().unwrap(); + assert!( + !pins3.is_empty(), + "must surface vis-2-sha behind hidden window" + ); + assert_eq!(pins3[0]["sha256_hex"], "vis-2-sha"); + } + + #[test] + fn test_truncated_cursor_does_not_leak_hidden_sha() { + // The token is AEAD-encrypted with XChaCha20Poly1305: the hidden + // sha256_hex must NOT be recoverable by a caller who knows the + // pinned_at and repo prefix. Unlike a stream-cipher XOR construction + // (where known plaintext at offset i reveals keystream[i] via + // keystream[i] = ciphertext[i] XOR plaintext[i]), the AEAD ciphertext + // is ChaCha20 encryption with a per-nonce block counter applied to + // 16-byte blocks, then authenticated by Poly1305 — so XOR at a single + // offset does not yield a reusable keystream byte and the tag prevents + // any chosen-ciphertext oracle. + // + // This test demonstrates the unrecoverability property by attempting a + // known-plaintext attack against the ciphertext suffix. + let seed = [0xab; 32]; // arbitrary test seed + let pinned_at = "2026-07-03T09:00:00Z"; + let hidden_sha = "ab".repeat(32); // 64-char hex — well-known hidden SHA + + let cursor = format!("{pinned_at}|{hidden_sha}"); + let token = create_opaque_cursor(&seed, &cursor); + + // Decode the raw token bytes — these are (nonce_24 || ciphertext). + let raw = URL_SAFE_NO_PAD.decode(token.as_bytes()).unwrap(); + let (_nonce, ciphertext) = raw.split_at(24); + + // Known plaintext: the first 19 chars of pinned_at "2026-07-03T09:00:00Z" + // plus "|" = 20 bytes we know at the start. + // In the XOR-from-stream-cipher world, XOR of known plaintext with the + // ciphertext yields the keystream for those positions. If the keystream + // were reused at the sha suffix (modulo 32), XOR of known suffix with + // the recovered keystream would yield the hidden sha. + let known_prefix = format!("{pinned_at}|"); + let known_bytes = known_prefix.as_bytes(); + + let attempted_keystream: Vec = known_bytes + .iter() + .zip(ciphertext.iter()) + .map(|(p, c)| p ^ c) + .collect(); + + // Use the "recovered keystream" at the same positions in the suffix + // (which would be valid only with a repeating XOR keystream). The + // suffix is the last 64 bytes of the ciphertext (hidden_sha length). + if ciphertext.len() >= known_bytes.len() + 64 { + let suffix_start = ciphertext.len() - 64; + let attempted_sha: String = ciphertext[suffix_start..] + .iter() + .zip(attempted_keystream.iter().cycle()) + .map(|(c, k)| (c ^ k) as char) + .collect(); + + // With a real AEAD the "recovered" suffix is garbage, not the sha. + assert_ne!( + attempted_sha, hidden_sha, + "XOR-based known-plaintext attack on AEAD must NOT recover the hidden sha" + ); + } + + // Substring check: the token bytes must not contain the sha256_hex in + // the clear. + let raw_str = std::str::from_utf8(&raw).unwrap_or(""); + assert!( + !raw_str.contains(&hidden_sha), + "truncated_cursor token MUST NOT contain hidden sha256_hex in the clear" + ); + + // Positive round-trip: correct seed decodes the full cursor. + let decoded = decode_opaque_cursor(&seed, &token).unwrap(); + assert_eq!(decoded.0, pinned_at); + assert_eq!(decoded.1, hidden_sha); + + // Wrong key must not decode. + let wrong_seed = [0xcd; 32]; + assert!(decode_opaque_cursor(&wrong_seed, &token).is_none()); + } + + #[sqlx::test] + async fn test_max_walks_plaintext_not_in_response_cursor(pool: PgPool) { + let app_state = test_state(pool.clone()).await; + + // ── Create one visible repo (no path-scoped rules) ──────────────── + sqlx::query( + "INSERT INTO repos (id, name, owner_did, description, is_public, default_branch, created_at, updated_at, disk_path) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", + ) + .bind("repo-walks-vis") + .bind("walksvis") + .bind("did:key:z6Mkwowner") + .bind("visible") + .bind(true) + .bind("main") + .bind("2026-07-03T00:00:00Z") + .bind("2026-07-03T00:00:00Z") + .bind("/srv/walksvis") + .execute(app_state.db.pool()) + .await + .unwrap(); + + // ── Seed > MAX_WALKS (50) path-scoped repos with hidden pins ───── + let num_wall_repos = 55usize; + for i in 0..num_wall_repos { + let repo_id = format!("repo-wall-{i}"); + let repo_name = format!("wall{i}"); + let disk_path = format!("/srv/{repo_name}"); + sqlx::query( + "INSERT INTO repos (id, name, owner_did, description, is_public, default_branch, created_at, updated_at, disk_path) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)", + ) + .bind(&repo_id) + .bind(&repo_name) + .bind("did:key:z6Mkwowner") + .bind("desc") + .bind(true) + .bind("main") + .bind("2026-07-03T00:00:00Z") + .bind("2026-07-03T00:00:00Z") + .bind(&disk_path) + .execute(app_state.db.pool()) + .await + .unwrap(); + + // Add a /secret/** deny rule so the repo is path-scoped. + sqlx::query( + "INSERT INTO visibility_rules (id, repo_id, path_glob, mode, reader_dids, created_by, created_at) + VALUES ($1, $2, $3, $4, $5, $6, $7)", + ) + .bind(format!("rule-wall-{i}")) + .bind(&repo_id) + .bind("/secret/**") + .bind("deny") + .bind("") + .bind("did:key:z6Mkwowner") + .bind("2026-07-03T00:00:00Z") + .execute(app_state.db.pool()) + .await + .unwrap(); + + // One hidden pin per wall repo. + let sha = format!("wallsha{i:04}"); + let slug = format!("z6Mkwowner/{repo_name}"); + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo, owner_did) + VALUES ($1, $2, $3, $4, $5)", + ) + .bind(&sha) + .bind(format!("cid-wall-{i}")) + .bind("2026-07-03T09:00:00Z") + .bind(&slug) + .bind("did:key:z6Mkwowner") + .execute(app_state.db.pool()) + .await + .unwrap(); + } + + // ── One visible pin (newest timestamp so it appears first) ──────── + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo, owner_did) + VALUES ('vis-walks-sha', 'vis-walks-cid', '2026-07-03T10:00:00Z', 'z6Mkwowner/walksvis', 'did:key:z6Mkwowner')", + ) + .execute(app_state.db.pool()) + .await + .unwrap(); + + let auth = Extension(AuthenticatedDid("did:key:z6Mkstranger".to_string())); + let res = list_pins( + State(app_state.clone()), + Query(ListPinsQuery { + limit: 50, + cursor: None, + truncated_cursor: None, + }), + Some(auth), + ) + .await + .unwrap() + .0; + + // The visible pin (newest) must be returned. + let pins = res["pins"].as_array().unwrap(); + assert_eq!(pins.len(), 1, "must return the visible pin"); + assert_eq!(pins[0]["sha256_hex"], "vis-walks-sha"); + + // The page is truncated (not filled) because MAX_WALKS was hit. + assert_eq!(res.get("truncated").and_then(|v| v.as_bool()), Some(true)); + // next_cursor IS present — it points to the VISIBLE pin shown to the + // caller (no leak). When response_cursor holds a visible pin the + // plaintext cursor is safe; the P1 leak only happened when skip_pos + // (an un-walked hidden pin) was put in response_cursor. + let nc = res["next_cursor"] + .as_str() + .expect("next_cursor must be present for visible pin pagination"); + let bytes = URL_SAFE_NO_PAD.decode(nc.as_bytes()).unwrap(); + let decoded = String::from_utf8(bytes).unwrap(); + assert!( + decoded.contains("vis-walks-sha"), + "next_cursor must reference the visible pin, not a hidden SHA: {decoded}" + ); + // No truncated_cursor — next_cursor handles pagination. + assert!( + res.get("truncated_cursor").is_none(), + "truncated_cursor must NOT be present when next_cursor suffices" + ); + + // ── Second request: skip past the visible pin into the hidden wall ── + // The response must use the AEAD token (no plaintext next_cursor) + // because no visible pin is in the returned batch. + let auth = Extension(AuthenticatedDid("did:key:z6Mkstranger".to_string())); + let res2 = list_pins( + State(app_state.clone()), + Query(ListPinsQuery { + limit: 50, + cursor: Some(nc.to_string()), + truncated_cursor: None, + }), + Some(auth), + ) + .await + .unwrap() + .0; + + let pins2 = res2["pins"].as_array().unwrap(); + assert!(pins2.is_empty(), "second page has no visible pins"); + assert_eq!(res2.get("truncated").and_then(|v| v.as_bool()), Some(true)); + // next_cursor must NOT be present — no visible pin in this batch. + assert!( + res2.get("next_cursor").is_none(), + "next_cursor must not be present when no visible pin is returned" + ); + // truncated_cursor MUST be present and AEAD-encrypted. + let token = res2["truncated_cursor"] + .as_str() + .expect("truncated_cursor must be present for hidden-only page"); + for i in 0..num_wall_repos { + let sha = format!("wallsha{i:04}"); + assert!( + !token.contains(&sha), + "truncated_cursor must not contain hidden sha256_hex in the clear: {sha}" + ); + } + } + + #[sqlx::test] + async fn test_structural_pin_included_withheld_blob_excluded(pool: PgPool) { + let app_state = test_state(pool.clone()).await; + + // ── Create a real on-disk bare repo with objects ────────────────── + let owner_did = "did:key:z6Mkwowner"; + let repo_name = "structest"; + let owner_slug = owner_did.replace([':', '/'], "_"); + let repo_path = std::path::PathBuf::from("/tmp") + .join(&owner_slug) + .join(format!("{repo_name}.git")); + + // Remove leftovers from a prior failed run, then init a bare repo. + let _ = std::fs::remove_dir_all(&repo_path); + crate::git::store::init_bare(&repo_path).unwrap(); + + // Create a blob: echo -n "secret content" | git hash-object -w --stdin + let mut blob_child = Command::new("git") + .args([ + "-C", + repo_path.to_str().unwrap(), + "hash-object", + "-w", + "--stdin", + ]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + blob_child + .stdin + .as_mut() + .unwrap() + .write_all(b"secret content") + .unwrap(); + // Drop stdin to close it so hash-object can finish. + drop(blob_child.stdin.take()); + let blob_output = blob_child.wait_with_output().unwrap(); + assert!( + blob_output.status.success(), + "git hash-object failed: {}", + String::from_utf8_lossy(&blob_output.stderr) + ); + let blob_sha = String::from_utf8_lossy(&blob_output.stdout) + .trim() + .to_string(); + assert!(!blob_sha.is_empty(), "blob sha must not be empty"); + + // Create a sub-tree for "secret/" containing the blob at "file.txt" + let sub_tree_input = format!("100644 blob {blob_sha}\tfile.txt"); + let mut sub_tree_child = Command::new("git") + .args(["-C", repo_path.to_str().unwrap(), "mktree"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + sub_tree_child + .stdin + .as_mut() + .unwrap() + .write_all(sub_tree_input.as_bytes()) + .unwrap(); + drop(sub_tree_child.stdin.take()); + let sub_tree_output = sub_tree_child.wait_with_output().unwrap(); + assert!( + sub_tree_output.status.success(), + "git mktree for secret/ failed: {}", + String::from_utf8_lossy(&sub_tree_output.stderr) + ); + let sub_tree_sha = String::from_utf8_lossy(&sub_tree_output.stdout) + .trim() + .to_string(); + assert!(!sub_tree_sha.is_empty(), "sub-tree sha must not be empty"); + + // Create the root tree containing the secret/ sub-tree at path "secret" + let root_tree_input = format!("040000 tree {sub_tree_sha}\tsecret"); + let mut root_tree_child = Command::new("git") + .args(["-C", repo_path.to_str().unwrap(), "mktree"]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + root_tree_child + .stdin + .as_mut() + .unwrap() + .write_all(root_tree_input.as_bytes()) + .unwrap(); + drop(root_tree_child.stdin.take()); + let root_tree_output = root_tree_child.wait_with_output().unwrap(); + assert!( + root_tree_output.status.success(), + "git mktree for root tree failed: {}", + String::from_utf8_lossy(&root_tree_output.stderr) + ); + let tree_sha = String::from_utf8_lossy(&root_tree_output.stdout) + .trim() + .to_string(); + assert!(!tree_sha.is_empty(), "root tree sha must not be empty"); + + // Create a commit pointing to the tree + let commit_output = Command::new("git") + .args([ + "-C", + repo_path.to_str().unwrap(), + "commit-tree", + &tree_sha, + "-m", + "initial", + ]) + .env("GIT_AUTHOR_NAME", "test") + .env("GIT_AUTHOR_EMAIL", "test@test.com") + .env("GIT_COMMITTER_NAME", "test") + .env("GIT_COMMITTER_EMAIL", "test@test.com") + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .unwrap(); + assert!( + commit_output.status.success(), + "git commit-tree failed: {}", + String::from_utf8_lossy(&commit_output.stderr) + ); + let commit_sha = String::from_utf8_lossy(&commit_output.stdout) + .trim() + .to_string(); + assert!(!commit_sha.is_empty(), "commit sha must not be empty"); + + // Update HEAD so the blob walk can reach the blob. + // In a bare repo HEAD is a symref to refs/heads/main, so we update the ref. + let update_output = Command::new("git") + .args([ + "-C", + repo_path.to_str().unwrap(), + "update-ref", + "refs/heads/main", + &commit_sha, + ]) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .output() + .unwrap(); + assert!( + update_output.status.success(), + "git update-ref failed: {}", + String::from_utf8_lossy(&update_output.stderr) + ); + + // ── Seed the DB ─────────────────────────────────────────────────── + // Slug must match what list_pins computes from normalize_owner_key: + // normalize_owner_key("did:key:z6Mkwowner") = "z6Mkwowner" + // slug = "z6Mkwowner/structest" + let repo_slug = format!("z6Mkwowner/{repo_name}"); + sqlx::query( + "INSERT INTO repos (id, name, owner_did, description, is_public, default_branch, created_at, updated_at, disk_path) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)" + ) + .bind("repo-structest") + .bind(repo_name) + .bind(owner_did) + .bind("structural test repo") + .bind(true) + .bind("main") + .bind("2026-07-03T00:00:00Z") + .bind("2026-07-03T00:00:00Z") + .bind(repo_path.to_str().unwrap()) + .execute(app_state.db.pool()) + .await + .unwrap(); + + // Add a /secret/** deny rule so the blob is withheld from strangers. + sqlx::query( + "INSERT INTO visibility_rules (id, repo_id, path_glob, mode, reader_dids, created_by, created_at) + VALUES ($1, $2, $3, $4, $5, $6, $7)" + ) + .bind("rule-structest") + .bind("repo-structest") + .bind("/secret/**") + .bind("deny") + .bind("") + .bind("did:key:z6Mkwowner") + .bind("2026-07-03T00:00:00Z") + .execute(app_state.db.pool()) + .await + .unwrap(); + + // Pin the blob (must be withheld under /secret/**). + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo, owner_did) + VALUES ($1, $2, $3, $4, $5)", + ) + .bind(&blob_sha) + .bind("blob-cid") + .bind("2026-07-03T12:00:00Z") + .bind(&repo_slug) + .bind(owner_did) + .execute(app_state.db.pool()) + .await + .unwrap(); + + // Pin the tree (structural — must be visible). + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo, owner_did) + VALUES ($1, $2, $3, $4, $5)", + ) + .bind(&tree_sha) + .bind("tree-cid") + .bind("2026-07-03T11:00:00Z") + .bind(&repo_slug) + .bind(owner_did) + .execute(app_state.db.pool()) + .await + .unwrap(); + + // Pin the commit (structural — must be visible). + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo, owner_did) + VALUES ($1, $2, $3, $4, $5)", + ) + .bind(&commit_sha) + .bind("commit-cid") + .bind("2026-07-03T10:00:00Z") + .bind(&repo_slug) + .bind(owner_did) + .execute(app_state.db.pool()) + .await + .unwrap(); + + // ── Call list_pins as a stranger ────────────────────────────────── + let stranger = Extension(AuthenticatedDid("did:key:z6Mkstranger".to_string())); + let res = list_pins( + State(app_state.clone()), + Query(ListPinsQuery { + limit: 50, + cursor: None, + truncated_cursor: None, + }), + Some(stranger), + ) + .await + .unwrap() + .0; + + let pins = res["pins"].as_array().unwrap(); + let sha_hexes: Vec<&str> = pins + .iter() + .filter_map(|p| p["sha256_hex"].as_str()) + .collect(); + + // The withheld blob at /secret/** must NOT appear. + assert!( + !sha_hexes.contains(&blob_sha.as_str()), + "withheld blob pin under /secret/** must NOT appear for stranger" + ); + // The tree and commit are structural objects not in the blob set — + // they MUST appear (KTD3). + assert!( + sha_hexes.contains(&tree_sha.as_str()), + "structural tree pin must appear for stranger" + ); + assert!( + sha_hexes.contains(&commit_sha.as_str()), + "structural commit pin must appear for stranger" + ); + + // Clean up the on-disk repo. + let _ = std::fs::remove_dir_all(&repo_path); + } + + #[sqlx::test] + async fn test_stranger_denied_private_repo_pins(pool: PgPool) { + let app_state = test_state(pool.clone()).await; + + // Seed a fully private repo (is_public = false). + sqlx::query( + "INSERT INTO repos (id, name, owner_did, description, is_public, default_branch, created_at, updated_at, disk_path) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)" + ) + .bind("repo-private") + .bind("privaterepo") + .bind("did:key:z6Mkwowner") + .bind("private repo") + .bind(false) + .bind("main") + .bind("2026-07-03T00:00:00Z") + .bind("2026-07-03T00:00:00Z") + .bind("/srv/privaterepo") + .execute(app_state.db.pool()) + .await + .unwrap(); + + // Insert a pin owned by the owner. + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo, owner_did) + VALUES ('priv-sha-1', 'priv-cid-1', '2026-07-03T12:00:00Z', 'z6Mkwowner/privaterepo', 'did:key:z6Mkwowner')", + ) + .execute(app_state.db.pool()) + .await + .unwrap(); + + // A stranger (not the owner, not a listed reader) must see no pins. + let stranger_auth = Extension(AuthenticatedDid("did:key:z6Mkstranger".to_string())); + let res = list_pins( + State(app_state.clone()), + Query(ListPinsQuery { + limit: 50, + cursor: None, + truncated_cursor: None, + }), + Some(stranger_auth), + ) + .await + .unwrap() + .0; + assert_eq!( + res["pins"].as_array().unwrap().len(), + 0, + "stranger must not see pins from a private repo" + ); + assert_eq!(res["count"].as_u64().unwrap(), 0); + } + + #[sqlx::test] + async fn test_orphan_empty_repo_pins_excluded(pool: PgPool) { + let app_state = test_state(pool.clone()).await; + + // Seed a public repo (so the caller has some readable repo context). + sqlx::query( + "INSERT INTO repos (id, name, owner_did, description, is_public, default_branch, created_at, updated_at, disk_path) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)" + ) + .bind("repo-public") + .bind("pubrepo") + .bind("did:key:z6Mkwowner") + .bind("public repo") + .bind(true) + .bind("main") + .bind("2026-07-03T00:00:00Z") + .bind("2026-07-03T00:00:00Z") + .bind("/srv/pubrepo") + .execute(app_state.db.pool()) + .await + .unwrap(); + + // Insert a legit pin for the public repo. + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo, owner_did) + VALUES ('legit-sha', 'legit-cid', '2026-07-03T12:00:00Z', 'z6Mkwowner/pubrepo', 'did:key:z6Mkwowner')", + ) + .execute(app_state.db.pool()) + .await + .unwrap(); + + // Insert a legacy orphan pin with repo = '' (empty string) and owner_did = ''. + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo, owner_did) + VALUES ('orphan-sha', 'orphan-cid', '2026-07-03T11:00:00Z', '', '')", + ) + .execute(app_state.db.pool()) + .await + .unwrap(); + + // Signed caller must see the legit pin but NOT the orphan. + let auth = Extension(AuthenticatedDid("did:key:z6Mkcaller".to_string())); + let res = list_pins( + State(app_state.clone()), + Query(ListPinsQuery { + limit: 50, + cursor: None, + truncated_cursor: None, + }), + Some(auth), + ) + .await + .unwrap() + .0; + let pins = res["pins"].as_array().unwrap(); + let sha_hexes: Vec<&str> = pins + .iter() + .filter_map(|p| p["sha256_hex"].as_str()) + .collect(); + assert!(sha_hexes.contains(&"legit-sha"), "legit pin must appear"); + assert!( + !sha_hexes.contains(&"orphan-sha"), + "orphan pin with repo='' must NOT appear" + ); + } + + /// Verifies the non-sybil (global) rate limiter sheds requests after + /// its cap is reached, even across distinct DIDs (P3). + #[sqlx::test] + async fn global_rate_limiter_sheds_after_budget_exhausted(pool: PgPool) { + let mut state = test_state(pool).await; + // Tighten the global limiter to max 2 with a singleton map so + // rotating DIDs cannot bypass the cap. + state.ipfs_list_global_limiter = + crate::rate_limit::RateLimiter::new_bounded(2, std::time::Duration::from_secs(3600), 1); + + // First two requests with distinct DIDs are within budget. + let r1 = list_pins( + State(state.clone()), + Query(ListPinsQuery { + limit: 1, + cursor: None, + truncated_cursor: None, + }), + Some(Extension(AuthenticatedDid("did:key:z6MkwA".into()))), + ) + .await; + assert!( + r1.is_ok() || matches!(r1, Err(AppError::Unauthorized(_))), + "first caller should not be refused by global limiter, got {r1:?}", + ); + + let r2 = list_pins( + State(state.clone()), + Query(ListPinsQuery { + limit: 1, + cursor: None, + truncated_cursor: None, + }), + Some(Extension(AuthenticatedDid("did:key:z6MkwB".into()))), + ) + .await; + assert!( + r2.is_ok() || matches!(r2, Err(AppError::Unauthorized(_))), + "second caller should not be refused by global limiter, got {r2:?}", + ); + + // Third request with a fresh DID — global bucket is empty. + let r3 = list_pins( + State(state.clone()), + Query(ListPinsQuery { + limit: 1, + cursor: None, + truncated_cursor: None, + }), + Some(Extension(AuthenticatedDid("did:key:z6MkwC".into()))), + ) + .await; + assert!( + matches!(r3, Err(AppError::TooManyRequests(_))), + "third caller should be refused by global limiter, got {r3:?}", + ); + } + + /// Single DID that exhausts its per-DID budget does NOT drain the + /// shared global bucket — the global check is charged only after the + /// per-DID check passes (P2, P3). + #[sqlx::test] + async fn single_did_over_budget_does_not_drain_global(pool: PgPool) { + let mut state = test_state(pool).await; + // Per-DID limit of 1 so the second request from the same DID is + // refused before the global limiter is charged. + state.ipfs_list_rate_limiter = crate::rate_limit::RateLimiter::new_bounded( + 1, + std::time::Duration::from_secs(3600), + 200_000, + ); + // Global limit of 3 — generous enough that two distinct DIDs can + // both pass even if the over-budget DID had drained the bucket. + state.ipfs_list_global_limiter = + crate::rate_limit::RateLimiter::new_bounded(3, std::time::Duration::from_secs(3600), 1); + + // DID A — first request passes per-DID and charges global. + let r1 = list_pins( + State(state.clone()), + Query(ListPinsQuery { + limit: 1, + cursor: None, + truncated_cursor: None, + }), + Some(Extension(AuthenticatedDid("did:key:z6MkwX".into()))), + ) + .await; + assert!( + r1.is_ok() || matches!(&r1, Err(AppError::Unauthorized(_))), + "DID A first request should not be refused, got {r1:?}", + ); + + // DID A — second request is refused by per-DID limiter (budget + // exhausted), BEFORE the global bucket would be charged. + let r2 = list_pins( + State(state.clone()), + Query(ListPinsQuery { + limit: 1, + cursor: None, + truncated_cursor: None, + }), + Some(Extension(AuthenticatedDid("did:key:z6MkwX".into()))), + ) + .await; + assert!( + matches!(r2, Err(AppError::TooManyRequests(_))), + "DID A second request should get per-DID 429, got {r2:?}", + ); + + // DID B — should still pass because the global bucket was charged + // only once (by DID A's first request, which passed per-DID). + let r3 = list_pins( + State(state.clone()), + Query(ListPinsQuery { + limit: 1, + cursor: None, + truncated_cursor: None, + }), + Some(Extension(AuthenticatedDid("did:key:z6MkwY".into()))), + ) + .await; + assert!( + r3.is_ok() || matches!(&r3, Err(AppError::Unauthorized(_))), + "DID B should not be refused (global bucket has 2 of 3 remaining), got {r3:?}", + ); + } } diff --git a/crates/gitlawb-node/src/api/repos.rs b/crates/gitlawb-node/src/api/repos.rs index b38b177b..28aaa2f2 100644 --- a/crates/gitlawb-node/src/api/repos.rs +++ b/crates/gitlawb-node/src/api/repos.rs @@ -3,6 +3,7 @@ use axum::http::StatusCode; use axum::response::Response; use axum::Json; use bytes::Bytes; +use std::sync::atomic::AtomicBool; use std::sync::Arc; use crate::auth::{caller_authorized_to_push, AuthenticatedDid}; @@ -65,8 +66,9 @@ async fn replication_withheld_set( Some(rules) => { let owner_did = owner_did.to_string(); tokio::task::spawn_blocking(move || { + let uncancelled = AtomicBool::new(false); crate::git::visibility_pack::withheld_blob_oids( - &disk_path, &rules, is_public, &owner_did, None, + &disk_path, &rules, is_public, &owner_did, None, &uncancelled, ) }) .await @@ -108,8 +110,9 @@ async fn fail_closed_full_scan_objects( candidates: Vec, ) -> Vec { tokio::task::spawn_blocking(move || -> anyhow::Result> { + let uncancelled = AtomicBool::new(false); let allowed = crate::git::visibility_pack::replicable_blob_set( - &disk_path, &rules, is_public, &owner_did, + &disk_path, &rules, is_public, &owner_did, &uncancelled, )?; let all_blobs = crate::git::push_delta::all_blob_oids(&disk_path)?; Ok(crate::git::visibility_pack::replicable_objects_fail_closed( @@ -659,12 +662,14 @@ pub async fn git_upload_pack( let caller_owned = caller.map(str::to_string); let is_public = record.is_public; tokio::task::spawn_blocking(move || { + let uncancelled = AtomicBool::new(false); visibility_pack::withheld_blob_oids( &path, &rules, is_public, &owner_did, caller_owned.as_deref(), + &uncancelled, ) }) .await @@ -1114,18 +1119,22 @@ pub async fn git_receive_pack( let rules_for_enc = rules_opt.clone(); let repo_id = record.id.clone(); let owner_did = record.owner_did.clone(); + let owner_short = crate::db::normalize_owner_key(&record.owner_did).to_string(); let is_public = record.is_public; let irys_url = state.config.irys_url.clone(); let http_client = std::sync::Arc::clone(&state.http_client); let node_did_str = state.node_did.to_string(); let node_seed = state.node_keypair.to_seed(); let repo_name = record.name.clone(); + let repo_slug = format!("{owner_short}/{repo_name}"); tokio::spawn(async move { let pinned = crate::ipfs_pin::pin_new_objects( &ipfs_api, &repo_path_clone, object_list_ipfs, &db_clone, + &repo_slug, + &owner_did, ) .await; if !pinned.is_empty() { @@ -1145,8 +1154,13 @@ pub async fn git_receive_pack( let p = repo_path_clone.clone(); let owner = owner_did.clone(); let recip = tokio::task::spawn_blocking(move || { + let uncancelled = AtomicBool::new(false); crate::git::visibility_pack::withheld_blob_recipients( - &p, &rules, is_public, &owner, + &p, + &rules, + is_public, + &owner, + &uncancelled, ) }) .await; @@ -1213,6 +1227,7 @@ pub async fn git_receive_pack( crate::db::normalize_owner_key(&record.owner_did), record.name ); + let owner_did_for_pinata = record.owner_did.clone(); let ref_updates_clone = ref_updates .iter() .map(|u| (u.ref_name.clone(), u.old_sha.clone(), u.new_sha.clone())) @@ -1236,6 +1251,8 @@ pub async fn git_receive_pack( &repo_path_clone, object_list_pinata, &db_clone, + &repo_slug, + &owner_did_for_pinata, ) .await } else { diff --git a/crates/gitlawb-node/src/auth/mod.rs b/crates/gitlawb-node/src/auth/mod.rs index 720fb3ae..6234873c 100644 --- a/crates/gitlawb-node/src/auth/mod.rs +++ b/crates/gitlawb-node/src/auth/mod.rs @@ -519,6 +519,9 @@ mod tests { push_limiter_trust: crate::rate_limit::TrustedProxy::None, sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + walk_semaphore: Arc::new(tokio::sync::Semaphore::new(4)), + ipfs_list_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), + ipfs_list_global_limiter: RateLimiter::new_bounded(1200, Duration::from_secs(3600), 1), shutdown_tx: tokio::sync::watch::channel(false).0, } } diff --git a/crates/gitlawb-node/src/config.rs b/crates/gitlawb-node/src/config.rs index fc2247d9..7f735dfc 100644 --- a/crates/gitlawb-node/src/config.rs +++ b/crates/gitlawb-node/src/config.rs @@ -234,6 +234,29 @@ pub struct Config { value_parser = clap::value_parser!(u64).range(1..) )] pub db_retry_max_secs: u64, + + /// Maximum number of concurrent visibility walks (git rev-list / ls-tree) + /// across all IPFS pin listing requests. Prevents a flood of signed + /// requests from exhausting the blocking-pool worker or leaving git + /// children running past their timeout (P1). + #[arg(long, env = "GITLAWB_WALK_CONCURRENCY_LIMIT", default_value_t = 4, value_parser = clap::value_parser!(u32).range(1..))] + pub walk_concurrency_limit: u32, + + /// Per-DID rate limit for IPFS pin listing — requests per hour per DID. + /// The listing performs expensive git walks and cat-file probes, so a + /// throwaway DID with a valid signature can otherwise exhaust resources. + #[arg(long, env = "GITLAWB_IPFS_LIST_RATE_LIMIT", default_value_t = 60)] + pub ipfs_list_rate_limit: usize, + + /// Global (non-sybil) rate limit for IPFS pin listing — total requests + /// per hour regardless of signed DID. Prevents DID-rotation attacks + /// from bypassing the per-DID limiter (P1). + #[arg( + long, + env = "GITLAWB_IPFS_LIST_GLOBAL_RATE_LIMIT", + default_value_t = 1200 + )] + pub ipfs_list_global_rate_limit: usize, } impl Config { diff --git a/crates/gitlawb-node/src/db/mod.rs b/crates/gitlawb-node/src/db/mod.rs index 33f5bd67..6d3dcb0a 100644 --- a/crates/gitlawb-node/src/db/mod.rs +++ b/crates/gitlawb-node/src/db/mod.rs @@ -160,6 +160,8 @@ pub struct PinnedCidRecord { pub cid: String, pub pinned_at: String, pub pinata_cid: Option, + pub repo: String, + pub owner_did: String, } #[derive(Debug, Clone, Serialize, Deserialize)] @@ -678,6 +680,15 @@ const MIGRATIONS: &[Migration] = &[ )"#, "CREATE INDEX IF NOT EXISTS idx_repo_replicas_repo ON repo_replicas(repo_id)", "CREATE INDEX IF NOT EXISTS idx_repo_replicas_did ON repo_replicas(replica_did)", + // ── Pinned CID repos (junction table for shared-object associations) ── + r#"CREATE TABLE IF NOT EXISTS pinned_cid_repos ( + sha256_hex TEXT NOT NULL, + repo TEXT NOT NULL, + owner_did TEXT NOT NULL, + pinned_at TEXT NOT NULL, + PRIMARY KEY (sha256_hex, repo) + )"#, + "CREATE INDEX IF NOT EXISTS idx_pinned_cid_repos_repo ON pinned_cid_repos(repo, owner_did)", // ── PR comments ───────────────────────────────────────────────── r#"CREATE TABLE IF NOT EXISTS pr_comments ( id TEXT NOT NULL PRIMARY KEY, @@ -856,6 +867,7 @@ const MIGRATIONS: &[Migration] = &[ "ALTER TABLE repos ADD COLUMN IF NOT EXISTS quarantined BOOLEAN NOT NULL DEFAULT FALSE", ], }, + Migration { version: 10, name: "ref_cert_unique_per_ref", @@ -883,6 +895,86 @@ const MIGRATIONS: &[Migration] = &[ "ALTER TABLE received_ref_updates ADD COLUMN IF NOT EXISTS owner_did TEXT", ], }, + Migration { + version: 12, + name: "pinned_cids_repo_owner", + stmts: &[ + "ALTER TABLE pinned_cids ADD COLUMN IF NOT EXISTS repo TEXT", + "ALTER TABLE pinned_cids ADD COLUMN IF NOT EXISTS owner_did TEXT", + // Backfill repo/owner only when both the CID AND the Git SHA match + // between pinned_cids and branch_cids. The CID alone does not encode + // the Git object type, so a private blob and a public commit with the + // same raw bytes would share a CID, and the plain CID-join would + // wrongly assign the private pin to the public repo (P1). Requiring + // p.sha256_hex = bc.sha ensures only the ref-target objects + // (commits) are matched — blobs/trees fall through to the empty- + // string fallback below, which is safer than a wrong assignment. + r#"UPDATE pinned_cids p + SET repo = m.repo, + owner_did = m.owner_did + FROM ( + SELECT DISTINCT + bc.cid, + bc.sha, + bc.repo, + r.owner_did + FROM branch_cids bc + JOIN repos r + ON r.name = split_part(bc.repo, '/', 2) + AND (CASE WHEN r.owner_did LIKE 'did:key:%' AND position(':' in substr(r.owner_did, 9)) = 0 THEN substr(r.owner_did, 9) ELSE r.owner_did END) + = split_part(bc.repo, '/', 1) + ) m + WHERE p.cid = m.cid + AND p.sha256_hex = m.sha"#, + // Fallback for remaining rows + "UPDATE pinned_cids SET repo = '' WHERE repo IS NULL", + "UPDATE pinned_cids SET owner_did = '' WHERE owner_did IS NULL", + // Default the new columns so a pre-v11 binary still running during + // a rolling deploy can INSERT (sha256_hex, cid, pinned_at) without + // hitting a NOT NULL violation (P2). + "ALTER TABLE pinned_cids ALTER COLUMN repo SET DEFAULT ''", + "ALTER TABLE pinned_cids ALTER COLUMN owner_did SET DEFAULT ''", + "ALTER TABLE pinned_cids ALTER COLUMN repo SET NOT NULL", + "ALTER TABLE pinned_cids ALTER COLUMN owner_did SET NOT NULL", + // New unique constraint for post-v11 ON CONFLICT(repo, sha256_hex) + "CREATE UNIQUE INDEX IF NOT EXISTS pinned_cids_repo_sha_hex_key ON pinned_cids (repo, sha256_hex)", + // Old PK on sha256_hex kept intact for pre-v11 ON CONFLICT(sha256_hex) + "CREATE INDEX IF NOT EXISTS idx_pinned_cids_repo_owner ON pinned_cids (repo, owner_did)", + ], + }, + Migration { + version: 13, + name: "arweave_anchors_repo_owner_index", + stmts: &[ + "CREATE INDEX IF NOT EXISTS idx_arweave_anchors_repo_owner_anchored ON arweave_anchors (repo, owner_did, anchored_at DESC)", + ], + }, + Migration { + version: 14, + name: "pinned_cid_repos_junction", + stmts: &[ + r#"CREATE TABLE IF NOT EXISTS pinned_cid_repos ( + sha256_hex TEXT NOT NULL, + repo TEXT NOT NULL, + owner_did TEXT NOT NULL, + pinned_at TEXT NOT NULL, + PRIMARY KEY (sha256_hex, repo) + )"#, + "CREATE INDEX IF NOT EXISTS idx_pinned_cid_repos_repo ON pinned_cid_repos(repo, owner_did)", + // Backfill from existing pinned_cids rows that have non-empty repo/owner + r#"INSERT INTO pinned_cid_repos (sha256_hex, repo, owner_did, pinned_at) + SELECT sha256_hex, repo, owner_did, pinned_at + FROM pinned_cids + WHERE repo IS NOT NULL AND repo != '' + AND owner_did IS NOT NULL AND owner_did != '' + ON CONFLICT (sha256_hex, repo) DO NOTHING"#, + // Note: legacy (unassociated) pinned_cids rows with empty repo + // or owner_did are intentionally not migrated to the junction + // table: the scoped listing requires a known (repo, owner_did) + // pair, and silently showing orphaned objects to every caller + // would leak SHA/CID pairs from before the migration., + ], + }, ]; // ── Repos ───────────────────────────────────────────────────────────────────── @@ -2177,15 +2269,80 @@ impl Db { Ok(row.get::("cnt") > 0) } - pub async fn record_pinned_cid(&self, sha256_hex: &str, cid: &str) -> Result<()> { + #[allow(dead_code)] + pub async fn get_pinned_cid(&self, sha256_hex: &str) -> Result> { + let row = sqlx::query("SELECT cid FROM pinned_cids WHERE sha256_hex = $1 LIMIT 1") + .bind(sha256_hex) + .fetch_optional(&self.pool) + .await?; + Ok(row.map(|r| r.get("cid"))) + } + + /// Record a pinned CID with explicit repo/owner_did association. + /// Phase 1 (expand): targets the kept sha256_hex PK so pre-v10 and + /// post-v10 writers share the same conflict target. Phase 2 (contract) + /// will switch to ON CONFLICT(repo, sha256_hex) after the old PK is + /// dropped and (repo, sha256_hex) becomes the new primary key. + pub async fn record_pinned_cid_full( + &self, + sha256_hex: &str, + cid: &str, + repo: &str, + owner_did: &str, + ) -> Result<()> { + let now = Utc::now().to_rfc3339(); sqlx::query( - "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) - VALUES ($1, $2, $3) - ON CONFLICT(sha256_hex) DO NOTHING", + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo, owner_did) + VALUES ($1, $2, $3, $4, $5) + ON CONFLICT(sha256_hex) DO UPDATE SET + repo = COALESCE(NULLIF(EXCLUDED.repo, ''), pinned_cids.repo), + owner_did = COALESCE(NULLIF(EXCLUDED.owner_did, ''), pinned_cids.owner_did)", ) .bind(sha256_hex) .bind(cid) - .bind(Utc::now().to_rfc3339()) + .bind(&now) + .bind(repo) + .bind(owner_did) + .execute(&self.pool) + .await?; + + // Also record the (repo, owner_did) association in the junction table + // so shared Git objects are visible to every repo's readers (P2). + if !repo.is_empty() && !owner_did.is_empty() { + sqlx::query( + "INSERT INTO pinned_cid_repos (sha256_hex, repo, owner_did, pinned_at) + VALUES ($1, $2, $3, $4) + ON CONFLICT (sha256_hex, repo) DO NOTHING", + ) + .bind(sha256_hex) + .bind(repo) + .bind(owner_did) + .bind(&now) + .execute(&self.pool) + .await?; + } + Ok(()) + } + + pub async fn update_pinned_cid_repo( + &self, + sha256_hex: &str, + repo: &str, + owner_did: &str, + ) -> Result<()> { + if repo.is_empty() || owner_did.is_empty() { + return Ok(()); + } + let now = Utc::now().to_rfc3339(); + sqlx::query( + "INSERT INTO pinned_cid_repos (sha256_hex, repo, owner_did, pinned_at) + VALUES ($1, $2, $3, $4) + ON CONFLICT (sha256_hex, repo) DO NOTHING", + ) + .bind(sha256_hex) + .bind(repo) + .bind(owner_did) + .bind(&now) .execute(&self.pool) .await?; Ok(()) @@ -2261,9 +2418,10 @@ impl Db { Ok(row.map(|r| r.get("recipients_tag"))) } + #[allow(dead_code)] pub async fn list_pinned_cids(&self) -> Result> { let rows = sqlx::query( - "SELECT sha256_hex, cid, pinned_at, pinata_cid FROM pinned_cids ORDER BY pinned_at DESC", + "SELECT sha256_hex, cid, pinned_at, pinata_cid, repo, owner_did FROM pinned_cids ORDER BY pinned_at DESC", ) .fetch_all(&self.pool) .await?; @@ -2274,6 +2432,97 @@ impl Db { cid: r.get("cid"), pinned_at: r.get("pinned_at"), pinata_cid: r.get("pinata_cid"), + repo: r.get("repo"), + owner_did: r.get("owner_did"), + }) + .collect()) + } + + #[allow(dead_code)] + pub async fn get_pinata_cid(&self, sha256_hex: &str) -> Result> { + let row = sqlx::query("SELECT pinata_cid FROM pinned_cids WHERE sha256_hex = $1 AND pinata_cid IS NOT NULL LIMIT 1") + .bind(sha256_hex) + .fetch_optional(&self.pool) + .await?; + Ok(row.map(|r| r.get("pinata_cid"))) + } + + /// Bounded global pin query: returns pins for any of the given (repo, owner_did) + /// pairs, ordered by pinned_at DESC, capped at `limit`. + /// Fetch pinned CIDs matching the readable (repo, owner_did) pairs, + /// ordered by (pinned_at DESC, sha256_hex DESC, repo DESC) for stable + /// keyset pagination. Multiple associations for the same SHA may be + /// returned (one per readable repo); the caller deduplicates by + /// sha256_hex after per-association visibility evaluation (P2). + /// When `cursor` is `Some((pinned_at, sha256_hex))`, only rows + /// strictly before that cursor are returned (no duplicates across + /// pages, since every association for a given SHA shares the same + /// (pinned_at, sha256_hex) pair). + pub async fn list_pinned_cids_for_repos( + &self, + repos: &[String], + owner_dids: &[String], + no_rules: &[bool], + limit: i64, + cursor: Option<(&str, &str)>, + ) -> Result> { + let rows = if let Some((pa, sha)) = cursor { + sqlx::query( + "SELECT p.sha256_hex, p.cid, p.pinned_at, p.pinata_cid, + COALESCE(pr.repo, p.repo) AS repo, + COALESCE(pr.owner_did, p.owner_did) AS owner_did + FROM pinned_cids p + LEFT JOIN pinned_cid_repos pr ON pr.sha256_hex = p.sha256_hex + JOIN UNNEST($1::text[], $2::text[], $3::bool[]) + AS pairs(repo, owner_did, no_rules) + ON (COALESCE(pr.repo, p.repo), COALESCE(pr.owner_did, p.owner_did)) + = (pairs.repo, pairs.owner_did) + WHERE (p.pinned_at, p.sha256_hex) + < ($4::text, $5::text) + ORDER BY p.pinned_at DESC, p.sha256_hex DESC, + COALESCE(pr.repo, p.repo) DESC + LIMIT $6", + ) + .bind(repos) + .bind(owner_dids) + .bind(no_rules) + .bind(pa) + .bind(sha) + .bind(limit) + .fetch_all(&self.pool) + .await? + } else { + sqlx::query( + "SELECT p.sha256_hex, p.cid, p.pinned_at, p.pinata_cid, + COALESCE(pr.repo, p.repo) AS repo, + COALESCE(pr.owner_did, p.owner_did) AS owner_did + FROM pinned_cids p + LEFT JOIN pinned_cid_repos pr ON pr.sha256_hex = p.sha256_hex + JOIN UNNEST($1::text[], $2::text[], $3::bool[]) + AS pairs(repo, owner_did, no_rules) + ON (COALESCE(pr.repo, p.repo), COALESCE(pr.owner_did, p.owner_did)) + = (pairs.repo, pairs.owner_did) + ORDER BY p.pinned_at DESC, p.sha256_hex DESC, + COALESCE(pr.repo, p.repo) DESC + LIMIT $4", + ) + .bind(repos) + .bind(owner_dids) + .bind(no_rules) + .bind(limit) + .fetch_all(&self.pool) + .await? + }; + + Ok(rows + .into_iter() + .map(|r| PinnedCidRecord { + sha256_hex: r.get("sha256_hex"), + cid: r.get("cid"), + pinned_at: r.get("pinned_at"), + pinata_cid: r.get("pinata_cid"), + repo: r.get("repo"), + owner_did: r.get("owner_did"), }) .collect()) } @@ -2290,20 +2539,44 @@ impl Db { } /// Record the Pinata CID for a git object. - /// Inserts the row if it doesn't exist (objects pinned directly to Pinata - /// without a prior local IPFS pin get cid = pinata_cid). - pub async fn record_pinata_cid(&self, sha256_hex: &str, pinata_cid: &str) -> Result<()> { + /// Record the Pinata CID with explicit repo/owner_did association. + pub async fn record_pinata_cid_full( + &self, + sha256_hex: &str, + pinata_cid: &str, + repo: &str, + owner_did: &str, + ) -> Result<()> { + let now = Utc::now().to_rfc3339(); sqlx::query( - "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid) - VALUES ($1, $2, $3, $4) - ON CONFLICT(sha256_hex) DO UPDATE SET pinata_cid = EXCLUDED.pinata_cid", + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, pinata_cid, repo, owner_did) + VALUES ($1, $2, $3, $4, $5, $6) + ON CONFLICT(sha256_hex) DO UPDATE SET pinata_cid = EXCLUDED.pinata_cid, + repo = COALESCE(NULLIF(EXCLUDED.repo, ''), pinned_cids.repo), + owner_did = COALESCE(NULLIF(EXCLUDED.owner_did, ''), pinned_cids.owner_did)", ) .bind(sha256_hex) .bind(pinata_cid) // fallback local cid if row is new - .bind(Utc::now().to_rfc3339()) + .bind(&now) .bind(pinata_cid) + .bind(repo) + .bind(owner_did) .execute(&self.pool) .await?; + + if !repo.is_empty() && !owner_did.is_empty() { + sqlx::query( + "INSERT INTO pinned_cid_repos (sha256_hex, repo, owner_did, pinned_at) + VALUES ($1, $2, $3, $4) + ON CONFLICT (sha256_hex, repo) DO NOTHING", + ) + .bind(sha256_hex) + .bind(repo) + .bind(owner_did) + .bind(&now) + .execute(&self.pool) + .await?; + } Ok(()) } } @@ -2737,6 +3010,50 @@ impl Db { }) .collect()) } + + /// List arweave anchors scoped to repos the caller can read. + /// Filtered by (repo, owner_did) pairs from the caller's readable set. + pub async fn list_arweave_anchors_for_repos( + &self, + repos: &[String], + owner_dids: &[String], + limit: i64, + ) -> Result> { + if repos.is_empty() { + return Ok(Vec::new()); + } + let rows = sqlx::query( + "SELECT id, repo, owner_did, ref_name, old_sha, new_sha, cid, irys_tx_id, arweave_url, node_did, anchored_at + FROM arweave_anchors + WHERE (repo, owner_did) IN ( + SELECT * FROM UNNEST($1::text[], $2::text[]) + ) + ORDER BY anchored_at DESC + LIMIT $3", + ) + .bind(repos) + .bind(owner_dids) + .bind(limit) + .fetch_all(&self.pool) + .await?; + + Ok(rows + .into_iter() + .map(|r| ArweaveAnchor { + id: r.get("id"), + repo: r.get("repo"), + owner_did: r.get("owner_did"), + ref_name: r.get("ref_name"), + old_sha: r.get("old_sha"), + new_sha: r.get("new_sha"), + cid: r.get("cid"), + irys_tx_id: r.get("irys_tx_id"), + arweave_url: r.get("arweave_url"), + node_did: r.get("node_did"), + anchored_at: r.get("anchored_at"), + }) + .collect()) + } } // ── Row helpers ─────────────────────────────────────────────────────────────── @@ -3467,7 +3784,8 @@ impl Db { #[cfg(test)] mod migration_tests { - use super::{MIGRATIONS, MIGRATION_V1_NAME}; + use super::{Db, MIGRATIONS, MIGRATION_V1_NAME}; + use sqlx::{PgPool, Row}; #[test] fn migrations_are_non_empty() { @@ -3654,6 +3972,282 @@ mod migration_tests { // (d) Re-run: idempotent — ADD COLUMN IF NOT EXISTS must not error. db.migrate().await.unwrap(); } + + #[sqlx::test] + async fn test_migration_v11_upgrade_path(pool: PgPool) { + let db = Db::for_testing(pool); + + // Run migrations up to version 9 + async fn run_migrations_up_to(db: &Db, version: i64) { + sqlx::query( + r#"CREATE TABLE IF NOT EXISTS schema_migrations ( + version BIGINT NOT NULL PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL + )"#, + ) + .execute(&db.pool) + .await + .unwrap(); + + for m in super::MIGRATIONS { + if m.version > version { + break; + } + let already: bool = sqlx::query( + "SELECT EXISTS(SELECT 1 FROM schema_migrations WHERE version = $1) AS applied", + ) + .bind(m.version) + .fetch_one(&db.pool) + .await + .unwrap() + .get::("applied"); + + if already { + continue; + } + + let mut tx = db.pool.begin().await.unwrap(); + for stmt in m.stmts { + sqlx::query(stmt).execute(&mut *tx).await.unwrap(); + } + sqlx::query( + "INSERT INTO schema_migrations (version, name, applied_at) VALUES ($1, $2, $3)", + ) + .bind(m.version) + .bind(m.name) + .bind(chrono::Utc::now().to_rfc3339()) + .execute(&mut *tx) + .await + .unwrap(); + tx.commit().await.unwrap(); + } + } + + run_migrations_up_to(&db, 9).await; + + // Seed a repo, branch_cids, and pinned_cids under v9 schema + sqlx::query( + "INSERT INTO repos (id, name, owner_did, description, is_public, default_branch, created_at, updated_at, disk_path) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)" + ) + .bind("repo-123") + .bind("myrepo") + .bind("did:key:z6Mkwowner") + .bind("desc") + .bind(true) + .bind("main") + .bind("2026-07-03T00:00:00Z") + .bind("2026-07-03T00:00:00Z") + .bind("/srv/repo-123") + .execute(&db.pool) + .await + .unwrap(); + + sqlx::query( + "INSERT INTO branch_cids (repo, ref_name, sha, cid, node_did, updated_at) + VALUES ($1, $2, $3, $4, $5, $6)", + ) + .bind("z6Mkwowner/myrepo") + .bind("refs/heads/main") + .bind("old-sha") + .bind("old-cid") + .bind("node-did") + .bind("2026-07-03T00:00:00Z") + .execute(&db.pool) + .await + .unwrap(); + + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) + VALUES ($1, $2, $3)", + ) + .bind("old-sha") + .bind("old-cid") + .bind("2026-07-03T00:00:00Z") + .execute(&db.pool) + .await + .unwrap(); + + // Run remaining migrations (v10 = ref_cert_dedup, v11 = pinned_cids) + db.run_migrations().await.unwrap(); + + // Verify backfilling of repo and owner_did columns + let row = sqlx::query( + "SELECT sha256_hex, cid, repo, owner_did FROM pinned_cids WHERE sha256_hex = 'old-sha'", + ) + .fetch_one(&db.pool) + .await + .unwrap(); + + assert_eq!(row.get::("repo"), "z6Mkwowner/myrepo"); + assert_eq!(row.get::("owner_did"), "did:key:z6Mkwowner"); + + // Phase 1 (expand): the old PK on sha256_hex still rejects duplicate + // SHA across repos — pre-v10 ON CONFLICT(sha256_hex) keeps working. + let res = sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo, owner_did) + VALUES ($1, $2, $3, $4, $5)", + ) + .bind("old-sha") + .bind("old-cid") + .bind("2026-07-03T00:00:00Z") + .bind("other-repo") + .bind("other-owner") + .execute(&db.pool) + .await; + + assert!( + res.is_err(), + "Phase 1: old PK on sha256_hex must reject duplicate SHA across repos" + ); + + // Phase 2 (contract): drop the old PK and UNIQUE, promote to compound PK. + // Once all pre-v10 writers are drained this step makes the migration + // complete — same SHA can appear in different repos. + sqlx::query("ALTER TABLE pinned_cids DROP CONSTRAINT pinned_cids_pkey") + .execute(&db.pool) + .await + .unwrap(); + sqlx::query("DROP INDEX IF EXISTS pinned_cids_repo_sha_hex_key") + .execute(&db.pool) + .await + .unwrap(); + sqlx::query("ALTER TABLE pinned_cids ADD PRIMARY KEY (repo, sha256_hex)") + .execute(&db.pool) + .await + .unwrap(); + + // Now the same SHA works in a different repo (compound PK allows it). + let res = sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at, repo, owner_did) + VALUES ($1, $2, $3, $4, $5)", + ) + .bind("old-sha") + .bind("old-cid") + .bind("2026-07-03T00:00:00Z") + .bind("other-repo") + .bind("other-owner") + .execute(&db.pool) + .await; + + assert!( + res.is_ok(), + "Phase 2: compound PK must allow same SHA in different repos" + ); + } + + /// A pinned CID whose SHA is not a current branch_cids ref tip falls back to + /// repo = '' after migration v11. This tests that the backfill does not + /// silently orphan such pins by leaving repo NULL/unqueryable; the empty + /// string is at least queryable by list_pinned_cids_for_repos callers. + #[sqlx::test] + async fn test_migration_v11_orphan_non_tip_pin(pool: PgPool) { + let db = Db::for_testing(pool); + + // Run migrations up to version 9 + async fn run_migrations_up_to(db: &Db, version: i64) { + sqlx::query( + r#"CREATE TABLE IF NOT EXISTS schema_migrations ( + version BIGINT NOT NULL PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL + )"#, + ) + .execute(&db.pool) + .await + .unwrap(); + + for m in super::MIGRATIONS { + if m.version > version { + break; + } + let already: bool = sqlx::query( + "SELECT EXISTS(SELECT 1 FROM schema_migrations WHERE version = $1) AS applied", + ) + .bind(m.version) + .fetch_one(&db.pool) + .await + .unwrap() + .get::("applied"); + + if already { + continue; + } + + let mut tx = db.pool.begin().await.unwrap(); + for stmt in m.stmts { + sqlx::query(stmt).execute(&mut *tx).await.unwrap(); + } + sqlx::query( + "INSERT INTO schema_migrations (version, name, applied_at) VALUES ($1, $2, $3)", + ) + .bind(m.version) + .bind(m.name) + .bind(chrono::Utc::now().to_rfc3339()) + .execute(&mut *tx) + .await + .unwrap(); + tx.commit().await.unwrap(); + } + } + + run_migrations_up_to(&db, 9).await; + + // Seed a repo and a pinned_cid, but no matching branch_cids entry. + sqlx::query( + "INSERT INTO repos (id, name, owner_did, description, is_public, default_branch, created_at, updated_at, disk_path) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)" + ) + .bind("repo-orphan") + .bind("orphan-repo") + .bind("did:key:z6Mkworphan") + .bind("desc") + .bind(true) + .bind("main") + .bind("2026-07-03T00:00:00Z") + .bind("2026-07-03T00:00:00Z") + .bind("/srv/orphan") + .execute(&db.pool) + .await + .unwrap(); + + // This CID is a pinned object that is NOT a current ref tip — + // no matching row in branch_cids exists. + sqlx::query( + "INSERT INTO pinned_cids (sha256_hex, cid, pinned_at) + VALUES ($1, $2, $3)", + ) + .bind("orphan-sha") + .bind("orphan-cid") + .bind("2026-07-03T00:00:00Z") + .execute(&db.pool) + .await + .unwrap(); + + // Run remaining migrations (v10 = ref_cert_dedup, v11 = pinned_cids) + db.run_migrations().await.unwrap(); + + // The orphan pin should have fallen back to repo = '' because + // branch_cids had no matching cid to backfill from. + let row = sqlx::query( + "SELECT sha256_hex, repo, owner_did FROM pinned_cids WHERE sha256_hex = 'orphan-sha'", + ) + .fetch_one(&db.pool) + .await + .unwrap(); + + assert_eq!( + row.get::("repo"), + "", + "non-tip pin must fall back to empty repo" + ); + assert_eq!( + row.get::("owner_did"), + "", + "non-tip pin must fall back to empty owner_did" + ); + } } #[cfg(test)] diff --git a/crates/gitlawb-node/src/git/visibility_pack.rs b/crates/gitlawb-node/src/git/visibility_pack.rs index cb70e39c..d42eac03 100644 --- a/crates/gitlawb-node/src/git/visibility_pack.rs +++ b/crates/gitlawb-node/src/git/visibility_pack.rs @@ -9,6 +9,7 @@ use crate::visibility::{visibility_check, Decision}; use anyhow::{Context, Result}; use std::collections::{BTreeSet, HashMap, HashSet}; use std::path::Path; +use std::sync::atomic::{AtomicBool, Ordering}; /// Fail closed unless every ref ultimately resolves to a commit (a ref pointing /// directly at a blob or tree, or an annotated tag — even a nested one — of such @@ -21,19 +22,13 @@ use std::path::Path; /// Full peeling is why this is not `for-each-ref %(*objecttype)`, which /// dereferences only one tag level and so misclassifies a tag-of-a-tag-of-a- /// commit as a non-commit. -fn assert_all_refs_are_commits(repo_path: &Path) -> Result<()> { - let refs = std::process::Command::new("git") - .args(["for-each-ref", "--format=%(refname)"]) - .current_dir(repo_path) - .output() - .context("git for-each-ref failed")?; - if !refs.status.success() { - anyhow::bail!( - "git for-each-ref failed: {}", - String::from_utf8_lossy(&refs.stderr) - ); - } - let refs_stdout = String::from_utf8_lossy(&refs.stdout); +fn assert_all_refs_are_commits(repo_path: &Path, cancelled: &AtomicBool) -> Result<()> { + let refs = run_git( + repo_path, + &["for-each-ref", "--format=%(refname)"], + cancelled, + )?; + let refs_stdout = String::from_utf8_lossy(&refs); let refnames: Vec<&str> = refs_stdout .lines() .map(str::trim) @@ -65,37 +60,55 @@ fn assert_all_refs_are_commits(repo_path: &Path) -> Result<()> { .stderr(std::process::Stdio::piped()) .spawn() .context("failed to spawn git cat-file")?; - // Feed stdin on a writer thread so this thread can drain stdout via - // wait_with_output concurrently; a None handle (the pipe vanished) becomes a - // broken-pipe write error. wait_with_output reaps the child unconditionally - // before any error is surfaced, so no path drops it unwaited (#53), and the - // writer is joined only after the drain so the join cannot deadlock. + // Feed stdin on a writer thread so this thread can poll the child for + // cancellation (wait_with_output would block indefinitely). let writer = child .stdin .take() .map(|mut stdin| std::thread::spawn(move || stdin.write_all(queries.as_bytes()))); - let peel_result = child.wait_with_output(); - let write_result = match writer { - Some(handle) => handle - .join() - .map_err(|_| anyhow::anyhow!("git cat-file stdin writer thread panicked"))?, - None => Err(std::io::Error::new( - std::io::ErrorKind::BrokenPipe, - "git cat-file stdin unavailable", - )), + // Read stdout on a background thread — the pipe blocks, so we poll + // through the child for cancellation. + let mut stdout = child.stdout.take().unwrap(); + let (stdout_tx, stdout_rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let mut buf = Vec::new(); + let _ = std::io::Read::read_to_end(&mut stdout, &mut buf); + let _ = stdout_tx.send(buf); + }); + let (peel_status, peel_stdout_bytes, peel_stderr) = loop { + if cancelled.load(Ordering::Relaxed) { + let _ = child.kill(); + let _ = child.wait(); + let _ = writer.map(|h| h.join()); + return Ok(()); + } + match child.try_wait() { + Ok(Some(status)) => { + let stdout_buf = stdout_rx.recv().unwrap_or_default(); + let mut stderr_buf = Vec::new(); + let _ = + std::io::Read::read_to_end(&mut child.stderr.take().unwrap(), &mut stderr_buf); + let _ = writer.map(|h| h.join()); + break (status, stdout_buf, stderr_buf); + } + Ok(None) => { + std::thread::sleep(std::time::Duration::from_millis(50)); + continue; + } + Err(e) => { + let _ = writer.map(|h| h.join()); + anyhow::bail!("git cat-file wait failed: {e}"); + } + } }; - // Surface a write error only if the process didn't already fail with a - // clearer status. - let peel = peel_result.context("git cat-file failed")?; - if !peel.status.success() { + if !peel_status.success() { anyhow::bail!( "git cat-file --batch-check failed: {}", - String::from_utf8_lossy(&peel.stderr) + String::from_utf8_lossy(&peel_stderr) ); } - write_result.context("failed to write to git cat-file stdin")?; - let peel_stdout = String::from_utf8_lossy(&peel.stdout); + let peel_stdout = String::from_utf8_lossy(&peel_stdout_bytes); let types: Vec<&str> = peel_stdout.lines().map(str::trim).collect(); // A short read means at least one ref went unclassified — fail closed. if types.len() != refnames.len() { @@ -147,65 +160,34 @@ fn assert_all_refs_are_commits(repo_path: &Path) -> Result<()> { /// Fails closed: if commit enumeration or any tree walk fails, returns an error so /// the caller aborts the serve/pin rather than producing a partial (under-withheld) /// set. -fn blob_paths(repo_path: &Path) -> Result> { - assert_all_refs_are_commits(repo_path)?; +fn blob_paths(repo_path: &Path, cancelled: &AtomicBool) -> Result> { + assert_all_refs_are_commits(repo_path, cancelled)?; - // Enumerate every reachable commit, not just ref tips. `--all` walks all refs; - // append HEAD so a detached HEAD (reachable by rev-list/upload-pack but in no - // ref) is still classified. When HEAD does not resolve (unborn branch on an - // empty repo) `--all` alone yields nothing, which is correct — no objects exist. + // Enumerate every reachable commit, not just ref tips. let head = store::head_commit(repo_path).context("resolve HEAD failed")?; let mut rev_args = vec!["rev-list", "--all"]; if head.is_some() { rev_args.push("HEAD"); } - let commits = std::process::Command::new("git") - .args(&rev_args) - .current_dir(repo_path) - .output() - .context("git rev-list --all failed")?; - if !commits.status.success() { - anyhow::bail!( - "git rev-list --all failed: {}", - String::from_utf8_lossy(&commits.stderr) - ); - } - let commits_stdout = String::from_utf8_lossy(&commits.stdout); + let commits = run_git(repo_path, &rev_args, cancelled)?; + let commits_stdout = String::from_utf8_lossy(&commits); let mut out: HashSet<(String, String)> = HashSet::new(); for commit in commits_stdout.lines() { + if cancelled.load(Ordering::Relaxed) { + return Ok(Vec::new()); + } let commit = commit.trim(); if commit.is_empty() { continue; } - let listing = std::process::Command::new("git") - .args(["ls-tree", "-rz", commit]) - .current_dir(repo_path) - .output() - .context("git ls-tree -rz failed")?; - if !listing.status.success() { - anyhow::bail!( - "git ls-tree -rz {commit} failed: {}", - String::from_utf8_lossy(&listing.stderr) - ); - } - // `-z` NUL-delimits records and emits paths raw; plain `git ls-tree -r` - // C-quotes any path with non-ASCII or special bytes (e.g. café.txt becomes - // "secret/caf\303\251.txt"), and that quoted literal would not match a - // visibility rule like "/secret/**", under-withholding the blob. The TAB - // field separator survives `-z`, so the per-record parse is unchanged. - // - // Parse strictly: a lossy decode would replace an invalid byte in a denied - // path (e.g. a non-UTF-8 directory name) with U+FFFD, and the mangled string - // would no longer match its deny rule — the same under-withholding class, one - // layer down. Fail closed instead so the caller aborts rather than leaks. - let Ok(listing_stdout) = std::str::from_utf8(&listing.stdout) else { + let listing = run_git(repo_path, &["ls-tree", "-rz", commit], cancelled)?; + let Ok(listing_stdout) = std::str::from_utf8(&listing) else { anyhow::bail!( "git ls-tree -rz {commit} returned a non-UTF-8 path; \ refusing to produce a partial (under-withheld) set" ); }; for record in listing_stdout.split('\0') { - // " blob \t" let Some((meta, path)) = record.split_once('\t') else { continue; }; @@ -223,6 +205,64 @@ fn blob_paths(repo_path: &Path) -> Result> { Ok(out.into_iter().collect()) } +/// Spawn a git subprocess and return its stdout, killing the child process if +/// the cancellation flag is set (P1). Reads stdout on a background thread +/// while polling cancellation in a loop so long-running `git rev-list` / +/// `git ls-tree` can be interrupted promptly. +fn run_git(repo_path: &Path, args: &[&str], cancelled: &AtomicBool) -> Result> { + let mut child = std::process::Command::new("git") + .args(args) + .current_dir(repo_path) + .stdout(std::process::Stdio::piped()) + .stderr(std::process::Stdio::piped()) + .spawn() + .context("failed to spawn git subprocess")?; + + let mut stdout = child.stdout.take().unwrap(); + + // Read stdout on a background thread — the pipe blocks, so we poll + // through the child instead. + let (tx, rx) = std::sync::mpsc::channel(); + std::thread::spawn(move || { + let mut buf = Vec::new(); + let _ = std::io::Read::read_to_end(&mut stdout, &mut buf); + let _ = tx.send(buf); + }); + + loop { + if cancelled.load(Ordering::Relaxed) { + let _ = child.kill(); + let _ = child.wait(); + return Ok(Vec::new()); + } + match child.try_wait() { + Ok(Some(status)) => { + let stdout_buf = rx.recv().unwrap_or_default(); + if !status.success() { + let mut stderr_buf = Vec::new(); + let _ = std::io::Read::read_to_end( + &mut child.stderr.take().unwrap(), + &mut stderr_buf, + ); + return Err(anyhow::anyhow!( + "git {} failed: {}", + args.join(" "), + String::from_utf8_lossy(&stderr_buf) + )); + } + return Ok(stdout_buf); + } + Ok(None) => { + std::thread::sleep(std::time::Duration::from_millis(50)); + continue; + } + Err(e) => { + return Err(anyhow::anyhow!("git wait failed: {e}")); + } + } + } +} + /// Blob OIDs the caller may not read. A blob is withheld only if visibility /// denies the caller at *every* path the blob appears at; a blob that is also /// reachable through an allowed path is sent (its content is public elsewhere). @@ -235,8 +275,9 @@ pub fn withheld_blob_oids( is_public: bool, owner_did: &str, caller: Option<&str>, + cancelled: &AtomicBool, ) -> Result> { - let pairs = blob_paths(repo_path)?; + let pairs = blob_paths(repo_path, cancelled)?; Ok(withheld_from_pairs( &pairs, rules, is_public, owner_did, caller, )) @@ -309,8 +350,9 @@ pub fn replicable_blob_set( rules: &[VisibilityRule], is_public: bool, owner_did: &str, + cancelled: &AtomicBool, ) -> Result> { - allowed_blob_set_for_caller(repo_path, rules, is_public, owner_did, None) + allowed_blob_set_for_caller(repo_path, rules, is_public, owner_did, None, cancelled) } /// Reachable blob OIDs that visibility ALLOWS `caller` at some path. The @@ -331,8 +373,9 @@ pub fn allowed_blob_set_for_caller( is_public: bool, owner_did: &str, caller: Option<&str>, + cancelled: &AtomicBool, ) -> Result> { - let pairs = blob_paths(repo_path)?; + let pairs = blob_paths(repo_path, cancelled)?; let mut allowed = HashSet::new(); for (oid, path) in &pairs { if visibility_check(rules, is_public, owner_did, caller, path) == Decision::Allow { @@ -370,9 +413,10 @@ pub fn withheld_blob_recipients( rules: &[VisibilityRule], is_public: bool, owner_did: &str, + cancelled: &AtomicBool, ) -> Result>> { // One history walk feeds both the withheld set and the recipient mapping. - let pairs = blob_paths(repo_path)?; + let pairs = blob_paths(repo_path, cancelled)?; let withheld = withheld_from_pairs(&pairs, rules, is_public, owner_did, None); if withheld.is_empty() { return Ok(HashMap::new()); @@ -473,7 +517,8 @@ mod tests { let (_td, bare, secret_oid, public_oid) = fixture(); let rules = [rule("/secret/**", &[])]; // caller = None models the public / any peer: what must not replicate. - let withheld = withheld_blob_oids(&bare, &rules, true, OWNER, None).unwrap(); + let withheld = + withheld_blob_oids(&bare, &rules, true, OWNER, None, &AtomicBool::new(false)).unwrap(); assert!( withheld.contains(&secret_oid), "secret blob must be withheld" @@ -490,8 +535,15 @@ mod tests { fn non_reader_withholds_only_the_private_blob() { let (_td, bare, secret, public) = fixture(); let rules = [rule("/secret/**", &["did:key:zFriend"])]; - let withheld = - withheld_blob_oids(&bare, &rules, true, OWNER, Some("did:key:zStranger")).unwrap(); + let withheld = withheld_blob_oids( + &bare, + &rules, + true, + OWNER, + Some("did:key:zStranger"), + &AtomicBool::new(false), + ) + .unwrap(); assert!(withheld.contains(&secret), "secret blob must be withheld"); assert!( !withheld.contains(&public), @@ -503,7 +555,15 @@ mod tests { fn owner_withholds_nothing() { let (_td, bare, secret, public) = fixture(); let rules = [rule("/secret/**", &["did:key:zFriend"])]; - let withheld = withheld_blob_oids(&bare, &rules, true, OWNER, Some(OWNER)).unwrap(); + let withheld = withheld_blob_oids( + &bare, + &rules, + true, + OWNER, + Some(OWNER), + &AtomicBool::new(false), + ) + .unwrap(); assert!(withheld.is_empty(), "owner sees everything"); let _ = (secret, public); } @@ -512,15 +572,23 @@ mod tests { fn listed_reader_withholds_nothing() { let (_td, bare, _secret, _public) = fixture(); let rules = [rule("/secret/**", &["did:key:zFriend"])]; - let withheld = - withheld_blob_oids(&bare, &rules, true, OWNER, Some("did:key:zFriend")).unwrap(); + let withheld = withheld_blob_oids( + &bare, + &rules, + true, + OWNER, + Some("did:key:zFriend"), + &AtomicBool::new(false), + ) + .unwrap(); assert!(withheld.is_empty(), "listed reader sees the subtree"); } #[test] fn no_subtree_rules_withholds_nothing() { let (_td, bare, _secret, _public) = fixture(); - let withheld = withheld_blob_oids(&bare, &[], true, OWNER, None).unwrap(); + let withheld = + withheld_blob_oids(&bare, &[], true, OWNER, None, &AtomicBool::new(false)).unwrap(); assert!( withheld.is_empty(), "public repo, no rules, nothing withheld" @@ -579,7 +647,9 @@ mod tests { ]; for (rules, caller) in cases { assert!(!has_path_scoped_rule(&rules)); - let withheld = withheld_blob_oids(&bare, &rules, true, OWNER, caller).unwrap(); + let withheld = + withheld_blob_oids(&bare, &rules, true, OWNER, caller, &AtomicBool::new(false)) + .unwrap(); assert!( withheld.is_empty(), "no path-scoped rule must withhold nothing for a gate-passing caller (caller={caller:?})" @@ -603,7 +673,15 @@ mod tests { // the walk is empty and the served set is complete. let root_only = vec![rule("/", &["did:key:zReader"])]; assert!(!has_path_scoped_rule(&root_only)); - let withheld_a = withheld_blob_oids(&bare, &root_only, true, OWNER, reader).unwrap(); + let withheld_a = withheld_blob_oids( + &bare, + &root_only, + true, + OWNER, + reader, + &AtomicBool::new(false), + ) + .unwrap(); assert!( withheld_a.is_empty(), "root-only rules withhold nothing for a gate-passing reader; the skip is safe" @@ -622,7 +700,9 @@ mod tests { rule("/secret/**", &["did:key:zOther"]), ]; assert!(has_path_scoped_rule(&scoped)); - let withheld_b = withheld_blob_oids(&bare, &scoped, true, OWNER, reader).unwrap(); + let withheld_b = + withheld_blob_oids(&bare, &scoped, true, OWNER, reader, &AtomicBool::new(false)) + .unwrap(); let served_b = replicable_objects(all, &withheld_b); assert!( !served_b.contains(&secret), @@ -636,7 +716,15 @@ mod tests { // Branch C — same path-scoped rules, but the caller is the owner. The // owner bypasses every rule, so the walk withholds nothing and the full // pack (secret included) is served even though a path-scoped rule exists. - let withheld_c = withheld_blob_oids(&bare, &scoped, true, OWNER, Some(OWNER)).unwrap(); + let withheld_c = withheld_blob_oids( + &bare, + &scoped, + true, + OWNER, + Some(OWNER), + &AtomicBool::new(false), + ) + .unwrap(); assert!( withheld_c.is_empty(), "the owner bypasses path-scoped rules and is served everything" @@ -742,7 +830,8 @@ mod tests { ); let rules: Vec = vec![]; - let allowed = replicable_blob_set(&work, &rules, true, OWNER).unwrap(); + let allowed = + replicable_blob_set(&work, &rules, true, OWNER, &AtomicBool::new(false)).unwrap(); assert!( !allowed.contains(&dangling_oid), "dangling blob is unreachable, so never in the allowed set" @@ -823,7 +912,15 @@ mod tests { // Every gate-relevant caller: anonymous, listed reader, owner. None of // them can put the dangling blob in the allowed set — it has no path. for caller in [None, Some(reader), Some(OWNER)] { - let allowed = allowed_blob_set_for_caller(&work, &rules, true, OWNER, caller).unwrap(); + let allowed = allowed_blob_set_for_caller( + &work, + &rules, + true, + OWNER, + caller, + &AtomicBool::new(false), + ) + .unwrap(); assert!( !allowed.contains(&dangling_oid), "dangling blob must be absent from allowed-set (caller={caller:?})" @@ -842,7 +939,8 @@ mod tests { let (_td, repo, secret_oid, public_oid) = fixture(); let reader = "did:key:zReader"; let rules = vec![rule("/secret/**", &[reader])]; - let map = withheld_blob_recipients(&repo, &rules, true, OWNER).unwrap(); + let map = + withheld_blob_recipients(&repo, &rules, true, OWNER, &AtomicBool::new(false)).unwrap(); let recips = map.get(&secret_oid).expect("secret blob has recipients"); assert!(recips.contains(OWNER)); @@ -895,7 +993,8 @@ mod tests { run(&["update-ref", "-d", &head_ref]); let rules = [rule("/secret/**", &[])]; - let withheld = withheld_blob_oids(&bare, &rules, true, OWNER, None).unwrap(); + let withheld = + withheld_blob_oids(&bare, &rules, true, OWNER, None, &AtomicBool::new(false)).unwrap(); assert!( withheld.contains(&secret_oid), "blob reachable only via refs/custom/* must still be withheld" @@ -940,7 +1039,8 @@ mod tests { run(&["update-ref", "-d", &head_ref]); let rules = [rule("/secret/**", &[])]; - let withheld = withheld_blob_oids(&bare, &rules, true, OWNER, None).unwrap(); + let withheld = + withheld_blob_oids(&bare, &rules, true, OWNER, None, &AtomicBool::new(false)).unwrap(); assert!( withheld.contains(&secret_oid), "blob reachable only via detached HEAD must still be withheld" @@ -1007,7 +1107,8 @@ mod tests { ); let rules = [rule("/secret/**", &[])]; - let withheld = withheld_blob_oids(&bare, &rules, true, OWNER, None).unwrap(); + let withheld = + withheld_blob_oids(&bare, &rules, true, OWNER, None, &AtomicBool::new(false)).unwrap(); assert!( withheld.contains(&secret_oid), "secret blob deleted at the tip but reachable in history must be withheld" @@ -1064,7 +1165,8 @@ mod tests { ); let rules = [rule("/secret/**", &[])]; - let withheld = withheld_blob_oids(&bare, &rules, true, OWNER, None).unwrap(); + let withheld = + withheld_blob_oids(&bare, &rules, true, OWNER, None, &AtomicBool::new(false)).unwrap(); assert!( withheld.contains(&secret_oid), "secret blob at a non-ASCII path must be withheld" @@ -1141,7 +1243,8 @@ mod tests { ); let rules = [rule(nfc_rule, &[])]; - let withheld = withheld_blob_oids(&bare, &rules, true, OWNER, None).unwrap(); + let withheld = + withheld_blob_oids(&bare, &rules, true, OWNER, None, &AtomicBool::new(false)).unwrap(); assert!( withheld.contains(&secret_oid), "NFC-authored deny rule must withhold the secret blob under the NFD-named directory" @@ -1217,7 +1320,8 @@ mod tests { ); let rules = [rule("/secret/**", &[])]; - let withheld = withheld_blob_oids(&bare, &rules, true, OWNER, None).unwrap(); + let withheld = + withheld_blob_oids(&bare, &rules, true, OWNER, None, &AtomicBool::new(false)).unwrap(); assert!( withheld.contains(&secret_oid), "secret blob at a path with TAB/newline must be withheld" @@ -1304,7 +1408,7 @@ mod tests { ); let rules = [rule("/s\u{fffd}cret/**", &[])]; - let result = withheld_blob_oids(&bare, &rules, true, OWNER, None); + let result = withheld_blob_oids(&bare, &rules, true, OWNER, None, &AtomicBool::new(false)); assert!( result.is_err(), "a non-UTF-8 path must fail closed (Err), not be lossy-decoded and leaked" @@ -1319,7 +1423,7 @@ mod tests { // ref and under-withholding. std::fs::write(bare.join("refs/heads/blobref"), format!("{secret}\n")).unwrap(); let rules = [rule("/secret/**", &[])]; - let result = withheld_blob_oids(&bare, &rules, true, OWNER, None); + let result = withheld_blob_oids(&bare, &rules, true, OWNER, None, &AtomicBool::new(false)); assert!( result.is_err(), "a ref that cannot be traversed must fail closed (Err)" @@ -1350,7 +1454,8 @@ mod tests { run(&["tag", "-a", "-m", "outer", "v2", "v1"]); let rules = [rule("/secret/**", &[])]; - let withheld = withheld_blob_oids(&bare, &rules, true, OWNER, None).unwrap(); + let withheld = + withheld_blob_oids(&bare, &rules, true, OWNER, None, &AtomicBool::new(false)).unwrap(); assert!( withheld.contains(&secret_oid), "secret blob must still be withheld with annotated and nested tags present" @@ -1378,7 +1483,7 @@ mod tests { run(&["tag", "-a", "-m", "blobtag", "blobtag", &secret]); let rules = [rule("/secret/**", &[])]; - let result = withheld_blob_oids(&bare, &rules, true, OWNER, None); + let result = withheld_blob_oids(&bare, &rules, true, OWNER, None, &AtomicBool::new(false)); assert!( result.is_err(), "an annotated tag of a blob must fail closed (Err)" @@ -1397,7 +1502,7 @@ mod tests { ) .unwrap(); let rules = [rule("/secret/**", &[])]; - let result = withheld_blob_oids(&bare, &rules, true, OWNER, None); + let result = withheld_blob_oids(&bare, &rules, true, OWNER, None, &AtomicBool::new(false)); assert!( result.is_err(), "a ref pointing at a missing object must fail closed (Err)" @@ -1425,7 +1530,9 @@ mod tests { let (tx, rx) = std::sync::mpsc::channel(); std::thread::spawn(move || { let rules = [rule("/secret/**", &[])]; - let is_err = withheld_blob_oids(&bare, &rules, true, OWNER, None).is_err(); + let is_err = + withheld_blob_oids(&bare, &rules, true, OWNER, None, &AtomicBool::new(false)) + .is_err(); let _ = tx.send(is_err); }); match rx.recv_timeout(std::time::Duration::from_secs(10)) { @@ -1487,7 +1594,8 @@ mod tests { ); let rules = [rule("/secret/**", &[])]; - let withheld = withheld_blob_oids(&bare, &rules, true, OWNER, None).unwrap(); + let withheld = + withheld_blob_oids(&bare, &rules, true, OWNER, None, &AtomicBool::new(false)).unwrap(); assert!( !withheld.contains(&shared_oid), "a blob also reachable via an allowed path must not be withheld" diff --git a/crates/gitlawb-node/src/ipfs_pin.rs b/crates/gitlawb-node/src/ipfs_pin.rs index 3b346190..7d7dd7a6 100644 --- a/crates/gitlawb-node/src/ipfs_pin.rs +++ b/crates/gitlawb-node/src/ipfs_pin.rs @@ -93,12 +93,18 @@ pub async fn cat(ipfs_api: &str, cid: &str) -> Result> { /// object's bytes. The twin in `pinata.rs` mirrors this shape — change both in /// lockstep. /// +/// `repo_slug` and `owner_did` are recorded alongside each pin so the scoped +/// listing query (`list_pinned_cids_for_repos`) can find them. Pass empty +/// strings if not available (the scoped listing will omit such pins). +/// /// Returns a list of `(sha256_hex, cid)` pairs for objects pinned this call. pub async fn pin_new_objects( ipfs_api: &str, repo_path: &std::path::Path, object_list: Vec, db: &crate::db::Db, + repo_slug: &str, + owner_did: &str, ) -> Vec<(String, String)> { if ipfs_api.is_empty() { return vec![]; @@ -107,9 +113,16 @@ pub async fn pin_new_objects( let mut pinned = Vec::new(); for sha in object_list { - // Skip if already pinned + // Check if already pinned. Even when pinned, update the + // repo/owner_did association so the scoped listing query can + // find this object under the current repo (P2). match db.is_pinned(&sha).await { - Ok(true) => continue, + Ok(true) => { + if !repo_slug.is_empty() { + let _ = db.update_pinned_cid_repo(&sha, repo_slug, owner_did).await; + } + continue; + } Ok(false) => {} Err(e) => { tracing::warn!(sha = %sha, err = %e, "DB error checking pinned status"); @@ -130,7 +143,10 @@ pub async fn pin_new_objects( // Pin to IPFS match pin_git_object(ipfs_api, &sha, &data).await { Ok(cid) if !cid.is_empty() => { - if let Err(e) = db.record_pinned_cid(&sha, &cid).await { + if let Err(e) = db + .record_pinned_cid_full(&sha, &cid, repo_slug, owner_did) + .await + { tracing::warn!(sha = %sha, err = %e, "failed to record pinned CID in DB"); } pinned.push((sha, cid)); diff --git a/crates/gitlawb-node/src/main.rs b/crates/gitlawb-node/src/main.rs index aa0483db..a03fbbca 100644 --- a/crates/gitlawb-node/src/main.rs +++ b/crates/gitlawb-node/src/main.rs @@ -377,6 +377,19 @@ async fn main() -> Result<()> { push_limiter_trust, sync_trigger_rate_limiter, peer_write_rate_limiter, + walk_semaphore: Arc::new(tokio::sync::Semaphore::new( + config.walk_concurrency_limit as usize, + )), + ipfs_list_rate_limiter: rate_limit::RateLimiter::new_bounded( + config.ipfs_list_rate_limit, + std::time::Duration::from_secs(3600), + 200_000, + ), + ipfs_list_global_limiter: rate_limit::RateLimiter::new_bounded( + config.ipfs_list_global_rate_limit, + std::time::Duration::from_secs(3600), + 1, + ), shutdown_tx: shutdown_tx.clone(), }; diff --git a/crates/gitlawb-node/src/pinata.rs b/crates/gitlawb-node/src/pinata.rs index 6c9c0bff..8edf00b2 100644 --- a/crates/gitlawb-node/src/pinata.rs +++ b/crates/gitlawb-node/src/pinata.rs @@ -76,6 +76,10 @@ pub async fn pin_object( /// this shape — change both in lockstep. Objects already recorded with a /// `pinata_cid` are skipped. Returns `(sha_hex, cid)` pairs for each newly /// pinned object. +/// +/// `repo_slug` and `owner_did` are recorded alongside each pin so the scoped +/// listing query can find them. Pass empty strings if not available. +#[allow(clippy::too_many_arguments)] pub async fn pin_new_objects( client: &reqwest::Client, upload_url: &str, @@ -83,6 +87,8 @@ pub async fn pin_new_objects( repo_path: &std::path::Path, object_list: Vec, db: &crate::db::Db, + repo_slug: &str, + owner_did: &str, ) -> Vec<(String, String)> { if jwt.is_empty() { return vec![]; @@ -92,7 +98,12 @@ pub async fn pin_new_objects( for sha in object_list { match db.has_pinata_cid(&sha).await { - Ok(true) => continue, + Ok(true) => { + if !repo_slug.is_empty() { + let _ = db.update_pinned_cid_repo(&sha, repo_slug, owner_did).await; + } + continue; + } Ok(false) => {} Err(e) => { tracing::warn!(sha = %sha, err = %e, "DB error checking pinata_cid"); @@ -111,7 +122,10 @@ pub async fn pin_new_objects( match pin_object(client, upload_url, jwt, &sha, &data).await { Ok(cid) if !cid.is_empty() => { - if let Err(e) = db.record_pinata_cid(&sha, &cid).await { + if let Err(e) = db + .record_pinata_cid_full(&sha, &cid, repo_slug, owner_did) + .await + { tracing::warn!(sha = %sha, err = %e, "failed to record pinata_cid in DB"); } pinned.push((sha, cid)); diff --git a/crates/gitlawb-node/src/server.rs b/crates/gitlawb-node/src/server.rs index f4c0d3e3..681f80e0 100644 --- a/crates/gitlawb-node/src/server.rs +++ b/crates/gitlawb-node/src/server.rs @@ -210,17 +210,19 @@ pub fn build_router(state: AppState) -> Router { .layer(axum::Extension(push_limiter)); // ── IPFS content-addressed retrieval and pin listing ────────────────── - // `/ipfs/{cid}` carries `optional_signature` so `get_by_cid` sees the caller - // identity and can apply per-repo visibility (#110); anonymous callers stay - // anonymous and still read genuinely public content. `/api/v1/ipfs/pins` - // stays unsigned — gating the pin index is tracked separately (#121). + // Both `/ipfs/{cid}` and `/api/v1/ipfs/pins` carry `optional_signature` + // so handlers see the caller identity and can apply per-repo visibility. + // Anonymous callers stay anonymous for public content; the pin listing + // requires authentication (handled in the handler itself). let ipfs_routes = Router::new() .route("/ipfs/{cid}", get(ipfs::get_by_cid)) - .layer(middleware::from_fn(auth::optional_signature)) - .merge(Router::new().route("/api/v1/ipfs/pins", get(ipfs::list_pins))); + .merge(Router::new().route("/api/v1/ipfs/pins", get(ipfs::list_pins))) + .layer(middleware::from_fn(auth::optional_signature)); // ── Arweave permanent anchors ────────────────────────────────────────── - let arweave_routes = Router::new().route("/api/v1/arweave/anchors", get(arweave::list_anchors)); + let arweave_routes = Router::new() + .route("/api/v1/arweave/anchors", get(arweave::list_anchors)) + .layer(middleware::from_fn(auth::optional_signature)); // ── Bounty routes (write — require HTTP Signature) ───────────────── let bounty_write_routes = add_auth_layers( diff --git a/crates/gitlawb-node/src/state.rs b/crates/gitlawb-node/src/state.rs index d3e53f3a..2231c19f 100644 --- a/crates/gitlawb-node/src/state.rs +++ b/crates/gitlawb-node/src/state.rs @@ -81,6 +81,16 @@ pub struct AppState { /// sink as trigger and accepts unsigned requests from known peers, so it is /// braked too; each peer's distinct IP gets its own bucket. pub peer_write_rate_limiter: RateLimiter, + /// Concurrency limiter for expensive visibility walks (git rev-list / + /// git ls-tree) triggered by the IPFS pin listing endpoint (P1). + /// Prevents a flood of signed requests from exhausting the blocking + /// pool or leaving git children running past their timeout. + pub walk_semaphore: Arc, + /// Per-DID rate limiter for the IPFS pin listing endpoint. + pub ipfs_list_rate_limiter: RateLimiter, + /// Global (non-sybil) rate limiter for the IPFS pin listing endpoint. + /// Keyed on a fixed value so rotating DIDs cannot bypass it (P1). + pub ipfs_list_global_limiter: RateLimiter, /// Process-wide graceful-shutdown signal. Sending `true` causes every /// task that holds a `watch::Receiver` to exit at its next await point. /// Used by: diff --git a/crates/gitlawb-node/src/test_support.rs b/crates/gitlawb-node/src/test_support.rs index be6fb7b8..24be94de 100644 --- a/crates/gitlawb-node/src/test_support.rs +++ b/crates/gitlawb-node/src/test_support.rs @@ -81,6 +81,9 @@ fn build_state(db: Arc, pool: PgPool) -> AppState { push_limiter_trust: crate::rate_limit::TrustedProxy::None, sync_trigger_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), peer_write_rate_limiter: RateLimiter::new(600, Duration::from_secs(3600)), + walk_semaphore: Arc::new(tokio::sync::Semaphore::new(4)), + ipfs_list_rate_limiter: RateLimiter::new(60, Duration::from_secs(3600)), + ipfs_list_global_limiter: RateLimiter::new_bounded(1200, Duration::from_secs(3600), 1), shutdown_tx: tokio::sync::watch::channel(false).0, } } diff --git a/crates/gl/Cargo.toml b/crates/gl/Cargo.toml index 9b11d573..f6b00fa0 100644 --- a/crates/gl/Cargo.toml +++ b/crates/gl/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "gl" description = "The gitlawb CLI — identity management, node control, MCP server" -version = "0.6.0" # x-release-please-version +version = "0.7.0" # x-release-please-version edition.workspace = true rust-version.workspace = true license.workspace = true @@ -13,6 +13,7 @@ path = "src/main.rs" [dependencies] gitlawb-core = { path = "../gitlawb-core" } icaptcha-client = { path = "../icaptcha-client" } +base64 = { workspace = true } tokio = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } diff --git a/crates/gl/src/http.rs b/crates/gl/src/http.rs index 4c30f512..4a8c3cc7 100644 --- a/crates/gl/src/http.rs +++ b/crates/gl/src/http.rs @@ -196,6 +196,30 @@ impl NodeClient { } } +/// Read a response body with a streaming byte cap so chunked responses don't +/// allocate unbounded memory before the check (P2). +/// +/// Returns an error if the body exceeds `max_bytes` *before* buffering the +/// full payload. Handles both `Content-Length` and chunked transfer-encoding. +pub async fn capped_response(mut resp: reqwest::Response, max_bytes: usize) -> Result> { + let mut body = Vec::new(); + loop { + let chunk = resp.chunk().await?; + match chunk { + Some(bytes) => { + if body.len() + bytes.len() > max_bytes { + anyhow::bail!( + "response body exceeds {max_bytes} byte limit (already read {})", + body.len() + ); + } + body.extend_from_slice(&bytes); + } + None => return Ok(body), + } + } +} + /// Run the (blocking) iCaptcha solve loop off the async runtime. async fn obtain_proof(cfg: IcaptchaCfg) -> Result { tokio::task::spawn_blocking(move || icaptcha_client::obtain_proof(&cfg, None)) diff --git a/crates/gl/src/ipfs_cmd.rs b/crates/gl/src/ipfs_cmd.rs index d4cb64fc..62e907df 100644 --- a/crates/gl/src/ipfs_cmd.rs +++ b/crates/gl/src/ipfs_cmd.rs @@ -9,7 +9,7 @@ use anyhow::{Context, Result}; use clap::{Args, Subcommand}; use serde_json::Value; -use crate::http::NodeClient; +use crate::http::{capped_response, NodeClient}; #[derive(Args)] pub struct IpfsArgs { @@ -49,24 +49,25 @@ async fn cmd_list(node: String, dir: Option) -> Result<()> { // error (it already names `gl identity new`) rather than a bare 401. let keypair = crate::identity::load_keypair_from_dir(dir.as_deref())?; let client = NodeClient::new(&node, Some(keypair)); - let resp = client.get_signed("/api/v1/ipfs/pins").await?; - if !resp.status().is_success() { - let status = resp.status(); - let body = resp.text().await.unwrap_or_default(); - anyhow::bail!("node returned {status} for pins listing: {body}"); - } - let resp: Value = resp.json().await.context("failed to parse pins response")?; + let (pins, incomplete) = list_pins_paginated(&client).await?; - let pins = resp["pins"].as_array().cloned().unwrap_or_default(); - let count = resp["count"].as_u64().unwrap_or(pins.len() as u64); + let count = pins.len(); if pins.is_empty() { - println!("No IPFS pins recorded on {node}"); - println!("(Push to a repo with GITLAWB_IPFS_API set to start pinning)"); + if incomplete { + println!("IPFS pins on {node}: listing incomplete — unable to enumerate all pins"); + } else { + println!("No IPFS pins recorded on {node}"); + println!("(Push to a repo with GITLAWB_IPFS_API set to start pinning)"); + } return Ok(()); } - println!("IPFS pins ({count}) on {node}"); + print!("IPFS pins ({count}) on {node}"); + if incomplete { + print!(" (truncated — too many results)"); + } + println!(); println!(); for pin in &pins { let cid = pin["cid"].as_str().unwrap_or("?"); @@ -86,6 +87,192 @@ async fn cmd_list(node: String, dir: Option) -> Result<()> { Ok(()) } +/// Sanitize a node response body for inclusion in CLI error messages. +/// Strips ANSI/OSC control sequences and non-printable bytes (except +/// newline and tab) so a malicious or compromised node cannot inject +/// terminal output through an otherwise bounded error body (P2). +/// Truncates on a char boundary (not a byte boundary) to avoid panicking +/// on multi-byte UTF-8 (P2). +fn sanitize_body(body: &str) -> String { + const MAX_BODY_CHARS: usize = 500; + body.chars() + .take(MAX_BODY_CHARS) + .filter(|&c| c.is_ascii_graphic() || c == ' ' || c == '\n' || c == '\t') + .collect() +} + +/// Paginate through the full pin listing, collecting all pins and handling +/// the expired-truncated_cursor retry (P2). The last_next_cursor restore +/// may cause a duplicate page (self-limiting via the cycle guard). +async fn list_pins_paginated(client: &NodeClient) -> Result<(Vec, bool)> { + let mut all_pins = Vec::new(); + let mut all_pins_bytes = 0usize; + let mut cursor: Option = None; + let mut truncated_cursor: Option = None; + // Persist the last next_cursor across the truncated leg so that an + // expired truncated_cursor (400) can resume from where we left off + // rather than restarting at page 1 (P2). Note: this may re-fetch + // the page before the truncated one (self-limiting via cycle guard). + let mut last_next_cursor: Option = None; + // Advancement guard: track every cursor value seen to detect cycles. + let mut seen_cursors: std::collections::HashSet = std::collections::HashSet::new(); + let mut incomplete = false; + let mut pages = 0u32; + // Consecutive empty pages without forward progress: a buggy or hostile + // node that returns empty pages with fresh cursors cannot loop + // indefinitely (P2). + let mut consecutive_empty_pages = 0u32; + const MAX_CONSECUTIVE_EMPTY: u32 = 5; + // Bounds: at most 10 000 pages, 1 000 000 rows total, 512 MiB + // aggregate retained JSON, or 64 MiB per response body — limits + // unbounded loops and prevents a single oversized page from + // exhausting memory before the row cap is checked (P2). + const MAX_PAGES: u32 = 10_000; + const MAX_ROWS: usize = 1_000_000; + const MAX_AGGREGATE_BYTES: usize = 512 * 1024 * 1024; + const MAX_RESPONSE_BYTES: usize = 64 * 1024 * 1024; + + loop { + pages += 1; + if pages > MAX_PAGES { + incomplete = true; + break; + } + + // Request the maximum page size to minimise page-turn requests + // against the per-DID quota (P2). + // Server clamps limit to 200 per page (P2). Request the max so we + // minimise page-turn requests against the per-DID quota. + let mut path = "/api/v1/ipfs/pins?limit=200".to_string(); + let mut params = Vec::new(); + let mut had_truncated = false; + if let Some(c) = cursor.take() { + params.push(format!("cursor={}", urlencoding::encode(&c))); + } + if let Some(tc) = truncated_cursor.take() { + had_truncated = true; + params.push(format!("truncated_cursor={}", urlencoding::encode(&tc))); + } + for p in ¶ms { + path.push('&'); + path.push_str(p); + } + + let resp = client.get_signed(&path).await?; + + if !resp.status().is_success() { + let status = resp.status(); + // P1: rate-limited — surface a partial result instead of failing. + if status == 429 { + incomplete = true; + break; + } + if status == 400 && had_truncated { + let body = String::from_utf8_lossy( + &capped_response(resp, MAX_RESPONSE_BYTES) + .await + .unwrap_or_default(), + ) + .to_string(); + // Only treat a 400 as expired-cursor when the server explicitly + // says so. Any other 400 — malformed cursor, protocol change, + // node bug — is surfaced as an error. + if body.contains("invalid or expired truncated_cursor") { + cursor = last_next_cursor.clone(); + continue; + } + anyhow::bail!( + "node returned 400 for pins listing: {}", + sanitize_body(&body) + ); + } + let body = String::from_utf8_lossy( + &capped_response(resp, MAX_RESPONSE_BYTES) + .await + .unwrap_or_default(), + ) + .to_string(); + anyhow::bail!( + "node returned {status} for pins listing: {}", + sanitize_body(&body) + ); + } + let body = capped_response(resp, MAX_RESPONSE_BYTES).await?; + let resp: Value = serde_json::from_slice(&body).with_context(|| { + format!( + "failed to parse pins response ({len} bytes)", + len = body.len() + ) + })?; + + let pins = resp["pins"].as_array().cloned().unwrap_or_default(); + + if pins.is_empty() { + consecutive_empty_pages += 1; + if consecutive_empty_pages >= MAX_CONSECUTIVE_EMPTY { + incomplete = true; + break; + } + } else { + consecutive_empty_pages = 0; + } + + if all_pins.len() + pins.len() > MAX_ROWS { + incomplete = true; + break; + } + + // Aggregate memory bound: track the serialised size of each pin + // JSON object to prevent a malicious node from exhausting the + // CLI's memory with many small pages of oversized fields (P2). + for pin in &pins { + all_pins_bytes += serde_json::to_string(pin).map(|s| s.len()).unwrap_or(256); + } + if all_pins_bytes > MAX_AGGREGATE_BYTES { + incomplete = true; + break; + } + + let next = resp["next_cursor"].as_str().map(String::from); + let new_trunc = resp["truncated_cursor"].as_str().map(String::from); + + // Detect cursor cycling: keys on the exact (next_cursor, truncated) + // pair, so a node that returns a fresh pair every page never trips it. + // MAX_PAGES provides the ultimate bound (10 K round-trips per listing). + let cycle_key = + next.as_deref().unwrap_or("").to_string() + "|" + new_trunc.as_deref().unwrap_or(""); + + // Bound cursor / cycle-key retained bytes alongside pin data so a + // node returning fresh near-64 MiB cursors per page cannot exhaust + // memory via the seen_cursors set (P3). Account for the cycle_key + // string that will be stored in seen_cursors plus HashSet entry + // overhead (~32 bytes per entry). + all_pins_bytes += cycle_key.len() + 32; + if all_pins_bytes > MAX_AGGREGATE_BYTES { + incomplete = true; + break; + } + + if !cycle_key.is_empty() && !seen_cursors.insert(cycle_key) { + incomplete = true; + break; + } + + all_pins.extend(pins); + + if next.is_none() && new_trunc.is_none() { + break; + } + if let Some(ref n) = next { + last_next_cursor = Some(n.clone()); + } + cursor = next; + truncated_cursor = new_trunc; + } + + Ok((all_pins, incomplete)) +} + async fn cmd_get(cid: String, node: String) -> Result<()> { let client = NodeClient::new(&node, None); let path = format!("/ipfs/{cid}"); @@ -144,7 +331,7 @@ mod tests { // Happy path: signed GET to /api/v1/ipfs/pins carrying the RFC 9421 // signature headers, node returns a populated pins body. let m = server - .mock("GET", "/api/v1/ipfs/pins") + .mock("GET", mockito::Matcher::Regex(r"^/api/v1/ipfs/pins".to_string())) .match_header("signature", mockito::Matcher::Any) .match_header("signature-input", mockito::Matcher::Any) .match_header("content-digest", mockito::Matcher::Any) @@ -169,7 +356,10 @@ mod tests { let keystore = seed_keystore(); let m = server - .mock("GET", "/api/v1/ipfs/pins") + .mock( + "GET", + mockito::Matcher::Regex(r"^/api/v1/ipfs/pins".to_string()), + ) .match_header("signature", mockito::Matcher::Any) .with_status(200) .with_header("content-type", "application/json") @@ -192,7 +382,10 @@ mod tests { // The endpoint must never be hit when there is no identity. let m = server - .mock("GET", "/api/v1/ipfs/pins") + .mock( + "GET", + mockito::Matcher::Regex(r"^/api/v1/ipfs/pins".to_string()), + ) .expect(0) .create_async() .await; @@ -217,7 +410,10 @@ mod tests { // A signed request the node rejects (401) must surface as an error, // not be silently parsed into an empty pin list. let m = server - .mock("GET", "/api/v1/ipfs/pins") + .mock( + "GET", + mockito::Matcher::Regex(r"^/api/v1/ipfs/pins".to_string()), + ) .match_header("signature", mockito::Matcher::Any) .with_status(401) .with_header("content-type", "application/json") diff --git a/crates/gl/src/node.rs b/crates/gl/src/node.rs index 367ba576..414b13a3 100644 --- a/crates/gl/src/node.rs +++ b/crates/gl/src/node.rs @@ -6,7 +6,7 @@ use gitlawb_core::identity::Keypair; use serde_json::Value; use std::path::PathBuf; -use crate::http::NodeClient; +use crate::http::{capped_response, NodeClient}; use crate::identity::load_keypair_from_dir; use crate::node_stake; @@ -189,8 +189,15 @@ async fn try_get_json(client: &NodeClient, path: &str) -> Option { /// sign in. A pins failure never aborts the dashboard. #[derive(Debug)] enum PinsPanel { - /// Signed read succeeded and returned pins (carries the resolved count). - Pins(u64), + /// Signed read succeeded and returned pins. + Pins { + count: u64, + /// True when the traversal hit its safety bounds (page cap, row cap, + /// or cursor cycle) before consuming all available data. The count + /// is an undercount; the dashboard signals that the listing was + /// truncated (P2). + incomplete: bool, + }, /// Signed read succeeded but the node has no pins recorded. Empty, /// Signed read was rejected (401/other) or errored. @@ -241,30 +248,156 @@ async fn fetch_pins(node: &str, auth: PinsAuth) -> PinsPanel { PinsAuth::DirUnusable(dir) => return PinsPanel::IdentityError(dir), }; let client = NodeClient::new(node, Some(kp)); - let resp = match client.get_signed("/api/v1/ipfs/pins").await { - Ok(r) => r, - Err(_) => return PinsPanel::Unavailable, - }; - if !resp.status().is_success() { - return PinsPanel::Unavailable; + let mut total: u64 = 0; + let mut cursor: Option = None; + let mut truncated_cursor: Option = None; + // Persist the last next_cursor across the truncated leg so that an + // expired truncated_cursor (400) can resume from where we left off + // rather than restarting at page 1 (P2). + let mut last_next_cursor: Option = None; + let mut seen_cursors: std::collections::HashSet = std::collections::HashSet::new(); + let mut incomplete = false; + let mut pages = 0u32; + // Consecutive empty pages without forward progress: a buggy or hostile + // node that returns empty pages with fresh cursors cannot loop + // indefinitely (P2). + let mut consecutive_empty_pages = 0u32; + const MAX_CONSECUTIVE_EMPTY: u32 = 5; + const MAX_PAGES: u32 = 10_000; + const MAX_ROWS: u64 = 1_000_000; + const MAX_RESPONSE_BYTES: usize = 64 * 1024 * 1024; + // Aggregate memory bound for retained cursor/cycle-key bytes (P3). + // Shared with the pin data budget from ipfs_cmd.rs. + const MAX_AGGREGATE_BYTES: usize = 512 * 1024 * 1024; + let mut all_cursor_bytes: usize = 0; + + loop { + pages += 1; + if pages > MAX_PAGES { + incomplete = true; + break; + } + + // Request the max page size to minimise page-turn requests + // against the per-DID quota (P2). The server clamps to 200. + let mut path = "/api/v1/ipfs/pins?limit=200".to_string(); + let mut had_truncated = false; + if let Some(c) = cursor.take() { + path.push_str(&format!("&cursor={}", urlencoding::encode(&c))); + } + if let Some(tc) = truncated_cursor.take() { + had_truncated = true; + path.push_str(&format!("&truncated_cursor={}", urlencoding::encode(&tc))); + } + + let resp = match client.get_signed(&path).await { + Ok(r) => r, + Err(_) => return PinsPanel::Unavailable, + }; + + if !resp.status().is_success() { + // P2: rate-limited — surface a partial result instead of failing. + if resp.status().as_u16() == 429 { + incomplete = true; + break; + } + if resp.status().as_u16() == 400 && had_truncated { + let body = String::from_utf8_lossy( + &capped_response(resp, MAX_RESPONSE_BYTES) + .await + .unwrap_or_default(), + ) + .to_string(); + if body.contains("invalid or expired truncated_cursor") { + cursor = last_next_cursor.clone(); + continue; + } + } + return PinsPanel::Unavailable; + } + let body = match capped_response(resp, MAX_RESPONSE_BYTES).await { + Ok(b) => b, + Err(_) => return PinsPanel::Unavailable, + }; + let Ok(body) = serde_json::from_slice::(&body) else { + return PinsPanel::Unavailable; + }; + + let page_pins = body["pins"].as_array().map(|a| a.len() as u64).unwrap_or(0); + + if total + page_pins > MAX_ROWS { + incomplete = true; + break; + } + total += page_pins; + + if page_pins == 0 { + consecutive_empty_pages += 1; + if consecutive_empty_pages >= MAX_CONSECUTIVE_EMPTY { + incomplete = true; + break; + } + } else { + consecutive_empty_pages = 0; + } + + let next = body["next_cursor"].as_str().map(String::from); + let new_trunc = body["truncated_cursor"].as_str().map(String::from); + + // Detect cursor cycling + let cycle_key = + next.as_deref().unwrap_or("").to_string() + "|" + new_trunc.as_deref().unwrap_or(""); + + // Bound retained cursor bytes against the same aggregate budget used + // by ipfs_cmd.rs so a node returning near-64 MiB cursors per page + // cannot exhaust memory (P3). Check before insert so cycle_key is + // not moved. + all_cursor_bytes += cycle_key.len() + 32; // +32 for HashSet entry overhead + if all_cursor_bytes > MAX_AGGREGATE_BYTES { + incomplete = true; + break; + } + + if !cycle_key.is_empty() && !seen_cursors.insert(cycle_key) { + incomplete = true; + break; + } + + if next.is_none() && new_trunc.is_none() { + break; + } + if let Some(ref n) = next { + last_next_cursor = Some(n.clone()); + } + cursor = next; + truncated_cursor = new_trunc; } - let Ok(body) = resp.json::().await else { - return PinsPanel::Unavailable; - }; - let count = body["count"] - .as_u64() - .unwrap_or_else(|| body["pins"].as_array().map(|a| a.len() as u64).unwrap_or(0)); + let count = total; if count == 0 { - PinsPanel::Empty + if incomplete { + // Buggy or capacity-constrained node: the listing was truncated + // before any row was emitted. Render as unavailable so an + // authoritative "Pinned CIDs: 0" is never shown for a partial + // result (P2). + PinsPanel::Unavailable + } else { + PinsPanel::Empty + } } else { - PinsPanel::Pins(count) + PinsPanel::Pins { count, incomplete } } } /// Render the one-line pins-panel status for the `gl node status` dashboard. fn pins_status_line(panel: &PinsPanel) -> String { match panel { - PinsPanel::Pins(count) => format!(" Pinned CIDs: {count}"), + PinsPanel::Pins { count, incomplete } => { + let mut s = format!(" Pinned CIDs: {count}"); + if *incomplete { + s.push_str(" (truncated)"); + } + s + } PinsPanel::Empty => " Pinned CIDs: 0".to_string(), PinsPanel::Unavailable => " IPFS pins: unavailable".to_string(), PinsPanel::NeedsIdentity => { @@ -523,7 +656,7 @@ mod tests { // A keyed fetch must sign the request (RFC 9421 headers) and, on a // populated 200 body, land in the Pins state carrying the pins. let m = server - .mock("GET", "/api/v1/ipfs/pins") + .mock("GET", mockito::Matcher::Regex(r"^/api/v1/ipfs/pins(\?.*)?$".to_string())) .match_header("signature", mockito::Matcher::Any) .match_header("signature-input", mockito::Matcher::Any) .match_header("content-digest", mockito::Matcher::Any) @@ -537,7 +670,7 @@ mod tests { let panel = fetch_pins(&server.url(), PinsAuth::Keyed(kp)).await; match panel { - PinsPanel::Pins(count) => assert_eq!(count, 1), + PinsPanel::Pins { count, .. } => assert_eq!(count, 1), other => panic!("expected Pins, got {other:?}"), } @@ -550,7 +683,10 @@ mod tests { let kp = Keypair::generate(); let m = server - .mock("GET", "/api/v1/ipfs/pins") + .mock( + "GET", + mockito::Matcher::Regex(r"^/api/v1/ipfs/pins(\?.*)?$".to_string()), + ) .match_header("signature", mockito::Matcher::Any) .with_status(200) .with_header("content-type", "application/json") @@ -575,7 +711,10 @@ mod tests { // Node rejects the signed read (401): the panel must degrade to // Unavailable without panicking, so cmd_status still completes. let m = server - .mock("GET", "/api/v1/ipfs/pins") + .mock( + "GET", + mockito::Matcher::Regex(r"^/api/v1/ipfs/pins(\?.*)?$".to_string()), + ) .match_header("signature", mockito::Matcher::Any) .with_status(401) .with_header("content-type", "application/json") @@ -620,7 +759,10 @@ mod tests { // 2xx but the body is not valid JSON: must degrade to Unavailable, // never panic. let m = server - .mock("GET", "/api/v1/ipfs/pins") + .mock( + "GET", + mockito::Matcher::Regex(r"^/api/v1/ipfs/pins(\?.*)?$".to_string()), + ) .match_header("signature", mockito::Matcher::Any) .with_status(200) .with_header("content-type", "application/json") @@ -656,7 +798,13 @@ mod tests { #[test] fn test_pins_status_line_renders_each_state() { - assert_eq!(pins_status_line(&PinsPanel::Pins(3)), " Pinned CIDs: 3"); + assert_eq!( + pins_status_line(&PinsPanel::Pins { + count: 3, + incomplete: false + }), + " Pinned CIDs: 3" + ); assert_eq!(pins_status_line(&PinsPanel::Empty), " Pinned CIDs: 0"); assert_eq!( pins_status_line(&PinsPanel::Unavailable),