From 59cfd7376dc9fd3060537f52629ea45bad916a0a Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Sun, 19 Jul 2026 09:20:21 -0500 Subject: [PATCH 1/4] fix(release): restore the unreleased 0.6.0 train --- CHANGELOG.md | 6 ++---- Cargo.lock | 10 +++++----- Cargo.toml | 2 +- README.md | 2 +- THREAT_MODEL.md | 2 +- fuzz/Cargo.lock | 2 +- ordvec-ffi/Cargo.toml | 2 +- ordvec-manifest-python/Cargo.toml | 2 +- ordvec-manifest-python/pyproject.toml | 2 +- .../python/ordvec_manifest/__init__.py | 2 +- ordvec-manifest/Cargo.toml | 4 ++-- ordvec-python/Cargo.toml | 2 +- ordvec-python/pyproject.toml | 2 +- ordvec-python/python/ordvec/__init__.py | 2 +- 14 files changed, 20 insertions(+), 22 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f3f7397..0906810 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,7 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 _No unreleased changes._ -## 0.7.0 - 2026-07-13 +## 0.6.0 - 2026-07-19 ### Added @@ -33,7 +33,7 @@ _No unreleased changes._ read core; the sqlite registry's private duplicate hasher is deduped onto the public helper. -### Changed +### Breaking changes - **BREAKING (`ordvec-manifest`): deterministic manifest schema v2.** The manifest schema version is now `ordvec.index_manifest.v2`. `manifest_id` @@ -59,8 +59,6 @@ _No unreleased changes._ while the rebuildable `active_manifest` pointer is reset and must be re-activated. -## 0.6.0 - 2026-07-04 - ### Performance - **Batched sign candidate generation now streams the corpus once per call.** diff --git a/Cargo.lock b/Cargo.lock index fda6c77..c505790 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -844,7 +844,7 @@ checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" [[package]] name = "ordvec" -version = "0.7.0" +version = "0.6.0" dependencies = [ "rand 0.10.2", "rand_chacha 0.10.0", @@ -854,14 +854,14 @@ dependencies = [ [[package]] name = "ordvec-ffi" -version = "0.7.0" +version = "0.6.0" dependencies = [ "ordvec", ] [[package]] name = "ordvec-manifest" -version = "0.7.0" +version = "0.6.0" dependencies = [ "chrono", "clap", @@ -877,7 +877,7 @@ dependencies = [ [[package]] name = "ordvec-manifest-python" -version = "0.7.0" +version = "0.6.0" dependencies = [ "ordvec-manifest", "pyo3", @@ -887,7 +887,7 @@ dependencies = [ [[package]] name = "ordvec-python" -version = "0.7.0" +version = "0.6.0" dependencies = [ "numpy", "ordvec", diff --git a/Cargo.toml b/Cargo.toml index 5b9395c..4c3ee3f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ordvec" -version = "0.7.0" +version = "0.6.0" edition = "2021" rust-version = "1.89" # AVX-512 intrinsics stabilized in 1.89.0; also clears the 1.87 floor from u64::is_multiple_of description = "Training-free ordinal & sign quantization for vector retrieval" diff --git a/README.md b/README.md index 0ee8b28..0442cd9 100644 --- a/README.md +++ b/README.md @@ -212,7 +212,7 @@ Details in [`docs/RANK_MODES.md`](docs/RANK_MODES.md). ```toml [dependencies] -ordvec = "0.7" +ordvec = "0.6" # Or, to track unreleased `main`, use a git dependency instead: # ordvec = { git = "https://github.com/Project-Navi/ordvec" } diff --git a/THREAT_MODEL.md b/THREAT_MODEL.md index 9f02937..a096f29 100644 --- a/THREAT_MODEL.md +++ b/THREAT_MODEL.md @@ -1,6 +1,6 @@ # Threat Model — `ordvec` -> **Status:** v0.7.0 (pre-1.0), 2026-06-15. This is the maintained threat model +> **Status:** v0.6.0 (pre-1.0), 2026-06-15. This is the maintained threat model > for the `ordvec` Rust crate, C ABI, Go wrapper, PyO3/maturin Python bindings, > and the `ordvec-manifest` sidecar verifier. It is reviewed when the > attack surface changes (new persistence formats, new `unsafe` kernels, new diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index 39b43de..ecb390e 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -231,7 +231,7 @@ checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "ordvec" -version = "0.7.0" +version = "0.6.0" dependencies = [ "rayon", ] diff --git a/ordvec-ffi/Cargo.toml b/ordvec-ffi/Cargo.toml index 756c076..177a92b 100644 --- a/ordvec-ffi/Cargo.toml +++ b/ordvec-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ordvec-ffi" -version = "0.7.0" +version = "0.6.0" edition = "2021" rust-version = "1.89" publish = false diff --git a/ordvec-manifest-python/Cargo.toml b/ordvec-manifest-python/Cargo.toml index 5eddd45..490ef70 100644 --- a/ordvec-manifest-python/Cargo.toml +++ b/ordvec-manifest-python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ordvec-manifest-python" -version = "0.7.0" +version = "0.6.0" edition = "2021" rust-version = "1.89" description = "Python bindings for ordvec-manifest index provenance verification" diff --git a/ordvec-manifest-python/pyproject.toml b/ordvec-manifest-python/pyproject.toml index 30e0dd7..1e2f099 100644 --- a/ordvec-manifest-python/pyproject.toml +++ b/ordvec-manifest-python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "maturin" [project] name = "ordvec-manifest" -version = "0.7.0" +version = "0.6.0" description = "Python bindings for ordvec index manifest verification" readme = "README.md" requires-python = ">=3.10" diff --git a/ordvec-manifest-python/python/ordvec_manifest/__init__.py b/ordvec-manifest-python/python/ordvec_manifest/__init__.py index 1cfae05..4d9790c 100644 --- a/ordvec-manifest-python/python/ordvec_manifest/__init__.py +++ b/ordvec-manifest-python/python/ordvec_manifest/__init__.py @@ -50,4 +50,4 @@ "create_manifest", ] -__version__ = "0.7.0" +__version__ = "0.6.0" diff --git a/ordvec-manifest/Cargo.toml b/ordvec-manifest/Cargo.toml index fdd69a6..88c6de0 100644 --- a/ordvec-manifest/Cargo.toml +++ b/ordvec-manifest/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ordvec-manifest" -version = "0.7.0" +version = "0.6.0" edition = "2021" rust-version = "1.89" license = "MIT OR Apache-2.0" @@ -29,7 +29,7 @@ required-features = ["cli"] chrono = { version = "0.4.44", default-features = false, features = ["clock", "std"] } clap = { version = "4.6.1", features = ["derive"], optional = true } hex = "0.4.3" -ordvec = { version = "0.7.0", path = ".." } +ordvec = { version = "0.6.0", path = ".." } rusqlite = { version = "0.40.0", optional = true } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/ordvec-python/Cargo.toml b/ordvec-python/Cargo.toml index 9e67885..fb0a3cd 100644 --- a/ordvec-python/Cargo.toml +++ b/ordvec-python/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "ordvec-python" -version = "0.7.0" +version = "0.6.0" edition = "2021" rust-version = "1.89" # inherits ordvec's AVX-512 MSRV floor description = "Python bindings for ordvec — training-free ordinal & sign vector quantization" diff --git a/ordvec-python/pyproject.toml b/ordvec-python/pyproject.toml index 918213c..79b26f7 100644 --- a/ordvec-python/pyproject.toml +++ b/ordvec-python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "maturin" [project] name = "ordvec" -version = "0.7.0" +version = "0.6.0" description = "Training-free ordinal & sign quantization for compressed vector retrieval" readme = "README.md" requires-python = ">=3.10" diff --git a/ordvec-python/python/ordvec/__init__.py b/ordvec-python/python/ordvec/__init__.py index 3788968..5cfa291 100644 --- a/ordvec-python/python/ordvec/__init__.py +++ b/ordvec-python/python/ordvec/__init__.py @@ -115,4 +115,4 @@ "SignBitmapIndex", ] -__version__ = "0.7.0" +__version__ = "0.6.0" From 7103c06890c97a4316abd388d3926d1f0a4d5512 Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Sun, 19 Jul 2026 09:23:01 -0500 Subject: [PATCH 2/4] docs(release): normalize the 0.6.0 release notes --- CHANGELOG.md | 49 +++++++++++++++++++++++-------------------------- RELEASING.md | 11 ++++++----- 2 files changed, 29 insertions(+), 31 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0906810..f994b1c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,32 +33,6 @@ _No unreleased changes._ read core; the sqlite registry's private duplicate hasher is deduped onto the public helper. -### Breaking changes - -- **BREAKING (`ordvec-manifest`): deterministic manifest schema v2.** The - manifest schema version is now `ordvec.index_manifest.v2`. `manifest_id` - and `created_at` are removed from `IndexManifest`, creation omits the - optional `build` field (serialized as absent, not `null`), and auxiliary - artifact entries are sorted by - `(name, path)`, so identical bundle content serializes to byte-identical - manifests and `sha256(manifest.json)` is the bundle's content address. - Existing v1 manifests no longer parse; loading one fails with an error - naming both schema versions (zero back-compat, pre-release). Embedded - paths must now be canonical — bundle-relative, forward slashes, no `.`, - `..`, or empty segments — enforced both at creation (non-embeddable - inputs fail `create` instead of minting a manifest that fails its own - verification) and at verification (`*_path_not_canonical` codes, now - also covering calibration and encoder-distortion profile refs). Absolute - paths and escaping `..` paths remain available behind the existing - `allow_absolute_paths` / `allow_path_escape` opt-ins. The - `write_manifest_file` serialization form is documented as the single - canonical byte form: hashing and signing operate on stored bytes, and any - serializer change is a schema-version event. The sqlite report registry - drops its `manifest_id` column: the cached `verification_reports` table is - migrated in place on open (rows preserved, under one atomic transaction), - while the rebuildable `active_manifest` pointer is reset and must be - re-activated. - ### Performance - **Batched sign candidate generation now streams the corpus once per call.** @@ -115,6 +89,29 @@ _No unreleased changes._ ### Changed +- **BREAKING (`ordvec-manifest`): deterministic manifest schema v2.** The + manifest schema version is now `ordvec.index_manifest.v2`. `manifest_id` + and `created_at` are removed from `IndexManifest`, creation omits the + optional `build` field (serialized as absent, not `null`), and auxiliary + artifact entries are sorted by + `(name, path)`, so identical bundle content serializes to byte-identical + manifests and `sha256(manifest.json)` is the bundle's content address. + This is an intentional pre-1.0 minor-version break: v1 manifests from + `0.5.x` do not parse, and loading one fails with an error naming both schema + versions. Embedded paths must now be canonical — bundle-relative, forward + slashes, no `.`, `..`, or empty segments — enforced both at creation + (non-embeddable inputs fail `create` instead of minting a manifest that + fails its own verification) and at verification (`*_path_not_canonical` + codes, now also covering calibration and encoder-distortion profile refs). + Absolute paths and escaping `..` paths remain available behind the existing + `allow_absolute_paths` / `allow_path_escape` opt-ins. The + `write_manifest_file` serialization form is documented as the single + canonical byte form: hashing and signing operate on stored bytes, and any + serializer change is a schema-version event. The sqlite report registry + drops its `manifest_id` column: the cached `verification_reports` table is + migrated in place on open (rows preserved, under one atomic transaction), + while the rebuildable `active_manifest` pointer is reset and must be + re-activated. - **ordvec-manifest: derived artifact size bounds.** Verification now bounds every artifact read by its manifest-declared `file_size_bytes` (the manifest itself remains hard-capped at 1 MiB and SHA-256 pins content); manifest diff --git a/RELEASING.md b/RELEASING.md index f56df20..9af5e7c 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -173,11 +173,12 @@ the OIDC exchange (no risk of a bad publish; just a failed run). lockstep versions, MSRV/docs drift, registry metadata parity, Python classifier/URL parity, docs.rs feature policy, package contents, and release workflow invariants. - - **Downstream un-patch (one-time, 0.6.0):** OrdinalDB's workspace - `Cargo.toml` carries a `[patch.crates-io]` block pointing `ordvec` and - `ordvec-manifest` at this repo's `integration/full-stack` git branch. - When 0.6.0 publishes, that block must be removed so OrdinalDB consumes - the published crates.io releases instead of the pre-release git branch. + - **Downstream un-patch (one-time, 0.6.0):** OrdinalDB's root workspace and + workspace-excluded standalone consumers carry `[patch.crates-io]` blocks + for pre-release `ordvec` and `ordvec-manifest` git revisions. When 0.6.0 + publishes, remove every one of those overrides, regenerate each + independent lockfile, and prove that OrdinalDB consumes the published + crates.io releases instead of a pre-release git revision. 4. Confirm CI is **green for current `main` HEAD**. `require-ci-green` checks `main` HEAD's SHA — which needs a **completed, successful** (not `cancelled`, not in-progress) run of `ci.yml`, `python.yml`, `fuzz.yml`, From e5a5d2d3b2aef5395c31dd1526243c1a44b7fad4 Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Thu, 23 Jul 2026 14:03:01 -0500 Subject: [PATCH 3/4] fix(manifest): harden v0.6.0 canonical writes and MSRV (#298) * fix(manifest): harden canonical writes for v0.6.0 * fix(manifest): close canonical write races * fix(manifest): canonicalize equivalent JSON numbers * test(manifest): make inode swap deterministic * fix(manifest): normalize typed signed zero * Harden v0.6.0 release gates and first-run surfaces (#299) * ci: harden v0.6 release gates * docs: polish v0.6.0 first-run surfaces * fix(release): fail closed across audit gaps * fix(ci): configure fuzz advisory manifest once --- .github/PULL_REQUEST_TEMPLATE.md | 2 +- .github/workflows/audit.yml | 20 +- .github/workflows/ci.yml | 13 +- .github/workflows/release.yml | 124 +++++- CHANGELOG.md | 39 ++ CONTRIBUTING.md | 2 +- Cargo.lock | 22 +- README.md | 125 ++++-- RELEASING.md | 22 +- THREAT_MODEL.md | 55 +-- benchmarks/beir-bench/Cargo.toml | 2 +- deny.toml | 5 +- docs/INDEX_PROVENANCE.md | 10 +- docs/PERSISTED_FORMAT.md | 4 +- docs/artifact-platform-matrix.md | 2 +- docs/compatibility-policy.md | 8 +- examples/quickstart.rs | 20 + fuzz/Cargo.lock | 4 +- ordvec-ffi/Cargo.toml | 2 +- ordvec-manifest-python/Cargo.toml | 2 +- ordvec-manifest-python/README.md | 35 +- ordvec-manifest-python/pyproject.toml | 2 + ordvec-manifest/Cargo.toml | 7 +- ordvec-manifest/README.md | 45 ++ ordvec-manifest/src/lib.rs | 479 +++++++++++++++++++-- ordvec-manifest/src/main.rs | 94 +++- ordvec-manifest/tests/deterministic.rs | 463 +++++++++++++++++++- ordvec-manifest/tests/manifest.rs | 34 +- ordvec-python/Cargo.toml | 2 +- ordvec-python/README.md | 63 ++- ordvec-python/pyproject.toml | 2 + ordvec-python/tests/test_rank_quant.py | 37 +- src/lib.rs | 20 +- tests/release_publish_invariants.py | 346 ++++++++++++++- tests/release_signed_release_invariants.sh | 9 + 35 files changed, 1901 insertions(+), 220 deletions(-) create mode 100644 examples/quickstart.rs diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 5174261..860f57f 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -16,7 +16,7 @@ if this touches public Rust, Python, C ABI, Go, Manifest, persisted-format, examples/docs, feature, or MSRV surfaces - [ ] `CHANGELOG.md` updated under `Unreleased` if user-facing -- [ ] `cargo deny check` passes (licenses / advisories / bans / sources) +- [ ] `cargo deny --workspace --all-features --locked check` passes (licenses / advisories / bans / sources) - [ ] If `ordvec-python/` changed: `cargo clippy -p ordvec-python --all-targets -- -D warnings`, then `maturin develop` + `pytest ordvec-python/tests` pass ## Notes diff --git a/.github/workflows/audit.yml b/.github/workflows/audit.yml index e9daa2d..8c54e25 100644 --- a/.github/workflows/audit.yml +++ b/.github/workflows/audit.yml @@ -10,8 +10,10 @@ name: audit # Read-only token, no `run:` steps (no injection surface). Because this runs # UNATTENDED on a cron schedule, the third-party actions are SHA-pinned (not the # mutable @v4/@v2 tags ci.yml uses on human-triggered push/PR) so a compromised -# tag update cannot auto-execute on a scheduled runner. Scoped to -# `check advisories`. +# tag update cannot auto-execute on a scheduled runner. Scoped to `check +# advisories`, with separate locked roots for the main workspace and the +# standalone fuzz crate so neither committed lockfile can fall outside the +# advisory graph. on: schedule: @@ -36,6 +38,16 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: EmbarkStudios/cargo-deny-action@bb137d7af7e4fb67e5f82a49c4fce4fad40782fe # v2.0.20 + - name: Audit root workspace advisories + uses: EmbarkStudios/cargo-deny-action@bb137d7af7e4fb67e5f82a49c4fce4fad40782fe # v2.0.20 with: - command: check advisories + command: check + arguments: --workspace --all-features --locked + command-arguments: advisories + - name: Audit standalone fuzz lockfile advisories + uses: EmbarkStudios/cargo-deny-action@bb137d7af7e4fb67e5f82a49c4fce4fad40782fe # v2.0.20 + with: + command: check + manifest-path: fuzz/Cargo.toml + arguments: --all-features --locked + command-arguments: advisories diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 07f5c90..a121ea6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -222,6 +222,8 @@ jobs: run: cargo build - name: cargo test run: cargo test + - name: cargo test -p ordvec-manifest --all-features + run: cargo test -p ordvec-manifest --all-features # ---------------------------------------------------------------------- # No-system-deps guarantee + packaging check. Fails if any forbidden @@ -348,9 +350,18 @@ jobs: - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - - uses: EmbarkStudios/cargo-deny-action@bb137d7af7e4fb67e5f82a49c4fce4fad40782fe # v2.0.20 + - name: Check root workspace policy + uses: EmbarkStudios/cargo-deny-action@bb137d7af7e4fb67e5f82a49c4fce4fad40782fe # v2.0.20 + with: + command: check + arguments: --workspace --all-features --locked + - name: Audit standalone fuzz lockfile advisories + uses: EmbarkStudios/cargo-deny-action@bb137d7af7e4fb67e5f82a49c4fce4fad40782fe # v2.0.20 with: command: check + manifest-path: fuzz/Cargo.toml + arguments: --all-features --locked + command-arguments: advisories # ---------------------------------------------------------------------- # AVX-512 coverage via Intel SDE. GitHub-hosted runners lack AVX-512, so we diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4052b3d..9edc611 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -161,6 +161,7 @@ jobs: needs: guard if: needs.guard.outputs.ok == 'true' runs-on: ubuntu-latest + timeout-minutes: 35 permissions: contents: read actions: read @@ -174,26 +175,105 @@ jobs: GH_TOKEN: ${{ github.token }} REPO: ${{ github.repository }} SHA: ${{ github.sha }} + POLL_INTERVAL_SECONDS: 30 + POLL_TIMEOUT_SECONDS: 1800 run: | set -euo pipefail - MAIN_SHA="$(gh api "repos/${REPO}/commits/main" --jq '.sha')" - echo "current main sha: ${MAIN_SHA}" - if [ "$SHA" != "$MAIN_SHA" ]; then - echo "::error::release tag points at ${SHA}, but current main is ${MAIN_SHA}. Move the tag to the current protected main HEAD and re-run." - exit 1 - fi - # Require a SUCCESSFUL push run for this SHA *on main* for each workflow. - # Filtering on branch as well as head_sha stops a green run for the same - # commit on an unrelated branch from satisfying the gate. - for wf in ci.yml python.yml fuzz.yml codeql.yml actionlint.yml zizmor.yml; do - ok="$(gh api \ - "repos/${REPO}/actions/workflows/${wf}/runs?head_sha=${SHA}&branch=main&event=push&status=success&per_page=20" \ - --jq '[.workflow_runs[] | select(.head_branch == "main" and .event == "push" and .conclusion == "success")] | length')" - echo "successful push ${wf} runs for ${SHA} on main: ${ok}" - if [ "${ok}" -lt 1 ]; then - echo "::error::no successful push ${wf} run for ${SHA} on main. Push to main, let CI pass, then re-tag." + deadline=$((SECONDS + POLL_TIMEOUT_SECONDS)) + while :; do + MAIN_SHA="$(gh api "repos/${REPO}/commits/main" --jq '.sha')" + echo "current main sha: ${MAIN_SHA}" + if [ "$SHA" != "$MAIN_SHA" ]; then + echo "::error::release tag points at ${SHA}, but current main is ${MAIN_SHA}. Move the tag to the current protected main HEAD and re-run." + exit 1 + fi + + all_green=true + for wf in ci.yml python.yml fuzz.yml codeql.yml actionlint.yml zizmor.yml; do + # Query every status, then inspect the latest exact-SHA push run. + # A success-only filter cannot distinguish normal run-visibility + # delay from terminal failure and was the source of issue #272. + api_error="$(mktemp)" + if runs_json="$(gh api \ + "repos/${REPO}/actions/workflows/${wf}/runs?head_sha=${SHA}&branch=main&event=push&per_page=100" \ + 2>"$api_error")"; then + : + else + api_exit_status=$? + api_message="$(<"$api_error")" + api_status="$(sed -nE 's/.*HTTP ([0-9]{3}).*/\1/p' "$api_error" | tail -n 1)" + retryable_api_error=false + case "$api_status" in + 408|429|5??) + retryable_api_error=true + ;; + "") + if [ "$api_exit_status" -ne 4 ] && grep -Eiq \ + 'timeout|timed out|connection (reset|refused|closed)|could not resolve|temporary failure in name resolution|network is unreachable|TLS handshake|unexpected EOF|error connecting to' \ + "$api_error"; then + retryable_api_error=true + fi + ;; + 403) + if grep -Eiq 'rate.?limit|secondary rate|abuse detection' "$api_error"; then + retryable_api_error=true + fi + ;; + esac + rm -f "$api_error" + if "$retryable_api_error"; then + echo "waiting for ${wf}: transient API query failure${api_status:+ (HTTP ${api_status})}: ${api_message}" + all_green=false + continue + fi + echo "::error::${wf}: API query failed permanently${api_status:+ (HTTP ${api_status})}: ${api_message}" + exit 1 + fi + rm -f "$api_error" + + latest="$(jq -r --arg sha "$SHA" ' + [.workflow_runs[] + | select(.head_sha == $sha and .head_branch == "main" and .event == "push")] + | sort_by([.run_number, .run_attempt, .id]) + | last + | if . == null then "" + else [.status, (.conclusion // ""), (.html_url // "")] | join("|") + end + ' <<<"$runs_json")" + if [ -z "$latest" ]; then + echo "waiting for ${wf}: no exact-SHA main push run is visible yet" + all_green=false + continue + fi + + IFS='|' read -r status conclusion run_url <<<"$latest" + if [ "$status" != "completed" ]; then + echo "waiting for ${wf}: latest run is ${status} (${run_url})" + all_green=false + elif [ "$conclusion" = "success" ]; then + echo "green ${wf}: ${run_url}" + else + echo "::error::${wf} completed with ${conclusion:-no conclusion} for ${SHA}: ${run_url}" + exit 1 + fi + done + + if "$all_green"; then + FINAL_MAIN_SHA="$(gh api "repos/${REPO}/commits/main" --jq '.sha')" + echo "final current main sha: ${FINAL_MAIN_SHA}" + if [ "$SHA" != "$FINAL_MAIN_SHA" ]; then + echo "::error::main advanced from release tag ${SHA} to ${FINAL_MAIN_SHA} while CI was being checked. Tag only the new protected main HEAD after its own CI is green." + exit 1 + fi + echo "all release-gated workflows are green for ${SHA} on main" + break + fi + if (( SECONDS >= deadline )); then + echo "::error::timed out after ${POLL_TIMEOUT_SECONDS}s waiting for exact-SHA main push CI. No artifacts or draft release were created." exit 1 fi + echo "CI is not settled; polling again in ${POLL_INTERVAL_SECONDS}s" + sleep "$POLL_INTERVAL_SECONDS" done release-avx512: @@ -264,7 +344,7 @@ jobs: notes: name: release notes (git-cliff) + draft Release - needs: guard + needs: [guard, require-ci-green] if: needs.guard.outputs.ok == 'true' runs-on: ubuntu-latest permissions: @@ -302,7 +382,7 @@ jobs: build-crate: name: build .crate + SBOM - needs: guard + needs: [guard, require-ci-green] if: needs.guard.outputs.ok == 'true' runs-on: ubuntu-latest steps: @@ -393,7 +473,7 @@ jobs: build-wheels: name: wheel ${{ matrix.platform.target }} (${{ matrix.platform.runner }}) - needs: guard + needs: [guard, require-ci-green] if: needs.guard.outputs.ok == 'true' runs-on: ${{ matrix.platform.runner }} strategy: @@ -536,7 +616,7 @@ jobs: build-sdist: name: build sdist + SBOM - needs: guard + needs: [guard, require-ci-green] if: needs.guard.outputs.ok == 'true' runs-on: ubuntu-latest steps: @@ -609,7 +689,7 @@ jobs: build-manifest-wheels: name: manifest wheel ${{ matrix.platform.target }} (${{ matrix.platform.runner }}) - needs: guard + needs: [guard, require-ci-green] if: needs.guard.outputs.ok == 'true' runs-on: ${{ matrix.platform.runner }} strategy: @@ -674,7 +754,7 @@ jobs: build-manifest-sdist: name: build manifest sdist + SBOM - needs: guard + needs: [guard, require-ci-green] if: needs.guard.outputs.ok == 'true' runs-on: ubuntu-latest steps: diff --git a/CHANGELOG.md b/CHANGELOG.md index f994b1c..2bc72fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -89,6 +89,14 @@ _No unreleased changes._ ### Changed +- **First-run and release surfaces are aligned for v0.6.0.** The root and PyPI + READMEs now lead with a tiny deterministic retrieval that prints a checked + result, show persistence/reopen, and route users to the Rust, Python, CLI, + and manifest packages. Both manifest packages include an end-to-end + create-and-verify path, and every `ordvec-manifest` command and option now + has CLI help text. Release invariants pin the current package version, + manifest schema, feature defaults, public metadata links, and first-run + examples so the next release cannot silently drift across surfaces. - **BREAKING (`ordvec-manifest`): deterministic manifest schema v2.** The manifest schema version is now `ordvec.index_manifest.v2`. `manifest_id` and `created_at` are removed from `IndexManifest`, creation omits the @@ -136,6 +144,37 @@ _No unreleased changes._ `sha256_file_bounded` no longer materialises the file in memory before hashing. +### Fixed + +- **`ordvec-manifest`: canonical content addresses are independent of + `serde_json` feature unification.** Nested extension and attestation maps are + recursively key-sorted and JSON numbers are normalized before serialization, + so downstream `preserve_order` / `arbitrary_precision` features cannot change + the same logical manifest's bytes. Manifest creation now rejects non-UTF-8 + artifact paths instead of embedding the lossy replacement character. +- **`ordvec-manifest`: manifest writes are atomic and fail before replacement.** + `write_manifest_file` writes and syncs a same-directory temporary file before + atomically replacing the destination, and rejects non-finite distortion + values before touching an existing manifest. New-file mode/umask behavior and + existing portable permission bits are preserved. +- **`ordvec-manifest`: restore the declared Rust 1.89 floor for SQLite builds.** + The optional `rusqlite` dependency returns to the compatible 0.39 line after + 0.40's build dependency used a library feature unavailable on Rust 1.89; the + all-features manifest suite now runs in the permanent MSRV lane. + +### Security + +- The release CI gate now waits, with a 30-minute fail-closed deadline, for + the latest exact-HEAD push runs instead of treating normal Actions + visibility/in-progress delay as an immediate failure. It rejects terminal + non-successes and a moving `main`, and every draft-release or artifact-build + job is directly blocked behind the gate. +- The advisory policy now roots `cargo-deny` at every workspace member and + enables every feature, closing the blind spot that omitted dev-only + benchmark dependencies. `anyhow` is updated from 1.0.102 to 1.0.103 to + clear RUSTSEC-2026-0190; the documented dev-only `bincode` 1.x + unmaintained advisory remains explicitly triaged. + ## 0.5.0 - 2026-06-19 ### Security diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index d7c7eab..a2340e8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -44,7 +44,7 @@ cargo test --no-default-features cargo +1.89.0 build # MSRV cargo build --locked RUSTFLAGS="-D warnings" cargo build -cargo deny check # licenses / advisories / bans / sources +cargo deny --workspace --all-features --locked check # full locked workspace graph ``` SIMD dispatches at runtime — AVX-512 / AVX2 on x86_64, NEON on aarch64, diff --git a/Cargo.lock b/Cargo.lock index c505790..b36e6f1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -95,9 +95,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" [[package]] name = "autocfg" @@ -503,17 +503,14 @@ name = "hashbrown" version = "0.17.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" -dependencies = [ - "foldhash 0.2.0", -] [[package]] name = "hashlink" -version = "0.12.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a5081f264ed7adee96ea4b4778b6bb9da0a7228b084587aa3bd3ff05da7c5a3b" +checksum = "824e001ac4f3012dd16a264bec811403a67ca9deb6c102fc5049b32c4574b35f" dependencies = [ - "hashbrown 0.17.1", + "hashbrown 0.16.1", ] [[package]] @@ -678,9 +675,9 @@ checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" [[package]] name = "libsqlite3-sys" -version = "0.38.1" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f6c19a05435c21ac299d71b6a9c13db3e3f47c520517d58990a462a1397a61db" +checksum = "b1f111c8c41e7c61a49cd34e44c7619462967221a6443b0ec299e0ac30cfb9b1" dependencies = [ "cc", "pkg-config", @@ -1182,9 +1179,9 @@ dependencies = [ [[package]] name = "rusqlite" -version = "0.40.1" +version = "0.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11438310b19e3109b6446c33d1ed5e889428cf2e278407bc7896bc4aaea43323" +checksum = "a0d2b0146dd9661bf67bb107c0bb2a55064d556eeb3fc314151b957f313bcd4e" dependencies = [ "bitflags", "fallible-iterator", @@ -1277,6 +1274,7 @@ version = "1.0.150" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8014e44b4736ed0538adeecded0fce2a272f22dc9578a7eb6b2d9993c74cfb9" dependencies = [ + "indexmap", "itoa", "memchr", "serde", diff --git a/README.md b/README.md index 0442cd9..453cfbd 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,84 @@ Training-free ordinal & sign quantization for vector retrieval. that quantizes the **ordinal (rank) and sign structure** of an embedding — no codebook, no learned rotation, no graph to build. +## Choose your surface + +| You want to... | Use | Install | +|---|---|---| +| Build retrieval in Rust | [`ordvec`](https://crates.io/crates/ordvec) | `cargo add ordvec@0.6` | +| Build retrieval in Python | [`ordvec`](https://pypi.org/project/ordvec/) | `python -m pip install ordvec` | +| Create or verify index manifests from Rust or a CLI | [`ordvec-manifest`](https://crates.io/crates/ordvec-manifest) | `cargo install ordvec-manifest` | +| Verify manifests from Python | [`ordvec-manifest`](https://pypi.org/project/ordvec-manifest/) | `python -m pip install ordvec-manifest` | + +## Quickstart: see a result in 30 seconds + +Python: + +```bash +python -m pip install --upgrade ordvec +``` + +```python +import numpy as np +from ordvec import RankQuant + +documents = np.array([ + [8, 7, 6, 5, 4, 3, 2, 1], + [1, 2, 3, 4, 5, 6, 7, 8], + [8, 1, 7, 2, 6, 3, 5, 4], +], dtype=np.float32) +query = np.array([[8, 7, 6, 5, 4, 3, 2, 1]], dtype=np.float32) + +index = RankQuant(dim=8, bits=1) +index.add(documents) +scores, ids = index.search_asymmetric(query, k=1) +print(f"top document: {ids[0, 0]} (score {scores[0, 0]:.3f})") +``` + +```text +top document: 0 (score 0.396) +``` + +Rust: + +```bash +cargo add ordvec@0.6 +``` + +Or add the dependency directly: + +```toml +[dependencies] +ordvec = "0.6" +``` + +```rust +use ordvec::RankQuant; + +let documents = [ + 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, + 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, + 8.0, 1.0, 7.0, 2.0, 6.0, 3.0, 5.0, 4.0, +]; +let query = [8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0]; + +let mut index = RankQuant::new(8, 1); +index.add(&documents); +let results = index.search_asymmetric(&query, 1); +assert_eq!(results.indices_for_query(0)[0], 0); +``` + +The runnable Rust version is [`examples/quickstart.rs`](examples/quickstart.rs). + +Persist and reopen the same index without keeping the original float corpus: + +```python +index.write("quickstart.ovrq") +reopened = RankQuant.load("quickstart.ovrq") +_, reopened_ids = reopened.search_asymmetric(query, k=1) +assert reopened_ids[0, 0] == 0 +``` + ## What is ordinal retrieval? Ordinal retrieval is a retrieval family where the index operates on order/sign @@ -78,7 +156,7 @@ and the gap widens over the committed subsampling sweep: layers are the right comparison target; this README does not claim public million-scale HNSW crossover or GPU bandwidth numbers until the underlying run artifacts are committed. -- **Reproducible on your machine, one command:** +- **Reproducible on your machine:** ```sh make bench-beir-setup # Python deps + CUDA llama-cpp-python (GGUF Q8 encoder) @@ -124,7 +202,7 @@ structure of each vector on its own: known before you see any data (256 B at dim = 1024, 2-bit), with `bits ∈ {1, 2, 4}` the size/recall knob. (`b = 8` is an opt-in evidence/refinement width — asymmetric scoring at any dim, symmetric only - when `dim % 256 == 0` — not a broad retrieval mode. In v0.5.0 it is + when `dim % 256 == 0` — not a broad retrieval mode. In v0.6.0 it is Rust-only, in-memory, not accepted by the Python `RankQuant` constructor, and not persistable to `.ovrq`; each prepared asymmetric query owns a `dim * 256` `f32` LUT, about 64 MiB at the maximum dimension.) @@ -148,7 +226,7 @@ large-scale serving rather than competing with one. - **`RankQuant`** — ranks bucketed into `1 << bits` equal-width bins, `bits` bits per coordinate (`dim * bits / 8` bytes/doc). Both a symmetric (Spearman) and asymmetric (float-query LUT) scorer. `bits ∈ - {1, 2, 4}` are the cross-language persisted retrieval widths in v0.5.0; + {1, 2, 4}` are the cross-language persisted retrieval widths in v0.6.0; `b = 8` is Rust-only and in-memory for evidence/refinement. - **`Bitmap`** — a top-bucket bitmap per document (one bit per coordinate); scoring is `popcount(Q AND D)`, a coarsened rank overlap. @@ -162,8 +240,8 @@ Two further paths, for callers who need them: scalar dispatch) for absolute-minimum stage-1 scan latency, at 2× the RankQuant b=2 footprint (`dim/2` bytes/doc) and 8-bit LUT scoring noise. It persists to `.ovfs` (magic `OVFS`) through direct - `RankQuantFastscan::{write,load}` calls. In v0.5.0, `.ovfs` is not yet part - of the `probe_index_metadata()` / `ordvec-manifest` v1 contract; bind it with + `RankQuantFastscan::{write,load}` calls. In v0.6.0, `.ovfs` is not yet part + of the `probe_index_metadata()` / `ordvec-manifest` v2 contract; bind it with an application-owned digest or attestation if it crosses a trust boundary. Reach for it only when scan latency at b=2 is the binding constraint; the headline retrieval surface is still `RankQuant` / `Bitmap` / two-stage. @@ -208,36 +286,6 @@ empirical contract to measure. Details in [`docs/RANK_MODES.md`](docs/RANK_MODES.md). -## Quickstart - -```toml -[dependencies] -ordvec = "0.6" - -# Or, to track unreleased `main`, use a git dependency instead: -# ordvec = { git = "https://github.com/Project-Navi/ordvec" } -``` - -```rust -use ordvec::RankQuant; - -let dim = 1024; -let n_docs = 10_000; -let mut index = RankQuant::new(dim, 2); // 2 bits/coord → 256 bytes/doc - -// `add` takes a flat, row-major buffer of `dim * n_docs` f32s. -// Replace this with your real embeddings. -let doc_embeddings: Vec = vec![0.0; dim * n_docs]; -index.add(&doc_embeddings); - -// Asymmetric scan: full-precision queries vs bucketed docs (recommended). -let query_embeddings: Vec = vec![0.0; dim * 4]; // 4 queries, row-major -let results = index.search_asymmetric(&query_embeddings, 10); - -let top_ids = results.indices_for_query(0); // top-10 doc ids for query 0 -let top_scores = results.scores_for_query(0); -``` - For the two-stage compressed-scan path (`Bitmap` / `SignBitmap` candidate generation → `RankQuant` rerank) and the full mode comparison, see [`docs/RANK_MODES.md`](docs/RANK_MODES.md). @@ -342,9 +390,6 @@ candidate slices passed to `Search` until the call returns. [`theorem-map`](https://github.com/Project-Navi/ordvec-formalization/blob/main/docs/theorem-map.md), and [`reviewer brief`](https://github.com/Project-Navi/ordvec-formalization/blob/main/docs/reviewer-brief.md). - **API docs:** , -- **Paper (OrdVec / RankQuant):** _link TBD — see - [Research collaboration](#research-collaboration)._ - ## Benchmarks ### BEIR retrieval (public datasets, reproducible) @@ -517,8 +562,8 @@ The probe/manifest-covered on-disk formats (`.ovr` / `.ovrq` / `.ovbm` / **no built-in checksum, MAC, or signature — by design.** The loaders validate *structure* (magic, version, bounds, exact-length payload) but not *origin*: a structurally valid file can still be untrusted. `RankQuantFastscan` also writes -and loads `.ovfs` directly, but in v0.5.0 that format is not yet covered by -`probe_index_metadata()` or `ordvec-manifest` v1. If an index file crosses a +and loads `.ovfs` directly, but in v0.6.0 that format is not yet covered by +`probe_index_metadata()` or `ordvec-manifest` v2. If an index file crosses a trust boundary (network transfer, shared storage), verify it before loading. `ordvec-manifest` binds supported index files to a JSON manifest by SHA-256, header metadata, row identity, named auxiliary sidecars, and attestation shape diff --git a/RELEASING.md b/RELEASING.md index 9af5e7c..fbed381 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -24,8 +24,12 @@ The unified `release.yml`: workflow but skip every job below the gate; - runs a **`require-ci-green`** gate confirming the tag points at current `main` HEAD and that per-commit CI is green on `main` for that SHA — `ci.yml`, - `python.yml`, `fuzz.yml`, `codeql.yml`, `actionlint.yml`, `zizmor.yml` (a - *successful* run for that exact SHA on `main`); + `python.yml`, `fuzz.yml`, `codeql.yml`, `actionlint.yml`, `zizmor.yml`. The + gate polls the latest exact-SHA push run every 30 seconds for at most 30 + minutes: missing, queued, and in-progress runs wait; a terminal non-success + fails immediately; and a moved `main` HEAD fails immediately. Every draft- + release and build-artifact job depends directly on this gate, so an unsettled + or failed gate creates no release artifacts and no draft GitHub Release; - publishes via **OIDC trusted publishing** (no long-lived crates.io / PyPI tokens in the repo) for both Rust crates and both Python distributions; - canonicalizes each Python dist before attestation and release upload: for a @@ -167,8 +171,11 @@ the OIDC exchange (no risk of a bad publish; just a failed run). `ordvec-manifest-python/Cargo.toml`, `ordvec-manifest-python/pyproject.toml`, `ordvec-manifest-python/python/ordvec_manifest/__init__.py`, and - `ordvec-ffi/Cargo.toml`) and update `CHANGELOG.md` with migration notes for - every intentional compatibility break. Commit on `main`. + `ordvec-ffi/Cargo.toml`). Also bump every internal path-dependency version: + `ordvec-manifest` → `ordvec`, both Python binding aliases, `ordvec-ffi` → + `ordvec`, and `benchmarks/beir-bench` → `ordvec`. Update `CHANGELOG.md` + with migration notes for every intentional compatibility break. Commit on + `main`. - Run `python tests/release_publish_invariants.py` after the bump; it checks lockstep versions, MSRV/docs drift, registry metadata parity, Python classifier/URL parity, docs.rs feature policy, package contents, and @@ -181,8 +188,11 @@ the OIDC exchange (no risk of a bad publish; just a failed run). crates.io releases instead of a pre-release git revision. 4. Confirm CI is **green for current `main` HEAD**. `require-ci-green` checks `main` HEAD's SHA — which needs a **completed, successful** (not - `cancelled`, not in-progress) run of `ci.yml`, `python.yml`, `fuzz.yml`, - `codeql.yml`, `actionlint.yml`, and `zizmor.yml`. + `cancelled`) latest run of `ci.yml`, `python.yml`, `fuzz.yml`, `codeql.yml`, + `actionlint.yml`, and `zizmor.yml`. A just-pushed tag may reach the release + workflow before Actions exposes or completes those runs, so the gate waits + up to 30 minutes and polls every 30 seconds. It fails immediately on a + terminal non-success or if `main` advances, and fails closed on timeout. - Routine `ci.yml` / `coverage.yml` runs may warn and skip SDE-dependent steps when Intel's downloadmirror challenges GitHub-hosted runners. That keeps external mirror outages from holding `main` red, but it does **not** diff --git a/THREAT_MODEL.md b/THREAT_MODEL.md index a096f29..aad3d55 100644 --- a/THREAT_MODEL.md +++ b/THREAT_MODEL.md @@ -1,6 +1,6 @@ # Threat Model — `ordvec` -> **Status:** v0.6.0 (pre-1.0), 2026-06-15. This is the maintained threat model +> **Status:** v0.6.0 (pre-1.0), 2026-07-19. This is the maintained threat model > for the `ordvec` Rust crate, C ABI, Go wrapper, PyO3/maturin Python bindings, > and the `ordvec-manifest` sidecar verifier. It is reviewed when the > attack surface changes (new persistence formats, new `unsafe` kernels, new @@ -55,10 +55,10 @@ merged, (2) enforceable by tests or CI, (3) local to the library boundary, and (4) unlikely to add operational burden downstream. Heavyweight controls (mandatory index signing, long-running fuzz farms, service-level admission control) are documented as **deployment guidance** unless the project has -maintainer capacity to own them. Release publication requires a non-triggering -approver through protected GitHub Environments; the residual release -supply-chain risk is approver account compromise / collusion, not a -single-owner project structure (see THREAT-SUPPLY-001). +maintainer capacity to own them. Release policy requires a non-triggering +approver through protected GitHub Environments, but that policy is only in +force when the pre-tag environment-settings audit passes. The live settings +currently fail that audit and block v0.6.0 (see THREAT-SUPPLY-001). --- @@ -306,7 +306,7 @@ applications must validate paths before calling"). ## 5. Supply-chain threats (THREAT-SUPPLY) -### 5.1 Existing controls (verified) +### 5.1 Workflow controls (verified) and environment audit status **Workflow code (all workflows):** third-party actions pinned by commit SHA (the one mandated exception is the SLSA reusable workflow, which the SLSA @@ -327,6 +327,11 @@ attestation fails the release closed. Each Rust publish job proves pre- and post-publish crates.io byte identity against the attested `.crate`; PyPI additionally gets **PEP 740** attestations via Trusted Publishing. +The environment protections are mutable GitHub configuration outside the +workflow. `tests/release_environment_settings.sh` audits them before a tag is +created. As of 2026-07-19 that audit fails; the exact drift and release gate are +recorded in THREAT-SUPPLY-001 below. + **Static / supply-chain analysis:** **CodeQL** scans Rust, Python, and Actions (no-build databases); **OpenSSF Scorecard** publishes SARIF to code scanning and the score badge; **zizmor** audits workflow hardening (pinned); a @@ -337,23 +342,25 @@ scoped to the wheel. ### 5.2 Risks -**THREAT-SUPPLY-001 (mitigated; residual = release-approver account -compromise / collusion): Release configuration and ownership.** The release -**environments** (`pypi`, `crates-io`) list `Fieldnote-Echo` and `toadkicker` as -required reviewers, enable **prevent self-review**, enforce a **30-minute wait -timer**, and restrict deployment to the **release-tag pattern -`v[0-9]*.[0-9]*.[0-9]*`** (the tag-triggered workflow runs on -`refs/tags/...`, not `refs/heads/main`, so a branch-only allowlist would -deadlock publishing — see RELEASING.md). The `require-ci-green` gate -independently verifies the tag SHA has a successful push-event CI run on `main`, -and `main` itself is branch-protected (PR review, no force-push) — so a release -cannot be cut from an unmerged or attacker branch, and no publish runs without -an explicit human approval by a listed release approver who did not trigger the -deployment. The remaining residual is compromise or misuse of an eligible -approver account, or collusion between release participants. *Mitigations:* -strong 2FA / passkeys on both approver accounts, a small reviewed approver list, -and the 30-minute deployment window for the non-triggering approver to inspect -or cancel a bad release. See [`RELEASING.md`](RELEASING.md). +**THREAT-SUPPLY-001 (open release blocker as of 2026-07-19): Release +configuration and ownership.** The intended release-environment policy for +both `pypi` and `crates-io` is: `Fieldnote-Echo` and `toadkicker` are eligible +reviewers, self-review is prevented, a 30-minute wait timer is enforced, and +deployment is restricted to the release-tag pattern +`v[0-9]*.[0-9]*.[0-9]*`. The live environments currently retain the correct +tag pattern but list only `Fieldnote-Echo`, permit self-review, and have no wait +timer. Consequently `tests/release_environment_settings.sh` fails and **no +v0.6.0 tag or publish should proceed**. Restore `toadkicker`, prevent +self-review, and the 30-minute wait on both environments, then rerun the audit +before tagging. The `require-ci-green` gate independently verifies the tag SHA +has a successful push-event CI run on `main`, and `main` itself is +branch-protected (PR review, no force-push), but those controls do not replace +the independent publication approval gate. Once the environment audit passes, +the remaining residual is compromise or misuse of an eligible approver account, +or collusion between release participants. *Mitigations:* strong 2FA / passkeys +on both approver accounts, a small reviewed approver list, and the 30-minute +deployment window for the non-triggering approver to inspect or cancel a bad +release. See [`RELEASING.md`](RELEASING.md). **THREAT-SUPPLY-002 (mitigated): Release immutability and tag integrity.** Published artifacts are **immutable by registry design** — crates.io is @@ -507,7 +514,7 @@ blast radius of a compromised dependency separately. | THREAT-FFI-003 | FFI | Binding | Accidental telemetry through ABI stats | Low | Low | **Mitigated** — caller-owned stats, no logging | | THREAT-FFI-004 | FFI | Binding | Concurrent input mutation during released-GIL call | Medium | Medium | **P2** — documented contract | | THREAT-FFI-005 | FFI | Binding | Unsanitized path forwarding | Medium | Medium | **P2** — documented contract | -| THREAT-SUPPLY-001 | Supply chain | Config | Release config / dual-approver gate | Low | Critical | **Mitigated** (two approvers, self-review blocked, 30-minute wait timer, `require-ci-green` main-SHA gate); residual = approver compromise / collusion | +| THREAT-SUPPLY-001 | Supply chain | Config | Release config / dual-approver gate | Medium | Critical | **Release blocked** until both environment settings pass the pre-tag audit; tag policy and `require-ci-green` remain in place | | THREAT-SUPPLY-002 | Supply chain | Config | Release immutability / tag integrity | Low | High | **Mitigated** — registries immutable; GitHub immutable releases on + `main` protected | | THREAT-SUPPLY-003 | Supply chain | Config | Typosquatting adjacent names | Medium | Medium | P3 | | THREAT-QUERY-001 | Resource | Deployment | Batch / `k` exhaustion in serving | Medium | Medium | **P2** — deployment docs | diff --git a/benchmarks/beir-bench/Cargo.toml b/benchmarks/beir-bench/Cargo.toml index 5122bf5..04349b6 100644 --- a/benchmarks/beir-bench/Cargo.toml +++ b/benchmarks/beir-bench/Cargo.toml @@ -13,7 +13,7 @@ name = "beir-bench" path = "src/main.rs" [dependencies] -ordvec = { path = "../.." } +ordvec = { version = "0.6.0", path = "../.." } # Pure-Rust HNSW (Malkov–Yashunin); no system/C++ deps. The faithful portable # stand-in for the C++ hnswlib (no maintained Rust binding to that exists). hnsw_rs = "0.3" diff --git a/deny.toml b/deny.toml index 17acba9..5e20903 100644 --- a/deny.toml +++ b/deny.toml @@ -1,6 +1,6 @@ # cargo-deny configuration for ordvec. # -# Run locally with: cargo deny check +# Run locally with: cargo deny --workspace --all-features --locked check # Schema validated against cargo-deny 0.19.x. # # ordvec ships no system/numerical dependencies (no BLAS, faer, ndarray, @@ -36,6 +36,8 @@ ignore = ["RUSTSEC-2025-0141"] # * BSD-2-Clause — zerocopy / zerocopy-derive are # `BSD-2-Clause OR Apache-2.0 OR MIT`; the OR resolves via Apache/MIT, but # allowing BSD-2-Clause keeps the choice explicit and audit-clear. +# * Zlib — foldhash, transitively used by the workspace's +# dev-only HNSW benchmark and the optional manifest SQLite implementation. # * Apache-2.0 WITH LLVM-exception — wasi's expression includes it; only # pulled on wasm targets, harmless to allow and keeps cross-target clean. allow = [ @@ -44,6 +46,7 @@ allow = [ "Apache-2.0 WITH LLVM-exception", "Unicode-3.0", "BSD-2-Clause", + "Zlib", ] # License text must match an SPDX template at >= this confidence to count. confidence-threshold = 0.8 diff --git a/docs/INDEX_PROVENANCE.md b/docs/INDEX_PROVENANCE.md index 03fa45e..4e92113 100644 --- a/docs/INDEX_PROVENANCE.md +++ b/docs/INDEX_PROVENANCE.md @@ -4,8 +4,8 @@ `.ovrq` / `.ovbm` / `.ovsb` files and reloads them through `Rank::load`, `RankQuant::load`, `Bitmap::load`, and `SignBitmap::load`. `RankQuantFastscan` also writes and loads `.ovfs` through its direct API, but in -v0.5.0 `.ovfs` is not covered by `probe_index_metadata()` or -`ordvec-manifest` v1. This note states exactly **what the loaders guarantee and +v0.6.0 `.ovfs` is not covered by `probe_index_metadata()` or +`ordvec-manifest` v2. This note states exactly **what the loaders guarantee and what they do not**, so you can decide whether an index file needs out-of-band verification before you load it. For the byte layout and versioning of the persisted formats themselves, see [`PERSISTED_FORMAT.md`](PERSISTED_FORMAT.md). @@ -105,12 +105,12 @@ The manifest verifier checks: least one subject SHA-256 matching the artifact when attestations are supplied. -The v1 verifier intentionally does not create or verify `.ovfs` FastScan +The v2 verifier intentionally does not create or verify `.ovfs` FastScan artifacts yet. If a `RankQuantFastscan` artifact crosses a trust boundary in -v0.5.0, bind the bytes with a caller-owned checksum, artifact-store control, or +v0.6.0, bind the bytes with a caller-owned checksum, artifact-store control, or attestation and load it directly only after that policy check succeeds. The direct `.ovfs` loader still rejects invalid nibbles, non-canonical block-tail -padding, and rows that violate b=2 constant composition; manifest v1 simply +padding, and rows that violate b=2 constant composition; manifest v2 simply does not bind or probe those bytes yet. Auxiliary artifacts are for application-owned sidecars such as metadata, diff --git a/docs/PERSISTED_FORMAT.md b/docs/PERSISTED_FORMAT.md index 3f19677..556b6b7 100644 --- a/docs/PERSISTED_FORMAT.md +++ b/docs/PERSISTED_FORMAT.md @@ -65,7 +65,7 @@ cache in their own manifests: `n_top`; - `file_size_bytes`: total observed file size. -In v0.5.0, `probe_index_metadata(path)` rejects `OVFS` with an unsupported +In v0.6.0, `probe_index_metadata(path)` rejects `OVFS` with an unsupported metadata-probe error rather than returning a partial descriptor. Load `.ovfs` only through `RankQuantFastscan::load` unless and until the FastScan metadata contract is promoted in a later minor release; the direct loader rejects @@ -227,7 +227,7 @@ payload invariants: - `SignBitmap::load`: no additional row invariant exists. `RankQuantFastscan::load` has its own direct loader path for `.ovfs`; it is not -covered by this probe-versus-load contract in v0.5.0. +covered by this probe-versus-load contract in v0.6.0. Loader success is the primitive binary-safety boundary. It is not a provenance or deployment-policy decision. diff --git a/docs/artifact-platform-matrix.md b/docs/artifact-platform-matrix.md index 8218093..d88a20b 100644 --- a/docs/artifact-platform-matrix.md +++ b/docs/artifact-platform-matrix.md @@ -13,7 +13,7 @@ environment matching a platform family is supported. | Surface | Published where | Platform/build contract | Release verification | | --- | --- | --- | --- | | `ordvec` Rust crate | crates.io package `ordvec`; GitHub Release `.crate` asset | Rust 1.89 MSRV; default features empty; pure Rust, no BLAS/LAPACK/system numeric dependency | `cargo package --locked`; GitHub/Sigstore/SLSA provenance; pre-publish and post-publish byte identity against crates.io | -| `ordvec-manifest` Rust crate | crates.io package `ordvec-manifest`; GitHub Release `.crate` asset | Rust 1.89 MSRV; default features empty; optional `cli`, `sqlite`, and `sqlite-bundled` features | Built after matching `ordvec` exists; GitHub/Sigstore/SLSA provenance; byte identity against crates.io | +| `ordvec-manifest` Rust crate | crates.io package `ordvec-manifest`; GitHub Release `.crate` asset | Rust 1.89 MSRV; default feature `cli`; optional `sqlite` and `sqlite-bundled` features; library-only consumers can disable defaults | Built after matching `ordvec` exists; GitHub/Sigstore/SLSA provenance; byte identity against crates.io | | Python `ordvec` | PyPI package `ordvec`; GitHub Release wheels and sdist | CPython 3.10+ abi3; `numpy>=2.2`; wheels for Linux x86_64 and Linux aarch64 are manylinux/glibc wheels; no musllinux/Alpine wheel is shipped yet; macOS aarch64 and Windows x64 wheels are also published; native extension modules are embedded in the wheel and do not load a separate `ordvec_ffi` library | Canonical wheel/sdist selection; linux/aarch64 native smoke; PyPI hash verification; PEP 740 attestation on fresh upload | | Python `ordvec-manifest` | PyPI package `ordvec-manifest`; GitHub Release wheels and sdist | CPython 3.10+ abi3; Linux wheels are manylinux/glibc for x86_64 and aarch64; no musllinux/Alpine wheel is shipped yet; macOS aarch64 and Windows x64 wheels are also published; native extension modules are embedded in the wheel | Canonical wheel/sdist selection; linux/aarch64 native smoke; PyPI hash verification; PEP 740 attestation on fresh upload | | Node/WASM | Not shipped; no npm package is published yet | Placeholder for issue #138; no JavaScript, TypeScript, or wasm package support is promised by this release | No release verification until a future packaging lane adds build jobs | diff --git a/docs/compatibility-policy.md b/docs/compatibility-policy.md index a471b51..a2e4491 100644 --- a/docs/compatibility-policy.md +++ b/docs/compatibility-policy.md @@ -63,9 +63,9 @@ The `experimental` feature is a default-off research surface. Today it exposes `RankQuantFastscan` is a stable, public (but specialized) type, covered by the normal pre-1.0 compatibility policy above. Its direct `.ovfs` -`RankQuantFastscan::{write,load}` path is supported, but in v0.5.0 `.ovfs` is +`RankQuantFastscan::{write,load}` path is supported, but in v0.6.0 `.ovfs` is not yet part of the primitive persisted-format, `probe_index_metadata()`, or -`ordvec-manifest` v1 contract. Feature-gated `#[doc(hidden)]` exports such as +`ordvec-manifest` v2 contract. Feature-gated `#[doc(hidden)]` exports such as `search_asymmetric_byte_lut` are reachable for internal benchmarks and parity tests only when explicitly enabled, and are not part of the stable default API. @@ -114,7 +114,7 @@ a new ABI version or clear migration notes. The Go wrapper follows the C ABI. Source-breaking Go API changes require the same compatibility classification in release notes. -The `ordvec-manifest` CLI, library API, and v1 JSON schema are treated as +The `ordvec-manifest` CLI, library API, and v2 JSON schema are treated as stable release surfaces. Patch releases should not introduce breaking changes to the CLI arguments, emitted error codes, library report shapes, or JSON schema structure. Minor releases may introduce schema, CLI, or library updates @@ -136,7 +136,7 @@ loaders. Writers no longer emit those magics. `RankQuantFastscan` writes and loads `.ovfs` / `OVFS` directly, but that specialized format is not included in the primitive probe/manifest contract for -v0.5.0. Promoting `.ovfs` into `probe_index_metadata()` and `ordvec-manifest` +v0.6.0. Promoting `.ovfs` into `probe_index_metadata()` and `ordvec-manifest` requires an explicit future compatibility review. Patch releases should keep valid files from the same minor series loadable. diff --git a/examples/quickstart.rs b/examples/quickstart.rs new file mode 100644 index 0000000..79a926b --- /dev/null +++ b/examples/quickstart.rs @@ -0,0 +1,20 @@ +use ordvec::RankQuant; + +fn main() { + // Three tiny embeddings with distinct coordinate orderings. + let documents = [ + 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, // document 0 + 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, // document 1 + 8.0, 1.0, 7.0, 2.0, 6.0, 3.0, 5.0, 4.0, // document 2 + ]; + let query = [8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0]; + + let mut index = RankQuant::new(8, 1); + index.add(&documents); + let results = index.search_asymmetric(&query, 1); + + let top_document = results.indices_for_query(0)[0]; + let top_score = results.scores_for_query(0)[0]; + assert_eq!(top_document, 0); + println!("top document: {top_document} (score {top_score:.3})"); +} diff --git a/fuzz/Cargo.lock b/fuzz/Cargo.lock index ecb390e..f67f073 100644 --- a/fuzz/Cargo.lock +++ b/fuzz/Cargo.lock @@ -4,9 +4,9 @@ version = 4 [[package]] name = "anyhow" -version = "1.0.102" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +checksum = "2a4385e2e34eb35d6b3efe798b9eb88096925d87726c0798709bf56d9ed84af3" [[package]] name = "arbitrary" diff --git a/ordvec-ffi/Cargo.toml b/ordvec-ffi/Cargo.toml index 177a92b..c5bd08d 100644 --- a/ordvec-ffi/Cargo.toml +++ b/ordvec-ffi/Cargo.toml @@ -11,4 +11,4 @@ name = "ordvec_ffi" crate-type = ["rlib", "cdylib", "staticlib"] [dependencies] -ordvec = { path = ".." } +ordvec = { version = "0.6.0", path = ".." } diff --git a/ordvec-manifest-python/Cargo.toml b/ordvec-manifest-python/Cargo.toml index 490ef70..946ae8b 100644 --- a/ordvec-manifest-python/Cargo.toml +++ b/ordvec-manifest-python/Cargo.toml @@ -14,7 +14,7 @@ name = "_ordvec_manifest" crate-type = ["cdylib"] [dependencies] -ordvec_manifest_core = { package = "ordvec-manifest", path = "../ordvec-manifest", default-features = false } +ordvec_manifest_core = { package = "ordvec-manifest", version = "0.6.0", path = "../ordvec-manifest", default-features = false } pyo3 = { version = "0.29.0", features = ["extension-module", "abi3-py310"] } serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" diff --git a/ordvec-manifest-python/README.md b/ordvec-manifest-python/README.md index 9fa3b11..674e8cc 100644 --- a/ordvec-manifest-python/README.md +++ b/ordvec-manifest-python/README.md @@ -2,24 +2,41 @@ Python bindings for the `ordvec-manifest` verifier. -Install from PyPI: +## First verified index ```bash -python -m pip install ordvec-manifest +python -m pip install --upgrade ordvec ordvec-manifest ``` -Import as `ordvec_manifest`. The package exposes the Rust manifest verifier as -dict-returning Python functions: - ```python +import numpy as np +from ordvec import RankQuant import ordvec_manifest -report = ordvec_manifest.verify_manifest("index.manifest.json") -if not report["ok"]: - raise RuntimeError(report["errors"]) +documents = np.array([ + [8, 7, 6, 5, 4, 3, 2, 1], + [1, 2, 3, 4, 5, 6, 7, 8], +], dtype=np.float32) +index = RankQuant(dim=8, bits=1) +index.add(documents) +index.write("quickstart.ovrq") + +ordvec_manifest.create_manifest( + "quickstart.ovrq", + "quickstart.manifest.json", + "quickstart-embedding-v1", + row_id_is_identity=True, +) +report = ordvec_manifest.verify_manifest("quickstart.manifest.json") +print(f"verified: {report['ok']}") +``` + +```text +verified: True ``` -Create manifests with caller-owned sidecars by passing dictionaries with +The package exposes the Rust manifest verifier as dict-returning Python +functions. To bind existing caller-owned sidecars, pass dictionaries with `name`, `path`, and optional `required`: ```python diff --git a/ordvec-manifest-python/pyproject.toml b/ordvec-manifest-python/pyproject.toml index 1e2f099..19f9f14 100644 --- a/ordvec-manifest-python/pyproject.toml +++ b/ordvec-manifest-python/pyproject.toml @@ -42,6 +42,8 @@ classifiers = [ Homepage = "https://github.com/Project-Navi/ordvec" Repository = "https://github.com/Project-Navi/ordvec" Issues = "https://github.com/Project-Navi/ordvec/issues" +Documentation = "https://github.com/Project-Navi/ordvec/blob/v0.6.0/ordvec-manifest-python/README.md" +Changelog = "https://github.com/Project-Navi/ordvec/blob/main/CHANGELOG.md" [tool.maturin] manifest-path = "Cargo.toml" diff --git a/ordvec-manifest/Cargo.toml b/ordvec-manifest/Cargo.toml index 88c6de0..40357d3 100644 --- a/ordvec-manifest/Cargo.toml +++ b/ordvec-manifest/Cargo.toml @@ -30,14 +30,15 @@ chrono = { version = "0.4.44", default-features = false, features = ["clock", "s clap = { version = "4.6.1", features = ["derive"], optional = true } hex = "0.4.3" ordvec = { version = "0.6.0", path = ".." } -rusqlite = { version = "0.40.0", optional = true } +rusqlite = { version = "0.39.0", optional = true } serde = { version = "1.0", features = ["derive"] } -serde_json = "1.0" +serde_json = { version = "1.0", features = ["arbitrary_precision"] } sha2 = "0.11.0" +tempfile = "3.27.0" uuid = { version = "1.23.2", features = ["v4"] } [dev-dependencies] -tempfile = "3.27.0" +serde_json = { version = "1.0", features = ["preserve_order"] } [features] default = ["cli"] diff --git a/ordvec-manifest/README.md b/ordvec-manifest/README.md index 87da107..332f90a 100644 --- a/ordvec-manifest/README.md +++ b/ordvec-manifest/README.md @@ -16,6 +16,44 @@ From a workspace checkout, use the CLI with `cargo run -p ordvec-manifest --`. Library-only consumers that do not need the CLI can depend on the crate with `default-features = false`. +## First verified index + +The shortest end-to-end check uses the Python core package to write a tiny real +ordvec index, then creates and verifies its manifest with the CLI: + +```sh +python -m pip install --upgrade ordvec +cargo install ordvec-manifest + +python - <<'PY' +import numpy as np +from ordvec import RankQuant + +documents = np.array([ + [8, 7, 6, 5, 4, 3, 2, 1], + [1, 2, 3, 4, 5, 6, 7, 8], +], dtype=np.float32) +index = RankQuant(dim=8, bits=1) +index.add(documents) +index.write("quickstart.ovrq") +PY + +ordvec-manifest create \ + --index quickstart.ovrq \ + --row-id-is-identity \ + --embedding-model quickstart-embedding-v1 \ + --out quickstart.manifest.json +ordvec-manifest verify --manifest quickstart.manifest.json +``` + +```text +quickstart.manifest.json +verified +``` + +For an existing bundle with a caller-owned ID sidecar, bind both files before +loading either one: + ```sh ordvec-manifest create \ --index path/to/index.ovrq \ @@ -38,6 +76,13 @@ by `(name, path)`, and embedded paths must be canonical (bundle-relative, forward slashes, no `.`, `..`, or empty segments). Relative paths resolve from the manifest file's directory, absolute paths are rejected by default, and relative paths may not escape the manifest directory unless explicitly allowed. +Nested extension and attestation JSON is recursively canonicalized, independent +of downstream `serde_json` ordering/precision features; non-UTF-8 paths and +non-finite distortion values fail before serialization. Manifest writes use a +synced same-directory temporary file and atomically replace the destination; +new Unix files retain `File::create`'s mode/umask behavior and replacements +preserve portable permission bits (not platform-specific ACLs or ownership). +Symlink and other non-regular destinations are rejected rather than replaced. `create` follows the same policy: by default it emits only paths that should verify with default settings. If an artifact or JSONL row map lives outside the manifest directory, pass `--allow-path-escape` at create time and again at diff --git a/ordvec-manifest/src/lib.rs b/ordvec-manifest/src/lib.rs index 180b7c0..6bdbb6a 100644 --- a/ordvec-manifest/src/lib.rs +++ b/ordvec-manifest/src/lib.rs @@ -13,6 +13,25 @@ //! verified snapshot of the canonical artifact path and related load metadata. //! The `ordvec-manifest` binary exposes the same bounded verification surfaces //! for command-line use. +//! +//! ```no_run +//! use ordvec_manifest::{verify_for_load, VerifyOptions}; +//! +//! # fn main() -> Result<(), Box> { +//! let plan = verify_for_load("quickstart.manifest.json", VerifyOptions::default())?; +//! println!( +//! "verified {} rows at {}", +//! plan.metadata().vector_count, +//! plan.artifact_path().display() +//! ); +//! // Load the index from plan.artifact_path() immediately, while the verified +//! // bytes remain under the caller's control. +//! # Ok(()) +//! # } +//! ``` +//! +//! See the crate README for a copy-and-run index creation and CLI verification +//! path. use chrono::{DateTime, SecondsFormat, Utc}; use ordvec::{ @@ -24,7 +43,7 @@ use sha2::{Digest, Sha256}; use std::collections::{BTreeMap, HashSet}; use std::fmt; use std::fs::{self, File}; -use std::io::{self, BufRead, BufReader, Read}; +use std::io::{self, BufRead, BufReader, Read, Write}; use std::path::{Component, Path, PathBuf}; use uuid::Uuid; @@ -199,7 +218,7 @@ fn read_bounded_file( ) })?; let mut bytes = Vec::new(); - let mut limited = file.by_ref().take(read_limit); + let mut limited = Read::by_ref(&mut file).take(read_limit); limited.read_to_end(&mut bytes)?; if bytes.len() > max_len { return Err(ManifestError::limit_exceeded( @@ -4435,20 +4454,350 @@ fn create_auxiliary_artifacts( /// serializer or its settings changes every manifest's identity and is a /// schema-version event, not a cosmetic change. /// -/// The canonical bytes depend on `serde_json`'s default feature set. Nested -/// [`serde_json::Value`] maps carried in `extensions` / `attestations` are -/// key-sorted only while `serde_json` is built without `preserve_order`; a -/// consumer whose dependency graph enables `serde_json/preserve_order` (or -/// `arbitrary_precision`) via feature unification would serialize those maps -/// in insertion order, changing the content address. Do not enable those -/// features in a build that produces content-addressed manifests. See -/// . +/// Nested [`serde_json::Value`] maps carried in `extensions` / `attestations` +/// are recursively key-sorted and their numbers are normalized before +/// serialization. The stored bytes therefore do not depend on downstream +/// feature unification of `serde_json/preserve_order` or +/// `serde_json/arbitrary_precision`. +/// +/// The destination is replaced atomically from a temporary file in the same +/// directory after the bytes and portable permission bits are synced. New +/// files use the same `0o666 & !umask` Unix mode as [`File::create`]; replacing +/// an existing file preserves its [`fs::Permissions`]. Platform-specific ACLs, +/// ownership, and extended attributes are outside this portable API contract. +/// Existing symlinks and other non-regular destinations observed before the +/// replacement are rejected. The destination is checked again immediately +/// before replacement and, on Unix, an existing file's device/inode identity +/// must still match. Callers must still serialize concurrent writers: portable +/// rename is atomic replacement, not a compare-and-swap primitive. A failed +/// validation or serialization never truncates an existing manifest. pub fn write_manifest_file( manifest: &IndexManifest, path: impl AsRef, ) -> Result<(), ManifestError> { - let file = File::create(path)?; - serde_json::to_writer_pretty(file, manifest)?; + let canonical = canonical_manifest_for_write(manifest)?; + let path = path.as_ref(); + let parent = path + .parent() + .filter(|parent| !parent.as_os_str().is_empty()) + .unwrap_or_else(|| Path::new(".")); + let destination = snapshot_manifest_destination(path)?; + let inherited_permissions = destination.permissions.clone(); + + // Open the directory before writing or replacing anything. On Unix this + // makes a predictable directory-handle permission failure happen before + // the destination changes; the same handle is synced after the rename. + #[cfg(unix)] + let parent_directory = File::open(parent)?; + + let mut builder = tempfile::Builder::new(); + builder.prefix(".ordvec-manifest-").suffix(".tmp"); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + + builder.permissions( + inherited_permissions + .clone() + .unwrap_or_else(|| fs::Permissions::from_mode(0o666)), + ); + } + #[cfg(not(unix))] + if let Some(permissions) = inherited_permissions.as_ref() { + builder.permissions(permissions.clone()); + } + let mut temporary = builder.tempfile_in(parent)?; + serde_json::to_writer_pretty(temporary.as_file_mut(), &canonical)?; + temporary.as_file_mut().flush()?; + if let Some(permissions) = inherited_permissions { + temporary.as_file().set_permissions(permissions)?; + } + temporary.as_file().sync_all()?; + persist_manifest_temporary(temporary, path, &destination)?; + #[cfg(unix)] + parent_directory.sync_all()?; + Ok(()) +} + +struct ManifestDestinationSnapshot { + permissions: Option, + #[cfg(unix)] + identity: Option<(u64, u64)>, +} + +fn snapshot_manifest_destination( + path: &Path, +) -> Result { + match fs::symlink_metadata(path) { + Ok(metadata) if metadata.file_type().is_file() => { + #[cfg(unix)] + let identity = { + use std::os::unix::fs::MetadataExt; + Some((metadata.dev(), metadata.ino())) + }; + Ok(ManifestDestinationSnapshot { + permissions: Some(metadata.permissions()), + #[cfg(unix)] + identity, + }) + } + Ok(_) => Err(ManifestError::invalid(format!( + "manifest destination {} exists and is not a regular file", + path.display() + ))), + Err(error) if error.kind() == io::ErrorKind::NotFound => Ok(ManifestDestinationSnapshot { + permissions: None, + #[cfg(unix)] + identity: None, + }), + Err(error) => Err(ManifestError::Io(error)), + } +} + +fn persist_manifest_temporary( + temporary: tempfile::NamedTempFile, + path: &Path, + initial: &ManifestDestinationSnapshot, +) -> Result<(), ManifestError> { + let current = snapshot_manifest_destination(path)?; + match ( + initial.permissions.is_some(), + current.permissions.is_some(), + ) { + (false, true) => { + return Err(ManifestError::invalid(format!( + "manifest destination {} appeared during the write; coordinate concurrent writers and retry", + path.display() + ))) + } + (true, false) => { + return Err(ManifestError::invalid(format!( + "manifest destination {} disappeared during the write; coordinate concurrent writers and retry", + path.display() + ))) + } + _ => {} + } + #[cfg(unix)] + if initial.identity != current.identity { + return Err(ManifestError::invalid(format!( + "manifest destination {} was replaced during the write; coordinate concurrent writers and retry", + path.display() + ))); + } + temporary + .persist(path) + .map_err(|error| ManifestError::Io(error.error))?; + Ok(()) +} + +fn canonical_manifest_for_write(manifest: &IndexManifest) -> Result { + validate_manifest_numbers_for_write(manifest)?; + let mut canonical = manifest.clone(); + normalize_typed_manifest_signed_zero(&mut canonical); + for (index, attestation) in canonical.attestations.iter_mut().enumerate() { + canonicalize_json_value(attestation, &format!("attestations[{index}]"))?; + } + for (name, extension) in &mut canonical.extensions { + canonicalize_json_value(extension, &format!("extensions[{name:?}]"))?; + } + Ok(canonical) +} + +fn normalize_typed_manifest_signed_zero(manifest: &mut IndexManifest) { + let Some(profile) = manifest.encoder_distortion.as_mut() else { + return; + }; + + // Keep this list aligned with every typed f64 field reachable from + // IndexManifest. Validation runs on the caller-owned manifest first; only + // this cloned canonical representation is changed. + normalize_optional_signed_zero(&mut profile.bounds.declared_lower_bound); + normalize_optional_signed_zero(&mut profile.bounds.declared_upper_bound); + normalize_optional_signed_zero(&mut profile.bounds.estimated_distortion); + normalize_optional_signed_zero(&mut profile.bounds.violation_rate); + normalize_optional_signed_zero(&mut profile.bounds.max_observed_violation); + normalize_optional_signed_zero(&mut profile.bounds.quantile_observed_violation); + normalize_optional_signed_zero(&mut profile.scope.confidence); + normalize_optional_signed_zero(&mut profile.scope.coverage); +} + +fn normalize_optional_signed_zero(value: &mut Option) { + if value.is_some_and(|value| value == 0.0) { + *value = Some(0.0); + } +} + +fn canonicalize_json_value( + value: &mut serde_json::Value, + context: &str, +) -> Result<(), ManifestError> { + match value { + serde_json::Value::Array(values) => { + for (index, nested) in values.iter_mut().enumerate() { + canonicalize_json_value(nested, &format!("{context}[{index}]"))?; + } + } + serde_json::Value::Object(values) => { + for (key, nested) in values.iter_mut() { + canonicalize_json_value(nested, &format!("{context}.{key}"))?; + } + values.sort_keys(); + } + serde_json::Value::Number(number) => { + let original = number.to_string(); + let identity = decimal_identity(&original).ok_or_else(|| { + ManifestError::invalid(format!( + "{context} contains a JSON number that cannot be represented canonically" + )) + })?; + let normalized = if let Some(integer) = canonical_integer_from_identity(&identity) { + integer + } else { + let value = number.as_f64().ok_or_else(|| { + ManifestError::invalid(format!( + "{context} contains a JSON number that cannot be represented canonically" + )) + })?; + if !value.is_finite() { + return Err(ManifestError::invalid(format!( + "{context} contains a non-finite JSON number" + ))); + } + let normalized = serde_json::Number::from_f64(value).ok_or_else(|| { + ManifestError::invalid(format!( + "{context} contains a JSON number that cannot be represented canonically" + )) + })?; + if decimal_identity(&normalized.to_string()).as_ref() != Some(&identity) { + return Err(ManifestError::invalid(format!( + "{context} contains a JSON number outside the exact canonical i64/u64/f64 domain" + ))); + } + normalized + }; + *number = normalized; + } + serde_json::Value::Null | serde_json::Value::Bool(_) | serde_json::Value::String(_) => {} + } + Ok(()) +} + +fn canonical_integer_from_identity(identity: &(bool, String, i64)) -> Option { + let (negative, digits, exponent) = identity; + if *exponent < 0 { + return None; + } + + // Accumulate rather than materializing exponent zeroes. Any value outside + // u64 overflows after at most twenty decimal digits, even for a hostile + // arbitrary-precision exponent. + let mut magnitude = 0_u64; + for digit in digits.bytes() { + magnitude = magnitude + .checked_mul(10)? + .checked_add(u64::from(digit - b'0'))?; + } + let mut remaining_exponent = *exponent; + while remaining_exponent > 0 { + magnitude = magnitude.checked_mul(10)?; + remaining_exponent -= 1; + } + + if *negative { + let i64_min_magnitude = (i64::MAX as u64) + 1; + if magnitude == i64_min_magnitude { + Some(serde_json::Number::from(i64::MIN)) + } else { + let signed = i64::try_from(magnitude).ok()?; + Some(serde_json::Number::from(-signed)) + } + } else { + Some(serde_json::Number::from(magnitude)) + } +} + +fn decimal_identity(value: &str) -> Option<(bool, String, i64)> { + let (negative, unsigned) = if let Some(unsigned) = value.strip_prefix('-') { + (true, unsigned) + } else { + (false, value) + }; + let mut exponent_split = unsigned.split(['e', 'E']); + let mantissa = exponent_split.next()?; + let explicit_exponent = exponent_split + .next() + .map(str::parse::) + .transpose() + .ok()? + .unwrap_or(0); + if exponent_split.next().is_some() { + return None; + } + let (integer, fraction) = mantissa.split_once('.').unwrap_or((mantissa, "")); + if integer.is_empty() + || !integer.bytes().all(|byte| byte.is_ascii_digit()) + || !fraction.bytes().all(|byte| byte.is_ascii_digit()) + { + return None; + } + let fraction_len = i64::try_from(fraction.len()).ok()?; + let mut exponent = explicit_exponent.checked_sub(fraction_len)?; + let mut digits = format!("{integer}{fraction}"); + let first_nonzero = digits.bytes().position(|byte| byte != b'0'); + let Some(first_nonzero) = first_nonzero else { + return Some((false, "0".to_string(), 0)); + }; + digits.drain(..first_nonzero); + while digits.ends_with('0') { + digits.pop(); + exponent = exponent.checked_add(1)?; + } + Some((negative, digits, exponent)) +} + +fn validate_manifest_numbers_for_write(manifest: &IndexManifest) -> Result<(), ManifestError> { + let Some(profile) = manifest.encoder_distortion.as_ref() else { + return Ok(()); + }; + let values = [ + ( + "encoder_distortion.bounds.declared_lower_bound", + profile.bounds.declared_lower_bound, + ), + ( + "encoder_distortion.bounds.declared_upper_bound", + profile.bounds.declared_upper_bound, + ), + ( + "encoder_distortion.bounds.estimated_distortion", + profile.bounds.estimated_distortion, + ), + ( + "encoder_distortion.bounds.violation_rate", + profile.bounds.violation_rate, + ), + ( + "encoder_distortion.bounds.max_observed_violation", + profile.bounds.max_observed_violation, + ), + ( + "encoder_distortion.bounds.quantile_observed_violation", + profile.bounds.quantile_observed_violation, + ), + ( + "encoder_distortion.scope.confidence", + profile.scope.confidence, + ), + ("encoder_distortion.scope.coverage", profile.scope.coverage), + ]; + for (name, value) in values { + if value.is_some_and(|value| !value.is_finite()) { + return Err(ManifestError::invalid(format!( + "{name} must be finite before the manifest can be written" + ))); + } + } Ok(()) } @@ -4734,7 +5083,7 @@ fn manifest_path_for_create( if relative.as_os_str().is_empty() { ".".to_string() } else { - path_to_manifest_string(relative) + path_to_manifest_string(relative)? } } else if !options.allow_path_escape { return Err(ManifestError::invalid(format!( @@ -4743,9 +5092,9 @@ fn manifest_path_for_create( canonical_base.display() ))); } else if let Some(relative) = relative_path_between(&canonical_base, &canonical_path) { - path_to_manifest_string(&relative) + path_to_manifest_string(&relative)? } else if options.allow_absolute_paths { - path_to_manifest_string(&canonical_path) + path_to_manifest_string(&canonical_path)? } else { return Err(ManifestError::invalid(format!( "{context} path {} cannot be expressed relative to manifest directory {}; use --allow-absolute-paths with --allow-path-escape", @@ -4832,9 +5181,14 @@ fn is_manifest_path_absolute(path: &str) -> bool { && (bytes[2] == b'/' || bytes[2] == b'\\') } -fn path_to_manifest_string(path: &Path) -> String { +fn path_to_manifest_string(path: &Path) -> Result { if path.is_absolute() { - let display = path.display().to_string(); + let display = path.to_str().ok_or_else(|| { + ManifestError::invalid(format!( + "path {:?} is not valid UTF-8 and cannot be embedded in a manifest", + path.as_os_str() + )) + })?; // fs::canonicalize returns verbatim paths on Windows; strip the // verbatim prefix so the embedded string round-trips through // PathBuf::from at verification time. @@ -4843,23 +5197,32 @@ fn path_to_manifest_string(path: &Path) -> String { } else if let Some(rest) = display.strip_prefix(r"\\?\") { rest.to_string() } else { - display + display.to_string() }; - return display.replace('\\', "/"); - } - let parts = path - .components() - .filter_map(|component| match component { - Component::Normal(part) => Some(part.to_string_lossy().into_owned()), - Component::CurDir => Some(".".to_string()), - Component::ParentDir => Some("..".to_string()), - Component::Prefix(_) | Component::RootDir => None, - }) - .collect::>(); + return Ok(display.replace('\\', "/")); + } + let mut parts = Vec::new(); + for component in path.components() { + match component { + Component::Normal(part) => parts.push( + part.to_str() + .ok_or_else(|| { + ManifestError::invalid(format!( + "path {:?} is not valid UTF-8 and cannot be embedded in a manifest", + path.as_os_str() + )) + })? + .to_string(), + ), + Component::CurDir => parts.push(".".to_string()), + Component::ParentDir => parts.push("..".to_string()), + Component::Prefix(_) | Component::RootDir => {} + } + } if parts.is_empty() { - ".".to_string() + Ok(".".to_string()) } else { - parts.join("/") + Ok(parts.join("/")) } } @@ -4944,6 +5307,60 @@ mod tests { assert!(!is_manifest_path_absolute("\\")); } + #[cfg(unix)] + #[test] + fn pre_persist_validation_rejects_a_destination_swapped_to_a_symlink() { + use std::io::Write as _; + use std::os::unix::fs::symlink; + + let directory = tempfile::tempdir().unwrap(); + let destination = directory.path().join("manifest.json"); + let target = directory.path().join("target.json"); + fs::write(&destination, b"original").unwrap(); + fs::write(&target, b"target").unwrap(); + let initial = snapshot_manifest_destination(&destination).unwrap(); + let mut temporary = tempfile::NamedTempFile::new_in(directory.path()).unwrap(); + temporary.write_all(b"replacement").unwrap(); + + fs::remove_file(&destination).unwrap(); + symlink(&target, &destination).unwrap(); + let error = persist_manifest_temporary(temporary, &destination, &initial).unwrap_err(); + + assert!(error.to_string().contains("not a regular file"), "{error}"); + assert!(fs::symlink_metadata(&destination) + .unwrap() + .file_type() + .is_symlink()); + assert_eq!(fs::read(target).unwrap(), b"target"); + } + + #[cfg(unix)] + #[test] + fn pre_persist_validation_rejects_a_replaced_regular_inode() { + use std::io::Write as _; + + let directory = tempfile::tempdir().unwrap(); + let destination = directory.path().join("manifest.json"); + let held_original = directory.path().join("original-manifest-inode"); + fs::write(&destination, b"original").unwrap(); + // Keep the original inode live so the filesystem cannot recycle its + // identity for the concurrent writer after `destination` is unlinked. + fs::hard_link(&destination, &held_original).unwrap(); + let initial = snapshot_manifest_destination(&destination).unwrap(); + let mut temporary = tempfile::NamedTempFile::new_in(directory.path()).unwrap(); + temporary.write_all(b"replacement").unwrap(); + + fs::remove_file(&destination).unwrap(); + fs::write(&destination, b"concurrent writer").unwrap(); + let error = persist_manifest_temporary(temporary, &destination, &initial).unwrap_err(); + + assert!( + error.to_string().contains("was replaced during the write"), + "{error}" + ); + assert_eq!(fs::read(destination).unwrap(), b"concurrent writer"); + } + #[test] fn manifest_kind_conversion_uses_format_registry_coverage() { for spec in FORMATS { diff --git a/ordvec-manifest/src/main.rs b/ordvec-manifest/src/main.rs index f4188e4..8b73fd7 100644 --- a/ordvec-manifest/src/main.rs +++ b/ordvec-manifest/src/main.rs @@ -14,66 +14,93 @@ const EXIT_USAGE_OR_CONFIG: i32 = 2; #[derive(Parser)] #[command(name = "ordvec-manifest")] -#[command(about = "Verify ordvec index manifests", version)] +#[command(about = "Create and verify ordvec index manifests", version)] +#[command(after_help = "Run `ordvec-manifest --help` for command options.")] struct Cli { + /// Manifest operation to run. #[command(subcommand)] command: Commands, } #[derive(Subcommand)] enum Commands { + /// Compute a file's SHA-256 digest and byte length. Hash { + /// File to hash. path: PathBuf, + /// Emit a machine-readable JSON object. #[arg(long)] json: bool, }, + /// Print a manifest summary without verifying its artifacts. Inspect { + /// Manifest JSON to inspect. manifest: PathBuf, #[command(flatten)] limits: LimitArgs, + /// Emit the parsed manifest as JSON. #[arg(long)] json: bool, }, + /// Verify a manifest and every declared artifact. Verify { + /// Manifest JSON to verify. #[arg(long)] manifest: PathBuf, + /// Override the primary index path declared by the manifest. #[arg(long)] index: Option, + /// Permit absolute artifact paths (disabled by default). #[arg(long)] allow_absolute_paths: bool, + /// Permit relative paths that escape the manifest directory. #[arg(long)] allow_path_escape: bool, + /// Permit duplicate db_id values in a JSONL row map. #[arg(long)] allow_duplicate_db_ids: bool, #[command(flatten)] limits: LimitArgs, + /// Emit the verification report as JSON. #[arg(long)] json: bool, }, + /// Create a deterministic manifest for an existing ordvec index. Create { + /// Existing ordvec index to bind. #[arg(long)] index: PathBuf, + /// JSONL row-identity map to bind. #[arg(long)] row_map: Option, + /// Declare row IDs as the zero-based index row numbers. #[arg(long)] row_id_is_identity: bool, + /// Bind a required caller-owned sidecar as NAME=PATH (repeatable). #[arg(long = "aux", value_name = "NAME=PATH", value_parser = parse_auxiliary_artifact_arg)] auxiliary_artifacts: Vec, + /// Bind an optional caller-owned sidecar as NAME=PATH (repeatable). #[arg(long = "optional-aux", value_name = "NAME=PATH", value_parser = parse_auxiliary_artifact_arg)] optional_auxiliary_artifacts: Vec, + /// Identifier of the embedding model that produced the index. #[arg(long)] embedding_model: String, + /// Destination manifest JSON path. #[arg(long)] out: PathBuf, + /// Permit absolute artifact paths in the emitted manifest. #[arg(long)] allow_absolute_paths: bool, + /// Permit emitted relative paths to escape the manifest directory. #[arg(long)] allow_path_escape: bool, #[command(flatten)] limits: LimitArgs, }, #[cfg(feature = "sqlite")] + /// Verify or activate manifests with a SQLite report cache. Sqlite { + /// SQLite-backed operation to run. #[command(subcommand)] command: SqliteCommands, }, @@ -104,7 +131,7 @@ fn parse_auxiliary_artifact_arg(value: &str) -> Result panic!("expected verify command"), } } + + #[test] + fn help_describes_commands_and_safety_relevant_options() { + let mut root = Cli::command(); + let root_help = root.render_long_help().to_string(); + for expected in [ + "Compute a file's SHA-256 digest", + "Print a manifest summary", + "Verify a manifest and every declared artifact", + "Create a deterministic manifest", + ] { + assert!( + root_help.contains(expected), + "missing help text: {expected}" + ); + } + + let mut verify = Cli::command() + .find_subcommand("verify") + .expect("verify subcommand") + .clone(); + let verify_help = verify.render_long_help().to_string(); + for expected in [ + "Manifest JSON to verify", + "Permit absolute artifact paths", + "Permit relative paths that escape", + "Emit the verification report as JSON", + ] { + assert!( + verify_help.contains(expected), + "missing verify help text: {expected}" + ); + } + } } #[cfg(feature = "sqlite")] #[derive(Subcommand)] enum SqliteCommands { + /// Verify a manifest, optionally reusing a valid cached report. Verify { + /// SQLite cache database path. #[arg(long)] db: PathBuf, + /// Manifest JSON to verify. #[arg(long)] manifest: PathBuf, + /// Reuse a matching cached report when available. #[arg(long)] use_cache: bool, + /// Override the primary index path declared by the manifest. #[arg(long)] index: Option, + /// Permit absolute artifact paths (disabled by default). #[arg(long)] allow_absolute_paths: bool, + /// Permit relative paths that escape the manifest directory. #[arg(long)] allow_path_escape: bool, + /// Permit duplicate db_id values in a JSONL row map. #[arg(long)] allow_duplicate_db_ids: bool, #[command(flatten)] limits: LimitArgs, + /// Emit the verification report as JSON. #[arg(long)] json: bool, }, + /// Verify and mark a manifest active in the SQLite cache. Activate { + /// SQLite cache database path. #[arg(long)] db: PathBuf, + /// Manifest JSON to verify and activate. #[arg(long)] manifest: PathBuf, + /// Activate even when verification reports errors. #[arg(long)] force: bool, + /// Override the primary index path declared by the manifest. #[arg(long)] index: Option, + /// Permit absolute artifact paths (disabled by default). #[arg(long)] allow_absolute_paths: bool, + /// Permit relative paths that escape the manifest directory. #[arg(long)] allow_path_escape: bool, + /// Permit duplicate db_id values in a JSONL row map. #[arg(long)] allow_duplicate_db_ids: bool, #[command(flatten)] limits: LimitArgs, + /// Emit the verification report as JSON. #[arg(long)] json: bool, }, @@ -198,26 +277,37 @@ enum SqliteCommands { #[derive(Args, Clone, Debug, Default)] struct LimitArgs { + /// Maximum manifest JSON bytes to read. #[arg(long)] max_manifest_bytes: Option, + /// Maximum bytes in one JSONL row-map line. #[arg(long)] max_row_map_line_bytes: Option, + /// Maximum JSONL row-map rows to inspect. #[arg(long)] max_row_map_rows: Option, + /// Maximum bytes retained while checking duplicate db_id values. #[arg(long)] max_row_map_tracked_id_bytes: Option, + /// Maximum number of declared auxiliary artifacts. #[arg(long)] max_auxiliary_artifacts: Option, + /// Maximum bytes permitted for each auxiliary artifact. #[arg(long)] max_auxiliary_artifact_bytes: Option, + /// Maximum bytes permitted for the primary index artifact. #[arg(long)] max_index_artifact_bytes: Option, + /// Maximum bytes permitted for a calibration profile. #[arg(long)] max_calibration_profile_bytes: Option, + /// Maximum bytes permitted for an encoder-distortion profile. #[arg(long)] max_encoder_distortion_profile_bytes: Option, + /// Maximum detail issues retained in a verification report. #[arg(long)] max_report_issues: Option, + /// Maximum bytes accepted for a cached SQLite report. #[arg(long)] max_cached_report_bytes: Option, } diff --git a/ordvec-manifest/tests/deterministic.rs b/ordvec-manifest/tests/deterministic.rs index a57bfc3..950e18c 100644 --- a/ordvec-manifest/tests/deterministic.rs +++ b/ordvec-manifest/tests/deterministic.rs @@ -2,9 +2,12 @@ use ordvec::RankQuant; use ordvec_manifest::{ create_manifest_for_index, create_manifest_for_index_with_options, load_manifest_file, sha256_file, verify_manifest_with_base, write_manifest_file, CreateAuxiliaryArtifact, - CreateManifestOptions, CreateRowIdentity, VerifyOptions, SCHEMA_VERSION, + CreateManifestOptions, CreateRowIdentity, DistortionBounds, DistortionEvidence, + DistortionEvidenceKind, DistortionScope, EncoderDistortionProfileRef, EncoderSpec, + IndexManifest, ManifestError, MetricSpec, VerifyOptions, ENCODER_DISTORTION_SCHEMA_VERSION, + SCHEMA_VERSION, }; -use serde_json::json; +use serde_json::{json, Map, Value}; use std::fs; use std::path::{Path, PathBuf}; @@ -51,6 +54,80 @@ fn build_manifest_bytes(dir: &Path, aux_names: &[&str]) -> Vec { fs::read(&manifest_path).unwrap() } +fn manifest_bytes_with_extension_number( + dir: &Path, + number: &str, +) -> Result, ManifestError> { + let index = write_index(dir); + let manifest_path = dir.join("manifest.json"); + let mut manifest = create_manifest_for_index( + &index, + CreateRowIdentity::RowIdIdentity, + "test-embedding", + &manifest_path, + )?; + manifest.extensions.insert( + "canonical_number".to_string(), + serde_json::from_str(number).map_err(ManifestError::Json)?, + ); + write_manifest_file(&manifest, &manifest_path)?; + fs::read(manifest_path).map_err(ManifestError::Io) +} + +fn distortion_profile_with_zero( + manifest: &IndexManifest, + zero: f64, +) -> EncoderDistortionProfileRef { + EncoderDistortionProfileRef { + schema_version: ENCODER_DISTORTION_SCHEMA_VERSION.to_string(), + profile_id: "urn:uuid:a8c39375-ae65-4924-92f5-8088adfab9b5".to_string(), + created_at: None, + encoder: EncoderSpec { + model: manifest.embedding.model.clone(), + dim: manifest.embedding.dim, + model_revision: manifest.embedding.model_revision.clone(), + normalization: manifest.embedding.normalization.clone(), + }, + tokenizer_revision: manifest.embedding.tokenizer_revision.clone(), + pooling: manifest.embedding.pooling.clone(), + source_metric: MetricSpec { + name: "qrel_distance".to_string(), + version: Some("v1".to_string()), + digest: None, + }, + embedding_metric: MetricSpec { + name: "cosine".to_string(), + version: None, + digest: None, + }, + bounds: DistortionBounds { + declared_lower_bound: Some(0.5), + declared_upper_bound: Some(2.0), + estimated_distortion: Some(4.0), + violation_rate: Some(zero), + max_observed_violation: Some(zero), + quantile_observed_violation: Some(zero), + }, + scope: DistortionScope { + corpus_digest: None, + query_set_digest: None, + pair_sample_digest: None, + domain: Some("test-corpus".to_string()), + sample_size: Some(1), + confidence: Some(zero), + coverage: Some(zero), + estimator_version: Some("test-estimator/1".to_string()), + }, + evidence: DistortionEvidence { + kind: DistortionEvidenceKind::CallerAsserted, + estimator_id: None, + estimator_hash: None, + }, + profile: None, + calibration_profile_id: None, + } +} + #[test] fn identical_inputs_produce_byte_identical_manifests() { let temp_a = tempfile::tempdir().unwrap(); @@ -132,6 +209,357 @@ fn auxiliary_declaration_order_does_not_change_manifest_bytes() { assert_eq!(bytes_a, bytes_b); } +#[test] +fn nested_json_order_and_number_spelling_do_not_change_manifest_bytes() { + let temp_a = tempfile::tempdir().unwrap(); + let temp_b = tempfile::tempdir().unwrap(); + let index_a = write_index(temp_a.path()); + let index_b = write_index(temp_b.path()); + let path_a = temp_a.path().join("manifest.json"); + let path_b = temp_b.path().join("manifest.json"); + let mut manifest_a = create_manifest_for_index( + &index_a, + CreateRowIdentity::RowIdIdentity, + "test-embedding", + &path_a, + ) + .unwrap(); + let mut manifest_b = create_manifest_for_index( + &index_b, + CreateRowIdentity::RowIdIdentity, + "test-embedding", + &path_b, + ) + .unwrap(); + + let mut descending = Map::new(); + descending.insert("z".to_string(), serde_json::from_str("1e0").unwrap()); + descending.insert("a".to_string(), json!({"y": true, "b": false})); + let mut ascending = Map::new(); + ascending.insert("a".to_string(), json!({"b": false, "y": true})); + ascending.insert("z".to_string(), serde_json::from_str("1.0").unwrap()); + manifest_a + .extensions + .insert("nested".to_string(), Value::Object(descending)); + manifest_b + .extensions + .insert("nested".to_string(), Value::Object(ascending)); + + write_manifest_file(&manifest_a, &path_a).unwrap(); + write_manifest_file(&manifest_b, &path_b).unwrap(); + assert_eq!(fs::read(path_a).unwrap(), fs::read(path_b).unwrap()); +} + +#[test] +fn equivalent_json_number_spellings_have_one_canonical_manifest_representation() { + let cases: &[(&str, &[&str], &str)] = &[ + ("zero", &["0", "-0", "0.0", "-0.0", "0e9"], "0"), + ("one", &["1", "1.0", "1e0", "10e-1"], "1"), + ("negative one", &["-1", "-1.0", "-1e0"], "-1"), + ("thousand", &["1000", "1000.0", "1e3"], "1000"), + ("fraction", &["0.5", "0.50", "5e-1"], "0.5"), + ( + "two to 53 minus one", + &["9007199254740991", "9007199254740991.0"], + "9007199254740991", + ), + ( + "two to 53", + &["9007199254740992", "9007199254740992.0"], + "9007199254740992", + ), + ( + "two to 53 plus one", + &[ + "9007199254740993", + "9007199254740993.0", + "9007199254740993e0", + ], + "9007199254740993", + ), + ( + "i64 min", + &[ + "-9223372036854775808", + "-9223372036854775808.0", + "-9.223372036854775808e18", + ], + "-9223372036854775808", + ), + ( + "u64 max", + &[ + "18446744073709551615", + "18446744073709551615.0", + "1.8446744073709551615e19", + ], + "18446744073709551615", + ), + ( + "out of integer range canonical f64", + &["100000000000000000000", "100000000000000000000.0", "1e20"], + "1e+20", + ), + ]; + + for (case, spellings, expected_number) in cases { + let mut canonical_bytes: Option> = None; + for spelling in *spellings { + let temp = tempfile::tempdir().unwrap(); + let bytes = manifest_bytes_with_extension_number(temp.path(), spelling).unwrap(); + let stored: Value = serde_json::from_slice(&bytes).unwrap(); + assert_eq!( + stored["extensions"]["canonical_number"].to_string(), + *expected_number, + "{case}: {spelling}" + ); + if let Some(expected_bytes) = &canonical_bytes { + assert_eq!(expected_bytes, &bytes, "{case}: {spelling}"); + } else { + canonical_bytes = Some(bytes); + } + } + } +} + +#[test] +fn typed_signed_zero_fields_have_one_canonical_manifest_representation() { + let temp_a = tempfile::tempdir().unwrap(); + let temp_b = tempfile::tempdir().unwrap(); + let index_a = write_index(temp_a.path()); + let index_b = write_index(temp_b.path()); + let path_a = temp_a.path().join("manifest.json"); + let path_b = temp_b.path().join("manifest.json"); + let mut manifest_a = create_manifest_for_index( + &index_a, + CreateRowIdentity::RowIdIdentity, + "test-embedding", + &path_a, + ) + .unwrap(); + let mut manifest_b = create_manifest_for_index( + &index_b, + CreateRowIdentity::RowIdIdentity, + "test-embedding", + &path_b, + ) + .unwrap(); + manifest_a.encoder_distortion = Some(distortion_profile_with_zero(&manifest_a, -0.0)); + manifest_b.encoder_distortion = Some(distortion_profile_with_zero(&manifest_b, 0.0)); + + write_manifest_file(&manifest_a, &path_a).unwrap(); + write_manifest_file(&manifest_b, &path_b).unwrap(); + let bytes_a = fs::read(path_a).unwrap(); + let bytes_b = fs::read(path_b).unwrap(); + assert_eq!(bytes_a, bytes_b); + + let caller_profile = manifest_a.encoder_distortion.as_ref().unwrap(); + for value in [ + caller_profile.bounds.violation_rate.unwrap(), + caller_profile.bounds.max_observed_violation.unwrap(), + caller_profile.bounds.quantile_observed_violation.unwrap(), + caller_profile.scope.confidence.unwrap(), + caller_profile.scope.coverage.unwrap(), + ] { + assert!( + value.is_sign_negative(), + "writer mutated its caller-owned manifest" + ); + } + + let stored: Value = serde_json::from_slice(&bytes_a).unwrap(); + for pointer in [ + "/encoder_distortion/bounds/violation_rate", + "/encoder_distortion/bounds/max_observed_violation", + "/encoder_distortion/bounds/quantile_observed_violation", + "/encoder_distortion/scope/confidence", + "/encoder_distortion/scope/coverage", + ] { + let value = stored.pointer(pointer).unwrap().as_f64().unwrap(); + assert_eq!(value, 0.0, "{pointer}"); + assert!(value.is_sign_positive(), "{pointer}"); + } +} + +#[test] +fn adjacent_large_json_integers_keep_distinct_manifest_addresses() { + for spellings in [ + ["9007199254740991.0", "9007199254740992.0"], + ["9007199254740992.0", "9007199254740993.0"], + ["18446744073709551614.0", "18446744073709551615.0"], + ] { + let temp_a = tempfile::tempdir().unwrap(); + let temp_b = tempfile::tempdir().unwrap(); + let bytes_a = manifest_bytes_with_extension_number(temp_a.path(), spellings[0]).unwrap(); + let bytes_b = manifest_bytes_with_extension_number(temp_b.path(), spellings[1]).unwrap(); + assert_ne!(bytes_a, bytes_b, "{spellings:?}"); + } +} + +#[test] +fn arbitrary_precision_numbers_that_would_alias_are_rejected() { + let temp = tempfile::tempdir().unwrap(); + let index = write_index(temp.path()); + for (case, number, expected_error) in [ + ( + "two_to_64_integer", + "18446744073709551616", + "outside the exact canonical", + ), + ( + "two_to_64_decimal", + "18446744073709551616.0", + "outside the exact canonical", + ), + ( + "two_to_64_exponent", + "1.8446744073709551616e19", + "outside the exact canonical", + ), + ( + "two_to_64_plus_one", + "18446744073709551617", + "outside the exact canonical", + ), + ( + "lossy_decimal", + "0.100000000000000005", + "outside the exact canonical", + ), + ("f64_overflow", "1e400", "cannot be represented canonically"), + ("f64_underflow", "1e-400", "outside the exact canonical"), + ] { + let manifest_path = temp.path().join(format!("{case}.json")); + let mut manifest = create_manifest_for_index( + &index, + CreateRowIdentity::RowIdIdentity, + "test-embedding", + &manifest_path, + ) + .unwrap(); + manifest + .extensions + .insert("precise".to_string(), serde_json::from_str(number).unwrap()); + let err = write_manifest_file(&manifest, &manifest_path).unwrap_err(); + assert!(err.to_string().contains(expected_error), "{number}: {err}"); + assert!(!manifest_path.exists()); + } +} + +#[test] +fn manifest_write_atomically_replaces_the_destination_inode() { + let temp = tempfile::tempdir().unwrap(); + let index = write_index(temp.path()); + let manifest_path = temp.path().join("manifest.json"); + let old_inode_link = temp.path().join("old-manifest-inode"); + let manifest = create_manifest_for_index( + &index, + CreateRowIdentity::RowIdIdentity, + "test-embedding", + &manifest_path, + ) + .unwrap(); + + fs::write(&manifest_path, b"old manifest bytes").unwrap(); + fs::hard_link(&manifest_path, &old_inode_link).unwrap(); + write_manifest_file(&manifest, &manifest_path).unwrap(); + + assert_eq!(fs::read(old_inode_link).unwrap(), b"old manifest bytes"); + assert_eq!( + load_manifest_file(manifest_path) + .unwrap() + .manifest + .schema_version, + SCHEMA_VERSION + ); +} + +#[cfg(unix)] +#[test] +fn manifest_write_matches_create_mode_and_preserves_existing_mode() { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir().unwrap(); + let index = write_index(temp.path()); + let manifest_path = temp.path().join("manifest.json"); + let control_path = temp.path().join("create-mode-control"); + fs::File::create(&control_path).unwrap(); + let create_mode = fs::metadata(&control_path).unwrap().permissions().mode() & 0o777; + let manifest = create_manifest_for_index( + &index, + CreateRowIdentity::RowIdIdentity, + "test-embedding", + &manifest_path, + ) + .unwrap(); + + write_manifest_file(&manifest, &manifest_path).unwrap(); + assert_eq!( + fs::metadata(&manifest_path).unwrap().permissions().mode() & 0o777, + create_mode + ); + + fs::set_permissions(&manifest_path, fs::Permissions::from_mode(0o640)).unwrap(); + write_manifest_file(&manifest, &manifest_path).unwrap(); + assert_eq!( + fs::metadata(&manifest_path).unwrap().permissions().mode() & 0o777, + 0o640 + ); +} + +#[cfg(unix)] +#[test] +fn manifest_write_opens_parent_before_replacing_destination() { + use std::os::unix::fs::PermissionsExt; + + let temp = tempfile::tempdir().unwrap(); + let index = write_index(temp.path()); + let manifest_path = temp.path().join("manifest.json"); + let manifest = create_manifest_for_index( + &index, + CreateRowIdentity::RowIdIdentity, + "test-embedding", + &manifest_path, + ) + .unwrap(); + fs::write(&manifest_path, b"original manifest").unwrap(); + + let original_permissions = fs::metadata(temp.path()).unwrap().permissions(); + fs::set_permissions(temp.path(), fs::Permissions::from_mode(0o300)).unwrap(); + let result = write_manifest_file(&manifest, &manifest_path); + fs::set_permissions(temp.path(), original_permissions).unwrap(); + + match result.unwrap_err() { + ManifestError::Io(error) => assert_eq!(error.kind(), std::io::ErrorKind::PermissionDenied), + error => panic!("expected parent-directory permission error, got {error}"), + } + assert_eq!(fs::read(manifest_path).unwrap(), b"original manifest"); +} + +#[cfg(unix)] +#[test] +fn manifest_write_does_not_replace_a_symlink_destination() { + use std::os::unix::fs::symlink; + + let temp = tempfile::tempdir().unwrap(); + let index = write_index(temp.path()); + let manifest_path = temp.path().join("manifest.json"); + let manifest = create_manifest_for_index( + &index, + CreateRowIdentity::RowIdIdentity, + "test-embedding", + &manifest_path, + ) + .unwrap(); + symlink("missing-manifest-target", &manifest_path).unwrap(); + + assert!(write_manifest_file(&manifest, &manifest_path).is_err()); + assert!(fs::symlink_metadata(manifest_path) + .unwrap() + .file_type() + .is_symlink()); +} + #[test] fn old_schema_manifest_fails_with_clear_schema_version_error() { let temp = tempfile::tempdir().unwrap(); @@ -349,3 +777,34 @@ fn create_rejects_paths_it_cannot_embed_canonically() { .unwrap_err(); assert!(err.to_string().contains("cannot be embedded"), "{err}"); } + +#[cfg(target_os = "linux")] +#[test] +fn create_rejects_non_utf8_paths_instead_of_embedding_them_lossily() { + use std::ffi::OsString; + use std::os::unix::ffi::OsStringExt; + + let temp = tempfile::tempdir().unwrap(); + let index = write_index(temp.path()); + let manifest_path = temp.path().join("manifest.json"); + let non_utf8 = temp + .path() + .join(OsString::from_vec(b"aux-\xff.bin".to_vec())); + fs::write(&non_utf8, b"aux").unwrap(); + let err = create_manifest_for_index_with_options( + &index, + CreateRowIdentity::RowIdIdentity, + "test-embedding", + &manifest_path, + CreateManifestOptions { + auxiliary_artifacts: vec![CreateAuxiliaryArtifact { + name: "aux".to_string(), + path: non_utf8, + required: true, + }], + ..CreateManifestOptions::default() + }, + ) + .unwrap_err(); + assert!(err.to_string().contains("not valid UTF-8"), "{err}"); +} diff --git a/ordvec-manifest/tests/manifest.rs b/ordvec-manifest/tests/manifest.rs index 28207de..4c5dbef 100644 --- a/ordvec-manifest/tests/manifest.rs +++ b/ordvec-manifest/tests/manifest.rs @@ -2,10 +2,10 @@ use ordvec::{Bitmap, Rank, RankQuant, SignBitmap}; use ordvec_manifest::{ create_manifest_for_index, create_manifest_for_index_with_options, load_manifest_file, load_manifest_file_with_options, sha256_file, verify_document_for_load, verify_for_load, - verify_index_manifest, verify_manifest_with_base, AuxiliaryArtifact, AuxiliaryArtifactState, - CalibrationOrdinalization, CalibrationProfileRef, CreateAuxiliaryArtifact, - CreateManifestOptions, CreateRowIdentity, DistortionBounds, DistortionEvidence, - DistortionEvidenceKind, DistortionProfileArtifactRef, DistortionScope, + verify_index_manifest, verify_manifest_with_base, write_manifest_file, AuxiliaryArtifact, + AuxiliaryArtifactState, CalibrationOrdinalization, CalibrationProfileRef, + CreateAuxiliaryArtifact, CreateManifestOptions, CreateRowIdentity, DistortionBounds, + DistortionEvidence, DistortionEvidenceKind, DistortionProfileArtifactRef, DistortionScope, EncoderDistortionProfileRef, EncoderSpec, ManifestIndexKind, ManifestIndexParams, MetricSpec, NullModelSpec, ProfileArtifactRef, ProfileParameterization, RequireAuxiliaryError, ResourceLimits, RowIdentity, VerificationCode, VerifiedLoadPlanError, VerifyOptions, @@ -27,6 +27,32 @@ fn write_index(dir: &Path) -> PathBuf { path } +#[test] +fn write_rejects_non_finite_numbers_without_replacing_the_destination() { + let temp = tempfile::tempdir().unwrap(); + let index = write_index(temp.path()); + let manifest_path = temp.path().join("manifest.json"); + let mut manifest = create_manifest_for_index( + &index, + CreateRowIdentity::RowIdIdentity, + "test-embedding", + &manifest_path, + ) + .unwrap(); + let mut profile = distortion_profile(&manifest, None, None, DistortionEvidenceKind::Certified); + profile.bounds.estimated_distortion = Some(f64::NAN); + manifest.encoder_distortion = Some(profile); + fs::write(&manifest_path, b"existing manifest bytes").unwrap(); + + let err = write_manifest_file(&manifest, &manifest_path).unwrap_err(); + assert!( + err.to_string() + .contains("encoder_distortion.bounds.estimated_distortion must be finite"), + "{err}" + ); + assert_eq!(fs::read(manifest_path).unwrap(), b"existing manifest bytes"); +} + fn write_rankquant_index(dir: &Path, rows: usize) -> PathBuf { let path = dir.join("index.ovrq"); let mut index = RankQuant::new(16, 2); diff --git a/ordvec-python/Cargo.toml b/ordvec-python/Cargo.toml index fb0a3cd..297f30f 100644 --- a/ordvec-python/Cargo.toml +++ b/ordvec-python/Cargo.toml @@ -20,6 +20,6 @@ bench-utils = ["ordvec_core/bench-utils"] [dependencies] # Alias the core crate as `ordvec_core` so binding code is unambiguous and never # mixes `ordvec::` with the Python-facing `ordvec` package name. -ordvec_core = { package = "ordvec", path = ".." } +ordvec_core = { package = "ordvec", version = "0.6.0", path = ".." } pyo3 = { version = "0.29.0", features = ["extension-module", "abi3-py310"] } numpy = "0.29.0" diff --git a/ordvec-python/README.md b/ordvec-python/README.md index 8375d28..f3d414d 100644 --- a/ordvec-python/README.md +++ b/ordvec-python/README.md @@ -5,14 +5,40 @@ training-free **ordinal & sign** vector-quantization library for compressed nearest-neighbour retrieval over high-dimensional embeddings. Pure-Rust core, zero system dependencies; SIMD-accelerated at runtime (AVX-512 / AVX2 / scalar). +## Quickstart + +```bash +python -m pip install --upgrade ordvec +``` + ```python import numpy as np -import ordvec +from ordvec import RankQuant + +documents = np.array([ + [8, 7, 6, 5, 4, 3, 2, 1], + [1, 2, 3, 4, 5, 6, 7, 8], + [8, 1, 7, 2, 6, 3, 5, 4], +], dtype=np.float32) +query = np.array([[8, 7, 6, 5, 4, 3, 2, 1]], dtype=np.float32) + +index = RankQuant(dim=8, bits=1) +index.add(documents) +scores, ids = index.search_asymmetric(query, k=1) +print(f"top document: {ids[0, 0]} (score {scores[0, 0]:.3f})") +``` + +```text +top document: 0 (score 0.396) +``` + +Persist and reopen without the original float corpus: -q = ordvec.RankQuant(1024, 2) # 1024-dim, 2 bits/coord -q.add(np.random.randn(10_000, 1024).astype(np.float32)) -# asymmetric: full-precision float queries vs bucketed docs (recommended) -scores, ids = q.search_asymmetric(np.random.randn(8, 1024).astype(np.float32), k=10) +```python +index.write("quickstart.ovrq") +reopened = RankQuant.load("quickstart.ovrq") +_, reopened_ids = reopened.search_asymmetric(query, k=1) +assert reopened_ids[0, 0] == 0 ``` ## Classes @@ -25,17 +51,28 @@ scores, ids = q.search_asymmetric(np.random.randn(8, 1024).astype(np.float32), k | `SignBitmap` | Sign bitmap for sign-cosine candidate generation; separate from the constant-weight bitmap theorem. | The Rust crate's `b = 8` RankQuant evidence/refinement width is not exposed -through the v0.5 Python `RankQuant` constructor and cannot be persisted to +through the v0.6.0 Python `RankQuant` constructor and cannot be persisted to `.ovrq`; use `bits` 1, 2, or 4 from Python. ## Two-stage retrieval (subset rerank) A `Bitmap` / `SignBitmap` probe yields a candidate shortlist that -`RankQuant.search_asymmetric_subset(query, candidates, k)` reranks exactly: +`RankQuant.search_asymmetric_subset(query, candidates, k)` reranks exactly. +Continuing from the quickstart's `documents` and `query`, tile the tiny rows to +the bitmap kernel's 64-coordinate minimum: ```python -cands = bm.top_m_candidates(query, m=256) # uint32 shortlist -scores, ids = rq.search_asymmetric_subset(query, cands, k=10) +from ordvec import Bitmap + +documents64 = np.tile(documents, (1, 8)) +query64 = np.tile(query, (1, 8)) +reranker = RankQuant(dim=64, bits=1) +reranker.add(documents64) +probe = Bitmap(dim=64, n_top=16) +probe.add(documents64) +candidates = probe.top_m_candidates(query64[0], m=2) +scores, ids = reranker.search_asymmetric_subset(query64[0], candidates, k=1) +assert ids[0] == 0 ``` Both returned arrays have length **`min(k, len(candidates))`**, not `k`. When @@ -56,11 +93,7 @@ calibrates that threshold by the hypergeometric upper tail. This is not a deployment guarantee for every encoder or corpus. Real-corpus recall, monotonicity, and null fit remain empirical diagnostics. -## Installation - -```bash -pip install ordvec -``` +## Installation details Wheels target CPython 3.10+ (abi3) and require `numpy>=2.2`. Building from source needs a Rust toolchain (MSRV 1.89) and @@ -74,7 +107,7 @@ buffers first, so ordinary Python in-place NumPy mutation from another thread cannot race the detached Rust scan. Large calls may temporarily require an additional input-sized buffer. The cross-language ownership and lifetime contract is maintained in -[`docs/bindings-safety.md`](https://github.com/Project-Navi/ordvec/blob/v0.5.0/docs/bindings-safety.md) +[`docs/bindings-safety.md`](https://github.com/Project-Navi/ordvec/blob/v0.6.0/docs/bindings-safety.md) for this release line. ## Type stubs diff --git a/ordvec-python/pyproject.toml b/ordvec-python/pyproject.toml index 79b26f7..240545e 100644 --- a/ordvec-python/pyproject.toml +++ b/ordvec-python/pyproject.toml @@ -49,6 +49,8 @@ dependencies = [ Homepage = "https://github.com/Project-Navi/ordvec" Repository = "https://github.com/Project-Navi/ordvec" Issues = "https://github.com/Project-Navi/ordvec/issues" +Documentation = "https://github.com/Project-Navi/ordvec/blob/v0.6.0/ordvec-python/README.md" +Changelog = "https://github.com/Project-Navi/ordvec/blob/main/CHANGELOG.md" Formalization = "https://github.com/Project-Navi/ordvec-formalization" [tool.maturin] diff --git a/ordvec-python/tests/test_rank_quant.py b/ordvec-python/tests/test_rank_quant.py index a6e1e2f..ca4fce3 100644 --- a/ordvec-python/tests/test_rank_quant.py +++ b/ordvec-python/tests/test_rank_quant.py @@ -16,7 +16,7 @@ import numpy as np import pytest -from ordvec import RankQuant, rankquant_eval_search +from ordvec import Bitmap, RankQuant, rankquant_eval_search def unit_vectors(n: int, dim: int, seed: int = 0) -> np.ndarray: @@ -117,6 +117,41 @@ def test_search_asymmetric_shape(bits): assert indices.shape == (3, 10) +def test_readme_quickstart_has_visible_deterministic_result(tmp_path): + documents = np.array( + [ + [8, 7, 6, 5, 4, 3, 2, 1], + [1, 2, 3, 4, 5, 6, 7, 8], + [8, 1, 7, 2, 6, 3, 5, 4], + ], + dtype=np.float32, + ) + query = np.array([[8, 7, 6, 5, 4, 3, 2, 1]], dtype=np.float32) + + index = RankQuant(dim=8, bits=1) + index.add(documents) + scores, ids = index.search_asymmetric(query, k=1) + + assert int(ids[0, 0]) == 0 + assert round(float(scores[0, 0]), 3) == 0.396 + + path = str(tmp_path / "quickstart.ovrq") + index.write(path) + reopened = RankQuant.load(path) + _, reopened_ids = reopened.search_asymmetric(query, k=1) + assert int(reopened_ids[0, 0]) == 0 + + documents64 = np.tile(documents, (1, 8)) + query64 = np.tile(query, (1, 8)) + reranker = RankQuant(dim=64, bits=1) + reranker.add(documents64) + probe = Bitmap(dim=64, n_top=16) + probe.add(documents64) + candidates = probe.top_m_candidates(query64[0], m=2) + _, subset_ids = reranker.search_asymmetric_subset(query64[0], candidates, k=1) + assert int(subset_ids[0]) == 0 + + def test_search_asymmetric_read_concurrent_results_match_baseline(): corpus = np.ascontiguousarray(unit_vectors(64, 128, seed=101)) queries = np.ascontiguousarray(unit_vectors(5, 128, seed=202)) diff --git a/src/lib.rs b/src/lib.rs index 754b6a4..d5d494b 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -53,16 +53,20 @@ //! in-model theorem, not a claim that real encoders automatically satisfy the //! quotient, symmetry, or null assumptions. //! -//! ```no_run -//! use ordvec::{Rank, RankQuant}; +//! ``` +//! use ordvec::RankQuant; //! -//! let mut idx = RankQuant::new(1024, 2); -//! let docs: Vec = vec![0.0; 1024 * 10_000]; -//! idx.add(&docs); +//! let documents = [ +//! 8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0, +//! 1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, +//! 8.0, 1.0, 7.0, 2.0, 6.0, 3.0, 5.0, 4.0, +//! ]; +//! let query = [8.0, 7.0, 6.0, 5.0, 4.0, 3.0, 2.0, 1.0]; //! -//! let queries: Vec = vec![0.0; 1024 * 4]; -//! let res = idx.search_asymmetric(&queries, 10); -//! assert_eq!(res.k, 10); +//! let mut index = RankQuant::new(8, 1); +//! index.add(&documents); +//! let results = index.search_asymmetric(&query, 1); +//! assert_eq!(results.indices_for_query(0)[0], 0); //! ``` // Every unsafe operation in the crate must sit inside an explicit `unsafe {}` diff --git a/tests/release_publish_invariants.py b/tests/release_publish_invariants.py index e4d39a8..e2f266f 100644 --- a/tests/release_publish_invariants.py +++ b/tests/release_publish_invariants.py @@ -23,6 +23,7 @@ WORKFLOW_PATH = os.environ.get("RELEASE_WORKFLOW_PATH", ".github/workflows/release.yml") CI_WORKFLOW_PATH = os.environ.get("CI_WORKFLOW_PATH", ".github/workflows/ci.yml") +AUDIT_WORKFLOW_PATH = os.environ.get("AUDIT_WORKFLOW_PATH", ".github/workflows/audit.yml") PYTHON_WORKFLOW_PATH = os.environ.get("PYTHON_WORKFLOW_PATH", ".github/workflows/python.yml") STRICT_STABLE_TAG_PATTERN = r"^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$" CHANGELOG_RELEASE_HEADING_RE = re.compile( @@ -464,30 +465,120 @@ def check_release_version_sync() -> None: if version != core_version: fail(f"{label} is {version}, expected lockstep version {core_version}") - manifest = load_toml("ordvec-manifest/Cargo.toml") - dependencies = mapping(manifest.get("dependencies"), "ordvec-manifest/Cargo.toml: dependencies") - ordvec_dep = mapping( - dependencies.get("ordvec"), "ordvec-manifest/Cargo.toml: dependencies.ordvec" + internal_requirements = ( + ("ordvec-manifest/Cargo.toml", "ordvec"), + ("ordvec-python/Cargo.toml", "ordvec_core"), + ("ordvec-manifest-python/Cargo.toml", "ordvec_manifest_core"), + ("ordvec-ffi/Cargo.toml", "ordvec"), + ("benchmarks/beir-bench/Cargo.toml", "ordvec"), ) - dep_version = ordvec_dep.get("version") - if dep_version != core_version: - fail( - "ordvec-manifest/Cargo.toml: dependencies.ordvec.version " - f"is {dep_version!r}, expected {core_version!r}" + for path, dependency_name in internal_requirements: + manifest = load_toml(path) + dependencies = mapping(manifest.get("dependencies"), f"{path}: dependencies") + dependency = mapping( + dependencies.get(dependency_name), + f"{path}: dependencies.{dependency_name}", ) + dep_version = dependency.get("version") + if dep_version != core_version: + fail( + f"{path}: dependencies.{dependency_name}.version is {dep_version!r}, " + f"expected lockstep version {core_version!r}" + ) changelog = read_text("CHANGELOG.md") - if not re.search(rf"^## \[?{re.escape(core_version)}\]? - \d{{4}}-\d{{2}}-\d{{2}}$", changelog, re.MULTILINE): + release_heading = re.search( + rf"^## \[?{re.escape(core_version)}\]? - (\d{{4}}-\d{{2}}-\d{{2}})$", + changelog, + re.MULTILINE, + ) + if release_heading is None: fail(f"CHANGELOG.md must contain a dated section for {core_version}") check_unreleased_section_empty_for_dated_version(changelog, core_version) threat_model = read_text("THREAT_MODEL.md") - if not re.search( - rf"^\>\s+\*\*Status:\*\*\s+v{re.escape(core_version)}\s+\(pre-1\.0\),", - threat_model, + release_date = release_heading.group(1) + expected_threat_status = ( + f"> **Status:** v{core_version} (pre-1.0), {release_date}." + ) + if expected_threat_status not in threat_model: + fail( + "THREAT_MODEL.md status must match the current changelog release " + f"and date: {expected_threat_status!r}" + ) + + semver_capture = r"v((?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*)\.(?:0|[1-9][0-9]*))" + release_contract_patterns = { + "README.md": ( + rf"\bIn {semver_capture} it is\s+Rust-only", + rf"cross-language persisted retrieval widths in {semver_capture}", + rf"`RankQuantFastscan::\{{write,load\}}` calls\. In {semver_capture}", + rf"loads `\.ovfs` directly, but in {semver_capture}", + ), + "ordvec-python/README.md": ( + rf"through the {semver_capture} Python `RankQuant` constructor", + rf"/blob/{semver_capture}/docs/bindings-safety\.md", + ), + "docs/INDEX_PROVENANCE.md": ( + rf"^{semver_capture} `\.ovfs` is not covered", + rf"crosses a trust boundary in\s+{semver_capture}, bind the bytes", + ), + "docs/PERSISTED_FORMAT.md": ( + rf"In {semver_capture}, `probe_index_metadata\(path\)` rejects", + rf"probe-versus-load contract in {semver_capture}\.", + ), + "docs/compatibility-policy.md": ( + rf"supported, but in {semver_capture} `\.ovfs`", + rf"^{semver_capture}\. Promoting `\.ovfs`", + ), + } + for path, patterns in release_contract_patterns.items(): + text = read_text(path) + for pattern in patterns: + match = re.search(pattern, text, re.MULTILINE) + if match is None: + fail(f"{path}: current-release contract marker is missing: {pattern!r}") + if match.group(1) != core_version: + fail( + f"{path}: current-release contract says v{match.group(1)}, " + f"expected v{core_version}" + ) + + schema_source = read_text("ordvec-manifest/src/lib.rs") + schema_match = re.search( + r'^pub const SCHEMA_VERSION: &str = "ordvec\.index_manifest\.v([1-9][0-9]*)";$', + schema_source, re.MULTILINE, - ): - fail(f"THREAT_MODEL.md status must mention v{core_version}") + ) + if schema_match is None: + fail("ordvec-manifest/src/lib.rs: could not read the manifest schema major") + schema_major = schema_match.group(1) + schema_contract_patterns = { + "README.md": ( + r"`probe_index_metadata\(\)` / `ordvec-manifest` v([1-9][0-9]*) contract", + r"`probe_index_metadata\(\)` or `ordvec-manifest` v([1-9][0-9]*)\.", + ), + "docs/INDEX_PROVENANCE.md": ( + r"`ordvec-manifest` v([1-9][0-9]*)\. This note", + r"The v([1-9][0-9]*) verifier intentionally does not create or verify `\.ovfs`", + r"manifest v([1-9][0-9]*) simply\s+does not bind or probe those bytes", + ), + "docs/compatibility-policy.md": ( + r"`ordvec-manifest` v([1-9][0-9]*) contract\. Feature-gated", + r"and v([1-9][0-9]*) JSON schema are treated as\s+stable release surfaces", + ), + } + for path, patterns in schema_contract_patterns.items(): + text = read_text(path) + for pattern in patterns: + match = re.search(pattern, text, re.MULTILINE) + if match is None: + fail(f"{path}: current-schema contract marker is missing: {pattern!r}") + if match.group(1) != schema_major: + fail( + f"{path}: current-schema contract says v{match.group(1)}, " + f"expected v{schema_major}" + ) fuzz_lock = read_text("fuzz/Cargo.lock") if not re.search( @@ -660,6 +751,29 @@ def check_manifest_cli_defaults() -> None: if "cargo install ordvec-manifest" not in readme: fail("ordvec-manifest/README.md: must document default cargo install") + matrix = read_text("docs/artifact-platform-matrix.md") + matrix_row = re.search( + r"^\| `ordvec-manifest` Rust crate \|.*$", matrix, re.MULTILINE + ) + if matrix_row is None: + fail("docs/artifact-platform-matrix.md: ordvec-manifest Rust crate row is missing") + row_text = matrix_row.group(0) + if re.search(r"default features? (?:are )?empty", row_text, re.IGNORECASE): + fail( + "docs/artifact-platform-matrix.md: ordvec-manifest defaults must not be " + "documented as empty" + ) + for required_fragment in ( + "default feature `cli`", + "optional `sqlite` and `sqlite-bundled`", + "disable defaults", + ): + if required_fragment not in row_text: + fail( + "docs/artifact-platform-matrix.md: ordvec-manifest row must mention " + f"{required_fragment!r}" + ) + def check_publication_model() -> None: expected_publish = { @@ -679,6 +793,7 @@ def check_publication_model() -> None: def check_python_package_metadata() -> None: + release_tag = f"v{package_version('Cargo.toml')}" pyproject = load_toml("ordvec-python/pyproject.toml") project = mapping(pyproject.get("project"), "ordvec-python/pyproject.toml: project") if project.get("name") != "ordvec": @@ -718,6 +833,11 @@ def check_python_package_metadata() -> None: "Homepage": "https://github.com/Project-Navi/ordvec", "Repository": "https://github.com/Project-Navi/ordvec", "Issues": "https://github.com/Project-Navi/ordvec/issues", + "Documentation": ( + f"https://github.com/Project-Navi/ordvec/blob/{release_tag}/" + "ordvec-python/README.md" + ), + "Changelog": "https://github.com/Project-Navi/ordvec/blob/main/CHANGELOG.md", "Formalization": "https://github.com/Project-Navi/ordvec-formalization", }.items(): if urls.get(key) != expected: @@ -779,6 +899,11 @@ def check_python_package_metadata() -> None: "Homepage": "https://github.com/Project-Navi/ordvec", "Repository": "https://github.com/Project-Navi/ordvec", "Issues": "https://github.com/Project-Navi/ordvec/issues", + "Documentation": ( + f"https://github.com/Project-Navi/ordvec/blob/{release_tag}/" + "ordvec-manifest-python/README.md" + ), + "Changelog": "https://github.com/Project-Navi/ordvec/blob/main/CHANGELOG.md", }.items(): if manifest_urls.get(key) != expected: fail( @@ -827,6 +952,69 @@ def check_python_package_metadata() -> None: fail(".github/dependabot.yml must keep the Python NumPy floor comment at >=2.2") +def check_first_run_docs() -> None: + readme = read_text("README.md") + quickstart_pos = readme.find("## Quickstart: see a result in 30 seconds") + benchmark_pos = readme.find("## Benchmark at a glance") + if quickstart_pos < 0 or benchmark_pos < 0 or quickstart_pos > benchmark_pos: + fail("README.md: the runnable quickstart must appear before benchmark detail") + for required in ( + "## Choose your surface", + "top document: 0 (score 0.396)", + 'index.write("quickstart.ovrq")', + 'RankQuant.load("quickstart.ovrq")', + "examples/quickstart.rs", + ): + if required not in readme: + fail(f"README.md: first-run path must include {required!r}") + if "link TBD" in readme: + fail("README.md: public release links must not contain a TBD placeholder") + + crate_docs = read_text("src/lib.rs") + for path, text in (("README.md", readme), ("src/lib.rs", crate_docs)): + example_pos = text.find("RankQuant::new") + if example_pos < 0: + fail(f"{path}: expected RankQuant::new in the first-run example") + if "10_000" in text[example_pos : example_pos + 800]: + fail(f"{path}: first example must not allocate a 10,000-row placeholder corpus") + + python_readme = read_text("ordvec-python/README.md") + install_pos = python_readme.find("python -m pip install --upgrade ordvec") + import_pos = python_readme.find("from ordvec import RankQuant") + if install_pos < 0 or import_pos < 0 or install_pos > import_pos: + fail("ordvec-python/README.md: installation must precede the first import") + for required in ( + "top document: 0 (score 0.396)", + 'index.write("quickstart.ovrq")', + "Continuing from the quickstart", + ): + if required not in python_readme: + fail(f"ordvec-python/README.md: first-run path must include {required!r}") + if "np.random" in python_readme: + fail("ordvec-python/README.md: first-run examples must be deterministic") + + manifest_readme = read_text("ordvec-manifest/README.md") + for required in ( + "## First verified index", + 'index.write("quickstart.ovrq")', + "quickstart.manifest.json\nverified", + ): + if required not in manifest_readme: + fail(f"ordvec-manifest/README.md: first-run path must include {required!r}") + + manifest_python_readme = read_text("ordvec-manifest-python/README.md") + for required in ( + "python -m pip install --upgrade ordvec ordvec-manifest", + 'index.write("quickstart.ovrq")', + "verified: True", + ): + if required not in manifest_python_readme: + fail( + "ordvec-manifest-python/README.md: first-run path must include " + f"{required!r}" + ) + + def check_release_docs_include_manifest_pypi_lane() -> None: releasing = read_text("RELEASING.md") normalized_releasing = " ".join(releasing.split()) @@ -950,6 +1138,7 @@ def check_package_contents() -> None: "docs/compatibility-policy.md", "docs/determinism.md", "examples/bench_rank.rs", + "examples/quickstart.rs", "src/lib.rs", "tests/index/main.rs", "tests/persistence_compat.rs", @@ -1306,10 +1495,14 @@ def check_release_security_gates(workflow: dict[str, Any], path: str) -> None: jobs = mapping(workflow.get("jobs"), f"{path}: jobs") require_job = mapping(jobs.get("require-ci-green"), f"{path}: jobs.require-ci-green") + timeout_minutes = require_job.get("timeout-minutes") + if not isinstance(timeout_minutes, int) or not 30 < timeout_minutes <= 60: + fail(f"{path}: require-ci-green must have a bounded 31-60 minute job timeout") steps = sequence(require_job.get("steps"), f"{path}: jobs.require-ci-green.steps") gated_workflows = ("ci.yml", "python.yml", "fuzz.yml", "codeql.yml", "actionlint.yml", "zizmor.yml") found_loop: tuple[str, ...] | None = None found_gate_run: str | None = None + found_gate_env: dict[str, Any] | None = None for index, raw_step in enumerate(steps): step = mapping(raw_step, f"{path}: jobs.require-ci-green.steps[{index}]") run = step.get("run") @@ -1319,6 +1512,9 @@ def check_release_security_gates(workflow: dict[str, Any], path: str) -> None: if match: found_loop = tuple(shlex.split(match.group(1))) found_gate_run = run + found_gate_env = mapping( + step.get("env"), f"{path}: jobs.require-ci-green.steps[{index}].env" + ) break if found_loop is None: fail(f"{path}: require-ci-green must loop over the release-gated workflow filenames") @@ -1328,12 +1524,56 @@ def check_release_security_gates(workflow: dict[str, Any], path: str) -> None: ) if found_gate_run is None or "event=push" not in found_gate_run or '.event == "push"' not in found_gate_run: fail(f"{path}: require-ci-green must require successful push workflow runs") + if "status=success" in found_gate_run: + fail(f"{path}: require-ci-green must query all run states before classifying them") if ( found_gate_run is None or "repos/${REPO}/commits/main" not in found_gate_run or "MAIN_SHA" not in found_gate_run ): fail(f"{path}: require-ci-green must verify the release tag points at current main") + if found_gate_env is None: + fail(f"{path}: require-ci-green poll environment was not found") + if found_gate_env.get("POLL_INTERVAL_SECONDS") != 30: + fail(f"{path}: require-ci-green must poll at the reviewed 30-second interval") + if found_gate_env.get("POLL_TIMEOUT_SECONDS") != 1800: + fail(f"{path}: require-ci-green must fail closed after the reviewed 30-minute deadline") + required_poll_fragments = ( + "while :; do", + ".head_sha == $sha", + 'sort_by([.run_number, .run_attempt, .id])', + "| last", + 'status\" != \"completed', + 'conclusion\" = \"success', + "SECONDS >= deadline", + 'sleep \"$POLL_INTERVAL_SECONDS\"', + 'api_status=\"$(sed -nE', + "api_exit_status=$?", + "408|429|5??)", + "connection (reset|refused|closed)", + "rate.?limit|secondary rate|abuse detection", + "retryable_api_error=true", + "API query failed permanently", + ) + for fragment in required_poll_fragments: + if fragment not in found_gate_run: + fail(f"{path}: require-ci-green is missing bounded-poll behavior {fragment!r}") + loop_index = found_gate_run.find("while :; do") + main_check_index = found_gate_run.find('MAIN_SHA="$(gh api', loop_index) + workflow_loop_index = found_gate_run.find("for wf in ", loop_index) + if main_check_index < loop_index or main_check_index > workflow_loop_index: + fail(f"{path}: require-ci-green must re-check current main before every workflow poll") + success_index = found_gate_run.find('if "$all_green"; then', workflow_loop_index) + final_main_index = found_gate_run.find('FINAL_MAIN_SHA="$(gh api', success_index) + success_break_index = found_gate_run.find("break", success_index) + if ( + success_index < workflow_loop_index + or final_main_index < success_index + or success_break_index < final_main_index + or '$SHA" != "$FINAL_MAIN_SHA' not in found_gate_run[final_main_index:success_break_index] + ): + fail(f"{path}: require-ci-green must re-check current main immediately before success") + allowed_id_token_jobs = { "attest", @@ -1377,6 +1617,74 @@ def check_release_security_gates(workflow: dict[str, Any], path: str) -> None: fail(f"{path}: jobs.{job_name} must use environment {environment!r}; got {actual!r}") +def check_cargo_deny_workspace_scope( + workflow: dict[str, Any], path: str, job_name: str, *, advisories_only: bool +) -> None: + jobs = mapping(workflow.get("jobs"), f"{path}: jobs") + job = mapping(jobs.get(job_name), f"{path}: jobs.{job_name}") + steps = sequence(job.get("steps"), f"{path}: jobs.{job_name}.steps") + deny_steps = [ + mapping(raw_step, f"{path}: jobs.{job_name}.steps[{index}]") + for index, raw_step in enumerate(steps) + if action_name(mapping(raw_step, f"{path}: jobs.{job_name}.steps[{index}]")) + == "embarkstudios/cargo-deny-action" + ] + if len(deny_steps) != 2: + fail( + f"{path}: jobs.{job_name} must use exactly two cargo-deny actions " + "(root workspace plus standalone fuzz lockfile)" + ) + + scoped_command_arguments: dict[str, str] = {} + for index, deny_step in enumerate(deny_steps): + inputs = mapping( + deny_step.get("with"), f"{path}: jobs.{job_name}.cargo-deny[{index}].with" + ) + if inputs.get("command") != "check": + fail(f"{path}: jobs.{job_name} cargo-deny command must be check") + argument_words = shlex.split(str(inputs.get("arguments", ""))) + if "--all-features" not in argument_words: + fail(f"{path}: jobs.{job_name} cargo-deny must activate all features") + if "--locked" not in argument_words: + fail(f"{path}: jobs.{job_name} cargo-deny must audit a committed lockfile") + + if "--manifest-path" in argument_words: + fail( + f"{path}: jobs.{job_name} cargo-deny must use the action's " + "manifest-path input instead of duplicating the CLI option" + ) + manifest_path = norm_path(inputs.get("manifest-path")) + has_fuzz_manifest = manifest_path == "fuzz/Cargo.toml" + if manifest_path and not has_fuzz_manifest: + fail( + f"{path}: jobs.{job_name} cargo-deny has unexpected manifest-path " + f"{manifest_path!r}" + ) + if "--workspace" in argument_words and not has_fuzz_manifest: + scope = "workspace" + elif has_fuzz_manifest and "--workspace" not in argument_words: + scope = "fuzz" + else: + fail( + f"{path}: jobs.{job_name} cargo-deny action must target either " + "--workspace or --manifest-path fuzz/Cargo.toml" + ) + if scope in scoped_command_arguments: + fail(f"{path}: jobs.{job_name} has duplicate cargo-deny scope {scope!r}") + scoped_command_arguments[scope] = str(inputs.get("command-arguments", "")).strip() + + if set(scoped_command_arguments) != {"workspace", "fuzz"}: + fail(f"{path}: jobs.{job_name} must audit both workspace and fuzz lockfile scopes") + expected_workspace_command = "advisories" if advisories_only else "" + if scoped_command_arguments["workspace"] != expected_workspace_command: + fail( + f"{path}: jobs.{job_name} workspace cargo-deny command arguments must be " + f"{expected_workspace_command!r}" + ) + if scoped_command_arguments["fuzz"] != "advisories": + fail(f"{path}: jobs.{job_name} fuzz cargo-deny must be scoped to advisories") + + def check_aarch64_smoke_selector(workflow: dict[str, Any], path: str) -> None: jobs = mapping(workflow.get("jobs"), f"{path}: jobs") job = mapping(jobs.get("smoke-linux-aarch64-wheel"), f"{path}: jobs.smoke-linux-aarch64-wheel") @@ -2057,6 +2365,7 @@ def check_sde_cache_invariants() -> None: def main() -> None: workflow = load_workflow(WORKFLOW_PATH) ci_workflow = load_workflow(CI_WORKFLOW_PATH) + audit_workflow = load_workflow(AUDIT_WORKFLOW_PATH) check_release_version_sync() check_release_compatibility_sync() check_python_binding_safety_docs_sync() @@ -2064,10 +2373,17 @@ def main() -> None: check_manifest_cli_defaults() check_publication_model() check_python_package_metadata() + check_first_run_docs() check_release_docs_include_manifest_pypi_lane() check_strict_release_tag_patterns(workflow, WORKFLOW_PATH) check_package_contents() check_ci_package_guards(ci_workflow, CI_WORKFLOW_PATH) + check_cargo_deny_workspace_scope( + ci_workflow, CI_WORKFLOW_PATH, "deny", advisories_only=False + ) + check_cargo_deny_workspace_scope( + audit_workflow, AUDIT_WORKFLOW_PATH, "advisories", advisories_only=True + ) check_hash_requirement_temp_paths( [WORKFLOW_PATH, PYTHON_WORKFLOW_PATH, CI_WORKFLOW_PATH, COVERAGE_WORKFLOW_PATH] ) diff --git a/tests/release_signed_release_invariants.sh b/tests/release_signed_release_invariants.sh index 2994fa7..be9c475 100755 --- a/tests/release_signed_release_invariants.sh +++ b/tests/release_signed_release_invariants.sh @@ -112,6 +112,15 @@ job_downloads_artifact_to_path() { ' } +# No draft Release, build artifact, wheel, or sdist may be created until the +# exact-head main CI gate has settled green. Keeping these dependencies direct +# makes the fail-closed front door visible and prevents a future DAG refactor +# from accidentally starting release work before the gate. +for job in notes build-crate build-wheels build-sdist build-manifest-wheels build-manifest-sdist; do + job_needs "$job" require-ci-green \ + || fail "$job must \`needs: require-ci-green\` before creating release artifacts" +done + # ---------------------------------------------------------------------- # (1) release-assets-draft needs attest + provenance + require-ci-green + notes # + fail-closed release AVX-512 proof + exact linux/aarch64 wheel smoke From 1605b6a3fe8e19905564e93df314cb2f6242f5dd Mon Sep 17 00:00:00 2001 From: Nelson Spence Date: Thu, 23 Jul 2026 17:37:21 -0500 Subject: [PATCH 4/4] docs(release): disclose Python wheel platform limits --- README.md | 5 ++++- ordvec-manifest-python/README.md | 7 +++++++ ordvec-python/README.md | 9 ++++++--- 3 files changed, 17 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 453cfbd..725e8da 100644 --- a/README.md +++ b/README.md @@ -340,7 +340,10 @@ Python — the bindings ship to PyPI as `ordvec`: pip install ordvec ``` -Wheels target CPython 3.10+ (abi3); to build from source instead, see +The v0.6.0 wheel matrix covers CPython 3.10+ (abi3) on manylinux/glibc +x86_64 and aarch64, macOS Apple Silicon, and Windows x64. Intel Mac and +musl/Alpine installations fall back to a source build and require Rust 1.89; +see the [artifact platform matrix](docs/artifact-platform-matrix.md) and [`ordvec-python/`](https://github.com/Project-Navi/ordvec/tree/main/ordvec-python). The runtime dependency floor is `numpy>=2.2`. diff --git a/ordvec-manifest-python/README.md b/ordvec-manifest-python/README.md index 674e8cc..9e75a77 100644 --- a/ordvec-manifest-python/README.md +++ b/ordvec-manifest-python/README.md @@ -8,6 +8,13 @@ Python bindings for the `ordvec-manifest` verifier. python -m pip install --upgrade ordvec ordvec-manifest ``` +The v0.6.0 wheel matrix covers CPython 3.10+ (abi3) on manylinux/glibc +x86_64 and aarch64, macOS Apple Silicon, and Windows x64. Intel Mac and +musl/Alpine installations fall back to a source build and require Rust 1.89 +plus [maturin](https://www.maturin.rs/). See the +[artifact platform matrix](https://github.com/Project-Navi/ordvec/blob/v0.6.0/docs/artifact-platform-matrix.md) +for the complete release surface. + ```python import numpy as np from ordvec import RankQuant diff --git a/ordvec-python/README.md b/ordvec-python/README.md index f3d414d..025f35b 100644 --- a/ordvec-python/README.md +++ b/ordvec-python/README.md @@ -95,9 +95,12 @@ recall, monotonicity, and null fit remain empirical diagnostics. ## Installation details -Wheels target CPython 3.10+ (abi3) and require `numpy>=2.2`. Building from -source needs a Rust toolchain (MSRV 1.89) and -[maturin](https://www.maturin.rs/). +The v0.6.0 wheel matrix covers CPython 3.10+ (abi3) on manylinux/glibc +x86_64 and aarch64, macOS Apple Silicon, and Windows x64, and requires +`numpy>=2.2`. Intel Mac and musl/Alpine installations fall back to a source +build and require Rust 1.89 plus [maturin](https://www.maturin.rs/). See the +[artifact platform matrix](https://github.com/Project-Navi/ordvec/blob/v0.6.0/docs/artifact-platform-matrix.md) +for the complete release surface. ## Safety contract