cargo: allow building from precompiled sources

.github/workflows/ci.yml

@@ -9,6 +9,10 @@ on:
     branches:
       - main
 
+permissions:
+  actions: read
+  contents: read
+
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
   cancel-in-progress: true
@@ -17,9 +21,6 @@ jobs:
   build-and-test:
     runs-on: ubuntu-latest
     env:
-      LBUG_VERSION: main
-      LBUG_SOURCE_DIR: lbug-src
-      LBUG_BUILD_FROM_SOURCE: 1
       CARGO_HOME: ${{ github.workspace }}/.cargo
       RUSTC_WRAPPER: sccache
       SCCACHE_GHA_ENABLED: "true"
@@ -28,6 +29,13 @@ jobs:
         with:
           fetch-depth: 1
 
+      - name: Checkout ladybug
+        uses: actions/checkout@v4
+        with:
+          repository: LadybugDB/ladybug
+          fetch-depth: 1
+          path: ladybug
+
       - name: Setup sccache
         uses: mozilla-actions/sccache-action@v0.0.9
 
@@ -52,8 +60,51 @@ jobs:
           restore-keys: |
             cargo-registry-${{ runner.os }}-
 
+      - name: Resolve compatible lbug artifact run
+        working-directory: ladybug
+        env:
+          GITHUB_TOKEN: ${{ github.token }}
+        run: |
+          SHA="$(git rev-parse HEAD)"
+          API_URL="https://api.github.com/repos/LadybugDB/ladybug/actions/workflows/build-and-deploy.yml/runs"
+          AUTH_HEADER="Authorization: Bearer $GITHUB_TOKEN"
+          ACCEPT_HEADER="Accept: application/vnd.github+json"
+          VERSION_HEADER="X-GitHub-Api-Version: 2022-11-28"
+
+          RUN_ID="$(
+            curl -fsSL \
+              -H "$AUTH_HEADER" \
+              -H "$ACCEPT_HEADER" \
+              -H "$VERSION_HEADER" \
+              "$API_URL?head_sha=$SHA&status=success&per_page=1" \
+            | python -c 'import json,sys; data=json.load(sys.stdin); runs=data.get("workflow_runs") or []; print(runs[0]["id"] if runs else "")'
+          )"
+
+          if [ -z "$RUN_ID" ]; then
+            RUN_ID="$(
+              curl -fsSL \
+                -H "$AUTH_HEADER" \
+                -H "$ACCEPT_HEADER" \
+                -H "$VERSION_HEADER" \
+                "$API_URL?branch=main&status=success&per_page=1" \
+              | python -c 'import json,sys; data=json.load(sys.stdin); runs=data.get("workflow_runs") or []; print(runs[0]["id"] if runs else "")'
+            )"
+          fi
+
+          if [ -z "$RUN_ID" ]; then
+            echo "Could not find a successful LadybugDB/ladybug build-and-deploy run." >&2
+            exit 1
+          fi
+
+          echo "Using Ladybug build-and-deploy RUN_ID=$RUN_ID for SHA=$SHA"
+          echo "LBUG_PRECOMPILED_RUN_ID=$RUN_ID" >> "$GITHUB_ENV"
+
       - name: Build
+        env:
+          GH_TOKEN: ${{ github.token }}
         run: cargo build --release
 
       - name: Run tests
+        env:
+          GH_TOKEN: ${{ github.token }}
         run: cargo test

build.rs

@@ -1,7 +1,7 @@
 use std::env;
 use std::path::{Path, PathBuf};
 
-const PREBUILT_LIB_DIR: &str = ".cache/lbug-prebuilt/lib";
+const PREBUILT_CACHE_DIR: &str = ".cache/lbug-prebuilt";
 
 fn link_mode() -> &'static str {
     if env::var("LBUG_SHARED").is_ok() {
@@ -15,7 +15,7 @@ fn get_target() -> String {
     env::var("PROFILE").unwrap()
 }
 
-fn link_libraries(link_bundled_deps: bool, prebuilt_lbug_root: Option<&Path>) {
+fn link_libraries(link_bundled_deps: bool) {
     // This also needs to be set by any crates using it if they want to use extensions
     if !cfg!(windows) && link_mode() == "static" {
         println!("cargo:rustc-link-arg=-rdynamic");
@@ -41,9 +41,6 @@ fn link_libraries(link_bundled_deps: bool, prebuilt_lbug_root: Option<&Path>) {
         }
 
         if !link_bundled_deps {
-            if let Some(lbug_root) = prebuilt_lbug_root {
-                link_prebuilt_libraries(lbug_root);
-            }
             return;
         }
 
@@ -75,36 +72,6 @@ fn link_libraries(link_bundled_deps: bool, prebuilt_lbug_root: Option<&Path>) {
     }
 }
 
-fn link_prebuilt_libraries(lbug_root: &Path) {
-    let build_dir = lbug_root.join("build").join("release");
-    for (dir, lib) in [
-        ("antlr4_cypher", "antlr4_cypher"),
-        ("antlr4_runtime", "antlr4_runtime"),
-        ("brotli", "brotlidec"),
-        ("brotli", "brotlicommon"),
-        ("utf8proc", "utf8proc"),
-        ("re2", "re2"),
-        ("fastpfor", "fastpfor"),
-        ("parquet", "parquet"),
-        ("snappy", "snappy"),
-        ("thrift", "thrift"),
-        ("yyjson", "yyjson"),
-        ("zstd", "zstd"),
-        ("miniz", "miniz"),
-        ("mbedtls", "mbedtls"),
-        ("lz4", "lz4"),
-        ("roaring_bitmap", "roaring_bitmap"),
-        ("simsimd", "simsimd"),
-    ] {
-        let lib_dir = build_dir.join("third_party").join(dir);
-        let lib_path = lib_dir.join(format!("lib{lib}.a"));
-        if lib_path.exists() {
-            println!("cargo:rustc-link-search=native={}", lib_dir.display());
-            println!("cargo:rustc-link-lib=static={lib}");
-        }
-    }
-}
-
 fn manifest_dir() -> PathBuf {
     PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap())
 }
@@ -117,7 +84,68 @@ fn static_lbug_file_name() -> &'static str {
     }
 }
 
+fn prebuilt_cache_key() -> String {
+    let source = if let Ok(run_id) = env::var("LBUG_PRECOMPILED_RUN_ID") {
+        format!("run-{run_id}")
+    } else if let Ok(version) = env::var("LBUG_VERSION") {
+        format!("version-{version}")
+    } else {
+        "latest".to_string()
+    };
+
+    source
+        .chars()
+        .map(|c| {
+            if c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '.' {
+                c
+            } else {
+                '-'
+            }
+        })
+        .collect()
+}
+
+fn prebuilt_lib_dir(manifest_dir: &Path) -> PathBuf {
+    manifest_dir
+        .join(PREBUILT_CACHE_DIR)
+        .join(prebuilt_cache_key())
+        .join("lib")
+}
+
+fn prebuilt_source_desc() -> String {
+    let repo =
+        env::var("LBUG_GITHUB_REPOSITORY").unwrap_or_else(|_| "LadybugDB/ladybug".to_string());
+    if let Ok(run_id) = env::var("LBUG_PRECOMPILED_RUN_ID") {
+        format!("run:{repo}/{run_id}")
+    } else if let Ok(version) = env::var("LBUG_VERSION") {
+        let version = version.strip_prefix('v').unwrap_or(&version);
+        format!("release:{repo}/v{version}")
+    } else {
+        format!("release:{repo}/latest")
+    }
+}
+
+fn emit_lbug_metadata(source: &str, lib_dir: &Path) {
+    println!("cargo:rustc-env=LBUG_PRECOMPILED_SOURCE={source}");
+    println!(
+        "cargo:rustc-env=LBUG_PRECOMPILED_LIBRARY_DIR={}",
+        lib_dir.display()
+    );
+}
+
 fn try_download_prebuilt_lbug(manifest_dir: &Path) -> bool {
+    for var in [
+        "LBUG_PRECOMPILED_RUN_ID",
+        "LBUG_VERSION",
+        "LBUG_GITHUB_REPOSITORY",
+        "LBUG_LINUX_VARIANT",
+        "LBUG_LIB_KIND",
+        "LBUG_BUILD_FROM_SOURCE",
+        "LBUG_RUST_BUILD_FROM_SOURCE",
+    ] {
+        println!("cargo:rerun-if-env-changed={var}");
+    }
+
     if link_mode() != "static" {
         return false;
     }
@@ -127,7 +155,7 @@ fn try_download_prebuilt_lbug(manifest_dir: &Path) -> bool {
         return false;
     }
 
-    let lib_dir = manifest_dir.join(PREBUILT_LIB_DIR);
+    let lib_dir = prebuilt_lib_dir(manifest_dir);
     let lib_path = lib_dir.join(static_lbug_file_name());
     if lib_path.exists() {
         return true;
@@ -138,9 +166,9 @@ fn try_download_prebuilt_lbug(manifest_dir: &Path) -> bool {
         return false;
     }
 
-    println!("cargo:warning=Downloading prebuilt liblbug static archive...");
     let status = std::process::Command::new("sh")
         .arg(&script)
+        .env("LBUG_TARGET_DIR", &lib_dir)
         .current_dir(manifest_dir)
         .status();
 
@@ -161,20 +189,16 @@ fn try_download_prebuilt_lbug(manifest_dir: &Path) -> bool {
     }
 }
 
-fn use_prebuilt_lbug(manifest_dir: &Path) -> Option<(Vec<PathBuf>, PathBuf)> {
+fn use_prebuilt_lbug(manifest_dir: &Path) -> Option<Vec<PathBuf>> {
     if !try_download_prebuilt_lbug(manifest_dir) {
         return None;
     }
 
-    let lib_dir = manifest_dir.join(PREBUILT_LIB_DIR);
-    let lbug_root = get_lbug_root();
+    let lib_dir = prebuilt_lib_dir(manifest_dir);
     println!("cargo:rustc-link-search=native={}", lib_dir.display());
     println!("cargo:rerun-if-changed={}", lib_dir.display());
-    println!(
-        "cargo:warning=Using prebuilt liblbug from {}",
-        lib_dir.join(static_lbug_file_name()).display()
-    );
-    Some((vec![lib_dir, lbug_root.join("src/include")], lbug_root))
+    emit_lbug_metadata(&prebuilt_source_desc(), &lib_dir);
+    Some(vec![lib_dir])
 }
 
 fn get_lbug_root() -> PathBuf {
@@ -340,14 +364,16 @@ fn build_ffi(
 
     println!("cargo:rerun-if-changed=include/lbug_rs.h");
     println!("cargo:rerun-if-changed=src/lbug_rs.cpp");
-    // Note that this should match the lbug-src/* entries in the package.include list in Cargo.toml
-    // Unfortunately they appear to need to be specified individually since the symlink is
-    // considered to be changed each time.
-    println!("cargo:rerun-if-changed=lbug-src/src");
-    println!("cargo:rerun-if-changed=lbug-src/cmake");
-    println!("cargo:rerun-if-changed=lbug-src/third_party");
-    println!("cargo:rerun-if-changed=lbug-src/CMakeLists.txt");
-    println!("cargo:rerun-if-changed=lbug-src/tools/CMakeLists.txt");
+    if bundled {
+        // Note that this should match the lbug-src/* entries in the package.include list in Cargo.toml
+        // Unfortunately they appear to need to be specified individually since the symlink is
+        // considered to be changed each time.
+        println!("cargo:rerun-if-changed=lbug-src/src");
+        println!("cargo:rerun-if-changed=lbug-src/cmake");
+        println!("cargo:rerun-if-changed=lbug-src/third_party");
+        println!("cargo:rerun-if-changed=lbug-src/CMakeLists.txt");
+        println!("cargo:rerun-if-changed=lbug-src/tools/CMakeLists.txt");
+    }
 
     if cfg!(windows) {
         build.flag("/std:c++20");
@@ -367,25 +393,26 @@ fn main() {
     let manifest_dir = manifest_dir();
     let mut bundled = false;
     let mut link_bundled_deps = false;
-    let mut prebuilt_lbug_root = None;
     let mut include_paths = vec![manifest_dir.join("include")];
 
     if let (Ok(lbug_lib_dir), Ok(lbug_include)) =
         (env::var("LBUG_LIBRARY_DIR"), env::var("LBUG_INCLUDE_DIR"))
     {
         println!("cargo:rustc-link-search=native={lbug_lib_dir}");
         println!("cargo:rustc-link-arg=-Wl,-rpath,{lbug_lib_dir}");
+        emit_lbug_metadata("external", Path::new(&lbug_lib_dir));
         include_paths.push(Path::new(&lbug_include).to_path_buf());
-    } else if let Some((prebuilt_include_paths, lbug_root)) = use_prebuilt_lbug(&manifest_dir) {
+    } else if let Some(prebuilt_include_paths) = use_prebuilt_lbug(&manifest_dir) {
         include_paths.extend(prebuilt_include_paths);
-        prebuilt_lbug_root = Some(lbug_root);
     } else {
         include_paths.extend(build_bundled_cmake());
         bundled = true;
         link_bundled_deps = true;
+        println!("cargo:rustc-env=LBUG_PRECOMPILED_SOURCE=source");
+        println!("cargo:rustc-env=LBUG_PRECOMPILED_LIBRARY_DIR=");
     }
     if link_mode() == "static" {
-        link_libraries(link_bundled_deps, prebuilt_lbug_root.as_deref());
+        link_libraries(link_bundled_deps);
     }
     build_ffi(
         "src/ffi.rs",
@@ -405,6 +432,6 @@ fn main() {
         );
     }
     if link_mode() == "dylib" {
-        link_libraries(link_bundled_deps, prebuilt_lbug_root.as_deref());
+        link_libraries(link_bundled_deps);
     }
 }

src/lib.rs

@@ -30,8 +30,8 @@
 //! variables can be used to configure the build process:
 //!
 //! - `LBUG_SHARED`: If set, link dynamically instead of statically
-//! - `LBUG_SOURCE_DIR`: Directory of a Lbug source checkout to use for C++ headers when linking a
-//!   precompiled archive or falling back to a source build. Defaults to `../ladybug` when present.
+//! - `LBUG_SOURCE_DIR`: Directory of a Lbug source checkout to use when falling back to a source
+//!   build. Defaults to `../ladybug` when present.
 //! - `LBUG_INCLUDE_DIR`: Directory of Lbug's headers
 //! - `LBUG_LIBRARY_DIR`: Directory containing Lbug's pre-built libraries.
 //! - `LBUG_BUILD_FROM_SOURCE` or `LBUG_RUST_BUILD_FROM_SOURCE`: If set, skip downloading a
@@ -87,7 +87,32 @@ mod value;
 
 /// The version of the Lbug crate as reported by Cargo's `CARGO_PKG_VERSION` environment variable
 pub const VERSION: &str = env!("CARGO_PKG_VERSION");
+/// The source of the linked Lbug library selected by the build script.
+///
+/// This is `external` when `LBUG_LIBRARY_DIR`/`LBUG_INCLUDE_DIR` were supplied, `source` when the
+/// bundled C++ source was built, or a value such as `run:LadybugDB/ladybug/25646256977` or
+/// `release:LadybugDB/ladybug/v0.16.0` when a precompiled archive was downloaded.
+pub const LBUG_LIBRARY_SOURCE: &str = env!("LBUG_PRECOMPILED_SOURCE");
+/// The directory containing the linked precompiled Lbug library, if one was used.
+pub const LBUG_LIBRARY_DIR: &str = env!("LBUG_PRECOMPILED_LIBRARY_DIR");
+
 /// Returns the storage version of the Lbug library
 pub fn get_storage_version() -> u64 {
     crate::ffi::ffi::get_storage_version()
 }
+
+/// Returns the source of the linked Lbug library selected by the build script.
+pub fn get_library_source() -> &'static str {
+    LBUG_LIBRARY_SOURCE
+}
+
+/// Returns the directory containing the linked precompiled Lbug library.
+///
+/// This returns `None` when Lbug was built from source as part of this crate build.
+pub fn get_library_dir() -> Option<&'static str> {
+    if LBUG_LIBRARY_DIR.is_empty() {
+        None
+    } else {
+        Some(LBUG_LIBRARY_DIR)
+    }
+}