diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 42b6f60..65c6bec 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -9,6 +9,10 @@ on: branches: - main +permissions: + actions: read + contents: read + concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true @@ -17,9 +21,6 @@ jobs: build-and-test: runs-on: ubuntu-latest env: - LBUG_VERSION: main - LBUG_SOURCE_DIR: lbug-src - LBUG_BUILD_FROM_SOURCE: 1 CARGO_HOME: ${{ github.workspace }}/.cargo RUSTC_WRAPPER: sccache SCCACHE_GHA_ENABLED: "true" @@ -28,6 +29,13 @@ jobs: with: fetch-depth: 1 + - name: Checkout ladybug + uses: actions/checkout@v4 + with: + repository: LadybugDB/ladybug + fetch-depth: 1 + path: ladybug + - name: Setup sccache uses: mozilla-actions/sccache-action@v0.0.9 @@ -52,8 +60,51 @@ jobs: restore-keys: | cargo-registry-${{ runner.os }}- + - name: Resolve compatible lbug artifact run + working-directory: ladybug + env: + GITHUB_TOKEN: ${{ github.token }} + run: | + SHA="$(git rev-parse HEAD)" + API_URL="https://api.github.com/repos/LadybugDB/ladybug/actions/workflows/build-and-deploy.yml/runs" + AUTH_HEADER="Authorization: Bearer $GITHUB_TOKEN" + ACCEPT_HEADER="Accept: application/vnd.github+json" + VERSION_HEADER="X-GitHub-Api-Version: 2022-11-28" + + RUN_ID="$( + curl -fsSL \ + -H "$AUTH_HEADER" \ + -H "$ACCEPT_HEADER" \ + -H "$VERSION_HEADER" \ + "$API_URL?head_sha=$SHA&status=success&per_page=1" \ + | python -c 'import json,sys; data=json.load(sys.stdin); runs=data.get("workflow_runs") or []; print(runs[0]["id"] if runs else "")' + )" + + if [ -z "$RUN_ID" ]; then + RUN_ID="$( + curl -fsSL \ + -H "$AUTH_HEADER" \ + -H "$ACCEPT_HEADER" \ + -H "$VERSION_HEADER" \ + "$API_URL?branch=main&status=success&per_page=1" \ + | python -c 'import json,sys; data=json.load(sys.stdin); runs=data.get("workflow_runs") or []; print(runs[0]["id"] if runs else "")' + )" + fi + + if [ -z "$RUN_ID" ]; then + echo "Could not find a successful LadybugDB/ladybug build-and-deploy run." >&2 + exit 1 + fi + + echo "Using Ladybug build-and-deploy RUN_ID=$RUN_ID for SHA=$SHA" + echo "LBUG_PRECOMPILED_RUN_ID=$RUN_ID" >> "$GITHUB_ENV" + - name: Build + env: + GH_TOKEN: ${{ github.token }} run: cargo build --release - name: Run tests + env: + GH_TOKEN: ${{ github.token }} run: cargo test diff --git a/build.rs b/build.rs index d7774ea..8bd4f53 100644 --- a/build.rs +++ b/build.rs @@ -1,7 +1,7 @@ use std::env; use std::path::{Path, PathBuf}; -const PREBUILT_LIB_DIR: &str = ".cache/lbug-prebuilt/lib"; +const PREBUILT_CACHE_DIR: &str = ".cache/lbug-prebuilt"; fn link_mode() -> &'static str { if env::var("LBUG_SHARED").is_ok() { @@ -15,7 +15,7 @@ fn get_target() -> String { env::var("PROFILE").unwrap() } -fn link_libraries(link_bundled_deps: bool, prebuilt_lbug_root: Option<&Path>) { +fn link_libraries(link_bundled_deps: bool) { // This also needs to be set by any crates using it if they want to use extensions if !cfg!(windows) && link_mode() == "static" { println!("cargo:rustc-link-arg=-rdynamic"); @@ -41,9 +41,6 @@ fn link_libraries(link_bundled_deps: bool, prebuilt_lbug_root: Option<&Path>) { } if !link_bundled_deps { - if let Some(lbug_root) = prebuilt_lbug_root { - link_prebuilt_libraries(lbug_root); - } return; } @@ -75,36 +72,6 @@ fn link_libraries(link_bundled_deps: bool, prebuilt_lbug_root: Option<&Path>) { } } -fn link_prebuilt_libraries(lbug_root: &Path) { - let build_dir = lbug_root.join("build").join("release"); - for (dir, lib) in [ - ("antlr4_cypher", "antlr4_cypher"), - ("antlr4_runtime", "antlr4_runtime"), - ("brotli", "brotlidec"), - ("brotli", "brotlicommon"), - ("utf8proc", "utf8proc"), - ("re2", "re2"), - ("fastpfor", "fastpfor"), - ("parquet", "parquet"), - ("snappy", "snappy"), - ("thrift", "thrift"), - ("yyjson", "yyjson"), - ("zstd", "zstd"), - ("miniz", "miniz"), - ("mbedtls", "mbedtls"), - ("lz4", "lz4"), - ("roaring_bitmap", "roaring_bitmap"), - ("simsimd", "simsimd"), - ] { - let lib_dir = build_dir.join("third_party").join(dir); - let lib_path = lib_dir.join(format!("lib{lib}.a")); - if lib_path.exists() { - println!("cargo:rustc-link-search=native={}", lib_dir.display()); - println!("cargo:rustc-link-lib=static={lib}"); - } - } -} - fn manifest_dir() -> PathBuf { PathBuf::from(std::env::var("CARGO_MANIFEST_DIR").unwrap()) } @@ -117,7 +84,68 @@ fn static_lbug_file_name() -> &'static str { } } +fn prebuilt_cache_key() -> String { + let source = if let Ok(run_id) = env::var("LBUG_PRECOMPILED_RUN_ID") { + format!("run-{run_id}") + } else if let Ok(version) = env::var("LBUG_VERSION") { + format!("version-{version}") + } else { + "latest".to_string() + }; + + source + .chars() + .map(|c| { + if c.is_ascii_alphanumeric() || c == '-' || c == '_' || c == '.' { + c + } else { + '-' + } + }) + .collect() +} + +fn prebuilt_lib_dir(manifest_dir: &Path) -> PathBuf { + manifest_dir + .join(PREBUILT_CACHE_DIR) + .join(prebuilt_cache_key()) + .join("lib") +} + +fn prebuilt_source_desc() -> String { + let repo = + env::var("LBUG_GITHUB_REPOSITORY").unwrap_or_else(|_| "LadybugDB/ladybug".to_string()); + if let Ok(run_id) = env::var("LBUG_PRECOMPILED_RUN_ID") { + format!("run:{repo}/{run_id}") + } else if let Ok(version) = env::var("LBUG_VERSION") { + let version = version.strip_prefix('v').unwrap_or(&version); + format!("release:{repo}/v{version}") + } else { + format!("release:{repo}/latest") + } +} + +fn emit_lbug_metadata(source: &str, lib_dir: &Path) { + println!("cargo:rustc-env=LBUG_PRECOMPILED_SOURCE={source}"); + println!( + "cargo:rustc-env=LBUG_PRECOMPILED_LIBRARY_DIR={}", + lib_dir.display() + ); +} + fn try_download_prebuilt_lbug(manifest_dir: &Path) -> bool { + for var in [ + "LBUG_PRECOMPILED_RUN_ID", + "LBUG_VERSION", + "LBUG_GITHUB_REPOSITORY", + "LBUG_LINUX_VARIANT", + "LBUG_LIB_KIND", + "LBUG_BUILD_FROM_SOURCE", + "LBUG_RUST_BUILD_FROM_SOURCE", + ] { + println!("cargo:rerun-if-env-changed={var}"); + } + if link_mode() != "static" { return false; } @@ -127,7 +155,7 @@ fn try_download_prebuilt_lbug(manifest_dir: &Path) -> bool { return false; } - let lib_dir = manifest_dir.join(PREBUILT_LIB_DIR); + let lib_dir = prebuilt_lib_dir(manifest_dir); let lib_path = lib_dir.join(static_lbug_file_name()); if lib_path.exists() { return true; @@ -138,9 +166,9 @@ fn try_download_prebuilt_lbug(manifest_dir: &Path) -> bool { return false; } - println!("cargo:warning=Downloading prebuilt liblbug static archive..."); let status = std::process::Command::new("sh") .arg(&script) + .env("LBUG_TARGET_DIR", &lib_dir) .current_dir(manifest_dir) .status(); @@ -161,20 +189,16 @@ fn try_download_prebuilt_lbug(manifest_dir: &Path) -> bool { } } -fn use_prebuilt_lbug(manifest_dir: &Path) -> Option<(Vec, PathBuf)> { +fn use_prebuilt_lbug(manifest_dir: &Path) -> Option> { if !try_download_prebuilt_lbug(manifest_dir) { return None; } - let lib_dir = manifest_dir.join(PREBUILT_LIB_DIR); - let lbug_root = get_lbug_root(); + let lib_dir = prebuilt_lib_dir(manifest_dir); println!("cargo:rustc-link-search=native={}", lib_dir.display()); println!("cargo:rerun-if-changed={}", lib_dir.display()); - println!( - "cargo:warning=Using prebuilt liblbug from {}", - lib_dir.join(static_lbug_file_name()).display() - ); - Some((vec![lib_dir, lbug_root.join("src/include")], lbug_root)) + emit_lbug_metadata(&prebuilt_source_desc(), &lib_dir); + Some(vec![lib_dir]) } fn get_lbug_root() -> PathBuf { @@ -340,14 +364,16 @@ fn build_ffi( println!("cargo:rerun-if-changed=include/lbug_rs.h"); println!("cargo:rerun-if-changed=src/lbug_rs.cpp"); - // Note that this should match the lbug-src/* entries in the package.include list in Cargo.toml - // Unfortunately they appear to need to be specified individually since the symlink is - // considered to be changed each time. - println!("cargo:rerun-if-changed=lbug-src/src"); - println!("cargo:rerun-if-changed=lbug-src/cmake"); - println!("cargo:rerun-if-changed=lbug-src/third_party"); - println!("cargo:rerun-if-changed=lbug-src/CMakeLists.txt"); - println!("cargo:rerun-if-changed=lbug-src/tools/CMakeLists.txt"); + if bundled { + // Note that this should match the lbug-src/* entries in the package.include list in Cargo.toml + // Unfortunately they appear to need to be specified individually since the symlink is + // considered to be changed each time. + println!("cargo:rerun-if-changed=lbug-src/src"); + println!("cargo:rerun-if-changed=lbug-src/cmake"); + println!("cargo:rerun-if-changed=lbug-src/third_party"); + println!("cargo:rerun-if-changed=lbug-src/CMakeLists.txt"); + println!("cargo:rerun-if-changed=lbug-src/tools/CMakeLists.txt"); + } if cfg!(windows) { build.flag("/std:c++20"); @@ -367,7 +393,6 @@ fn main() { let manifest_dir = manifest_dir(); let mut bundled = false; let mut link_bundled_deps = false; - let mut prebuilt_lbug_root = None; let mut include_paths = vec![manifest_dir.join("include")]; if let (Ok(lbug_lib_dir), Ok(lbug_include)) = @@ -375,17 +400,19 @@ fn main() { { println!("cargo:rustc-link-search=native={lbug_lib_dir}"); println!("cargo:rustc-link-arg=-Wl,-rpath,{lbug_lib_dir}"); + emit_lbug_metadata("external", Path::new(&lbug_lib_dir)); include_paths.push(Path::new(&lbug_include).to_path_buf()); - } else if let Some((prebuilt_include_paths, lbug_root)) = use_prebuilt_lbug(&manifest_dir) { + } else if let Some(prebuilt_include_paths) = use_prebuilt_lbug(&manifest_dir) { include_paths.extend(prebuilt_include_paths); - prebuilt_lbug_root = Some(lbug_root); } else { include_paths.extend(build_bundled_cmake()); bundled = true; link_bundled_deps = true; + println!("cargo:rustc-env=LBUG_PRECOMPILED_SOURCE=source"); + println!("cargo:rustc-env=LBUG_PRECOMPILED_LIBRARY_DIR="); } if link_mode() == "static" { - link_libraries(link_bundled_deps, prebuilt_lbug_root.as_deref()); + link_libraries(link_bundled_deps); } build_ffi( "src/ffi.rs", @@ -405,6 +432,6 @@ fn main() { ); } if link_mode() == "dylib" { - link_libraries(link_bundled_deps, prebuilt_lbug_root.as_deref()); + link_libraries(link_bundled_deps); } } diff --git a/src/lib.rs b/src/lib.rs index 77eb2d5..1b74844 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -30,8 +30,8 @@ //! variables can be used to configure the build process: //! //! - `LBUG_SHARED`: If set, link dynamically instead of statically -//! - `LBUG_SOURCE_DIR`: Directory of a Lbug source checkout to use for C++ headers when linking a -//! precompiled archive or falling back to a source build. Defaults to `../ladybug` when present. +//! - `LBUG_SOURCE_DIR`: Directory of a Lbug source checkout to use when falling back to a source +//! build. Defaults to `../ladybug` when present. //! - `LBUG_INCLUDE_DIR`: Directory of Lbug's headers //! - `LBUG_LIBRARY_DIR`: Directory containing Lbug's pre-built libraries. //! - `LBUG_BUILD_FROM_SOURCE` or `LBUG_RUST_BUILD_FROM_SOURCE`: If set, skip downloading a @@ -87,7 +87,32 @@ mod value; /// The version of the Lbug crate as reported by Cargo's `CARGO_PKG_VERSION` environment variable pub const VERSION: &str = env!("CARGO_PKG_VERSION"); +/// The source of the linked Lbug library selected by the build script. +/// +/// This is `external` when `LBUG_LIBRARY_DIR`/`LBUG_INCLUDE_DIR` were supplied, `source` when the +/// bundled C++ source was built, or a value such as `run:LadybugDB/ladybug/25646256977` or +/// `release:LadybugDB/ladybug/v0.16.0` when a precompiled archive was downloaded. +pub const LBUG_LIBRARY_SOURCE: &str = env!("LBUG_PRECOMPILED_SOURCE"); +/// The directory containing the linked precompiled Lbug library, if one was used. +pub const LBUG_LIBRARY_DIR: &str = env!("LBUG_PRECOMPILED_LIBRARY_DIR"); + /// Returns the storage version of the Lbug library pub fn get_storage_version() -> u64 { crate::ffi::ffi::get_storage_version() } + +/// Returns the source of the linked Lbug library selected by the build script. +pub fn get_library_source() -> &'static str { + LBUG_LIBRARY_SOURCE +} + +/// Returns the directory containing the linked precompiled Lbug library. +/// +/// This returns `None` when Lbug was built from source as part of this crate build. +pub fn get_library_dir() -> Option<&'static str> { + if LBUG_LIBRARY_DIR.is_empty() { + None + } else { + Some(LBUG_LIBRARY_DIR) + } +}