diff --git a/c/include/cuvs/neighbors/hnsw.h b/c/include/cuvs/neighbors/hnsw.h index 15eb1b0569..823656410c 100644 --- a/c/include/cuvs/neighbors/hnsw.h +++ b/c/include/cuvs/neighbors/hnsw.h @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ @@ -109,7 +109,7 @@ CUVS_EXPORT cuvsError_t cuvsHnswAceParamsDestroy(cuvsHnswAceParams_t params); struct cuvsHnswIndexParams { /* hierarchy of the hnsw index */ enum cuvsHnswHierarchy hierarchy; - /** Size of the candidate list during hierarchy construction when hierarchy is `CPU`*/ + /** Maximum candidate list size used during index construction. */ int ef_construction; /** Number of host threads to use to construct hierarchy when hierarchy is `CPU` or `GPU`. When the value is 0, the number of threads is automatically determined to the @@ -119,15 +119,15 @@ struct cuvsHnswIndexParams { is parallelized with the help of CPU threads. */ int num_threads; - /** HNSW M parameter: number of bi-directional links per node (used when building with ACE). - * graph_degree = m * 2, intermediate_graph_degree = m * 3. + /** HNSW M parameter: number of bi-directional links per node. When the graph is built on the GPU, + * this parameter is used to derive the internal CAGRA graph build parameters. */ size_t M; /** Distance type for the index. */ cuvsDistanceType metric; /** - * Optional: specify ACE parameters for building HNSW index using ACE algorithm. - * Set to nullptr for default behavior (from_cagra conversion). + * Optional ACE parameters for out-of-core graph construction. + * Set to nullptr to select the graph build algorithm automatically. */ cuvsHnswAceParams_t ace_params; }; @@ -285,22 +285,20 @@ CUVS_EXPORT cuvsError_t cuvsHnswFromCagraWithDataset(cuvsResources_t res, */ /** - * @defgroup hnsw_c_index_build Build HNSW index using ACE algorithm + * @defgroup hnsw_c_index_build Build an HNSW index * @{ */ /** - * @brief Build an HNSW index using ACE (Augmented Core Extraction) algorithm. + * @brief Build an HNSW index from HNSW parameters. * - * ACE enables building HNSW indexes for datasets too large to fit in GPU memory by: - * 1. Partitioning the dataset using balanced k-means into core and augmented partitions - * 2. Building sub-indexes for each partition independently - * 3. Concatenating sub-graphs into a final unified index + * The graph is built on the GPU and converted to an HNSW index that can be searched on the CPU. + * The graph build algorithm is selected automatically unless explicit ACE parameters are provided. * * NOTE: This function requires CUDA to be available at runtime. * * @param[in] res cuvsResources_t opaque C handle - * @param[in] params cuvsHnswIndexParams_t with ACE parameters configured + * @param[in] params cuvsHnswIndexParams_t with HNSW build parameters * @param[in] dataset DLManagedTensor* host dataset to build index from * @param[out] index cuvsHnswIndex_t to return the built HNSW index * @@ -314,18 +312,10 @@ CUVS_EXPORT cuvsError_t cuvsHnswFromCagraWithDataset(cuvsResources_t res, * cuvsResources_t res; * cuvsResourcesCreate(&res); * - * // Create ACE parameters - * cuvsHnswAceParams_t ace_params; - * cuvsHnswAceParamsCreate(&ace_params); - * ace_params->npartitions = 4; - * ace_params->use_disk = true; - * ace_params->build_dir = "/tmp/hnsw_ace_build"; - * * // Create index parameters * cuvsHnswIndexParams_t params; * cuvsHnswIndexParamsCreate(¶ms); * params->hierarchy = GPU; - * params->ace_params = ace_params; * params->M = 32; * params->ef_construction = 120; * @@ -340,7 +330,6 @@ CUVS_EXPORT cuvsError_t cuvsHnswFromCagraWithDataset(cuvsResources_t res, * cuvsHnswBuild(res, params, &dataset, hnsw_index); * * // Clean up - * cuvsHnswAceParamsDestroy(ace_params); * cuvsHnswIndexParamsDestroy(params); * cuvsHnswIndexDestroy(hnsw_index); * cuvsResourcesDestroy(res); diff --git a/c/src/neighbors/hnsw.cpp b/c/src/neighbors/hnsw.cpp index 02bd2918f8..1f19f33875 100644 --- a/c/src/neighbors/hnsw.cpp +++ b/c/src/neighbors/hnsw.cpp @@ -38,15 +38,17 @@ void _build(cuvsResources_t res, cpp_params.M = params->M; cpp_params.metric = static_cast(params->metric); - // Configure ACE parameters - RAFT_EXPECTS(params->ace_params != nullptr, "ACE parameters must be set for hnsw::build"); - auto ace_params = cuvs::neighbors::hnsw::graph_build_params::ace_params(); - ace_params.npartitions = params->ace_params->npartitions; - ace_params.build_dir = params->ace_params->build_dir ? params->ace_params->build_dir : "/tmp/hnsw_ace_build"; - ace_params.use_disk = params->ace_params->use_disk; - ace_params.max_host_memory_gb = params->ace_params->max_host_memory_gb; - ace_params.max_gpu_memory_gb = params->ace_params->max_gpu_memory_gb; - cpp_params.graph_build_params = ace_params; + if (params->ace_params != nullptr) { + auto ace_params = cuvs::neighbors::hnsw::graph_build_params::ace_params(); + ace_params.npartitions = params->ace_params->npartitions; + ace_params.build_dir = params->ace_params->build_dir + ? params->ace_params->build_dir + : "/tmp/hnsw_ace_build"; + ace_params.use_disk = params->ace_params->use_disk; + ace_params.max_host_memory_gb = params->ace_params->max_host_memory_gb; + ace_params.max_gpu_memory_gb = params->ace_params->max_gpu_memory_gb; + cpp_params.graph_build_params = ace_params; + } using dataset_mdspan_type = raft::host_matrix_view; auto dataset_mds = cuvs::core::from_dlpack(dataset_tensor); diff --git a/c/tests/neighbors/ann_hnsw_c.cu b/c/tests/neighbors/ann_hnsw_c.cu index cdd8788530..1386bac3c2 100644 --- a/c/tests/neighbors/ann_hnsw_c.cu +++ b/c/tests/neighbors/ann_hnsw_c.cu @@ -136,6 +136,37 @@ TEST(CagraHnswC, BuildSearch) cuvsResourcesDestroy(res); } +TEST(HnswC, BuildWithoutAce) +{ + cuvsResources_t res; + cuvsResourcesCreate(&res); + + DLManagedTensor dataset_tensor; + dataset_tensor.dl_tensor.data = dataset; + dataset_tensor.dl_tensor.device.device_type = kDLCPU; + dataset_tensor.dl_tensor.ndim = 2; + dataset_tensor.dl_tensor.dtype.code = kDLFloat; + dataset_tensor.dl_tensor.dtype.bits = 32; + dataset_tensor.dl_tensor.dtype.lanes = 1; + int64_t dataset_shape[2] = {4, 2}; + dataset_tensor.dl_tensor.shape = dataset_shape; + dataset_tensor.dl_tensor.strides = nullptr; + + cuvsHnswIndexParams_t hnsw_params; + cuvsHnswIndexParamsCreate(&hnsw_params); + hnsw_params->M = 2; + hnsw_params->ef_construction = 100; + + cuvsHnswIndex_t hnsw_index; + cuvsHnswIndexCreate(&hnsw_index); + + ASSERT_EQ(cuvsHnswBuild(res, hnsw_params, &dataset_tensor, hnsw_index), CUVS_SUCCESS); + + cuvsHnswIndexParamsDestroy(hnsw_params); + cuvsHnswIndexDestroy(hnsw_index); + cuvsResourcesDestroy(res); +} + TEST(HnswAceC, BuildSearch) { // create cuvsResources_t diff --git a/fern/pages/c_api/c-api-neighbors-hnsw.md b/fern/pages/c_api/c-api-neighbors-hnsw.md index 317c9aa4f0..497fe89584 100644 --- a/fern/pages/c_api/c-api-neighbors-hnsw.md +++ b/fern/pages/c_api/c-api-neighbors-hnsw.md @@ -282,12 +282,12 @@ cuvsHnswIndex_t hnsw_index); [`cuvsError_t`](/api-reference/c-api-core-c-api#cuvserror-t) -## Build HNSW index using ACE algorithm +## Build an HNSW index ### cuvsHnswBuild -Build an HNSW index using ACE (Augmented Core Extraction) algorithm. +Build an HNSW index from HNSW parameters. ```c cuvsError_t cuvsHnswBuild(cuvsResources_t res, @@ -296,11 +296,8 @@ DLManagedTensor* dataset, cuvsHnswIndex_t index); ``` -ACE enables building HNSW indexes for datasets too large to fit in GPU memory by: - -1. Partitioning the dataset using balanced k-means into core and augmented partitions -2. Building sub-indexes for each partition independently -3. Concatenating sub-graphs into a final unified index +The graph is built on the GPU and converted to an HNSW index that can be searched on the CPU. +The graph build algorithm is selected automatically unless explicit ACE parameters are provided. NOTE: This function requires CUDA to be available at runtime. @@ -309,7 +306,7 @@ NOTE: This function requires CUDA to be available at runtime. | Name | Direction | Type | Description | | --- | --- | --- | --- | | `res` | in | [`cuvsResources_t`](/api-reference/c-api-core-c-api#cuvsresources-t) | cuvsResources_t opaque C handle | -| `params` | in | `cuvsHnswIndexParams_t` | cuvsHnswIndexParams_t with ACE parameters configured | +| `params` | in | `cuvsHnswIndexParams_t` | cuvsHnswIndexParams_t with HNSW build parameters | | `dataset` | in | `DLManagedTensor*` | DLManagedTensor* host dataset to build index from | | `index` | out | [`cuvsHnswIndex_t`](/api-reference/c-api-neighbors-hnsw#cuvshnswindex) | cuvsHnswIndex_t to return the built HNSW index | diff --git a/fern/pages/java_api/java-api-com-nvidia-cuvs-hnswindex.md b/fern/pages/java_api/java-api-com-nvidia-cuvs-hnswindex.md index df40dd733f..36d2623a2e 100644 --- a/fern/pages/java_api/java-api-com-nvidia-cuvs-hnswindex.md +++ b/fern/pages/java_api/java-api-com-nvidia-cuvs-hnswindex.md @@ -101,21 +101,16 @@ _Source: `java/cuvs-java/src/main/java/com/nvidia/cuvs/HnswIndex.java:53`_ static HnswIndex build(CuVSResources resources, HnswIndexParams hnswParams, CuVSMatrix dataset) throws Throwable ``` -Builds an HNSW index using the ACE (Augmented Core Extraction) algorithm. - -ACE enables building HNSW indexes for datasets too large to fit in GPU -memory by partitioning the dataset and building sub-indexes for each -partition independently. - -NOTE: This method requires `hnswParams.getAceParams()` to be set with -an instance of HnswAceParams. +Builds an HNSW index from HNSW parameters. The graph is built on the GPU and converted to an +HNSW index that can be searched on the CPU. The graph build algorithm is selected automatically +unless explicit ACE parameters are provided. **Parameters** | Name | Description | | --- | --- | | `resources` | The CuVS resources | -| `hnswParams` | Parameters for the HNSW index with ACE configuration | +| `hnswParams` | Parameters for the HNSW index | | `dataset` | The dataset to build the index from | **Returns** diff --git a/fern/pages/java_api/java-api-com-nvidia-cuvs-hnswindexparams.md b/fern/pages/java_api/java-api-com-nvidia-cuvs-hnswindexparams.md index 518043dc8d..7b1951fc08 100644 --- a/fern/pages/java_api/java-api-com-nvidia-cuvs-hnswindexparams.md +++ b/fern/pages/java_api/java-api-com-nvidia-cuvs-hnswindexparams.md @@ -83,8 +83,7 @@ public long getM() ``` Gets the HNSW M parameter: number of bi-directional links per node -(used when building with ACE). graph_degree = m * 2, -intermediate_graph_degree = m * 3. +used to derive the internal graph build parameters for GPU construction. **Returns** @@ -112,7 +111,8 @@ _Source: `java/cuvs-java/src/main/java/com/nvidia/cuvs/HnswIndexParams.java:142` public HnswAceParams getAceParams() ``` -Gets the ACE parameters for building HNSW index using ACE algorithm. +Gets the optional ACE parameters for explicit out-of-core graph construction. When not set, the +graph build algorithm is selected automatically. **Returns** @@ -159,14 +159,13 @@ _Source: `java/cuvs-java/src/main/java/com/nvidia/cuvs/HnswIndexParams.java:202` public Builder withEfConstruction(int efConstruction) ``` -Sets the size of the candidate list during hierarchy construction when -hierarchy is `CPU`. +Sets the maximum candidate list size used during index construction. **Parameters** | Name | Description | | --- | --- | -| `efConstruction` | the size of the candidate list during hierarchy construction when hierarchy is `CPU` | +| `efConstruction` | the maximum candidate list size used during construction | **Returns** @@ -222,8 +221,7 @@ public Builder withM(long m) ``` Sets the HNSW M parameter: number of bi-directional links per node -(used when building with ACE). graph_degree = m * 2, -intermediate_graph_degree = m * 3. +used to derive the internal graph build parameters for GPU construction. **Parameters** @@ -263,7 +261,8 @@ _Source: `java/cuvs-java/src/main/java/com/nvidia/cuvs/HnswIndexParams.java:262` public Builder withAceParams(HnswAceParams aceParams) ``` -Sets the ACE parameters for building HNSW index using ACE algorithm. +Sets optional ACE parameters for explicit out-of-core graph construction. When not set, the graph +build algorithm is selected automatically. **Parameters** diff --git a/fern/pages/java_api/java-api-com-nvidia-cuvs-spi-cuvsprovider.md b/fern/pages/java_api/java-api-com-nvidia-cuvs-spi-cuvsprovider.md index 17557e5e57..5cc6c3eeb0 100644 --- a/fern/pages/java_api/java-api-com-nvidia-cuvs-spi-cuvsprovider.md +++ b/fern/pages/java_api/java-api-com-nvidia-cuvs-spi-cuvsprovider.md @@ -229,14 +229,14 @@ _Source: `java/cuvs-java/src/main/java/com/nvidia/cuvs/spi/CuVSProvider.java:123 HnswIndex hnswIndexBuild(CuVSResources resources, HnswIndexParams hnswParams, CuVSMatrix dataset) throws Throwable ``` -Builds an HNSW index using the ACE (Augmented Core Extraction) algorithm. +Builds an HNSW index from HNSW parameters using GPU graph construction. **Parameters** | Name | Description | | --- | --- | | `resources` | The CuVS resources | -| `hnswParams` | Parameters for the HNSW index with ACE configuration | +| `hnswParams` | Parameters for the HNSW index | | `dataset` | The dataset to build the index from | **Returns** diff --git a/fern/pages/python_api/python-api-neighbors-hnsw.md b/fern/pages/python_api/python-api-neighbors-hnsw.md index a27d58918e..d4578b81ad 100644 --- a/fern/pages/python_api/python-api-neighbors-hnsw.md +++ b/fern/pages/python_api/python-api-neighbors-hnsw.md @@ -87,11 +87,11 @@ Parameters to build index for HNSW nearest neighbor search | Name | Type | Description | | --- | --- | --- | | `hierarchy` | `string, default = "gpu" (optional)` | The hierarchy of the HNSW index.
Valid values are ["none", "cpu", "gpu"].
- "none": No hierarchy is built.
- "cpu": Hierarchy is built using CPU.
- "gpu": Hierarchy is built using GPU. | -| `ef_construction` | `int, default = 200 (optional)` | Maximum number of candidate list size used during construction when hierarchy is `cpu`. | +| `ef_construction` | `int, default = 200 (optional)` | Maximum candidate list size used during index construction. | | `num_threads` | `int, default = 0 (optional)` | Number of CPU threads used to increase construction parallelism when hierarchy is `cpu` or `gpu`. When the value is 0, the number of threads is automatically determined to the maximum number of threads available.
NOTE: When hierarchy is `gpu`, while the majority of the work is done on the GPU, initialization of the HNSW index itself and some other work is parallelized with the help of CPU threads. | -| `M` | `int, default = 32 (optional)` | HNSW M parameter: number of bi-directional links per node (used when building with ACE). graph_degree = m * 2, intermediate_graph_degree = m * 3. | +| `M` | `int, default = 32 (optional)` | HNSW M parameter: number of bi-directional links per node. When the graph is built on the GPU, this parameter is used to derive the internal CAGRA graph build parameters. | | `metric` | `string, default = "sqeuclidean" (optional)` | Distance metric to use.
Valid values: ["sqeuclidean", "inner_product"] | -| `ace_params` | `AceParams, default = None (optional)` | ACE parameters for building HNSW index using ACE algorithm. If set, enables the build() function to use ACE for index construction. | +| `ace_params` | `AceParams, default = None (optional)` | Explicit ACE parameters for out-of-core graph construction. When not set, the graph build algorithm is selected automatically. | **Constructor** @@ -200,20 +200,17 @@ def num_threads(self) def build(IndexParams index_params, dataset, resources=None) ``` -Build an HNSW index using the ACE (Augmented Core Extraction) algorithm. +Build an HNSW index from HNSW parameters. -ACE enables building HNSW indices for datasets too large to fit in GPU -memory by partitioning the dataset and building sub-indices for each -partition independently. - -NOTE: This function requires `index_params.ace_params` to be set with -an instance of AceParams. +The graph is built on the GPU and converted to an HNSW index that can be +searched on the CPU. The graph build algorithm is selected automatically +unless explicit ACE parameters are provided. **Parameters** | Name | Type | Description | | --- | --- | --- | -| `index_params` | `IndexParams` | Parameters for the HNSW index with ACE configuration. Must have `ace_params` set. | +| `index_params` | `IndexParams` | Parameters for the HNSW index. | | `dataset` | `Host array interface compliant matrix shape (n_samples, dim)` | Supported dtype [float32, float16, int8, uint8] | | `resources` | `cuvs.common.Resources, optional` | | @@ -234,17 +231,9 @@ an instance of AceParams. >>> dataset = np.random.random_sample((n_samples, n_features), ... dtype=np.float32) >>> ->>> # Create ACE parameters ->>> ace_params = hnsw.AceParams( -... npartitions=4, -... use_disk=True, -... build_dir="/tmp/hnsw_ace_build" -... ) ->>> ->>> # Create index parameters with ACE +>>> # Create HNSW index parameters >>> index_params = hnsw.IndexParams( ... hierarchy="gpu", -... ace_params=ace_params, ... ef_construction=120, ... M=32, ... metric="sqeuclidean" diff --git a/java/cuvs-java/src/main/java/com/nvidia/cuvs/HnswIndex.java b/java/cuvs-java/src/main/java/com/nvidia/cuvs/HnswIndex.java index 3eef491b62..50216729a8 100644 --- a/java/cuvs-java/src/main/java/com/nvidia/cuvs/HnswIndex.java +++ b/java/cuvs-java/src/main/java/com/nvidia/cuvs/HnswIndex.java @@ -1,5 +1,5 @@ /* - * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION. + * SPDX-FileCopyrightText: Copyright (c) 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. * SPDX-License-Identifier: Apache-2.0 */ package com.nvidia.cuvs; @@ -58,17 +58,12 @@ static HnswIndex fromCagra(HnswIndexParams hnswParams, CagraIndex cagraIndex) th } /** - * Builds an HNSW index using the ACE (Augmented Core Extraction) algorithm. - * - * ACE enables building HNSW indexes for datasets too large to fit in GPU - * memory by partitioning the dataset and building sub-indexes for each - * partition independently. - * - * NOTE: This method requires `hnswParams.getAceParams()` to be set with - * an instance of HnswAceParams. + * Builds an HNSW index from HNSW parameters. The graph is built on the GPU and converted to an + * HNSW index that can be searched on the CPU. The graph build algorithm is selected automatically + * unless explicit ACE parameters are provided. * * @param resources The CuVS resources - * @param hnswParams Parameters for the HNSW index with ACE configuration + * @param hnswParams Parameters for the HNSW index * @param dataset The dataset to build the index from * @return A new HNSW index ready for search * @throws Throwable if an error occurs during building @@ -78,7 +73,6 @@ static HnswIndex build(CuVSResources resources, HnswIndexParams hnswParams, CuVS Objects.requireNonNull(resources); Objects.requireNonNull(hnswParams); Objects.requireNonNull(dataset); - Objects.requireNonNull(hnswParams.getAceParams(), "ACE parameters must be set for build()"); return CuVSProvider.provider().hnswIndexBuild(resources, hnswParams, dataset); } diff --git a/java/cuvs-java/src/main/java/com/nvidia/cuvs/HnswIndexParams.java b/java/cuvs-java/src/main/java/com/nvidia/cuvs/HnswIndexParams.java index 294e21cce5..ea2394a33d 100644 --- a/java/cuvs-java/src/main/java/com/nvidia/cuvs/HnswIndexParams.java +++ b/java/cuvs-java/src/main/java/com/nvidia/cuvs/HnswIndexParams.java @@ -126,8 +126,7 @@ public int getVectorDimension() { /** * Gets the HNSW M parameter: number of bi-directional links per node - * (used when building with ACE). graph_degree = m * 2, - * intermediate_graph_degree = m * 3. + * used to derive the internal graph build parameters for GPU construction. * * @return the M parameter */ @@ -145,7 +144,8 @@ public CuvsDistanceType getMetric() { } /** - * Gets the ACE parameters for building HNSW index using ACE algorithm. + * Gets the optional ACE parameters for explicit out-of-core graph construction. When not set, the + * graph build algorithm is selected automatically. * * @return the ACE parameters, or null if not set */ @@ -206,11 +206,9 @@ public Builder withHierarchy(CuvsHnswHierarchy hierarchy) { } /** - * Sets the size of the candidate list during hierarchy construction when - * hierarchy is `CPU`. + * Sets the maximum candidate list size used during index construction. * - * @param efConstruction the size of the candidate list during hierarchy - * construction when hierarchy is `CPU` + * @param efConstruction the maximum candidate list size used during construction * @return an instance of Builder */ public Builder withEfConstruction(int efConstruction) { @@ -242,9 +240,8 @@ public Builder withVectorDimension(int vectorDimension) { } /** - * Sets the HNSW M parameter: number of bi-directional links per node - * (used when building with ACE). graph_degree = m * 2, - * intermediate_graph_degree = m * 3. + * Sets the HNSW M parameter: number of bi-directional links per node used to derive the internal + * graph build parameters for GPU construction. * * @param m the M parameter * @return an instance of Builder @@ -266,7 +263,8 @@ public Builder withMetric(CuvsDistanceType metric) { } /** - * Sets the ACE parameters for building HNSW index using ACE algorithm. + * Sets optional ACE parameters for explicit out-of-core graph construction. When not set, the + * graph build algorithm is selected automatically. * * @param aceParams the ACE parameters * @return an instance of Builder diff --git a/java/cuvs-java/src/main/java/com/nvidia/cuvs/spi/CuVSProvider.java b/java/cuvs-java/src/main/java/com/nvidia/cuvs/spi/CuVSProvider.java index 05bbc4ab0f..42f9776458 100644 --- a/java/cuvs-java/src/main/java/com/nvidia/cuvs/spi/CuVSProvider.java +++ b/java/cuvs-java/src/main/java/com/nvidia/cuvs/spi/CuVSProvider.java @@ -150,10 +150,10 @@ HnswIndex.Builder newHnswIndexBuilder(CuVSResources cuVSResources) HnswIndex hnswIndexFromCagra(HnswIndexParams hnswParams, CagraIndex cagraIndex) throws Throwable; /** - * Builds an HNSW index using the ACE (Augmented Core Extraction) algorithm. + * Builds an HNSW index from HNSW parameters using GPU graph construction. * * @param resources The CuVS resources - * @param hnswParams Parameters for the HNSW index with ACE configuration + * @param hnswParams Parameters for the HNSW index * @param dataset The dataset to build the index from * @return A new HNSW index ready for search * @throws Throwable if an error occurs during building diff --git a/java/cuvs-java/src/main/java22/com/nvidia/cuvs/internal/HnswIndexImpl.java b/java/cuvs-java/src/main/java22/com/nvidia/cuvs/internal/HnswIndexImpl.java index 58457e0405..6f32e8deab 100644 --- a/java/cuvs-java/src/main/java22/com/nvidia/cuvs/internal/HnswIndexImpl.java +++ b/java/cuvs-java/src/main/java22/com/nvidia/cuvs/internal/HnswIndexImpl.java @@ -243,10 +243,10 @@ public static HnswIndex.Builder newBuilder(CuVSResources cuvsResources) { } /** - * Builds an HNSW index using the ACE algorithm. + * Builds an HNSW index from HNSW parameters using GPU graph construction. * * @param resources The CuVS resources - * @param hnswParams Parameters for the HNSW index with ACE configuration + * @param hnswParams Parameters for the HNSW index * @param dataset The dataset to build the index from * @return A new HNSW index ready for search * @throws Throwable if an error occurs during building @@ -256,7 +256,6 @@ public static HnswIndex build( Objects.requireNonNull(resources); Objects.requireNonNull(hnswParams); Objects.requireNonNull(dataset); - Objects.requireNonNull(hnswParams.getAceParams(), "ACE parameters must be set for build()"); // Create HNSW index MemorySegment hnswIndex = createHnswIndexHandle(); @@ -268,7 +267,7 @@ public static HnswIndex build( MemorySegment hnswParamsMemorySegment = hnswParamsHandle.handle(); - // Link ACE params to HNSW index params + // Link optional ACE params to HNSW index params cuvsHnswIndexParams.ace_params(hnswParamsMemorySegment, aceParamsHandle.handle()); // Prepare dataset tensor @@ -303,6 +302,10 @@ private static CloseableHandle createHnswIndexParamsForBuild( } private static CloseableHandle createHnswAceParams(Arena arena, HnswAceParams aceParams) { + if (aceParams == null) { + return CloseableHandle.NULL; + } + var params = createHnswAceParamsNative(); MemorySegment seg = params.handle(); diff --git a/java/cuvs-java/src/test/java/com/nvidia/cuvs/HnswBuildAndSearchIT.java b/java/cuvs-java/src/test/java/com/nvidia/cuvs/HnswBuildAndSearchIT.java index 3ac5b9dd0e..7cb4f0c3fa 100644 --- a/java/cuvs-java/src/test/java/com/nvidia/cuvs/HnswBuildAndSearchIT.java +++ b/java/cuvs-java/src/test/java/com/nvidia/cuvs/HnswBuildAndSearchIT.java @@ -128,6 +128,38 @@ public void testIndexingAndSearchingFlow() throws Throwable { } } + @Test + public void testBuildWithoutAce() throws Throwable { + List> expectedResults = + Arrays.asList( + Map.of(3, 0.038782578f), + Map.of(0, 0.12472608f), + Map.of(3, 0.047766715f), + Map.of(1, 0.15224178f)); + + HnswIndexParams indexParams = + new HnswIndexParams.Builder() + .withHierarchy(HnswHierarchy.GPU) + .withM(2) + .withEfConstruction(100) + .withMetric(HnswIndexParams.CuvsDistanceType.L2Expanded) + .withVectorDimension(2) + .build(); + + try (CuVSResources resources = CheckedCuVSResources.create(); + CuVSMatrix datasetMatrix = CuVSMatrix.ofArray(dataset); + HnswIndex index = HnswIndex.build(resources, indexParams, datasetMatrix)) { + HnswQuery query = + new HnswQuery.Builder(resources) + .withQueryVectors(queries) + .withSearchParams(new HnswSearchParams.Builder().withEF(100).build()) + .withTopK(1) + .build(); + + checkResults(expectedResults, index.search(query).getResults()); + } + } + @Test public void testIndexingAndSearchingWithFunctionMapping() throws Throwable { // Expected search results diff --git a/python/cuvs/cuvs/neighbors/hnsw/hnsw.pyx b/python/cuvs/cuvs/neighbors/hnsw/hnsw.pyx index 6757695de3..91256a0e18 100644 --- a/python/cuvs/cuvs/neighbors/hnsw/hnsw.pyx +++ b/python/cuvs/cuvs/neighbors/hnsw/hnsw.pyx @@ -1,5 +1,5 @@ # -# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # cython: language_level=3 @@ -129,8 +129,7 @@ cdef class IndexParams: - "cpu": Hierarchy is built using CPU. - "gpu": Hierarchy is built using GPU. ef_construction : int, default = 200 (optional) - Maximum number of candidate list size used during construction - when hierarchy is `cpu`. + Maximum candidate list size used during index construction. num_threads : int, default = 0 (optional) Number of CPU threads used to increase construction parallelism when hierarchy is `cpu` or `gpu`. When the value is 0, the number of @@ -140,14 +139,14 @@ cdef class IndexParams: on the GPU, initialization of the HNSW index itself and some other work is parallelized with the help of CPU threads. M : int, default = 32 (optional) - HNSW M parameter: number of bi-directional links per node - (used when building with ACE). graph_degree = m * 2, - intermediate_graph_degree = m * 3. + HNSW M parameter: number of bi-directional links per node. When the + graph is built on the GPU, this parameter is used to derive the + internal CAGRA graph build parameters. metric : string, default = "sqeuclidean" (optional) Distance metric to use. Valid values: ["sqeuclidean", "inner_product"] ace_params : AceParams, default = None (optional) - ACE parameters for building HNSW index using ACE algorithm. If set, - enables the build() function to use ACE for index construction. + Explicit ACE parameters for out-of-core graph construction. When not + set, the graph build algorithm is selected automatically. """ cdef cuvsHnswIndexParams* params @@ -471,20 +470,16 @@ def from_cagra(IndexParams index_params, cagra.Index cagra_index, @auto_sync_resources def build(IndexParams index_params, dataset, resources=None): """ - Build an HNSW index using the ACE (Augmented Core Extraction) algorithm. + Build an HNSW index from HNSW parameters. - ACE enables building HNSW indices for datasets too large to fit in GPU - memory by partitioning the dataset and building sub-indices for each - partition independently. - - NOTE: This function requires `index_params.ace_params` to be set with - an instance of AceParams. + The graph is built on the GPU and converted to an HNSW index that can be + searched on the CPU. The graph build algorithm is selected automatically + unless explicit ACE parameters are provided. Parameters ---------- index_params : IndexParams - Parameters for the HNSW index with ACE configuration. - Must have `ace_params` set. + Parameters for the HNSW index. dataset : Host array interface compliant matrix shape (n_samples, dim) Supported dtype [float32, float16, int8, uint8] {resources_docstring} @@ -504,17 +499,9 @@ def build(IndexParams index_params, dataset, resources=None): >>> dataset = np.random.random_sample((n_samples, n_features), ... dtype=np.float32) >>> - >>> # Create ACE parameters - >>> ace_params = hnsw.AceParams( - ... npartitions=4, - ... use_disk=True, - ... build_dir="/tmp/hnsw_ace_build" - ... ) - >>> - >>> # Create index parameters with ACE + >>> # Create HNSW index parameters >>> index_params = hnsw.IndexParams( ... hierarchy="gpu", - ... ace_params=ace_params, ... ef_construction=120, ... M=32, ... metric="sqeuclidean" @@ -532,10 +519,6 @@ def build(IndexParams index_params, dataset, resources=None): ... k=10 ... ) """ - if index_params.ace_params is None: - raise ValueError("index_params.ace_params must be set for hnsw.build(). " - "Use AceParams to configure ACE algorithm parameters.") - dataset_ai = wrap_array(dataset) _check_input_array(dataset_ai, [np.dtype('float32'), np.dtype('float16'), diff --git a/python/cuvs/cuvs/tests/test_hnsw.py b/python/cuvs/cuvs/tests/test_hnsw.py index a5fb71f48b..86e12e96f5 100644 --- a/python/cuvs/cuvs/tests/test_hnsw.py +++ b/python/cuvs/cuvs/tests/test_hnsw.py @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # @@ -91,6 +91,29 @@ def test_hnsw(dtype, k, ef, num_threads, metric, build_algo, hierarchy): ) +def test_hnsw_build_hnsw_first_api(): + dataset = generate_data((2000, 16), np.float32) + queries = generate_data((100, 16), np.float32) + k = 10 + + index_params = hnsw.IndexParams( + hierarchy="gpu", + M=16, + ef_construction=120, + metric="sqeuclidean", + ) + index = hnsw.build(index_params, dataset) + + _, neighbors = hnsw.search(hnsw.SearchParams(ef=100), index, queries, k) + + reference = NearestNeighbors( + n_neighbors=k, algorithm="brute", metric="sqeuclidean" + ).fit(dataset) + _, reference_neighbors = reference.kneighbors(queries) + + assert calc_recall(neighbors, reference_neighbors) >= 0.9 + + def run_hnsw_extend_test( n_rows=10000, add_rows=2000,