From 81506a69fd2069e5e40ca071a8d5cbef13336423 Mon Sep 17 00:00:00 2001 From: alon Date: Sat, 31 Jan 2026 13:54:03 +0200 Subject: [PATCH 1/3] add tiered metrics - wip --- src/VecSim/algorithms/hnsw/hnsw_tiered.h | 17 ++++++++ src/VecSim/vec_sim_common.h | 3 ++ src/VecSim/vec_sim_index.h | 2 + src/VecSim/vec_sim_tiered_index.h | 2 + tests/unit/test_hnsw_tiered.cpp | 53 ++++++++++++++++++++++++ 5 files changed, 77 insertions(+) diff --git a/src/VecSim/algorithms/hnsw/hnsw_tiered.h b/src/VecSim/algorithms/hnsw/hnsw_tiered.h index f9d94dc52..a4d5e08e4 100644 --- a/src/VecSim/algorithms/hnsw/hnsw_tiered.h +++ b/src/VecSim/algorithms/hnsw/hnsw_tiered.h @@ -94,6 +94,11 @@ class TieredHNSWIndex : public VecSimTieredIndex { // associated swap jobs. std::mutex idToRepairJobsGuard; + // Counter for vectors inserted directly into HNSW by the main thread (bypassing flat buffer). + // This happens in WriteInPlace mode or when the flat buffer is full. + // Not atomic since it's only accessed from the main thread. + size_t directHNSWInsertions{0}; + void executeInsertJob(HNSWInsertJob *job); void executeRepairJob(HNSWRepairJob *job); @@ -211,6 +216,7 @@ class TieredHNSWIndex : public VecSimTieredIndex { // needed. VecSimIndexDebugInfo debugInfo() const override; VecSimIndexBasicInfo basicInfo() const override; + VecSimIndexStatsInfo statisticInfo() const override; VecSimDebugInfoIterator *debugInfoIterator() const override; VecSimBatchIterator *newBatchIterator(const void *queryBlob, VecSimQueryParams *queryParams) const override { @@ -729,6 +735,8 @@ int TieredHNSWIndex::addVector(const void *blob, labelType l this->lockMainIndexGuard(); hnsw_index->addVector(storage_blob.get(), label); this->unlockMainIndexGuard(); + // Track direct insertion to HNSW (bypassing flat buffer) + ++this->directHNSWInsertions; return ret; } if (this->frontendIndex->indexSize() >= this->flatBufferLimit) { @@ -746,6 +754,8 @@ int TieredHNSWIndex::addVector(const void *blob, labelType l // index. auto storage_blob = this->frontendIndex->preprocessForStorage(blob); this->insertVectorToHNSW(hnsw_index, label, storage_blob.get()); + // Track direct insertion to HNSW (flat buffer was full) + ++this->directHNSWInsertions; return ret; } // Otherwise, we fall back to the "regular" insertion into the flat buffer @@ -1151,6 +1161,13 @@ void TieredHNSWIndex::TieredHNSW_BatchIterator::filter_irrel results.resize(cur_end - results.begin()); } +template +VecSimIndexStatsInfo TieredHNSWIndex::statisticInfo() const { + auto stats = VecSimTieredIndex::statisticInfo(); + stats.directHNSWInsertions = this->directHNSWInsertions; + return stats; +} + template VecSimIndexDebugInfo TieredHNSWIndex::debugInfo() const { auto info = VecSimTieredIndex::debugInfo(); diff --git a/src/VecSim/vec_sim_common.h b/src/VecSim/vec_sim_common.h index fa136b7fe..945053f45 100644 --- a/src/VecSim/vec_sim_common.h +++ b/src/VecSim/vec_sim_common.h @@ -342,6 +342,9 @@ typedef struct { size_t memory; size_t numberOfMarkedDeleted; // The number of vectors that are marked as deleted (HNSW/tiered // only). + size_t directHNSWInsertions; // Count of vectors inserted directly into HNSW by main thread + // (bypassing flat buffer). Tiered HNSW only. + size_t flatBufferSize; // Current flat buffer size. Tiered indexes only. } VecSimIndexStatsInfo; typedef struct { diff --git a/src/VecSim/vec_sim_index.h b/src/VecSim/vec_sim_index.h index 017935ce9..88eabea69 100644 --- a/src/VecSim/vec_sim_index.h +++ b/src/VecSim/vec_sim_index.h @@ -198,6 +198,8 @@ struct VecSimIndexAbstract : public VecSimIndexInterface { return VecSimIndexStatsInfo{ .memory = this->getAllocationSize(), .numberOfMarkedDeleted = 0, + .directHNSWInsertions = 0, + .flatBufferSize = 0, }; } diff --git a/src/VecSim/vec_sim_tiered_index.h b/src/VecSim/vec_sim_tiered_index.h index 0a36b1f86..4517cf5ca 100644 --- a/src/VecSim/vec_sim_tiered_index.h +++ b/src/VecSim/vec_sim_tiered_index.h @@ -320,6 +320,8 @@ VecSimIndexStatsInfo VecSimTieredIndex::statisticInfo() cons auto stats = VecSimIndexStatsInfo{ .memory = this->getAllocationSize(), .numberOfMarkedDeleted = this->getNumMarkedDeleted(), + .directHNSWInsertions = 0, // Base tiered index returns 0; TieredHNSWIndex overrides + .flatBufferSize = this->frontendIndex->indexSize(), }; return stats; diff --git a/tests/unit/test_hnsw_tiered.cpp b/tests/unit/test_hnsw_tiered.cpp index 04b1f18b9..c74de8bc7 100644 --- a/tests/unit/test_hnsw_tiered.cpp +++ b/tests/unit/test_hnsw_tiered.cpp @@ -2761,6 +2761,9 @@ TYPED_TEST(HNSWTieredIndexTest, testInfo) { EXPECT_EQ(info.tieredInfo.backgroundIndexing, false); EXPECT_EQ(info.tieredInfo.bufferLimit, 1000); EXPECT_EQ(info.tieredInfo.specificTieredBackendInfo.hnswTieredInfo.pendingSwapJobsThreshold, 1); + // Verify new tiered-specific stats + EXPECT_EQ(stats.flatBufferSize, 0); + EXPECT_EQ(stats.directHNSWInsertions, 0); // Validate that Static info returns the right restricted info as well. VecSimIndexBasicInfo s_info = VecSimIndex_BasicInfo(tiered_index); @@ -2787,6 +2790,9 @@ TYPED_TEST(HNSWTieredIndexTest, testInfo) { info.tieredInfo.frontendCommonInfo.memory); EXPECT_EQ(info.commonInfo.memory, stats.memory); EXPECT_EQ(info.tieredInfo.backgroundIndexing, true); + // Vector is in flat buffer, no direct insertions yet + EXPECT_EQ(stats.flatBufferSize, 1); + EXPECT_EQ(stats.directHNSWInsertions, 0); mock_thread_pool.thread_iteration(); info = tiered_index->debugInfo(); @@ -2803,6 +2809,9 @@ TYPED_TEST(HNSWTieredIndexTest, testInfo) { info.tieredInfo.frontendCommonInfo.memory); EXPECT_EQ(info.commonInfo.memory, stats.memory); EXPECT_EQ(info.tieredInfo.backgroundIndexing, false); + // Vector moved from flat buffer to HNSW by background thread + EXPECT_EQ(stats.flatBufferSize, 0); + EXPECT_EQ(stats.directHNSWInsertions, 0); if (TypeParam::isMulti()) { GenerateAndAddVector(tiered_index, dim, 1, 1); @@ -2839,6 +2848,50 @@ TYPED_TEST(HNSWTieredIndexTest, testInfo) { EXPECT_EQ(info.tieredInfo.backgroundIndexing, false); } +TYPED_TEST(HNSWTieredIndexTest, testDirectHNSWInsertionsStats) { + // Test that directHNSWInsertions counter is incremented when flat buffer is full. + size_t dim = 4; + size_t buffer_limit = 5; + HNSWParams params = {.type = TypeParam::get_index_type(), + .dim = dim, + .metric = VecSimMetric_L2, + .multi = TypeParam::isMulti()}; + VecSimParams hnsw_params = CreateParams(params); + auto mock_thread_pool = tieredIndexMock(); + + // Create index with small buffer limit + auto *tiered_index = + this->CreateTieredHNSWIndex(hnsw_params, mock_thread_pool, 1, buffer_limit); + + // Fill the flat buffer + for (size_t i = 0; i < buffer_limit; i++) { + GenerateAndAddVector(tiered_index, dim, i, i); + } + + VecSimIndexStatsInfo stats = tiered_index->statisticInfo(); + EXPECT_EQ(stats.flatBufferSize, buffer_limit); + EXPECT_EQ(stats.directHNSWInsertions, 0); + + // Add more vectors - these should go directly to HNSW + size_t extra_vectors = 3; + for (size_t i = buffer_limit; i < buffer_limit + extra_vectors; i++) { + GenerateAndAddVector(tiered_index, dim, i, i); + } + + stats = tiered_index->statisticInfo(); + EXPECT_EQ(stats.flatBufferSize, buffer_limit); + EXPECT_EQ(stats.directHNSWInsertions, extra_vectors); + + // Drain the flat buffer by starting threads and waiting for them to finish + mock_thread_pool.init_threads(); + mock_thread_pool.thread_pool_join(); + + stats = tiered_index->statisticInfo(); + EXPECT_EQ(stats.flatBufferSize, 0); + // Direct insertions counter should be preserved + EXPECT_EQ(stats.directHNSWInsertions, extra_vectors); +} + TYPED_TEST(HNSWTieredIndexTest, testInfoIterator) { // Create TieredHNSW index instance with a mock queue. size_t dim = 4; From 69190676d8e07c002601f2f2b961a1cb95d83ed2 Mon Sep 17 00:00:00 2001 From: alon Date: Sun, 1 Feb 2026 09:45:35 +0200 Subject: [PATCH 2/3] format --- src/VecSim/vec_sim_tiered_index.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/VecSim/vec_sim_tiered_index.h b/src/VecSim/vec_sim_tiered_index.h index 4517cf5ca..61294b90b 100644 --- a/src/VecSim/vec_sim_tiered_index.h +++ b/src/VecSim/vec_sim_tiered_index.h @@ -320,7 +320,7 @@ VecSimIndexStatsInfo VecSimTieredIndex::statisticInfo() cons auto stats = VecSimIndexStatsInfo{ .memory = this->getAllocationSize(), .numberOfMarkedDeleted = this->getNumMarkedDeleted(), - .directHNSWInsertions = 0, // Base tiered index returns 0; TieredHNSWIndex overrides + .directHNSWInsertions = 0, // Base tiered index returns 0; TieredHNSWIndex overrides .flatBufferSize = this->frontendIndex->indexSize(), }; From 4ff06e89605ed319a7497e1934734c2b411b2bfc Mon Sep 17 00:00:00 2001 From: alon Date: Mon, 2 Feb 2026 11:33:47 +0200 Subject: [PATCH 3/3] add a test with write in place --- tests/unit/test_hnsw_tiered.cpp | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tests/unit/test_hnsw_tiered.cpp b/tests/unit/test_hnsw_tiered.cpp index c74de8bc7..ffcac02ea 100644 --- a/tests/unit/test_hnsw_tiered.cpp +++ b/tests/unit/test_hnsw_tiered.cpp @@ -2890,6 +2890,30 @@ TYPED_TEST(HNSWTieredIndexTest, testDirectHNSWInsertionsStats) { EXPECT_EQ(stats.flatBufferSize, 0); // Direct insertions counter should be preserved EXPECT_EQ(stats.directHNSWInsertions, extra_vectors); + + // Test write-in-place mode: vectors should go directly to HNSW even when buffer is not full + VecSim_SetWriteMode(VecSim_WriteInPlace); + + size_t write_in_place_vectors = 4; + size_t label_offset = buffer_limit + extra_vectors; + for (size_t i = 0; i < write_in_place_vectors; i++) { + GenerateAndAddVector(tiered_index, dim, label_offset + i, label_offset + i); + } + + stats = tiered_index->statisticInfo(); + // Flat buffer should still be empty (vectors went directly to HNSW) + EXPECT_EQ(stats.flatBufferSize, 0); + // Direct insertions counter should include the write-in-place vectors + EXPECT_EQ(stats.directHNSWInsertions, extra_vectors + write_in_place_vectors); + + // Verify all vectors are in the backend index + VecSimIndexDebugInfo info = tiered_index->debugInfo(); + EXPECT_EQ(info.tieredInfo.backendCommonInfo.indexSize, + buffer_limit + extra_vectors + write_in_place_vectors); + EXPECT_EQ(info.tieredInfo.frontendCommonInfo.indexSize, 0); + + // Reset to async mode + VecSim_SetWriteMode(VecSim_WriteAsync); } TYPED_TEST(HNSWTieredIndexTest, testInfoIterator) {