feature: Per index logger and new logger unit test for each index (#87)

This PR intends to add per index logging feature

Author: yuejiaointel

include/svs/index/flat/flat.h

@@ -17,6 +17,7 @@
 #pragma once
 
 // Flat index utilities
+#include "svs/core/logging.h"
 #include "svs/index/flat/inserters.h"
 #include "svs/index/index.h"
 
@@ -145,6 +146,8 @@ class FlatIndex {
     data_storage_type data_;
     [[no_unique_address]] distance_type distance_;
     threads::ThreadPoolHandle threadpool_;
+    // SVS logger for per index logging
+    svs::logging::logger_ptr logger_;
 
     // Constructs controlling the iteration strategy over the data and queries.
     search_parameters_type search_parameters_{};
@@ -171,6 +174,9 @@ class FlatIndex {
     }
 
   public:
+    /// @brief Getter method for logger
+    svs::logging::logger_ptr get_logger() const { return logger_; }
+
     search_parameters_type get_search_parameters() const { return search_parameters_; }
 
     void set_search_parameters(const search_parameters_type& search_parameters) {
@@ -189,22 +195,35 @@ class FlatIndex {
     ///     instance or an integer specifying the number of threads to use. In the latter
     ///     case, a new default thread pool will be constructed using ``threadpool_proto``
     ///     as the number of threads to create.
+    /// @param logger_ Spd logger for per-index logging customization.
     ///
     /// @copydoc threadpool_requirements
     ///
     template <typename ThreadPoolProto>
-    FlatIndex(Data data, Dist distance, ThreadPoolProto threadpool_proto)
+    FlatIndex(
+        Data data,
+        Dist distance,
+        ThreadPoolProto threadpool_proto,
+        svs::logging::logger_ptr logger = svs::logging::get()
+    )
         requires std::is_same_v<Ownership, OwnsMembers>
         : data_{std::move(data)}
         , distance_{std::move(distance)}
-        , threadpool_{threads::as_threadpool(std::move(threadpool_proto))} {}
+        , threadpool_{threads::as_threadpool(std::move(threadpool_proto))}
+        , logger_{std::move(logger)} {}
 
     template <typename ThreadPoolProto>
-    FlatIndex(Data& data, Dist distance, ThreadPoolProto threadpool_proto)
+    FlatIndex(
+        Data& data,
+        Dist distance,
+        ThreadPoolProto threadpool_proto,
+        svs::logging::logger_ptr logger = svs::logging::get()
+    )
         requires std::is_same_v<Ownership, ReferencesMembers>
         : data_{data}
         , distance_{std::move(distance)}
-        , threadpool_{threads::as_threadpool(std::move(threadpool_proto))} {}
+        , threadpool_{threads::as_threadpool(std::move(threadpool_proto))}
+        , logger_{std::move(logger)} {}
 
     ////// Dataset Interface
 
@@ -462,6 +481,7 @@ class FlatIndex {
 ///     instance or an integer specifying the number of threads to use. In the latter case,
 ///     a new default thread pool will be constructed using ``threadpool_proto`` as the
 ///     number of threads to create.
+/// @param logger_ Spd logger for per-index logging customization.
 ///
 /// This method provides much of the heavy lifting for constructing a Flat index from
 /// a data file on disk or a dataset in memory.
@@ -472,11 +492,16 @@ class FlatIndex {
 ///
 template <typename DataProto, typename Distance, typename ThreadPoolProto>
 auto auto_assemble(
-    DataProto&& data_proto, Distance distance, ThreadPoolProto threadpool_proto
+    DataProto&& data_proto,
+    Distance distance,
+    ThreadPoolProto threadpool_proto,
+    svs::logging::logger_ptr logger = svs::logging::get()
 ) {
     auto threadpool = threads::as_threadpool(std::move(threadpool_proto));
     auto data = svs::detail::dispatch_load(std::forward<DataProto>(data_proto), threadpool);
-    return FlatIndex(std::move(data), std::move(distance), std::move(threadpool));
+    return FlatIndex(
+        std::move(data), std::move(distance), std::move(threadpool), std::move(logger)
+    );
 }
 
 /// @brief Alias for a short-lived flat index.

include/svs/index/inverted/clustering.h

@@ -801,7 +801,8 @@ Clustering<I> cluster_with(
     const Data& data,
     std::span<const I> centroid_ids,
     const ClusteringParameters& params,
-    Index& primary_index
+    Index& primary_index,
+    svs::logging::logger_ptr logger = svs::logging::get()
 ) {
     for (auto id : centroid_ids) {
         if (id >= data.size()) {
@@ -820,7 +821,6 @@ Clustering<I> cluster_with(
     size_t start = 0;
     size_t datasize = data.size();
     auto timer = lib::Timer();
-    auto logger = svs::logging::get();
 
     while (start < datasize) {
         size_t stop = std::min(start + batchsize, datasize);

include/svs/index/inverted/memory_based.h

@@ -339,12 +339,17 @@ template <typename Index, typename Cluster> class InvertedIndex {
 
     template <threads::ThreadPool Pool>
     InvertedIndex(
-        Index index, Cluster cluster, translator_type index_local_to_global, Pool threadpool
+        Index index,
+        Cluster cluster,
+        translator_type index_local_to_global,
+        Pool threadpool,
+        svs::logging::logger_ptr logger = svs::logging::get()
     )
         : index_{std::move(index)}
         , cluster_{std::move(cluster)}
         , index_local_to_global_{std::move(index_local_to_global)}
-        , threadpool_{std::move(threadpool)} {
+        , threadpool_{std::move(threadpool)}
+        , logger_{std::move(logger)} {
         // Clear out the threadpool in the inner index - prefer to handle threading
         // ourselves.
         index_.set_threadpool(threads::SequentialThreadPool());
@@ -492,6 +497,10 @@ template <typename Index, typename Cluster> class InvertedIndex {
         index_.save(index_config, graph, data);
     }
 
+    ///// Accessors
+    /// @brief Getter method for logger
+    svs::logging::logger_ptr get_logger() const { return logger_; }
+
   private:
     // Tunable Parameters
     double refinement_epsilon_ = 10.0;
@@ -503,6 +512,9 @@ template <typename Index, typename Cluster> class InvertedIndex {
 
     // Transient parameters.
     threads::ThreadPoolHandle threadpool_;
+
+    // SVS logger for per index logging
+    svs::logging::logger_ptr logger_;
 };
 
 struct PickRandomly {
@@ -548,7 +560,8 @@ auto auto_build(
     // Customizations
     Strategy strategy = {},
     CentroidPicker centroid_picker = {},
-    ClusteringOp clustering_op = {}
+    ClusteringOp clustering_op = {},
+    svs::logging::logger_ptr logger = svs::logging::get()
 ) {
     // Perform clustering.
     auto threadpool = threads::as_threadpool(std::move(threadpool_proto));
@@ -569,7 +582,11 @@ auto auto_build(
 
     // Cluster the dataset with the help of the primary index.
     auto clustering = cluster_with(
-        data, lib::as_const_span(centroids), parameters.clustering_parameters_, index
+        data,
+        lib::as_const_span(centroids),
+        parameters.clustering_parameters_,
+        index,
+        logger
     );
 
     // Perform any post-proceseccing on the clustering.
@@ -585,7 +602,8 @@ auto auto_build(
         std::move(index),
         strategy(data, clustering, HugepageAllocator<std::byte>()),
         std::move(centroids),
-        std::move(primary_threadpool)};
+        std::move(primary_threadpool),
+        std::move(logger)};
 }
 
 ///// Auto Assembling.
@@ -601,7 +619,8 @@ auto assemble_from_clustering(
     Strategy strategy,
     const std::filesystem::path& index_config,
     const std::filesystem::path& graph,
-    ThreadPoolProto threadpool_proto
+    ThreadPoolProto threadpool_proto,
+    svs::logging::logger_ptr logger = svs::logging::get()
 ) {
     auto threadpool = threads::as_threadpool(std::move(threadpool_proto));
     auto original = svs::detail::dispatch_load(std::move(data_proto), threadpool);
@@ -621,15 +640,17 @@ auto assemble_from_clustering(
             return local_data;
         }),
         distance,
-        1
+        1,
+        logger
     );
 
     // Create the clustering and return the final results.
     return InvertedIndex(
         std::move(index),
         strategy(original, clustering, HugepageAllocator<std::byte>()),
         std::move(ids),
-        std::move(threadpool)
+        std::move(threadpool),
+        std::move(logger)
     );
 }
 

include/svs/index/vamana/calibrate.h

@@ -176,9 +176,9 @@ VamanaSearchParameters optimize_split_buffer(
     double target_recall,
     VamanaSearchParameters current,
     const F& compute_recall,
-    const DoSearch& do_search
+    const DoSearch& do_search,
+    svs::logging::logger_ptr logger = svs::logging::get()
 ) {
-    auto logger = svs::logging::get();
     svs::logging::trace(logger, "Entering split buffer optimization routine");
     assert(
         current.buffer_config_.get_search_window_size() ==
@@ -252,11 +252,11 @@ std::pair<VamanaSearchParameters, bool> optimize_search_buffer(
     size_t num_neighbors,
     double target_recall,
     const ComputeRecall& compute_recall,
-    const DoSearch& do_search
+    const DoSearch& do_search,
+    svs::logging::logger_ptr logger = svs::logging::get()
 ) {
     using enum CalibrationParameters::SearchBufferOptimization;
     using dataset_type = typename Index::data_type;
-    auto logger = svs::logging::get();
 
     double max_recall = std::numeric_limits<double>::lowest();
     const size_t current_capacity = current.buffer_config_.get_total_capacity();
@@ -345,9 +345,9 @@ VamanaSearchParameters tune_prefetch(
     const CalibrationParameters& calibration_parameters,
     Index& index,
     VamanaSearchParameters search_parameters,
-    const DoSearch& do_search
+    const DoSearch& do_search,
+    svs::logging::logger_ptr logger = svs::logging::get()
 ) {
-    auto logger = svs::logging::get();
     svs::logging::trace(logger, "Tuning prefetch parameters");
     const auto& prefetch_steps = calibration_parameters.prefetch_steps_;
     size_t max_lookahead = index.max_degree();

include/svs/index/vamana/dynamic_index.h

@@ -157,6 +157,9 @@ class MutableVamanaIndex {
     float alpha_ = 1.2;
     bool use_full_search_history_ = true;
 
+    // SVS logger for per index logging
+    svs::logging::logger_ptr logger_;
+
     // Methods
   public:
     // Constructors
@@ -167,7 +170,9 @@ class MutableVamanaIndex {
         Idx entry_point,
         Dist distance_function,
         const ExternalIds& external_ids,
-        ThreadPoolProto threadpool_proto
+        ThreadPoolProto threadpool_proto,
+        // Optional logger parameter
+        svs::logging::logger_ptr logger = svs::logging::get()
     )
         : graph_{std::move(graph)}
         , data_{std::move(data)}
@@ -178,7 +183,9 @@ class MutableVamanaIndex {
         , distance_{std::move(distance_function)}
         , threadpool_{threads::as_threadpool(std::move(threadpool_proto))}
         , search_parameters_{vamana::construct_default_search_parameters(data_)}
-        , construction_window_size_{2 * graph.max_degree()} {
+        , construction_window_size_{2 * graph.max_degree()}
+        // Ctor accept logger in parameter
+        , logger_{std::move(logger)} {
         translator_.insert(external_ids, threads::UnitRange<Idx>(0, external_ids.size()));
     }
 
@@ -191,7 +198,8 @@ class MutableVamanaIndex {
         Data data,
         const ExternalIds& external_ids,
         Dist distance_function,
-        ThreadPoolProto threadpool_proto
+        ThreadPoolProto threadpool_proto,
+        svs::logging::logger_ptr logger = svs::logging::get()
     )
         : graph_(Graph{data.size(), parameters.graph_max_degree})
         , data_(std::move(data))
@@ -206,7 +214,8 @@ class MutableVamanaIndex {
         , max_candidates_(parameters.max_candidate_pool_size)
         , prune_to_(parameters.prune_to)
         , alpha_(parameters.alpha)
-        , use_full_search_history_{parameters.use_full_search_history} {
+        , use_full_search_history_{parameters.use_full_search_history}
+        , logger_{std::move(logger)} {
         // Setup the initial translation of external to internal ids.
         translator_.insert(external_ids, threads::UnitRange<Idx>(0, external_ids.size()));
 
@@ -220,8 +229,8 @@ class MutableVamanaIndex {
         auto builder = VamanaBuilder(
             graph_, data_, distance_, parameters, threadpool_, prefetch_parameters
         );
-        builder.construct(1.0f, entry_point_[0]);
-        builder.construct(parameters.alpha, entry_point_[0]);
+        builder.construct(1.0f, entry_point_[0], logging::Level::Info, logger_);
+        builder.construct(parameters.alpha, entry_point_[0], logging::Level::Info, logger_);
     }
 
     /// @brief Post re-load constructor.
@@ -240,7 +249,8 @@ class MutableVamanaIndex {
         graph_type graph,
         const Dist& distance_function,
         IDTranslator translator,
-        Pool threadpool
+        Pool threadpool,
+        svs::logging::logger_ptr logger = svs::logging::get()
     )
         : graph_{std::move(graph)}
         , data_{std::move(data)}
@@ -255,7 +265,8 @@ class MutableVamanaIndex {
         , max_candidates_{config.build_parameters.max_candidate_pool_size}
         , prune_to_{config.build_parameters.prune_to}
         , alpha_{config.build_parameters.alpha}
-        , use_full_search_history_{config.build_parameters.use_full_search_history} {}
+        , use_full_search_history_{config.build_parameters.use_full_search_history}
+        , logger_{std::move(logger)} {}
 
     ///// Scratchspace
     scratchspace_type scratchspace(const search_parameters_type& sp) const {
@@ -272,6 +283,8 @@ class MutableVamanaIndex {
     scratchspace_type scratchspace() const { return scratchspace(get_search_parameters()); }
 
     ///// Accessors
+    /// @brief Getter method for logger
+    svs::logging::logger_ptr get_logger() const { return logger_; }
 
     /// @brief Get the alpha value used for pruning while mutating the graph.
     float get_alpha() const { return alpha_; }
@@ -1200,6 +1213,17 @@ template <typename Data, typename Dist, typename ExternalIds, threads::ThreadPoo
 MutableVamanaIndex(const VamanaBuildParameters&, Data, const ExternalIds&, Dist, Pool)
     -> MutableVamanaIndex<graphs::SimpleBlockedGraph<uint32_t>, Data, Dist>;
 
+// Guide with logging
+template <typename Data, typename Dist, typename ExternalIds, threads::ThreadPool Pool>
+MutableVamanaIndex(
+    const VamanaBuildParameters&,
+    Data,
+    const ExternalIds&,
+    Dist,
+    Pool,
+    svs::logging::logger_ptr
+) -> MutableVamanaIndex<graphs::SimpleBlockedGraph<uint32_t>, Data, Dist>;
+
 namespace detail {
 
 struct VamanaStateLoader {
@@ -1251,7 +1275,8 @@ auto auto_dynamic_assemble(
     // to easily benchmark the static versus dynamic implementation.
     //
     // This is an internal API and should not be considered officially supported nor stable.
-    bool debug_load_from_static = false
+    bool debug_load_from_static = false,
+    svs::logging::logger_ptr logger = svs::logging::get()
 ) {
     // Load the dataset
     auto threadpool = threads::as_threadpool(std::move(threadpool_proto));
@@ -1317,7 +1342,8 @@ auto auto_dynamic_assemble(
         std::move(graph),
         std::move(distance),
         std::move(translator),
-        std::move(threadpool)};
+        std::move(threadpool),
+        std::move(logger)};
 }
 
 } // namespace svs::index::vamana