[net] Move asmap into NetGroupManager

Author: jnewbery

Diff for: src/addrdb.cpp

@@ -13,6 +13,7 @@
 #include <hash.h>
 #include <logging/timer.h>
 #include <netbase.h>
+#include <netgroup.h>
 #include <random.h>
 #include <streams.h>
 #include <tinyformat.h>
@@ -182,10 +183,10 @@ void ReadFromStream(AddrMan& addr, CDataStream& ssPeers)
     DeserializeDB(ssPeers, addr, false);
 }
 
-std::optional<bilingual_str> LoadAddrman(const std::vector<bool>& asmap, const ArgsManager& args, std::unique_ptr<AddrMan>& addrman)
+std::optional<bilingual_str> LoadAddrman(const NetGroupManager& netgroupman, const ArgsManager& args, std::unique_ptr<AddrMan>& addrman)
 {
     auto check_addrman = std::clamp<int32_t>(args.GetIntArg("-checkaddrman", DEFAULT_ADDRMAN_CONSISTENCY_CHECKS), 0, 1000000);
-    addrman = std::make_unique<AddrMan>(asmap, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman);
+    addrman = std::make_unique<AddrMan>(netgroupman, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman);
 
     int64_t nStart = GetTimeMillis();
     const auto path_addr{args.GetDataDirNet() / "peers.dat"};
@@ -194,7 +195,7 @@ std::optional<bilingual_str> LoadAddrman(const std::vector<bool>& asmap, const A
         LogPrintf("Loaded %i addresses from peers.dat  %dms\n", addrman->size(), GetTimeMillis() - nStart);
     } catch (const DbNotFoundError&) {
         // Addrman can be in an inconsistent state after failure, reset it
-        addrman = std::make_unique<AddrMan>(asmap, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman);
+        addrman = std::make_unique<AddrMan>(netgroupman, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman);
         LogPrintf("Creating peers.dat because the file was not found (%s)\n", fs::quoted(fs::PathToString(path_addr)));
         DumpPeerAddresses(args, *addrman);
     } catch (const InvalidAddrManVersionError&) {
@@ -203,7 +204,7 @@ std::optional<bilingual_str> LoadAddrman(const std::vector<bool>& asmap, const A
             return strprintf(_("Failed to rename invalid peers.dat file. Please move or delete it and try again."));
         }
         // Addrman can be in an inconsistent state after failure, reset it
-        addrman = std::make_unique<AddrMan>(asmap, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman);
+        addrman = std::make_unique<AddrMan>(netgroupman, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman);
         LogPrintf("Creating new peers.dat because the file version was not compatible (%s). Original backed up to peers.dat.bak\n", fs::quoted(fs::PathToString(path_addr)));
         DumpPeerAddresses(args, *addrman);
     } catch (const std::exception& e) {

Diff for: src/addrdb.h

@@ -17,6 +17,7 @@ class ArgsManager;
 class AddrMan;
 class CAddress;
 class CDataStream;
+class NetGroupManager;
 struct bilingual_str;
 
 bool DumpPeerAddresses(const ArgsManager& args, const AddrMan& addr);
@@ -48,7 +49,7 @@ class CBanDB
 };
 
 /** Returns an error string on failure */
-std::optional<bilingual_str> LoadAddrman(const std::vector<bool>& asmap, const ArgsManager& args, std::unique_ptr<AddrMan>& addrman);
+std::optional<bilingual_str> LoadAddrman(const NetGroupManager& netgroupman, const ArgsManager& args, std::unique_ptr<AddrMan>& addrman);
 
 /**
  * Dump the anchor IP address database (anchors.dat)

Diff for: src/addrman.cpp

@@ -99,11 +99,11 @@ double AddrInfo::GetChance(int64_t nNow) const
     return fChance;
 }
 
-AddrManImpl::AddrManImpl(std::vector<bool>&& asmap, bool deterministic, int32_t consistency_check_ratio)
+AddrManImpl::AddrManImpl(const NetGroupManager& netgroupman, bool deterministic, int32_t consistency_check_ratio)
     : insecure_rand{deterministic}
     , nKey{deterministic ? uint256{1} : insecure_rand.rand256()}
     , m_consistency_check_ratio{consistency_check_ratio}
-    , m_asmap{std::move(asmap)}
+    , m_netgroupman{netgroupman}
 {
     for (auto& bucket : vvNew) {
         for (auto& entry : bucket) {
@@ -219,8 +219,8 @@ void AddrManImpl::Serialize(Stream& s_) const
     // Store asmap checksum after bucket entries so that it
     // can be ignored by older clients for backward compatibility.
     uint256 asmap_checksum;
-    if (m_asmap.size() != 0) {
-        asmap_checksum = SerializeHash(m_asmap);
+    if (m_netgroupman.GetAsmap().size() != 0) {
+        asmap_checksum = SerializeHash(m_netgroupman.GetAsmap());
     }
     s << asmap_checksum;
 }
@@ -298,7 +298,7 @@ void AddrManImpl::Unserialize(Stream& s_)
     for (int n = 0; n < nTried; n++) {
         AddrInfo info;
         s >> info;
-        int nKBucket = info.GetTriedBucket(nKey, m_asmap);
+        int nKBucket = info.GetTriedBucket(nKey, m_netgroupman.GetAsmap());
         int nKBucketPos = info.GetBucketPosition(nKey, false, nKBucket);
         if (info.IsValid()
                 && vvTried[nKBucket][nKBucketPos] == -1) {
@@ -336,8 +336,8 @@ void AddrManImpl::Unserialize(Stream& s_)
     // to restore the entries to the buckets/positions they were in before
     // serialization.
     uint256 supplied_asmap_checksum;
-    if (m_asmap.size() != 0) {
-        supplied_asmap_checksum = SerializeHash(m_asmap);
+    if (m_netgroupman.GetAsmap().size() != 0) {
+        supplied_asmap_checksum = SerializeHash(m_netgroupman.GetAsmap());
     }
     uint256 serialized_asmap_checksum;
     if (format >= Format::V2_ASMAP) {
@@ -371,7 +371,7 @@ void AddrManImpl::Unserialize(Stream& s_)
         } else {
             // In case the new table data cannot be used (bucket count wrong or new asmap),
             // try to give them a reference based on their primary source address.
-            bucket = info.GetNewBucket(nKey, m_asmap);
+            bucket = info.GetNewBucket(nKey, m_netgroupman.GetAsmap());
             bucket_position = info.GetBucketPosition(nKey, true, bucket);
             if (vvNew[bucket][bucket_position] == -1) {
                 vvNew[bucket][bucket_position] = entry_index;
@@ -495,7 +495,7 @@ void AddrManImpl::MakeTried(AddrInfo& info, int nId)
     AssertLockHeld(cs);
 
     // remove the entry from all new buckets
-    const int start_bucket{info.GetNewBucket(nKey, m_asmap)};
+    const int start_bucket{info.GetNewBucket(nKey, m_netgroupman.GetAsmap())};
     for (int n = 0; n < ADDRMAN_NEW_BUCKET_COUNT; ++n) {
         const int bucket{(start_bucket + n) % ADDRMAN_NEW_BUCKET_COUNT};
         const int pos{info.GetBucketPosition(nKey, true, bucket)};
@@ -510,7 +510,7 @@ void AddrManImpl::MakeTried(AddrInfo& info, int nId)
     assert(info.nRefCount == 0);
 
     // which tried bucket to move the entry to
-    int nKBucket = info.GetTriedBucket(nKey, m_asmap);
+    int nKBucket = info.GetTriedBucket(nKey, m_netgroupman.GetAsmap());
     int nKBucketPos = info.GetBucketPosition(nKey, false, nKBucket);
 
     // first make space to add it (the existing tried entry there is moved to new, deleting whatever is there).
@@ -526,7 +526,7 @@ void AddrManImpl::MakeTried(AddrInfo& info, int nId)
         nTried--;
 
         // find which new bucket it belongs to
-        int nUBucket = infoOld.GetNewBucket(nKey, m_asmap);
+        int nUBucket = infoOld.GetNewBucket(nKey, m_netgroupman.GetAsmap());
         int nUBucketPos = infoOld.GetBucketPosition(nKey, true, nUBucket);
         ClearNew(nUBucket, nUBucketPos);
         assert(vvNew[nUBucket][nUBucketPos] == -1);
@@ -594,7 +594,7 @@ bool AddrManImpl::AddSingle(const CAddress& addr, const CNetAddr& source, int64_
         nNew++;
     }
 
-    int nUBucket = pinfo->GetNewBucket(nKey, source, m_asmap);
+    int nUBucket = pinfo->GetNewBucket(nKey, source, m_netgroupman.GetAsmap());
     int nUBucketPos = pinfo->GetBucketPosition(nKey, true, nUBucket);
     bool fInsert = vvNew[nUBucket][nUBucketPos] == -1;
     if (vvNew[nUBucket][nUBucketPos] != nId) {
@@ -610,7 +610,7 @@ bool AddrManImpl::AddSingle(const CAddress& addr, const CNetAddr& source, int64_
             pinfo->nRefCount++;
             vvNew[nUBucket][nUBucketPos] = nId;
             LogPrint(BCLog::ADDRMAN, "Added %s mapped to AS%i to new[%i][%i]\n",
-                     addr.ToString(), addr.GetMappedAS(m_asmap), nUBucket, nUBucketPos);
+                     addr.ToString(), addr.GetMappedAS(m_netgroupman.GetAsmap()), nUBucket, nUBucketPos);
         } else {
             if (pinfo->nRefCount == 0) {
                 Delete(nId);
@@ -650,7 +650,7 @@ bool AddrManImpl::Good_(const CService& addr, bool test_before_evict, int64_t nT
 
 
     // which tried bucket to move the entry to
-    int tried_bucket = info.GetTriedBucket(nKey, m_asmap);
+    int tried_bucket = info.GetTriedBucket(nKey, m_netgroupman.GetAsmap());
     int tried_bucket_pos = info.GetBucketPosition(nKey, false, tried_bucket);
 
     // Will moving this address into tried evict another entry?
@@ -669,7 +669,7 @@ bool AddrManImpl::Good_(const CService& addr, bool test_before_evict, int64_t nT
         // move nId to the tried tables
         MakeTried(info, nId);
         LogPrint(BCLog::ADDRMAN, "Moved %s mapped to AS%i to tried[%i][%i]\n",
-                 addr.ToString(), addr.GetMappedAS(m_asmap), tried_bucket, tried_bucket_pos);
+                 addr.ToString(), addr.GetMappedAS(m_netgroupman.GetAsmap()), tried_bucket, tried_bucket_pos);
         return true;
     }
 }
@@ -863,7 +863,7 @@ void AddrManImpl::ResolveCollisions_()
             AddrInfo& info_new = mapInfo[id_new];
 
             // Which tried bucket to move the entry to.
-            int tried_bucket = info_new.GetTriedBucket(nKey, m_asmap);
+            int tried_bucket = info_new.GetTriedBucket(nKey, m_netgroupman.GetAsmap());
             int tried_bucket_pos = info_new.GetBucketPosition(nKey, false, tried_bucket);
             if (!info_new.IsValid()) { // id_new may no longer map to a valid address
                 erase_collision = true;
@@ -929,7 +929,7 @@ std::pair<CAddress, int64_t> AddrManImpl::SelectTriedCollision_()
     const AddrInfo& newInfo = mapInfo[id_new];
 
     // which tried bucket to move the entry to
-    int tried_bucket = newInfo.GetTriedBucket(nKey, m_asmap);
+    int tried_bucket = newInfo.GetTriedBucket(nKey, m_netgroupman.GetAsmap());
     int tried_bucket_pos = newInfo.GetBucketPosition(nKey, false, tried_bucket);
 
     const AddrInfo& info_old = mapInfo[vvTried[tried_bucket][tried_bucket_pos]];
@@ -945,13 +945,13 @@ std::optional<AddressPosition> AddrManImpl::FindAddressEntry_(const CAddress& ad
     if (!addr_info) return std::nullopt;
 
     if(addr_info->fInTried) {
-        int bucket{addr_info->GetTriedBucket(nKey, m_asmap)};
+        int bucket{addr_info->GetTriedBucket(nKey, m_netgroupman.GetAsmap())};
         return AddressPosition(/*tried_in=*/true,
                                /*multiplicity_in=*/1,
                                /*bucket_in=*/bucket,
                                /*position_in=*/addr_info->GetBucketPosition(nKey, false, bucket));
     } else {
-        int bucket{addr_info->GetNewBucket(nKey, m_asmap)};
+        int bucket{addr_info->GetNewBucket(nKey, m_netgroupman.GetAsmap())};
         return AddressPosition(/*tried_in=*/false,
                                /*multiplicity_in=*/addr_info->nRefCount,
                                /*bucket_in=*/bucket,
@@ -1026,7 +1026,7 @@ int AddrManImpl::CheckAddrman() const
                 if (!setTried.count(vvTried[n][i]))
                     return -11;
                 const auto it{mapInfo.find(vvTried[n][i])};
-                if (it == mapInfo.end() || it->second.GetTriedBucket(nKey, m_asmap) != n) {
+                if (it == mapInfo.end() || it->second.GetTriedBucket(nKey, m_netgroupman.GetAsmap()) != n) {
                     return -17;
                 }
                 if (it->second.GetBucketPosition(nKey, false, n) != i) {
@@ -1154,13 +1154,8 @@ std::optional<AddressPosition> AddrManImpl::FindAddressEntry(const CAddress& add
     return entry;
 }
 
-const std::vector<bool>& AddrManImpl::GetAsmap() const
-{
-    return m_asmap;
-}
-
-AddrMan::AddrMan(std::vector<bool> asmap, bool deterministic, int32_t consistency_check_ratio)
-    : m_impl(std::make_unique<AddrManImpl>(std::move(asmap), deterministic, consistency_check_ratio)) {}
+AddrMan::AddrMan(const NetGroupManager& netgroupman, bool deterministic, int32_t consistency_check_ratio)
+    : m_impl(std::make_unique<AddrManImpl>(netgroupman, deterministic, consistency_check_ratio)) {}
 
 AddrMan::~AddrMan() = default;
 
@@ -1235,11 +1230,6 @@ void AddrMan::SetServices(const CService& addr, ServiceFlags nServices)
     m_impl->SetServices(addr, nServices);
 }
 
-const std::vector<bool>& AddrMan::GetAsmap() const
-{
-    return m_impl->GetAsmap();
-}
-
 std::optional<AddressPosition> AddrMan::FindAddressEntry(const CAddress& addr)
 {
     return m_impl->FindAddressEntry(addr);

Diff for: src/addrman.h

@@ -7,6 +7,7 @@
 #define BITCOIN_ADDRMAN_H
 
 #include <netaddress.h>
+#include <netgroup.h>
 #include <protocol.h>
 #include <streams.h>
 #include <timedata.h>
@@ -88,7 +89,7 @@ class AddrMan
     const std::unique_ptr<AddrManImpl> m_impl;
 
 public:
-    explicit AddrMan(std::vector<bool> asmap, bool deterministic, int32_t consistency_check_ratio);
+    explicit AddrMan(const NetGroupManager& netgroupman, bool deterministic, int32_t consistency_check_ratio);
 
     ~AddrMan();
 
@@ -172,8 +173,6 @@ class AddrMan
     //! Update an entry's service bits.
     void SetServices(const CService& addr, ServiceFlags nServices);
 
-    const std::vector<bool>& GetAsmap() const;
-
     /** Test-only function
      * Find the address record in AddrMan and return information about its
      * position.

Diff for: src/addrman_impl.h

@@ -100,7 +100,7 @@ class AddrInfo : public CAddress
 class AddrManImpl
 {
 public:
-    AddrManImpl(std::vector<bool>&& asmap, bool deterministic, int32_t consistency_check_ratio);
+    AddrManImpl(const NetGroupManager& netgroupman, bool deterministic, int32_t consistency_check_ratio);
 
     ~AddrManImpl();
 
@@ -140,8 +140,6 @@ class AddrManImpl
     std::optional<AddressPosition> FindAddressEntry(const CAddress& addr)
         EXCLUSIVE_LOCKS_REQUIRED(!cs);
 
-    const std::vector<bool>& GetAsmap() const;
-
     friend class AddrManDeterministic;
 
 private:
@@ -212,21 +210,8 @@ class AddrManImpl
     /** Perform consistency checks every m_consistency_check_ratio operations (if non-zero). */
     const int32_t m_consistency_check_ratio;
 
-    // Compressed IP->ASN mapping, loaded from a file when a node starts.
-    // Should be always empty if no file was provided.
-    // This mapping is then used for bucketing nodes in Addrman.
-    //
-    // If asmap is provided, nodes will be bucketed by
-    // AS they belong to, in order to make impossible for a node
-    // to connect to several nodes hosted in a single AS.
-    // This is done in response to Erebus attack, but also to generally
-    // diversify the connections every node creates,
-    // especially useful when a large fraction of nodes
-    // operate under a couple of cloud providers.
-    //
-    // If a new asmap was provided, the existing records
-    // would be re-bucketed accordingly.
-    const std::vector<bool> m_asmap;
+    /** Reference to the netgroup manager. netgroupman must be constructed before addrman and destructed after. */
+    const NetGroupManager& m_netgroupman;
 
     //! Find an entry.
     AddrInfo* Find(const CService& addr, int* pnId = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs);

Diff for: src/bench/addrman.cpp

@@ -4,6 +4,7 @@
 
 #include <addrman.h>
 #include <bench/bench.h>
+#include <netgroup.h>
 #include <random.h>
 #include <util/check.h>
 #include <util/time.h>
@@ -16,7 +17,7 @@
 static constexpr size_t NUM_SOURCES = 64;
 static constexpr size_t NUM_ADDRESSES_PER_SOURCE = 256;
 
-static const std::vector<bool> EMPTY_ASMAP;
+static NetGroupManager EMPTY_NETGROUPMAN{std::vector<bool>()};
 static constexpr uint32_t ADDRMAN_CONSISTENCY_CHECK_RATIO{0};
 
 static std::vector<CAddress> g_sources;
@@ -77,14 +78,14 @@ static void AddrManAdd(benchmark::Bench& bench)
     CreateAddresses();
 
     bench.run([&] {
-        AddrMan addrman{EMPTY_ASMAP, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO};
+        AddrMan addrman{EMPTY_NETGROUPMAN, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO};
         AddAddressesToAddrMan(addrman);
     });
 }
 
 static void AddrManSelect(benchmark::Bench& bench)
 {
-    AddrMan addrman{EMPTY_ASMAP, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO};
+    AddrMan addrman{EMPTY_NETGROUPMAN, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO};
 
     FillAddrMan(addrman);
 
@@ -96,7 +97,7 @@ static void AddrManSelect(benchmark::Bench& bench)
 
 static void AddrManGetAddr(benchmark::Bench& bench)
 {
-    AddrMan addrman{EMPTY_ASMAP, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO};
+    AddrMan addrman{EMPTY_NETGROUPMAN, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO};
 
     FillAddrMan(addrman);
 
@@ -125,7 +126,7 @@ static void AddrManAddThenGood(benchmark::Bench& bench)
         //
         // This has some overhead (exactly the result of AddrManAdd benchmark), but that overhead is constant so improvements in
         // AddrMan::Good() will still be noticeable.
-        AddrMan addrman{EMPTY_ASMAP, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO};
+        AddrMan addrman{EMPTY_NETGROUPMAN, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO};
         AddAddressesToAddrMan(addrman);
 
         markSomeAsGood(addrman);

Diff for: src/init.cpp

@@ -1256,20 +1256,22 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
 
         // Initialize netgroup manager
         assert(!node.netgroupman);
-        node.netgroupman = std::make_unique<NetGroupManager>();
+        node.netgroupman = std::make_unique<NetGroupManager>(std::move(asmap));
 
         // Initialize addrman
         assert(!node.addrman);
         uiInterface.InitMessage(_("Loading P2P addresses…").translated);
-        if (const auto error{LoadAddrman(asmap, args, node.addrman)}) {
+        if (const auto error{LoadAddrman(*node.netgroupman, args, node.addrman)}) {
             return InitError(*error);
         }
     }
 
     assert(!node.banman);
     node.banman = std::make_unique<BanMan>(gArgs.GetDataDirNet() / "banlist", &uiInterface, args.GetIntArg("-bantime", DEFAULT_MISBEHAVING_BANTIME));
     assert(!node.connman);
-    node.connman = std::make_unique<CConnman>(GetRand(std::numeric_limits<uint64_t>::max()), GetRand(std::numeric_limits<uint64_t>::max()), *node.addrman, args.GetBoolArg("-networkactive", true));
+    node.connman = std::make_unique<CConnman>(GetRand(std::numeric_limits<uint64_t>::max()),
+                                              GetRand(std::numeric_limits<uint64_t>::max()),
+                                              *node.addrman, *node.netgroupman, args.GetBoolArg("-networkactive", true));
 
     assert(!node.fee_estimator);
     // Don't initialize fee estimation with old data if we don't relay transactions,