Skip to content

Commit f2e673f

Browse files
fjahrfjahr
committed
WIP: Use span of bytes for asmap data
1 parent 6302f95 commit f2e673f

15 files changed

+144
-153
lines changed

‎src/bench/addrman.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
static constexpr size_t NUM_SOURCES = 64;
2525
static constexpr size_t NUM_ADDRESSES_PER_SOURCE = 256;
2626

27-
static NetGroupManager EMPTY_NETGROUPMAN{std::vector<bool>()};
27+
static NetGroupManager EMPTY_NETGROUPMAN{NetGroupManager::NoAsmap()};
2828
static constexpr uint32_t ADDRMAN_CONSISTENCY_CHECK_RATIO{0};
2929

3030
static std::vector<CAddress> g_sources;

‎src/init.cpp

+13-10
Original file line numberDiff line numberDiff line change
@@ -1400,9 +1400,10 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
14001400
ApplyArgsManOptions(args, peerman_opts);
14011401

14021402
{
1403-
1404-
// Read asmap file or embedded data if configured
1405-
std::vector<bool> asmap;
1403+
// Read asmap file or embedded data if configured and initialize
1404+
// Netgroupman with or without it
1405+
assert(!node.netgroupman);
1406+
uint256 asmap_version;
14061407
if (args.IsArgSet("-asmap")) {
14071408
const bool asmap_file_set{args.GetPathArg("-asmap") != ""};
14081409
fs::path asmap_path = args.GetPathArg("-asmap", DEFAULT_ASMAP_FILENAME);
@@ -1420,7 +1421,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
14201421
if (fs::exists(asmap_path)) {
14211422
// If a file exists at the path (could be passed or the default
14221423
// location), try to read the file
1423-
asmap = DecodeAsmap(asmap_path);
1424+
std::vector<std::byte> asmap{DecodeAsmap(asmap_path)};
14241425
if (asmap.empty()) {
14251426
// If the file could not be read, print the error depending
14261427
// on if it was passed or the default location
@@ -1431,14 +1432,19 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
14311432
}
14321433
return false;
14331434
}
1435+
node.netgroupman = std::make_unique<NetGroupManager>(NetGroupManager::WithLoadedAsmap(asmap));
1436+
asmap_version = AsmapChecksum(asmap);
14341437
} else {
14351438
#ifdef ENABLE_EMBEDDED_ASMAP
14361439
// If the file doesn't exist, try to use the embedded data
1437-
asmap = DecodeAsmap(node::data::ip_asn);
1440+
std::span<const std::byte> asmap{CheckAsmap(node::data::ip_asn)};
14381441
if (asmap.empty()) {
14391442
InitError(strprintf(_("Could not read embedded asmap data")));
14401443
return false;
14411444
}
1445+
node.netgroupman = std::make_unique<NetGroupManager>(NetGroupManager::WithEmbeddedAsmap(asmap));
1446+
asmap_version = AsmapChecksum(asmap);
1447+
LogInfo("Opened asmap data (%zu bytes) from embedded byte array\n", asmap.size());
14421448
#else
14431449
// If there is no embedded data, fail and report the default
14441450
// file as missing since we only end up here if the no
@@ -1447,16 +1453,13 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info)
14471453
return false;
14481454
#endif
14491455
}
1450-
const uint256 asmap_version = (HashWriter{} << asmap).GetHash();
1456+
14511457
LogPrintf("Using asmap version %s for IP bucketing\n", asmap_version.ToString());
14521458
} else {
1459+
node.netgroupman = std::make_unique<NetGroupManager>(NetGroupManager::NoAsmap());
14531460
LogPrintf("Using /16 prefix for IP bucketing\n");
14541461
}
14551462

1456-
// Initialize netgroup manager
1457-
assert(!node.netgroupman);
1458-
node.netgroupman = std::make_unique<NetGroupManager>(std::move(asmap));
1459-
14601463
// Initialize addrman
14611464
assert(!node.addrman);
14621465
uiInterface.InitMessage(_("Loading P2P addresses…"));

‎src/netgroup.cpp

+8-13
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,13 @@
66

77
#include <hash.h>
88
#include <logging.h>
9+
#include <uint256.h>
910
#include <util/asmap.h>
1011

1112
uint256 NetGroupManager::GetAsmapChecksum() const
1213
{
1314
if (!m_asmap.size()) return {};
14-
15-
return (HashWriter{} << m_asmap).GetHash();
15+
return AsmapChecksum(m_asmap);
1616
}
1717

1818
std::vector<unsigned char> NetGroupManager::GetGroup(const CNetAddr& address) const
@@ -81,30 +81,25 @@ std::vector<unsigned char> NetGroupManager::GetGroup(const CNetAddr& address) co
8181
uint32_t NetGroupManager::GetMappedAS(const CNetAddr& address) const
8282
{
8383
uint32_t net_class = address.GetNetClass();
84-
if (m_asmap.size() == 0 || (net_class != NET_IPV4 && net_class != NET_IPV6)) {
84+
if (m_asmap.empty() || (net_class != NET_IPV4 && net_class != NET_IPV6)) {
8585
return 0; // Indicates not found, safe because AS0 is reserved per RFC7607.
8686
}
87-
std::vector<bool> ip_bits(128);
87+
std::vector<std::byte> ip_bits(16);
8888
if (address.HasLinkedIPv4()) {
8989
// For lookup, treat as if it was just an IPv4 address (IPV4_IN_IPV6_PREFIX + IPv4 bits)
9090
for (int8_t byte_i = 0; byte_i < 12; ++byte_i) {
91-
for (uint8_t bit_i = 0; bit_i < 8; ++bit_i) {
92-
ip_bits[byte_i * 8 + bit_i] = (IPV4_IN_IPV6_PREFIX[byte_i] >> (7 - bit_i)) & 1;
93-
}
91+
ip_bits[byte_i] = static_cast<std::byte>(IPV4_IN_IPV6_PREFIX[byte_i]);
9492
}
9593
uint32_t ipv4 = address.GetLinkedIPv4();
96-
for (int i = 0; i < 32; ++i) {
97-
ip_bits[96 + i] = (ipv4 >> (31 - i)) & 1;
94+
for (int i = 0; i < 4; ++i) {
95+
ip_bits[12 + i] = static_cast<std::byte>((ipv4 >> (24 - i * 8)) & 0xFF);
9896
}
9997
} else {
10098
// Use all 128 bits of the IPv6 address otherwise
10199
assert(address.IsIPv6());
102100
auto addr_bytes = address.GetAddrBytes();
103101
for (int8_t byte_i = 0; byte_i < 16; ++byte_i) {
104-
uint8_t cur_byte = addr_bytes[byte_i];
105-
for (uint8_t bit_i = 0; bit_i < 8; ++bit_i) {
106-
ip_bits[byte_i * 8 + bit_i] = (cur_byte >> (7 - bit_i)) & 1;
107-
}
102+
ip_bits[byte_i] = static_cast<std::byte>(addr_bytes[byte_i]);
108103
}
109104
}
110105
uint32_t mapped_as = Interpret(m_asmap, ip_bits);

‎src/netgroup.h

+21-4
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,18 @@
1515
*/
1616
class NetGroupManager {
1717
public:
18-
explicit NetGroupManager(std::vector<bool> asmap)
19-
: m_asmap{std::move(asmap)}
20-
{}
18+
static NetGroupManager WithEmbeddedAsmap(std::span<const std::byte> asmap) {
19+
return NetGroupManager(asmap, {});
20+
}
21+
22+
static NetGroupManager WithLoadedAsmap(std::vector<std::byte> loaded_asmap) {
23+
std::span<const std::byte> asmap_span(loaded_asmap);
24+
return NetGroupManager(asmap_span, std::move(loaded_asmap));
25+
}
26+
27+
static NetGroupManager NoAsmap() {
28+
return NetGroupManager({}, {});
29+
}
2130

2231
/** Get a checksum identifying the asmap being used. */
2332
uint256 GetAsmapChecksum() const;
@@ -70,7 +79,15 @@ class NetGroupManager {
7079
*
7180
* This is initialized in the constructor, const, and therefore is
7281
* thread-safe. */
73-
const std::vector<bool> m_asmap;
82+
const std::span<const std::byte> m_asmap;
83+
84+
// TODO: document
85+
std::vector<std::byte> m_loaded_asmap;
86+
87+
explicit NetGroupManager(std::span<const std::byte> asmap, std::vector<std::byte> loaded_asmap)
88+
: m_asmap(asmap.empty() ? std::span<const std::byte>() : asmap),
89+
m_loaded_asmap(std::move(loaded_asmap))
90+
{}
7491
};
7592

7693
#endif // BITCOIN_NETGROUP_H

‎src/test/addrman_tests.cpp

+4-21
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ using namespace std::literals;
2424
using node::NodeContext;
2525
using util::ToString;
2626

27-
static NetGroupManager EMPTY_NETGROUPMAN{std::vector<bool>()};
27+
static NetGroupManager EMPTY_NETGROUPMAN{NetGroupManager::NoAsmap()};
2828
static const bool DETERMINISTIC{true};
2929

3030
static int32_t GetCheckRatio(const NodeContext& node_ctx)
@@ -46,20 +46,6 @@ static CService ResolveService(const std::string& ip, uint16_t port = 0)
4646
return serv.value_or(CService{});
4747
}
4848

49-
50-
static std::vector<bool> FromBytes(std::span<const std::byte> source)
51-
{
52-
int vector_size(source.size() * 8);
53-
std::vector<bool> result(vector_size);
54-
for (int byte_i = 0; byte_i < vector_size / 8; ++byte_i) {
55-
uint8_t cur_byte{std::to_integer<uint8_t>(source[byte_i])};
56-
for (int bit_i = 0; bit_i < 8; ++bit_i) {
57-
result[byte_i * 8 + bit_i] = (cur_byte >> bit_i) & 1;
58-
}
59-
}
60-
return result;
61-
}
62-
6349
BOOST_FIXTURE_TEST_SUITE(addrman_tests, BasicTestingSetup)
6450

6551
BOOST_AUTO_TEST_CASE(addrman_simple)
@@ -592,8 +578,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket_legacy)
592578
// 101.8.0.0/16 AS8
593579
BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket)
594580
{
595-
std::vector<bool> asmap = FromBytes(test::data::asmap);
596-
NetGroupManager ngm_asmap{asmap};
581+
NetGroupManager ngm_asmap{NetGroupManager::WithEmbeddedAsmap(test::data::asmap)};
597582

598583
CAddress addr1 = CAddress(ResolveService("250.1.1.1", 8333), NODE_NONE);
599584
CAddress addr2 = CAddress(ResolveService("250.1.1.1", 9999), NODE_NONE);
@@ -646,8 +631,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket)
646631

647632
BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket)
648633
{
649-
std::vector<bool> asmap = FromBytes(test::data::asmap);
650-
NetGroupManager ngm_asmap{asmap};
634+
NetGroupManager ngm_asmap{NetGroupManager::WithEmbeddedAsmap(test::data::asmap)};
651635

652636
CAddress addr1 = CAddress(ResolveService("250.1.2.1", 8333), NODE_NONE);
653637
CAddress addr2 = CAddress(ResolveService("250.1.2.1", 9999), NODE_NONE);
@@ -724,8 +708,7 @@ BOOST_AUTO_TEST_CASE(caddrinfo_get_new_bucket)
724708

725709
BOOST_AUTO_TEST_CASE(addrman_serialization)
726710
{
727-
std::vector<bool> asmap1 = FromBytes(test::data::asmap);
728-
NetGroupManager netgroupman{asmap1};
711+
NetGroupManager netgroupman{NetGroupManager::WithEmbeddedAsmap(test::data::asmap)};
729712

730713
const auto ratio = GetCheckRatio(m_node);
731714
auto addrman_asmap1 = std::make_unique<AddrMan>(netgroupman, DETERMINISTIC, ratio);

‎src/test/fuzz/asmap.cpp

+12-18
Original file line numberDiff line numberDiff line change
@@ -11,22 +11,13 @@
1111
#include <vector>
1212

1313
//! asmap code that consumes nothing
14-
static const std::vector<bool> IPV6_PREFIX_ASMAP = {};
14+
static const std::vector<std::byte> IPV6_PREFIX_ASMAP = {};
1515

1616
//! asmap code that consumes the 96 prefix bits of ::ffff:0/96 (IPv4-in-IPv6 map)
17-
static const std::vector<bool> IPV4_PREFIX_ASMAP = {
18-
true, true, false, true, true, true, true, true, true, true, false, false, false, false, false, false, false, false, // Match 0x00
19-
true, true, false, true, true, true, true, true, true, true, false, false, false, false, false, false, false, false, // Match 0x00
20-
true, true, false, true, true, true, true, true, true, true, false, false, false, false, false, false, false, false, // Match 0x00
21-
true, true, false, true, true, true, true, true, true, true, false, false, false, false, false, false, false, false, // Match 0x00
22-
true, true, false, true, true, true, true, true, true, true, false, false, false, false, false, false, false, false, // Match 0x00
23-
true, true, false, true, true, true, true, true, true, true, false, false, false, false, false, false, false, false, // Match 0x00
24-
true, true, false, true, true, true, true, true, true, true, false, false, false, false, false, false, false, false, // Match 0x00
25-
true, true, false, true, true, true, true, true, true, true, false, false, false, false, false, false, false, false, // Match 0x00
26-
true, true, false, true, true, true, true, true, true, true, false, false, false, false, false, false, false, false, // Match 0x00
27-
true, true, false, true, true, true, true, true, true, true, false, false, false, false, false, false, false, false, // Match 0x00
28-
true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true, // Match 0xFF
29-
true, true, false, true, true, true, true, true, true, true, true, true, true, true, true, true, true, true // Match 0xFF
17+
static const std::vector<std::byte> IPV4_PREFIX_ASMAP = {
18+
std::byte{0x00}, std::byte{0x00}, std::byte{0x00}, std::byte{0x00},
19+
std::byte{0x00}, std::byte{0x00}, std::byte{0x00}, std::byte{0x00},
20+
std::byte{0x00}, std::byte{0x00}, std::byte{0xFF}, std::byte{0xFF}
3021
};
3122

3223
FUZZ_TARGET(asmap)
@@ -37,13 +28,16 @@ FUZZ_TARGET(asmap)
3728
bool ipv6 = buffer[0] & 128;
3829
const size_t addr_size = ipv6 ? ADDR_IPV6_SIZE : ADDR_IPV4_SIZE;
3930
if (buffer.size() < size_t(1 + asmap_size + addr_size)) return;
40-
std::vector<bool> asmap = ipv6 ? IPV6_PREFIX_ASMAP : IPV4_PREFIX_ASMAP;
41-
asmap.reserve(asmap.size() + 8 * asmap_size);
31+
32+
std::vector<std::byte> asmap_vec = ipv6 ? IPV6_PREFIX_ASMAP : IPV4_PREFIX_ASMAP;
4233
for (int i = 0; i < asmap_size; ++i) {
34+
uint8_t byte = buffer[1 + i];
4335
for (int j = 0; j < 8; ++j) {
44-
asmap.push_back((buffer[1 + i] >> j) & 1);
36+
asmap_vec.push_back(static_cast<std::byte>((byte >> j) & 1));
4537
}
4638
}
39+
std::span<const std::byte> asmap(asmap_vec);
40+
4741
if (!SanityCheckASMap(asmap, 128)) return;
4842

4943
const uint8_t* addr_data = buffer.data() + 1 + asmap_size;
@@ -57,6 +51,6 @@ FUZZ_TARGET(asmap)
5751
memcpy(&ipv4, addr_data, addr_size);
5852
net_addr.SetIP(CNetAddr{ipv4});
5953
}
60-
NetGroupManager netgroupman{asmap};
54+
NetGroupManager netgroupman{NetGroupManager::WithEmbeddedAsmap(asmap)};
6155
(void)netgroupman.GetMappedAS(net_addr);
6256
}

‎src/test/fuzz/asmap_direct.cpp

+9-7
Original file line numberDiff line numberDiff line change
@@ -31,19 +31,21 @@ FUZZ_TARGET(asmap_direct)
3131
if (buffer.size() - sep_pos - 1 > 128) return; // At most 128 bits in IP address
3232

3333
// Checks on asmap
34-
std::vector<bool> asmap(buffer.begin(), buffer.begin() + sep_pos);
35-
if (SanityCheckASMap(asmap, buffer.size() - 1 - sep_pos)) {
34+
std::vector<std::byte> asmap(reinterpret_cast<const std::byte*>(buffer.data()),
35+
reinterpret_cast<const std::byte*>(buffer.data() + sep_pos));
36+
if (SanityCheckASMap(std::span<const std::byte>(asmap), buffer.size() - 1 - sep_pos)) {
3637
// Verify that for valid asmaps, no prefix (except up to 7 zero padding bits) is valid.
37-
std::vector<bool> asmap_prefix = asmap;
38-
while (!asmap_prefix.empty() && asmap_prefix.size() + 7 > asmap.size() && asmap_prefix.back() == false) {
38+
std::vector<std::byte> asmap_prefix = asmap;
39+
while (!asmap_prefix.empty() && asmap_prefix.size() + 7 > asmap.size() && asmap_prefix.back() == std::byte{0}) {
3940
asmap_prefix.pop_back();
4041
}
4142
while (!asmap_prefix.empty()) {
4243
asmap_prefix.pop_back();
43-
assert(!SanityCheckASMap(asmap_prefix, buffer.size() - 1 - sep_pos));
44+
assert(!SanityCheckASMap(std::span<const std::byte>(asmap_prefix), buffer.size() - 1 - sep_pos));
4445
}
4546
// No address input should trigger assertions in interpreter
46-
std::vector<bool> addr(buffer.begin() + sep_pos + 1, buffer.end());
47-
(void)Interpret(asmap, addr);
47+
std::vector<std::byte> addr(reinterpret_cast<const std::byte*>(buffer.data() + sep_pos + 1),
48+
reinterpret_cast<const std::byte*>(buffer.data() + buffer.size()));
49+
(void)Interpret(std::span<const std::byte>(asmap), std::span<const std::byte>(addr));
4850
}
4951
}

‎src/test/fuzz/p2p_handshake.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ FUZZ_TARGET(p2p_handshake, .init = ::initialize)
4848
chainman.ResetIbd();
4949

5050
node::Warnings warnings{};
51-
NetGroupManager netgroupman{{}};
51+
NetGroupManager netgroupman{NetGroupManager::NoAsmap()};
5252
AddrMan addrman{netgroupman, /*deterministic=*/true, 0};
5353
auto peerman = PeerManager::make(connman, addrman,
5454
/*banman=*/nullptr, chainman,

‎src/test/fuzz/util.h

-5
Original file line numberDiff line numberDiff line change
@@ -65,11 +65,6 @@ template<typename B = uint8_t>
6565
return ret;
6666
}
6767

68-
[[nodiscard]] inline std::vector<bool> ConsumeRandomLengthBitVector(FuzzedDataProvider& fuzzed_data_provider, const std::optional<size_t>& max_length = std::nullopt) noexcept
69-
{
70-
return BytesToBits(ConsumeRandomLengthByteVector(fuzzed_data_provider, max_length));
71-
}
72-
7368
[[nodiscard]] inline DataStream ConsumeDataStream(FuzzedDataProvider& fuzzed_data_provider, const std::optional<size_t>& max_length = std::nullopt) noexcept
7469
{
7570
return DataStream{ConsumeRandomLengthByteVector(fuzzed_data_provider, max_length)};

‎src/test/fuzz/util/net.h

+5-3
Original file line numberDiff line numberDiff line change
@@ -200,9 +200,11 @@ class FuzzedSock : public Sock
200200

201201
[[nodiscard]] inline NetGroupManager ConsumeNetGroupManager(FuzzedDataProvider& fuzzed_data_provider) noexcept
202202
{
203-
std::vector<bool> asmap = ConsumeRandomLengthBitVector(fuzzed_data_provider);
204-
if (!SanityCheckASMap(asmap, 128)) asmap.clear();
205-
return NetGroupManager(asmap);
203+
std::vector<std::byte> asmap{ConsumeRandomLengthByteVector<std::byte>(fuzzed_data_provider)};
204+
if (!SanityCheckASMap(std::span<std::byte>(asmap), 128)) {
205+
return NetGroupManager::NoAsmap();
206+
}
207+
return NetGroupManager::WithLoadedAsmap(asmap);
206208
}
207209

208210
inline CSubNet ConsumeSubNet(FuzzedDataProvider& fuzzed_data_provider) noexcept

‎src/test/netbase_tests.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -323,7 +323,7 @@ BOOST_AUTO_TEST_CASE(subnet_test)
323323

324324
BOOST_AUTO_TEST_CASE(netbase_getgroup)
325325
{
326-
NetGroupManager netgroupman{std::vector<bool>()}; // use /16
326+
NetGroupManager netgroupman{NetGroupManager::NoAsmap()}; // use /16
327327
BOOST_CHECK(netgroupman.GetGroup(ResolveIP("127.0.0.1")) == std::vector<unsigned char>({0})); // Local -> !Routable()
328328
BOOST_CHECK(netgroupman.GetGroup(ResolveIP("257.0.0.1")) == std::vector<unsigned char>({0})); // !Valid -> !Routable()
329329
BOOST_CHECK(netgroupman.GetGroup(ResolveIP("10.0.0.1")) == std::vector<unsigned char>({0})); // RFC1918 -> !Routable()

‎src/test/util/setup_common.cpp

+1-1
Original file line numberDiff line numberDiff line change
@@ -321,7 +321,7 @@ TestingSetup::TestingSetup(
321321

322322
if (!opts.setup_net) return;
323323

324-
m_node.netgroupman = std::make_unique<NetGroupManager>(/*asmap=*/std::vector<bool>());
324+
m_node.netgroupman = std::make_unique<NetGroupManager>(NetGroupManager::NoAsmap());
325325
m_node.addrman = std::make_unique<AddrMan>(*m_node.netgroupman,
326326
/*deterministic=*/false,
327327
m_node.args->GetIntArg("-checkaddrman", 0));

0 commit comments

Comments
 (0)