Merge #36: Add support to filter nodes by node flags

sipa · sipa · commit ab0ac73e5f48 · 2016-06-07T17:57:12.000+02:00
c9679dc Add whitelist for service filter, use a map for cacheTime (Jonas Schnelli) 5c8b9e3 IP-Query: Use a vector instead a set and random pick an IP (Jonas Schnelli) 839db15 Add support to filter nodes by node flags (Jonas Schnelli)
diff --git a/db.cpp b/db.cpp
@@ -166,7 +166,7 @@ void CAddrDb::Add_(const CAddress &addr, bool force) {
   nDirty++;
 }
 
-void CAddrDb::GetIPs_(set<CNetAddr>& ips, int max, const bool* nets) {
+void CAddrDb::GetIPs_(set<CNetAddr>& ips, uint64_t requestedFlags, int max, const bool* nets) {
   if (goodId.size() == 0) {
     int id = -1;
     if (ourId.size() == 0) {
@@ -175,23 +175,28 @@ void CAddrDb::GetIPs_(set<CNetAddr>& ips, int max, const bool* nets) {
     } else {
       id = *ourId.begin();
     }
-    if (id >= 0) {
+    if (id >= 0 && idToInfo[id].services & requestedFlags) {
       ips.insert(idToInfo[id].ip);
     }
     return;
   }
-  if (max > goodId.size() / 2)
-    max = goodId.size() / 2;
+  std::vector<int> goodIdFiltered;
+  for (std::set<int>::const_iterator it = goodId.begin(); it != goodId.end(); it++) {
+    if ((idToInfo[*it].services & requestedFlags) == requestedFlags)
+      goodIdFiltered.push_back(*it);
+  }
+
+  if (!goodIdFiltered.size())
+    return;
+
+  if (max > goodIdFiltered.size() / 2)
+    max = goodIdFiltered.size() / 2;
   if (max < 1)
     max = 1;
-  int low = *goodId.begin();
-  int high = *goodId.rbegin();
+
   set<int> ids;
   while (ids.size() < max) {
-    int range = high-low+1;
-    int pos = low + (rand() % range);
-    int id = *(goodId.lower_bound(pos));
-    ids.insert(id);
+    ids.insert(goodIdFiltered[rand() % goodIdFiltered.size()]);
   }
   for (set<int>::const_iterator it = ids.begin(); it != ids.end(); it++) {
     CService &ip = idToInfo[*it].ip;
diff --git a/db.h b/db.h
@@ -219,7 +219,7 @@ class CAddrDb {
   void Bad_(const CService &ip, int ban);  // mark an IP as bad (and optionally ban it) (must have been returned by Get_)
   void Skipped_(const CService &ip);       // mark an IP as skipped (must have been returned by Get_)
   int Lookup_(const CService &ip);         // look up id of an IP
-  void GetIPs_(std::set<CNetAddr>& ips, int max, const bool *nets); // get a random set of IPs (shared lock only)
+  void GetIPs_(std::set<CNetAddr>& ips, uint64_t requestedFlags, int max, const bool *nets); // get a random set of IPs (shared lock only)
 
 public:
   std::map<CService, time_t> banned; // nodes that are banned, with their unban time (a)
@@ -351,8 +351,8 @@ class CAddrDb {
       }
     }
   }
-  void GetIPs(std::set<CNetAddr>& ips, int max, const bool *nets) {
+  void GetIPs(std::set<CNetAddr>& ips, uint64_t requestedFlags, int max, const bool *nets) {
     SHARED_CRITICAL_BLOCK(cs)
-      GetIPs_(ips, max, nets);
+      GetIPs_(ips, requestedFlags, max, nets);
   }
 };
diff --git a/dns.c b/dns.c
@@ -327,7 +327,7 @@ ssize_t static dnshandle(dns_opt_t *opt, const unsigned char *inbuf, size_t insi
   // A/AAAA records
   if ((typ == TYPE_A || typ == TYPE_AAAA || typ == QTYPE_ANY) && (cls == CLASS_IN || cls == QCLASS_ANY)) {
     addr_t addr[32];
-    int naddr = opt->cb((void*)opt, addr, 32, typ == TYPE_A || typ == QTYPE_ANY, typ == TYPE_AAAA || typ == QTYPE_ANY);
+    int naddr = opt->cb((void*)opt, name, addr, 32, typ == TYPE_A || typ == QTYPE_ANY, typ == TYPE_AAAA || typ == QTYPE_ANY);
     int n = 0;
     while (n < naddr) {
       int ret = 1;
diff --git a/dns.h b/dns.h
@@ -18,7 +18,7 @@ typedef struct {
   const char *host;
   const char *ns;
   const char *mbox;
-  int (*cb)(void *opt, addr_t *addr, int max, int ipv4, int ipv6);
+  int (*cb)(void *opt, char *requested_hostname, addr_t *addr, int max, int ipv4, int ipv6);
   // stats
   uint64_t nRequests;
 } dns_opt_t;
diff --git a/main.cpp b/main.cpp
@@ -15,6 +15,17 @@ using namespace std;
 
 bool fTestNet = false;
 
+uint64_t filter_whitelist[] = {
+  0x0000000000000001,
+  0x0000000000000003,
+  0x0000000000000005,
+  0x0000000000000007,
+  0x0000000000000009,
+  0x000000000000000B,
+  0x000000000000000D,
+  0x000000000000000F,
+};
+
 class CDnsSeedOpts {
 public:
   int nThreads;
@@ -167,53 +178,54 @@ extern "C" void* ThreadCrawler(void* data) {
   } while(1);
 }
 
-extern "C" int GetIPList(void *thread, addr_t *addr, int max, int ipv4, int ipv6);
+extern "C" int GetIPList(void *thread, char *requestedHostname, addr_t *addr, int max, int ipv4, int ipv6);
 
 class CDnsThread {
 public:
   dns_opt_t dns_opt; // must be first
   const int id;
-  vector<addr_t> cache;
+  std::map<uint64_t, vector<addr_t> > cache;
   int nIPv4, nIPv6;
-  time_t cacheTime;
+  std::map<uint64_t, time_t> cacheTime;
   unsigned int cacheHits;
   uint64_t dbQueries;
+  std::vector<uint64_t> filterWhitelist;
 
-  void cacheHit(bool force = false) {
+  void cacheHit(uint64_t requestedFlags, bool force = false) {
     static bool nets[NET_MAX] = {};
     if (!nets[NET_IPV4]) {
         nets[NET_IPV4] = true;
         nets[NET_IPV6] = true;
     }
     time_t now = time(NULL);
     cacheHits++;
-    if (force || cacheHits > (cache.size()*cache.size()/400) || (cacheHits*cacheHits > cache.size() / 20 && (now - cacheTime > 5))) {
+    if (force || cacheHits > (cache[requestedFlags].size()*cache[requestedFlags].size()/400) || (cacheHits*cacheHits > cache[requestedFlags].size() / 20 && (now - cacheTime[requestedFlags] > 5))) {
       set<CNetAddr> ips;
-      db.GetIPs(ips, 1000, nets);
+      db.GetIPs(ips, requestedFlags, 1000, nets);
       dbQueries++;
-      cache.clear();
+      cache[requestedFlags].clear();
       nIPv4 = 0;
       nIPv6 = 0;
-      cache.reserve(ips.size());
+      cache[requestedFlags].reserve(ips.size());
       for (set<CNetAddr>::iterator it = ips.begin(); it != ips.end(); it++) {
         struct in_addr addr;
         struct in6_addr addr6;
         if ((*it).GetInAddr(&addr)) {
           addr_t a;
           a.v = 4;
           memcpy(&a.data.v4, &addr, 4);
-          cache.push_back(a);
+          cache[requestedFlags].push_back(a);
           nIPv4++;
         } else if ((*it).GetIn6Addr(&addr6)) {
           addr_t a;
           a.v = 6;
           memcpy(&a.data.v6, &addr6, 16);
-          cache.push_back(a);
+          cache[requestedFlags].push_back(a);
           nIPv6++;
         }
       }
       cacheHits = 0;
-      cacheTime = now;
+      cacheTime[requestedFlags] = now;
     }
   }
 
@@ -227,24 +239,36 @@ class CDnsThread {
     dns_opt.port = opts->nPort;
     dns_opt.nRequests = 0;
     cache.clear();
-    cache.reserve(1000);
-    cacheTime = 0;
+    cacheTime.clear();
     cacheHits = 0;
     dbQueries = 0;
     nIPv4 = 0;
     nIPv6 = 0;
-    cacheHit(true);
+    filterWhitelist = std::vector<uint64_t>(filter_whitelist, filter_whitelist + (sizeof filter_whitelist / sizeof filter_whitelist[0]));
   }
 
   void run() {
     dnsserver(&dns_opt);
   }
 };
 
-extern "C" int GetIPList(void *data, addr_t* addr, int max, int ipv4, int ipv6) {
+extern "C" int GetIPList(void *data, char *requestedHostname, addr_t* addr, int max, int ipv4, int ipv6) {
   CDnsThread *thread = (CDnsThread*)data;
-  thread->cacheHit();
-  unsigned int size = thread->cache.size();
+
+  uint64_t requestedFlags = 0;
+  int hostlen = strlen(requestedHostname);
+  if (hostlen > 1 && requestedHostname[0] == 'x' && requestedHostname[1] != '0') {
+    char *pEnd;
+    uint64_t flags = (uint64_t)strtoull(requestedHostname+1, &pEnd, 16);
+    if (*pEnd == '.' && pEnd <= requestedHostname+17 && std::find(thread->filterWhitelist.begin(), thread->filterWhitelist.end(), flags) != thread->filterWhitelist.end())
+      requestedFlags = flags;
+    else
+      return 0;
+  }
+  else if (strcasecmp(requestedHostname, thread->dns_opt.host))
+    return 0;
+  thread->cacheHit(requestedFlags);
+  unsigned int size = thread->cache[requestedFlags].size();
   unsigned int maxmax = (ipv4 ? thread->nIPv4 : 0) + (ipv6 ? thread->nIPv6 : 0);
   if (max > size)
     max = size;
@@ -254,16 +278,16 @@ extern "C" int GetIPList(void *data, addr_t* addr, int max, int ipv4, int ipv6)
   while (i<max) {
     int j = i + (rand() % (size - i));
     do {
-        bool ok = (ipv4 && thread->cache[j].v == 4) || 
-                  (ipv6 && thread->cache[j].v == 6);
+        bool ok = (ipv4 && thread->cache[requestedFlags][j].v == 4) ||
+                  (ipv6 && thread->cache[requestedFlags][j].v == 6);
         if (ok) break;
         j++;
         if (j==size)
             j=i;
     } while(1);
-    addr[i] = thread->cache[j];
-    thread->cache[j] = thread->cache[i];
-    thread->cache[i] = addr[i];
+    addr[i] = thread->cache[requestedFlags][j];
+    thread->cache[requestedFlags][j] = thread->cache[requestedFlags][i];
+    thread->cache[requestedFlags][i] = addr[i];
     i++;
   }
   return max;
@@ -306,11 +330,11 @@ extern "C" void* ThreadDumper(void*) {
         rename("dnsseed.dat.new", "dnsseed.dat");
       }
       FILE *d = fopen("dnsseed.dump", "w");
-      fprintf(d, "# address                                        good  lastSuccess    %%(2h)   %%(8h)   %%(1d)   %%(7d)  %%(30d)  blocks      svcs  version\n");
+      fprintf(d, "# address                                        servicebits good  lastSuccess    %%(2h)   %%(8h)   %%(1d)   %%(7d)  %%(30d)  blocks      svcs  version\n");
       double stat[5]={0,0,0,0,0};
       for (vector<CAddrReport>::const_iterator it = v.begin(); it < v.end(); it++) {
         CAddrReport rep = *it;
-        fprintf(d, "%-47s  %4d  %11"PRId64"  %6.2f%% %6.2f%% %6.2f%% %6.2f%% %6.2f%%  %6i  %08"PRIx64"  %5i \"%s\"\n", rep.ip.ToString().c_str(), (int)rep.fGood, rep.lastSuccess, 100.0*rep.uptime[0], 100.0*rep.uptime[1], 100.0*rep.uptime[2], 100.0*rep.uptime[3], 100.0*rep.uptime[4], rep.blocks, rep.services, rep.clientVersion, rep.clientSubVersion.c_str());
+        fprintf(d, "%-47s  %8lld %4d  %11"PRId64"  %6.2f%% %6.2f%% %6.2f%% %6.2f%% %6.2f%%  %6i  %08"PRIx64"  %5i \"%s\"\n", rep.ip.ToString().c_str(), (uint64_t)rep.services, (int)rep.fGood, rep.lastSuccess, 100.0*rep.uptime[0], 100.0*rep.uptime[1], 100.0*rep.uptime[2], 100.0*rep.uptime[3], 100.0*rep.uptime[4], rep.blocks, rep.services, rep.clientVersion, rep.clientSubVersion.c_str());
         stat[0] += rep.uptime[0];
         stat[1] += rep.uptime[1];
         stat[2] += rep.uptime[2];

Original file line number	Diff line number	Diff line change
`@@ -219,7 +219,7 @@ class CAddrDb {`
`219`	`219`	`void Bad_(const CService &ip, int ban); // mark an IP as bad (and optionally ban it) (must have been returned by Get_)`
`220`	`220`	`void Skipped_(const CService &ip); // mark an IP as skipped (must have been returned by Get_)`
`221`	`221`	`int Lookup_(const CService &ip); // look up id of an IP`
`222`		`- void GetIPs_(std::set<CNetAddr>& ips, int max, const bool *nets); // get a random set of IPs (shared lock only)`
	`222`	`+ void GetIPs_(std::set<CNetAddr>& ips, uint64_t requestedFlags, int max, const bool *nets); // get a random set of IPs (shared lock only)`
`223`	`223`
`224`	`224`	`public:`
`225`	`225`	`std::map<CService, time_t> banned; // nodes that are banned, with their unban time (a)`
`@@ -351,8 +351,8 @@ class CAddrDb {`
`351`	`351`	`}`
`352`	`352`	`}`
`353`	`353`	`}`
`354`		`- void GetIPs(std::set<CNetAddr>& ips, int max, const bool *nets) {`
	`354`	`+ void GetIPs(std::set<CNetAddr>& ips, uint64_t requestedFlags, int max, const bool *nets) {`
`355`	`355`	`SHARED_CRITICAL_BLOCK(cs)`
`356`		`- GetIPs_(ips, max, nets);`
	`356`	`+ GetIPs_(ips, requestedFlags, max, nets);`
`357`	`357`	`}`
`358`	`358`	`};`