Skip to content

Commit e689ea0

Browse files
ayeowchayeowch
committed
Deserialize I2P and CJDNS addresses
1 parent ce07c19 commit e689ea0

9 files changed

+186
-75
lines changed

‎crawl.py

+47-22
Original file line numberDiff line numberDiff line change
@@ -32,33 +32,33 @@
3232

3333
monkey.patch_all()
3434

35-
import geoip2.database
36-
import gevent
3735
import json
3836
import logging
3937
import os
4038
import random
41-
import redis.connection
4239
import socket
4340
import sys
4441
import time
45-
from binascii import hexlify
46-
from binascii import unhexlify
4742
from collections import Counter
4843
from configparser import ConfigParser
44+
from ipaddress import ip_address, ip_network
45+
46+
import geoip2.database
47+
import gevent
48+
import redis.connection
49+
from binascii import hexlify, unhexlify
4950
from geoip2.errors import AddressNotFoundError
50-
from ipaddress import ip_address
51-
from ipaddress import ip_network
52-
53-
from protocol import Connection
54-
from protocol import ConnectionError
55-
from protocol import ProtocolError
56-
from protocol import TO_SERVICES
57-
from utils import conf_list
58-
from utils import get_keys
59-
from utils import http_get_txt
60-
from utils import ip_to_network
61-
from utils import new_redis_conn
51+
52+
from protocol import (
53+
CJDNS_NETWORK,
54+
Connection,
55+
ConnectionError,
56+
I2P_SUFFIX,
57+
ONION_SUFFIX,
58+
ProtocolError,
59+
TO_SERVICES,
60+
)
61+
from utils import conf_list, get_keys, http_get_txt, ip_to_network, new_redis_conn
6262

6363
redis.connection.socket = gevent.socket
6464

@@ -112,7 +112,13 @@ def get_peers(conn):
112112
age = now - timestamp # seconds
113113
if age < 0 or age > CONF["max_age"]:
114114
continue
115-
address = peer["ipv4"] or peer["ipv6"] or peer["onion"]
115+
address = (
116+
peer["ipv4"]
117+
or peer["ipv6"]
118+
or peer["onion"]
119+
or peer["i2p"]
120+
or peer["cjdns"]
121+
)
116122
port = peer["port"] if peer["port"] > 0 else CONF["port"]
117123
services = peer["services"]
118124
if not address:
@@ -180,7 +186,7 @@ def connect(key, redis_conn):
180186
height = int(height)
181187

182188
proxy = None
183-
if address.endswith(".onion") and CONF["onion"]:
189+
if address.endswith(ONION_SUFFIX) and CONF["onion"]:
184190
proxy = random.choice(CONF["tor_proxies"])
185191

186192
conn = Connection(
@@ -230,6 +236,15 @@ def connect(key, redis_conn):
230236

231237
peers = get_cached_peers(conn, redis_conn)
232238
for peer in peers:
239+
# I2P and CJDNS peers are cached but not crawled.
240+
address = peer[0]
241+
if address.endswith(I2P_SUFFIX):
242+
continue
243+
if (
244+
not address.endswith(ONION_SUFFIX)
245+
and ip_address(address) in CJDNS_NETWORK
246+
):
247+
continue
233248
redis_pipe.sadd("pending", str(peer))
234249
redis_pipe.set(key, "")
235250
redis_pipe.sadd("up", key)
@@ -370,7 +385,7 @@ def task(redis_conn):
370385
continue
371386

372387
# Skip .onion node.
373-
if node[0].endswith(".onion") and not CONF["onion"]:
388+
if node[0].endswith(ONION_SUFFIX) and not CONF["onion"]:
374389
continue
375390

376391
key = f"node:{node[0]}-{node[1]}-{node[2]}"
@@ -429,6 +444,8 @@ def is_excluded(address):
429444
430445
In priority order, the rules are:
431446
- Include onion address
447+
- Include I2P address
448+
- Include CJDNS address
432449
- Exclude private address
433450
- Exclude address without ASN when include_asns/exclude_asns is set
434451
- Exclude if address is in exclude_asns
@@ -437,10 +454,18 @@ def is_excluded(address):
437454
- Exclude if address is not in include_asns
438455
- Include address
439456
"""
440-
if address.endswith(".onion"):
457+
if address.endswith(ONION_SUFFIX):
458+
return False
459+
460+
if address.endswith(I2P_SUFFIX):
461+
return False
462+
463+
ip_obj = ip_address(address)
464+
465+
if ip_obj in CJDNS_NETWORK:
441466
return False
442467

443-
if ip_address(address).is_private:
468+
if ip_obj.is_private:
444469
return True
445470

446471
include_asns = CONF["current_include_asns"]

‎ping.py

+9-13
Original file line numberDiff line numberDiff line change
@@ -32,27 +32,23 @@
3232

3333
monkey.patch_all()
3434

35-
import gevent
36-
import gevent.pool
3735
import glob
3836
import json
3937
import logging
4038
import os
4139
import random
42-
import redis.connection
4340
import socket
4441
import sys
4542
import time
46-
from binascii import hexlify
47-
from binascii import unhexlify
4843
from configparser import ConfigParser
4944

50-
from protocol import Connection
51-
from protocol import ConnectionError
52-
from protocol import ProtocolError
53-
from utils import get_keys
54-
from utils import ip_to_network
55-
from utils import new_redis_conn
45+
import gevent
46+
import gevent.pool
47+
import redis.connection
48+
from binascii import hexlify, unhexlify
49+
50+
from protocol import Connection, ConnectionError, ONION_SUFFIX, ProtocolError
51+
from utils import get_keys, ip_to_network, new_redis_conn
5652

5753
redis.connection.socket = gevent.socket
5854

@@ -225,7 +221,7 @@ def task(redis_conn):
225221

226222
relay = CONF["relay"]
227223
proxy = None
228-
if address.endswith(".onion") and CONF["onion"]:
224+
if address.endswith(ONION_SUFFIX) and CONF["onion"]:
229225
relay = CONF["onion_relay"]
230226
proxy = random.choice(CONF["tor_proxies"])
231227

@@ -258,7 +254,7 @@ def task(redis_conn):
258254
redis_conn.srem("open", str(node))
259255
return
260256

261-
if address.endswith(".onion"):
257+
if address.endswith(ONION_SUFFIX):
262258
# Map local port to .onion node.
263259
local_port = conn.socket.getsockname()[1]
264260
logging.debug(f"Local port {conn.to_addr}: {local_port}")

‎protocol.py

+33-25
Original file line numberDiff line numberDiff line change
@@ -148,6 +148,7 @@
148148
from base64 import b32decode, b32encode
149149
from collections import deque
150150
from io import BytesIO, SEEK_CUR
151+
from ipaddress import ip_network
151152

152153
import gevent
153154
import socks
@@ -160,7 +161,7 @@
160161
FROM_SERVICES = 0
161162
TO_SERVICES = 1 # NODE_NETWORK
162163
USER_AGENT = "/bitnodes.io:0.3/"
163-
HEIGHT = 754565
164+
HEIGHT = 872447
164165
RELAY = 0 # Set to 1 to receive all txs.
165166

166167
SOCKET_BUFSIZE = 8192
@@ -187,15 +188,14 @@
187188
NETWORK_CJDNS: 16,
188189
}
189190

190-
SUPPORTED_NETWORKS = [
191-
NETWORK_IPV4,
192-
NETWORK_IPV6,
193-
NETWORK_TORV2,
194-
NETWORK_TORV3,
195-
]
196-
197191
ONION_V3_LEN = 62
198192

193+
ONION_SUFFIX = ".onion"
194+
I2P_SUFFIX = ".b32.i2p"
195+
196+
# CJDNS address uses fc00::/8 reserved IPv6 range.
197+
CJDNS_NETWORK = ip_network("fc00::/8")
198+
199199

200200
class ProtocolError(Exception):
201201
pass
@@ -229,10 +229,6 @@ class UnknownNetworkIdError(ProtocolError):
229229
pass
230230

231231

232-
class UnsupportedNetworkIdError(ProtocolError):
233-
pass
234-
235-
236232
class InvalidAddrLenError(ProtocolError):
237233
pass
238234

@@ -257,19 +253,26 @@ def addr_to_onion_v2(addr):
257253
"""
258254
Returns .onion address for the specified v2 onion addr.
259255
"""
260-
return (b32encode(addr).lower() + b".onion").decode()
256+
return b32encode(addr).lower().decode() + ONION_SUFFIX
261257

262258

263259
def addr_to_onion_v3(addr):
264260
"""
265261
Returns .onion address for the specified v3 onion addr (PUBKEY).
266262
267263
onion_address = base32(PUBKEY | CHECKSUM | VERSION) + '.onion'
268-
See https://gitweb.torproject.org/torspec.git/tree/rend-spec-v3.txt#n2135
264+
See https://spec.torproject.org/rend-spec/encoding-onion-addresses.html
269265
"""
270266
version = b"\x03"
271267
checksum = hashlib.sha3_256(b".onion checksum" + addr + version).digest()[:2]
272-
return (b32encode(addr + checksum + version).lower() + b".onion").decode()
268+
return b32encode(addr + checksum + version).lower().decode() + ONION_SUFFIX
269+
270+
271+
def addr_to_i2p(addr):
272+
"""
273+
Returns .b32.i2p address (base32(SHA256) + '.b32.i2p').
274+
"""
275+
return b32encode(addr).decode().replace("=", "").lower() + I2P_SUFFIX
273276

274277

275278
def unpack(fmt, string):
@@ -283,7 +286,7 @@ def unpack(fmt, string):
283286

284287

285288
def create_connection(address, timeout=SOCKET_TIMEOUT, source_address=None, proxy=None):
286-
if address[0].endswith(".onion") and proxy is None:
289+
if address[0].endswith(ONION_SUFFIX) and proxy is None:
287290
raise ProxyRequired("tor proxy is required to connect to .onion address")
288291
if proxy:
289292
socks.setdefaultproxy(socks.PROXY_TYPE_SOCKS5, proxy[0], proxy[1])
@@ -640,7 +643,7 @@ def serialize_network_address(self, addr, version=None):
640643
else:
641644
(services, ip_address, port) = addr
642645

643-
if ip_address.endswith(".onion"):
646+
if ip_address.endswith(ONION_SUFFIX):
644647
if len(ip_address) == ONION_V3_LEN:
645648
network_id = NETWORK_TORV3
646649
else:
@@ -690,6 +693,8 @@ def deserialize_network_address(self, data, has_timestamp=False, version=None):
690693
ipv4 = ""
691694
ipv6 = ""
692695
onion = ""
696+
i2p = ""
697+
cjdns = ""
693698

694699
timestamp = None
695700
if has_timestamp:
@@ -703,22 +708,23 @@ def deserialize_network_address(self, data, has_timestamp=False, version=None):
703708
if network_id not in NETWORK_LENGTHS.keys():
704709
raise UnknownNetworkIdError(f"unknown network id {network_id}")
705710

706-
if network_id not in SUPPORTED_NETWORKS:
707-
raise UnsupportedNetworkIdError(f"unsupported network id {network_id}")
708-
709711
addr_len = self.deserialize_int(data)
710712
if addr_len != NETWORK_LENGTHS[network_id]:
711713
raise InvalidAddrLenError
712714

713715
addr = data.read(addr_len)
714-
if network_id == NETWORK_TORV2:
716+
if network_id == NETWORK_IPV4:
717+
ipv4 = socket.inet_ntop(socket.AF_INET, addr)
718+
elif network_id == NETWORK_IPV6:
719+
ipv6 = socket.inet_ntop(socket.AF_INET6, addr)
720+
elif network_id == NETWORK_TORV2:
715721
onion = addr_to_onion_v2(addr)
716722
elif network_id == NETWORK_TORV3:
717723
onion = addr_to_onion_v3(addr)
718-
elif network_id == NETWORK_IPV6:
719-
ipv6 = socket.inet_ntop(socket.AF_INET6, addr)
720-
elif network_id == NETWORK_IPV4:
721-
ipv4 = socket.inet_ntop(socket.AF_INET, addr)
724+
elif network_id == NETWORK_I2P:
725+
i2p = addr_to_i2p(addr)
726+
elif network_id == NETWORK_CJDNS:
727+
cjdns = socket.inet_ntop(socket.AF_INET6, addr)
722728

723729
port = unpack(">H", data.read(2))
724730
else:
@@ -750,6 +756,8 @@ def deserialize_network_address(self, data, has_timestamp=False, version=None):
750756
"ipv4": ipv4,
751757
"ipv6": ipv6,
752758
"onion": onion,
759+
"i2p": i2p,
760+
"cjdns": cjdns,
753761
"port": port,
754762
}
755763

‎resolve.py

+10-10
Original file line numberDiff line numberDiff line change
@@ -32,23 +32,23 @@
3232

3333
monkey.patch_all()
3434

35-
import gevent
36-
import gevent.pool
3735
import logging
3836
import os
39-
import redis.connection
4037
import socket
4138
import sys
4239
import time
43-
from binascii import hexlify
44-
from binascii import unhexlify
4540
from collections import defaultdict
4641
from configparser import ConfigParser
4742
from decimal import Decimal
43+
44+
import gevent
45+
import gevent.pool
46+
import redis.connection
47+
from binascii import hexlify, unhexlify
4848
from geoip2.errors import AddressNotFoundError
4949

50-
from utils import GeoIp
51-
from utils import new_redis_conn
50+
from protocol import ONION_SUFFIX
51+
from utils import GeoIp, new_redis_conn
5252

5353
redis.connection.socket = gevent.socket
5454

@@ -87,7 +87,7 @@ def resolve_addresses(self):
8787
if ttl < 0.1 * CONF["ttl"]: # Less than 10% of initial TTL.
8888
expiring = True
8989

90-
if expiring and idx < 1000 and not address.endswith(".onion"):
90+
if expiring and idx < 1000 and not address.endswith(ONION_SUFFIX):
9191
self.resolved["hostname"][address] = None
9292
idx += 1
9393

@@ -181,7 +181,7 @@ def raw_geoip(self, address):
181181
asn = None
182182
org = None
183183

184-
if not address.endswith(".onion"):
184+
if not address.endswith(ONION_SUFFIX):
185185
try:
186186
gcountry = self.geoip.country(address)
187187
except AddressNotFoundError:
@@ -203,7 +203,7 @@ def raw_geoip(self, address):
203203
lng = float(Decimal(gcity.location.longitude).quantize(GEO_PREC))
204204
timezone = gcity.location.time_zone
205205

206-
if address.endswith(".onion"):
206+
if address.endswith(ONION_SUFFIX):
207207
asn = "TOR"
208208
org = "Tor network"
209209
else:

0 commit comments

Comments
 (0)