|
| 1 | +#!/usr/bin/env python3 |
| 2 | + |
| 3 | +""" Demonstration of eBPF limitations and the effect on USDT with the |
| 4 | + net:inbound_message and net:outbound_message tracepoints. """ |
| 5 | + |
| 6 | +# This script shows a limitation of eBPF when data larger than 32kb is passed to |
| 7 | +# user-space. It uses BCC (https://github.com/iovisor/bcc) to load a sandboxed |
| 8 | +# eBPF program into the Linux kernel (root privileges are required). The eBPF |
| 9 | +# program attaches to two statically defined tracepoints. The tracepoint |
| 10 | +# 'net:inbound_message' is called when a new P2P message is received, and |
| 11 | +# 'net:outbound_message' is called on outbound P2P messages. The eBPF program |
| 12 | +# submits the P2P messages to this script via a BPF ring buffer. The submitted |
| 13 | +# messages are printed. |
| 14 | + |
| 15 | +# eBPF Limitations: |
| 16 | +# |
| 17 | +# Bitcoin P2P messages can be larger than 32kb (e.g. tx, block, ...). The eBPF |
| 18 | +# VM's stack is limited to 512 bytes, and we can't allocate more than about 32kb |
| 19 | +# for a P2P message in the eBPF VM. The message data is cut off when the message |
| 20 | +# is larger than MAX_MSG_DATA_LENGTH (see definition below). This can be detected |
| 21 | +# in user-space by comparing the data length to the message length variable. The |
| 22 | +# message is cut off when the data length is smaller than the message length. |
| 23 | +# A warning is included with the printed message data. |
| 24 | +# |
| 25 | +# Data is submitted to user-space (i.e. to this script) via a ring buffer. The |
| 26 | +# throughput of the ring buffer is limited. Each p2p_message is about 32kb in |
| 27 | +# size. In- or outbound messages submitted to the ring buffer in rapid |
| 28 | +# succession fill the ring buffer faster than it can be read. Some messages are |
| 29 | +# lost. |
| 30 | +# |
| 31 | +# BCC prints: "Possibly lost 2 samples" on lost messages. |
| 32 | + |
| 33 | +import sys |
| 34 | +from bcc import BPF, USDT |
| 35 | + |
| 36 | +# BCC: The C program to be compiled to an eBPF program (by BCC) and loaded into |
| 37 | +# a sandboxed Linux kernel VM. |
| 38 | +program = """ |
| 39 | +#include <uapi/linux/ptrace.h> |
| 40 | +
|
| 41 | +#define MIN(a,b) ({ __typeof__ (a) _a = (a); __typeof__ (b) _b = (b); _a < _b ? _a : _b; }) |
| 42 | +
|
| 43 | +// Maximum possible allocation size |
| 44 | +// from include/linux/percpu.h in the Linux kernel |
| 45 | +#define PCPU_MIN_UNIT_SIZE (32 << 10) |
| 46 | +
|
| 47 | +// Tor v3 addresses are 62 chars + 6 chars for the port (':12345'). |
| 48 | +#define MAX_PEER_ADDR_LENGTH 62 + 6 |
| 49 | +#define MAX_PEER_CONN_TYPE_LENGTH 20 |
| 50 | +#define MAX_MSG_TYPE_LENGTH 20 |
| 51 | +#define MAX_MSG_DATA_LENGTH PCPU_MIN_UNIT_SIZE - 200 |
| 52 | +
|
| 53 | +struct p2p_message |
| 54 | +{ |
| 55 | + u64 peer_id; |
| 56 | + char peer_addr[MAX_PEER_ADDR_LENGTH]; |
| 57 | + char peer_conn_type[MAX_PEER_CONN_TYPE_LENGTH]; |
| 58 | + char msg_type[MAX_MSG_TYPE_LENGTH]; |
| 59 | + u64 msg_size; |
| 60 | + u8 msg[MAX_MSG_DATA_LENGTH]; |
| 61 | +}; |
| 62 | +
|
| 63 | +// We can't store the p2p_message struct on the eBPF stack as it is limited to |
| 64 | +// 512 bytes and P2P message can be bigger than 512 bytes. However, we can use |
| 65 | +// an BPF-array with a length of 1 to allocate up to 32768 bytes (this is |
| 66 | +// defined by PCPU_MIN_UNIT_SIZE in include/linux/percpu.h in the Linux kernel). |
| 67 | +// Also see https://github.com/iovisor/bcc/issues/2306 |
| 68 | +BPF_ARRAY(msg_arr, struct p2p_message, 1); |
| 69 | +
|
| 70 | +// Two BPF perf buffers for pushing data (here P2P messages) to user-space. |
| 71 | +BPF_PERF_OUTPUT(inbound_messages); |
| 72 | +BPF_PERF_OUTPUT(outbound_messages); |
| 73 | +
|
| 74 | +int trace_inbound_message(struct pt_regs *ctx) { |
| 75 | + int idx = 0; |
| 76 | + struct p2p_message *msg = msg_arr.lookup(&idx); |
| 77 | +
|
| 78 | + // lookup() does not return a NULL pointer. However, the BPF verifier |
| 79 | + // requires an explicit check that that the `msg` pointer isn't a NULL |
| 80 | + // pointer. See https://github.com/iovisor/bcc/issues/2595 |
| 81 | + if (msg == NULL) return 1; |
| 82 | +
|
| 83 | + bpf_usdt_readarg(1, ctx, &msg->peer_id); |
| 84 | + bpf_usdt_readarg_p(2, ctx, &msg->peer_addr, MAX_PEER_ADDR_LENGTH); |
| 85 | + bpf_usdt_readarg_p(3, ctx, &msg->peer_conn_type, MAX_PEER_CONN_TYPE_LENGTH); |
| 86 | + bpf_usdt_readarg_p(4, ctx, &msg->msg_type, MAX_MSG_TYPE_LENGTH); |
| 87 | + bpf_usdt_readarg(5, ctx, &msg->msg_size); |
| 88 | + bpf_usdt_readarg_p(6, ctx, &msg->msg, MIN(msg->msg_size, MAX_MSG_DATA_LENGTH)); |
| 89 | +
|
| 90 | + inbound_messages.perf_submit(ctx, msg, sizeof(*msg)); |
| 91 | + return 0; |
| 92 | +}; |
| 93 | +
|
| 94 | +int trace_outbound_message(struct pt_regs *ctx) { |
| 95 | + int idx = 0; |
| 96 | + struct p2p_message *msg = msg_arr.lookup(&idx); |
| 97 | +
|
| 98 | + // lookup() does not return a NULL pointer. However, the BPF verifier |
| 99 | + // requires an explicit check that that the `msg` pointer isn't a NULL |
| 100 | + // pointer. See https://github.com/iovisor/bcc/issues/2595 |
| 101 | + if (msg == NULL) return 1; |
| 102 | +
|
| 103 | + bpf_usdt_readarg(1, ctx, &msg->peer_id); |
| 104 | + bpf_usdt_readarg_p(2, ctx, &msg->peer_addr, MAX_PEER_ADDR_LENGTH); |
| 105 | + bpf_usdt_readarg_p(3, ctx, &msg->peer_conn_type, MAX_PEER_CONN_TYPE_LENGTH); |
| 106 | + bpf_usdt_readarg_p(4, ctx, &msg->msg_type, MAX_MSG_TYPE_LENGTH); |
| 107 | + bpf_usdt_readarg(5, ctx, &msg->msg_size); |
| 108 | + bpf_usdt_readarg_p(6, ctx, &msg->msg, MIN(msg->msg_size, MAX_MSG_DATA_LENGTH)); |
| 109 | +
|
| 110 | + outbound_messages.perf_submit(ctx, msg, sizeof(*msg)); |
| 111 | + return 0; |
| 112 | +}; |
| 113 | +""" |
| 114 | + |
| 115 | + |
| 116 | +def print_message(event, inbound): |
| 117 | + print(f"%s %s msg '%s' from peer %d (%s, %s) with %d bytes: %s" % |
| 118 | + ( |
| 119 | + f"Warning: incomplete message (only %d out of %d bytes)!" % ( |
| 120 | + len(event.msg), event.msg_size) if len(event.msg) < event.msg_size else "", |
| 121 | + "inbound" if inbound else "outbound", |
| 122 | + event.msg_type.decode("utf-8"), |
| 123 | + event.peer_id, |
| 124 | + event.peer_conn_type.decode("utf-8"), |
| 125 | + event.peer_addr.decode("utf-8"), |
| 126 | + event.msg_size, |
| 127 | + bytes(event.msg[:event.msg_size]).hex(), |
| 128 | + ) |
| 129 | + ) |
| 130 | + |
| 131 | + |
| 132 | +def main(bitcoind_path): |
| 133 | + bitcoind_with_usdts = USDT(path=str(bitcoind_path)) |
| 134 | + |
| 135 | + # attaching the trace functions defined in the BPF program to the tracepoints |
| 136 | + bitcoind_with_usdts.enable_probe( |
| 137 | + probe="inbound_message", fn_name="trace_inbound_message") |
| 138 | + bitcoind_with_usdts.enable_probe( |
| 139 | + probe="outbound_message", fn_name="trace_outbound_message") |
| 140 | + bpf = BPF(text=program, usdt_contexts=[bitcoind_with_usdts]) |
| 141 | + |
| 142 | + # BCC: perf buffer handle function for inbound_messages |
| 143 | + def handle_inbound(_, data, size): |
| 144 | + """ Inbound message handler. |
| 145 | +
|
| 146 | + Called each time a message is submitted to the inbound_messages BPF table.""" |
| 147 | + |
| 148 | + event = bpf["inbound_messages"].event(data) |
| 149 | + print_message(event, True) |
| 150 | + |
| 151 | + # BCC: perf buffer handle function for outbound_messages |
| 152 | + |
| 153 | + def handle_outbound(_, data, size): |
| 154 | + """ Outbound message handler. |
| 155 | +
|
| 156 | + Called each time a message is submitted to the outbound_messages BPF table.""" |
| 157 | + |
| 158 | + event = bpf["outbound_messages"].event(data) |
| 159 | + print_message(event, False) |
| 160 | + |
| 161 | + # BCC: add handlers to the inbound and outbound perf buffers |
| 162 | + bpf["inbound_messages"].open_perf_buffer(handle_inbound) |
| 163 | + bpf["outbound_messages"].open_perf_buffer(handle_outbound) |
| 164 | + |
| 165 | + print("Logging raw P2P messages.") |
| 166 | + print("Messages larger that about 32kb will be cut off!") |
| 167 | + print("Some messages might be lost!") |
| 168 | + while True: |
| 169 | + try: |
| 170 | + bpf.perf_buffer_poll() |
| 171 | + except KeyboardInterrupt: |
| 172 | + exit() |
| 173 | + |
| 174 | + |
| 175 | +if __name__ == "__main__": |
| 176 | + if len(sys.argv) < 2: |
| 177 | + print("USAGE:", sys.argv[0], "path/to/bitcoind") |
| 178 | + exit() |
| 179 | + path = sys.argv[1] |
| 180 | + main(path) |
0 commit comments