bpftune: add adaptive sampling

One approach to reducing overhead is to introduce sampling; i.e.
a high-frequency program bails early and only does more expensive
work for 1 out of every N invokations.  bpftune has adaptive sampling
to support this.  It works as follows.

In your BPF object, add a struct bpftune_sample variable for the
BPF program you wish to sample.  For example in tcp_buffer_tuner.bpf.c
we have

```
struct bpftune_sample rcv_space_sample = { };
```

Since tcp_rcv_space_adjust() is called frequently, add the following
early in function lifetime:

```
        bpftune_sample(rcv_space_sample);
```

The above will bail if it is not the Nth invokation of the program.
Note that N is usually bpftune_sample_rate - default 4 - but will
be adaptively adjusted if the program runs too frequently.  Too
frequently is 2N instances - i.e. collecting data twice - in a 10 msec
interval.  In such cases we double the sample rate.  Similarly if
we fall outside that range we lower the sample rate towards
bpftune_sample_rate, so over time it should adjust to handle the
rate of invokations adaptively.

To add reporting to your tuner on exit (how many times the program
was invoked and what fraction of these we collected data for),
in the init method add

```
                bpftuner_bpf_sample_add(tcp_buffer, tuner, rcv_space_sample);
```

Then on tuner fini, you will see something like:

```
bpftune: Sample 'rcv_space_sample': associated program was called 598663 times, collected data every 2048 of these.
```

In this case the frequency dictated we increase the fractional rate of
data collection from 1 in 4 to 1 in 2048.

With this in place, overheads recorded via stress-ng are much reduced.

Signed-off-by: Alan Maguire <[email protected]>

Author: alan-maguire

CONTRIBUTING.md

@@ -232,6 +232,52 @@ $ TUNER=tcp_buffer_tuner.so sh qperf_test.sh
 
 Replace TUNER value with the name of the tuner you want to assess.
 
+One approach to reducing overhead is to introduce sampling; i.e.
+a high-frequency program bails early and only does more expensive
+work for 1 out of every N invokations.  bpftune has adaptive sampling
+to support this.  It works as follows.
+
+In your BPF object, add a struct bpftune_sample variable for the
+BPF program you wish to sample.  For example in tcp_buffer_tuner.bpf.c
+we have
+
+```
+struct bpftune_sample rcv_space_sample = { };
+```
+
+Since tcp_rcv_space_adjust() is called frequently, add the following
+early in function lifetime:
+
+```
+	bpftune_sample(rcv_space_sample);
+```
+
+The above will bail if it is not the Nth invokation of the program.
+Note that N is usually bpftune_sample_rate - default 4 - but will
+be adaptively adjusted if the program runs too frequently.  Too
+frequently is 2N instances - i.e. collecting data twice - in a 10 msec
+interval.  In such cases we double the sample rate.  Similarly if
+we fall outside that range we lower the sample rate towards
+bpftune_sample_rate, so over time it should adjust to handle the
+rate of invokations adaptively.
+
+To add reporting to your tuner on exit (how many times the program
+was invoked and what fraction of these we collected data for),
+in the init method add
+
+```
+	        bpftuner_bpf_sample_add(tcp_buffer, tuner, rcv_space_sample);
+```
+
+Then on tuner fini, you will see something like:
+
+```
+bpftune: Sample 'rcv_space_sample': associated program was called 598663 times, collected data every 2048 of these.
+```
+
+In this case the frequency dictated we increase the fractional rate of
+data collection from 1 in 4 to 1 in 2048.
+
 ## Tests
 
 Tests are mandatory for tuners; in the test directory you can see

include/bpftune/bpftune.bpf.h

@@ -274,8 +274,31 @@ unsigned long long bpftune_init_net;
 bool debug;
 
 unsigned int bpftune_sample_rate = 4;
-
-#define bpftune_skip_sample(count)	((++count % bpftune_sample_rate) != 0)
+ 
+#define TEN_MSEC	((__u64)10000000)
+
+/* Auto-tune sample rate. Start with bpftune_sample_rate, and only
+ * sample every bpftune_sample_rate samples (default 4).  However
+ * if we see multiple events in a 10msec window, double the sample
+ * rate such that we only sample 1/8 etc.  Similarly lower the sample
+ * rate if we have previously increased it and not seen as many samples
+ * in that window.
+ */
+#define bpftune_sample(sample)					\
+do {								\
+	__u64 last_ts = 0;					\
+								\
+	if (!sample.rate)					\
+		sample.rate = bpftune_sample_rate;		\
+	if (((++sample.count) % sample.rate) != 0)		\
+		return 0;					\
+	last_ts = sample.ts;					\
+	sample.ts = bpf_ktime_get_ns();				\
+	if ((sample.ts - last_ts) < TEN_MSEC)			\
+		sample.rate = sample.rate << 1;			\
+	else if (sample.rate > bpftune_sample_rate)		\
+		sample.rate = sample.rate >> 1;			\
+} while (0)
 
 #define __barrier asm volatile("" ::: "memory")
 

include/bpftune/bpftune.h

@@ -160,6 +160,19 @@ enum bpftune_support_level {
 	BPFTUNE_SUPPORT_NORMAL
 };
 
+struct bpftune_sample {
+	__u64 count;
+	__u64 ts;
+	__u64 rate;
+};
+
+struct bpftune_sample_desc {
+	const char *name;
+	struct bpftune_sample *sample;
+};
+
+#define BPFTUNE_MAX_SAMPLES	8
+
 struct bpftuner {
 	unsigned int id;
 	enum bpftune_state state;
@@ -189,6 +202,8 @@ struct bpftuner {
 	struct bpftunable *tunables;
 	unsigned int num_scenarios;
 	struct bpftunable_scenario *scenarios;
+	unsigned int num_samples;
+	struct bpftune_sample_desc samples[BPFTUNE_MAX_SAMPLES];
 };
 
 /* from include/linux/log2.h */

include/bpftune/libbpftune.h

@@ -312,6 +312,32 @@ void bpftuner_tunables_fini(struct bpftuner *tuner);
 #define bpftuner_bpf_map_get(tuner_name, tuner, map)			     \
 	bpftuner_bpf_skel_val(tuner_name, tuner, maps.map)
 
+#define bpftuner_bpf_sample_add(tuner_name, tuner, s)		     	     \
+	do {								     \
+		struct tuner_name##_tuner_bpf *__skel = tuner->skel;	     \
+		struct tuner_name##_tuner_bpf_legacy *__lskel = tuner->skel; \
+		struct tuner_name##_tuner_bpf_nobtf *__nskel = tuner->skel;  \
+		struct bpftune_sample_desc *d;				     \
+		d = &tuner->samples[tuner->num_samples];		     \
+		d->name = #s;						     \
+		switch (tuner->bpf_support) {				     \
+                case BPFTUNE_SUPPORT_NORMAL:				     \
+                        d->sample = &__skel->bss->s;			     \
+                        break;                                               \
+                case BPFTUNE_SUPPORT_LEGACY:                                 \
+                        d->sample = &__lskel->bss->s;			     \
+                        break;                                               \
+                case BPFTUNE_SUPPORT_NOBTF:                                  \
+                        d->sample = &__nskel->bss->s;			     \
+                default:						     \
+                        break;                                               \
+                }                                                            \
+		tuner->num_samples++;					     \
+                bpftune_log(LOG_DEBUG, "%s: added sample '%s'\n",	     \
+                            #tuner_name, #s);				     \
+	} while (0)
+
+
 enum bpftune_support_level bpftune_bpf_support(void);
 bool bpftune_have_vmlinux_btf(void);
 void bpftune_force_bpf_support(enum bpftune_support_level);

src/libbpftune.c

@@ -791,6 +791,13 @@ void bpftuner_fini(struct bpftuner *tuner, enum bpftune_state state)
 
 	bpftune_log(LOG_DEBUG, "cleaning up tuner %s with %d tunables, %d scenarios\n",
 		    tuner->name, tuner->num_tunables, tuner->num_scenarios);
+	/* Show sample data before destroying BPF skeleton */
+	for (i = 0; i < tuner->num_samples; i++) {
+		bpftune_log(BPFTUNE_LOG_LEVEL, "Sample '%s': associated program was called %lu times, collected data every %lu of these.\n",
+			    tuner->samples[i].name,
+			    tuner->samples[i].sample->count,
+			    tuner->samples[i].sample->rate);
+	}
 	if (tuner->fini)
 		tuner->fini(tuner);
 	/* report summary of events for tuner */

src/net_buffer_tuner.bpf.c

@@ -33,6 +33,8 @@ int netdev_max_backlog = 0;
 int netdev_budget = 0;
 int netdev_budget_usecs = 0;
 
+struct bpftune_sample drop_sample = {};
+
 #ifdef BPFTUNE_LEGACY
 SEC("kretprobe/enqueue_to_backlog")
 int BPF_KRETPROBE(bpftune_enqueue_to_backlog, int ret)
@@ -55,9 +57,8 @@ int BPF_PROG(bpftune_enqueue_to_backlog, struct sk_buff *skb, int cpu,
 	drop_count++;
 
 	/* only sample subset of drops to reduce overhead. */
-	if (bpftune_skip_sample(drop_count))
-		return 0;
-	
+	bpftune_sample(drop_sample);
+
 	/* if we drop more than 1/16 of the backlog queue size/min,
 	 * increase backlog queue size.  This means as the queue size
 	 * increases, the likliehood of hitting that limit decreases.
@@ -92,14 +93,14 @@ int BPF_PROG(bpftune_enqueue_to_backlog, struct sk_buff *skb, int cpu,
 	return 0;
 }
 
+struct bpftune_sample rx_action_sample = {};
+
 #ifndef BPFTUNE_LEGACY
 
 BPF_MAP_DEF(time_squeeze_map, BPF_MAP_TYPE_PERCPU_ARRAY, unsigned int, unsigned int, 1, 0);
 
 extern const struct softnet_data softnet_data __ksym;
 
-__u64 rx_count = 0;
-
 SEC("fexit/net_rx_action")
 int BPF_PROG(net_rx_action)
 {
@@ -111,8 +112,8 @@ int BPF_PROG(net_rx_action)
 	unsigned int *last_time_squeezep = NULL;
 	unsigned int zero = 0;
 
-	if (bpftune_skip_sample(rx_count))
-		return 0;
+	bpftune_sample(rx_action_sample);
+
 	sd = (struct softnet_data *)bpf_this_cpu_ptr(&softnet_data);
 	if (!sd)
 		return 0;

src/net_buffer_tuner.c

@@ -62,6 +62,8 @@ int init(struct bpftuner *tuner)
 			     budget);
 	bpftuner_bpf_var_set(net_buffer, tuner, netdev_budget_usecs,
 			     budget_usecs);
+	bpftuner_bpf_sample_add(net_buffer, tuner, drop_sample);
+	bpftuner_bpf_sample_add(net_buffer, tuner, rx_action_sample);
 	err = bpftuner_bpf_attach(net_buffer, tuner);
 	if (err)
 		return err;

src/tcp_buffer_tuner.bpf.c

@@ -26,11 +26,6 @@
 bool under_memory_pressure = false;
 bool near_memory_pressure = false;
 bool near_memory_exhaustion = false;
-/* use global tcp sock count since tcp memory pressure/exhaustion are
- * computed as fraction of total system memory.
- */
-__s64 tcp_sock_count = 0;
-__s64 tcp_max_sock_count = 0;
 
 /* set from userspace */
 int kernel_page_size;
@@ -227,19 +222,27 @@ BPF_FENTRY(tcp_sndbuf_expand, struct sock *sk)
 	return 0;
 }
 
+__u64 rcv_space_count = 0;
+
+struct bpftune_sample rcv_space_sample = { };
+
 /* sadly tcp_rcv_space_adjust() has checks internal to it so it is called
  * regardless of if we are under memory pressure or not; so use the variable
  * we set when memory pressure is triggered.
  */
 BPF_FENTRY(tcp_rcv_space_adjust, struct sock *sk)
 {
 	struct bpftune_event event = { 0 };
-	struct net *net = BPFTUNE_CORE_READ(sk, sk_net.net);
 	struct tcp_sock *tp = (struct tcp_sock *)sk;
 	long rmem[3], rmem_new[3];
 	__u8 sk_userlocks = 0;
+	struct net *net;
 	long rcvbuf;
 
+	/* only sample subset of events to reduce overhead. */
+	bpftune_sample(rcv_space_sample);
+
+	net = BPFTUNE_CORE_READ(sk, sk_net.net);
 	if (!sk || !net)
 		return 0;
 
@@ -288,17 +291,7 @@ BPF_FENTRY(tcp_init_sock, struct sock *sk)
 {
 	struct bpftune_event event = { 0 };
 
-	if (sk) {
-		if (++tcp_sock_count > tcp_max_sock_count)
-			tcp_max_sock_count = tcp_sock_count;
+	if (sk)
 		(void) tcp_nearly_out_of_memory(sk, &event);
-	}
-	return 0;
-}
-
-BPF_FENTRY(tcp_release_cb, struct sock *sk)
-{
-	if (tcp_sock_count > 0)
-		tcp_sock_count--;
 	return 0;
 }

src/tcp_buffer_tuner.c

@@ -164,6 +164,7 @@ int init(struct bpftuner *tuner)
 			     ilog2(SK_MEM_QUANTUM));
 	bpftuner_bpf_var_set(tcp_buffer, tuner, nr_free_buffer_pages,
 			     nr_free_buffer_pages(true));
+	bpftuner_bpf_sample_add(tcp_buffer, tuner, rcv_space_sample);
 	err = bpftuner_bpf_attach(tcp_buffer, tuner);
 	if (err)
 		return err;

test/Makefile

@@ -24,7 +24,7 @@ TUNER_TESTS =	support_test log_test service_test inotify_test cap_test \
 		sample_test sample_legacy_test \
 		strategy_test strategy_legacy_test \
 		rollback_test rollback_legacy_test \
-		query_test \
+		query_test rate_test \
 		many_netns_test many_netns_legacy_test \
 		podman_globalonly_test podman_globalonly_legacy_test \
 		sysctl_test sysctl_legacy_test sysctl_netns_test \