conn_tuner: use remote gateway associated with route to aggregate cong alg info

this allows us to aggregate info on a default route versus locally-destined traffic
and reduces overhead from a per-remote host to per route gateway; we can also
get the gateway via the dest cache from the socket.

Sample retransmit sock op events to avoid overheads, and also no need to send
events for cong alg choice any more (we have summary on exit or explicit request).

Also add support for retrieving/setting string sysctl values; we will need that
for cong alg settings (allowed, default)

Signed-off-by: Alan Maguire <[email protected]>

Author: alan-maguire

docs/bpftune-tcp-conn.rst

@@ -19,7 +19,7 @@ DESCRIPTION
         such cases, BBR is a good fit since it continuously estimates bottleneck
         bandwidth and attempts to fit the congestion algorithm to it.
 
-        When we have limited information about a remote host - i.e. we have
+        When we have limited information about a destination - i.e. we have
         not had > REMOTE_HOST_MIN_INSTANCES connections involving it,
         the only auto-selection involved is to use BBR in cases where
         loss rates exceed 1/(2^DROP_SHIFT) (1.5%) of the packet sent rate -
@@ -37,7 +37,7 @@ DESCRIPTION
          BDP = BottleneckBandwidth * MinRoundTripTime
 
         The algorithm works as follows; BPF maintains a map of metrics keyed
-        by remote IP address.  For each remote IP address, we track the
+        by remote gateway.  For each remote gateway, we track the
         minimum RTT observed across all TCP connections and the max bandwidth
         observed.  The former tells us - as closely as we can determine -
         what the true RTT of the link is.  The latter estimates the

include/bpftune/bpftune.bpf.h

@@ -277,6 +277,10 @@ unsigned long long bpftune_init_net;
 #define NTF_EXT_LEARNED	0x10
 #endif
 
+#ifndef RTF_GATEWAY
+#define RTF_GATEWAY	0x02
+#endif
+
 #define EINVAL		22
 #define ENOMEM		12
 #define EAGAIN		11

include/bpftune/bpftune.h

@@ -92,6 +92,7 @@ struct bpftunable_scenario {
 
 #define BPFTUNABLE_NAMESPACED	0x1	/* settable in non-global namespace? */
 #define BPFTUNABLE_OPTIONAL	0x2	/* do not fail it tunable not found (e.g. ipv6 */
+#define BPFTUNABLE_STRING	0x4	/* tunable is a string, not numeric */
 
 struct bpftunable_desc {
 	unsigned int id;
@@ -106,11 +107,19 @@ struct bpftunable_stats {
 	unsigned long nonglobal_ns[BPFTUNE_MAX_SCENARIOS];
 };
 
+#define BPFTUNE_MAX_STR		128
+
 struct bpftunable {
 	struct bpftunable_desc desc;
 	enum bpftune_state state;
-	long initial_values[BPFTUNE_MAX_VALUES];
-	long current_values[BPFTUNE_MAX_VALUES];
+	union {
+		long initial_values[BPFTUNE_MAX_VALUES];
+		char initial_str[BPFTUNE_MAX_STR];
+	};
+	union {
+		long current_values[BPFTUNE_MAX_VALUES];
+		char current_str[BPFTUNE_MAX_STR];
+	};
 	struct bpftunable_stats stats;
 };
 

include/bpftune/libbpftune.h

@@ -158,7 +158,7 @@ int bpftuner_tunable_sysctl_write(struct bpftuner *tuner,
 				  unsigned int tunable,
 				  unsigned int scenario,
 				  unsigned long netns_cookie,
-				  __u8 num_values, long *values,
+				  __u8 num_values, void *values,
 				  const char *fmt, ...);
 
 int bpftuner_tunable_update(struct bpftuner *tuner,
@@ -349,7 +349,9 @@ void bpftune_ring_buffer_fini(void *ring_buffer);
 
 void bpftune_sysctl_name_to_path(const char *name, char *path, size_t path_sz);
 int bpftune_sysctl_read(int netns_fd, const char *name, long *values);
+int bpftune_sysctl_read_string(int netns_fd, const char *name, char *val);
 int bpftune_sysctl_write(int netns_fd, const char *name, __u8 num_values, long *values);
+int bpftune_sysctl_write_string(int netns_fd, const char *name, char *val);
 long long bpftune_ksym_addr(char type, const char *name);
 int bpftune_snmpstat_read(unsigned long netns_cookie, int family, const char *linename, const char *name, long *value);
 int bpftune_netstat_read(unsigned long netns_cookie, int family, const char *linename, const char *name, long *value);

src/libbpftune.c

@@ -806,6 +806,7 @@ static void bpftuner_rollback(struct bpftuner *tuner, bool log_only)
 		char newvals[PATH_MAX] = { };
 		bool changes = false;
 		char s[PATH_MAX];
+		void *val;
 
 		if (t->desc.type != BPFTUNABLE_SYSCTL)
 			continue;
@@ -823,13 +824,20 @@ static void bpftuner_rollback(struct bpftuner *tuner, bool log_only)
 		/* nothing to rollback? */
 		if (!changes)
 			continue;
-		for (j = 0; j < t->desc.num_values; j++) {
-			snprintf(s, sizeof(s), "%ld ",
-				 t->initial_values[j]);
-			strcat(oldvals, s);
-			snprintf(s, sizeof(s), "%ld ",
-				 t->current_values[j]);
-			strcat(newvals, s);
+		if (t->desc.flags & BPFTUNABLE_STRING) {
+			strncpy(oldvals, t->initial_str, sizeof(oldvals));
+			strncpy(newvals, t->current_str, sizeof(newvals));
+			val = t->initial_str;
+		} else {
+			for (j = 0; j < t->desc.num_values; j++) {
+				snprintf(s, sizeof(s), "%ld ",
+					 t->initial_values[j]);
+				strcat(oldvals, s);
+				snprintf(s, sizeof(s), "%ld ",
+					 t->current_values[j]);
+				strcat(newvals, s);
+			}
+			val = t->initial_values;
 		}
 		if (log_only) {
 			bpftune_log(BPFTUNE_LOG_LEVEL, "# To roll back changes to '%s', run the following as a privileged user in a terminal:\n",
@@ -840,7 +848,7 @@ static void bpftuner_rollback(struct bpftuner *tuner, bool log_only)
 			bpftuner_tunable_sysctl_write(tuner, i, k,
 				0,
 				t->desc.num_values,
-				t->initial_values,
+				val,
 				"Rolling back sysctl values for '%s' from (%s) to original values (%s)...\n",
 				t->desc.name,
 				newvals, oldvals);
@@ -1001,9 +1009,11 @@ void bpftune_sysctl_name_to_path(const char *name, char *path, size_t path_sz)
 			path[i] = '/';
 }
 
-int bpftune_sysctl_read(int netns_fd, const char *name, long *values)
+static int __bpftune_sysctl_read(int netns_fd, const char *name, void *val,
+				 bool isstr)
 {
 	int i, orig_netns_fd = 0, num_values = 0;
+	long *values = val;
 	char path[PATH_MAX];
 	int err = 0;	
 	FILE *fp;
@@ -1025,25 +1035,33 @@ int bpftune_sysctl_read(int netns_fd, const char *name, long *values)
 			    path, netns_fd, strerror(-err));
 		goto out;
 	}
-	num_values = fscanf(fp, "%ld %ld %ld",
-			    &values[0], &values[1], &values[2]);
-	if (num_values == 0)
-		err = -ENOENT;
-	else if (num_values < 0)
-		err = -errno;
+	if (isstr) {
+		num_values = fscanf(fp, "%[^\n]", (char *)val);
+		if (num_values != 1)
+			err = -ENOENT;
+		else
+			bpftune_log(LOG_DEBUG, "Read %s = %s\n", name, (char *)val);
+	} else {
+		num_values = fscanf(fp, "%ld %ld %ld",
+				    &values[0], &values[1], &values[2]);
+		if (num_values == 0) {
+			err = -ENOENT;
+		} else if (num_values < 0) {
+			err = -errno;
+		} else {
+			for (i = 0; i < num_values; i++) {
+				bpftune_log(LOG_DEBUG, "Read %s[%d] = %ld\n",
+					    name, i, values[i]);
+			}
+		}
+	}
 	fclose(fp);
 
 	if (err) {
 		bpftune_log(LOG_ERR, "could not read from %s: %s\n", path,
 			    strerror(-err));
 		goto out;
 	}
-
-	for (i = 0; i < num_values; i++) {
-		bpftune_log(LOG_DEBUG, "Read %s[%d] = %ld\n",
-			    name, i, values[i]);
-	}
-
 out:
 	bpftune_netns_set(orig_netns_fd, NULL, true);
 out_unset:
@@ -1053,11 +1071,24 @@ int bpftune_sysctl_read(int netns_fd, const char *name, long *values)
 	return err ? err : num_values;
 }
 
-int bpftune_sysctl_write(int netns_fd, const char *name, __u8 num_values, long *values)
+int bpftune_sysctl_read(int netns_fd, const char *name, long *values)
+{
+	return __bpftune_sysctl_read(netns_fd, name, values, false);
+}
+
+int bpftune_sysctl_read_string(int netns_fd, const char *name, char *val)
+{
+	return __bpftune_sysctl_read(netns_fd, name, val, true);
+}
+
+static int __bpftune_sysctl_write(int netns_fd, const char *name, __u8 num_values,
+				  void *val, bool isstr)
 {
 	long old_values[BPFTUNE_MAX_VALUES] = {};
+	char old_value_str[PATH_MAX] = {};
 	int i, err = 0, orig_netns_fd = 0;
 	int old_num_values;
+	long *values = val;
 	char path[PATH_MAX];
 	FILE *fp;
 
@@ -1074,18 +1105,27 @@ int bpftune_sysctl_write(int netns_fd, const char *name, __u8 num_values, long *
 		goto out_unset;
 
 	/* If value is already set to val, do nothing. */
-	old_num_values = bpftune_sysctl_read(0, name, old_values);
+	if (isstr) {
+		old_num_values = bpftune_sysctl_read_string(0, name, old_value_str);
+	} else {
+		old_num_values = bpftune_sysctl_read(0, name, old_values);
+	}
 	if (old_num_values < 0) {
 		err = old_num_values;
 		goto out;
 	}
 	if (num_values == old_num_values) {
-		for (i = 0; i < num_values; i++) {
-			if (old_values[i] != values[i])
-				break;
+		if (isstr) {
+			if (strcmp(old_value_str, val) == 0)
+				goto out;
+		} else {
+			for (i = 0; i < num_values; i++) {
+				if (old_values[i] != values[i])
+					break;
+			}
+			if (i == num_values)
+				goto out;
 		}
-		if (i == num_values)
-			goto out;
 	}
         fp = fopen(path, "w");
         if (!fp) {
@@ -1095,14 +1135,18 @@ int bpftune_sysctl_write(int netns_fd, const char *name, __u8 num_values, long *
 		goto out;
 	}
 
-	for (i = 0; i < num_values; i++)
-		fprintf(fp, "%ld ", values[i]);
+	if (isstr) {
+		fprintf(fp, "%s", (char *)val);
+		bpftune_log(LOG_DEBUG, "Wrote %s = %s\n", name, val);
+	} else {
+		for (i = 0; i < num_values; i++) {
+			fprintf(fp, "%ld ", values[i]);
+			bpftune_log(LOG_DEBUG, "Wrote %s[%d] = %ld\n",
+				    name, i, values[i]);
+		}
+	}
         fclose(fp);
 
-	for (i = 0; i < num_values; i++) {
-		bpftune_log(LOG_DEBUG, "Wrote %s[%d] = %ld\n",
-			    name, i, values[i]);
-	}
 out:
 	bpftune_netns_set(orig_netns_fd, NULL, true);
 out_unset:
@@ -1112,6 +1156,17 @@ int bpftune_sysctl_write(int netns_fd, const char *name, __u8 num_values, long *
         return err;
 }
 
+int bpftune_sysctl_write(int netns_fd, const char *name, __u8 num_values, long *values)
+{
+	return __bpftune_sysctl_write(netns_fd, name, num_values, values, false);
+}
+
+int bpftune_sysctl_write_string(int netns_fd, const char *name, char *val)
+{
+	return __bpftune_sysctl_write(netns_fd, name, 1, val, true);
+}
+
+
 long long bpftune_ksym_addr(char type, const char *name)
 {
 	long long ret = -ENOENT;
@@ -1302,8 +1357,12 @@ int bpftuner_tunables_init(struct bpftuner *tuner, unsigned int num_descs,
 
 		if (descs[i].type != BPFTUNABLE_SYSCTL)
 			continue;
-		num_values = bpftune_sysctl_read(0, descs[i].name,
-				tuner->tunables[i].current_values);
+		if (descs[i].flags & BPFTUNABLE_STRING)
+			num_values = bpftune_sysctl_read_string(0, descs[i].name,
+								tuner->tunables[i].current_str);
+		else
+			num_values = bpftune_sysctl_read(0, descs[i].name,
+					tuner->tunables[i].current_values);
 		if (num_values < 0) {
 			if (descs[i].flags & BPFTUNABLE_OPTIONAL) {
 				bpftune_log(LOG_DEBUG, "error reading optional tunable '%s': %s\n",
@@ -1319,9 +1378,15 @@ int bpftuner_tunables_init(struct bpftuner *tuner, unsigned int num_descs,
 				    descs[i].num_values, num_values);
 			return -EINVAL;
 		}
-		memcpy(tuner->tunables[i].initial_values,
-		       tuner->tunables[i].current_values,
-		       sizeof(tuner->tunables[i].initial_values));
+		if (descs[i].flags & BPFTUNABLE_STRING) {
+			strncpy(tuner->tunables[i].initial_str,
+				tuner->tunables[i].current_str,
+				sizeof(tuner->tunables[i].initial_str));
+		} else {
+			memcpy(tuner->tunables[i].initial_values,
+			       tuner->tunables[i].current_values,
+			       sizeof(tuner->tunables[i].initial_values));
+		}
 	}
 
 	return 0;
@@ -1387,13 +1452,18 @@ static void __bpftuner_scenario_log(struct bpftuner *tuner, unsigned int tunable
 			char s[PATH_MAX];
 			__u8 i;
 
-			for (i = 0; i < t->desc.num_values; i++) {
-				snprintf(s, sizeof(s), "%ld ",
-					 t->initial_values[i]);
-				strcat(oldvals, s);
-				snprintf(s, sizeof(s), "%ld ",
-					 t->current_values[i]);
-				strcat(newvals, s);
+			if (t->desc.flags & BPFTUNABLE_STRING) {
+				strncpy(oldvals, t->initial_str, sizeof(oldvals));
+				strncpy(newvals, t->current_str, sizeof(newvals));
+			} else {
+				for (i = 0; i < t->desc.num_values; i++) {
+					snprintf(s, sizeof(s), "%ld ",
+						 t->initial_values[i]);
+					strcat(oldvals, s);
+					snprintf(s, sizeof(s), "%ld ",
+						 t->current_values[i]);
+					strcat(newvals, s);
+				}
 			}
 			bpftune_log(BPFTUNE_LOG_LEVEL, "# sysctl '%s' changed from (%s) -> (%s)\n",
 				    t->desc.name, oldvals, newvals);
@@ -1415,7 +1485,7 @@ static void __bpftuner_scenario_log(struct bpftuner *tuner, unsigned int tunable
 
 int bpftuner_tunable_sysctl_write(struct bpftuner *tuner, unsigned int tunable,
 				  unsigned int scenario, unsigned long netns_cookie,
-				  __u8 num_values, long *values,
+				  __u8 num_values, void *values,
 				  const char *fmt, ...)
 {
 	struct bpftunable *t = bpftuner_tunable(tuner, tunable);
@@ -1450,16 +1520,23 @@ int bpftuner_tunable_sysctl_write(struct bpftuner *tuner, unsigned int tunable,
 	}
 	global_ns = fd == 0;
 
-	ret = bpftune_sysctl_write(fd, t->desc.name, num_values, values);
+	if (t->desc.flags & BPFTUNABLE_STRING)
+		ret = bpftune_sysctl_write_string(fd, t->desc.name, values);
+	else
+		ret = bpftune_sysctl_write(fd, t->desc.name, num_values, values);
 	if (!ret) {
 		__u8 i;
 
 		bpftuner_scenario_log_fmt(tuner, tunable, scenario, fd, false, fmt);
 
 		/* only cache values for rollback for global ns */
 		if (global_ns) {
-			for (i = 0; i < t->desc.num_values; i++)
-				t->current_values[i] = values[i];
+			if (t->desc.flags & BPFTUNABLE_STRING) {
+				strncpy(t->current_str, values, sizeof(t->current_str));
+			} else {
+				for (i = 0; i < t->desc.num_values; i++)
+					t->current_values[i] = ((long *)values)[i];
+			}
 		}
 	} else if (ret < 0) {
 		/* If sysctl update failed, mark non-global netns as gone to

src/libbpftune.map

@@ -55,7 +55,9 @@ LIBBPFTUNE_0.1.1 {
 		bpftune_ring_buffer_fini;
 		bpftune_sysctl_name_to_path;
 		bpftune_sysctl_read;
+		bpftune_sysctl_read_string;
 		bpftune_sysctl_write;
+		bpftune_sysctl_write_string;
 		bpftune_ksym_addr;
 		bpftune_netstat_read;
 		bpftune_snmpstat_read;