1 // SPDX-License-Identifier: GPL-2.0
2 /* Copyright (c) 2018 Facebook */
3 
4 #include <stdlib.h>
5 #include <unistd.h>
6 #include <stdbool.h>
7 #include <string.h>
8 #include <errno.h>
9 #include <assert.h>
10 #include <fcntl.h>
11 #include <linux/bpf.h>
12 #include <linux/err.h>
13 #include <linux/types.h>
14 #include <linux/if_ether.h>
15 #include <sys/types.h>
16 #include <sys/epoll.h>
17 #include <sys/socket.h>
18 #include <netinet/in.h>
19 #include <bpf/bpf.h>
20 #include <bpf/libbpf.h>
21 #include "bpf_rlimit.h"
22 #include "bpf_util.h"
23 #include "test_select_reuseport_common.h"
24 
25 #define MIN_TCPHDR_LEN 20
26 #define UDPHDR_LEN 8
27 
28 #define TCP_SYNCOOKIE_SYSCTL "/proc/sys/net/ipv4/tcp_syncookies"
29 #define TCP_FO_SYSCTL "/proc/sys/net/ipv4/tcp_fastopen"
30 #define REUSEPORT_ARRAY_SIZE 32
31 
32 static int result_map, tmp_index_ovr_map, linum_map, data_check_map;
33 static enum result expected_results[NR_RESULTS];
34 static int sk_fds[REUSEPORT_ARRAY_SIZE];
35 static int reuseport_array, outer_map;
36 static int select_by_skb_data_prog;
37 static int saved_tcp_syncookie;
38 static struct bpf_object *obj;
39 static int saved_tcp_fo;
40 static __u32 index_zero;
41 static int epfd;
42 
43 static union sa46 {
44 	struct sockaddr_in6 v6;
45 	struct sockaddr_in v4;
46 	sa_family_t family;
47 } srv_sa;
48 
49 #define CHECK(condition, tag, format...) ({				\
50 	int __ret = !!(condition);					\
51 	if (__ret) {							\
52 		printf("%s(%d):FAIL:%s ", __func__, __LINE__, tag);	\
53 		printf(format);						\
54 		exit(-1);						\
55 	}								\
56 })
57 
create_maps(void)58 static void create_maps(void)
59 {
60 	struct bpf_create_map_attr attr = {};
61 
62 	/* Creating reuseport_array */
63 	attr.name = "reuseport_array";
64 	attr.map_type = BPF_MAP_TYPE_REUSEPORT_SOCKARRAY;
65 	attr.key_size = sizeof(__u32);
66 	attr.value_size = sizeof(__u32);
67 	attr.max_entries = REUSEPORT_ARRAY_SIZE;
68 
69 	reuseport_array = bpf_create_map_xattr(&attr);
70 	CHECK(reuseport_array == -1, "creating reuseport_array",
71 	      "reuseport_array:%d errno:%d\n", reuseport_array, errno);
72 
73 	/* Creating outer_map */
74 	attr.name = "outer_map";
75 	attr.map_type = BPF_MAP_TYPE_ARRAY_OF_MAPS;
76 	attr.key_size = sizeof(__u32);
77 	attr.value_size = sizeof(__u32);
78 	attr.max_entries = 1;
79 	attr.inner_map_fd = reuseport_array;
80 	outer_map = bpf_create_map_xattr(&attr);
81 	CHECK(outer_map == -1, "creating outer_map",
82 	      "outer_map:%d errno:%d\n", outer_map, errno);
83 }
84 
prepare_bpf_obj(void)85 static void prepare_bpf_obj(void)
86 {
87 	struct bpf_program *prog;
88 	struct bpf_map *map;
89 	int err;
90 	struct bpf_object_open_attr attr = {
91 		.file = "test_select_reuseport_kern.o",
92 		.prog_type = BPF_PROG_TYPE_SK_REUSEPORT,
93 	};
94 
95 	obj = bpf_object__open_xattr(&attr);
96 	CHECK(IS_ERR_OR_NULL(obj), "open test_select_reuseport_kern.o",
97 	      "obj:%p PTR_ERR(obj):%ld\n", obj, PTR_ERR(obj));
98 
99 	prog = bpf_program__next(NULL, obj);
100 	CHECK(!prog, "get first bpf_program", "!prog\n");
101 	bpf_program__set_type(prog, attr.prog_type);
102 
103 	map = bpf_object__find_map_by_name(obj, "outer_map");
104 	CHECK(!map, "find outer_map", "!map\n");
105 	err = bpf_map__reuse_fd(map, outer_map);
106 	CHECK(err, "reuse outer_map", "err:%d\n", err);
107 
108 	err = bpf_object__load(obj);
109 	CHECK(err, "load bpf_object", "err:%d\n", err);
110 
111 	select_by_skb_data_prog = bpf_program__fd(prog);
112 	CHECK(select_by_skb_data_prog == -1, "get prog fd",
113 	      "select_by_skb_data_prog:%d\n", select_by_skb_data_prog);
114 
115 	map = bpf_object__find_map_by_name(obj, "result_map");
116 	CHECK(!map, "find result_map", "!map\n");
117 	result_map = bpf_map__fd(map);
118 	CHECK(result_map == -1, "get result_map fd",
119 	      "result_map:%d\n", result_map);
120 
121 	map = bpf_object__find_map_by_name(obj, "tmp_index_ovr_map");
122 	CHECK(!map, "find tmp_index_ovr_map", "!map\n");
123 	tmp_index_ovr_map = bpf_map__fd(map);
124 	CHECK(tmp_index_ovr_map == -1, "get tmp_index_ovr_map fd",
125 	      "tmp_index_ovr_map:%d\n", tmp_index_ovr_map);
126 
127 	map = bpf_object__find_map_by_name(obj, "linum_map");
128 	CHECK(!map, "find linum_map", "!map\n");
129 	linum_map = bpf_map__fd(map);
130 	CHECK(linum_map == -1, "get linum_map fd",
131 	      "linum_map:%d\n", linum_map);
132 
133 	map = bpf_object__find_map_by_name(obj, "data_check_map");
134 	CHECK(!map, "find data_check_map", "!map\n");
135 	data_check_map = bpf_map__fd(map);
136 	CHECK(data_check_map == -1, "get data_check_map fd",
137 	      "data_check_map:%d\n", data_check_map);
138 }
139 
sa46_init_loopback(union sa46 * sa,sa_family_t family)140 static void sa46_init_loopback(union sa46 *sa, sa_family_t family)
141 {
142 	memset(sa, 0, sizeof(*sa));
143 	sa->family = family;
144 	if (sa->family == AF_INET6)
145 		sa->v6.sin6_addr = in6addr_loopback;
146 	else
147 		sa->v4.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
148 }
149 
sa46_init_inany(union sa46 * sa,sa_family_t family)150 static void sa46_init_inany(union sa46 *sa, sa_family_t family)
151 {
152 	memset(sa, 0, sizeof(*sa));
153 	sa->family = family;
154 	if (sa->family == AF_INET6)
155 		sa->v6.sin6_addr = in6addr_any;
156 	else
157 		sa->v4.sin_addr.s_addr = INADDR_ANY;
158 }
159 
read_int_sysctl(const char * sysctl)160 static int read_int_sysctl(const char *sysctl)
161 {
162 	char buf[16];
163 	int fd, ret;
164 
165 	fd = open(sysctl, 0);
166 	CHECK(fd == -1, "open(sysctl)", "sysctl:%s fd:%d errno:%d\n",
167 	      sysctl, fd, errno);
168 
169 	ret = read(fd, buf, sizeof(buf));
170 	CHECK(ret <= 0, "read(sysctl)", "sysctl:%s ret:%d errno:%d\n",
171 	      sysctl, ret, errno);
172 	close(fd);
173 
174 	return atoi(buf);
175 }
176 
write_int_sysctl(const char * sysctl,int v)177 static void write_int_sysctl(const char *sysctl, int v)
178 {
179 	int fd, ret, size;
180 	char buf[16];
181 
182 	fd = open(sysctl, O_RDWR);
183 	CHECK(fd == -1, "open(sysctl)", "sysctl:%s fd:%d errno:%d\n",
184 	      sysctl, fd, errno);
185 
186 	size = snprintf(buf, sizeof(buf), "%d", v);
187 	ret = write(fd, buf, size);
188 	CHECK(ret != size, "write(sysctl)",
189 	      "sysctl:%s ret:%d size:%d errno:%d\n", sysctl, ret, size, errno);
190 	close(fd);
191 }
192 
restore_sysctls(void)193 static void restore_sysctls(void)
194 {
195 	write_int_sysctl(TCP_FO_SYSCTL, saved_tcp_fo);
196 	write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, saved_tcp_syncookie);
197 }
198 
enable_fastopen(void)199 static void enable_fastopen(void)
200 {
201 	int fo;
202 
203 	fo = read_int_sysctl(TCP_FO_SYSCTL);
204 	write_int_sysctl(TCP_FO_SYSCTL, fo | 7);
205 }
206 
enable_syncookie(void)207 static void enable_syncookie(void)
208 {
209 	write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, 2);
210 }
211 
disable_syncookie(void)212 static void disable_syncookie(void)
213 {
214 	write_int_sysctl(TCP_SYNCOOKIE_SYSCTL, 0);
215 }
216 
get_linum(void)217 static __u32 get_linum(void)
218 {
219 	__u32 linum;
220 	int err;
221 
222 	err = bpf_map_lookup_elem(linum_map, &index_zero, &linum);
223 	CHECK(err == -1, "lookup_elem(linum_map)", "err:%d errno:%d\n",
224 	      err, errno);
225 
226 	return linum;
227 }
228 
check_data(int type,sa_family_t family,const struct cmd * cmd,int cli_fd)229 static void check_data(int type, sa_family_t family, const struct cmd *cmd,
230 		       int cli_fd)
231 {
232 	struct data_check expected = {}, result;
233 	union sa46 cli_sa;
234 	socklen_t addrlen;
235 	int err;
236 
237 	addrlen = sizeof(cli_sa);
238 	err = getsockname(cli_fd, (struct sockaddr *)&cli_sa,
239 			  &addrlen);
240 	CHECK(err == -1, "getsockname(cli_fd)", "err:%d errno:%d\n",
241 	      err, errno);
242 
243 	err = bpf_map_lookup_elem(data_check_map, &index_zero, &result);
244 	CHECK(err == -1, "lookup_elem(data_check_map)", "err:%d errno:%d\n",
245 	      err, errno);
246 
247 	if (type == SOCK_STREAM) {
248 		expected.len = MIN_TCPHDR_LEN;
249 		expected.ip_protocol = IPPROTO_TCP;
250 	} else {
251 		expected.len = UDPHDR_LEN;
252 		expected.ip_protocol = IPPROTO_UDP;
253 	}
254 
255 	if (family == AF_INET6) {
256 		expected.eth_protocol = htons(ETH_P_IPV6);
257 		expected.bind_inany = !srv_sa.v6.sin6_addr.s6_addr32[3] &&
258 			!srv_sa.v6.sin6_addr.s6_addr32[2] &&
259 			!srv_sa.v6.sin6_addr.s6_addr32[1] &&
260 			!srv_sa.v6.sin6_addr.s6_addr32[0];
261 
262 		memcpy(&expected.skb_addrs[0], cli_sa.v6.sin6_addr.s6_addr32,
263 		       sizeof(cli_sa.v6.sin6_addr));
264 		memcpy(&expected.skb_addrs[4], &in6addr_loopback,
265 		       sizeof(in6addr_loopback));
266 		expected.skb_ports[0] = cli_sa.v6.sin6_port;
267 		expected.skb_ports[1] = srv_sa.v6.sin6_port;
268 	} else {
269 		expected.eth_protocol = htons(ETH_P_IP);
270 		expected.bind_inany = !srv_sa.v4.sin_addr.s_addr;
271 
272 		expected.skb_addrs[0] = cli_sa.v4.sin_addr.s_addr;
273 		expected.skb_addrs[1] = htonl(INADDR_LOOPBACK);
274 		expected.skb_ports[0] = cli_sa.v4.sin_port;
275 		expected.skb_ports[1] = srv_sa.v4.sin_port;
276 	}
277 
278 	if (memcmp(&result, &expected, offsetof(struct data_check,
279 						equal_check_end))) {
280 		printf("unexpected data_check\n");
281 		printf("  result: (0x%x, %u, %u)\n",
282 		       result.eth_protocol, result.ip_protocol,
283 		       result.bind_inany);
284 		printf("expected: (0x%x, %u, %u)\n",
285 		       expected.eth_protocol, expected.ip_protocol,
286 		       expected.bind_inany);
287 		CHECK(1, "data_check result != expected",
288 		      "bpf_prog_linum:%u\n", get_linum());
289 	}
290 
291 	CHECK(!result.hash, "data_check result.hash empty",
292 	      "result.hash:%u", result.hash);
293 
294 	expected.len += cmd ? sizeof(*cmd) : 0;
295 	if (type == SOCK_STREAM)
296 		CHECK(expected.len > result.len, "expected.len > result.len",
297 		      "expected.len:%u result.len:%u bpf_prog_linum:%u\n",
298 		      expected.len, result.len, get_linum());
299 	else
300 		CHECK(expected.len != result.len, "expected.len != result.len",
301 		      "expected.len:%u result.len:%u bpf_prog_linum:%u\n",
302 		      expected.len, result.len, get_linum());
303 }
304 
check_results(void)305 static void check_results(void)
306 {
307 	__u32 results[NR_RESULTS];
308 	__u32 i, broken = 0;
309 	int err;
310 
311 	for (i = 0; i < NR_RESULTS; i++) {
312 		err = bpf_map_lookup_elem(result_map, &i, &results[i]);
313 		CHECK(err == -1, "lookup_elem(result_map)",
314 		      "i:%u err:%d errno:%d\n", i, err, errno);
315 	}
316 
317 	for (i = 0; i < NR_RESULTS; i++) {
318 		if (results[i] != expected_results[i]) {
319 			broken = i;
320 			break;
321 		}
322 	}
323 
324 	if (i == NR_RESULTS)
325 		return;
326 
327 	printf("unexpected result\n");
328 	printf(" result: [");
329 	printf("%u", results[0]);
330 	for (i = 1; i < NR_RESULTS; i++)
331 		printf(", %u", results[i]);
332 	printf("]\n");
333 
334 	printf("expected: [");
335 	printf("%u", expected_results[0]);
336 	for (i = 1; i < NR_RESULTS; i++)
337 		printf(", %u", expected_results[i]);
338 	printf("]\n");
339 
340 	CHECK(expected_results[broken] != results[broken],
341 	      "unexpected result",
342 	      "expected_results[%u] != results[%u] bpf_prog_linum:%u\n",
343 	      broken, broken, get_linum());
344 }
345 
send_data(int type,sa_family_t family,void * data,size_t len,enum result expected)346 static int send_data(int type, sa_family_t family, void *data, size_t len,
347 		     enum result expected)
348 {
349 	union sa46 cli_sa;
350 	int fd, err;
351 
352 	fd = socket(family, type, 0);
353 	CHECK(fd == -1, "socket()", "fd:%d errno:%d\n", fd, errno);
354 
355 	sa46_init_loopback(&cli_sa, family);
356 	err = bind(fd, (struct sockaddr *)&cli_sa, sizeof(cli_sa));
357 	CHECK(fd == -1, "bind(cli_sa)", "err:%d errno:%d\n", err, errno);
358 
359 	err = sendto(fd, data, len, MSG_FASTOPEN, (struct sockaddr *)&srv_sa,
360 		     sizeof(srv_sa));
361 	CHECK(err != len && expected >= PASS,
362 	      "sendto()", "family:%u err:%d errno:%d expected:%d\n",
363 	      family, err, errno, expected);
364 
365 	return fd;
366 }
367 
do_test(int type,sa_family_t family,struct cmd * cmd,enum result expected)368 static void do_test(int type, sa_family_t family, struct cmd *cmd,
369 		    enum result expected)
370 {
371 	int nev, srv_fd, cli_fd;
372 	struct epoll_event ev;
373 	struct cmd rcv_cmd;
374 	ssize_t nread;
375 
376 	cli_fd = send_data(type, family, cmd, cmd ? sizeof(*cmd) : 0,
377 			   expected);
378 	nev = epoll_wait(epfd, &ev, 1, expected >= PASS ? 5 : 0);
379 	CHECK((nev <= 0 && expected >= PASS) ||
380 	      (nev > 0 && expected < PASS),
381 	      "nev <> expected",
382 	      "nev:%d expected:%d type:%d family:%d data:(%d, %d)\n",
383 	      nev, expected, type, family,
384 	      cmd ? cmd->reuseport_index : -1,
385 	      cmd ? cmd->pass_on_failure : -1);
386 	check_results();
387 	check_data(type, family, cmd, cli_fd);
388 
389 	if (expected < PASS)
390 		return;
391 
392 	CHECK(expected != PASS_ERR_SK_SELECT_REUSEPORT &&
393 	      cmd->reuseport_index != ev.data.u32,
394 	      "check cmd->reuseport_index",
395 	      "cmd:(%u, %u) ev.data.u32:%u\n",
396 	      cmd->pass_on_failure, cmd->reuseport_index, ev.data.u32);
397 
398 	srv_fd = sk_fds[ev.data.u32];
399 	if (type == SOCK_STREAM) {
400 		int new_fd = accept(srv_fd, NULL, 0);
401 
402 		CHECK(new_fd == -1, "accept(srv_fd)",
403 		      "ev.data.u32:%u new_fd:%d errno:%d\n",
404 		      ev.data.u32, new_fd, errno);
405 
406 		nread = recv(new_fd, &rcv_cmd, sizeof(rcv_cmd), MSG_DONTWAIT);
407 		CHECK(nread != sizeof(rcv_cmd),
408 		      "recv(new_fd)",
409 		      "ev.data.u32:%u nread:%zd sizeof(rcv_cmd):%zu errno:%d\n",
410 		      ev.data.u32, nread, sizeof(rcv_cmd), errno);
411 
412 		close(new_fd);
413 	} else {
414 		nread = recv(srv_fd, &rcv_cmd, sizeof(rcv_cmd), MSG_DONTWAIT);
415 		CHECK(nread != sizeof(rcv_cmd),
416 		      "recv(sk_fds)",
417 		      "ev.data.u32:%u nread:%zd sizeof(rcv_cmd):%zu errno:%d\n",
418 		      ev.data.u32, nread, sizeof(rcv_cmd), errno);
419 	}
420 
421 	close(cli_fd);
422 }
423 
test_err_inner_map(int type,sa_family_t family)424 static void test_err_inner_map(int type, sa_family_t family)
425 {
426 	struct cmd cmd = {
427 		.reuseport_index = 0,
428 		.pass_on_failure = 0,
429 	};
430 
431 	printf("%s: ", __func__);
432 	expected_results[DROP_ERR_INNER_MAP]++;
433 	do_test(type, family, &cmd, DROP_ERR_INNER_MAP);
434 	printf("OK\n");
435 }
436 
test_err_skb_data(int type,sa_family_t family)437 static void test_err_skb_data(int type, sa_family_t family)
438 {
439 	printf("%s: ", __func__);
440 	expected_results[DROP_ERR_SKB_DATA]++;
441 	do_test(type, family, NULL, DROP_ERR_SKB_DATA);
442 	printf("OK\n");
443 }
444 
test_err_sk_select_port(int type,sa_family_t family)445 static void test_err_sk_select_port(int type, sa_family_t family)
446 {
447 	struct cmd cmd = {
448 		.reuseport_index = REUSEPORT_ARRAY_SIZE,
449 		.pass_on_failure = 0,
450 	};
451 
452 	printf("%s: ", __func__);
453 	expected_results[DROP_ERR_SK_SELECT_REUSEPORT]++;
454 	do_test(type, family, &cmd, DROP_ERR_SK_SELECT_REUSEPORT);
455 	printf("OK\n");
456 }
457 
test_pass(int type,sa_family_t family)458 static void test_pass(int type, sa_family_t family)
459 {
460 	struct cmd cmd;
461 	int i;
462 
463 	printf("%s: ", __func__);
464 	cmd.pass_on_failure = 0;
465 	for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++) {
466 		expected_results[PASS]++;
467 		cmd.reuseport_index = i;
468 		do_test(type, family, &cmd, PASS);
469 	}
470 	printf("OK\n");
471 }
472 
test_syncookie(int type,sa_family_t family)473 static void test_syncookie(int type, sa_family_t family)
474 {
475 	int err, tmp_index = 1;
476 	struct cmd cmd = {
477 		.reuseport_index = 0,
478 		.pass_on_failure = 0,
479 	};
480 
481 	if (type != SOCK_STREAM)
482 		return;
483 
484 	printf("%s: ", __func__);
485 	/*
486 	 * +1 for TCP-SYN and
487 	 * +1 for the TCP-ACK (ack the syncookie)
488 	 */
489 	expected_results[PASS] += 2;
490 	enable_syncookie();
491 	/*
492 	 * Simulate TCP-SYN and TCP-ACK are handled by two different sk:
493 	 * TCP-SYN: select sk_fds[tmp_index = 1] tmp_index is from the
494 	 *          tmp_index_ovr_map
495 	 * TCP-ACK: select sk_fds[reuseport_index = 0] reuseport_index
496 	 *          is from the cmd.reuseport_index
497 	 */
498 	err = bpf_map_update_elem(tmp_index_ovr_map, &index_zero,
499 				  &tmp_index, BPF_ANY);
500 	CHECK(err == -1, "update_elem(tmp_index_ovr_map, 0, 1)",
501 	      "err:%d errno:%d\n", err, errno);
502 	do_test(type, family, &cmd, PASS);
503 	err = bpf_map_lookup_elem(tmp_index_ovr_map, &index_zero,
504 				  &tmp_index);
505 	CHECK(err == -1 || tmp_index != -1,
506 	      "lookup_elem(tmp_index_ovr_map)",
507 	      "err:%d errno:%d tmp_index:%d\n",
508 	      err, errno, tmp_index);
509 	disable_syncookie();
510 	printf("OK\n");
511 }
512 
test_pass_on_err(int type,sa_family_t family)513 static void test_pass_on_err(int type, sa_family_t family)
514 {
515 	struct cmd cmd = {
516 		.reuseport_index = REUSEPORT_ARRAY_SIZE,
517 		.pass_on_failure = 1,
518 	};
519 
520 	printf("%s: ", __func__);
521 	expected_results[PASS_ERR_SK_SELECT_REUSEPORT] += 1;
522 	do_test(type, family, &cmd, PASS_ERR_SK_SELECT_REUSEPORT);
523 	printf("OK\n");
524 }
525 
prepare_sk_fds(int type,sa_family_t family,bool inany)526 static void prepare_sk_fds(int type, sa_family_t family, bool inany)
527 {
528 	const int first = REUSEPORT_ARRAY_SIZE - 1;
529 	int i, err, optval = 1;
530 	struct epoll_event ev;
531 	socklen_t addrlen;
532 
533 	if (inany)
534 		sa46_init_inany(&srv_sa, family);
535 	else
536 		sa46_init_loopback(&srv_sa, family);
537 	addrlen = sizeof(srv_sa);
538 
539 	/*
540 	 * The sk_fds[] is filled from the back such that the order
541 	 * is exactly opposite to the (struct sock_reuseport *)reuse->socks[].
542 	 */
543 	for (i = first; i >= 0; i--) {
544 		sk_fds[i] = socket(family, type, 0);
545 		CHECK(sk_fds[i] == -1, "socket()", "sk_fds[%d]:%d errno:%d\n",
546 		      i, sk_fds[i], errno);
547 		err = setsockopt(sk_fds[i], SOL_SOCKET, SO_REUSEPORT,
548 				 &optval, sizeof(optval));
549 		CHECK(err == -1, "setsockopt(SO_REUSEPORT)",
550 		      "sk_fds[%d] err:%d errno:%d\n",
551 		      i, err, errno);
552 
553 		if (i == first) {
554 			err = setsockopt(sk_fds[i], SOL_SOCKET,
555 					 SO_ATTACH_REUSEPORT_EBPF,
556 					 &select_by_skb_data_prog,
557 					 sizeof(select_by_skb_data_prog));
558 			CHECK(err == -1, "setsockopt(SO_ATTACH_REUEPORT_EBPF)",
559 			      "err:%d errno:%d\n", err, errno);
560 		}
561 
562 		err = bind(sk_fds[i], (struct sockaddr *)&srv_sa, addrlen);
563 		CHECK(err == -1, "bind()", "sk_fds[%d] err:%d errno:%d\n",
564 		      i, err, errno);
565 
566 		if (type == SOCK_STREAM) {
567 			err = listen(sk_fds[i], 10);
568 			CHECK(err == -1, "listen()",
569 			      "sk_fds[%d] err:%d errno:%d\n",
570 			      i, err, errno);
571 		}
572 
573 		err = bpf_map_update_elem(reuseport_array, &i, &sk_fds[i],
574 					  BPF_NOEXIST);
575 		CHECK(err == -1, "update_elem(reuseport_array)",
576 		      "sk_fds[%d] err:%d errno:%d\n", i, err, errno);
577 
578 		if (i == first) {
579 			socklen_t addrlen = sizeof(srv_sa);
580 
581 			err = getsockname(sk_fds[i], (struct sockaddr *)&srv_sa,
582 					  &addrlen);
583 			CHECK(err == -1, "getsockname()",
584 			      "sk_fds[%d] err:%d errno:%d\n", i, err, errno);
585 		}
586 	}
587 
588 	epfd = epoll_create(1);
589 	CHECK(epfd == -1, "epoll_create(1)",
590 	      "epfd:%d errno:%d\n", epfd, errno);
591 
592 	ev.events = EPOLLIN;
593 	for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++) {
594 		ev.data.u32 = i;
595 		err = epoll_ctl(epfd, EPOLL_CTL_ADD, sk_fds[i], &ev);
596 		CHECK(err, "epoll_ctl(EPOLL_CTL_ADD)", "sk_fds[%d]\n", i);
597 	}
598 }
599 
setup_per_test(int type,unsigned short family,bool inany)600 static void setup_per_test(int type, unsigned short family, bool inany)
601 {
602 	int ovr = -1, err;
603 
604 	prepare_sk_fds(type, family, inany);
605 	err = bpf_map_update_elem(tmp_index_ovr_map, &index_zero, &ovr,
606 				  BPF_ANY);
607 	CHECK(err == -1, "update_elem(tmp_index_ovr_map, 0, -1)",
608 	      "err:%d errno:%d\n", err, errno);
609 }
610 
cleanup_per_test(void)611 static void cleanup_per_test(void)
612 {
613 	int i, err;
614 
615 	for (i = 0; i < REUSEPORT_ARRAY_SIZE; i++)
616 		close(sk_fds[i]);
617 	close(epfd);
618 
619 	err = bpf_map_delete_elem(outer_map, &index_zero);
620 	CHECK(err == -1, "delete_elem(outer_map)",
621 	      "err:%d errno:%d\n", err, errno);
622 }
623 
cleanup(void)624 static void cleanup(void)
625 {
626 	close(outer_map);
627 	close(reuseport_array);
628 	bpf_object__close(obj);
629 }
630 
test_all(void)631 static void test_all(void)
632 {
633 	/* Extra SOCK_STREAM to test bind_inany==true */
634 	const int types[] = { SOCK_STREAM, SOCK_DGRAM, SOCK_STREAM };
635 	const char * const type_strings[] = { "TCP", "UDP", "TCP" };
636 	const char * const family_strings[] = { "IPv6", "IPv4" };
637 	const unsigned short families[] = { AF_INET6, AF_INET };
638 	const bool bind_inany[] = { false, false, true };
639 	int t, f, err;
640 
641 	for (f = 0; f < ARRAY_SIZE(families); f++) {
642 		unsigned short family = families[f];
643 
644 		for (t = 0; t < ARRAY_SIZE(types); t++) {
645 			bool inany = bind_inany[t];
646 			int type = types[t];
647 
648 			printf("######## %s/%s %s ########\n",
649 			       family_strings[f], type_strings[t],
650 				inany ? " INANY  " : "LOOPBACK");
651 
652 			setup_per_test(type, family, inany);
653 
654 			test_err_inner_map(type, family);
655 
656 			/* Install reuseport_array to the outer_map */
657 			err = bpf_map_update_elem(outer_map, &index_zero,
658 						  &reuseport_array, BPF_ANY);
659 			CHECK(err == -1, "update_elem(outer_map)",
660 			      "err:%d errno:%d\n", err, errno);
661 
662 			test_err_skb_data(type, family);
663 			test_err_sk_select_port(type, family);
664 			test_pass(type, family);
665 			test_syncookie(type, family);
666 			test_pass_on_err(type, family);
667 
668 			cleanup_per_test();
669 			printf("\n");
670 		}
671 	}
672 }
673 
main(int argc,const char ** argv)674 int main(int argc, const char **argv)
675 {
676 	create_maps();
677 	prepare_bpf_obj();
678 	saved_tcp_fo = read_int_sysctl(TCP_FO_SYSCTL);
679 	saved_tcp_syncookie = read_int_sysctl(TCP_SYNCOOKIE_SYSCTL);
680 	enable_fastopen();
681 	disable_syncookie();
682 	atexit(restore_sysctls);
683 
684 	test_all();
685 
686 	cleanup();
687 	return 0;
688 }
689