1 // SPDX-License-Identifier: GPL-2.0
2 // Copyright (c) 2017-2018 Covalent IO, Inc. http://covalent.io
3 #include <stdio.h>
4 #include <stdlib.h>
5 #include <sys/socket.h>
6 #include <sys/ioctl.h>
7 #include <sys/select.h>
8 #include <netinet/in.h>
9 #include <arpa/inet.h>
10 #include <unistd.h>
11 #include <string.h>
12 #include <errno.h>
13 #include <sys/ioctl.h>
14 #include <stdbool.h>
15 #include <signal.h>
16 #include <fcntl.h>
17 #include <sys/wait.h>
18 #include <time.h>
19 #include <sched.h>
20
21 #include <sys/time.h>
22 #include <sys/resource.h>
23 #include <sys/types.h>
24 #include <sys/sendfile.h>
25
26 #include <linux/netlink.h>
27 #include <linux/socket.h>
28 #include <linux/sock_diag.h>
29 #include <linux/bpf.h>
30 #include <linux/if_link.h>
31 #include <linux/tls.h>
32 #include <assert.h>
33 #include <libgen.h>
34
35 #include <getopt.h>
36
37 #include <bpf/bpf.h>
38 #include <bpf/libbpf.h>
39
40 #include "bpf_util.h"
41 #include "bpf_rlimit.h"
42 #include "cgroup_helpers.h"
43
44 int running;
45 static void running_handler(int a);
46
47 #ifndef TCP_ULP
48 # define TCP_ULP 31
49 #endif
50 #ifndef SOL_TLS
51 # define SOL_TLS 282
52 #endif
53
54 /* randomly selected ports for testing on lo */
55 #define S1_PORT 10000
56 #define S2_PORT 10001
57
58 #define BPF_SOCKMAP_FILENAME "test_sockmap_kern.o"
59 #define BPF_SOCKHASH_FILENAME "test_sockhash_kern.o"
60 #define CG_PATH "/sockmap"
61
62 /* global sockets */
63 int s1, s2, c1, c2, p1, p2;
64 int test_cnt;
65 int passed;
66 int failed;
67 int map_fd[8];
68 struct bpf_map *maps[8];
69 int prog_fd[11];
70
71 int txmsg_pass;
72 int txmsg_noisy;
73 int txmsg_redir;
74 int txmsg_redir_noisy;
75 int txmsg_drop;
76 int txmsg_apply;
77 int txmsg_cork;
78 int txmsg_start;
79 int txmsg_end;
80 int txmsg_start_push;
81 int txmsg_end_push;
82 int txmsg_ingress;
83 int txmsg_skb;
84 int ktls;
85 int peek_flag;
86
87 static const struct option long_options[] = {
88 {"help", no_argument, NULL, 'h' },
89 {"cgroup", required_argument, NULL, 'c' },
90 {"rate", required_argument, NULL, 'r' },
91 {"verbose", no_argument, NULL, 'v' },
92 {"iov_count", required_argument, NULL, 'i' },
93 {"length", required_argument, NULL, 'l' },
94 {"test", required_argument, NULL, 't' },
95 {"data_test", no_argument, NULL, 'd' },
96 {"txmsg", no_argument, &txmsg_pass, 1 },
97 {"txmsg_noisy", no_argument, &txmsg_noisy, 1 },
98 {"txmsg_redir", no_argument, &txmsg_redir, 1 },
99 {"txmsg_redir_noisy", no_argument, &txmsg_redir_noisy, 1},
100 {"txmsg_drop", no_argument, &txmsg_drop, 1 },
101 {"txmsg_apply", required_argument, NULL, 'a'},
102 {"txmsg_cork", required_argument, NULL, 'k'},
103 {"txmsg_start", required_argument, NULL, 's'},
104 {"txmsg_end", required_argument, NULL, 'e'},
105 {"txmsg_start_push", required_argument, NULL, 'p'},
106 {"txmsg_end_push", required_argument, NULL, 'q'},
107 {"txmsg_ingress", no_argument, &txmsg_ingress, 1 },
108 {"txmsg_skb", no_argument, &txmsg_skb, 1 },
109 {"ktls", no_argument, &ktls, 1 },
110 {"peek", no_argument, &peek_flag, 1 },
111 {0, 0, NULL, 0 }
112 };
113
usage(char * argv[])114 static void usage(char *argv[])
115 {
116 int i;
117
118 printf(" Usage: %s --cgroup <cgroup_path>\n", argv[0]);
119 printf(" options:\n");
120 for (i = 0; long_options[i].name != 0; i++) {
121 printf(" --%-12s", long_options[i].name);
122 if (long_options[i].flag != NULL)
123 printf(" flag (internal value:%d)\n",
124 *long_options[i].flag);
125 else
126 printf(" -%c\n", long_options[i].val);
127 }
128 printf("\n");
129 }
130
sock_to_string(int s)131 char *sock_to_string(int s)
132 {
133 if (s == c1)
134 return "client1";
135 else if (s == c2)
136 return "client2";
137 else if (s == s1)
138 return "server1";
139 else if (s == s2)
140 return "server2";
141 else if (s == p1)
142 return "peer1";
143 else if (s == p2)
144 return "peer2";
145 else
146 return "unknown";
147 }
148
sockmap_init_ktls(int verbose,int s)149 static int sockmap_init_ktls(int verbose, int s)
150 {
151 struct tls12_crypto_info_aes_gcm_128 tls_tx = {
152 .info = {
153 .version = TLS_1_2_VERSION,
154 .cipher_type = TLS_CIPHER_AES_GCM_128,
155 },
156 };
157 struct tls12_crypto_info_aes_gcm_128 tls_rx = {
158 .info = {
159 .version = TLS_1_2_VERSION,
160 .cipher_type = TLS_CIPHER_AES_GCM_128,
161 },
162 };
163 int so_buf = 6553500;
164 int err;
165
166 err = setsockopt(s, 6, TCP_ULP, "tls", sizeof("tls"));
167 if (err) {
168 fprintf(stderr, "setsockopt: TCP_ULP(%s) failed with error %i\n", sock_to_string(s), err);
169 return -EINVAL;
170 }
171 err = setsockopt(s, SOL_TLS, TLS_TX, (void *)&tls_tx, sizeof(tls_tx));
172 if (err) {
173 fprintf(stderr, "setsockopt: TLS_TX(%s) failed with error %i\n", sock_to_string(s), err);
174 return -EINVAL;
175 }
176 err = setsockopt(s, SOL_TLS, TLS_RX, (void *)&tls_rx, sizeof(tls_rx));
177 if (err) {
178 fprintf(stderr, "setsockopt: TLS_RX(%s) failed with error %i\n", sock_to_string(s), err);
179 return -EINVAL;
180 }
181 err = setsockopt(s, SOL_SOCKET, SO_SNDBUF, &so_buf, sizeof(so_buf));
182 if (err) {
183 fprintf(stderr, "setsockopt: (%s) failed sndbuf with error %i\n", sock_to_string(s), err);
184 return -EINVAL;
185 }
186 err = setsockopt(s, SOL_SOCKET, SO_RCVBUF, &so_buf, sizeof(so_buf));
187 if (err) {
188 fprintf(stderr, "setsockopt: (%s) failed rcvbuf with error %i\n", sock_to_string(s), err);
189 return -EINVAL;
190 }
191
192 if (verbose)
193 fprintf(stdout, "socket(%s) kTLS enabled\n", sock_to_string(s));
194 return 0;
195 }
sockmap_init_sockets(int verbose)196 static int sockmap_init_sockets(int verbose)
197 {
198 int i, err, one = 1;
199 struct sockaddr_in addr;
200 int *fds[4] = {&s1, &s2, &c1, &c2};
201
202 s1 = s2 = p1 = p2 = c1 = c2 = 0;
203
204 /* Init sockets */
205 for (i = 0; i < 4; i++) {
206 *fds[i] = socket(AF_INET, SOCK_STREAM, 0);
207 if (*fds[i] < 0) {
208 perror("socket s1 failed()");
209 return errno;
210 }
211 }
212
213 /* Allow reuse */
214 for (i = 0; i < 2; i++) {
215 err = setsockopt(*fds[i], SOL_SOCKET, SO_REUSEADDR,
216 (char *)&one, sizeof(one));
217 if (err) {
218 perror("setsockopt failed()");
219 return errno;
220 }
221 }
222
223 /* Non-blocking sockets */
224 for (i = 0; i < 2; i++) {
225 err = ioctl(*fds[i], FIONBIO, (char *)&one);
226 if (err < 0) {
227 perror("ioctl s1 failed()");
228 return errno;
229 }
230 }
231
232 /* Bind server sockets */
233 memset(&addr, 0, sizeof(struct sockaddr_in));
234 addr.sin_family = AF_INET;
235 addr.sin_addr.s_addr = inet_addr("127.0.0.1");
236
237 addr.sin_port = htons(S1_PORT);
238 err = bind(s1, (struct sockaddr *)&addr, sizeof(addr));
239 if (err < 0) {
240 perror("bind s1 failed()\n");
241 return errno;
242 }
243
244 addr.sin_port = htons(S2_PORT);
245 err = bind(s2, (struct sockaddr *)&addr, sizeof(addr));
246 if (err < 0) {
247 perror("bind s2 failed()\n");
248 return errno;
249 }
250
251 /* Listen server sockets */
252 addr.sin_port = htons(S1_PORT);
253 err = listen(s1, 32);
254 if (err < 0) {
255 perror("listen s1 failed()\n");
256 return errno;
257 }
258
259 addr.sin_port = htons(S2_PORT);
260 err = listen(s2, 32);
261 if (err < 0) {
262 perror("listen s1 failed()\n");
263 return errno;
264 }
265
266 /* Initiate Connect */
267 addr.sin_port = htons(S1_PORT);
268 err = connect(c1, (struct sockaddr *)&addr, sizeof(addr));
269 if (err < 0 && errno != EINPROGRESS) {
270 perror("connect c1 failed()\n");
271 return errno;
272 }
273
274 addr.sin_port = htons(S2_PORT);
275 err = connect(c2, (struct sockaddr *)&addr, sizeof(addr));
276 if (err < 0 && errno != EINPROGRESS) {
277 perror("connect c2 failed()\n");
278 return errno;
279 } else if (err < 0) {
280 err = 0;
281 }
282
283 /* Accept Connecrtions */
284 p1 = accept(s1, NULL, NULL);
285 if (p1 < 0) {
286 perror("accept s1 failed()\n");
287 return errno;
288 }
289
290 p2 = accept(s2, NULL, NULL);
291 if (p2 < 0) {
292 perror("accept s1 failed()\n");
293 return errno;
294 }
295
296 if (verbose) {
297 printf("connected sockets: c1 <-> p1, c2 <-> p2\n");
298 printf("cgroups binding: c1(%i) <-> s1(%i) - - - c2(%i) <-> s2(%i)\n",
299 c1, s1, c2, s2);
300 }
301 return 0;
302 }
303
304 struct msg_stats {
305 size_t bytes_sent;
306 size_t bytes_recvd;
307 struct timespec start;
308 struct timespec end;
309 };
310
311 struct sockmap_options {
312 int verbose;
313 bool base;
314 bool sendpage;
315 bool data_test;
316 bool drop_expected;
317 int iov_count;
318 int iov_length;
319 int rate;
320 };
321
msg_loop_sendpage(int fd,int iov_length,int cnt,struct msg_stats * s,struct sockmap_options * opt)322 static int msg_loop_sendpage(int fd, int iov_length, int cnt,
323 struct msg_stats *s,
324 struct sockmap_options *opt)
325 {
326 bool drop = opt->drop_expected;
327 unsigned char k = 0;
328 FILE *file;
329 int i, fp;
330
331 file = fopen(".sendpage_tst.tmp", "w+");
332 for (i = 0; i < iov_length * cnt; i++, k++)
333 fwrite(&k, sizeof(char), 1, file);
334 fflush(file);
335 fseek(file, 0, SEEK_SET);
336 fclose(file);
337
338 fp = open(".sendpage_tst.tmp", O_RDONLY);
339 clock_gettime(CLOCK_MONOTONIC, &s->start);
340 for (i = 0; i < cnt; i++) {
341 int sent = sendfile(fd, fp, NULL, iov_length);
342
343 if (!drop && sent < 0) {
344 perror("send loop error:");
345 close(fp);
346 return sent;
347 } else if (drop && sent >= 0) {
348 printf("sendpage loop error expected: %i\n", sent);
349 close(fp);
350 return -EIO;
351 }
352
353 if (sent > 0)
354 s->bytes_sent += sent;
355 }
356 clock_gettime(CLOCK_MONOTONIC, &s->end);
357 close(fp);
358 return 0;
359 }
360
msg_free_iov(struct msghdr * msg)361 static void msg_free_iov(struct msghdr *msg)
362 {
363 int i;
364
365 for (i = 0; i < msg->msg_iovlen; i++)
366 free(msg->msg_iov[i].iov_base);
367 free(msg->msg_iov);
368 msg->msg_iov = NULL;
369 msg->msg_iovlen = 0;
370 }
371
msg_alloc_iov(struct msghdr * msg,int iov_count,int iov_length,bool data,bool xmit)372 static int msg_alloc_iov(struct msghdr *msg,
373 int iov_count, int iov_length,
374 bool data, bool xmit)
375 {
376 unsigned char k = 0;
377 struct iovec *iov;
378 int i;
379
380 iov = calloc(iov_count, sizeof(struct iovec));
381 if (!iov)
382 return errno;
383
384 for (i = 0; i < iov_count; i++) {
385 unsigned char *d = calloc(iov_length, sizeof(char));
386
387 if (!d) {
388 fprintf(stderr, "iov_count %i/%i OOM\n", i, iov_count);
389 goto unwind_iov;
390 }
391 iov[i].iov_base = d;
392 iov[i].iov_len = iov_length;
393
394 if (data && xmit) {
395 int j;
396
397 for (j = 0; j < iov_length; j++)
398 d[j] = k++;
399 }
400 }
401
402 msg->msg_iov = iov;
403 msg->msg_iovlen = iov_count;
404
405 return 0;
406 unwind_iov:
407 for (i--; i >= 0 ; i--)
408 free(msg->msg_iov[i].iov_base);
409 return -ENOMEM;
410 }
411
msg_verify_data(struct msghdr * msg,int size,int chunk_sz)412 static int msg_verify_data(struct msghdr *msg, int size, int chunk_sz)
413 {
414 int i, j, bytes_cnt = 0;
415 unsigned char k = 0;
416
417 for (i = 0; i < msg->msg_iovlen; i++) {
418 unsigned char *d = msg->msg_iov[i].iov_base;
419
420 for (j = 0;
421 j < msg->msg_iov[i].iov_len && size; j++) {
422 if (d[j] != k++) {
423 fprintf(stderr,
424 "detected data corruption @iov[%i]:%i %02x != %02x, %02x ?= %02x\n",
425 i, j, d[j], k - 1, d[j+1], k);
426 return -EIO;
427 }
428 bytes_cnt++;
429 if (bytes_cnt == chunk_sz) {
430 k = 0;
431 bytes_cnt = 0;
432 }
433 size--;
434 }
435 }
436 return 0;
437 }
438
msg_loop(int fd,int iov_count,int iov_length,int cnt,struct msg_stats * s,bool tx,struct sockmap_options * opt)439 static int msg_loop(int fd, int iov_count, int iov_length, int cnt,
440 struct msg_stats *s, bool tx,
441 struct sockmap_options *opt)
442 {
443 struct msghdr msg = {0}, msg_peek = {0};
444 int err, i, flags = MSG_NOSIGNAL;
445 bool drop = opt->drop_expected;
446 bool data = opt->data_test;
447
448 err = msg_alloc_iov(&msg, iov_count, iov_length, data, tx);
449 if (err)
450 goto out_errno;
451 if (peek_flag) {
452 err = msg_alloc_iov(&msg_peek, iov_count, iov_length, data, tx);
453 if (err)
454 goto out_errno;
455 }
456
457 if (tx) {
458 clock_gettime(CLOCK_MONOTONIC, &s->start);
459 for (i = 0; i < cnt; i++) {
460 int sent = sendmsg(fd, &msg, flags);
461
462 if (!drop && sent < 0) {
463 perror("send loop error:");
464 goto out_errno;
465 } else if (drop && sent >= 0) {
466 printf("send loop error expected: %i\n", sent);
467 errno = -EIO;
468 goto out_errno;
469 }
470 if (sent > 0)
471 s->bytes_sent += sent;
472 }
473 clock_gettime(CLOCK_MONOTONIC, &s->end);
474 } else {
475 int slct, recvp = 0, recv, max_fd = fd;
476 int fd_flags = O_NONBLOCK;
477 struct timeval timeout;
478 float total_bytes;
479 fd_set w;
480
481 fcntl(fd, fd_flags);
482 total_bytes = (float)iov_count * (float)iov_length * (float)cnt;
483 err = clock_gettime(CLOCK_MONOTONIC, &s->start);
484 if (err < 0)
485 perror("recv start time: ");
486 while (s->bytes_recvd < total_bytes) {
487 if (txmsg_cork) {
488 timeout.tv_sec = 0;
489 timeout.tv_usec = 300000;
490 } else {
491 timeout.tv_sec = 1;
492 timeout.tv_usec = 0;
493 }
494
495 /* FD sets */
496 FD_ZERO(&w);
497 FD_SET(fd, &w);
498
499 slct = select(max_fd + 1, &w, NULL, NULL, &timeout);
500 if (slct == -1) {
501 perror("select()");
502 clock_gettime(CLOCK_MONOTONIC, &s->end);
503 goto out_errno;
504 } else if (!slct) {
505 if (opt->verbose)
506 fprintf(stderr, "unexpected timeout\n");
507 errno = -EIO;
508 clock_gettime(CLOCK_MONOTONIC, &s->end);
509 goto out_errno;
510 }
511
512 errno = 0;
513 if (peek_flag) {
514 flags |= MSG_PEEK;
515 recvp = recvmsg(fd, &msg_peek, flags);
516 if (recvp < 0) {
517 if (errno != EWOULDBLOCK) {
518 clock_gettime(CLOCK_MONOTONIC, &s->end);
519 goto out_errno;
520 }
521 }
522 flags = 0;
523 }
524
525 recv = recvmsg(fd, &msg, flags);
526 if (recv < 0) {
527 if (errno != EWOULDBLOCK) {
528 clock_gettime(CLOCK_MONOTONIC, &s->end);
529 perror("recv failed()\n");
530 goto out_errno;
531 }
532 }
533
534 s->bytes_recvd += recv;
535
536 if (data) {
537 int chunk_sz = opt->sendpage ?
538 iov_length * cnt :
539 iov_length * iov_count;
540
541 errno = msg_verify_data(&msg, recv, chunk_sz);
542 if (errno) {
543 perror("data verify msg failed\n");
544 goto out_errno;
545 }
546 if (recvp) {
547 errno = msg_verify_data(&msg_peek,
548 recvp,
549 chunk_sz);
550 if (errno) {
551 perror("data verify msg_peek failed\n");
552 goto out_errno;
553 }
554 }
555 }
556 }
557 clock_gettime(CLOCK_MONOTONIC, &s->end);
558 }
559
560 msg_free_iov(&msg);
561 msg_free_iov(&msg_peek);
562 return err;
563 out_errno:
564 msg_free_iov(&msg);
565 msg_free_iov(&msg_peek);
566 return errno;
567 }
568
569 static float giga = 1000000000;
570
sentBps(struct msg_stats s)571 static inline float sentBps(struct msg_stats s)
572 {
573 return s.bytes_sent / (s.end.tv_sec - s.start.tv_sec);
574 }
575
recvdBps(struct msg_stats s)576 static inline float recvdBps(struct msg_stats s)
577 {
578 return s.bytes_recvd / (s.end.tv_sec - s.start.tv_sec);
579 }
580
sendmsg_test(struct sockmap_options * opt)581 static int sendmsg_test(struct sockmap_options *opt)
582 {
583 float sent_Bps = 0, recvd_Bps = 0;
584 int rx_fd, txpid, rxpid, err = 0;
585 struct msg_stats s = {0};
586 int iov_count = opt->iov_count;
587 int iov_buf = opt->iov_length;
588 int rx_status, tx_status;
589 int cnt = opt->rate;
590
591 errno = 0;
592
593 if (opt->base)
594 rx_fd = p1;
595 else
596 rx_fd = p2;
597
598 if (ktls) {
599 /* Redirecting into non-TLS socket which sends into a TLS
600 * socket is not a valid test. So in this case lets not
601 * enable kTLS but still run the test.
602 */
603 if (!txmsg_redir || (txmsg_redir && txmsg_ingress)) {
604 err = sockmap_init_ktls(opt->verbose, rx_fd);
605 if (err)
606 return err;
607 }
608 err = sockmap_init_ktls(opt->verbose, c1);
609 if (err)
610 return err;
611 }
612
613 rxpid = fork();
614 if (rxpid == 0) {
615 if (opt->drop_expected)
616 exit(0);
617
618 if (opt->sendpage)
619 iov_count = 1;
620 err = msg_loop(rx_fd, iov_count, iov_buf,
621 cnt, &s, false, opt);
622 if (err && opt->verbose)
623 fprintf(stderr,
624 "msg_loop_rx: iov_count %i iov_buf %i cnt %i err %i\n",
625 iov_count, iov_buf, cnt, err);
626 if (s.end.tv_sec - s.start.tv_sec) {
627 sent_Bps = sentBps(s);
628 recvd_Bps = recvdBps(s);
629 }
630 if (opt->verbose)
631 fprintf(stdout,
632 "rx_sendmsg: TX: %zuB %fB/s %fGB/s RX: %zuB %fB/s %fGB/s %s\n",
633 s.bytes_sent, sent_Bps, sent_Bps/giga,
634 s.bytes_recvd, recvd_Bps, recvd_Bps/giga,
635 peek_flag ? "(peek_msg)" : "");
636 if (err && txmsg_cork)
637 err = 0;
638 exit(err ? 1 : 0);
639 } else if (rxpid == -1) {
640 perror("msg_loop_rx: ");
641 return errno;
642 }
643
644 txpid = fork();
645 if (txpid == 0) {
646 if (opt->sendpage)
647 err = msg_loop_sendpage(c1, iov_buf, cnt, &s, opt);
648 else
649 err = msg_loop(c1, iov_count, iov_buf,
650 cnt, &s, true, opt);
651
652 if (err)
653 fprintf(stderr,
654 "msg_loop_tx: iov_count %i iov_buf %i cnt %i err %i\n",
655 iov_count, iov_buf, cnt, err);
656 if (s.end.tv_sec - s.start.tv_sec) {
657 sent_Bps = sentBps(s);
658 recvd_Bps = recvdBps(s);
659 }
660 if (opt->verbose)
661 fprintf(stdout,
662 "tx_sendmsg: TX: %zuB %fB/s %f GB/s RX: %zuB %fB/s %fGB/s\n",
663 s.bytes_sent, sent_Bps, sent_Bps/giga,
664 s.bytes_recvd, recvd_Bps, recvd_Bps/giga);
665 exit(err ? 1 : 0);
666 } else if (txpid == -1) {
667 perror("msg_loop_tx: ");
668 return errno;
669 }
670
671 assert(waitpid(rxpid, &rx_status, 0) == rxpid);
672 assert(waitpid(txpid, &tx_status, 0) == txpid);
673 if (WIFEXITED(rx_status)) {
674 err = WEXITSTATUS(rx_status);
675 if (err) {
676 fprintf(stderr, "rx thread exited with err %d. ", err);
677 goto out;
678 }
679 }
680 if (WIFEXITED(tx_status)) {
681 err = WEXITSTATUS(tx_status);
682 if (err)
683 fprintf(stderr, "tx thread exited with err %d. ", err);
684 }
685 out:
686 return err;
687 }
688
forever_ping_pong(int rate,struct sockmap_options * opt)689 static int forever_ping_pong(int rate, struct sockmap_options *opt)
690 {
691 struct timeval timeout;
692 char buf[1024] = {0};
693 int sc;
694
695 timeout.tv_sec = 10;
696 timeout.tv_usec = 0;
697
698 /* Ping/Pong data from client to server */
699 sc = send(c1, buf, sizeof(buf), 0);
700 if (sc < 0) {
701 perror("send failed()\n");
702 return sc;
703 }
704
705 do {
706 int s, rc, i, max_fd = p2;
707 fd_set w;
708
709 /* FD sets */
710 FD_ZERO(&w);
711 FD_SET(c1, &w);
712 FD_SET(c2, &w);
713 FD_SET(p1, &w);
714 FD_SET(p2, &w);
715
716 s = select(max_fd + 1, &w, NULL, NULL, &timeout);
717 if (s == -1) {
718 perror("select()");
719 break;
720 } else if (!s) {
721 fprintf(stderr, "unexpected timeout\n");
722 break;
723 }
724
725 for (i = 0; i <= max_fd && s > 0; ++i) {
726 if (!FD_ISSET(i, &w))
727 continue;
728
729 s--;
730
731 rc = recv(i, buf, sizeof(buf), 0);
732 if (rc < 0) {
733 if (errno != EWOULDBLOCK) {
734 perror("recv failed()\n");
735 return rc;
736 }
737 }
738
739 if (rc == 0) {
740 close(i);
741 break;
742 }
743
744 sc = send(i, buf, rc, 0);
745 if (sc < 0) {
746 perror("send failed()\n");
747 return sc;
748 }
749 }
750
751 if (rate)
752 sleep(rate);
753
754 if (opt->verbose) {
755 printf(".");
756 fflush(stdout);
757
758 }
759 } while (running);
760
761 return 0;
762 }
763
764 enum {
765 PING_PONG,
766 SENDMSG,
767 BASE,
768 BASE_SENDPAGE,
769 SENDPAGE,
770 };
771
run_options(struct sockmap_options * options,int cg_fd,int test)772 static int run_options(struct sockmap_options *options, int cg_fd, int test)
773 {
774 int i, key, next_key, err, tx_prog_fd = -1, zero = 0;
775
776 /* If base test skip BPF setup */
777 if (test == BASE || test == BASE_SENDPAGE)
778 goto run;
779
780 /* Attach programs to sockmap */
781 err = bpf_prog_attach(prog_fd[0], map_fd[0],
782 BPF_SK_SKB_STREAM_PARSER, 0);
783 if (err) {
784 fprintf(stderr,
785 "ERROR: bpf_prog_attach (sockmap %i->%i): %d (%s)\n",
786 prog_fd[0], map_fd[0], err, strerror(errno));
787 return err;
788 }
789
790 err = bpf_prog_attach(prog_fd[1], map_fd[0],
791 BPF_SK_SKB_STREAM_VERDICT, 0);
792 if (err) {
793 fprintf(stderr, "ERROR: bpf_prog_attach (sockmap): %d (%s)\n",
794 err, strerror(errno));
795 return err;
796 }
797
798 /* Attach to cgroups */
799 err = bpf_prog_attach(prog_fd[2], cg_fd, BPF_CGROUP_SOCK_OPS, 0);
800 if (err) {
801 fprintf(stderr, "ERROR: bpf_prog_attach (groups): %d (%s)\n",
802 err, strerror(errno));
803 return err;
804 }
805
806 run:
807 err = sockmap_init_sockets(options->verbose);
808 if (err) {
809 fprintf(stderr, "ERROR: test socket failed: %d\n", err);
810 goto out;
811 }
812
813 /* Attach txmsg program to sockmap */
814 if (txmsg_pass)
815 tx_prog_fd = prog_fd[3];
816 else if (txmsg_noisy)
817 tx_prog_fd = prog_fd[4];
818 else if (txmsg_redir)
819 tx_prog_fd = prog_fd[5];
820 else if (txmsg_redir_noisy)
821 tx_prog_fd = prog_fd[6];
822 else if (txmsg_drop)
823 tx_prog_fd = prog_fd[9];
824 /* apply and cork must be last */
825 else if (txmsg_apply)
826 tx_prog_fd = prog_fd[7];
827 else if (txmsg_cork)
828 tx_prog_fd = prog_fd[8];
829 else
830 tx_prog_fd = 0;
831
832 if (tx_prog_fd) {
833 int redir_fd, i = 0;
834
835 err = bpf_prog_attach(tx_prog_fd,
836 map_fd[1], BPF_SK_MSG_VERDICT, 0);
837 if (err) {
838 fprintf(stderr,
839 "ERROR: bpf_prog_attach (txmsg): %d (%s)\n",
840 err, strerror(errno));
841 goto out;
842 }
843
844 err = bpf_map_update_elem(map_fd[1], &i, &c1, BPF_ANY);
845 if (err) {
846 fprintf(stderr,
847 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
848 err, strerror(errno));
849 goto out;
850 }
851
852 if (txmsg_redir || txmsg_redir_noisy)
853 redir_fd = c2;
854 else
855 redir_fd = c1;
856
857 err = bpf_map_update_elem(map_fd[2], &i, &redir_fd, BPF_ANY);
858 if (err) {
859 fprintf(stderr,
860 "ERROR: bpf_map_update_elem (txmsg): %d (%s\n",
861 err, strerror(errno));
862 goto out;
863 }
864
865 if (txmsg_apply) {
866 err = bpf_map_update_elem(map_fd[3],
867 &i, &txmsg_apply, BPF_ANY);
868 if (err) {
869 fprintf(stderr,
870 "ERROR: bpf_map_update_elem (apply_bytes): %d (%s\n",
871 err, strerror(errno));
872 goto out;
873 }
874 }
875
876 if (txmsg_cork) {
877 err = bpf_map_update_elem(map_fd[4],
878 &i, &txmsg_cork, BPF_ANY);
879 if (err) {
880 fprintf(stderr,
881 "ERROR: bpf_map_update_elem (cork_bytes): %d (%s\n",
882 err, strerror(errno));
883 goto out;
884 }
885 }
886
887 if (txmsg_start) {
888 err = bpf_map_update_elem(map_fd[5],
889 &i, &txmsg_start, BPF_ANY);
890 if (err) {
891 fprintf(stderr,
892 "ERROR: bpf_map_update_elem (txmsg_start): %d (%s)\n",
893 err, strerror(errno));
894 goto out;
895 }
896 }
897
898 if (txmsg_end) {
899 i = 1;
900 err = bpf_map_update_elem(map_fd[5],
901 &i, &txmsg_end, BPF_ANY);
902 if (err) {
903 fprintf(stderr,
904 "ERROR: bpf_map_update_elem (txmsg_end): %d (%s)\n",
905 err, strerror(errno));
906 goto out;
907 }
908 }
909
910 if (txmsg_start_push) {
911 i = 2;
912 err = bpf_map_update_elem(map_fd[5],
913 &i, &txmsg_start_push, BPF_ANY);
914 if (err) {
915 fprintf(stderr,
916 "ERROR: bpf_map_update_elem (txmsg_start_push): %d (%s)\n",
917 err, strerror(errno));
918 goto out;
919 }
920 }
921
922 if (txmsg_end_push) {
923 i = 3;
924 err = bpf_map_update_elem(map_fd[5],
925 &i, &txmsg_end_push, BPF_ANY);
926 if (err) {
927 fprintf(stderr,
928 "ERROR: bpf_map_update_elem %i@%i (txmsg_end_push): %d (%s)\n",
929 txmsg_end_push, i, err, strerror(errno));
930 goto out;
931 }
932 }
933
934 if (txmsg_ingress) {
935 int in = BPF_F_INGRESS;
936
937 i = 0;
938 err = bpf_map_update_elem(map_fd[6], &i, &in, BPF_ANY);
939 if (err) {
940 fprintf(stderr,
941 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
942 err, strerror(errno));
943 }
944 i = 1;
945 err = bpf_map_update_elem(map_fd[1], &i, &p1, BPF_ANY);
946 if (err) {
947 fprintf(stderr,
948 "ERROR: bpf_map_update_elem (p1 txmsg): %d (%s)\n",
949 err, strerror(errno));
950 }
951 err = bpf_map_update_elem(map_fd[2], &i, &p1, BPF_ANY);
952 if (err) {
953 fprintf(stderr,
954 "ERROR: bpf_map_update_elem (p1 redir): %d (%s)\n",
955 err, strerror(errno));
956 }
957
958 i = 2;
959 err = bpf_map_update_elem(map_fd[2], &i, &p2, BPF_ANY);
960 if (err) {
961 fprintf(stderr,
962 "ERROR: bpf_map_update_elem (p2 txmsg): %d (%s)\n",
963 err, strerror(errno));
964 }
965 }
966
967 if (txmsg_skb) {
968 int skb_fd = (test == SENDMSG || test == SENDPAGE) ?
969 p2 : p1;
970 int ingress = BPF_F_INGRESS;
971
972 i = 0;
973 err = bpf_map_update_elem(map_fd[7],
974 &i, &ingress, BPF_ANY);
975 if (err) {
976 fprintf(stderr,
977 "ERROR: bpf_map_update_elem (txmsg_ingress): %d (%s)\n",
978 err, strerror(errno));
979 }
980
981 i = 3;
982 err = bpf_map_update_elem(map_fd[0],
983 &i, &skb_fd, BPF_ANY);
984 if (err) {
985 fprintf(stderr,
986 "ERROR: bpf_map_update_elem (c1 sockmap): %d (%s)\n",
987 err, strerror(errno));
988 }
989 }
990 }
991
992 if (txmsg_drop)
993 options->drop_expected = true;
994
995 if (test == PING_PONG)
996 err = forever_ping_pong(options->rate, options);
997 else if (test == SENDMSG) {
998 options->base = false;
999 options->sendpage = false;
1000 err = sendmsg_test(options);
1001 } else if (test == SENDPAGE) {
1002 options->base = false;
1003 options->sendpage = true;
1004 err = sendmsg_test(options);
1005 } else if (test == BASE) {
1006 options->base = true;
1007 options->sendpage = false;
1008 err = sendmsg_test(options);
1009 } else if (test == BASE_SENDPAGE) {
1010 options->base = true;
1011 options->sendpage = true;
1012 err = sendmsg_test(options);
1013 } else
1014 fprintf(stderr, "unknown test\n");
1015 out:
1016 /* Detatch and zero all the maps */
1017 bpf_prog_detach2(prog_fd[2], cg_fd, BPF_CGROUP_SOCK_OPS);
1018 bpf_prog_detach2(prog_fd[0], map_fd[0], BPF_SK_SKB_STREAM_PARSER);
1019 bpf_prog_detach2(prog_fd[1], map_fd[0], BPF_SK_SKB_STREAM_VERDICT);
1020 if (tx_prog_fd >= 0)
1021 bpf_prog_detach2(tx_prog_fd, map_fd[1], BPF_SK_MSG_VERDICT);
1022
1023 for (i = 0; i < 8; i++) {
1024 key = next_key = 0;
1025 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1026 while (bpf_map_get_next_key(map_fd[i], &key, &next_key) == 0) {
1027 bpf_map_update_elem(map_fd[i], &key, &zero, BPF_ANY);
1028 key = next_key;
1029 }
1030 }
1031
1032 close(s1);
1033 close(s2);
1034 close(p1);
1035 close(p2);
1036 close(c1);
1037 close(c2);
1038 return err;
1039 }
1040
test_to_str(int test)1041 static char *test_to_str(int test)
1042 {
1043 switch (test) {
1044 case SENDMSG:
1045 return "sendmsg";
1046 case SENDPAGE:
1047 return "sendpage";
1048 }
1049 return "unknown";
1050 }
1051
1052 #define OPTSTRING 60
test_options(char * options)1053 static void test_options(char *options)
1054 {
1055 char tstr[OPTSTRING];
1056
1057 memset(options, 0, OPTSTRING);
1058
1059 if (txmsg_pass)
1060 strncat(options, "pass,", OPTSTRING);
1061 if (txmsg_noisy)
1062 strncat(options, "pass_noisy,", OPTSTRING);
1063 if (txmsg_redir)
1064 strncat(options, "redir,", OPTSTRING);
1065 if (txmsg_redir_noisy)
1066 strncat(options, "redir_noisy,", OPTSTRING);
1067 if (txmsg_drop)
1068 strncat(options, "drop,", OPTSTRING);
1069 if (txmsg_apply) {
1070 snprintf(tstr, OPTSTRING, "apply %d,", txmsg_apply);
1071 strncat(options, tstr, OPTSTRING);
1072 }
1073 if (txmsg_cork) {
1074 snprintf(tstr, OPTSTRING, "cork %d,", txmsg_cork);
1075 strncat(options, tstr, OPTSTRING);
1076 }
1077 if (txmsg_start) {
1078 snprintf(tstr, OPTSTRING, "start %d,", txmsg_start);
1079 strncat(options, tstr, OPTSTRING);
1080 }
1081 if (txmsg_end) {
1082 snprintf(tstr, OPTSTRING, "end %d,", txmsg_end);
1083 strncat(options, tstr, OPTSTRING);
1084 }
1085 if (txmsg_ingress)
1086 strncat(options, "ingress,", OPTSTRING);
1087 if (txmsg_skb)
1088 strncat(options, "skb,", OPTSTRING);
1089 if (ktls)
1090 strncat(options, "ktls,", OPTSTRING);
1091 if (peek_flag)
1092 strncat(options, "peek,", OPTSTRING);
1093 }
1094
__test_exec(int cgrp,int test,struct sockmap_options * opt)1095 static int __test_exec(int cgrp, int test, struct sockmap_options *opt)
1096 {
1097 char *options = calloc(OPTSTRING, sizeof(char));
1098 int err;
1099
1100 if (test == SENDPAGE)
1101 opt->sendpage = true;
1102 else
1103 opt->sendpage = false;
1104
1105 if (txmsg_drop)
1106 opt->drop_expected = true;
1107 else
1108 opt->drop_expected = false;
1109
1110 test_options(options);
1111
1112 fprintf(stdout,
1113 "[TEST %i]: (%i, %i, %i, %s, %s): ",
1114 test_cnt, opt->rate, opt->iov_count, opt->iov_length,
1115 test_to_str(test), options);
1116 fflush(stdout);
1117 err = run_options(opt, cgrp, test);
1118 fprintf(stdout, "%s\n", !err ? "PASS" : "FAILED");
1119 test_cnt++;
1120 !err ? passed++ : failed++;
1121 free(options);
1122 return err;
1123 }
1124
test_exec(int cgrp,struct sockmap_options * opt)1125 static int test_exec(int cgrp, struct sockmap_options *opt)
1126 {
1127 int err = __test_exec(cgrp, SENDMSG, opt);
1128
1129 if (err)
1130 goto out;
1131
1132 err = __test_exec(cgrp, SENDPAGE, opt);
1133 out:
1134 return err;
1135 }
1136
test_loop(int cgrp)1137 static int test_loop(int cgrp)
1138 {
1139 struct sockmap_options opt;
1140
1141 int err, i, l, r;
1142
1143 opt.verbose = 0;
1144 opt.base = false;
1145 opt.sendpage = false;
1146 opt.data_test = false;
1147 opt.drop_expected = false;
1148 opt.iov_count = 0;
1149 opt.iov_length = 0;
1150 opt.rate = 0;
1151
1152 r = 1;
1153 for (i = 1; i < 100; i += 33) {
1154 for (l = 1; l < 100; l += 33) {
1155 opt.rate = r;
1156 opt.iov_count = i;
1157 opt.iov_length = l;
1158 err = test_exec(cgrp, &opt);
1159 if (err)
1160 goto out;
1161 }
1162 }
1163 sched_yield();
1164 out:
1165 return err;
1166 }
1167
test_txmsg(int cgrp)1168 static int test_txmsg(int cgrp)
1169 {
1170 int err;
1171
1172 txmsg_pass = txmsg_noisy = txmsg_redir_noisy = txmsg_drop = 0;
1173 txmsg_apply = txmsg_cork = 0;
1174 txmsg_ingress = txmsg_skb = 0;
1175
1176 txmsg_pass = 1;
1177 err = test_loop(cgrp);
1178 txmsg_pass = 0;
1179 if (err)
1180 goto out;
1181
1182 txmsg_redir = 1;
1183 err = test_loop(cgrp);
1184 txmsg_redir = 0;
1185 if (err)
1186 goto out;
1187
1188 txmsg_drop = 1;
1189 err = test_loop(cgrp);
1190 txmsg_drop = 0;
1191 if (err)
1192 goto out;
1193
1194 txmsg_redir = 1;
1195 txmsg_ingress = 1;
1196 err = test_loop(cgrp);
1197 txmsg_redir = 0;
1198 txmsg_ingress = 0;
1199 if (err)
1200 goto out;
1201 out:
1202 txmsg_pass = 0;
1203 txmsg_redir = 0;
1204 txmsg_drop = 0;
1205 return err;
1206 }
1207
test_send(struct sockmap_options * opt,int cgrp)1208 static int test_send(struct sockmap_options *opt, int cgrp)
1209 {
1210 int err;
1211
1212 opt->iov_length = 1;
1213 opt->iov_count = 1;
1214 opt->rate = 1;
1215 err = test_exec(cgrp, opt);
1216 if (err)
1217 goto out;
1218
1219 opt->iov_length = 1;
1220 opt->iov_count = 1024;
1221 opt->rate = 1;
1222 err = test_exec(cgrp, opt);
1223 if (err)
1224 goto out;
1225
1226 opt->iov_length = 1024;
1227 opt->iov_count = 1;
1228 opt->rate = 1;
1229 err = test_exec(cgrp, opt);
1230 if (err)
1231 goto out;
1232
1233 opt->iov_length = 1;
1234 opt->iov_count = 1;
1235 opt->rate = 512;
1236 err = test_exec(cgrp, opt);
1237 if (err)
1238 goto out;
1239
1240 opt->iov_length = 256;
1241 opt->iov_count = 1024;
1242 opt->rate = 2;
1243 err = test_exec(cgrp, opt);
1244 if (err)
1245 goto out;
1246
1247 opt->rate = 100;
1248 opt->iov_count = 1;
1249 opt->iov_length = 5;
1250 err = test_exec(cgrp, opt);
1251 if (err)
1252 goto out;
1253 out:
1254 sched_yield();
1255 return err;
1256 }
1257
test_mixed(int cgrp)1258 static int test_mixed(int cgrp)
1259 {
1260 struct sockmap_options opt = {0};
1261 int err;
1262
1263 txmsg_pass = txmsg_noisy = txmsg_redir_noisy = txmsg_drop = 0;
1264 txmsg_apply = txmsg_cork = 0;
1265 txmsg_start = txmsg_end = 0;
1266 txmsg_start_push = txmsg_end_push = 0;
1267
1268 /* Test small and large iov_count values with pass/redir/apply/cork */
1269 txmsg_pass = 1;
1270 txmsg_redir = 0;
1271 txmsg_apply = 1;
1272 txmsg_cork = 0;
1273 err = test_send(&opt, cgrp);
1274 if (err)
1275 goto out;
1276
1277 txmsg_pass = 1;
1278 txmsg_redir = 0;
1279 txmsg_apply = 0;
1280 txmsg_cork = 1;
1281 err = test_send(&opt, cgrp);
1282 if (err)
1283 goto out;
1284
1285 txmsg_pass = 1;
1286 txmsg_redir = 0;
1287 txmsg_apply = 1;
1288 txmsg_cork = 1;
1289 err = test_send(&opt, cgrp);
1290 if (err)
1291 goto out;
1292
1293 txmsg_pass = 1;
1294 txmsg_redir = 0;
1295 txmsg_apply = 1024;
1296 txmsg_cork = 0;
1297 err = test_send(&opt, cgrp);
1298 if (err)
1299 goto out;
1300
1301 txmsg_pass = 1;
1302 txmsg_redir = 0;
1303 txmsg_apply = 0;
1304 txmsg_cork = 1024;
1305 err = test_send(&opt, cgrp);
1306 if (err)
1307 goto out;
1308
1309 txmsg_pass = 1;
1310 txmsg_redir = 0;
1311 txmsg_apply = 1024;
1312 txmsg_cork = 1024;
1313 err = test_send(&opt, cgrp);
1314 if (err)
1315 goto out;
1316
1317 txmsg_pass = 1;
1318 txmsg_redir = 0;
1319 txmsg_cork = 4096;
1320 txmsg_apply = 4096;
1321 err = test_send(&opt, cgrp);
1322 if (err)
1323 goto out;
1324
1325 txmsg_pass = 0;
1326 txmsg_redir = 1;
1327 txmsg_apply = 1;
1328 txmsg_cork = 0;
1329 err = test_send(&opt, cgrp);
1330 if (err)
1331 goto out;
1332
1333 txmsg_pass = 0;
1334 txmsg_redir = 1;
1335 txmsg_apply = 0;
1336 txmsg_cork = 1;
1337 err = test_send(&opt, cgrp);
1338 if (err)
1339 goto out;
1340
1341 txmsg_pass = 0;
1342 txmsg_redir = 1;
1343 txmsg_apply = 1024;
1344 txmsg_cork = 0;
1345 err = test_send(&opt, cgrp);
1346 if (err)
1347 goto out;
1348
1349 txmsg_pass = 0;
1350 txmsg_redir = 1;
1351 txmsg_apply = 0;
1352 txmsg_cork = 1024;
1353 err = test_send(&opt, cgrp);
1354 if (err)
1355 goto out;
1356
1357 txmsg_pass = 0;
1358 txmsg_redir = 1;
1359 txmsg_apply = 1024;
1360 txmsg_cork = 1024;
1361 err = test_send(&opt, cgrp);
1362 if (err)
1363 goto out;
1364
1365 txmsg_pass = 0;
1366 txmsg_redir = 1;
1367 txmsg_cork = 4096;
1368 txmsg_apply = 4096;
1369 err = test_send(&opt, cgrp);
1370 if (err)
1371 goto out;
1372 out:
1373 return err;
1374 }
1375
test_start_end(int cgrp)1376 static int test_start_end(int cgrp)
1377 {
1378 struct sockmap_options opt = {0};
1379 int err, i;
1380
1381 /* Test basic start/end with lots of iov_count and iov_lengths */
1382 txmsg_start = 1;
1383 txmsg_end = 2;
1384 txmsg_start_push = 1;
1385 txmsg_end_push = 2;
1386 err = test_txmsg(cgrp);
1387 if (err)
1388 goto out;
1389
1390 /* Test start/end with cork */
1391 opt.rate = 16;
1392 opt.iov_count = 1;
1393 opt.iov_length = 100;
1394 txmsg_cork = 1600;
1395
1396 for (i = 99; i <= 1600; i += 500) {
1397 txmsg_start = 0;
1398 txmsg_end = i;
1399 txmsg_start_push = 0;
1400 txmsg_end_push = i;
1401 err = test_exec(cgrp, &opt);
1402 if (err)
1403 goto out;
1404 }
1405
1406 /* Test start/end with cork but pull data in middle */
1407 for (i = 199; i <= 1600; i += 500) {
1408 txmsg_start = 100;
1409 txmsg_end = i;
1410 txmsg_start_push = 100;
1411 txmsg_end_push = i;
1412 err = test_exec(cgrp, &opt);
1413 if (err)
1414 goto out;
1415 }
1416
1417 /* Test start/end with cork pulling last sg entry */
1418 txmsg_start = 1500;
1419 txmsg_end = 1600;
1420 txmsg_start_push = 1500;
1421 txmsg_end_push = 1600;
1422 err = test_exec(cgrp, &opt);
1423 if (err)
1424 goto out;
1425
1426 /* Test start/end pull of single byte in last page */
1427 txmsg_start = 1111;
1428 txmsg_end = 1112;
1429 txmsg_start_push = 1111;
1430 txmsg_end_push = 1112;
1431 err = test_exec(cgrp, &opt);
1432 if (err)
1433 goto out;
1434
1435 /* Test start/end with end < start */
1436 txmsg_start = 1111;
1437 txmsg_end = 0;
1438 txmsg_start_push = 1111;
1439 txmsg_end_push = 0;
1440 err = test_exec(cgrp, &opt);
1441 if (err)
1442 goto out;
1443
1444 /* Test start/end with end > data */
1445 txmsg_start = 0;
1446 txmsg_end = 1601;
1447 txmsg_start_push = 0;
1448 txmsg_end_push = 1601;
1449 err = test_exec(cgrp, &opt);
1450 if (err)
1451 goto out;
1452
1453 /* Test start/end with start > data */
1454 txmsg_start = 1601;
1455 txmsg_end = 1600;
1456 txmsg_start_push = 1601;
1457 txmsg_end_push = 1600;
1458 err = test_exec(cgrp, &opt);
1459
1460 out:
1461 txmsg_start = 0;
1462 txmsg_end = 0;
1463 sched_yield();
1464 return err;
1465 }
1466
1467 char *map_names[] = {
1468 "sock_map",
1469 "sock_map_txmsg",
1470 "sock_map_redir",
1471 "sock_apply_bytes",
1472 "sock_cork_bytes",
1473 "sock_bytes",
1474 "sock_redir_flags",
1475 "sock_skb_opts",
1476 };
1477
1478 int prog_attach_type[] = {
1479 BPF_SK_SKB_STREAM_PARSER,
1480 BPF_SK_SKB_STREAM_VERDICT,
1481 BPF_CGROUP_SOCK_OPS,
1482 BPF_SK_MSG_VERDICT,
1483 BPF_SK_MSG_VERDICT,
1484 BPF_SK_MSG_VERDICT,
1485 BPF_SK_MSG_VERDICT,
1486 BPF_SK_MSG_VERDICT,
1487 BPF_SK_MSG_VERDICT,
1488 BPF_SK_MSG_VERDICT,
1489 };
1490
1491 int prog_type[] = {
1492 BPF_PROG_TYPE_SK_SKB,
1493 BPF_PROG_TYPE_SK_SKB,
1494 BPF_PROG_TYPE_SOCK_OPS,
1495 BPF_PROG_TYPE_SK_MSG,
1496 BPF_PROG_TYPE_SK_MSG,
1497 BPF_PROG_TYPE_SK_MSG,
1498 BPF_PROG_TYPE_SK_MSG,
1499 BPF_PROG_TYPE_SK_MSG,
1500 BPF_PROG_TYPE_SK_MSG,
1501 BPF_PROG_TYPE_SK_MSG,
1502 };
1503
populate_progs(char * bpf_file)1504 static int populate_progs(char *bpf_file)
1505 {
1506 struct bpf_program *prog;
1507 struct bpf_object *obj;
1508 int i = 0;
1509 long err;
1510
1511 obj = bpf_object__open(bpf_file);
1512 err = libbpf_get_error(obj);
1513 if (err) {
1514 char err_buf[256];
1515
1516 libbpf_strerror(err, err_buf, sizeof(err_buf));
1517 printf("Unable to load eBPF objects in file '%s' : %s\n",
1518 bpf_file, err_buf);
1519 return -1;
1520 }
1521
1522 bpf_object__for_each_program(prog, obj) {
1523 bpf_program__set_type(prog, prog_type[i]);
1524 bpf_program__set_expected_attach_type(prog,
1525 prog_attach_type[i]);
1526 i++;
1527 }
1528
1529 i = bpf_object__load(obj);
1530 i = 0;
1531 bpf_object__for_each_program(prog, obj) {
1532 prog_fd[i] = bpf_program__fd(prog);
1533 i++;
1534 }
1535
1536 for (i = 0; i < sizeof(map_fd)/sizeof(int); i++) {
1537 maps[i] = bpf_object__find_map_by_name(obj, map_names[i]);
1538 map_fd[i] = bpf_map__fd(maps[i]);
1539 if (map_fd[i] < 0) {
1540 fprintf(stderr, "load_bpf_file: (%i) %s\n",
1541 map_fd[i], strerror(errno));
1542 return -1;
1543 }
1544 }
1545
1546 return 0;
1547 }
1548
__test_suite(int cg_fd,char * bpf_file)1549 static int __test_suite(int cg_fd, char *bpf_file)
1550 {
1551 int err, cleanup = cg_fd;
1552
1553 err = populate_progs(bpf_file);
1554 if (err < 0) {
1555 fprintf(stderr, "ERROR: (%i) load bpf failed\n", err);
1556 return err;
1557 }
1558
1559 if (cg_fd < 0) {
1560 if (setup_cgroup_environment()) {
1561 fprintf(stderr, "ERROR: cgroup env failed\n");
1562 return -EINVAL;
1563 }
1564
1565 cg_fd = create_and_get_cgroup(CG_PATH);
1566 if (cg_fd < 0) {
1567 fprintf(stderr,
1568 "ERROR: (%i) open cg path failed: %s\n",
1569 cg_fd, optarg);
1570 return cg_fd;
1571 }
1572
1573 if (join_cgroup(CG_PATH)) {
1574 fprintf(stderr, "ERROR: failed to join cgroup\n");
1575 return -EINVAL;
1576 }
1577 }
1578
1579 /* Tests basic commands and APIs with range of iov values */
1580 txmsg_start = txmsg_end = txmsg_start_push = txmsg_end_push = 0;
1581 err = test_txmsg(cg_fd);
1582 if (err)
1583 goto out;
1584
1585 /* Tests interesting combinations of APIs used together */
1586 err = test_mixed(cg_fd);
1587 if (err)
1588 goto out;
1589
1590 /* Tests pull_data API using start/end API */
1591 err = test_start_end(cg_fd);
1592 if (err)
1593 goto out;
1594
1595 out:
1596 printf("Summary: %i PASSED %i FAILED\n", passed, failed);
1597 if (cleanup < 0) {
1598 cleanup_cgroup_environment();
1599 close(cg_fd);
1600 }
1601 return err;
1602 }
1603
test_suite(int cg_fd)1604 static int test_suite(int cg_fd)
1605 {
1606 int err;
1607
1608 err = __test_suite(cg_fd, BPF_SOCKMAP_FILENAME);
1609 if (err)
1610 goto out;
1611 err = __test_suite(cg_fd, BPF_SOCKHASH_FILENAME);
1612 out:
1613 if (cg_fd > -1)
1614 close(cg_fd);
1615 return err;
1616 }
1617
main(int argc,char ** argv)1618 int main(int argc, char **argv)
1619 {
1620 int iov_count = 1, length = 1024, rate = 1;
1621 struct sockmap_options options = {0};
1622 int opt, longindex, err, cg_fd = 0;
1623 char *bpf_file = BPF_SOCKMAP_FILENAME;
1624 int test = PING_PONG;
1625
1626 if (argc < 2)
1627 return test_suite(-1);
1628
1629 while ((opt = getopt_long(argc, argv, ":dhvc:r:i:l:t:p:q:",
1630 long_options, &longindex)) != -1) {
1631 switch (opt) {
1632 case 's':
1633 txmsg_start = atoi(optarg);
1634 break;
1635 case 'e':
1636 txmsg_end = atoi(optarg);
1637 break;
1638 case 'p':
1639 txmsg_start_push = atoi(optarg);
1640 break;
1641 case 'q':
1642 txmsg_end_push = atoi(optarg);
1643 break;
1644 case 'a':
1645 txmsg_apply = atoi(optarg);
1646 break;
1647 case 'k':
1648 txmsg_cork = atoi(optarg);
1649 break;
1650 case 'c':
1651 cg_fd = open(optarg, O_DIRECTORY, O_RDONLY);
1652 if (cg_fd < 0) {
1653 fprintf(stderr,
1654 "ERROR: (%i) open cg path failed: %s\n",
1655 cg_fd, optarg);
1656 return cg_fd;
1657 }
1658 break;
1659 case 'r':
1660 rate = atoi(optarg);
1661 break;
1662 case 'v':
1663 options.verbose = 1;
1664 break;
1665 case 'i':
1666 iov_count = atoi(optarg);
1667 break;
1668 case 'l':
1669 length = atoi(optarg);
1670 break;
1671 case 'd':
1672 options.data_test = true;
1673 break;
1674 case 't':
1675 if (strcmp(optarg, "ping") == 0) {
1676 test = PING_PONG;
1677 } else if (strcmp(optarg, "sendmsg") == 0) {
1678 test = SENDMSG;
1679 } else if (strcmp(optarg, "base") == 0) {
1680 test = BASE;
1681 } else if (strcmp(optarg, "base_sendpage") == 0) {
1682 test = BASE_SENDPAGE;
1683 } else if (strcmp(optarg, "sendpage") == 0) {
1684 test = SENDPAGE;
1685 } else {
1686 usage(argv);
1687 return -1;
1688 }
1689 break;
1690 case 0:
1691 break;
1692 case 'h':
1693 default:
1694 usage(argv);
1695 return -1;
1696 }
1697 }
1698
1699 if (argc <= 3 && cg_fd)
1700 return test_suite(cg_fd);
1701
1702 if (!cg_fd) {
1703 fprintf(stderr, "%s requires cgroup option: --cgroup <path>\n",
1704 argv[0]);
1705 return -1;
1706 }
1707
1708 err = populate_progs(bpf_file);
1709 if (err) {
1710 fprintf(stderr, "populate program: (%s) %s\n",
1711 bpf_file, strerror(errno));
1712 return 1;
1713 }
1714 running = 1;
1715
1716 /* catch SIGINT */
1717 signal(SIGINT, running_handler);
1718
1719 options.iov_count = iov_count;
1720 options.iov_length = length;
1721 options.rate = rate;
1722
1723 err = run_options(&options, cg_fd, test);
1724 close(cg_fd);
1725 return err;
1726 }
1727
running_handler(int a)1728 void running_handler(int a)
1729 {
1730 running = 0;
1731 }
1732