1#!/usr/bin/env python
2# Copyright (c) PLUMgrid, Inc.
3# Licensed under the Apache License, Version 2.0 (the "License")
4
5# This program implements a topology likes below:
6#   pem: physical endpoint manager, implemented as a bpf program
7#
8#     vm1 <--------+  +----> bridge1 <----+
9#                  V  V                   V
10#                  pem                  router
11#                  ^  ^                   ^
12#     vm2 <--------+  +----> bridge2 <----+
13#
14# The vm1, vm2 and router are implemented as namespaces.
15# The linux bridge device is used to provice bridge functionality.
16# pem bpf will be attached to related network devices for vm1, vm1, bridge1 and bridge2.
17#
18# vm1 and vm2 are in different subnet. For vm1 to communicate to vm2,
19# the packet will have to travel from vm1 to pem, bridge1, router, bridge2, pem, and
20# then come to vm2.
21#
22# When this test is run with verbose mode (ctest -R <test_name> -V),
23# the following printout is observed on my local box:
24#
25# ......
26# 9: PING 200.1.1.1 (200.1.1.1) 56(84) bytes of data.
27# 9: 64 bytes from 200.1.1.1: icmp_req=1 ttl=63 time=0.090 ms
28# 9: 64 bytes from 200.1.1.1: icmp_req=2 ttl=63 time=0.032 ms
29# 9:
30# 9: --- 200.1.1.1 ping statistics ---
31# 9: 2 packets transmitted, 2 received, 0% packet loss, time 999ms
32# 9: rtt min/avg/max/mdev = 0.032/0.061/0.090/0.029 ms
33# 9: [ ID] Interval       Transfer     Bandwidth
34# 9: [  5]  0.0- 1.0 sec  3.80 GBytes  32.6 Gbits/sec
35# 9: Starting netserver with host 'IN(6)ADDR_ANY' port '12865' and family AF_UNSPEC
36# 9: MIGRATED TCP STREAM TEST from 0.0.0.0 (0.0.0.0) port 0 AF_INET to 200.1.1.1 (200.1.1.1) port 0 AF_INET : demo
37# 9: Recv   Send    Send
38# 9: Socket Socket  Message  Elapsed
39# 9: Size   Size    Size     Time     Throughput
40# 9: bytes  bytes   bytes    secs.    10^6bits/sec
41# 9:
42# 9:  87380  16384  65160    1.00     39940.46
43# 9: MIGRATED TCP REQUEST/RESPONSE TEST from 0.0.0.0 (0.0.0.0) port 0 AF_INET to 200.1.1.1 (200.1.1.1) port 0 AF_INET : demo : first burst 0
44# 9: Local /Remote
45# 9: Socket Size   Request  Resp.   Elapsed  Trans.
46# 9: Send   Recv   Size     Size    Time     Rate
47# 9: bytes  Bytes  bytes    bytes   secs.    per sec
48# 9:
49# 9: 16384  87380  1        1       1.00     46387.80
50# 9: 16384  87380
51# 9: .
52# 9: ----------------------------------------------------------------------
53# 9: Ran 1 test in 7.495s
54# 9:
55# 9: OK
56
57from ctypes import c_uint
58from bcc import BPF
59from pyroute2 import IPRoute, NetNS, IPDB, NSPopen
60from utils import NSPopenWithCheck
61import sys
62from time import sleep
63from unittest import main, TestCase
64import subprocess
65from simulation import Simulation
66
67arg1 = sys.argv.pop(1)
68ipr = IPRoute()
69ipdb = IPDB(nl=ipr)
70sim = Simulation(ipdb)
71
72allocated_interfaces = set(ipdb.interfaces.keys())
73
74def get_next_iface(prefix):
75    i = 0
76    while True:
77        iface = "{0}{1}".format(prefix, i)
78        if iface not in allocated_interfaces:
79            allocated_interfaces.add(iface)
80            return iface
81        i += 1
82
83class TestBPFSocket(TestCase):
84    def setup_br(self, br, veth_rt_2_br, veth_pem_2_br, veth_br_2_pem):
85        # create veth which connecting pem and br
86        with ipdb.create(ifname=veth_pem_2_br, kind="veth", peer=veth_br_2_pem) as v:
87            v.up()
88        ipdb.interfaces[veth_br_2_pem].up().commit()
89        subprocess.call(["sysctl", "-q", "-w", "net.ipv6.conf." + veth_pem_2_br + ".disable_ipv6=1"])
90        subprocess.call(["sysctl", "-q", "-w", "net.ipv6.conf." + veth_br_2_pem + ".disable_ipv6=1"])
91
92        # set up the bridge and add router interface as one of its slaves
93        with ipdb.create(ifname=br, kind="bridge") as br1:
94            br1.add_port(ipdb.interfaces[veth_pem_2_br])
95            br1.add_port(ipdb.interfaces[veth_rt_2_br])
96            br1.up()
97        subprocess.call(["sysctl", "-q", "-w", "net.ipv6.conf." + br + ".disable_ipv6=1"])
98
99    def set_default_const(self):
100        self.ns1            = "ns1"
101        self.ns2            = "ns2"
102        self.ns_router      = "ns_router"
103        self.br1            = get_next_iface("br")
104        self.veth_pem_2_br1 = "v20"
105        self.veth_br1_2_pem = "v21"
106        self.br2            = get_next_iface("br")
107        self.veth_pem_2_br2 = "v22"
108        self.veth_br2_2_pem = "v23"
109
110        self.vm1_ip         = "100.1.1.1"
111        self.vm2_ip         = "200.1.1.1"
112        self.vm1_rtr_ip     = "100.1.1.254"
113        self.vm2_rtr_ip     = "200.1.1.254"
114        self.vm1_rtr_mask   = "100.1.1.0/24"
115        self.vm2_rtr_mask   = "200.1.1.0/24"
116
117    def attach_filter(self, ifname, fd, name):
118        ifindex = ipdb.interfaces[ifname].index
119        ipr.tc("add", "ingress", ifindex, "ffff:")
120        ipr.tc("add-filter", "bpf", ifindex, ":1", fd=fd, name=name,
121              parent="ffff:", action="drop", classid=1)
122
123    def config_maps(self):
124        # pem just relays packets between VM and its corresponding
125        # slave link in the bridge interface
126        ns1_ifindex = self.ns1_eth_out.index
127        ns2_ifindex = self.ns2_eth_out.index
128        br1_ifindex = ipdb.interfaces[self.veth_br1_2_pem].index
129        br2_ifindex = ipdb.interfaces[self.veth_br2_2_pem].index
130        self.pem_dest[c_uint(ns1_ifindex)] = c_uint(br1_ifindex)
131        self.pem_dest[c_uint(br1_ifindex)] = c_uint(ns1_ifindex)
132        self.pem_dest[c_uint(ns2_ifindex)] = c_uint(br2_ifindex)
133        self.pem_dest[c_uint(br2_ifindex)] = c_uint(ns2_ifindex)
134
135        # tc filter setup with bpf programs attached
136        self.attach_filter(self.veth_br1_2_pem, self.pem_fn.fd, self.pem_fn.name)
137        self.attach_filter(self.veth_br2_2_pem, self.pem_fn.fd, self.pem_fn.name)
138
139    def test_brb2(self):
140        try:
141            b = BPF(src_file=arg1, debug=0)
142            self.pem_fn = b.load_func("pem", BPF.SCHED_CLS)
143            self.pem_dest= b.get_table("pem_dest")
144            self.pem_stats = b.get_table("pem_stats")
145
146            # set up the topology
147            self.set_default_const()
148            (ns1_ipdb, self.ns1_eth_out, _) = sim._create_ns(self.ns1, ipaddr=self.vm1_ip+'/24',
149                                                             fn=self.pem_fn, action='drop',
150                                                             disable_ipv6=True)
151            (ns2_ipdb, self.ns2_eth_out, _) = sim._create_ns(self.ns2, ipaddr=self.vm2_ip+'/24',
152                                                             fn=self.pem_fn, action='drop',
153                                                             disable_ipv6=True)
154            ns1_ipdb.routes.add({'dst': self.vm2_rtr_mask, 'gateway': self.vm1_rtr_ip}).commit()
155            ns2_ipdb.routes.add({'dst': self.vm1_rtr_mask, 'gateway': self.vm2_rtr_ip}).commit()
156
157            (_, self.nsrtr_eth0_out, _) = sim._create_ns(self.ns_router, ipaddr=self.vm1_rtr_ip+'/24',
158                                                         disable_ipv6=True)
159            (rt_ipdb, self.nsrtr_eth1_out, _) = sim._ns_add_ifc(self.ns_router, "eth1", "ns_router2",
160                                                                ipaddr=self.vm2_rtr_ip+'/24',
161                                                                disable_ipv6=True)
162            # enable ip forwarding in router ns
163            nsp = NSPopen(rt_ipdb.nl.netns, ["sysctl", "-w", "net.ipv4.ip_forward=1"])
164            nsp.wait(); nsp.release()
165
166            # for each VM connecting to pem, there will be a corresponding veth connecting to the bridge
167            self.setup_br(self.br1, self.nsrtr_eth0_out.ifname, self.veth_pem_2_br1, self.veth_br1_2_pem)
168            self.setup_br(self.br2, self.nsrtr_eth1_out.ifname, self.veth_pem_2_br2, self.veth_br2_2_pem)
169
170            # load the program and configure maps
171            self.config_maps()
172
173            # ping
174            nsp = NSPopen(ns1_ipdb.nl.netns, ["ping", self.vm2_ip, "-c", "2"]); nsp.wait(); nsp.release()
175            # one arp request/reply, 2 icmp request/reply per VM, total 6 packets per VM, 12 packets total
176            self.assertEqual(self.pem_stats[c_uint(0)].value, 12)
177
178            nsp_server = NSPopenWithCheck(ns2_ipdb.nl.netns, ["iperf", "-s", "-xSC"])
179            sleep(1)
180            nsp = NSPopen(ns1_ipdb.nl.netns, ["iperf", "-c", self.vm2_ip, "-t", "1", "-xSC"])
181            nsp.wait(); nsp.release()
182            nsp_server.kill(); nsp_server.wait(); nsp_server.release()
183
184            nsp_server = NSPopenWithCheck(ns2_ipdb.nl.netns, ["netserver", "-D"])
185            sleep(1)
186            nsp = NSPopenWithCheck(ns1_ipdb.nl.netns, ["netperf", "-l", "1", "-H", self.vm2_ip, "--", "-m", "65160"])
187            nsp.wait(); nsp.release()
188            nsp = NSPopen(ns1_ipdb.nl.netns, ["netperf", "-l", "1", "-H", self.vm2_ip, "-t", "TCP_RR"])
189            nsp.wait(); nsp.release()
190            nsp_server.kill(); nsp_server.wait(); nsp_server.release()
191
192        finally:
193            if self.br1 in ipdb.interfaces: ipdb.interfaces[self.br1].remove().commit()
194            if self.br2 in ipdb.interfaces: ipdb.interfaces[self.br2].remove().commit()
195            if self.veth_pem_2_br1 in ipdb.interfaces: ipdb.interfaces[self.veth_pem_2_br1].remove().commit()
196            if self.veth_pem_2_br2 in ipdb.interfaces: ipdb.interfaces[self.veth_pem_2_br2].remove().commit()
197            sim.release()
198            ipdb.release()
199
200
201if __name__ == "__main__":
202    main()
203