1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef __BPF_HELPERS_H 3 #define __BPF_HELPERS_H 4 5 /* helper macro to place programs, maps, license in 6 * different sections in elf_bpf file. Section names 7 * are interpreted by elf_bpf loader 8 */ 9 #define SEC(NAME) __attribute__((section(NAME), used)) 10 11 /* helper functions called from eBPF programs written in C */ 12 static void *(*bpf_map_lookup_elem)(void *map, void *key) = 13 (void *) BPF_FUNC_map_lookup_elem; 14 static int (*bpf_map_update_elem)(void *map, void *key, void *value, 15 unsigned long long flags) = 16 (void *) BPF_FUNC_map_update_elem; 17 static int (*bpf_map_delete_elem)(void *map, void *key) = 18 (void *) BPF_FUNC_map_delete_elem; 19 static int (*bpf_map_push_elem)(void *map, void *value, 20 unsigned long long flags) = 21 (void *) BPF_FUNC_map_push_elem; 22 static int (*bpf_map_pop_elem)(void *map, void *value) = 23 (void *) BPF_FUNC_map_pop_elem; 24 static int (*bpf_map_peek_elem)(void *map, void *value) = 25 (void *) BPF_FUNC_map_peek_elem; 26 static int (*bpf_probe_read)(void *dst, int size, void *unsafe_ptr) = 27 (void *) BPF_FUNC_probe_read; 28 static unsigned long long (*bpf_ktime_get_ns)(void) = 29 (void *) BPF_FUNC_ktime_get_ns; 30 static int (*bpf_trace_printk)(const char *fmt, int fmt_size, ...) = 31 (void *) BPF_FUNC_trace_printk; 32 static void (*bpf_tail_call)(void *ctx, void *map, int index) = 33 (void *) BPF_FUNC_tail_call; 34 static unsigned long long (*bpf_get_smp_processor_id)(void) = 35 (void *) BPF_FUNC_get_smp_processor_id; 36 static unsigned long long (*bpf_get_current_pid_tgid)(void) = 37 (void *) BPF_FUNC_get_current_pid_tgid; 38 static unsigned long long (*bpf_get_current_uid_gid)(void) = 39 (void *) BPF_FUNC_get_current_uid_gid; 40 static int (*bpf_get_current_comm)(void *buf, int buf_size) = 41 (void *) BPF_FUNC_get_current_comm; 42 static unsigned long long (*bpf_perf_event_read)(void *map, 43 unsigned long long flags) = 44 (void *) BPF_FUNC_perf_event_read; 45 static int (*bpf_clone_redirect)(void *ctx, int ifindex, int flags) = 46 (void *) BPF_FUNC_clone_redirect; 47 static int (*bpf_redirect)(int ifindex, int flags) = 48 (void *) BPF_FUNC_redirect; 49 static int (*bpf_redirect_map)(void *map, int key, int flags) = 50 (void *) BPF_FUNC_redirect_map; 51 static int (*bpf_perf_event_output)(void *ctx, void *map, 52 unsigned long long flags, void *data, 53 int size) = 54 (void *) BPF_FUNC_perf_event_output; 55 static int (*bpf_get_stackid)(void *ctx, void *map, int flags) = 56 (void *) BPF_FUNC_get_stackid; 57 static int (*bpf_probe_write_user)(void *dst, void *src, int size) = 58 (void *) BPF_FUNC_probe_write_user; 59 static int (*bpf_current_task_under_cgroup)(void *map, int index) = 60 (void *) BPF_FUNC_current_task_under_cgroup; 61 static int (*bpf_skb_get_tunnel_key)(void *ctx, void *key, int size, int flags) = 62 (void *) BPF_FUNC_skb_get_tunnel_key; 63 static int (*bpf_skb_set_tunnel_key)(void *ctx, void *key, int size, int flags) = 64 (void *) BPF_FUNC_skb_set_tunnel_key; 65 static int (*bpf_skb_get_tunnel_opt)(void *ctx, void *md, int size) = 66 (void *) BPF_FUNC_skb_get_tunnel_opt; 67 static int (*bpf_skb_set_tunnel_opt)(void *ctx, void *md, int size) = 68 (void *) BPF_FUNC_skb_set_tunnel_opt; 69 static unsigned long long (*bpf_get_prandom_u32)(void) = 70 (void *) BPF_FUNC_get_prandom_u32; 71 static int (*bpf_xdp_adjust_head)(void *ctx, int offset) = 72 (void *) BPF_FUNC_xdp_adjust_head; 73 static int (*bpf_xdp_adjust_meta)(void *ctx, int offset) = 74 (void *) BPF_FUNC_xdp_adjust_meta; 75 static int (*bpf_get_socket_cookie)(void *ctx) = 76 (void *) BPF_FUNC_get_socket_cookie; 77 static int (*bpf_setsockopt)(void *ctx, int level, int optname, void *optval, 78 int optlen) = 79 (void *) BPF_FUNC_setsockopt; 80 static int (*bpf_getsockopt)(void *ctx, int level, int optname, void *optval, 81 int optlen) = 82 (void *) BPF_FUNC_getsockopt; 83 static int (*bpf_sock_ops_cb_flags_set)(void *ctx, int flags) = 84 (void *) BPF_FUNC_sock_ops_cb_flags_set; 85 static int (*bpf_sk_redirect_map)(void *ctx, void *map, int key, int flags) = 86 (void *) BPF_FUNC_sk_redirect_map; 87 static int (*bpf_sk_redirect_hash)(void *ctx, void *map, void *key, int flags) = 88 (void *) BPF_FUNC_sk_redirect_hash; 89 static int (*bpf_sock_map_update)(void *map, void *key, void *value, 90 unsigned long long flags) = 91 (void *) BPF_FUNC_sock_map_update; 92 static int (*bpf_sock_hash_update)(void *map, void *key, void *value, 93 unsigned long long flags) = 94 (void *) BPF_FUNC_sock_hash_update; 95 static int (*bpf_perf_event_read_value)(void *map, unsigned long long flags, 96 void *buf, unsigned int buf_size) = 97 (void *) BPF_FUNC_perf_event_read_value; 98 static int (*bpf_perf_prog_read_value)(void *ctx, void *buf, 99 unsigned int buf_size) = 100 (void *) BPF_FUNC_perf_prog_read_value; 101 static int (*bpf_override_return)(void *ctx, unsigned long rc) = 102 (void *) BPF_FUNC_override_return; 103 static int (*bpf_msg_redirect_map)(void *ctx, void *map, int key, int flags) = 104 (void *) BPF_FUNC_msg_redirect_map; 105 static int (*bpf_msg_redirect_hash)(void *ctx, 106 void *map, void *key, int flags) = 107 (void *) BPF_FUNC_msg_redirect_hash; 108 static int (*bpf_msg_apply_bytes)(void *ctx, int len) = 109 (void *) BPF_FUNC_msg_apply_bytes; 110 static int (*bpf_msg_cork_bytes)(void *ctx, int len) = 111 (void *) BPF_FUNC_msg_cork_bytes; 112 static int (*bpf_msg_pull_data)(void *ctx, int start, int end, int flags) = 113 (void *) BPF_FUNC_msg_pull_data; 114 static int (*bpf_msg_push_data)(void *ctx, int start, int end, int flags) = 115 (void *) BPF_FUNC_msg_push_data; 116 static int (*bpf_bind)(void *ctx, void *addr, int addr_len) = 117 (void *) BPF_FUNC_bind; 118 static int (*bpf_xdp_adjust_tail)(void *ctx, int offset) = 119 (void *) BPF_FUNC_xdp_adjust_tail; 120 static int (*bpf_skb_get_xfrm_state)(void *ctx, int index, void *state, 121 int size, int flags) = 122 (void *) BPF_FUNC_skb_get_xfrm_state; 123 static int (*bpf_sk_select_reuseport)(void *ctx, void *map, void *key, __u32 flags) = 124 (void *) BPF_FUNC_sk_select_reuseport; 125 static int (*bpf_get_stack)(void *ctx, void *buf, int size, int flags) = 126 (void *) BPF_FUNC_get_stack; 127 static int (*bpf_fib_lookup)(void *ctx, struct bpf_fib_lookup *params, 128 int plen, __u32 flags) = 129 (void *) BPF_FUNC_fib_lookup; 130 static int (*bpf_lwt_push_encap)(void *ctx, unsigned int type, void *hdr, 131 unsigned int len) = 132 (void *) BPF_FUNC_lwt_push_encap; 133 static int (*bpf_lwt_seg6_store_bytes)(void *ctx, unsigned int offset, 134 void *from, unsigned int len) = 135 (void *) BPF_FUNC_lwt_seg6_store_bytes; 136 static int (*bpf_lwt_seg6_action)(void *ctx, unsigned int action, void *param, 137 unsigned int param_len) = 138 (void *) BPF_FUNC_lwt_seg6_action; 139 static int (*bpf_lwt_seg6_adjust_srh)(void *ctx, unsigned int offset, 140 unsigned int len) = 141 (void *) BPF_FUNC_lwt_seg6_adjust_srh; 142 static int (*bpf_rc_repeat)(void *ctx) = 143 (void *) BPF_FUNC_rc_repeat; 144 static int (*bpf_rc_keydown)(void *ctx, unsigned int protocol, 145 unsigned long long scancode, unsigned int toggle) = 146 (void *) BPF_FUNC_rc_keydown; 147 static unsigned long long (*bpf_get_current_cgroup_id)(void) = 148 (void *) BPF_FUNC_get_current_cgroup_id; 149 static void *(*bpf_get_local_storage)(void *map, unsigned long long flags) = 150 (void *) BPF_FUNC_get_local_storage; 151 static unsigned long long (*bpf_skb_cgroup_id)(void *ctx) = 152 (void *) BPF_FUNC_skb_cgroup_id; 153 static unsigned long long (*bpf_skb_ancestor_cgroup_id)(void *ctx, int level) = 154 (void *) BPF_FUNC_skb_ancestor_cgroup_id; 155 static struct bpf_sock *(*bpf_sk_lookup_tcp)(void *ctx, 156 struct bpf_sock_tuple *tuple, 157 int size, unsigned long long netns_id, 158 unsigned long long flags) = 159 (void *) BPF_FUNC_sk_lookup_tcp; 160 static struct bpf_sock *(*bpf_sk_lookup_udp)(void *ctx, 161 struct bpf_sock_tuple *tuple, 162 int size, unsigned long long netns_id, 163 unsigned long long flags) = 164 (void *) BPF_FUNC_sk_lookup_udp; 165 static int (*bpf_sk_release)(struct bpf_sock *sk) = 166 (void *) BPF_FUNC_sk_release; 167 static int (*bpf_skb_vlan_push)(void *ctx, __be16 vlan_proto, __u16 vlan_tci) = 168 (void *) BPF_FUNC_skb_vlan_push; 169 static int (*bpf_skb_vlan_pop)(void *ctx) = 170 (void *) BPF_FUNC_skb_vlan_pop; 171 172 /* llvm builtin functions that eBPF C program may use to 173 * emit BPF_LD_ABS and BPF_LD_IND instructions 174 */ 175 struct sk_buff; 176 unsigned long long load_byte(void *skb, 177 unsigned long long off) asm("llvm.bpf.load.byte"); 178 unsigned long long load_half(void *skb, 179 unsigned long long off) asm("llvm.bpf.load.half"); 180 unsigned long long load_word(void *skb, 181 unsigned long long off) asm("llvm.bpf.load.word"); 182 183 /* a helper structure used by eBPF C program 184 * to describe map attributes to elf_bpf loader 185 */ 186 struct bpf_map_def { 187 unsigned int type; 188 unsigned int key_size; 189 unsigned int value_size; 190 unsigned int max_entries; 191 unsigned int map_flags; 192 unsigned int inner_map_idx; 193 unsigned int numa_node; 194 }; 195 196 #define BPF_ANNOTATE_KV_PAIR(name, type_key, type_val) \ 197 struct ____btf_map_##name { \ 198 type_key key; \ 199 type_val value; \ 200 }; \ 201 struct ____btf_map_##name \ 202 __attribute__ ((section(".maps." #name), used)) \ 203 ____btf_map_##name = { } 204 205 static int (*bpf_skb_load_bytes)(void *ctx, int off, void *to, int len) = 206 (void *) BPF_FUNC_skb_load_bytes; 207 static int (*bpf_skb_load_bytes_relative)(void *ctx, int off, void *to, int len, __u32 start_header) = 208 (void *) BPF_FUNC_skb_load_bytes_relative; 209 static int (*bpf_skb_store_bytes)(void *ctx, int off, void *from, int len, int flags) = 210 (void *) BPF_FUNC_skb_store_bytes; 211 static int (*bpf_l3_csum_replace)(void *ctx, int off, int from, int to, int flags) = 212 (void *) BPF_FUNC_l3_csum_replace; 213 static int (*bpf_l4_csum_replace)(void *ctx, int off, int from, int to, int flags) = 214 (void *) BPF_FUNC_l4_csum_replace; 215 static int (*bpf_csum_diff)(void *from, int from_size, void *to, int to_size, int seed) = 216 (void *) BPF_FUNC_csum_diff; 217 static int (*bpf_skb_under_cgroup)(void *ctx, void *map, int index) = 218 (void *) BPF_FUNC_skb_under_cgroup; 219 static int (*bpf_skb_change_head)(void *, int len, int flags) = 220 (void *) BPF_FUNC_skb_change_head; 221 static int (*bpf_skb_pull_data)(void *, int len) = 222 (void *) BPF_FUNC_skb_pull_data; 223 224 /* Scan the ARCH passed in from ARCH env variable (see Makefile) */ 225 #if defined(__TARGET_ARCH_x86) 226 #define bpf_target_x86 227 #define bpf_target_defined 228 #elif defined(__TARGET_ARCH_s930x) 229 #define bpf_target_s930x 230 #define bpf_target_defined 231 #elif defined(__TARGET_ARCH_arm64) 232 #define bpf_target_arm64 233 #define bpf_target_defined 234 #elif defined(__TARGET_ARCH_mips) 235 #define bpf_target_mips 236 #define bpf_target_defined 237 #elif defined(__TARGET_ARCH_powerpc) 238 #define bpf_target_powerpc 239 #define bpf_target_defined 240 #elif defined(__TARGET_ARCH_sparc) 241 #define bpf_target_sparc 242 #define bpf_target_defined 243 #else 244 #undef bpf_target_defined 245 #endif 246 247 /* Fall back to what the compiler says */ 248 #ifndef bpf_target_defined 249 #if defined(__x86_64__) 250 #define bpf_target_x86 251 #elif defined(__s390x__) 252 #define bpf_target_s930x 253 #elif defined(__aarch64__) 254 #define bpf_target_arm64 255 #elif defined(__mips__) 256 #define bpf_target_mips 257 #elif defined(__powerpc__) 258 #define bpf_target_powerpc 259 #elif defined(__sparc__) 260 #define bpf_target_sparc 261 #endif 262 #endif 263 264 #if defined(bpf_target_x86) 265 266 #define PT_REGS_PARM1(x) ((x)->di) 267 #define PT_REGS_PARM2(x) ((x)->si) 268 #define PT_REGS_PARM3(x) ((x)->dx) 269 #define PT_REGS_PARM4(x) ((x)->cx) 270 #define PT_REGS_PARM5(x) ((x)->r8) 271 #define PT_REGS_RET(x) ((x)->sp) 272 #define PT_REGS_FP(x) ((x)->bp) 273 #define PT_REGS_RC(x) ((x)->ax) 274 #define PT_REGS_SP(x) ((x)->sp) 275 #define PT_REGS_IP(x) ((x)->ip) 276 277 #elif defined(bpf_target_s390x) 278 279 #define PT_REGS_PARM1(x) ((x)->gprs[2]) 280 #define PT_REGS_PARM2(x) ((x)->gprs[3]) 281 #define PT_REGS_PARM3(x) ((x)->gprs[4]) 282 #define PT_REGS_PARM4(x) ((x)->gprs[5]) 283 #define PT_REGS_PARM5(x) ((x)->gprs[6]) 284 #define PT_REGS_RET(x) ((x)->gprs[14]) 285 #define PT_REGS_FP(x) ((x)->gprs[11]) /* Works only with CONFIG_FRAME_POINTER */ 286 #define PT_REGS_RC(x) ((x)->gprs[2]) 287 #define PT_REGS_SP(x) ((x)->gprs[15]) 288 #define PT_REGS_IP(x) ((x)->psw.addr) 289 290 #elif defined(bpf_target_arm64) 291 292 #define PT_REGS_PARM1(x) ((x)->regs[0]) 293 #define PT_REGS_PARM2(x) ((x)->regs[1]) 294 #define PT_REGS_PARM3(x) ((x)->regs[2]) 295 #define PT_REGS_PARM4(x) ((x)->regs[3]) 296 #define PT_REGS_PARM5(x) ((x)->regs[4]) 297 #define PT_REGS_RET(x) ((x)->regs[30]) 298 #define PT_REGS_FP(x) ((x)->regs[29]) /* Works only with CONFIG_FRAME_POINTER */ 299 #define PT_REGS_RC(x) ((x)->regs[0]) 300 #define PT_REGS_SP(x) ((x)->sp) 301 #define PT_REGS_IP(x) ((x)->pc) 302 303 #elif defined(bpf_target_mips) 304 305 #define PT_REGS_PARM1(x) ((x)->regs[4]) 306 #define PT_REGS_PARM2(x) ((x)->regs[5]) 307 #define PT_REGS_PARM3(x) ((x)->regs[6]) 308 #define PT_REGS_PARM4(x) ((x)->regs[7]) 309 #define PT_REGS_PARM5(x) ((x)->regs[8]) 310 #define PT_REGS_RET(x) ((x)->regs[31]) 311 #define PT_REGS_FP(x) ((x)->regs[30]) /* Works only with CONFIG_FRAME_POINTER */ 312 #define PT_REGS_RC(x) ((x)->regs[1]) 313 #define PT_REGS_SP(x) ((x)->regs[29]) 314 #define PT_REGS_IP(x) ((x)->cp0_epc) 315 316 #elif defined(bpf_target_powerpc) 317 318 #define PT_REGS_PARM1(x) ((x)->gpr[3]) 319 #define PT_REGS_PARM2(x) ((x)->gpr[4]) 320 #define PT_REGS_PARM3(x) ((x)->gpr[5]) 321 #define PT_REGS_PARM4(x) ((x)->gpr[6]) 322 #define PT_REGS_PARM5(x) ((x)->gpr[7]) 323 #define PT_REGS_RC(x) ((x)->gpr[3]) 324 #define PT_REGS_SP(x) ((x)->sp) 325 #define PT_REGS_IP(x) ((x)->nip) 326 327 #elif defined(bpf_target_sparc) 328 329 #define PT_REGS_PARM1(x) ((x)->u_regs[UREG_I0]) 330 #define PT_REGS_PARM2(x) ((x)->u_regs[UREG_I1]) 331 #define PT_REGS_PARM3(x) ((x)->u_regs[UREG_I2]) 332 #define PT_REGS_PARM4(x) ((x)->u_regs[UREG_I3]) 333 #define PT_REGS_PARM5(x) ((x)->u_regs[UREG_I4]) 334 #define PT_REGS_RET(x) ((x)->u_regs[UREG_I7]) 335 #define PT_REGS_RC(x) ((x)->u_regs[UREG_I0]) 336 #define PT_REGS_SP(x) ((x)->u_regs[UREG_FP]) 337 338 /* Should this also be a bpf_target check for the sparc case? */ 339 #if defined(__arch64__) 340 #define PT_REGS_IP(x) ((x)->tpc) 341 #else 342 #define PT_REGS_IP(x) ((x)->pc) 343 #endif 344 345 #endif 346 347 #ifdef bpf_target_powerpc 348 #define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ (ip) = (ctx)->link; }) 349 #define BPF_KRETPROBE_READ_RET_IP BPF_KPROBE_READ_RET_IP 350 #elif bpf_target_sparc 351 #define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ (ip) = PT_REGS_RET(ctx); }) 352 #define BPF_KRETPROBE_READ_RET_IP BPF_KPROBE_READ_RET_IP 353 #else 354 #define BPF_KPROBE_READ_RET_IP(ip, ctx) ({ \ 355 bpf_probe_read(&(ip), sizeof(ip), (void *)PT_REGS_RET(ctx)); }) 356 #define BPF_KRETPROBE_READ_RET_IP(ip, ctx) ({ \ 357 bpf_probe_read(&(ip), sizeof(ip), \ 358 (void *)(PT_REGS_FP(ctx) + sizeof(ip))); }) 359 #endif 360 361 #endif 362