1 // SPDX-License-Identifier: GPL-2.0 2 /* XSKMAP used for AF_XDP sockets 3 * Copyright(c) 2018 Intel Corporation. 4 */ 5 6 #include <linux/bpf.h> 7 #include <linux/capability.h> 8 #include <net/xdp_sock.h> 9 #include <linux/slab.h> 10 #include <linux/sched.h> 11 12 #include "xsk.h" 13 14 static struct xsk_map_node *xsk_map_node_alloc(struct xsk_map *map, 15 struct xdp_sock **map_entry) 16 { 17 struct xsk_map_node *node; 18 19 node = bpf_map_kzalloc(&map->map, sizeof(*node), 20 GFP_ATOMIC | __GFP_NOWARN); 21 if (!node) 22 return ERR_PTR(-ENOMEM); 23 24 bpf_map_inc(&map->map); 25 26 node->map = map; 27 node->map_entry = map_entry; 28 return node; 29 } 30 31 static void xsk_map_node_free(struct xsk_map_node *node) 32 { 33 bpf_map_put(&node->map->map); 34 kfree(node); 35 } 36 37 static void xsk_map_sock_add(struct xdp_sock *xs, struct xsk_map_node *node) 38 { 39 spin_lock_bh(&xs->map_list_lock); 40 list_add_tail(&node->node, &xs->map_list); 41 spin_unlock_bh(&xs->map_list_lock); 42 } 43 44 static void xsk_map_sock_delete(struct xdp_sock *xs, 45 struct xdp_sock **map_entry) 46 { 47 struct xsk_map_node *n, *tmp; 48 49 spin_lock_bh(&xs->map_list_lock); 50 list_for_each_entry_safe(n, tmp, &xs->map_list, node) { 51 if (map_entry == n->map_entry) { 52 list_del(&n->node); 53 xsk_map_node_free(n); 54 } 55 } 56 spin_unlock_bh(&xs->map_list_lock); 57 } 58 59 static struct bpf_map *xsk_map_alloc(union bpf_attr *attr) 60 { 61 struct xsk_map *m; 62 int numa_node; 63 u64 size; 64 65 if (!capable(CAP_NET_ADMIN)) 66 return ERR_PTR(-EPERM); 67 68 if (attr->max_entries == 0 || attr->key_size != 4 || 69 attr->value_size != 4 || 70 attr->map_flags & ~(BPF_F_NUMA_NODE | BPF_F_RDONLY | BPF_F_WRONLY)) 71 return ERR_PTR(-EINVAL); 72 73 numa_node = bpf_map_attr_numa_node(attr); 74 size = struct_size(m, xsk_map, attr->max_entries); 75 76 m = bpf_map_area_alloc(size, numa_node); 77 if (!m) 78 return ERR_PTR(-ENOMEM); 79 80 bpf_map_init_from_attr(&m->map, attr); 81 spin_lock_init(&m->lock); 82 83 return &m->map; 84 } 85 86 static void xsk_map_free(struct bpf_map *map) 87 { 88 struct xsk_map *m = container_of(map, struct xsk_map, map); 89 90 bpf_clear_redirect_map(map); 91 synchronize_net(); 92 bpf_map_area_free(m); 93 } 94 95 static int xsk_map_get_next_key(struct bpf_map *map, void *key, void *next_key) 96 { 97 struct xsk_map *m = container_of(map, struct xsk_map, map); 98 u32 index = key ? *(u32 *)key : U32_MAX; 99 u32 *next = next_key; 100 101 if (index >= m->map.max_entries) { 102 *next = 0; 103 return 0; 104 } 105 106 if (index == m->map.max_entries - 1) 107 return -ENOENT; 108 *next = index + 1; 109 return 0; 110 } 111 112 static int xsk_map_gen_lookup(struct bpf_map *map, struct bpf_insn *insn_buf) 113 { 114 const int ret = BPF_REG_0, mp = BPF_REG_1, index = BPF_REG_2; 115 struct bpf_insn *insn = insn_buf; 116 117 *insn++ = BPF_LDX_MEM(BPF_W, ret, index, 0); 118 *insn++ = BPF_JMP_IMM(BPF_JGE, ret, map->max_entries, 5); 119 *insn++ = BPF_ALU64_IMM(BPF_LSH, ret, ilog2(sizeof(struct xsk_sock *))); 120 *insn++ = BPF_ALU64_IMM(BPF_ADD, mp, offsetof(struct xsk_map, xsk_map)); 121 *insn++ = BPF_ALU64_REG(BPF_ADD, ret, mp); 122 *insn++ = BPF_LDX_MEM(BPF_SIZEOF(struct xsk_sock *), ret, ret, 0); 123 *insn++ = BPF_JMP_IMM(BPF_JA, 0, 0, 1); 124 *insn++ = BPF_MOV64_IMM(ret, 0); 125 return insn - insn_buf; 126 } 127 128 static void *xsk_map_lookup_elem(struct bpf_map *map, void *key) 129 { 130 WARN_ON_ONCE(!rcu_read_lock_held()); 131 return __xsk_map_lookup_elem(map, *(u32 *)key); 132 } 133 134 static void *xsk_map_lookup_elem_sys_only(struct bpf_map *map, void *key) 135 { 136 return ERR_PTR(-EOPNOTSUPP); 137 } 138 139 static int xsk_map_update_elem(struct bpf_map *map, void *key, void *value, 140 u64 map_flags) 141 { 142 struct xsk_map *m = container_of(map, struct xsk_map, map); 143 struct xdp_sock *xs, *old_xs, **map_entry; 144 u32 i = *(u32 *)key, fd = *(u32 *)value; 145 struct xsk_map_node *node; 146 struct socket *sock; 147 int err; 148 149 if (unlikely(map_flags > BPF_EXIST)) 150 return -EINVAL; 151 if (unlikely(i >= m->map.max_entries)) 152 return -E2BIG; 153 154 sock = sockfd_lookup(fd, &err); 155 if (!sock) 156 return err; 157 158 if (sock->sk->sk_family != PF_XDP) { 159 sockfd_put(sock); 160 return -EOPNOTSUPP; 161 } 162 163 xs = (struct xdp_sock *)sock->sk; 164 165 map_entry = &m->xsk_map[i]; 166 node = xsk_map_node_alloc(m, map_entry); 167 if (IS_ERR(node)) { 168 sockfd_put(sock); 169 return PTR_ERR(node); 170 } 171 172 spin_lock_bh(&m->lock); 173 old_xs = READ_ONCE(*map_entry); 174 if (old_xs == xs) { 175 err = 0; 176 goto out; 177 } else if (old_xs && map_flags == BPF_NOEXIST) { 178 err = -EEXIST; 179 goto out; 180 } else if (!old_xs && map_flags == BPF_EXIST) { 181 err = -ENOENT; 182 goto out; 183 } 184 xsk_map_sock_add(xs, node); 185 WRITE_ONCE(*map_entry, xs); 186 if (old_xs) 187 xsk_map_sock_delete(old_xs, map_entry); 188 spin_unlock_bh(&m->lock); 189 sockfd_put(sock); 190 return 0; 191 192 out: 193 spin_unlock_bh(&m->lock); 194 sockfd_put(sock); 195 xsk_map_node_free(node); 196 return err; 197 } 198 199 static int xsk_map_delete_elem(struct bpf_map *map, void *key) 200 { 201 struct xsk_map *m = container_of(map, struct xsk_map, map); 202 struct xdp_sock *old_xs, **map_entry; 203 int k = *(u32 *)key; 204 205 if (k >= map->max_entries) 206 return -EINVAL; 207 208 spin_lock_bh(&m->lock); 209 map_entry = &m->xsk_map[k]; 210 old_xs = xchg(map_entry, NULL); 211 if (old_xs) 212 xsk_map_sock_delete(old_xs, map_entry); 213 spin_unlock_bh(&m->lock); 214 215 return 0; 216 } 217 218 void xsk_map_try_sock_delete(struct xsk_map *map, struct xdp_sock *xs, 219 struct xdp_sock **map_entry) 220 { 221 spin_lock_bh(&map->lock); 222 if (READ_ONCE(*map_entry) == xs) { 223 WRITE_ONCE(*map_entry, NULL); 224 xsk_map_sock_delete(xs, map_entry); 225 } 226 spin_unlock_bh(&map->lock); 227 } 228 229 static bool xsk_map_meta_equal(const struct bpf_map *meta0, 230 const struct bpf_map *meta1) 231 { 232 return meta0->max_entries == meta1->max_entries && 233 bpf_map_meta_equal(meta0, meta1); 234 } 235 236 static int xsk_map_btf_id; 237 const struct bpf_map_ops xsk_map_ops = { 238 .map_meta_equal = xsk_map_meta_equal, 239 .map_alloc = xsk_map_alloc, 240 .map_free = xsk_map_free, 241 .map_get_next_key = xsk_map_get_next_key, 242 .map_lookup_elem = xsk_map_lookup_elem, 243 .map_gen_lookup = xsk_map_gen_lookup, 244 .map_lookup_elem_sys_only = xsk_map_lookup_elem_sys_only, 245 .map_update_elem = xsk_map_update_elem, 246 .map_delete_elem = xsk_map_delete_elem, 247 .map_check_btf = map_check_no_btf, 248 .map_btf_name = "xsk_map", 249 .map_btf_id = &xsk_map_btf_id, 250 }; 251