1 // SPDX-License-Identifier: GPL-2.0 2 /* Copyright (c) 2019 Facebook */ 3 4 #include <linux/init.h> 5 #include <linux/types.h> 6 #include <linux/bpf_verifier.h> 7 #include <linux/bpf.h> 8 #include <linux/btf.h> 9 #include <linux/btf_ids.h> 10 #include <linux/filter.h> 11 #include <net/tcp.h> 12 #include <net/bpf_sk_storage.h> 13 14 /* "extern" is to avoid sparse warning. It is only used in bpf_struct_ops.c. */ 15 extern struct bpf_struct_ops bpf_tcp_congestion_ops; 16 17 static u32 unsupported_ops[] = { 18 offsetof(struct tcp_congestion_ops, get_info), 19 }; 20 21 static const struct btf_type *tcp_sock_type; 22 static u32 tcp_sock_id, sock_id; 23 24 static int bpf_tcp_ca_init(struct btf *btf) 25 { 26 s32 type_id; 27 28 type_id = btf_find_by_name_kind(btf, "sock", BTF_KIND_STRUCT); 29 if (type_id < 0) 30 return -EINVAL; 31 sock_id = type_id; 32 33 type_id = btf_find_by_name_kind(btf, "tcp_sock", BTF_KIND_STRUCT); 34 if (type_id < 0) 35 return -EINVAL; 36 tcp_sock_id = type_id; 37 tcp_sock_type = btf_type_by_id(btf, tcp_sock_id); 38 39 return 0; 40 } 41 42 static bool is_unsupported(u32 member_offset) 43 { 44 unsigned int i; 45 46 for (i = 0; i < ARRAY_SIZE(unsupported_ops); i++) { 47 if (member_offset == unsupported_ops[i]) 48 return true; 49 } 50 51 return false; 52 } 53 54 extern struct btf *btf_vmlinux; 55 56 static bool bpf_tcp_ca_is_valid_access(int off, int size, 57 enum bpf_access_type type, 58 const struct bpf_prog *prog, 59 struct bpf_insn_access_aux *info) 60 { 61 if (!bpf_tracing_btf_ctx_access(off, size, type, prog, info)) 62 return false; 63 64 if (base_type(info->reg_type) == PTR_TO_BTF_ID && 65 !bpf_type_has_unsafe_modifiers(info->reg_type) && 66 info->btf_id == sock_id) 67 /* promote it to tcp_sock */ 68 info->btf_id = tcp_sock_id; 69 70 return true; 71 } 72 73 static int bpf_tcp_ca_btf_struct_access(struct bpf_verifier_log *log, 74 const struct bpf_reg_state *reg, 75 int off, int size, enum bpf_access_type atype, 76 u32 *next_btf_id, enum bpf_type_flag *flag) 77 { 78 const struct btf_type *t; 79 size_t end; 80 81 if (atype == BPF_READ) 82 return btf_struct_access(log, reg, off, size, atype, next_btf_id, flag); 83 84 t = btf_type_by_id(reg->btf, reg->btf_id); 85 if (t != tcp_sock_type) { 86 bpf_log(log, "only read is supported\n"); 87 return -EACCES; 88 } 89 90 switch (off) { 91 case offsetof(struct sock, sk_pacing_rate): 92 end = offsetofend(struct sock, sk_pacing_rate); 93 break; 94 case offsetof(struct sock, sk_pacing_status): 95 end = offsetofend(struct sock, sk_pacing_status); 96 break; 97 case bpf_ctx_range(struct inet_connection_sock, icsk_ca_priv): 98 end = offsetofend(struct inet_connection_sock, icsk_ca_priv); 99 break; 100 case offsetof(struct inet_connection_sock, icsk_ack.pending): 101 end = offsetofend(struct inet_connection_sock, 102 icsk_ack.pending); 103 break; 104 case offsetof(struct tcp_sock, snd_cwnd): 105 end = offsetofend(struct tcp_sock, snd_cwnd); 106 break; 107 case offsetof(struct tcp_sock, snd_cwnd_cnt): 108 end = offsetofend(struct tcp_sock, snd_cwnd_cnt); 109 break; 110 case offsetof(struct tcp_sock, snd_ssthresh): 111 end = offsetofend(struct tcp_sock, snd_ssthresh); 112 break; 113 case offsetof(struct tcp_sock, ecn_flags): 114 end = offsetofend(struct tcp_sock, ecn_flags); 115 break; 116 default: 117 bpf_log(log, "no write support to tcp_sock at off %d\n", off); 118 return -EACCES; 119 } 120 121 if (off + size > end) { 122 bpf_log(log, 123 "write access at off %d with size %d beyond the member of tcp_sock ended at %zu\n", 124 off, size, end); 125 return -EACCES; 126 } 127 128 return 0; 129 } 130 131 BPF_CALL_2(bpf_tcp_send_ack, struct tcp_sock *, tp, u32, rcv_nxt) 132 { 133 /* bpf_tcp_ca prog cannot have NULL tp */ 134 __tcp_send_ack((struct sock *)tp, rcv_nxt); 135 return 0; 136 } 137 138 static const struct bpf_func_proto bpf_tcp_send_ack_proto = { 139 .func = bpf_tcp_send_ack, 140 .gpl_only = false, 141 /* In case we want to report error later */ 142 .ret_type = RET_INTEGER, 143 .arg1_type = ARG_PTR_TO_BTF_ID, 144 .arg1_btf_id = &tcp_sock_id, 145 .arg2_type = ARG_ANYTHING, 146 }; 147 148 static u32 prog_ops_moff(const struct bpf_prog *prog) 149 { 150 const struct btf_member *m; 151 const struct btf_type *t; 152 u32 midx; 153 154 midx = prog->expected_attach_type; 155 t = bpf_tcp_congestion_ops.type; 156 m = &btf_type_member(t)[midx]; 157 158 return __btf_member_bit_offset(t, m) / 8; 159 } 160 161 static const struct bpf_func_proto * 162 bpf_tcp_ca_get_func_proto(enum bpf_func_id func_id, 163 const struct bpf_prog *prog) 164 { 165 switch (func_id) { 166 case BPF_FUNC_tcp_send_ack: 167 return &bpf_tcp_send_ack_proto; 168 case BPF_FUNC_sk_storage_get: 169 return &bpf_sk_storage_get_proto; 170 case BPF_FUNC_sk_storage_delete: 171 return &bpf_sk_storage_delete_proto; 172 case BPF_FUNC_setsockopt: 173 /* Does not allow release() to call setsockopt. 174 * release() is called when the current bpf-tcp-cc 175 * is retiring. It is not allowed to call 176 * setsockopt() to make further changes which 177 * may potentially allocate new resources. 178 */ 179 if (prog_ops_moff(prog) != 180 offsetof(struct tcp_congestion_ops, release)) 181 return &bpf_sk_setsockopt_proto; 182 return NULL; 183 case BPF_FUNC_getsockopt: 184 /* Since get/setsockopt is usually expected to 185 * be available together, disable getsockopt for 186 * release also to avoid usage surprise. 187 * The bpf-tcp-cc already has a more powerful way 188 * to read tcp_sock from the PTR_TO_BTF_ID. 189 */ 190 if (prog_ops_moff(prog) != 191 offsetof(struct tcp_congestion_ops, release)) 192 return &bpf_sk_getsockopt_proto; 193 return NULL; 194 case BPF_FUNC_ktime_get_coarse_ns: 195 return &bpf_ktime_get_coarse_ns_proto; 196 default: 197 return bpf_base_func_proto(func_id); 198 } 199 } 200 201 BTF_SET8_START(bpf_tcp_ca_check_kfunc_ids) 202 BTF_ID_FLAGS(func, tcp_reno_ssthresh) 203 BTF_ID_FLAGS(func, tcp_reno_cong_avoid) 204 BTF_ID_FLAGS(func, tcp_reno_undo_cwnd) 205 BTF_ID_FLAGS(func, tcp_slow_start) 206 BTF_ID_FLAGS(func, tcp_cong_avoid_ai) 207 BTF_SET8_END(bpf_tcp_ca_check_kfunc_ids) 208 209 static const struct btf_kfunc_id_set bpf_tcp_ca_kfunc_set = { 210 .owner = THIS_MODULE, 211 .set = &bpf_tcp_ca_check_kfunc_ids, 212 }; 213 214 static const struct bpf_verifier_ops bpf_tcp_ca_verifier_ops = { 215 .get_func_proto = bpf_tcp_ca_get_func_proto, 216 .is_valid_access = bpf_tcp_ca_is_valid_access, 217 .btf_struct_access = bpf_tcp_ca_btf_struct_access, 218 }; 219 220 static int bpf_tcp_ca_init_member(const struct btf_type *t, 221 const struct btf_member *member, 222 void *kdata, const void *udata) 223 { 224 const struct tcp_congestion_ops *utcp_ca; 225 struct tcp_congestion_ops *tcp_ca; 226 u32 moff; 227 228 utcp_ca = (const struct tcp_congestion_ops *)udata; 229 tcp_ca = (struct tcp_congestion_ops *)kdata; 230 231 moff = __btf_member_bit_offset(t, member) / 8; 232 switch (moff) { 233 case offsetof(struct tcp_congestion_ops, flags): 234 if (utcp_ca->flags & ~TCP_CONG_MASK) 235 return -EINVAL; 236 tcp_ca->flags = utcp_ca->flags; 237 return 1; 238 case offsetof(struct tcp_congestion_ops, name): 239 if (bpf_obj_name_cpy(tcp_ca->name, utcp_ca->name, 240 sizeof(tcp_ca->name)) <= 0) 241 return -EINVAL; 242 if (tcp_ca_find(utcp_ca->name)) 243 return -EEXIST; 244 return 1; 245 } 246 247 return 0; 248 } 249 250 static int bpf_tcp_ca_check_member(const struct btf_type *t, 251 const struct btf_member *member, 252 const struct bpf_prog *prog) 253 { 254 if (is_unsupported(__btf_member_bit_offset(t, member) / 8)) 255 return -ENOTSUPP; 256 return 0; 257 } 258 259 static int bpf_tcp_ca_reg(void *kdata) 260 { 261 return tcp_register_congestion_control(kdata); 262 } 263 264 static void bpf_tcp_ca_unreg(void *kdata) 265 { 266 tcp_unregister_congestion_control(kdata); 267 } 268 269 struct bpf_struct_ops bpf_tcp_congestion_ops = { 270 .verifier_ops = &bpf_tcp_ca_verifier_ops, 271 .reg = bpf_tcp_ca_reg, 272 .unreg = bpf_tcp_ca_unreg, 273 .check_member = bpf_tcp_ca_check_member, 274 .init_member = bpf_tcp_ca_init_member, 275 .init = bpf_tcp_ca_init, 276 .name = "tcp_congestion_ops", 277 }; 278 279 static int __init bpf_tcp_ca_kfunc_init(void) 280 { 281 return register_btf_kfunc_id_set(BPF_PROG_TYPE_STRUCT_OPS, &bpf_tcp_ca_kfunc_set); 282 } 283 late_initcall(bpf_tcp_ca_kfunc_init); 284