1 // SPDX-License-Identifier: GPL-2.0-only 2 /* Copyright (c) 2020 Facebook */ 3 4 #include <linux/fs.h> 5 #include <linux/anon_inodes.h> 6 #include <linux/filter.h> 7 #include <linux/bpf.h> 8 9 struct bpf_iter_target_info { 10 struct list_head list; 11 const struct bpf_iter_reg *reg_info; 12 u32 btf_id; /* cached value */ 13 }; 14 15 struct bpf_iter_link { 16 struct bpf_link link; 17 struct bpf_iter_aux_info aux; 18 struct bpf_iter_target_info *tinfo; 19 }; 20 21 struct bpf_iter_priv_data { 22 struct bpf_iter_target_info *tinfo; 23 const struct bpf_iter_seq_info *seq_info; 24 struct bpf_prog *prog; 25 u64 session_id; 26 u64 seq_num; 27 bool done_stop; 28 u8 target_private[] __aligned(8); 29 }; 30 31 static struct list_head targets = LIST_HEAD_INIT(targets); 32 static DEFINE_MUTEX(targets_mutex); 33 34 /* protect bpf_iter_link changes */ 35 static DEFINE_MUTEX(link_mutex); 36 37 /* incremented on every opened seq_file */ 38 static atomic64_t session_id; 39 40 static int prepare_seq_file(struct file *file, struct bpf_iter_link *link, 41 const struct bpf_iter_seq_info *seq_info); 42 43 static void bpf_iter_inc_seq_num(struct seq_file *seq) 44 { 45 struct bpf_iter_priv_data *iter_priv; 46 47 iter_priv = container_of(seq->private, struct bpf_iter_priv_data, 48 target_private); 49 iter_priv->seq_num++; 50 } 51 52 static void bpf_iter_dec_seq_num(struct seq_file *seq) 53 { 54 struct bpf_iter_priv_data *iter_priv; 55 56 iter_priv = container_of(seq->private, struct bpf_iter_priv_data, 57 target_private); 58 iter_priv->seq_num--; 59 } 60 61 static void bpf_iter_done_stop(struct seq_file *seq) 62 { 63 struct bpf_iter_priv_data *iter_priv; 64 65 iter_priv = container_of(seq->private, struct bpf_iter_priv_data, 66 target_private); 67 iter_priv->done_stop = true; 68 } 69 70 static bool bpf_iter_support_resched(struct seq_file *seq) 71 { 72 struct bpf_iter_priv_data *iter_priv; 73 74 iter_priv = container_of(seq->private, struct bpf_iter_priv_data, 75 target_private); 76 return iter_priv->tinfo->reg_info->feature & BPF_ITER_RESCHED; 77 } 78 79 /* maximum visited objects before bailing out */ 80 #define MAX_ITER_OBJECTS 1000000 81 82 /* bpf_seq_read, a customized and simpler version for bpf iterator. 83 * no_llseek is assumed for this file. 84 * The following are differences from seq_read(): 85 * . fixed buffer size (PAGE_SIZE) 86 * . assuming no_llseek 87 * . stop() may call bpf program, handling potential overflow there 88 */ 89 static ssize_t bpf_seq_read(struct file *file, char __user *buf, size_t size, 90 loff_t *ppos) 91 { 92 struct seq_file *seq = file->private_data; 93 size_t n, offs, copied = 0; 94 int err = 0, num_objs = 0; 95 bool can_resched; 96 void *p; 97 98 mutex_lock(&seq->lock); 99 100 if (!seq->buf) { 101 seq->size = PAGE_SIZE << 3; 102 seq->buf = kvmalloc(seq->size, GFP_KERNEL); 103 if (!seq->buf) { 104 err = -ENOMEM; 105 goto done; 106 } 107 } 108 109 if (seq->count) { 110 n = min(seq->count, size); 111 err = copy_to_user(buf, seq->buf + seq->from, n); 112 if (err) { 113 err = -EFAULT; 114 goto done; 115 } 116 seq->count -= n; 117 seq->from += n; 118 copied = n; 119 goto done; 120 } 121 122 seq->from = 0; 123 p = seq->op->start(seq, &seq->index); 124 if (!p) 125 goto stop; 126 if (IS_ERR(p)) { 127 err = PTR_ERR(p); 128 seq->op->stop(seq, p); 129 seq->count = 0; 130 goto done; 131 } 132 133 err = seq->op->show(seq, p); 134 if (err > 0) { 135 /* object is skipped, decrease seq_num, so next 136 * valid object can reuse the same seq_num. 137 */ 138 bpf_iter_dec_seq_num(seq); 139 seq->count = 0; 140 } else if (err < 0 || seq_has_overflowed(seq)) { 141 if (!err) 142 err = -E2BIG; 143 seq->op->stop(seq, p); 144 seq->count = 0; 145 goto done; 146 } 147 148 can_resched = bpf_iter_support_resched(seq); 149 while (1) { 150 loff_t pos = seq->index; 151 152 num_objs++; 153 offs = seq->count; 154 p = seq->op->next(seq, p, &seq->index); 155 if (pos == seq->index) { 156 pr_info_ratelimited("buggy seq_file .next function %ps " 157 "did not updated position index\n", 158 seq->op->next); 159 seq->index++; 160 } 161 162 if (IS_ERR_OR_NULL(p)) 163 break; 164 165 /* got a valid next object, increase seq_num */ 166 bpf_iter_inc_seq_num(seq); 167 168 if (seq->count >= size) 169 break; 170 171 if (num_objs >= MAX_ITER_OBJECTS) { 172 if (offs == 0) { 173 err = -EAGAIN; 174 seq->op->stop(seq, p); 175 goto done; 176 } 177 break; 178 } 179 180 err = seq->op->show(seq, p); 181 if (err > 0) { 182 bpf_iter_dec_seq_num(seq); 183 seq->count = offs; 184 } else if (err < 0 || seq_has_overflowed(seq)) { 185 seq->count = offs; 186 if (offs == 0) { 187 if (!err) 188 err = -E2BIG; 189 seq->op->stop(seq, p); 190 goto done; 191 } 192 break; 193 } 194 195 if (can_resched) 196 cond_resched(); 197 } 198 stop: 199 offs = seq->count; 200 /* bpf program called if !p */ 201 seq->op->stop(seq, p); 202 if (!p) { 203 if (!seq_has_overflowed(seq)) { 204 bpf_iter_done_stop(seq); 205 } else { 206 seq->count = offs; 207 if (offs == 0) { 208 err = -E2BIG; 209 goto done; 210 } 211 } 212 } 213 214 n = min(seq->count, size); 215 err = copy_to_user(buf, seq->buf, n); 216 if (err) { 217 err = -EFAULT; 218 goto done; 219 } 220 copied = n; 221 seq->count -= n; 222 seq->from = n; 223 done: 224 if (!copied) 225 copied = err; 226 else 227 *ppos += copied; 228 mutex_unlock(&seq->lock); 229 return copied; 230 } 231 232 static const struct bpf_iter_seq_info * 233 __get_seq_info(struct bpf_iter_link *link) 234 { 235 const struct bpf_iter_seq_info *seq_info; 236 237 if (link->aux.map) { 238 seq_info = link->aux.map->ops->iter_seq_info; 239 if (seq_info) 240 return seq_info; 241 } 242 243 return link->tinfo->reg_info->seq_info; 244 } 245 246 static int iter_open(struct inode *inode, struct file *file) 247 { 248 struct bpf_iter_link *link = inode->i_private; 249 250 return prepare_seq_file(file, link, __get_seq_info(link)); 251 } 252 253 static int iter_release(struct inode *inode, struct file *file) 254 { 255 struct bpf_iter_priv_data *iter_priv; 256 struct seq_file *seq; 257 258 seq = file->private_data; 259 if (!seq) 260 return 0; 261 262 iter_priv = container_of(seq->private, struct bpf_iter_priv_data, 263 target_private); 264 265 if (iter_priv->seq_info->fini_seq_private) 266 iter_priv->seq_info->fini_seq_private(seq->private); 267 268 bpf_prog_put(iter_priv->prog); 269 seq->private = iter_priv; 270 271 return seq_release_private(inode, file); 272 } 273 274 const struct file_operations bpf_iter_fops = { 275 .open = iter_open, 276 .llseek = no_llseek, 277 .read = bpf_seq_read, 278 .release = iter_release, 279 }; 280 281 /* The argument reg_info will be cached in bpf_iter_target_info. 282 * The common practice is to declare target reg_info as 283 * a const static variable and passed as an argument to 284 * bpf_iter_reg_target(). 285 */ 286 int bpf_iter_reg_target(const struct bpf_iter_reg *reg_info) 287 { 288 struct bpf_iter_target_info *tinfo; 289 290 tinfo = kmalloc(sizeof(*tinfo), GFP_KERNEL); 291 if (!tinfo) 292 return -ENOMEM; 293 294 tinfo->reg_info = reg_info; 295 INIT_LIST_HEAD(&tinfo->list); 296 297 mutex_lock(&targets_mutex); 298 list_add(&tinfo->list, &targets); 299 mutex_unlock(&targets_mutex); 300 301 return 0; 302 } 303 304 void bpf_iter_unreg_target(const struct bpf_iter_reg *reg_info) 305 { 306 struct bpf_iter_target_info *tinfo; 307 bool found = false; 308 309 mutex_lock(&targets_mutex); 310 list_for_each_entry(tinfo, &targets, list) { 311 if (reg_info == tinfo->reg_info) { 312 list_del(&tinfo->list); 313 kfree(tinfo); 314 found = true; 315 break; 316 } 317 } 318 mutex_unlock(&targets_mutex); 319 320 WARN_ON(found == false); 321 } 322 323 static void cache_btf_id(struct bpf_iter_target_info *tinfo, 324 struct bpf_prog *prog) 325 { 326 tinfo->btf_id = prog->aux->attach_btf_id; 327 } 328 329 bool bpf_iter_prog_supported(struct bpf_prog *prog) 330 { 331 const char *attach_fname = prog->aux->attach_func_name; 332 u32 prog_btf_id = prog->aux->attach_btf_id; 333 const char *prefix = BPF_ITER_FUNC_PREFIX; 334 struct bpf_iter_target_info *tinfo; 335 int prefix_len = strlen(prefix); 336 bool supported = false; 337 338 if (strncmp(attach_fname, prefix, prefix_len)) 339 return false; 340 341 mutex_lock(&targets_mutex); 342 list_for_each_entry(tinfo, &targets, list) { 343 if (tinfo->btf_id && tinfo->btf_id == prog_btf_id) { 344 supported = true; 345 break; 346 } 347 if (!strcmp(attach_fname + prefix_len, tinfo->reg_info->target)) { 348 cache_btf_id(tinfo, prog); 349 supported = true; 350 break; 351 } 352 } 353 mutex_unlock(&targets_mutex); 354 355 if (supported) { 356 prog->aux->ctx_arg_info_size = tinfo->reg_info->ctx_arg_info_size; 357 prog->aux->ctx_arg_info = tinfo->reg_info->ctx_arg_info; 358 } 359 360 return supported; 361 } 362 363 static void bpf_iter_link_release(struct bpf_link *link) 364 { 365 struct bpf_iter_link *iter_link = 366 container_of(link, struct bpf_iter_link, link); 367 368 if (iter_link->tinfo->reg_info->detach_target) 369 iter_link->tinfo->reg_info->detach_target(&iter_link->aux); 370 } 371 372 static void bpf_iter_link_dealloc(struct bpf_link *link) 373 { 374 struct bpf_iter_link *iter_link = 375 container_of(link, struct bpf_iter_link, link); 376 377 kfree(iter_link); 378 } 379 380 static int bpf_iter_link_replace(struct bpf_link *link, 381 struct bpf_prog *new_prog, 382 struct bpf_prog *old_prog) 383 { 384 int ret = 0; 385 386 mutex_lock(&link_mutex); 387 if (old_prog && link->prog != old_prog) { 388 ret = -EPERM; 389 goto out_unlock; 390 } 391 392 if (link->prog->type != new_prog->type || 393 link->prog->expected_attach_type != new_prog->expected_attach_type || 394 link->prog->aux->attach_btf_id != new_prog->aux->attach_btf_id) { 395 ret = -EINVAL; 396 goto out_unlock; 397 } 398 399 old_prog = xchg(&link->prog, new_prog); 400 bpf_prog_put(old_prog); 401 402 out_unlock: 403 mutex_unlock(&link_mutex); 404 return ret; 405 } 406 407 static void bpf_iter_link_show_fdinfo(const struct bpf_link *link, 408 struct seq_file *seq) 409 { 410 struct bpf_iter_link *iter_link = 411 container_of(link, struct bpf_iter_link, link); 412 bpf_iter_show_fdinfo_t show_fdinfo; 413 414 seq_printf(seq, 415 "target_name:\t%s\n", 416 iter_link->tinfo->reg_info->target); 417 418 show_fdinfo = iter_link->tinfo->reg_info->show_fdinfo; 419 if (show_fdinfo) 420 show_fdinfo(&iter_link->aux, seq); 421 } 422 423 static int bpf_iter_link_fill_link_info(const struct bpf_link *link, 424 struct bpf_link_info *info) 425 { 426 struct bpf_iter_link *iter_link = 427 container_of(link, struct bpf_iter_link, link); 428 char __user *ubuf = u64_to_user_ptr(info->iter.target_name); 429 bpf_iter_fill_link_info_t fill_link_info; 430 u32 ulen = info->iter.target_name_len; 431 const char *target_name; 432 u32 target_len; 433 434 if (!ulen ^ !ubuf) 435 return -EINVAL; 436 437 target_name = iter_link->tinfo->reg_info->target; 438 target_len = strlen(target_name); 439 info->iter.target_name_len = target_len + 1; 440 441 if (ubuf) { 442 if (ulen >= target_len + 1) { 443 if (copy_to_user(ubuf, target_name, target_len + 1)) 444 return -EFAULT; 445 } else { 446 char zero = '\0'; 447 448 if (copy_to_user(ubuf, target_name, ulen - 1)) 449 return -EFAULT; 450 if (put_user(zero, ubuf + ulen - 1)) 451 return -EFAULT; 452 return -ENOSPC; 453 } 454 } 455 456 fill_link_info = iter_link->tinfo->reg_info->fill_link_info; 457 if (fill_link_info) 458 return fill_link_info(&iter_link->aux, info); 459 460 return 0; 461 } 462 463 static const struct bpf_link_ops bpf_iter_link_lops = { 464 .release = bpf_iter_link_release, 465 .dealloc = bpf_iter_link_dealloc, 466 .update_prog = bpf_iter_link_replace, 467 .show_fdinfo = bpf_iter_link_show_fdinfo, 468 .fill_link_info = bpf_iter_link_fill_link_info, 469 }; 470 471 bool bpf_link_is_iter(struct bpf_link *link) 472 { 473 return link->ops == &bpf_iter_link_lops; 474 } 475 476 int bpf_iter_link_attach(const union bpf_attr *attr, struct bpf_prog *prog) 477 { 478 union bpf_iter_link_info __user *ulinfo; 479 struct bpf_link_primer link_primer; 480 struct bpf_iter_target_info *tinfo; 481 union bpf_iter_link_info linfo; 482 struct bpf_iter_link *link; 483 u32 prog_btf_id, linfo_len; 484 bool existed = false; 485 int err; 486 487 if (attr->link_create.target_fd || attr->link_create.flags) 488 return -EINVAL; 489 490 memset(&linfo, 0, sizeof(union bpf_iter_link_info)); 491 492 ulinfo = u64_to_user_ptr(attr->link_create.iter_info); 493 linfo_len = attr->link_create.iter_info_len; 494 if (!ulinfo ^ !linfo_len) 495 return -EINVAL; 496 497 if (ulinfo) { 498 err = bpf_check_uarg_tail_zero(ulinfo, sizeof(linfo), 499 linfo_len); 500 if (err) 501 return err; 502 linfo_len = min_t(u32, linfo_len, sizeof(linfo)); 503 if (copy_from_user(&linfo, ulinfo, linfo_len)) 504 return -EFAULT; 505 } 506 507 prog_btf_id = prog->aux->attach_btf_id; 508 mutex_lock(&targets_mutex); 509 list_for_each_entry(tinfo, &targets, list) { 510 if (tinfo->btf_id == prog_btf_id) { 511 existed = true; 512 break; 513 } 514 } 515 mutex_unlock(&targets_mutex); 516 if (!existed) 517 return -ENOENT; 518 519 link = kzalloc(sizeof(*link), GFP_USER | __GFP_NOWARN); 520 if (!link) 521 return -ENOMEM; 522 523 bpf_link_init(&link->link, BPF_LINK_TYPE_ITER, &bpf_iter_link_lops, prog); 524 link->tinfo = tinfo; 525 526 err = bpf_link_prime(&link->link, &link_primer); 527 if (err) { 528 kfree(link); 529 return err; 530 } 531 532 if (tinfo->reg_info->attach_target) { 533 err = tinfo->reg_info->attach_target(prog, &linfo, &link->aux); 534 if (err) { 535 bpf_link_cleanup(&link_primer); 536 return err; 537 } 538 } 539 540 return bpf_link_settle(&link_primer); 541 } 542 543 static void init_seq_meta(struct bpf_iter_priv_data *priv_data, 544 struct bpf_iter_target_info *tinfo, 545 const struct bpf_iter_seq_info *seq_info, 546 struct bpf_prog *prog) 547 { 548 priv_data->tinfo = tinfo; 549 priv_data->seq_info = seq_info; 550 priv_data->prog = prog; 551 priv_data->session_id = atomic64_inc_return(&session_id); 552 priv_data->seq_num = 0; 553 priv_data->done_stop = false; 554 } 555 556 static int prepare_seq_file(struct file *file, struct bpf_iter_link *link, 557 const struct bpf_iter_seq_info *seq_info) 558 { 559 struct bpf_iter_priv_data *priv_data; 560 struct bpf_iter_target_info *tinfo; 561 struct bpf_prog *prog; 562 u32 total_priv_dsize; 563 struct seq_file *seq; 564 int err = 0; 565 566 mutex_lock(&link_mutex); 567 prog = link->link.prog; 568 bpf_prog_inc(prog); 569 mutex_unlock(&link_mutex); 570 571 tinfo = link->tinfo; 572 total_priv_dsize = offsetof(struct bpf_iter_priv_data, target_private) + 573 seq_info->seq_priv_size; 574 priv_data = __seq_open_private(file, seq_info->seq_ops, 575 total_priv_dsize); 576 if (!priv_data) { 577 err = -ENOMEM; 578 goto release_prog; 579 } 580 581 if (seq_info->init_seq_private) { 582 err = seq_info->init_seq_private(priv_data->target_private, &link->aux); 583 if (err) 584 goto release_seq_file; 585 } 586 587 init_seq_meta(priv_data, tinfo, seq_info, prog); 588 seq = file->private_data; 589 seq->private = priv_data->target_private; 590 591 return 0; 592 593 release_seq_file: 594 seq_release_private(file->f_inode, file); 595 file->private_data = NULL; 596 release_prog: 597 bpf_prog_put(prog); 598 return err; 599 } 600 601 int bpf_iter_new_fd(struct bpf_link *link) 602 { 603 struct bpf_iter_link *iter_link; 604 struct file *file; 605 unsigned int flags; 606 int err, fd; 607 608 if (link->ops != &bpf_iter_link_lops) 609 return -EINVAL; 610 611 flags = O_RDONLY | O_CLOEXEC; 612 fd = get_unused_fd_flags(flags); 613 if (fd < 0) 614 return fd; 615 616 file = anon_inode_getfile("bpf_iter", &bpf_iter_fops, NULL, flags); 617 if (IS_ERR(file)) { 618 err = PTR_ERR(file); 619 goto free_fd; 620 } 621 622 iter_link = container_of(link, struct bpf_iter_link, link); 623 err = prepare_seq_file(file, iter_link, __get_seq_info(iter_link)); 624 if (err) 625 goto free_file; 626 627 fd_install(fd, file); 628 return fd; 629 630 free_file: 631 fput(file); 632 free_fd: 633 put_unused_fd(fd); 634 return err; 635 } 636 637 struct bpf_prog *bpf_iter_get_info(struct bpf_iter_meta *meta, bool in_stop) 638 { 639 struct bpf_iter_priv_data *iter_priv; 640 struct seq_file *seq; 641 void *seq_priv; 642 643 seq = meta->seq; 644 if (seq->file->f_op != &bpf_iter_fops) 645 return NULL; 646 647 seq_priv = seq->private; 648 iter_priv = container_of(seq_priv, struct bpf_iter_priv_data, 649 target_private); 650 651 if (in_stop && iter_priv->done_stop) 652 return NULL; 653 654 meta->session_id = iter_priv->session_id; 655 meta->seq_num = iter_priv->seq_num; 656 657 return iter_priv->prog; 658 } 659 660 int bpf_iter_run_prog(struct bpf_prog *prog, void *ctx) 661 { 662 int ret; 663 664 rcu_read_lock(); 665 migrate_disable(); 666 ret = BPF_PROG_RUN(prog, ctx); 667 migrate_enable(); 668 rcu_read_unlock(); 669 670 /* bpf program can only return 0 or 1: 671 * 0 : okay 672 * 1 : retry the same object 673 * The bpf_iter_run_prog() return value 674 * will be seq_ops->show() return value. 675 */ 676 return ret == 0 ? 0 : -EAGAIN; 677 } 678