1 // SPDX-License-Identifier: GPL-2.0 2 /* Watch queue and general notification mechanism, built on pipes 3 * 4 * Copyright (C) 2020 Red Hat, Inc. All Rights Reserved. 5 * Written by David Howells (dhowells@redhat.com) 6 * 7 * See Documentation/watch_queue.rst 8 */ 9 10 #define pr_fmt(fmt) "watchq: " fmt 11 #include <linux/module.h> 12 #include <linux/init.h> 13 #include <linux/sched.h> 14 #include <linux/slab.h> 15 #include <linux/printk.h> 16 #include <linux/miscdevice.h> 17 #include <linux/fs.h> 18 #include <linux/mm.h> 19 #include <linux/pagemap.h> 20 #include <linux/poll.h> 21 #include <linux/uaccess.h> 22 #include <linux/vmalloc.h> 23 #include <linux/file.h> 24 #include <linux/security.h> 25 #include <linux/cred.h> 26 #include <linux/sched/signal.h> 27 #include <linux/watch_queue.h> 28 #include <linux/pipe_fs_i.h> 29 30 MODULE_DESCRIPTION("Watch queue"); 31 MODULE_AUTHOR("Red Hat, Inc."); 32 MODULE_LICENSE("GPL"); 33 34 #define WATCH_QUEUE_NOTE_SIZE 128 35 #define WATCH_QUEUE_NOTES_PER_PAGE (PAGE_SIZE / WATCH_QUEUE_NOTE_SIZE) 36 37 /* 38 * This must be called under the RCU read-lock, which makes 39 * sure that the wqueue still exists. It can then take the lock, 40 * and check that the wqueue hasn't been destroyed, which in 41 * turn makes sure that the notification pipe still exists. 42 */ 43 static inline bool lock_wqueue(struct watch_queue *wqueue) 44 { 45 spin_lock_bh(&wqueue->lock); 46 if (unlikely(wqueue->defunct)) { 47 spin_unlock_bh(&wqueue->lock); 48 return false; 49 } 50 return true; 51 } 52 53 static inline void unlock_wqueue(struct watch_queue *wqueue) 54 { 55 spin_unlock_bh(&wqueue->lock); 56 } 57 58 static void watch_queue_pipe_buf_release(struct pipe_inode_info *pipe, 59 struct pipe_buffer *buf) 60 { 61 struct watch_queue *wqueue = (struct watch_queue *)buf->private; 62 struct page *page; 63 unsigned int bit; 64 65 /* We need to work out which note within the page this refers to, but 66 * the note might have been maximum size, so merely ANDing the offset 67 * off doesn't work. OTOH, the note must've been more than zero size. 68 */ 69 bit = buf->offset + buf->len; 70 if ((bit & (WATCH_QUEUE_NOTE_SIZE - 1)) == 0) 71 bit -= WATCH_QUEUE_NOTE_SIZE; 72 bit /= WATCH_QUEUE_NOTE_SIZE; 73 74 page = buf->page; 75 bit += page->index; 76 77 set_bit(bit, wqueue->notes_bitmap); 78 generic_pipe_buf_release(pipe, buf); 79 } 80 81 // No try_steal function => no stealing 82 #define watch_queue_pipe_buf_try_steal NULL 83 84 /* New data written to a pipe may be appended to a buffer with this type. */ 85 static const struct pipe_buf_operations watch_queue_pipe_buf_ops = { 86 .release = watch_queue_pipe_buf_release, 87 .try_steal = watch_queue_pipe_buf_try_steal, 88 .get = generic_pipe_buf_get, 89 }; 90 91 /* 92 * Post a notification to a watch queue. 93 * 94 * Must be called with the RCU lock for reading, and the 95 * watch_queue lock held, which guarantees that the pipe 96 * hasn't been released. 97 */ 98 static bool post_one_notification(struct watch_queue *wqueue, 99 struct watch_notification *n) 100 { 101 void *p; 102 struct pipe_inode_info *pipe = wqueue->pipe; 103 struct pipe_buffer *buf; 104 struct page *page; 105 unsigned int head, tail, mask, note, offset, len; 106 bool done = false; 107 108 if (!pipe) 109 return false; 110 111 spin_lock_irq(&pipe->rd_wait.lock); 112 113 mask = pipe->ring_size - 1; 114 head = pipe->head; 115 tail = pipe->tail; 116 if (pipe_full(head, tail, pipe->ring_size)) 117 goto lost; 118 119 note = find_first_bit(wqueue->notes_bitmap, wqueue->nr_notes); 120 if (note >= wqueue->nr_notes) 121 goto lost; 122 123 page = wqueue->notes[note / WATCH_QUEUE_NOTES_PER_PAGE]; 124 offset = note % WATCH_QUEUE_NOTES_PER_PAGE * WATCH_QUEUE_NOTE_SIZE; 125 get_page(page); 126 len = n->info & WATCH_INFO_LENGTH; 127 p = kmap_atomic(page); 128 memcpy(p + offset, n, len); 129 kunmap_atomic(p); 130 131 buf = &pipe->bufs[head & mask]; 132 buf->page = page; 133 buf->private = (unsigned long)wqueue; 134 buf->ops = &watch_queue_pipe_buf_ops; 135 buf->offset = offset; 136 buf->len = len; 137 buf->flags = PIPE_BUF_FLAG_WHOLE; 138 smp_store_release(&pipe->head, head + 1); /* vs pipe_read() */ 139 140 if (!test_and_clear_bit(note, wqueue->notes_bitmap)) { 141 spin_unlock_irq(&pipe->rd_wait.lock); 142 BUG(); 143 } 144 wake_up_interruptible_sync_poll_locked(&pipe->rd_wait, EPOLLIN | EPOLLRDNORM); 145 done = true; 146 147 out: 148 spin_unlock_irq(&pipe->rd_wait.lock); 149 if (done) 150 kill_fasync(&pipe->fasync_readers, SIGIO, POLL_IN); 151 return done; 152 153 lost: 154 buf = &pipe->bufs[(head - 1) & mask]; 155 buf->flags |= PIPE_BUF_FLAG_LOSS; 156 goto out; 157 } 158 159 /* 160 * Apply filter rules to a notification. 161 */ 162 static bool filter_watch_notification(const struct watch_filter *wf, 163 const struct watch_notification *n) 164 { 165 const struct watch_type_filter *wt; 166 unsigned int st_bits = sizeof(wt->subtype_filter[0]) * 8; 167 unsigned int st_index = n->subtype / st_bits; 168 unsigned int st_bit = 1U << (n->subtype % st_bits); 169 int i; 170 171 if (!test_bit(n->type, wf->type_filter)) 172 return false; 173 174 for (i = 0; i < wf->nr_filters; i++) { 175 wt = &wf->filters[i]; 176 if (n->type == wt->type && 177 (wt->subtype_filter[st_index] & st_bit) && 178 (n->info & wt->info_mask) == wt->info_filter) 179 return true; 180 } 181 182 return false; /* If there is a filter, the default is to reject. */ 183 } 184 185 /** 186 * __post_watch_notification - Post an event notification 187 * @wlist: The watch list to post the event to. 188 * @n: The notification record to post. 189 * @cred: The creds of the process that triggered the notification. 190 * @id: The ID to match on the watch. 191 * 192 * Post a notification of an event into a set of watch queues and let the users 193 * know. 194 * 195 * The size of the notification should be set in n->info & WATCH_INFO_LENGTH and 196 * should be in units of sizeof(*n). 197 */ 198 void __post_watch_notification(struct watch_list *wlist, 199 struct watch_notification *n, 200 const struct cred *cred, 201 u64 id) 202 { 203 const struct watch_filter *wf; 204 struct watch_queue *wqueue; 205 struct watch *watch; 206 207 if (((n->info & WATCH_INFO_LENGTH) >> WATCH_INFO_LENGTH__SHIFT) == 0) { 208 WARN_ON(1); 209 return; 210 } 211 212 rcu_read_lock(); 213 214 hlist_for_each_entry_rcu(watch, &wlist->watchers, list_node) { 215 if (watch->id != id) 216 continue; 217 n->info &= ~WATCH_INFO_ID; 218 n->info |= watch->info_id; 219 220 wqueue = rcu_dereference(watch->queue); 221 wf = rcu_dereference(wqueue->filter); 222 if (wf && !filter_watch_notification(wf, n)) 223 continue; 224 225 if (security_post_notification(watch->cred, cred, n) < 0) 226 continue; 227 228 if (lock_wqueue(wqueue)) { 229 post_one_notification(wqueue, n); 230 unlock_wqueue(wqueue); 231 } 232 } 233 234 rcu_read_unlock(); 235 } 236 EXPORT_SYMBOL(__post_watch_notification); 237 238 /* 239 * Allocate sufficient pages to preallocation for the requested number of 240 * notifications. 241 */ 242 long watch_queue_set_size(struct pipe_inode_info *pipe, unsigned int nr_notes) 243 { 244 struct watch_queue *wqueue = pipe->watch_queue; 245 struct page **pages; 246 unsigned long *bitmap; 247 unsigned long user_bufs; 248 unsigned int bmsize; 249 int ret, i, nr_pages; 250 251 if (!wqueue) 252 return -ENODEV; 253 if (wqueue->notes) 254 return -EBUSY; 255 256 if (nr_notes < 1 || 257 nr_notes > 512) /* TODO: choose a better hard limit */ 258 return -EINVAL; 259 260 nr_pages = (nr_notes + WATCH_QUEUE_NOTES_PER_PAGE - 1); 261 nr_pages /= WATCH_QUEUE_NOTES_PER_PAGE; 262 user_bufs = account_pipe_buffers(pipe->user, pipe->nr_accounted, nr_pages); 263 264 if (nr_pages > pipe->max_usage && 265 (too_many_pipe_buffers_hard(user_bufs) || 266 too_many_pipe_buffers_soft(user_bufs)) && 267 pipe_is_unprivileged_user()) { 268 ret = -EPERM; 269 goto error; 270 } 271 272 nr_notes = nr_pages * WATCH_QUEUE_NOTES_PER_PAGE; 273 ret = pipe_resize_ring(pipe, roundup_pow_of_two(nr_notes)); 274 if (ret < 0) 275 goto error; 276 277 pages = kcalloc(sizeof(struct page *), nr_pages, GFP_KERNEL); 278 if (!pages) 279 goto error; 280 281 for (i = 0; i < nr_pages; i++) { 282 pages[i] = alloc_page(GFP_KERNEL); 283 if (!pages[i]) 284 goto error_p; 285 pages[i]->index = i * WATCH_QUEUE_NOTES_PER_PAGE; 286 } 287 288 bmsize = (nr_notes + BITS_PER_LONG - 1) / BITS_PER_LONG; 289 bmsize *= sizeof(unsigned long); 290 bitmap = kmalloc(bmsize, GFP_KERNEL); 291 if (!bitmap) 292 goto error_p; 293 294 memset(bitmap, 0xff, bmsize); 295 wqueue->notes = pages; 296 wqueue->notes_bitmap = bitmap; 297 wqueue->nr_pages = nr_pages; 298 wqueue->nr_notes = nr_notes; 299 return 0; 300 301 error_p: 302 while (--i >= 0) 303 __free_page(pages[i]); 304 kfree(pages); 305 error: 306 (void) account_pipe_buffers(pipe->user, nr_pages, pipe->nr_accounted); 307 return ret; 308 } 309 310 /* 311 * Set the filter on a watch queue. 312 */ 313 long watch_queue_set_filter(struct pipe_inode_info *pipe, 314 struct watch_notification_filter __user *_filter) 315 { 316 struct watch_notification_type_filter *tf; 317 struct watch_notification_filter filter; 318 struct watch_type_filter *q; 319 struct watch_filter *wfilter; 320 struct watch_queue *wqueue = pipe->watch_queue; 321 int ret, nr_filter = 0, i; 322 323 if (!wqueue) 324 return -ENODEV; 325 326 if (!_filter) { 327 /* Remove the old filter */ 328 wfilter = NULL; 329 goto set; 330 } 331 332 /* Grab the user's filter specification */ 333 if (copy_from_user(&filter, _filter, sizeof(filter)) != 0) 334 return -EFAULT; 335 if (filter.nr_filters == 0 || 336 filter.nr_filters > 16 || 337 filter.__reserved != 0) 338 return -EINVAL; 339 340 tf = memdup_user(_filter->filters, filter.nr_filters * sizeof(*tf)); 341 if (IS_ERR(tf)) 342 return PTR_ERR(tf); 343 344 ret = -EINVAL; 345 for (i = 0; i < filter.nr_filters; i++) { 346 if ((tf[i].info_filter & ~tf[i].info_mask) || 347 tf[i].info_mask & WATCH_INFO_LENGTH) 348 goto err_filter; 349 /* Ignore any unknown types */ 350 if (tf[i].type >= WATCH_TYPE__NR) 351 continue; 352 nr_filter++; 353 } 354 355 /* Now we need to build the internal filter from only the relevant 356 * user-specified filters. 357 */ 358 ret = -ENOMEM; 359 wfilter = kzalloc(struct_size(wfilter, filters, nr_filter), GFP_KERNEL); 360 if (!wfilter) 361 goto err_filter; 362 wfilter->nr_filters = nr_filter; 363 364 q = wfilter->filters; 365 for (i = 0; i < filter.nr_filters; i++) { 366 if (tf[i].type >= WATCH_TYPE__NR) 367 continue; 368 369 q->type = tf[i].type; 370 q->info_filter = tf[i].info_filter; 371 q->info_mask = tf[i].info_mask; 372 q->subtype_filter[0] = tf[i].subtype_filter[0]; 373 __set_bit(q->type, wfilter->type_filter); 374 q++; 375 } 376 377 kfree(tf); 378 set: 379 pipe_lock(pipe); 380 wfilter = rcu_replace_pointer(wqueue->filter, wfilter, 381 lockdep_is_held(&pipe->mutex)); 382 pipe_unlock(pipe); 383 if (wfilter) 384 kfree_rcu(wfilter, rcu); 385 return 0; 386 387 err_filter: 388 kfree(tf); 389 return ret; 390 } 391 392 static void __put_watch_queue(struct kref *kref) 393 { 394 struct watch_queue *wqueue = 395 container_of(kref, struct watch_queue, usage); 396 struct watch_filter *wfilter; 397 int i; 398 399 for (i = 0; i < wqueue->nr_pages; i++) 400 __free_page(wqueue->notes[i]); 401 kfree(wqueue->notes); 402 bitmap_free(wqueue->notes_bitmap); 403 404 wfilter = rcu_access_pointer(wqueue->filter); 405 if (wfilter) 406 kfree_rcu(wfilter, rcu); 407 kfree_rcu(wqueue, rcu); 408 } 409 410 /** 411 * put_watch_queue - Dispose of a ref on a watchqueue. 412 * @wqueue: The watch queue to unref. 413 */ 414 void put_watch_queue(struct watch_queue *wqueue) 415 { 416 kref_put(&wqueue->usage, __put_watch_queue); 417 } 418 EXPORT_SYMBOL(put_watch_queue); 419 420 static void free_watch(struct rcu_head *rcu) 421 { 422 struct watch *watch = container_of(rcu, struct watch, rcu); 423 424 put_watch_queue(rcu_access_pointer(watch->queue)); 425 atomic_dec(&watch->cred->user->nr_watches); 426 put_cred(watch->cred); 427 kfree(watch); 428 } 429 430 static void __put_watch(struct kref *kref) 431 { 432 struct watch *watch = container_of(kref, struct watch, usage); 433 434 call_rcu(&watch->rcu, free_watch); 435 } 436 437 /* 438 * Discard a watch. 439 */ 440 static void put_watch(struct watch *watch) 441 { 442 kref_put(&watch->usage, __put_watch); 443 } 444 445 /** 446 * init_watch - Initialise a watch 447 * @watch: The watch to initialise. 448 * @wqueue: The queue to assign. 449 * 450 * Initialise a watch and set the watch queue. 451 */ 452 void init_watch(struct watch *watch, struct watch_queue *wqueue) 453 { 454 kref_init(&watch->usage); 455 INIT_HLIST_NODE(&watch->list_node); 456 INIT_HLIST_NODE(&watch->queue_node); 457 rcu_assign_pointer(watch->queue, wqueue); 458 } 459 460 /** 461 * add_watch_to_object - Add a watch on an object to a watch list 462 * @watch: The watch to add 463 * @wlist: The watch list to add to 464 * 465 * @watch->queue must have been set to point to the queue to post notifications 466 * to and the watch list of the object to be watched. @watch->cred must also 467 * have been set to the appropriate credentials and a ref taken on them. 468 * 469 * The caller must pin the queue and the list both and must hold the list 470 * locked against racing watch additions/removals. 471 */ 472 int add_watch_to_object(struct watch *watch, struct watch_list *wlist) 473 { 474 struct watch_queue *wqueue = rcu_access_pointer(watch->queue); 475 struct watch *w; 476 477 hlist_for_each_entry(w, &wlist->watchers, list_node) { 478 struct watch_queue *wq = rcu_access_pointer(w->queue); 479 if (wqueue == wq && watch->id == w->id) 480 return -EBUSY; 481 } 482 483 watch->cred = get_current_cred(); 484 rcu_assign_pointer(watch->watch_list, wlist); 485 486 if (atomic_inc_return(&watch->cred->user->nr_watches) > 487 task_rlimit(current, RLIMIT_NOFILE)) { 488 atomic_dec(&watch->cred->user->nr_watches); 489 put_cred(watch->cred); 490 return -EAGAIN; 491 } 492 493 if (lock_wqueue(wqueue)) { 494 kref_get(&wqueue->usage); 495 kref_get(&watch->usage); 496 hlist_add_head(&watch->queue_node, &wqueue->watches); 497 unlock_wqueue(wqueue); 498 } 499 500 hlist_add_head_rcu(&watch->list_node, &wlist->watchers); 501 return 0; 502 } 503 EXPORT_SYMBOL(add_watch_to_object); 504 505 /** 506 * remove_watch_from_object - Remove a watch or all watches from an object. 507 * @wlist: The watch list to remove from 508 * @wq: The watch queue of interest (ignored if @all is true) 509 * @id: The ID of the watch to remove (ignored if @all is true) 510 * @all: True to remove all objects 511 * 512 * Remove a specific watch or all watches from an object. A notification is 513 * sent to the watcher to tell them that this happened. 514 */ 515 int remove_watch_from_object(struct watch_list *wlist, struct watch_queue *wq, 516 u64 id, bool all) 517 { 518 struct watch_notification_removal n; 519 struct watch_queue *wqueue; 520 struct watch *watch; 521 int ret = -EBADSLT; 522 523 rcu_read_lock(); 524 525 again: 526 spin_lock(&wlist->lock); 527 hlist_for_each_entry(watch, &wlist->watchers, list_node) { 528 if (all || 529 (watch->id == id && rcu_access_pointer(watch->queue) == wq)) 530 goto found; 531 } 532 spin_unlock(&wlist->lock); 533 goto out; 534 535 found: 536 ret = 0; 537 hlist_del_init_rcu(&watch->list_node); 538 rcu_assign_pointer(watch->watch_list, NULL); 539 spin_unlock(&wlist->lock); 540 541 /* We now own the reference on watch that used to belong to wlist. */ 542 543 n.watch.type = WATCH_TYPE_META; 544 n.watch.subtype = WATCH_META_REMOVAL_NOTIFICATION; 545 n.watch.info = watch->info_id | watch_sizeof(n.watch); 546 n.id = id; 547 if (id != 0) 548 n.watch.info = watch->info_id | watch_sizeof(n); 549 550 wqueue = rcu_dereference(watch->queue); 551 552 if (lock_wqueue(wqueue)) { 553 post_one_notification(wqueue, &n.watch); 554 555 if (!hlist_unhashed(&watch->queue_node)) { 556 hlist_del_init_rcu(&watch->queue_node); 557 put_watch(watch); 558 } 559 560 unlock_wqueue(wqueue); 561 } 562 563 if (wlist->release_watch) { 564 void (*release_watch)(struct watch *); 565 566 release_watch = wlist->release_watch; 567 rcu_read_unlock(); 568 (*release_watch)(watch); 569 rcu_read_lock(); 570 } 571 put_watch(watch); 572 573 if (all && !hlist_empty(&wlist->watchers)) 574 goto again; 575 out: 576 rcu_read_unlock(); 577 return ret; 578 } 579 EXPORT_SYMBOL(remove_watch_from_object); 580 581 /* 582 * Remove all the watches that are contributory to a queue. This has the 583 * potential to race with removal of the watches by the destruction of the 584 * objects being watched or with the distribution of notifications. 585 */ 586 void watch_queue_clear(struct watch_queue *wqueue) 587 { 588 struct watch_list *wlist; 589 struct watch *watch; 590 bool release; 591 592 rcu_read_lock(); 593 spin_lock_bh(&wqueue->lock); 594 595 /* Prevent new notifications from being stored. */ 596 wqueue->defunct = true; 597 598 while (!hlist_empty(&wqueue->watches)) { 599 watch = hlist_entry(wqueue->watches.first, struct watch, queue_node); 600 hlist_del_init_rcu(&watch->queue_node); 601 /* We now own a ref on the watch. */ 602 spin_unlock_bh(&wqueue->lock); 603 604 /* We can't do the next bit under the queue lock as we need to 605 * get the list lock - which would cause a deadlock if someone 606 * was removing from the opposite direction at the same time or 607 * posting a notification. 608 */ 609 wlist = rcu_dereference(watch->watch_list); 610 if (wlist) { 611 void (*release_watch)(struct watch *); 612 613 spin_lock(&wlist->lock); 614 615 release = !hlist_unhashed(&watch->list_node); 616 if (release) { 617 hlist_del_init_rcu(&watch->list_node); 618 rcu_assign_pointer(watch->watch_list, NULL); 619 620 /* We now own a second ref on the watch. */ 621 } 622 623 release_watch = wlist->release_watch; 624 spin_unlock(&wlist->lock); 625 626 if (release) { 627 if (release_watch) { 628 rcu_read_unlock(); 629 /* This might need to call dput(), so 630 * we have to drop all the locks. 631 */ 632 (*release_watch)(watch); 633 rcu_read_lock(); 634 } 635 put_watch(watch); 636 } 637 } 638 639 put_watch(watch); 640 spin_lock_bh(&wqueue->lock); 641 } 642 643 spin_unlock_bh(&wqueue->lock); 644 rcu_read_unlock(); 645 } 646 647 /** 648 * get_watch_queue - Get a watch queue from its file descriptor. 649 * @fd: The fd to query. 650 */ 651 struct watch_queue *get_watch_queue(int fd) 652 { 653 struct pipe_inode_info *pipe; 654 struct watch_queue *wqueue = ERR_PTR(-EINVAL); 655 struct fd f; 656 657 f = fdget(fd); 658 if (f.file) { 659 pipe = get_pipe_info(f.file, false); 660 if (pipe && pipe->watch_queue) { 661 wqueue = pipe->watch_queue; 662 kref_get(&wqueue->usage); 663 } 664 fdput(f); 665 } 666 667 return wqueue; 668 } 669 EXPORT_SYMBOL(get_watch_queue); 670 671 /* 672 * Initialise a watch queue 673 */ 674 int watch_queue_init(struct pipe_inode_info *pipe) 675 { 676 struct watch_queue *wqueue; 677 678 wqueue = kzalloc(sizeof(*wqueue), GFP_KERNEL); 679 if (!wqueue) 680 return -ENOMEM; 681 682 wqueue->pipe = pipe; 683 kref_init(&wqueue->usage); 684 spin_lock_init(&wqueue->lock); 685 INIT_HLIST_HEAD(&wqueue->watches); 686 687 pipe->watch_queue = wqueue; 688 return 0; 689 } 690