1 /* 2 * An async IO implementation for Linux 3 * Written by Benjamin LaHaise <bcrl@kvack.org> 4 * 5 * Implements an efficient asynchronous io interface. 6 * 7 * Copyright 2000, 2001, 2002 Red Hat, Inc. All Rights Reserved. 8 * Copyright 2018 Christoph Hellwig. 9 * 10 * See ../COPYING for licensing terms. 11 */ 12 #define pr_fmt(fmt) "%s: " fmt, __func__ 13 14 #include <linux/kernel.h> 15 #include <linux/init.h> 16 #include <linux/errno.h> 17 #include <linux/time.h> 18 #include <linux/aio_abi.h> 19 #include <linux/export.h> 20 #include <linux/syscalls.h> 21 #include <linux/backing-dev.h> 22 #include <linux/refcount.h> 23 #include <linux/uio.h> 24 25 #include <linux/sched/signal.h> 26 #include <linux/fs.h> 27 #include <linux/file.h> 28 #include <linux/mm.h> 29 #include <linux/mman.h> 30 #include <linux/mmu_context.h> 31 #include <linux/percpu.h> 32 #include <linux/slab.h> 33 #include <linux/timer.h> 34 #include <linux/aio.h> 35 #include <linux/highmem.h> 36 #include <linux/workqueue.h> 37 #include <linux/security.h> 38 #include <linux/eventfd.h> 39 #include <linux/blkdev.h> 40 #include <linux/compat.h> 41 #include <linux/migrate.h> 42 #include <linux/ramfs.h> 43 #include <linux/percpu-refcount.h> 44 #include <linux/mount.h> 45 46 #include <asm/kmap_types.h> 47 #include <linux/uaccess.h> 48 #include <linux/nospec.h> 49 50 #include "internal.h" 51 52 #define KIOCB_KEY 0 53 54 #define AIO_RING_MAGIC 0xa10a10a1 55 #define AIO_RING_COMPAT_FEATURES 1 56 #define AIO_RING_INCOMPAT_FEATURES 0 57 struct aio_ring { 58 unsigned id; /* kernel internal index number */ 59 unsigned nr; /* number of io_events */ 60 unsigned head; /* Written to by userland or under ring_lock 61 * mutex by aio_read_events_ring(). */ 62 unsigned tail; 63 64 unsigned magic; 65 unsigned compat_features; 66 unsigned incompat_features; 67 unsigned header_length; /* size of aio_ring */ 68 69 70 struct io_event io_events[0]; 71 }; /* 128 bytes + ring size */ 72 73 #define AIO_RING_PAGES 8 74 75 struct kioctx_table { 76 struct rcu_head rcu; 77 unsigned nr; 78 struct kioctx __rcu *table[]; 79 }; 80 81 struct kioctx_cpu { 82 unsigned reqs_available; 83 }; 84 85 struct ctx_rq_wait { 86 struct completion comp; 87 atomic_t count; 88 }; 89 90 struct kioctx { 91 struct percpu_ref users; 92 atomic_t dead; 93 94 struct percpu_ref reqs; 95 96 unsigned long user_id; 97 98 struct __percpu kioctx_cpu *cpu; 99 100 /* 101 * For percpu reqs_available, number of slots we move to/from global 102 * counter at a time: 103 */ 104 unsigned req_batch; 105 /* 106 * This is what userspace passed to io_setup(), it's not used for 107 * anything but counting against the global max_reqs quota. 108 * 109 * The real limit is nr_events - 1, which will be larger (see 110 * aio_setup_ring()) 111 */ 112 unsigned max_reqs; 113 114 /* Size of ringbuffer, in units of struct io_event */ 115 unsigned nr_events; 116 117 unsigned long mmap_base; 118 unsigned long mmap_size; 119 120 struct page **ring_pages; 121 long nr_pages; 122 123 struct rcu_work free_rwork; /* see free_ioctx() */ 124 125 /* 126 * signals when all in-flight requests are done 127 */ 128 struct ctx_rq_wait *rq_wait; 129 130 struct { 131 /* 132 * This counts the number of available slots in the ringbuffer, 133 * so we avoid overflowing it: it's decremented (if positive) 134 * when allocating a kiocb and incremented when the resulting 135 * io_event is pulled off the ringbuffer. 136 * 137 * We batch accesses to it with a percpu version. 138 */ 139 atomic_t reqs_available; 140 } ____cacheline_aligned_in_smp; 141 142 struct { 143 spinlock_t ctx_lock; 144 struct list_head active_reqs; /* used for cancellation */ 145 } ____cacheline_aligned_in_smp; 146 147 struct { 148 struct mutex ring_lock; 149 wait_queue_head_t wait; 150 } ____cacheline_aligned_in_smp; 151 152 struct { 153 unsigned tail; 154 unsigned completed_events; 155 spinlock_t completion_lock; 156 } ____cacheline_aligned_in_smp; 157 158 struct page *internal_pages[AIO_RING_PAGES]; 159 struct file *aio_ring_file; 160 161 unsigned id; 162 }; 163 164 struct fsync_iocb { 165 struct work_struct work; 166 struct file *file; 167 bool datasync; 168 }; 169 170 struct poll_iocb { 171 struct file *file; 172 struct wait_queue_head *head; 173 __poll_t events; 174 bool woken; 175 bool cancelled; 176 struct wait_queue_entry wait; 177 struct work_struct work; 178 }; 179 180 struct aio_kiocb { 181 union { 182 struct kiocb rw; 183 struct fsync_iocb fsync; 184 struct poll_iocb poll; 185 }; 186 187 struct kioctx *ki_ctx; 188 kiocb_cancel_fn *ki_cancel; 189 190 struct iocb __user *ki_user_iocb; /* user's aiocb */ 191 __u64 ki_user_data; /* user's data for completion */ 192 193 struct list_head ki_list; /* the aio core uses this 194 * for cancellation */ 195 refcount_t ki_refcnt; 196 197 /* 198 * If the aio_resfd field of the userspace iocb is not zero, 199 * this is the underlying eventfd context to deliver events to. 200 */ 201 struct eventfd_ctx *ki_eventfd; 202 }; 203 204 /*------ sysctl variables----*/ 205 static DEFINE_SPINLOCK(aio_nr_lock); 206 unsigned long aio_nr; /* current system wide number of aio requests */ 207 unsigned long aio_max_nr = 0x10000; /* system wide maximum number of aio requests */ 208 /*----end sysctl variables---*/ 209 210 static struct kmem_cache *kiocb_cachep; 211 static struct kmem_cache *kioctx_cachep; 212 213 static struct vfsmount *aio_mnt; 214 215 static const struct file_operations aio_ring_fops; 216 static const struct address_space_operations aio_ctx_aops; 217 218 static struct file *aio_private_file(struct kioctx *ctx, loff_t nr_pages) 219 { 220 struct file *file; 221 struct inode *inode = alloc_anon_inode(aio_mnt->mnt_sb); 222 if (IS_ERR(inode)) 223 return ERR_CAST(inode); 224 225 inode->i_mapping->a_ops = &aio_ctx_aops; 226 inode->i_mapping->private_data = ctx; 227 inode->i_size = PAGE_SIZE * nr_pages; 228 229 file = alloc_file_pseudo(inode, aio_mnt, "[aio]", 230 O_RDWR, &aio_ring_fops); 231 if (IS_ERR(file)) 232 iput(inode); 233 return file; 234 } 235 236 static struct dentry *aio_mount(struct file_system_type *fs_type, 237 int flags, const char *dev_name, void *data) 238 { 239 struct dentry *root = mount_pseudo(fs_type, "aio:", NULL, NULL, 240 AIO_RING_MAGIC); 241 242 if (!IS_ERR(root)) 243 root->d_sb->s_iflags |= SB_I_NOEXEC; 244 return root; 245 } 246 247 /* aio_setup 248 * Creates the slab caches used by the aio routines, panic on 249 * failure as this is done early during the boot sequence. 250 */ 251 static int __init aio_setup(void) 252 { 253 static struct file_system_type aio_fs = { 254 .name = "aio", 255 .mount = aio_mount, 256 .kill_sb = kill_anon_super, 257 }; 258 aio_mnt = kern_mount(&aio_fs); 259 if (IS_ERR(aio_mnt)) 260 panic("Failed to create aio fs mount."); 261 262 kiocb_cachep = KMEM_CACHE(aio_kiocb, SLAB_HWCACHE_ALIGN|SLAB_PANIC); 263 kioctx_cachep = KMEM_CACHE(kioctx,SLAB_HWCACHE_ALIGN|SLAB_PANIC); 264 return 0; 265 } 266 __initcall(aio_setup); 267 268 static void put_aio_ring_file(struct kioctx *ctx) 269 { 270 struct file *aio_ring_file = ctx->aio_ring_file; 271 struct address_space *i_mapping; 272 273 if (aio_ring_file) { 274 truncate_setsize(file_inode(aio_ring_file), 0); 275 276 /* Prevent further access to the kioctx from migratepages */ 277 i_mapping = aio_ring_file->f_mapping; 278 spin_lock(&i_mapping->private_lock); 279 i_mapping->private_data = NULL; 280 ctx->aio_ring_file = NULL; 281 spin_unlock(&i_mapping->private_lock); 282 283 fput(aio_ring_file); 284 } 285 } 286 287 static void aio_free_ring(struct kioctx *ctx) 288 { 289 int i; 290 291 /* Disconnect the kiotx from the ring file. This prevents future 292 * accesses to the kioctx from page migration. 293 */ 294 put_aio_ring_file(ctx); 295 296 for (i = 0; i < ctx->nr_pages; i++) { 297 struct page *page; 298 pr_debug("pid(%d) [%d] page->count=%d\n", current->pid, i, 299 page_count(ctx->ring_pages[i])); 300 page = ctx->ring_pages[i]; 301 if (!page) 302 continue; 303 ctx->ring_pages[i] = NULL; 304 put_page(page); 305 } 306 307 if (ctx->ring_pages && ctx->ring_pages != ctx->internal_pages) { 308 kfree(ctx->ring_pages); 309 ctx->ring_pages = NULL; 310 } 311 } 312 313 static int aio_ring_mremap(struct vm_area_struct *vma) 314 { 315 struct file *file = vma->vm_file; 316 struct mm_struct *mm = vma->vm_mm; 317 struct kioctx_table *table; 318 int i, res = -EINVAL; 319 320 spin_lock(&mm->ioctx_lock); 321 rcu_read_lock(); 322 table = rcu_dereference(mm->ioctx_table); 323 for (i = 0; i < table->nr; i++) { 324 struct kioctx *ctx; 325 326 ctx = rcu_dereference(table->table[i]); 327 if (ctx && ctx->aio_ring_file == file) { 328 if (!atomic_read(&ctx->dead)) { 329 ctx->user_id = ctx->mmap_base = vma->vm_start; 330 res = 0; 331 } 332 break; 333 } 334 } 335 336 rcu_read_unlock(); 337 spin_unlock(&mm->ioctx_lock); 338 return res; 339 } 340 341 static const struct vm_operations_struct aio_ring_vm_ops = { 342 .mremap = aio_ring_mremap, 343 #if IS_ENABLED(CONFIG_MMU) 344 .fault = filemap_fault, 345 .map_pages = filemap_map_pages, 346 .page_mkwrite = filemap_page_mkwrite, 347 #endif 348 }; 349 350 static int aio_ring_mmap(struct file *file, struct vm_area_struct *vma) 351 { 352 vma->vm_flags |= VM_DONTEXPAND; 353 vma->vm_ops = &aio_ring_vm_ops; 354 return 0; 355 } 356 357 static const struct file_operations aio_ring_fops = { 358 .mmap = aio_ring_mmap, 359 }; 360 361 #if IS_ENABLED(CONFIG_MIGRATION) 362 static int aio_migratepage(struct address_space *mapping, struct page *new, 363 struct page *old, enum migrate_mode mode) 364 { 365 struct kioctx *ctx; 366 unsigned long flags; 367 pgoff_t idx; 368 int rc; 369 370 /* 371 * We cannot support the _NO_COPY case here, because copy needs to 372 * happen under the ctx->completion_lock. That does not work with the 373 * migration workflow of MIGRATE_SYNC_NO_COPY. 374 */ 375 if (mode == MIGRATE_SYNC_NO_COPY) 376 return -EINVAL; 377 378 rc = 0; 379 380 /* mapping->private_lock here protects against the kioctx teardown. */ 381 spin_lock(&mapping->private_lock); 382 ctx = mapping->private_data; 383 if (!ctx) { 384 rc = -EINVAL; 385 goto out; 386 } 387 388 /* The ring_lock mutex. The prevents aio_read_events() from writing 389 * to the ring's head, and prevents page migration from mucking in 390 * a partially initialized kiotx. 391 */ 392 if (!mutex_trylock(&ctx->ring_lock)) { 393 rc = -EAGAIN; 394 goto out; 395 } 396 397 idx = old->index; 398 if (idx < (pgoff_t)ctx->nr_pages) { 399 /* Make sure the old page hasn't already been changed */ 400 if (ctx->ring_pages[idx] != old) 401 rc = -EAGAIN; 402 } else 403 rc = -EINVAL; 404 405 if (rc != 0) 406 goto out_unlock; 407 408 /* Writeback must be complete */ 409 BUG_ON(PageWriteback(old)); 410 get_page(new); 411 412 rc = migrate_page_move_mapping(mapping, new, old, NULL, mode, 1); 413 if (rc != MIGRATEPAGE_SUCCESS) { 414 put_page(new); 415 goto out_unlock; 416 } 417 418 /* Take completion_lock to prevent other writes to the ring buffer 419 * while the old page is copied to the new. This prevents new 420 * events from being lost. 421 */ 422 spin_lock_irqsave(&ctx->completion_lock, flags); 423 migrate_page_copy(new, old); 424 BUG_ON(ctx->ring_pages[idx] != old); 425 ctx->ring_pages[idx] = new; 426 spin_unlock_irqrestore(&ctx->completion_lock, flags); 427 428 /* The old page is no longer accessible. */ 429 put_page(old); 430 431 out_unlock: 432 mutex_unlock(&ctx->ring_lock); 433 out: 434 spin_unlock(&mapping->private_lock); 435 return rc; 436 } 437 #endif 438 439 static const struct address_space_operations aio_ctx_aops = { 440 .set_page_dirty = __set_page_dirty_no_writeback, 441 #if IS_ENABLED(CONFIG_MIGRATION) 442 .migratepage = aio_migratepage, 443 #endif 444 }; 445 446 static int aio_setup_ring(struct kioctx *ctx, unsigned int nr_events) 447 { 448 struct aio_ring *ring; 449 struct mm_struct *mm = current->mm; 450 unsigned long size, unused; 451 int nr_pages; 452 int i; 453 struct file *file; 454 455 /* Compensate for the ring buffer's head/tail overlap entry */ 456 nr_events += 2; /* 1 is required, 2 for good luck */ 457 458 size = sizeof(struct aio_ring); 459 size += sizeof(struct io_event) * nr_events; 460 461 nr_pages = PFN_UP(size); 462 if (nr_pages < 0) 463 return -EINVAL; 464 465 file = aio_private_file(ctx, nr_pages); 466 if (IS_ERR(file)) { 467 ctx->aio_ring_file = NULL; 468 return -ENOMEM; 469 } 470 471 ctx->aio_ring_file = file; 472 nr_events = (PAGE_SIZE * nr_pages - sizeof(struct aio_ring)) 473 / sizeof(struct io_event); 474 475 ctx->ring_pages = ctx->internal_pages; 476 if (nr_pages > AIO_RING_PAGES) { 477 ctx->ring_pages = kcalloc(nr_pages, sizeof(struct page *), 478 GFP_KERNEL); 479 if (!ctx->ring_pages) { 480 put_aio_ring_file(ctx); 481 return -ENOMEM; 482 } 483 } 484 485 for (i = 0; i < nr_pages; i++) { 486 struct page *page; 487 page = find_or_create_page(file->f_mapping, 488 i, GFP_HIGHUSER | __GFP_ZERO); 489 if (!page) 490 break; 491 pr_debug("pid(%d) page[%d]->count=%d\n", 492 current->pid, i, page_count(page)); 493 SetPageUptodate(page); 494 unlock_page(page); 495 496 ctx->ring_pages[i] = page; 497 } 498 ctx->nr_pages = i; 499 500 if (unlikely(i != nr_pages)) { 501 aio_free_ring(ctx); 502 return -ENOMEM; 503 } 504 505 ctx->mmap_size = nr_pages * PAGE_SIZE; 506 pr_debug("attempting mmap of %lu bytes\n", ctx->mmap_size); 507 508 if (down_write_killable(&mm->mmap_sem)) { 509 ctx->mmap_size = 0; 510 aio_free_ring(ctx); 511 return -EINTR; 512 } 513 514 ctx->mmap_base = do_mmap_pgoff(ctx->aio_ring_file, 0, ctx->mmap_size, 515 PROT_READ | PROT_WRITE, 516 MAP_SHARED, 0, &unused, NULL); 517 up_write(&mm->mmap_sem); 518 if (IS_ERR((void *)ctx->mmap_base)) { 519 ctx->mmap_size = 0; 520 aio_free_ring(ctx); 521 return -ENOMEM; 522 } 523 524 pr_debug("mmap address: 0x%08lx\n", ctx->mmap_base); 525 526 ctx->user_id = ctx->mmap_base; 527 ctx->nr_events = nr_events; /* trusted copy */ 528 529 ring = kmap_atomic(ctx->ring_pages[0]); 530 ring->nr = nr_events; /* user copy */ 531 ring->id = ~0U; 532 ring->head = ring->tail = 0; 533 ring->magic = AIO_RING_MAGIC; 534 ring->compat_features = AIO_RING_COMPAT_FEATURES; 535 ring->incompat_features = AIO_RING_INCOMPAT_FEATURES; 536 ring->header_length = sizeof(struct aio_ring); 537 kunmap_atomic(ring); 538 flush_dcache_page(ctx->ring_pages[0]); 539 540 return 0; 541 } 542 543 #define AIO_EVENTS_PER_PAGE (PAGE_SIZE / sizeof(struct io_event)) 544 #define AIO_EVENTS_FIRST_PAGE ((PAGE_SIZE - sizeof(struct aio_ring)) / sizeof(struct io_event)) 545 #define AIO_EVENTS_OFFSET (AIO_EVENTS_PER_PAGE - AIO_EVENTS_FIRST_PAGE) 546 547 void kiocb_set_cancel_fn(struct kiocb *iocb, kiocb_cancel_fn *cancel) 548 { 549 struct aio_kiocb *req = container_of(iocb, struct aio_kiocb, rw); 550 struct kioctx *ctx = req->ki_ctx; 551 unsigned long flags; 552 553 if (WARN_ON_ONCE(!list_empty(&req->ki_list))) 554 return; 555 556 spin_lock_irqsave(&ctx->ctx_lock, flags); 557 list_add_tail(&req->ki_list, &ctx->active_reqs); 558 req->ki_cancel = cancel; 559 spin_unlock_irqrestore(&ctx->ctx_lock, flags); 560 } 561 EXPORT_SYMBOL(kiocb_set_cancel_fn); 562 563 /* 564 * free_ioctx() should be RCU delayed to synchronize against the RCU 565 * protected lookup_ioctx() and also needs process context to call 566 * aio_free_ring(). Use rcu_work. 567 */ 568 static void free_ioctx(struct work_struct *work) 569 { 570 struct kioctx *ctx = container_of(to_rcu_work(work), struct kioctx, 571 free_rwork); 572 pr_debug("freeing %p\n", ctx); 573 574 aio_free_ring(ctx); 575 free_percpu(ctx->cpu); 576 percpu_ref_exit(&ctx->reqs); 577 percpu_ref_exit(&ctx->users); 578 kmem_cache_free(kioctx_cachep, ctx); 579 } 580 581 static void free_ioctx_reqs(struct percpu_ref *ref) 582 { 583 struct kioctx *ctx = container_of(ref, struct kioctx, reqs); 584 585 /* At this point we know that there are no any in-flight requests */ 586 if (ctx->rq_wait && atomic_dec_and_test(&ctx->rq_wait->count)) 587 complete(&ctx->rq_wait->comp); 588 589 /* Synchronize against RCU protected table->table[] dereferences */ 590 INIT_RCU_WORK(&ctx->free_rwork, free_ioctx); 591 queue_rcu_work(system_wq, &ctx->free_rwork); 592 } 593 594 /* 595 * When this function runs, the kioctx has been removed from the "hash table" 596 * and ctx->users has dropped to 0, so we know no more kiocbs can be submitted - 597 * now it's safe to cancel any that need to be. 598 */ 599 static void free_ioctx_users(struct percpu_ref *ref) 600 { 601 struct kioctx *ctx = container_of(ref, struct kioctx, users); 602 struct aio_kiocb *req; 603 604 spin_lock_irq(&ctx->ctx_lock); 605 606 while (!list_empty(&ctx->active_reqs)) { 607 req = list_first_entry(&ctx->active_reqs, 608 struct aio_kiocb, ki_list); 609 req->ki_cancel(&req->rw); 610 list_del_init(&req->ki_list); 611 } 612 613 spin_unlock_irq(&ctx->ctx_lock); 614 615 percpu_ref_kill(&ctx->reqs); 616 percpu_ref_put(&ctx->reqs); 617 } 618 619 static int ioctx_add_table(struct kioctx *ctx, struct mm_struct *mm) 620 { 621 unsigned i, new_nr; 622 struct kioctx_table *table, *old; 623 struct aio_ring *ring; 624 625 spin_lock(&mm->ioctx_lock); 626 table = rcu_dereference_raw(mm->ioctx_table); 627 628 while (1) { 629 if (table) 630 for (i = 0; i < table->nr; i++) 631 if (!rcu_access_pointer(table->table[i])) { 632 ctx->id = i; 633 rcu_assign_pointer(table->table[i], ctx); 634 spin_unlock(&mm->ioctx_lock); 635 636 /* While kioctx setup is in progress, 637 * we are protected from page migration 638 * changes ring_pages by ->ring_lock. 639 */ 640 ring = kmap_atomic(ctx->ring_pages[0]); 641 ring->id = ctx->id; 642 kunmap_atomic(ring); 643 return 0; 644 } 645 646 new_nr = (table ? table->nr : 1) * 4; 647 spin_unlock(&mm->ioctx_lock); 648 649 table = kzalloc(sizeof(*table) + sizeof(struct kioctx *) * 650 new_nr, GFP_KERNEL); 651 if (!table) 652 return -ENOMEM; 653 654 table->nr = new_nr; 655 656 spin_lock(&mm->ioctx_lock); 657 old = rcu_dereference_raw(mm->ioctx_table); 658 659 if (!old) { 660 rcu_assign_pointer(mm->ioctx_table, table); 661 } else if (table->nr > old->nr) { 662 memcpy(table->table, old->table, 663 old->nr * sizeof(struct kioctx *)); 664 665 rcu_assign_pointer(mm->ioctx_table, table); 666 kfree_rcu(old, rcu); 667 } else { 668 kfree(table); 669 table = old; 670 } 671 } 672 } 673 674 static void aio_nr_sub(unsigned nr) 675 { 676 spin_lock(&aio_nr_lock); 677 if (WARN_ON(aio_nr - nr > aio_nr)) 678 aio_nr = 0; 679 else 680 aio_nr -= nr; 681 spin_unlock(&aio_nr_lock); 682 } 683 684 /* ioctx_alloc 685 * Allocates and initializes an ioctx. Returns an ERR_PTR if it failed. 686 */ 687 static struct kioctx *ioctx_alloc(unsigned nr_events) 688 { 689 struct mm_struct *mm = current->mm; 690 struct kioctx *ctx; 691 int err = -ENOMEM; 692 693 /* 694 * Store the original nr_events -- what userspace passed to io_setup(), 695 * for counting against the global limit -- before it changes. 696 */ 697 unsigned int max_reqs = nr_events; 698 699 /* 700 * We keep track of the number of available ringbuffer slots, to prevent 701 * overflow (reqs_available), and we also use percpu counters for this. 702 * 703 * So since up to half the slots might be on other cpu's percpu counters 704 * and unavailable, double nr_events so userspace sees what they 705 * expected: additionally, we move req_batch slots to/from percpu 706 * counters at a time, so make sure that isn't 0: 707 */ 708 nr_events = max(nr_events, num_possible_cpus() * 4); 709 nr_events *= 2; 710 711 /* Prevent overflows */ 712 if (nr_events > (0x10000000U / sizeof(struct io_event))) { 713 pr_debug("ENOMEM: nr_events too high\n"); 714 return ERR_PTR(-EINVAL); 715 } 716 717 if (!nr_events || (unsigned long)max_reqs > aio_max_nr) 718 return ERR_PTR(-EAGAIN); 719 720 ctx = kmem_cache_zalloc(kioctx_cachep, GFP_KERNEL); 721 if (!ctx) 722 return ERR_PTR(-ENOMEM); 723 724 ctx->max_reqs = max_reqs; 725 726 spin_lock_init(&ctx->ctx_lock); 727 spin_lock_init(&ctx->completion_lock); 728 mutex_init(&ctx->ring_lock); 729 /* Protect against page migration throughout kiotx setup by keeping 730 * the ring_lock mutex held until setup is complete. */ 731 mutex_lock(&ctx->ring_lock); 732 init_waitqueue_head(&ctx->wait); 733 734 INIT_LIST_HEAD(&ctx->active_reqs); 735 736 if (percpu_ref_init(&ctx->users, free_ioctx_users, 0, GFP_KERNEL)) 737 goto err; 738 739 if (percpu_ref_init(&ctx->reqs, free_ioctx_reqs, 0, GFP_KERNEL)) 740 goto err; 741 742 ctx->cpu = alloc_percpu(struct kioctx_cpu); 743 if (!ctx->cpu) 744 goto err; 745 746 err = aio_setup_ring(ctx, nr_events); 747 if (err < 0) 748 goto err; 749 750 atomic_set(&ctx->reqs_available, ctx->nr_events - 1); 751 ctx->req_batch = (ctx->nr_events - 1) / (num_possible_cpus() * 4); 752 if (ctx->req_batch < 1) 753 ctx->req_batch = 1; 754 755 /* limit the number of system wide aios */ 756 spin_lock(&aio_nr_lock); 757 if (aio_nr + ctx->max_reqs > aio_max_nr || 758 aio_nr + ctx->max_reqs < aio_nr) { 759 spin_unlock(&aio_nr_lock); 760 err = -EAGAIN; 761 goto err_ctx; 762 } 763 aio_nr += ctx->max_reqs; 764 spin_unlock(&aio_nr_lock); 765 766 percpu_ref_get(&ctx->users); /* io_setup() will drop this ref */ 767 percpu_ref_get(&ctx->reqs); /* free_ioctx_users() will drop this */ 768 769 err = ioctx_add_table(ctx, mm); 770 if (err) 771 goto err_cleanup; 772 773 /* Release the ring_lock mutex now that all setup is complete. */ 774 mutex_unlock(&ctx->ring_lock); 775 776 pr_debug("allocated ioctx %p[%ld]: mm=%p mask=0x%x\n", 777 ctx, ctx->user_id, mm, ctx->nr_events); 778 return ctx; 779 780 err_cleanup: 781 aio_nr_sub(ctx->max_reqs); 782 err_ctx: 783 atomic_set(&ctx->dead, 1); 784 if (ctx->mmap_size) 785 vm_munmap(ctx->mmap_base, ctx->mmap_size); 786 aio_free_ring(ctx); 787 err: 788 mutex_unlock(&ctx->ring_lock); 789 free_percpu(ctx->cpu); 790 percpu_ref_exit(&ctx->reqs); 791 percpu_ref_exit(&ctx->users); 792 kmem_cache_free(kioctx_cachep, ctx); 793 pr_debug("error allocating ioctx %d\n", err); 794 return ERR_PTR(err); 795 } 796 797 /* kill_ioctx 798 * Cancels all outstanding aio requests on an aio context. Used 799 * when the processes owning a context have all exited to encourage 800 * the rapid destruction of the kioctx. 801 */ 802 static int kill_ioctx(struct mm_struct *mm, struct kioctx *ctx, 803 struct ctx_rq_wait *wait) 804 { 805 struct kioctx_table *table; 806 807 spin_lock(&mm->ioctx_lock); 808 if (atomic_xchg(&ctx->dead, 1)) { 809 spin_unlock(&mm->ioctx_lock); 810 return -EINVAL; 811 } 812 813 table = rcu_dereference_raw(mm->ioctx_table); 814 WARN_ON(ctx != rcu_access_pointer(table->table[ctx->id])); 815 RCU_INIT_POINTER(table->table[ctx->id], NULL); 816 spin_unlock(&mm->ioctx_lock); 817 818 /* free_ioctx_reqs() will do the necessary RCU synchronization */ 819 wake_up_all(&ctx->wait); 820 821 /* 822 * It'd be more correct to do this in free_ioctx(), after all 823 * the outstanding kiocbs have finished - but by then io_destroy 824 * has already returned, so io_setup() could potentially return 825 * -EAGAIN with no ioctxs actually in use (as far as userspace 826 * could tell). 827 */ 828 aio_nr_sub(ctx->max_reqs); 829 830 if (ctx->mmap_size) 831 vm_munmap(ctx->mmap_base, ctx->mmap_size); 832 833 ctx->rq_wait = wait; 834 percpu_ref_kill(&ctx->users); 835 return 0; 836 } 837 838 /* 839 * exit_aio: called when the last user of mm goes away. At this point, there is 840 * no way for any new requests to be submited or any of the io_* syscalls to be 841 * called on the context. 842 * 843 * There may be outstanding kiocbs, but free_ioctx() will explicitly wait on 844 * them. 845 */ 846 void exit_aio(struct mm_struct *mm) 847 { 848 struct kioctx_table *table = rcu_dereference_raw(mm->ioctx_table); 849 struct ctx_rq_wait wait; 850 int i, skipped; 851 852 if (!table) 853 return; 854 855 atomic_set(&wait.count, table->nr); 856 init_completion(&wait.comp); 857 858 skipped = 0; 859 for (i = 0; i < table->nr; ++i) { 860 struct kioctx *ctx = 861 rcu_dereference_protected(table->table[i], true); 862 863 if (!ctx) { 864 skipped++; 865 continue; 866 } 867 868 /* 869 * We don't need to bother with munmap() here - exit_mmap(mm) 870 * is coming and it'll unmap everything. And we simply can't, 871 * this is not necessarily our ->mm. 872 * Since kill_ioctx() uses non-zero ->mmap_size as indicator 873 * that it needs to unmap the area, just set it to 0. 874 */ 875 ctx->mmap_size = 0; 876 kill_ioctx(mm, ctx, &wait); 877 } 878 879 if (!atomic_sub_and_test(skipped, &wait.count)) { 880 /* Wait until all IO for the context are done. */ 881 wait_for_completion(&wait.comp); 882 } 883 884 RCU_INIT_POINTER(mm->ioctx_table, NULL); 885 kfree(table); 886 } 887 888 static void put_reqs_available(struct kioctx *ctx, unsigned nr) 889 { 890 struct kioctx_cpu *kcpu; 891 unsigned long flags; 892 893 local_irq_save(flags); 894 kcpu = this_cpu_ptr(ctx->cpu); 895 kcpu->reqs_available += nr; 896 897 while (kcpu->reqs_available >= ctx->req_batch * 2) { 898 kcpu->reqs_available -= ctx->req_batch; 899 atomic_add(ctx->req_batch, &ctx->reqs_available); 900 } 901 902 local_irq_restore(flags); 903 } 904 905 static bool get_reqs_available(struct kioctx *ctx) 906 { 907 struct kioctx_cpu *kcpu; 908 bool ret = false; 909 unsigned long flags; 910 911 local_irq_save(flags); 912 kcpu = this_cpu_ptr(ctx->cpu); 913 if (!kcpu->reqs_available) { 914 int old, avail = atomic_read(&ctx->reqs_available); 915 916 do { 917 if (avail < ctx->req_batch) 918 goto out; 919 920 old = avail; 921 avail = atomic_cmpxchg(&ctx->reqs_available, 922 avail, avail - ctx->req_batch); 923 } while (avail != old); 924 925 kcpu->reqs_available += ctx->req_batch; 926 } 927 928 ret = true; 929 kcpu->reqs_available--; 930 out: 931 local_irq_restore(flags); 932 return ret; 933 } 934 935 /* refill_reqs_available 936 * Updates the reqs_available reference counts used for tracking the 937 * number of free slots in the completion ring. This can be called 938 * from aio_complete() (to optimistically update reqs_available) or 939 * from aio_get_req() (the we're out of events case). It must be 940 * called holding ctx->completion_lock. 941 */ 942 static void refill_reqs_available(struct kioctx *ctx, unsigned head, 943 unsigned tail) 944 { 945 unsigned events_in_ring, completed; 946 947 /* Clamp head since userland can write to it. */ 948 head %= ctx->nr_events; 949 if (head <= tail) 950 events_in_ring = tail - head; 951 else 952 events_in_ring = ctx->nr_events - (head - tail); 953 954 completed = ctx->completed_events; 955 if (events_in_ring < completed) 956 completed -= events_in_ring; 957 else 958 completed = 0; 959 960 if (!completed) 961 return; 962 963 ctx->completed_events -= completed; 964 put_reqs_available(ctx, completed); 965 } 966 967 /* user_refill_reqs_available 968 * Called to refill reqs_available when aio_get_req() encounters an 969 * out of space in the completion ring. 970 */ 971 static void user_refill_reqs_available(struct kioctx *ctx) 972 { 973 spin_lock_irq(&ctx->completion_lock); 974 if (ctx->completed_events) { 975 struct aio_ring *ring; 976 unsigned head; 977 978 /* Access of ring->head may race with aio_read_events_ring() 979 * here, but that's okay since whether we read the old version 980 * or the new version, and either will be valid. The important 981 * part is that head cannot pass tail since we prevent 982 * aio_complete() from updating tail by holding 983 * ctx->completion_lock. Even if head is invalid, the check 984 * against ctx->completed_events below will make sure we do the 985 * safe/right thing. 986 */ 987 ring = kmap_atomic(ctx->ring_pages[0]); 988 head = ring->head; 989 kunmap_atomic(ring); 990 991 refill_reqs_available(ctx, head, ctx->tail); 992 } 993 994 spin_unlock_irq(&ctx->completion_lock); 995 } 996 997 /* aio_get_req 998 * Allocate a slot for an aio request. 999 * Returns NULL if no requests are free. 1000 */ 1001 static inline struct aio_kiocb *aio_get_req(struct kioctx *ctx) 1002 { 1003 struct aio_kiocb *req; 1004 1005 if (!get_reqs_available(ctx)) { 1006 user_refill_reqs_available(ctx); 1007 if (!get_reqs_available(ctx)) 1008 return NULL; 1009 } 1010 1011 req = kmem_cache_alloc(kiocb_cachep, GFP_KERNEL|__GFP_ZERO); 1012 if (unlikely(!req)) 1013 goto out_put; 1014 1015 percpu_ref_get(&ctx->reqs); 1016 INIT_LIST_HEAD(&req->ki_list); 1017 refcount_set(&req->ki_refcnt, 0); 1018 req->ki_ctx = ctx; 1019 return req; 1020 out_put: 1021 put_reqs_available(ctx, 1); 1022 return NULL; 1023 } 1024 1025 static struct kioctx *lookup_ioctx(unsigned long ctx_id) 1026 { 1027 struct aio_ring __user *ring = (void __user *)ctx_id; 1028 struct mm_struct *mm = current->mm; 1029 struct kioctx *ctx, *ret = NULL; 1030 struct kioctx_table *table; 1031 unsigned id; 1032 1033 if (get_user(id, &ring->id)) 1034 return NULL; 1035 1036 rcu_read_lock(); 1037 table = rcu_dereference(mm->ioctx_table); 1038 1039 if (!table || id >= table->nr) 1040 goto out; 1041 1042 id = array_index_nospec(id, table->nr); 1043 ctx = rcu_dereference(table->table[id]); 1044 if (ctx && ctx->user_id == ctx_id) { 1045 if (percpu_ref_tryget_live(&ctx->users)) 1046 ret = ctx; 1047 } 1048 out: 1049 rcu_read_unlock(); 1050 return ret; 1051 } 1052 1053 static inline void iocb_put(struct aio_kiocb *iocb) 1054 { 1055 if (refcount_read(&iocb->ki_refcnt) == 0 || 1056 refcount_dec_and_test(&iocb->ki_refcnt)) { 1057 percpu_ref_put(&iocb->ki_ctx->reqs); 1058 kmem_cache_free(kiocb_cachep, iocb); 1059 } 1060 } 1061 1062 /* aio_complete 1063 * Called when the io request on the given iocb is complete. 1064 */ 1065 static void aio_complete(struct aio_kiocb *iocb, long res, long res2) 1066 { 1067 struct kioctx *ctx = iocb->ki_ctx; 1068 struct aio_ring *ring; 1069 struct io_event *ev_page, *event; 1070 unsigned tail, pos, head; 1071 unsigned long flags; 1072 1073 /* 1074 * Add a completion event to the ring buffer. Must be done holding 1075 * ctx->completion_lock to prevent other code from messing with the tail 1076 * pointer since we might be called from irq context. 1077 */ 1078 spin_lock_irqsave(&ctx->completion_lock, flags); 1079 1080 tail = ctx->tail; 1081 pos = tail + AIO_EVENTS_OFFSET; 1082 1083 if (++tail >= ctx->nr_events) 1084 tail = 0; 1085 1086 ev_page = kmap_atomic(ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]); 1087 event = ev_page + pos % AIO_EVENTS_PER_PAGE; 1088 1089 event->obj = (u64)(unsigned long)iocb->ki_user_iocb; 1090 event->data = iocb->ki_user_data; 1091 event->res = res; 1092 event->res2 = res2; 1093 1094 kunmap_atomic(ev_page); 1095 flush_dcache_page(ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]); 1096 1097 pr_debug("%p[%u]: %p: %p %Lx %lx %lx\n", 1098 ctx, tail, iocb, iocb->ki_user_iocb, iocb->ki_user_data, 1099 res, res2); 1100 1101 /* after flagging the request as done, we 1102 * must never even look at it again 1103 */ 1104 smp_wmb(); /* make event visible before updating tail */ 1105 1106 ctx->tail = tail; 1107 1108 ring = kmap_atomic(ctx->ring_pages[0]); 1109 head = ring->head; 1110 ring->tail = tail; 1111 kunmap_atomic(ring); 1112 flush_dcache_page(ctx->ring_pages[0]); 1113 1114 ctx->completed_events++; 1115 if (ctx->completed_events > 1) 1116 refill_reqs_available(ctx, head, tail); 1117 spin_unlock_irqrestore(&ctx->completion_lock, flags); 1118 1119 pr_debug("added to ring %p at [%u]\n", iocb, tail); 1120 1121 /* 1122 * Check if the user asked us to deliver the result through an 1123 * eventfd. The eventfd_signal() function is safe to be called 1124 * from IRQ context. 1125 */ 1126 if (iocb->ki_eventfd) { 1127 eventfd_signal(iocb->ki_eventfd, 1); 1128 eventfd_ctx_put(iocb->ki_eventfd); 1129 } 1130 1131 /* 1132 * We have to order our ring_info tail store above and test 1133 * of the wait list below outside the wait lock. This is 1134 * like in wake_up_bit() where clearing a bit has to be 1135 * ordered with the unlocked test. 1136 */ 1137 smp_mb(); 1138 1139 if (waitqueue_active(&ctx->wait)) 1140 wake_up(&ctx->wait); 1141 iocb_put(iocb); 1142 } 1143 1144 /* aio_read_events_ring 1145 * Pull an event off of the ioctx's event ring. Returns the number of 1146 * events fetched 1147 */ 1148 static long aio_read_events_ring(struct kioctx *ctx, 1149 struct io_event __user *event, long nr) 1150 { 1151 struct aio_ring *ring; 1152 unsigned head, tail, pos; 1153 long ret = 0; 1154 int copy_ret; 1155 1156 /* 1157 * The mutex can block and wake us up and that will cause 1158 * wait_event_interruptible_hrtimeout() to schedule without sleeping 1159 * and repeat. This should be rare enough that it doesn't cause 1160 * peformance issues. See the comment in read_events() for more detail. 1161 */ 1162 sched_annotate_sleep(); 1163 mutex_lock(&ctx->ring_lock); 1164 1165 /* Access to ->ring_pages here is protected by ctx->ring_lock. */ 1166 ring = kmap_atomic(ctx->ring_pages[0]); 1167 head = ring->head; 1168 tail = ring->tail; 1169 kunmap_atomic(ring); 1170 1171 /* 1172 * Ensure that once we've read the current tail pointer, that 1173 * we also see the events that were stored up to the tail. 1174 */ 1175 smp_rmb(); 1176 1177 pr_debug("h%u t%u m%u\n", head, tail, ctx->nr_events); 1178 1179 if (head == tail) 1180 goto out; 1181 1182 head %= ctx->nr_events; 1183 tail %= ctx->nr_events; 1184 1185 while (ret < nr) { 1186 long avail; 1187 struct io_event *ev; 1188 struct page *page; 1189 1190 avail = (head <= tail ? tail : ctx->nr_events) - head; 1191 if (head == tail) 1192 break; 1193 1194 pos = head + AIO_EVENTS_OFFSET; 1195 page = ctx->ring_pages[pos / AIO_EVENTS_PER_PAGE]; 1196 pos %= AIO_EVENTS_PER_PAGE; 1197 1198 avail = min(avail, nr - ret); 1199 avail = min_t(long, avail, AIO_EVENTS_PER_PAGE - pos); 1200 1201 ev = kmap(page); 1202 copy_ret = copy_to_user(event + ret, ev + pos, 1203 sizeof(*ev) * avail); 1204 kunmap(page); 1205 1206 if (unlikely(copy_ret)) { 1207 ret = -EFAULT; 1208 goto out; 1209 } 1210 1211 ret += avail; 1212 head += avail; 1213 head %= ctx->nr_events; 1214 } 1215 1216 ring = kmap_atomic(ctx->ring_pages[0]); 1217 ring->head = head; 1218 kunmap_atomic(ring); 1219 flush_dcache_page(ctx->ring_pages[0]); 1220 1221 pr_debug("%li h%u t%u\n", ret, head, tail); 1222 out: 1223 mutex_unlock(&ctx->ring_lock); 1224 1225 return ret; 1226 } 1227 1228 static bool aio_read_events(struct kioctx *ctx, long min_nr, long nr, 1229 struct io_event __user *event, long *i) 1230 { 1231 long ret = aio_read_events_ring(ctx, event + *i, nr - *i); 1232 1233 if (ret > 0) 1234 *i += ret; 1235 1236 if (unlikely(atomic_read(&ctx->dead))) 1237 ret = -EINVAL; 1238 1239 if (!*i) 1240 *i = ret; 1241 1242 return ret < 0 || *i >= min_nr; 1243 } 1244 1245 static long read_events(struct kioctx *ctx, long min_nr, long nr, 1246 struct io_event __user *event, 1247 ktime_t until) 1248 { 1249 long ret = 0; 1250 1251 /* 1252 * Note that aio_read_events() is being called as the conditional - i.e. 1253 * we're calling it after prepare_to_wait() has set task state to 1254 * TASK_INTERRUPTIBLE. 1255 * 1256 * But aio_read_events() can block, and if it blocks it's going to flip 1257 * the task state back to TASK_RUNNING. 1258 * 1259 * This should be ok, provided it doesn't flip the state back to 1260 * TASK_RUNNING and return 0 too much - that causes us to spin. That 1261 * will only happen if the mutex_lock() call blocks, and we then find 1262 * the ringbuffer empty. So in practice we should be ok, but it's 1263 * something to be aware of when touching this code. 1264 */ 1265 if (until == 0) 1266 aio_read_events(ctx, min_nr, nr, event, &ret); 1267 else 1268 wait_event_interruptible_hrtimeout(ctx->wait, 1269 aio_read_events(ctx, min_nr, nr, event, &ret), 1270 until); 1271 return ret; 1272 } 1273 1274 /* sys_io_setup: 1275 * Create an aio_context capable of receiving at least nr_events. 1276 * ctxp must not point to an aio_context that already exists, and 1277 * must be initialized to 0 prior to the call. On successful 1278 * creation of the aio_context, *ctxp is filled in with the resulting 1279 * handle. May fail with -EINVAL if *ctxp is not initialized, 1280 * if the specified nr_events exceeds internal limits. May fail 1281 * with -EAGAIN if the specified nr_events exceeds the user's limit 1282 * of available events. May fail with -ENOMEM if insufficient kernel 1283 * resources are available. May fail with -EFAULT if an invalid 1284 * pointer is passed for ctxp. Will fail with -ENOSYS if not 1285 * implemented. 1286 */ 1287 SYSCALL_DEFINE2(io_setup, unsigned, nr_events, aio_context_t __user *, ctxp) 1288 { 1289 struct kioctx *ioctx = NULL; 1290 unsigned long ctx; 1291 long ret; 1292 1293 ret = get_user(ctx, ctxp); 1294 if (unlikely(ret)) 1295 goto out; 1296 1297 ret = -EINVAL; 1298 if (unlikely(ctx || nr_events == 0)) { 1299 pr_debug("EINVAL: ctx %lu nr_events %u\n", 1300 ctx, nr_events); 1301 goto out; 1302 } 1303 1304 ioctx = ioctx_alloc(nr_events); 1305 ret = PTR_ERR(ioctx); 1306 if (!IS_ERR(ioctx)) { 1307 ret = put_user(ioctx->user_id, ctxp); 1308 if (ret) 1309 kill_ioctx(current->mm, ioctx, NULL); 1310 percpu_ref_put(&ioctx->users); 1311 } 1312 1313 out: 1314 return ret; 1315 } 1316 1317 #ifdef CONFIG_COMPAT 1318 COMPAT_SYSCALL_DEFINE2(io_setup, unsigned, nr_events, u32 __user *, ctx32p) 1319 { 1320 struct kioctx *ioctx = NULL; 1321 unsigned long ctx; 1322 long ret; 1323 1324 ret = get_user(ctx, ctx32p); 1325 if (unlikely(ret)) 1326 goto out; 1327 1328 ret = -EINVAL; 1329 if (unlikely(ctx || nr_events == 0)) { 1330 pr_debug("EINVAL: ctx %lu nr_events %u\n", 1331 ctx, nr_events); 1332 goto out; 1333 } 1334 1335 ioctx = ioctx_alloc(nr_events); 1336 ret = PTR_ERR(ioctx); 1337 if (!IS_ERR(ioctx)) { 1338 /* truncating is ok because it's a user address */ 1339 ret = put_user((u32)ioctx->user_id, ctx32p); 1340 if (ret) 1341 kill_ioctx(current->mm, ioctx, NULL); 1342 percpu_ref_put(&ioctx->users); 1343 } 1344 1345 out: 1346 return ret; 1347 } 1348 #endif 1349 1350 /* sys_io_destroy: 1351 * Destroy the aio_context specified. May cancel any outstanding 1352 * AIOs and block on completion. Will fail with -ENOSYS if not 1353 * implemented. May fail with -EINVAL if the context pointed to 1354 * is invalid. 1355 */ 1356 SYSCALL_DEFINE1(io_destroy, aio_context_t, ctx) 1357 { 1358 struct kioctx *ioctx = lookup_ioctx(ctx); 1359 if (likely(NULL != ioctx)) { 1360 struct ctx_rq_wait wait; 1361 int ret; 1362 1363 init_completion(&wait.comp); 1364 atomic_set(&wait.count, 1); 1365 1366 /* Pass requests_done to kill_ioctx() where it can be set 1367 * in a thread-safe way. If we try to set it here then we have 1368 * a race condition if two io_destroy() called simultaneously. 1369 */ 1370 ret = kill_ioctx(current->mm, ioctx, &wait); 1371 percpu_ref_put(&ioctx->users); 1372 1373 /* Wait until all IO for the context are done. Otherwise kernel 1374 * keep using user-space buffers even if user thinks the context 1375 * is destroyed. 1376 */ 1377 if (!ret) 1378 wait_for_completion(&wait.comp); 1379 1380 return ret; 1381 } 1382 pr_debug("EINVAL: invalid context id\n"); 1383 return -EINVAL; 1384 } 1385 1386 static void aio_remove_iocb(struct aio_kiocb *iocb) 1387 { 1388 struct kioctx *ctx = iocb->ki_ctx; 1389 unsigned long flags; 1390 1391 spin_lock_irqsave(&ctx->ctx_lock, flags); 1392 list_del(&iocb->ki_list); 1393 spin_unlock_irqrestore(&ctx->ctx_lock, flags); 1394 } 1395 1396 static void aio_complete_rw(struct kiocb *kiocb, long res, long res2) 1397 { 1398 struct aio_kiocb *iocb = container_of(kiocb, struct aio_kiocb, rw); 1399 1400 if (!list_empty_careful(&iocb->ki_list)) 1401 aio_remove_iocb(iocb); 1402 1403 if (kiocb->ki_flags & IOCB_WRITE) { 1404 struct inode *inode = file_inode(kiocb->ki_filp); 1405 1406 /* 1407 * Tell lockdep we inherited freeze protection from submission 1408 * thread. 1409 */ 1410 if (S_ISREG(inode->i_mode)) 1411 __sb_writers_acquired(inode->i_sb, SB_FREEZE_WRITE); 1412 file_end_write(kiocb->ki_filp); 1413 } 1414 1415 fput(kiocb->ki_filp); 1416 aio_complete(iocb, res, res2); 1417 } 1418 1419 static int aio_prep_rw(struct kiocb *req, struct iocb *iocb) 1420 { 1421 int ret; 1422 1423 req->ki_filp = fget(iocb->aio_fildes); 1424 if (unlikely(!req->ki_filp)) 1425 return -EBADF; 1426 req->ki_complete = aio_complete_rw; 1427 req->ki_pos = iocb->aio_offset; 1428 req->ki_flags = iocb_flags(req->ki_filp); 1429 if (iocb->aio_flags & IOCB_FLAG_RESFD) 1430 req->ki_flags |= IOCB_EVENTFD; 1431 req->ki_hint = ki_hint_validate(file_write_hint(req->ki_filp)); 1432 if (iocb->aio_flags & IOCB_FLAG_IOPRIO) { 1433 /* 1434 * If the IOCB_FLAG_IOPRIO flag of aio_flags is set, then 1435 * aio_reqprio is interpreted as an I/O scheduling 1436 * class and priority. 1437 */ 1438 ret = ioprio_check_cap(iocb->aio_reqprio); 1439 if (ret) { 1440 pr_debug("aio ioprio check cap error: %d\n", ret); 1441 fput(req->ki_filp); 1442 return ret; 1443 } 1444 1445 req->ki_ioprio = iocb->aio_reqprio; 1446 } else 1447 req->ki_ioprio = IOPRIO_PRIO_VALUE(IOPRIO_CLASS_NONE, 0); 1448 1449 ret = kiocb_set_rw_flags(req, iocb->aio_rw_flags); 1450 if (unlikely(ret)) 1451 fput(req->ki_filp); 1452 return ret; 1453 } 1454 1455 static int aio_setup_rw(int rw, struct iocb *iocb, struct iovec **iovec, 1456 bool vectored, bool compat, struct iov_iter *iter) 1457 { 1458 void __user *buf = (void __user *)(uintptr_t)iocb->aio_buf; 1459 size_t len = iocb->aio_nbytes; 1460 1461 if (!vectored) { 1462 ssize_t ret = import_single_range(rw, buf, len, *iovec, iter); 1463 *iovec = NULL; 1464 return ret; 1465 } 1466 #ifdef CONFIG_COMPAT 1467 if (compat) 1468 return compat_import_iovec(rw, buf, len, UIO_FASTIOV, iovec, 1469 iter); 1470 #endif 1471 return import_iovec(rw, buf, len, UIO_FASTIOV, iovec, iter); 1472 } 1473 1474 static inline void aio_rw_done(struct kiocb *req, ssize_t ret) 1475 { 1476 switch (ret) { 1477 case -EIOCBQUEUED: 1478 break; 1479 case -ERESTARTSYS: 1480 case -ERESTARTNOINTR: 1481 case -ERESTARTNOHAND: 1482 case -ERESTART_RESTARTBLOCK: 1483 /* 1484 * There's no easy way to restart the syscall since other AIO's 1485 * may be already running. Just fail this IO with EINTR. 1486 */ 1487 ret = -EINTR; 1488 /*FALLTHRU*/ 1489 default: 1490 aio_complete_rw(req, ret, 0); 1491 } 1492 } 1493 1494 static ssize_t aio_read(struct kiocb *req, struct iocb *iocb, bool vectored, 1495 bool compat) 1496 { 1497 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; 1498 struct iov_iter iter; 1499 struct file *file; 1500 ssize_t ret; 1501 1502 ret = aio_prep_rw(req, iocb); 1503 if (ret) 1504 return ret; 1505 file = req->ki_filp; 1506 1507 ret = -EBADF; 1508 if (unlikely(!(file->f_mode & FMODE_READ))) 1509 goto out_fput; 1510 ret = -EINVAL; 1511 if (unlikely(!file->f_op->read_iter)) 1512 goto out_fput; 1513 1514 ret = aio_setup_rw(READ, iocb, &iovec, vectored, compat, &iter); 1515 if (ret) 1516 goto out_fput; 1517 ret = rw_verify_area(READ, file, &req->ki_pos, iov_iter_count(&iter)); 1518 if (!ret) 1519 aio_rw_done(req, call_read_iter(file, req, &iter)); 1520 kfree(iovec); 1521 out_fput: 1522 if (unlikely(ret)) 1523 fput(file); 1524 return ret; 1525 } 1526 1527 static ssize_t aio_write(struct kiocb *req, struct iocb *iocb, bool vectored, 1528 bool compat) 1529 { 1530 struct iovec inline_vecs[UIO_FASTIOV], *iovec = inline_vecs; 1531 struct iov_iter iter; 1532 struct file *file; 1533 ssize_t ret; 1534 1535 ret = aio_prep_rw(req, iocb); 1536 if (ret) 1537 return ret; 1538 file = req->ki_filp; 1539 1540 ret = -EBADF; 1541 if (unlikely(!(file->f_mode & FMODE_WRITE))) 1542 goto out_fput; 1543 ret = -EINVAL; 1544 if (unlikely(!file->f_op->write_iter)) 1545 goto out_fput; 1546 1547 ret = aio_setup_rw(WRITE, iocb, &iovec, vectored, compat, &iter); 1548 if (ret) 1549 goto out_fput; 1550 ret = rw_verify_area(WRITE, file, &req->ki_pos, iov_iter_count(&iter)); 1551 if (!ret) { 1552 /* 1553 * Open-code file_start_write here to grab freeze protection, 1554 * which will be released by another thread in 1555 * aio_complete_rw(). Fool lockdep by telling it the lock got 1556 * released so that it doesn't complain about the held lock when 1557 * we return to userspace. 1558 */ 1559 if (S_ISREG(file_inode(file)->i_mode)) { 1560 __sb_start_write(file_inode(file)->i_sb, SB_FREEZE_WRITE, true); 1561 __sb_writers_release(file_inode(file)->i_sb, SB_FREEZE_WRITE); 1562 } 1563 req->ki_flags |= IOCB_WRITE; 1564 aio_rw_done(req, call_write_iter(file, req, &iter)); 1565 } 1566 kfree(iovec); 1567 out_fput: 1568 if (unlikely(ret)) 1569 fput(file); 1570 return ret; 1571 } 1572 1573 static void aio_fsync_work(struct work_struct *work) 1574 { 1575 struct fsync_iocb *req = container_of(work, struct fsync_iocb, work); 1576 int ret; 1577 1578 ret = vfs_fsync(req->file, req->datasync); 1579 fput(req->file); 1580 aio_complete(container_of(req, struct aio_kiocb, fsync), ret, 0); 1581 } 1582 1583 static int aio_fsync(struct fsync_iocb *req, struct iocb *iocb, bool datasync) 1584 { 1585 if (unlikely(iocb->aio_buf || iocb->aio_offset || iocb->aio_nbytes || 1586 iocb->aio_rw_flags)) 1587 return -EINVAL; 1588 1589 req->file = fget(iocb->aio_fildes); 1590 if (unlikely(!req->file)) 1591 return -EBADF; 1592 if (unlikely(!req->file->f_op->fsync)) { 1593 fput(req->file); 1594 return -EINVAL; 1595 } 1596 1597 req->datasync = datasync; 1598 INIT_WORK(&req->work, aio_fsync_work); 1599 schedule_work(&req->work); 1600 return 0; 1601 } 1602 1603 static inline void aio_poll_complete(struct aio_kiocb *iocb, __poll_t mask) 1604 { 1605 struct file *file = iocb->poll.file; 1606 1607 aio_complete(iocb, mangle_poll(mask), 0); 1608 fput(file); 1609 } 1610 1611 static void aio_poll_complete_work(struct work_struct *work) 1612 { 1613 struct poll_iocb *req = container_of(work, struct poll_iocb, work); 1614 struct aio_kiocb *iocb = container_of(req, struct aio_kiocb, poll); 1615 struct poll_table_struct pt = { ._key = req->events }; 1616 struct kioctx *ctx = iocb->ki_ctx; 1617 __poll_t mask = 0; 1618 1619 if (!READ_ONCE(req->cancelled)) 1620 mask = vfs_poll(req->file, &pt) & req->events; 1621 1622 /* 1623 * Note that ->ki_cancel callers also delete iocb from active_reqs after 1624 * calling ->ki_cancel. We need the ctx_lock roundtrip here to 1625 * synchronize with them. In the cancellation case the list_del_init 1626 * itself is not actually needed, but harmless so we keep it in to 1627 * avoid further branches in the fast path. 1628 */ 1629 spin_lock_irq(&ctx->ctx_lock); 1630 if (!mask && !READ_ONCE(req->cancelled)) { 1631 add_wait_queue(req->head, &req->wait); 1632 spin_unlock_irq(&ctx->ctx_lock); 1633 return; 1634 } 1635 list_del_init(&iocb->ki_list); 1636 spin_unlock_irq(&ctx->ctx_lock); 1637 1638 aio_poll_complete(iocb, mask); 1639 } 1640 1641 /* assumes we are called with irqs disabled */ 1642 static int aio_poll_cancel(struct kiocb *iocb) 1643 { 1644 struct aio_kiocb *aiocb = container_of(iocb, struct aio_kiocb, rw); 1645 struct poll_iocb *req = &aiocb->poll; 1646 1647 spin_lock(&req->head->lock); 1648 WRITE_ONCE(req->cancelled, true); 1649 if (!list_empty(&req->wait.entry)) { 1650 list_del_init(&req->wait.entry); 1651 schedule_work(&aiocb->poll.work); 1652 } 1653 spin_unlock(&req->head->lock); 1654 1655 return 0; 1656 } 1657 1658 static int aio_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync, 1659 void *key) 1660 { 1661 struct poll_iocb *req = container_of(wait, struct poll_iocb, wait); 1662 struct aio_kiocb *iocb = container_of(req, struct aio_kiocb, poll); 1663 __poll_t mask = key_to_poll(key); 1664 1665 req->woken = true; 1666 1667 /* for instances that support it check for an event match first: */ 1668 if (mask) { 1669 if (!(mask & req->events)) 1670 return 0; 1671 1672 /* try to complete the iocb inline if we can: */ 1673 if (spin_trylock(&iocb->ki_ctx->ctx_lock)) { 1674 list_del(&iocb->ki_list); 1675 spin_unlock(&iocb->ki_ctx->ctx_lock); 1676 1677 list_del_init(&req->wait.entry); 1678 aio_poll_complete(iocb, mask); 1679 return 1; 1680 } 1681 } 1682 1683 list_del_init(&req->wait.entry); 1684 schedule_work(&req->work); 1685 return 1; 1686 } 1687 1688 struct aio_poll_table { 1689 struct poll_table_struct pt; 1690 struct aio_kiocb *iocb; 1691 int error; 1692 }; 1693 1694 static void 1695 aio_poll_queue_proc(struct file *file, struct wait_queue_head *head, 1696 struct poll_table_struct *p) 1697 { 1698 struct aio_poll_table *pt = container_of(p, struct aio_poll_table, pt); 1699 1700 /* multiple wait queues per file are not supported */ 1701 if (unlikely(pt->iocb->poll.head)) { 1702 pt->error = -EINVAL; 1703 return; 1704 } 1705 1706 pt->error = 0; 1707 pt->iocb->poll.head = head; 1708 add_wait_queue(head, &pt->iocb->poll.wait); 1709 } 1710 1711 static ssize_t aio_poll(struct aio_kiocb *aiocb, struct iocb *iocb) 1712 { 1713 struct kioctx *ctx = aiocb->ki_ctx; 1714 struct poll_iocb *req = &aiocb->poll; 1715 struct aio_poll_table apt; 1716 __poll_t mask; 1717 1718 /* reject any unknown events outside the normal event mask. */ 1719 if ((u16)iocb->aio_buf != iocb->aio_buf) 1720 return -EINVAL; 1721 /* reject fields that are not defined for poll */ 1722 if (iocb->aio_offset || iocb->aio_nbytes || iocb->aio_rw_flags) 1723 return -EINVAL; 1724 1725 INIT_WORK(&req->work, aio_poll_complete_work); 1726 req->events = demangle_poll(iocb->aio_buf) | EPOLLERR | EPOLLHUP; 1727 req->file = fget(iocb->aio_fildes); 1728 if (unlikely(!req->file)) 1729 return -EBADF; 1730 1731 apt.pt._qproc = aio_poll_queue_proc; 1732 apt.pt._key = req->events; 1733 apt.iocb = aiocb; 1734 apt.error = -EINVAL; /* same as no support for IOCB_CMD_POLL */ 1735 1736 /* initialized the list so that we can do list_empty checks */ 1737 INIT_LIST_HEAD(&req->wait.entry); 1738 init_waitqueue_func_entry(&req->wait, aio_poll_wake); 1739 1740 /* one for removal from waitqueue, one for this function */ 1741 refcount_set(&aiocb->ki_refcnt, 2); 1742 1743 mask = vfs_poll(req->file, &apt.pt) & req->events; 1744 if (unlikely(!req->head)) { 1745 /* we did not manage to set up a waitqueue, done */ 1746 goto out; 1747 } 1748 1749 spin_lock_irq(&ctx->ctx_lock); 1750 spin_lock(&req->head->lock); 1751 if (req->woken) { 1752 /* wake_up context handles the rest */ 1753 mask = 0; 1754 apt.error = 0; 1755 } else if (mask || apt.error) { 1756 /* if we get an error or a mask we are done */ 1757 WARN_ON_ONCE(list_empty(&req->wait.entry)); 1758 list_del_init(&req->wait.entry); 1759 } else { 1760 /* actually waiting for an event */ 1761 list_add_tail(&aiocb->ki_list, &ctx->active_reqs); 1762 aiocb->ki_cancel = aio_poll_cancel; 1763 } 1764 spin_unlock(&req->head->lock); 1765 spin_unlock_irq(&ctx->ctx_lock); 1766 1767 out: 1768 if (unlikely(apt.error)) { 1769 fput(req->file); 1770 return apt.error; 1771 } 1772 1773 if (mask) 1774 aio_poll_complete(aiocb, mask); 1775 iocb_put(aiocb); 1776 return 0; 1777 } 1778 1779 static int io_submit_one(struct kioctx *ctx, struct iocb __user *user_iocb, 1780 bool compat) 1781 { 1782 struct aio_kiocb *req; 1783 struct iocb iocb; 1784 ssize_t ret; 1785 1786 if (unlikely(copy_from_user(&iocb, user_iocb, sizeof(iocb)))) 1787 return -EFAULT; 1788 1789 /* enforce forwards compatibility on users */ 1790 if (unlikely(iocb.aio_reserved2)) { 1791 pr_debug("EINVAL: reserve field set\n"); 1792 return -EINVAL; 1793 } 1794 1795 /* prevent overflows */ 1796 if (unlikely( 1797 (iocb.aio_buf != (unsigned long)iocb.aio_buf) || 1798 (iocb.aio_nbytes != (size_t)iocb.aio_nbytes) || 1799 ((ssize_t)iocb.aio_nbytes < 0) 1800 )) { 1801 pr_debug("EINVAL: overflow check\n"); 1802 return -EINVAL; 1803 } 1804 1805 req = aio_get_req(ctx); 1806 if (unlikely(!req)) 1807 return -EAGAIN; 1808 1809 if (iocb.aio_flags & IOCB_FLAG_RESFD) { 1810 /* 1811 * If the IOCB_FLAG_RESFD flag of aio_flags is set, get an 1812 * instance of the file* now. The file descriptor must be 1813 * an eventfd() fd, and will be signaled for each completed 1814 * event using the eventfd_signal() function. 1815 */ 1816 req->ki_eventfd = eventfd_ctx_fdget((int) iocb.aio_resfd); 1817 if (IS_ERR(req->ki_eventfd)) { 1818 ret = PTR_ERR(req->ki_eventfd); 1819 req->ki_eventfd = NULL; 1820 goto out_put_req; 1821 } 1822 } 1823 1824 ret = put_user(KIOCB_KEY, &user_iocb->aio_key); 1825 if (unlikely(ret)) { 1826 pr_debug("EFAULT: aio_key\n"); 1827 goto out_put_req; 1828 } 1829 1830 req->ki_user_iocb = user_iocb; 1831 req->ki_user_data = iocb.aio_data; 1832 1833 switch (iocb.aio_lio_opcode) { 1834 case IOCB_CMD_PREAD: 1835 ret = aio_read(&req->rw, &iocb, false, compat); 1836 break; 1837 case IOCB_CMD_PWRITE: 1838 ret = aio_write(&req->rw, &iocb, false, compat); 1839 break; 1840 case IOCB_CMD_PREADV: 1841 ret = aio_read(&req->rw, &iocb, true, compat); 1842 break; 1843 case IOCB_CMD_PWRITEV: 1844 ret = aio_write(&req->rw, &iocb, true, compat); 1845 break; 1846 case IOCB_CMD_FSYNC: 1847 ret = aio_fsync(&req->fsync, &iocb, false); 1848 break; 1849 case IOCB_CMD_FDSYNC: 1850 ret = aio_fsync(&req->fsync, &iocb, true); 1851 break; 1852 case IOCB_CMD_POLL: 1853 ret = aio_poll(req, &iocb); 1854 break; 1855 default: 1856 pr_debug("invalid aio operation %d\n", iocb.aio_lio_opcode); 1857 ret = -EINVAL; 1858 break; 1859 } 1860 1861 /* 1862 * If ret is 0, we'd either done aio_complete() ourselves or have 1863 * arranged for that to be done asynchronously. Anything non-zero 1864 * means that we need to destroy req ourselves. 1865 */ 1866 if (ret) 1867 goto out_put_req; 1868 return 0; 1869 out_put_req: 1870 put_reqs_available(ctx, 1); 1871 percpu_ref_put(&ctx->reqs); 1872 if (req->ki_eventfd) 1873 eventfd_ctx_put(req->ki_eventfd); 1874 kmem_cache_free(kiocb_cachep, req); 1875 return ret; 1876 } 1877 1878 /* sys_io_submit: 1879 * Queue the nr iocbs pointed to by iocbpp for processing. Returns 1880 * the number of iocbs queued. May return -EINVAL if the aio_context 1881 * specified by ctx_id is invalid, if nr is < 0, if the iocb at 1882 * *iocbpp[0] is not properly initialized, if the operation specified 1883 * is invalid for the file descriptor in the iocb. May fail with 1884 * -EFAULT if any of the data structures point to invalid data. May 1885 * fail with -EBADF if the file descriptor specified in the first 1886 * iocb is invalid. May fail with -EAGAIN if insufficient resources 1887 * are available to queue any iocbs. Will return 0 if nr is 0. Will 1888 * fail with -ENOSYS if not implemented. 1889 */ 1890 SYSCALL_DEFINE3(io_submit, aio_context_t, ctx_id, long, nr, 1891 struct iocb __user * __user *, iocbpp) 1892 { 1893 struct kioctx *ctx; 1894 long ret = 0; 1895 int i = 0; 1896 struct blk_plug plug; 1897 1898 if (unlikely(nr < 0)) 1899 return -EINVAL; 1900 1901 ctx = lookup_ioctx(ctx_id); 1902 if (unlikely(!ctx)) { 1903 pr_debug("EINVAL: invalid context id\n"); 1904 return -EINVAL; 1905 } 1906 1907 if (nr > ctx->nr_events) 1908 nr = ctx->nr_events; 1909 1910 blk_start_plug(&plug); 1911 for (i = 0; i < nr; i++) { 1912 struct iocb __user *user_iocb; 1913 1914 if (unlikely(get_user(user_iocb, iocbpp + i))) { 1915 ret = -EFAULT; 1916 break; 1917 } 1918 1919 ret = io_submit_one(ctx, user_iocb, false); 1920 if (ret) 1921 break; 1922 } 1923 blk_finish_plug(&plug); 1924 1925 percpu_ref_put(&ctx->users); 1926 return i ? i : ret; 1927 } 1928 1929 #ifdef CONFIG_COMPAT 1930 COMPAT_SYSCALL_DEFINE3(io_submit, compat_aio_context_t, ctx_id, 1931 int, nr, compat_uptr_t __user *, iocbpp) 1932 { 1933 struct kioctx *ctx; 1934 long ret = 0; 1935 int i = 0; 1936 struct blk_plug plug; 1937 1938 if (unlikely(nr < 0)) 1939 return -EINVAL; 1940 1941 ctx = lookup_ioctx(ctx_id); 1942 if (unlikely(!ctx)) { 1943 pr_debug("EINVAL: invalid context id\n"); 1944 return -EINVAL; 1945 } 1946 1947 if (nr > ctx->nr_events) 1948 nr = ctx->nr_events; 1949 1950 blk_start_plug(&plug); 1951 for (i = 0; i < nr; i++) { 1952 compat_uptr_t user_iocb; 1953 1954 if (unlikely(get_user(user_iocb, iocbpp + i))) { 1955 ret = -EFAULT; 1956 break; 1957 } 1958 1959 ret = io_submit_one(ctx, compat_ptr(user_iocb), true); 1960 if (ret) 1961 break; 1962 } 1963 blk_finish_plug(&plug); 1964 1965 percpu_ref_put(&ctx->users); 1966 return i ? i : ret; 1967 } 1968 #endif 1969 1970 /* lookup_kiocb 1971 * Finds a given iocb for cancellation. 1972 */ 1973 static struct aio_kiocb * 1974 lookup_kiocb(struct kioctx *ctx, struct iocb __user *iocb) 1975 { 1976 struct aio_kiocb *kiocb; 1977 1978 assert_spin_locked(&ctx->ctx_lock); 1979 1980 /* TODO: use a hash or array, this sucks. */ 1981 list_for_each_entry(kiocb, &ctx->active_reqs, ki_list) { 1982 if (kiocb->ki_user_iocb == iocb) 1983 return kiocb; 1984 } 1985 return NULL; 1986 } 1987 1988 /* sys_io_cancel: 1989 * Attempts to cancel an iocb previously passed to io_submit. If 1990 * the operation is successfully cancelled, the resulting event is 1991 * copied into the memory pointed to by result without being placed 1992 * into the completion queue and 0 is returned. May fail with 1993 * -EFAULT if any of the data structures pointed to are invalid. 1994 * May fail with -EINVAL if aio_context specified by ctx_id is 1995 * invalid. May fail with -EAGAIN if the iocb specified was not 1996 * cancelled. Will fail with -ENOSYS if not implemented. 1997 */ 1998 SYSCALL_DEFINE3(io_cancel, aio_context_t, ctx_id, struct iocb __user *, iocb, 1999 struct io_event __user *, result) 2000 { 2001 struct kioctx *ctx; 2002 struct aio_kiocb *kiocb; 2003 int ret = -EINVAL; 2004 u32 key; 2005 2006 if (unlikely(get_user(key, &iocb->aio_key))) 2007 return -EFAULT; 2008 if (unlikely(key != KIOCB_KEY)) 2009 return -EINVAL; 2010 2011 ctx = lookup_ioctx(ctx_id); 2012 if (unlikely(!ctx)) 2013 return -EINVAL; 2014 2015 spin_lock_irq(&ctx->ctx_lock); 2016 kiocb = lookup_kiocb(ctx, iocb); 2017 if (kiocb) { 2018 ret = kiocb->ki_cancel(&kiocb->rw); 2019 list_del_init(&kiocb->ki_list); 2020 } 2021 spin_unlock_irq(&ctx->ctx_lock); 2022 2023 if (!ret) { 2024 /* 2025 * The result argument is no longer used - the io_event is 2026 * always delivered via the ring buffer. -EINPROGRESS indicates 2027 * cancellation is progress: 2028 */ 2029 ret = -EINPROGRESS; 2030 } 2031 2032 percpu_ref_put(&ctx->users); 2033 2034 return ret; 2035 } 2036 2037 static long do_io_getevents(aio_context_t ctx_id, 2038 long min_nr, 2039 long nr, 2040 struct io_event __user *events, 2041 struct timespec64 *ts) 2042 { 2043 ktime_t until = ts ? timespec64_to_ktime(*ts) : KTIME_MAX; 2044 struct kioctx *ioctx = lookup_ioctx(ctx_id); 2045 long ret = -EINVAL; 2046 2047 if (likely(ioctx)) { 2048 if (likely(min_nr <= nr && min_nr >= 0)) 2049 ret = read_events(ioctx, min_nr, nr, events, until); 2050 percpu_ref_put(&ioctx->users); 2051 } 2052 2053 return ret; 2054 } 2055 2056 /* io_getevents: 2057 * Attempts to read at least min_nr events and up to nr events from 2058 * the completion queue for the aio_context specified by ctx_id. If 2059 * it succeeds, the number of read events is returned. May fail with 2060 * -EINVAL if ctx_id is invalid, if min_nr is out of range, if nr is 2061 * out of range, if timeout is out of range. May fail with -EFAULT 2062 * if any of the memory specified is invalid. May return 0 or 2063 * < min_nr if the timeout specified by timeout has elapsed 2064 * before sufficient events are available, where timeout == NULL 2065 * specifies an infinite timeout. Note that the timeout pointed to by 2066 * timeout is relative. Will fail with -ENOSYS if not implemented. 2067 */ 2068 SYSCALL_DEFINE5(io_getevents, aio_context_t, ctx_id, 2069 long, min_nr, 2070 long, nr, 2071 struct io_event __user *, events, 2072 struct timespec __user *, timeout) 2073 { 2074 struct timespec64 ts; 2075 int ret; 2076 2077 if (timeout && unlikely(get_timespec64(&ts, timeout))) 2078 return -EFAULT; 2079 2080 ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &ts : NULL); 2081 if (!ret && signal_pending(current)) 2082 ret = -EINTR; 2083 return ret; 2084 } 2085 2086 struct __aio_sigset { 2087 const sigset_t __user *sigmask; 2088 size_t sigsetsize; 2089 }; 2090 2091 SYSCALL_DEFINE6(io_pgetevents, 2092 aio_context_t, ctx_id, 2093 long, min_nr, 2094 long, nr, 2095 struct io_event __user *, events, 2096 struct timespec __user *, timeout, 2097 const struct __aio_sigset __user *, usig) 2098 { 2099 struct __aio_sigset ksig = { NULL, }; 2100 sigset_t ksigmask, sigsaved; 2101 struct timespec64 ts; 2102 int ret; 2103 2104 if (timeout && unlikely(get_timespec64(&ts, timeout))) 2105 return -EFAULT; 2106 2107 if (usig && copy_from_user(&ksig, usig, sizeof(ksig))) 2108 return -EFAULT; 2109 2110 if (ksig.sigmask) { 2111 if (ksig.sigsetsize != sizeof(sigset_t)) 2112 return -EINVAL; 2113 if (copy_from_user(&ksigmask, ksig.sigmask, sizeof(ksigmask))) 2114 return -EFAULT; 2115 sigdelsetmask(&ksigmask, sigmask(SIGKILL) | sigmask(SIGSTOP)); 2116 sigprocmask(SIG_SETMASK, &ksigmask, &sigsaved); 2117 } 2118 2119 ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &ts : NULL); 2120 if (signal_pending(current)) { 2121 if (ksig.sigmask) { 2122 current->saved_sigmask = sigsaved; 2123 set_restore_sigmask(); 2124 } 2125 2126 if (!ret) 2127 ret = -ERESTARTNOHAND; 2128 } else { 2129 if (ksig.sigmask) 2130 sigprocmask(SIG_SETMASK, &sigsaved, NULL); 2131 } 2132 2133 return ret; 2134 } 2135 2136 #ifdef CONFIG_COMPAT 2137 COMPAT_SYSCALL_DEFINE5(io_getevents, compat_aio_context_t, ctx_id, 2138 compat_long_t, min_nr, 2139 compat_long_t, nr, 2140 struct io_event __user *, events, 2141 struct old_timespec32 __user *, timeout) 2142 { 2143 struct timespec64 t; 2144 int ret; 2145 2146 if (timeout && get_old_timespec32(&t, timeout)) 2147 return -EFAULT; 2148 2149 ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &t : NULL); 2150 if (!ret && signal_pending(current)) 2151 ret = -EINTR; 2152 return ret; 2153 } 2154 2155 2156 struct __compat_aio_sigset { 2157 compat_sigset_t __user *sigmask; 2158 compat_size_t sigsetsize; 2159 }; 2160 2161 COMPAT_SYSCALL_DEFINE6(io_pgetevents, 2162 compat_aio_context_t, ctx_id, 2163 compat_long_t, min_nr, 2164 compat_long_t, nr, 2165 struct io_event __user *, events, 2166 struct old_timespec32 __user *, timeout, 2167 const struct __compat_aio_sigset __user *, usig) 2168 { 2169 struct __compat_aio_sigset ksig = { NULL, }; 2170 sigset_t ksigmask, sigsaved; 2171 struct timespec64 t; 2172 int ret; 2173 2174 if (timeout && get_old_timespec32(&t, timeout)) 2175 return -EFAULT; 2176 2177 if (usig && copy_from_user(&ksig, usig, sizeof(ksig))) 2178 return -EFAULT; 2179 2180 if (ksig.sigmask) { 2181 if (ksig.sigsetsize != sizeof(compat_sigset_t)) 2182 return -EINVAL; 2183 if (get_compat_sigset(&ksigmask, ksig.sigmask)) 2184 return -EFAULT; 2185 sigdelsetmask(&ksigmask, sigmask(SIGKILL) | sigmask(SIGSTOP)); 2186 sigprocmask(SIG_SETMASK, &ksigmask, &sigsaved); 2187 } 2188 2189 ret = do_io_getevents(ctx_id, min_nr, nr, events, timeout ? &t : NULL); 2190 if (signal_pending(current)) { 2191 if (ksig.sigmask) { 2192 current->saved_sigmask = sigsaved; 2193 set_restore_sigmask(); 2194 } 2195 if (!ret) 2196 ret = -ERESTARTNOHAND; 2197 } else { 2198 if (ksig.sigmask) 2199 sigprocmask(SIG_SETMASK, &sigsaved, NULL); 2200 } 2201 2202 return ret; 2203 } 2204 #endif 2205