1 /* 2 FUSE: Filesystem in Userspace 3 Copyright (C) 2001-2008 Miklos Szeredi <miklos@szeredi.hu> 4 5 This program can be distributed under the terms of the GNU GPL. 6 See the file COPYING. 7 */ 8 9 #include "fuse_i.h" 10 11 #include <linux/pagemap.h> 12 #include <linux/slab.h> 13 #include <linux/kernel.h> 14 #include <linux/sched.h> 15 #include <linux/sched/signal.h> 16 #include <linux/module.h> 17 #include <linux/swap.h> 18 #include <linux/falloc.h> 19 #include <linux/uio.h> 20 #include <linux/fs.h> 21 22 static int fuse_send_open(struct fuse_mount *fm, u64 nodeid, 23 unsigned int open_flags, int opcode, 24 struct fuse_open_out *outargp) 25 { 26 struct fuse_open_in inarg; 27 FUSE_ARGS(args); 28 29 memset(&inarg, 0, sizeof(inarg)); 30 inarg.flags = open_flags & ~(O_CREAT | O_EXCL | O_NOCTTY); 31 if (!fm->fc->atomic_o_trunc) 32 inarg.flags &= ~O_TRUNC; 33 34 if (fm->fc->handle_killpriv_v2 && 35 (inarg.flags & O_TRUNC) && !capable(CAP_FSETID)) { 36 inarg.open_flags |= FUSE_OPEN_KILL_SUIDGID; 37 } 38 39 args.opcode = opcode; 40 args.nodeid = nodeid; 41 args.in_numargs = 1; 42 args.in_args[0].size = sizeof(inarg); 43 args.in_args[0].value = &inarg; 44 args.out_numargs = 1; 45 args.out_args[0].size = sizeof(*outargp); 46 args.out_args[0].value = outargp; 47 48 return fuse_simple_request(fm, &args); 49 } 50 51 struct fuse_release_args { 52 struct fuse_args args; 53 struct fuse_release_in inarg; 54 struct inode *inode; 55 }; 56 57 struct fuse_file *fuse_file_alloc(struct fuse_mount *fm) 58 { 59 struct fuse_file *ff; 60 61 ff = kzalloc(sizeof(struct fuse_file), GFP_KERNEL_ACCOUNT); 62 if (unlikely(!ff)) 63 return NULL; 64 65 ff->fm = fm; 66 ff->release_args = kzalloc(sizeof(*ff->release_args), 67 GFP_KERNEL_ACCOUNT); 68 if (!ff->release_args) { 69 kfree(ff); 70 return NULL; 71 } 72 73 INIT_LIST_HEAD(&ff->write_entry); 74 mutex_init(&ff->readdir.lock); 75 refcount_set(&ff->count, 1); 76 RB_CLEAR_NODE(&ff->polled_node); 77 init_waitqueue_head(&ff->poll_wait); 78 79 ff->kh = atomic64_inc_return(&fm->fc->khctr); 80 81 return ff; 82 } 83 84 void fuse_file_free(struct fuse_file *ff) 85 { 86 kfree(ff->release_args); 87 mutex_destroy(&ff->readdir.lock); 88 kfree(ff); 89 } 90 91 static struct fuse_file *fuse_file_get(struct fuse_file *ff) 92 { 93 refcount_inc(&ff->count); 94 return ff; 95 } 96 97 static void fuse_release_end(struct fuse_mount *fm, struct fuse_args *args, 98 int error) 99 { 100 struct fuse_release_args *ra = container_of(args, typeof(*ra), args); 101 102 iput(ra->inode); 103 kfree(ra); 104 } 105 106 static void fuse_file_put(struct fuse_file *ff, bool sync, bool isdir) 107 { 108 if (refcount_dec_and_test(&ff->count)) { 109 struct fuse_args *args = &ff->release_args->args; 110 111 if (isdir ? ff->fm->fc->no_opendir : ff->fm->fc->no_open) { 112 /* Do nothing when client does not implement 'open' */ 113 fuse_release_end(ff->fm, args, 0); 114 } else if (sync) { 115 fuse_simple_request(ff->fm, args); 116 fuse_release_end(ff->fm, args, 0); 117 } else { 118 args->end = fuse_release_end; 119 if (fuse_simple_background(ff->fm, args, 120 GFP_KERNEL | __GFP_NOFAIL)) 121 fuse_release_end(ff->fm, args, -ENOTCONN); 122 } 123 kfree(ff); 124 } 125 } 126 127 struct fuse_file *fuse_file_open(struct fuse_mount *fm, u64 nodeid, 128 unsigned int open_flags, bool isdir) 129 { 130 struct fuse_conn *fc = fm->fc; 131 struct fuse_file *ff; 132 int opcode = isdir ? FUSE_OPENDIR : FUSE_OPEN; 133 134 ff = fuse_file_alloc(fm); 135 if (!ff) 136 return ERR_PTR(-ENOMEM); 137 138 ff->fh = 0; 139 /* Default for no-open */ 140 ff->open_flags = FOPEN_KEEP_CACHE | (isdir ? FOPEN_CACHE_DIR : 0); 141 if (isdir ? !fc->no_opendir : !fc->no_open) { 142 struct fuse_open_out outarg; 143 int err; 144 145 err = fuse_send_open(fm, nodeid, open_flags, opcode, &outarg); 146 if (!err) { 147 ff->fh = outarg.fh; 148 ff->open_flags = outarg.open_flags; 149 150 } else if (err != -ENOSYS) { 151 fuse_file_free(ff); 152 return ERR_PTR(err); 153 } else { 154 if (isdir) 155 fc->no_opendir = 1; 156 else 157 fc->no_open = 1; 158 } 159 } 160 161 if (isdir) 162 ff->open_flags &= ~FOPEN_DIRECT_IO; 163 164 ff->nodeid = nodeid; 165 166 return ff; 167 } 168 169 int fuse_do_open(struct fuse_mount *fm, u64 nodeid, struct file *file, 170 bool isdir) 171 { 172 struct fuse_file *ff = fuse_file_open(fm, nodeid, file->f_flags, isdir); 173 174 if (!IS_ERR(ff)) 175 file->private_data = ff; 176 177 return PTR_ERR_OR_ZERO(ff); 178 } 179 EXPORT_SYMBOL_GPL(fuse_do_open); 180 181 static void fuse_link_write_file(struct file *file) 182 { 183 struct inode *inode = file_inode(file); 184 struct fuse_inode *fi = get_fuse_inode(inode); 185 struct fuse_file *ff = file->private_data; 186 /* 187 * file may be written through mmap, so chain it onto the 188 * inodes's write_file list 189 */ 190 spin_lock(&fi->lock); 191 if (list_empty(&ff->write_entry)) 192 list_add(&ff->write_entry, &fi->write_files); 193 spin_unlock(&fi->lock); 194 } 195 196 void fuse_finish_open(struct inode *inode, struct file *file) 197 { 198 struct fuse_file *ff = file->private_data; 199 struct fuse_conn *fc = get_fuse_conn(inode); 200 201 if (ff->open_flags & FOPEN_STREAM) 202 stream_open(inode, file); 203 else if (ff->open_flags & FOPEN_NONSEEKABLE) 204 nonseekable_open(inode, file); 205 206 if (fc->atomic_o_trunc && (file->f_flags & O_TRUNC)) { 207 struct fuse_inode *fi = get_fuse_inode(inode); 208 209 spin_lock(&fi->lock); 210 fi->attr_version = atomic64_inc_return(&fc->attr_version); 211 i_size_write(inode, 0); 212 spin_unlock(&fi->lock); 213 file_update_time(file); 214 fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE); 215 } 216 if ((file->f_mode & FMODE_WRITE) && fc->writeback_cache) 217 fuse_link_write_file(file); 218 } 219 220 int fuse_open_common(struct inode *inode, struct file *file, bool isdir) 221 { 222 struct fuse_mount *fm = get_fuse_mount(inode); 223 struct fuse_conn *fc = fm->fc; 224 int err; 225 bool is_wb_truncate = (file->f_flags & O_TRUNC) && 226 fc->atomic_o_trunc && 227 fc->writeback_cache; 228 bool dax_truncate = (file->f_flags & O_TRUNC) && 229 fc->atomic_o_trunc && FUSE_IS_DAX(inode); 230 231 if (fuse_is_bad(inode)) 232 return -EIO; 233 234 err = generic_file_open(inode, file); 235 if (err) 236 return err; 237 238 if (is_wb_truncate || dax_truncate) 239 inode_lock(inode); 240 241 if (dax_truncate) { 242 filemap_invalidate_lock(inode->i_mapping); 243 err = fuse_dax_break_layouts(inode, 0, 0); 244 if (err) 245 goto out_inode_unlock; 246 } 247 248 if (is_wb_truncate || dax_truncate) 249 fuse_set_nowrite(inode); 250 251 err = fuse_do_open(fm, get_node_id(inode), file, isdir); 252 if (!err) 253 fuse_finish_open(inode, file); 254 255 if (is_wb_truncate || dax_truncate) 256 fuse_release_nowrite(inode); 257 if (!err) { 258 struct fuse_file *ff = file->private_data; 259 260 if (fc->atomic_o_trunc && (file->f_flags & O_TRUNC)) 261 truncate_pagecache(inode, 0); 262 else if (!(ff->open_flags & FOPEN_KEEP_CACHE)) 263 invalidate_inode_pages2(inode->i_mapping); 264 } 265 if (dax_truncate) 266 filemap_invalidate_unlock(inode->i_mapping); 267 out_inode_unlock: 268 if (is_wb_truncate || dax_truncate) 269 inode_unlock(inode); 270 271 return err; 272 } 273 274 static void fuse_prepare_release(struct fuse_inode *fi, struct fuse_file *ff, 275 unsigned int flags, int opcode) 276 { 277 struct fuse_conn *fc = ff->fm->fc; 278 struct fuse_release_args *ra = ff->release_args; 279 280 /* Inode is NULL on error path of fuse_create_open() */ 281 if (likely(fi)) { 282 spin_lock(&fi->lock); 283 list_del(&ff->write_entry); 284 spin_unlock(&fi->lock); 285 } 286 spin_lock(&fc->lock); 287 if (!RB_EMPTY_NODE(&ff->polled_node)) 288 rb_erase(&ff->polled_node, &fc->polled_files); 289 spin_unlock(&fc->lock); 290 291 wake_up_interruptible_all(&ff->poll_wait); 292 293 ra->inarg.fh = ff->fh; 294 ra->inarg.flags = flags; 295 ra->args.in_numargs = 1; 296 ra->args.in_args[0].size = sizeof(struct fuse_release_in); 297 ra->args.in_args[0].value = &ra->inarg; 298 ra->args.opcode = opcode; 299 ra->args.nodeid = ff->nodeid; 300 ra->args.force = true; 301 ra->args.nocreds = true; 302 } 303 304 void fuse_file_release(struct inode *inode, struct fuse_file *ff, 305 unsigned int open_flags, fl_owner_t id, bool isdir) 306 { 307 struct fuse_inode *fi = get_fuse_inode(inode); 308 struct fuse_release_args *ra = ff->release_args; 309 int opcode = isdir ? FUSE_RELEASEDIR : FUSE_RELEASE; 310 311 fuse_prepare_release(fi, ff, open_flags, opcode); 312 313 if (ff->flock) { 314 ra->inarg.release_flags |= FUSE_RELEASE_FLOCK_UNLOCK; 315 ra->inarg.lock_owner = fuse_lock_owner_id(ff->fm->fc, id); 316 } 317 /* Hold inode until release is finished */ 318 ra->inode = igrab(inode); 319 320 /* 321 * Normally this will send the RELEASE request, however if 322 * some asynchronous READ or WRITE requests are outstanding, 323 * the sending will be delayed. 324 * 325 * Make the release synchronous if this is a fuseblk mount, 326 * synchronous RELEASE is allowed (and desirable) in this case 327 * because the server can be trusted not to screw up. 328 */ 329 fuse_file_put(ff, ff->fm->fc->destroy, isdir); 330 } 331 332 void fuse_release_common(struct file *file, bool isdir) 333 { 334 fuse_file_release(file_inode(file), file->private_data, file->f_flags, 335 (fl_owner_t) file, isdir); 336 } 337 338 static int fuse_open(struct inode *inode, struct file *file) 339 { 340 return fuse_open_common(inode, file, false); 341 } 342 343 static int fuse_release(struct inode *inode, struct file *file) 344 { 345 struct fuse_conn *fc = get_fuse_conn(inode); 346 347 /* 348 * Dirty pages might remain despite write_inode_now() call from 349 * fuse_flush() due to writes racing with the close. 350 */ 351 if (fc->writeback_cache) 352 write_inode_now(inode, 1); 353 354 fuse_release_common(file, false); 355 356 /* return value is ignored by VFS */ 357 return 0; 358 } 359 360 void fuse_sync_release(struct fuse_inode *fi, struct fuse_file *ff, 361 unsigned int flags) 362 { 363 WARN_ON(refcount_read(&ff->count) > 1); 364 fuse_prepare_release(fi, ff, flags, FUSE_RELEASE); 365 /* 366 * iput(NULL) is a no-op and since the refcount is 1 and everything's 367 * synchronous, we are fine with not doing igrab() here" 368 */ 369 fuse_file_put(ff, true, false); 370 } 371 EXPORT_SYMBOL_GPL(fuse_sync_release); 372 373 /* 374 * Scramble the ID space with XTEA, so that the value of the files_struct 375 * pointer is not exposed to userspace. 376 */ 377 u64 fuse_lock_owner_id(struct fuse_conn *fc, fl_owner_t id) 378 { 379 u32 *k = fc->scramble_key; 380 u64 v = (unsigned long) id; 381 u32 v0 = v; 382 u32 v1 = v >> 32; 383 u32 sum = 0; 384 int i; 385 386 for (i = 0; i < 32; i++) { 387 v0 += ((v1 << 4 ^ v1 >> 5) + v1) ^ (sum + k[sum & 3]); 388 sum += 0x9E3779B9; 389 v1 += ((v0 << 4 ^ v0 >> 5) + v0) ^ (sum + k[sum>>11 & 3]); 390 } 391 392 return (u64) v0 + ((u64) v1 << 32); 393 } 394 395 struct fuse_writepage_args { 396 struct fuse_io_args ia; 397 struct rb_node writepages_entry; 398 struct list_head queue_entry; 399 struct fuse_writepage_args *next; 400 struct inode *inode; 401 struct fuse_sync_bucket *bucket; 402 }; 403 404 static struct fuse_writepage_args *fuse_find_writeback(struct fuse_inode *fi, 405 pgoff_t idx_from, pgoff_t idx_to) 406 { 407 struct rb_node *n; 408 409 n = fi->writepages.rb_node; 410 411 while (n) { 412 struct fuse_writepage_args *wpa; 413 pgoff_t curr_index; 414 415 wpa = rb_entry(n, struct fuse_writepage_args, writepages_entry); 416 WARN_ON(get_fuse_inode(wpa->inode) != fi); 417 curr_index = wpa->ia.write.in.offset >> PAGE_SHIFT; 418 if (idx_from >= curr_index + wpa->ia.ap.num_pages) 419 n = n->rb_right; 420 else if (idx_to < curr_index) 421 n = n->rb_left; 422 else 423 return wpa; 424 } 425 return NULL; 426 } 427 428 /* 429 * Check if any page in a range is under writeback 430 * 431 * This is currently done by walking the list of writepage requests 432 * for the inode, which can be pretty inefficient. 433 */ 434 static bool fuse_range_is_writeback(struct inode *inode, pgoff_t idx_from, 435 pgoff_t idx_to) 436 { 437 struct fuse_inode *fi = get_fuse_inode(inode); 438 bool found; 439 440 spin_lock(&fi->lock); 441 found = fuse_find_writeback(fi, idx_from, idx_to); 442 spin_unlock(&fi->lock); 443 444 return found; 445 } 446 447 static inline bool fuse_page_is_writeback(struct inode *inode, pgoff_t index) 448 { 449 return fuse_range_is_writeback(inode, index, index); 450 } 451 452 /* 453 * Wait for page writeback to be completed. 454 * 455 * Since fuse doesn't rely on the VM writeback tracking, this has to 456 * use some other means. 457 */ 458 static void fuse_wait_on_page_writeback(struct inode *inode, pgoff_t index) 459 { 460 struct fuse_inode *fi = get_fuse_inode(inode); 461 462 wait_event(fi->page_waitq, !fuse_page_is_writeback(inode, index)); 463 } 464 465 /* 466 * Wait for all pending writepages on the inode to finish. 467 * 468 * This is currently done by blocking further writes with FUSE_NOWRITE 469 * and waiting for all sent writes to complete. 470 * 471 * This must be called under i_mutex, otherwise the FUSE_NOWRITE usage 472 * could conflict with truncation. 473 */ 474 static void fuse_sync_writes(struct inode *inode) 475 { 476 fuse_set_nowrite(inode); 477 fuse_release_nowrite(inode); 478 } 479 480 static int fuse_flush(struct file *file, fl_owner_t id) 481 { 482 struct inode *inode = file_inode(file); 483 struct fuse_mount *fm = get_fuse_mount(inode); 484 struct fuse_file *ff = file->private_data; 485 struct fuse_flush_in inarg; 486 FUSE_ARGS(args); 487 int err; 488 489 if (fuse_is_bad(inode)) 490 return -EIO; 491 492 if (ff->open_flags & FOPEN_NOFLUSH && !fm->fc->writeback_cache) 493 return 0; 494 495 err = write_inode_now(inode, 1); 496 if (err) 497 return err; 498 499 inode_lock(inode); 500 fuse_sync_writes(inode); 501 inode_unlock(inode); 502 503 err = filemap_check_errors(file->f_mapping); 504 if (err) 505 return err; 506 507 err = 0; 508 if (fm->fc->no_flush) 509 goto inval_attr_out; 510 511 memset(&inarg, 0, sizeof(inarg)); 512 inarg.fh = ff->fh; 513 inarg.lock_owner = fuse_lock_owner_id(fm->fc, id); 514 args.opcode = FUSE_FLUSH; 515 args.nodeid = get_node_id(inode); 516 args.in_numargs = 1; 517 args.in_args[0].size = sizeof(inarg); 518 args.in_args[0].value = &inarg; 519 args.force = true; 520 521 err = fuse_simple_request(fm, &args); 522 if (err == -ENOSYS) { 523 fm->fc->no_flush = 1; 524 err = 0; 525 } 526 527 inval_attr_out: 528 /* 529 * In memory i_blocks is not maintained by fuse, if writeback cache is 530 * enabled, i_blocks from cached attr may not be accurate. 531 */ 532 if (!err && fm->fc->writeback_cache) 533 fuse_invalidate_attr_mask(inode, STATX_BLOCKS); 534 return err; 535 } 536 537 int fuse_fsync_common(struct file *file, loff_t start, loff_t end, 538 int datasync, int opcode) 539 { 540 struct inode *inode = file->f_mapping->host; 541 struct fuse_mount *fm = get_fuse_mount(inode); 542 struct fuse_file *ff = file->private_data; 543 FUSE_ARGS(args); 544 struct fuse_fsync_in inarg; 545 546 memset(&inarg, 0, sizeof(inarg)); 547 inarg.fh = ff->fh; 548 inarg.fsync_flags = datasync ? FUSE_FSYNC_FDATASYNC : 0; 549 args.opcode = opcode; 550 args.nodeid = get_node_id(inode); 551 args.in_numargs = 1; 552 args.in_args[0].size = sizeof(inarg); 553 args.in_args[0].value = &inarg; 554 return fuse_simple_request(fm, &args); 555 } 556 557 static int fuse_fsync(struct file *file, loff_t start, loff_t end, 558 int datasync) 559 { 560 struct inode *inode = file->f_mapping->host; 561 struct fuse_conn *fc = get_fuse_conn(inode); 562 int err; 563 564 if (fuse_is_bad(inode)) 565 return -EIO; 566 567 inode_lock(inode); 568 569 /* 570 * Start writeback against all dirty pages of the inode, then 571 * wait for all outstanding writes, before sending the FSYNC 572 * request. 573 */ 574 err = file_write_and_wait_range(file, start, end); 575 if (err) 576 goto out; 577 578 fuse_sync_writes(inode); 579 580 /* 581 * Due to implementation of fuse writeback 582 * file_write_and_wait_range() does not catch errors. 583 * We have to do this directly after fuse_sync_writes() 584 */ 585 err = file_check_and_advance_wb_err(file); 586 if (err) 587 goto out; 588 589 err = sync_inode_metadata(inode, 1); 590 if (err) 591 goto out; 592 593 if (fc->no_fsync) 594 goto out; 595 596 err = fuse_fsync_common(file, start, end, datasync, FUSE_FSYNC); 597 if (err == -ENOSYS) { 598 fc->no_fsync = 1; 599 err = 0; 600 } 601 out: 602 inode_unlock(inode); 603 604 return err; 605 } 606 607 void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos, 608 size_t count, int opcode) 609 { 610 struct fuse_file *ff = file->private_data; 611 struct fuse_args *args = &ia->ap.args; 612 613 ia->read.in.fh = ff->fh; 614 ia->read.in.offset = pos; 615 ia->read.in.size = count; 616 ia->read.in.flags = file->f_flags; 617 args->opcode = opcode; 618 args->nodeid = ff->nodeid; 619 args->in_numargs = 1; 620 args->in_args[0].size = sizeof(ia->read.in); 621 args->in_args[0].value = &ia->read.in; 622 args->out_argvar = true; 623 args->out_numargs = 1; 624 args->out_args[0].size = count; 625 } 626 627 static void fuse_release_user_pages(struct fuse_args_pages *ap, 628 bool should_dirty) 629 { 630 unsigned int i; 631 632 for (i = 0; i < ap->num_pages; i++) { 633 if (should_dirty) 634 set_page_dirty_lock(ap->pages[i]); 635 put_page(ap->pages[i]); 636 } 637 } 638 639 static void fuse_io_release(struct kref *kref) 640 { 641 kfree(container_of(kref, struct fuse_io_priv, refcnt)); 642 } 643 644 static ssize_t fuse_get_res_by_io(struct fuse_io_priv *io) 645 { 646 if (io->err) 647 return io->err; 648 649 if (io->bytes >= 0 && io->write) 650 return -EIO; 651 652 return io->bytes < 0 ? io->size : io->bytes; 653 } 654 655 /** 656 * In case of short read, the caller sets 'pos' to the position of 657 * actual end of fuse request in IO request. Otherwise, if bytes_requested 658 * == bytes_transferred or rw == WRITE, the caller sets 'pos' to -1. 659 * 660 * An example: 661 * User requested DIO read of 64K. It was split into two 32K fuse requests, 662 * both submitted asynchronously. The first of them was ACKed by userspace as 663 * fully completed (req->out.args[0].size == 32K) resulting in pos == -1. The 664 * second request was ACKed as short, e.g. only 1K was read, resulting in 665 * pos == 33K. 666 * 667 * Thus, when all fuse requests are completed, the minimal non-negative 'pos' 668 * will be equal to the length of the longest contiguous fragment of 669 * transferred data starting from the beginning of IO request. 670 */ 671 static void fuse_aio_complete(struct fuse_io_priv *io, int err, ssize_t pos) 672 { 673 int left; 674 675 spin_lock(&io->lock); 676 if (err) 677 io->err = io->err ? : err; 678 else if (pos >= 0 && (io->bytes < 0 || pos < io->bytes)) 679 io->bytes = pos; 680 681 left = --io->reqs; 682 if (!left && io->blocking) 683 complete(io->done); 684 spin_unlock(&io->lock); 685 686 if (!left && !io->blocking) { 687 ssize_t res = fuse_get_res_by_io(io); 688 689 if (res >= 0) { 690 struct inode *inode = file_inode(io->iocb->ki_filp); 691 struct fuse_conn *fc = get_fuse_conn(inode); 692 struct fuse_inode *fi = get_fuse_inode(inode); 693 694 spin_lock(&fi->lock); 695 fi->attr_version = atomic64_inc_return(&fc->attr_version); 696 spin_unlock(&fi->lock); 697 } 698 699 io->iocb->ki_complete(io->iocb, res); 700 } 701 702 kref_put(&io->refcnt, fuse_io_release); 703 } 704 705 static struct fuse_io_args *fuse_io_alloc(struct fuse_io_priv *io, 706 unsigned int npages) 707 { 708 struct fuse_io_args *ia; 709 710 ia = kzalloc(sizeof(*ia), GFP_KERNEL); 711 if (ia) { 712 ia->io = io; 713 ia->ap.pages = fuse_pages_alloc(npages, GFP_KERNEL, 714 &ia->ap.descs); 715 if (!ia->ap.pages) { 716 kfree(ia); 717 ia = NULL; 718 } 719 } 720 return ia; 721 } 722 723 static void fuse_io_free(struct fuse_io_args *ia) 724 { 725 kfree(ia->ap.pages); 726 kfree(ia); 727 } 728 729 static void fuse_aio_complete_req(struct fuse_mount *fm, struct fuse_args *args, 730 int err) 731 { 732 struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args); 733 struct fuse_io_priv *io = ia->io; 734 ssize_t pos = -1; 735 736 fuse_release_user_pages(&ia->ap, io->should_dirty); 737 738 if (err) { 739 /* Nothing */ 740 } else if (io->write) { 741 if (ia->write.out.size > ia->write.in.size) { 742 err = -EIO; 743 } else if (ia->write.in.size != ia->write.out.size) { 744 pos = ia->write.in.offset - io->offset + 745 ia->write.out.size; 746 } 747 } else { 748 u32 outsize = args->out_args[0].size; 749 750 if (ia->read.in.size != outsize) 751 pos = ia->read.in.offset - io->offset + outsize; 752 } 753 754 fuse_aio_complete(io, err, pos); 755 fuse_io_free(ia); 756 } 757 758 static ssize_t fuse_async_req_send(struct fuse_mount *fm, 759 struct fuse_io_args *ia, size_t num_bytes) 760 { 761 ssize_t err; 762 struct fuse_io_priv *io = ia->io; 763 764 spin_lock(&io->lock); 765 kref_get(&io->refcnt); 766 io->size += num_bytes; 767 io->reqs++; 768 spin_unlock(&io->lock); 769 770 ia->ap.args.end = fuse_aio_complete_req; 771 ia->ap.args.may_block = io->should_dirty; 772 err = fuse_simple_background(fm, &ia->ap.args, GFP_KERNEL); 773 if (err) 774 fuse_aio_complete_req(fm, &ia->ap.args, err); 775 776 return num_bytes; 777 } 778 779 static ssize_t fuse_send_read(struct fuse_io_args *ia, loff_t pos, size_t count, 780 fl_owner_t owner) 781 { 782 struct file *file = ia->io->iocb->ki_filp; 783 struct fuse_file *ff = file->private_data; 784 struct fuse_mount *fm = ff->fm; 785 786 fuse_read_args_fill(ia, file, pos, count, FUSE_READ); 787 if (owner != NULL) { 788 ia->read.in.read_flags |= FUSE_READ_LOCKOWNER; 789 ia->read.in.lock_owner = fuse_lock_owner_id(fm->fc, owner); 790 } 791 792 if (ia->io->async) 793 return fuse_async_req_send(fm, ia, count); 794 795 return fuse_simple_request(fm, &ia->ap.args); 796 } 797 798 static void fuse_read_update_size(struct inode *inode, loff_t size, 799 u64 attr_ver) 800 { 801 struct fuse_conn *fc = get_fuse_conn(inode); 802 struct fuse_inode *fi = get_fuse_inode(inode); 803 804 spin_lock(&fi->lock); 805 if (attr_ver >= fi->attr_version && size < inode->i_size && 806 !test_bit(FUSE_I_SIZE_UNSTABLE, &fi->state)) { 807 fi->attr_version = atomic64_inc_return(&fc->attr_version); 808 i_size_write(inode, size); 809 } 810 spin_unlock(&fi->lock); 811 } 812 813 static void fuse_short_read(struct inode *inode, u64 attr_ver, size_t num_read, 814 struct fuse_args_pages *ap) 815 { 816 struct fuse_conn *fc = get_fuse_conn(inode); 817 818 /* 819 * If writeback_cache is enabled, a short read means there's a hole in 820 * the file. Some data after the hole is in page cache, but has not 821 * reached the client fs yet. So the hole is not present there. 822 */ 823 if (!fc->writeback_cache) { 824 loff_t pos = page_offset(ap->pages[0]) + num_read; 825 fuse_read_update_size(inode, pos, attr_ver); 826 } 827 } 828 829 static int fuse_do_readpage(struct file *file, struct page *page) 830 { 831 struct inode *inode = page->mapping->host; 832 struct fuse_mount *fm = get_fuse_mount(inode); 833 loff_t pos = page_offset(page); 834 struct fuse_page_desc desc = { .length = PAGE_SIZE }; 835 struct fuse_io_args ia = { 836 .ap.args.page_zeroing = true, 837 .ap.args.out_pages = true, 838 .ap.num_pages = 1, 839 .ap.pages = &page, 840 .ap.descs = &desc, 841 }; 842 ssize_t res; 843 u64 attr_ver; 844 845 /* 846 * Page writeback can extend beyond the lifetime of the 847 * page-cache page, so make sure we read a properly synced 848 * page. 849 */ 850 fuse_wait_on_page_writeback(inode, page->index); 851 852 attr_ver = fuse_get_attr_version(fm->fc); 853 854 /* Don't overflow end offset */ 855 if (pos + (desc.length - 1) == LLONG_MAX) 856 desc.length--; 857 858 fuse_read_args_fill(&ia, file, pos, desc.length, FUSE_READ); 859 res = fuse_simple_request(fm, &ia.ap.args); 860 if (res < 0) 861 return res; 862 /* 863 * Short read means EOF. If file size is larger, truncate it 864 */ 865 if (res < desc.length) 866 fuse_short_read(inode, attr_ver, res, &ia.ap); 867 868 SetPageUptodate(page); 869 870 return 0; 871 } 872 873 static int fuse_read_folio(struct file *file, struct folio *folio) 874 { 875 struct page *page = &folio->page; 876 struct inode *inode = page->mapping->host; 877 int err; 878 879 err = -EIO; 880 if (fuse_is_bad(inode)) 881 goto out; 882 883 err = fuse_do_readpage(file, page); 884 fuse_invalidate_atime(inode); 885 out: 886 unlock_page(page); 887 return err; 888 } 889 890 static void fuse_readpages_end(struct fuse_mount *fm, struct fuse_args *args, 891 int err) 892 { 893 int i; 894 struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args); 895 struct fuse_args_pages *ap = &ia->ap; 896 size_t count = ia->read.in.size; 897 size_t num_read = args->out_args[0].size; 898 struct address_space *mapping = NULL; 899 900 for (i = 0; mapping == NULL && i < ap->num_pages; i++) 901 mapping = ap->pages[i]->mapping; 902 903 if (mapping) { 904 struct inode *inode = mapping->host; 905 906 /* 907 * Short read means EOF. If file size is larger, truncate it 908 */ 909 if (!err && num_read < count) 910 fuse_short_read(inode, ia->read.attr_ver, num_read, ap); 911 912 fuse_invalidate_atime(inode); 913 } 914 915 for (i = 0; i < ap->num_pages; i++) { 916 struct page *page = ap->pages[i]; 917 918 if (!err) 919 SetPageUptodate(page); 920 else 921 SetPageError(page); 922 unlock_page(page); 923 put_page(page); 924 } 925 if (ia->ff) 926 fuse_file_put(ia->ff, false, false); 927 928 fuse_io_free(ia); 929 } 930 931 static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file) 932 { 933 struct fuse_file *ff = file->private_data; 934 struct fuse_mount *fm = ff->fm; 935 struct fuse_args_pages *ap = &ia->ap; 936 loff_t pos = page_offset(ap->pages[0]); 937 size_t count = ap->num_pages << PAGE_SHIFT; 938 ssize_t res; 939 int err; 940 941 ap->args.out_pages = true; 942 ap->args.page_zeroing = true; 943 ap->args.page_replace = true; 944 945 /* Don't overflow end offset */ 946 if (pos + (count - 1) == LLONG_MAX) { 947 count--; 948 ap->descs[ap->num_pages - 1].length--; 949 } 950 WARN_ON((loff_t) (pos + count) < 0); 951 952 fuse_read_args_fill(ia, file, pos, count, FUSE_READ); 953 ia->read.attr_ver = fuse_get_attr_version(fm->fc); 954 if (fm->fc->async_read) { 955 ia->ff = fuse_file_get(ff); 956 ap->args.end = fuse_readpages_end; 957 err = fuse_simple_background(fm, &ap->args, GFP_KERNEL); 958 if (!err) 959 return; 960 } else { 961 res = fuse_simple_request(fm, &ap->args); 962 err = res < 0 ? res : 0; 963 } 964 fuse_readpages_end(fm, &ap->args, err); 965 } 966 967 static void fuse_readahead(struct readahead_control *rac) 968 { 969 struct inode *inode = rac->mapping->host; 970 struct fuse_conn *fc = get_fuse_conn(inode); 971 unsigned int i, max_pages, nr_pages = 0; 972 973 if (fuse_is_bad(inode)) 974 return; 975 976 max_pages = min_t(unsigned int, fc->max_pages, 977 fc->max_read / PAGE_SIZE); 978 979 for (;;) { 980 struct fuse_io_args *ia; 981 struct fuse_args_pages *ap; 982 983 if (fc->num_background >= fc->congestion_threshold && 984 rac->ra->async_size >= readahead_count(rac)) 985 /* 986 * Congested and only async pages left, so skip the 987 * rest. 988 */ 989 break; 990 991 nr_pages = readahead_count(rac) - nr_pages; 992 if (nr_pages > max_pages) 993 nr_pages = max_pages; 994 if (nr_pages == 0) 995 break; 996 ia = fuse_io_alloc(NULL, nr_pages); 997 if (!ia) 998 return; 999 ap = &ia->ap; 1000 nr_pages = __readahead_batch(rac, ap->pages, nr_pages); 1001 for (i = 0; i < nr_pages; i++) { 1002 fuse_wait_on_page_writeback(inode, 1003 readahead_index(rac) + i); 1004 ap->descs[i].length = PAGE_SIZE; 1005 } 1006 ap->num_pages = nr_pages; 1007 fuse_send_readpages(ia, rac->file); 1008 } 1009 } 1010 1011 static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to) 1012 { 1013 struct inode *inode = iocb->ki_filp->f_mapping->host; 1014 struct fuse_conn *fc = get_fuse_conn(inode); 1015 1016 /* 1017 * In auto invalidate mode, always update attributes on read. 1018 * Otherwise, only update if we attempt to read past EOF (to ensure 1019 * i_size is up to date). 1020 */ 1021 if (fc->auto_inval_data || 1022 (iocb->ki_pos + iov_iter_count(to) > i_size_read(inode))) { 1023 int err; 1024 err = fuse_update_attributes(inode, iocb->ki_filp, STATX_SIZE); 1025 if (err) 1026 return err; 1027 } 1028 1029 return generic_file_read_iter(iocb, to); 1030 } 1031 1032 static void fuse_write_args_fill(struct fuse_io_args *ia, struct fuse_file *ff, 1033 loff_t pos, size_t count) 1034 { 1035 struct fuse_args *args = &ia->ap.args; 1036 1037 ia->write.in.fh = ff->fh; 1038 ia->write.in.offset = pos; 1039 ia->write.in.size = count; 1040 args->opcode = FUSE_WRITE; 1041 args->nodeid = ff->nodeid; 1042 args->in_numargs = 2; 1043 if (ff->fm->fc->minor < 9) 1044 args->in_args[0].size = FUSE_COMPAT_WRITE_IN_SIZE; 1045 else 1046 args->in_args[0].size = sizeof(ia->write.in); 1047 args->in_args[0].value = &ia->write.in; 1048 args->in_args[1].size = count; 1049 args->out_numargs = 1; 1050 args->out_args[0].size = sizeof(ia->write.out); 1051 args->out_args[0].value = &ia->write.out; 1052 } 1053 1054 static unsigned int fuse_write_flags(struct kiocb *iocb) 1055 { 1056 unsigned int flags = iocb->ki_filp->f_flags; 1057 1058 if (iocb_is_dsync(iocb)) 1059 flags |= O_DSYNC; 1060 if (iocb->ki_flags & IOCB_SYNC) 1061 flags |= O_SYNC; 1062 1063 return flags; 1064 } 1065 1066 static ssize_t fuse_send_write(struct fuse_io_args *ia, loff_t pos, 1067 size_t count, fl_owner_t owner) 1068 { 1069 struct kiocb *iocb = ia->io->iocb; 1070 struct file *file = iocb->ki_filp; 1071 struct fuse_file *ff = file->private_data; 1072 struct fuse_mount *fm = ff->fm; 1073 struct fuse_write_in *inarg = &ia->write.in; 1074 ssize_t err; 1075 1076 fuse_write_args_fill(ia, ff, pos, count); 1077 inarg->flags = fuse_write_flags(iocb); 1078 if (owner != NULL) { 1079 inarg->write_flags |= FUSE_WRITE_LOCKOWNER; 1080 inarg->lock_owner = fuse_lock_owner_id(fm->fc, owner); 1081 } 1082 1083 if (ia->io->async) 1084 return fuse_async_req_send(fm, ia, count); 1085 1086 err = fuse_simple_request(fm, &ia->ap.args); 1087 if (!err && ia->write.out.size > count) 1088 err = -EIO; 1089 1090 return err ?: ia->write.out.size; 1091 } 1092 1093 bool fuse_write_update_attr(struct inode *inode, loff_t pos, ssize_t written) 1094 { 1095 struct fuse_conn *fc = get_fuse_conn(inode); 1096 struct fuse_inode *fi = get_fuse_inode(inode); 1097 bool ret = false; 1098 1099 spin_lock(&fi->lock); 1100 fi->attr_version = atomic64_inc_return(&fc->attr_version); 1101 if (written > 0 && pos > inode->i_size) { 1102 i_size_write(inode, pos); 1103 ret = true; 1104 } 1105 spin_unlock(&fi->lock); 1106 1107 fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE); 1108 1109 return ret; 1110 } 1111 1112 static ssize_t fuse_send_write_pages(struct fuse_io_args *ia, 1113 struct kiocb *iocb, struct inode *inode, 1114 loff_t pos, size_t count) 1115 { 1116 struct fuse_args_pages *ap = &ia->ap; 1117 struct file *file = iocb->ki_filp; 1118 struct fuse_file *ff = file->private_data; 1119 struct fuse_mount *fm = ff->fm; 1120 unsigned int offset, i; 1121 bool short_write; 1122 int err; 1123 1124 for (i = 0; i < ap->num_pages; i++) 1125 fuse_wait_on_page_writeback(inode, ap->pages[i]->index); 1126 1127 fuse_write_args_fill(ia, ff, pos, count); 1128 ia->write.in.flags = fuse_write_flags(iocb); 1129 if (fm->fc->handle_killpriv_v2 && !capable(CAP_FSETID)) 1130 ia->write.in.write_flags |= FUSE_WRITE_KILL_SUIDGID; 1131 1132 err = fuse_simple_request(fm, &ap->args); 1133 if (!err && ia->write.out.size > count) 1134 err = -EIO; 1135 1136 short_write = ia->write.out.size < count; 1137 offset = ap->descs[0].offset; 1138 count = ia->write.out.size; 1139 for (i = 0; i < ap->num_pages; i++) { 1140 struct page *page = ap->pages[i]; 1141 1142 if (err) { 1143 ClearPageUptodate(page); 1144 } else { 1145 if (count >= PAGE_SIZE - offset) 1146 count -= PAGE_SIZE - offset; 1147 else { 1148 if (short_write) 1149 ClearPageUptodate(page); 1150 count = 0; 1151 } 1152 offset = 0; 1153 } 1154 if (ia->write.page_locked && (i == ap->num_pages - 1)) 1155 unlock_page(page); 1156 put_page(page); 1157 } 1158 1159 return err; 1160 } 1161 1162 static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia, 1163 struct address_space *mapping, 1164 struct iov_iter *ii, loff_t pos, 1165 unsigned int max_pages) 1166 { 1167 struct fuse_args_pages *ap = &ia->ap; 1168 struct fuse_conn *fc = get_fuse_conn(mapping->host); 1169 unsigned offset = pos & (PAGE_SIZE - 1); 1170 size_t count = 0; 1171 int err; 1172 1173 ap->args.in_pages = true; 1174 ap->descs[0].offset = offset; 1175 1176 do { 1177 size_t tmp; 1178 struct page *page; 1179 pgoff_t index = pos >> PAGE_SHIFT; 1180 size_t bytes = min_t(size_t, PAGE_SIZE - offset, 1181 iov_iter_count(ii)); 1182 1183 bytes = min_t(size_t, bytes, fc->max_write - count); 1184 1185 again: 1186 err = -EFAULT; 1187 if (fault_in_iov_iter_readable(ii, bytes)) 1188 break; 1189 1190 err = -ENOMEM; 1191 page = grab_cache_page_write_begin(mapping, index); 1192 if (!page) 1193 break; 1194 1195 if (mapping_writably_mapped(mapping)) 1196 flush_dcache_page(page); 1197 1198 tmp = copy_page_from_iter_atomic(page, offset, bytes, ii); 1199 flush_dcache_page(page); 1200 1201 if (!tmp) { 1202 unlock_page(page); 1203 put_page(page); 1204 goto again; 1205 } 1206 1207 err = 0; 1208 ap->pages[ap->num_pages] = page; 1209 ap->descs[ap->num_pages].length = tmp; 1210 ap->num_pages++; 1211 1212 count += tmp; 1213 pos += tmp; 1214 offset += tmp; 1215 if (offset == PAGE_SIZE) 1216 offset = 0; 1217 1218 /* If we copied full page, mark it uptodate */ 1219 if (tmp == PAGE_SIZE) 1220 SetPageUptodate(page); 1221 1222 if (PageUptodate(page)) { 1223 unlock_page(page); 1224 } else { 1225 ia->write.page_locked = true; 1226 break; 1227 } 1228 if (!fc->big_writes) 1229 break; 1230 } while (iov_iter_count(ii) && count < fc->max_write && 1231 ap->num_pages < max_pages && offset == 0); 1232 1233 return count > 0 ? count : err; 1234 } 1235 1236 static inline unsigned int fuse_wr_pages(loff_t pos, size_t len, 1237 unsigned int max_pages) 1238 { 1239 return min_t(unsigned int, 1240 ((pos + len - 1) >> PAGE_SHIFT) - 1241 (pos >> PAGE_SHIFT) + 1, 1242 max_pages); 1243 } 1244 1245 static ssize_t fuse_perform_write(struct kiocb *iocb, 1246 struct address_space *mapping, 1247 struct iov_iter *ii, loff_t pos) 1248 { 1249 struct inode *inode = mapping->host; 1250 struct fuse_conn *fc = get_fuse_conn(inode); 1251 struct fuse_inode *fi = get_fuse_inode(inode); 1252 int err = 0; 1253 ssize_t res = 0; 1254 1255 if (inode->i_size < pos + iov_iter_count(ii)) 1256 set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); 1257 1258 do { 1259 ssize_t count; 1260 struct fuse_io_args ia = {}; 1261 struct fuse_args_pages *ap = &ia.ap; 1262 unsigned int nr_pages = fuse_wr_pages(pos, iov_iter_count(ii), 1263 fc->max_pages); 1264 1265 ap->pages = fuse_pages_alloc(nr_pages, GFP_KERNEL, &ap->descs); 1266 if (!ap->pages) { 1267 err = -ENOMEM; 1268 break; 1269 } 1270 1271 count = fuse_fill_write_pages(&ia, mapping, ii, pos, nr_pages); 1272 if (count <= 0) { 1273 err = count; 1274 } else { 1275 err = fuse_send_write_pages(&ia, iocb, inode, 1276 pos, count); 1277 if (!err) { 1278 size_t num_written = ia.write.out.size; 1279 1280 res += num_written; 1281 pos += num_written; 1282 1283 /* break out of the loop on short write */ 1284 if (num_written != count) 1285 err = -EIO; 1286 } 1287 } 1288 kfree(ap->pages); 1289 } while (!err && iov_iter_count(ii)); 1290 1291 fuse_write_update_attr(inode, pos, res); 1292 clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); 1293 1294 return res > 0 ? res : err; 1295 } 1296 1297 static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from) 1298 { 1299 struct file *file = iocb->ki_filp; 1300 struct address_space *mapping = file->f_mapping; 1301 ssize_t written = 0; 1302 ssize_t written_buffered = 0; 1303 struct inode *inode = mapping->host; 1304 ssize_t err; 1305 struct fuse_conn *fc = get_fuse_conn(inode); 1306 loff_t endbyte = 0; 1307 1308 if (fc->writeback_cache) { 1309 /* Update size (EOF optimization) and mode (SUID clearing) */ 1310 err = fuse_update_attributes(mapping->host, file, 1311 STATX_SIZE | STATX_MODE); 1312 if (err) 1313 return err; 1314 1315 if (fc->handle_killpriv_v2 && 1316 should_remove_suid(file_dentry(file))) { 1317 goto writethrough; 1318 } 1319 1320 return generic_file_write_iter(iocb, from); 1321 } 1322 1323 writethrough: 1324 inode_lock(inode); 1325 1326 /* We can write back this queue in page reclaim */ 1327 current->backing_dev_info = inode_to_bdi(inode); 1328 1329 err = generic_write_checks(iocb, from); 1330 if (err <= 0) 1331 goto out; 1332 1333 err = file_remove_privs(file); 1334 if (err) 1335 goto out; 1336 1337 err = file_update_time(file); 1338 if (err) 1339 goto out; 1340 1341 if (iocb->ki_flags & IOCB_DIRECT) { 1342 loff_t pos = iocb->ki_pos; 1343 written = generic_file_direct_write(iocb, from); 1344 if (written < 0 || !iov_iter_count(from)) 1345 goto out; 1346 1347 pos += written; 1348 1349 written_buffered = fuse_perform_write(iocb, mapping, from, pos); 1350 if (written_buffered < 0) { 1351 err = written_buffered; 1352 goto out; 1353 } 1354 endbyte = pos + written_buffered - 1; 1355 1356 err = filemap_write_and_wait_range(file->f_mapping, pos, 1357 endbyte); 1358 if (err) 1359 goto out; 1360 1361 invalidate_mapping_pages(file->f_mapping, 1362 pos >> PAGE_SHIFT, 1363 endbyte >> PAGE_SHIFT); 1364 1365 written += written_buffered; 1366 iocb->ki_pos = pos + written_buffered; 1367 } else { 1368 written = fuse_perform_write(iocb, mapping, from, iocb->ki_pos); 1369 if (written >= 0) 1370 iocb->ki_pos += written; 1371 } 1372 out: 1373 current->backing_dev_info = NULL; 1374 inode_unlock(inode); 1375 if (written > 0) 1376 written = generic_write_sync(iocb, written); 1377 1378 return written ? written : err; 1379 } 1380 1381 static inline unsigned long fuse_get_user_addr(const struct iov_iter *ii) 1382 { 1383 return (unsigned long)ii->iov->iov_base + ii->iov_offset; 1384 } 1385 1386 static inline size_t fuse_get_frag_size(const struct iov_iter *ii, 1387 size_t max_size) 1388 { 1389 return min(iov_iter_single_seg_count(ii), max_size); 1390 } 1391 1392 static int fuse_get_user_pages(struct fuse_args_pages *ap, struct iov_iter *ii, 1393 size_t *nbytesp, int write, 1394 unsigned int max_pages) 1395 { 1396 size_t nbytes = 0; /* # bytes already packed in req */ 1397 ssize_t ret = 0; 1398 1399 /* Special case for kernel I/O: can copy directly into the buffer */ 1400 if (iov_iter_is_kvec(ii)) { 1401 unsigned long user_addr = fuse_get_user_addr(ii); 1402 size_t frag_size = fuse_get_frag_size(ii, *nbytesp); 1403 1404 if (write) 1405 ap->args.in_args[1].value = (void *) user_addr; 1406 else 1407 ap->args.out_args[0].value = (void *) user_addr; 1408 1409 iov_iter_advance(ii, frag_size); 1410 *nbytesp = frag_size; 1411 return 0; 1412 } 1413 1414 while (nbytes < *nbytesp && ap->num_pages < max_pages) { 1415 unsigned npages; 1416 size_t start; 1417 ret = iov_iter_get_pages2(ii, &ap->pages[ap->num_pages], 1418 *nbytesp - nbytes, 1419 max_pages - ap->num_pages, 1420 &start); 1421 if (ret < 0) 1422 break; 1423 1424 nbytes += ret; 1425 1426 ret += start; 1427 npages = DIV_ROUND_UP(ret, PAGE_SIZE); 1428 1429 ap->descs[ap->num_pages].offset = start; 1430 fuse_page_descs_length_init(ap->descs, ap->num_pages, npages); 1431 1432 ap->num_pages += npages; 1433 ap->descs[ap->num_pages - 1].length -= 1434 (PAGE_SIZE - ret) & (PAGE_SIZE - 1); 1435 } 1436 1437 ap->args.user_pages = true; 1438 if (write) 1439 ap->args.in_pages = true; 1440 else 1441 ap->args.out_pages = true; 1442 1443 *nbytesp = nbytes; 1444 1445 return ret < 0 ? ret : 0; 1446 } 1447 1448 ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter, 1449 loff_t *ppos, int flags) 1450 { 1451 int write = flags & FUSE_DIO_WRITE; 1452 int cuse = flags & FUSE_DIO_CUSE; 1453 struct file *file = io->iocb->ki_filp; 1454 struct inode *inode = file->f_mapping->host; 1455 struct fuse_file *ff = file->private_data; 1456 struct fuse_conn *fc = ff->fm->fc; 1457 size_t nmax = write ? fc->max_write : fc->max_read; 1458 loff_t pos = *ppos; 1459 size_t count = iov_iter_count(iter); 1460 pgoff_t idx_from = pos >> PAGE_SHIFT; 1461 pgoff_t idx_to = (pos + count - 1) >> PAGE_SHIFT; 1462 ssize_t res = 0; 1463 int err = 0; 1464 struct fuse_io_args *ia; 1465 unsigned int max_pages; 1466 1467 max_pages = iov_iter_npages(iter, fc->max_pages); 1468 ia = fuse_io_alloc(io, max_pages); 1469 if (!ia) 1470 return -ENOMEM; 1471 1472 if (!cuse && fuse_range_is_writeback(inode, idx_from, idx_to)) { 1473 if (!write) 1474 inode_lock(inode); 1475 fuse_sync_writes(inode); 1476 if (!write) 1477 inode_unlock(inode); 1478 } 1479 1480 io->should_dirty = !write && user_backed_iter(iter); 1481 while (count) { 1482 ssize_t nres; 1483 fl_owner_t owner = current->files; 1484 size_t nbytes = min(count, nmax); 1485 1486 err = fuse_get_user_pages(&ia->ap, iter, &nbytes, write, 1487 max_pages); 1488 if (err && !nbytes) 1489 break; 1490 1491 if (write) { 1492 if (!capable(CAP_FSETID)) 1493 ia->write.in.write_flags |= FUSE_WRITE_KILL_SUIDGID; 1494 1495 nres = fuse_send_write(ia, pos, nbytes, owner); 1496 } else { 1497 nres = fuse_send_read(ia, pos, nbytes, owner); 1498 } 1499 1500 if (!io->async || nres < 0) { 1501 fuse_release_user_pages(&ia->ap, io->should_dirty); 1502 fuse_io_free(ia); 1503 } 1504 ia = NULL; 1505 if (nres < 0) { 1506 iov_iter_revert(iter, nbytes); 1507 err = nres; 1508 break; 1509 } 1510 WARN_ON(nres > nbytes); 1511 1512 count -= nres; 1513 res += nres; 1514 pos += nres; 1515 if (nres != nbytes) { 1516 iov_iter_revert(iter, nbytes - nres); 1517 break; 1518 } 1519 if (count) { 1520 max_pages = iov_iter_npages(iter, fc->max_pages); 1521 ia = fuse_io_alloc(io, max_pages); 1522 if (!ia) 1523 break; 1524 } 1525 } 1526 if (ia) 1527 fuse_io_free(ia); 1528 if (res > 0) 1529 *ppos = pos; 1530 1531 return res > 0 ? res : err; 1532 } 1533 EXPORT_SYMBOL_GPL(fuse_direct_io); 1534 1535 static ssize_t __fuse_direct_read(struct fuse_io_priv *io, 1536 struct iov_iter *iter, 1537 loff_t *ppos) 1538 { 1539 ssize_t res; 1540 struct inode *inode = file_inode(io->iocb->ki_filp); 1541 1542 res = fuse_direct_io(io, iter, ppos, 0); 1543 1544 fuse_invalidate_atime(inode); 1545 1546 return res; 1547 } 1548 1549 static ssize_t fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter); 1550 1551 static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to) 1552 { 1553 ssize_t res; 1554 1555 if (!is_sync_kiocb(iocb) && iocb->ki_flags & IOCB_DIRECT) { 1556 res = fuse_direct_IO(iocb, to); 1557 } else { 1558 struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); 1559 1560 res = __fuse_direct_read(&io, to, &iocb->ki_pos); 1561 } 1562 1563 return res; 1564 } 1565 1566 static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from) 1567 { 1568 struct inode *inode = file_inode(iocb->ki_filp); 1569 struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb); 1570 ssize_t res; 1571 1572 /* Don't allow parallel writes to the same file */ 1573 inode_lock(inode); 1574 res = generic_write_checks(iocb, from); 1575 if (res > 0) { 1576 if (!is_sync_kiocb(iocb) && iocb->ki_flags & IOCB_DIRECT) { 1577 res = fuse_direct_IO(iocb, from); 1578 } else { 1579 res = fuse_direct_io(&io, from, &iocb->ki_pos, 1580 FUSE_DIO_WRITE); 1581 fuse_write_update_attr(inode, iocb->ki_pos, res); 1582 } 1583 } 1584 inode_unlock(inode); 1585 1586 return res; 1587 } 1588 1589 static ssize_t fuse_file_read_iter(struct kiocb *iocb, struct iov_iter *to) 1590 { 1591 struct file *file = iocb->ki_filp; 1592 struct fuse_file *ff = file->private_data; 1593 struct inode *inode = file_inode(file); 1594 1595 if (fuse_is_bad(inode)) 1596 return -EIO; 1597 1598 if (FUSE_IS_DAX(inode)) 1599 return fuse_dax_read_iter(iocb, to); 1600 1601 if (!(ff->open_flags & FOPEN_DIRECT_IO)) 1602 return fuse_cache_read_iter(iocb, to); 1603 else 1604 return fuse_direct_read_iter(iocb, to); 1605 } 1606 1607 static ssize_t fuse_file_write_iter(struct kiocb *iocb, struct iov_iter *from) 1608 { 1609 struct file *file = iocb->ki_filp; 1610 struct fuse_file *ff = file->private_data; 1611 struct inode *inode = file_inode(file); 1612 1613 if (fuse_is_bad(inode)) 1614 return -EIO; 1615 1616 if (FUSE_IS_DAX(inode)) 1617 return fuse_dax_write_iter(iocb, from); 1618 1619 if (!(ff->open_flags & FOPEN_DIRECT_IO)) 1620 return fuse_cache_write_iter(iocb, from); 1621 else 1622 return fuse_direct_write_iter(iocb, from); 1623 } 1624 1625 static void fuse_writepage_free(struct fuse_writepage_args *wpa) 1626 { 1627 struct fuse_args_pages *ap = &wpa->ia.ap; 1628 int i; 1629 1630 if (wpa->bucket) 1631 fuse_sync_bucket_dec(wpa->bucket); 1632 1633 for (i = 0; i < ap->num_pages; i++) 1634 __free_page(ap->pages[i]); 1635 1636 if (wpa->ia.ff) 1637 fuse_file_put(wpa->ia.ff, false, false); 1638 1639 kfree(ap->pages); 1640 kfree(wpa); 1641 } 1642 1643 static void fuse_writepage_finish(struct fuse_mount *fm, 1644 struct fuse_writepage_args *wpa) 1645 { 1646 struct fuse_args_pages *ap = &wpa->ia.ap; 1647 struct inode *inode = wpa->inode; 1648 struct fuse_inode *fi = get_fuse_inode(inode); 1649 struct backing_dev_info *bdi = inode_to_bdi(inode); 1650 int i; 1651 1652 for (i = 0; i < ap->num_pages; i++) { 1653 dec_wb_stat(&bdi->wb, WB_WRITEBACK); 1654 dec_node_page_state(ap->pages[i], NR_WRITEBACK_TEMP); 1655 wb_writeout_inc(&bdi->wb); 1656 } 1657 wake_up(&fi->page_waitq); 1658 } 1659 1660 /* Called under fi->lock, may release and reacquire it */ 1661 static void fuse_send_writepage(struct fuse_mount *fm, 1662 struct fuse_writepage_args *wpa, loff_t size) 1663 __releases(fi->lock) 1664 __acquires(fi->lock) 1665 { 1666 struct fuse_writepage_args *aux, *next; 1667 struct fuse_inode *fi = get_fuse_inode(wpa->inode); 1668 struct fuse_write_in *inarg = &wpa->ia.write.in; 1669 struct fuse_args *args = &wpa->ia.ap.args; 1670 __u64 data_size = wpa->ia.ap.num_pages * PAGE_SIZE; 1671 int err; 1672 1673 fi->writectr++; 1674 if (inarg->offset + data_size <= size) { 1675 inarg->size = data_size; 1676 } else if (inarg->offset < size) { 1677 inarg->size = size - inarg->offset; 1678 } else { 1679 /* Got truncated off completely */ 1680 goto out_free; 1681 } 1682 1683 args->in_args[1].size = inarg->size; 1684 args->force = true; 1685 args->nocreds = true; 1686 1687 err = fuse_simple_background(fm, args, GFP_ATOMIC); 1688 if (err == -ENOMEM) { 1689 spin_unlock(&fi->lock); 1690 err = fuse_simple_background(fm, args, GFP_NOFS | __GFP_NOFAIL); 1691 spin_lock(&fi->lock); 1692 } 1693 1694 /* Fails on broken connection only */ 1695 if (unlikely(err)) 1696 goto out_free; 1697 1698 return; 1699 1700 out_free: 1701 fi->writectr--; 1702 rb_erase(&wpa->writepages_entry, &fi->writepages); 1703 fuse_writepage_finish(fm, wpa); 1704 spin_unlock(&fi->lock); 1705 1706 /* After fuse_writepage_finish() aux request list is private */ 1707 for (aux = wpa->next; aux; aux = next) { 1708 next = aux->next; 1709 aux->next = NULL; 1710 fuse_writepage_free(aux); 1711 } 1712 1713 fuse_writepage_free(wpa); 1714 spin_lock(&fi->lock); 1715 } 1716 1717 /* 1718 * If fi->writectr is positive (no truncate or fsync going on) send 1719 * all queued writepage requests. 1720 * 1721 * Called with fi->lock 1722 */ 1723 void fuse_flush_writepages(struct inode *inode) 1724 __releases(fi->lock) 1725 __acquires(fi->lock) 1726 { 1727 struct fuse_mount *fm = get_fuse_mount(inode); 1728 struct fuse_inode *fi = get_fuse_inode(inode); 1729 loff_t crop = i_size_read(inode); 1730 struct fuse_writepage_args *wpa; 1731 1732 while (fi->writectr >= 0 && !list_empty(&fi->queued_writes)) { 1733 wpa = list_entry(fi->queued_writes.next, 1734 struct fuse_writepage_args, queue_entry); 1735 list_del_init(&wpa->queue_entry); 1736 fuse_send_writepage(fm, wpa, crop); 1737 } 1738 } 1739 1740 static struct fuse_writepage_args *fuse_insert_writeback(struct rb_root *root, 1741 struct fuse_writepage_args *wpa) 1742 { 1743 pgoff_t idx_from = wpa->ia.write.in.offset >> PAGE_SHIFT; 1744 pgoff_t idx_to = idx_from + wpa->ia.ap.num_pages - 1; 1745 struct rb_node **p = &root->rb_node; 1746 struct rb_node *parent = NULL; 1747 1748 WARN_ON(!wpa->ia.ap.num_pages); 1749 while (*p) { 1750 struct fuse_writepage_args *curr; 1751 pgoff_t curr_index; 1752 1753 parent = *p; 1754 curr = rb_entry(parent, struct fuse_writepage_args, 1755 writepages_entry); 1756 WARN_ON(curr->inode != wpa->inode); 1757 curr_index = curr->ia.write.in.offset >> PAGE_SHIFT; 1758 1759 if (idx_from >= curr_index + curr->ia.ap.num_pages) 1760 p = &(*p)->rb_right; 1761 else if (idx_to < curr_index) 1762 p = &(*p)->rb_left; 1763 else 1764 return curr; 1765 } 1766 1767 rb_link_node(&wpa->writepages_entry, parent, p); 1768 rb_insert_color(&wpa->writepages_entry, root); 1769 return NULL; 1770 } 1771 1772 static void tree_insert(struct rb_root *root, struct fuse_writepage_args *wpa) 1773 { 1774 WARN_ON(fuse_insert_writeback(root, wpa)); 1775 } 1776 1777 static void fuse_writepage_end(struct fuse_mount *fm, struct fuse_args *args, 1778 int error) 1779 { 1780 struct fuse_writepage_args *wpa = 1781 container_of(args, typeof(*wpa), ia.ap.args); 1782 struct inode *inode = wpa->inode; 1783 struct fuse_inode *fi = get_fuse_inode(inode); 1784 struct fuse_conn *fc = get_fuse_conn(inode); 1785 1786 mapping_set_error(inode->i_mapping, error); 1787 /* 1788 * A writeback finished and this might have updated mtime/ctime on 1789 * server making local mtime/ctime stale. Hence invalidate attrs. 1790 * Do this only if writeback_cache is not enabled. If writeback_cache 1791 * is enabled, we trust local ctime/mtime. 1792 */ 1793 if (!fc->writeback_cache) 1794 fuse_invalidate_attr_mask(inode, FUSE_STATX_MODIFY); 1795 spin_lock(&fi->lock); 1796 rb_erase(&wpa->writepages_entry, &fi->writepages); 1797 while (wpa->next) { 1798 struct fuse_mount *fm = get_fuse_mount(inode); 1799 struct fuse_write_in *inarg = &wpa->ia.write.in; 1800 struct fuse_writepage_args *next = wpa->next; 1801 1802 wpa->next = next->next; 1803 next->next = NULL; 1804 next->ia.ff = fuse_file_get(wpa->ia.ff); 1805 tree_insert(&fi->writepages, next); 1806 1807 /* 1808 * Skip fuse_flush_writepages() to make it easy to crop requests 1809 * based on primary request size. 1810 * 1811 * 1st case (trivial): there are no concurrent activities using 1812 * fuse_set/release_nowrite. Then we're on safe side because 1813 * fuse_flush_writepages() would call fuse_send_writepage() 1814 * anyway. 1815 * 1816 * 2nd case: someone called fuse_set_nowrite and it is waiting 1817 * now for completion of all in-flight requests. This happens 1818 * rarely and no more than once per page, so this should be 1819 * okay. 1820 * 1821 * 3rd case: someone (e.g. fuse_do_setattr()) is in the middle 1822 * of fuse_set_nowrite..fuse_release_nowrite section. The fact 1823 * that fuse_set_nowrite returned implies that all in-flight 1824 * requests were completed along with all of their secondary 1825 * requests. Further primary requests are blocked by negative 1826 * writectr. Hence there cannot be any in-flight requests and 1827 * no invocations of fuse_writepage_end() while we're in 1828 * fuse_set_nowrite..fuse_release_nowrite section. 1829 */ 1830 fuse_send_writepage(fm, next, inarg->offset + inarg->size); 1831 } 1832 fi->writectr--; 1833 fuse_writepage_finish(fm, wpa); 1834 spin_unlock(&fi->lock); 1835 fuse_writepage_free(wpa); 1836 } 1837 1838 static struct fuse_file *__fuse_write_file_get(struct fuse_inode *fi) 1839 { 1840 struct fuse_file *ff; 1841 1842 spin_lock(&fi->lock); 1843 ff = list_first_entry_or_null(&fi->write_files, struct fuse_file, 1844 write_entry); 1845 if (ff) 1846 fuse_file_get(ff); 1847 spin_unlock(&fi->lock); 1848 1849 return ff; 1850 } 1851 1852 static struct fuse_file *fuse_write_file_get(struct fuse_inode *fi) 1853 { 1854 struct fuse_file *ff = __fuse_write_file_get(fi); 1855 WARN_ON(!ff); 1856 return ff; 1857 } 1858 1859 int fuse_write_inode(struct inode *inode, struct writeback_control *wbc) 1860 { 1861 struct fuse_inode *fi = get_fuse_inode(inode); 1862 struct fuse_file *ff; 1863 int err; 1864 1865 /* 1866 * Inode is always written before the last reference is dropped and 1867 * hence this should not be reached from reclaim. 1868 * 1869 * Writing back the inode from reclaim can deadlock if the request 1870 * processing itself needs an allocation. Allocations triggering 1871 * reclaim while serving a request can't be prevented, because it can 1872 * involve any number of unrelated userspace processes. 1873 */ 1874 WARN_ON(wbc->for_reclaim); 1875 1876 ff = __fuse_write_file_get(fi); 1877 err = fuse_flush_times(inode, ff); 1878 if (ff) 1879 fuse_file_put(ff, false, false); 1880 1881 return err; 1882 } 1883 1884 static struct fuse_writepage_args *fuse_writepage_args_alloc(void) 1885 { 1886 struct fuse_writepage_args *wpa; 1887 struct fuse_args_pages *ap; 1888 1889 wpa = kzalloc(sizeof(*wpa), GFP_NOFS); 1890 if (wpa) { 1891 ap = &wpa->ia.ap; 1892 ap->num_pages = 0; 1893 ap->pages = fuse_pages_alloc(1, GFP_NOFS, &ap->descs); 1894 if (!ap->pages) { 1895 kfree(wpa); 1896 wpa = NULL; 1897 } 1898 } 1899 return wpa; 1900 1901 } 1902 1903 static void fuse_writepage_add_to_bucket(struct fuse_conn *fc, 1904 struct fuse_writepage_args *wpa) 1905 { 1906 if (!fc->sync_fs) 1907 return; 1908 1909 rcu_read_lock(); 1910 /* Prevent resurrection of dead bucket in unlikely race with syncfs */ 1911 do { 1912 wpa->bucket = rcu_dereference(fc->curr_bucket); 1913 } while (unlikely(!atomic_inc_not_zero(&wpa->bucket->count))); 1914 rcu_read_unlock(); 1915 } 1916 1917 static int fuse_writepage_locked(struct page *page) 1918 { 1919 struct address_space *mapping = page->mapping; 1920 struct inode *inode = mapping->host; 1921 struct fuse_conn *fc = get_fuse_conn(inode); 1922 struct fuse_inode *fi = get_fuse_inode(inode); 1923 struct fuse_writepage_args *wpa; 1924 struct fuse_args_pages *ap; 1925 struct page *tmp_page; 1926 int error = -ENOMEM; 1927 1928 set_page_writeback(page); 1929 1930 wpa = fuse_writepage_args_alloc(); 1931 if (!wpa) 1932 goto err; 1933 ap = &wpa->ia.ap; 1934 1935 tmp_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM); 1936 if (!tmp_page) 1937 goto err_free; 1938 1939 error = -EIO; 1940 wpa->ia.ff = fuse_write_file_get(fi); 1941 if (!wpa->ia.ff) 1942 goto err_nofile; 1943 1944 fuse_writepage_add_to_bucket(fc, wpa); 1945 fuse_write_args_fill(&wpa->ia, wpa->ia.ff, page_offset(page), 0); 1946 1947 copy_highpage(tmp_page, page); 1948 wpa->ia.write.in.write_flags |= FUSE_WRITE_CACHE; 1949 wpa->next = NULL; 1950 ap->args.in_pages = true; 1951 ap->num_pages = 1; 1952 ap->pages[0] = tmp_page; 1953 ap->descs[0].offset = 0; 1954 ap->descs[0].length = PAGE_SIZE; 1955 ap->args.end = fuse_writepage_end; 1956 wpa->inode = inode; 1957 1958 inc_wb_stat(&inode_to_bdi(inode)->wb, WB_WRITEBACK); 1959 inc_node_page_state(tmp_page, NR_WRITEBACK_TEMP); 1960 1961 spin_lock(&fi->lock); 1962 tree_insert(&fi->writepages, wpa); 1963 list_add_tail(&wpa->queue_entry, &fi->queued_writes); 1964 fuse_flush_writepages(inode); 1965 spin_unlock(&fi->lock); 1966 1967 end_page_writeback(page); 1968 1969 return 0; 1970 1971 err_nofile: 1972 __free_page(tmp_page); 1973 err_free: 1974 kfree(wpa); 1975 err: 1976 mapping_set_error(page->mapping, error); 1977 end_page_writeback(page); 1978 return error; 1979 } 1980 1981 static int fuse_writepage(struct page *page, struct writeback_control *wbc) 1982 { 1983 struct fuse_conn *fc = get_fuse_conn(page->mapping->host); 1984 int err; 1985 1986 if (fuse_page_is_writeback(page->mapping->host, page->index)) { 1987 /* 1988 * ->writepages() should be called for sync() and friends. We 1989 * should only get here on direct reclaim and then we are 1990 * allowed to skip a page which is already in flight 1991 */ 1992 WARN_ON(wbc->sync_mode == WB_SYNC_ALL); 1993 1994 redirty_page_for_writepage(wbc, page); 1995 unlock_page(page); 1996 return 0; 1997 } 1998 1999 if (wbc->sync_mode == WB_SYNC_NONE && 2000 fc->num_background >= fc->congestion_threshold) 2001 return AOP_WRITEPAGE_ACTIVATE; 2002 2003 err = fuse_writepage_locked(page); 2004 unlock_page(page); 2005 2006 return err; 2007 } 2008 2009 struct fuse_fill_wb_data { 2010 struct fuse_writepage_args *wpa; 2011 struct fuse_file *ff; 2012 struct inode *inode; 2013 struct page **orig_pages; 2014 unsigned int max_pages; 2015 }; 2016 2017 static bool fuse_pages_realloc(struct fuse_fill_wb_data *data) 2018 { 2019 struct fuse_args_pages *ap = &data->wpa->ia.ap; 2020 struct fuse_conn *fc = get_fuse_conn(data->inode); 2021 struct page **pages; 2022 struct fuse_page_desc *descs; 2023 unsigned int npages = min_t(unsigned int, 2024 max_t(unsigned int, data->max_pages * 2, 2025 FUSE_DEFAULT_MAX_PAGES_PER_REQ), 2026 fc->max_pages); 2027 WARN_ON(npages <= data->max_pages); 2028 2029 pages = fuse_pages_alloc(npages, GFP_NOFS, &descs); 2030 if (!pages) 2031 return false; 2032 2033 memcpy(pages, ap->pages, sizeof(struct page *) * ap->num_pages); 2034 memcpy(descs, ap->descs, sizeof(struct fuse_page_desc) * ap->num_pages); 2035 kfree(ap->pages); 2036 ap->pages = pages; 2037 ap->descs = descs; 2038 data->max_pages = npages; 2039 2040 return true; 2041 } 2042 2043 static void fuse_writepages_send(struct fuse_fill_wb_data *data) 2044 { 2045 struct fuse_writepage_args *wpa = data->wpa; 2046 struct inode *inode = data->inode; 2047 struct fuse_inode *fi = get_fuse_inode(inode); 2048 int num_pages = wpa->ia.ap.num_pages; 2049 int i; 2050 2051 wpa->ia.ff = fuse_file_get(data->ff); 2052 spin_lock(&fi->lock); 2053 list_add_tail(&wpa->queue_entry, &fi->queued_writes); 2054 fuse_flush_writepages(inode); 2055 spin_unlock(&fi->lock); 2056 2057 for (i = 0; i < num_pages; i++) 2058 end_page_writeback(data->orig_pages[i]); 2059 } 2060 2061 /* 2062 * Check under fi->lock if the page is under writeback, and insert it onto the 2063 * rb_tree if not. Otherwise iterate auxiliary write requests, to see if there's 2064 * one already added for a page at this offset. If there's none, then insert 2065 * this new request onto the auxiliary list, otherwise reuse the existing one by 2066 * swapping the new temp page with the old one. 2067 */ 2068 static bool fuse_writepage_add(struct fuse_writepage_args *new_wpa, 2069 struct page *page) 2070 { 2071 struct fuse_inode *fi = get_fuse_inode(new_wpa->inode); 2072 struct fuse_writepage_args *tmp; 2073 struct fuse_writepage_args *old_wpa; 2074 struct fuse_args_pages *new_ap = &new_wpa->ia.ap; 2075 2076 WARN_ON(new_ap->num_pages != 0); 2077 new_ap->num_pages = 1; 2078 2079 spin_lock(&fi->lock); 2080 old_wpa = fuse_insert_writeback(&fi->writepages, new_wpa); 2081 if (!old_wpa) { 2082 spin_unlock(&fi->lock); 2083 return true; 2084 } 2085 2086 for (tmp = old_wpa->next; tmp; tmp = tmp->next) { 2087 pgoff_t curr_index; 2088 2089 WARN_ON(tmp->inode != new_wpa->inode); 2090 curr_index = tmp->ia.write.in.offset >> PAGE_SHIFT; 2091 if (curr_index == page->index) { 2092 WARN_ON(tmp->ia.ap.num_pages != 1); 2093 swap(tmp->ia.ap.pages[0], new_ap->pages[0]); 2094 break; 2095 } 2096 } 2097 2098 if (!tmp) { 2099 new_wpa->next = old_wpa->next; 2100 old_wpa->next = new_wpa; 2101 } 2102 2103 spin_unlock(&fi->lock); 2104 2105 if (tmp) { 2106 struct backing_dev_info *bdi = inode_to_bdi(new_wpa->inode); 2107 2108 dec_wb_stat(&bdi->wb, WB_WRITEBACK); 2109 dec_node_page_state(new_ap->pages[0], NR_WRITEBACK_TEMP); 2110 wb_writeout_inc(&bdi->wb); 2111 fuse_writepage_free(new_wpa); 2112 } 2113 2114 return false; 2115 } 2116 2117 static bool fuse_writepage_need_send(struct fuse_conn *fc, struct page *page, 2118 struct fuse_args_pages *ap, 2119 struct fuse_fill_wb_data *data) 2120 { 2121 WARN_ON(!ap->num_pages); 2122 2123 /* 2124 * Being under writeback is unlikely but possible. For example direct 2125 * read to an mmaped fuse file will set the page dirty twice; once when 2126 * the pages are faulted with get_user_pages(), and then after the read 2127 * completed. 2128 */ 2129 if (fuse_page_is_writeback(data->inode, page->index)) 2130 return true; 2131 2132 /* Reached max pages */ 2133 if (ap->num_pages == fc->max_pages) 2134 return true; 2135 2136 /* Reached max write bytes */ 2137 if ((ap->num_pages + 1) * PAGE_SIZE > fc->max_write) 2138 return true; 2139 2140 /* Discontinuity */ 2141 if (data->orig_pages[ap->num_pages - 1]->index + 1 != page->index) 2142 return true; 2143 2144 /* Need to grow the pages array? If so, did the expansion fail? */ 2145 if (ap->num_pages == data->max_pages && !fuse_pages_realloc(data)) 2146 return true; 2147 2148 return false; 2149 } 2150 2151 static int fuse_writepages_fill(struct page *page, 2152 struct writeback_control *wbc, void *_data) 2153 { 2154 struct fuse_fill_wb_data *data = _data; 2155 struct fuse_writepage_args *wpa = data->wpa; 2156 struct fuse_args_pages *ap = &wpa->ia.ap; 2157 struct inode *inode = data->inode; 2158 struct fuse_inode *fi = get_fuse_inode(inode); 2159 struct fuse_conn *fc = get_fuse_conn(inode); 2160 struct page *tmp_page; 2161 int err; 2162 2163 if (!data->ff) { 2164 err = -EIO; 2165 data->ff = fuse_write_file_get(fi); 2166 if (!data->ff) 2167 goto out_unlock; 2168 } 2169 2170 if (wpa && fuse_writepage_need_send(fc, page, ap, data)) { 2171 fuse_writepages_send(data); 2172 data->wpa = NULL; 2173 } 2174 2175 err = -ENOMEM; 2176 tmp_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM); 2177 if (!tmp_page) 2178 goto out_unlock; 2179 2180 /* 2181 * The page must not be redirtied until the writeout is completed 2182 * (i.e. userspace has sent a reply to the write request). Otherwise 2183 * there could be more than one temporary page instance for each real 2184 * page. 2185 * 2186 * This is ensured by holding the page lock in page_mkwrite() while 2187 * checking fuse_page_is_writeback(). We already hold the page lock 2188 * since clear_page_dirty_for_io() and keep it held until we add the 2189 * request to the fi->writepages list and increment ap->num_pages. 2190 * After this fuse_page_is_writeback() will indicate that the page is 2191 * under writeback, so we can release the page lock. 2192 */ 2193 if (data->wpa == NULL) { 2194 err = -ENOMEM; 2195 wpa = fuse_writepage_args_alloc(); 2196 if (!wpa) { 2197 __free_page(tmp_page); 2198 goto out_unlock; 2199 } 2200 fuse_writepage_add_to_bucket(fc, wpa); 2201 2202 data->max_pages = 1; 2203 2204 ap = &wpa->ia.ap; 2205 fuse_write_args_fill(&wpa->ia, data->ff, page_offset(page), 0); 2206 wpa->ia.write.in.write_flags |= FUSE_WRITE_CACHE; 2207 wpa->next = NULL; 2208 ap->args.in_pages = true; 2209 ap->args.end = fuse_writepage_end; 2210 ap->num_pages = 0; 2211 wpa->inode = inode; 2212 } 2213 set_page_writeback(page); 2214 2215 copy_highpage(tmp_page, page); 2216 ap->pages[ap->num_pages] = tmp_page; 2217 ap->descs[ap->num_pages].offset = 0; 2218 ap->descs[ap->num_pages].length = PAGE_SIZE; 2219 data->orig_pages[ap->num_pages] = page; 2220 2221 inc_wb_stat(&inode_to_bdi(inode)->wb, WB_WRITEBACK); 2222 inc_node_page_state(tmp_page, NR_WRITEBACK_TEMP); 2223 2224 err = 0; 2225 if (data->wpa) { 2226 /* 2227 * Protected by fi->lock against concurrent access by 2228 * fuse_page_is_writeback(). 2229 */ 2230 spin_lock(&fi->lock); 2231 ap->num_pages++; 2232 spin_unlock(&fi->lock); 2233 } else if (fuse_writepage_add(wpa, page)) { 2234 data->wpa = wpa; 2235 } else { 2236 end_page_writeback(page); 2237 } 2238 out_unlock: 2239 unlock_page(page); 2240 2241 return err; 2242 } 2243 2244 static int fuse_writepages(struct address_space *mapping, 2245 struct writeback_control *wbc) 2246 { 2247 struct inode *inode = mapping->host; 2248 struct fuse_conn *fc = get_fuse_conn(inode); 2249 struct fuse_fill_wb_data data; 2250 int err; 2251 2252 err = -EIO; 2253 if (fuse_is_bad(inode)) 2254 goto out; 2255 2256 if (wbc->sync_mode == WB_SYNC_NONE && 2257 fc->num_background >= fc->congestion_threshold) 2258 return 0; 2259 2260 data.inode = inode; 2261 data.wpa = NULL; 2262 data.ff = NULL; 2263 2264 err = -ENOMEM; 2265 data.orig_pages = kcalloc(fc->max_pages, 2266 sizeof(struct page *), 2267 GFP_NOFS); 2268 if (!data.orig_pages) 2269 goto out; 2270 2271 err = write_cache_pages(mapping, wbc, fuse_writepages_fill, &data); 2272 if (data.wpa) { 2273 WARN_ON(!data.wpa->ia.ap.num_pages); 2274 fuse_writepages_send(&data); 2275 } 2276 if (data.ff) 2277 fuse_file_put(data.ff, false, false); 2278 2279 kfree(data.orig_pages); 2280 out: 2281 return err; 2282 } 2283 2284 /* 2285 * It's worthy to make sure that space is reserved on disk for the write, 2286 * but how to implement it without killing performance need more thinking. 2287 */ 2288 static int fuse_write_begin(struct file *file, struct address_space *mapping, 2289 loff_t pos, unsigned len, struct page **pagep, void **fsdata) 2290 { 2291 pgoff_t index = pos >> PAGE_SHIFT; 2292 struct fuse_conn *fc = get_fuse_conn(file_inode(file)); 2293 struct page *page; 2294 loff_t fsize; 2295 int err = -ENOMEM; 2296 2297 WARN_ON(!fc->writeback_cache); 2298 2299 page = grab_cache_page_write_begin(mapping, index); 2300 if (!page) 2301 goto error; 2302 2303 fuse_wait_on_page_writeback(mapping->host, page->index); 2304 2305 if (PageUptodate(page) || len == PAGE_SIZE) 2306 goto success; 2307 /* 2308 * Check if the start this page comes after the end of file, in which 2309 * case the readpage can be optimized away. 2310 */ 2311 fsize = i_size_read(mapping->host); 2312 if (fsize <= (pos & PAGE_MASK)) { 2313 size_t off = pos & ~PAGE_MASK; 2314 if (off) 2315 zero_user_segment(page, 0, off); 2316 goto success; 2317 } 2318 err = fuse_do_readpage(file, page); 2319 if (err) 2320 goto cleanup; 2321 success: 2322 *pagep = page; 2323 return 0; 2324 2325 cleanup: 2326 unlock_page(page); 2327 put_page(page); 2328 error: 2329 return err; 2330 } 2331 2332 static int fuse_write_end(struct file *file, struct address_space *mapping, 2333 loff_t pos, unsigned len, unsigned copied, 2334 struct page *page, void *fsdata) 2335 { 2336 struct inode *inode = page->mapping->host; 2337 2338 /* Haven't copied anything? Skip zeroing, size extending, dirtying. */ 2339 if (!copied) 2340 goto unlock; 2341 2342 pos += copied; 2343 if (!PageUptodate(page)) { 2344 /* Zero any unwritten bytes at the end of the page */ 2345 size_t endoff = pos & ~PAGE_MASK; 2346 if (endoff) 2347 zero_user_segment(page, endoff, PAGE_SIZE); 2348 SetPageUptodate(page); 2349 } 2350 2351 if (pos > inode->i_size) 2352 i_size_write(inode, pos); 2353 2354 set_page_dirty(page); 2355 2356 unlock: 2357 unlock_page(page); 2358 put_page(page); 2359 2360 return copied; 2361 } 2362 2363 static int fuse_launder_folio(struct folio *folio) 2364 { 2365 int err = 0; 2366 if (folio_clear_dirty_for_io(folio)) { 2367 struct inode *inode = folio->mapping->host; 2368 2369 /* Serialize with pending writeback for the same page */ 2370 fuse_wait_on_page_writeback(inode, folio->index); 2371 err = fuse_writepage_locked(&folio->page); 2372 if (!err) 2373 fuse_wait_on_page_writeback(inode, folio->index); 2374 } 2375 return err; 2376 } 2377 2378 /* 2379 * Write back dirty data/metadata now (there may not be any suitable 2380 * open files later for data) 2381 */ 2382 static void fuse_vma_close(struct vm_area_struct *vma) 2383 { 2384 int err; 2385 2386 err = write_inode_now(vma->vm_file->f_mapping->host, 1); 2387 mapping_set_error(vma->vm_file->f_mapping, err); 2388 } 2389 2390 /* 2391 * Wait for writeback against this page to complete before allowing it 2392 * to be marked dirty again, and hence written back again, possibly 2393 * before the previous writepage completed. 2394 * 2395 * Block here, instead of in ->writepage(), so that the userspace fs 2396 * can only block processes actually operating on the filesystem. 2397 * 2398 * Otherwise unprivileged userspace fs would be able to block 2399 * unrelated: 2400 * 2401 * - page migration 2402 * - sync(2) 2403 * - try_to_free_pages() with order > PAGE_ALLOC_COSTLY_ORDER 2404 */ 2405 static vm_fault_t fuse_page_mkwrite(struct vm_fault *vmf) 2406 { 2407 struct page *page = vmf->page; 2408 struct inode *inode = file_inode(vmf->vma->vm_file); 2409 2410 file_update_time(vmf->vma->vm_file); 2411 lock_page(page); 2412 if (page->mapping != inode->i_mapping) { 2413 unlock_page(page); 2414 return VM_FAULT_NOPAGE; 2415 } 2416 2417 fuse_wait_on_page_writeback(inode, page->index); 2418 return VM_FAULT_LOCKED; 2419 } 2420 2421 static const struct vm_operations_struct fuse_file_vm_ops = { 2422 .close = fuse_vma_close, 2423 .fault = filemap_fault, 2424 .map_pages = filemap_map_pages, 2425 .page_mkwrite = fuse_page_mkwrite, 2426 }; 2427 2428 static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma) 2429 { 2430 struct fuse_file *ff = file->private_data; 2431 2432 /* DAX mmap is superior to direct_io mmap */ 2433 if (FUSE_IS_DAX(file_inode(file))) 2434 return fuse_dax_mmap(file, vma); 2435 2436 if (ff->open_flags & FOPEN_DIRECT_IO) { 2437 /* Can't provide the coherency needed for MAP_SHARED */ 2438 if (vma->vm_flags & VM_MAYSHARE) 2439 return -ENODEV; 2440 2441 invalidate_inode_pages2(file->f_mapping); 2442 2443 return generic_file_mmap(file, vma); 2444 } 2445 2446 if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE)) 2447 fuse_link_write_file(file); 2448 2449 file_accessed(file); 2450 vma->vm_ops = &fuse_file_vm_ops; 2451 return 0; 2452 } 2453 2454 static int convert_fuse_file_lock(struct fuse_conn *fc, 2455 const struct fuse_file_lock *ffl, 2456 struct file_lock *fl) 2457 { 2458 switch (ffl->type) { 2459 case F_UNLCK: 2460 break; 2461 2462 case F_RDLCK: 2463 case F_WRLCK: 2464 if (ffl->start > OFFSET_MAX || ffl->end > OFFSET_MAX || 2465 ffl->end < ffl->start) 2466 return -EIO; 2467 2468 fl->fl_start = ffl->start; 2469 fl->fl_end = ffl->end; 2470 2471 /* 2472 * Convert pid into init's pid namespace. The locks API will 2473 * translate it into the caller's pid namespace. 2474 */ 2475 rcu_read_lock(); 2476 fl->fl_pid = pid_nr_ns(find_pid_ns(ffl->pid, fc->pid_ns), &init_pid_ns); 2477 rcu_read_unlock(); 2478 break; 2479 2480 default: 2481 return -EIO; 2482 } 2483 fl->fl_type = ffl->type; 2484 return 0; 2485 } 2486 2487 static void fuse_lk_fill(struct fuse_args *args, struct file *file, 2488 const struct file_lock *fl, int opcode, pid_t pid, 2489 int flock, struct fuse_lk_in *inarg) 2490 { 2491 struct inode *inode = file_inode(file); 2492 struct fuse_conn *fc = get_fuse_conn(inode); 2493 struct fuse_file *ff = file->private_data; 2494 2495 memset(inarg, 0, sizeof(*inarg)); 2496 inarg->fh = ff->fh; 2497 inarg->owner = fuse_lock_owner_id(fc, fl->fl_owner); 2498 inarg->lk.start = fl->fl_start; 2499 inarg->lk.end = fl->fl_end; 2500 inarg->lk.type = fl->fl_type; 2501 inarg->lk.pid = pid; 2502 if (flock) 2503 inarg->lk_flags |= FUSE_LK_FLOCK; 2504 args->opcode = opcode; 2505 args->nodeid = get_node_id(inode); 2506 args->in_numargs = 1; 2507 args->in_args[0].size = sizeof(*inarg); 2508 args->in_args[0].value = inarg; 2509 } 2510 2511 static int fuse_getlk(struct file *file, struct file_lock *fl) 2512 { 2513 struct inode *inode = file_inode(file); 2514 struct fuse_mount *fm = get_fuse_mount(inode); 2515 FUSE_ARGS(args); 2516 struct fuse_lk_in inarg; 2517 struct fuse_lk_out outarg; 2518 int err; 2519 2520 fuse_lk_fill(&args, file, fl, FUSE_GETLK, 0, 0, &inarg); 2521 args.out_numargs = 1; 2522 args.out_args[0].size = sizeof(outarg); 2523 args.out_args[0].value = &outarg; 2524 err = fuse_simple_request(fm, &args); 2525 if (!err) 2526 err = convert_fuse_file_lock(fm->fc, &outarg.lk, fl); 2527 2528 return err; 2529 } 2530 2531 static int fuse_setlk(struct file *file, struct file_lock *fl, int flock) 2532 { 2533 struct inode *inode = file_inode(file); 2534 struct fuse_mount *fm = get_fuse_mount(inode); 2535 FUSE_ARGS(args); 2536 struct fuse_lk_in inarg; 2537 int opcode = (fl->fl_flags & FL_SLEEP) ? FUSE_SETLKW : FUSE_SETLK; 2538 struct pid *pid = fl->fl_type != F_UNLCK ? task_tgid(current) : NULL; 2539 pid_t pid_nr = pid_nr_ns(pid, fm->fc->pid_ns); 2540 int err; 2541 2542 if (fl->fl_lmops && fl->fl_lmops->lm_grant) { 2543 /* NLM needs asynchronous locks, which we don't support yet */ 2544 return -ENOLCK; 2545 } 2546 2547 /* Unlock on close is handled by the flush method */ 2548 if ((fl->fl_flags & FL_CLOSE_POSIX) == FL_CLOSE_POSIX) 2549 return 0; 2550 2551 fuse_lk_fill(&args, file, fl, opcode, pid_nr, flock, &inarg); 2552 err = fuse_simple_request(fm, &args); 2553 2554 /* locking is restartable */ 2555 if (err == -EINTR) 2556 err = -ERESTARTSYS; 2557 2558 return err; 2559 } 2560 2561 static int fuse_file_lock(struct file *file, int cmd, struct file_lock *fl) 2562 { 2563 struct inode *inode = file_inode(file); 2564 struct fuse_conn *fc = get_fuse_conn(inode); 2565 int err; 2566 2567 if (cmd == F_CANCELLK) { 2568 err = 0; 2569 } else if (cmd == F_GETLK) { 2570 if (fc->no_lock) { 2571 posix_test_lock(file, fl); 2572 err = 0; 2573 } else 2574 err = fuse_getlk(file, fl); 2575 } else { 2576 if (fc->no_lock) 2577 err = posix_lock_file(file, fl, NULL); 2578 else 2579 err = fuse_setlk(file, fl, 0); 2580 } 2581 return err; 2582 } 2583 2584 static int fuse_file_flock(struct file *file, int cmd, struct file_lock *fl) 2585 { 2586 struct inode *inode = file_inode(file); 2587 struct fuse_conn *fc = get_fuse_conn(inode); 2588 int err; 2589 2590 if (fc->no_flock) { 2591 err = locks_lock_file_wait(file, fl); 2592 } else { 2593 struct fuse_file *ff = file->private_data; 2594 2595 /* emulate flock with POSIX locks */ 2596 ff->flock = true; 2597 err = fuse_setlk(file, fl, 1); 2598 } 2599 2600 return err; 2601 } 2602 2603 static sector_t fuse_bmap(struct address_space *mapping, sector_t block) 2604 { 2605 struct inode *inode = mapping->host; 2606 struct fuse_mount *fm = get_fuse_mount(inode); 2607 FUSE_ARGS(args); 2608 struct fuse_bmap_in inarg; 2609 struct fuse_bmap_out outarg; 2610 int err; 2611 2612 if (!inode->i_sb->s_bdev || fm->fc->no_bmap) 2613 return 0; 2614 2615 memset(&inarg, 0, sizeof(inarg)); 2616 inarg.block = block; 2617 inarg.blocksize = inode->i_sb->s_blocksize; 2618 args.opcode = FUSE_BMAP; 2619 args.nodeid = get_node_id(inode); 2620 args.in_numargs = 1; 2621 args.in_args[0].size = sizeof(inarg); 2622 args.in_args[0].value = &inarg; 2623 args.out_numargs = 1; 2624 args.out_args[0].size = sizeof(outarg); 2625 args.out_args[0].value = &outarg; 2626 err = fuse_simple_request(fm, &args); 2627 if (err == -ENOSYS) 2628 fm->fc->no_bmap = 1; 2629 2630 return err ? 0 : outarg.block; 2631 } 2632 2633 static loff_t fuse_lseek(struct file *file, loff_t offset, int whence) 2634 { 2635 struct inode *inode = file->f_mapping->host; 2636 struct fuse_mount *fm = get_fuse_mount(inode); 2637 struct fuse_file *ff = file->private_data; 2638 FUSE_ARGS(args); 2639 struct fuse_lseek_in inarg = { 2640 .fh = ff->fh, 2641 .offset = offset, 2642 .whence = whence 2643 }; 2644 struct fuse_lseek_out outarg; 2645 int err; 2646 2647 if (fm->fc->no_lseek) 2648 goto fallback; 2649 2650 args.opcode = FUSE_LSEEK; 2651 args.nodeid = ff->nodeid; 2652 args.in_numargs = 1; 2653 args.in_args[0].size = sizeof(inarg); 2654 args.in_args[0].value = &inarg; 2655 args.out_numargs = 1; 2656 args.out_args[0].size = sizeof(outarg); 2657 args.out_args[0].value = &outarg; 2658 err = fuse_simple_request(fm, &args); 2659 if (err) { 2660 if (err == -ENOSYS) { 2661 fm->fc->no_lseek = 1; 2662 goto fallback; 2663 } 2664 return err; 2665 } 2666 2667 return vfs_setpos(file, outarg.offset, inode->i_sb->s_maxbytes); 2668 2669 fallback: 2670 err = fuse_update_attributes(inode, file, STATX_SIZE); 2671 if (!err) 2672 return generic_file_llseek(file, offset, whence); 2673 else 2674 return err; 2675 } 2676 2677 static loff_t fuse_file_llseek(struct file *file, loff_t offset, int whence) 2678 { 2679 loff_t retval; 2680 struct inode *inode = file_inode(file); 2681 2682 switch (whence) { 2683 case SEEK_SET: 2684 case SEEK_CUR: 2685 /* No i_mutex protection necessary for SEEK_CUR and SEEK_SET */ 2686 retval = generic_file_llseek(file, offset, whence); 2687 break; 2688 case SEEK_END: 2689 inode_lock(inode); 2690 retval = fuse_update_attributes(inode, file, STATX_SIZE); 2691 if (!retval) 2692 retval = generic_file_llseek(file, offset, whence); 2693 inode_unlock(inode); 2694 break; 2695 case SEEK_HOLE: 2696 case SEEK_DATA: 2697 inode_lock(inode); 2698 retval = fuse_lseek(file, offset, whence); 2699 inode_unlock(inode); 2700 break; 2701 default: 2702 retval = -EINVAL; 2703 } 2704 2705 return retval; 2706 } 2707 2708 /* 2709 * All files which have been polled are linked to RB tree 2710 * fuse_conn->polled_files which is indexed by kh. Walk the tree and 2711 * find the matching one. 2712 */ 2713 static struct rb_node **fuse_find_polled_node(struct fuse_conn *fc, u64 kh, 2714 struct rb_node **parent_out) 2715 { 2716 struct rb_node **link = &fc->polled_files.rb_node; 2717 struct rb_node *last = NULL; 2718 2719 while (*link) { 2720 struct fuse_file *ff; 2721 2722 last = *link; 2723 ff = rb_entry(last, struct fuse_file, polled_node); 2724 2725 if (kh < ff->kh) 2726 link = &last->rb_left; 2727 else if (kh > ff->kh) 2728 link = &last->rb_right; 2729 else 2730 return link; 2731 } 2732 2733 if (parent_out) 2734 *parent_out = last; 2735 return link; 2736 } 2737 2738 /* 2739 * The file is about to be polled. Make sure it's on the polled_files 2740 * RB tree. Note that files once added to the polled_files tree are 2741 * not removed before the file is released. This is because a file 2742 * polled once is likely to be polled again. 2743 */ 2744 static void fuse_register_polled_file(struct fuse_conn *fc, 2745 struct fuse_file *ff) 2746 { 2747 spin_lock(&fc->lock); 2748 if (RB_EMPTY_NODE(&ff->polled_node)) { 2749 struct rb_node **link, *parent; 2750 2751 link = fuse_find_polled_node(fc, ff->kh, &parent); 2752 BUG_ON(*link); 2753 rb_link_node(&ff->polled_node, parent, link); 2754 rb_insert_color(&ff->polled_node, &fc->polled_files); 2755 } 2756 spin_unlock(&fc->lock); 2757 } 2758 2759 __poll_t fuse_file_poll(struct file *file, poll_table *wait) 2760 { 2761 struct fuse_file *ff = file->private_data; 2762 struct fuse_mount *fm = ff->fm; 2763 struct fuse_poll_in inarg = { .fh = ff->fh, .kh = ff->kh }; 2764 struct fuse_poll_out outarg; 2765 FUSE_ARGS(args); 2766 int err; 2767 2768 if (fm->fc->no_poll) 2769 return DEFAULT_POLLMASK; 2770 2771 poll_wait(file, &ff->poll_wait, wait); 2772 inarg.events = mangle_poll(poll_requested_events(wait)); 2773 2774 /* 2775 * Ask for notification iff there's someone waiting for it. 2776 * The client may ignore the flag and always notify. 2777 */ 2778 if (waitqueue_active(&ff->poll_wait)) { 2779 inarg.flags |= FUSE_POLL_SCHEDULE_NOTIFY; 2780 fuse_register_polled_file(fm->fc, ff); 2781 } 2782 2783 args.opcode = FUSE_POLL; 2784 args.nodeid = ff->nodeid; 2785 args.in_numargs = 1; 2786 args.in_args[0].size = sizeof(inarg); 2787 args.in_args[0].value = &inarg; 2788 args.out_numargs = 1; 2789 args.out_args[0].size = sizeof(outarg); 2790 args.out_args[0].value = &outarg; 2791 err = fuse_simple_request(fm, &args); 2792 2793 if (!err) 2794 return demangle_poll(outarg.revents); 2795 if (err == -ENOSYS) { 2796 fm->fc->no_poll = 1; 2797 return DEFAULT_POLLMASK; 2798 } 2799 return EPOLLERR; 2800 } 2801 EXPORT_SYMBOL_GPL(fuse_file_poll); 2802 2803 /* 2804 * This is called from fuse_handle_notify() on FUSE_NOTIFY_POLL and 2805 * wakes up the poll waiters. 2806 */ 2807 int fuse_notify_poll_wakeup(struct fuse_conn *fc, 2808 struct fuse_notify_poll_wakeup_out *outarg) 2809 { 2810 u64 kh = outarg->kh; 2811 struct rb_node **link; 2812 2813 spin_lock(&fc->lock); 2814 2815 link = fuse_find_polled_node(fc, kh, NULL); 2816 if (*link) { 2817 struct fuse_file *ff; 2818 2819 ff = rb_entry(*link, struct fuse_file, polled_node); 2820 wake_up_interruptible_sync(&ff->poll_wait); 2821 } 2822 2823 spin_unlock(&fc->lock); 2824 return 0; 2825 } 2826 2827 static void fuse_do_truncate(struct file *file) 2828 { 2829 struct inode *inode = file->f_mapping->host; 2830 struct iattr attr; 2831 2832 attr.ia_valid = ATTR_SIZE; 2833 attr.ia_size = i_size_read(inode); 2834 2835 attr.ia_file = file; 2836 attr.ia_valid |= ATTR_FILE; 2837 2838 fuse_do_setattr(file_dentry(file), &attr, file); 2839 } 2840 2841 static inline loff_t fuse_round_up(struct fuse_conn *fc, loff_t off) 2842 { 2843 return round_up(off, fc->max_pages << PAGE_SHIFT); 2844 } 2845 2846 static ssize_t 2847 fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter) 2848 { 2849 DECLARE_COMPLETION_ONSTACK(wait); 2850 ssize_t ret = 0; 2851 struct file *file = iocb->ki_filp; 2852 struct fuse_file *ff = file->private_data; 2853 loff_t pos = 0; 2854 struct inode *inode; 2855 loff_t i_size; 2856 size_t count = iov_iter_count(iter), shortened = 0; 2857 loff_t offset = iocb->ki_pos; 2858 struct fuse_io_priv *io; 2859 2860 pos = offset; 2861 inode = file->f_mapping->host; 2862 i_size = i_size_read(inode); 2863 2864 if ((iov_iter_rw(iter) == READ) && (offset >= i_size)) 2865 return 0; 2866 2867 io = kmalloc(sizeof(struct fuse_io_priv), GFP_KERNEL); 2868 if (!io) 2869 return -ENOMEM; 2870 spin_lock_init(&io->lock); 2871 kref_init(&io->refcnt); 2872 io->reqs = 1; 2873 io->bytes = -1; 2874 io->size = 0; 2875 io->offset = offset; 2876 io->write = (iov_iter_rw(iter) == WRITE); 2877 io->err = 0; 2878 /* 2879 * By default, we want to optimize all I/Os with async request 2880 * submission to the client filesystem if supported. 2881 */ 2882 io->async = ff->fm->fc->async_dio; 2883 io->iocb = iocb; 2884 io->blocking = is_sync_kiocb(iocb); 2885 2886 /* optimization for short read */ 2887 if (io->async && !io->write && offset + count > i_size) { 2888 iov_iter_truncate(iter, fuse_round_up(ff->fm->fc, i_size - offset)); 2889 shortened = count - iov_iter_count(iter); 2890 count -= shortened; 2891 } 2892 2893 /* 2894 * We cannot asynchronously extend the size of a file. 2895 * In such case the aio will behave exactly like sync io. 2896 */ 2897 if ((offset + count > i_size) && io->write) 2898 io->blocking = true; 2899 2900 if (io->async && io->blocking) { 2901 /* 2902 * Additional reference to keep io around after 2903 * calling fuse_aio_complete() 2904 */ 2905 kref_get(&io->refcnt); 2906 io->done = &wait; 2907 } 2908 2909 if (iov_iter_rw(iter) == WRITE) { 2910 ret = fuse_direct_io(io, iter, &pos, FUSE_DIO_WRITE); 2911 fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE); 2912 } else { 2913 ret = __fuse_direct_read(io, iter, &pos); 2914 } 2915 iov_iter_reexpand(iter, iov_iter_count(iter) + shortened); 2916 2917 if (io->async) { 2918 bool blocking = io->blocking; 2919 2920 fuse_aio_complete(io, ret < 0 ? ret : 0, -1); 2921 2922 /* we have a non-extending, async request, so return */ 2923 if (!blocking) 2924 return -EIOCBQUEUED; 2925 2926 wait_for_completion(&wait); 2927 ret = fuse_get_res_by_io(io); 2928 } 2929 2930 kref_put(&io->refcnt, fuse_io_release); 2931 2932 if (iov_iter_rw(iter) == WRITE) { 2933 fuse_write_update_attr(inode, pos, ret); 2934 if (ret < 0 && offset + count > i_size) 2935 fuse_do_truncate(file); 2936 } 2937 2938 return ret; 2939 } 2940 2941 static int fuse_writeback_range(struct inode *inode, loff_t start, loff_t end) 2942 { 2943 int err = filemap_write_and_wait_range(inode->i_mapping, start, LLONG_MAX); 2944 2945 if (!err) 2946 fuse_sync_writes(inode); 2947 2948 return err; 2949 } 2950 2951 static long fuse_file_fallocate(struct file *file, int mode, loff_t offset, 2952 loff_t length) 2953 { 2954 struct fuse_file *ff = file->private_data; 2955 struct inode *inode = file_inode(file); 2956 struct fuse_inode *fi = get_fuse_inode(inode); 2957 struct fuse_mount *fm = ff->fm; 2958 FUSE_ARGS(args); 2959 struct fuse_fallocate_in inarg = { 2960 .fh = ff->fh, 2961 .offset = offset, 2962 .length = length, 2963 .mode = mode 2964 }; 2965 int err; 2966 bool lock_inode = !(mode & FALLOC_FL_KEEP_SIZE) || 2967 (mode & (FALLOC_FL_PUNCH_HOLE | 2968 FALLOC_FL_ZERO_RANGE)); 2969 2970 bool block_faults = FUSE_IS_DAX(inode) && lock_inode; 2971 2972 if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE | 2973 FALLOC_FL_ZERO_RANGE)) 2974 return -EOPNOTSUPP; 2975 2976 if (fm->fc->no_fallocate) 2977 return -EOPNOTSUPP; 2978 2979 if (lock_inode) { 2980 inode_lock(inode); 2981 if (block_faults) { 2982 filemap_invalidate_lock(inode->i_mapping); 2983 err = fuse_dax_break_layouts(inode, 0, 0); 2984 if (err) 2985 goto out; 2986 } 2987 2988 if (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE)) { 2989 loff_t endbyte = offset + length - 1; 2990 2991 err = fuse_writeback_range(inode, offset, endbyte); 2992 if (err) 2993 goto out; 2994 } 2995 } 2996 2997 if (!(mode & FALLOC_FL_KEEP_SIZE) && 2998 offset + length > i_size_read(inode)) { 2999 err = inode_newsize_ok(inode, offset + length); 3000 if (err) 3001 goto out; 3002 } 3003 3004 if (!(mode & FALLOC_FL_KEEP_SIZE)) 3005 set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); 3006 3007 args.opcode = FUSE_FALLOCATE; 3008 args.nodeid = ff->nodeid; 3009 args.in_numargs = 1; 3010 args.in_args[0].size = sizeof(inarg); 3011 args.in_args[0].value = &inarg; 3012 err = fuse_simple_request(fm, &args); 3013 if (err == -ENOSYS) { 3014 fm->fc->no_fallocate = 1; 3015 err = -EOPNOTSUPP; 3016 } 3017 if (err) 3018 goto out; 3019 3020 /* we could have extended the file */ 3021 if (!(mode & FALLOC_FL_KEEP_SIZE)) { 3022 if (fuse_write_update_attr(inode, offset + length, length)) 3023 file_update_time(file); 3024 } 3025 3026 if (mode & (FALLOC_FL_PUNCH_HOLE | FALLOC_FL_ZERO_RANGE)) 3027 truncate_pagecache_range(inode, offset, offset + length - 1); 3028 3029 fuse_invalidate_attr_mask(inode, FUSE_STATX_MODSIZE); 3030 3031 out: 3032 if (!(mode & FALLOC_FL_KEEP_SIZE)) 3033 clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state); 3034 3035 if (block_faults) 3036 filemap_invalidate_unlock(inode->i_mapping); 3037 3038 if (lock_inode) 3039 inode_unlock(inode); 3040 3041 fuse_flush_time_update(inode); 3042 3043 return err; 3044 } 3045 3046 static ssize_t __fuse_copy_file_range(struct file *file_in, loff_t pos_in, 3047 struct file *file_out, loff_t pos_out, 3048 size_t len, unsigned int flags) 3049 { 3050 struct fuse_file *ff_in = file_in->private_data; 3051 struct fuse_file *ff_out = file_out->private_data; 3052 struct inode *inode_in = file_inode(file_in); 3053 struct inode *inode_out = file_inode(file_out); 3054 struct fuse_inode *fi_out = get_fuse_inode(inode_out); 3055 struct fuse_mount *fm = ff_in->fm; 3056 struct fuse_conn *fc = fm->fc; 3057 FUSE_ARGS(args); 3058 struct fuse_copy_file_range_in inarg = { 3059 .fh_in = ff_in->fh, 3060 .off_in = pos_in, 3061 .nodeid_out = ff_out->nodeid, 3062 .fh_out = ff_out->fh, 3063 .off_out = pos_out, 3064 .len = len, 3065 .flags = flags 3066 }; 3067 struct fuse_write_out outarg; 3068 ssize_t err; 3069 /* mark unstable when write-back is not used, and file_out gets 3070 * extended */ 3071 bool is_unstable = (!fc->writeback_cache) && 3072 ((pos_out + len) > inode_out->i_size); 3073 3074 if (fc->no_copy_file_range) 3075 return -EOPNOTSUPP; 3076 3077 if (file_inode(file_in)->i_sb != file_inode(file_out)->i_sb) 3078 return -EXDEV; 3079 3080 inode_lock(inode_in); 3081 err = fuse_writeback_range(inode_in, pos_in, pos_in + len - 1); 3082 inode_unlock(inode_in); 3083 if (err) 3084 return err; 3085 3086 inode_lock(inode_out); 3087 3088 err = file_modified(file_out); 3089 if (err) 3090 goto out; 3091 3092 /* 3093 * Write out dirty pages in the destination file before sending the COPY 3094 * request to userspace. After the request is completed, truncate off 3095 * pages (including partial ones) from the cache that have been copied, 3096 * since these contain stale data at that point. 3097 * 3098 * This should be mostly correct, but if the COPY writes to partial 3099 * pages (at the start or end) and the parts not covered by the COPY are 3100 * written through a memory map after calling fuse_writeback_range(), 3101 * then these partial page modifications will be lost on truncation. 3102 * 3103 * It is unlikely that someone would rely on such mixed style 3104 * modifications. Yet this does give less guarantees than if the 3105 * copying was performed with write(2). 3106 * 3107 * To fix this a mapping->invalidate_lock could be used to prevent new 3108 * faults while the copy is ongoing. 3109 */ 3110 err = fuse_writeback_range(inode_out, pos_out, pos_out + len - 1); 3111 if (err) 3112 goto out; 3113 3114 if (is_unstable) 3115 set_bit(FUSE_I_SIZE_UNSTABLE, &fi_out->state); 3116 3117 args.opcode = FUSE_COPY_FILE_RANGE; 3118 args.nodeid = ff_in->nodeid; 3119 args.in_numargs = 1; 3120 args.in_args[0].size = sizeof(inarg); 3121 args.in_args[0].value = &inarg; 3122 args.out_numargs = 1; 3123 args.out_args[0].size = sizeof(outarg); 3124 args.out_args[0].value = &outarg; 3125 err = fuse_simple_request(fm, &args); 3126 if (err == -ENOSYS) { 3127 fc->no_copy_file_range = 1; 3128 err = -EOPNOTSUPP; 3129 } 3130 if (err) 3131 goto out; 3132 3133 truncate_inode_pages_range(inode_out->i_mapping, 3134 ALIGN_DOWN(pos_out, PAGE_SIZE), 3135 ALIGN(pos_out + outarg.size, PAGE_SIZE) - 1); 3136 3137 file_update_time(file_out); 3138 fuse_write_update_attr(inode_out, pos_out + outarg.size, outarg.size); 3139 3140 err = outarg.size; 3141 out: 3142 if (is_unstable) 3143 clear_bit(FUSE_I_SIZE_UNSTABLE, &fi_out->state); 3144 3145 inode_unlock(inode_out); 3146 file_accessed(file_in); 3147 3148 fuse_flush_time_update(inode_out); 3149 3150 return err; 3151 } 3152 3153 static ssize_t fuse_copy_file_range(struct file *src_file, loff_t src_off, 3154 struct file *dst_file, loff_t dst_off, 3155 size_t len, unsigned int flags) 3156 { 3157 ssize_t ret; 3158 3159 ret = __fuse_copy_file_range(src_file, src_off, dst_file, dst_off, 3160 len, flags); 3161 3162 if (ret == -EOPNOTSUPP || ret == -EXDEV) 3163 ret = generic_copy_file_range(src_file, src_off, dst_file, 3164 dst_off, len, flags); 3165 return ret; 3166 } 3167 3168 static const struct file_operations fuse_file_operations = { 3169 .llseek = fuse_file_llseek, 3170 .read_iter = fuse_file_read_iter, 3171 .write_iter = fuse_file_write_iter, 3172 .mmap = fuse_file_mmap, 3173 .open = fuse_open, 3174 .flush = fuse_flush, 3175 .release = fuse_release, 3176 .fsync = fuse_fsync, 3177 .lock = fuse_file_lock, 3178 .get_unmapped_area = thp_get_unmapped_area, 3179 .flock = fuse_file_flock, 3180 .splice_read = generic_file_splice_read, 3181 .splice_write = iter_file_splice_write, 3182 .unlocked_ioctl = fuse_file_ioctl, 3183 .compat_ioctl = fuse_file_compat_ioctl, 3184 .poll = fuse_file_poll, 3185 .fallocate = fuse_file_fallocate, 3186 .copy_file_range = fuse_copy_file_range, 3187 }; 3188 3189 static const struct address_space_operations fuse_file_aops = { 3190 .read_folio = fuse_read_folio, 3191 .readahead = fuse_readahead, 3192 .writepage = fuse_writepage, 3193 .writepages = fuse_writepages, 3194 .launder_folio = fuse_launder_folio, 3195 .dirty_folio = filemap_dirty_folio, 3196 .bmap = fuse_bmap, 3197 .direct_IO = fuse_direct_IO, 3198 .write_begin = fuse_write_begin, 3199 .write_end = fuse_write_end, 3200 }; 3201 3202 void fuse_init_file_inode(struct inode *inode, unsigned int flags) 3203 { 3204 struct fuse_inode *fi = get_fuse_inode(inode); 3205 3206 inode->i_fop = &fuse_file_operations; 3207 inode->i_data.a_ops = &fuse_file_aops; 3208 3209 INIT_LIST_HEAD(&fi->write_files); 3210 INIT_LIST_HEAD(&fi->queued_writes); 3211 fi->writectr = 0; 3212 init_waitqueue_head(&fi->page_waitq); 3213 fi->writepages = RB_ROOT; 3214 3215 if (IS_ENABLED(CONFIG_FUSE_DAX)) 3216 fuse_dax_inode_init(inode, flags); 3217 } 3218